[ { "path": ".github/ISSUE_TEMPLATE/1_issue_report.yml", "content": "name: Bug / Issue Report\ndescription: Report a bug or an issue.\ntitle: \"[Issue]: \"\nlabels: [bug]\nbody:\n - type: markdown\n attributes:\n value: >\n **Before opening an issue, please make sure you are running the latest version of iSubRip,\n and that there isn't an already-existing open issue for your issue under the [issues tab](https://github.com/MichaelYochpaz/iSubRip/labels/bug).**\n - type: checkboxes\n id: check-confirmation\n attributes:\n label: Confirmations\n options:\n - label: \"I have checked the issues tab, and couldn't find an existing open issue for the issue I want to report.\"\n required: true\n - type: dropdown\n id: os-type\n attributes:\n label: OS Type\n description: The operation system that's being used to run iSubRip.\n options:\n - Windows\n - MacOS\n - Linux\n validations:\n required: true\n - type: input\n id: python-version\n attributes:\n label: Python Version\n description: |\n The Python version that's being used to run iSubRip.\n Can be checked by running `python --version`.\n placeholder: |\n Example: \"3.10.6\"\n validations:\n required: true\n - type: input\n id: version\n attributes:\n label: Package Version\n description: |\n iSubRip's version that's being used.\n Can be checked by running `python -m pip show isubrip`.\n placeholder: |\n Example: \"2.3.2\"\n validations:\n required: true\n - type: textarea\n id: description\n attributes:\n label: Description\n description: |\n A summary of the issue.\n Include as much information as possible, and steps to reproduce (if they're known).\n Log files (see README for more information) can be attached by clicking the area to highlight it, and then dragging & dropping files in.\n validations:\n required: true\n - type: textarea\n id: output-log\n attributes:\n label: Output Log\n description: |\n iSubRip's output when the issue occurred.\n Please include the command that was used to run iSubRip.\n render: Text\n placeholder: |\n Example:\n\n isubrip https://itunes.apple.com/us/movie/can-you-hear-us-now/id1617191490\n Scraping https://itunes.apple.com/us/movie/can-you-hear-us-now/id1617191490...\n Found movie: Can You Hear Us Now?\n Traceback (most recent call last):\n File \"%appdata%\\local\\programs\\python\\python38-32\\lib\\runpy.py\", line 193, in _run_module_as_main\n return _run_code(code, main_globals, None,\n File \"%appdata%\\local\\programs\\python\\python38-32\\lib\\runpy.py\", line 86, in _run_code\n exec(code, run_globals)\n File \"%appdata%\\local\\programs\\python\\python38-32\\scripts\\isubrip.exe\\__main__.py\", line 7, in \n File \"%appdata%\\local\\programs\\python\\python38-32\\lib\\site-packages\\isubrip\\__main__.py\", line 91, in main\n os.makedirs(current_download_path, exist_ok=True)\n File \"%appdata%\\local\\programs\\python\\python38-32\\lib\\os.py\", line 221, in makedirs\n mkdir(name, mode)\n OSError: [WinError 123] The filename, directory name, or volume label syntax is incorrect: 'C:\\\\%appdata%\\\\Local\\\\Temp\\\\iSubRip\\\\Can.You.Hear.Us.Now?.iT.WEB'\n validations:\n required: true\n - type: textarea\n id: config\n attributes:\n label: Config\n description: |\n The iSubRip config file you are using.\n **Leave empty only if there is no config file in use.**\n render: TOML\n placeholder: |\n Example:\n\n [downloads]\n folder = \"C:\\\\Subtitles\\\\iTunes\"\n languages = [\"en-US\", \"fr-FR\", \"he\"]\n zip = false\n\n [subtitles]\n convert-to-srt = true\n fix-rtl = true\n validations:\n required: false" }, { "path": ".github/ISSUE_TEMPLATE/2_feature_request.yml", "content": "name: Feature Request\ndescription: Request a new feature or improvement to an existing one.\ntitle: \"[Feature Request]: \"\nlabels: [feature-request]\nbody:\n - type: markdown\n attributes:\n value: >\n **Before opening an issue, please make sure there isn't an already-existing issue, open or closed,\n for this feature request under the [issues tab](https://github.com/MichaelYochpaz/iSubRip/issues?q=label%3Afeature-request).**\n - type: checkboxes\n id: check-confirmation\n attributes:\n label: Confirmations\n options:\n - label: \"I have checked the issues tab, and couldn't find an existing issue with my feature request.\"\n required: true\n - type: textarea\n id: description\n attributes:\n label: Description\n description: A summary of the feature you want to request.\n validations:\n required: true" }, { "path": ".github/ISSUE_TEMPLATE/3_question.yml", "content": "name: Ask a question\ndescription: Ask a question regarding iSubRip.\ntitle: \"[Question]: \"\nlabels: [question]\nbody:\n - type: markdown\n attributes:\n value: |\n **Please use this template only for questions.\n For issue / bug reports and feature requests, use one of the other templates.**\n - type: textarea\n id: description\n attributes:\n label: Question\n description: The question you want to ask.\n validations:\n required: true" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: false" }, { "path": ".gitignore", "content": "# From: https://github.com/github/gitignore/blob/main/Python.gitignore\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\n.pybuilder/\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n# For a library or package, you might want to ignore these files since the code is\n# intended to run in multiple environments; otherwise, check them in:\n# .python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n# PyCharm\n# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can\n# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore\n# and can be added to the global gitignore or merged into this file. For a more nuclear\n# option (not recommended) you can uncomment the following to ignore the entire idea folder.\n#.idea/\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\n## 2.6.8 [2025-10-14]\n### Changes:\n* Removed Python 3.9 support, added Python 3.14 support.\n\n### Bug Fixes:\n* Fixed a regression issue where the source type (e.g. \"WEB\") was missing from zip file names following the changes in version 2.6.7. ([Issue #100](https://github.com/MichaelYochpaz/iSubRip/issues/100))\n---\n## 2.6.7 [2025-10-10]\n### Added:\n* Updated M3U8 playlists loading to use alternative playlist URLs as fallback in case of an empty response.\n* Added playlist URLs to debug-level logs. ([Issue #94](https://github.com/MichaelYochpaz/iSubRip/issues/94))\n---\n## 2.6.6 [2025-08-13]\n### Changes:\n* Updated the date format in logs for unreleased content that has a release date. ([Issue #91](https://github.com/MichaelYochpaz/iSubRip/issues/91))\n\n### Bug Fixes:\n* Fixed an issue where in some cases, when converting from WebVTT to SubRip with the `subrip-alignment-conversion` setting enabled, top-aligned captions were not detected correctly. ([Issue #90](https://github.com/MichaelYochpaz/iSubRip/issues/90))\n---\n## 2.6.5 [2025-06-20]\n### Changes:\n* Added missing languages to the list of RTL languages (relevant if the `languages.fix-rtl` config setting is enabled).\n* Minor improvements to the download progress bar.\n\n### Bug Fixes:\n* Fixed an issue where in some cases, the \"Downloaded subtitles (X/Y)\" progress log line would print repeatedly. ([Issue #87](https://github.com/MichaelYochpaz/iSubRip/issues/87))\n---\n## 2.6.4 [2025-06-13]\n### Changes:\n* Minor improvements to the download progress bar.\n\n### Bug Fixes:\n* Fixed an issue where if there is no config file, an error is raised instead of using default settings. ([Issue #86](https://github.com/MichaelYochpaz/iSubRip/issues/86))\n---\n## 2.6.3 [2025-03-17]\n### Bug Fixes:\n* Fixed an issue where logs containing the percentage character (`%`) would raise an error. ([Issue #82](https://github.com/MichaelYochpaz/iSubRip/issues/82))\n---\n## 2.6.2 [2025-02-04]\n### Bug Fixes:\n* Fixed an issue where AppleTV API calls would fail due to changes on AppleTV requiring a missing `utsk` parameter. ([Issue #80](https://github.com/MichaelYochpaz/iSubRip/issues/80))\n* Fixed an issue where iTunes URLs would not work due to iTunes no longer redirecting to AppleTV. A different method will be used now to find corresponding AppleTV URLs. Also added a retry mechanism as it appears to be a bit unreliable at times. (thanks @yonatand1230 for suggesting this method!). ([Issue #78](https://github.com/MichaelYochpaz/iSubRip/issues/78))\n* Removed progress bar when where there are no matching subtitles to download (previously, it would just show 0/0 with 0% progress).\n---\n## 2.6.1 [2025-01-31]\n### Bug Fixes:\n* Fixed a backwards compatibility issue in code, which would cause errors when running on Python versions lower than 3.12. ([Issue #78](https://github.com/MichaelYochpaz/iSubRip/issues/78))\n---\n## 2.6.0 [2025-01-28]\n**The following update contains breaking changes to the config file. \nIf you are using one, please update your config file accordingly.**\n\n### Added:\n* Added a new `general.log-level` config setting, the log level of stdout (console) output. Set to `info` by default. Can be changed to `debug`, `warning`, or `error`. See the updated [example config](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml) for an example.\n\n### Changes:\n* Console output has been overhauled and improved, with colorful interactive output.\n* Config file is now parsed and validated in a more reliable and efficient manner. Configuration errors will now be more readable and descriptive.\n* **Breaking config changes** - the `scrapers` config category has been updated. Settings that should apply for all scrapers are now under the `scrapers.default` category instead of straight under `scrapers`. See the updated [example config](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml) for examples.\n* Updated AppleTV scraper request parameters.\n* Minor improvements to logs.\n* Python 3.8 is no longer supported. Minimum supported version has been updated to 3.9.\n\n### Bug Fixes:\n* Fixed an issue where if `verify-ssl` is set to `false`, and the `urllib3` package (which isn't a dependency of iSubRip) is not installed, an error could be thrown.\n---\n## 2.5.6 [2024-07-07]\n### Bug Fixes:\n* Fixed an issue where the update message from version `2.5.4` to `2.5.5` would still appear after updating. ([Issue #73](https://github.com/MichaelYochpaz/iSubRip/issues/73))\n---\n## 2.5.5 [2024-07-06]\n### Added:\n* Added new `timeout` setting to the config file, for the option to change the timeout for all / specific scrapers. See the updated [example config](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml) for usage examples. ([Issue #71](https://github.com/MichaelYochpaz/iSubRip/issues/71))\n\n### Changes:\n* Default timeout for requests has been updated from 5 seconds to 10 seconds. ([Issue #71](https://github.com/MichaelYochpaz/iSubRip/issues/71))\n---\n## 2.5.4 [2024-04-28]\n### Bug Fixes:\n* Fixed an issue where if the `logs` directory does not exist, the folder isn't created, causing an error. ([Issue #67](https://github.com/MichaelYochpaz/iSubRip/issues/67))\n* Fixed an issue where the summary log of successful and failed download would not account for failed downloads. ([Issue #68](https://github.com/MichaelYochpaz/iSubRip/issues/68))\n---\n## 2.5.3 [2024-04-09]\n### Added:\n* Added new `proxy` and `verify-ssl` settings to the config file, for allowing the usage of a proxy when making requests, and disabling SSL verification. See the updated [example config](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml) for usage examples.\n\n### Changes:\n* `subtitles.rtl-languages` config setting is no longer supported, and its values are now hardcoded and can't be modified.\n\n### Bug Fixes:\n* Fixed an issue where in some cases, `STYLE` blocks would repeat throughout the subtitles file, and cause inaccurate cue count. ([Issue #63](https://github.com/MichaelYochpaz/iSubRip/issues/63))\n* Fixed an issue where the WebVTT Style blocks would have their `STYLE` tag replaced with a `REGION` tag in downloaded subtitles.\n* Fixed an issue where an empty playlist (with a size of 0 bytes) would be reported as a valid playlist with no matching subtitles. ([Issue #65](https://github.com/MichaelYochpaz/iSubRip/issues/65))\n---\n## 2.5.2 [2024-01-06]\n### Bug Fixes:\n* Fixed an issue where errors would not be handled gracefully, and cause an unexpected crash. ([Issue #55](https://github.com/MichaelYochpaz/iSubRip/issues/55))\n---\n## 2.5.1 [2023-12-23]\n### Bug Fixes:\n* Fixed an issue where source abbreviation was missing from file names of downloaded subtitles files. ([Issue #53](https://github.com/MichaelYochpaz/iSubRip/issues/53))\n---\n## 2.5.0 [2023-12-16]\n### Added:\n* Added logs. See the new [Logs section in the README](https://github.com/MichaelYochpaz/iSubRip#logs) for more information.\n* Added a new `subtitles.webvtt.subrip-alignment-conversion` config setting (which is off by default), which if set to true, will add the `{\\an8}` tag at the start of lines that are annotated at the top (with the `line:0.00%` WebVTT setting) when converting to SubRip. ([Issue #35](https://github.com/MichaelYochpaz/iSubRip/issues/35))\n* Implemented caching for AppleTV's storefront configuration data, which should reduce the amount of requests used when scraping multiple AppleTV URLs from the same storefront.\n\n### Changes:\n* Big backend changes to the structure of the code, mostly to improve modularity and allow for easier development in the future, and improve performance.\n* Updated the CLI output to utilize logs and print with colors according to log-level.\n* Improved error handling in some cases where an invalid URL is used.\n\n### Bug Fixes:\n* Fixed an issue where if a movie is a pre-order with a set release date, a message with availability date wouldn't be printed in some cases.\n---\n## 2.4.3 [2023-06-18]\n### Bug Fixes:\n* Fixed an issue where some AppleTV URLs (or iTunes links that refer to such URLs) would not be matched in some cases, resulting in a \"No matching scraper was found...\" error. ([Issue #46](https://github.com/MichaelYochpaz/iSubRip/issues/46))\n---\n## 2.4.2 [2023-06-02]\n### Changes:\n* Improved error handling for subtitles downloads. ([Issue #44](https://github.com/MichaelYochpaz/iSubRip/issues/44))\n\n### Bug Fixes:\n* Fixed an issue where using a ZIP file, and saving to a different drive than the OS drive would fail. ([Issue #43](https://github.com/MichaelYochpaz/iSubRip/issues/43))\n---\n## 2.4.1 [2023-05-25]\n### Bug Fixes:\n* Fixed an issue where saving subtitles to a different drive than the OS drive would fail. ([Issue #41](https://github.com/MichaelYochpaz/iSubRip/issues/41))\n* Fixed AppleTV URLs with multiple iTunes playlists causing an error. ([Issue #42](https://github.com/MichaelYochpaz/iSubRip/issues/42))\n---\n## 2.4.0 [2023-05-23]\n### Added:\n- iTunes links will now redirect to AppleTV and scrape metadata from there, as AppleTV has additional and more accurate metadata.\n- Improved error messages to be more informative and case-specific:\n - If a movie is a pre-order and has no available playlist, a proper error message will be printed with its release date (if available).\n - If trying to scrape AppleTV+ content or series (which aren't currently supported), a proper error will be printed.\n\n### Changes:\n- A major refactor to the code, to make it more modular and allow for easier development of new features in the future.\n- Multiple changes (with some breaking changes) to the config file:\n - The `downloads.format` setting is deprecated, and replaced by the `subtitles.convert-to-srt` setting.\n - The `downloads.merge-playlists` setting is deprecated, with no replacement. \n If an AppleTV link has multiple playlists, they will be downloaded separately.\n - The `downloads.user-agent` setting is deprecated, with no replacement.\n The user-agent used by the scraper, will be used for downloads as well.\n - The `scraping` config category no longer exists, and is replaced by a `scrapers` category, which has a sub-category with settings for each scraper (for example, a `scrapers.itunes` sub-category).\n- Old config paths that were previously deprecated are no longer supported and will no longer work.\n The updated config settings can be found in the [example config](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml).\n\n### Notes:\n* This release includes a major rewrite of the code, which may have introduced new bugs to some core features. If you encountered one, [please report it](https://github.com/MichaelYochpaz/iSubRip/issues/new/choose).\n* Minimum supported Python version bumped to 3.8.\n* `beautifulsoup4` and `lxml` packages are no longer required or used.\n---\n## 2.3.3 [2022-10-09]\n### Changes:\n* Added release year to zip file names. ([Issue #31](https://github.com/MichaelYochpaz/iSubRip/issues/31))\n* If the generated path for a zip file is already taken, a number will be appended at the end of the file's name to avoid overwriting. ([Issue #34](https://github.com/MichaelYochpaz/iSubRip/issues/34))\n\n### Bug Fixes:\n* Fixed an exception being thrown if the path to downloads folder on the config is invalid.\n* Fixed AppleTV URLs without a movie title not working. ([Issue #29](https://github.com/MichaelYochpaz/iSubRip/issues/29))\n* Fixed issues for movies with specific characters (`/`, `:`), and Windows reserved names in their title. ([Issue #30](https://github.com/MichaelYochpaz/iSubRip/issues/30))\n---\n## 2.3.2 [2022-08-06]\n### Changes:\n* Changed config paths to the following locations: \nWindows: `%USERPROFILE%\\.isubrip\\config.json` \nLinux / macOS: `$HOME/.isubrip/config.json` \nMore info under Notes (and examples on the [README](https://github.com/MichaelYochpaz/iSubRip#configuration) file).\n\n### Bug Fixes:\n* Fixed an error with AppleTV links for movies released before 1970 (Epoch time). ([Issue #21](https://github.com/MichaelYochpaz/iSubRip/issues/21))\n* Fixed config file not being loaded on macOS. ([Issue #22](https://github.com/MichaelYochpaz/iSubRip/issues/22))\n* Fixed AppleTV scraping from the same storefront. ([Issue #24](https://github.com/MichaelYochpaz/iSubRip/issues/24))\n\n### Notes:\n* Running iSubRip with a config file in the previous locations will still work, but support for them will be dropped in the future. \n* `xdg` package is no longer required or used.\n---\n## 2.3.1 [2022-07-15]\n### Changes:\n* Improved AppleTV scraping to utilize AppleTV's API instead of scraping HTML.\n\n### Bug Fixes:\n* Fixed HTML escaped (for non-English) characters not matching AppleTV's URL RegEx. ([Issue #15](https://github.com/MichaelYochpaz/iSubRip/issues/15))\n---\n## 2.3.0 [2022-06-23]\n### Added:\n* AppleTV movie URLs are now supported.\n* Added a `merge-playlists` config option to treat multiple playlists that can be found on AppleTV pages as one (more info on the example config).\n\n### Changes:\n* Improved subtitles parser to perserve additional WebVTT data.\n* The config value `user-agent` under `scraping` is now separated to 2 different values: `itunes-user-agent`, and `appletv-user-agent`.\n\n### Bug Fixes:\n* Fixed movie titles with invalid Windows file-name characters (example: '?') causing a crash. ([Issue #14](https://github.com/MichaelYochpaz/iSubRip/issues/14))\n* Fixed iTunes store URLs without a movie title not working. ([Issue #13](https://github.com/MichaelYochpaz/iSubRip/issues/13))\n---\n## 2.2.0 [2022-04-25]\n### Added:\n* Replaced FFmpeg usage for parsing with a native subtitles parser (downloads are much faster now).\n* Added a `remove-duplicates` configuration remove duplicate paragraphs. (Was previously automatically fixed by FFmpeg.)\n* Added `fix-rtl` and `rtl-languages` configuration to fix RTL in RTL-languaged subtitles (has to be enabled in the config).\n\n### Changes:\n* FFmpeg is no longer required or used, and all FFmpeg-related settings are deprecated.\n\n### Notes:\n* `fix-rtl` is off by default and has to be enabled on the config. Check the `config.toml` example file for more info.\n* Minimum supported Python version bumped to 3.7.\n---\n## 2.1.2 [2022-04-03]\n### Bug Fixes:\n* Fixed subtitles being downloaded twice, which causes long (doubled) download times.\n---\n## 2.1.1 [2022-03-28]\n### Bug Fixes:\n* Fixed a compatibility issue with Python versions that are lower than 3.10.\n* Fixed downloading subtitles to an archive file not working properly.\n* Fixed a bug where the code continues to run if subtitles download failed, as if the download was successful.\n---\n## 2.1.0 [2022-03-19]\n### Added:\n* A note will be printed if a newer version is available on PyPI (can be disabled on the config).\n* Config will now be checked for errors before running.\n\n### Changes:\n* Big improvements to scraping, which is now far more reliable.\n* Added release year to subtitles file names.\n* Config structure slightly changed.\n\n### Notes:\n* If you use a user-config, it might need to be updated to match the new config structure.\n Example of an updated valid structure can be found [here](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml).\n---\n## 2.0.0 [2022-01-30]\nThe script is now a Python package that can be installed using pip.\n\n### Added:\n* Added a config file for changing configurations. (Example can be found [here](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml))\n* Added an option to choose subtitles format (vtt / srt).\n* Added an option to choose whether to zip subtitles files or not.\n* Multiple links can be passed for downloading subtitles for multiple movies one after another.\n* Temporary files are automatically removed if the script stops unexpectedly.\n\n### Changes:\n* A complete code overhaul from a single python script file to a package, while utilizing OOP and classes.\n* Improved scraping algorithm for faster playlist scraping.\n* FFmpeg will now automatically overwrite existing subtitles with the same file name.\n\n### Bug Fixes:\n* Fixed a bug where in some cases, no subtitles were found since the title has HTML escaped characters, which causes bad matching when checking if a valid playlist was found.\n---\n## 1.0.6 [2021-07-23]\n### Bug Fixes:\n* Fixed an issue where in some cases subtitles won't download when using `DOWNLOAD_FILTER` because of letter casing not matching.\n* Fixed and improved error handling, and added more descriptive error messages. ([Issue #9](https://github.com/MichaelYochpaz/iSubRip/issues/9))\n---\n## 1.0.5 [2021-05-27]\n### Bug Fixes:\n* Fixed subtitles for some movies not being found after previous release. ([Issue #8](https://github.com/MichaelYochpaz/iSubRip/issues/8))\n---\n## 1.0.4 [2021-05-25]\n### Bug Fixes:\n* Fixed the script not working after iTunes webpage data orientation slightly changed. ([Issue #6](https://github.com/MichaelYochpaz/iSubRip/issues/6) , [Issue #7](https://github.com/MichaelYochpaz/iSubRip/issues/7))\n---\n## 1.0.3 [2021-04-30]\n### Bug Fixes:\n* Fixed a bug where subtitles for suggested movies are being downloaded if movie's main playlist is not found. ([Issue #2](https://github.com/MichaelYochpaz/iSubRip/issues/2))\n* Added a \"cc\" tag to closed-caption subtitles' filename to avoid a collision with non-cc subtitles. ([Issue #3](https://github.com/MichaelYochpaz/iSubRip/issues/3))\n---\n## 1.0.2 [2021-04-15]\n### Added:\n* Added a User-Agent for sessions to avoid being blocked.\n\n### Changes:\n* `DOWNLOAD_FILTER` is no longer case-sensitive.\n* Added `lxml` to `requirements.txt`. ([Issue #1](https://github.com/MichaelYochpaz/iSubRip/issues/1))\n\n### Bug Fixes:\n* Fixed the script not working after iTunes webpage data orientation slightly changed. ([Issue #1](https://github.com/MichaelYochpaz/iSubRip/issues/1))\n---\n## 1.0.1 [2020-12-13]\n### Changes:\n* Improved error handling.\n \n### Bug Fixes:\n* Fixed file name formatting.\n---\n## 1.0.0 [2020-11-02]\n* Initial release." }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2022 Michael Yochpaz\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "> [!CAUTION]\n> iSubRip is currently not working due to changes on Apple's backend. \n> The future of this project is currently unknown. See [#103](https://github.com/MichaelYochpaz/iSubRip/issues/103) for details.\n\n# iSubRip\n**iSubRip** is a Python command-line tool for scraping and downloading subtitles from AppleTV and iTunes movie pages.\n\n
\n \"Python\n \"PyPI\n \"License\"\n\n \"Monthly\n \"Total\n \"Repo\n \"Issues\"\n
\n\n
\n\n
\n \n
\n\n---\n\n## ✨ Features\n- Scrape subtitles from AppleTV and iTunes movies without needing a purchase or account.\n- Retrieve the expected streaming release date (if available) for unreleased movies.\n- Utilize asynchronous downloading to speed up the download of chunked subtitles.\n- Automatically convert subtitles to SubRip (SRT) format.\n- Fix right-to-left (RTL) alignment in RTL language subtitles automatically.\n- Configure settings such as download folder, preferred languages, and toggling features.\n\n## 🚀 Quick Start\n### Installation\n```shell\npip install isubrip\n```\n\n### Usage\n```shell\nisubrip [URL...]\n```\n(URL can be either an AppleTV or iTunes movie URL)\n\n
\n\n> [!WARNING]\n> iSubRip is not recommended for use as a library in other projects. \n> The API frequently changes, and breaking changes to the API are common, even in minor versions.\n>\n> Support will not be provided for issues arising from using this package as a library.\n## 🛠 Configuration\nA [TOML](https://toml.io) configuration file can be created to customize various options and features.\n\nThe configuration file will be searched for in one of the following paths based on your operating system:\n\n- **Windows**: `%USERPROFILE%\\.isubrip\\config.toml`\n- **Linux / macOS**: `$HOME/.isubrip/config.toml`\n\n### Path Examples\n- **Windows**: `C:\\Users\\Michael\\.isubrip\\config.toml`\n- **Linux**: `/home/Michael/.isubrip/config.toml`\n- **macOS**: `/Users/Michael/.isubrip/config.toml`\n\n\n### Example Configuration\n```toml\n[downloads]\nfolder = \"C:\\\\Subtitles\\\\iTunes\"\nlanguages = [\"en-US\", \"fr-FR\", \"he\"]\nzip = false\n\n[subtitles]\nconvert-to-srt = true\nfix-rtl = true\n\n[subtitles.webvtt]\nsubrip-alignment-conversion = true\n```\n\nAn example config with details and explanations for all available settings can be found [here](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml).\n\n## 📜 Logs\nLog files are created for each run in the following paths, depending on your operating system:\n\n**Windows**: `%USERPROFILE%\\.isubrip\\logs` \n**Linux / macOS**: `$HOME/.isubrip/logs` \n\nLog rotation (deletion of old files once a certain number of files is reached) can be configured in the configuration file using the `general.log-rotation-size` setting. The default value is `15`.\n\nFor more details, see the [example configuration](https://github.com/MichaelYochpaz/iSubRip/blob/main/example-config.toml).\n\n\n## 📓 Changelog\nThe changelog for the latest, and all previous versions, can be found [here](https://github.com/MichaelYochpaz/iSubRip/blob/main/CHANGELOG.md).\n\n## 👨🏽‍💻 Contributing\nThis project is open-source but currently lacks the infrastructure to fully support external contributions.\n\nIf you wish to contribute, please open an issue first to discuss your proposed changes to avoid working on something that might not be accepted.\n\n## 🙏🏽 Support\nIf you find this project helpful, please consider supporting it by:\n- 🌟 Starring the repository\n- 💖 [Sponsoring the project](https://github.com/sponsors/MichaelYochpaz)\n\n## 📝 End User License Agreement\nBy using iSubRip, you agree to the following terms:\n\n1. **Disclaimer of Affiliation**: iSubRip is an independent, open-source project. It is not affiliated with, endorsed by, or in any way officially connected to Apple Inc., iTunes, or AppleTV.\n2. **Educational Purpose**: This tool is developed and provided for educational and research purposes only. It demonstrates techniques for accessing and processing publicly available, unencrypted subtitle data from HLS playlists.\n3. **User Responsibility and Compliance**: Any use of iSubRip is solely at the user's own risk and discretion. Users are responsible for ensuring that their use of the tool complies with all applicable laws, regulations, and terms of service of the content providers. This includes adhering to local, state, national, and international laws and regulations.\n4. **Limitation of Liability**: The developers of iSubRip shall not be held responsible for any legal consequences arising from the use of this tool. This includes, but is not limited to, claims of copyright infringement, intellectual property violations, or breaches of terms of service of content providers. Users assume all risks associated with acquiring and using subtitle data through this tool.\n\nBy using iSubRip, you acknowledge that you have read, understood, and agree to be bound by this agreement's terms and conditions.\n\n## ⚖️ License\nThis project is licensed under the MIT License. For more details, see the [LICENSE file](https://github.com/MichaelYochpaz/iSubRip/blob/main/LICENSE).\n" }, { "path": "example-config.toml", "content": "# ---------------- ⚠️ IMPORTANT - READ BEFORE USING ⚠️ ----------------\n# This is an example config file with all available settings and their default values (if they have one).\n# All settings are optional, and setting them in the config file will override their default values.\n#\n# In your config file, set only settings you wish to change from their default values.\n# Do NOT copy this file and use it as your config, as it will override ALL settings with the values specified here.\n# Use this file only as a reference to understand what different settings do,\n# and to decide which settings you should use in your config.\n#\n# Your config file should be saved in the following path (according to OS):\n# - Windows: %USERPROFILE%\\.isubrip\\config.toml\n# - Linux / macOS: $HOME/.isubrip/config.toml\n# ---------------------------------------------------------------------\n\n[general]\n# Check for updates before running, and show a note if a new version exists.\n# Value can be either 'true' or 'false'.\ncheck-for-updates = true\n\n# Maximum number of log files to keep in the logs folder.\n# Once the maximum number is reached, the oldest logs files will be deleted in rotation\n# until the number of files equals the maximum.\nlog-rotation-size = 15\n\n# Log level to use for stdout (console) output.\n# Value can be one of: \"debug\", \"info\", \"error\", \"warning\", \"critical\".\nlog-level = \"info\"\n\n\n[downloads]\n# Folder to downloads files to.\n# The default \".\" value means it will download to the same folder the script ran from.\n# Use double backslashes in path to avoid escaping characters. Example: \"C:\\\\Users\\\\\\\\Downloads\\\\\"\nfolder = \".\"\n\n# A list of iTunes language codes to download.\n# An empty array (like the one currently being used) will result in downloading all of the available subtitles.\n# Example: [\"en-US\", \"fr-FR\", \"he\"]\nlanguages = []\n\n# Whether to overwrite existing subtitles files.\n# If set to false, names of existing subtitles will have a number appended to them to avoid overwriting.\n# Value can be either 'true' or 'false'.\noverwrite-existing = false\n\n# Save files into a zip archive if there is more than one matching subtitles.\n# Value can be either 'true' or 'false'.\nzip = false\n\n\n[subtitles]\n# Fix RTL for RTL languages (Arabic & Hebrew).\n# Value can be either 'true' or 'false'.\n#\n# NOTE: This is off by default as some subtitles use other methods to fix RTL (like writing punctuations backwards).\n# Using this option on these type of subtitles can break the already-fixed RTL issues.\nfix-rtl = false\n\n# Remove duplicate paragraphs (same text and timestamps).\n# Value can be either 'true' or 'false'.\nremove-duplicates = true\n\n# Whether to convert subtitles to SRT format.\n# NOTE: This can cause loss of subtitles metadata that is not supported by SRT format.\nconvert-to-srt = false\n\n[subtitles.webvtt]\n# Whether to add a '{\\an8}' tag to lines that are aligned at the top when converting format from WebVTT to SubRip.\n# Relevant only if 'subtitles.convert-to-srt' is set to 'true'.\n# Value can be either 'true' or 'false'.\nsubrip-alignment-conversion = false\n\n\n[scrapers.default]\n# A subcategory to set default values for all scrapers.\n# These settings will be overridden by scraper-specific configuration, if set,\n# These settings will not apply if the scraper has a different specific default value.\n\n# Timeout in seconds for requests sent by all scrapers.\ntimeout = 10\n\n# User-Agent to use by default for requests sent by all scrapers.\nuser-agent = \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36\"\n\n# Proxy to use by default for requests sent by all scrapers.\nproxy = \"http://127.0.0.1:8080\"\n\n# Whether to verify SSL certificates when making requests for all scrapers.\n# Value can be either 'true' or 'false'.\nverify-ssl = true\n\n\n[scrapers.scraper-name]\n# Scraper-specific settings (set for each scraper separately).\n# Will override any default values previously set.\n# Replace 'scraper-name' with the name of the scraper to configure.\n# Available scrapers: itunes, appletv\n\n# Timeout in seconds for requests sent by the scraper.\ntimeout = 10\n\n# User-Agent to use for requests sent by the scraper.\nuser-agent = \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36\"\n\n# Proxy to use for requests sent by the scraper.\nproxy = \"http://127.0.0.1:8080\"\n\n# Whether to verify SSL certificates when making requests for the scraper.\n# Value can be either 'true' or 'false'.\nverify-ssl = true\n" }, { "path": "isubrip/__init__.py", "content": "" }, { "path": "isubrip/__main__.py", "content": "from __future__ import annotations\n\nimport asyncio\nimport logging\nimport sys\nfrom typing import TYPE_CHECKING\n\nimport httpx\nfrom pydantic import ValidationError\n\nfrom isubrip.cli import console\nfrom isubrip.commands.download import download\nfrom isubrip.config import Config\nfrom isubrip.constants import (\n PACKAGE_NAME,\n PACKAGE_VERSION,\n data_folder_path,\n log_files_path,\n user_config_file_path,\n)\nfrom isubrip.logger import logger, setup_loggers\nfrom isubrip.scrapers.scraper import Scraper, ScraperFactory\nfrom isubrip.subtitle_formats.webvtt import WebVTTCaptionBlock\nfrom isubrip.utils import (\n convert_log_level,\n format_config_validation_error,\n get_model_field,\n raise_for_status,\n single_string_to_list,\n)\n\nif sys.version_info >= (3, 11):\n import tomllib\nelse:\n import tomli as tomllib\n\nif TYPE_CHECKING:\n from pathlib import Path\n\n\nlog_rotation_size: int = 15 # Default size, before being updated by the config file.\n\n\ndef main() -> None:\n \"\"\"A wrapper for the actual main function that handles exceptions and cleanup.\"\"\"\n try:\n asyncio.run(_main())\n\n except Exception as ex:\n logger.error(f\"Error: {ex}\")\n logger.debug(\"Debug information:\", exc_info=True)\n exit(1)\n \n except KeyboardInterrupt:\n logger.debug(\"Keyboard interrupt detected, exiting...\")\n exit(0)\n\n finally:\n if log_rotation_size > 0:\n handle_log_rotation(rotation_size=log_rotation_size)\n\n for scraper in ScraperFactory.get_initialized_scrapers():\n logger.debug(f\"Requests count for '{scraper.name}' scraper: {scraper.requests_count}\") \n\n\nasync def _main() -> None:\n # Assure at least one argument was passed\n if len(sys.argv) < 2:\n logger.info(f\"Usage: {PACKAGE_NAME} [iTunes movie URL...]\")\n exit(0)\n\n # Generate the data folder if it doesn't previously exist\n if not data_folder_path().is_dir():\n data_folder_path().mkdir(parents=True)\n\n # If config file exists, parse it. Otherwise, create a config with default values\n if user_config_file_path().is_file():\n config = parse_config(config_file_location=user_config_file_path())\n\n else:\n config = Config()\n\n setup_loggers(\n stdout_loglevel=convert_log_level(log_level=config.general.log_level),\n stdout_console=console,\n logfile_output=True,\n logfile_output_path=log_files_path(),\n logfile_loglevel=logging.DEBUG,\n )\n\n cli_args = \" \".join(sys.argv[1:])\n logger.debug(f\"CLI Command: {PACKAGE_NAME} {cli_args}\")\n logger.debug(f\"Python version: {sys.version}\")\n logger.debug(f\"Package version: {PACKAGE_VERSION}\")\n logger.debug(f\"OS: {sys.platform}\")\n\n update_settings(config=config)\n\n if config.general.check_for_updates:\n check_for_updates(current_package_version=PACKAGE_VERSION)\n\n try:\n await download(\n *single_string_to_list(item=sys.argv[1:]),\n download_path=config.downloads.folder,\n language_filter=config.downloads.languages,\n convert_to_srt=config.subtitles.convert_to_srt,\n overwrite_existing=config.downloads.overwrite_existing,\n zip=config.downloads.zip,\n )\n \n finally:\n async_cleanup_coroutines = []\n\n for scraper in ScraperFactory.get_initialized_scrapers():\n async_cleanup_coroutines.append(scraper.async_close())\n \n if async_cleanup_coroutines:\n try:\n await asyncio.gather(*async_cleanup_coroutines)\n except Exception as e:\n logger.warning(f\"Error during async cleanup: {e}\")\n logger.debug(\"Cleanup debug info:\", exc_info=True)\n\n\ndef check_for_updates(current_package_version: str) -> None:\n \"\"\"\n Check and print if a newer version of the package is available, and log accordingly.\n\n Args:\n current_package_version (str): The current version of the package.\n \"\"\"\n api_url = f\"https://pypi.org/pypi/{PACKAGE_NAME}/json\"\n logger.debug(\"Checking for package updates on PyPI...\")\n try:\n response = httpx.get(\n url=api_url,\n headers={\"Accept\": \"application/json\"},\n timeout=5,\n )\n raise_for_status(response)\n response_data = response.json()\n\n pypi_latest_version = response_data[\"info\"][\"version\"]\n\n if pypi_latest_version != current_package_version:\n logger.warning(f\"You are currently using version '{current_package_version}' of '{PACKAGE_NAME}', \"\n f\"however version '{pypi_latest_version}' is available.\"\n f'\\nConsider upgrading by running \"pip install --upgrade {PACKAGE_NAME}\"')\n\n else:\n logger.debug(f\"Latest version of '{PACKAGE_NAME}' ({current_package_version}) is currently installed.\")\n\n except Exception as e:\n logger.warning(f\"Update check failed: {e}\")\n logger.debug(\"Debug information:\", exc_info=True)\n return\n\n\ndef handle_log_rotation(rotation_size: int) -> None:\n \"\"\"\n Handle log rotation and remove old log files if needed.\n\n Args:\n rotation_size (int): Maximum amount of log files to keep.\n \"\"\"\n sorted_log_files = sorted(log_files_path().glob(\"*.log\"), key=lambda file: file.stat().st_mtime, reverse=True)\n\n if len(sorted_log_files) > rotation_size:\n for log_file in sorted_log_files[rotation_size:]:\n log_file.unlink()\n\n\ndef parse_config(config_file_location: Path) -> Config:\n \"\"\"\n Parse the configuration file and return a Config instance.\n Exit the program (with code 1) if an error occurs while parsing the configuration file.\n\n Args:\n config_file_location (Path): The location of the configuration file.\n\n Returns:\n Config: An instance of the Config.\n \"\"\"\n try:\n with config_file_location.open('rb') as file:\n config_data = tomllib.load(file)\n\n return Config.model_validate(config_data)\n\n except ValidationError as e:\n logger.error(\"Invalid configuration - the following errors were found in the configuration file:\\n\" +\n format_config_validation_error(exc=e) +\n \"\\nPlease update your configuration to resolve this issue.\")\n logger.debug(\"Debug information:\", exc_info=True)\n exit(1)\n\n\n except tomllib.TOMLDecodeError as e:\n logger.error(f\"Error parsing config file: {e}\")\n logger.debug(\"Debug information:\", exc_info=True)\n exit(1)\n\n\n except Exception as e:\n logger.error(f\"Error loading configuration: {e}\")\n logger.debug(\"Debug information:\", exc_info=True)\n exit(1)\n\n\ndef update_settings(config: Config) -> None:\n \"\"\"\n Update settings according to config.\n\n Args:\n config (Config): An instance of a config to set settings according to.\n \"\"\"\n if config.general.log_level.casefold() == \"debug\":\n console.is_interactive = False\n\n Scraper.subtitles_fix_rtl = config.subtitles.fix_rtl\n Scraper.subtitles_remove_duplicates = config.subtitles.remove_duplicates\n\n Scraper.default_timeout = config.scrapers.default.timeout\n Scraper.default_user_agent = config.scrapers.default.user_agent\n Scraper.default_proxy = config.scrapers.default.proxy\n Scraper.default_verify_ssl = config.scrapers.default.verify_ssl\n\n for scraper in ScraperFactory.get_scraper_classes():\n if scraper_config := get_model_field(model=config.scrapers, field=scraper.id):\n scraper.config = scraper_config\n\n WebVTTCaptionBlock.subrip_alignment_conversion = (\n config.subtitles.webvtt.subrip_alignment_conversion\n )\n\n if config.general.log_rotation_size:\n global log_rotation_size\n log_rotation_size = config.general.log_rotation_size\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "isubrip/cli.py", "content": "from collections.abc import Iterator\nfrom contextlib import contextmanager\nfrom typing import Any\n\nfrom rich.console import Console\nfrom rich.live import Live\n\nconsole = Console(\n highlight=False,\n)\n\n@contextmanager \ndef conditional_live(renderable: Any) -> Iterator[Live | None]:\n \"\"\"\n A context manager that conditionally enables Rich's Live display based on console interactivity.\n \n When console.is_interactive is True, this behaves like Rich's Live display.\n When console.is_interactive is False, live updates are disabled.\n\n Args:\n renderable: The Rich renderable object to display in live mode.\n\n Yields:\n Optional[Live]: The Live display object if console is interactive, None otherwise.\n\n Example:\n ```python\n with conditional_live(progress) as live:\n # Your code here\n if live: # Optional: Check if live display is active\n live.update(...)\n ```\n \"\"\"\n if console.is_interactive:\n with Live(renderable, console=console) as live:\n yield live\n else:\n yield None\n" }, { "path": "isubrip/commands/__init__.py", "content": "" }, { "path": "isubrip/commands/download.py", "content": "from __future__ import annotations\n\nfrom pathlib import Path\nimport shutil\n\nfrom rich.console import Group\nfrom rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn\nfrom rich.text import Text\n\nfrom isubrip.cli import conditional_live, console\nfrom isubrip.data_structures import (\n Episode,\n MediaData,\n Movie,\n ScrapedMediaResponse,\n Season,\n Series,\n SubtitlesData,\n SubtitlesDownloadResults,\n)\nfrom isubrip.logger import logger\nfrom isubrip.scrapers.scraper import PlaylistLoadError, Scraper, ScraperError, ScraperFactory, SubtitlesDownloadError\nfrom isubrip.ui import MinsAndSecsTimeElapsedColumn\nfrom isubrip.utils import (\n TemporaryDirectory,\n download_subtitles_to_file,\n format_list,\n format_media_description,\n format_release_name,\n generate_non_conflicting_path,\n)\n\n\nasync def download(*urls: str,\n download_path: Path,\n language_filter: list[str] | None = None,\n convert_to_srt: bool = False,\n overwrite_existing: bool = True,\n zip: bool = False) -> None:\n \"\"\"\n Download subtitles from given URLs.\n\n Args:\n urls (list[str]): A list of URLs to download subtitles from.\n download_path (Path): Path to a folder where the subtitles will be downloaded to.\n language_filter (list[str] | None): List of specific languages to download. None for all languages (no filter).\n Defaults to None.\n convert_to_srt (bool, optional): Whether to convert the subtitles to SRT format. Defaults to False.\n overwrite_existing (bool, optional): Whether to overwrite existing subtitles. Defaults to True.\n zip (bool, optional): Whether to zip multiple subtitles. Defaults to False.\n \"\"\"\n for url in urls:\n try:\n logger.info(f\"Scraping [blue]{url}[/blue]\")\n\n scraper = ScraperFactory.get_scraper_instance(url=url)\n\n try:\n logger.debug(f\"Fetching {url}\")\n scraper_response: ScrapedMediaResponse = await scraper.get_data(url=url)\n\n except ScraperError as e:\n logger.error(f\"Error: {e}\")\n logger.debug(\"Debug information:\", exc_info=True)\n continue\n\n media_data = scraper_response.media_data\n playlist_scraper = ScraperFactory.get_scraper_instance(scraper_id=scraper_response.playlist_scraper)\n\n if not media_data:\n logger.error(f\"Error: No supported media was found for {url}.\")\n continue\n\n for media_item in media_data:\n try:\n logger.info(f\"Found {media_item.media_type}: \"\n f\"[cyan]{format_media_description(media_data=media_item)}[/cyan]\")\n await download_media(scraper=playlist_scraper,\n media_item=media_item,\n download_path=download_path,\n language_filter=language_filter,\n convert_to_srt=convert_to_srt,\n overwrite_existing=overwrite_existing,\n zip=zip)\n\n except Exception as e:\n if len(media_data) > 1:\n logger.warning(f\"Error scraping media item \"\n f\"'{format_media_description(media_data=media_item)}': {e}\\n\"\n f\"Skipping to next media item...\")\n logger.debug(\"Debug information:\", exc_info=True)\n continue\n\n raise\n\n except Exception as e:\n logger.error(f\"Error while scraping '{url}': {e}\")\n logger.debug(\"Debug information:\", exc_info=True)\n continue\n\n\nasync def download_media(scraper: Scraper, media_item: MediaData, download_path: Path,\n language_filter: list[str] | None = None, convert_to_srt: bool = False,\n overwrite_existing: bool = True, zip: bool = False) -> None:\n \"\"\"\n Download a media item.\n\n Args:\n scraper (Scraper): A Scraper object to use for downloading subtitles.\n media_item (MediaData): A media data item to download subtitles for.\n download_path (Path): Path to a folder where the subtitles will be downloaded to.\n language_filter (list[str] | None): List of specific languages to download. None for all languages (no filter).\n Defaults to None.\n convert_to_srt (bool, optional): Whether to convert the subtitles to SRT format. Defaults to False.\n overwrite_existing (bool, optional): Whether to overwrite existing subtitles. Defaults to True.\n zip (bool, optional): Whether to zip multiple subtitles. Defaults to False.\n \"\"\"\n if isinstance(media_item, Series):\n for season in media_item.seasons:\n await download_media(media_item=season, scraper=scraper, download_path=download_path,\n language_filter=language_filter, convert_to_srt=convert_to_srt,\n overwrite_existing=overwrite_existing, zip=zip)\n\n elif isinstance(media_item, Season):\n for episode in media_item.episodes:\n logger.info(f\"{format_media_description(media_data=episode, shortened=True)}:\")\n await download_media_item(media_item=episode, scraper=scraper, download_path=download_path,\n language_filter=language_filter, convert_to_srt=convert_to_srt,\n overwrite_existing=overwrite_existing, zip=zip)\n\n elif isinstance(media_item, (Movie | Episode)):\n await download_media_item(media_item=media_item, scraper=scraper, download_path=download_path,\n language_filter=language_filter, convert_to_srt=convert_to_srt,\n overwrite_existing=overwrite_existing, zip=zip)\n\n\nasync def download_media_item(scraper: Scraper, media_item: Movie | Episode, download_path: Path,\n language_filter: list[str] | None = None, convert_to_srt: bool = False,\n overwrite_existing: bool = True, zip: bool = False) -> None:\n \"\"\"\n Download subtitles for a single media item.\n\n Args:\n scraper (Scraper): A Scraper object to use for downloading subtitles.\n media_item (Movie | Episode): A movie or episode data object.\n download_path (Path): Path to a folder where the subtitles will be downloaded to.\n language_filter (list[str] | None): List of specific languages to download. None for all languages (no filter).\n Defaults to None.\n convert_to_srt (bool, optional): Whether to convert the subtitles to SRT format. Defaults to False.\n overwrite_existing (bool, optional): Whether to overwrite existing subtitles. Defaults to True.\n zip (bool, optional): Whether to zip multiple subtitles. Defaults to False.\n \"\"\"\n ex: Exception | None = None\n\n if media_item.playlist:\n try:\n results = await download_subtitles(\n scraper=scraper,\n media_data=media_item,\n download_path=download_path,\n language_filter=language_filter,\n convert_to_srt=convert_to_srt,\n overwrite_existing=overwrite_existing,\n zip=zip,\n )\n\n success_count = len(results.successful_subtitles)\n failed_count = len(results.failed_subtitles)\n\n if success_count or failed_count:\n logger.info(f\"{success_count}/{success_count + failed_count} subtitles were successfully downloaded.\")\n\n else:\n logger.info(\"No matching subtitles were found.\")\n\n return # noqa: TRY300\n\n except PlaylistLoadError as e:\n ex = e\n\n # We get here if there is no playlist, or there is one, but it failed to load\n if isinstance(media_item, Movie) and media_item.preorder_availability_date:\n logger.info(f\"[gold1]'{media_item.name}' is currently unavailable on {scraper.name}, \"\n f\"and will be available on {media_item.preorder_availability_date.strftime(r'%B %e, %Y')}.[/gold1]\")\n\n else:\n if ex:\n logger.error(f\"Error: {ex}\")\n\n else:\n logger.error(\"Error: No valid playlist was found.\")\n\n\nasync def download_subtitles(scraper: Scraper, media_data: Movie | Episode, download_path: Path,\n language_filter: list[str] | None = None, convert_to_srt: bool = False,\n overwrite_existing: bool = True, zip: bool = False) -> SubtitlesDownloadResults:\n \"\"\"\n Download subtitles for the given media data.\n\n Args:\n scraper (Scraper): A Scraper object to use for downloading subtitles.\n media_data (Movie | Episode): A movie or episode data object.\n download_path (Path): Path to a folder where the subtitles will be downloaded to.\n language_filter (list[str] | None): List of specific languages to download. None for all languages (no filter).\n Defaults to None.\n convert_to_srt (bool, optional): Whether to convert the subtitles to SRT format. Defaults to False.\n overwrite_existing (bool, optional): Whether to overwrite existing subtitles. Defaults to True.\n zip (bool, optional): Whether to zip multiple subtitles. Defaults to False.\n\n Returns:\n SubtitlesDownloadResults: A SubtitlesDownloadResults object containing the results of the download.\n \"\"\"\n temp_dir_name = format_release_name(\n title=media_data.name if isinstance(media_data, Movie) else media_data.series_name,\n release_date=media_data.release_date,\n season_number=None if isinstance(media_data, Movie) else media_data.season_number,\n episode_number=None if isinstance(media_data, Movie) else media_data.episode_number,\n episode_name=None if isinstance(media_data, Movie) else media_data.episode_name,\n media_source=scraper.abbreviation,\n )\n successful_downloads: list[SubtitlesData] = []\n failed_downloads: list[SubtitlesDownloadError] = []\n\n with TemporaryDirectory(directory_name=temp_dir_name) as temp_download_path:\n temp_downloads: list[Path] = []\n\n if not media_data.playlist:\n raise PlaylistLoadError(\"No playlist was found for provided media data.\")\n\n main_playlist = await scraper.load_playlist(url=media_data.playlist) # type: ignore[func-returns-value]\n\n if not main_playlist:\n raise PlaylistLoadError(\"Failed to load the main playlist.\")\n\n matching_subtitles = scraper.find_matching_subtitles(main_playlist=main_playlist, # type: ignore[var-annotated]\n language_filter=language_filter)\n\n # If no matching subtitles were found, there's no need to continue\n if not matching_subtitles:\n return SubtitlesDownloadResults(\n media_data=media_data,\n successful_subtitles=successful_downloads,\n failed_subtitles=failed_downloads,\n is_zip=zip,\n )\n\n logger.info(f\"{len(matching_subtitles)} matching subtitles were found.\", extra={\"hide_when_interactive\": True})\n downloaded_subtitles: list[str] = []\n\n progress_log = Text(f\"Downloaded subtitles ({len(downloaded_subtitles)}/{len(matching_subtitles)}):\")\n downloads_list = Text()\n progress_bar = Progress(\n SpinnerColumn(),\n TextColumn(\"[progress.description]{task.description}\"),\n BarColumn(),\n TextColumn(\"[progress.percentage][yellow]{task.percentage:>3.0f}%[/yellow]\"),\n TextColumn(\"[yellow]{task.completed}/{task.total}[/yellow]\"),\n MinsAndSecsTimeElapsedColumn(),\n console=console,\n )\n task = progress_bar.add_task(\"Starting download\", total=len(matching_subtitles))\n\n with conditional_live(\n Group(progress_log, downloads_list, Text(), progress_bar), # Empty 'Text' for line spacing\n ) as live:\n for matching_subtitles_item in matching_subtitles:\n language_info = scraper.format_subtitles_description(\n subtitles_media=matching_subtitles_item,\n )\n\n if live:\n progress_bar.update(task, advance=1, description=f\"Processing [magenta]{language_info}[/magenta]\")\n\n try:\n subtitles_data = await scraper.download_subtitles(media_data=matching_subtitles_item,\n subrip_conversion=convert_to_srt)\n\n except Exception as e:\n if isinstance(e, SubtitlesDownloadError):\n failed_downloads.append(e)\n original_error = e.original_exc\n\n else:\n original_error = e\n\n logger.warning(f\"Failed to download '{language_info}' subtitles: {original_error}\")\n logger.debug(\"Debug information:\", exc_info=original_error)\n continue\n\n try:\n temp_downloads.append(download_subtitles_to_file(\n media_data=media_data,\n subtitles_data=subtitles_data,\n output_path=temp_download_path,\n source_abbreviation=scraper.abbreviation,\n overwrite=overwrite_existing,\n ))\n\n downloaded_subtitles.append(f\"• {language_info}\")\n\n if live:\n progress_log.plain = (f\"Downloaded subtitles \"\n f\"({len(downloaded_subtitles)}/{len(matching_subtitles)}):\")\n downloads_list.plain = f\"{format_list(downloaded_subtitles, width=live.console.width)}\"\n\n logger.info(f\"{language_info} subtitles were successfully downloaded.\",\n extra={\"hide_when_interactive\": True})\n successful_downloads.append(subtitles_data)\n\n except Exception as e:\n logger.warning(f\"Failed to save '{language_info}' subtitles: {e}\")\n logger.debug(\"Debug information:\", exc_info=True)\n failed_downloads.append(\n SubtitlesDownloadError(\n language_code=subtitles_data.language_code,\n language_name=subtitles_data.language_name,\n special_type=subtitles_data.special_type,\n original_exc=e,\n ),\n )\n\n if live:\n progress_bar.update(task, visible=False)\n\n if not zip or len(temp_downloads) == 1:\n for file_path in temp_downloads:\n if overwrite_existing:\n new_path = download_path / file_path.name\n\n else:\n new_path = generate_non_conflicting_path(file_path=download_path / file_path.name)\n\n shutil.move(src=file_path, dst=new_path)\n\n elif len(temp_downloads) > 0:\n zip_path = Path(shutil.make_archive(\n base_name=str(temp_download_path.parent / temp_download_path.name),\n format=\"zip\",\n root_dir=temp_download_path,\n ))\n\n file_name = format_release_name(\n title=media_data.name if isinstance(media_data, Movie) else media_data.series_name,\n release_date=media_data.release_date,\n season_number=None if isinstance(media_data, Movie) else media_data.season_number,\n episode_number=None if isinstance(media_data, Movie) else media_data.episode_number,\n episode_name=None if isinstance(media_data, Movie) else media_data.episode_name,\n media_source=scraper.abbreviation,\n file_format=\"zip\",\n )\n\n if overwrite_existing:\n destination_path = download_path / file_name\n\n else:\n destination_path = generate_non_conflicting_path(file_path=download_path / file_name)\n\n shutil.move(src=zip_path, dst=destination_path)\n\n return SubtitlesDownloadResults(\n media_data=media_data,\n successful_subtitles=successful_downloads,\n failed_subtitles=failed_downloads,\n is_zip=zip,\n )\n" }, { "path": "isubrip/config.py", "content": "from __future__ import annotations\n\nfrom abc import ABC\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Literal\n\nfrom pydantic import AliasGenerator, BaseModel, ConfigDict, Field, create_model, field_validator\nfrom pydantic_core import PydanticCustomError\nfrom pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict, TomlConfigSettingsSource\n\nfrom isubrip.scrapers.scraper import DefaultScraperConfig, ScraperFactory\n\n\nclass ConfigCategory(BaseModel, ABC):\n \"\"\"A base class for settings categories.\"\"\"\n model_config = ConfigDict(\n extra='allow',\n alias_generator=AliasGenerator(\n validation_alias=lambda field_name: field_name.replace('_', '-'),\n ),\n )\n\n\nclass GeneralCategory(ConfigCategory):\n check_for_updates: bool = Field(default=True)\n verbose: bool = Field(default=False)\n log_level: Literal[\"debug\", \"info\", \"warning\", \"error\", \"critical\"] = Field(default=\"info\")\n log_rotation_size: int = Field(default=15)\n\n\nclass DownloadsCategory(ConfigCategory):\n folder: Path = Field(default=Path.cwd().resolve())\n languages: list[str] = Field(default=[])\n overwrite_existing: bool = Field(default=False)\n zip: bool = Field(default=False)\n\n @field_validator('folder')\n @classmethod\n def assure_path_exists(cls, value: Path) -> Path:\n if value.exists():\n if not value.is_dir():\n raise PydanticCustomError(\n \"invalid_path\",\n \"Path is not a directory.\",\n )\n\n else:\n raise PydanticCustomError(\n \"invalid_path\",\n \"Path does not exist.\")\n\n return value\n\n\nclass WebVTTSubcategory(ConfigCategory):\n subrip_alignment_conversion: bool = Field(default=False)\n\n\nclass SubtitlesCategory(ConfigCategory):\n fix_rtl: bool = Field(default=False)\n remove_duplicates: bool = Field(default=True)\n convert_to_srt: bool = Field(default=False)\n webvtt: WebVTTSubcategory = WebVTTSubcategory()\n\n\nclass ScrapersCategory(ConfigCategory):\n default: DefaultScraperConfig = Field(default_factory=DefaultScraperConfig)\n\n\n# Resolve mypy errors as mypy doesn't support dynamic models.\nif TYPE_CHECKING:\n DynamicScrapersCategory = ScrapersCategory\n\nelse:\n # A config model that's dynamically created based on the available scrapers and their configurations.\n DynamicScrapersCategory = create_model(\n 'DynamicScrapersCategory',\n __base__=ScrapersCategory,\n **{\n scraper.id: (scraper.ScraperConfig, Field(default_factory=scraper.ScraperConfig))\n for scraper in ScraperFactory.get_scraper_classes()\n }, # type: ignore[call-overload]\n )\n\n\nclass Config(BaseSettings):\n model_config = SettingsConfigDict(\n extra='forbid',\n )\n\n general: GeneralCategory = Field(default_factory=GeneralCategory)\n downloads: DownloadsCategory = Field(default_factory=DownloadsCategory)\n subtitles: SubtitlesCategory = Field(default_factory=SubtitlesCategory)\n scrapers: DynamicScrapersCategory = Field(default_factory=DynamicScrapersCategory)\n\n @classmethod\n def settings_customise_sources(\n cls,\n settings_cls: type[BaseSettings],\n init_settings: PydanticBaseSettingsSource,\n env_settings: PydanticBaseSettingsSource,\n dotenv_settings: PydanticBaseSettingsSource,\n file_secret_settings: PydanticBaseSettingsSource,\n ) -> tuple[PydanticBaseSettingsSource, ...]:\n return (\n init_settings,\n TomlConfigSettingsSource(settings_cls),\n env_settings,\n dotenv_settings,\n file_secret_settings,\n )\n" }, { "path": "isubrip/constants.py", "content": "from __future__ import annotations\n\nfrom functools import lru_cache\nfrom pathlib import Path\nfrom tempfile import gettempdir\n\n# General\nPACKAGE_NAME = \"isubrip\"\nPACKAGE_VERSION = \"2.6.8\"\n\nSCRAPER_MODULES_SUFFIX = \"_scraper\"\nUSER_CONFIG_FILE_NAME = \"config.toml\"\n\n@lru_cache(maxsize=1)\ndef data_folder_path() -> Path:\n return Path.home() / f\".{PACKAGE_NAME}\"\n\n@lru_cache(maxsize=1)\ndef temp_folder_path() -> Path:\n return Path(gettempdir()) / PACKAGE_NAME\n\n@lru_cache(maxsize=1)\ndef user_config_file_path() -> Path:\n return data_folder_path() / USER_CONFIG_FILE_NAME\n\n# Logging Paths\n@lru_cache(maxsize=1)\ndef log_files_path() -> Path:\n return data_folder_path() / \"logs\"\n\n# Other\nWINDOWS_RESERVED_FILE_NAMES = frozenset(\n [\"CON\", \"PRN\", \"AUX\", \"NUL\", \"COM1\", \"COM2\", \"COM3\", \"COM4\", \"COM5\", \"COM6\", \"COM7\", \"COM8\", \"COM9\", \"LPT1\",\n \"LPT2\", \"LPT3\", \"LPT4\", \"LPT5\", \"LPT6\", \"LPT7\", \"LPT8\", \"LPT9\"],\n )\n\nRTL_LANGUAGES = frozenset([\"ar\", \"arc\", \"az\", \"dv\", \"he\", \"ks\", \"ku\", \"fa\", \"ur\", \"yi\"])\n" }, { "path": "isubrip/data_structures.py", "content": "from __future__ import annotations\n\nfrom abc import ABC\nimport datetime as dt # noqa: TC003\nfrom enum import Enum\nfrom typing import TYPE_CHECKING, Generic, Literal, NamedTuple, TypeVar\n\nimport m3u8\nfrom pydantic import BaseModel\n\nif TYPE_CHECKING:\n from isubrip.scrapers.scraper import SubtitlesDownloadError\n\nT = TypeVar(\"T\")\nMainPlaylist = TypeVar(\"MainPlaylist\", bound=m3u8.M3U8)\nPlaylistMediaItem = TypeVar(\"PlaylistMediaItem\", bound=m3u8.Media)\n\nMediaData = TypeVar(\"MediaData\", bound=\"MediaBase\")\n\n\nclass SubtitlesDownloadResults(NamedTuple):\n \"\"\"\n A named tuple containing download results.\n\n Attributes:\n media_data (Movie | Episode): An object containing metadata about the media the subtitles were downloaded for.\n successful_subtitles (list[SubtitlesData]): List of subtitles that were successfully downloaded.\n failed_subtitles (list[SubtitlesData]): List of subtitles that failed to download.\n is_zip (bool): Whether the subtitles were saved in a zip file.\n \"\"\"\n media_data: Movie | Episode\n successful_subtitles: list[SubtitlesData]\n failed_subtitles: list[SubtitlesDownloadError]\n is_zip: bool\n\n\nclass SubtitlesFormat(BaseModel):\n \"\"\"\n An object containing subtitles format data.\n\n Attributes:\n name (str): Name of the format.\n file_extension (str): File extension of the format.\n \"\"\"\n name: str\n file_extension: str\n\n\nclass SubtitlesFormatType(Enum):\n \"\"\"\n An Enum representing subtitles formats.\n\n Attributes:\n SUBRIP (SubtitlesFormat): SubRip format.\n WEBVTT (SubtitlesFormat): WebVTT format.\n \"\"\"\n SUBRIP = SubtitlesFormat(name=\"SubRip\", file_extension=\"srt\")\n WEBVTT = SubtitlesFormat(name=\"WebVTT\", file_extension=\"vtt\")\n\n\nclass SubtitlesType(Enum):\n \"\"\"\n Subtitles special type.\n\n Attributes:\n CC (SubtitlesType): Closed captions.\n FORCED (SubtitlesType): Forced subtitles.\n \"\"\"\n CC = \"CC\"\n FORCED = \"Forced\"\n\n\nclass SubtitlesData(BaseModel):\n \"\"\"\n An object containing subtitles data and metadata.\n\n Attributes:\n language_code (str): Language code of the language the subtitles are in.\n language_name (str | None, optional): Name of the language the subtitles are in.\n subtitles_format (SubtitlesFormatType): Format of the subtitles.\n content (bytes): Content of the subtitles in binary format.\n content_encoding (str): Encoding of subtitles content (ex. \"utf-8\").\n special_type (SubtitlesType | None, optional): Type of the subtitles, if they're not regular. Defaults to None.\n \"\"\"\n language_code: str\n subtitles_format: SubtitlesFormatType\n content: bytes\n content_encoding: str\n language_name: str | None = None\n special_type: SubtitlesType | None = None\n\n class ConfigDict:\n str_strip_whitespace = True\n\n\nclass MediaBase(BaseModel, ABC):\n \"\"\"A base class for media objects.\"\"\"\n\n\nclass Movie(MediaBase):\n \"\"\"\n An object containing movie metadata.\n\n Attributes:\n id (str | None, optional): ID of the movie on the service it was scraped from. Defaults to None.\n referrer_id (str | None, optional): ID of the movie on the original referring service. Defaults to None.\n name (str): Title of the movie.\n release_date (datetime | int | None, optional): Release date (datetime), or year (int) of the movie.\n Defaults to None.\n duration (timedelta | None, optional): Duration of the movie. Defaults to None.\n preorder_availability_date (datetime | None, optional):\n Date when the movie will be available for pre-order on the service it was scraped from.\n None if not a pre-order. Defaults to None.\n playlist (str | None, optional): Main playlist URL(s).\n \"\"\"\n media_type: Literal[\"movie\"] = \"movie\"\n name: str\n release_date: dt.datetime | int\n id: str | None = None\n referrer_id: str | None = None\n duration: dt.timedelta | None = None\n preorder_availability_date: dt.datetime | None = None\n playlist: str | list[str] | None = None\n\n\nclass Episode(MediaBase):\n \"\"\"\n An object containing episode metadata.\n\n Attributes:\n id (str | None, optional): ID of the episode on the service it was scraped from. Defaults to None.\n referrer_id (str | None, optional): ID of the episode on the original referring service. Defaults to None.\n series_name (str): Name of the series the episode is from.\n series_release_date (datetime | int | None, optional): Release date (datetime), or year (int) of the series.\n Defaults to None.\n season_number (int): Season number.\n season_name (str | None, optional): Season name. Defaults to None.\n episode_number (int): Episode number.\n episode_name (str | None, optional): Episode name. Defaults to None.\n episode_release_date (datetime | None): Release date of the episode. Defaults to None.\n episode_duration (timedelta | None, optional): Duration of the episode. Defaults to None.\n playlist (str | None, optional): Main playlist URL(s).\n \"\"\"\n media_type: Literal[\"episode\"] = \"episode\"\n series_name: str\n season_number: int\n episode_number: int\n id: str | None = None\n referrer_id: str | None = None\n series_release_date: dt.datetime | int | None = None\n season_name: str | None = None\n release_date: dt.datetime | None = None\n duration: dt.timedelta | None = None\n episode_name: str | None = None\n episode_release_date: dt.datetime | None = None\n episode_duration: dt.timedelta | None = None\n playlist: str | list[str] | None = None\n\n\nclass Season(MediaBase):\n \"\"\"\n An object containing season metadata.\n\n Attributes:\n id (str | None, optional): ID of the season on the service it was scraped from. Defaults to None.\n referrer_id (str | None, optional): ID of the season on the original referring service. Defaults to None.\n series_name (str): Name of the series the season is from.\n season_number (int): Season number.\n series_release_date (datetime | int | None, optional): Release date (datetime), or year (int) of the series.\n Defaults to None.\n season_name (str | None, optional): Season name. Defaults to None.\n season_release_date (datetime | None, optional): Release date of the season, or release year. Defaults to None.\n episodes (list[Episode]): A list of episode objects containing metadata about episodes of the season.\n \"\"\"\n media_type: Literal[\"season\"] = \"season\"\n series_name: str\n season_number: int\n id: str | None = None\n referrer_id: str | None = None\n series_release_date: dt.datetime | int | None = None\n season_name: str | None = None\n season_release_date: dt.datetime | int | None = None\n episodes: list[Episode] = []\n\n\nclass Series(MediaBase):\n \"\"\"\n An object containing series metadata.\n\n Attributes:\n id (str | None, optional): ID of the series on the service it was scraped from. Defaults to None.\n series_name (str): Series name.\n referrer_id (str | None, optional): ID of the series on the original referring service. Defaults to None.\n series_release_date (datetime | int | None, optional): Release date (datetime), or year (int) of the series.\n Defaults to None.\n seasons (list[Season]): A list of season objects containing metadata about seasons of the series.\n \"\"\"\n media_type: Literal[\"series\"] = \"series\"\n series_name: str\n seasons: list[Season] = []\n id: str | None = None\n referrer_id: str | None = None\n series_release_date: dt.datetime | int | None = None\n\n\nclass ScrapedMediaResponse(BaseModel, Generic[MediaData]):\n \"\"\"\n An object containing scraped media data and metadata.\n\n Attributes:\n media_data (list[Movie] | list[Episode] | list[Season] | list[Series]):\n An object containing the scraped media data.\n metadata_scraper (str): ID of the scraper that was used to scrape metadata.\n playlist_scraper (str): ID of the scraper that should be used to parse and scrape the playlist.\n original_data (dict): Original raw data from the API that was used to extract media's data.\n \"\"\"\n media_data: list[MediaData]\n metadata_scraper: str\n playlist_scraper: str\n original_data: dict\n" }, { "path": "isubrip/logger.py", "content": "from __future__ import annotations\n\nimport datetime as dt\nfrom functools import lru_cache\nimport logging\nimport re\nfrom typing import TYPE_CHECKING, ClassVar\n\nfrom rich.highlighter import NullHighlighter\nfrom rich.logging import RichHandler\n\nfrom isubrip.cli import console\nfrom isubrip.constants import (\n PACKAGE_NAME,\n)\n\nif TYPE_CHECKING:\n from pathlib import Path\n\n from rich.console import Console\n\n\nBBCOE_REGEX = re.compile(\n r\"(?i)(?P\\[(?P[a-z#@][^[]*?)])(?P.*)(?P\\[/(?P=tag_name)])\")\nLOG_FILE_METADATA = \"[%(asctime)s | %(levelname)s | %(threadName)s | %(filename)s::%(funcName)s::%(lineno)d] \"\n\n\ndef set_logger(_logger: logging.Logger) -> None:\n \"\"\"\n Set an external logger to be used by the package.\n\n Args:\n _logger (logging.Logger): A logger instance to be used by the package.\n \"\"\"\n global logger\n logger = _logger\n\n\nclass CustomStdoutFormatter(RichHandler):\n \"\"\"\n Custom formatter for stdout logging with Rich integration.\n \n This formatter adds color to log messages based on their level and\n supports hiding messages in interactive mode.\n \"\"\"\n LEVEL_COLORS: ClassVar[dict[int, str]] = {\n logging.ERROR: \"red\",\n logging.WARNING: \"dark_orange\",\n logging.DEBUG: \"grey54\",\n }\n \n def __init__(self, console: Console | None = None, debug_mode: bool = False) -> None:\n \"\"\"\n Initialize the stdout formatter.\n \n Args:\n console (Console | None, optional): Rich console instance to use for output. Defaults to None.\n debug_mode (bool, optional): Whether to show additional debug information. Defaults to False.\n \"\"\"\n super().__init__(\n console=console,\n show_time=debug_mode,\n show_level=debug_mode,\n show_path=debug_mode,\n highlighter=NullHighlighter(),\n markup=True,\n log_time_format=\"%H:%M:%S\",\n rich_tracebacks=debug_mode,\n tracebacks_extra_lines=0,\n )\n self._console = console\n\n def emit(self, record: logging.LogRecord) -> None:\n \"\"\"\n Emit a log record, respecting the 'hide_when_interactive' flag.\n \n Args:\n record (LogRecord): The log record to emit.\n \"\"\"\n # Skip emission if record is marked to be hidden in interactive mode\n if getattr(record, 'hide_when_interactive', False) and self._console and self._console.is_interactive:\n return\n super().emit(record)\n\n def format(self, record: logging.LogRecord) -> str:\n \"\"\"\n Format the log record with appropriate color based on level.\n \n Args:\n record (LogRecord): The log record to format.\n \n Returns:\n str: Formatted log message with Rich markup.\n \"\"\"\n # Get the message once\n message = record.getMessage()\n \n # Apply color based on log level using the class variable mapping\n if color := self.LEVEL_COLORS.get(record.levelno):\n record.msg = f\"[{color}]{message}[/{color}]\"\n \n return super().format(record)\n\n\nclass CustomLogFileFormatter(logging.Formatter):\n \"\"\"\n Custom formatter for log files that removes Rich markup tags.\n \"\"\"\n def __init__(self) -> None:\n \"\"\"\n Initialize the formatter with metadata format but without message part.\n We'll append the message manually to avoid issues with special characters.\n \"\"\"\n super().__init__(\n fmt=LOG_FILE_METADATA,\n datefmt=r\"%Y-%m-%d %H:%M:%S\",\n )\n \n @staticmethod\n @lru_cache(maxsize=64)\n def _remove_rich_markup(text: str) -> str:\n \"\"\"\n Remove Rich markup tags from text efficiently with caching.\n \n Args:\n text: Text containing Rich markup tags\n \n Returns:\n Text with Rich markup tags removed\n \"\"\"\n while match := BBCOE_REGEX.search(text):\n text = text[:match.start()] + match.group('content') + text[match.end():]\n return text\n \n def format(self, record: logging.LogRecord) -> str:\n \"\"\"\n Format the log record for file output, removing Rich markup.\n This implementation uses the standard formatter for the metadata part\n and then appends the message without formatting to avoid issues with\n special characters within the log message.\n \n Args:\n record: The log record to format\n \n Returns:\n Formatted log message suitable for file output\n \"\"\"\n message = record.getMessage()\n clean_message = self._remove_rich_markup(message)\n \n # Store the original message\n original_msg = record.msg\n original_args = record.args\n \n # Temporarily set an empty message to format just the metadata\n record.msg = \"\"\n record.args = None\n \n # Format the metadata part using the standard formatter\n metadata = super().format(record)\n \n # Restore the original message and args\n record.msg = original_msg\n record.args = original_args\n \n # Combine metadata and message without formatting the message\n return metadata + clean_message\n\n\ndef setup_loggers(stdout_output: bool = True, stdout_console: Console | None = None,\n stdout_loglevel: int = logging.INFO, logfile_output: bool = False,\n logfile_output_path: Path | None = None, logfile_loglevel: int = logging.DEBUG) -> None:\n \"\"\"\n Configure loggers for both stdout and file output.\n\n Args:\n stdout_output (bool, optional): Whether to output logs to STDOUT. Defaults to True.\n stdout_console (Console | None, optional): A Rich console instance to be used for STDOUT logging.\n Relevant only if `stdout_output` is True. Defaults to None.\n stdout_loglevel (int, optional): Log level for STDOUT logger. Relevant only if `stdout_output` is True.\n Defaults to logging.INFO.\n logfile_output (bool, optional): Whether to output logs to a logfile. Defaults to True.\n logfile_output_path (Path | None, optional): Path to the directory where log files will be saved.\n Required only if `logfile_output` is True. Defaults to None.\n logfile_loglevel (int, optional): Log level for logfile logger. Relevant only if `logfile_output` is True.\n Defaults to logging.DEBUG.\n \"\"\"\n logger.handlers.clear() # Remove and reset existing handlers\n logger.setLevel(logging.DEBUG)\n\n if stdout_output:\n debug_mode = (stdout_loglevel == logging.DEBUG)\n stdout_handler = CustomStdoutFormatter(\n debug_mode=debug_mode,\n console=stdout_console,\n )\n stdout_handler.setLevel(stdout_loglevel)\n logger.addHandler(stdout_handler)\n\n if logfile_output:\n if not logfile_output_path:\n raise ValueError(\"Missing required 'logfile_output_path' argument (required when 'logfile_output' is True.\")\n\n if not logfile_output_path.is_dir():\n logger.debug(\"Logs directory could not be found and will be created.\")\n logfile_output_path.mkdir()\n\n logfile_path = logfile_output_path / f\"{PACKAGE_NAME}_{dt.datetime.now().strftime(r'%Y-%m-%d_%H-%M-%S')}.log\"\n\n logfile_handler = logging.FileHandler(filename=logfile_path, encoding=\"utf-8\")\n logfile_handler.setLevel(logfile_loglevel)\n logfile_handler.setFormatter(CustomLogFileFormatter())\n logger.debug(f\"Log file location: '{logfile_path}'\")\n logger.addHandler(logfile_handler)\n\n\nlogger = logging.getLogger(PACKAGE_NAME)\n\n# Temporarily set the logger to INFO level until the config is loaded and the logger is properly set up\nlogger.setLevel(logging.INFO)\nlogger.addHandler(CustomStdoutFormatter(console=console))\n" }, { "path": "isubrip/scrapers/__init__.py", "content": "" }, { "path": "isubrip/scrapers/appletv_scraper.py", "content": "from __future__ import annotations\n\nimport datetime as dt\nfrom enum import Enum\nimport fnmatch\nimport re\nfrom typing import Any\n\nfrom httpx import HTTPError\n\nfrom isubrip.data_structures import Episode, Movie, ScrapedMediaResponse, Season, Series\nfrom isubrip.logger import logger\nfrom isubrip.scrapers.scraper import HLSScraper, ScraperError\nfrom isubrip.subtitle_formats.webvtt import WebVTTSubtitles\nfrom isubrip.utils import convert_epoch_to_datetime, parse_url_params, raise_for_status\n\n\nclass AppleTVScraper(HLSScraper):\n \"\"\"An Apple TV scraper.\"\"\"\n id = \"appletv\"\n name = \"Apple TV\"\n abbreviation = \"ATV\"\n url_regex = re.compile(r\"(?i)(?Phttps?://tv\\.apple\\.com/(?:(?P[a-z]{2})/)?(?Pmovie|episode|season|show)/(?:(?P[\\w\\-%]+)/)?(?Pumc\\.cmc\\.[a-z\\d]{23,25}))(?:\\?(?P.*))?\")\n subtitles_class = WebVTTSubtitles\n is_movie_scraper = True\n is_series_scraper = True\n uses_scrapers = [\"itunes\"]\n default_storefront = \"US\"\n storefronts_mapping = {\n \"AE\": \"143481\", \"AG\": \"143540\", \"AI\": \"143538\", \"AM\": \"143524\", \"AR\": \"143505\", \"AT\": \"143445\", \"AU\": \"143460\",\n \"AZ\": \"143568\", \"BE\": \"143446\", \"BG\": \"143526\", \"BH\": \"143559\", \"BM\": \"143542\", \"BN\": \"143560\", \"BO\": \"143556\",\n \"BR\": \"143503\", \"BS\": \"143539\", \"BW\": \"143525\", \"BY\": \"143565\", \"BZ\": \"143555\", \"CA\": \"143455\", \"CH\": \"143459\",\n \"CL\": \"143483\", \"CO\": \"143501\", \"CR\": \"143495\", \"CV\": \"143580\", \"CY\": \"143557\", \"CZ\": \"143489\", \"DE\": \"143443\",\n \"DK\": \"143458\", \"DM\": \"143545\", \"DO\": \"143508\", \"EC\": \"143509\", \"EE\": \"143518\", \"EG\": \"143516\", \"ES\": \"143454\",\n \"FI\": \"143447\", \"FJ\": \"143583\", \"FM\": \"143591\", \"FR\": \"143442\", \"GB\": \"143444\", \"GD\": \"143546\", \"GH\": \"143573\",\n \"GM\": \"143584\", \"GR\": \"143448\", \"GT\": \"143504\", \"GW\": \"143585\", \"HK\": \"143463\", \"HN\": \"143510\", \"HU\": \"143482\",\n \"ID\": \"143476\", \"IE\": \"143449\", \"IL\": \"143491\", \"IN\": \"143467\", \"IT\": \"143450\", \"JO\": \"143528\", \"JP\": \"143462\",\n \"KH\": \"143579\", \"KN\": \"143548\", \"KR\": \"143466\", \"KY\": \"143544\", \"LA\": \"143587\", \"LB\": \"143497\", \"LK\": \"143486\",\n \"LT\": \"143520\", \"LU\": \"143451\", \"LV\": \"143519\", \"MD\": \"143523\", \"MN\": \"143592\", \"MO\": \"143515\", \"MT\": \"143521\",\n \"MU\": \"143533\", \"MX\": \"143468\", \"MY\": \"143473\", \"MZ\": \"143593\", \"NA\": \"143594\", \"NE\": \"143534\", \"NI\": \"143512\",\n \"NL\": \"143452\", \"NO\": \"143457\", \"NZ\": \"143461\", \"OM\": \"143562\", \"PA\": \"143485\", \"PE\": \"143507\", \"PH\": \"143474\",\n \"PL\": \"143478\", \"PT\": \"143453\", \"PY\": \"143513\", \"QA\": \"143498\", \"RU\": \"143469\", \"SA\": \"143479\", \"SE\": \"143456\",\n \"SG\": \"143464\", \"SI\": \"143499\", \"SK\": \"143496\", \"SV\": \"143506\", \"SZ\": \"143602\", \"TH\": \"143475\", \"TJ\": \"143603\",\n \"TM\": \"143604\", \"TR\": \"143480\", \"TT\": \"143551\", \"TW\": \"143470\", \"UA\": \"143492\", \"UG\": \"143537\", \"US\": \"143441\",\n \"VE\": \"143502\", \"VG\": \"143543\", \"VN\": \"143471\", \"ZA\": \"143472\", \"ZW\": \"143605\",\n }\n\n _api_base_url = \"https://tv.apple.com/api/uts/v3\"\n _api_base_params = {\n \"utscf\": \"OjAAAAAAAAA~\",\n \"caller\": \"web\",\n \"v\": \"84\",\n \"pfm\": \"web\",\n }\n\n class Channel(Enum):\n \"\"\"\n An Enum representing AppleTV channels.\n Value represents the channel ID as used by the API.\n \"\"\"\n APPLE_TV_PLUS = \"tvs.sbd.4000\"\n DISNEY_PLUS = \"tvs.sbd.1000216\"\n ITUNES = \"tvs.sbd.9001\"\n HULU = \"tvs.sbd.10000\"\n MAX = \"tvs.sbd.9050\"\n NETFLIX = \"tvs.sbd.9000\"\n PRIME_VIDEO = \"tvs.sbd.12962\"\n STARZ = \"tvs.sbd.1000308\"\n\n def __init__(self, *args: Any, **kwargs: Any) -> None:\n super().__init__(*args, **kwargs)\n self._storefronts_request_params_cache: dict[str, dict[str, str]] = {}\n\n def _decide_locale(self, preferred_locales: str | list[str], default_locale: str, locales: list[str]) -> str:\n \"\"\"\n Decide which locale to use.\n\n Args:\n preferred_locales (str | list[str]): The preferred locales to use.\n default_locale (str): The default locale to use if there is no match.\n locales (list[str]): The locales to search in.\n\n Returns:\n str: The locale to use.\n \"\"\"\n if isinstance(preferred_locales, str):\n preferred_locales = [preferred_locales]\n\n for locale in preferred_locales:\n if locale in locales:\n return locale.replace(\"_\", \"-\")\n\n if result := fnmatch.filter(locales, \"en_*\"):\n return result[0].replace(\"_\", \"-\")\n\n return default_locale\n\n async def _fetch_api_data(self, storefront_id: str, endpoint: str, additional_params: dict | None = None) -> dict:\n \"\"\"\n Send a request to AppleTV's API and return the JSON response.\n\n Args:\n endpoint (str): The endpoint to send the request to.\n additional_params (dict[str, str]): Additional parameters to send with the request.\n\n Returns:\n dict: The JSON response.\n\n Raises:\n HttpError: If an HTTP error response is received.\n \"\"\"\n request_params = await self._fetch_request_params(storefront_id=storefront_id)\n\n if additional_params:\n request_params.update(additional_params)\n\n response = await self._client.get(url=f\"{self._api_base_url}{endpoint}\", params=request_params)\n\n try:\n raise_for_status(response)\n\n except HTTPError as e:\n if response.status_code == 404:\n raise ScraperError(\n \"Media not found. This could indicate that the provided URL is invalid.\",\n ) from e\n\n raise\n\n response_json: dict = response.json()\n response_data: dict = response_json.get(\"data\", {})\n\n return response_data\n \n async def _fetch_request_params(self, storefront_id: str) -> dict[str, str]:\n \"\"\"\n Fetch from the API request parameters for the given storefront ID.\n Uses caching with `self._storefronts_request_params_cache` for efficiency.\n\n Args:\n storefront_id (str): The ID of the storefront to fetch the request parameters for.\n\n Returns:\n dict: The request parameters for the given storefront ID. If returned from cache, a copy is returned.\n \"\"\"\n if storefront_cached_params := self._storefronts_request_params_cache.get(storefront_id):\n logger.debug(f\"Using cached request parameters for storefront '{storefront_id}':\"\n f\"'{storefront_cached_params}'.\")\n return storefront_cached_params.copy()\n\n configuration_data = await self._get_configuration_data(storefront_id=storefront_id)\n request_params: dict[str, str] = configuration_data[\"applicationProps\"][\"requiredParamsMap\"][\"Default\"]\n default_locale: str = configuration_data[\"applicationProps\"][\"storefront\"][\"defaultLocale\"]\n available_locales: list[str] = configuration_data[\"applicationProps\"][\"storefront\"][\"localesSupported\"]\n\n logger.debug(f\"Available locales for storefront '{storefront_id}': {available_locales}'. \"\n f\"Storefront's default locale: '{default_locale}'.\")\n\n locale = self._decide_locale(\n preferred_locales=[\"en_US\", \"en_GB\"],\n default_locale=default_locale,\n locales=available_locales,\n )\n\n request_params[\"sf\"] = storefront_id\n request_params[\"locale\"] = locale\n\n logger.debug(f\"Using and caching request parameters for storefront '{storefront_id}': {request_params}\")\n self._storefronts_request_params_cache[storefront_id] = request_params.copy()\n\n return request_params\n\n async def _get_configuration_data(self, storefront_id: str) -> dict:\n \"\"\"\n Get configuration data for the given storefront ID.\n\n Args:\n storefront_id (str): The ID of the storefront to get the configuration data for.\n\n Returns:\n dict: Configuration data as returned by the API for the given storefront ID.\n \"\"\"\n logger.debug(f\"Fetching configuration data for storefront '{storefront_id}'...\")\n url = f\"{self._api_base_url}/configurations\"\n\n params = self._api_base_params.copy()\n params[\"sf\"] = storefront_id\n\n response = await self._client.get(url=url, params=params)\n raise_for_status(response)\n logger.debug(\"Configuration data fetched successfully.\")\n\n response_data: dict = response.json()[\"data\"]\n return response_data\n\n def _map_playables_by_channel(self, playables: list[dict]) -> dict[str, dict]:\n \"\"\"\n Map playables by channel name.\n\n Args:\n playables (list[dict]): Playables data to map.\n\n Returns:\n dict: The mapped playables (in a `channel_name (str): [playables]` format).\n \"\"\"\n mapped_playables: dict = {}\n\n for playable in playables:\n if channel_id := playable.get(\"channelId\"):\n mapped_playables.setdefault(channel_id, []).append(playable)\n\n return mapped_playables\n\n async def get_movie_data(self, storefront_id: str, movie_id: str) -> ScrapedMediaResponse[Movie]:\n data = await self._fetch_api_data(\n storefront_id=storefront_id,\n endpoint=f\"/movies/{movie_id}\",\n )\n\n mapped_playables = self._map_playables_by_channel(playables=data[\"playables\"].values())\n logger.debug(f\"Available channels for movie '{movie_id}': \"\n f\"{' '.join(list(mapped_playables.keys()))}\")\n\n if self.Channel.ITUNES.value not in mapped_playables:\n if self.Channel.APPLE_TV_PLUS.value in mapped_playables:\n raise ScraperError(\"Scraping AppleTV+ content is not currently supported.\")\n\n raise ScraperError(\"No iTunes playables could be found.\")\n\n return_data = []\n\n for playable_data in mapped_playables[self.Channel.ITUNES.value]:\n return_data.append(self._extract_itunes_movie_data(playable_data))\n\n if len(return_data) > 1:\n logger.debug(f\"{len(return_data)} iTunes playables were found for movie '{movie_id}'.\")\n\n return ScrapedMediaResponse(\n media_data=return_data,\n metadata_scraper=self.id,\n playlist_scraper=\"itunes\",\n original_data=data,\n )\n\n def _extract_itunes_movie_data(self, playable_data: dict) -> Movie:\n \"\"\"\n Extract movie data from an AppleTV's API iTunes playable data.\n\n Args:\n playable_data (dict): The playable data from the AppleTV API.\n\n Returns:\n Movie: A Movie object.\n \"\"\"\n itunes_movie_id = playable_data[\"itunesMediaApiData\"][\"id\"]\n appletv_movie_id = playable_data[\"canonicalId\"]\n movie_title = playable_data[\"canonicalMetadata\"][\"movieTitle\"]\n movie_release_date = convert_epoch_to_datetime(playable_data[\"canonicalMetadata\"][\"releaseDate\"] // 1000)\n\n movie_playlists = []\n movie_duration = None\n\n if offers := playable_data[\"itunesMediaApiData\"].get(\"offers\"):\n for offer in offers:\n if (playlist := offer.get(\"hlsUrl\")) and offer[\"hlsUrl\"] not in movie_playlists:\n movie_playlists.append(playlist)\n\n if movie_duration_int := offers[0].get(\"durationInMilliseconds\"):\n movie_duration = dt.timedelta(milliseconds=movie_duration_int)\n\n if movie_expected_release_date := playable_data[\"itunesMediaApiData\"].get(\"futureRentalAvailabilityDate\"):\n movie_expected_release_date = dt.datetime.strptime(movie_expected_release_date, \"%Y-%m-%d\")\n\n return Movie(\n id=itunes_movie_id,\n referrer_id=appletv_movie_id,\n name=movie_title,\n release_date=movie_release_date,\n duration=movie_duration,\n preorder_availability_date=movie_expected_release_date,\n playlist=movie_playlists if movie_playlists else None,\n )\n\n async def get_episode_data(self, storefront_id: str, episode_id: str) -> ScrapedMediaResponse[Episode]:\n raise NotImplementedError(\"Series scraping is not currently supported.\")\n\n async def get_season_data(self, storefront_id: str, season_id: str, show_id: str) -> ScrapedMediaResponse[Season]:\n raise NotImplementedError(\"Series scraping is not currently supported.\")\n\n async def get_show_data(self, storefront_id: str, show_id: str) -> ScrapedMediaResponse[Series]:\n raise NotImplementedError(\"Series scraping is not currently supported.\")\n\n async def get_data(self, url: str) -> ScrapedMediaResponse:\n regex_match = self.match_url(url=url, raise_error=True)\n url_data = regex_match.groupdict()\n\n media_type = url_data[\"media_type\"]\n\n if storefront_code := url_data.get(\"country_code\"):\n storefront_code = storefront_code.upper()\n\n else:\n storefront_code = self.default_storefront\n\n media_id = url_data[\"media_id\"]\n\n if storefront_code not in self.storefronts_mapping:\n raise ScraperError(f\"ID mapping for storefront '{storefront_code}' could not be found.\")\n\n storefront_id = self.storefronts_mapping[storefront_code]\n\n if media_type == \"movie\":\n return await self.get_movie_data(storefront_id=storefront_id, movie_id=media_id)\n\n if media_type == \"episode\":\n return await self.get_episode_data(storefront_id=storefront_id, episode_id=media_id)\n\n if media_type == \"season\":\n if (url_params := url_data.get(\"url_params\")) and (show_id := parse_url_params(url_params).get(\"showId\")):\n return await self.get_season_data(storefront_id=storefront_id, season_id=media_id, show_id=show_id)\n\n raise ScraperError(\"Invalid AppleTV URL: Missing 'showId' parameter.\")\n\n if media_type == \"show\":\n return await self.get_show_data(storefront_id=storefront_id, show_id=media_id)\n\n raise ScraperError(f\"Invalid media type '{media_type}'.\")\n" }, { "path": "isubrip/scrapers/itunes_scraper.py", "content": "from __future__ import annotations\n\nimport asyncio\nimport re\nfrom typing import TYPE_CHECKING, Any\n\nfrom isubrip.logger import logger\nfrom isubrip.scrapers.scraper import HLSScraper, ScraperError, ScraperFactory\nfrom isubrip.subtitle_formats.webvtt import WebVTTSubtitles\n\nif TYPE_CHECKING:\n from m3u8.model import Media\n\n from isubrip.data_structures import Movie, ScrapedMediaResponse\n\n\nREDIRECT_MAX_RETRIES = 5\nREDIRECT_SLEEP_TIME = 2\n\nclass ItunesScraper(HLSScraper):\n \"\"\"An iTunes movie data scraper.\"\"\"\n id = \"itunes\"\n name = \"iTunes\"\n abbreviation = \"iT\"\n url_regex = re.compile(r\"(?i)(?Phttps?://itunes\\.apple\\.com/(?:(?P[a-z]{2})/)?(?Pmovie|tv-show|tv-season|show)/(?:(?P[\\w\\-%]+)/)?(?Pid\\d{9,10}))(?:\\?(?P.*))?\")\n subtitles_class = WebVTTSubtitles\n is_movie_scraper = True\n uses_scrapers = [\"appletv\"]\n\n _subtitles_filters = {\n HLSScraper.M3U8Attribute.GROUP_ID.value: [\"subtitles_ak\", \"subtitles_vod-ak-amt.tv.apple.com\"],\n **HLSScraper._subtitles_filters, # noqa: SLF001\n }\n\n def __init__(self, *args: Any, **kwargs: Any) -> None:\n super().__init__(*args, **kwargs)\n self._appletv_scraper = ScraperFactory.get_scraper_instance(\n scraper_id=\"appletv\",\n raise_error=True,\n )\n\n async def get_data(self, url: str) -> ScrapedMediaResponse[Movie]:\n \"\"\"\n Scrape iTunes to find info about a movie, and it's M3U8 main_playlist.\n\n Args:\n url (str): An iTunes store movie URL.\n\n Raises:\n InvalidURL: `itunes_url` is not a valid iTunes store movie URL.\n PageLoadError: HTML page did not load properly.\n HTTPError: HTTP request failed.\n\n Returns:\n Movie: A Movie (NamedTuple) object with movie's name, and an M3U8 object of the main_playlist\n if the main_playlist is found. None otherwise.\n \"\"\"\n regex_match = self.match_url(url, raise_error=True)\n url_data = regex_match.groupdict()\n country_code: str = url_data[\"country_code\"]\n media_id: str = url_data[\"media_id\"]\n appletv_redirect_finding_url = f\"https://tv.apple.com/{country_code}/movie/{media_id}\"\n\n logger.debug(\"Attempting to fetch redirect location from: \" + appletv_redirect_finding_url)\n\n retries = 0\n while True:\n response = await self._client.get(url=appletv_redirect_finding_url, follow_redirects=False)\n if response.status_code != 301 and retries < REDIRECT_MAX_RETRIES:\n retries += 1\n logger.debug(f\"AppleTV redirect URL not found (Response code: {response.status_code}),\"\n f\" retrying... ({retries}/{REDIRECT_MAX_RETRIES})\")\n await asyncio.sleep(REDIRECT_SLEEP_TIME)\n continue\n break\n\n redirect_location = response.headers.get(\"Location\")\n\n if response.status_code != 301 or not redirect_location:\n raise ScraperError(f\"AppleTV redirect URL not found (Response code: {response.status_code}).\")\n\n # Add 'https:' if redirect_location starts with '//'\n if redirect_location.startswith('//'):\n redirect_location = \"https:\" + redirect_location\n\n logger.debug(f\"Redirect URL: {redirect_location}\")\n\n if not self._appletv_scraper.match_url(redirect_location):\n raise ScraperError(\"Redirect URL is not a valid AppleTV URL.\")\n\n return await self._appletv_scraper.get_data(url=redirect_location)\n\n @staticmethod\n def parse_language_name(media_data: Media) -> str | None:\n name: str | None = media_data.name\n\n if name:\n return name.replace(' (forced)', '').strip()\n\n return None\n" }, { "path": "isubrip/scrapers/scraper.py", "content": "from __future__ import annotations\n\nfrom abc import ABC, abstractmethod\nimport asyncio\nfrom enum import Enum\nimport importlib\nimport inspect\nfrom pathlib import Path\nimport re\nimport sys\nfrom typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, overload\n\nimport httpx\nimport m3u8\nfrom pydantic import AliasGenerator, BaseModel, ConfigDict, Field, create_model\n\nfrom isubrip.constants import PACKAGE_NAME, SCRAPER_MODULES_SUFFIX\nfrom isubrip.data_structures import (\n MainPlaylist,\n PlaylistMediaItem,\n ScrapedMediaResponse,\n SubtitlesData,\n SubtitlesFormatType,\n SubtitlesType,\n)\nfrom isubrip.logger import logger\nfrom isubrip.utils import (\n SingletonMeta,\n format_subtitles_description,\n get_model_field,\n merge_dict_values,\n return_first_valid,\n single_string_to_list,\n)\n\nif TYPE_CHECKING:\n\n from isubrip.subtitle_formats.subtitles import Subtitles\n\n\nScraperT = TypeVar(\"ScraperT\", bound=\"Scraper\")\n\n\nclass ScraperConfigBase(BaseModel, ABC):\n \"\"\"\n A Pydantic BaseModel for base class for scraper's configuration classes.\n Also serves for setting default configuration settings for all scrapers.\n\n Attributes:\n timeout (int | float): Timeout to use when making requests.\n user_agent (st): User agent to use when making requests.\n proxy (str | None): Proxy to use when making requests.\n verify_ssl (bool): Whether to verify SSL certificates.\n \"\"\"\n model_config = ConfigDict(\n extra='forbid',\n alias_generator=AliasGenerator(\n validation_alias=lambda field_name: field_name.replace('_', '-'),\n ),\n )\n\n timeout: int | float | None = Field(default=None)\n user_agent: str | None = Field(default=None)\n proxy: str | None = Field(default=None)\n verify_ssl: bool | None = Field(default=None)\n\n\nclass DefaultScraperConfig(ScraperConfigBase):\n \"\"\"\n A Pydantic BaseModel for scraper's configuration classes.\n Also serves as a default configuration for all scrapers.\n\n Attributes:\n timeout (int | float): Timeout to use when making requests.\n user_agent (st): User agent to use when making requests.\n proxy (str | None): Proxy to use when making requests.\n verify_ssl (bool): Whether to verify SSL certificates.\n \"\"\"\n timeout: int | float = Field(default=10)\n user_agent: str = Field(\n default=\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36\", # noqa: E501\n )\n proxy: str | None = Field(default=None)\n verify_ssl: bool = Field(default=True)\n\n\nclass ScraperConfigSubcategory(BaseModel, ABC):\n \"\"\"A Pydantic BaseModel for a scraper's configuration subcategory (which can be set under 'ScraperConfig').\"\"\"\n model_config = ConfigDict(\n extra='forbid',\n alias_generator=AliasGenerator(\n validation_alias=lambda field_name: field_name.replace('_', '-'),\n ),\n )\n\nclass Scraper(ABC, metaclass=SingletonMeta):\n \"\"\"\n A base class for scrapers.\n\n Attributes:\n default_user_agent (str): [Class Attribute]\n Default user agent to use if no other user agent is specified when making requests.\n default_proxy (str | None): [Class Attribute] Default proxy to use when making requests.\n default_verify_ssl (bool): [Class Attribute] Whether to verify SSL certificates by default.\n subtitles_fix_rtl (bool): [Class Attribute] Whether to fix RTL from downloaded subtitles.\n subtitles_remove_duplicates (bool): [Class Attribute]\n Whether to remove duplicate lines from downloaded subtitles.\n\n id (str): [Class Attribute] ID of the scraper (must be unique).\n name (str): [Class Attribute] Name of the scraper.\n abbreviation (str): [Class Attribute] Abbreviation of the scraper.\n url_regex (re.Pattern | list[re.Pattern]): [Class Attribute] A RegEx pattern to find URLs matching the service.\n subtitles_class (type[Subtitles]): [Class Attribute] Class of the subtitles format returned by the scraper.\n is_movie_scraper (bool): [Class Attribute] Whether the scraper is for movies.\n is_series_scraper (bool): [Class Attribute] Whether the scraper is for series.\n uses_scrapers (list[str]): [Class Attribute] A list of IDs for other scraper classes that this scraper uses.\n This assures that the config data for the other scrapers is passed as well.\n config (ScraperConfig | None): [Class Attribute] A ScraperConfig instance for the scraper,\n containing configurations.\n _session (httpx.Client): A synchronous HTTP client session.\n _async_session (httpx.AsyncClient): An asynchronous HTTP client session.\n\n Notes:\n Each scraper implements its own `ScraperConfig` class (which can be overridden and updated),\n inheriting from `ScraperConfigBase`, which sets configurable options for the scraper.\n \"\"\"\n\n class ScraperConfig(ScraperConfigBase):\n \"\"\"A class representing scraper's configuration settings.\n Can be overridden to create a custom configuration with overridden default values,\n and additional settings.\"\"\"\n \n default_timeout: ClassVar[int | float] = 10\n default_user_agent: ClassVar[str] = httpx._client.USER_AGENT # noqa: SLF001\n default_proxy: ClassVar[str | None] = None\n default_verify_ssl: ClassVar[bool] = True\n subtitles_fix_rtl: ClassVar[bool] = False\n subtitles_remove_duplicates: ClassVar[bool] = True\n\n id: ClassVar[str]\n name: ClassVar[str]\n abbreviation: ClassVar[str]\n url_regex: ClassVar[re.Pattern | list[re.Pattern]]\n subtitles_class: ClassVar[type[Subtitles]]\n is_movie_scraper: ClassVar[bool] = False\n is_series_scraper: ClassVar[bool] = False\n uses_scrapers: ClassVar[list[str]] = []\n config: ClassVar[ScraperConfig | None] = None\n\n def __init__(self, timeout: int | float | None = None, user_agent: str | None = None,\n proxy: str | None = None, verify_ssl: bool | None = None):\n \"\"\"\n Initialize a Scraper object.\n\n Args:\n timeout (int | float | None, optional): A timeout to use when making requests. Defaults to None.\n user_agent (str | None, optional): A user agent to use when making requests. Defaults to None.\n proxy (str | None, optional): A proxy to use when making requests. Defaults to None.\n verify_ssl (bool | None, optional): Whether to verify SSL certificates. Defaults to None.\n \"\"\"\n self._timeout = return_first_valid(timeout,\n get_model_field(model=self.config, field='timeout'),\n self.default_timeout,\n raise_error=True)\n self._user_agent = return_first_valid(user_agent,\n get_model_field(model=self.config, field='user_agent'),\n self.default_user_agent,\n raise_error=True)\n self._proxy = return_first_valid(proxy,\n get_model_field(model=self.config, field='proxy'),\n self.default_proxy)\n self._verify_ssl = return_first_valid(verify_ssl,\n get_model_field(model=self.config, field='verify_ssl'),\n self.default_verify_ssl,\n raise_error=True)\n\n if self._timeout != self.default_timeout:\n logger.debug(f\"Initializing '{self.name}' scraper with custom timeout: '{self._timeout}'.\")\n\n if self._user_agent != self.default_user_agent:\n logger.debug(f\"Initializing '{self.name}' scraper with custom user-agent: '{self._user_agent}'.\")\n\n if self._proxy != self.default_proxy:\n logger.debug(f\"Initializing '{self.name}' scraper with proxy: '{self._proxy}'.\")\n\n if self._verify_ssl != self.default_verify_ssl:\n logger.debug(f\"Initializing '{self.name}' scraper with SSL verification set to: '{self._verify_ssl}'.\")\n\n self._requests_counter = 0\n clients_params: dict[str, Any] = {\n \"headers\": {\"User-Agent\": self._user_agent},\n \"verify\": self._verify_ssl,\n \"proxy\": self._proxy,\n \"timeout\": float(self._timeout),\n }\n self._client = httpx.AsyncClient(\n **clients_params,\n event_hooks={\n \"request\": [self._async_increment_requests_counter],\n },\n )\n\n # Update session settings according to configurations\n self._client.headers.update({\"User-Agent\": self._user_agent})\n\n def _increment_requests_counter(self, request: httpx.Request) -> None: # noqa: ARG002\n self._requests_counter += 1\n\n async def _async_increment_requests_counter(self, request: httpx.Request) -> None: # noqa: ARG002\n self._requests_counter += 1\n\n @property\n def requests_count(self) -> int:\n return self._requests_counter\n\n @classmethod\n @overload\n def match_url(cls, url: str, raise_error: Literal[True] = ...) -> re.Match:\n ...\n\n @classmethod\n @overload\n def match_url(cls, url: str, raise_error: Literal[False] = ...) -> re.Match | None:\n ...\n\n @classmethod\n def match_url(cls, url: str, raise_error: bool = False) -> re.Match | None:\n \"\"\"\n Checks if a URL matches scraper's url regex.\n\n Args:\n url (str): A URL to check against the regex.\n raise_error (bool, optional): Whether to raise an error instead of returning None if the URL doesn't match.\n\n Returns:\n re.Match | None: A Match object if the URL matches the regex, None otherwise (if raise_error is False).\n\n Raises:\n ValueError: If the URL doesn't match the regex and raise_error is True.\n \"\"\"\n if isinstance(cls.url_regex, re.Pattern) and (match_result := re.fullmatch(pattern=cls.url_regex, string=url)):\n return match_result\n\n if isinstance(cls.url_regex, list):\n for url_regex_item in cls.url_regex:\n if result := re.fullmatch(pattern=url_regex_item, string=url):\n return result\n\n if raise_error:\n raise ValueError(f\"URL '{url}' doesn't match the URL regex of {cls.name}.\")\n\n return None\n\n async def async_close(self) -> None:\n await self._client.aclose()\n\n @abstractmethod\n async def get_data(self, url: str) -> ScrapedMediaResponse:\n \"\"\"\n Scrape media information about the media on a URL.\n\n Args:\n url (str): A URL to get media information about.\n\n Returns:\n ScrapedMediaResponse: A ScrapedMediaResponse object containing scraped media information.\n \"\"\"\n\n @abstractmethod\n async def download_subtitles(self, media_data: PlaylistMediaItem, subrip_conversion: bool = False) -> SubtitlesData:\n \"\"\"\n Download subtitles from a media object.\n\n Args:\n media_data (PlaylistMediaItem): A media object to download subtitles from.\n subrip_conversion (bool, optional): Whether to convert the subtitles to SubRip format. Defaults to False.\n\n Returns:\n SubtitlesData: A SubtitlesData object containing downloaded subtitles.\n \n Raises:\n SubtitlesDownloadError: If the subtitles failed to download.\n \"\"\"\n\n @abstractmethod\n def find_matching_media(self, main_playlist: MainPlaylist,\n filters: dict[str, str | list[str]] | None = None) -> list:\n \"\"\"\n Find media items that match the given filters in the main playlist (or all media items if no filters are given).\n\n Args:\n main_playlist (MainPlaylist): Main playlist to search for media items in.\n filters (dict[str, str | list[str]] | None, optional): A dictionary of filters to match media items against.\n Defaults to None.\n\n Returns:\n list: A list of media items that match the given filters.\n \"\"\"\n\n @abstractmethod\n def find_matching_subtitles(self, main_playlist: MainPlaylist,\n language_filter: list[str] | None = None) -> list[PlaylistMediaItem]:\n \"\"\"\n Find subtitles that match the given language filter in the main playlist.\n\n Args:\n main_playlist (MainPlaylist): Main playlist to search for subtitles in.\n language_filter (list[str] | None, optional): A list of language codes to filter subtitles by.\n Defaults to None.\n\n Returns:\n list[PlaylistMediaItem]: A list of subtitles media objects that match the given language filter.\n \"\"\"\n\n @abstractmethod\n async def load_playlist(self, url: str | list[str], headers: dict | None = None) -> MainPlaylist | None:\n \"\"\"\n Load a playlist from a URL to a representing object.\n Multiple URLs can be given, in which case the first one that loads successfully will be returned.\n\n Args:\n url (str | list[str]): URL of the M3U8 playlist to load. Can also be a list of URLs (for redundancy).\n headers (dict | None, optional): A dictionary of headers to use when making the request.\n Defaults to None (results in using session's configured headers).\n\n Returns:\n MainPlaylist | None: A playlist object (matching the type), or None if the playlist couldn't be loaded.\n \"\"\"\n\n\n @staticmethod\n @abstractmethod\n def detect_subtitles_type(subtitles_media: PlaylistMediaItem) -> SubtitlesType | None:\n \"\"\"\n Detect the subtitles type (Closed Captions, Forced, etc.) from a media object.\n\n Args:\n subtitles_media (PlaylistMediaItem): Subtitles media object to detect the type of.\n\n Returns:\n SubtitlesType | None: The type of the subtitles, None for regular subtitles.\n \"\"\"\n\n\n @classmethod\n @abstractmethod\n def format_subtitles_description(cls, subtitles_media: PlaylistMediaItem) -> str:\n \"\"\"\n Format a description of the subtitles media object.\n \n Args:\n subtitles_media (PlaylistMediaItem): Subtitles media object to format the description of.\n \n Returns:\n str: A formatted description of the subtitles media object.\n\n Raises:\n ValueError: If minimal required data is missing from the media object.\n \"\"\"\n\n\nclass HLSScraper(Scraper, ABC):\n \"\"\"A base class for HLS (m3u8) scrapers.\"\"\"\n class M3U8Attribute(Enum):\n \"\"\"\n An enum representing all possible M3U8 attributes.\n Names / Keys represent M3U8 Media object attributes (should be converted to lowercase),\n and values represent the name of the key for config usage.\n \"\"\"\n ASSOC_LANGUAGE = \"assoc-language\"\n AUTOSELECT = \"autoselect\"\n CHARACTERISTICS = \"characteristics\"\n CHANNELS = \"channels\"\n DEFAULT = \"default\"\n FORCED = \"forced\"\n GROUP_ID = \"group-id\"\n INSTREAM_ID = \"instream-id\"\n LANGUAGE = \"language\"\n NAME = \"name\"\n STABLE_RENDITION_ID = \"stable-rendition-id\"\n TYPE = \"type\"\n\n default_playlist_filters: ClassVar[dict[str, str | list[str] | None] | None] = None\n\n _subtitles_filters: dict[str, str | list[str]] = {\n M3U8Attribute.TYPE.value: \"SUBTITLES\",\n }\n\n # Resolve mypy errors as mypy doesn't support dynamic models.\n if TYPE_CHECKING:\n PlaylistFiltersSubcategory = ScraperConfigSubcategory\n\n else:\n PlaylistFiltersSubcategory = create_model(\n \"PlaylistFiltersSubcategory\",\n __base__=ScraperConfigSubcategory,\n **{\n m3u8_attribute.value: (str | list[str] | None,\n Field(default=None))\n for m3u8_attribute in M3U8Attribute\n }, # type: ignore[call-overload]\n )\n\n\n class ScraperConfig(Scraper.ScraperConfig):\n playlist_filters: HLSScraper.PlaylistFiltersSubcategory = Field( # type: ignore[valid-type]\n default_factory=lambda: HLSScraper.PlaylistFiltersSubcategory(),\n )\n\n\n def __init__(self, playlist_filters: dict[str, str | list[str] | None] | None = None,\n *args: Any, **kwargs: Any) -> None:\n super().__init__(*args, **kwargs)\n self._playlist_filters = return_first_valid(playlist_filters,\n get_model_field(model=self.config,\n field='playlist_filters',\n convert_to_dict=True,\n exclude_none=True),\n self.default_playlist_filters)\n\n if self._playlist_filters:\n logger.debug(f\"Scraper '{self.name}' initialized with playlist filters: {self._playlist_filters}.\")\n\n @staticmethod\n def parse_language_name(media_data: m3u8.Media) -> str | None:\n \"\"\"\n Parse the language name from an M3U8 Media object.\n Can be overridden in subclasses for normalization.\n\n Args:\n media_data (m3u8.Media): Media object to parse the language name from.\n\n Returns:\n str | None: The language name if found, None otherwise.\n \"\"\"\n name: str | None = media_data.name\n return name\n\n async def load_playlist(self, url: str | list[str], headers: dict[str, str] | None = None) -> m3u8.M3U8 | None:\n _headers = headers or self._client.headers\n result: m3u8.M3U8 | None = None\n\n urls = single_string_to_list(item=url)\n response: httpx.Response | None = None\n\n for idx, url_item in enumerate(urls):\n try:\n logger.debug(f\"Loading M3U8 playlist from {url_item} ({idx + 1} of {len(urls)})\")\n response = await self._client.get(url=url_item, headers=_headers, timeout=5)\n\n if not response.text:\n logger.debug(\"Received an empty response for the playlist.\")\n continue\n\n except Exception as e:\n logger.debug(f\"Failed to load playlist: {e}\")\n continue\n\n if not response:\n raise PlaylistLoadError(\"Failed to load playlists from server.\")\n\n result = m3u8.loads(content=response.text, uri=url_item)\n break\n\n return result\n\n @staticmethod\n def detect_subtitles_type(subtitles_media: m3u8.Media) -> SubtitlesType | None:\n \"\"\"\n Detect the subtitles type (Closed Captions, Forced, etc.) from an M3U8 Media object.\n\n Args:\n subtitles_media (m3u8.Media): Subtitles Media object to detect the type of.\n\n Returns:\n SubtitlesType | None: The type of the subtitles, None for regular subtitles.\n \"\"\"\n if subtitles_media.forced == \"YES\":\n return SubtitlesType.FORCED\n\n if subtitles_media.characteristics is not None and \"public.accessibility\" in subtitles_media.characteristics:\n return SubtitlesType.CC\n\n return None\n\n async def download_subtitles(self, media_data: m3u8.Media, subrip_conversion: bool = False) -> SubtitlesData:\n try:\n playlist_m3u8 = await self.load_playlist(url=media_data.absolute_uri)\n\n if playlist_m3u8 is None:\n raise PlaylistLoadError(\"Could not load subtitles M3U8 playlist.\") # noqa: TRY301\n\n if not media_data.language:\n raise ValueError(\"Language code not found in media data.\") # noqa: TRY301\n\n downloaded_segments = await self.download_segments(playlist=playlist_m3u8)\n subtitles = self.subtitles_class(data=downloaded_segments[0], language_code=media_data.language)\n\n if len(downloaded_segments) > 1:\n for segment_data in downloaded_segments[1:]:\n segment_subtitles_obj = self.subtitles_class(data=segment_data, language_code=media_data.language)\n subtitles.append_subtitles(segment_subtitles_obj)\n\n subtitles.polish(\n fix_rtl=self.subtitles_fix_rtl,\n remove_duplicates=self.subtitles_remove_duplicates,\n )\n\n if subrip_conversion:\n subtitles_format = SubtitlesFormatType.SUBRIP\n content = subtitles.to_srt().dump()\n\n else:\n subtitles_format = SubtitlesFormatType.WEBVTT\n content = subtitles.dump()\n\n return SubtitlesData(\n language_code=media_data.language,\n language_name=self.parse_language_name(media_data=media_data),\n subtitles_format=subtitles_format,\n content=content,\n content_encoding=subtitles.encoding,\n special_type=self.detect_subtitles_type(subtitles_media=media_data),\n )\n \n except Exception as e:\n raise SubtitlesDownloadError(\n language_code=media_data.language,\n language_name=self.parse_language_name(media_data=media_data),\n special_type=self.detect_subtitles_type(subtitles_media=media_data),\n original_exc=e,\n ) from e\n\n async def download_segments(self, playlist: m3u8.M3U8) -> list[bytes]:\n responses = await asyncio.gather(\n *[\n self._client.get(url=segment.absolute_uri)\n for segment in playlist.segments\n ],\n )\n\n responses_data = []\n\n for result in responses:\n try:\n result.raise_for_status()\n responses_data.append(result.content)\n\n except Exception as e:\n raise DownloadError(\"One of the subtitles segments failed to download.\") from e\n\n return responses_data\n\n def find_matching_media(self, main_playlist: m3u8.M3U8,\n filters: dict[str, str | list[str]] | None = None) -> list[m3u8.Media]:\n results: list[m3u8.Media] = []\n playlist_filters: dict[str, str | list[str]] | None\n\n if self._playlist_filters:\n # Merge filtering dictionaries into a single dictionary\n playlist_filters = merge_dict_values(\n *[dict_item for dict_item in (filters, self._playlist_filters)\n if dict_item is not None],\n )\n\n else:\n playlist_filters = filters\n\n for media in main_playlist.media:\n if not playlist_filters:\n results.append(media)\n continue\n\n is_valid = True\n\n for filter_name, filter_value in playlist_filters.items():\n # Skip filter if its value is None\n if filter_value is None:\n continue\n\n try:\n filter_name_enum = HLSScraper.M3U8Attribute(filter_name)\n attribute_value = getattr(media, filter_name_enum.name.lower(), None)\n\n if (attribute_value is None) or (\n isinstance(filter_value, list) and\n attribute_value.casefold() not in (x.casefold() for x in filter_value)\n ) or (\n isinstance(filter_value, str) and filter_value.casefold() != attribute_value.casefold()\n ):\n is_valid = False\n break\n\n except Exception:\n is_valid = False\n\n if is_valid:\n results.append(media)\n\n return results\n\n def find_matching_subtitles(self, main_playlist: m3u8.M3U8,\n language_filter: list[str] | None = None) -> list[m3u8.Media]:\n _filters = self._subtitles_filters\n\n if language_filter:\n _filters[self.M3U8Attribute.LANGUAGE.value] = language_filter\n\n return self.find_matching_media(main_playlist=main_playlist, filters=_filters)\n \n @classmethod\n def format_subtitles_description(cls, subtitles_media: m3u8.Media) -> str:\n return format_subtitles_description(\n language_code=subtitles_media.language,\n language_name=cls.parse_language_name(media_data=subtitles_media),\n special_type=cls.detect_subtitles_type(subtitles_media=subtitles_media),\n )\n\n\nclass ScraperFactory:\n _scraper_classes_cache: list[type[Scraper]] | None = None\n _scraper_instances_cache: dict[type[Scraper], Scraper] = {}\n _currently_initializing: list[type[Scraper]] = [] # Used to prevent infinite recursion\n\n @classmethod\n def get_initialized_scrapers(cls) -> list[Scraper]:\n \"\"\"\n Get a list of all previously initialized scrapers.\n\n Returns:\n list[Scraper]: A list of initialized scrapers.\n \"\"\"\n return list(cls._scraper_instances_cache.values())\n\n @classmethod\n def get_scraper_classes(cls) -> list[type[Scraper]]:\n \"\"\"\n Find all scraper classes in the scrapers directory.\n\n Returns:\n list[Scraper]: A Scraper subclass.\n \"\"\"\n if cls._scraper_classes_cache is not None:\n return cls._scraper_classes_cache\n\n cls._scraper_classes_cache = []\n scraper_modules_paths = Path(__file__).parent.glob(f\"*{SCRAPER_MODULES_SUFFIX}.py\")\n\n for scraper_module_path in scraper_modules_paths:\n sys.path.append(str(scraper_module_path))\n\n module = importlib.import_module(f\"{PACKAGE_NAME}.scrapers.{scraper_module_path.stem}\")\n\n # Find all 'Scraper' subclasses\n for _, obj in inspect.getmembers(module,\n predicate=lambda x: inspect.isclass(x) and issubclass(x, Scraper)):\n # Skip object if it's an abstract or imported from another module\n if not inspect.isabstract(obj) and obj.__module__ == module.__name__:\n cls._scraper_classes_cache.append(obj)\n\n return cls._scraper_classes_cache\n\n @classmethod\n def _get_scraper_instance(cls, scraper_class: type[ScraperT], kwargs: dict | None = None) -> ScraperT:\n \"\"\"\n Initialize and return a scraper instance.\n\n Args:\n scraper_class (type[ScraperT]): A scraper class to initialize.\n kwargs (dict | None, optional): A dictionary containing parameters to pass to the scraper's constructor.\n Defaults to None.\n\n Returns:\n Scraper: An instance of the given scraper class.\n \"\"\"\n logger.debug(f\"Initializing '{scraper_class.name}' scraper...\")\n kwargs = kwargs or {}\n\n if scraper_class not in cls._scraper_instances_cache:\n logger.debug(f\"'{scraper_class.name}' scraper not found in cache, creating a new instance...\")\n\n if scraper_class in cls._currently_initializing:\n raise ScraperError(f\"'{scraper_class.name}' scraper is already being initialized.\\n\"\n f\"Make sure there are no circular dependencies between scrapers.\")\n\n cls._currently_initializing.append(scraper_class)\n\n cls._scraper_instances_cache[scraper_class] = scraper_class(**kwargs)\n cls._currently_initializing.remove(scraper_class)\n\n else:\n logger.debug(f\"Cached '{scraper_class.name}' scraper instance found and will be used.\")\n\n return cls._scraper_instances_cache[scraper_class] # type: ignore[return-value]\n\n @classmethod\n @overload\n def get_scraper_instance(cls, scraper_class: type[ScraperT], scraper_id: str | None = ...,\n url: str | None = ..., kwargs: dict | None = ...,\n raise_error: Literal[True] = ...) -> ScraperT:\n ...\n\n @classmethod\n @overload\n def get_scraper_instance(cls, scraper_class: type[ScraperT], scraper_id: str | None = ...,\n url: str | None = ..., kwargs: dict | None = ...,\n raise_error: Literal[False] = ...) -> ScraperT | None:\n ...\n\n @classmethod\n @overload\n def get_scraper_instance(cls, scraper_class: None = ..., scraper_id: str | None = ...,\n url: str | None = ..., kwargs: dict | None = ...,\n raise_error: Literal[True] = ...) -> Scraper:\n ...\n\n @classmethod\n @overload\n def get_scraper_instance(cls, scraper_class: None = ..., scraper_id: str | None = ...,\n url: str | None = ..., kwargs: dict | None = ...,\n raise_error: Literal[False] = ...) -> Scraper | None:\n ...\n\n @classmethod\n def get_scraper_instance(cls, scraper_class: type[Scraper] | None = None, scraper_id: str | None = None,\n url: str | None = None, kwargs: dict | None = None,\n raise_error: bool = True) -> Scraper | None:\n \"\"\"\n Find, initialize and return a scraper that matches the given URL or ID.\n\n Args:\n scraper_class (type[ScraperT] | None, optional): A scraper class to initialize. Defaults to None.\n scraper_id (str | None, optional): ID of a scraper to initialize. Defaults to None.\n url (str | None, optional): A URL to match a scraper for to initialize. Defaults to None.\n kwargs (dict | None, optional): A dictionary containing parameters to pass to the scraper's constructor.\n Defaults to None.\n raise_error (bool, optional): Whether to raise an error if no scraper was found. Defaults to False.\n\n Returns:\n ScraperT | Scraper | None: An instance of a scraper that matches the given URL or ID,\n None otherwise (if raise_error is False).\n\n Raises:\n ValueError: If no scraper was found and 'raise_error' is True.\n \"\"\"\n if not any((scraper_class, scraper_id, url)):\n raise ValueError(\"At least one of: 'scraper_class', 'scraper_id', or 'url' must be provided.\")\n\n if scraper_class:\n return cls._get_scraper_instance(\n scraper_class=scraper_class,\n kwargs=kwargs,\n )\n\n if scraper_id:\n logger.debug(f\"Searching for a scraper object with ID '{scraper_id}'...\")\n for scraper in cls.get_scraper_classes():\n if scraper.id == scraper_id:\n return cls._get_scraper_instance(\n scraper_class=scraper,\n kwargs=kwargs,\n )\n\n elif url:\n logger.debug(f\"Searching for a scraper object that matches URL '{url}'...\")\n for scraper in cls.get_scraper_classes():\n if scraper.match_url(url) is not None:\n return cls._get_scraper_instance(\n scraper_class=scraper,\n kwargs=kwargs,\n )\n\n error_message = \"No matching scraper was found.\"\n\n if raise_error:\n raise ValueError(error_message)\n\n logger.debug(error_message)\n return None\n\n\nclass ScraperError(Exception):\n pass\n\n\nclass DownloadError(ScraperError):\n pass\n\n\nclass PlaylistLoadError(ScraperError):\n pass\n\n\nclass SubtitlesDownloadError(ScraperError):\n def __init__(self, language_code: str | None, language_name: str | None = None,\n special_type: SubtitlesType | None = None, original_exc: Exception | None = None,\n *args: Any, **kwargs: dict[str, Any]):\n \"\"\"\n Initialize a SubtitlesDownloadError instance.\n\n Args:\n language_code (str | None, optional): Language code of the subtitles that failed to download.\n language_name (str | None, optional): Language name of the subtitles that failed to download.\n special_type (SubtitlesType | None, optional): Type of the subtitles that failed to download.\n original_exc (Exception | None, optional): The original exception that caused the error.\n \"\"\"\n super().__init__(*args, **kwargs)\n self.language_code = language_code\n self.language_name = language_name\n self.special_type = special_type\n self.original_exc = original_exc\n" }, { "path": "isubrip/subtitle_formats/__init__.py", "content": "" }, { "path": "isubrip/subtitle_formats/subrip.py", "content": "from __future__ import annotations\n\nfrom typing import Any\n\nfrom isubrip.data_structures import SubtitlesFormatType\nfrom isubrip.subtitle_formats.subtitles import Subtitles, SubtitlesCaptionBlock\n\n\nclass SubRipCaptionBlock(SubtitlesCaptionBlock):\n \"\"\"A subtitles caption block based on the SUBRIP format.\"\"\"\n def __eq__(self, other: Any) -> bool:\n return isinstance(other, type(self)) and \\\n self.start_time == other.start_time and self.end_time == other.end_time and self.payload == other.payload\n\n def __str__(self) -> str:\n result_str = \"\"\n time_format = \"%H:%M:%S,%f\"\n\n result_str += f\"{self.start_time.strftime(time_format)[:-3]} --> {self.end_time.strftime(time_format)[:-3]}\\n\"\n result_str += f\"{self.payload}\"\n\n return result_str\n\n def to_srt(self) -> SubRipCaptionBlock:\n return self\n\n\nclass SubRipSubtitles(Subtitles[SubRipCaptionBlock]):\n \"\"\"An object representing a SubRip subtitles file.\"\"\"\n format = SubtitlesFormatType.SUBRIP\n\n def _dumps(self) -> str:\n subtitles_str = \"\"\n\n for i, block in enumerate(iterable=self.blocks, start=1):\n subtitles_str += f\"{i}\\n{str(block)}\\n\\n\"\n\n return subtitles_str.rstrip('\\n')\n\n def _loads(self, data: str) -> None:\n raise NotImplementedError(\"SubRip subtitles loading is not supported.\")\n" }, { "path": "isubrip/subtitle_formats/subtitles.py", "content": "from __future__ import annotations\n\nfrom abc import ABC, abstractmethod\nfrom copy import deepcopy\nfrom datetime import time\nfrom typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar\n\nfrom isubrip.constants import RTL_LANGUAGES\nfrom isubrip.logger import logger\n\nif TYPE_CHECKING:\n from isubrip.data_structures import SubtitlesFormatType\n from isubrip.subtitle_formats.subrip import SubRipCaptionBlock, SubRipSubtitles\n\n\nRTL_CONTROL_CHARS = ('\\u200e', '\\u200f', '\\u202a', '\\u202b', '\\u202c', '\\u202d', '\\u202e')\nRTL_CHAR = '\\u202b'\n\nSubtitlesT = TypeVar('SubtitlesT', bound='Subtitles')\nSubtitlesBlockT = TypeVar('SubtitlesBlockT', bound='SubtitlesBlock')\n\n\nclass SubtitlesBlock(ABC):\n \"\"\"\n Abstract base class for subtitles blocks.\n\n Attributes:\n modified (bool): Whether the block has been modified.\n \"\"\"\n\n def __init__(self) -> None:\n self.modified: bool = False\n\n @abstractmethod\n def __copy__(self) -> SubtitlesBlock:\n \"\"\"Create a copy of the block.\"\"\"\n\n @abstractmethod\n def __eq__(self, other: Any) -> bool:\n \"\"\"Check if two objects are equal.\"\"\"\n\n @abstractmethod\n def __str__(self) -> str:\n \"\"\"Return a string representation of the block.\"\"\"\n\n\nclass SubtitlesCaptionBlock(SubtitlesBlock, ABC):\n \"\"\"\n A base class for subtitles caption blocks.\n\n Attributes:\n start_time (time): Start timestamp of the caption block.\n end_time (time): End timestamp of the caption block.\n payload (str): Caption block's payload.\n \"\"\"\n\n def __init__(self, start_time: time, end_time: time, payload: str):\n \"\"\"\n Initialize a new SubtitlesCaptionBlock object.\n\n Args:\n start_time: Start timestamp of the caption block.\n end_time: End timestamp of the caption block.\n payload: Caption block's payload.\n \"\"\"\n super().__init__()\n self.start_time = start_time\n self.end_time = end_time\n self.payload = payload\n\n def __copy__(self) -> SubtitlesCaptionBlock:\n copy = self.__class__(self.start_time, self.end_time, self.payload)\n copy.modified = self.modified\n return copy\n\n def fix_rtl(self) -> None:\n \"\"\"Fix payload's text direction to RTL.\"\"\"\n previous_payload = self.payload\n\n # Remove previous RTL-related formatting\n for char in RTL_CONTROL_CHARS:\n self.payload = self.payload.replace(char, '')\n\n # Add RLM char at the start of every line\n self.payload = RTL_CHAR + self.payload.replace(\"\\n\", f\"\\n{RTL_CHAR}\")\n\n if self.payload != previous_payload:\n self.modified = True\n\n @abstractmethod\n def to_srt(self) -> SubRipCaptionBlock:\n \"\"\"\n Convert WebVTT caption block to SRT caption block.\n\n Returns:\n SubRipCaptionBlock: The caption block in SRT format.\n \"\"\"\n ...\n\n\nclass Subtitles(Generic[SubtitlesBlockT], ABC):\n \"\"\"\n An object representing subtitles, made out of blocks.\n\n Attributes:\n _modified (bool): Whether the subtitles have been modified.\n format (SubtitlesFormatType): [Class Attribute] Format of the subtitles (contains name and file extension).\n language_code (str): Language code of the subtitles.\n blocks (list[SubtitlesBlock]): A list of subtitles blocks that make up the subtitles.\n encoding (str): Encoding of the subtitles.\n raw_data (bytes | None): Raw data of the subtitles.\n \"\"\"\n format: ClassVar[SubtitlesFormatType]\n\n def __init__(self, data: bytes | None, language_code: str, encoding: str = \"utf-8\"):\n \"\"\"\n Initialize a new Subtitles object.\n\n Args:\n data (bytes | None): Raw data of the subtitles.\n language_code (str): Language code of the subtitles.\n encoding (str, optional): Encoding of the subtitles. Defaults to \"utf-8\".\n \"\"\"\n self._modified = False\n self.raw_data = None\n\n self.blocks: list[SubtitlesBlockT] = []\n\n self.language_code = language_code\n self.encoding = encoding\n\n if data:\n self.raw_data = data\n self._load(data=data)\n\n def __add__(self: SubtitlesT, obj: SubtitlesBlockT | SubtitlesT) -> SubtitlesT:\n \"\"\"\n Add a new subtitles block, or append blocks from another subtitles object.\n\n Args:\n obj (SubtitlesBlock | Subtitles): A subtitles block or another subtitles object.\n\n Returns:\n Subtitles: The current subtitles object.\n \"\"\"\n if isinstance(obj, SubtitlesBlock):\n self.add_blocks(obj)\n\n elif isinstance(obj, self.__class__):\n self.append_subtitles(obj)\n\n else:\n logger.warning(f\"Cannot add object of type '{type(obj)}' to '{type(self)}' object. Skipping...\")\n\n return self\n\n def __copy__(self: SubtitlesT) -> SubtitlesT:\n \"\"\"Create a copy of the subtitles object.\"\"\"\n copy = self.__class__(data=None, language_code=self.language_code, encoding=self.encoding)\n copy.raw_data = self.raw_data\n copy.blocks = [block.__copy__() for block in self.blocks]\n copy._modified = self.modified() # noqa: SLF001\n return copy\n\n def __eq__(self, other: Any) -> bool:\n return isinstance(other, type(self)) and self.blocks == other.blocks\n\n def __str__(self) -> str:\n return self.dumps()\n\n def _dump(self) -> bytes:\n \"\"\"\n Dump subtitles object to bytes representing the subtitles.\n\n Returns:\n bytes: The subtitles in a bytes object.\n \"\"\"\n return self._dumps().encode(encoding=self.encoding)\n\n @abstractmethod\n def _dumps(self) -> str:\n \"\"\"\n Dump subtitles object to a string representing the subtitles.\n\n Returns:\n str: The subtitles in a string format.\n \"\"\"\n ...\n\n def _load(self, data: bytes) -> None:\n \"\"\"\n Load and parse subtitles data from bytes.\n\n Args:\n data (bytes): Subtitles data to load.\n \"\"\"\n parsed_data = data.decode(encoding=self.encoding)\n self._loads(data=parsed_data)\n\n @abstractmethod\n def _loads(self, data: str) -> None:\n \"\"\"\n Load and parse subtitles data from a string.\n\n Args:\n data (bytes): Subtitles data to load.\n \"\"\"\n ...\n\n def dump(self) -> bytes:\n \"\"\"\n Dump subtitles to a bytes object representing the subtitles.\n Returns the original raw subtitles data if they have not been modified, and raw data is available.\n\n Returns:\n bytes: The subtitles in a bytes object.\n \"\"\"\n if self.raw_data is not None and not self.modified():\n logger.debug(\"Returning original raw data as subtitles have not been modified.\")\n return self.raw_data\n\n return self._dump()\n\n def dumps(self) -> str:\n \"\"\"\n Dump subtitles to a string representing the subtitles.\n Returns the original raw subtitles data if they have not been modified, and raw data is available.\n\n Returns:\n\n \"\"\"\n if self.raw_data is not None and not self.modified():\n logger.debug(\"Returning original raw data (decoded) as subtitles have not been modified.\")\n return self.raw_data.decode(encoding=self.encoding)\n\n return self._dumps()\n\n def add_blocks(self: SubtitlesT,\n blocks: SubtitlesBlockT | list[SubtitlesBlockT],\n set_modified: bool = True) -> SubtitlesT:\n \"\"\"\n Add a new subtitles block to current subtitles.\n\n Args:\n blocks (SubtitlesBlock | list[SubtitlesBlock]):\n A block object or a list of block objects to append.\n set_modified (bool, optional): Whether to set the subtitles as modified. Defaults to True.\n\n Returns:\n Subtitles: The current subtitles object.\n \"\"\"\n if isinstance(blocks, list):\n if not blocks:\n return self\n\n self.blocks.extend(blocks)\n\n else:\n self.blocks.append(blocks)\n\n if set_modified:\n self._modified = True\n\n return self\n\n def append_subtitles(self: SubtitlesT,\n subtitles: SubtitlesT) -> SubtitlesT:\n \"\"\"\n Append subtitles to an existing subtitles object.\n\n Args:\n subtitles (Subtitles): Subtitles object to append to current subtitles.\n\n Returns:\n Subtitles: The current subtitles object.\n \"\"\"\n if subtitles.blocks:\n self.add_blocks(deepcopy(subtitles.blocks))\n\n if subtitles.modified():\n self._modified = True\n\n return self\n\n def polish(self: SubtitlesT,\n fix_rtl: bool = False,\n remove_duplicates: bool = True,\n ) -> SubtitlesT:\n \"\"\"\n Apply various fixes to subtitles.\n\n Args:\n fix_rtl (bool, optional): Whether to fix text direction of RTL languages. Defaults to False.\n remove_duplicates (bool, optional): Whether to remove duplicate captions. Defaults to True.\n\n Returns:\n Subtitles: The current subtitles object.\n \"\"\"\n fix_rtl = (fix_rtl and self.language_code.split('-')[0] in RTL_LANGUAGES)\n\n if not any((\n fix_rtl,\n remove_duplicates,\n )):\n return self\n\n previous_block: SubtitlesBlockT | None = None\n\n for block in self.blocks:\n if fix_rtl:\n block.fix_rtl()\n\n if remove_duplicates and previous_block is not None and block == previous_block:\n self.blocks.remove(previous_block)\n self._modified = True\n\n previous_block = block\n\n return self\n\n def modified(self) -> bool:\n \"\"\"\n Check if the subtitles have been modified (by checking if any of its blocks have been modified).\n\n Returns:\n bool: True if the subtitles have been modified, False otherwise.\n \"\"\"\n return self._modified or any(block.modified for block in self.blocks)\n\n def to_srt(self) -> SubRipSubtitles:\n \"\"\"\n Convert subtitles to SRT format.\n\n Returns:\n SubRipSubtitles: The subtitles in SRT format.\n \"\"\"\n from isubrip.subtitle_formats.subrip import SubRipSubtitles\n\n subrip_subtitles = SubRipSubtitles(\n data=None,\n language_code=self.language_code,\n encoding=self.encoding,\n )\n subrip_blocks = [block.to_srt() for block in self.blocks if isinstance(block, SubtitlesCaptionBlock)]\n subrip_subtitles.add_blocks(subrip_blocks)\n\n return subrip_subtitles\n\n\ndef split_timestamp(timestamp: str) -> tuple[time, time]:\n \"\"\"\n Split a subtitles timestamp into start and end.\n\n Args:\n timestamp (str): A subtitles timestamp. For example: \"00:00:00.000 --> 00:00:00.000\"\n\n Returns:\n tuple(time, time): A tuple containing start and end times as a datetime object.\n \"\"\"\n # Support ',' character in timestamp's milliseconds (used in SubRip format).\n timestamp = timestamp.replace(',', '.')\n\n start_time, end_time = timestamp.split(\" --> \")\n return time.fromisoformat(start_time), time.fromisoformat(end_time)\n" }, { "path": "isubrip/subtitle_formats/webvtt.py", "content": "from __future__ import annotations\n\nfrom abc import ABCMeta\nfrom copy import deepcopy\nimport re\nfrom typing import TYPE_CHECKING, Any, ClassVar\n\nfrom isubrip.data_structures import SubtitlesFormatType\nfrom isubrip.subtitle_formats.subrip import SubRipCaptionBlock\nfrom isubrip.subtitle_formats.subtitles import RTL_CHAR, Subtitles, SubtitlesBlock, SubtitlesCaptionBlock\nfrom isubrip.utils import split_subtitles_timestamp\n\nif TYPE_CHECKING:\n from datetime import time\n\n\n# WebVTT Documentation:\n# https://www.w3.org/TR/webvtt1/#cues\n# https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#webvtt_cues\n\nbottom_line_alignment_regex = re.compile(r\"line:0+(?:\\.0+)?%\")\n\n\nclass WebVTTBlock(SubtitlesBlock, metaclass=ABCMeta):\n \"\"\"\n Abstract base class for WEBVTT cue blocks.\n \"\"\"\n is_caption_block: bool = False\n\n\nclass WebVTTCaptionBlock(SubtitlesCaptionBlock, WebVTTBlock):\n \"\"\"An object representing a WebVTT caption block.\"\"\"\n subrip_alignment_conversion: ClassVar[bool] = False\n\n is_caption_block: bool = True\n\n def __init__(self, start_time: time, end_time: time, payload: str, settings: str = \"\", identifier: str = \"\"):\n \"\"\"\n Initialize a new object representing a WebVTT caption block.\n\n Args:\n start_time (time): Cue start time.\n end_time (time): Cue end time.\n settings (str): Cue settings.\n payload (str): Cue payload.\n \"\"\"\n super().__init__(start_time=start_time, end_time=end_time, payload=payload)\n self.identifier = identifier\n self.settings = settings\n\n def __copy__(self) -> WebVTTCaptionBlock:\n copy = self.__class__(start_time=self.start_time, end_time=self.end_time, payload=self.payload,\n settings=self.settings, identifier=self.identifier)\n copy.modified = self.modified\n return copy\n\n def to_srt(self) -> SubRipCaptionBlock:\n # Add a {\\an8} tag at the start of the payload if the `line` setting is set to 0%\n if self.subrip_alignment_conversion and WEBVTT_BOTTOM_LINE_ALIGNMENT_REGEX.search(self.settings):\n # If the payload starts with an RTL control char, add the tag after it\n if self.payload.startswith(RTL_CHAR):\n payload = RTL_CHAR + WEBVTT_ALIGN_TOP_TAG + self.payload[len(RTL_CHAR):]\n\n else:\n payload = WEBVTT_ALIGN_TOP_TAG + self.payload\n\n else:\n payload = self.payload\n\n return SubRipCaptionBlock(start_time=self.start_time, end_time=self.end_time, payload=payload)\n\n def __eq__(self, other: Any) -> bool:\n return isinstance(other, type(self)) and \\\n self.start_time == other.start_time and self.end_time == other.end_time and self.payload == other.payload\n\n def __str__(self) -> str:\n result_str = \"\"\n time_format = \"%H:%M:%S.%f\"\n\n # Add identifier (if it exists)\n if self.identifier:\n result_str += f\"{self.identifier}\\n\"\n\n result_str += f\"{self.start_time.strftime(time_format)[:-3]} --> {self.end_time.strftime(time_format)[:-3]}\"\n\n if self.settings:\n result_str += f\" {self.settings}\"\n\n result_str += f\"\\n{self.payload}\"\n\n return result_str\n\n\nclass WebVTTCommentBlock(WebVTTBlock):\n \"\"\"An object representing a WebVTT comment block.\"\"\"\n header = \"NOTE\"\n\n def __init__(self, payload: str, inline: bool = False) -> None:\n \"\"\"\n Initialize a new object representing a WebVTT comment block.\n\n Args:\n payload (str): Comment payload.\n \"\"\"\n super().__init__()\n self.payload = payload\n self.inline = inline\n\n def __copy__(self) -> WebVTTCommentBlock:\n copy = self.__class__(payload=self.payload, inline=self.inline)\n copy.modified = self.modified\n return copy\n\n def __eq__(self, other: Any) -> bool:\n return isinstance(other, type(self)) and self.inline == other.inline and self.payload == other.payload\n\n def __str__(self) -> str:\n if self.inline:\n return f\"{self.header} {self.payload}\"\n\n if self.payload:\n return f\"{self.header}\\n{self.payload}\"\n\n return self.header\n\n\nclass WebVTTStyleBlock(WebVTTBlock):\n \"\"\"An object representing a WebVTT style block.\"\"\"\n header = \"STYLE\"\n\n def __init__(self, payload: str) -> None:\n \"\"\"\n Initialize a new object representing a WebVTT style block.\n\n Args:\n payload (str): Style payload.\n \"\"\"\n super().__init__()\n self.payload = payload\n\n def __copy__(self) -> WebVTTStyleBlock:\n copy = self.__class__(payload=self.payload)\n copy.modified = self.modified\n return copy\n\n def __eq__(self, other: Any) -> bool:\n return isinstance(other, type(self)) and self.payload == other.payload\n\n def __str__(self) -> str:\n return f\"{self.header}\\n{self.payload}\"\n\n\nclass WebVTTRegionBlock(WebVTTBlock):\n \"\"\"An object representing a WebVTT region block.\"\"\"\n header = \"REGION\"\n\n def __init__(self, payload: str) -> None:\n \"\"\"\n Initialize a new object representing a WebVTT region block.\n\n Args:\n payload (str): Region payload.\n \"\"\"\n super().__init__()\n self.payload = payload\n\n def __copy__(self) -> WebVTTRegionBlock:\n copy = self.__class__(payload=self.payload)\n copy.modified = self.modified\n return copy\n\n def __eq__(self, other: Any) -> bool:\n return isinstance(other, type(self)) and self.payload == other.payload\n\n def __str__(self) -> str:\n return f\"{self.header} {self.payload}\"\n\n\nclass WebVTTSubtitles(Subtitles[WebVTTBlock]):\n \"\"\"An object representing a WebVTT subtitles file.\"\"\"\n format = SubtitlesFormatType.WEBVTT\n\n def _dumps(self) -> str:\n \"\"\"\n Dump subtitles to a string representing the subtitles in a WebVTT format.\n\n Returns:\n str: The subtitles in a string using a WebVTT format.\n \"\"\"\n subtitles_str = \"WEBVTT\\n\\n\"\n\n for block in self.blocks:\n subtitles_str += str(block) + \"\\n\\n\"\n\n return subtitles_str.rstrip('\\n')\n\n def _loads(self, data: str) -> None:\n \"\"\"\n Load and parse WebVTT subtitles data from a string.\n\n Args:\n data (bytes): Subtitles data to load.\n \"\"\"\n prev_line: str = \"\"\n lines_iterator = iter(data.splitlines())\n\n for line in lines_iterator:\n # If the line is a timestamp\n if caption_block_regex := re.match(WEBVTT_CAPTION_BLOCK_REGEX, line):\n # If previous line wasn't empty, add it as an identifier\n if prev_line:\n caption_identifier = prev_line\n\n else:\n caption_identifier = \"\"\n\n caption_timestamps = split_subtitles_timestamp(caption_block_regex.group(1))\n caption_settings = caption_block_regex.group(2)\n caption_payload = \"\"\n\n for additional_line in lines_iterator:\n if not additional_line:\n line = additional_line\n break\n\n caption_payload += additional_line + \"\\n\"\n\n caption_payload = caption_payload.rstrip(\"\\n\")\n self.blocks.append(WebVTTCaptionBlock(\n identifier=caption_identifier,\n start_time=caption_timestamps[0],\n end_time=caption_timestamps[1],\n settings=caption_settings,\n payload=caption_payload))\n\n elif comment_block_regex := re.match(WEBVTT_COMMENT_HEADER_REGEX, line):\n comment_payload = \"\"\n inline = False\n\n if comment_block_regex.group(1) is not None:\n comment_payload += comment_block_regex.group(1) + \"\\n\"\n inline = True\n\n for additional_line in lines_iterator:\n if not additional_line:\n line = additional_line\n break\n\n comment_payload += additional_line + \"\\n\"\n\n self.blocks.append(WebVTTCommentBlock(comment_payload.rstrip(\"\\n\"), inline=inline))\n\n elif line.rstrip(' \\t') == WebVTTRegionBlock.header:\n region_payload = \"\"\n\n for additional_line in lines_iterator:\n if not additional_line:\n line = additional_line\n break\n\n region_payload += additional_line + \"\\n\"\n\n self.blocks.append(WebVTTRegionBlock(region_payload.rstrip(\"\\n\")))\n\n elif line.rstrip(' \\t') == WebVTTStyleBlock.header:\n style_payload = \"\"\n\n for additional_line in lines_iterator:\n if not additional_line:\n line = additional_line\n break\n\n style_payload += additional_line + \"\\n\"\n\n self.blocks.append(WebVTTStyleBlock(style_payload.rstrip(\"\\n\")))\n\n prev_line = line\n\n def append_subtitles(self: WebVTTSubtitles,\n subtitles: WebVTTSubtitles) -> WebVTTSubtitles:\n if subtitles.blocks:\n subtitles_copy = deepcopy(subtitles)\n\n # Remove head blocks from the subtitles that will be appended\n subtitles_copy.remove_head_blocks()\n\n self.add_blocks(subtitles_copy.blocks)\n\n if subtitles_copy.modified():\n self._modified = True\n\n return self\n\n def remove_head_blocks(self) -> None:\n \"\"\"\n Remove all head blocks (Style / Region) from the subtitles.\n\n NOTE:\n Comment blocks are removed as well if they are before the first caption block (since they're probably\n related to the head blocks).\n \"\"\"\n for block in self.blocks:\n if isinstance(block, WebVTTCaptionBlock):\n break\n\n if isinstance(block, WebVTTCommentBlock | WebVTTStyleBlock | WebVTTRegionBlock):\n self.blocks.remove(block)\n\n\n# --- Constants ---\nWEBVTT_PERCENTAGE_REGEX = r\"\\d{1,3}(?:\\.\\d+)?%\"\nWEBVTT_CAPTION_TIMINGS_REGEX = \\\n r\"(?:[0-5]\\d:)?[0-5]\\d:[0-5]\\d[\\.,]\\d{3}[ \\t]+-->[ \\t]+(?:[0-5]\\d:)?[0-5]\\d:[0-5]\\d[\\.,]\\d{3}\"\n\nWEBVTT_CAPTION_SETTING_ALIGNMENT_REGEX = r\"align:(?:start|center|middle|end|left|right)\"\nWEBVTT_CAPTION_SETTING_LINE_REGEX = rf\"line:(?:{WEBVTT_PERCENTAGE_REGEX}|-?\\d+%)(?:,(?:start|center|middle|end))?\"\nWEBVTT_CAPTION_SETTING_POSITION_REGEX = rf\"position:{WEBVTT_PERCENTAGE_REGEX}(?:,(?:start|center|middle|end))?\"\nWEBVTT_CAPTION_SETTING_REGION_REGEX = r\"region:(?:(?!(?:-->)|\\t)\\S)+\"\nWEBVTT_CAPTION_SETTING_SIZE_REGEX = rf\"size:{WEBVTT_PERCENTAGE_REGEX}\"\nWEBVTT_CAPTION_SETTING_VERTICAL_REGEX = r\"vertical:(?:lr|rl)\"\n\nWEBVTT_CAPTION_SETTINGS_REGEX = (\"(?:\"\n f\"(?:{WEBVTT_CAPTION_SETTING_ALIGNMENT_REGEX})|\"\n f\"(?:{WEBVTT_CAPTION_SETTING_LINE_REGEX})|\"\n f\"(?:{WEBVTT_CAPTION_SETTING_POSITION_REGEX})|\"\n f\"(?:{WEBVTT_CAPTION_SETTING_REGION_REGEX})|\"\n f\"(?:{WEBVTT_CAPTION_SETTING_SIZE_REGEX})|\"\n f\"(?:{WEBVTT_CAPTION_SETTING_VERTICAL_REGEX})|\"\n f\"(?:[ \\t]+)\"\n \")*\")\n\nWEBVTT_CAPTION_BLOCK_REGEX = re.compile(rf\"^({WEBVTT_CAPTION_TIMINGS_REGEX})[ \\t]*({WEBVTT_CAPTION_SETTINGS_REGEX})?\")\nWEBVTT_COMMENT_HEADER_REGEX = re.compile(rf\"^{WebVTTCommentBlock.header}(?:$|[ \\t])(.+)?\")\nWEBVTT_BOTTOM_LINE_ALIGNMENT_REGEX = re.compile(r\"line:0+(?:\\.0+)?%\")\n\nWEBVTT_ALIGN_TOP_TAG = r\"{\\an8}\"\n" }, { "path": "isubrip/ui.py", "content": "from __future__ import annotations\n\nimport math\nfrom typing import TYPE_CHECKING\n\nfrom rich.progress import TimeElapsedColumn\nfrom rich.text import Text\n\nif TYPE_CHECKING:\n from rich.progress import Task\n\n\nclass MinsAndSecsTimeElapsedColumn(TimeElapsedColumn):\n \"\"\"Renders time elapsed in minutes and seconds.\"\"\"\n\n def render(self, task: Task) -> Text:\n \"\"\"Show time elapsed.\"\"\"\n elapsed = task.finished_time if task.finished else task.elapsed\n\n if elapsed is None:\n return Text(\"-:--\", style=\"progress.elapsed\")\n\n minutes, seconds = divmod(math.ceil(elapsed), 60)\n\n return Text(f\"{minutes:02d}:{seconds:02d}\", style=\"progress.elapsed\")\n" }, { "path": "isubrip/utils.py", "content": "from __future__ import annotations\n\nfrom abc import ABCMeta\nimport datetime as dt\nfrom functools import lru_cache\nimport logging\nfrom pathlib import Path\nimport re\nimport secrets\nimport shutil\nimport sys\nfrom typing import TYPE_CHECKING, Any, Literal, cast, overload\n\nfrom wcwidth import wcswidth\n\nfrom isubrip.constants import WINDOWS_RESERVED_FILE_NAMES, temp_folder_path\nfrom isubrip.data_structures import (\n Episode,\n MediaBase,\n Movie,\n Season,\n Series,\n SubtitlesData,\n SubtitlesFormatType,\n SubtitlesType,\n T,\n)\nfrom isubrip.logger import logger\n\nif TYPE_CHECKING:\n from os import PathLike\n from types import TracebackType\n\n import httpx\n from pydantic import BaseModel, ValidationError\n\n\nclass SingletonMeta(ABCMeta):\n \"\"\"\n A metaclass that implements the Singleton pattern.\n When a class using this metaclass is initialized, it will return the same instance every time.\n \"\"\"\n _instances: dict[object, object] = {}\n\n def __call__(cls, *args: Any, **kwargs: Any) -> object:\n if cls._instances.get(cls) is None:\n cls._instances[cls] = super().__call__(*args, **kwargs)\n\n return cls._instances[cls]\n\n\nclass TemporaryDirectory:\n \"\"\"\n A context manager for creating and managing a temporary directory.\n\n Args:\n directory_name (str | None, optional): Name of the directory to generate.\n If not specified, a random string will be generated. Defaults to None.\n \"\"\"\n def __init__(self, directory_name: str | None = None):\n if directory_name:\n self.directory_name = sanitize_path_segment(directory_name)\n else:\n self.directory_name = secrets.token_hex(5)\n\n self.path = temp_folder_path() / self.directory_name\n\n def __enter__(self) -> Path:\n \"\"\"Create the temporary directory and return its path.\"\"\"\n if self.path.is_dir():\n logger.debug(f\"Temporary directory '{self.path}' already exists. \"\n f\"Emptying directory from all contents...\")\n shutil.rmtree(self.path)\n\n self.path.mkdir(parents=True)\n logger.debug(f\"Temporary directory has been generated: '{self.path}'\")\n return self.path\n\n def __exit__(self, exc_type: type[BaseException] | None,\n exc_val: BaseException | None, exc_tb: TracebackType | None) -> None:\n \"\"\"Clean up the temporary directory.\"\"\"\n self.cleanup()\n\n def cleanup(self) -> None:\n \"\"\"Remove the temporary directory.\"\"\"\n if not self.path.exists():\n return\n\n logger.debug(f\"Removing temporary directory: '{self.path}'\")\n try:\n shutil.rmtree(self.path)\n except Exception as e:\n logger.warning(f\"Failed to remove temporary directory '{self.path}': {e}\")\n\n\ndef convert_epoch_to_datetime(epoch_timestamp: int) -> dt.datetime:\n \"\"\"\n Convert an epoch timestamp to a datetime object.\n\n Args:\n epoch_timestamp (int): Epoch timestamp.\n\n Returns:\n datetime: A datetime object representing the timestamp.\n \"\"\"\n if epoch_timestamp >= 0:\n return dt.datetime.fromtimestamp(epoch_timestamp)\n\n return dt.datetime(1970, 1, 1) + dt.timedelta(seconds=epoch_timestamp)\n\n\ndef convert_log_level(log_level: str) -> int:\n \"\"\"\n Convert a log level string to a logging level.\n\n Args:\n log_level (str): Log level string.\n\n Returns:\n int: Logging level.\n \n Raises:\n ValueError: If the log level is invalid.\n \"\"\"\n log_level_upper = log_level.upper()\n if log_level_upper not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):\n raise ValueError(f\"Invalid log level: {log_level}\")\n\n return cast(\"int\", getattr(logging, log_level_upper))\n\n\ndef download_subtitles_to_file(media_data: Movie | Episode, subtitles_data: SubtitlesData, output_path: str | PathLike,\n source_abbreviation: str | None = None, overwrite: bool = False) -> Path:\n \"\"\"\n Download subtitles to a file.\n\n Args:\n media_data (Movie | Episode): An object containing media data.\n subtitles_data (SubtitlesData): A SubtitlesData object containing subtitles data.\n output_path (str | PathLike): Path to the output folder.\n source_abbreviation (str | None, optional): Abbreviation of the source the subtitles are downloaded from.\n Defaults to None.\n overwrite (bool, optional): Whether to overwrite files if they already exist. Defaults to True.\n\n Returns:\n Path: Path to the downloaded subtitles file.\n\n Raises:\n ValueError: If the path in `output_path` does not exist.\n \"\"\"\n output_path = Path(output_path)\n\n if not output_path.is_dir():\n raise ValueError(f\"Invalid path: {output_path}\")\n\n if isinstance(media_data, Movie):\n file_name = format_release_name(title=media_data.name,\n release_date=media_data.release_date,\n media_source=source_abbreviation,\n language_code=subtitles_data.language_code,\n subtitles_type=subtitles_data.special_type,\n file_format=subtitles_data.subtitles_format)\n else: # isinstance(media_data, Episode):\n file_name = format_release_name(title=media_data.series_name,\n release_date=media_data.release_date,\n season_number=media_data.season_number,\n episode_number=media_data.episode_number,\n episode_name=media_data.episode_name,\n media_source=source_abbreviation,\n language_code=subtitles_data.language_code,\n subtitles_type=subtitles_data.special_type,\n file_format=subtitles_data.subtitles_format)\n\n file_path = output_path / file_name\n\n if file_path.exists() and not overwrite:\n file_path = generate_non_conflicting_path(file_path=file_path)\n\n with file_path.open('wb') as f:\n f.write(subtitles_data.content)\n\n return file_path\n\ndef format_config_validation_error(exc: ValidationError) -> str:\n \"\"\"\n Format a Pydantic ValidationError into a human-readable string.\n\n Args:\n exc (ValidationError): The ValidationError instance containing validation errors.\n\n Returns:\n str: A formatted string describing the validation errors, including the location,\n type, value, and error messages for each invalid field.\n \"\"\"\n validation_errors = exc.errors()\n error_str = \"\"\n\n consolidated_errors: dict[str, dict[str, Any]] = {}\n\n for validation_error in validation_errors:\n value: Any = validation_error['input']\n value_type: str = type(value).__name__\n location: list[str] = [str(item) for item in validation_error['loc']]\n error_msg: str = validation_error['msg']\n\n # When the expected type is a union, Pydantic returns several errors for each type,\n # with the type being the last item in the location list\n if (\n isinstance(location[-1], str) and\n (location[-1].endswith(']') or location[-1] in ('str', 'int', 'float', 'bool'))\n ):\n location.pop()\n\n if len(location) > 1:\n location_str = \".\".join(location)\n\n else:\n location_str = location[0]\n\n if location_str in consolidated_errors:\n consolidated_errors[location_str]['errors'].append(error_msg)\n\n else:\n consolidated_errors[location_str] = {}\n consolidated_errors[location_str]['info'] = {\n \"value\": value,\n \"type\": value_type,\n }\n consolidated_errors[location_str]['errors'] = [error_msg]\n\n for error_loc, error_data in consolidated_errors.items():\n error_type = error_data['info']['type']\n error_value = error_data['info']['value']\n error_str += f\"'{error_loc}' (type: '{error_type}', value: '{error_value}'):\\n\"\n \n for error in error_data['errors']:\n error_str += f\" {error}\\n\"\n\n return error_str\n\n\ndef format_list(items: list[str], width: int = 80) -> str:\n \"\"\"\n Format a list of strings into a grid-like display with dynamic column widths.\n \n The function automatically calculates the optimal number of columns based on the maximum item width \n and the desired total width. It properly handles Unicode characters by using their display width.\n\n Args:\n items (list[str]): List of strings to format\n width (int, optional): Maximum width of the output in characters. Defaults to 80.\n\n Returns:\n str: A formatted string with items arranged in columns\n\n Example:\n >>> items = [\"Item 1\", \"Long Item 2\", \"Item 3\", \"Item 4\"]\n >>> print(format_list(items, width=40))\n Item 1 Long Item 2\n Item 3 Item 4\n \"\"\"\n if not items:\n return \"\"\n \n # Calculate true display width for each item and add spacing\n item_widths = [(s, wcswidth(s)) for s in items]\n column_width = max(width for _, width in item_widths) + 4 # Add spacing between columns\n columns = max(1, width // column_width) # At least one column\n \n # Build rows with proper spacing\n rows = []\n for i in range(0, len(item_widths), columns):\n row_items = item_widths[i:i + columns]\n cols = []\n for text, text_width in row_items:\n padding = \" \" * (column_width - text_width)\n cols.append(f\"{text}{padding}\")\n rows.append(\"\".join(cols).rstrip())\n \n return \"\\n\".join(rows)\n\n\ndef format_media_description(media_data: MediaBase, shortened: bool = False) -> str:\n \"\"\"\n Generate a short description string of a media object.\n\n Args:\n media_data (MediaBase): An object containing media data.\n shortened (bool, optional): Whether to generate a shortened description. Defaults to False.\n\n Returns:\n str: A short description string of the media object.\n \"\"\"\n if isinstance(media_data, Movie):\n release_year = (\n media_data.release_date.year\n if isinstance(media_data.release_date, dt.datetime)\n else media_data.release_date\n )\n description_str = f\"{media_data.name.strip()} [{release_year}]\"\n\n if media_data.id:\n description_str += f\" (ID: {media_data.id})\"\n\n return description_str\n\n if isinstance(media_data, Series):\n description_str = f\"{media_data.series_name.strip()}\"\n\n if media_data.series_release_date:\n if isinstance(media_data.series_release_date, dt.datetime):\n description_str += f\" [{media_data.series_release_date.year}]\"\n\n else:\n description_str += f\" [{media_data.series_release_date}]\"\n\n if media_data.id:\n description_str += f\" (ID: {media_data.id})\"\n\n return description_str\n\n if isinstance(media_data, Season):\n description_str = \"\"\n\n if not shortened:\n description_str = f\"{media_data.series_name.strip()} - \"\n\n description_str += f\"Season {media_data.season_number}\"\n\n if media_data.season_name:\n description_str += f\" - {media_data.season_name.strip()}\"\n\n if media_data.id:\n description_str += f\" (ID: {media_data.id})\"\n\n return description_str\n\n if isinstance(media_data, Episode):\n description_str = \"\"\n\n if not shortened:\n description_str = f\"{media_data.series_name.strip()} - \"\n \n description_str += f\"S{media_data.season_number:02d}E{media_data.episode_number:02d}\"\n\n if media_data.episode_name:\n description_str += f\" - {media_data.episode_name.strip()}\"\n\n if media_data.id:\n description_str += f\" (ID: {media_data.id})\"\n\n return description_str\n\n raise ValueError(f\"Unsupported media type: '{type(media_data)}'\")\n\n\ndef format_release_name(title: str,\n release_date: dt.datetime | int | None = None,\n season_number: int | None = None,\n episode_number: int | None = None,\n episode_name: str | None = None,\n media_source: str | None = None,\n source_type: str | None = \"WEB\",\n additional_info: str | list[str] | None = None,\n language_code: str | None = None,\n subtitles_type: SubtitlesType | None = None,\n file_format: str | SubtitlesFormatType | None = None) -> str:\n \"\"\"\n Format a release name.\n\n Args:\n title (str): Media title.\n release_date (int | None, optional): Release date (datetime), or year (int) of the media. Defaults to None.\n season_number (int | None, optional): Season number. Defaults to None.\n episode_number (int | None, optional): Episode number. Defaults to None.\n episode_name (str | None, optional): Episode name. Defaults to None.\n media_source (str | None, optional): Media source name (full or abbreviation). Defaults to None.\n source_type(str | None, optional): General source type (WEB, BluRay, etc.). Defaults to None.\n additional_info (list[str] | str | None, optional): Additional info to add to the file name. Defaults to None.\n language_code (str | None, optional): Language code. Defaults to None.\n subtitles_type (SubtitlesType | None, optional): Subtitles type. Defaults to None.\n file_format (SubtitlesFormat | str | None, optional): File format to use. Defaults to None.\n\n Returns:\n str: Generated file name.\n \"\"\"\n file_name = slugify_title(title=title, separator=\".\")\n\n if release_date is not None:\n if isinstance(release_date, dt.datetime):\n release_year = release_date.year\n else:\n release_year = release_date\n file_name += f\".{release_year}\"\n\n if season_number is not None and episode_number is not None:\n file_name += f\".S{season_number:02}E{episode_number:02}\"\n\n if episode_name is not None:\n file_name += f\".{slugify_title(title=episode_name, separator='.')}\"\n\n if media_source is not None:\n file_name += f\".{media_source}\"\n\n if source_type is not None:\n file_name += f\".{source_type}\"\n\n if additional_info is not None:\n if isinstance(additional_info, list | tuple):\n additional_info = '.'.join(additional_info)\n file_name += f\".{additional_info}\"\n\n if language_code is not None:\n file_name += f\".{language_code}\"\n\n if subtitles_type is not None:\n file_name += f\".{subtitles_type.value.lower()}\"\n\n sanitized_basename = sanitize_path_segment(file_name)\n\n if file_format is not None:\n if isinstance(file_format, SubtitlesFormatType):\n file_format_str = file_format.value.file_extension\n else:\n file_format_str = file_format.lstrip('.')\n\n return f\"{sanitized_basename}.{file_format_str}\"\n\n return sanitized_basename\n\n\n@lru_cache\ndef format_subtitles_description(language_code: str | None = None, language_name: str | None = None,\n special_type: SubtitlesType | None = None) -> str:\n \"\"\"\n Format a subtitles description using its attributes.\n\n Args:\n language_code (str | None, optional): Language code. Defaults to None.\n language_name (str | None, optional): Language name. Defaults to None.\n special_type (SubtitlesType | None, optional): Subtitles type. Defaults to None.\n\n Returns:\n str: Formatted subtitles description.\n \n Raises:\n ValueError: If neither `language_code` nor `language_name` is provided.\n \"\"\"\n if language_name and language_code:\n language_str = f\"{language_name} ({language_code})\"\n\n elif result := (language_name or language_code):\n language_str = result\n\n else:\n raise ValueError(\"Either 'language_code' or 'language_name' must be provided.\")\n \n if special_type:\n language_str += f\" [{special_type.value}]\"\n\n return language_str\n\n\ndef get_model_field(model: BaseModel | None, field: str, convert_to_dict: bool = False, **kwargs: Any) -> Any:\n \"\"\"\n Get a field from a Pydantic model.\n\n Args:\n model (BaseModel | None): A Pydantic model.\n field (str): Field name to retrieve.\n convert_to_dict (bool, optional): Whether to convert the field value to a dictionary. Defaults to False.\n **kwargs: Additional keyword arguments to pass to the serialization method (`model_dump`).\n Relevant only if `convert_to_dict` is True, and `field_value` is a Pydantic model.\n\n Returns:\n Any: The field value.\n \"\"\"\n if model and hasattr(model, field):\n field_value = getattr(model, field)\n\n if convert_to_dict and hasattr(field_value, 'model_dump'):\n return field_value.model_dump(**kwargs)\n\n return field_value\n\n return None\n\n\ndef generate_non_conflicting_path(file_path: Path, has_extension: bool = True) -> Path:\n \"\"\"\n Generate a non-conflicting path for a file.\n If the file already exists, a number will be added to the end of the file name.\n\n Args:\n file_path (Path): Path to a file.\n has_extension (bool, optional): Whether the name of the file includes file extension. Defaults to True.\n\n Returns:\n Path: A non-conflicting file path.\n \"\"\"\n if isinstance(file_path, str):\n file_path = Path(file_path)\n\n if not file_path.exists():\n return file_path\n\n i = 1\n while True:\n if has_extension:\n new_file_path = file_path.parent / f\"{file_path.stem}-{i}{file_path.suffix}\"\n\n else:\n new_file_path = file_path.parent / f\"{file_path}-{i}\"\n\n if not new_file_path.exists():\n return new_file_path\n\n i += 1\n\n\n\n\ndef merge_dict_values(*dictionaries: dict) -> dict:\n \"\"\"\n A function for merging the values of multiple dictionaries using the same keys.\n If a key already exists, the value will be added to a list of values mapped to that key.\n\n Examples:\n merge_dict_values({'a': 1, 'b': 3}, {'a': 2, 'b': 4}) -> {'a': [1, 2], 'b': [3, 4]}\n merge_dict_values({'a': 1, 'b': 2}, {'a': 1, 'b': [2, 3]}) -> {'a': 1, 'b': [2, 3]}\n\n Note:\n This function support only merging of lists or single items (no tuples or other iterables),\n and without any nesting (lists within lists).\n\n Args:\n *dictionaries (dict): Dictionaries to merge.\n\n Returns:\n dict: A merged dictionary.\n \"\"\"\n _dictionaries: list[dict] = [d for d in dictionaries if d]\n\n if len(_dictionaries) == 0:\n return {}\n\n if len(_dictionaries) == 1:\n return _dictionaries[0]\n\n result: dict = {}\n\n for _dict in _dictionaries:\n for key, value in _dict.items():\n if key in result:\n if isinstance(result[key], list):\n if isinstance(value, list):\n result[key].extend(value)\n else:\n result[key].append(value)\n else:\n if isinstance(value, list):\n result[key] = [result[key], *value]\n else:\n result[key] = [result[key], value]\n else:\n result[key] = value\n\n return result\n\n\ndef raise_for_status(response: httpx.Response) -> None:\n \"\"\"\n Raise an exception if the response status code is invalid.\n Uses 'response.raise_for_status()' internally, with additional logging.\n\n Args:\n response (httpx.Response): A response object.\n \"\"\"\n truncation_threshold = 1500\n\n if not response.is_error:\n return\n\n if len(response.text) > truncation_threshold:\n # Truncate the response as in some cases there could be an unexpected long HTML response\n response_text = response.text[:truncation_threshold].rstrip() + \" \"\n\n else:\n response_text = response.text\n\n logger.debug(f\"Response status code: {response.status_code}\")\n\n if response.headers.get('Content-Type'):\n logger.debug(f\"Response type: {response.headers['Content-Type']}\")\n\n logger.debug(f\"Response text: {response_text}\")\n\n response.raise_for_status()\n\n\ndef parse_url_params(url_params: str) -> dict:\n \"\"\"\n Parse GET parameters from a URL to a dictionary.\n\n Args:\n url_params (str): URL parameters. (e.g. 'param1=value1¶m2=value2')\n\n Returns:\n dict: A dictionary containing the URL parameters.\n \"\"\"\n url_params = url_params.split('?')[-1].rstrip('&')\n params_list = url_params.split('&')\n\n if len(params_list) == 0 or \\\n (len(params_list) == 1 and '=' not in params_list[0]):\n return {}\n\n return {key: value for key, value in (param.split('=') for param in params_list)}\n\n\n@overload\ndef return_first_valid(*values: T | None, raise_error: Literal[True] = ...) -> T:\n ...\n\n\n@overload\ndef return_first_valid(*values: T | None, raise_error: Literal[False] = ...) -> T | None:\n ...\n\n\ndef return_first_valid(*values: T | None, raise_error: bool = False) -> T | None:\n \"\"\"\n Return the first non-None value from a list of values.\n\n Args:\n *values (T): Values to check.\n raise_error (bool, optional): Whether to raise an error if all values are None. Defaults to False.\n\n Returns:\n T | None: The first non-None value, or None if all values are None and `raise_error` is False.\n\n Raises:\n ValueError: If all values are None and `raise_error` is True.\n \"\"\"\n for value in values:\n if value is not None:\n return value\n\n if raise_error:\n raise ValueError(\"No valid value found.\")\n\n return None\n\ndef single_string_to_list(item: str | list[str]) -> list[str]:\n \"\"\"\n Convert a single string to a list containing the string.\n If None is passed, an empty list will be returned.\n\n Args:\n item (str | list[str]): A string or a list of strings.\n\n Returns:\n list[str]: A list containing the string, or an empty list if None was passed.\n \"\"\"\n if item is None:\n return []\n\n if isinstance(item, list):\n return item\n\n return [item]\n\n\ndef split_subtitles_timestamp(timestamp: str) -> tuple[dt.time, dt.time]:\n \"\"\"\n Split a subtitles timestamp into start and end.\n\n Args:\n timestamp (str): A subtitles timestamp. For example: \"00:00:00.000 --> 00:00:00.000\"\n\n Returns:\n tuple(time, time): A tuple containing start and end times as a datetime object.\n \"\"\"\n # Support ',' character in timestamp's milliseconds (used in SubRip format).\n timestamp = timestamp.replace(',', '.')\n\n start_time, end_time = timestamp.split(\" --> \")\n return dt.time.fromisoformat(start_time), dt.time.fromisoformat(end_time)\n\n\n_REMOVED_CHARS_SLUG = str.maketrans(\"\", \"\", '<>():\"?*')\n_REMOVED_CHARS_FS = str.maketrans(\"\", \"\", '<>:\"/\\\\|?*')\n_REPLACE_WITH_SEPARATOR = [\" - \", \" & \", \",\", \":\", \"|\", \"/\", \" \"]\n\n\n@lru_cache\ndef slugify_title(title: str, separator: str = \" \") -> str:\n \"\"\"\n Normalize a title into a slug-like string for creating name components.\n This function is for normalization, not for filesystem safety.\n\n Args:\n title (str): A media title.\n separator (str, optional): A separator to use between words. Defaults to \" \".\n\n Returns:\n str: A slugified title.\n \"\"\"\n title = title.strip()\n title = title.replace(\"…\", \"...\")\n\n # Replace multi-character sequences first\n for item in _REPLACE_WITH_SEPARATOR:\n if separator == \" \" and item in (\" & \", \" - \"):\n continue\n if item == \" & \":\n title = title.replace(item, f\"{separator}&{separator}\")\n else:\n title = title.replace(item, separator)\n\n # Remove invalid characters for a slug\n title = title.translate(_REMOVED_CHARS_SLUG)\n\n # Replace multiple separators with a single one\n if separator:\n title = re.sub(f\"[{re.escape(separator)}]+\", separator, title)\n\n return title\n\n\n@lru_cache\ndef sanitize_path_segment(segment: str, platform: str | None = None) -> str:\n \"\"\"\n Sanitize a file or directory name (path segment) for a given OS.\n\n Args:\n segment (str): The path segment to sanitize (file or directory name).\n platform (str | None, optional): Target platform ('win32', 'linux', 'darwin'). \n Defaults to 'sys.platform' (current platform).\n\n Returns:\n str: A sanitized path segment safe for use on the target filesystem.\n \"\"\"\n if platform is None:\n platform = sys.platform\n\n # Remove characters illegal on all filesystems\n segment = segment.translate(_REMOVED_CHARS_FS)\n\n if platform == \"win32\":\n # On Windows, remove trailing dots and spaces\n segment = segment.rstrip(\". \")\n\n # Handle reserved device names (match first segment before dot)\n base_name, _, _ = segment.partition('.')\n if base_name.upper() in WINDOWS_RESERVED_FILE_NAMES:\n segment = f\"_{segment}\"\n\n # Ensure the segment is not empty\n if not segment:\n return \"_\"\n\n return segment\n" }, { "path": "pyproject.toml", "content": "[project]\nname = \"isubrip\"\nversion = \"2.6.8\"\ndescription = \"A Python package for scraping and downloading subtitles from AppleTV / iTunes movie pages.\"\nauthors = [\n {name = \"Michael Yochpaz\"}\n]\nreadme = \"README.md\"\nkeywords = [\n \"iTunes\",\n \"AppleTV\",\n \"movies\",\n \"subtitles\",\n \"scrape\",\n \"scraper\",\n \"download\",\n \"m3u8\"\n]\nclassifiers = [\n \"Development Status :: 5 - Production/Stable\",\n \"Intended Audience :: End Users/Desktop\",\n \"Operating System :: Microsoft :: Windows\",\n \"Operating System :: MacOS\",\n \"Operating System :: POSIX :: Linux\",\n \"Topic :: Utilities\",\n \"License :: OSI Approved :: MIT License\",\n \"Programming Language :: Python :: 3.10\",\n \"Programming Language :: Python :: 3.11\",\n \"Programming Language :: Python :: 3.12\",\n \"Programming Language :: Python :: 3.13\",\n \"Programming Language :: Python :: 3.14\",\n]\nrequires-python = \">= 3.10\"\ndependencies = [\n \"httpx[http2]>=0.28.1\",\n \"m3u8>=6.0.0\",\n \"pydantic>=2.12.0\",\n \"pydantic-settings>=2.11.0\",\n \"pygments>=2.19.1\", # Used by 'rich'. Specified here as version 2.18 appears to cause issues.\n \"rich>=14.2.0\",\n \"tomli>=2.3.0\",\n \"wcwidth>=0.2.14\",\n]\n\n[project.urls]\nHomepage = \"https://github.com/MichaelYochpaz/iSubRip\"\nRepository = \"https://github.com/MichaelYochpaz/iSubRip\"\nIssues = \"https://github.com/MichaelYochpaz/iSubRip/issues\"\nChangelog = \"https://github.com/MichaelYochpaz/iSubRip/blob/main/CHANGELOG.md\"\n\n[project.scripts]\nisubrip = \"isubrip.__main__:main\"\n\n[build-system]\nrequires = [\"hatchling\"]\nbuild-backend = \"hatchling.build\"\n\n[tool.uv]\ndev-dependencies = [\n \"mypy>=1.14.1\",\n \"pyperf>=2.9.0\",\n \"pytest>=8.4.2\",\n \"ruff>=0.9.3\",\n]\n\n[tool.mypy]\ncheck_untyped_defs = true\ndisallow_untyped_defs = true\nexplicit_package_bases = true\nignore_missing_imports = true\npython_version = \"3.10\"\nwarn_return_any = true\nexclude = [\"tests/mock_data\"]\nplugins = [\"pydantic.mypy\"]\n\n[tool.ruff]\nline-length = 120\ntarget-version = \"py310\"\n\n[tool.ruff.lint]\nselect = [\n \"ARG\",\n \"ASYNC\",\n \"B\",\n \"C4\",\n \"COM\",\n \"E\",\n \"F\",\n \"FA\",\n \"I\",\n \"INP\",\n \"ISC\",\n \"N\",\n \"PIE\",\n \"PGH\",\n \"PT\",\n \"PTH\",\n \"Q\",\n \"RSE\",\n \"RET\",\n \"RUF\",\n \"S\",\n \"SIM\",\n \"SLF\",\n \"T20\",\n \"TCH\",\n \"TID\",\n \"TRY\",\n \"UP\",\n]\nignore = [\n \"C416\",\n \"Q000\",\n \"RUF010\",\n \"RUF012\",\n \"SIM108\",\n \"TD002\",\n \"TD003\",\n \"TRY003\",\n]\nunfixable = [\"ARG\"]\n\n[tool.ruff.lint.per-file-ignores]\n\"tests/*.py\" = [\"S101\"]\n\n[tool.ruff.lint.flake8-tidy-imports]\nban-relative-imports = \"all\"\n\n[tool.ruff.lint.flake8-quotes]\ndocstring-quotes = \"double\"\n\n[tool.ruff.lint.isort]\nforce-sort-within-sections = true\n\n[tool.ruff.lint.pyupgrade]\nkeep-runtime-typing = true\n" }, { "path": "tests/.gitignore", "content": "mock_data/" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/benchmarks/__init__.py", "content": "" }, { "path": "tests/benchmarks/download_benchmark.py", "content": "import pyperf\n\nfrom tests.benchmarks.download_benchmark_module import benchmark\n\n# Note: Run this script with the `-o` flag to output results to a file.\n\ndef main() -> None:\n runner = pyperf.Runner(\n processes=5,\n values=10,\n warmups=1,\n )\n\n runner.bench_async_func(\n name=\"Download Benchmark\",\n func=benchmark,\n )\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tests/benchmarks/download_benchmark_module.py", "content": "import asyncio\nimport logging\nfrom pathlib import Path\nfrom unittest.mock import patch\n\nfrom isubrip.cli import console\nfrom isubrip.commands.download import download\nfrom isubrip.logger import logger, setup_loggers\nfrom isubrip.utils import TemporaryDirectory\nfrom tests.tools.mock_loader import MockLoader\n\n\nasync def benchmark() -> None:\n setup_loggers(\n stdout_loglevel=logging.INFO,\n stdout_console=console,\n logfile_output=False,\n )\n url = \"https://tv.apple.com/il/movie/interstellar/umc.cmc.1vrwat5k1ucm5k42q97ioqyq3\"\n mock_data_path = Path(\"tests/mock_data/appletv/il/umc.cmc.1vrwat5k1ucm5k42q97ioqyq3\")\n mock_loader = MockLoader(mock_data_path, logger=logger)\n\n with TemporaryDirectory() as temp_dir:\n logger.info(f\"Temporary directory created at: {temp_dir}\")\n\n with patch(\"httpx.AsyncClient.send\", side_effect=mock_loader.mock_send_handler):\n await download(url, download_path=temp_dir)\n\n\nif __name__ == \"__main__\":\n asyncio.run(benchmark())\n" }, { "path": "tests/test_utils.py", "content": "import datetime as dt\n\nimport pytest\n\nfrom isubrip.data_structures import Episode, Movie, Season, Series, SubtitlesFormatType, SubtitlesType\nfrom isubrip.utils import (\n format_media_description,\n format_release_name,\n sanitize_path_segment,\n slugify_title,\n)\n\n\nclass TestSlugifyTitle:\n @pytest.mark.parametrize(\n (\"input_title\", \"separator\", \"expected_output\"),\n [\n (\"The Lord of the Rings: The Fellowship of the Ring\", \".\",\n \"The.Lord.of.the.Rings.The.Fellowship.of.the.Ring\"),\n (\"The Lord of the Rings: The Fellowship of the Ring\", \" \",\n \"The Lord of the Rings The Fellowship of the Ring\"),\n\n (\"Once Upon a Time... in Hollywood\", \".\", \"Once.Upon.a.Time.in.Hollywood\"),\n (\"Once Upon a Time... in Hollywood\", \" \", \"Once Upon a Time... in Hollywood\"),\n\n (\"Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb?\", \".\",\n \"Dr.Strangelove.or.How.I.Learned.to.Stop.Worrying.and.Love.the.Bomb\"),\n (\"Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb?\", \" \",\n \"Dr. Strangelove or How I Learned to Stop Worrying and Love the Bomb\"),\n\n (\"Mission: Impossible - The Final Reckoning\", \".\", \"Mission.Impossible.The.Final.Reckoning\"),\n (\"Mission: Impossible - The Final Reckoning\", \" \", \"Mission Impossible - The Final Reckoning\"),\n\n (\"Deadpool & Wolverine\", \".\", \"Deadpool.&.Wolverine\"),\n (\"Deadpool & Wolverine\", \" \", \"Deadpool & Wolverine\"),\n\n (\"50/50\", \".\", \"50.50\"),\n (\"50/50\", \" \", \"50 50\"),\n (\"50 / 50\", \".\", \"50.50\"),\n (\"50 / 50\", \" \", \"50 50\"),\n\n (\"V/H/S\", \".\", \"V.H.S\"),\n (\"V/H/S\", \" \", \"V H S\"),\n\n (\"What If...?\", \".\", \"What.If.\"),\n (\"What If...?\", \" \", \"What If...\"),\n ],\n )\n def test_slugify_title(self, input_title: str, separator: str, expected_output: str) -> None:\n assert slugify_title(title=input_title, separator=separator) == expected_output\n\n\nclass TestSanitizePath:\n @pytest.mark.parametrize(\n (\"name\", \"expected_name\"),\n [\n (\"A/B/C\", \"ABC\"),\n (\"AC\", \"ABC\"),\n ('A\"B\"C', \"ABC\"),\n (\"A:B:C\", \"ABC\"),\n (\"A|B|C\", \"ABC\"),\n (\"A?B?C\", \"ABC\"),\n (\"A*B*C\", \"ABC\"),\n ],\n )\n def test_sanitize_common_illegal_chars(self, name: str, expected_name: str) -> None:\n assert sanitize_path_segment(name) == expected_name\n\n @pytest.mark.parametrize(\n (\"name\", \"expected_unix\", \"expected_windows\"),\n [\n (\"name.\", \"name.\", \"name\"),\n (\"name..\", \"name..\", \"name\"),\n (\"name \", \"name \", \"name\"),\n (\" name\", \" name\", \" name\"),\n (\"name. \", \"name. \", \"name\"),\n (\"COM1\", \"COM1\", \"_COM1\"),\n (\"Con.Air\", \"Con.Air\", \"_Con.Air\"),\n (\"aux.txt\", \"aux.txt\", \"_aux.txt\"),\n (\"\", \"_\", \"_\"),\n ],\n )\n def test_sanitize_platform_specific(self, name: str, expected_unix: str, expected_windows: str) -> None:\n # Test Unix-like behavior\n assert sanitize_path_segment(name, platform='linux') == expected_unix\n\n # Test Windows behavior\n assert sanitize_path_segment(name, platform='win32') == expected_windows\n\n\nclass TestFormatMediaDescription:\n def test_movie_with_datetime_and_id(self) -> None:\n movie = Movie(name=\"Inception\", release_date=dt.datetime(2010, 7, 16), id=\"ID123\")\n assert format_media_description(media_data=movie) == \"Inception [2010] (ID: ID123)\"\n\n def test_series_with_year_no_id(self) -> None:\n series = Series(series_name=\"The Office\", series_release_date=2005)\n assert format_media_description(media_data=series) == \"The Office [2005]\"\n\n def test_season_full_with_name_and_id(self) -> None:\n season = Season(series_name=\"True Detective\", series_release_date=2024,\n season_number=4, season_name=\"Night Country\", id=\"ID321\")\n assert format_media_description(media_data=season) == \"True Detective - Season 4 - Night Country (ID: ID321)\"\n\n def test_season_full_with_name_and_id_and_extra_spaces(self) -> None:\n season = Season(series_name=\" True Detective \", series_release_date=2024,\n season_number=4, season_name=\" Night Country \", id=\"ID321\")\n assert format_media_description(media_data=season) == \"True Detective - Season 4 - Night Country (ID: ID321)\"\n\n def test_season_shortened_no_id(self) -> None:\n season = Season(series_name=\"Stranger Things\", season_number=3)\n assert format_media_description(media_data=season, shortened=True) == \"Season 3\"\n\n def test_episode_full_with_name_and_id(self) -> None:\n ep = Episode(series_name=\"Breaking Bad\",\n season_number=5, episode_number=14, episode_name=\"Ozymandias\", id=\"ID111\")\n assert format_media_description(media_data=ep) == \"Breaking Bad - S05E14 - Ozymandias (ID: ID111)\"\n\n def test_episode_shortened_no_id(self) -> None:\n ep = Episode(series_name=\"Breaking Bad\",\n season_number=5, episode_number=14, episode_name=\"Ozymandias\", id=\"ID111\")\n assert format_media_description(media_data=ep, shortened=True) == \"S05E14 - Ozymandias (ID: ID111)\"\n\n\nclass TestFormatReleaseName:\n def test_movie_with_source_and_web_default(self) -> None:\n assert format_release_name(\n title=\"Interstellar\",\n release_date=2014,\n media_source=\"iT\",\n ) == \"Interstellar.2014.iT.WEB\"\n\n def test_movie_with_source_type_none(self) -> None:\n assert format_release_name(\n title=\"Interstellar\",\n release_date=2014,\n media_source=\"iT\",\n source_type=None,\n ) == \"Interstellar.2014.iT\"\n\n def test_episode_with_source_web(self) -> None:\n assert format_release_name(\n title=\"Breaking Bad\",\n season_number=5,\n episode_number=14,\n media_source=\"iT\",\n ) == \"Breaking.Bad.S05E14.iT.WEB\"\n\n def test_episode_with_name_included(self) -> None:\n assert format_release_name(\n title=\"Breaking Bad\",\n season_number=1,\n episode_number=1,\n episode_name=\"Pilot\",\n media_source=\"iT\",\n ) == \"Breaking.Bad.S01E01.Pilot.iT.WEB\"\n\n def test_additional_info_language_and_subtitles_type_and_format_enum(self) -> None:\n assert format_release_name(\n title=\"Interstellar\",\n release_date=2014,\n media_source=\"iT\",\n additional_info=[\"HDR\", \"DV\"],\n language_code=\"en\",\n subtitles_type=SubtitlesType.FORCED,\n file_format=SubtitlesFormatType.SUBRIP,\n ) == \"Interstellar.2014.iT.WEB.HDR.DV.en.forced.srt\"\n\n def test_movie_zip_with_source_and_web(self) -> None:\n assert format_release_name(\n title=\"Interstellar\",\n release_date=2014,\n media_source=\"iT\",\n file_format=\"zip\",\n ) == \"Interstellar.2014.iT.WEB.zip\"\n" }, { "path": "tests/tools/__init__.py", "content": "" }, { "path": "tests/tools/generate_mock_data.py", "content": "from __future__ import annotations\n\nfrom abc import ABC, abstractmethod\nimport argparse\nimport asyncio\nimport hashlib\nimport inspect\nimport json\nimport logging\nfrom pathlib import Path\nimport shutil\nimport typing\nfrom typing import TYPE_CHECKING, Any, ClassVar\n\nfrom isubrip.cli import console\nfrom isubrip.logger import logger, setup_loggers\nfrom isubrip.scrapers.scraper import HLSScraper, PlaylistLoadError, Scraper, ScraperFactory\n\nif TYPE_CHECKING:\n import httpx\n import m3u8\n\n from isubrip.scrapers.appletv_scraper import AppleTVScraper\n\n\nsetup_loggers(\n stdout_loglevel=logging.DEBUG,\n stdout_console=console,\n logfile_output=False,\n)\n\nMOCK_GENERATOR_MAPPING: dict[str, type[MockDataGenerator]] = {}\n\n\nclass MockDataGenerator(ABC):\n \"\"\"Abstract base class for mock data generators.\"\"\"\n MOCK_DATA_ROOT: ClassVar[Path] = Path(__file__).parent.parent / \"mock_data\"\n\n def __init_subclass__(cls, **kwargs: Any) -> None:\n \"\"\"Automatically register mock data generator subclasses.\"\"\"\n super().__init_subclass__(**kwargs)\n\n if not inspect.isabstract(cls):\n type_hints = typing.get_type_hints(cls)\n scraper_class = type_hints.get(\"scraper\")\n\n if scraper_class and hasattr(scraper_class, \"id\"):\n MOCK_GENERATOR_MAPPING[scraper_class.id] = cls\n\n else:\n logger.warning(\n f\"MockDataGenerator subclass '{cls.__name__}' does not have a valid scraper class defined.\")\n\n def __init__(self, scraper: Scraper):\n self.scraper = scraper\n\n @abstractmethod\n def output_path(self, url: str) -> Path:\n \"\"\"\n Generate a relative path for the output file, based on the URL.\n\n Args:\n url: The URL to get the output directory for.\n\n Returns:\n A relative Path object for the output file.\n \"\"\"\n\n async def generate(self, url: str, languages: list[str] | None = None, force: bool = False) -> None:\n \"\"\"\n Generate mock data for a given URL.\n\n Args:\n url: The URL to generate mock data for.\n languages: A list of languages to download subtitles for.\n force: If True, delete existing mock data before generating new data.\n \"\"\"\n output_dir = self.MOCK_DATA_ROOT / self.output_path(url)\n manifest_path = output_dir / \"manifest.json\"\n\n if output_dir.is_dir() and manifest_path.is_file():\n if force:\n logger.info(f\"Removing existing mock data at: {output_dir}\")\n shutil.rmtree(output_dir)\n else:\n logger.error(f\"Mock data already exists at: {output_dir}. Use the --force flag to overwrite.\")\n return\n\n output_dir.mkdir(parents=True, exist_ok=True)\n manifest: dict[str, str] = {}\n\n # Define a hook to save responses\n async def save_response_hook(response: httpx.Response) -> None:\n await response.aread()\n url = str(response.request.url)\n\n if url in manifest:\n logger.debug(f\"URL already processed, skipping: {url}\")\n return\n\n logger.info(f\"Intercepted response from: {url}\")\n filename = hashlib.sha256(url.encode('utf-8')).hexdigest()\n file_path = output_dir / filename\n file_path.write_bytes(response.content)\n manifest[url] = filename\n logger.debug(f\"Saved response to: {file_path}\")\n\n try:\n # Attach the hook to the scraper's HTTP client\n self.scraper._client.event_hooks['response'].append(save_response_hook) # noqa: SLF001\n\n logger.info(\"Fetching media data...\")\n scraped_data = await self.scraper.get_data(url=url)\n\n if not scraped_data or not scraped_data.media_data:\n logger.error(f\"Could not retrieve media data for {url}\")\n return\n\n media_item = scraped_data.media_data[0]\n playlist_url = getattr(media_item, 'playlist', None)\n\n if not playlist_url:\n logger.error(f\"No playlist URL found in scraped data for {url}\")\n return\n\n main_playlist_url = playlist_url[0] if isinstance(playlist_url, list) else playlist_url\n logger.info(f\"Loading main playlist from: {main_playlist_url}\")\n main_playlist: m3u8.M3U8 | None = await self.scraper.load_playlist(url=main_playlist_url)\n\n if not main_playlist:\n logger.error(f\"Could not load main playlist from {main_playlist_url}\")\n return\n\n logger.info(f\"Searching for subtitle playlists (Languages: {languages or 'all'})...\")\n subtitle_media_items: list[m3u8.Media] = self.scraper.find_matching_subtitles(\n main_playlist, language_filter=languages)\n\n if not subtitle_media_items:\n logger.warning(f\"No matching subtitles found for languages: {languages or 'all'}\")\n return\n\n for sub_media in subtitle_media_items:\n try:\n logger.info(f\"Loading subtitle playlist: {sub_media.absolute_uri}\")\n subtitle_playlist: m3u8.M3U8 | None = await self.scraper.load_playlist(\n url=sub_media.absolute_uri)\n if subtitle_playlist and subtitle_playlist.segments:\n logger.info(f\"Downloading {len(subtitle_playlist.segments)} segments...\")\n if isinstance(self.scraper, HLSScraper):\n await self.scraper.download_segments(subtitle_playlist)\n\n except PlaylistLoadError as e:\n logger.error(f\"Failed to load subtitle playlist {sub_media.absolute_uri}: {e}\")\n except Exception:\n logger.error(f\"Unexpected error processing playlist {sub_media.absolute_uri}\", exc_info=True)\n\n except Exception as e:\n logger.error(f\"An error occurred during data generation: {e}\", exc_info=True)\n\n finally:\n if manifest:\n manifest_path = output_dir / \"manifest.json\"\n manifest_path.write_text(json.dumps(manifest, indent=4, sort_keys=True), encoding=\"utf-8\")\n logger.info(f\"Successfully generated {len(manifest)} mock data entries.\")\n logger.info(f\"Manifest written to: {manifest_path.resolve()}\")\n\n\nclass AppleTVMockDataGenerator(MockDataGenerator):\n \"\"\"A mock data generator for the Apple TV scraper.\"\"\"\n scraper: AppleTVScraper\n\n def output_path(self, url: str) -> Path:\n url_regex_match: dict[str, Any] = self.scraper.match_url(url=url, raise_error=True).groupdict()\n media_id: str = url_regex_match[\"media_id\"]\n storefront: str = url_regex_match[\"country_code\"]\n return Path(self.scraper.id) / storefront / media_id\n\n\nasync def main() -> None:\n parser = argparse.ArgumentParser(\n description=\"Generate mock HTTP response data for iSubRip testing. \"\n \"This script runs a scraper against a live URL and saves all \"\n \"HTTP responses to be used in offline tests.\",\n )\n parser.add_argument(\n \"-u\", \"--url\",\n required=True,\n metavar=\"URL\",\n help=\"The full URL of the media item to fetch data for.\",\n )\n parser.add_argument(\n \"-l\", \"--languages\",\n nargs=\"*\",\n default=None,\n help=\"Optional: language code(s) to filter subtitles (e.g., 'en', 'es'). Fetches all if not set.\",\n )\n parser.add_argument(\n \"-f\", \"--force\",\n action=\"store_true\",\n help=\"Force regeneration of mock data, deleting any existing data.\",\n )\n args = parser.parse_args()\n\n scraper = ScraperFactory.get_scraper_instance(url=args.url, raise_error=True)\n\n if not scraper:\n return\n\n if generator_class := MOCK_GENERATOR_MAPPING.get(scraper.id):\n generator = generator_class(scraper=scraper)\n\n logger.info(f\"Starting mock data generation for URL: {args.url}\")\n await generator.generate(url=args.url, languages=args.languages, force=args.force)\n logger.info(\"Mock data generation process complete.\")\n\n else:\n logger.error(f\"No mock data generator found for scraper: {scraper.id}\")\n\n await scraper.async_close()\n\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n" }, { "path": "tests/tools/mock_loader.py", "content": "from __future__ import annotations\n\nimport json\nimport logging\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\n\n\nclass MockLoader:\n \"\"\"\n An asynchronous mock loader for HTTP requests, designed to load mock data from a specified directory.\n \"\"\"\n def __init__(self, mock_data_dir: Path | str, logger: logging.Logger | None = None):\n self.mock_data_dir = Path(mock_data_dir)\n self.logger = logger or logging.getLogger(__name__)\n self._manifest: dict[str, Path] = {}\n self.logger.info(f\"Mock data initialized with using the data on: {self.mock_data_dir}\")\n\n manifest_paths = list(self.mock_data_dir.rglob(\"manifest.json\"))\n\n if not manifest_paths:\n raise FileNotFoundError(f\"No manifest file was found in {self.mock_data_dir}.\")\n\n for manifest_path in manifest_paths:\n self.logger.info(f\"Loading manifest from {manifest_path}...\")\n\n try:\n manifest_data = json.loads(manifest_path.read_text(encoding=\"utf-8\"))\n \n for url, filename in manifest_data.items():\n self._manifest[url] = manifest_path.parent / filename\n\n self.logger.info(f\"Manifest {manifest_path} loaded successfully.\")\n\n except json.JSONDecodeError:\n self.logger.exception(f\"Failed to decode manifest file: {manifest_path}\")\n\n except Exception:\n self.logger.exception(f\"Failed to load manifest file: {manifest_path}\", exc_info=True)\n\n if not self._manifest:\n raise ValueError(\"No valid mock data found in any manifest files.\")\n\n self.logger.info(f\"Loaded {len(self._manifest)} mock data entries from {len(manifest_paths)} manifests.\")\n\n async def mock_send_handler(self, request: httpx.Request, *args: Any, **kwargs: Any) -> httpx.Response: # noqa: ARG002\n \"\"\"An async handler to be used as a side_effect for a patched httpx.AsyncClient.send.\"\"\"\n url_str = str(request.url)\n\n if url_str not in self._manifest:\n raise KeyError(f\"Mock data not found for URL: {url_str}\")\n\n response_path = self._manifest[url_str]\n response_content = response_path.read_bytes()\n\n return httpx.Response(\n status_code=200,\n content=response_content,\n request=request,\n )\n" } ]