Repository: guilatrova/gracy Branch: main Commit: 1eee02d9d765 Files: 54 Total size: 239.4 KB Directory structure: gitextract_x0fjf4du/ ├── .flake8 ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .gracy/ │ └── pokeapi.sqlite3 ├── .pre-commit-config.yaml ├── .vscode/ │ └── settings.json ├── CHANGELOG.md ├── LICENSE ├── README.md ├── docker-compose.yml ├── examples/ │ ├── httpbin_post.py │ ├── memory.py │ ├── pokeapi.py │ ├── pokeapi_limit_concurrency.py │ ├── pokeapi_namespaces.py │ ├── pokeapi_replay.py │ ├── pokeapi_replay_mongo.py │ ├── pokeapi_retry.py │ ├── pokeapi_throttle.py │ └── pokestarwarsapi.py ├── pyproject.toml ├── src/ │ ├── gracy/ │ │ ├── __init__.py │ │ ├── _configs.py │ │ ├── _core.py │ │ ├── _general.py │ │ ├── _loggers.py │ │ ├── _models.py │ │ ├── _paginator.py │ │ ├── _reports/ │ │ │ ├── _builders.py │ │ │ ├── _models.py │ │ │ └── _printers.py │ │ ├── _types.py │ │ ├── _validators.py │ │ ├── common_hooks.py │ │ ├── exceptions.py │ │ ├── py.typed │ │ └── replays/ │ │ ├── _wrappers.py │ │ └── storages/ │ │ ├── _base.py │ │ ├── _sqlite_schema.py │ │ ├── pymongo.py │ │ └── sqlite.py │ └── tests/ │ ├── conftest.py │ ├── generate_test_db.py │ ├── test_generators.py │ ├── test_gracy_httpx.py │ ├── test_hooks.py │ ├── test_loggers.py │ ├── test_namespaces.py │ ├── test_parsers.py │ ├── test_retry.py │ └── test_validators.py └── todo.md ================================================ FILE CONTENTS ================================================ ================================================ FILE: .flake8 ================================================ [flake8] max-line-length=120 extend-ignore=E203 exclude= __init__.py ================================================ FILE: .github/FUNDING.yml ================================================ github: guilatrova custom: https://www.paypal.com/donate/?business=SUQKVABPUHUUQ&no_recurring=0&item_name=Thank+you+very+much+for+considering+supporting+my+work.+%E2%9D%A4%EF%B8%8F+It+keeps+me+motivated+to+keep+producing+value+for+you.¤cy_code=USD ================================================ FILE: .github/workflows/ci.yml ================================================ name: CI on: push: paths-ignore: - "docs/**" - "*.md" pull_request: paths-ignore: - "docs/**" - "*.md" jobs: test: # We want to run on external PRs, but not on our own internal PRs as they'll be run # by the push to the branch. Without this if check, checks are duplicated since # internal PRs match both the push and pull_request events. if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository strategy: fail-fast: false matrix: python-version: [3.8, 3.9, "3.10", 3.11] os: [ubuntu-latest] #, macOS-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - name: Install poetry shell: bash run: pipx install poetry - name: Set up Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: poetry # key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }} - name: Install dependencies run: poetry install --with=dev - name: Lint uses: pre-commit/action@v3.0.0 - name: Unit tests run: poetry run pytest -vvv ================================================ FILE: .github/workflows/release.yml ================================================ name: Semantic Release on: push: branches: - main jobs: release: runs-on: ubuntu-latest concurrency: release steps: - uses: actions/checkout@v2 with: fetch-depth: 0 token: ${{ secrets.GRACY_GITHUB_TOKEN }} - name: Install poetry shell: bash run: pipx install poetry - name: Set up Python 3.10 uses: actions/setup-python@v4 with: python-version: "3.10" cache: poetry - name: Install dependencies run: poetry install --with=dev - name: Python Semantic Release run: | git config --global user.name "github-actions" git config --global user.email "action@github.com" poetry run semantic-release publish -D commit_author="github-actions " env: GH_TOKEN: ${{secrets.GRACY_GITHUB_TOKEN}} PYPI_TOKEN: ${{secrets.PYPI_TOKEN}} ================================================ FILE: .gitignore ================================================ *.egg-info *.pyc *.log .DS_Store dist/ .mongo .venv/ ================================================ FILE: .pre-commit-config.yaml ================================================ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.5 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - id: ruff-format - repo: local hooks: - id: pyright name: pyright language: system types: [python] entry: "poetry run pyright" require_serial: true # use require_serial so that script is only called once per commit verbose: true # print the number of files as a sanity-check ================================================ FILE: .vscode/settings.json ================================================ { "python.analysis.typeCheckingMode": "basic", "editor.formatOnSave": true, "[python]": { "editor.rulers": [ 120 ], "editor.formatOnSave": true, "editor.defaultFormatter": "charliermarsh.ruff", "editor.codeActionsOnSave": { "source.fixAll.ruff": "explicit", "source.organizeImports.ruff": "explicit" }, }, "python.testing.pytestArgs": [ "src/tests" ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "files.exclude": { "**/.git": true, "**/.svn": true, "**/.hg": true, "**/CVS": true, "**/__pycache__": true, ".mypy_cache": true, ".pytest_cache": true, "**/.DS_Store": true, "**/Thumbs.db": true, ".venv/**": true, } } ================================================ FILE: CHANGELOG.md ================================================ # Changelog ## v1.34.0 (2024-11-27) ### Feature * Garbage collector improvements ([`36e537e`](https://github.com/guilatrova/gracy/commit/36e537ec85fd12c273a74945cf7962cbdd7494bd)) ## v1.33.1 (2024-10-07) ### Fix * Don't stop if ctx token reset fails ([`4ddad79`](https://github.com/guilatrova/gracy/commit/4ddad79627a5fd528a199d8bf6e58906d75cbec1)) ## v1.33.0 (2024-02-20) ### Feature * Enhance response typing ([`95f8c16`](https://github.com/guilatrova/gracy/commit/95f8c162de7ab64a925bf008e81e11b138426320)) ### Fix * Add typing_extensions to support deprecated ([`f6b2b7b`](https://github.com/guilatrova/gracy/commit/f6b2b7b1baf7a39c5f6f21cced7aeeeac23cbfe7)) ### Documentation * Update on dynamic type hinting ([`6d7975d`](https://github.com/guilatrova/gracy/commit/6d7975d1b383d040f2c960d1dc13482530b81571)) ## v1.32.2 (2024-01-31) ### Fix * Define correct typing ([`7c889fe`](https://github.com/guilatrova/gracy/commit/7c889fe782e7705843d84ec6dc1b3f0098e779e0)) ## v1.32.1 (2024-01-15) ### Fix * **retry:** Verify original exc for failed requests ([`31e79d7`](https://github.com/guilatrova/gracy/commit/31e79d75e6bbeedfdbbc7f69f2c254017c21e22c)) * Format original exception name properly ([`56254b6`](https://github.com/guilatrova/gracy/commit/56254b6e99325b550d3a33512918ab6bb0a56b9a)) ## v1.32.0 (2023-12-09) ### Feature * Improve concurrency logging ([`5292faf`](https://github.com/guilatrova/gracy/commit/5292faf9eff54e9290ab2db836677f44c15b4673)) * Add plotly printer ([`5b8a460`](https://github.com/guilatrova/gracy/commit/5b8a460ffeb8a363b2dfcbef95b3b6134c1f205a)) * Track requests timeline ([`9c30ba2`](https://github.com/guilatrova/gracy/commit/9c30ba2d84a9485d44530df9b82a7bcaf04cd01e)) * Replace manual impl with semaphore ([`57df089`](https://github.com/guilatrova/gracy/commit/57df08936d9f4a808cddd3857058f0f23ba3350b)) ### Documentation * Add explanation ([`b1ea801`](https://github.com/guilatrova/gracy/commit/b1ea80122d6915a207e95780928918c3253d532a)) * Explain concurrent request limit ([`63efb40`](https://github.com/guilatrova/gracy/commit/63efb4039a77c697510a6eb9c47c0fd3319e0017)) * Update badges/description ([`157daaf`](https://github.com/guilatrova/gracy/commit/157daafc3f12b4fa6641a5287688cd632a2ea851)) ## v1.31.0 (2023-12-03) ### Feature * Auto instantiate namespaces ([`3d26863`](https://github.com/guilatrova/gracy/commit/3d26863444a2a959a0ca6b4665c4791547acfbec)) ## v1.30.0 (2023-12-02) ### Feature * Add paginator ([`f8a1db8`](https://github.com/guilatrova/gracy/commit/f8a1db8108b2662c88c182679a71f2b4bb4476fd)) ## v1.29.0 (2023-12-02) ### Feature * Add generated_parsed_response ([`d9add8e`](https://github.com/guilatrova/gracy/commit/d9add8e208c54f72a08df5bfd4b66192c7aba751)) ### Fix * Specify parsed response type properly ([`2ef4566`](https://github.com/guilatrova/gracy/commit/2ef4566069e8f9a6bec129f393eb8b12c946b139)) ### Documentation * Explain parsed_response ([`792fd32`](https://github.com/guilatrova/gracy/commit/792fd32af1bed7ed2584f166082c6ca35d9cdaae)) ## v1.28.1 (2023-11-30) ### Fix * Resolve deploy issue ([`2ae3717`](https://github.com/guilatrova/gracy/commit/2ae3717aa4620e114e750b01eb363f49b0f0fd97)) ## v1.28.0 (2023-11-30) ### Feature * Introduce namespaces ([`67a70f3`](https://github.com/guilatrova/gracy/commit/67a70f3550a77a37d309fcb65800b1b6759fb8f8)) ### Documentation * Write about namespaces ([`468e205`](https://github.com/guilatrova/gracy/commit/468e205fddb1166770bddb4c8c241668b27d03a7)) ## v1.27.1 (2023-08-08) ### Fix * Use Union for python 3.8 ([`02415ac`](https://github.com/guilatrova/gracy/commit/02415aca8009e7bcae70f93b0c0b54c5a7d61473)) ## v1.27.0 (2023-08-08) ### Feature * Add concurrent calls ([`69976d3`](https://github.com/guilatrova/gracy/commit/69976d36d25fe45ae7b6cefe5e8800bf926aa4f8)) ### Documentation * Add note about concurrent requests ([`487dbed`](https://github.com/guilatrova/gracy/commit/487dbedbf707dbe38caf4529be1488ad72a0f2d9)) ## v1.26.0 (2023-08-06) ### Feature * Count ongoing requests ([`9495d4e`](https://github.com/guilatrova/gracy/commit/9495d4ed98839562fc470da2964d370638b6f727)) ### Fix * Use int over HttpStatus enum ([`abebe93`](https://github.com/guilatrova/gracy/commit/abebe938191b760f64490477abad595259537c79)) ### Documentation * Add common hooks ([`3a85897`](https://github.com/guilatrova/gracy/commit/3a85897baf65222e156e9762cd3f09d8bc35e2e4)) ## v1.25.0 (2023-06-30) ### Feature * **replays:** Support log events for replays ([`f9c6b80`](https://github.com/guilatrova/gracy/commit/f9c6b8071302b2262b2973830e7c3d378ae64288)) ### Fix * **reports:** Hide replays if display disabled ([`68692e0`](https://github.com/guilatrova/gracy/commit/68692e0e94ed1ec1d81ea343787abadd34ce3340)) ## v1.24.1 (2023-06-30) ### Fix * **replays:** Pass discard flag ([`2083a52`](https://github.com/guilatrova/gracy/commit/2083a524ed515758ccf1ff6b814d41eafd64ebf5)) ## v1.24.0 (2023-06-30) ### Feature * **replays:** Allow to discard bad status ([`9926ae3`](https://github.com/guilatrova/gracy/commit/9926ae3282ab9632261af204734cbfcf2fce721e)) * **loggers:** Add REPLAY placeholder ([`911a9ec`](https://github.com/guilatrova/gracy/commit/911a9ecdcb83bf877350b96d325047cea80edbd5)) * **reports:** Show replays ([`fc82bd7`](https://github.com/guilatrova/gracy/commit/fc82bd7322f35a61506bc2b42bc53c51d0ae1029)) * Track replays per request ([`e7322a3`](https://github.com/guilatrova/gracy/commit/e7322a3b31aa55d31e93e71cda43ba1ee55c53c2)) * **hooks:** Don't wait if replayed ([`785959f`](https://github.com/guilatrova/gracy/commit/785959f92a8bcacba9d7523a15c7682294bf2a00)) * **replays:** Add flag when replayed ([`8ae8394`](https://github.com/guilatrova/gracy/commit/8ae839489e338b8fbde18e96771dc7765587a6b2)) ## v1.23.0 (2023-06-28) ### Feature * **hooks:** Make reporter optional + dryrun ([`ea0ec00`](https://github.com/guilatrova/gracy/commit/ea0ec007ff5cbd19d1371fe157e9d563d50230bb)) * **hooks:** Add RateLimitBackOffHook ([`d941c68`](https://github.com/guilatrova/gracy/commit/d941c6880494c41171792fe54b9b6189bba964b9)) * Allow to modify retry-after ([`a5f043d`](https://github.com/guilatrova/gracy/commit/a5f043dc4a93fa2ade28967d8b019e483d52c6f7)) * **hooks:** Return hook result ([`ee8f102`](https://github.com/guilatrova/gracy/commit/ee8f10235bced2ed6fe8af1abe219e51950d943f)) ### Fix * **retry:** Validate log correctly ([`dbddcf5`](https://github.com/guilatrova/gracy/commit/dbddcf5f881f2873f3149bd25e4f06ad930b6c51)) ## v1.22.0 (2023-06-27) ### Feature * **retry:** Add RETRY_CAUSE to log ([`3bf1c72`](https://github.com/guilatrova/gracy/commit/3bf1c72776874b3b77f5620ce57e34a912e0a197)) * **retry:** Add cause ([`2b468fd`](https://github.com/guilatrova/gracy/commit/2b468fde1c1a8d3374268c54814bdfb4f992f191)) ### Fix * **retry:** Check override for status ([`1b9383d`](https://github.com/guilatrova/gracy/commit/1b9383d0e340c3a422e87f69c17ab1e097ca7112)) * Make it compatible with py3.8 ([`96e3e61`](https://github.com/guilatrova/gracy/commit/96e3e610eee324047c63adcddd5a6fa534cb51a8)) * **retry:** Set last rep for overriden retry ([`d0241e5`](https://github.com/guilatrova/gracy/commit/d0241e54a4f6cbd5f0b5541366cae3c3a64eb6c3)) * Typo and wrong attrs ([`07d30e0`](https://github.com/guilatrova/gracy/commit/07d30e09a4767084bbfcca5654682486b579bb42)) ## v1.21.1 (2023-06-26) ### Fix * Expose OverrideRetryOn ([`0f14782`](https://github.com/guilatrova/gracy/commit/0f14782aa870efcec829cd9efb2bc72ea101305e)) ## v1.21.0 (2023-06-26) ### Feature * **hooks:** Implement flag for diff locks ([`dcce610`](https://github.com/guilatrova/gracy/commit/dcce6104f1129ee4710b50fc034c1a8629f57cf2)) ## v1.20.0 (2023-06-26) ### Feature * **hooks:** Handle dates for retry_after header ([`3feef65`](https://github.com/guilatrova/gracy/commit/3feef652b1de9aa32b551c54307965f24a92e1b7)) * **hooks:** Add a common hook ([`04734ad`](https://github.com/guilatrova/gracy/commit/04734adcbea3ecc3cde93e1330f58992bb20b3be)) * **retry:** Implement retry override ([`0045413`](https://github.com/guilatrova/gracy/commit/004541385ca1956d6d3cc3c4807ec09cf9999177)) ## v1.19.0 (2023-06-11) ### Feature * Include cause/context to the req exception ([`b2a96f1`](https://github.com/guilatrova/gracy/commit/b2a96f1caba229ef8ae2b102f160b107bde1f9dd)) ## v1.18.0 (2023-06-10) ### Feature * Create request failed exc ([`8087365`](https://github.com/guilatrova/gracy/commit/8087365ebbefa1de8801fc232553e1a5f5d954b5)) ### Fix * **replays:** Handle content type none ([`0229218`](https://github.com/guilatrova/gracy/commit/02292182f99d925ea3cb3d916ba940489a6984e0)) ## v1.17.3 (2023-05-31) ### Fix * **deps:** Requires httpx>=0.23 ([`c3fc7c0`](https://github.com/guilatrova/gracy/commit/c3fc7c08387204eee825a5b6bd17d10d41706d78)) ## v1.17.2 (2023-05-18) ### Fix * Don't pass every kwargs to build_request ([`192838e`](https://github.com/guilatrova/gracy/commit/192838ed0dada8f0dbf30aa7a15a42efe7ad50b0)) ## v1.17.1 (2023-05-17) ### Fix * Support py38+ ([`1ac2f56`](https://github.com/guilatrova/gracy/commit/1ac2f56741094e5c0b4586093f8fa8cb26720d02)) ## v1.17.0 (2023-05-13) ### Feature * Count/display data about replays ([`658b5f5`](https://github.com/guilatrova/gracy/commit/658b5f52adf34d5d6e577abc7e6b9cd33a745919)) * Implement skip throttle ([`efb259c`](https://github.com/guilatrova/gracy/commit/efb259c151ee0807af325901155b67184de062e2)) ## v1.16.0 (2023-05-13) ### Feature * Pass retry state to after hook ([`218e510`](https://github.com/guilatrova/gracy/commit/218e510d84a3acf1fbee84141ec5f7123672cd6b)) * Implement gracy hooks ([`243dddb`](https://github.com/guilatrova/gracy/commit/243dddbd58845cbc92a0b84eaf44c612a125daf5)) * Implement hooks ([`7a88e4c`](https://github.com/guilatrova/gracy/commit/7a88e4c93e3517b9ed095f179582f0fb2809e48a)) ### Fix * Resolve recursion for hooks ([`e1be02b`](https://github.com/guilatrova/gracy/commit/e1be02bb20395c207353de2ea3bae8d839a34c03)) ### Documentation * **hooks:** Add example about hook ([`3cb52db`](https://github.com/guilatrova/gracy/commit/3cb52db7600213252bb36d6de8442bd487fd57b7)) ## v1.15.0 (2023-05-11) ### Feature * Show 'Aborts' as title ([`8485409`](https://github.com/guilatrova/gracy/commit/8485409e899e5d4591754ad62e35cfa4a128f124)) * **reports:** Show retries/throttles ([`f6de12a`](https://github.com/guilatrova/gracy/commit/f6de12a51a95b7c0ac8d0302004a3ad8c0d2e146)) ## v1.14.0 (2023-05-11) ### Feature * Default safe format + retry status code ([`5d7f834`](https://github.com/guilatrova/gracy/commit/5d7f834db146284813341d55979e25b373855606)) * Display aborted requests ([`67ac1ed`](https://github.com/guilatrova/gracy/commit/67ac1ed103248a8f65890826fc6732ec20adb683)) ### Documentation * Add note about graceful request ([`7e14c80`](https://github.com/guilatrova/gracy/commit/7e14c80205bd56df9297d2a169c3529397b4f05a)) ## v1.13.0 (2023-05-10) ### Feature * Track broken requests ([`e40d8b8`](https://github.com/guilatrova/gracy/commit/e40d8b8774c86f766c69c0cd8f0d5d5b65f09d0f)) * Capture broken requests (without a response) ([`bf0ac44`](https://github.com/guilatrova/gracy/commit/bf0ac44e87f96d7acc41f7f0e63411ac0f113a67)) ## v1.12.0 (2023-05-04) ### Feature * Improve decorator typing ([`72233d6`](https://github.com/guilatrova/gracy/commit/72233d60dd84cfddf2778b585b1260833f357c1e)) ## v1.11.4 (2023-05-04) ### Fix * Add support for `graceful_generator` ([`22ecf9a`](https://github.com/guilatrova/gracy/commit/22ecf9ac91064fcc4288f38ff73a77f4e165b98d)) ## v1.11.3 (2023-03-24) ### Fix * Make exception pickable ([`16d6a62`](https://github.com/guilatrova/gracy/commit/16d6a6248fd46a565c411743a3bf0f74dac94363)) ### Documentation * Show custom request timeout ([`e2a069b`](https://github.com/guilatrova/gracy/commit/e2a069b46a01cbcbf5bd2a9507d7d25505ecbd83)) ## v1.11.2 (2023-03-03) ### Fix * Log exhausted when appropriate ([`8c5d622`](https://github.com/guilatrova/gracy/commit/8c5d622fef7aa6dd2514cfaaf867445f56d7b04a)) * Retry considers last validation result ([`595177f`](https://github.com/guilatrova/gracy/commit/595177f50e396f4ca7b2dcc1c8ed535928a0aca7)) * Handle retry edge case ([`077e6f4`](https://github.com/guilatrova/gracy/commit/077e6f49d80cb6d886c31aa010a4f814a6953445)) * Retry result is used as response ([`8687156`](https://github.com/guilatrova/gracy/commit/8687156991058fa24043dc39658f0a12377a21f6)) ### Documentation * Add httpbin example ([`1babd10`](https://github.com/guilatrova/gracy/commit/1babd1098a46c4d0bc24ed228d76bb094260ad5e)) ## v1.11.1 (2023-02-23) ### Fix * **retry:** Don't retry when successful ([`b334c22`](https://github.com/guilatrova/gracy/commit/b334c227a4a8a688029130c736118b6dcb4f8f3b)) * **pymongo:** Adjust filter ([`5ee9f0c`](https://github.com/guilatrova/gracy/commit/5ee9f0c6aa523530929bd69d19a9ff637c46705c)) * **pymongo:** Use correct methods/kwargs ([`4a191d8`](https://github.com/guilatrova/gracy/commit/4a191d81e083772add036bc3d9d5937ccbf6d31c)) ### Documentation * Update examples ([`26420da`](https://github.com/guilatrova/gracy/commit/26420da78776862a0cb7569b5f64b610ed212ff6)) ## v1.11.0 (2023-02-23) ### Feature * Enable config debugging flag ([`07c6339`](https://github.com/guilatrova/gracy/commit/07c633923a20343329aa884ddc109f3cde0e5be0)) ## v1.10.1 (2023-02-23) ### Fix * Error log ([`6f63941`](https://github.com/guilatrova/gracy/commit/6f6394181ed024f738605a4743af2eea788ce4f7)) ## v1.10.0 (2023-02-22) ### Feature * Allow custom validators ([`50818f8`](https://github.com/guilatrova/gracy/commit/50818f89fe2a03800fde18fa38686a04853cb54a)) ### Fix * Implement proper validate/retry/parse logic ([`0b2fa75`](https://github.com/guilatrova/gracy/commit/0b2fa75228c9340efb8595fee801c0cfa3303619)) * Raise exception correctly ([`10a90b5`](https://github.com/guilatrova/gracy/commit/10a90b5159a2fce3e24c1bfac7f4b9e0cb58d059)) ### Documentation * Add exception details to retry params ([`8d69234`](https://github.com/guilatrova/gracy/commit/8d692346369b5c83d05e746ec1b7e9f924d02cbd)) * Enhance custom validator example ([`d5e02eb`](https://github.com/guilatrova/gracy/commit/d5e02eb032739639f9ceb655b5b88c39f8c9a0f6)) * Add validators ([`e3e8fa6`](https://github.com/guilatrova/gracy/commit/e3e8fa672e5f95d02f60dc3af762b6e6cd189d4d)) ## v1.9.1 (2023-02-21) ### Fix * Create tuples ([`f648f85`](https://github.com/guilatrova/gracy/commit/f648f85a5787b2cd86934051640e666815fe5864)) ## v1.9.0 (2023-02-21) ### Feature * Make exceptions pickable ([`5ab62c5`](https://github.com/guilatrova/gracy/commit/5ab62c59ac273078e7a1ef3122e76bf0c6901e70)) ### Documentation * Reword ([`0ca061f`](https://github.com/guilatrova/gracy/commit/0ca061f1b1e73c73b01808e2d9f0258f03e0fefa)) * Add a emoji ([`8da07ae`](https://github.com/guilatrova/gracy/commit/8da07aecd8da6642edf01a94475ff49f297c1886)) * Reword ([`a54f1f7`](https://github.com/guilatrova/gracy/commit/a54f1f7bac2b7a5fb52485b31c746e58734066d0)) * Reorder logging customization ([`f6d9d76`](https://github.com/guilatrova/gracy/commit/f6d9d765daee63e7e863426519f8acda5bc2c5f0)) ## v1.8.1 (2023-02-17) ### Fix * Retry logic triggers only once ([`0fc2358`](https://github.com/guilatrova/gracy/commit/0fc2358b1631eacc0587a59afe1d21b419f8679e)) ## v1.8.0 (2023-02-17) ### Feature * Calculate throttling await properly ([`ba520e0`](https://github.com/guilatrova/gracy/commit/ba520e034bab88b2b5a258473f8a2ba7ff7c5879)) * Lock throttling logs properly ([`a8ebd69`](https://github.com/guilatrova/gracy/commit/a8ebd69df0e5184a6a806870a12888c202ba37d8)) * Prevent floats for max_requests ([`b9aed74`](https://github.com/guilatrova/gracy/commit/b9aed746bdfcd672920baeb047cf02b31e146503)) * Format rule time range ([`514cbae`](https://github.com/guilatrova/gracy/commit/514cbaeeb2d02de12f60a62e8285ce0ba1ad0437)) * Allow custom time windows for throttling ([`7fc35f0`](https://github.com/guilatrova/gracy/commit/7fc35f09e4a5e8df50a746cf95d112b08d4dd9bc)) ### Fix * Correct kwargs ([`0db5925`](https://github.com/guilatrova/gracy/commit/0db59254081d479a20c411ab346cad605e3a2efb)) ### Documentation * Add `THROTTLE_TIME_RANGE` ([`299c200`](https://github.com/guilatrova/gracy/commit/299c2008b5da43e7a52035dc285375b0b1dfc093)) * **throttling:** Add timedelta example ([`74c20ef`](https://github.com/guilatrova/gracy/commit/74c20ef91c521165b72c999c7212268ca83ec7cc)) * Enhance throttling example ([`200b3c5`](https://github.com/guilatrova/gracy/commit/200b3c5adac8a16f3af002d56f2e3c8b84f3f0d3)) ## v1.7.1 (2023-02-14) ### Fix * **retry:** Remove duplicated default msg ([`963d7e8`](https://github.com/guilatrova/gracy/commit/963d7e8237a85c5f5692a01d7a3d1c0eb733b752)) ### Documentation * Fix reports/replay order ([`b4ddf79`](https://github.com/guilatrova/gracy/commit/b4ddf792fe29ae49e981fda5b1fca0bec4aca0f9)) ## v1.7.0 (2023-02-12) ### Feature * Handle missing replays ([`4395b83`](https://github.com/guilatrova/gracy/commit/4395b832cd9f75a88d696d5cba2eb7bd9f7ce61d)) * Report show replay mode ([`b488975`](https://github.com/guilatrova/gracy/commit/b4889755c75c3f3a14507b27b3d57ba243b5c828)) * Implement replay load w/ sqlite ([`4fa4cf6`](https://github.com/guilatrova/gracy/commit/4fa4cf6983ed64d82560d47c813bfeb4cfa5ed66)) * Implement replay (store only) w/ sqlite ([`797c2b9`](https://github.com/guilatrova/gracy/commit/797c2b95334f5ebfd9b17555278b1be44b7eeef2)) ### Fix * Handle 0 requests for logger printer ([`09e471c`](https://github.com/guilatrova/gracy/commit/09e471c791e34c9b30427c6903bb19c8c25338aa)) ### Documentation * Add details about custom replay storage ([`f03407f`](https://github.com/guilatrova/gracy/commit/f03407fbd66a40850d679541b0616fc7847c8b5c)) * Add brief explanation about replay ([`edd1a24`](https://github.com/guilatrova/gracy/commit/edd1a24fb255d8ed23288f277769b923e6af218b)) ## v1.6.1 (2023-02-11) ### Fix * Gracy supports Python >=3.8 ([`a3623a9`](https://github.com/guilatrova/gracy/commit/a3623a98a7459dcba3dc78ca11917be5c6c5a82d)) ## v1.6.0 (2023-02-07) ### Feature * Handle parsing failures ([`ac48952`](https://github.com/guilatrova/gracy/commit/ac489522a98412d65b85ac3317dbe6083d8819ad)) ### Documentation * Fix syntax ([`9996b39`](https://github.com/guilatrova/gracy/commit/9996b39f505d3221f2e63d78bd311e90f2608349)) ## v1.5.0 (2023-02-05) ### Feature * Protect lambda custom msg from unknown keys ([`d6da853`](https://github.com/guilatrova/gracy/commit/d6da8536d1b561fd606d2911749d99309aa92460)) * Implement lambda for loggers ([`e7d9248`](https://github.com/guilatrova/gracy/commit/e7d9248475ce9dab92913cc7fa7eb6554c9676d7)) ### Fix * Use correct typing for coroutine ([`65296cd`](https://github.com/guilatrova/gracy/commit/65296cdddf925126ea47e591f7def242b0e6b6da)) ### Documentation * Add report examples ([`269810c`](https://github.com/guilatrova/gracy/commit/269810c4d205e5356672287f08c3d34d3bc0c3f0)) ## v1.4.0 (2023-02-05) ### Feature * Implement the logger printer ([`40298f5`](https://github.com/guilatrova/gracy/commit/40298f5204a499730f93d2d79bbfed43dc754b0c)) * Implement the list printer ([`9adee2d`](https://github.com/guilatrova/gracy/commit/9adee2d9ea78a569ab1541724b86fb73b06a4f2e)) * Split rich as optional dep ([`ae169df`](https://github.com/guilatrova/gracy/commit/ae169df066871d4095b95c032e7ec06b85ab3249)) ### Documentation * Fix bad information ([`e1a6746`](https://github.com/guilatrova/gracy/commit/e1a67466a9403dc87719cde9079a0f2b0ed7b16f)) * Fix bad syntax example ([`116b9bf`](https://github.com/guilatrova/gracy/commit/116b9bf0e1ed6fabdb9e5d365ade7d92ab8d3429)) ## v1.3.0 (2023-02-01) ### Feature * Use locks for throttled requests ([`b2db6a7`](https://github.com/guilatrova/gracy/commit/b2db6a760b097b27142f17bf533d760e4e99605c)) ### Fix * Throttling/allowed not working ([`cb0251b`](https://github.com/guilatrova/gracy/commit/cb0251b49c43f9376783e6f457073410f6d326a1)) ## v1.2.1 (2023-02-01) ### Fix * Handle scenarios for just 1 request per url ([`f4f799b`](https://github.com/guilatrova/gracy/commit/f4f799bbc03ae318fba69dd299fb423800a18651)) ## v1.2.0 (2023-02-01) ### Feature * Simplify req/s rate to the user ([`1b428c7`](https://github.com/guilatrova/gracy/commit/1b428c788f192e0e23c49b27d9a46438d20d230a)) * Include req rate in report ([`e387a25`](https://github.com/guilatrova/gracy/commit/e387a25f831a27f031ebc1625ac642beb3895678)) * Clear base urls with ending slash ([`51fb8ee`](https://github.com/guilatrova/gracy/commit/51fb8ee9e369eecd951fb31da92edc3317e63483)) * Implement retry logging ([`f2d3238`](https://github.com/guilatrova/gracy/commit/f2d3238830bbda163b8b55f874f2ae7ecb11d6df)) ### Fix * Consider retry is unset ([`0ca1ed9`](https://github.com/guilatrova/gracy/commit/0ca1ed9e65faa8e1e7efd024a7264dbc328a3259)) * Retry must start with 1 ([`3e3e750`](https://github.com/guilatrova/gracy/commit/3e3e75003092bca7f4181c17b68a873ec77c31d1)) ### Documentation * Fix download badge ([`22a9d7a`](https://github.com/guilatrova/gracy/commit/22a9d7a132b86c6da084b6f59ddba74f64814238)) * Improve examples ([`4ca1f7d`](https://github.com/guilatrova/gracy/commit/4ca1f7df80b6b1bba9f255983a6be5b906b09a85)) * Add new placeholders ([`8eba619`](https://github.com/guilatrova/gracy/commit/8eba619dd73544861960b0a9a381fe97d2c5468f)) * Add some notes for custom exceptions ([`225f008`](https://github.com/guilatrova/gracy/commit/225f00828697d8a611bb596e1f3119570a1b363e)) ## v1.1.0 (2023-01-30) ### Feature * Change api to be public ([`3b0c828`](https://github.com/guilatrova/gracy/commit/3b0c8281c3e164d9a7f01770c698fa825afe562a)) ### Documentation * Fix examples/info ([`0193f11`](https://github.com/guilatrova/gracy/commit/0193f112807f4621f5fd35acc9fbec32c4a2554c)) ## v1.0.0 (2023-01-30) ### Feature * Drop python 3.7 support ([`0f69e5b`](https://github.com/guilatrova/gracy/commit/0f69e5be00f8202ea2aa98b71630ae167c6431f1)) ### Breaking * drop python 3.7 support ([`0f69e5b`](https://github.com/guilatrova/gracy/commit/0f69e5be00f8202ea2aa98b71630ae167c6431f1)) ### Documentation * Add remaining sections ([`4335b5a`](https://github.com/guilatrova/gracy/commit/4335b5a3313a56c36b7b54c9ec44a07b2e6b4bd0)) * Add throttling ([`6fc9583`](https://github.com/guilatrova/gracy/commit/6fc958328fcbc5304e745c29918f8ffb2f8fa1a4)) * Add retry ([`aa8a828`](https://github.com/guilatrova/gracy/commit/aa8a82844a8c77f99897512d23b01eb216b8e0ff)) * Add credits/settings section ([`113bf48`](https://github.com/guilatrova/gracy/commit/113bf4886ae50418ddaef62d6f4880171f98240f)) * Write about parsing ([`c133cda`](https://github.com/guilatrova/gracy/commit/c133cda6444058861a5129db5da0a4fd7a12965e)) * Remove colspans ([`3ef5fd7`](https://github.com/guilatrova/gracy/commit/3ef5fd77dbec659144a034405c815fa5a060d747)) * Add logging details ([`09e923c`](https://github.com/guilatrova/gracy/commit/09e923cb9bb14b858f8c6ab975fb50ffab8fd42a)) * Fix badge ([`fea301a`](https://github.com/guilatrova/gracy/commit/fea301a63db98398101ae796f3a14f35882922f7)) * Add empty topics ([`887b46c`](https://github.com/guilatrova/gracy/commit/887b46ca3a61d20fcc942e18868a159ffaded0f1)) * Improve top description ([`e745403`](https://github.com/guilatrova/gracy/commit/e745403116483c651ebcd9f7e26fe99ab468ad03)) ## v0.6.0 (2023-01-29) ### Feature * Implement throttling ([`8691045`](https://github.com/guilatrova/gracy/commit/869104595b7c6954ea31b159e89a1efe8028215c)) ### Fix * **throttling:** Resolve bugs ([`4c41326`](https://github.com/guilatrova/gracy/commit/4c4132608b61256b8949dcbc46558641bccceedf)) * **throttling:** Handle some scenarios ([`f9d4fbc`](https://github.com/guilatrova/gracy/commit/f9d4fbc5c2e6e378cdfdd7dc8a930852f9620477)) ### Documentation * Improve prop description ([`27f9e01`](https://github.com/guilatrova/gracy/commit/27f9e01dd5004827a8df471034138ad1bf18b10c)) ## v0.5.0 (2023-01-29) ### Feature * Implement custom exceptions ([`2d89ebd`](https://github.com/guilatrova/gracy/commit/2d89ebd4c862c60bfc816774c3102c8e9e43ed2a)) * Implement retry pass ([`45e8ce6`](https://github.com/guilatrova/gracy/commit/45e8ce6124127ef69f5a9704a6ae0dc4a48d1f45)) ## v0.4.0 (2023-01-29) ### Feature * Implement parser ([`ab48cd9`](https://github.com/guilatrova/gracy/commit/ab48cd937cfa37e4455260defa94a8d41620f878)) ### Documentation * Add custom logo ([`19f6bf8`](https://github.com/guilatrova/gracy/commit/19f6bf86b4daf68ee50908cf2833912b0f3de852)) ## v0.3.0 (2023-01-29) ### Feature * Improve client customization ([`1372b4f`](https://github.com/guilatrova/gracy/commit/1372b4fb9ba7fc6d2c9b8f5e3064f4e2c9fd9ab5)) ## v0.2.0 (2023-01-28) ### Feature * Calculate footer totals ([`eb77c71`](https://github.com/guilatrova/gracy/commit/eb77c7138c50511cb1d4edfbd7c6f77b52ca6989)) * Sort table by requests made desc ([`fced5eb`](https://github.com/guilatrova/gracy/commit/fced5eb47dcc87abc97f2d91b1905140ad4d65d9)) * Add custom color to status ([`9964723`](https://github.com/guilatrova/gracy/commit/99647237155aa0f8b6d236ffbaa71d6d616c4ea7)) * Fold correct column ([`4a0bff0`](https://github.com/guilatrova/gracy/commit/4a0bff08a67e3c549897cac3ce9ec6d94603c2e7)) ## v0.1.0 (2023-01-28) ### Feature * Fold url column ([`a4b0ed0`](https://github.com/guilatrova/gracy/commit/a4b0ed0c1b2fe2b313c113e9ecdb6020b2f949a4)) * Display status range in metrics ([`8d01476`](https://github.com/guilatrova/gracy/commit/8d0147613c83c708064d05033ba1e7a24d3fa6cf)) * Add custom color to failed requests ([`65c9ab7`](https://github.com/guilatrova/gracy/commit/65c9ab7c2db0f90b1a3f48c4ab74eb2d3a96dd42)) * Add rich table to display metrics ([`44944f7`](https://github.com/guilatrova/gracy/commit/44944f7874f474fc33b4f532260a65720df0c051)) * Implement logs ([`9caee55`](https://github.com/guilatrova/gracy/commit/9caee5576f9d8cf3f9a17429b54e5dd26df9fb15)) * Add stub for report ([`b394afe`](https://github.com/guilatrova/gracy/commit/b394afe66a5fadb3c4831f2ceb75842b717465b4)) * Narrow down retry logic ([`e444281`](https://github.com/guilatrova/gracy/commit/e444281be9f0e8d9752e0ae847a768fddd1c1586)) * Make gracy async ([`5edacca`](https://github.com/guilatrova/gracy/commit/5edacca8781c02b7046a636020a7847faf716e8e)) * Implement retry ([`f0a794a`](https://github.com/guilatrova/gracy/commit/f0a794a40a6d351b02b516fa3a0004798a0710c2)) * Implement strict/allowed status code ([`171688b`](https://github.com/guilatrova/gracy/commit/171688b591c0c88b825f5ff1590f55c5cf0e1a9d)) ### Fix * Use enum value for _str_ ([`345464f`](https://github.com/guilatrova/gracy/commit/345464f44a48d864d5a39e56dfadf94f6f55da16)) ### Documentation * Reword some stuff ([`546f3fc`](https://github.com/guilatrova/gracy/commit/546f3fc6188c312196c9ca69a5fb80e172b6738f)) * Slightly improve readme ([`8a56b3d`](https://github.com/guilatrova/gracy/commit/8a56b3d961cf3ad343d7c95412ce49184e914608)) * Fill with some gracy stuff ([`8183d26`](https://github.com/guilatrova/gracy/commit/8183d2686f8f3a4cdfc50bf8e13465edfc54ef6d)) ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2021 Guilherme Latrova Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================

Python's most graceful API Client Framework

Actions Status PyPI Downloads GitHub Code style: black try/except style: tryceratops Types: pyright Follow guilatrova Sponsor guilatrova

Gracy handles failures, logging, retries, throttling, parsing, and reporting for all your HTTP interactions. Gracy uses [httpx](https://github.com/encode/httpx) under the hood. > "Let Gracy do the boring stuff while you focus on your application" --- **Summary** - [🧑‍💻 Get started](#-get-started) - [Installation](#installation) - [Usage](#usage) - [Simple example](#simple-example) - [More examples](#more-examples) - [Settings](#settings) - [Strict/Allowed status code](#strictallowed-status-code) - [Custom Validators](#custom-validators) - [Parsing](#parsing) - [Parsing Typing](#parsing-typing) - [Retry](#retry) - [Throttling](#throttling) - [Concurrent Requests](#concurrent-requests) - [Logging](#logging) - [Custom Exceptions](#custom-exceptions) - [Reports](#reports) - [Logger](#logger) - [List](#list) - [Table](#table) - [Plotly](#plotly) - [Replay requests](#replay-requests) - [Recording](#recording) - [Replay](#replay) - [Resource Namespacing](#resource-namespacing) - [Pagination](#pagination) - [Advanced Usage](#advanced-usage) - [Customizing/Overriding configs per method](#customizingoverriding-configs-per-method) - [Customizing HTTPx client](#customizing-httpx-client) - [Overriding default request timeout](#overriding-default-request-timeout) - [Creating a custom Replay data source](#creating-a-custom-replay-data-source) - [Hooks before/after request](#hooks-beforeafter-request) - [Common Hooks](#common-hooks) - [`HttpHeaderRetryAfterBackOffHook`](#httpheaderretryafterbackoffhook) - [`RateLimitBackOffHook`](#ratelimitbackoffhook) - [📚 Extra Resources](#-extra-resources) - [Change log](#change-log) - [License](#license) - [Credits](#credits) ## 🧑‍💻 Get started ### Installation ``` pip install gracy ``` OR ``` poetry add gracy ``` ### Usage Examples will be shown using the [PokeAPI](https://pokeapi.co). #### Simple example ```py # 0. Import import asyncio import typing as t from gracy import BaseEndpoint, Gracy, GracyConfig, LogEvent, LogLevel # 1. Define your endpoints class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" # 👈 Put placeholders as needed # 2. Define your Graceful API class GracefulPokeAPI(Gracy[str]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" # 👈 Optional BASE_URL # 👇 Define settings to apply for every request SETTINGS = GracyConfig( log_request=LogEvent(LogLevel.DEBUG), log_response=LogEvent(LogLevel.INFO, "{URL} took {ELAPSED}"), parser={ "default": lambda r: r.json() } ) async def get_pokemon(self, name: str) -> t.Awaitable[dict]: return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) pokeapi = GracefulPokeAPI() async def main(): try: pokemon = await pokeapi.get_pokemon("pikachu") print(pokemon) finally: pokeapi.report_status("rich") asyncio.run(main()) ``` #### More examples - [PokeAPI with retries, parsers, logs](./examples/pokeapi.py) - [PokeAPI with throttling](./examples/pokeapi_throttle.py) - [PokeAPI with SQLite replay](./examples/pokeapi_replay.py) - [PokeAPI with Mongo replay](./examples/pokeapi_replay_mongo.py) ## Settings ### Strict/Allowed status code By default Gracy considers any successful status code (200-299) as successful. **Strict** You can modify this behavior by defining a strict status code or increase the range of allowed status codes: ```py from http import HTTPStatus GracyConfig( strict_status_code=HTTPStatus.CREATED ) ``` or a list of values: ```py from http import HTTPStatus GracyConfig( strict_status_code={HTTPStatus.OK, HTTPStatus.CREATED} ) ``` Using `strict_status_code` means that any other code not specified will raise an error regardless of being successful or not. **Allowed** You can also keep the behavior, but extend the range of allowed codes. ```py from http import HTTPStatus GracyConfig( allowed_status_code=HTTPStatus.NOT_FOUND ) ``` or a list of values ```py from http import HTTPStatus GracyConfig( allowed_status_code={HTTPStatus.NOT_FOUND, HTTPStatus.FORBIDDEN} ) ``` Using `allowed_status_code` means that all successful codes plus your defined codes will be considered successful. This is quite useful for parsing as you'll see soon. ⚠️ Note that `strict_status_code` takes precedence over `allowed_status_code`, probably you don't want to combine those. Prefer one or the other. ### Custom Validators You can implement your own custom validator to do further checks on the response and decide whether to consider the request failed (and as consequence trigger retries if they're set). ```py from gracy import GracefulValidator class MyException(Exception): pass class MyCustomValidator(GracefulValidator): def check(self, response: httpx.Response) -> None: jsonified = response.json() if jsonified.get('error', None): raise MyException("Error is not expected") return None ... class Config: SETTINGS = GracyConfig( ..., retry=GracefulRetry(retry_on=MyException, ...), # Set up retry to work whenever our validator fails validators=MyCustomValidator(), # Set up validator ) ``` ### Parsing Parsing allows you to handle the request based on the status code returned. The basic example is parsing `json`: ```py GracyConfig( parser={ "default": lambda r: r.json() } ) ``` In this example all successful requests will automatically return the `json()` result. You can also narrow it down to handle specific status codes. ```py class Config: SETTINGS = GracyConfig( ..., allowed_status_code=HTTPStatusCode.NOT_FOUND, parser={ "default": lambda r: r.json() HTTPStatusCode.NOT_FOUND: None } ) async def get_pokemon(self, name: str) -> dict| None: # 👇 Returns either dict or None return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) ``` Or even customize [exceptions to improve your code readability](https://guicommits.com/handling-exceptions-in-python-like-a-pro/): ```py class PokemonNotFound(GracyUserDefinedException): ... # More on exceptions below class Config: GracyConfig( ..., allowed_status_code=HTTPStatusCode.NOT_FOUND, parser={ "default": lambda r: r.json() HTTPStatusCode.NOT_FOUND: PokemonNotFound } ) async def get_pokemon(self, name: str) -> Awaitable[dict]: # 👇 Returns either dict or raises PokemonNotFound return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) ``` ### Parsing Typing Because parsers allow you to dynamically parse a payload based on the status code your IDE will not identify the return type by itself. To avoid boring `typing.cast` for every method, Gracy provides typed http methods, so you can define a specific return type: ```py async def list(self, offset: int = 0, limit: int = 20): params = dict(offset=offset, limit=limit) return await self.get[ResourceList]( # Specifies this method return a `ResourceList` PokeApiEndpoint.BERRY_LIST, params=params ) async def get_one(self, name_or_id: str | int): return await self.get[models.Berry | None]( PokeApiEndpoint.BERRY_GET, params=dict(KEY=str(name_or_id)) ) ``` ### Retry Who doesn't hate flaky APIs? 🙋 Yet there're many of them. Using tenacity, backoff, retry, aiohttp_retry, and any other retry libs is **NOT easy enough**. 🙅 You still would need to code the implementation for each request which is annoying. Here's how Gracy allows you to implement your retry logic: ```py class Config: GracyConfig( retry=GracefulRetry( delay=1, max_attempts=3, delay_modifier=1.5, retry_on=None, log_before=None, log_after=LogEvent(LogLevel.WARNING), log_exhausted=LogEvent(LogLevel.CRITICAL), behavior="break", ) ) ``` | Parameter | Description | Example | | ---------------- | --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | | `delay` | How many seconds to wait between retries | `2` would wait 2 seconds, `1.5` would wait 1.5 seconds, and so on | | `max_attempts` | How many times should Gracy retry the request? | `10` means 1 regular request with additional 10 retries in case they keep failing. `1` should be the minimum | | `delay_modifier` | Allows you to specify increasing delay times by multiplying this value to `delay` | Setting `1` means no delay change. Setting `2` means delay will be doubled every retry | | `retry_on` | Should we retry for which status codes/exceptions? `None` means for any non successful status code or exception | `HTTPStatus.BAD_REQUEST`, or `{HTTPStatus.BAD_REQUEST, HTTPStatus.FORBIDDEN}`, or `Exception` or `{Exception, HTTPStatus.NOT_FOUND}` | | `log_before` | Specify log level. `None` means don't log | More on logging later | | `log_after` | Specify log level. `None` means don't log | More on logging later | | `log_exhausted` | Specify log level. `None` means don't log | More on logging later | | `behavior` | Allows you to define how to deal if the retry fails. `pass` will accept any retry failure | `pass` or `break` (default) | | `overrides` | Allows to override `delay` based on last response status code | `{HTTPStatus.BAD_REQUEST: OverrideRetryOn(delay=0), HTTPStatus.INTERNAL_SERVER_ERROR: OverrideRetryOn(delay=10)}` | ### Throttling Rate limiting issues? No more. Gracy helps you proactively deal with it before any API throws 429 in your face. **Creating rules** You can define rules per endpoint using regex: ```py SIMPLE_RULE = ThrottleRule( url_pattern=r".*", max_requests=2 ) print(SIMPLE_RULE) # Output: "2 requests per second for URLs matching re.compile('.*')" COMPLEX_RULE = ThrottleRule( url_pattern=r".*\/pokemon\/.*", max_requests=10, per_time=timedelta(minutes=1, seconds=30), ) print(COMPLEX_RULE) # Output: 10 requests per 90 seconds for URLs matching re.compile('.*\\/pokemon\\/.*') ``` **Setting throttling** You can set up logging and assign rules as: ```py class Config: GracyConfig( throttling=GracefulThrottle( rules=ThrottleRule(r".*", 2), # 2 reqs/s for any endpoint log_limit_reached=LogEvent(LogLevel.ERROR), log_wait_over=LogEvent(LogLevel.WARNING), ), ) ``` ### Concurrent Requests Maybe the API you're hitting have some slow endpoints and you want to ensure that no more than a custom number of requests are being made concurrently. You can define a `ConcurrentRequestLimit` config. The simplest usage is: ```py from gracy import ConcurrentRequestLimit class Config: GracyConfig( concurrent_requests=ConcurrentRequestLimit( limit=1, # How many concurrent requests log_limit_reached=LogEvent(LogLevel.WARNING), log_limit_freed=LogEvent(LogLevel.INFO), ), ) ``` But you can also define it easily per method as: ```py class MyApiClient(Gracy[Endpoint]): @graceful(concurrent_requests=5) async def get_concurrently_five(self, name: str): ... ``` ### Logging You can **define and customize logs** for events by using `LogEvent` and `LogLevel`: ```py verbose_log = LogEvent(LogLevel.CRITICAL) custom_warn_log = LogEvent(LogLevel.WARNING, custom_message="{METHOD} {URL} is quite slow and flaky") custom_error_log = LogEvent(LogLevel.INFO, custom_message="{URL} returned a bad status code {STATUS}, but that's fine") ``` Note that placeholders are formatted and replaced later on by Gracy based on the event type, like: **Placeholders per event** | Placeholder | Description | Example | Supported Events | | ----------------------- | ------------------------------------------------------------- | -------------------------------------------------------------- | -------------------- | | `{URL}` | Full url being targetted | `https://pokeapi.co/api/v2/pokemon/pikachu` | *All* | | `{UURL}` | Full **Unformatted** url being targetted | `https://pokeapi.co/api/v2/pokemon/{NAME}` | *All* | | `{ENDPOINT}` | Endpoint being targetted | `/pokemon/pikachu` | *All* | | `{UENDPOINT}` | **Unformatted** endpoint being targetted | `/pokemon/{NAME}` | *All* | | `{METHOD}` | HTTP Request being used | `GET`, `POST` | *All* | | `{STATUS}` | Status code returned by the response | `200`, `404`, `501` | *After Request* | | `{ELAPSED}` | Amount of seconds taken for the request to complete | *Numeric* | *After Request* | | `{REPLAY}` | A placeholder that is displayed only when request is replayed | `REPLAYED` when replay, otherwise it's a blank str (``) | *After Request* | | `{IS_REPLAY}` | Boolean value to show whether it's replayed or not | String with `TRUE` when replayed or `FALSE` | *After Request* | | `{RETRY_DELAY}` | How long Gracy will wait before repeating the request | *Numeric* | *Any Retry event* | | `{RETRY_CAUSE}` | What caused the retry logic to trigger | `[Bad Status Code: 404]`, `[Request Error: ConnectionTimeout]` | *Any Retry event* | | `{CUR_ATTEMPT}` | Current attempt count for the current request | *Numeric* | *Any Retry event* | | `{MAX_ATTEMPT}` | Max attempt defined for the current request | *Numeric* | *Any Retry event* | | `{THROTTLE_LIMIT}` | How many reqs/s is defined for the current request | *Numeric* | *Any Throttle event* | | `{THROTTLE_TIME}` | How long Gracy will wait before calling the request | *Numeric* | *Any Throttle event* | | `{THROTTLE_TIME_RANGE}` | Time range defined by the throttling rule | `second`, `90 seconds` | *Any Throttle event* | and you can set up the log events as follows: **Requests** 1. Before request 2. After response 3. Response has non successful errors ```py GracyConfig( log_request=LogEvent(), log_response=LogEvent(), log_errors=LogEvent(), ) ``` **Retry** 1. Before retry 2. After retry 3. When retry exhausted ```py GracefulRetry( ..., log_before=LogEvent(), log_after=LogEvent(), log_exhausted=LogEvent(), ) ``` **Throttling** 1. When reqs/s limit is reached 2. When limit decreases again ```py GracefulThrottle( ..., log_limit_reached=LogEvent() log_wait_over=LogEvent() ) ``` **Dynamic Customization** You can customize it even further by passing a lambda: ```py LogEvent( LogLevel.ERROR, lambda r: "Request failed with {STATUS}" f" and it was {'redirected' if r.is_redirect else 'NOT redirected'}" if r else "", ) ``` Consider that: - Not all log events have the response available, so you need to guard yourself against it - Placeholders still works (e.g. `{STATUS}`) - You need to watch out for some attrs that might break the formatting logic (e.g. `r.headers`) ### Custom Exceptions You can define custom exceptions for more [fine grained control over your exception messages/types](https://guicommits.com/how-to-structure-exception-in-python-like-a-pro/). The simplest you can do is: ```py from gracy import Gracy, GracyConfig from gracy.exceptions import GracyUserDefinedException class MyCustomException(GracyUserDefinedException): pass class MyApi(Gracy[str]): class Config: SETTINGS = GracyConfig( ..., parser={ HTTPStatus.BAD_REQUEST: MyCustomException } ) ``` This will raise your custom exception under the conditions defined in your parser. You can improve it even further by customizing your message: ```py class PokemonNotFound(GracyUserDefinedException): BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" def _format_message(self, request_context: GracyRequestContext, response: httpx.Response) -> str: format_args = self._build_default_args() name = request_context.endpoint_args.get("NAME", "Unknown") return self.BASE_MESSAGE.format(NAME=name, **format_args) ``` ## Reports ### Logger Recommended for production environments. Gracy reports a short summary using `logger.info`. ```python pokeapi = GracefulPokeAPI() # do stuff with your API pokeapi.report_status("logger") # OUTPUT ❯ Gracy tracked that 'https://pokeapi.co/api/v2/pokemon/{NAME}' was hit 1 time(s) with a success rate of 100.00%, avg latency of 0.45s, and a rate of 1.0 reqs/s. ❯ Gracy tracked a total of 2 requests with a success rate of 100.00%, avg latency of 0.24s, and a rate of 1.0 reqs/s. ``` ### List Uses `print` to generate a short list with all attributes: ```python pokeapi = GracefulPokeAPI() # do stuff with your API pokeapi.report_status("list") # OUTPUT ____ / ___|_ __ __ _ ___ _ _ | | _| '__/ _` |/ __| | | | | |_| | | | (_| | (__| |_| | \____|_| \__,_|\___|\__, | |___/ Requests Summary Report 1. https://pokeapi.co/api/v2/pokemon/{NAME} Total Reqs (#): 1 Success (%): 100.00% Fail (%): 0.00% Avg Latency (s): 0.39 Max Latency (s): 0.39 2xx Resps: 1 3xx Resps: 0 4xx Resps: 0 5xx Resps: 0 Avg Reqs/sec: 1.0 reqs/s 2. https://pokeapi.co/api/v2/generation/{ID}/ Total Reqs (#): 1 Success (%): 100.00% Fail (%): 0.00% Avg Latency (s): 0.04 Max Latency (s): 0.04 2xx Resps: 1 3xx Resps: 0 4xx Resps: 0 5xx Resps: 0 Avg Reqs/sec: 1.0 reqs/s TOTAL Total Reqs (#): 2 Success (%): 100.00% Fail (%): 0.00% Avg Latency (s): 0.21 Max Latency (s): 0.00 2xx Resps: 2 3xx Resps: 0 4xx Resps: 0 5xx Resps: 0 Avg Reqs/sec: 1.0 reqs/s ``` ### Table It requires you to install [Rich](https://github.com/Textualize/rich). ```py pokeapi = GracefulPokeAPI() # do stuff with your API pokeapi.report_status("rich") ``` Here's an example of how it looks: ![Report](https://raw.githubusercontent.com/guilatrova/gracy/main/img/report-rich-example.png) ### Plotly It requires you to install [plotly 📊](https://github.com/plotly/plotly.py) and [pandas 🐼](https://github.com/pandas-dev/pandas). ```py pokeapi = GracefulPokeAPI() # do stuff with your API plotly_fig = pokeapi.report_status("plotly") plotly_fig.show() ``` Here's an example of how it looks: ![Report](https://raw.githubusercontent.com/guilatrova/gracy/main/img/report-plotly-example.png) ## Replay requests Gracy allows you to replay requests and responses from previous interactions. This is powerful because it allows you to test APIs without latency or consuming your rate limit. Now writing unit tests that relies on third-party APIs is doable. It works in two steps: | **Step** | **Description** | **Hits the API?** | | ------------ | ------------------------------------------------------------------------------ | ----------------- | | 1. Recording | Stores all requests/responses to be later replayed | **Yes** | | 2. Replay | Returns all previously generated responses based on your request as a "replay" | No | ### Recording The effort to record requests/responses is ZERO. You just need to pass a recording config to your Graceful API: ```py from gracy import GracyReplay from gracy.replays.storages.sqlite import SQLiteReplayStorage record_mode = GracyReplay("record", SQLiteReplayStorage("pokeapi.sqlite3")) pokeapi = GracefulPokeAPI(record_mode) ``` **Every request** will be recorded to the defined data source. ### Replay Once you have recorded all your requests you can enable the replay mode: ```py from gracy import GracyReplay from gracy.replays.storages.sqlite import SQLiteReplayStorage replay_mode = GracyReplay("replay", SQLiteReplayStorage("pokeapi.sqlite3")) pokeapi = GracefulPokeAPI(replay_mode) ``` **Every request** will be routed to the defined data source resulting in faster responses. **⚠️ Note that parsers, retries, throttling, and similar configs will work as usual**. ## Resource Namespacing You can have multiple namespaces to organize your API endpoints as you wish. To do so, you just have to inherit from `GracyNamespace` and instantiate it within the `GracyAPI`: ```py from gracy import Gracy, GracyNamespace, GracyConfig class PokemonNamespace(GracyNamespace[PokeApiEndpoint]): async def get_one(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) class BerryNamespace(GracyNamespace[PokeApiEndpoint]): async def get_one(self, name: str): return await self.get(PokeApiEndpoint.GET_BERRY, {"NAME": name}) class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( retry=RETRY, allowed_status_code={HTTPStatus.NOT_FOUND}, parser={HTTPStatus.NOT_FOUND: None}, ) # These will be automatically assigned on init berry: BerryNamespace pokemon: PokemonNamespace ``` And the usage will work as: ```py await pokeapi.pokemon.get_one("pikachu") await pokeapi.berry.get_one("cheri") ``` Note all configs are propagated to namespaces, but namespaces can still have their own which would cause merges when instantiatedg. ## Pagination There're endpoints that may require pagination. For that you can use `GracyPaginator`. For a simple case where you pass `offset` and `limit`, you can use `GracyOffsetPaginator`: ```py from gracy import GracyOffsetPaginator class BerryNamespace(GracyNamespace[PokeApiEndpoint]): @parsed_response(ResourceList) async def list(self, offset: int = 0, limit: int = 20): params = dict(offset=offset, limit=limit) return await self.get(PokeApiEndpoint.BERRY_LIST, params=params) def paginate(self, limit: int = 20) -> GracyOffsetPaginator[ResourceList]: return GracyOffsetPaginator[ResourceList]( gracy_func=self.list, has_next=lambda r: bool(r["next"]) if r else True, page_size=limit, ) ``` and then use it as: ```py async def main(): api = PokeApi() paginator = api.berry.paginate(2) # Just grabs the next page first = await paginator.next_page() print(first) # Resets current page to 0 paginator.set_page(0) # Loop throught it all async for page in paginator: print(page) ``` ## Advanced Usage ### Customizing/Overriding configs per method APIs may return different responses/conditions/payloads based on the endpoint. You can override any `GracyConfig` on a per method basis by using the `@graceful` decorator. NOTE: Use `@graceful_generator` if your function uses `yield`. ```python from gracy import Gracy, GracyConfig, GracefulRetry, graceful, graceful_generator retry = GracefulRetry(...) class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( retry=retry, log_errors=LogEvent( LogLevel.ERROR, "How can I become a pokemon master if {URL} keeps failing with {STATUS}" ), ) @graceful( retry=None, # 👈 Disables retry set in Config log_errors=None, # 👈 Disables log_errors set in Config allowed_status_code=HTTPStatus.NOT_FOUND, parser={ "default": lambda r: r.json()["order"], HTTPStatus.NOT_FOUND: None, }, ) async def maybe_get_pokemon_order(self, name: str): val: str | None = await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) return val @graceful( # 👈 Retry and log_errors are still set for this one strict_status_code=HTTPStatus.OK, parser={"default": lambda r: r.json()["order"]}, ) async def get_pokemon_order(self, name: str): val: str = await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) return val @graceful_generator( # 👈 Retry and log_errors are still set for this one parser={"default": lambda r: r.json()["order"]}, ) async def get_2_pokemons(self): names = ["charmander", "pikachu"] for name in names: r = await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) yield r ``` ### Customizing HTTPx client You might want to modify the HTTPx client settings, do so by: ```py class YourAPIClient(Gracy[str]): class Config: ... def __init__(self, token: token) -> None: self._token = token super().__init__() # 👇 Implement your logic here def _create_client(self) -> httpx.AsyncClient: client = super()._create_client() client.headers = {"Authorization": f"token {self._token}"} # type: ignore return client ``` ### Overriding default request timeout As default Gracy won't enforce a request timeout. You can define your own by setting it on Config as: ```py class GracefulAPI(GracyApi[str]): class Config: BASE_URL = "https://example.com" REQUEST_TIMEOUT = 10.2 # 👈 Here ``` ### Creating a custom Replay data source Gracy was built with extensibility in mind. You can create your own storage to store/load anywhere (e.g. SQL Database), here's an example: ```py import httpx from gracy import GracyReplayStorage class MyCustomStorage(GracyReplayStorage): def prepare(self) -> None: # (Optional) Executed upon API instance creation. ... async def record(self, response: httpx.Response) -> None: ... # REQUIRED. Your logic to store the response object. Note the httpx.Response has request data. async def _load(self, request: httpx.Request) -> httpx.Response: ... # REQUIRED. Your logic to load a response object based on the request. # Usage record_mode = GracyReplay("record", MyCustomStorage()) replay_mode = GracyReplay("replay", MyCustomStorage()) pokeapi = GracefulPokeAPI(record_mode) ``` ### Hooks before/after request You can set up hooks simply by defining `async def before` and `async def after` methods. ⚠️ NOTE: Gracy configs are disabled within these methods which means that retries/parsers/throttling won't take effect inside it. ```py class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( retry=RETRY, allowed_status_code={HTTPStatus.NOT_FOUND}, parser={HTTPStatus.NOT_FOUND: None}, ) def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: self.before_count = 0 self.after_status_counter = defaultdict[HTTPStatus, int](int) self.after_aborts = 0 self.after_retries_counter = 0 super().__init__(*args, **kwargs) async def before(self, context: GracyRequestContext): self.before_count += 1 async def after( self, context: GracyRequestContext, # Current request context response_or_exc: httpx.Response | Exception, # Either the request or an error retry_state: GracefulRetryState | None, # Set when this is generated from a retry ): if retry_state: self.after_retries_counter += 1 if isinstance(response_or_exc, httpx.Response): self.after_status_counter[HTTPStatus(response_or_exc.status_code)] += 1 else: self.after_aborts += 1 async def get_pokemon(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) ``` In the example above invoking `get_pokemon()` will trigger `before()`/`after()` hooks in sequence. #### Common Hooks ##### `HttpHeaderRetryAfterBackOffHook` This hook checks for 429 (TOO MANY REQUESTS), and then reads the [`retry-after` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After). If the value is set, then Gracy pauses **ALL** client requests until the time is over. This behavior can be modified to happen on a per-endpoint basis if `lock_per_endpoint` is True. Example Usage: ```py from gracy.common_hooks import HttpHeaderRetryAfterBackOffHook class GracefulAPI(GracyAPI[Endpoint]): def __init__(self): self._retry_after_hook = HttpHeaderRetryAfterBackOffHook( self._reporter, lock_per_endpoint=True, log_event=LogEvent( LogLevel.WARNING, custom_message=( "{ENDPOINT} produced {STATUS} and requested to wait {RETRY_AFTER}s " "- waiting {RETRY_AFTER_ACTUAL_WAIT}s" ), ), # Wait +10s to avoid this from happening again too soon seconds_processor=lambda secs_requested: secs_requested + 10, ) super().__init__() async def before(self, context: GracyRequestContext): await self._retry_after_hook.before(context) async def after( self, context: GracyRequestContext, response_or_exc: httpx.Response | Exception, retry_state: GracefulRetryState | None, ): retry_after_result = await self._retry_after_hook.after(context, response_or_exc) ``` ##### `RateLimitBackOffHook` This hook checks for 429 (TOO MANY REQUESTS) and locks requests for an arbitrary amount of time defined by you. If the value is set, then Gracy pauses **ALL** client requests until the time is over. This behavior can be modified to happen on a per-endpoint basis if `lock_per_endpoint` is True. ```py from gracy.common_hooks import RateLimitBackOffHook class GracefulAPI(GracyAPI[Endpoint]): def __init__(self): self._ratelimit_backoff_hook = RateLimitBackOffHook( 30, self._reporter, lock_per_endpoint=True, log_event=LogEvent( LogLevel.INFO, custom_message="{UENDPOINT} got rate limited, waiting for {WAIT_TIME}s", ), ) super().__init__() async def before(self, context: GracyRequestContext): await self._ratelimit_backoff_hook.before(context) async def after( self, context: GracyRequestContext, response_or_exc: httpx.Response | Exception, retry_state: GracefulRetryState | None, ): backoff_result = await self._ratelimit_backoff_hook.after(context, response_or_exc) ``` ```py from gracy.common_hooks import HttpHeaderRetryAfterBackOffHook, RateLimitBackOffHook ``` ## 📚 Extra Resources Some good practices I learned over the past years guided Gracy's philosophy, you might benefit by reading: - [How to log](https://guicommits.com/how-to-log-in-python-like-a-pro/) - [How to handle exceptions](https://guicommits.com/handling-exceptions-in-python-like-a-pro/) - [How to structure exceptions](https://guicommits.com/how-to-structure-exception-in-python-like-a-pro/) - [How to use Async correctly](https://guicommits.com/effective-python-async-like-a-pro/) - [Book: Python like a PRO](https://guilatrova.gumroad.com/l/python-like-a-pro) - [Book: Effective Python](https://amzn.to/3bEVHpG) ## Change log See [CHANGELOG](CHANGELOG.md). ## License MIT ## Credits Thanks to the last three startups I worked which forced me to do the same things and resolve the same problems over and over again. I got sick of it and built this lib. Most importantly: **Thanks to God**, who allowed me (a random 🇧🇷 guy) to work for many different 🇺🇸 startups. This is ironic since due to God's grace, I was able to build Gracy. 🙌 Also, thanks to the [httpx](https://github.com/encode/httpx) and [rich](https://github.com/Textualize/rich) projects for the beautiful and simple APIs that powers Gracy. ================================================ FILE: docker-compose.yml ================================================ version: "3.9" services: gracy_mongo: image: mongo:latest restart: always environment: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: example ports: - 27017:27017 volumes: - .mongo/data:/data/db ================================================ FILE: examples/httpbin_post.py ================================================ from __future__ import annotations import asyncio import httpx import typing as t from gracy import Gracy class GracefulHttpbin(Gracy[str]): class Config: BASE_URL = "https://httpbin.org/" async def post_json_example(self): res = await self.post( "post", None, json={"test": "json"}, headers={"header1": "1"} ) return res async def post_data_example(self): res = await self.post("post", None, data="data", headers={"header2": "2"}) return res async def main(): api = GracefulHttpbin() json_res = t.cast(httpx.Response, await api.post_json_example()) data_res = t.cast(httpx.Response, await api.post_data_example()) print(json_res.json()) print("-" * 100) print(data_res.json()) if __name__ == "__main__": asyncio.run(main()) ================================================ FILE: examples/memory.py ================================================ from __future__ import annotations import httpx from dataclasses import dataclass from time import sleep from gracy import ( BaseEndpoint, Gracy, GracyRequestContext, ) from gracy.exceptions import GracyUserDefinedException class PokemonNotFound(GracyUserDefinedException): BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" def _format_message( self, request_context: GracyRequestContext, response: httpx.Response ) -> str: format_args = self._build_default_args() name = request_context.endpoint_args.get("NAME", "Unknown") return self.BASE_MESSAGE.format(NAME=name, **format_args) class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" GET_GENERATION = "/generation/{ID}" class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" pass @dataclass class Test: pass def main(): while True: GracefulPokeAPI() sleep(1) if __name__ == "__main__": main() ================================================ FILE: examples/pokeapi.py ================================================ from __future__ import annotations import asyncio import httpx import typing as t from http import HTTPStatus from gracy import ( BaseEndpoint, GracefulRetry, Gracy, GracyRequestContext, LogEvent, LogLevel, graceful, ) from gracy.exceptions import GracyUserDefinedException retry = GracefulRetry( delay=1, max_attempts=3, delay_modifier=1.5, retry_on=None, log_before=LogEvent(LogLevel.WARNING), log_after=LogEvent(LogLevel.WARNING), log_exhausted=LogEvent(LogLevel.CRITICAL), behavior="pass", ) class PokemonNotFound(GracyUserDefinedException): BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" def _format_message( self, request_context: GracyRequestContext, response: httpx.Response ) -> str: format_args = self._build_default_args() name = request_context.endpoint_args.get("NAME", "Unknown") return self.BASE_MESSAGE.format(NAME=name, **format_args) class ServerIsOutError(Exception): pass class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" GET_GENERATION = "/generation/{ID}" class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" @graceful( strict_status_code={HTTPStatus.OK}, retry=retry, log_request=LogEvent(LogLevel.WARNING), log_errors=LogEvent( LogLevel.ERROR, lambda r: "Request failed with {STATUS}" f" and it was {'' if r.is_redirect else 'NOT'} redirected" if r else "", ), parser={ "default": lambda r: r.json()["name"], HTTPStatus.NOT_FOUND: PokemonNotFound, HTTPStatus.INTERNAL_SERVER_ERROR: ServerIsOutError, }, ) async def get_pokemon(self, name: str): self.get return await self.get[t.Optional[str]]( PokeApiEndpoint.GET_POKEMON, {"NAME": name} ) async def get_generation(self, gen: int): return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) pokeapi = GracefulPokeAPI() pokeapi_two = GracefulPokeAPI() async def main(): try: p1 = await pokeapi.get_pokemon("pikachu") try: p2 = await pokeapi_two.get_pokemon("doesnt-exist") except PokemonNotFound as ex: p2 = str(ex) await pokeapi.get_generation(1) print("P1: result of get_pokemon:", p1) print("P2: result of get_pokemon:", p2) finally: pokeapi.report_status("list") asyncio.run(main()) ================================================ FILE: examples/pokeapi_limit_concurrency.py ================================================ from __future__ import annotations import asyncio import logging import time from datetime import timedelta from http import HTTPStatus from rich import print from rich.logging import RichHandler logging.basicConfig( level=logging.INFO, format="%(message)s", datefmt="[%X]", handlers=[RichHandler()], ) from gracy import ( # noqa: E402 BaseEndpoint, ConcurrentRequestLimit, GracefulRetry, Gracy, GracyConfig, LogEvent, LogLevel, graceful, ) CONCURRENCY = ( ConcurrentRequestLimit( 2, limit_per_uurl=False, log_limit_reached=LogEvent( LogLevel.ERROR, custom_message="{URL} hit {CONCURRENT_REQUESTS} ongoing concurrent request", ), log_limit_freed=LogEvent(LogLevel.INFO, "{URL} is free to request"), ), ) RETRY = GracefulRetry( delay=0, # Force throttling to work max_attempts=3, retry_on=None, log_after=LogEvent(LogLevel.INFO), log_exhausted=LogEvent(LogLevel.ERROR), behavior="pass", ) class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" GET_GENERATION = "/generation/{ID}" class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( strict_status_code={HTTPStatus.OK}, retry=RETRY, concurrent_requests=CONCURRENCY, parser={ "default": lambda r: r.json(), HTTPStatus.NOT_FOUND: None, }, ) @graceful( parser={"default": lambda r: r.json()["order"], HTTPStatus.NOT_FOUND: None} ) async def get_pokemon(self, name: str): await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) async def get_generation(self, gen: int): return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) @graceful(parser={"default": lambda r: r}) async def slow_req(self, s: int): await self.get("https://httpbin.org/delay/{DELAY}", dict(DELAY=str(s))) pokeapi = GracefulPokeAPI() async def main(): pokemon_names = [ "bulbasaur", "charmander", "squirtle", "pikachu", "jigglypuff", "mewtwo", "gyarados", "dragonite", "mew", "chikorita", "cyndaquil", "totodile", "pichu", "togepi", "ampharos", "typhlosion", "feraligatr", "espeon", "umbreon", "lugia", "ho-oh", "treecko", "torchic", "mudkip", "gardevoir", "sceptile", "blaziken", "swampert", "rayquaza", "latias", "latios", "lucario", "garchomp", "darkrai", "giratina", # (1) this fails, so good to test retry "arceus", "snivy", "tepig", "oshawott", "zekrom", "reshiram", "victini", "chespin", "fennekin", "froakie", "xerneas", "yveltal", "zygarde", # (2) this fails, so good to test retry "decidueye", "incineroar", ] # pokemon_names = pokemon_names[:10] try: start = time.time() pokemon_reqs = [ asyncio.create_task(pokeapi.get_pokemon(name)) for name in pokemon_names[:10] ] slow_reqs = [asyncio.create_task(pokeapi.slow_req(s)) for s in range(3)] pokemon_reqs += [ asyncio.create_task(pokeapi.get_pokemon(name)) for name in pokemon_names[10:20] ] slow_reqs += [asyncio.create_task(pokeapi.slow_req(s)) for s in range(3)] pokemon_reqs += [ asyncio.create_task(pokeapi.get_pokemon(name)) for name in pokemon_names[20:] ] gen_reqs = [ asyncio.create_task(pokeapi.get_generation(gen)) for gen in range(1, 4) ] await asyncio.gather(*pokemon_reqs, *gen_reqs, *slow_reqs) await pokeapi.get_pokemon("hitmonchan") elapsed = time.time() - start print(f"All requests took {timedelta(seconds=elapsed)}s to finish") finally: plotly = pokeapi.report_status("plotly") plotly.show() asyncio.run(main()) ================================================ FILE: examples/pokeapi_namespaces.py ================================================ from __future__ import annotations import asyncio import typing as t from http import HTTPStatus from gracy import ( BaseEndpoint, Gracy, GracyConfig, GracyNamespace, LogEvent, LogLevel, ) from rich import print RESP_TYPE = t.Union[t.Dict[str, t.Any], None] class PokeApiEndpoint(BaseEndpoint): BERRY = "/berry/{KEY}" BERRY_FLAVOR = "/berry-flavor/{KEY}" BERRY_FIRMNESS = "/berry-firmness/{KEY}" POKEMON = "/pokemon/{KEY}" POKEMON_COLOR = "/pokemon-color/{KEY}" POKEMON_FORM = "/pokemon-form/{KEY}" class PokeApiBerryNamespace(GracyNamespace[PokeApiEndpoint]): async def get_this(self, name_or_id: t.Union[str, int]): return await self.get[RESP_TYPE]( PokeApiEndpoint.BERRY, dict(KEY=str(name_or_id)) ) async def get_flavor(self, name_or_id: t.Union[str, int]): return await self.get[RESP_TYPE]( PokeApiEndpoint.BERRY_FLAVOR, dict(KEY=str(name_or_id)) ) async def get_firmness(self, name_or_id: t.Union[str, int]): return await self.get[RESP_TYPE]( PokeApiEndpoint.BERRY_FIRMNESS, dict(KEY=str(name_or_id)) ) class PokeApiPokemonNamespace(GracyNamespace[PokeApiEndpoint]): async def get_this(self, name_or_id: t.Union[str, int]): return await self.get[RESP_TYPE]( PokeApiEndpoint.POKEMON, dict(KEY=str(name_or_id)) ) async def get_color(self, name_or_id: t.Union[str, int]): return await self.get[RESP_TYPE]( PokeApiEndpoint.POKEMON_COLOR, dict(KEY=str(name_or_id)) ) async def get_form(self, name_or_id: t.Union[str, int]): return await self.get[RESP_TYPE]( PokeApiEndpoint.POKEMON_FORM, dict(KEY=str(name_or_id)) ) class PokeApi(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" REQUEST_TIMEOUT = 5.0 SETTINGS = GracyConfig( parser={ HTTPStatus.OK: lambda resp: resp.json(), HTTPStatus.NOT_FOUND: None, }, allowed_status_code=HTTPStatus.NOT_FOUND, log_errors=LogEvent(LogLevel.ERROR), ) berry: PokeApiBerryNamespace pokemon: PokeApiPokemonNamespace async def main(): api = PokeApi() berry = api.berry.get_this("cheri") berry_flavor = api.berry.get_flavor("spicy") pikachu = api.pokemon.get_this("pikachu") black = api.pokemon.get_color("black") results = await asyncio.gather(berry, berry_flavor, pikachu, black) for content in results: print(content) api.report_status("rich") if __name__ == "__main__": asyncio.run(main()) ================================================ FILE: examples/pokeapi_replay.py ================================================ from __future__ import annotations import asyncio import httpx from http import HTTPStatus from gracy import ( BaseEndpoint, GracefulRetry, Gracy, GracyReplay, GracyRequestContext, LogEvent, LogLevel, ReplayLogEvent, graceful, ) from gracy.exceptions import GracyUserDefinedException from gracy.replays.storages.sqlite import SQLiteReplayStorage retry = GracefulRetry( delay=1, max_attempts=3, delay_modifier=1.5, retry_on=None, log_before=LogEvent(LogLevel.WARNING), log_after=LogEvent(LogLevel.WARNING), log_exhausted=LogEvent(LogLevel.CRITICAL), behavior="pass", ) record_mode = GracyReplay( "record", SQLiteReplayStorage("pokeapi.sqlite3"), ) replay_mode = GracyReplay( "replay", SQLiteReplayStorage("pokeapi.sqlite3"), log_replay=ReplayLogEvent(LogLevel.WARNING, frequency=1), ) class PokemonNotFound(GracyUserDefinedException): BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" def _format_message( self, request_context: GracyRequestContext, response: httpx.Response ) -> str: format_args = self._build_default_args() name = request_context.endpoint_args.get("NAME", "Unknown") return self.BASE_MESSAGE.format(NAME=name, **format_args) class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" GET_GENERATION = "/generation/{ID}" class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" @graceful( strict_status_code={HTTPStatus.OK}, retry=retry, log_errors=LogEvent( LogLevel.ERROR, lambda r: "Request failed with {STATUS}" f" and it was {'' if r.is_redirect else 'NOT'} redirected" if r else "", ), log_response=LogEvent(LogLevel.INFO), parser={ "default": lambda r: r.json()["name"], HTTPStatus.NOT_FOUND: PokemonNotFound, HTTPStatus.INTERNAL_SERVER_ERROR: PokemonNotFound, }, ) async def get_pokemon(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) async def get_generation(self, gen: int): return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) async def main(replay_mode: GracyReplay): pokeapi = GracefulPokeAPI(replay_mode) poke_names = {"pikachu", "elekid", "charmander", "blaziken", "hitmonchan"} try: get_pokemons = [ asyncio.create_task(pokeapi.get_pokemon(name)) for name in poke_names ] get_gens = [ asyncio.create_task(pokeapi.get_generation(gen_id)) for gen_id in range(1, 3) ] await asyncio.gather(*(get_pokemons + get_gens)) finally: pokeapi.report_status("rich") print("-" * 100) pokeapi.report_status("list") print("-" * 100) pokeapi.report_status("logger") asyncio.run(main(replay_mode)) ================================================ FILE: examples/pokeapi_replay_mongo.py ================================================ from __future__ import annotations import asyncio import httpx from http import HTTPStatus from gracy import ( BaseEndpoint, GracefulRetry, Gracy, GracyReplay, GracyRequestContext, LogEvent, LogLevel, graceful, ) from gracy.exceptions import GracyUserDefinedException from gracy.replays.storages.pymongo import MongoCredentials, MongoReplayStorage retry = GracefulRetry( delay=1, max_attempts=3, delay_modifier=1.5, retry_on=None, log_before=LogEvent(LogLevel.WARNING), log_after=LogEvent(LogLevel.WARNING), log_exhausted=LogEvent(LogLevel.CRITICAL), behavior="pass", ) mongo_container = MongoCredentials( host="localhost", username="root", password="example" ) record_mode = GracyReplay("record", MongoReplayStorage(mongo_container)) replay_mode = GracyReplay("replay", MongoReplayStorage(mongo_container)) class PokemonNotFound(GracyUserDefinedException): BASE_MESSAGE = "Unable to find a pokemon with the name [{NAME}] at {URL} due to {STATUS} status" def _format_message( self, request_context: GracyRequestContext, response: httpx.Response ) -> str: format_args = self._build_default_args() name = request_context.endpoint_args.get("NAME", "Unknown") return self.BASE_MESSAGE.format(NAME=name, **format_args) class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" GET_GENERATION = "/generation/{ID}" class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" @graceful( strict_status_code={HTTPStatus.OK}, retry=retry, log_errors=LogEvent( LogLevel.ERROR, lambda r: "Request failed with {STATUS}" f" and it was {'' if r.is_redirect else 'NOT'} redirected" if r else "", ), parser={ "default": lambda r: r.json()["name"], HTTPStatus.NOT_FOUND: PokemonNotFound, HTTPStatus.INTERNAL_SERVER_ERROR: PokemonNotFound, }, ) async def get_pokemon(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) async def get_generation(self, gen: int): return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) async def main(replay_mode: GracyReplay): pokeapi = GracefulPokeAPI(replay_mode) poke_names = {"pikachu", "elekid", "charmander", "blaziken", "hitmonchan"} try: get_pokemons = [ asyncio.create_task(pokeapi.get_pokemon(name)) for name in poke_names ] get_gens = [ asyncio.create_task(pokeapi.get_generation(gen_id)) for gen_id in range(1, 3) ] await asyncio.gather(*(get_pokemons + get_gens)) finally: pokeapi.report_status("rich") print("-" * 100) pokeapi.report_status("list") print("-" * 100) pokeapi.report_status("logger") asyncio.run(main(replay_mode)) ================================================ FILE: examples/pokeapi_retry.py ================================================ from __future__ import annotations import asyncio from http import HTTPStatus from gracy import ( BaseEndpoint, GracefulRetry, Gracy, GracyReplay, LogEvent, LogLevel, graceful, ) from gracy.replays.storages.sqlite import SQLiteReplayStorage retry = GracefulRetry( delay=1, max_attempts=3, delay_modifier=1.2, retry_on=None, log_before=LogEvent(LogLevel.WARNING), log_after=LogEvent(LogLevel.WARNING), log_exhausted=LogEvent(LogLevel.CRITICAL), behavior="pass", ) class ServerIsOutError(Exception): pass class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" @graceful( strict_status_code={HTTPStatus.OK}, retry=retry, log_errors=LogEvent(LogLevel.ERROR), parser={ "default": lambda r: r.json()["name"], HTTPStatus.NOT_FOUND: None, HTTPStatus.INTERNAL_SERVER_ERROR: ServerIsOutError, }, ) async def get_pokemon(self, name: str): return await self.get[str](PokeApiEndpoint.GET_POKEMON, {"NAME": name}) record = GracyReplay("record", SQLiteReplayStorage("pokeapi.sqlite3")) pokeapi = GracefulPokeAPI(record) async def main(): try: p1: str | None = await pokeapi.get_pokemon("pikachu") # 1 req = 200 print("P1: result of get_pokemon:", p1) p2: str | None = await pokeapi.get_pokemon("doesnt-exist") # 1+3 req = 404 print("P2: result of get_pokemon:", p2) finally: pokeapi.report_status("rich") asyncio.run(main()) ================================================ FILE: examples/pokeapi_throttle.py ================================================ from __future__ import annotations import asyncio import time import typing as t from datetime import timedelta from http import HTTPStatus from gracy import ( BaseEndpoint, GracefulRetry, GracefulThrottle, Gracy, GracyConfig, LogEvent, LogLevel, ThrottleRule, graceful, ) from rich import print RETRY = GracefulRetry( delay=0, # Force throttling to work max_attempts=3, retry_on=None, log_after=LogEvent(LogLevel.WARNING), log_exhausted=LogEvent(LogLevel.CRITICAL), behavior="pass", ) THROTTLE_RULE = ThrottleRule(r".*", 4, timedelta(seconds=2)) class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" GET_GENERATION = "/generation/{ID}" class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( strict_status_code={HTTPStatus.OK}, retry=RETRY, parser={ "default": lambda r: r.json(), HTTPStatus.NOT_FOUND: None, }, throttling=GracefulThrottle( rules=THROTTLE_RULE, log_limit_reached=LogEvent(LogLevel.ERROR), log_wait_over=LogEvent(LogLevel.WARNING), ), ) @graceful( parser={"default": lambda r: r.json()["order"], HTTPStatus.NOT_FOUND: None} ) async def get_pokemon(self, name: str): val = t.cast( t.Optional[str], await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) ) if val: print(f"{name} is #{val} in the pokedex") else: print(f"{name} was not found") async def get_generation(self, gen: int): return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) pokeapi = GracefulPokeAPI() async def main(): pokemon_names = [ "bulbasaur", "charmander", "squirtle", "pikachu", "jigglypuff", "mewtwo", "gyarados", "dragonite", "mew", "chikorita", "cyndaquil", "totodile", "pichu", "togepi", "ampharos", "typhlosion", "feraligatr", "espeon", "umbreon", "lugia", "ho-oh", "treecko", "torchic", "mudkip", "gardevoir", "sceptile", "blaziken", "swampert", "rayquaza", "latias", "latios", "lucario", "garchomp", "darkrai", "giratina", # (1) this fails, so good to test retry "arceus", "snivy", "tepig", "oshawott", "zekrom", "reshiram", "victini", "chespin", "fennekin", "froakie", "xerneas", "yveltal", "zygarde", # (2) this fails, so good to test retry "decidueye", "incineroar", ] # pokemon_names = pokemon_names[:10] print( f"Will query {len(pokemon_names)} pokemons concurrently - {str(THROTTLE_RULE)}" ) try: start = time.time() pokemon_reqs = [ asyncio.create_task(pokeapi.get_pokemon(name)) for name in pokemon_names ] gen_reqs = [ asyncio.create_task(pokeapi.get_generation(gen)) for gen in range(1, 4) ] await asyncio.gather(*pokemon_reqs, *gen_reqs) elapsed = time.time() - start print(f"All requests took {timedelta(seconds=elapsed)}s to finish") finally: pokeapi.report_status("rich") pokeapi.report_status("list") pokeapi._throttle_controller.debug_print() # type: ignore asyncio.run(main()) ================================================ FILE: examples/pokestarwarsapi.py ================================================ from __future__ import annotations import asyncio from http import HTTPStatus from gracy import BaseEndpoint, Gracy, LogEvent, LogLevel, graceful class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" @graceful( strict_status_code={HTTPStatus.OK}, log_request=LogEvent(LogLevel.INFO), parser={ "default": lambda r: r.json()["name"], HTTPStatus.NOT_FOUND: None, }, ) async def get_pokemon(self, name: str): return await self.get[str](PokeApiEndpoint.GET_POKEMON, {"NAME": name}) class StarWarsAPI(Gracy[str]): class Config: BASE_URL = "https://swapi.dev/api/" @graceful( strict_status_code=HTTPStatus.OK, log_request=LogEvent(LogLevel.INFO), parser={"default": lambda r: r.json()["name"]}, ) async def get_person(self, person_id: int): return await self.get[str]("people/{PERSON_ID}", {"PERSON_ID": str(person_id)}) pokeapi = GracefulPokeAPI() swapi = StarWarsAPI() async def main(): try: pk: str | None = await pokeapi.get_pokemon("pikachu") sw: str = await swapi.get_person(1) print("PK: result of get_pokemon:", pk) print("SW: result of get_person:", sw) finally: pokeapi.report_status("rich") asyncio.run(main()) ================================================ FILE: pyproject.toml ================================================ [tool.poetry] name = "gracy" version = "1.34.0" description = "Gracefully manage your API interactions" authors = ["Guilherme Latrova "] license = "MIT" keywords = ["api", "throttling", "http", "https", "async", "retry"] readme = "README.md" homepage = "https://github.com/guilatrova/gracy" repository = "https://github.com/guilatrova/gracy" include = ["LICENSE", "py.typed"] classifiers = [ "Development Status :: 4 - Beta", "Environment :: Web Environment", "Framework :: AsyncIO", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Topic :: Internet :: WWW/HTTP", ] packages = [{ include = "gracy", from = "src" }] [tool.poetry.urls] "Changelog" = "https://github.com/guilatrova/gracy/blob/main/CHANGELOG.md" [tool.semantic_release] version_variable = ["src/gracy/__init__.py:__version__"] version_toml = ["pyproject.toml:tool.poetry.version"] branch = "main" upload_to_pypi = true upload_to_release = true build_command = "pip install poetry && poetry build" [tool.poetry.dependencies] python = ">=3.8.1,<4.0" httpx = ">=0.23.0" rich = { version = "*", optional = true } pymongo = { version = "*", optional = true } typing-extensions = "^4.9.0" # It should be python = "<3.10" if we didn't use the 'deprecated' import from PEP 702 [tool.poetry.group.dev.dependencies] python-semantic-release = "^7.33.0" pre-commit = "^3.5.0" rich = "^13.2.0" pymongo = "^4.3.3" pytest = "^7.2.1" pytest-asyncio = "^0.20.3" ruff = "^0.1.6" pyright = "^1.1.351" [tool.poetry.extras] rich = ["rich"] pymongo = ["pymongo"] plotly = ["plotly", "pandas"] [tool.ruff.lint.isort] extra-standard-library = ["pytest", "httpx"] required-imports = ["from __future__ import annotations"] # https://microsoft.github.io/pyright/#/configuration [tool.pyright] include = ["src"] pythonVersion = "3.8" pythonPlatform = "All" reportMissingImports = "warning" reportIncompatibleVariableOverride = "none" [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = "src/tests" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" ================================================ FILE: src/gracy/__init__.py ================================================ """Gracefully manage your API interactions""" from __future__ import annotations import logging from . import common_hooks, exceptions, replays from ._core import Gracy, GracyNamespace, graceful, graceful_generator from ._models import ( DEFAULT_CONFIG, BaseEndpoint, ConcurrentRequestLimit, GracefulRetry, GracefulRetryState, GracefulThrottle, GracefulValidator, GracyConfig, GracyRequestContext, LogEvent, LogLevel, OverrideRetryOn, ThrottleRule, ) from ._paginator import GracyOffsetPaginator, GracyPaginator from ._reports._models import GracyAggregatedRequest, GracyAggregatedTotal, GracyReport from ._types import generated_parsed_response, parsed_response from .replays.storages._base import GracyReplay, GracyReplayStorage, ReplayLogEvent logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") __version__ = "1.34.0" __all__ = [ "exceptions", # Core "Gracy", "GracyNamespace", "graceful", "graceful_generator", # Paginatior "GracyPaginator", "GracyOffsetPaginator", # Models "BaseEndpoint", "GracefulRetry", "OverrideRetryOn", "GracefulRetryState", "GracefulValidator", "GracyRequestContext", "LogEvent", "LogLevel", "GracefulThrottle", "ThrottleRule", "GracyConfig", "DEFAULT_CONFIG", "ConcurrentRequestLimit", # Replays "replays", "GracyReplay", "GracyReplayStorage", "ReplayLogEvent", # Reports "GracyReport", "GracyAggregatedTotal", "GracyAggregatedRequest", # Hooks "common_hooks", # Types "parsed_response", "generated_parsed_response", ] ================================================ FILE: src/gracy/_configs.py ================================================ from __future__ import annotations from contextlib import contextmanager from contextvars import ContextVar from ._models import GracyConfig custom_config_context: ContextVar[GracyConfig | None] = ContextVar( "gracy_context", default=None ) within_hook_context: ContextVar[bool] = ContextVar("within_hook_context", default=False) @contextmanager def custom_gracy_config(config: GracyConfig): token = custom_config_context.set(config) try: yield finally: try: custom_config_context.reset(token) except Exception: pass # Best effort @contextmanager def within_hook(): token = within_hook_context.set(True) try: yield finally: within_hook_context.reset(token) ================================================ FILE: src/gracy/_core.py ================================================ from __future__ import annotations import asyncio import httpx import inspect import logging import sys import typing as t import weakref from asyncio import sleep from contextlib import asynccontextmanager from http import HTTPStatus from time import time from gracy.replays._wrappers import record_mode, replay_mode, smart_replay_mode from ._configs import ( custom_config_context, custom_gracy_config, within_hook, within_hook_context, ) from ._general import extract_request_kwargs from ._loggers import ( DefaultLogMessage, process_log_after_request, process_log_before_request, process_log_concurrency_freed, process_log_concurrency_limit, process_log_retry, process_log_throttle, ) from ._models import ( CONCURRENT_REQUEST_TYPE, DEFAULT_CONFIG, LOG_EVENT_TYPE, PARSER_TYPE, THROTTLE_LOCKER, UNSET_VALUE, ConcurrentRequestLimit, Endpoint, GracefulRequest, GracefulRetry, GracefulRetryState, GracefulValidator, GracyConfig, GracyRequestContext, LogEvent, ThrottleController, ThrottleRule, Unset, ) from ._reports._builders import ReportBuilder from ._reports._printers import PRINTERS, print_report from ._validators import AllowedStatusValidator, DefaultValidator, StrictStatusValidator from .exceptions import GracyParseFailed, GracyRequestFailed from .replays.storages._base import GracyReplay if sys.version_info >= (3, 10): from typing import ParamSpec else: from typing_extensions import ParamSpec logger = logging.getLogger("gracy") ANY_COROUTINE = t.Coroutine[t.Any, t.Any, t.Any] P = ParamSpec("P") HTTP_T = t.TypeVar("HTTP_T") GRACEFUL_T = t.TypeVar("GRACEFUL_T", bound=ANY_COROUTINE) GRACEFUL_GEN_T = t.TypeVar("GRACEFUL_GEN_T", bound=t.AsyncGenerator[t.Any, t.Any]) BEFORE_HOOK_TYPE = t.Callable[[GracyRequestContext], t.Awaitable[None]] AFTER_HOOK_TYPE = t.Callable[ [ GracyRequestContext, t.Union[httpx.Response, Exception], t.Optional[GracefulRetryState], ], t.Awaitable[None], ] async def _gracefully_throttle( report: ReportBuilder, controller: ThrottleController, request_context: GracyRequestContext, ): if isinstance(request_context.active_config.throttling, Unset): return if throttling := request_context.active_config.throttling: has_been_throttled = True while has_been_throttled: wait_per_rule: list[tuple[ThrottleRule, float]] = [ (rule, wait_time) for rule in throttling.rules if (wait_time := rule.calculate_await_time(controller)) > 0.0 ] if wait_per_rule: rule: ThrottleRule await_time: float rule, await_time = max(wait_per_rule, key=lambda x: x[1]) # type: ignore if THROTTLE_LOCKER.is_rule_throttled(rule): report.throttled(request_context) await asyncio.sleep(await_time) continue with THROTTLE_LOCKER.lock_rule(rule): if throttling.log_limit_reached: process_log_throttle( throttling.log_limit_reached, DefaultLogMessage.THROTTLE_HIT, await_time, rule, request_context, ) report.throttled(request_context) await asyncio.sleep(await_time) if throttling.log_wait_over: process_log_throttle( throttling.log_wait_over, DefaultLogMessage.THROTTLE_DONE, await_time, rule, request_context, ) else: has_been_throttled = False async def _gracefully_retry( report: ReportBuilder, throttle_controller: ThrottleController, last_response: t.Optional[httpx.Response], last_err: t.Optional[Exception], before_hook: BEFORE_HOOK_TYPE, after_hook: AFTER_HOOK_TYPE, request: GracefulRequest, request_context: GracyRequestContext, validators: t.List[GracefulValidator], ) -> GracefulRetryState: config = request_context.active_config retry = t.cast(GracefulRetry, config.retry) state = retry.create_state(last_response, last_err) response = last_response resulting_exc: t.Optional[Exception] = None failing = True while failing: state.increment(response) if state.cant_retry: break if retry.log_before: process_log_retry( retry.log_before, DefaultLogMessage.RETRY_BEFORE, request_context, state ) await sleep(state.delay) await _gracefully_throttle(report, throttle_controller, request_context) throttle_controller.init_request(request_context) start = 0 try: await before_hook(request_context) start = time() response = await request() except Exception as request_err: resulting_exc = GracyRequestFailed(request_context, request_err) report.track(request_context, request_err, start) await after_hook(request_context, request_err, state) else: report.track(request_context, response, start) await after_hook(request_context, response, state) finally: report.retried(request_context) if response: resulting_exc = None for validator in validators: try: validator.check(response) except Exception as ex: resulting_exc = ex break state.last_response = response state.last_exc = resulting_exc # Even if all validators are passing, we check whether # it should retry for cases like: # e.g. Allow = 404 (so it's a success), # but Retry it up to 3 times to see whether it becomes 200 if config.should_retry(response, resulting_exc) is False: state.success = True failing = False if retry.log_after: process_log_retry( retry.log_after, DefaultLogMessage.RETRY_AFTER, request_context, state, response, ) if ( state.cant_retry and config.should_retry(state.last_response, resulting_exc) and retry.log_exhausted ): process_log_retry( retry.log_exhausted, DefaultLogMessage.RETRY_EXHAUSTED, request_context, state, response, ) return state def _maybe_parse_result( active_config: GracyConfig, request_context: GracyRequestContext, result: httpx.Response, ): if active_config.parser and not isinstance(active_config.parser, Unset): default_fallback = active_config.parser.get("default", UNSET_VALUE) parse_result = active_config.parser.get(result.status_code, default_fallback) if not isinstance(parse_result, Unset): if isinstance(parse_result, type) and issubclass(parse_result, Exception): raise parse_result(request_context, result) elif callable(parse_result): try: return parse_result(result) except Exception as ex: raise GracyParseFailed(result) from ex else: return parse_result return result async def _gracify( report: ReportBuilder, throttle_controller: ThrottleController, replay: t.Optional[GracyReplay], before_hook: BEFORE_HOOK_TYPE, after_hook: AFTER_HOOK_TYPE, request: GracefulRequest, request_context: GracyRequestContext, ): active_config = request_context.active_config if isinstance(active_config.log_request, LogEvent): process_log_before_request(active_config.log_request, request_context) resulting_exc: t.Optional[Exception] = None do_throttle = True if replay and replay.disable_throttling: replay_available = await replay.has_replay(request.request) if replay_available: do_throttle = False if do_throttle: await _gracefully_throttle(report, throttle_controller, request_context) throttle_controller.init_request(request_context) start = 0 try: await before_hook(request_context) start = time() response = await request() except Exception as request_err: resulting_exc = GracyRequestFailed(request_context, request_err) response = None report.track(request_context, resulting_exc, start) await after_hook(request_context, resulting_exc, None) else: report.track(request_context, response, start) await after_hook(request_context, response, None) if active_config.log_response and isinstance(active_config.log_response, LogEvent): process_log_after_request( active_config.log_response, DefaultLogMessage.AFTER, request_context, response, ) validators: list[GracefulValidator] = [] if active_config.strict_status_code and not isinstance( active_config.strict_status_code, Unset ): validators.append(StrictStatusValidator(active_config.strict_status_code)) elif active_config.allowed_status_code and not isinstance( active_config.allowed_status_code, Unset ): validators.append(AllowedStatusValidator(active_config.allowed_status_code)) else: validators.append(DefaultValidator()) if isinstance(active_config.validators, GracefulValidator): validators.append(active_config.validators) elif isinstance(active_config.validators, t.Iterable): validators += active_config.validators if response: for validator in validators: try: validator.check(response) except Exception as ex: resulting_exc = ex break retry_result: t.Optional[GracefulRetryState] = None if active_config.should_retry(response, resulting_exc): retry_result = await _gracefully_retry( report, throttle_controller, response, resulting_exc, before_hook, after_hook, request, request_context, validators, ) response = retry_result.last_response resulting_exc = retry_result.last_exc did_request_fail = bool(resulting_exc) if did_request_fail: if active_config.log_errors and isinstance(active_config.log_errors, LogEvent): process_log_after_request( active_config.log_errors, DefaultLogMessage.ERRORS, request_context, response, ) must_break = True if ( isinstance(active_config.retry, GracefulRetry) and active_config.retry.behavior == "pass" ): must_break = False if resulting_exc and must_break: raise resulting_exc final_result = ( _maybe_parse_result(active_config, request_context, response) if response else None ) return final_result class OngoingRequestsTracker: def __init__(self) -> None: self._count = 0 self._previously_limited = False @property def count(self) -> int: return self._count @asynccontextmanager async def request( self, context: GracyRequestContext, concurrent_request: t.Optional[ConcurrentRequestLimit], ): has_been_limited = False semaphore = None try: if concurrent_request is None: self._count += 1 yield return semaphore = concurrent_request.get_semaphore(context) has_been_limited = semaphore.locked() await semaphore.acquire() self._count += 1 if has_been_limited and self._previously_limited is False: if isinstance(concurrent_request.log_limit_reached, LogEvent): process_log_concurrency_limit( concurrent_request.log_limit_reached, concurrent_request.limit, context, ) if self._previously_limited and has_been_limited is False: if isinstance(concurrent_request.log_limit_freed, LogEvent): process_log_concurrency_freed( concurrent_request.log_limit_freed, context ) yield finally: if semaphore: semaphore.release() self._previously_limited = has_been_limited self._count -= 1 DISABLED_GRACY_CONFIG: t.Final = GracyConfig( strict_status_code=None, allowed_status_code=None, validators=None, retry=None, log_request=None, log_response=None, log_errors=None, parser=None, ) class Gracy(t.Generic[Endpoint]): """Helper class that provides a standard way to create an Requester using inheritance. """ _reporter: ReportBuilder = ReportBuilder() _throttle_controller: ThrottleController = ThrottleController() class Config: BASE_URL: str = "" REQUEST_TIMEOUT: t.Optional[float] = None SETTINGS: GracyConfig = DEFAULT_CONFIG def __init__( self, replay: t.Optional[GracyReplay] = None, DEBUG_ENABLED: bool = False, **kwargs: t.Any, ) -> None: self.DEBUG_ENABLED = DEBUG_ENABLED self._base_config = t.cast( GracyConfig, getattr(self.Config, "SETTINGS", DEFAULT_CONFIG) ) self._client = self._create_client(**kwargs) self.replays = replay self._ongoing_tracker = OngoingRequestsTracker() self._post_init() self._init_typed_http_methods() def _init_typed_http_methods(self): gracy_ref = weakref.ref(self) class HTTPMethod(t.Generic[HTTP_T]): def __new__( cls, endpoint: t.Union[Endpoint, str], endpoint_args: t.Optional[t.Dict[str, str]] = None, *args: t.Any, **kwargs: t.Any, ): myself_instance = super().__new__(cls) return myself_instance.execute(endpoint, endpoint_args, *args, **kwargs) def _get_gracy_instance(self): gracy_instance = gracy_ref() if gracy_instance is None: raise ReferenceError( "Gracy instance has been garbage collected - Should never happen" ) return gracy_instance async def execute( self, endpoint: t.Union[Endpoint, str], endpoint_args: t.Optional[t.Dict[str, str]] = None, *args: t.Any, **kwargs: t.Any, ): method_name = type(self).__name__.upper() coro = await self._get_gracy_instance()._request( method_name, endpoint, endpoint_args, *args, **kwargs ) return t.cast(HTTP_T, coro) class Get(t.Generic[HTTP_T], HTTPMethod[HTTP_T]): pass class Post(t.Generic[HTTP_T], HTTPMethod[HTTP_T]): pass class Put(t.Generic[HTTP_T], HTTPMethod[HTTP_T]): pass class Patch(t.Generic[HTTP_T], HTTPMethod[HTTP_T]): pass class Delete(t.Generic[HTTP_T], HTTPMethod[HTTP_T]): pass class Head(t.Generic[HTTP_T], HTTPMethod[HTTP_T]): pass class Options(t.Generic[HTTP_T], HTTPMethod[HTTP_T]): pass self.get = Get self.post = Post self.put = Put self.patch = Patch self.delete = Delete self.head = Head self.options = Options def _post_init(self): """Initializes namespaces and replays after init""" if self.replays: self.replays.storage.prepare() self._instantiate_namespaces() def _instantiate_namespaces(self): annotations = self.__annotations__ for attr_name, attr_type in annotations.items(): if isinstance(attr_type, str): resolved_module = __import__(self.__module__, fromlist=[attr_type]) klass = getattr(resolved_module, attr_type, None) elif inspect.isclass(attr_type): klass = attr_type else: klass = None if klass and issubclass(klass, GracyNamespace): setattr(self, attr_name, klass(self)) @property def ongoing_requests_count(self) -> int: return self._ongoing_tracker.count def _create_client(self, **kwargs: t.Any) -> httpx.AsyncClient: base_url = getattr(self.Config, "BASE_URL", "") request_timeout = getattr(self.Config, "REQUEST_TIMEOUT", None) return httpx.AsyncClient(base_url=str(base_url), timeout=request_timeout) async def _request( self, method: str, endpoint: t.Union[Endpoint, str], endpoint_args: t.Optional[t.Dict[str, str]] = None, *args: t.Any, **kwargs: t.Any, ): custom_config = custom_config_context.get() active_config = self._base_config if custom_config: active_config = GracyConfig.merge_config(self._base_config, custom_config) if self.DEBUG_ENABLED: logger.debug(f"Active Config for {endpoint}: {active_config}") request_context = GracyRequestContext( method, str(self._client.base_url), endpoint, endpoint_args, active_config ) httpx_request_func = self._client.request if replays := self.replays: if replays.mode == "record": httpx_request_func = record_mode(replays, httpx_request_func) elif replays.mode == "replay": httpx_request_func = replay_mode( replays, self._client, httpx_request_func ) else: httpx_request_func = smart_replay_mode( replays, self._client, httpx_request_func ) request_kwargs = extract_request_kwargs(kwargs) request = self._client.build_request( request_context.method, request_context.endpoint, **request_kwargs ) graceful_request = _gracify( Gracy._reporter, Gracy._throttle_controller, replays, self._before, self._after, GracefulRequest( request, httpx_request_func, request_context.method, request_context.endpoint, *args, **kwargs, ), request_context, ) concurrent = active_config.get_concurrent_limit(request_context) async with self._ongoing_tracker.request(request_context, concurrent): return await graceful_request async def before(self, context: GracyRequestContext): ... async def _before(self, context: GracyRequestContext): if within_hook_context.get(): return with custom_gracy_config(DISABLED_GRACY_CONFIG), within_hook(): try: await self.before(context) except Exception: logger.exception("Gracy before hook raised an unexpected exception") async def after( self, context: GracyRequestContext, response_or_exc: t.Union[httpx.Response, Exception], retry_state: t.Optional[GracefulRetryState], ): ... async def _after( self, context: GracyRequestContext, response_or_exc: t.Union[httpx.Response, Exception], retry_state: t.Optional[GracefulRetryState], ): if within_hook_context.get(): return with custom_gracy_config(DISABLED_GRACY_CONFIG), within_hook(): try: await self.after(context, response_or_exc, retry_state) except Exception: logger.exception("Gracy after hook raised an unexpected exception") def get_report(self): return self._reporter.build(self._throttle_controller, self.replays) def report_status(self, printer: PRINTERS): report = self.get_report() return print_report(report, printer) @classmethod def dangerously_reset_report(cls): """ Doing this will reset throttling rules and metrics. So be sure you know what you're doing. """ cls._throttle_controller = ThrottleController() cls._reporter = ReportBuilder() class GracyNamespace(t.Generic[Endpoint], Gracy[Endpoint]): Config = None # type: ignore """Resetted to rely on parent""" def __init__(self, parent: Gracy[Endpoint], **kwargs: t.Any) -> None: self.DEBUG_ENABLED = parent.DEBUG_ENABLED self.replays = parent.replays self._parent = parent self._ongoing_tracker = parent._ongoing_tracker self._init_typed_http_methods() self._client = self._get_namespace_client(parent, **kwargs) self._setup_namespace_config(parent) def _get_namespace_client( self, parent: Gracy[Endpoint], **kwargs: t.Any ) -> httpx.AsyncClient: return parent._client def _setup_namespace_config(self, parent: Gracy[Endpoint]): if self.Config is None: # type: ignore self.Config = parent.Config self._base_config = parent._base_config else: parent_config = parent.Config if not hasattr(self.Config, "BASE_URL"): self.Config.BASE_URL = parent_config.BASE_URL if not hasattr(self.Config, "REQUEST_TIMEOUT"): self.Config.REQUEST_TIMEOUT = parent_config.REQUEST_TIMEOUT if hasattr(self.Config, "SETTINGS"): settings_config = GracyConfig.merge_config( self.Config.SETTINGS, parent_config.SETTINGS ) else: settings_config = parent_config.SETTINGS self._base_config = settings_config parent_settings = parent._base_config parent_config = parent.Config namespace_config = self.Config namespace_config.BASE_URL = parent_config.BASE_URL if hasattr(self.Config, "SETTINGS"): self._base_config = GracyConfig.merge_config( parent_settings, self.Config.SETTINGS ) else: self._base_config = parent_settings def graceful( strict_status_code: t.Union[ t.Iterable[HTTPStatus], HTTPStatus, None, Unset ] = UNSET_VALUE, allowed_status_code: t.Union[ t.Iterable[HTTPStatus], HTTPStatus, None, Unset ] = UNSET_VALUE, validators: t.Union[ t.Iterable[GracefulValidator], GracefulValidator, None, Unset ] = UNSET_VALUE, retry: t.Union[GracefulRetry, Unset, None] = UNSET_VALUE, log_request: LOG_EVENT_TYPE = UNSET_VALUE, log_response: LOG_EVENT_TYPE = UNSET_VALUE, log_errors: LOG_EVENT_TYPE = UNSET_VALUE, parser: PARSER_TYPE = UNSET_VALUE, concurrent_requests: t.Union[CONCURRENT_REQUEST_TYPE, int] = UNSET_VALUE, ): concurrent_requests_config: CONCURRENT_REQUEST_TYPE if isinstance(concurrent_requests, int): concurrent_requests_config = ConcurrentRequestLimit(concurrent_requests) else: concurrent_requests_config = concurrent_requests config = GracyConfig( strict_status_code=strict_status_code, allowed_status_code=allowed_status_code, validators=validators, retry=retry, log_request=log_request, log_response=log_response, log_errors=log_errors, parser=parser, concurrent_requests=concurrent_requests_config, ) def _wrapper( wrapped_function: t.Callable[P, GRACEFUL_T], ) -> t.Callable[P, GRACEFUL_T]: async def _inner_wrapper(*args: P.args, **kwargs: P.kwargs): with custom_gracy_config(config): res = await wrapped_function(*args, **kwargs) return res return t.cast(t.Callable[P, GRACEFUL_T], _inner_wrapper) return _wrapper def graceful_generator( strict_status_code: t.Union[ t.Iterable[HTTPStatus], HTTPStatus, None, Unset ] = UNSET_VALUE, allowed_status_code: t.Union[ t.Iterable[HTTPStatus], HTTPStatus, None, Unset ] = UNSET_VALUE, validators: t.Union[ t.Iterable[GracefulValidator], GracefulValidator, None, Unset ] = UNSET_VALUE, retry: t.Union[GracefulRetry, Unset, None] = UNSET_VALUE, log_request: LOG_EVENT_TYPE = UNSET_VALUE, log_response: LOG_EVENT_TYPE = UNSET_VALUE, log_errors: LOG_EVENT_TYPE = UNSET_VALUE, parser: PARSER_TYPE = UNSET_VALUE, concurrent_requests: t.Union[CONCURRENT_REQUEST_TYPE, int] = UNSET_VALUE, ): concurrent_requests_config: CONCURRENT_REQUEST_TYPE if isinstance(concurrent_requests, int): concurrent_requests_config = ConcurrentRequestLimit(concurrent_requests) else: concurrent_requests_config = concurrent_requests config = GracyConfig( strict_status_code=strict_status_code, allowed_status_code=allowed_status_code, validators=validators, retry=retry, log_request=log_request, log_response=log_response, log_errors=log_errors, parser=parser, concurrent_requests=concurrent_requests_config, ) def _wrapper( wrapped_function: t.Callable[P, GRACEFUL_GEN_T], ) -> t.Callable[P, GRACEFUL_GEN_T]: async def _inner_wrapper(*args: P.args, **kwargs: P.kwargs): with custom_gracy_config(config): async for res in wrapped_function(*args, **kwargs): yield res return t.cast(t.Callable[P, GRACEFUL_GEN_T], _inner_wrapper) return _wrapper ================================================ FILE: src/gracy/_general.py ================================================ from __future__ import annotations import typing as t VALID_BUILD_REQUEST_KEYS = { "content", "data", "files", "json", "params", "headers", "cookies", "timeout", "extensions", } """ There're some kwargs that are handled by httpx request, but only a few are properly handled by https build_request. Defined in httpx._client:322 """ def extract_request_kwargs(kwargs: dict[str, t.Any]) -> dict[str, t.Any]: return {k: v for k, v in kwargs.items() if k in VALID_BUILD_REQUEST_KEYS} ================================================ FILE: src/gracy/_loggers.py ================================================ from __future__ import annotations import httpx import logging import typing as t from enum import Enum from ._models import GracefulRetryState, GracyRequestContext, LogEvent, ThrottleRule logger = logging.getLogger("gracy") def is_replay(resp: httpx.Response) -> bool: return getattr(resp, "_gracy_replayed", False) class SafeDict(t.Dict[str, str]): def __missing__(self, key: str): return "{" + key + "}" class DefaultLogMessage(str, Enum): BEFORE = "Request on {URL} is ongoing" AFTER = "{REPLAY}[{METHOD}] {URL} returned {STATUS}" ERRORS = "[{METHOD}] {URL} returned a bad status ({STATUS})" THROTTLE_HIT = "{URL} hit {THROTTLE_LIMIT} reqs/{THROTTLE_TIME_RANGE}" THROTTLE_DONE = "Done waiting {THROTTLE_TIME}s to hit {URL}" RETRY_BEFORE = ( "GracefulRetry: {URL} will wait {RETRY_DELAY}s before next attempt due to " "{RETRY_CAUSE} ({CUR_ATTEMPT} out of {MAX_ATTEMPT})" ) RETRY_AFTER = ( "GracefulRetry: {URL} replied {STATUS} ({CUR_ATTEMPT} out of {MAX_ATTEMPT})" ) RETRY_EXHAUSTED = "GracefulRetry: {URL} exhausted the maximum attempts of {MAX_ATTEMPT} due to {RETRY_CAUSE}" REPLAY_RECORDED = "Gracy Replay: Recorded {RECORDED_COUNT} requests" REPLAY_REPLAYED = "Gracy Replay: Replayed {REPLAYED_COUNT} requests" CONCURRENT_REQUEST_LIMIT_HIT = ( "{UURL} hit {CONCURRENT_REQUESTS} ongoing concurrent requests" ) CONCURRENT_REQUEST_LIMIT_FREED = "{UURL} concurrency has been freed" def do_log( logevent: LogEvent, defaultmsg: str, format_args: dict[str, t.Any], response: httpx.Response | None = None, ): # Let's protect ourselves against potential customizations with undefined {key} safe_format_args = SafeDict(**format_args) if logevent.custom_message: if isinstance(logevent.custom_message, str): message = logevent.custom_message.format_map(safe_format_args) else: message = logevent.custom_message(response).format_map(safe_format_args) else: message = defaultmsg.format_map(safe_format_args) logger.log(logevent.level, message, extra=format_args) def extract_base_format_args(request_context: GracyRequestContext) -> dict[str, str]: return dict( URL=request_context.url, ENDPOINT=request_context.endpoint, UURL=request_context.unformatted_url, UENDPOINT=request_context.unformatted_endpoint, METHOD=request_context.method, ) def extract_response_format_args(response: httpx.Response | None) -> dict[str, str]: status_code = response.status_code if response else "ABORTED" elapsed = response.elapsed if response else "UNKNOWN" if response and is_replay(response): replayed = "TRUE" replayed_str = "REPLAYED" else: replayed = "FALSE" replayed_str = "" return dict( STATUS=str(status_code), ELAPSED=str(elapsed), IS_REPLAY=replayed, REPLAY=replayed_str, ) def process_log_before_request( logevent: LogEvent, request_context: GracyRequestContext ) -> None: format_args = extract_base_format_args(request_context) do_log(logevent, DefaultLogMessage.BEFORE, format_args) def process_log_throttle( logevent: LogEvent, default_message: str, await_time: float, rule: ThrottleRule, request_context: GracyRequestContext, ): format_args = dict( **extract_base_format_args(request_context), THROTTLE_TIME=await_time, THROTTLE_LIMIT=rule.max_requests, THROTTLE_TIME_RANGE=rule.readable_time_range, ) do_log(logevent, default_message, format_args) def process_log_retry( logevent: LogEvent, defaultmsg: str, request_context: GracyRequestContext, state: GracefulRetryState, response: httpx.Response | None = None, ): maybe_response_args: dict[str, str] = {} if response: maybe_response_args = extract_response_format_args(response) format_args = dict( **extract_base_format_args(request_context), **maybe_response_args, RETRY_DELAY=state.delay, RETRY_CAUSE=state.cause, CUR_ATTEMPT=state.cur_attempt, MAX_ATTEMPT=state.max_attempts, ) do_log(logevent, defaultmsg, format_args, response) def process_log_after_request( logevent: LogEvent, defaultmsg: str, request_context: GracyRequestContext, response: httpx.Response | None, ) -> None: format_args: dict[str, str] = dict( **extract_base_format_args(request_context), **extract_response_format_args(response), ) do_log(logevent, defaultmsg, format_args, response) def process_log_concurrency_limit( logevent: LogEvent, count: int, request_context: GracyRequestContext ): format_args: t.Dict[str, str] = dict( CONCURRENT_REQUESTS=f"{count:,}", **extract_base_format_args(request_context), ) do_log(logevent, DefaultLogMessage.CONCURRENT_REQUEST_LIMIT_HIT, format_args) def process_log_concurrency_freed( logevent: LogEvent, request_context: GracyRequestContext ): format_args: t.Dict[str, str] = dict( **extract_base_format_args(request_context), ) do_log(logevent, DefaultLogMessage.CONCURRENT_REQUEST_LIMIT_FREED, format_args) ================================================ FILE: src/gracy/_models.py ================================================ from __future__ import annotations import asyncio import copy import httpx import inspect import itertools import logging import re import typing as t from abc import ABC, abstractmethod from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta from enum import Enum, IntEnum from http import HTTPStatus from threading import Lock from ._types import PARSER_TYPE, UNSET_VALUE, Unset class LogLevel(IntEnum): CRITICAL = logging.CRITICAL ERROR = logging.ERROR WARNING = logging.WARNING INFO = logging.INFO DEBUG = logging.DEBUG NOTSET = logging.NOTSET @dataclass class LogEvent: level: LogLevel custom_message: t.Callable[[httpx.Response | None], str] | str | None = None """You can add some placeholders to be injected in the log. e.g. - `{URL} executed` - `API replied {STATUS} and took {ELAPSED}` - `{METHOD} {URL} returned {STATUS}` - `Becareful because {URL} is flaky` Placeholders may change depending on the context. Check the docs to see all available placeholder. """ LOG_EVENT_TYPE = t.Union[None, Unset, LogEvent] class GracefulRetryState: cur_attempt: int = 0 success: bool = False last_exc: Exception | None = None last_response: httpx.Response | None def __init__(self, retry_config: GracefulRetry) -> None: self._retry_config = retry_config self._delay = retry_config.delay self._override_delay: float | None = None @property def delay(self) -> float: if self._override_delay is not None: return self._override_delay return self._delay @property def failed(self) -> bool: return not self.success @property def max_attempts(self): return self._retry_config.max_attempts @property def can_retry(self): return self.cur_attempt <= self.max_attempts @property def cant_retry(self): return not self.can_retry @property def cause(self) -> str: """Describes why the Retry was triggered""" # Importing here to avoid cyclic imports from gracy.exceptions import ( GracyRequestFailed, GracyUserDefinedException, NonOkResponse, UnexpectedResponse, ) if self.success: return "SUCCESSFUL" exc = self.last_exc if self.last_response: if isinstance(exc, NonOkResponse) or isinstance(exc, UnexpectedResponse): return f"[Bad Status Code: {self.last_response.status_code}]" if isinstance(exc, GracyUserDefinedException): return f"[User Error: {type(exc).__name__}]" if exc: if isinstance(exc, GracyRequestFailed): return f"[Request Error: {type(exc.original_exc).__name__}]" return f"[{type(exc).__name__}]" # This final block is unlikely to ever happen resp = t.cast(httpx.Response, self.last_response) return f"[Bad Status Code: {resp.status_code}]" def increment(self, response: httpx.Response | None): self.cur_attempt += 1 if self.cur_attempt > 1: self._delay *= self._retry_config.delay_modifier self._override_delay = None if ( response and self._retry_config.overrides and self._retry_config.overrides.get(response.status_code) ): self._override_delay = self._retry_config.overrides[ response.status_code ].delay STATUS_OR_EXCEPTION = t.Union[int, t.Type[Exception]] @dataclass class OverrideRetryOn: delay: float @dataclass class GracefulRetry: delay: float max_attempts: int delay_modifier: float = 1 retry_on: STATUS_OR_EXCEPTION | t.Iterable[STATUS_OR_EXCEPTION] | None = None log_before: LogEvent | None = None log_after: LogEvent | None = None log_exhausted: LogEvent | None = None behavior: t.Literal["break", "pass"] = "break" overrides: t.Union[t.Dict[int, OverrideRetryOn], None] = None def needs_retry(self, response_result: int) -> bool: if self.retry_on is None: return True retry_on_status = self.retry_on if not isinstance(retry_on_status, t.Iterable): retry_on_status = {retry_on_status} return response_result in retry_on_status def create_state( self, result: httpx.Response | None, exc: Exception | None ) -> GracefulRetryState: state = GracefulRetryState(self) state.last_response = result state.last_exc = exc return state class ThrottleRule: url_pattern: t.Pattern[str] """ Which URLs do you want to account for this? e.g. Strict values: - `"https://myapi.com/endpoint"` Regex values: - `"https://.*"` - `"http(s)?://myapi.com/.*"` """ max_requests: int """ How many requests should be run `per_time_range` """ per_time_range: timedelta """ Used in combination with `max_requests` to measure throttle """ def __init__( self, url_pattern: str, max_requests: int, per_time_range: timedelta = timedelta(seconds=1), ) -> None: self.url_pattern = re.compile(url_pattern) self.max_requests = max_requests self.per_time_range = per_time_range if isinstance(max_requests, float): raise TypeError(f"{max_requests=} should be an integer") @property def readable_time_range(self) -> str: seconds = self.per_time_range.total_seconds() periods = { ("hour", 3600), ("minute", 60), ("second", 1), } parts: list[str] = [] for period_name, period_seconds in periods: if seconds >= period_seconds: period_value, seconds = divmod(seconds, period_seconds) if period_value == 1: parts.append(period_name) else: parts.append(f"{int(period_value)} {period_name}s") if seconds < 1: break if len(parts) == 1: return parts[0] else: return ", ".join(parts[:-1]) + " and " + parts[-1] def __str__(self) -> str: return f"{self.max_requests} requests per {self.readable_time_range} for URLs matching {self.url_pattern}" def calculate_await_time(self, controller: ThrottleController) -> float: """ Checks current reqs/second and awaits if limit is reached. Returns whether limit was hit or not. """ rate_limit = self.max_requests cur_rate = controller.calculate_requests_per_rule( self.url_pattern, self.per_time_range ) if cur_rate >= rate_limit: time_diff = (rate_limit - cur_rate) or 1 waiting_time = self.per_time_range.total_seconds() / time_diff return waiting_time return 0.0 class ThrottleLocker: def __init__(self) -> None: self._regex_lock = t.DefaultDict[t.Pattern[str], Lock](Lock) self._generic_lock = Lock() @contextmanager def lock_rule(self, rule: ThrottleRule): with self._regex_lock[rule.url_pattern] as lock: yield lock @contextmanager def lock_check(self): with self._generic_lock as lock: yield lock def is_rule_throttled(self, rule: ThrottleRule) -> bool: return self._regex_lock[rule.url_pattern].locked() THROTTLE_LOCKER: t.Final = ThrottleLocker() class GracefulThrottle: rules: list[ThrottleRule] = [] log_limit_reached: LogEvent | None = None log_wait_over: LogEvent | None = None def __init__( self, rules: list[ThrottleRule] | ThrottleRule, log_limit_reached: LogEvent | None = None, log_wait_over: LogEvent | None = None, ) -> None: self.rules = rules if isinstance(rules, t.Iterable) else [rules] self.log_limit_reached = log_limit_reached self.log_wait_over = log_wait_over class ThrottleController: def __init__(self) -> None: self._control = t.DefaultDict[str, t.List[datetime]](list) def init_request(self, request_context: GracyRequestContext): with THROTTLE_LOCKER.lock_check(): self._control[request_context.url].append( datetime.now() ) # This should always keep it sorted asc def calculate_requests_per_rule( self, url_pattern: t.Pattern[str], range: timedelta ) -> float: with THROTTLE_LOCKER.lock_check(): past_time_window = datetime.now() - range request_rate = 0.0 request_times = sorted( itertools.chain( *[ started_ats for url, started_ats in self._control.items() if url_pattern.match(url) ], ), reverse=True, ) req_idx = 0 total_reqs = len(request_times) while req_idx < total_reqs: # e.g. Limit 4 requests per 2 seconds, now is 09:55 # request_time=09:54 >= past_time_window=09:53 if request_times[req_idx] >= past_time_window: request_rate += 1 else: # Because it's sorted desc there's no need to keep iterating return request_rate req_idx += 1 return request_rate def calculate_requests_per_sec(self, url_pattern: t.Pattern[str]) -> float: with THROTTLE_LOCKER.lock_check(): requests_per_second = 0.0 coalesced_started_ats = sorted( itertools.chain( *[ started_ats for url, started_ats in self._control.items() if url_pattern.match(url) ] ) ) if coalesced_started_ats: # Best effort to measure rate if we just performed 1 request last = ( coalesced_started_ats[-1] if len(coalesced_started_ats) > 1 else datetime.now() ) start = coalesced_started_ats[0] elapsed = last - start if elapsed.seconds > 0: requests_per_second = len(coalesced_started_ats) / elapsed.seconds return requests_per_second def debug_print(self): # Intended only for local development from rich.console import Console from rich.table import Table console = Console() table = Table(title="Throttling Summary") table.add_column("URL", overflow="fold") table.add_column("Count", justify="right") table.add_column("Times", justify="right") for url, times in self._control.items(): human_times = [time.strftime("%H:%M:%S.%f") for time in times] table.add_row(url, f"{len(times):,}", f"[yellow]{human_times}[/yellow]") console.print(table) class GracefulValidator(ABC): """ Run `check` raises exceptions in case it's not passing. """ @abstractmethod def check(self, response: httpx.Response) -> None: """Returns `None` to pass or raise exception""" pass @dataclass class RequestTimeline: url: str start: float end: float @classmethod def build(cls, start: float, resp: httpx.Response): end = start + resp.elapsed.total_seconds() return cls( url=str(resp.url), start=start, end=end, ) @dataclass class ConcurrentRequestLimit: """ Limits how many concurrent calls for a specific endpoint can be active. e.g. If you limit 10 requests to the endpoing /xyz """ limit: int uurl_pattern: t.Pattern[str] = re.compile(".*") blocking_args: t.Optional[t.Iterable[str]] = None """ Combine endpoint args to decide whether to limit. Optional, leaving it blank means that any request to the endpoint will be blocked """ limit_per_uurl: bool = True """ Whether Gracy should limit requests per UURL or the whole api. If True, UURLs will be grouped, so: Limit = 1 #1. GET /test/{VALUE} (0/1) - RUNNING #3. GET /another/{VALUE} (0/1) - RUNNING #2. GET /test/{VALUE} (1/1) - WAITING [Grouped with #1] If False, every UURL will be matched, so: Limit = 1 #1. GET /test/{VALUE} (0/1) - RUNNING #3. GET /another/{VALUE} (0/1) - WAITING [Grouped with ALL] #2. GET /test/{VALUE} (1/1) - WAITING [Grouped with ALL] """ log_limit_reached: LOG_EVENT_TYPE = None """ Log event for the first time the limit is reached. It's only triggered again if the limit slows down. """ log_limit_freed: LOG_EVENT_TYPE = None def __post_init__(self): self._arg_semaphore_map: t.Dict[ t.Tuple[str, ...], asyncio.BoundedSemaphore ] = {} def _get_blocking_key( self, request_context: GracyRequestContext ) -> t.Tuple[str, ...]: uurl_arg = request_context.unformatted_url if self.limit_per_uurl else "global" args: t.List[str] = [] if self.blocking_args: args = [ request_context.endpoint_args.get(arg, "") for arg in self.blocking_args ] return (uurl_arg, *args) def get_semaphore( self, request_context: GracyRequestContext ) -> asyncio.BoundedSemaphore: key = self._get_blocking_key(request_context) if key not in self._arg_semaphore_map: self._arg_semaphore_map[key] = asyncio.BoundedSemaphore(self.limit) return self._arg_semaphore_map[key] CONCURRENT_REQUEST_TYPE = t.Union[ t.Iterable[ConcurrentRequestLimit], ConcurrentRequestLimit, None, Unset ] @dataclass class GracyConfig: log_request: LOG_EVENT_TYPE = UNSET_VALUE log_response: LOG_EVENT_TYPE = UNSET_VALUE log_errors: LOG_EVENT_TYPE = UNSET_VALUE retry: GracefulRetry | None | Unset = UNSET_VALUE strict_status_code: t.Iterable[HTTPStatus] | HTTPStatus | None | Unset = UNSET_VALUE """Strictly enforces only one or many HTTP Status code to be considered as successful. e.g. Setting it to 201 would raise exceptions for both 204 or 200""" allowed_status_code: t.Iterable[ HTTPStatus ] | HTTPStatus | None | Unset = UNSET_VALUE """Adds one or many HTTP Status code that would normally be considered an error e.g. 404 would consider any 200-299 and 404 as successful. NOTE: `strict_status_code` takes precedence. """ validators: t.Iterable[ GracefulValidator ] | GracefulValidator | None | Unset = UNSET_VALUE """Adds one or many validators to be run for the response to decide whether it was successful or not. NOTE: `strict_status_code` or `allowed_status_code` are executed before. If none is set, it will first check whether the response has a successful code. """ parser: PARSER_TYPE = UNSET_VALUE """ Tell Gracy how to deal with the responses for you. Examples: - `"default": lambda response: response.json()` - `HTTPStatus.OK: lambda response: response.json()["ok_data"]` - `HTTPStatus.NOT_FOUND: None` - `HTTPStatus.INTERNAL_SERVER_ERROR: UserDefinedServerException` """ throttling: GracefulThrottle | None | Unset = UNSET_VALUE concurrent_requests: CONCURRENT_REQUEST_TYPE = UNSET_VALUE def should_retry( self, response: httpx.Response | None, req_or_validation_exc: Exception | None ) -> bool: """Only checks if given status requires retry. Does not consider attempts.""" if self.has_retry: retry = t.cast(GracefulRetry, self.retry) retry_on: t.Iterable[STATUS_OR_EXCEPTION] if ( not isinstance(retry.retry_on, t.Iterable) and retry.retry_on is not None ): retry_on = [retry.retry_on] elif retry.retry_on is None: retry_on = [] else: retry_on = retry.retry_on if response is None: if retry.retry_on is None: return True for maybe_exc in retry_on: if inspect.isclass(maybe_exc): if isinstance(req_or_validation_exc, maybe_exc): return True # Importing here to avoid cyclic imports from .exceptions import GracyRequestFailed if isinstance( req_or_validation_exc, GracyRequestFailed ) and isinstance(req_or_validation_exc.original_exc, maybe_exc): return True return False response_status = response.status_code if retry.retry_on is None: if req_or_validation_exc or response.is_success is False: return True if isinstance(retry.retry_on, t.Iterable): if response_status in retry.retry_on: return True for maybe_exc in retry.retry_on: if inspect.isclass(maybe_exc) and isinstance( req_or_validation_exc, maybe_exc ): return True elif inspect.isclass(retry.retry_on): return isinstance(req_or_validation_exc, retry.retry_on) else: return retry.retry_on == response_status return False @property def has_retry(self) -> bool: return self.retry is not None and self.retry != UNSET_VALUE @classmethod def merge_config(cls, base: GracyConfig, modifier: GracyConfig): new_obj = copy.copy(base) for key, value in vars(modifier).items(): if getattr(base, key) == UNSET_VALUE: setattr(new_obj, key, value) elif value != UNSET_VALUE: setattr(new_obj, key, value) return new_obj def get_concurrent_limit( self, context: GracyRequestContext ) -> t.Optional[ConcurrentRequestLimit]: if ( isinstance(self.concurrent_requests, Unset) or self.concurrent_requests is None ): return None if isinstance(self.concurrent_requests, ConcurrentRequestLimit): if self.concurrent_requests.uurl_pattern.match(context.unformatted_url): return self.concurrent_requests return None for rule in self.concurrent_requests: if rule.uurl_pattern.match(context.unformatted_url): return rule return None DEFAULT_CONFIG: t.Final = GracyConfig( log_request=None, log_response=None, log_errors=LogEvent(LogLevel.ERROR), strict_status_code=None, allowed_status_code=None, retry=None, ) class BaseEndpoint(str, Enum): def __str__(self) -> str: return self.value Endpoint = t.TypeVar("Endpoint", bound=t.Union[BaseEndpoint, str]) # , default=str) class GracefulRequest: request: httpx.Request request_func: t.Callable[..., t.Awaitable[httpx.Response]] """Can't use coroutine because we need to retrigger it during retries, and coro can't be awaited twice""" args: tuple[t.Any, ...] kwargs: dict[str, t.Any] def __init__( self, request: httpx.Request, request_func: t.Callable[..., t.Awaitable[httpx.Response]], *args: t.Any, **kwargs: t.Any, ) -> None: self.request = request self.request_func = request_func self.args = args self.kwargs = kwargs def __call__(self) -> t.Awaitable[httpx.Response]: return self.request_func(*self.args, **self.kwargs) class GracyRequestContext: def __init__( self, method: str, base_url: str, endpoint: str, endpoint_args: t.Union[t.Dict[str, str], None], active_config: GracyConfig, ) -> None: if base_url.endswith("/"): base_url = base_url[:-1] final_endpoint = endpoint.format(**endpoint_args) if endpoint_args else endpoint self.endpoint_args = endpoint_args or {} self.endpoint = final_endpoint self.unformatted_endpoint = endpoint self.url = f"{base_url}{self.endpoint}" self.unformatted_url = f"{base_url}{self.unformatted_endpoint}" self.method = method self._active_config = active_config @property def active_config(self) -> GracyConfig: return self._active_config ================================================ FILE: src/gracy/_paginator.py ================================================ from __future__ import annotations import typing as t RESP_T = t.TypeVar("RESP_T") TOKEN_T = t.TypeVar("TOKEN_T") class GracyPaginator(t.Generic[RESP_T, TOKEN_T]): def __init__( self, gracy_func: t.Callable[..., t.Awaitable[RESP_T]], has_next: t.Callable[[t.Optional[RESP_T]], bool], initial_token: TOKEN_T, page_size: int = 20, get_next_token: t.Optional[t.Callable[[RESP_T, TOKEN_T], TOKEN_T]] = None, get_prev_token: t.Optional[t.Callable[[TOKEN_T], TOKEN_T]] = None, prepare_params: t.Optional[ t.Callable[[TOKEN_T, int], t.Dict[str, t.Any]] ] = None, has_prev: t.Optional[t.Callable[[TOKEN_T], bool]] = None, ): self.has_next = has_next self._endpoint_func = gracy_func self._custom_has_prev = has_prev self._prepare_endpoint_params = prepare_params self._get_next_token = get_next_token self._get_prev_token = get_prev_token self._token = initial_token self._page_size = page_size self._cur_resp: t.Optional[RESP_T] = None def _prepare_params( self, token: TOKEN_T, page_size: int, ) -> t.Dict[str, t.Any]: if self._prepare_endpoint_params: return self._prepare_endpoint_params(token, page_size) params = dict(token=token, limit=self._page_size) return params async def _fetch_page(self) -> RESP_T: params = self._prepare_params(self._token, page_size=20) self._cur_resp = await self._endpoint_func(**params) return self._cur_resp def has_prev(self, token: TOKEN_T) -> bool: if self._custom_has_prev: return self._custom_has_prev(token) return False def _calculate_next_token(self, resp: RESP_T, token: TOKEN_T) -> TOKEN_T: if self._get_next_token: return self._get_next_token(resp, token) raise NotImplementedError("GracyPaginator requires you to setup get_next_token") # noqa: TRY003 def _calculate_prev_token(self, token: TOKEN_T) -> TOKEN_T: if self._get_prev_token: return self._get_prev_token(token) raise NotImplementedError("GracyPaginator requires you to setup get_prev_token") # noqa: TRY003 def set_page(self, token: TOKEN_T) -> None: self._token = token async def next_page(self) -> RESP_T | None: if not self.has_next(self._cur_resp): return None page_result = await self._fetch_page() self._token = self._calculate_next_token(page_result, self._token) return page_result async def prev_page(self): if not self.has_prev(self._token): return None self._token = self._calculate_prev_token(self._token) page_result = await self._fetch_page() return page_result def __aiter__(self): return self async def __anext__(self): page = await self.next_page() if page is None: raise StopAsyncIteration return page class GracyOffsetPaginator(t.Generic[RESP_T], GracyPaginator[RESP_T, int]): def __init__( self, gracy_func: t.Callable[..., t.Awaitable[RESP_T]], has_next: t.Callable[[RESP_T | None], bool], page_size: int = 20, prepare_params: t.Callable[[int, int], t.Dict[str, t.Any]] | None = None, has_prev: t.Callable[[int], bool] | None = None, ): super().__init__( gracy_func, has_next, initial_token=0, page_size=page_size, prepare_params=prepare_params, has_prev=has_prev, ) def _prepare_params( self, token: int, page_size: int, ) -> t.Dict[str, t.Any]: if self._prepare_endpoint_params: return self._prepare_endpoint_params(token, page_size) params = dict(offset=token, limit=self._page_size) return params def has_prev(self, token: int) -> bool: if self._custom_has_prev: return self._custom_has_prev(token) return token > 0 def _calculate_next_token(self, resp: RESP_T, token: int) -> int: return token + self._page_size def _calculate_prev_token(self, token: int) -> int: return token - self._page_size ================================================ FILE: src/gracy/_reports/_builders.py ================================================ from __future__ import annotations import httpx import re import typing as t from collections import defaultdict from statistics import mean from .._models import GracyRequestContext, RequestTimeline, ThrottleController from ..replays.storages._base import GracyReplay, is_replay from ._models import ( GracyAggregatedRequest, GracyReport, GracyRequestCounters, GracyRequestResult, ) ANY_REGEX: t.Final = r".+" REQUEST_ERROR_STATUS: t.Final = 0 REQUEST_SUM_KEY = t.Union[int, t.Literal["total", "retries", "throttles", "replays", 0]] REQUEST_SUM_PER_STATUS_TYPE = t.Dict[str, t.DefaultDict[REQUEST_SUM_KEY, int]] class ReportBuilder: def __init__(self) -> None: self._results: t.List[GracyRequestResult] = [] self._counters = t.DefaultDict[str, GracyRequestCounters](GracyRequestCounters) self._request_history = t.DefaultDict[str, t.List[RequestTimeline]](list) def track( self, request_context: GracyRequestContext, response_or_exc: t.Union[httpx.Response, Exception], request_start: float, ): self._results.append( GracyRequestResult(request_context.unformatted_url, response_or_exc) ) if isinstance(response_or_exc, httpx.Response): if is_replay(response_or_exc): self._replayed(request_context) request_entry = RequestTimeline.build(request_start, response_or_exc) self._request_history[request_context.unformatted_url].append(request_entry) def retried(self, request_context: GracyRequestContext): self._counters[request_context.unformatted_url].retries += 1 def throttled(self, request_context: GracyRequestContext): self._counters[request_context.unformatted_url].throttles += 1 def _replayed(self, request_context: GracyRequestContext): self._counters[request_context.unformatted_url].replays += 1 def _calculate_req_rate_for_url( self, unformatted_url: str, throttle_controller: ThrottleController ) -> float: pattern = re.compile(re.sub(r"{(\w+)}", ANY_REGEX, unformatted_url)) rate = throttle_controller.calculate_requests_per_sec(pattern) return rate def build( self, throttle_controller: ThrottleController, replay_settings: GracyReplay | None, ) -> GracyReport: requests_by_uurl = t.DefaultDict[ str, t.Set[t.Union[httpx.Response, Exception]] ](set) requests_sum: REQUEST_SUM_PER_STATUS_TYPE = defaultdict( lambda: defaultdict(int) ) for result in self._results: requests_by_uurl[result.uurl].add(result.response) requests_sum[result.uurl]["total"] += 1 if isinstance(result.response, httpx.Response): requests_sum[result.uurl][result.response.status_code] += 1 else: requests_sum[result.uurl][REQUEST_ERROR_STATUS] += 1 for uurl, counters in self._counters.items(): requests_sum[uurl]["throttles"] = counters.throttles requests_sum[uurl]["retries"] = counters.retries requests_sum[uurl]["replays"] = counters.replays requests_sum = dict( sorted( requests_sum.items(), key=lambda item: item[1]["total"], reverse=True ) ) report = GracyReport(replay_settings, self._request_history) for uurl, data in requests_sum.items(): all_requests = { req for req in requests_by_uurl[uurl] if isinstance(req, httpx.Response) } total_requests = data["total"] url_latency = [r.elapsed.total_seconds() for r in all_requests] # Rate # Use min to handle scenarios like: # 10 reqs in a 2 millisecond window would produce a number >1,000 leading the user to think that we're # producing 1,000 requests which isn't true. rate = min( self._calculate_req_rate_for_url(uurl, throttle_controller), total_requests, ) resp_2xx = 0 resp_3xx = 0 resp_4xx = 0 resp_5xx = 0 aborted = 0 retries = 0 throttles = 0 replays = 0 for maybe_status, count in data.items(): if maybe_status == "total": continue if maybe_status == REQUEST_ERROR_STATUS: aborted += count continue if maybe_status == "throttles": throttles += count continue if maybe_status == "retries": retries += count continue if maybe_status == "replays": replays += count continue status = maybe_status if 200 <= status < 300: resp_2xx += count elif 300 <= status < 400: resp_3xx += count elif 400 <= status < 500: resp_4xx += count elif 500 <= status: resp_5xx += count report_request = GracyAggregatedRequest( uurl, total_requests, # Responses resp_2xx=resp_2xx, resp_3xx=resp_3xx, resp_4xx=resp_4xx, resp_5xx=resp_5xx, reqs_aborted=aborted, retries=retries, throttles=throttles, replays=replays, # General avg_latency=mean(url_latency) if url_latency else 0, max_latency=max(url_latency) if url_latency else 0, req_rate_per_sec=rate, ) report.add_request(report_request) return report ================================================ FILE: src/gracy/_reports/_models.py ================================================ from __future__ import annotations import httpx import typing as t from dataclasses import dataclass, field from statistics import mean from .._models import RequestTimeline from ..replays.storages._base import GracyReplay @dataclass(frozen=True) class GracyRequestResult: __slots__ = ("uurl", "response") uurl: str response: httpx.Response | Exception @dataclass class GracyRequestCounters: throttles: int = 0 retries: int = 0 replays: int = 0 @dataclass class ReportGenericAggregatedRequest: uurl: str """unformatted url""" total_requests: int resp_2xx: int resp_3xx: int resp_4xx: int resp_5xx: int reqs_aborted: int retries: int throttles: int replays: int max_latency: float @property def success_rate(self) -> float: if self.total_requests: return (self.resp_2xx / self.total_requests) * 100 return 0 @property def failed_rate(self) -> float: if self.total_requests: return 100.00 - self.success_rate return 0 @dataclass class GracyAggregatedRequest(ReportGenericAggregatedRequest): avg_latency: float = 0 req_rate_per_sec: float = 0 @dataclass class GracyAggregatedTotal(ReportGenericAggregatedRequest): all_avg_latency: list[float] = field(default_factory=list) all_req_rates: list[float] = field(default_factory=list) @property def avg_latency(self) -> float: entries = self.all_avg_latency or [0] return mean(entries) @property def req_rate_per_sec(self) -> float: entries = self.all_req_rates or [0] return mean(entries) def increment_result(self, row: GracyAggregatedRequest) -> None: self.total_requests += row.total_requests self.resp_2xx += row.resp_2xx self.resp_3xx += row.resp_3xx self.resp_4xx += row.resp_4xx self.resp_5xx += row.resp_5xx self.reqs_aborted += row.reqs_aborted self.throttles += row.throttles self.retries += row.retries self.replays += row.replays self.all_avg_latency.append(row.avg_latency) if row.req_rate_per_sec > 0: self.all_req_rates.append(row.req_rate_per_sec) class GracyReport: def __init__( self, replay_settings: t.Optional[GracyReplay], requests_timeline: t.Dict[str, t.List[RequestTimeline]], ) -> None: self.requests: list[GracyAggregatedRequest | GracyAggregatedTotal] = [] self.total = GracyAggregatedTotal( "TOTAL", # serves as title total_requests=0, resp_2xx=0, resp_3xx=0, resp_4xx=0, resp_5xx=0, reqs_aborted=0, retries=0, throttles=0, replays=0, max_latency=0, ) self.replay_settings = replay_settings self.requests_timeline = requests_timeline def add_request(self, request: GracyAggregatedRequest) -> None: self.requests.append(request) self.total.increment_result(request) ================================================ FILE: src/gracy/_reports/_printers.py ================================================ from __future__ import annotations import logging import typing as t from abc import ABC, abstractmethod from datetime import datetime from ..replays.storages._base import GracyReplay from ._models import GracyAggregatedTotal, GracyReport logger = logging.getLogger("gracy") PRINTERS = t.Literal["rich", "list", "logger", "plotly"] class Titles: url: t.Final = "URL" total_requests: t.Final = "Total Reqs (#)" success_rate: t.Final = "Success (%)" failed_rate: t.Final = "Fail (%)" avg_latency: t.Final = "Avg Latency (s)" max_latency: t.Final = "Max Latency (s)" resp_2xx: t.Final = "2xx Resps" resp_3xx: t.Final = "3xx Resps" resp_4xx: t.Final = "4xx Resps" resp_5xx: t.Final = "5xx Resps" reqs_aborted: t.Final = "Aborts" retries: t.Final = "Retries" throttles: t.Final = "Throttles" replays: t.Final = "Replays" req_rate_per_sec: t.Final = "Avg Reqs/sec" def _getreplays_warn(replay_settings: GracyReplay | None) -> str: res = "" if replay_settings and replay_settings.display_report: if replay_settings.records_made: res = f"{replay_settings.records_made:,} Requests Recorded" if replay_settings.replays_made: if res: res += " / " res += f"{replay_settings.replays_made:,} Requests Replayed" if res: return f"({res})" return res def _format_value( val: float, color: str | None = None, isset_color: str | None = None, precision: int = 2, bold: bool = False, prefix: str = "", suffix: str = "", padprefix: int = 0, ) -> str: cur = f"{prefix.rjust(padprefix)}{val:,.{precision}f}{suffix}" if bold: cur = f"[bold]{cur}[/bold]" if val and isset_color: cur = f"[{isset_color}]{cur}[/{isset_color}]" elif color: cur = f"[{color}]{cur}[/{color}]" return cur def _format_int( val: int, color: str | None = None, isset_color: str | None = None, bold: bool = False, prefix: str = "", suffix: str = "", padprefix: int = 0, ) -> str: cur = f"{prefix.rjust(padprefix)}{val:,}{suffix}" if bold: cur = f"[bold]{cur}[/bold]" if val and isset_color: cur = f"[{isset_color}]{cur}[/{isset_color}]" elif color: cur = f"[{color}]{cur}[/{color}]" return cur def _print_header(report: GracyReport): print(" ____") print(" / ___|_ __ __ _ ___ _ _") print(" | | _| '__/ _` |/ __| | | |") print(" | |_| | | | (_| | (__| |_| |") print(" \\____|_| \\__,_|\\___|\\__, |") print( f" |___/ Requests Summary Report {_getreplays_warn(report.replay_settings)}" ) class BasePrinter(ABC): @abstractmethod def print_report(self, report: GracyReport) -> t.Any: pass class RichPrinter(BasePrinter): def print_report(self, report: GracyReport) -> None: # Dynamic import so we don't have to require it as dependency from rich.console import Console from rich.table import Table in_replay_mode = ( report.replay_settings and report.replay_settings.display_report ) console = Console() title_warn = ( f"[yellow]{_getreplays_warn(report.replay_settings)}[/yellow]" if in_replay_mode else "" ) table = Table(title=f"Gracy Requests Summary {title_warn}") table.add_column(Titles.url, overflow="fold") table.add_column(Titles.total_requests, justify="right") table.add_column(Titles.success_rate, justify="right") table.add_column(Titles.failed_rate, justify="right") table.add_column(Titles.avg_latency, justify="right") table.add_column(Titles.max_latency, justify="right") table.add_column(Titles.resp_2xx, justify="right") table.add_column(Titles.resp_3xx, justify="right") table.add_column(Titles.resp_4xx, justify="right") table.add_column(Titles.resp_5xx, justify="right") table.add_column(Titles.reqs_aborted, justify="right") table.add_column(Titles.retries, justify="right") table.add_column(Titles.throttles, justify="right") if in_replay_mode: table.add_column(Titles.replays, justify="right") table.add_column(Titles.req_rate_per_sec, justify="right") rows = report.requests report.total.uurl = f"[bold]{report.total.uurl}[/bold]" rows.append(report.total) for idx, request_row in enumerate(rows): is_last_line_before_footer = idx < len(rows) - 1 and isinstance( rows[idx + 1], GracyAggregatedTotal ) row_values: tuple[str, ...] = ( _format_int(request_row.total_requests, bold=True), _format_value(request_row.success_rate, "green", suffix="%"), _format_value( request_row.failed_rate, None, "red", bold=True, suffix="%" ), _format_value(request_row.avg_latency), _format_value(request_row.max_latency), _format_int(request_row.resp_2xx), _format_int(request_row.resp_3xx), _format_int(request_row.resp_4xx, isset_color="red"), _format_int(request_row.resp_5xx, isset_color="red"), _format_int(request_row.reqs_aborted, isset_color="red"), _format_int(request_row.retries, isset_color="yellow"), _format_int(request_row.throttles, isset_color="yellow"), ) if in_replay_mode: row_values = ( *row_values, _format_int(request_row.replays, isset_color="yellow"), ) table.add_row( request_row.uurl, *row_values, _format_value( request_row.req_rate_per_sec, precision=1, suffix=" reqs/s" ), end_section=is_last_line_before_footer, ) console.print(table) class PlotlyPrinter(BasePrinter): def print_report(self, report: GracyReport): # Dynamic import so we don't have to require it as dependency import pandas as pd # pyright: ignore[reportMissingImports] import plotly.express as px df = pd.DataFrame( [ dict( Uurl=uurl, Url=entry.url, Start=datetime.utcfromtimestamp(entry.start), Finish=datetime.utcfromtimestamp(entry.end), ) for uurl, requests in report.requests_timeline.items() for entry in requests ] ) fig = px.timeline( df, x_start="Start", x_end="Finish", y="Uurl", color="Url", ) fig.update_yaxes(autorange="reversed") fig.update_xaxes(tickformat="%H:%M:%S.%f") fig.update_layout(barmode="group") return fig class ListPrinter(BasePrinter): def print_report(self, report: GracyReport) -> None: _print_header(report) entries = report.requests entries.append(report.total) in_replay_mode = ( report.replay_settings and report.replay_settings.display_report ) PAD_PREFIX: t.Final = 20 for idx, entry in enumerate(entries, 1): title = entry.uurl if idx == len(entries) else f"{idx}. {entry.uurl}" print(f"\n\n{title}") print( _format_int( entry.total_requests, padprefix=PAD_PREFIX, prefix=f"{Titles.total_requests}: ", ) ) print( _format_value( entry.success_rate, padprefix=PAD_PREFIX, prefix=f"{Titles.success_rate}: ", suffix="%", ) ) print( _format_value( entry.failed_rate, padprefix=PAD_PREFIX, prefix=f"{Titles.failed_rate}: ", suffix="%", ) ) print( _format_value( entry.avg_latency, padprefix=PAD_PREFIX, prefix=f"{Titles.avg_latency}: ", ) ) print( _format_value( entry.max_latency, padprefix=PAD_PREFIX, prefix=f"{Titles.max_latency}: ", ) ) print( _format_int( entry.resp_2xx, padprefix=PAD_PREFIX, prefix=f"{Titles.resp_2xx}: " ) ) print( _format_int( entry.resp_3xx, padprefix=PAD_PREFIX, prefix=f"{Titles.resp_3xx}: " ) ) print( _format_int( entry.resp_4xx, padprefix=PAD_PREFIX, prefix=f"{Titles.resp_4xx}: " ) ) print( _format_int( entry.resp_5xx, padprefix=PAD_PREFIX, prefix=f"{Titles.resp_5xx}: " ) ) print( _format_int( entry.reqs_aborted, padprefix=PAD_PREFIX, prefix=f"{Titles.reqs_aborted}: ", ) ) print( _format_int( entry.retries, padprefix=PAD_PREFIX, prefix=f"{Titles.retries}: " ) ) print( _format_int( entry.throttles, padprefix=PAD_PREFIX, prefix=f"{Titles.throttles}: ", ) ) if in_replay_mode: print( _format_int( entry.replays, padprefix=PAD_PREFIX, prefix=f"{Titles.replays}: ", ) ) print( _format_value( entry.req_rate_per_sec, precision=1, padprefix=PAD_PREFIX, prefix=f"{Titles.req_rate_per_sec}: ", suffix=" reqs/s", ) ) class LoggerPrinter(BasePrinter): def print_report(self, report: GracyReport) -> None: # the first entry should be the most frequent URL hit if not report.requests: logger.warning("No requests were triggered") return first_entry, *_ = report.requests total = report.total logger.info( f"Gracy tracked that '{first_entry.uurl}' was hit {_format_int(first_entry.total_requests)} time(s) " f"with a success rate of {_format_value(first_entry.success_rate, suffix='%')}, " f"avg latency of {_format_value(first_entry.avg_latency)}s, " f"and a rate of {_format_value(first_entry.req_rate_per_sec, precision=1, suffix=' reqs/s')}." ) logger.info( f"Gracy tracked a total of {_format_int(total.total_requests)} requests " f"with a success rate of {_format_value(total.success_rate, suffix='%')}, " f"avg latency of {_format_value(total.avg_latency)}s, " f"and a rate of {_format_value(total.req_rate_per_sec, precision=1, suffix=' reqs/s')}." ) if replay := report.replay_settings: if replay.mode == "record": logger.info("All requests were recorded with GracyReplay") else: logger.warning( "All requests were REPLAYED (no HTTP interaction) with GracyReplay" ) def print_report(report: GracyReport, method: PRINTERS) -> t.Any: printer: t.Optional[BasePrinter] = None if method == "rich": printer = RichPrinter() elif method == "list": printer = ListPrinter() elif method == "logger": printer = LoggerPrinter() elif method == "plotly": printer = PlotlyPrinter() if printer: return printer.print_report(report) ================================================ FILE: src/gracy/_types.py ================================================ from __future__ import annotations import httpx import sys import typing as t from http import HTTPStatus from typing_extensions import deprecated if sys.version_info >= (3, 10): from typing import ParamSpec else: from typing_extensions import ParamSpec class Unset: """ The default "unset" state indicates that whatever default is set on the client should be used. This is different to setting `None`, which explicitly disables the parameter, possibly overriding a client default. """ def __bool__(self): return False PARSER_KEY = t.Union[HTTPStatus, int, t.Literal["default"]] PARSER_VALUE = t.Union[t.Type[Exception], t.Callable[[httpx.Response], t.Any], None] PARSER_TYPE = t.Union[t.Dict[PARSER_KEY, PARSER_VALUE], Unset, None] UNSET_VALUE: t.Final = Unset() P = ParamSpec("P") T = t.TypeVar("T") @deprecated("Use typed http methods instead e.g. `self.get[DesiredType]()`") def parsed_response(return_type: t.Type[T]): # type: ignore def _decorated( func: t.Callable[P, t.Any] ) -> t.Callable[P, t.Coroutine[t.Any, t.Any, T]]: async def _gracy_method(*args: P.args, **kwargs: P.kwargs) -> T: return await func(*args, **kwargs) return _gracy_method return _decorated @deprecated("Use typed http methods instead e.g. `self.get[DesiredType]()`") def generated_parsed_response(return_type: t.Type[T]): # type: ignore def _decorated( func: t.Callable[P, t.AsyncGenerator[t.Any, t.Any]] ) -> t.Callable[P, t.AsyncGenerator[T, t.Any]]: async def _gracy_method( *args: P.args, **kwargs: P.kwargs ) -> t.AsyncGenerator[T, t.Any]: async for i in func(*args, **kwargs): yield i return _gracy_method return _decorated ================================================ FILE: src/gracy/_validators.py ================================================ from __future__ import annotations import typing as t import httpx from ._models import GracefulValidator from .exceptions import NonOkResponse, UnexpectedResponse class DefaultValidator(GracefulValidator): def check(self, response: httpx.Response) -> None: if response.is_success: return None raise NonOkResponse(str(response.url), response) class StrictStatusValidator(GracefulValidator): def __init__(self, status_code: t.Union[int, t.Iterable[int]]) -> None: if isinstance(status_code, t.Iterable): self._status_codes = status_code else: self._status_codes = {status_code} def check(self, response: httpx.Response) -> None: if response.status_code in self._status_codes: return None raise UnexpectedResponse(str(response.url), response, self._status_codes) class AllowedStatusValidator(GracefulValidator): def __init__(self, status_code: t.Union[int, t.Iterable[int]]) -> None: if isinstance(status_code, t.Iterable): self._status_codes = status_code else: self._status_codes = {status_code} def check(self, response: httpx.Response) -> None: if response.is_success: return None if response.status_code in self._status_codes: return None raise NonOkResponse(str(response.url), response) ================================================ FILE: src/gracy/common_hooks.py ================================================ from __future__ import annotations import asyncio import logging import typing as t from asyncio import Lock from dataclasses import dataclass from datetime import datetime from http import HTTPStatus import httpx from ._loggers import do_log, extract_base_format_args, extract_response_format_args from ._models import GracyRequestContext, LogEvent from ._reports._builders import ReportBuilder from .replays.storages._base import is_replay logger = logging.getLogger("gracy") @dataclass class HookResult: executed: bool awaited: float = 0 dry_run: bool = False class HttpHeaderRetryAfterBackOffHook: """ Provides two methods `before()` and `after()` to be used as hooks by Gracy. This hook checks for 429 (TOO MANY REQUESTS), and then reads the `retry-after` header (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After). If the value is set, then Gracy pauses **ALL** client requests until the time is over. This behavior can be modified to happen on a per-endpoint basis if `lock_per_endpoint` is True. ### ⚠️ Retry This doesn't replace `GracefulRetry`. Make sure you implement a proper retry logic, otherwise the 429 will break the client. ### Throttling If you pass the reporter, it will count every await as "throttled" for that UURL. ### Processor You can optionally pass in a lambda, that can be used to modify/increase the wait time from the header. ### Log Event You can optionally define a log event. It provides the response and the context, but also `RETRY_AFTER` that contains the header value. `RETRY_AFTER_ACTUAL_WAIT` is also available in case you modify the original value. """ DEFAULT_LOG_MESSAGE: t.Final = ( "[{METHOD}] {URL} requested to wait for {RETRY_AFTER}s" ) ALL_CLIENT_LOCK: t.Final = "CLIENT" def __init__( self, reporter: ReportBuilder | None = None, lock_per_endpoint: bool = False, log_event: LogEvent | None = None, seconds_processor: None | t.Callable[[float], float] = None, *, dry_run: bool = False, ) -> None: self._reporter = reporter self._lock_per_endpoint = lock_per_endpoint self._lock_manager = t.DefaultDict[str, Lock](Lock) self._log_event = log_event self._processor = seconds_processor or (lambda x: x) self._dry_run = dry_run def _process_log( self, request_context: GracyRequestContext, response: httpx.Response, retry_after: float, actual_wait: float, ) -> None: if event := self._log_event: format_args: t.Dict[str, str] = dict( **extract_base_format_args(request_context), **extract_response_format_args(response), RETRY_AFTER=str(retry_after), RETRY_AFTER_ACTUAL_WAIT=str(actual_wait), ) do_log(event, self.DEFAULT_LOG_MESSAGE, format_args, response) def _parse_retry_after_as_seconds(self, response: httpx.Response) -> float: retry_after_value = response.headers.get("retry-after") if retry_after_value is None: return 0 if retry_after_value.isdigit(): return int(retry_after_value) try: # It might be a date as: Wed, 21 Oct 2015 07:28:00 GMT date_time = datetime.strptime(retry_after_value, "%a, %d %b %Y %H:%M:%S %Z") date_as_seconds = (date_time - datetime.now()).total_seconds() except Exception: logger.exception( f"Unable to parse {retry_after_value} as date within {type(self).__name__}" ) return 0 else: return date_as_seconds async def before(self, context: GracyRequestContext) -> HookResult: return HookResult(False) async def after( self, context: GracyRequestContext, response_or_exc: httpx.Response | Exception, ) -> HookResult: if ( isinstance(response_or_exc, httpx.Response) and response_or_exc.status_code == HTTPStatus.TOO_MANY_REQUESTS ): if is_replay(response_or_exc): return HookResult(executed=False, dry_run=self._dry_run) retry_after_seconds = self._parse_retry_after_as_seconds(response_or_exc) actual_wait = self._processor(retry_after_seconds) if retry_after_seconds > 0: lock_name = ( context.unformatted_url if self._lock_per_endpoint else self.ALL_CLIENT_LOCK ) async with self._lock_manager[lock_name]: self._process_log( context, response_or_exc, retry_after_seconds, actual_wait ) if self._reporter: self._reporter.throttled(context) if self._dry_run is False: await asyncio.sleep(actual_wait) return HookResult(True, actual_wait, self._dry_run) return HookResult(False, dry_run=self._dry_run) class RateLimitBackOffHook: """ Provides two methods `before()` and `after()` to be used as hooks by Gracy. This hook checks for 429 (TOO MANY REQUESTS) and locks requests for an arbitrary amount of time defined by you. If the value is set, then Gracy pauses **ALL** client requests until the time is over. This behavior can be modified to happen on a per-endpoint basis if `lock_per_endpoint` is True. ### ⚠️ Retry This doesn't replace `GracefulRetry`. Make sure you implement a proper retry logic, otherwise the 429 will break the client. ### Throttling If you pass the reporter, it will count every await as "throttled" for that UURL. ### Log Event You can optionally define a log event. It provides the response and the context, but also `WAIT_TIME` that contains the wait value. """ DEFAULT_LOG_MESSAGE: t.Final = ( "[{METHOD}] {UENDPOINT} got rate limited, waiting for {WAIT_TIME}s" ) ALL_CLIENT_LOCK: t.Final = "CLIENT" def __init__( self, delay: float, reporter: ReportBuilder | None = None, lock_per_endpoint: bool = False, log_event: LogEvent | None = None, *, dry_run: bool = False, ) -> None: self._reporter = reporter self._lock_per_endpoint = lock_per_endpoint self._lock_manager = t.DefaultDict[str, Lock](Lock) self._log_event = log_event self._delay = delay self._dry_run = dry_run def _process_log( self, request_context: GracyRequestContext, response: httpx.Response ) -> None: if event := self._log_event: format_args: t.Dict[str, str] = dict( **extract_base_format_args(request_context), **extract_response_format_args(response), WAIT_TIME=str(self._delay), ) do_log(event, self.DEFAULT_LOG_MESSAGE, format_args, response) async def before(self, context: GracyRequestContext) -> HookResult: return HookResult(False) async def after( self, context: GracyRequestContext, response_or_exc: httpx.Response | Exception, ) -> HookResult: if ( isinstance(response_or_exc, httpx.Response) and response_or_exc.status_code == HTTPStatus.TOO_MANY_REQUESTS ): if is_replay(response_or_exc): return HookResult(executed=False, dry_run=self._dry_run) lock_name = ( context.unformatted_url if self._lock_per_endpoint else self.ALL_CLIENT_LOCK ) async with self._lock_manager[lock_name]: self._process_log(context, response_or_exc) if self._reporter: self._reporter.throttled(context) if self._dry_run is False: await asyncio.sleep(self._delay) return HookResult(True, self._delay, self._dry_run) return HookResult(False, dry_run=self._dry_run) ================================================ FILE: src/gracy/exceptions.py ================================================ from __future__ import annotations import httpx import typing as t from abc import ABC, abstractmethod from ._models import GracyRequestContext REDUCE_PICKABLE_RETURN = t.Tuple[t.Type[Exception], t.Tuple[t.Any, ...]] class GracyException(Exception, ABC): @abstractmethod def __reduce__(self) -> REDUCE_PICKABLE_RETURN: """ `__reduce__` is required to avoid Gracy from breaking in different environments that pickles the results (e.g. inside ThreadPools). More context: https://stackoverflow.com/a/36342588/2811539 """ pass class GracyRequestFailed(GracyException): """ Sometimes the httpx's request fails for whatever reason (TCP, SSL, etc errors), this is a wrapper exception so the client can be easily "retried" for any failed requests. NOTE: Consider that failed requests means NO RESPONSE because the request never completed Maybe this would be an `ExceptionGroup` if Gracy ever deprecates Python < 3.11 """ def __init__(self, context: GracyRequestContext, original_exc: Exception) -> None: self.original_exc = original_exc self.request_context = context original_exc_name = self._get_exc_name(original_exc) super().__init__( f"The request for [{context.method}] {context.url} never got a response due to {original_exc_name} " ) # Inspired by https://stackoverflow.com/a/54716092/2811539 # We include the original exception as part of the stack trace by doing that. self.__cause__ = original_exc self.__context__ = original_exc @staticmethod def _get_exc_name(exc: Exception) -> str: """ Formats the exception as "module.ClassType" e.g. httpx.ReadTimeout """ exc_type = type(exc) module = exc_type.__module__ if module is not None and module != "__main__": return module + "." + exc_type.__qualname__ return exc_type.__qualname__ def __reduce__(self) -> REDUCE_PICKABLE_RETURN: return (GracyRequestFailed, (self.request_context, self.original_exc)) class GracyParseFailed(GracyException): def __init__(self, response: httpx.Response) -> None: msg = ( f"Unable to parse result from [{response.request.method}] {response.url} ({response.status_code}). " f"Response content is: {response.text}" ) self.url = response.request.url self.response = response super().__init__(msg) def __reduce__(self) -> REDUCE_PICKABLE_RETURN: return (GracyParseFailed, (self.response,)) class BadResponse(GracyException): def __init__( self, message: str | None, url: str, response: httpx.Response, expected: str | int | t.Iterable[int], ) -> None: self.url = url self.response = response self._args = ( message, url, response, expected, ) if isinstance(expected, str): expectedstr = expected elif isinstance(expected, int): expectedstr = str(expected) else: expectedstr = ", ".join([str(s) for s in expected]) curmsg = ( message or f"{url} raised {response.status_code}, but it was expecting {expectedstr}" ) super().__init__(curmsg) def __reduce__(self) -> REDUCE_PICKABLE_RETURN: return (BadResponse, self._args) class UnexpectedResponse(BadResponse): def __init__( self, url: str, response: httpx.Response, expected: str | int | t.Iterable[int] ) -> None: super().__init__(None, url, response, expected) self.url = url self.response = response self.expected = expected def __reduce__(self) -> REDUCE_PICKABLE_RETURN: return (UnexpectedResponse, (self.url, self.response, self.expected)) class NonOkResponse(BadResponse): def __init__(self, url: str, response: httpx.Response) -> None: super().__init__(None, url, response, "any successful status code") self.arg1 = url self.arg2 = response def __reduce__(self) -> REDUCE_PICKABLE_RETURN: return (NonOkResponse, (self.arg1, self.arg2)) class GracyUserDefinedException(GracyException): BASE_MESSAGE: str = "[{METHOD}] {URL} returned {}" def __init__( self, request_context: GracyRequestContext, response: httpx.Response ) -> None: self._request_context = request_context self._response = response super().__init__(self._format_message(request_context, response)) def _build_default_args(self) -> dict[str, t.Any]: request_context = self._request_context return dict( # Context ENDPOINT=request_context.endpoint, UURL=request_context.unformatted_url, UENDPOINT=request_context.unformatted_endpoint, # Response URL=self.response.request.url, METHOD=self.response.request.method, STATUS=self.response.status_code, ELAPSED=self.response.elapsed, ) def _format_message( self, request_context: GracyRequestContext, response: httpx.Response ) -> str: format_args = self._build_default_args() return self.BASE_MESSAGE.format(**format_args) @property def url(self): return self._request_context.url @property def endpoint(self): return self._request_context.endpoint @property def response(self): return self._response def __reduce__(self) -> REDUCE_PICKABLE_RETURN: return (GracyUserDefinedException, (self._request_context, self._response)) class GracyReplayRequestNotFound(GracyException): def __init__(self, request: httpx.Request) -> None: self.request = request msg = f"Gracy was unable to replay {request.method} {request.url} - did you forget to record it?" super().__init__(msg) def __reduce__(self) -> REDUCE_PICKABLE_RETURN: return (GracyReplayRequestNotFound, (self.request,)) ================================================ FILE: src/gracy/py.typed ================================================ ================================================ FILE: src/gracy/replays/_wrappers.py ================================================ from __future__ import annotations import typing as t from functools import wraps import httpx from gracy._general import extract_request_kwargs from gracy.exceptions import GracyReplayRequestNotFound from .storages._base import GracyReplay httpx_func_type = t.Callable[..., t.Awaitable[httpx.Response]] def record_mode(replay: GracyReplay, httpx_request_func: httpx_func_type): @wraps(httpx_request_func) async def _wrapper(*args: t.Any, **kwargs: t.Any): httpx_response = await httpx_request_func(*args, **kwargs) await replay.storage.record(httpx_response) replay.inc_record() return httpx_response return _wrapper def replay_mode( replay: GracyReplay, client: httpx.AsyncClient, httpx_request_func: httpx_func_type ): @wraps(httpx_request_func) async def _wrapper(*args: t.Any, **kwargs: t.Any): request_kwargs = extract_request_kwargs(kwargs) request = client.build_request(*args, **request_kwargs) stored_response = await replay.storage.load( request, replay.discard_replays_older_than, replay.discard_bad_responses, ) replay.inc_replay() return stored_response return _wrapper def smart_replay_mode( replay: GracyReplay, client: httpx.AsyncClient, httpx_request_func: httpx_func_type ): @wraps(httpx_request_func) async def _wrapper(*args: t.Any, **kwargs: t.Any): request_kwargs = extract_request_kwargs(kwargs) request = client.build_request(*args, **request_kwargs) try: stored_response = await replay.storage.load( request, replay.discard_replays_older_than, replay.discard_bad_responses, ) except GracyReplayRequestNotFound: httpx_response = await httpx_request_func(*args, **kwargs) await replay.storage.record(httpx_response) response = httpx_response replay.inc_record() else: response = stored_response replay.inc_replay() return response return _wrapper ================================================ FILE: src/gracy/replays/storages/_base.py ================================================ from __future__ import annotations import logging import typing as t from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime import httpx from gracy.exceptions import GracyReplayRequestNotFound from ..._loggers import DefaultLogMessage, do_log from ..._models import LogEvent logger = logging.getLogger(__name__) REPLAY_FLAG: t.Final = "_gracy_replayed" def is_replay(resp: httpx.Response) -> bool: return getattr(resp, REPLAY_FLAG, False) class GracyReplayStorage(ABC): def prepare(self) -> None: """(Optional) Executed upon API instance creation.""" pass @abstractmethod async def record(self, response: httpx.Response) -> None: """Logic to store the response object. Note the httpx.Response has request data""" pass @abstractmethod async def find_replay( self, request: httpx.Request, discard_before: datetime | None ) -> t.Any | None: pass @abstractmethod async def _load( self, request: httpx.Request, discard_before: datetime | None ) -> httpx.Response: """Logic to load a response object based on the request. Raises `GracyReplayRequestNotFound` if missing""" pass async def load( self, request: httpx.Request, discard_before: datetime | None, discard_bad_responses: bool = False, ) -> httpx.Response: """Logic to load a response object based on the request. Raises `GracyReplayRequestNotFound` if missing""" resp = await self._load(request, discard_before) setattr(resp, REPLAY_FLAG, True) if discard_bad_responses and resp.is_success is False: raise GracyReplayRequestNotFound(request) return resp def flush(self) -> None: """(Optional) Executed during close (preferably once all requests were made).""" pass @dataclass class ReplayLogEvent(LogEvent): frequency: int = 1_000 """Defines how often to log when request is recorded/replayed""" @dataclass class GracyReplay: mode: t.Literal["record", "replay", "smart-replay"] """ `record`: Will record all requests made to the API `replay`: Will read all responses from the defined storage `smart-replay`: Will read all responses (like `replay`), but if it's missing it will `record` for future replays """ storage: GracyReplayStorage """Where to read/write requests and responses""" discard_replays_older_than: datetime | None = None """If set, Gracy will treat all replays older than defined value as not found""" discard_bad_responses: bool = False """If set True, then Gracy will discard bad requests (e.g. non 2xx)""" disable_throttling: bool = False """Only applicable to `smart-replay` and `replay` modes. If a replay exists then don't throttle the request""" display_report: bool = True """Whether to display records made and replays made to the final report""" log_record: ReplayLogEvent | None = None """Whether to log and how often to upon recording requests. The only available placeholder is `RECORDED_COUNT`""" log_replay: ReplayLogEvent | None = None """Whether to log and how often to upon replaying requests. The only available placeholder is `REPLAYED_COUNT`""" records_made: int = 0 replays_made: int = 0 async def has_replay(self, request: httpx.Request) -> bool: replay = await self.storage.find_replay( request, self.discard_replays_older_than ) return bool(replay) def inc_record(self): self.records_made += 1 if log_ev := self.log_record: if self.records_made % log_ev.frequency == 0: args = dict(RECORDED_COUNT=f"{self.records_made:,}") do_log(log_ev, DefaultLogMessage.REPLAY_RECORDED, args) def inc_replay(self): self.replays_made += 1 if log_ev := self.log_replay: if self.replays_made % log_ev.frequency == 0: args = dict(REPLAYED_COUNT=f"{self.replays_made:,}") do_log(log_ev, DefaultLogMessage.REPLAY_REPLAYED, args) ================================================ FILE: src/gracy/replays/storages/_sqlite_schema.py ================================================ from __future__ import annotations import typing as t TABLE_NAME: t.Final = "gracy_recordings" CREATE_RECORDINGS_TABLE: t.Final = f""" CREATE TABLE {TABLE_NAME}( url VARCHAR(255) NOT NULL, method VARCHAR(20) NOT NULL, request_body BLOB NULL, response BLOB NOT NULL, updated_at DATETIME NOT NULL ) """ INDEX_RECORDINGS_TABLE: t.Final = f""" CREATE UNIQUE INDEX idx_gracy_request ON {TABLE_NAME}(url, method, request_body) """ INDEX_RECORDINGS_TABLE_WITHOUT_REQUEST_BODY: t.Final = f""" CREATE UNIQUE INDEX idx_gracy_request_empty_req_body ON {TABLE_NAME}(url, method) WHERE request_body IS NULL """ INSERT_RECORDING_BASE: t.Final = f""" INSERT OR REPLACE INTO {TABLE_NAME} VALUES (?, ?, ?, ?, ?) """ FIND_REQUEST_WITH_REQ_BODY: t.Final = f""" SELECT response, updated_at FROM {TABLE_NAME} WHERE url = ? AND method = ? AND request_body = ? """ FIND_REQUEST_WITHOUT_REQ_BODY: t.Final = f""" SELECT response, updated_at FROM {TABLE_NAME} WHERE url = ? AND method = ? AND request_body IS NULL """ ================================================ FILE: src/gracy/replays/storages/pymongo.py ================================================ from __future__ import annotations import httpx import json import pickle import typing as t from dataclasses import asdict, dataclass from datetime import datetime from threading import Lock from gracy.exceptions import GracyReplayRequestNotFound from ._base import GracyReplayStorage try: import pymongo except ModuleNotFoundError: pass @dataclass class MongoCredentials: host: str | None = None """Can be a full URI""" port: int = 27017 username: str | None = None password: str | None = None class MongoReplayDocument(t.TypedDict): url: str method: str request_body: bytes | None response: bytes response_content: dict[str, t.Any] | str | None """Useful for debugging since Mongo supports unstructured data""" updated_at: datetime def get_unique_keys_from_doc( replay_doc: MongoReplayDocument, ) -> t.Dict[str, bytes | None | str]: return { "url": replay_doc["url"], "method": replay_doc["method"], "request_body": replay_doc["request_body"], } def get_unique_keys_from_request( request: httpx.Request, ) -> t.Dict[str, bytes | None | str]: return { "url": str(request.url), "method": request.method, "request_body": request.content or None, } batch_lock = Lock() class MongoReplayStorage(GracyReplayStorage): def __init__( self, creds: MongoCredentials, database_name: str = "gracy", collection_name: str = "gracy-replay", batch_size: int | None = None, ) -> None: creds_kwargs = asdict(creds) client = pymongo.MongoClient(**creds_kwargs, document_class=MongoReplayDocument) # pyright: ignore[reportPossiblyUnboundVariable] mongo_db = client[database_name] self._collection = mongo_db[collection_name] self._batch = batch_size self._batch_ops: t.List[pymongo.ReplaceOne[MongoReplayDocument]] = [] def _flush_batch(self) -> None: if self._batch_ops: with batch_lock: self._collection.bulk_write(self._batch_ops) # type: ignore self._batch_ops = [] def _create_or_batch(self, doc: MongoReplayDocument) -> None: filter = get_unique_keys_from_doc(doc) if self._batch and self._batch > 1: with batch_lock: self._batch_ops.append(pymongo.ReplaceOne(filter, doc, upsert=True)) # pyright: ignore[reportPossiblyUnboundVariable] if len(self._batch_ops) >= self._batch: self._flush_batch() else: self._collection.replace_one(filter, doc, upsert=True) def prepare(self) -> None: self._collection.create_index( [("url", 1), ("method", 1), ("request_body", 1)], background=True, unique=True, ) async def record(self, response: httpx.Response) -> None: response_serialized = pickle.dumps(response) response_content = response.text or None content_type = response.headers.get("Content-Type") if content_type and "json" in content_type: try: jsonified_content = response.json() except json.decoder.JSONDecodeError: pass else: response_content = jsonified_content doc = MongoReplayDocument( url=str(response.url), method=str(response.request.method), request_body=response.request.content or None, response=response_serialized, response_content=response_content, updated_at=datetime.now(), ) self._create_or_batch(doc) async def find_replay( self, request: httpx.Request, discard_before: datetime | None ) -> MongoReplayDocument | None: filter = get_unique_keys_from_request(request) doc = self._collection.find_one(filter) if doc is None: return None if discard_before and doc["updated_at"] < discard_before: return None return doc async def _load( self, request: httpx.Request, discard_before: datetime | None ) -> httpx.Response: doc = await self.find_replay(request, discard_before) if doc is None: raise GracyReplayRequestNotFound(request) serialized_response = doc["response"] response: httpx.Response = pickle.loads(serialized_response) return response def flush(self) -> None: self._flush_batch() ================================================ FILE: src/gracy/replays/storages/sqlite.py ================================================ from __future__ import annotations import httpx import logging import pickle import sqlite3 import typing as t from dataclasses import dataclass from datetime import datetime from pathlib import Path from gracy.exceptions import GracyReplayRequestNotFound from . import _sqlite_schema as schema from ._base import GracyReplayStorage logger = logging.getLogger(__name__) @dataclass class GracyRecording: url: str method: str request_body: bytes | None response: bytes updated_at: datetime class SQLiteReplayStorage(GracyReplayStorage): def __init__( self, db_name: str = "gracy-records.sqlite3", dir: str = ".gracy" ) -> None: self.db_dir = Path(dir) self.db_file = self.db_dir / db_name self._con: sqlite3.Connection = None # type: ignore def _create_db(self) -> None: logger.info("Creating Gracy Replay sqlite database") con = sqlite3.connect(str(self.db_file)) cur = con.cursor() cur.execute(schema.CREATE_RECORDINGS_TABLE) cur.execute(schema.INDEX_RECORDINGS_TABLE) cur.execute(schema.INDEX_RECORDINGS_TABLE_WITHOUT_REQUEST_BODY) def _insert_into_db(self, recording: GracyRecording) -> None: cur = self._con.cursor() params = ( recording.url, recording.method, recording.request_body, recording.response, datetime.now(), ) cur.execute(schema.INSERT_RECORDING_BASE, params) self._con.commit() def prepare(self) -> None: self.db_dir.mkdir(parents=True, exist_ok=True) if self.db_file.exists() is False: self._create_db() self._con = sqlite3.connect(str(self.db_file)) async def record(self, response: httpx.Response) -> None: response_serialized = pickle.dumps(response) recording = GracyRecording( str(response.url), str(response.request.method), response.request.content or None, response_serialized, datetime.now(), ) self._insert_into_db(recording) def _find_record(self, request: httpx.Request): cur = self._con.cursor() params: t.Iterable[str | bytes] if bool(request.content): params = (str(request.url), request.method, request.content) cur.execute(schema.FIND_REQUEST_WITH_REQ_BODY, params) else: params = (str(request.url), request.method) cur.execute(schema.FIND_REQUEST_WITHOUT_REQ_BODY, params) fetch_res = cur.fetchone() return fetch_res async def find_replay( self, request: httpx.Request, discard_before: datetime | None ) -> t.Any | None: fetch_res = self._find_record(request) if fetch_res is None: return None updated_at: datetime = fetch_res[1] if discard_before and updated_at < discard_before: return None return fetch_res async def _load( self, request: httpx.Request, discard_before: datetime | None ) -> httpx.Response: fetch_res = await self.find_replay(request, discard_before) if fetch_res is None: raise GracyReplayRequestNotFound(request) serialized_response: bytes = fetch_res[0] response: httpx.Response = pickle.loads(serialized_response) return response ================================================ FILE: src/tests/conftest.py ================================================ from __future__ import annotations import typing as t import httpx from gracy import BaseEndpoint, Gracy, GracyReplay from gracy.replays.storages.sqlite import SQLiteReplayStorage MISSING_NAME: t.Final = "doesnt-exist" """Should match what we recorded previously to successfully replay""" PRESENT_POKEMON_NAME: t.Final = "charmander" """Should match what we recorded previously to successfully replay""" PRESENT_BERRY_NAME: t.Final = "cheri" """Should match what we recorded previously to successfully replay""" REPLAY: t.Final = GracyReplay("replay", SQLiteReplayStorage("pokeapi.sqlite3")) class FakeReplayStorage(SQLiteReplayStorage): """Completely ignores the request defined to return a response matching the urls in the order specified""" def __init__(self, force_urls: t.List[str]) -> None: self._force_urls = force_urls self._response_idx = 0 super().__init__("pokeapi.sqlite3") def _find_record(self, request: httpx.Request): cur = self._con.cursor() url = self._force_urls[self._response_idx] self._response_idx += 1 cur.execute( """ SELECT response, updated_at FROM gracy_recordings WHERE url = ?""", (url,), ) return cur.fetchone() class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" GET_BERRY = "/berry/{NAME}" def assert_one_request_made(gracy_api: Gracy[PokeApiEndpoint]): report = gracy_api.get_report() assert len(report.requests) == 1 def assert_requests_made( gracy_api: Gracy[PokeApiEndpoint], total_requests: int, endpoints_count: int = 1 ): report = gracy_api.get_report() assert len(report.requests) == endpoints_count assert report.requests[0].total_requests == total_requests def assert_muiti_endpoints_requests_made( gracy_api: Gracy[PokeApiEndpoint], endpoints_count: int, *total_requests: int, ): report = gracy_api.get_report() assert len(report.requests) == endpoints_count for i, expected_total in enumerate(total_requests): assert report.requests[i].total_requests == expected_total ================================================ FILE: src/tests/generate_test_db.py ================================================ from __future__ import annotations import asyncio import typing as t from gracy import BaseEndpoint, Gracy, GracyReplay from gracy.replays.storages.sqlite import SQLiteReplayStorage class PokeApiEndpoint(BaseEndpoint): GET_POKEMON = "/pokemon/{NAME}" GET_BERRY = "/berry/{NAME}" GET_GENERATION = "/generation/{ID}" class GracefulPokeAPIRecorder(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" def __init__(self) -> None: record_mode: t.Final = GracyReplay( "record", SQLiteReplayStorage("pokeapi.sqlite3"), ) super().__init__(record_mode) async def get_pokemon(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) async def get_berry(self, name: str): return await self.get(PokeApiEndpoint.GET_BERRY, {"NAME": name}) async def get_generation(self, gen: int): return await self.get(PokeApiEndpoint.GET_GENERATION, {"ID": str(gen)}) async def main(): pokeapi = GracefulPokeAPIRecorder() poke_names = {"pikachu", "elekid", "charmander", "blaziken", "hitmonchan"} try: get_pokemons = [ asyncio.create_task(pokeapi.get_pokemon(name)) for name in poke_names ] get_gens = [ asyncio.create_task(pokeapi.get_generation(gen_id)) for gen_id in range(1, 3) ] get_berries = [asyncio.create_task(pokeapi.get_berry("cheri"))] await asyncio.gather(*(get_pokemons + get_gens + get_berries)) finally: pokeapi.report_status("rich") if __name__ == "__main__": asyncio.run(main()) ================================================ FILE: src/tests/test_generators.py ================================================ from __future__ import annotations import pytest import typing as t from gracy import Gracy, graceful_generator from tests.conftest import REPLAY, PokeApiEndpoint class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" @graceful_generator(parser={"default": lambda r: r.json()}) async def get_2_yield_graceful(self): names = ["charmander", "pikachu"] for name in names: r = await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) yield r @pytest.fixture() def make_pokeapi(): def factory(): Gracy.dangerously_reset_report() return GracefulPokeAPI(REPLAY) return factory async def test_pokemon_ok_json(make_pokeapi: t.Callable[[], GracefulPokeAPI]): pokeapi = make_pokeapi() count = 0 async for _ in pokeapi.get_2_yield_graceful(): count += 1 assert count == 2 ================================================ FILE: src/tests/test_gracy_httpx.py ================================================ from __future__ import annotations import pytest import typing as t from http import HTTPStatus from gracy import GracefulRetry, Gracy, GracyConfig from tests.conftest import PRESENT_POKEMON_NAME, REPLAY, PokeApiEndpoint RETRY: t.Final = GracefulRetry( delay=0.001, max_attempts=2, retry_on={HTTPStatus.NOT_FOUND}, behavior="break", ) @pytest.fixture() def make_pokeapi(): def factory(): Gracy.dangerously_reset_report() return GracefulPokeAPI(REPLAY) return factory class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( retry=RETRY, allowed_status_code={HTTPStatus.NOT_FOUND}, parser={HTTPStatus.NOT_FOUND: None}, ) MAKE_POKEAPI_TYPE = t.Callable[[], GracefulPokeAPI] async def test_pass_kwargs(make_pokeapi: MAKE_POKEAPI_TYPE): pokeapi = make_pokeapi() await pokeapi.get( PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME), follow_redirects=True, ) ================================================ FILE: src/tests/test_hooks.py ================================================ from __future__ import annotations import httpx import pytest import typing as t from http import HTTPStatus from unittest.mock import patch from gracy import ( GracefulRetry, GracefulRetryState, Gracy, GracyConfig, GracyRequestContext, ) from gracy.exceptions import GracyRequestFailed from tests.conftest import ( MISSING_NAME, PRESENT_POKEMON_NAME, REPLAY, PokeApiEndpoint, assert_requests_made, ) RETRY: t.Final = GracefulRetry( delay=0.001, max_attempts=2, retry_on={HTTPStatus.NOT_FOUND}, behavior="break", ) @pytest.fixture() def make_pokeapi(): def factory(): Gracy.dangerously_reset_report() return GracefulPokeAPI(REPLAY) return factory class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( retry=RETRY, allowed_status_code={HTTPStatus.NOT_FOUND}, parser={HTTPStatus.NOT_FOUND: None}, ) def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: self.before_count = 0 self.after_status_counter = t.DefaultDict[int, int](int) self.after_aborts = 0 self.after_retries_counter = 0 super().__init__(*args, **kwargs) async def before(self, context: GracyRequestContext): self.before_count += 1 async def after( self, context: GracyRequestContext, response_or_exc: httpx.Response | Exception, retry_state: GracefulRetryState | None, ): if retry_state: self.after_retries_counter += 1 if isinstance(response_or_exc, httpx.Response): self.after_status_counter[response_or_exc.status_code] += 1 else: self.after_aborts += 1 async def get_pokemon(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) class GracefulPokeAPIWithRequestHooks(GracefulPokeAPI): async def before(self, context: GracyRequestContext): await super().before(context) # This shouldn't re-trigger any hook! await self.get_pokemon(PRESENT_POKEMON_NAME) async def after( self, context: GracyRequestContext, response_or_exc: httpx.Response | Exception, retry_state: GracefulRetryState | None, ): await super().after(context, response_or_exc, retry_state) # This shouldn't re-trigger any hook! await self.get_pokemon(PRESENT_POKEMON_NAME) MAKE_POKEAPI_TYPE = t.Callable[[], GracefulPokeAPI] async def test_before_hook_counts(make_pokeapi: MAKE_POKEAPI_TYPE): pokeapi = make_pokeapi() assert pokeapi.before_count == 0 await pokeapi.get(PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME)) assert pokeapi.before_count == 1 await pokeapi.get(PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME)) assert pokeapi.before_count == 2 async def test_after_hook_counts_statuses(make_pokeapi: MAKE_POKEAPI_TYPE): pokeapi = make_pokeapi() assert pokeapi.after_status_counter[HTTPStatus.OK] == 0 assert pokeapi.after_status_counter[HTTPStatus.NOT_FOUND] == 0 await pokeapi.get( PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME) ) # 200 await pokeapi.get( PokeApiEndpoint.GET_POKEMON, dict(NAME=PRESENT_POKEMON_NAME) ) # 200 await pokeapi.get( PokeApiEndpoint.GET_POKEMON, dict(NAME=MISSING_NAME) ) # 404 + retry 2x await pokeapi.get( PokeApiEndpoint.GET_POKEMON, dict(NAME=MISSING_NAME) ) # 404 + retry 2x assert pokeapi.after_status_counter[HTTPStatus.OK] == 2 assert pokeapi.after_status_counter[HTTPStatus.NOT_FOUND] == 6 assert pokeapi.after_retries_counter == 4 async def test_after_hook_counts_aborts(): Gracy.dangerously_reset_report() pokeapi = GracefulPokeAPI() class SomeRequestException(Exception): pass mock: t.Any with patch.object(pokeapi, "_client", autospec=True) as mock: mock.request.side_effect = SomeRequestException("Request failed") with pytest.raises(GracyRequestFailed): await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) assert pokeapi.after_status_counter[HTTPStatus.OK] == 0 assert pokeapi.after_status_counter[HTTPStatus.NOT_FOUND] == 0 assert pokeapi.after_retries_counter == 0 assert pokeapi.after_aborts == 1 async def test_hook_has_no_recursion(): Gracy.dangerously_reset_report() pokeapi = GracefulPokeAPIWithRequestHooks(REPLAY) EXPECTED_REQS: t.Final = 1 + 2 # This + Before hook + After hook await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) assert_requests_made(pokeapi, EXPECTED_REQS) async def test_hook_with_retries_has_no_recursion(): Gracy.dangerously_reset_report() pokeapi = GracefulPokeAPIWithRequestHooks(REPLAY) # (1 This + 2 Retries) + 2 hooks for each (3) EXPECTED_REQS: t.Final = (1 + 2) + (2 * 3) await pokeapi.get_pokemon(MISSING_NAME) assert pokeapi.before_count == 3 assert pokeapi.after_status_counter[HTTPStatus.NOT_FOUND] == 3 assert pokeapi.after_retries_counter == 2 assert_requests_made(pokeapi, EXPECTED_REQS) ================================================ FILE: src/tests/test_loggers.py ================================================ from __future__ import annotations import httpx import logging import pytest import typing as t from gracy import GracefulValidator, Gracy, GracyConfig, LogEvent, LogLevel from gracy.exceptions import NonOkResponse from tests.conftest import MISSING_NAME, PRESENT_POKEMON_NAME, REPLAY, PokeApiEndpoint class CustomValidator(GracefulValidator): def check(self, response: httpx.Response) -> None: if response.json()["order"] != 47: raise ValueError("Pokemon #order should be 47") # noqa: TRY003 def assert_log(record: logging.LogRecord, expected_event: LogEvent): assert record.levelno == expected_event.level assert record.message == expected_event.custom_message # No formatting set # NOTE: captest only captures >=warning ON_REQUEST: t.Final = LogEvent(LogLevel.WARNING, "LOG_REQUEST") ON_RESPONSE: t.Final = LogEvent(LogLevel.ERROR, "LOG_RESPONSE") ON_ERROR: t.Final = LogEvent(LogLevel.CRITICAL, "LOG_ERROR") class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( log_request=ON_REQUEST, log_response=ON_RESPONSE, log_errors=ON_ERROR, ) async def get_pokemon(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @pytest.fixture() def pokeapi(): Gracy.dangerously_reset_report() return GracefulPokeAPI(REPLAY) async def test_pokemon_log_request_response( pokeapi: GracefulPokeAPI, caplog: pytest.LogCaptureFixture ): await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) assert len(caplog.records) == 2 assert_log(caplog.records[0], ON_REQUEST) assert_log(caplog.records[1], ON_RESPONSE) async def test_pokemon_log_request_response_error( pokeapi: GracefulPokeAPI, caplog: pytest.LogCaptureFixture ): with pytest.raises(NonOkResponse): await pokeapi.get_pokemon(MISSING_NAME) assert len(caplog.records) == 3 assert_log(caplog.records[0], ON_REQUEST) assert_log(caplog.records[1], ON_RESPONSE) assert_log(caplog.records[2], ON_ERROR) ================================================ FILE: src/tests/test_namespaces.py ================================================ from __future__ import annotations import pytest import typing as t from http import HTTPStatus from gracy import GracefulRetry, Gracy, GracyConfig, GracyNamespace from tests.conftest import ( PRESENT_BERRY_NAME, PRESENT_POKEMON_NAME, REPLAY, PokeApiEndpoint, assert_muiti_endpoints_requests_made, ) RETRY: t.Final = GracefulRetry( delay=0.001, max_attempts=2, retry_on={HTTPStatus.NOT_FOUND}, behavior="break", ) class PokemonNamespace(GracyNamespace[PokeApiEndpoint]): async def get_one(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) class BerryNamespace(GracyNamespace[PokeApiEndpoint]): async def get_one(self, name: str): return await self.get(PokeApiEndpoint.GET_BERRY, {"NAME": name}) class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( retry=RETRY, allowed_status_code={HTTPStatus.NOT_FOUND}, parser={HTTPStatus.NOT_FOUND: None}, ) berry: BerryNamespace pokemon: PokemonNamespace @pytest.fixture() def make_pokeapi(): def factory(): Gracy.dangerously_reset_report() return GracefulPokeAPI(REPLAY) return factory MAKE_POKEAPI_TYPE = t.Callable[[], GracefulPokeAPI] async def test_get_from_namespaces(make_pokeapi: MAKE_POKEAPI_TYPE): pokeapi = make_pokeapi() await pokeapi.pokemon.get_one(PRESENT_POKEMON_NAME) await pokeapi.berry.get_one(PRESENT_BERRY_NAME) EXPECTED_ENDPOINTS = 2 EXPECTED_REQUESTS = (1, 1) assert_muiti_endpoints_requests_made( pokeapi, EXPECTED_ENDPOINTS, *EXPECTED_REQUESTS ) ================================================ FILE: src/tests/test_parsers.py ================================================ from __future__ import annotations import httpx import pytest import typing as t from http import HTTPStatus from gracy import Gracy, GracyConfig, graceful from gracy.exceptions import GracyParseFailed from tests.conftest import ( MISSING_NAME, PRESENT_POKEMON_NAME, REPLAY, PokeApiEndpoint, assert_one_request_made, ) class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig(allowed_status_code=HTTPStatus.NOT_FOUND) @graceful(parser={"default": lambda r: r.json()}) async def get_pokemon(self, name: str): return await self.get[t.Dict[str, t.Any]]( PokeApiEndpoint.GET_POKEMON, {"NAME": name} ) @graceful(parser={HTTPStatus.NOT_FOUND: lambda r: None}) async def get_pokemon_not_found_as_none(self, name: str): return await self.get[t.Optional[httpx.Response]]( PokeApiEndpoint.GET_POKEMON, {"NAME": name} ) @pytest.fixture() def make_pokeapi(): def factory(): Gracy.dangerously_reset_report() return GracefulPokeAPI(REPLAY) return factory async def test_pokemon_ok_json(make_pokeapi: t.Callable[[], GracefulPokeAPI]): pokeapi = make_pokeapi() result: dict[str, t.Any] = await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) assert isinstance(result, dict) assert "name" in result assert result["name"] == PRESENT_POKEMON_NAME assert_one_request_made(pokeapi) async def test_pokemon_bad_json(make_pokeapi: t.Callable[[], GracefulPokeAPI]): pokeapi = make_pokeapi() with pytest.raises(GracyParseFailed): await pokeapi.get_pokemon(MISSING_NAME) assert_one_request_made(pokeapi) async def test_pokemon_not_found_as_none(make_pokeapi: t.Callable[[], GracefulPokeAPI]): pokeapi = make_pokeapi() result = await pokeapi.get_pokemon_not_found_as_none(MISSING_NAME) assert result is None assert_one_request_made(pokeapi) ================================================ FILE: src/tests/test_retry.py ================================================ from __future__ import annotations import httpx import logging import pytest import typing as t from http import HTTPStatus from unittest.mock import patch from gracy import ( GracefulRetry, GracefulValidator, Gracy, GracyConfig, GracyReplay, LogEvent, LogLevel, OverrideRetryOn, graceful, ) from gracy.exceptions import GracyRequestFailed, NonOkResponse from tests.conftest import ( MISSING_NAME, PRESENT_POKEMON_NAME, REPLAY, FakeReplayStorage, PokeApiEndpoint, assert_requests_made, ) RETRY: t.Final = GracefulRetry( delay=0.001, max_attempts=0, retry_on={HTTPStatus.NOT_FOUND, ValueError}, behavior="pass", ) """NOTE: Max attempts will be patched later in fixture""" RETRY_ON_NONE: t.Final = GracefulRetry( delay=0.001, max_attempts=1, retry_on=None, behavior="pass" ) RETRY_LOG_BEFORE = LogEvent(LogLevel.WARNING, "LOG_BEFORE") RETRY_LOG_AFTER = LogEvent(LogLevel.ERROR, "LOG_AFTER") RETRY_LOG_EXHAUSTED = LogEvent(LogLevel.CRITICAL, "LOG_EXHAUSTED") RETRY_3_TIMES_LOG: t.Final = GracefulRetry( delay=0.001, max_attempts=3, retry_on=HTTPStatus.NOT_FOUND, log_before=RETRY_LOG_BEFORE, log_after=RETRY_LOG_AFTER, log_exhausted=RETRY_LOG_EXHAUSTED, ) RETRY_3_TIMES_OVERRIDE_PLACEHOLDER_LOG: t.Final = GracefulRetry( delay=90, # Will be overriden max_attempts=3, retry_on=HTTPStatus.NOT_FOUND, overrides={HTTPStatus.NOT_FOUND: OverrideRetryOn(delay=0.001)}, log_before=LogEvent(LogLevel.WARNING, "BEFORE: {RETRY_DELAY} {RETRY_CAUSE}"), log_after=LogEvent(LogLevel.WARNING, "AFTER: {RETRY_CAUSE}"), ) def assert_log(record: logging.LogRecord, expected_event: LogEvent): assert record.levelno == expected_event.level assert record.message == expected_event.custom_message # No formatting set class CustomValidator(GracefulValidator): def check(self, response: httpx.Response) -> None: if response.json()["order"] != 47: raise ValueError("Pokemon #order should be 47") # noqa: TRY003 class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" SETTINGS = GracyConfig( retry=RETRY, allowed_status_code={HTTPStatus.NOT_FOUND}, parser={HTTPStatus.NOT_FOUND: None}, ) async def get_pokemon(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(retry=None) async def get_pokemon_without_retry(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(retry=None, parser=None, allowed_status_code=None) async def get_pokemon_without_retry_or_parser(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(strict_status_code={HTTPStatus.OK}) async def get_pokemon_with_strict_status(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(allowed_status_code=None, parser={"default": lambda r: r.json()}) async def get_pokemon_without_allowed_status(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(validators=CustomValidator()) async def get_pokemon_with_custom_validator(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(retry=RETRY_ON_NONE) async def get_pokemon_with_retry_on_none(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(retry=RETRY_ON_NONE, validators=CustomValidator()) async def get_pokemon_with_retry_on_none_and_validator(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(retry=RETRY_3_TIMES_LOG, allowed_status_code=None) async def get_pokemon_with_log_retry_3_times(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(retry=RETRY_3_TIMES_OVERRIDE_PLACEHOLDER_LOG, allowed_status_code=None) async def get_pokemon_with_retry_overriden_log_placeholder(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @pytest.fixture() def make_pokeapi(): def factory( max_attempts: int, break_or_pass: str = "pass", replay_enabled: bool = True ): Gracy.dangerously_reset_report() api = GracefulPokeAPI(REPLAY) if replay_enabled else GracefulPokeAPI() api._base_config.retry.max_attempts = max_attempts # type: ignore api._base_config.retry.behavior = break_or_pass # type: ignore return api return factory @pytest.fixture() def make_flaky_pokeapi(): def factory( flaky_requests: int, max_attempts: int | None = None, break_or_pass: str = "break", ): Gracy.dangerously_reset_report() force_urls = ( ["https://pokeapi.co/api/v2/pokemon/doesnt-exist"] * flaky_requests ) + (["https://pokeapi.co/api/v2/pokemon/charmander"]) mock_storage = FakeReplayStorage(force_urls) fake_replay = GracyReplay("replay", mock_storage) api = GracefulPokeAPI(fake_replay) if max_attempts: api._base_config.retry.max_attempts = max_attempts # type: ignore api._base_config.retry.behavior = break_or_pass # type: ignore return api return factory class PokeApiFactory(t.Protocol): def __call__( self, max_attempts: int, break_or_pass: str = "pass", replay_enabled: bool = True, ) -> GracefulPokeAPI: ... class FlakyPokeApiFactory(t.Protocol): def __call__( self, flaky_requests: int, max_attempts: int | None = None, break_or_pass: str = "pass", ) -> GracefulPokeAPI: ... async def test_ensure_replay_is_enabled(make_pokeapi: PokeApiFactory): pokeapi = make_pokeapi(0) result = await pokeapi.get_pokemon(MISSING_NAME) report = pokeapi.get_report() assert result is None assert report.replay_settings is not None assert report.replay_settings.mode == "replay" assert len(report.requests) == 1 assert report.requests[0].total_requests == 1 @pytest.mark.parametrize("max_retries", [2, 4, 6]) async def test_pokemon_not_found(max_retries: int, make_pokeapi: PokeApiFactory): EXPECTED_REQS: t.Final = 1 + max_retries # First request + Retries (2) = 3 requests pokeapi = make_pokeapi(max_retries) result = await pokeapi.get_pokemon(MISSING_NAME) assert result is None assert_requests_made(pokeapi, EXPECTED_REQS) @pytest.mark.parametrize("max_retries", [2, 4, 6]) async def test_pokemon_not_found_without_allowed( max_retries: int, make_pokeapi: t.Callable[[int, str], GracefulPokeAPI] ): EXPECTED_REQS: t.Final = 1 + max_retries # First request + Retries (2) = 3 requests pokeapi = make_pokeapi(max_retries, "break") with pytest.raises(NonOkResponse): await pokeapi.get_pokemon_without_allowed_status(MISSING_NAME) assert_requests_made(pokeapi, EXPECTED_REQS) @pytest.mark.parametrize("max_retries", [2, 4, 6]) async def test_pokemon_not_found_with_strict_status( max_retries: int, make_pokeapi: PokeApiFactory ): EXPECTED_REQS: t.Final = 1 + max_retries # First request + Retries (2) = 3 requests pokeapi = make_pokeapi(max_retries) result = await pokeapi.get_pokemon_with_strict_status(MISSING_NAME) assert result is None assert_requests_made(pokeapi, EXPECTED_REQS) async def test_pokemon_with_bad_parser_break_wont_run(make_pokeapi: PokeApiFactory): MAX_RETRIES: t.Final = 2 EXPECTED_REQS: t.Final = 1 + MAX_RETRIES # First request + Retries (2) = 3 requests pokeapi = make_pokeapi(MAX_RETRIES, "break") with pytest.raises(NonOkResponse): await pokeapi.get_pokemon_without_allowed_status(MISSING_NAME) assert_requests_made(pokeapi, EXPECTED_REQS) async def test_retry_with_failing_custom_validation(make_pokeapi: PokeApiFactory): MAX_RETRIES: t.Final = 2 EXPECTED_REQS: t.Final = 1 + MAX_RETRIES # First request + Retries (2) = 3 requests pokeapi = make_pokeapi(MAX_RETRIES) result = await pokeapi.get_pokemon_with_custom_validator(PRESENT_POKEMON_NAME) assert result is not None assert_requests_made(pokeapi, EXPECTED_REQS) async def test_failing_without_retry(make_pokeapi: PokeApiFactory): EXPECTED_REQS: t.Final = 1 pokeapi = make_pokeapi(0) # Won't have effect result = await pokeapi.get_pokemon_without_retry(MISSING_NAME) assert result is None assert_requests_made(pokeapi, EXPECTED_REQS) async def test_failing_without_retry_or_parser(make_pokeapi: PokeApiFactory): EXPECTED_REQS: t.Final = 1 pokeapi = make_pokeapi(0) # Won't have effect with pytest.raises(NonOkResponse): await pokeapi.get_pokemon_without_retry_or_parser(MISSING_NAME) assert_requests_made(pokeapi, EXPECTED_REQS) async def test_retry_none_for_successful_request(make_pokeapi: PokeApiFactory): EXPECTED_REQS: t.Final = 1 pokeapi = make_pokeapi(0) # Won't have effect result = await pokeapi.get_pokemon_with_retry_on_none(PRESENT_POKEMON_NAME) assert result is not None assert_requests_made(pokeapi, EXPECTED_REQS) async def test_retry_none_for_failing_request(make_pokeapi: PokeApiFactory): EXPECTED_REQS: t.Final = 2 pokeapi = make_pokeapi(0) # Won't have effect result = await pokeapi.get_pokemon_with_retry_on_none(MISSING_NAME) assert result is None assert_requests_made(pokeapi, EXPECTED_REQS) async def test_retry_none_for_failing_validator(make_pokeapi: PokeApiFactory): EXPECTED_REQS: t.Final = 2 pokeapi = make_pokeapi(0) # Won't have effect response = await pokeapi.get_pokemon_with_retry_on_none_and_validator( PRESENT_POKEMON_NAME ) assert response is not None assert_requests_made(pokeapi, EXPECTED_REQS) async def test_retry_eventually_recovers(make_flaky_pokeapi: FlakyPokeApiFactory): RETRY_ATTEMPTS: t.Final = 4 EXPECTED_REQS: t.Final = 1 + RETRY_ATTEMPTS # Scenario: 1 + 3 Retry attemps fail + Last attempt works pokeapi = make_flaky_pokeapi(4, RETRY_ATTEMPTS) result = await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) # Test assert result is not None assert_requests_made(pokeapi, EXPECTED_REQS) async def test_retry_eventually_recovers_with_strict( make_flaky_pokeapi: FlakyPokeApiFactory, ): RETRY_ATTEMPTS: t.Final = 4 EXPECTED_REQS: t.Final = 1 + RETRY_ATTEMPTS # Scenario: 1 + 3 Retry attempts fail + last attempt works pokeapi = make_flaky_pokeapi(4, RETRY_ATTEMPTS) result = await pokeapi.get_pokemon_with_strict_status(PRESENT_POKEMON_NAME) # Test assert result is not None assert_requests_made(pokeapi, EXPECTED_REQS) async def test_retry_logs( make_flaky_pokeapi: FlakyPokeApiFactory, caplog: pytest.LogCaptureFixture ): FLAKY_REQUESTS: t.Final = 3 EXPECTED_REQS: t.Final = FLAKY_REQUESTS + 1 pokeapi = make_flaky_pokeapi(FLAKY_REQUESTS) result = await pokeapi.get_pokemon_with_log_retry_3_times(PRESENT_POKEMON_NAME) # Test assert result is not None assert_requests_made(pokeapi, EXPECTED_REQS) assert len(caplog.records) == 6 assert_log(caplog.records[0], RETRY_LOG_BEFORE) assert_log(caplog.records[1], RETRY_LOG_AFTER) assert_log(caplog.records[2], RETRY_LOG_BEFORE) assert_log(caplog.records[3], RETRY_LOG_AFTER) assert_log(caplog.records[4], RETRY_LOG_BEFORE) assert_log(caplog.records[5], RETRY_LOG_AFTER) async def test_retry_logs_fail_reason( make_flaky_pokeapi: FlakyPokeApiFactory, caplog: pytest.LogCaptureFixture ): FLAKY_REQUESTS: t.Final = 2 EXPECTED_REQS: t.Final = FLAKY_REQUESTS + 1 pokeapi = make_flaky_pokeapi(FLAKY_REQUESTS) result = await pokeapi.get_pokemon_with_retry_overriden_log_placeholder( PRESENT_POKEMON_NAME ) # Test assert result is not None assert_requests_made(pokeapi, EXPECTED_REQS) assert len(caplog.records) == 4 assert caplog.records[0].message == "BEFORE: 0.001 [Bad Status Code: 404]" assert caplog.records[1].message == "AFTER: [Bad Status Code: 404]" assert caplog.records[2].message == "BEFORE: 0.001 [Bad Status Code: 404]" assert caplog.records[3].message == "AFTER: SUCCESSFUL" async def test_retry_logs_exhausts( make_pokeapi: PokeApiFactory, caplog: pytest.LogCaptureFixture ): EXPECTED_REQS: t.Final = 3 + 1 # Retry's value from graceful + 1 pokeapi = make_pokeapi(0) # Won't take effect due to @graceful with pytest.raises(NonOkResponse): await pokeapi.get_pokemon_with_log_retry_3_times(MISSING_NAME) # Test assert_requests_made(pokeapi, EXPECTED_REQS) assert len(caplog.records) == 7 assert_log(caplog.records[0], RETRY_LOG_BEFORE) assert_log(caplog.records[1], RETRY_LOG_AFTER) assert_log(caplog.records[2], RETRY_LOG_BEFORE) assert_log(caplog.records[3], RETRY_LOG_AFTER) assert_log(caplog.records[4], RETRY_LOG_BEFORE) assert_log(caplog.records[5], RETRY_LOG_AFTER) assert_log(caplog.records[6], RETRY_LOG_EXHAUSTED) async def test_retry_without_replay_request_without_response_generic( make_pokeapi: PokeApiFactory, ): EXPECTED_REQS: t.Final = 3 + 1 class SomeRequestException(Exception): pass # Regardless of replay being disabled, no request will be triggered as we're mocking httpx pokeapi = make_pokeapi(3, break_or_pass="break", replay_enabled=False) pokeapi._base_config.retry.retry_on.add(GracyRequestFailed) # type: ignore mock: t.Any with patch.object(pokeapi, "_client", autospec=True) as mock: mock.request.side_effect = SomeRequestException("Request failed") with pytest.raises(GracyRequestFailed): await pokeapi.get_pokemon(PRESENT_POKEMON_NAME) assert_requests_made(pokeapi, EXPECTED_REQS) ================================================ FILE: src/tests/test_validators.py ================================================ from __future__ import annotations import httpx import pytest import typing as t from http import HTTPStatus from gracy import GracefulValidator, Gracy, graceful from gracy.exceptions import NonOkResponse, UnexpectedResponse from tests.conftest import ( MISSING_NAME, PRESENT_POKEMON_NAME, REPLAY, PokeApiEndpoint, assert_one_request_made, ) class CustomValidator(GracefulValidator): def check(self, response: httpx.Response) -> None: if response.json()["order"] != 47: raise ValueError("Pokemon #order should be 47") # noqa: TRY003 class GracefulPokeAPI(Gracy[PokeApiEndpoint]): class Config: BASE_URL = "https://pokeapi.co/api/v2/" async def get_pokemon(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(strict_status_code=HTTPStatus.INTERNAL_SERVER_ERROR) async def get_pokemon_with_wrong_strict_status(self, name: str): return await self.get(PokeApiEndpoint.GET_POKEMON, {"NAME": name}) @graceful(strict_status_code=HTTPStatus.OK) async def get_pokemon_with_correct_strict_status(self, name: str): return await self.get[httpx.Response]( PokeApiEndpoint.GET_POKEMON, {"NAME": name} ) @graceful(allowed_status_code=HTTPStatus.NOT_FOUND) async def get_pokemon_allow_404(self, name: str): return await self.get[httpx.Response]( PokeApiEndpoint.GET_POKEMON, {"NAME": name} ) @pytest.fixture() def make_pokeapi(): def factory(): Gracy.dangerously_reset_report() return GracefulPokeAPI(REPLAY) return factory async def test_pokemon_ok_default(make_pokeapi: t.Callable[[], GracefulPokeAPI]): pokeapi = make_pokeapi() result = t.cast(httpx.Response, await pokeapi.get_pokemon(PRESENT_POKEMON_NAME)) assert result.status_code == HTTPStatus.OK assert_one_request_made(pokeapi) async def test_pokemon_not_found_default(make_pokeapi: t.Callable[[], GracefulPokeAPI]): pokeapi = make_pokeapi() try: _ = await pokeapi.get_pokemon(MISSING_NAME) except NonOkResponse as ex: assert ex.response.status_code == HTTPStatus.NOT_FOUND else: pytest.fail("NonOkResponse was expected") assert_one_request_made(pokeapi) async def test_pokemon_strict_status_fail( make_pokeapi: t.Callable[[], GracefulPokeAPI] ): pokeapi = make_pokeapi() try: _ = await pokeapi.get_pokemon_with_wrong_strict_status(PRESENT_POKEMON_NAME) except UnexpectedResponse as ex: assert ex.response.status_code == HTTPStatus.OK else: pytest.fail("UnexpectedResponse was expected") assert_one_request_made(pokeapi) async def test_pokemon_strict_status_success( make_pokeapi: t.Callable[[], GracefulPokeAPI] ): pokeapi = make_pokeapi() result = await pokeapi.get_pokemon_with_correct_strict_status(PRESENT_POKEMON_NAME) assert result.status_code == HTTPStatus.OK assert_one_request_made(pokeapi) async def test_pokemon_allow_404(make_pokeapi: t.Callable[[], GracefulPokeAPI]): pokeapi = make_pokeapi() result = await pokeapi.get_pokemon_allow_404(MISSING_NAME) assert result.status_code == HTTPStatus.NOT_FOUND assert_one_request_made(pokeapi) ================================================ FILE: todo.md ================================================ # Todo - [x] Strict status code - [x] Allowed status code - [x] Retry - [x] Retry but pass - [x] Retry logging - [x] Metrics (% of successful calls) - [x] Status codes - [x] % per status code - [x] avg ms taken to execute - [x] Parsing - [x] default = lambda r: r.json() - [x] 200 = lambda r: r.text() - [x] 404 = None - [x] 401 = InadequatePermissions - [x] Throttle - [x] URL regex support - [ ] Authorization - [ ] Validate if token is still valid - [ ] Auto refresh - [ ] Docs - [x] Readme - [x] Methods without `_` - [ ] Contributing - [ ] Allow to specify status ranges - [x] Custom exception for parsing errors - [x] Replay/Record payloads - [x] SQLite - [ ] Mongo - [ ] Custom Storage