SYMBOL INDEX (2556 symbols across 422 files) FILE: docs/deployment/code_examples/apify/crawler_as_actor_example.py function main (line 8) | async def main() -> None: FILE: docs/deployment/code_examples/apify/get_public_url.py function main (line 6) | async def main() -> None: FILE: docs/deployment/code_examples/apify/log_with_config_example.py function main (line 6) | async def main() -> None: FILE: docs/deployment/code_examples/apify/proxy_advanced_example.py function main (line 6) | async def main() -> None: FILE: docs/deployment/code_examples/apify/proxy_example.py function main (line 6) | async def main() -> None: FILE: docs/deployment/code_examples/aws/beautifulsoup_crawler_lambda.py function main (line 13) | async def main() -> str: function lambda_handler (line 58) | def lambda_handler(_event: dict[str, Any], _context: LambdaContext) -> d... FILE: docs/deployment/code_examples/aws/playwright_crawler_lambda.py function main (line 13) | async def main() -> str: function lambda_handler (line 70) | def lambda_handler(_event: dict[str, Any], _context: LambdaContext) -> d... FILE: docs/deployment/code_examples/google/cloud_run_example.py function main (line 12) | async def main() -> str: FILE: docs/deployment/code_examples/google/google_example.py function main (line 12) | async def main() -> str: function crawlee_run (line 51) | def crawlee_run(request: Request) -> Response: FILE: docs/examples/code_examples/adaptive_playwright_crawler.py function main (line 13) | async def main() -> None: FILE: docs/examples/code_examples/add_data_to_dataset_bs.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/add_data_to_dataset_dataset.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/add_data_to_dataset_pw.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/beautifulsoup_crawler.py function main (line 11) | async def main() -> None: FILE: docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py function main (line 7) | async def main() -> None: FILE: docs/examples/code_examples/beautifulsoup_crawler_stop.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/capture_screenshot_using_playwright.py function main (line 7) | async def main() -> None: FILE: docs/examples/code_examples/configure_json_logging.py class InterceptHandler (line 18) | class InterceptHandler(logging.Handler): method emit (line 19) | def emit(self, record: logging.LogRecord) -> None: function formatter (line 54) | def formatter(record: Record) -> str: function main (line 74) | async def main() -> None: FILE: docs/examples/code_examples/crawl_all_links_on_website_bs.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/crawl_all_links_on_website_pw.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/crawl_multiple_urls_bs.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/crawl_multiple_urls_pw.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/crawl_specific_links_on_website_bs.py function main (line 7) | async def main() -> None: FILE: docs/examples/code_examples/crawl_specific_links_on_website_pw.py function main (line 7) | async def main() -> None: FILE: docs/examples/code_examples/crawl_website_with_relative_links_all_links.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/export_entire_dataset_to_file_csv.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/export_entire_dataset_to_file_json.py function main (line 6) | async def main() -> None: FILE: docs/examples/code_examples/extract_and_add_specific_links_on_website_bs.py function main (line 7) | async def main() -> None: FILE: docs/examples/code_examples/extract_and_add_specific_links_on_website_pw.py function main (line 7) | async def main() -> None: FILE: docs/examples/code_examples/fill_and_submit_web_form_crawler.py function main (line 8) | async def main() -> None: FILE: docs/examples/code_examples/fill_and_submit_web_form_request.py function main (line 7) | async def main() -> None: FILE: docs/examples/code_examples/parsel_crawler.py function main (line 9) | async def main() -> None: FILE: docs/examples/code_examples/parsel_crawler_with_error_snapshotter.py function main (line 8) | async def main() -> None: FILE: docs/examples/code_examples/playwright_block_requests.py function main (line 10) | async def main() -> None: FILE: docs/examples/code_examples/playwright_crawler.py function main (line 10) | async def main() -> None: FILE: docs/examples/code_examples/playwright_crawler_with_camoufox.py class CamoufoxPlugin (line 15) | class CamoufoxPlugin(PlaywrightBrowserPlugin): method new_browser (line 21) | async def new_browser(self) -> PlaywrightBrowserController: function main (line 36) | async def main() -> None: FILE: docs/examples/code_examples/playwright_crawler_with_error_snapshotter.py function main (line 8) | async def main() -> None: FILE: docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py function main (line 11) | async def main() -> None: FILE: docs/examples/code_examples/respect_robots_on_skipped_request.py function main (line 10) | async def main() -> None: FILE: docs/examples/code_examples/respect_robots_txt_file.py function main (line 9) | async def main() -> None: FILE: docs/examples/code_examples/resuming_paused_crawl.py function main (line 17) | async def main() -> None: FILE: docs/examples/code_examples/run_parallel_crawlers.py function main (line 14) | async def main() -> None: FILE: docs/examples/code_examples/using_browser_profiles_chrome.py function main (line 16) | async def main() -> None: FILE: docs/examples/code_examples/using_browser_profiles_firefox.py function main (line 17) | async def main() -> None: FILE: docs/examples/code_examples/using_sitemap_request_loader.py function create_transform_request (line 14) | def create_transform_request( function main (line 38) | async def main() -> None: FILE: docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py function main (line 10) | async def main() -> None: FILE: docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py function archive_response (line 11) | async def archive_response(context: ParselCrawlingContext, writer: WARCW... function main (line 32) | async def main() -> None: FILE: docs/guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py function archive_response (line 18) | async def archive_response( function main (line 46) | async def main() -> None: FILE: docs/guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/error_handling/change_handle_error_status.py function main (line 14) | async def main() -> None: FILE: docs/guides/code_examples/error_handling/disable_retry.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/error_handling/handle_proxy_error.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/http_clients/parsel_curl_impersonate_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/http_clients/parsel_httpx_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/http_clients/parsel_impit_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/beautifulsoup_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/http_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/lexbor_parser.py function main (line 11) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/lxml_parser.py function main (line 10) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/lxml_saxonche_parser.py function main (line 11) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/parsel_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/pyquery_parser.py function main (line 11) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/scrapling_parser.py function main (line 11) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/selectolax_adaptive_run.py function main (line 11) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/selectolax_context.py class SelectolaxLexborContext (line 12) | class SelectolaxLexborContext(ParsedHttpCrawlingContext[LexborHTMLParser]): method parser (line 20) | def parser(self) -> LexborHTMLParser: method from_parsed_http_crawling_context (line 25) | def from_parsed_http_crawling_context( FILE: docs/guides/code_examples/http_crawlers/selectolax_crawler.py class SelectolaxLexborCrawler (line 23) | class SelectolaxLexborCrawler( method __init__ (line 28) | def __init__( FILE: docs/guides/code_examples/http_crawlers/selectolax_crawler_run.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/http_crawlers/selectolax_parser.py class SelectolaxLexborParser (line 17) | class SelectolaxLexborParser(AbstractHttpParser[LexborHTMLParser, Lexbor... method parse (line 21) | async def parse(self, response: HttpResponse) -> LexborHTMLParser: method parse_text (line 28) | async def parse_text(self, text: str) -> LexborHTMLParser: method select (line 33) | async def select( method is_matching_selector (line 40) | def is_matching_selector( method find_links (line 47) | def find_links( FILE: docs/guides/code_examples/login_crawler/http_login.py function main (line 13) | async def main() -> None: FILE: docs/guides/code_examples/login_crawler/playwright_login.py function main (line 12) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler/browser_configuration_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler/browser_pool_page_hooks_example.py function main (line 19) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler/multiple_launch_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler/navigation_hooks_example.py function main (line 12) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler_adaptive/handler.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py class CustomRenderingTypePredictor (line 13) | class CustomRenderingTypePredictor(RenderingTypePredictor): method __init__ (line 14) | def __init__(self) -> None: method predict (line 19) | def predict(self, request: Request) -> RenderingTypePrediction: method store_result (line 36) | def store_result(self, request: Request, rendering_type: RenderingType... function result_checker (line 44) | def result_checker(result: RequestHandlerRunResult) -> bool: function result_comparator (line 50) | def result_comparator( function main (line 61) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py function main (line 11) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler_stagehand/browser_classes.py class StagehandBrowserController (line 26) | class StagehandBrowserController(PlaywrightBrowserController): method __init__ (line 28) | def __init__( method new_page (line 38) | async def new_page( class StagehandPlugin (line 74) | class StagehandPlugin(PlaywrightBrowserPlugin): method __init__ (line 78) | def __init__(self, stagehand: Stagehand, **kwargs: Any) -> None: method new_browser (line 84) | async def new_browser(self) -> StagehandBrowserController: FILE: docs/guides/code_examples/playwright_crawler_stagehand/stagehand_run.py function main (line 17) | async def main() -> None: FILE: docs/guides/code_examples/playwright_crawler_stagehand/support_classes.py class CrawleeStagehandPage (line 11) | class CrawleeStagehandPage: method __init__ (line 14) | def __init__(self, page: StagehandPage) -> None: method goto (line 17) | async def goto( method __getattr__ (line 34) | def __getattr__(self, name: str) -> Any: method __aenter__ (line 38) | async def __aenter__(self) -> CrawleeStagehandPage: method __aexit__ (line 42) | async def __aexit__( class CrawleeStagehand (line 51) | class CrawleeStagehand(Stagehand): method init (line 54) | async def init(self) -> None: FILE: docs/guides/code_examples/proxy_management/inspecting_bs_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/proxy_management/inspecting_pw_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/proxy_management/integration_bs_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/proxy_management/integration_pw_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/proxy_management/quick_start_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/proxy_management/session_bs_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/proxy_management/session_pw_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/proxy_management/tiers_bs_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/proxy_management/tiers_pw_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/request_loaders/rl_basic_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/request_loaders/rl_basic_example_with_persist.py function main (line 19) | async def main() -> None: FILE: docs/guides/code_examples/request_loaders/rl_tandem_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/request_loaders/rl_tandem_example_explicit.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/request_loaders/sitemap_basic_example.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/request_loaders/sitemap_example_with_persist.py function main (line 20) | async def main() -> None: FILE: docs/guides/code_examples/request_loaders/sitemap_tandem_example.py function main (line 9) | async def main() -> None: FILE: docs/guides/code_examples/request_loaders/sitemap_tandem_example_explicit.py function main (line 10) | async def main() -> None: FILE: docs/guides/code_examples/request_router/adaptive_crawler_handlers.py function main (line 11) | async def main() -> None: FILE: docs/guides/code_examples/request_router/basic_request_handlers.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/request_router/custom_router_default_only.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/request_router/error_handler.py function main (line 10) | async def main() -> None: FILE: docs/guides/code_examples/request_router/failed_request_handler.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/request_router/http_pre_navigation.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/request_router/playwright_pre_navigation.py function main (line 10) | async def main() -> None: FILE: docs/guides/code_examples/request_router/simple_default_handler.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/running_in_web_server/crawler.py class State (line 11) | class State(TypedDict): function lifespan (line 19) | async def lifespan(app: FastAPI) -> AsyncIterator[State]: FILE: docs/guides/code_examples/running_in_web_server/server.py function index (line 18) | def index() -> str: function scrape_url (line 36) | async def scrape_url(request: Request, url: str | None = None) -> dict: FILE: docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_conflicts.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_crawler_configuration.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_crawler_event_manager.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_crawler_storage_client.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_locator_configuration.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_locator_event_manager.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_locator_storage_client.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_storage_configuration.py function main (line 10) | async def main() -> None: FILE: docs/guides/code_examples/service_locator/service_storage_storage_client.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/session_management/multi_sessions_http.py function create_session_function (line 15) | def create_session_function() -> Callable[[], Session]: function main (line 30) | async def main() -> None: FILE: docs/guides/code_examples/session_management/one_session_http.py function main (line 10) | async def main() -> None: FILE: docs/guides/code_examples/session_management/sm_basic.py function main (line 9) | async def main() -> None: FILE: docs/guides/code_examples/session_management/sm_beautifulsoup.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/session_management/sm_http.py function main (line 9) | async def main() -> None: FILE: docs/guides/code_examples/session_management/sm_parsel.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/session_management/sm_playwright.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/session_management/sm_standalone.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storage_clients/custom_storage_client_example.py class CustomDatasetClient (line 18) | class CustomDatasetClient(DatasetClient): class CustomKeyValueStoreClient (line 23) | class CustomKeyValueStoreClient(KeyValueStoreClient): class CustomRequestQueueClient (line 28) | class CustomRequestQueueClient(RequestQueueClient): class CustomStorageClient (line 36) | class CustomStorageClient(StorageClient): method create_dataset_client (line 37) | async def create_dataset_client( method create_kvs_client (line 47) | async def create_kvs_client( method create_rq_client (line 57) | async def create_rq_client( FILE: docs/guides/code_examples/storage_clients/registering_storage_clients_example.py function main (line 9) | async def main() -> None: FILE: docs/guides/code_examples/storage_clients/sql_storage_client_basic_example.py function main (line 5) | async def main() -> None: FILE: docs/guides/code_examples/storage_clients/sql_storage_client_configuration_example.py function main (line 8) | async def main() -> None: FILE: docs/guides/code_examples/storages/cleaning_do_not_purge_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/dataset_basic_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/dataset_with_crawler_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/storages/helper_add_requests_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/helper_enqueue_links_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/kvs_basic_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/kvs_with_crawler_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/storages/opening.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/rq_basic_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/rq_with_crawler_example.py function main (line 6) | async def main() -> None: FILE: docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py function main (line 7) | async def main() -> None: FILE: docs/guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py function instrument_crawler (line 14) | def instrument_crawler() -> None: function main (line 35) | async def main() -> None: FILE: docs/introduction/code_examples/02_bs.py function main (line 8) | async def main() -> None: FILE: docs/introduction/code_examples/02_bs_better.py function main (line 7) | async def main() -> None: FILE: docs/introduction/code_examples/02_request_queue.py function main (line 6) | async def main() -> None: FILE: docs/introduction/code_examples/03_enqueue_strategy.py function main (line 6) | async def main() -> None: FILE: docs/introduction/code_examples/03_finding_new_links.py function main (line 6) | async def main() -> None: FILE: docs/introduction/code_examples/03_globs.py function main (line 7) | async def main() -> None: FILE: docs/introduction/code_examples/03_original_code.py function main (line 6) | async def main() -> None: FILE: docs/introduction/code_examples/03_transform_request.py function transform_request (line 9) | def transform_request( function main (line 31) | async def main() -> None: FILE: docs/introduction/code_examples/04_sanity_check.py function main (line 7) | async def main() -> None: FILE: docs/introduction/code_examples/05_crawling_detail.py function main (line 6) | async def main() -> None: FILE: docs/introduction/code_examples/05_crawling_listing.py function main (line 6) | async def main() -> None: FILE: docs/introduction/code_examples/06_scraping.py function main (line 6) | async def main() -> None: FILE: docs/introduction/code_examples/07_final_code.py function main (line 6) | async def main() -> None: FILE: docs/introduction/code_examples/07_first_code.py function main (line 9) | async def main() -> None: FILE: docs/introduction/code_examples/08_main.py function main (line 8) | async def main() -> None: FILE: docs/introduction/code_examples/08_routes.py function default_handler (line 8) | async def default_handler(context: PlaywrightCrawlingContext) -> None: function category_handler (line 21) | async def category_handler(context: PlaywrightCrawlingContext) -> None: function detail_handler (line 42) | async def detail_handler(context: PlaywrightCrawlingContext) -> None: FILE: docs/introduction/code_examples/09_apify_sdk.py function main (line 11) | async def main() -> None: FILE: docs/quick-start/code_examples/beautifulsoup_crawler_example.py function main (line 6) | async def main() -> None: FILE: docs/quick-start/code_examples/parsel_crawler_example.py function main (line 6) | async def main() -> None: FILE: docs/quick-start/code_examples/playwright_crawler_example.py function main (line 6) | async def main() -> None: FILE: docs/quick-start/code_examples/playwright_crawler_headful_example.py function main (line 6) | async def main() -> None: FILE: src/crawlee/_autoscaling/_types.py class LoadRatioInfo (line 17) | class LoadRatioInfo: method is_overloaded (line 28) | def is_overloaded(self) -> bool: class SystemInfo (line 34) | class SystemInfo: method is_system_idle (line 53) | def is_system_idle(self) -> bool: method __str__ (line 62) | def __str__(self) -> str: class CpuSnapshot (line 74) | class CpuSnapshot: method is_overloaded (line 87) | def is_overloaded(self) -> bool: class MemorySnapshot (line 93) | class MemorySnapshot: method is_overloaded (line 115) | def is_overloaded(self) -> bool: class EventLoopSnapshot (line 126) | class EventLoopSnapshot: method max_delay_exceeded (line 139) | def max_delay_exceeded(self) -> timedelta: method is_overloaded (line 144) | def is_overloaded(self) -> bool: class ClientSnapshot (line 150) | class ClientSnapshot: method is_overloaded (line 166) | def is_overloaded(self) -> bool: class Ratio (line 175) | class Ratio: FILE: src/crawlee/_autoscaling/autoscaled_pool.py class AbortError (line 24) | class AbortError(Exception): class _AutoscaledPoolRun (line 28) | class _AutoscaledPoolRun: method __init__ (line 29) | def __init__(self) -> None: class AutoscaledPool (line 39) | class AutoscaledPool: method __init__ (line 64) | def __init__( method run (line 105) | async def run(self) -> None: method abort (line 155) | async def abort(self) -> None: method pause (line 163) | def pause(self) -> None: method resume (line 167) | def resume(self) -> None: method desired_concurrency (line 172) | def desired_concurrency(self) -> int: method current_concurrency (line 177) | def current_concurrency(self) -> int: method _autoscale (line 184) | def _autoscale(self) -> None: method _log_system_status (line 204) | def _log_system_status(self) -> None: method _worker_task_orchestrator (line 213) | async def _worker_task_orchestrator(self, run: _AutoscaledPoolRun) -> ... method _reap_worker_task (line 262) | def _reap_worker_task(self, task: asyncio.Task, run: _AutoscaledPoolRu... method _worker_task (line 275) | async def _worker_task(self) -> None: FILE: src/crawlee/_autoscaling/snapshotter.py function _warn_once (line 31) | def _warn_once(warning_message: str) -> None: class SortedSnapshotList (line 36) | class SortedSnapshotList(list[T]): method add (line 39) | def add(self, item: T) -> None: class Snapshotter (line 45) | class Snapshotter: method __init__ (line 72) | def __init__( method from_config (line 119) | def from_config(cls, config: Configuration | None = None) -> Snapshotter: method _get_sorted_list_by_created_at (line 145) | def _get_sorted_list_by_created_at(input_list: list[T]) -> SortedSnaps... method active (line 155) | def active(self) -> bool: method __aenter__ (line 159) | async def __aenter__(self) -> Snapshotter: method __aexit__ (line 176) | async def __aexit__( method get_memory_sample (line 201) | def get_memory_sample(self, duration: timedelta | None = None) -> list... method get_event_loop_sample (line 214) | def get_event_loop_sample(self, duration: timedelta | None = None) -> ... method get_cpu_sample (line 227) | def get_cpu_sample(self, duration: timedelta | None = None) -> list[Sn... method get_client_sample (line 240) | def get_client_sample(self, duration: timedelta | None = None) -> list... method _get_sample (line 253) | def _get_sample(snapshots: list[Snapshot], duration: timedelta | None ... method _snapshot_cpu (line 264) | async def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None: method _snapshot_memory (line 285) | async def _snapshot_memory(self, event_data: EventSystemInfoData) -> N... method _snapshot_event_loop (line 346) | async def _snapshot_event_loop(self) -> None: method _snapshot_client (line 367) | async def _snapshot_client(self) -> None: method _prune_snapshots (line 391) | def _prune_snapshots(self, snapshots: list[Snapshot], now: datetime) -... method _evaluate_memory_load (line 416) | def _evaluate_memory_load( FILE: src/crawlee/_autoscaling/system_status.py class SystemStatus (line 21) | class SystemStatus: method __init__ (line 39) | def __init__( method get_current_system_info (line 71) | def get_current_system_info(self) -> SystemInfo: method get_historical_system_info (line 82) | def get_historical_system_info(self) -> SystemInfo: method _get_system_info (line 93) | def _get_system_info(self, *, sample_duration: timedelta | None = None... method _is_cpu_overloaded (line 115) | def _is_cpu_overloaded(self, sample_duration: timedelta | None = None)... method _is_memory_overloaded (line 128) | def _is_memory_overloaded(self, sample_duration: timedelta | None = No... method _is_event_loop_overloaded (line 141) | def _is_event_loop_overloaded(self, sample_duration: timedelta | None ... method _is_client_overloaded (line 154) | def _is_client_overloaded(self, sample_duration: timedelta | None = No... method _is_sample_overloaded (line 167) | def _is_sample_overloaded(self, sample: list[Snapshot], threshold: flo... FILE: src/crawlee/_cli.py function callback (line 39) | def callback( function _prompt_for_project_name (line 56) | def _prompt_for_project_name(initial_project_name: str | None) -> str: function _prompt_text (line 84) | def _prompt_text(message: str, default: str) -> str: function _prompt_choice (line 98) | def _prompt_choice(message: str, choices: list[str]) -> str: function _prompt_bool (line 112) | def _prompt_bool(message: str, *, default: bool) -> bool: function create (line 126) | def create( FILE: src/crawlee/_log_config.py function string_to_log_level (line 41) | def string_to_log_level(level: LogLevel) -> int: function get_configured_log_level (line 57) | def get_configured_log_level() -> int: function configure_logger (line 69) | def configure_logger(logger: logging.Logger, *, remove_old_handlers: boo... class CrawleeLogFormatter (line 84) | class CrawleeLogFormatter(logging.Formatter): method __init__ (line 100) | def __init__( method _get_extra_fields (line 116) | def _get_extra_fields(self, record: logging.LogRecord) -> dict[str, Any]: method format (line 124) | def format(self, record: logging.LogRecord) -> str: FILE: src/crawlee/_request.py class RequestState (line 21) | class RequestState(IntEnum): class CrawleeRequestData (line 34) | class CrawleeRequestData(BaseModel): class UserData (line 65) | class UserData(BaseModel, MutableMapping[str, JsonSerializable]): method __getitem__ (line 81) | def __getitem__(self, key: str) -> JsonSerializable: method __setitem__ (line 84) | def __setitem__(self, key: str, value: JsonSerializable) -> None: method __delitem__ (line 93) | def __delitem__(self, key: str) -> None: method __iter__ (line 96) | def __iter__(self) -> Iterator[str]: # ty: ignore[invalid-method-over... method __len__ (line 99) | def __len__(self) -> int: method __eq__ (line 102) | def __eq__(self, other: object) -> bool: method __hash__ (line 111) | def __hash__(self) -> int: class RequestOptions (line 121) | class RequestOptions(TypedDict): class Request (line 145) | class Request(BaseModel): method from_url (line 243) | def from_url( method get_query_param_from_url (line 344) | def get_query_param_from_url(self, param: str, *, default: str | None ... method label (line 350) | def label(self) -> str | None: method session_id (line 355) | def session_id(self) -> str | None: method crawlee_data (line 360) | def crawlee_data(self) -> CrawleeRequestData: method crawl_depth (line 369) | def crawl_depth(self) -> int: method crawl_depth (line 374) | def crawl_depth(self, new_value: int) -> None: method state (line 378) | def state(self) -> RequestState: method state (line 383) | def state(self, new_state: RequestState) -> None: method max_retries (line 387) | def max_retries(self) -> int | None: method session_rotation_count (line 392) | def session_rotation_count(self) -> int | None: method session_rotation_count (line 397) | def session_rotation_count(self, new_session_rotation_count: int) -> N... method enqueue_strategy (line 401) | def enqueue_strategy(self) -> EnqueueStrategy: method enqueue_strategy (line 406) | def enqueue_strategy(self, new_enqueue_strategy: EnqueueStrategy) -> N... method last_proxy_tier (line 410) | def last_proxy_tier(self) -> int | None: method last_proxy_tier (line 415) | def last_proxy_tier(self, new_value: int) -> None: method forefront (line 419) | def forefront(self) -> bool: method forefront (line 424) | def forefront(self, new_value: bool) -> None: method was_already_handled (line 428) | def was_already_handled(self) -> bool: class RequestWithLock (line 433) | class RequestWithLock(Request): FILE: src/crawlee/_service_locator.py class ServiceLocator (line 20) | class ServiceLocator: method __init__ (line 28) | def __init__( method get_configuration (line 38) | def get_configuration(self) -> Configuration: method set_configuration (line 46) | def set_configuration(self, configuration: Configuration) -> None: method get_event_manager (line 63) | def get_event_manager(self) -> EventManager: method set_event_manager (line 76) | def set_event_manager(self, event_manager: EventManager) -> None: method get_storage_client (line 93) | def get_storage_client(self) -> StorageClient: method set_storage_client (line 106) | def set_storage_client(self, storage_client: StorageClient) -> None: method storage_instance_manager (line 124) | def storage_instance_manager(self) -> StorageInstanceManager: FILE: src/crawlee/_types.py function _normalize_headers (line 52) | def _normalize_headers(headers: Mapping[str, str]) -> dict[str, str]: class HttpHeaders (line 60) | class HttpHeaders(RootModel, Mapping[str, str]): method __getitem__ (line 75) | def __getitem__(self, key: str) -> str: method __setitem__ (line 78) | def __setitem__(self, key: str, value: str) -> None: method __delitem__ (line 81) | def __delitem__(self, key: str) -> None: method __or__ (line 84) | def __or__(self, other: HttpHeaders) -> HttpHeaders: method __ror__ (line 89) | def __ror__(self, other: HttpHeaders) -> HttpHeaders: method __iter__ (line 94) | def __iter__(self) -> Iterator[str]: # ty: ignore[invalid-method-over... method __len__ (line 97) | def __len__(self) -> int: class ConcurrencySettings (line 102) | class ConcurrencySettings: method __init__ (line 105) | def __init__( class EnqueueLinksKwargs (line 144) | class EnqueueLinksKwargs(TypedDict): class AddRequestsKwargs (line 175) | class AddRequestsKwargs(EnqueueLinksKwargs): class PushDataKwargs (line 193) | class PushDataKwargs(TypedDict): class PushDataFunctionCall (line 197) | class PushDataFunctionCall(PushDataKwargs): class KeyValueStoreInterface (line 204) | class KeyValueStoreInterface(Protocol): method get_value (line 208) | async def get_value(self, key: str) -> Any: ... method get_value (line 211) | async def get_value(self, key: str, default_value: T) -> T: ... method get_value (line 214) | async def get_value(self, key: str, default_value: T | None = None) ->... method get_value (line 216) | async def get_value(self, key: str, default_value: T | None = None) ->... method set_value (line 218) | async def set_value( class KeyValueStoreValue (line 227) | class KeyValueStoreValue: class KeyValueStoreChangeRecords (line 232) | class KeyValueStoreChangeRecords: method __init__ (line 233) | def __init__(self, actual_key_value_store: KeyValueStore) -> None: method set_value (line 237) | async def set_value( method get_value (line 246) | async def get_value(self, key: str) -> Any: ... method get_value (line 249) | async def get_value(self, key: str, default_value: T) -> T: ... method get_value (line 252) | async def get_value(self, key: str, default_value: T | None = None) ->... method get_value (line 254) | async def get_value(self, key: str, default_value: T | None = None) ->... class RequestHandlerRunResult (line 261) | class RequestHandlerRunResult: method __init__ (line 264) | def __init__( method request (line 279) | def request(self) -> Request: method add_requests (line 282) | async def add_requests( method push_data (line 298) | async def push_data( method get_key_value_store (line 317) | async def get_key_value_store( method apply_request_changes (line 331) | def apply_request_changes(self, target: Request) -> None: class AddRequestsFunction (line 341) | class AddRequestsFunction(Protocol): method __call__ (line 348) | def __call__( class EnqueueLinksFunction (line 371) | class EnqueueLinksFunction(Protocol): method __call__ (line 386) | def __call__( method __call__ (line 401) | def __call__( method __call__ (line 411) | def __call__( class ExtractLinksFunction (line 452) | class ExtractLinksFunction(Protocol): method __call__ (line 459) | def __call__( class GetKeyValueStoreFunction (line 489) | class GetKeyValueStoreFunction(Protocol): method __call__ (line 495) | def __call__( class GetKeyValueStoreFromRequestHandlerFunction (line 511) | class GetKeyValueStoreFromRequestHandlerFunction(Protocol): method __call__ (line 517) | def __call__( class PushDataFunction (line 534) | class PushDataFunction(Protocol): method __call__ (line 541) | def __call__( class SendRequestFunction (line 561) | class SendRequestFunction(Protocol): method __call__ (line 568) | def __call__( class PageSnapshot (line 591) | class PageSnapshot: method __bool__ (line 600) | def __bool__(self) -> bool: class UseStateFunction (line 605) | class UseStateFunction(Protocol): method __call__ (line 614) | def __call__( class BasicCrawlingContext (line 630) | class BasicCrawlingContext: method get_snapshot (line 664) | async def get_snapshot(self) -> PageSnapshot: method __hash__ (line 668) | def __hash__(self) -> int: method create_modified_copy (line 672) | def create_modified_copy( class GetDataKwargs (line 691) | class GetDataKwargs(TypedDict): class ExportToKwargs (line 728) | class ExportToKwargs(TypedDict): class ExportDataJsonKwargs (line 750) | class ExportDataJsonKwargs(TypedDict): class ExportDataCsvKwargs (line 786) | class ExportDataCsvKwargs(TypedDict): FILE: src/crawlee/_utils/byte_size.py class ByteSize (line 13) | class ByteSize: method __post_init__ (line 18) | def __post_init__(self) -> None: method validate (line 23) | def validate(cls, value: Any) -> ByteSize: method from_kb (line 33) | def from_kb(cls, kb: float) -> ByteSize: method from_mb (line 37) | def from_mb(cls, mb: float) -> ByteSize: method from_gb (line 41) | def from_gb(cls, gb: float) -> ByteSize: method from_tb (line 45) | def from_tb(cls, tb: float) -> ByteSize: method to_kb (line 48) | def to_kb(self) -> float: method to_mb (line 51) | def to_mb(self) -> float: method to_gb (line 54) | def to_gb(self) -> float: method to_tb (line 57) | def to_tb(self) -> float: method __str__ (line 60) | def __str__(self) -> str: method __eq__ (line 71) | def __eq__(self, other: object) -> bool: method __hash__ (line 76) | def __hash__(self) -> int: method __lt__ (line 80) | def __lt__(self, other: object) -> bool: method __le__ (line 85) | def __le__(self, other: object) -> bool: method __gt__ (line 90) | def __gt__(self, other: object) -> bool: method __ge__ (line 95) | def __ge__(self, other: object) -> bool: method __add__ (line 100) | def __add__(self, other: object) -> ByteSize: method __sub__ (line 105) | def __sub__(self, other: object) -> ByteSize: method __mul__ (line 113) | def __mul__(self, other: object) -> ByteSize: method __truediv__ (line 119) | def __truediv__(self, other: object) -> float: method __rmul__ (line 127) | def __rmul__(self, other: object) -> ByteSize: FILE: src/crawlee/_utils/console.py function make_table (line 11) | def make_table(rows: Sequence[Sequence[str]], width: int = 100) -> str: FILE: src/crawlee/_utils/context.py function ensure_context (line 11) | def ensure_context(method: T) -> T: FILE: src/crawlee/_utils/crypto.py function compute_short_hash (line 7) | def compute_short_hash(data: bytes, *, length: int = 8) -> str: function crypto_random_object_id (line 21) | def crypto_random_object_id(length: int = 17) -> str: FILE: src/crawlee/_utils/docs.py function docs_group (line 31) | def docs_group(group_name: GroupName) -> Callable[[T], T]: # noqa: ARG001 FILE: src/crawlee/_utils/file.py function _write_file (line 22) | def _write_file(path: Path, data: str | bytes) -> None: function _write_file (line 36) | def _write_file(path: Path, data: str | bytes) -> None: function infer_mime_type (line 63) | def infer_mime_type(value: Any) -> str: function json_dumps (line 88) | async def json_dumps(obj: Any) -> str: function atomic_write (line 101) | async def atomic_write( function atomic_write (line 110) | async def atomic_write( function atomic_write (line 118) | async def atomic_write( function export_json_to_stream (line 152) | async def export_json_to_stream( function export_csv_to_stream (line 161) | async def export_csv_to_stream( FILE: src/crawlee/_utils/globs.py class Glob (line 11) | class Glob: method __init__ (line 14) | def __init__(self, glob: str) -> None: function _translate (line 19) | def _translate( function _fnmatch_translate (line 76) | def _fnmatch_translate(pat: str, star: str, question_mark: str) -> list[... FILE: src/crawlee/_utils/models.py function _timedelta_to_ms (line 15) | def _timedelta_to_ms(td: timedelta | None) -> float | None: function _timedelta_to_secs (line 23) | def _timedelta_to_secs(td: timedelta | None) -> float | None: function _timedelta_from_ms (line 34) | def _timedelta_from_ms(value: float | timedelta | Any | None, handler: C... function _timedelta_from_secs (line 49) | def _timedelta_from_secs( FILE: src/crawlee/_utils/raise_if_too_many_kwargs.py function raise_if_too_many_kwargs (line 4) | def raise_if_too_many_kwargs(max_kwargs: int = 1, **kwargs: Any) -> None: FILE: src/crawlee/_utils/recoverable_state.py class RecoverableState (line 19) | class RecoverableState(Generic[TStateModel]): method __init__ (line 34) | def __init__( method initialize (line 93) | async def initialize(self) -> TStateModel: method teardown (line 121) | async def teardown(self) -> None: method current_value (line 139) | def current_value(self) -> TStateModel: method is_initialized (line 147) | def is_initialized(self) -> bool: method has_persisted_state (line 151) | async def has_persisted_state(self) -> bool: method reset (line 161) | async def reset(self) -> None: method persist_state (line 175) | async def persist_state(self, event_data: EventPersistStateData | None... method _load_saved_state (line 200) | async def _load_saved_state(self) -> None: FILE: src/crawlee/_utils/recurring_task.py class RecurringTask (line 18) | class RecurringTask: method __init__ (line 27) | def __init__(self, func: Callable, delay: timedelta) -> None: method __aenter__ (line 37) | async def __aenter__(self) -> Self: method __aexit__ (line 41) | async def __aexit__( method _wrapper (line 49) | async def _wrapper(self) -> None: method start (line 60) | def start(self) -> None: method stop (line 68) | async def stop(self) -> None: FILE: src/crawlee/_utils/requests.py function normalize_url (line 16) | def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: function compute_unique_key (line 50) | def compute_unique_key( function _get_payload_hash (line 113) | def _get_payload_hash(payload: HttpPayload | None) -> str: function _get_headers_hash (line 118) | def _get_headers_hash(headers: HttpHeaders | None) -> str: FILE: src/crawlee/_utils/robots.py class RobotsTxtFile (line 22) | class RobotsTxtFile: method __init__ (line 23) | def __init__( method from_content (line 32) | async def from_content(cls, url: str, content: str) -> Self: method find (line 43) | async def find(cls, url: str, http_client: HttpClient, proxy_info: Pro... method load (line 55) | async def load(cls, url: str, http_client: HttpClient, proxy_info: Pro... method is_allowed (line 80) | def is_allowed(self, url: str, user_agent: str = '*') -> bool: method get_sitemaps (line 92) | def get_sitemaps(self) -> list[str]: method get_crawl_delay (line 96) | def get_crawl_delay(self, user_agent: str = '*') -> int | None: method parse_sitemaps (line 106) | async def parse_sitemaps(self) -> Sitemap: method parse_urls_from_sitemaps (line 114) | async def parse_urls_from_sitemaps(self) -> list[str]: FILE: src/crawlee/_utils/sitemap.py class SitemapUrl (line 40) | class SitemapUrl: class NestedSitemap (line 49) | class NestedSitemap: class ParseSitemapOptions (line 54) | class ParseSitemapOptions(TypedDict, total=False): class SitemapSource (line 61) | class SitemapSource(TypedDict): class _SitemapItem (line 68) | class _SitemapItem(TypedDict, total=False): class _XMLSaxSitemapHandler (line 77) | class _XMLSaxSitemapHandler(ContentHandler): method __init__ (line 78) | def __init__(self) -> None: method items (line 87) | def items(self) -> list[_SitemapItem]: method startElement (line 91) | def startElement(self, name: str, attrs: AttributesImpl) -> None: method characters (line 99) | def characters(self, content: str) -> None: method endElement (line 104) | def endElement(self, name: str) -> None: class _TxtSitemapParser (line 132) | class _TxtSitemapParser: method __init__ (line 135) | def __init__(self) -> None: method process_chunk (line 138) | async def process_chunk(self, chunk: str) -> AsyncGenerator[_SitemapIt... method flush (line 153) | async def flush(self) -> AsyncGenerator[_SitemapItem, None]: method close (line 161) | def close(self) -> None: class _XmlSitemapParser (line 166) | class _XmlSitemapParser: method __init__ (line 169) | def __init__(self) -> None: method process_chunk (line 174) | async def process_chunk(self, chunk: str) -> AsyncGenerator[_SitemapIt... method flush (line 188) | async def flush(self) -> AsyncGenerator[_SitemapItem, None]: method close (line 201) | def close(self) -> None: function _get_parser (line 207) | def _get_parser(content_type: str = '', url: str | None = None) -> _XmlS... function _get_origin_url (line 215) | def _get_origin_url(source: SitemapSource) -> str: function _process_sitemap_item (line 225) | async def _process_sitemap_item( function _process_raw_source (line 268) | async def _process_raw_source( function _fetch_and_process_sitemap (line 306) | async def _fetch_and_process_sitemap( class Sitemap (line 383) | class Sitemap: method __init__ (line 384) | def __init__(self, urls: list[str]) -> None: method urls (line 388) | def urls(self) -> list[str]: method try_common_names (line 392) | async def try_common_names(cls, url: str, http_client: HttpClient, pro... method load (line 398) | async def load( method from_xml_string (line 412) | async def from_xml_string(cls, content: str) -> Sitemap: method parse (line 416) | async def parse( function parse_sitemap (line 427) | async def parse_sitemap( function _merge_async_generators (line 496) | async def _merge_async_generators(*generators: AsyncGenerator) -> AsyncG... function _discover_for_hostname (line 526) | async def _discover_for_hostname( function discover_valid_sitemaps (line 581) | async def discover_valid_sitemaps( FILE: src/crawlee/_utils/system.py function _get_used_memory (line 28) | def _get_used_memory(process: psutil.Process) -> int: function _get_used_memory (line 32) | def _get_used_memory(process: psutil.Process) -> int: class CpuInfo (line 36) | class CpuInfo(BaseModel): class MemoryUsageInfo (line 59) | class MemoryUsageInfo(BaseModel): class MemoryInfo (line 87) | class MemoryInfo(MemoryUsageInfo): function get_cpu_info (line 106) | def get_cpu_info() -> CpuInfo: function get_memory_info (line 117) | def get_memory_info() -> MemoryInfo: FILE: src/crawlee/_utils/time.py class TimerResult (line 20) | class TimerResult: function measure_time (line 26) | def measure_time() -> Iterator[TimerResult]: class SharedTimeout (line 41) | class SharedTimeout: method __init__ (line 47) | def __init__(self, timeout: timedelta) -> None: method __aenter__ (line 52) | async def __aenter__(self) -> timedelta: method __aexit__ (line 61) | async def __aexit__( function format_duration (line 78) | def format_duration(duration: timedelta | None) -> str: FILE: src/crawlee/_utils/try_import.py function try_import (line 10) | def try_import(module_name: str, *symbol_names: str) -> Iterator[None]: function install_import_hook (line 22) | def install_import_hook(module_name: str) -> None: class FailedImport (line 28) | class FailedImport: class ImportWrapper (line 35) | class ImportWrapper(ModuleType): method __getattribute__ (line 38) | def __getattribute__(self, name: str) -> Any: FILE: src/crawlee/_utils/urls.py function is_url_absolute (line 13) | def is_url_absolute(url: str) -> bool: function convert_to_absolute_url (line 21) | def convert_to_absolute_url(base_url: str, relative_url: str) -> str: function to_absolute_url_iterator (line 26) | def to_absolute_url_iterator(base_url: str, urls: Iterator[str], logger:... function validate_http_url (line 44) | def validate_http_url(value: str | None) -> str | None: FILE: src/crawlee/_utils/wait.py function wait_for (line 15) | async def wait_for( function wait_for_all_tasks_for_finish (line 49) | async def wait_for_all_tasks_for_finish( FILE: src/crawlee/_utils/web.py function is_status_code_client_error (line 6) | def is_status_code_client_error(value: int) -> bool: function is_status_code_server_error (line 11) | def is_status_code_server_error(value: int) -> bool: function is_status_code_successful (line 16) | def is_status_code_successful(value: int) -> bool: FILE: src/crawlee/browsers/_browser_controller.py class BrowserController (line 21) | class BrowserController(ABC): method pages (line 29) | def pages(self) -> list[Page]: method total_opened_pages (line 34) | def total_opened_pages(self) -> int: method pages_count (line 39) | def pages_count(self) -> int: method last_page_opened_at (line 44) | def last_page_opened_at(self) -> datetime: method idle_time (line 49) | def idle_time(self) -> timedelta: method has_free_capacity (line 54) | def has_free_capacity(self) -> bool: method is_browser_connected (line 59) | def is_browser_connected(self) -> bool: method browser_type (line 64) | def browser_type(self) -> BrowserType: method new_page (line 68) | async def new_page( method close (line 89) | async def close(self, *, force: bool = False) -> None: FILE: src/crawlee/browsers/_browser_plugin.py class BrowserPlugin (line 19) | class BrowserPlugin(ABC): method active (line 31) | def active(self) -> bool: method browser_type (line 36) | def browser_type(self) -> BrowserType: method browser_launch_options (line 41) | def browser_launch_options(self) -> Mapping[str, Any]: method browser_new_context_options (line 51) | def browser_new_context_options(self) -> Mapping[str, Any]: method max_open_pages_per_browser (line 61) | def max_open_pages_per_browser(self) -> int: method __aenter__ (line 65) | async def __aenter__(self) -> BrowserPlugin: method __aexit__ (line 73) | async def __aexit__( method new_browser (line 86) | async def new_browser(self) -> BrowserController: FILE: src/crawlee/browsers/_browser_pool.py class BrowserPool (line 34) | class BrowserPool: method __init__ (line 48) | def __init__( method with_default_plugin (line 113) | def with_default_plugin( method plugins (line 170) | def plugins(self) -> Sequence[BrowserPlugin]: method active_browsers (line 175) | def active_browsers(self) -> Sequence[BrowserController]: method inactive_browsers (line 180) | def inactive_browsers(self) -> Sequence[BrowserController]: method pages (line 185) | def pages(self) -> Mapping[str, CrawleePage]: method total_pages_count (line 190) | def total_pages_count(self) -> int: method active (line 195) | def active(self) -> bool: method __aenter__ (line 199) | async def __aenter__(self) -> BrowserPool: method __aexit__ (line 223) | async def __aexit__( method new_page (line 251) | async def new_page( method new_page_with_each_plugin (line 281) | async def new_page_with_each_plugin(self) -> Sequence[CrawleePage]: method _get_new_page (line 294) | async def _get_new_page( method _pick_browser_with_free_capacity (line 342) | def _pick_browser_with_free_capacity( method _retire_browser (line 353) | def _retire_browser(self, browser: BrowserController) -> None: method _launch_new_browser (line 359) | async def _launch_new_browser(self, plugin: BrowserPlugin) -> BrowserC... method _identify_inactive_browsers (line 365) | def _identify_inactive_browsers(self) -> None: method _close_inactive_browsers (line 372) | async def _close_inactive_browsers(self) -> None: method _execute_hooks (line 379) | async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[Non... method _override_page_close (line 384) | def _override_page_close(self, crawlee_page: CrawleePage, browser_cont... method pre_page_create_hook (line 398) | def pre_page_create_hook( method post_page_create_hook (line 413) | def post_page_create_hook( method pre_page_close_hook (line 425) | def pre_page_close_hook( method post_page_close_hook (line 437) | def post_page_close_hook( FILE: src/crawlee/browsers/_playwright_browser.py class PlaywrightPersistentBrowser (line 22) | class PlaywrightPersistentBrowser(Browser): method __init__ (line 32) | def __init__( method browser_type (line 47) | def browser_type(self) -> BrowserType: method contexts (line 51) | def contexts(self) -> list[BrowserContext]: method is_connected (line 54) | def is_connected(self) -> bool: method new_context (line 57) | async def new_context(self, **context_options: Any) -> BrowserContext: method _delete_temp_dir (line 79) | async def _delete_temp_dir(self, _: BrowserContext | None) -> None: method close (line 85) | async def close(self, **kwargs: Any) -> None: method version (line 96) | def version(self) -> str: method new_page (line 99) | async def new_page(self, **kwargs: Any) -> Page: method new_browser_cdp_session (line 103) | async def new_browser_cdp_session(self) -> CDPSession: method start_tracing (line 106) | async def start_tracing(self, **kwargs: Any) -> None: method stop_tracing (line 109) | async def stop_tracing(self, **kwargs: Any) -> bytes: FILE: src/crawlee/browsers/_playwright_browser_controller.py class PlaywrightBrowserController (line 32) | class PlaywrightBrowserController(BrowserController): method __init__ (line 42) | def __init__( method _get_context_creation_lock (line 86) | async def _get_context_creation_lock(self) -> Lock: method pages (line 99) | def pages(self) -> list[Page]: method total_opened_pages (line 104) | def total_opened_pages(self) -> int: method pages_count (line 109) | def pages_count(self) -> int: method last_page_opened_at (line 114) | def last_page_opened_at(self) -> datetime: method idle_time (line 119) | def idle_time(self) -> timedelta: method has_free_capacity (line 124) | def has_free_capacity(self) -> bool: method is_browser_connected (line 129) | def is_browser_connected(self) -> bool: method browser_type (line 134) | def browser_type(self) -> BrowserType: method new_page (line 138) | async def new_page( method close (line 192) | async def close(self, *, force: bool = False) -> None: method _on_page_close (line 208) | def _on_page_close(self, page: Page) -> None: method _create_browser_context (line 212) | async def _create_browser_context( FILE: src/crawlee/browsers/_playwright_browser_plugin.py class PlaywrightBrowserPlugin (line 32) | class PlaywrightBrowserPlugin(BrowserPlugin): method __init__ (line 44) | def __init__( method active (line 117) | def active(self) -> bool: method browser_type (line 122) | def browser_type(self) -> BrowserType: method browser_launch_options (line 127) | def browser_launch_options(self) -> Mapping[str, Any]: method browser_new_context_options (line 138) | def browser_new_context_options(self) -> Mapping[str, Any]: method max_open_pages_per_browser (line 149) | def max_open_pages_per_browser(self) -> int: method __aenter__ (line 153) | async def __aenter__(self) -> PlaywrightBrowserPlugin: method __aexit__ (line 162) | async def __aexit__( method new_browser (line 177) | async def new_browser(self) -> PlaywrightBrowserController: FILE: src/crawlee/browsers/_types.py class CrawleePage (line 13) | class CrawleePage: FILE: src/crawlee/configuration.py class Configuration (line 20) | class Configuration(BaseSettings): method get_global_configuration (line 216) | def get_global_configuration(cls) -> Self: FILE: src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py class HttpCrawlerOptions (line 37) | class HttpCrawlerOptions( class AbstractHttpCrawler (line 51) | class AbstractHttpCrawler( method __init__ (line 70) | def __init__( method create_parsed_http_crawler_class (line 93) | def create_parsed_http_crawler_class( method _create_static_content_crawler_pipeline (line 118) | def _create_static_content_crawler_pipeline(self) -> ContextPipeline[P... method _execute_pre_navigation_hooks (line 130) | async def _execute_pre_navigation_hooks( method _execute_post_navigation_hooks (line 145) | async def _execute_post_navigation_hooks( method _parse_http_response (line 153) | async def _parse_http_response( method _create_extract_links_function (line 173) | def _create_extract_links_function( method _make_http_request (line 256) | async def _make_http_request(self, context: BasicCrawlingContext) -> A... method _handle_status_code_response (line 277) | async def _handle_status_code_response( method _handle_blocked_request_by_content (line 299) | async def _handle_blocked_request_by_content( method pre_navigation_hook (line 317) | def pre_navigation_hook(self, hook: Callable[[BasicCrawlingContext], A... method post_navigation_hook (line 325) | def post_navigation_hook(self, hook: Callable[[HttpCrawlingContext], A... FILE: src/crawlee/crawlers/_abstract_http/_abstract_http_parser.py class AbstractHttpParser (line 19) | class AbstractHttpParser(ABC, Generic[TParseResult, TSelectResult]): method parse (line 23) | async def parse(self, response: HttpResponse) -> TParseResult: method parse_text (line 34) | async def parse_text(self, text: str) -> TParseResult: method select (line 45) | async def select(self, parsed_content: TParseResult, selector: str) ->... method is_blocked (line 56) | def is_blocked(self, parsed_content: TParseResult) -> BlockedInfo: method is_matching_selector (line 84) | def is_matching_selector(self, parsed_content: TParseResult, selector:... method find_links (line 96) | def find_links(self, parsed_content: TParseResult, selector: str, attr... FILE: src/crawlee/crawlers/_abstract_http/_http_crawling_context.py class HttpCrawlingContext (line 18) | class HttpCrawlingContext(BasicCrawlingContext, HttpCrawlingResult): method from_basic_crawling_context (line 22) | def from_basic_crawling_context(cls, context: BasicCrawlingContext, ht... method get_snapshot (line 27) | async def get_snapshot(self) -> PageSnapshot: class ParsedHttpCrawlingContext (line 34) | class ParsedHttpCrawlingContext(HttpCrawlingContext, Generic[TParseResul... method from_http_crawling_context (line 45) | def from_http_crawling_context( FILE: src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py class _NonPersistentStatistics (line 57) | class _NonPersistentStatistics(Statistics): method __init__ (line 63) | def __init__(self) -> None: method __aenter__ (line 66) | async def __aenter__(self) -> Self: method __aexit__ (line 71) | async def __aexit__( class AdaptivePlaywrightCrawler (line 81) | class AdaptivePlaywrightCrawler( method __init__ (line 119) | def __init__( method with_beautifulsoup_static_parser (line 235) | def with_beautifulsoup_static_parser( method with_parsel_static_parser (line 261) | def with_parsel_static_parser( method _crawl_one (line 285) | async def _crawl_one( method _pipeline_call_factory (line 340) | def _pipeline_call_factory( method _run_request_handler (line 371) | async def _run_request_handler(self, context: BasicCrawlingContext) ->... method pre_navigation_hook (line 437) | def pre_navigation_hook( method post_navigation_hook (line 462) | def post_navigation_hook( method track_http_only_request_handler_runs (line 488) | def track_http_only_request_handler_runs(self) -> None: method track_browser_request_handler_runs (line 491) | def track_browser_request_handler_runs(self) -> None: method track_rendering_type_mispredictions (line 494) | def track_rendering_type_mispredictions(self) -> None: class SubCrawlerRun (line 499) | class SubCrawlerRun: FILE: src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler_statistics.py class AdaptivePlaywrightCrawlerStatisticState (line 12) | class AdaptivePlaywrightCrawlerStatisticState(StatisticsState): FILE: src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py class AdaptiveContextError (line 29) | class AdaptiveContextError(RuntimeError): class AdaptivePlaywrightCrawlingContext (line 35) | class AdaptivePlaywrightCrawlingContext( method page (line 50) | def page(self) -> Page: method infinite_scroll (line 60) | def infinite_scroll(self) -> Callable[[], Awaitable[None]]: method response (line 71) | def response(self) -> Response: method wait_for_selector (line 80) | async def wait_for_selector(self, selector: str, timeout: timedelta = ... method query_selector_one (line 93) | async def query_selector_one( method query_selector_all (line 111) | async def query_selector_all( method parse_with_static_parser (line 149) | async def parse_with_static_parser( method from_parsed_http_crawling_context (line 170) | def from_parsed_http_crawling_context( method from_playwright_crawling_context (line 179) | async def from_playwright_crawling_context( class AdaptivePlaywrightPreNavCrawlingContext (line 208) | class AdaptivePlaywrightPreNavCrawlingContext(BasicCrawlingContext): method page (line 222) | def page(self) -> Page: method from_pre_navigation_context (line 236) | def from_pre_navigation_context(cls, context: BasicCrawlingContext) ->... class AdaptivePlaywrightPostNavCrawlingContext (line 254) | class AdaptivePlaywrightPostNavCrawlingContext(HttpCrawlingContext): method page (line 264) | def page(self) -> Page: method response (line 274) | def response(self) -> Response: method from_post_navigation_context (line 284) | async def from_post_navigation_context( FILE: src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py class RenderingTypePredictorState (line 34) | class RenderingTypePredictorState(BaseModel): class RenderingTypePrediction (line 49) | class RenderingTypePrediction: class RenderingTypePredictor (line 62) | class RenderingTypePredictor(ABC): method __init__ (line 65) | def __init__(self) -> None: method predict (line 71) | def predict(self, request: Request) -> RenderingTypePrediction: method store_result (line 79) | def store_result(self, request: Request, rendering_type: RenderingType... method initialize (line 87) | async def initialize(self) -> None: method clear (line 93) | async def clear(self) -> None: method __aenter__ (line 99) | async def __aenter__(self) -> RenderingTypePredictor: method __aexit__ (line 104) | async def __aexit__( class DefaultRenderingTypePredictor (line 115) | class DefaultRenderingTypePredictor(RenderingTypePredictor): method __init__ (line 122) | def __init__( method initialize (line 161) | async def initialize(self) -> None: method clear (line 169) | async def clear(self) -> None: method predict (line 177) | def predict(self, request: Request) -> RenderingTypePrediction: method store_result (line 209) | def store_result(self, request: Request, rendering_type: RenderingType... method _retrain (line 222) | def _retrain(self) -> None: method _calculate_mean_similarity (line 235) | def _calculate_mean_similarity(self, url: UrlComponents, label: str, r... method _calculate_feature_vector (line 243) | def _calculate_feature_vector(self, url: UrlComponents, label: str) ->... function get_url_components (line 250) | def get_url_components(url: str) -> UrlComponents: function calculate_url_similarity (line 258) | def calculate_url_similarity(url_1: UrlComponents, url_2: UrlComponents)... FILE: src/crawlee/crawlers/_adaptive_playwright/_result_comparator.py function create_default_comparator (line 11) | def create_default_comparator( function full_result_comparator (line 22) | def full_result_comparator(result_1: RequestHandlerRunResult, result_2: ... function push_data_only_comparator (line 37) | def push_data_only_comparator(result_1: RequestHandlerRunResult, result_... FILE: src/crawlee/crawlers/_adaptive_playwright/_utils.py function sklearn_model_validator (line 7) | def sklearn_model_validator(v: LogisticRegression | dict[str, Any]) -> L... function sklearn_model_serializer (line 21) | def sklearn_model_serializer(model: LogisticRegression) -> dict[str, Any]: FILE: src/crawlee/crawlers/_basic/_basic_crawler.py class _BasicCrawlerOptions (line 113) | class _BasicCrawlerOptions(TypedDict): class _BasicCrawlerOptionsGeneric (line 221) | class _BasicCrawlerOptionsGeneric(TypedDict, Generic[TCrawlingContext, T... class BasicCrawlerOptions (line 235) | class BasicCrawlerOptions( class BasicCrawler (line 247) | class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]): method __init__ (line 275) | def __init__( method log (line 511) | def log(self) -> logging.Logger: method router (line 516) | def router(self) -> Router[TCrawlingContext]: method router (line 524) | def router(self, router: Router[TCrawlingContext]) -> None: method statistics (line 531) | def statistics(self) -> Statistics[TStatisticsState]: method stop (line 535) | def stop(self, reason: str = 'Stop was called externally.') -> None: method _wrap_handler_with_error_context (line 546) | def _wrap_handler_with_error_context( method _stop_if_max_requests_count_exceeded (line 564) | def _stop_if_max_requests_count_exceeded(self) -> None: method _get_session (line 574) | async def _get_session(self) -> Session | None: method _get_session_by_id (line 588) | async def _get_session_by_id(self, session_id: str | None) -> Session ... method _get_proxy_info (line 602) | async def _get_proxy_info(self, request: Request, session: Session | N... method get_request_manager (line 613) | async def get_request_manager(self) -> RequestManager: method get_dataset (line 623) | async def get_dataset( method get_key_value_store (line 639) | async def get_key_value_store( method error_handler (line 655) | def error_handler( method failed_request_handler (line 665) | def failed_request_handler( method on_skipped_request (line 675) | def on_skipped_request(self, callback: SkippedRequestCallback) -> Skip... method run (line 683) | async def run( method _run_crawler (line 770) | async def _run_crawler(self) -> None: method add_requests (line 795) | async def add_requests( method use_state (line 843) | async def use_state( method _save_crawler_state (line 850) | async def _save_crawler_state(self) -> None: method get_data (line 854) | async def get_data( method export_data (line 884) | async def export_data( method _push_data (line 928) | async def _push_data( method _should_retry_request (line 951) | def _should_retry_request(self, context: BasicCrawlingContext, error: ... method _check_url_after_redirects (line 968) | async def _check_url_after_redirects(self, context: TCrawlingContext) ... method _create_enqueue_links_function (line 984) | def _create_enqueue_links_function( method _enqueue_links_filter_iterator (line 1042) | def _enqueue_links_filter_iterator( method _check_enqueue_strategy (line 1080) | def _check_enqueue_strategy( method _check_url_patterns (line 1115) | def _check_url_patterns( method _handle_request_retries (line 1145) | async def _handle_request_retries( method _handle_request_error (line 1184) | async def _handle_request_error(self, context: TCrawlingContext | Basi... method _handle_failed_request (line 1210) | async def _handle_failed_request(self, context: TCrawlingContext | Bas... method _handle_skipped_request (line 1223) | async def _handle_skipped_request( method _get_message_from_error (line 1238) | def _get_message_from_error(self, error: Exception) -> str: method _get_only_inner_most_exception (line 1256) | def _get_only_inner_most_exception(self, error: BaseException) -> Base... method _prepare_send_request_function (line 1265) | def _prepare_send_request_function( method _convert_url_to_request_iterator (line 1288) | def _convert_url_to_request_iterator(self, urls: Sequence[str | Reques... method _add_requests (line 1301) | async def _add_requests( method _commit_request_handler_result (line 1335) | async def _commit_request_handler_result(self, context: BasicCrawlingC... method _commit_key_value_store_changes (line 1350) | async def _commit_key_value_store_changes( method __is_finished_function (line 1359) | async def __is_finished_function(self) -> bool: method __is_task_ready_function (line 1375) | async def __is_task_ready_function(self) -> bool: method __run_task_function (line 1387) | async def __run_task_function(self) -> None: method _run_request_handler (line 1512) | async def _run_request_handler(self, context: BasicCrawlingContext) ->... method _raise_for_error_status_code (line 1525) | def _raise_for_error_status_code(self, status_code: int) -> None: method _raise_for_session_blocked_status_code (line 1547) | def _raise_for_session_blocked_status_code(self, session: Session | No... method _check_request_collision (line 1563) | def _check_request_collision(self, request: Request, session: Session ... method _is_allowed_based_on_robots_txt_file (line 1578) | async def _is_allowed_based_on_robots_txt_file(self, url: str) -> bool: method _get_robots_txt_file_for_url (line 1589) | async def _get_robots_txt_file_for_url(self, url: str) -> RobotsTxtFil... method _find_txt_file_for_url (line 1613) | async def _find_txt_file_for_url(self, url: str) -> RobotsTxtFile: method _log_status_message (line 1621) | def _log_status_message(self, message: str, level: LogLevel = 'DEBUG')... method _crawler_state_task (line 1631) | async def _crawler_state_task(self) -> None: method _mark_request_as_handled (line 1671) | async def _mark_request_as_handled(self, request: Request) -> None: FILE: src/crawlee/crawlers/_basic/_context_pipeline.py class _Middleware (line 24) | class _Middleware(Generic[TMiddlewareCrawlingContext, TCrawlingContext]): method __init__ (line 27) | def __init__( method action (line 39) | async def action(self) -> TMiddlewareCrawlingContext: method cleanup (line 43) | async def cleanup(self, final_consumer_exception: Exception | None) ->... class ContextPipeline (line 57) | class ContextPipeline(Generic[TCrawlingContext]): method __init__ (line 63) | def __init__( method _middleware_chain (line 76) | def _middleware_chain(self) -> Generator[ContextPipeline[Any], None, N... method __call__ (line 82) | async def __call__( method compose (line 125) | def compose( FILE: src/crawlee/crawlers/_basic/_context_utils.py function swapped_context (line 15) | def swapped_context( FILE: src/crawlee/crawlers/_basic/_logging_utils.py function _get_only_innermost_exception (line 8) | def _get_only_innermost_exception(error: BaseException) -> BaseException: function _get_filtered_traceback_parts_for_asyncio_timeout_error (line 28) | def _get_filtered_traceback_parts_for_asyncio_timeout_error(traceback_pa... function _strip_pep657_highlighting (line 42) | def _strip_pep657_highlighting(traceback_part: str) -> str: function reduce_asyncio_timeout_error_to_relevant_traceback_parts (line 48) | def reduce_asyncio_timeout_error_to_relevant_traceback_parts( function _get_traceback_parts_for_innermost_exception (line 55) | def _get_traceback_parts_for_innermost_exception(error: Exception) -> li... function get_one_line_error_summary_if_possible (line 62) | def get_one_line_error_summary_if_possible(error: Exception) -> str: FILE: src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawler.py class BeautifulSoupCrawler (line 22) | class BeautifulSoupCrawler(AbstractHttpCrawler[BeautifulSoupCrawlingCont... method __init__ (line 57) | def __init__( FILE: src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawling_context.py class BeautifulSoupCrawlingContext (line 14) | class BeautifulSoupCrawlingContext(ParsedHttpCrawlingContext[BeautifulSo... method soup (line 21) | def soup(self) -> BeautifulSoup: method from_parsed_http_crawling_context (line 26) | def from_parsed_http_crawling_context(cls, context: ParsedHttpCrawling... method html_to_text (line 30) | def html_to_text(self) -> str: FILE: src/crawlee/crawlers/_beautifulsoup/_beautifulsoup_parser.py class BeautifulSoupParser (line 18) | class BeautifulSoupParser(AbstractHttpParser[BeautifulSoup, Tag]): method __init__ (line 21) | def __init__(self, parser: BeautifulSoupParserType = 'lxml') -> None: method parse (line 25) | async def parse(self, response: HttpResponse) -> BeautifulSoup: method parse_text (line 29) | async def parse_text(self, text: str) -> BeautifulSoup: method is_matching_selector (line 33) | def is_matching_selector(self, parsed_content: Tag, selector: str) -> ... method select (line 37) | async def select(self, parsed_content: Tag, selector: str) -> Sequence... method find_links (line 41) | def find_links(self, parsed_content: Tag, selector: str, attribute: st... FILE: src/crawlee/crawlers/_beautifulsoup/_utils.py function html_to_text (line 20) | def html_to_text(source: str | Tag) -> str: FILE: src/crawlee/crawlers/_http/_http_crawler.py class HttpCrawler (line 17) | class HttpCrawler(AbstractHttpCrawler[ParsedHttpCrawlingContext[bytes], ... method __init__ (line 49) | def __init__( FILE: src/crawlee/crawlers/_http/_http_parser.py class NoParser (line 18) | class NoParser(AbstractHttpParser[bytes, bytes]): method parse (line 26) | async def parse(self, response: HttpResponse) -> bytes: method parse_text (line 30) | async def parse_text(self, text: str) -> bytes: method select (line 34) | async def select(self, parsed_content: bytes, selector: str) -> Sequen... method is_blocked (line 38) | def is_blocked(self, parsed_content: bytes) -> BlockedInfo: # Intenti... method is_matching_selector (line 42) | def is_matching_selector(self, parsed_content: bytes, selector: str) -... method find_links (line 46) | def find_links( FILE: src/crawlee/crawlers/_parsel/_parsel_crawler.py class ParselCrawler (line 22) | class ParselCrawler(AbstractHttpCrawler[ParselCrawlingContext, Selector,... method __init__ (line 57) | def __init__( FILE: src/crawlee/crawlers/_parsel/_parsel_crawling_context.py class ParselCrawlingContext (line 14) | class ParselCrawlingContext(ParsedHttpCrawlingContext[Selector]): method selector (line 21) | def selector(self) -> Selector: method from_parsed_http_crawling_context (line 26) | def from_parsed_http_crawling_context(cls, context: ParsedHttpCrawling... method html_to_text (line 30) | def html_to_text(self) -> str: FILE: src/crawlee/crawlers/_parsel/_parsel_parser.py class ParselParser (line 19) | class ParselParser(AbstractHttpParser[Selector, Selector]): method parse (line 23) | async def parse(self, response: HttpResponse) -> Selector: method parse_text (line 28) | async def parse_text(self, text: str) -> Selector: method select (line 32) | async def select(self, parsed_content: Selector, selector: str) -> Seq... method is_matching_selector (line 36) | def is_matching_selector(self, parsed_content: Selector, selector: str... method find_links (line 40) | def find_links(self, parsed_content: Selector, selector: str, attribut... FILE: src/crawlee/crawlers/_parsel/_utils.py function html_to_text (line 16) | def html_to_text(source: str | Selector) -> str: FILE: src/crawlee/crawlers/_playwright/_playwright_crawler.py class PlaywrightCrawler (line 61) | class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, Statisti... method __init__ (line 101) | def __init__( method _open_page (line 219) | async def _open_page( method _prepare_request_interceptor (line 260) | def _prepare_request_interceptor( method _navigate (line 281) | async def _navigate( method _create_extract_links_function (line 356) | def _create_extract_links_function(self, context: PlaywrightPreNavCraw... method _handle_status_code_response (line 438) | async def _handle_status_code_response( method _handle_blocked_request_by_content (line 460) | async def _handle_blocked_request_by_content( method _execute_post_navigation_hooks (line 489) | async def _execute_post_navigation_hooks( method _create_crawling_context (line 496) | async def _create_crawling_context( method pre_navigation_hook (line 528) | def pre_navigation_hook(self, hook: Callable[[PlaywrightPreNavCrawling... method post_navigation_hook (line 536) | def post_navigation_hook(self, hook: Callable[[PlaywrightPostNavCrawli... method _get_cookies (line 544) | async def _get_cookies(self, page: Page) -> list[PlaywrightCookieParam]: method _update_cookies (line 549) | async def _update_cookies(self, page: Page, cookies: list[PlaywrightCo... method _find_txt_file_for_url (line 553) | async def _find_txt_file_for_url(self, url: str) -> RobotsTxtFile: class _PlaywrightCrawlerAdditionalOptions (line 564) | class _PlaywrightCrawlerAdditionalOptions(TypedDict): class PlaywrightCrawlerOptions (line 596) | class PlaywrightCrawlerOptions( FILE: src/crawlee/crawlers/_playwright/_playwright_crawling_context.py class PlaywrightCrawlingContext (line 18) | class PlaywrightCrawlingContext(PlaywrightPostNavCrawlingContext): FILE: src/crawlee/crawlers/_playwright/_playwright_http_client.py function browser_page_context (line 30) | async def browser_page_context(page: Page) -> AsyncGenerator[None, None]: class PlaywrightHttpClient (line 39) | class PlaywrightHttpClient(HttpClient): method __init__ (line 50) | def __init__(self) -> None: method crawl (line 55) | async def crawl( method send_request (line 67) | async def send_request( method stream (line 102) | def stream( method cleanup (line 115) | async def cleanup(self) -> None: FILE: src/crawlee/crawlers/_playwright/_playwright_post_nav_crawling_context.py class PlaywrightPostNavCrawlingContext (line 16) | class PlaywrightPostNavCrawlingContext(PlaywrightPreNavCrawlingContext): FILE: src/crawlee/crawlers/_playwright/_playwright_pre_nav_crawling_context.py class PlaywrightPreNavCrawlingContext (line 17) | class PlaywrightPreNavCrawlingContext(BasicCrawlingContext): method get_snapshot (line 32) | async def get_snapshot(self) -> PageSnapshot: FILE: src/crawlee/crawlers/_playwright/_types.py class BlockRequestsFunction (line 19) | class BlockRequestsFunction(Protocol): method __call__ (line 26) | async def __call__( class PlaywrightHttpResponse (line 38) | class PlaywrightHttpResponse: method read (line 46) | async def read(self) -> bytes: method read_stream (line 49) | async def read_stream(self) -> AsyncGenerator[bytes, None]: method from_playwright_response (line 55) | async def from_playwright_response(cls, response: Response | APIRespon... class GotoOptions (line 68) | class GotoOptions(TypedDict): FILE: src/crawlee/crawlers/_playwright/_utils.py function infinite_scroll (line 25) | async def infinite_scroll(page: Page) -> None: function block_requests (line 81) | async def block_requests( FILE: src/crawlee/crawlers/_types.py class BlockedInfo (line 7) | class BlockedInfo: method __bool__ (line 12) | def __bool__(self) -> bool: FILE: src/crawlee/errors.py class UserDefinedErrorHandlerError (line 28) | class UserDefinedErrorHandlerError(Exception): class UserHandlerTimeoutError (line 32) | class UserHandlerTimeoutError(UserDefinedErrorHandlerError): class SessionError (line 37) | class SessionError(Exception): class ServiceConflictError (line 45) | class ServiceConflictError(Exception): method __init__ (line 48) | def __init__(self, service: type, new_value: object, existing_value: o... class ProxyError (line 56) | class ProxyError(SessionError): class HttpStatusCodeError (line 61) | class HttpStatusCodeError(Exception): method __init__ (line 64) | def __init__(self, message: str, status_code: int) -> None: class HttpClientStatusCodeError (line 71) | class HttpClientStatusCodeError(HttpStatusCodeError): class RequestHandlerError (line 76) | class RequestHandlerError(Exception, Generic[TCrawlingContext]): method __init__ (line 79) | def __init__(self, wrapped_exception: Exception, crawling_context: TCr... class ContextPipelineInitializationError (line 86) | class ContextPipelineInitializationError(Exception): method __init__ (line 92) | def __init__(self, wrapped_exception: Exception, crawling_context: Bas... class ContextPipelineFinalizationError (line 99) | class ContextPipelineFinalizationError(Exception): method __init__ (line 105) | def __init__(self, wrapped_exception: Exception, crawling_context: Bas... class ContextPipelineInterruptedError (line 112) | class ContextPipelineInterruptedError(Exception): class RequestCollisionError (line 117) | class RequestCollisionError(Exception): FILE: src/crawlee/events/_event_manager.py class EventManagerOptions (line 41) | class EventManagerOptions(TypedDict): class EventManager (line 55) | class EventManager: method __init__ (line 63) | def __init__( method active (line 100) | def active(self) -> bool: method __aenter__ (line 104) | async def __aenter__(self) -> EventManager: method __aexit__ (line 117) | async def __aexit__( method on (line 143) | def on(self, *, event: Literal[Event.PERSIST_STATE], listener: EventLi... method on (line 145) | def on(self, *, event: Literal[Event.SYSTEM_INFO], listener: EventList... method on (line 147) | def on(self, *, event: Literal[Event.MIGRATING], listener: EventListen... method on (line 149) | def on(self, *, event: Literal[Event.ABORTING], listener: EventListene... method on (line 151) | def on(self, *, event: Literal[Event.EXIT], listener: EventListener[Ev... method on (line 153) | def on(self, *, event: Literal[Event.CRAWLER_STATUS], listener: EventL... method on (line 155) | def on(self, *, event: Event, listener: EventListener[None]) -> None: ... method on (line 157) | def on(self, *, event: Event, listener: EventListener[Any]) -> None: method off (line 207) | def off(self, *, event: Event, listener: EventListener[Any] | None = N... method emit (line 224) | def emit(self, *, event: Literal[Event.PERSIST_STATE], event_data: Eve... method emit (line 226) | def emit(self, *, event: Literal[Event.SYSTEM_INFO], event_data: Event... method emit (line 228) | def emit(self, *, event: Literal[Event.MIGRATING], event_data: EventMi... method emit (line 230) | def emit(self, *, event: Literal[Event.ABORTING], event_data: EventAbo... method emit (line 232) | def emit(self, *, event: Literal[Event.EXIT], event_data: EventExitDat... method emit (line 234) | def emit(self, *, event: Literal[Event.CRAWLER_STATUS], event_data: Ev... method emit (line 236) | def emit(self, *, event: Event, event_data: Any) -> None: ... method emit (line 239) | def emit(self, *, event: Event, event_data: EventData) -> None: method wait_for_all_listeners_to_complete (line 249) | async def wait_for_all_listeners_to_complete(self, *, timeout: timedel... method _emit_persist_state_event (line 268) | async def _emit_persist_state_event(self) -> None: FILE: src/crawlee/events/_local_event_manager.py class LocalEventManager (line 26) | class LocalEventManager(EventManager): method __init__ (line 34) | def __init__( method from_config (line 59) | def from_config(cls, config: Configuration | None = None) -> LocalEven... method __aenter__ (line 72) | async def __aenter__(self) -> LocalEventManager: method __aexit__ (line 81) | async def __aexit__( method _emit_system_info_event (line 94) | async def _emit_system_info_event(self) -> None: FILE: src/crawlee/events/_types.py class Event (line 15) | class Event(str, Enum): class EventPersistStateData (line 40) | class EventPersistStateData(BaseModel): class EventSystemInfoData (line 49) | class EventSystemInfoData(BaseModel): class EventMigratingData (line 62) | class EventMigratingData(BaseModel): class EventAbortingData (line 73) | class EventAbortingData(BaseModel): class EventExitData (line 80) | class EventExitData(BaseModel): class EventCrawlerStatusData (line 87) | class EventCrawlerStatusData(BaseModel): FILE: src/crawlee/fingerprint_suite/_browserforge_adapter.py class PatchedHeaderGenerator (line 30) | class PatchedHeaderGenerator(bf_HeaderGenerator): method _get_accept_language_header (line 33) | def _get_accept_language_header(self, locales: tuple[str, ...] | list[... method generate (line 56) | def generate( method _contains_all_sec_headers (line 120) | def _contains_all_sec_headers(self, headers: dict[str, str]) -> bool: method _get_expected_browser_keywords (line 123) | def _get_expected_browser_keywords(self, browser: str | None) -> set[s... method _get_single_browser_type (line 130) | def _get_single_browser_type(self, browser: Iterable[str | Browser] | ... class PatchedFingerprintGenerator (line 154) | class PatchedFingerprintGenerator(bf_FingerprintGenerator): method __init__ (line 157) | def __init__( class BrowserforgeFingerprintGenerator (line 181) | class BrowserforgeFingerprintGenerator(FingerprintGenerator): method __init__ (line 187) | def __init__( method generate (line 227) | def generate(self) -> bf_Fingerprint: class BrowserforgeHeaderGenerator (line 242) | class BrowserforgeHeaderGenerator: method __init__ (line 245) | def __init__(self) -> None: method generate (line 248) | def generate(self, browser_type: SupportedBrowserType = 'chrome') -> d... function get_available_header_network (line 253) | def get_available_header_network() -> dict: function get_available_header_values (line 258) | def get_available_header_values(header_network: dict, node_name: str | s... FILE: src/crawlee/fingerprint_suite/_fingerprint_generator.py class FingerprintGenerator (line 13) | class FingerprintGenerator(ABC): method generate (line 17) | def generate(self) -> Fingerprint: FILE: src/crawlee/fingerprint_suite/_header_generator.py function fingerprint_browser_type_from_playwright_browser_type (line 13) | def fingerprint_browser_type_from_playwright_browser_type( class HeaderGenerator (line 26) | class HeaderGenerator: method __init__ (line 29) | def __init__(self) -> None: method _select_specific_headers (line 32) | def _select_specific_headers(self, all_headers: dict[str, str], header... method get_specific_headers (line 35) | def get_specific_headers( method get_common_headers (line 48) | def get_common_headers(self) -> HttpHeaders: method get_random_user_agent_header (line 57) | def get_random_user_agent_header(self) -> HttpHeaders: method get_user_agent_header (line 62) | def get_user_agent_header( method get_sec_ch_ua_headers (line 73) | def get_sec_ch_ua_headers( FILE: src/crawlee/fingerprint_suite/_types.py class ScreenOptions (line 13) | class ScreenOptions(BaseModel): class HeaderGeneratorOptions (line 31) | class HeaderGeneratorOptions(BaseModel): FILE: src/crawlee/http_clients/_base.py class HttpResponse (line 23) | class HttpResponse(Protocol): method http_version (line 27) | def http_version(self) -> str: method status_code (line 31) | def status_code(self) -> int: method headers (line 35) | def headers(self) -> HttpHeaders: method read (line 38) | async def read(self) -> bytes: method read_stream (line 48) | def read_stream(self) -> AsyncIterator[bytes]: class HttpCrawlingResult (line 63) | class HttpCrawlingResult: class HttpClient (line 75) | class HttpClient(ABC): method __init__ (line 79) | def __init__( method active (line 95) | def active(self) -> bool: method crawl (line 100) | async def crawl( method send_request (line 128) | async def send_request( method stream (line 160) | def stream( method cleanup (line 193) | async def cleanup(self) -> None: method __aenter__ (line 201) | async def __aenter__(self) -> HttpClient: method __aexit__ (line 213) | async def __aexit__( FILE: src/crawlee/http_clients/_curl_impersonate.py class _EmptyCookies (line 41) | class _EmptyCookies(CurlCookies): method get_cookies_for_curl (line 43) | def get_cookies_for_curl(self, request: CurlRequest) -> list[CurlMorsel]: method update_cookies_from_curl (line 47) | def update_cookies_from_curl(self, morsels: list[CurlMorsel]) -> None: class _AsyncSession (line 51) | class _AsyncSession(AsyncSession): method __init__ (line 53) | def __init__(self, *args: Any, **kwargs: Any) -> None: class _CurlImpersonateResponse (line 58) | class _CurlImpersonateResponse: method __init__ (line 61) | def __init__(self, response: Response) -> None: method http_version (line 65) | def http_version(self) -> str: method status_code (line 84) | def status_code(self) -> int: method headers (line 88) | def headers(self) -> HttpHeaders: method read (line 91) | async def read(self) -> bytes: method read_stream (line 97) | async def read_stream(self) -> AsyncGenerator[bytes, None]: class CurlImpersonateHttpClient (line 109) | class CurlImpersonateHttpClient(HttpClient): method __init__ (line 128) | def __init__( method crawl (line 148) | async def crawl( method send_request (line 189) | async def send_request( method stream (line 230) | async def stream( method _get_client (line 273) | def _get_client(self, proxy_url: str | None) -> AsyncSession: method _convert_method (line 297) | def _convert_method(self, method: HttpMethod) -> CurlHttpMethod: method _is_proxy_error (line 332) | def _is_proxy_error(error: CurlRequestError) -> bool: method _get_cookies (line 347) | def _get_cookies(curl: Curl) -> list[Cookie]: method cleanup (line 360) | async def cleanup(self) -> None: FILE: src/crawlee/http_clients/_httpx.py class _HttpxResponse (line 32) | class _HttpxResponse: method __init__ (line 35) | def __init__(self, response: httpx.Response) -> None: method http_version (line 39) | def http_version(self) -> str: method status_code (line 43) | def status_code(self) -> int: method headers (line 47) | def headers(self) -> HttpHeaders: method read (line 50) | async def read(self) -> bytes: method read_stream (line 55) | async def read_stream(self) -> AsyncIterator[bytes]: class _HttpxTransport (line 63) | class _HttpxTransport(httpx.AsyncHTTPTransport): method handle_async_request (line 72) | async def handle_async_request(self, request: httpx.Request) -> httpx.... class HttpxHttpClient (line 86) | class HttpxHttpClient(HttpClient): method __init__ (line 107) | def __init__( method crawl (line 143) | async def crawl( method send_request (line 184) | async def send_request( method stream (line 220) | async def stream( method _build_request (line 253) | def _build_request( method _get_client (line 278) | def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: method _combine_headers (line 320) | def _combine_headers(self, explicit_headers: HttpHeaders | None) -> Ht... method _is_proxy_error (line 335) | def _is_proxy_error(error: httpx.TransportError) -> bool: method cleanup (line 349) | async def cleanup(self) -> None: FILE: src/crawlee/http_clients/_impit.py class _ClientCacheEntry (line 33) | class _ClientCacheEntry(TypedDict): class _ImpitResponse (line 40) | class _ImpitResponse: method __init__ (line 43) | def __init__(self, response: Response) -> None: method http_version (line 47) | def http_version(self) -> str: method status_code (line 51) | def status_code(self) -> int: method headers (line 55) | def headers(self) -> HttpHeaders: method read (line 58) | async def read(self) -> bytes: method read_stream (line 63) | async def read_stream(self) -> AsyncIterator[bytes]: class ImpitHttpClient (line 72) | class ImpitHttpClient(HttpClient): method __init__ (line 91) | def __init__( method crawl (line 121) | async def crawl( method send_request (line 155) | async def send_request( method stream (line 190) | async def stream( method _get_client (line 225) | def _get_client(self, proxy_url: str | None, cookie_jar: CookieJar | N... method _is_proxy_error (line 257) | def _is_proxy_error(error: HTTPError) -> bool: method cleanup (line 271) | async def cleanup(self) -> None: FILE: src/crawlee/otel/crawler_instrumentor.py class CrawlerInstrumentor (line 25) | class CrawlerInstrumentor(BaseInstrumentor): method __init__ (line 28) | def __init__( method instrumentation_dependencies (line 120) | def instrumentation_dependencies(self) -> list[str]: method _instrument_all_public_methods (line 124) | def _instrument_all_public_methods(self, on_class: type) -> None: method _instrument (line 144) | def _instrument(self, **_: Any) -> None: method _uninstrument (line 148) | def _uninstrument(self, **_: Any) -> None: FILE: src/crawlee/project_template/templates/main.py function main (line 32) | async def main() -> None: FILE: src/crawlee/project_template/templates/main_playwright_camoufox.py class CamoufoxPlugin (line 13) | class CamoufoxPlugin(PlaywrightBrowserPlugin): method new_browser (line 19) | async def new_browser(self) -> PlaywrightBrowserController: FILE: src/crawlee/project_template/templates/routes_beautifulsoup.py function default_handler (line 8) | async def default_handler(context: BeautifulSoupCrawlingContext) -> None: FILE: src/crawlee/project_template/templates/routes_parsel.py function default_handler (line 8) | async def default_handler(context: ParselCrawlingContext) -> None: FILE: src/crawlee/project_template/templates/routes_playwright.py function default_handler (line 8) | async def default_handler(context: PlaywrightCrawlingContext) -> None: FILE: src/crawlee/proxy_configuration.py class ProxyInfo (line 26) | class ProxyInfo: class ProxyConfiguration (line 56) | class ProxyConfiguration: method __init__ (line 68) | def __init__( method _create_url (line 106) | def _create_url(self, url: str | None) -> URL | None: method new_proxy_info (line 114) | async def new_proxy_info( method new_url (line 159) | async def new_url( method _pick_url (line 176) | async def _pick_url( class _ProxyTierTracker (line 224) | class _ProxyTierTracker: method __init__ (line 227) | def __init__(self, tiered_proxy_urls: list[list[URL | None]]) -> None: method all_urls (line 233) | def all_urls(self) -> Sequence[URL | None]: method get_tier_urls (line 236) | def get_tier_urls(self, tier_number: int) -> Sequence[URL | None]: method add_error (line 239) | def add_error(self, domain: str, tier: int) -> None: method predict_tier (line 242) | def predict_tier(self, domain: str) -> int: class _NewUrlFunction (line 263) | class _NewUrlFunction(Protocol): method __call__ (line 264) | def __call__( FILE: src/crawlee/request_loaders/_request_list.py class RequestListState (line 19) | class RequestListState(BaseModel): class RequestListData (line 27) | class RequestListData(BaseModel): class RequestList (line 32) | class RequestList(RequestLoader): method __init__ (line 35) | def __init__( method _get_state (line 92) | async def _get_state(self) -> RequestListState: method name (line 140) | def name(self) -> str | None: method get_handled_count (line 144) | async def get_handled_count(self) -> int: method get_total_count (line 148) | async def get_total_count(self) -> int: method is_empty (line 152) | async def is_empty(self) -> bool: method is_finished (line 157) | async def is_finished(self) -> bool: method fetch_next_request (line 162) | async def fetch_next_request(self) -> Request | None: method mark_request_as_handled (line 184) | async def mark_request_as_handled(self, request: Request) -> None: method _ensure_next_request (line 189) | async def _ensure_next_request(self) -> None: method _dequeue_requests (line 204) | async def _dequeue_requests(self, count: int) -> AsyncGenerator[Reques... method _iterate_in_threadpool (line 211) | async def _iterate_in_threadpool(self, iterable: Iterable[str | Reques... FILE: src/crawlee/request_loaders/_request_loader.py class RequestLoader (line 17) | class RequestLoader(ABC): method get_handled_count (line 29) | async def get_handled_count(self) -> int: method get_total_count (line 33) | async def get_total_count(self) -> int: method is_empty (line 37) | async def is_empty(self) -> bool: method is_finished (line 41) | async def is_finished(self) -> bool: method fetch_next_request (line 45) | async def fetch_next_request(self) -> Request | None: method mark_request_as_handled (line 53) | async def mark_request_as_handled(self, request: Request) -> Processed... method to_tandem (line 56) | async def to_tandem(self, request_manager: RequestManager | None = Non... method _transform_request (line 72) | def _transform_request(self, request: str | Request) -> Request: method _transform_requests (line 82) | def _transform_requests(self, requests: Sequence[str | Request]) -> li... FILE: src/crawlee/request_loaders/_request_manager.py class RequestManager (line 18) | class RequestManager(RequestLoader, ABC): method drop (line 22) | async def drop(self) -> None: method add_request (line 26) | async def add_request( method add_requests (line 43) | async def add_requests( method reclaim_request (line 71) | async def reclaim_request(self, request: Request, *, forefront: bool =... FILE: src/crawlee/request_loaders/_request_manager_tandem.py class RequestManagerTandem (line 24) | class RequestManagerTandem(RequestManager): method __init__ (line 31) | def __init__(self, request_loader: RequestLoader, request_manager: Req... method get_handled_count (line 36) | async def get_handled_count(self) -> int: method get_total_count (line 40) | async def get_total_count(self) -> int: method is_empty (line 44) | async def is_empty(self) -> bool: method is_finished (line 48) | async def is_finished(self) -> bool: method add_request (line 52) | async def add_request(self, request: str | Request, *, forefront: bool... method add_requests (line 56) | async def add_requests( method fetch_next_request (line 76) | async def fetch_next_request(self) -> Request | None: method reclaim_request (line 99) | async def reclaim_request(self, request: Request, *, forefront: bool =... method mark_request_as_handled (line 103) | async def mark_request_as_handled(self, request: Request) -> None: method drop (line 107) | async def drop(self) -> None: FILE: src/crawlee/request_loaders/_sitemap_request_loader.py class SitemapRequestLoaderState (line 33) | class SitemapRequestLoaderState(BaseModel): class SitemapRequestLoader (line 91) | class SitemapRequestLoader(RequestLoader): method __init__ (line 106) | def __init__( method _get_state (line 161) | async def _get_state(self) -> SitemapRequestLoaderState: method _check_url_patterns (line 189) | def _check_url_patterns( method _load_sitemaps (line 219) | async def _load_sitemaps(self) -> None: method get_total_count (line 288) | async def get_total_count(self) -> int: method get_handled_count (line 294) | async def get_handled_count(self) -> int: method is_empty (line 300) | async def is_empty(self) -> bool: method is_finished (line 306) | async def is_finished(self) -> bool: method fetch_next_request (line 312) | async def fetch_next_request(self) -> Request | None: method mark_request_as_handled (line 340) | async def mark_request_as_handled(self, request: Request) -> Processed... method abort_loading (line 348) | async def abort_loading(self) -> None: method start (line 355) | async def start(self) -> None: method close (line 361) | async def close(self) -> None: method __aenter__ (line 366) | async def __aenter__(self) -> SitemapRequestLoader: method __aexit__ (line 371) | async def __aexit__( FILE: src/crawlee/router.py class Router (line 20) | class Router(Generic[TCrawlingContext]): method __init__ (line 59) | def __init__(self) -> None: method default_handler (line 63) | def default_handler(self: Router, handler: RequestHandler[TCrawlingCon... method handler (line 76) | def handler( method __call__ (line 94) | async def __call__(self, context: TCrawlingContext) -> None: FILE: src/crawlee/sessions/_cookies.py class CookieParam (line 17) | class CookieParam(TypedDict, total=False): class PlaywrightCookieParam (line 45) | class PlaywrightCookieParam(TypedDict, total=False): class SessionCookies (line 60) | class SessionCookies: method __init__ (line 63) | def __init__(self, cookies: SessionCookies | CookieJar | dict[str, str... method jar (line 84) | def jar(self) -> CookieJar: method set (line 88) | def set( method _convert_cookie_to_dict (line 138) | def _convert_cookie_to_dict(self, cookie: Cookie) -> CookieParam: method _to_playwright (line 161) | def _to_playwright(self, cookie_dict: CookieParam) -> PlaywrightCookie... method _from_playwright (line 174) | def _from_playwright(self, cookie_dict: PlaywrightCookieParam) -> Cook... method get_cookies_as_dicts (line 188) | def get_cookies_as_dicts(self) -> list[CookieParam]: method store_cookie (line 192) | def store_cookie(self, cookie: Cookie) -> None: method store_cookies (line 200) | def store_cookies(self, cookies: list[Cookie]) -> None: method set_cookies (line 210) | def set_cookies(self, cookie_dicts: list[CookieParam]) -> None: method get_cookies_as_playwright_format (line 220) | def get_cookies_as_playwright_format(self) -> list[PlaywrightCookiePar... method set_cookies_from_playwright_format (line 224) | def set_cookies_from_playwright_format(self, pw_cookies: list[Playwrig... method __deepcopy__ (line 231) | def __deepcopy__(self, memo: dict[int, Any] | None) -> SessionCookies: method __len__ (line 236) | def __len__(self) -> int: method __setitem__ (line 239) | def __setitem__(self, name: str, value: str) -> None: method __getitem__ (line 242) | def __getitem__(self, name: str) -> str | None: method __iter__ (line 248) | def __iter__(self) -> Iterator[CookieParam]: method __repr__ (line 251) | def __repr__(self) -> str: method __bool__ (line 257) | def __bool__(self) -> bool: method __eq__ (line 262) | def __eq__(self, other: object) -> bool: method __hash__ (line 274) | def __hash__(self) -> int: method _is_valid_same_site (line 279) | def _is_valid_same_site(self, value: str | None) -> TypeGuard[Literal[... FILE: src/crawlee/sessions/_models.py class SessionModel (line 20) | class SessionModel(BaseModel): class SessionPoolModel (line 38) | class SessionPoolModel(BaseModel): method session_count (line 68) | def session_count(self) -> int: method usable_session_count (line 74) | def usable_session_count(self) -> int: method retired_session_count (line 80) | def retired_session_count(self) -> int: FILE: src/crawlee/sessions/_session.py class Session (line 22) | class Session: method __init__ (line 34) | def __init__( method from_model (line 77) | def from_model(cls, model: SessionModel) -> Session: method __repr__ (line 82) | def __repr__(self) -> str: method __eq__ (line 86) | def __eq__(self, other: object) -> bool: method __hash__ (line 92) | def __hash__(self) -> int: method id (line 115) | def id(self) -> str: method user_data (line 120) | def user_data(self) -> dict: method cookies (line 125) | def cookies(self) -> SessionCookies: method error_score (line 130) | def error_score(self) -> float: method usage_count (line 135) | def usage_count(self) -> float: method expires_at (line 140) | def expires_at(self) -> datetime: method is_blocked (line 145) | def is_blocked(self) -> bool: method is_expired (line 150) | def is_expired(self) -> bool: method is_max_usage_count_reached (line 155) | def is_max_usage_count_reached(self) -> bool: method is_usable (line 160) | def is_usable(self) -> bool: method get_state (line 165) | def get_state(self, *, as_dict: Literal[True]) -> dict: ... method get_state (line 168) | def get_state(self, *, as_dict: Literal[False]) -> SessionModel: ... method get_state (line 170) | def get_state(self, *, as_dict: bool = False) -> SessionModel | dict: method mark_good (line 191) | def mark_good(self) -> None: method mark_bad (line 202) | def mark_bad(self) -> None: method retire (line 211) | def retire(self) -> None: method is_blocked_status_code (line 223) | def is_blocked_status_code( FILE: src/crawlee/sessions/_session_pool.py class SessionPool (line 28) | class SessionPool: method __init__ (line 36) | def __init__( method __repr__ (line 86) | def __repr__(self) -> str: method session_count (line 91) | def session_count(self) -> int: method usable_session_count (line 96) | def usable_session_count(self) -> int: method retired_session_count (line 101) | def retired_session_count(self) -> int: method active (line 106) | def active(self) -> bool: method __aenter__ (line 110) | async def __aenter__(self) -> SessionPool: method __aexit__ (line 130) | async def __aexit__( method get_state (line 149) | def get_state(self, *, as_dict: Literal[True]) -> dict: ... method get_state (line 152) | def get_state(self, *, as_dict: Literal[False]) -> SessionPoolModel: ... method get_state (line 155) | def get_state(self, *, as_dict: bool = False) -> SessionPoolModel | dict: method add_session (line 163) | def add_session(self, session: Session) -> None: method get_session (line 180) | async def get_session(self) -> Session: method get_session_by_id (line 200) | async def get_session_by_id(self, session_id: str) -> Session | None: method reset_store (line 225) | async def reset_store(self) -> None: method _create_new_session (line 229) | async def _create_new_session(self) -> Session: method _fill_sessions_to_max (line 238) | async def _fill_sessions_to_max(self) -> None: method _get_random_session (line 243) | def _get_random_session(self) -> Session: method _remove_retired_sessions (line 250) | def _remove_retired_sessions(self) -> None: FILE: src/crawlee/statistics/_error_snapshotter.py class ErrorSnapshotter (line 15) | class ErrorSnapshotter: method __init__ (line 23) | def __init__(self, *, snapshot_kvs_name: str | None = None) -> None: method capture_snapshot (line 26) | async def capture_snapshot( method _save_html (line 61) | async def _save_html(self, kvs: KeyValueStore, html: str, base_name: s... method _save_screenshot (line 65) | async def _save_screenshot(self, kvs: KeyValueStore, screenshot: bytes... method _sanitize_filename (line 69) | def _sanitize_filename(self, filename: str) -> str: method _get_snapshot_base_name (line 72) | def _get_snapshot_base_name(self, error_message: str, file_and_line: s... FILE: src/crawlee/statistics/_error_tracker.py class ErrorTracker (line 23) | class ErrorTracker: method __init__ (line 26) | def __init__( method add (line 46) | async def add( method _capture_error_snapshot (line 108) | async def _capture_error_snapshot( method _get_file_and_line (line 119) | def _get_file_and_line(self, error: Exception) -> str: method _get_error_message (line 126) | def _get_error_message(self, error: Exception) -> str: method unique_error_count (line 136) | def unique_error_count(self) -> int: method total (line 145) | def total(self) -> int: method get_most_common_errors (line 153) | def get_most_common_errors(self, n: int = 3) -> list[tuple[str | None,... method _get_error_repr (line 164) | def _get_error_repr(self, file_and_line: str | None, name: str | None,... method _create_generic_message (line 172) | def _create_generic_message(message_1: str | None, message_2: str | No... FILE: src/crawlee/statistics/_models.py class FinalStatistics (line 22) | class FinalStatistics: method to_table (line 36) | def to_table(self) -> str: method to_dict (line 47) | def to_dict(self) -> dict[str, float | int | list[int]]: method __str__ (line 51) | def __str__(self) -> str: class StatisticsState (line 58) | class StatisticsState(BaseModel): method model_post_init (line 110) | def model_post_init(self, /, __context: Any) -> None: method crawler_runtime (line 114) | def crawler_runtime(self) -> timedelta: method crawler_runtime (line 121) | def crawler_runtime(self, value: timedelta) -> None: method crawler_runtime_for_serialization (line 132) | def crawler_runtime_for_serialization(self) -> timedelta: method request_total_duration (line 140) | def request_total_duration(self) -> timedelta: method request_avg_failed_duration (line 145) | def request_avg_failed_duration(self) -> timedelta | None: method request_avg_finished_duration (line 150) | def request_avg_finished_duration(self) -> timedelta | None: method requests_total (line 155) | def requests_total(self) -> int: FILE: src/crawlee/statistics/_statistics.py class RequestProcessingRecord (line 31) | class RequestProcessingRecord: method __init__ (line 34) | def __init__(self) -> None: method run (line 39) | def run(self) -> int: method finish (line 45) | def finish(self) -> timedelta: method retry_count (line 54) | def retry_count(self) -> int: class Statistics (line 60) | class Statistics(Generic[TStatisticsState]): method __init__ (line 71) | def __init__( method replace_state_model (line 113) | def replace_state_model(self, state_model: type[TNewStatisticsState]) ... method with_default_state (line 127) | def with_default_state( method active (line 154) | def active(self) -> bool: method __aenter__ (line 158) | async def __aenter__(self) -> Self: method __aexit__ (line 180) | async def __aexit__( method state (line 204) | def state(self) -> TStatisticsState: method register_status_code (line 208) | def register_status_code(self, code: int) -> None: method record_request_processing_start (line 215) | def record_request_processing_start(self, request_id_or_key: str) -> N... method record_request_processing_finish (line 222) | def record_request_processing_finish(self, request_id_or_key: str) -> ... method record_request_processing_failure (line 244) | def record_request_processing_failure(self, request_id_or_key: str) ->... method calculate (line 258) | def calculate(self) -> FinalStatistics: method reset (line 277) | async def reset(self) -> None: method _log (line 284) | def _log(self) -> None: method _save_retry_count_for_request (line 291) | def _save_retry_count_for_request(self, record: RequestProcessingRecor... FILE: src/crawlee/storage_clients/_base/_dataset_client.py class DatasetClient (line 13) | class DatasetClient(ABC): method get_metadata (line 27) | async def get_metadata(self) -> DatasetMetadata: method drop (line 31) | async def drop(self) -> None: method purge (line 38) | async def purge(self) -> None: method push_data (line 45) | async def push_data(self, data: list[Any] | dict[str, Any]) -> None: method get_data (line 52) | async def get_data( method iterate_items (line 73) | async def iterate_items( FILE: src/crawlee/storage_clients/_base/_key_value_store_client.py class KeyValueStoreClient (line 12) | class KeyValueStoreClient(ABC): method get_metadata (line 26) | async def get_metadata(self) -> KeyValueStoreMetadata: method drop (line 30) | async def drop(self) -> None: method purge (line 37) | async def purge(self) -> None: method get_value (line 44) | async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: method set_value (line 51) | async def set_value(self, *, key: str, value: Any, content_type: str |... method delete_value (line 58) | async def delete_value(self, *, key: str) -> None: method iterate_keys (line 65) | async def iterate_keys( method get_public_url (line 82) | async def get_public_url(self, *, key: str) -> str: method record_exists (line 89) | async def record_exists(self, *, key: str) -> bool: FILE: src/crawlee/storage_clients/_base/_request_queue_client.py class RequestQueueClient (line 13) | class RequestQueueClient(ABC): method get_metadata (line 21) | async def get_metadata(self) -> RequestQueueMetadata: method drop (line 25) | async def drop(self) -> None: method purge (line 32) | async def purge(self) -> None: method add_batch_of_requests (line 39) | async def add_batch_of_requests( method get_request (line 66) | async def get_request(self, unique_key: str) -> Request | None: method fetch_next_request (line 77) | async def fetch_next_request(self) -> Request | None: method mark_request_as_handled (line 94) | async def mark_request_as_handled(self, request: Request) -> Processed... method reclaim_request (line 107) | async def reclaim_request( method is_empty (line 126) | async def is_empty(self) -> bool: FILE: src/crawlee/storage_clients/_base/_storage_client.py class StorageClient (line 19) | class StorageClient(ABC): method get_storage_client_cache_key (line 33) | def get_storage_client_cache_key(self, configuration: Configuration) -... method create_dataset_client (line 42) | async def create_dataset_client( method create_kvs_client (line 53) | async def create_kvs_client( method create_rq_client (line 64) | async def create_rq_client( method get_rate_limit_errors (line 74) | def get_rate_limit_errors(self) -> dict[int, int]: method _purge_if_needed (line 78) | async def _purge_if_needed( FILE: src/crawlee/storage_clients/_file_system/_dataset_client.py class FileSystemDatasetClient (line 29) | class FileSystemDatasetClient(DatasetClient): method __init__ (line 56) | def __init__( method get_metadata (line 76) | async def get_metadata(self) -> DatasetMetadata: method path_to_dataset (line 80) | def path_to_dataset(self) -> Path: method path_to_metadata (line 85) | def path_to_metadata(self) -> Path: method open (line 90) | async def open( method drop (line 205) | async def drop(self) -> None: method purge (line 211) | async def purge(self) -> None: method push_data (line 223) | async def push_data(self, data: list[dict[str, Any]] | dict[str, Any])... method get_data (line 242) | async def get_data( method iterate_items (line 341) | async def iterate_items( method _update_metadata (line 414) | async def _update_metadata( method _push_item (line 444) | async def _push_item(self, item: dict[str, Any], item_id: int) -> None: method _get_sorted_data_files (line 465) | async def _get_sorted_data_files(self) -> list[Path]: FILE: src/crawlee/storage_clients/_file_system/_key_value_store_client.py class FileSystemKeyValueStoreClient (line 32) | class FileSystemKeyValueStoreClient(KeyValueStoreClient): method __init__ (line 56) | def __init__( method get_metadata (line 76) | async def get_metadata(self) -> KeyValueStoreMetadata: method path_to_kvs (line 80) | def path_to_kvs(self) -> Path: method path_to_metadata (line 85) | def path_to_metadata(self) -> Path: method open (line 90) | async def open( method drop (line 204) | async def drop(self) -> None: method purge (line 211) | async def purge(self) -> None: method get_value (line 224) | async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: method set_value (line 303) | async def set_value(self, *, key: str, value: Any, content_type: str |... method delete_value (line 344) | async def delete_value(self, *, key: str) -> None: method iterate_keys (line 367) | async def iterate_keys( method get_public_url (line 428) | async def get_public_url(self, *, key: str) -> str: method record_exists (line 442) | async def record_exists(self, *, key: str) -> bool: method _update_metadata (line 461) | async def _update_metadata( method _encode_key (line 487) | def _encode_key(self, key: str) -> str: method _decode_key (line 491) | def _decode_key(self, encoded_key: str) -> str: FILE: src/crawlee/storage_clients/_file_system/_request_queue_client.py class RequestQueueState (line 40) | class RequestQueueState(BaseModel): class FileSystemRequestQueueClient (line 62) | class FileSystemRequestQueueClient(RequestQueueClient): method __init__ (line 91) | def __init__( method get_metadata (line 124) | async def get_metadata(self) -> RequestQueueMetadata: method path_to_rq (line 128) | def path_to_rq(self) -> Path: method path_to_metadata (line 133) | def path_to_metadata(self) -> Path: method _create_recoverable_state (line 138) | async def _create_recoverable_state(cls, id: str, configuration: Confi... method open (line 154) | async def open( method drop (line 283) | async def drop(self) -> None: method purge (line 299) | async def purge(self) -> None: method add_batch_of_requests (line 323) | async def add_batch_of_requests( method get_request (line 448) | async def get_request(self, unique_key: str) -> Request | None: method fetch_next_request (line 461) | async def fetch_next_request(self) -> Request | None: method mark_request_as_handled (line 484) | async def mark_request_as_handled(self, request: Request) -> Processed... method reclaim_request (line 527) | async def reclaim_request( method is_empty (line 588) | async def is_empty(self) -> bool: method _get_request_path (line 624) | def _get_request_path(self, unique_key: str) -> Path: method _update_metadata (line 635) | async def _update_metadata( method _refresh_cache (line 685) | async def _refresh_cache(self) -> None: method _get_request_files (line 747) | async def _get_request_files(cls, path_to_rq: Path) -> list[Path]: method _parse_request_file (line 768) | async def _parse_request_file(cls, file_path: Path) -> Request | None: method _discover_existing_requests (line 800) | async def _discover_existing_requests(self) -> None: method _get_file_base_name_from_unique_key (line 821) | def _get_file_base_name_from_unique_key(unique_key: str) -> str: FILE: src/crawlee/storage_clients/_file_system/_storage_client.py class FileSystemStorageClient (line 20) | class FileSystemStorageClient(StorageClient): method get_storage_client_cache_key (line 38) | def get_storage_client_cache_key(self, configuration: Configuration) -... method create_dataset_client (line 43) | async def create_dataset_client( method create_kvs_client (line 57) | async def create_kvs_client( method create_rq_client (line 71) | async def create_rq_client( FILE: src/crawlee/storage_clients/_memory/_dataset_client.py class MemoryDatasetClient (line 20) | class MemoryDatasetClient(DatasetClient): method __init__ (line 33) | def __init__( method get_metadata (line 48) | async def get_metadata(self) -> DatasetMetadata: method open (line 52) | async def open( method drop (line 98) | async def drop(self) -> None: method purge (line 107) | async def purge(self) -> None: method push_data (line 116) | async def push_data(self, data: list[dict[str, Any]] | dict[str, Any])... method get_data (line 135) | async def get_data( method iterate_items (line 194) | async def iterate_items( method _update_metadata (line 240) | async def _update_metadata( method _push_item (line 263) | async def _push_item(self, item: dict[str, Any]) -> None: FILE: src/crawlee/storage_clients/_memory/_key_value_store_client.py class MemoryKeyValueStoreClient (line 19) | class MemoryKeyValueStoreClient(KeyValueStoreClient): method __init__ (line 31) | def __init__( method get_metadata (line 46) | async def get_metadata(self) -> KeyValueStoreMetadata: method open (line 50) | async def open( method drop (line 95) | async def drop(self) -> None: method purge (line 100) | async def purge(self) -> None: method get_value (line 105) | async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: method set_value (line 112) | async def set_value(self, *, key: str, value: Any, content_type: str |... method delete_value (line 129) | async def delete_value(self, *, key: str) -> None: method iterate_keys (line 135) | async def iterate_keys( method get_public_url (line 164) | async def get_public_url(self, *, key: str) -> str: method record_exists (line 168) | async def record_exists(self, *, key: str) -> bool: method _update_metadata (line 172) | async def _update_metadata( FILE: src/crawlee/storage_clients/_memory/_request_queue_client.py class MemoryRequestQueueClient (line 23) | class MemoryRequestQueueClient(RequestQueueClient): method __init__ (line 34) | def __init__( method get_metadata (line 58) | async def get_metadata(self) -> RequestQueueMetadata: method open (line 62) | async def open( method drop (line 111) | async def drop(self) -> None: method purge (line 126) | async def purge(self) -> None: method add_batch_of_requests (line 141) | async def add_batch_of_requests( method fetch_next_request (line 232) | async def fetch_next_request(self) -> Request | None: method get_request (line 251) | async def get_request(self, unique_key: str) -> Request | None: method mark_request_as_handled (line 256) | async def mark_request_as_handled(self, request: Request) -> Processed... method reclaim_request (line 288) | async def reclaim_request( method is_empty (line 317) | async def is_empty(self) -> bool: method _update_metadata (line 328) | async def _update_metadata( FILE: src/crawlee/storage_clients/_memory/_storage_client.py class MemoryStorageClient (line 15) | class MemoryStorageClient(StorageClient): method create_dataset_client (line 31) | async def create_dataset_client( method create_kvs_client (line 45) | async def create_kvs_client( method create_rq_client (line 59) | async def create_rq_client( FILE: src/crawlee/storage_clients/_redis/_client_mixin.py class MetadataUpdateParams (line 27) | class MetadataUpdateParams(TypedDict, total=False): class RedisClientMixin (line 34) | class RedisClientMixin: method __init__ (line 49) | def __init__(self, storage_name: str, storage_id: str, redis: Redis) -... method redis (line 57) | def redis(self) -> Redis: method metadata_key (line 62) | def metadata_key(self) -> str: method _get_metadata_by_name (line 67) | async def _get_metadata_by_name(cls, name: str, redis: Redis, *, with_... method _get_metadata_name_by_id (line 88) | async def _get_metadata_name_by_id(cls, id: str, redis: Redis) -> str ... method _open (line 103) | async def _open( method _load_scripts (line 167) | async def _load_scripts(self) -> None: method _ensure_scripts_loaded (line 171) | async def _ensure_scripts_loaded(self) -> None: method _get_pipeline (line 178) | async def _get_pipeline(self, *, with_execute: bool = True) -> AsyncIt... method _create_storage (line 188) | async def _create_storage(self, pipeline: Pipeline) -> None: method _create_script (line 191) | async def _create_script(self, script_name: str) -> AsyncScript: method _create_metadata_and_storage (line 197) | async def _create_metadata_and_storage(self, storage_name: str, metada... method _drop (line 220) | async def _drop(self, extra_keys: list[str]) -> None: method _purge (line 229) | async def _purge(self, extra_keys: list[str], metadata_kwargs: Metadat... method _get_metadata (line 237) | async def _get_metadata(self, metadata_model: type[DatasetMetadata]) -... method _get_metadata (line 239) | async def _get_metadata(self, metadata_model: type[KeyValueStoreMetada... method _get_metadata (line 241) | async def _get_metadata(self, metadata_model: type[RequestQueueMetadat... method _get_metadata (line 243) | async def _get_metadata( method _specific_update_metadata (line 255) | async def _specific_update_metadata(self, pipeline: Pipeline, **kwargs... method _update_metadata (line 265) | async def _update_metadata( FILE: src/crawlee/storage_clients/_redis/_dataset_client.py class _DatasetMetadataUpdateParams (line 23) | class _DatasetMetadataUpdateParams(MetadataUpdateParams): class RedisDatasetClient (line 30) | class RedisDatasetClient(DatasetClient, RedisClientMixin): method __init__ (line 54) | def __init__(self, storage_name: str, storage_id: str, redis: Redis) -... method _items_key (line 67) | def _items_key(self) -> str: method open (line 72) | async def open( method get_metadata (line 106) | async def get_metadata(self) -> DatasetMetadata: method drop (line 110) | async def drop(self) -> None: method purge (line 114) | async def purge(self) -> None: method push_data (line 123) | async def push_data(self, data: list[dict[str, Any]] | dict[str, Any])... method get_data (line 137) | async def get_data( method iterate_items (line 221) | async def iterate_items( method _create_storage (line 303) | async def _create_storage(self, pipeline: Pipeline) -> None: method _specific_update_metadata (line 309) | async def _specific_update_metadata( FILE: src/crawlee/storage_clients/_redis/_key_value_store_client.py class RedisKeyValueStoreClient (line 24) | class RedisKeyValueStoreClient(KeyValueStoreClient, RedisClientMixin): method __init__ (line 52) | def __init__(self, storage_name: str, storage_id: str, redis: Redis) -... method _items_key (line 60) | def _items_key(self) -> str: method _metadata_items_key (line 65) | def _metadata_items_key(self) -> str: method open (line 70) | async def open( method get_metadata (line 104) | async def get_metadata(self) -> KeyValueStoreMetadata: method drop (line 108) | async def drop(self) -> None: method purge (line 112) | async def purge(self) -> None: method set_value (line 119) | async def set_value(self, *, key: str, value: Any, content_type: str |... method get_value (line 159) | async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: method delete_value (line 204) | async def delete_value(self, *, key: str) -> None: method iterate_keys (line 211) | async def iterate_keys( method get_public_url (line 251) | async def get_public_url(self, *, key: str) -> str: method record_exists (line 255) | async def record_exists(self, *, key: str) -> bool: FILE: src/crawlee/storage_clients/_redis/_request_queue_client.py class _QueueMetadataUpdateParams (line 29) | class _QueueMetadataUpdateParams(MetadataUpdateParams): class RedisRequestQueueClient (line 42) | class RedisRequestQueueClient(RequestQueueClient, RedisClientMixin): method __init__ (line 87) | def __init__( method _added_filter_key (line 121) | def _added_filter_key(self) -> str: method _handled_filter_key (line 128) | def _handled_filter_key(self) -> str: method _pending_set_key (line 135) | def _pending_set_key(self) -> str: method _handled_set_key (line 142) | def _handled_set_key(self) -> str: method _queue_key (line 149) | def _queue_key(self) -> str: method _data_key (line 154) | def _data_key(self) -> str: method _in_progress_key (line 159) | def _in_progress_key(self) -> str: method open (line 164) | async def open( method get_metadata (line 211) | async def get_metadata(self) -> RequestQueueMetadata: method drop (line 215) | async def drop(self) -> None: method purge (line 226) | async def purge(self) -> None: method add_batch_of_requests (line 246) | async def add_batch_of_requests( method fetch_next_request (line 353) | async def fetch_next_request(self) -> Request | None: method get_request (line 381) | async def get_request(self, unique_key: str) -> Request | None: method mark_request_as_handled (line 390) | async def mark_request_as_handled(self, request: Request) -> Processed... method reclaim_request (line 428) | async def reclaim_request( method is_empty (line 473) | async def is_empty(self) -> bool: method _load_scripts (line 491) | async def _load_scripts(self) -> None: method _create_storage (line 501) | async def _create_storage(self, pipeline: Pipeline) -> None: method _reclaim_stale_requests (line 515) | async def _reclaim_stale_requests(self) -> None: method _specific_update_metadata (line 527) | async def _specific_update_metadata( FILE: src/crawlee/storage_clients/_redis/_storage_client.py class RedisStorageClient (line 19) | class RedisStorageClient(StorageClient): method __init__ (line 39) | def __init__( method create_dataset_client (line 87) | async def create_dataset_client( method create_kvs_client (line 108) | async def create_kvs_client( method create_rq_client (line 129) | async def create_rq_client( FILE: src/crawlee/storage_clients/_redis/_utils.py function await_redis_response (line 9) | async def await_redis_response(response: Awaitable[T]) -> T: ... function await_redis_response (line 11) | async def await_redis_response(response: T) -> T: ... function await_redis_response (line 14) | async def await_redis_response(response: Awaitable[T] | T) -> T: function read_lua_script (line 21) | def read_lua_script(script_name: str) -> str: FILE: src/crawlee/storage_clients/_sql/_client_mixin.py class MetadataUpdateParams (line 45) | class MetadataUpdateParams(TypedDict, total=False): class SqlClientMixin (line 52) | class SqlClientMixin(ABC): method __init__ (line 78) | def __init__(self, *, id: str, storage_client: SqlStorageClient) -> None: method _open (line 83) | async def _open( method _safely_open (line 138) | async def _safely_open( method get_session (line 197) | async def get_session(self, *, with_simple_commit: bool = False) -> As... method _build_insert_stmt_with_ignore (line 211) | def _build_insert_stmt_with_ignore( method _build_upsert_stmt (line 236) | def _build_upsert_stmt( method _purge (line 274) | async def _purge(self, metadata_kwargs: MetadataUpdateParams) -> None: method _drop (line 288) | async def _drop(self) -> None: method _get_metadata (line 305) | async def _get_metadata(self, metadata_model: type[DatasetMetadata]) -... method _get_metadata (line 307) | async def _get_metadata(self, metadata_model: type[KeyValueStoreMetada... method _get_metadata (line 309) | async def _get_metadata(self, metadata_model: type[RequestQueueMetadat... method _get_metadata (line 311) | async def _get_metadata( method _specific_update_metadata (line 326) | def _specific_update_metadata(self, **kwargs: Any) -> dict[str, Any]: method _prepare_buffer_data (line 336) | def _prepare_buffer_data(self, **kwargs: Any) -> dict[str, Any]: method _apply_buffer_updates (line 340) | async def _apply_buffer_updates(self, session: AsyncSession, max_buffe... method _update_metadata (line 348) | async def _update_metadata( method _add_buffer_record (line 381) | async def _add_buffer_record( method _try_acquire_buffer_lock (line 405) | async def _try_acquire_buffer_lock(self, session: AsyncSession) -> bool: method _release_buffer_lock (line 467) | async def _release_buffer_lock(self, session: AsyncSession) -> None: method _has_pending_buffer_updates (line 477) | async def _has_pending_buffer_updates(self, session: AsyncSession) -> ... method _process_buffers (line 494) | async def _process_buffers(self) -> None: FILE: src/crawlee/storage_clients/_sql/_dataset_client.py class _DatasetMetadataUpdateParams (line 30) | class _DatasetMetadataUpdateParams(MetadataUpdateParams): class SqlDatasetClient (line 37) | class SqlDatasetClient(DatasetClient, SqlClientMixin): method __init__ (line 68) | def __init__( method open (line 81) | async def open( method get_metadata (line 113) | async def get_metadata(self) -> DatasetMetadata: method drop (line 118) | async def drop(self) -> None: method purge (line 126) | async def purge(self) -> None: method push_data (line 141) | async def push_data(self, data: list[dict[str, Any]] | dict[str, Any])... method get_data (line 154) | async def get_data( method iterate_items (line 201) | async def iterate_items( method _prepare_get_stmt (line 234) | def _prepare_get_stmt( method _specific_update_metadata (line 279) | def _specific_update_metadata( method _prepare_buffer_data (line 303) | def _prepare_buffer_data(self, delta_item_count: int | None = None, **... method _apply_buffer_updates (line 316) | async def _apply_buffer_updates(self, session: AsyncSession, max_buffe... FILE: src/crawlee/storage_clients/_sql/_db_models.py class AwareDateTime (line 17) | class AwareDateTime(TypeDecorator): method process_result_value (line 28) | def process_result_value(self, value: datetime | None, dialect: Dialec... class JsonField (line 35) | class JsonField(TypeDecorator): method load_dialect_impl (line 41) | def load_dialect_impl(self, dialect: Dialect) -> TypeEngine[JSON | JSO... class Base (line 48) | class Base(DeclarativeBase): class StorageMetadataDb (line 52) | class StorageMetadataDb: class DatasetMetadataDb (line 74) | class DatasetMetadataDb(StorageMetadataDb, Base): class RequestQueueMetadataDb (line 94) | class RequestQueueMetadataDb(StorageMetadataDb, Base): class KeyValueStoreMetadataDb (line 127) | class KeyValueStoreMetadataDb(StorageMetadataDb, Base): class KeyValueStoreRecordDb (line 144) | class KeyValueStoreRecordDb(Base): class DatasetItemDb (line 177) | class DatasetItemDb(Base): class RequestDb (line 202) | class RequestDb(Base): class RequestQueueStateDb (line 251) | class RequestQueueStateDb(Base): class VersionDb (line 271) | class VersionDb(Base): class MetadataBufferDb (line 279) | class MetadataBufferDb: class KeyValueStoreMetadataBufferDb (line 293) | class KeyValueStoreMetadataBufferDb(MetadataBufferDb, Base): class DatasetMetadataBufferDb (line 306) | class DatasetMetadataBufferDb(MetadataBufferDb, Base): class RequestQueueMetadataBufferDb (line 323) | class RequestQueueMetadataBufferDb(MetadataBufferDb, Base): FILE: src/crawlee/storage_clients/_sql/_key_value_store_client.py class SqlKeyValueStoreClient (line 34) | class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin): method __init__ (line 72) | def __init__( method open (line 85) | async def open( method get_metadata (line 121) | async def get_metadata(self) -> KeyValueStoreMetadata: method drop (line 126) | async def drop(self) -> None: method purge (line 134) | async def purge(self) -> None: method set_value (line 143) | async def set_value(self, *, key: str, value: Any, content_type: str |... method get_value (line 184) | async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: method delete_value (line 230) | async def delete_value(self, *, key: str) -> None: method iterate_keys (line 244) | async def iterate_keys( method record_exists (line 278) | async def record_exists(self, *, key: str) -> bool: method get_public_url (line 291) | async def get_public_url(self, *, key: str) -> str: method _specific_update_metadata (line 295) | def _specific_update_metadata(self, **_kwargs: dict[str, Any]) -> dict... method _prepare_buffer_data (line 299) | def _prepare_buffer_data(self, **_kwargs: Any) -> dict[str, Any]: method _apply_buffer_updates (line 308) | async def _apply_buffer_updates(self, session: AsyncSession, max_buffe... FILE: src/crawlee/storage_clients/_sql/_request_queue_client.py class _QueueMetadataUpdateParams (line 41) | class _QueueMetadataUpdateParams(MetadataUpdateParams): class SqlRequestQueueClient (line 54) | class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin): method __init__ (line 103) | def __init__( method open (line 125) | async def open( method get_metadata (line 166) | async def get_metadata(self) -> RequestQueueMetadata: method drop (line 173) | async def drop(self) -> None: method purge (line 183) | async def purge(self) -> None: method add_batch_of_requests (line 204) | async def add_batch_of_requests( method get_request (line 395) | async def get_request(self, unique_key: str) -> Request | None: method fetch_next_request (line 414) | async def fetch_next_request(self) -> Request | None: method mark_request_as_handled (line 496) | async def mark_request_as_handled(self, request: Request) -> Processed... method reclaim_request (line 527) | async def reclaim_request( method is_empty (line 584) | async def is_empty(self) -> bool: method _get_state (line 627) | async def _get_state(self, session: AsyncSession) -> RequestQueueStateDb: method _specific_update_metadata (line 644) | def _specific_update_metadata( method _get_int_id_from_unique_key (line 729) | def _get_int_id_from_unique_key(unique_key: str) -> int: method _prepare_buffer_data (line 743) | def _prepare_buffer_data( method _apply_buffer_updates (line 779) | async def _apply_buffer_updates(self, session: AsyncSession, max_buffe... FILE: src/crawlee/storage_clients/_sql/_storage_client.py class SqlStorageClient (line 32) | class SqlStorageClient(StorageClient): method __init__ (line 54) | def __init__( method __aenter__ (line 86) | async def __aenter__(self) -> SqlStorageClient: method __aexit__ (line 90) | async def __aexit__( method engine (line 100) | def engine(self) -> AsyncEngine: method get_dialect_name (line 106) | def get_dialect_name(self) -> str | None: method initialize (line 110) | async def initialize(self, configuration: Configuration) -> None: method close (line 161) | async def close(self) -> None: method create_session (line 167) | def create_session(self) -> AsyncSession: method create_dataset_client (line 178) | async def create_dataset_client( method create_kvs_client (line 200) | async def create_kvs_client( method create_rq_client (line 222) | async def create_rq_client( method _get_or_create_engine (line 243) | def _get_or_create_engine(self, configuration: Configuration) -> Async... FILE: src/crawlee/storage_clients/models.py class StorageMetadata (line 17) | class StorageMetadata(BaseModel): class DatasetMetadata (line 42) | class DatasetMetadata(StorageMetadata): class KeyValueStoreMetadata (line 52) | class KeyValueStoreMetadata(StorageMetadata): class RequestQueueMetadata (line 59) | class RequestQueueMetadata(StorageMetadata): class KeyValueStoreRecordMetadata (line 78) | class KeyValueStoreRecordMetadata(BaseModel): class KeyValueStoreRecord (line 100) | class KeyValueStoreRecord(KeyValueStoreRecordMetadata, Generic[KvsValueT... class DatasetItemsListPage (line 110) | class DatasetItemsListPage(BaseModel): class ProcessedRequest (line 140) | class ProcessedRequest(BaseModel): class UnprocessedRequest (line 154) | class UnprocessedRequest(BaseModel): class AddRequestsResponse (line 165) | class AddRequestsResponse(BaseModel): FILE: src/crawlee/storages/_base.py class Storage (line 15) | class Storage(ABC): method id (line 20) | def id(self) -> str: method name (line 25) | def name(self) -> str | None: method get_metadata (line 29) | async def get_metadata(self) -> DatasetMetadata | KeyValueStoreMetadat... method open (line 34) | async def open( method drop (line 57) | async def drop(self) -> None: method purge (line 61) | async def purge(self) -> None: FILE: src/crawlee/storages/_dataset.py class Dataset (line 33) | class Dataset(Storage): method __init__ (line 69) | def __init__(self, client: DatasetClient, id: str, name: str | None) -... method id (line 87) | def id(self) -> str: method name (line 92) | def name(self) -> str | None: method get_metadata (line 96) | async def get_metadata(self) -> DatasetMetadata: method open (line 101) | async def open( method drop (line 128) | async def drop(self) -> None: method purge (line 134) | async def purge(self) -> None: method push_data (line 137) | async def push_data(self, data: list[dict[str, Any]] | dict[str, Any])... method get_data (line 150) | async def get_data( method iterate_items (line 201) | async def iterate_items( method list_items (line 249) | async def list_items( method export_to (line 300) | async def export_to( method export_to (line 312) | async def export_to( method export_to (line 323) | async def export_to( FILE: src/crawlee/storages/_key_value_store.py class AutosavedValue (line 35) | class AutosavedValue(RootModel): class KeyValueStore (line 40) | class KeyValueStore(Storage): method __init__ (line 78) | def __init__(self, client: KeyValueStoreClient, id: str, name: str | N... method id (line 99) | def id(self) -> str: method name (line 104) | def name(self) -> str | None: method get_metadata (line 108) | async def get_metadata(self) -> KeyValueStoreMetadata: method open (line 113) | async def open( method drop (line 140) | async def drop(self) -> None: method purge (line 148) | async def purge(self) -> None: method get_value (line 152) | async def get_value(self, key: str) -> Any: ... method get_value (line 155) | async def get_value(self, key: str, default_value: T) -> T: ... method get_value (line 158) | async def get_value(self, key: str, default_value: T | None = None) ->... method get_value (line 160) | async def get_value(self, key: str, default_value: T | None = None) ->... method set_value (line 173) | async def set_value( method delete_value (line 188) | async def delete_value(self, key: str) -> None: method iterate_keys (line 196) | async def iterate_keys( method list_keys (line 216) | async def list_keys( method record_exists (line 240) | async def record_exists(self, key: str) -> bool: method get_public_url (line 251) | async def get_public_url(self, key: str) -> str: method get_auto_saved_value (line 262) | async def get_auto_saved_value( method persist_autosaved_values (line 299) | async def persist_autosaved_values(self) -> None: method _clear_cache (line 306) | async def _clear_cache(self) -> None: FILE: src/crawlee/storages/_request_queue.py class RequestQueue (line 33) | class RequestQueue(Storage, RequestManager): method __init__ (line 74) | def __init__(self, client: RequestQueueClient, id: str, name: str | No... method id (line 95) | def id(self) -> str: method name (line 100) | def name(self) -> str | None: method get_metadata (line 104) | async def get_metadata(self) -> RequestQueueMetadata: method get_handled_count (line 108) | async def get_handled_count(self) -> int: method get_total_count (line 113) | async def get_total_count(self) -> int: method open (line 119) | async def open( method drop (line 144) | async def drop(self) -> None: method purge (line 152) | async def purge(self) -> None: method add_request (line 156) | async def add_request( method add_requests (line 180) | async def add_requests( method fetch_next_request (line 230) | async def fetch_next_request(self) -> Request | None: method get_request (line 247) | async def get_request(self, unique_key: str) -> Request | None: method mark_request_as_handled (line 258) | async def mark_request_as_handled(self, request: Request) -> Processed... method reclaim_request (line 273) | async def reclaim_request( method is_empty (line 295) | async def is_empty(self) -> bool: method is_finished (line 307) | async def is_finished(self) -> bool: method _process_batch (line 327) | async def _process_batch( FILE: src/crawlee/storages/_storage_instance_manager.py class _StorageCache (line 22) | class _StorageCache: method remove_from_cache (line 40) | def remove_from_cache(self, storage_instance: Storage) -> None: class StorageInstanceManager (line 69) | class StorageInstanceManager: method __init__ (line 79) | def __init__(self) -> None: method open_storage_instance (line 83) | async def open_storage_instance( method remove_from_cache (line 195) | def remove_from_cache(self, storage_instance: Storage) -> None: method clear_cache (line 203) | def clear_cache(self) -> None: method _get_from_cache (line 207) | def _get_from_cache( method _check_name_alias_conflict (line 234) | def _check_name_alias_conflict( FILE: src/crawlee/storages/_utils.py function validate_storage_name (line 6) | def validate_storage_name(name: str | None) -> None: FILE: tests/e2e/conftest.py function pytest_configure (line 11) | def pytest_configure(config: Config) -> None: function crawlee_wheel_path (line 31) | def crawlee_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun... FILE: tests/e2e/project_template/test_static_crawlers_templates.py function test_static_crawler_actor_at_apify (line 47) | async def test_static_crawler_actor_at_apify( FILE: tests/e2e/project_template/utils.py function patch_crawlee_version_in_project (line 8) | def patch_crawlee_version_in_project( function _patch_crawlee_version_in_requirements_txt_based_project (line 21) | def _patch_crawlee_version_in_requirements_txt_based_project(project_pat... function _patch_crawlee_version_in_pyproject_toml_based_project (line 58) | def _patch_crawlee_version_in_pyproject_toml_based_project(project_path:... FILE: tests/unit/_autoscaling/test_autoscaled_pool.py function system_status (line 24) | def system_status() -> SystemStatus | Mock: function future (line 31) | def future(value: T, /) -> Awaitable[T]: function test_runs_concurrently (line 38) | async def test_runs_concurrently(system_status: SystemStatus | Mock) -> ... function test_abort_works (line 66) | async def test_abort_works(system_status: SystemStatus | Mock) -> None: function test_propagates_exceptions (line 93) | async def test_propagates_exceptions(system_status: SystemStatus | Mock)... function test_propagates_exceptions_after_finished (line 121) | async def test_propagates_exceptions_after_finished(system_status: Syste... function test_autoscales (line 152) | async def test_autoscales( function test_autoscales_uses_desired_concurrency_ratio (line 223) | async def test_autoscales_uses_desired_concurrency_ratio( function test_max_tasks_per_minute_works (line 283) | async def test_max_tasks_per_minute_works(system_status: SystemStatus | ... function test_allows_multiple_run_calls (line 314) | async def test_allows_multiple_run_calls(system_status: SystemStatus | M... FILE: tests/unit/_autoscaling/test_snapshotter.py function event_manager (line 35) | async def event_manager() -> AsyncGenerator[LocalEventManager, None]: function snapshotter (line 56) | async def snapshotter(event_manager: LocalEventManager) -> AsyncGenerato... function default_cpu_info (line 64) | def default_cpu_info() -> CpuInfo: function default_memory_info (line 69) | def default_memory_info() -> MemoryInfo: function event_system_data_info (line 78) | def event_system_data_info(default_cpu_info: CpuInfo, default_memory_inf... function test_start_stop_lifecycle (line 85) | async def test_start_stop_lifecycle() -> None: function test_snapshot_cpu (line 92) | async def test_snapshot_cpu( function test_snapshot_memory (line 102) | async def test_snapshot_memory( function test_snapshot_memory_with_memory_info_sets_system_wide_fields (line 112) | async def test_snapshot_memory_with_memory_info_sets_system_wide_fields( function test_snapshot_event_loop (line 139) | def test_snapshot_event_loop(snapshotter: Snapshotter) -> None: function test_snapshot_client (line 145) | def test_snapshot_client(snapshotter: Snapshotter) -> None: function test_snapshot_client_overloaded (line 151) | def test_snapshot_client_overloaded() -> None: function test_get_cpu_sample (line 159) | async def test_get_cpu_sample( function test_methods_raise_error_when_not_active (line 196) | async def test_methods_raise_error_when_not_active() -> None: function test_snapshot_pruning_removes_outdated_records (line 225) | async def test_snapshot_pruning_removes_outdated_records( function test_memory_load_evaluation_logs_warning_on_high_usage (line 260) | async def test_memory_load_evaluation_logs_warning_on_high_usage( function test_memory_load_evaluation_silent_on_acceptable_usage (line 300) | async def test_memory_load_evaluation_silent_on_acceptable_usage( function test_snapshots_time_ordered (line 329) | async def test_snapshots_time_ordered(snapshotter: Snapshotter, event_ma... function test_sorted_snapshot_list_add_maintains_order (line 359) | def test_sorted_snapshot_list_add_maintains_order() -> None: function test_dynamic_memory (line 391) | async def test_dynamic_memory( FILE: tests/unit/_autoscaling/test_system_status.py function snapshotter (line 25) | async def snapshotter() -> AsyncGenerator[Snapshotter, None]: function now (line 32) | def now() -> datetime: function test_start_stop_lifecycle (line 36) | async def test_start_stop_lifecycle() -> None: function test_cpu_is_overloaded (line 45) | def test_cpu_is_overloaded(snapshotter: Snapshotter, now: datetime) -> N... function test_cpu_is_not_overloaded (line 61) | def test_cpu_is_not_overloaded(snapshotter: Snapshotter, now: datetime) ... function test_get_system_info (line 77) | def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: function test_client_overloaded (line 195) | def test_client_overloaded( function test_memory_overloaded_system_wide (line 217) | def test_memory_overloaded_system_wide(snapshotter: Snapshotter, now: da... FILE: tests/unit/_statistics/test_error_tracker.py function test_error_tracker_counts (line 20) | async def test_error_tracker_counts(error_tracker: ErrorTracker, expecte... function test_error_tracker_similar_messages_full_stack (line 54) | async def test_error_tracker_similar_messages_full_stack( function test_show_full_message (line 90) | async def test_show_full_message(*, show_full_message: bool, expected_me... function test_error_tracker_with_errors_chain (line 104) | async def test_error_tracker_with_errors_chain() -> None: FILE: tests/unit/_statistics/test_periodic_logging.py function test_periodic_logging (line 14) | async def test_periodic_logging(caplog: pytest.LogCaptureFixture) -> None: FILE: tests/unit/_statistics/test_persistence.py function test_basic_persistence (line 6) | async def test_basic_persistence() -> None: FILE: tests/unit/_statistics/test_request_max_duration.py function test_request_max_duration_tracks_maximum (line 8) | async def test_request_max_duration_tracks_maximum() -> None: FILE: tests/unit/_statistics/test_request_processing_record.py function test_tracking_time_resolution (line 6) | def test_tracking_time_resolution() -> None: FILE: tests/unit/_utils/test_byte_size.py function test_initializations (line 8) | def test_initializations() -> None: function test_conversions (line 19) | def test_conversions() -> None: function test_string_representation (line 27) | def test_string_representation() -> None: function test_comparisons (line 35) | def test_comparisons() -> None: function test_additions (line 47) | def test_additions() -> None: function test_subtractions (line 62) | def test_subtractions() -> None: function test_multiplication (line 81) | def test_multiplication() -> None: function test_divisions (line 98) | def test_divisions() -> None: FILE: tests/unit/_utils/test_console.py function test_empty_input (line 6) | def test_empty_input() -> None: function test_empty_row (line 10) | def test_empty_row() -> None: function test_single_column (line 14) | def test_single_column() -> None: function test_two_columns (line 21) | def test_two_columns() -> None: function test_long_content_truncation (line 34) | def test_long_content_truncation() -> None: FILE: tests/unit/_utils/test_crypto.py function test_crypto_random_object_id_default_length (line 6) | def test_crypto_random_object_id_default_length() -> None: function test_crypto_random_object_id_custom_length (line 11) | def test_crypto_random_object_id_custom_length() -> None: function test_crypto_random_object_id_character_set (line 17) | def test_crypto_random_object_id_character_set() -> None: function test_compute_short_hash_with_known_input (line 24) | def test_compute_short_hash_with_known_input() -> None: function test_compute_short_hash_with_empty_input (line 30) | def test_compute_short_hash_with_empty_input() -> None: function test_compute_short_hash_output_length (line 36) | def test_compute_short_hash_output_length() -> None: function test_compute_short_hash_differentiates_input (line 41) | def test_compute_short_hash_differentiates_input() -> None: FILE: tests/unit/_utils/test_file.py function test_json_dumps (line 8) | async def test_json_dumps() -> None: FILE: tests/unit/_utils/test_globs.py function test_asterisk (line 6) | def test_asterisk() -> None: function test_double_asteritsk (line 13) | def test_double_asteritsk() -> None: FILE: tests/unit/_utils/test_html_to_text.py function test_html_to_text (line 187) | def test_html_to_text(source: str, expected_text: str, html_to_text: Cal... function test_html_to_text_raises_on_wrong_input_type (line 192) | def test_html_to_text_raises_on_wrong_input_type(html_to_text: Callable[... function test_html_to_text_parsel (line 198) | def test_html_to_text_parsel() -> None: function test_html_to_text_beautifulsoup (line 202) | def test_html_to_text_beautifulsoup() -> None: FILE: tests/unit/_utils/test_measure_time.py function test_measure_time_wall_sync (line 9) | def test_measure_time_wall_sync() -> None: function test_measure_time_cpu_sync (line 18) | def test_measure_time_cpu_sync() -> None: function test_measure_time_wall_async (line 33) | async def test_measure_time_wall_async() -> None: FILE: tests/unit/_utils/test_raise_if_too_many_kwargs.py function test_limit_kwargs_default (line 22) | def test_limit_kwargs_default(kwargs: dict[str, Any], *, should_raise: b... function test_limit_kwargs (line 35) | def test_limit_kwargs(kwargs: dict[str, Any], *, should_raise: bool) -> ... FILE: tests/unit/_utils/test_recurring_task.py function function (line 13) | def function() -> AsyncMock: function delay (line 20) | def delay() -> timedelta: function test_init (line 24) | async def test_init(function: AsyncMock, delay: timedelta) -> None: function test_start_and_stop (line 31) | async def test_start_and_stop(function: AsyncMock, delay: timedelta) -> ... function test_execution (line 45) | async def test_execution(function: AsyncMock, delay: timedelta) -> None: FILE: tests/unit/_utils/test_requests.py function test_normalize_url (line 36) | def test_normalize_url(url: str, expected_output: str, *, keep_url_fragm... function test_compute_unique_key_basic (line 41) | def test_compute_unique_key_basic() -> None: function test_compute_unique_key_handles_fragments (line 48) | def test_compute_unique_key_handles_fragments() -> None: function test_compute_unique_key_handles_payload (line 57) | def test_compute_unique_key_handles_payload() -> None: function test_compute_unique_key_handles_headers (line 74) | def test_compute_unique_key_handles_headers() -> None: function test_compute_unique_key_complex (line 91) | def test_compute_unique_key_complex() -> None: function test_compute_unique_key_post_with_none_payload (line 117) | def test_compute_unique_key_post_with_none_payload() -> None: function test_compute_unique_key_with_whitespace_in_headers (line 124) | def test_compute_unique_key_with_whitespace_in_headers() -> None: FILE: tests/unit/_utils/test_robots.py function test_generation_robots_txt_url (line 13) | async def test_generation_robots_txt_url(server_url: URL, http_client: H... function test_allow_disallow_robots_txt (line 18) | async def test_allow_disallow_robots_txt(server_url: URL, http_client: H... function test_extract_sitemaps_urls (line 26) | async def test_extract_sitemaps_urls(server_url: URL, http_client: HttpC... function test_parse_from_content (line 32) | async def test_parse_from_content() -> None: function test_bind_robots_txt_url (line 45) | async def test_bind_robots_txt_url() -> None: FILE: tests/unit/_utils/test_shared_timeout.py function test_shared_timeout_tracks_elapsed_time (line 9) | async def test_shared_timeout_tracks_elapsed_time() -> None: function test_shared_timeout_expires (line 23) | async def test_shared_timeout_expires() -> None: function test_shared_timeout_cannot_be_nested (line 35) | async def test_shared_timeout_cannot_be_nested() -> None: function test_shared_timeout_multiple_sequential_uses (line 45) | async def test_shared_timeout_multiple_sequential_uses() -> None: FILE: tests/unit/_utils/test_sitemap.py function _make_mock_client (line 51) | def _make_mock_client(url_map: dict[str, tuple[int, bytes]]) -> AsyncMock: function compress_gzip (line 68) | def compress_gzip(data: str) -> bytes: function encode_base64 (line 73) | def encode_base64(data: bytes) -> str: function test_sitemap (line 78) | async def test_sitemap(server_url: URL, http_client: HttpClient) -> None: function test_extract_metadata_sitemap (line 89) | async def test_extract_metadata_sitemap(server_url: URL, http_client: Ht... function test_gzipped_sitemap (line 106) | async def test_gzipped_sitemap(server_url: URL, http_client: HttpClient)... function test_gzipped_sitemap_with_invalid_data (line 115) | async def test_gzipped_sitemap_with_invalid_data(server_url: URL, http_c... function test_gz_sitemap_with_non_gzipped (line 126) | async def test_gz_sitemap_with_non_gzipped(server_url: URL, http_client:... function test_gzipped_sitemap_with_bad_type (line 137) | async def test_gzipped_sitemap_with_bad_type(server_url: URL, http_clien... function test_xml_sitemap_with_gzipped_data (line 149) | async def test_xml_sitemap_with_gzipped_data(server_url: URL, http_clien... function test_parent_sitemap (line 159) | async def test_parent_sitemap(server_url: URL, http_client: HttpClient) ... function test_non_sitemap_url (line 186) | async def test_non_sitemap_url(server_url: URL, http_client: HttpClient)... function test_cdata_sitemap (line 194) | async def test_cdata_sitemap(server_url: URL, http_client: HttpClient) -... function test_txt_sitemap (line 213) | async def test_txt_sitemap(server_url: URL, http_client: HttpClient) -> ... function test_sitemap_pretty (line 231) | async def test_sitemap_pretty(server_url: URL, http_client: HttpClient) ... function test_sitemap_from_string (line 262) | async def test_sitemap_from_string() -> None: function test_discover_sitemap_from_robots_txt (line 270) | async def test_discover_sitemap_from_robots_txt() -> None: function test_discover_sitemap_from_common_paths (line 280) | async def test_discover_sitemap_from_common_paths() -> None: function test_discover_sitemap_from_input_url (line 295) | async def test_discover_sitemap_from_input_url() -> None: function test_discover_sitemap_deduplication (line 304) | async def test_discover_sitemap_deduplication() -> None: function test_discover_sitemaps_multiple_domains (line 319) | async def test_discover_sitemaps_multiple_domains() -> None: function test_discover_sitemap_url_without_host_skipped (line 342) | async def test_discover_sitemap_url_without_host_skipped() -> None: FILE: tests/unit/_utils/test_system.py function test_get_memory_info_returns_valid_values (line 17) | def test_get_memory_info_returns_valid_values() -> None: function test_get_cpu_info_returns_valid_values (line 24) | def test_get_cpu_info_returns_valid_values() -> None: function test_memory_estimation_does_not_overestimate_due_to_shared_memory (line 30) | def test_memory_estimation_does_not_overestimate_due_to_shared_memory() ... FILE: tests/unit/_utils/test_timedelta_ms.py class _ModelWithTimedeltaMs (line 12) | class _ModelWithTimedeltaMs(BaseModel): function test_model_with_timedelta_ms_input_types (line 30) | def test_model_with_timedelta_ms_input_types( FILE: tests/unit/_utils/test_urls.py function test_is_url_absolute (line 9) | def test_is_url_absolute() -> None: function test_convert_to_absolute_url (line 19) | def test_convert_to_absolute_url() -> None: function test_validate_http_url (line 36) | def test_validate_http_url() -> None: FILE: tests/unit/browsers/test_browser_pool.py function test_default_plugin_new_page_creation (line 23) | async def test_default_plugin_new_page_creation(server_url: URL) -> None: function test_multiple_plugins_new_page_creation (line 43) | async def test_multiple_plugins_new_page_creation(server_url: URL) -> None: function test_new_page_with_each_plugin (line 79) | async def test_new_page_with_each_plugin(server_url: URL) -> None: function test_with_default_plugin_constructor (line 106) | async def test_with_default_plugin_constructor(server_url: URL) -> None: function test_new_page_with_existing_id (line 125) | async def test_new_page_with_existing_id() -> None: function test_new_page_with_invalid_plugin (line 132) | async def test_new_page_with_invalid_plugin() -> None: function test_resource_management (line 140) | async def test_resource_management(server_url: URL) -> None: function test_methods_raise_error_when_not_active (line 154) | async def test_methods_raise_error_when_not_active() -> None: function test_with_plugin_contains_page_options (line 174) | async def test_with_plugin_contains_page_options(server_url: URL) -> None: function test_browser_pool_retire_browser_after_page_count (line 190) | async def test_browser_pool_retire_browser_after_page_count( function test_pre_page_create_hook_is_called (line 209) | async def test_pre_page_create_hook_is_called() -> None: function test_post_page_create_hook_is_called (line 243) | async def test_post_page_create_hook_is_called() -> None: function test_pre_page_close_hook (line 272) | async def test_pre_page_close_hook() -> None: function test_post_page_close_hook (line 291) | async def test_post_page_close_hook() -> None: function test_page_hooks_execution_order (line 312) | async def test_page_hooks_execution_order() -> None: function test_multiple_hooks_all_called (line 344) | async def test_multiple_hooks_all_called() -> None: FILE: tests/unit/browsers/test_playwright_browser.py function playwright (line 18) | async def playwright() -> AsyncGenerator[Playwright, None]: function test_init (line 23) | async def test_init(playwright: Playwright) -> None: function test_delete_temp_folder_with_close_browser (line 35) | async def test_delete_temp_folder_with_close_browser(playwright: Playwri... FILE: tests/unit/browsers/test_playwright_browser_controller.py function playwright (line 21) | async def playwright() -> AsyncGenerator[Playwright, None]: function browser (line 27) | async def browser(playwright: Playwright) -> AsyncGenerator[Browser, None]: function controller (line 34) | async def controller(browser: Browser) -> AsyncGenerator[PlaywrightBrows... function test_initial_state (line 40) | async def test_initial_state(browser: Browser) -> None: function test_open_and_close_page (line 52) | async def test_open_and_close_page(controller: PlaywrightBrowserControll... function test_max_open_pages_limit (line 66) | async def test_max_open_pages_limit(controller: PlaywrightBrowserControl... function test_idle_time (line 91) | async def test_idle_time(controller: PlaywrightBrowserController) -> None: function test_close_browser_with_open_pages (line 98) | async def test_close_browser_with_open_pages(browser: Browser) -> None: function test_memory_leak_on_concurrent_context_creation (line 114) | async def test_memory_leak_on_concurrent_context_creation() -> None: function test_max_open_pages_limit_on_concurrent_creation (line 143) | async def test_max_open_pages_limit_on_concurrent_creation(controller: P... function test_max_open_pages_limit_error_on_concurrent_creation (line 152) | async def test_max_open_pages_limit_error_on_concurrent_creation(control... function test_browser_with_pre_existing_context (line 158) | async def test_browser_with_pre_existing_context(tmp_path: Path) -> None: FILE: tests/unit/browsers/test_playwright_browser_plugin.py function plugin (line 16) | async def plugin() -> AsyncGenerator[PlaywrightBrowserPlugin, None]: function test_initial_state (line 21) | async def test_initial_state() -> None: function test_new_browser (line 37) | async def test_new_browser(plugin: PlaywrightBrowserPlugin, server_url: ... function test_multiple_new_browsers (line 51) | async def test_multiple_new_browsers(plugin: PlaywrightBrowserPlugin) ->... function test_methods_raise_error_when_not_active (line 58) | async def test_methods_raise_error_when_not_active() -> None: function raise_error_if_chrome_and_executable_path (line 74) | async def raise_error_if_chrome_and_executable_path() -> None: FILE: tests/unit/conftest.py function suppress_user_warning (line 33) | async def suppress_user_warning() -> AsyncGenerator[None, None]: function prepare_test_env (line 44) | def prepare_test_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) ->... function _isolate_test_environment (line 84) | def _isolate_test_environment(prepare_test_env: Callable[[], None]) -> N... function _set_crawler_log_level (line 97) | def _set_crawler_log_level(pytestconfig: pytest.Config, monkeypatch: pyt... function proxy_info (line 106) | async def proxy_info(unused_tcp_port: int) -> ProxyInfo: function proxy (line 121) | async def proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, None]: function disabled_proxy (line 136) | async def disabled_proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyI... function header_network (line 152) | def header_network() -> dict: function key_value_store (line 157) | async def key_value_store() -> AsyncGenerator[KeyValueStore, None]: function http_server (line 164) | def http_server(unused_tcp_port_factory: Callable[[], int]) -> Iterator[... function server_url (line 172) | def server_url(http_server: TestServer) -> URL: function redirect_http_server (line 179) | def redirect_http_server(unused_tcp_port_factory: Callable[[], int]) -> ... function redirect_server_url (line 197) | def redirect_server_url(redirect_http_server: TestServer) -> URL: function http_client (line 209) | async def http_client(request: pytest.FixtureRequest) -> AsyncGenerator[... function redis_client (line 226) | def redis_client() -> FakeAsyncRedis: FILE: tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py function test_urls (line 65) | def test_urls(server_url: URL) -> list[str]: function key_value_store (line 74) | async def key_value_store() -> AsyncGenerator[KeyValueStore, None]: class _SimpleRenderingTypePredictor (line 80) | class _SimpleRenderingTypePredictor(RenderingTypePredictor): method __init__ (line 83) | def __init__( method predict (line 94) | def predict(self, request: Request) -> RenderingTypePrediction: method store_result (line 98) | def store_result(self, request: Request, rendering_type: RenderingType... class TestInput (line 103) | class TestInput: function test_adaptive_crawling (line 154) | async def test_adaptive_crawling( function test_adaptive_crawling_parsel (line 208) | async def test_adaptive_crawling_parsel(test_urls: list[str]) -> None: function test_adaptive_crawling_pre_nav_change_to_context (line 240) | async def test_adaptive_crawling_pre_nav_change_to_context(test_urls: li... function test_playwright_only_pre_navigation_hook (line 271) | async def test_playwright_only_pre_navigation_hook(test_urls: list[str])... function test_adaptive_crawling_post_nav_change_to_context (line 304) | async def test_adaptive_crawling_post_nav_change_to_context(test_urls: l... function test_playwright_only_post_navigation_hook (line 335) | async def test_playwright_only_post_navigation_hook(test_urls: list[str]... function test_adaptive_crawling_result (line 368) | async def test_adaptive_crawling_result(test_urls: list[str]) -> None: function test_adaptive_crawling_predictor_calls (line 399) | async def test_adaptive_crawling_predictor_calls( function test_adaptive_crawling_result_use_state_isolation (line 438) | async def test_adaptive_crawling_result_use_state_isolation( function test_adaptive_crawling_statistics (line 468) | async def test_adaptive_crawling_statistics(test_urls: list[str]) -> None: function test_adaptive_crawler_exceptions_in_sub_crawlers (line 502) | async def test_adaptive_crawler_exceptions_in_sub_crawlers(*, error_in_p... function test_adaptive_playwright_crawler_statistics_in_init (line 543) | async def test_adaptive_playwright_crawler_statistics_in_init() -> None: function test_adaptive_playwright_crawler_timeout_in_sub_crawler (line 572) | async def test_adaptive_playwright_crawler_timeout_in_sub_crawler(test_u... function test_adaptive_playwright_crawler_default_predictor (line 611) | async def test_adaptive_playwright_crawler_default_predictor(test_urls: ... function test_adaptive_context_query_selector_beautiful_soup (line 634) | async def test_adaptive_context_query_selector_beautiful_soup(test_urls:... function test_adaptive_context_query_selector_parsel (line 680) | async def test_adaptive_context_query_selector_parsel(test_urls: list[st... function test_adaptive_context_parse_with_static_parser_parsel (line 718) | async def test_adaptive_context_parse_with_static_parser_parsel(test_url... function test_adaptive_context_helpers_on_changed_selector (line 755) | async def test_adaptive_context_helpers_on_changed_selector(test_urls: l... function test_adaptive_context_query_non_existing_element (line 786) | async def test_adaptive_context_query_non_existing_element(test_urls: li... function test_change_context_state_after_handling (line 841) | async def test_change_context_state_after_handling(test_input: TestInput... function test_adaptive_playwright_crawler_with_sql_storage (line 888) | async def test_adaptive_playwright_crawler_with_sql_storage(test_urls: l... FILE: tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler_statistics.py function test_predictor_state_persistence (line 7) | async def test_predictor_state_persistence() -> None: FILE: tests/unit/crawlers/_adaptive_playwright/test_predictor.py function test_predictor_same_label (line 27) | async def test_predictor_same_label(url: str, expected_prediction: Rende... function test_predictor_new_label_increased_detection_probability_recommendation (line 45) | async def test_predictor_new_label_increased_detection_probability_recom... function test_unreliable_prediction (line 89) | async def test_unreliable_prediction() -> None: function test_no_learning_data_prediction (line 114) | async def test_no_learning_data_prediction() -> None: function test_persistent_no_learning_data_prediction (line 126) | async def test_persistent_no_learning_data_prediction() -> None: function test_persistent_prediction (line 140) | async def test_persistent_prediction() -> None: function test_persistent_prediction_recovery (line 164) | async def test_persistent_prediction_recovery(*, persistence_enabled: bo... function test_url_similarity (line 213) | def test_url_similarity(url_1: str, url_2: str, expected_rounded_similar... FILE: tests/unit/crawlers/_basic/test_basic_crawler.py function test_processes_requests_from_explicit_queue (line 47) | async def test_processes_requests_from_explicit_queue() -> None: function test_processes_requests_from_request_source_tandem (line 63) | async def test_processes_requests_from_request_source_tandem() -> None: function test_processes_requests_from_run_args (line 89) | async def test_processes_requests_from_run_args() -> None: function test_allows_multiple_run_calls (line 102) | async def test_allows_multiple_run_calls() -> None: function test_retries_failed_requests (line 123) | async def test_retries_failed_requests() -> None: function test_respects_no_retry (line 146) | async def test_respects_no_retry() -> None: function test_respects_request_specific_max_retries (line 174) | async def test_respects_request_specific_max_retries() -> None: function test_calls_error_handler (line 199) | async def test_calls_error_handler() -> None: function test_calls_error_handler_for_session_errors (line 233) | async def test_calls_error_handler_for_session_errors() -> None: function test_handles_error_in_error_handler (line 253) | async def test_handles_error_in_error_handler() -> None: function test_calls_failed_request_handler (line 269) | async def test_calls_failed_request_handler() -> None: function test_handlers_use_context_helpers (line 290) | async def test_handlers_use_context_helpers(tmp_path: Path, handler: str... function test_handles_error_in_failed_request_handler (line 329) | async def test_handles_error_in_failed_request_handler() -> None: function test_send_request_works (line 352) | async def test_send_request_works(server_url: URL, method: HttpMethod, p... class AddRequestsTestInput (line 378) | class AddRequestsTestInput: function test_enqueue_strategy (line 559) | async def test_enqueue_strategy(test_input: AddRequestsTestInput) -> None: function test_session_rotation (line 583) | async def test_session_rotation(server_url: URL) -> None: function test_final_statistics (line 608) | async def test_final_statistics() -> None: function test_crawler_get_storages (line 652) | async def test_crawler_get_storages() -> None: function test_crawler_run_requests (line 665) | async def test_crawler_run_requests() -> None: function test_context_push_and_get_data (line 685) | async def test_context_push_and_get_data() -> None: function test_context_push_and_get_data_handler_error (line 706) | async def test_context_push_and_get_data_handler_error() -> None: function test_crawler_push_and_export_data (line 722) | async def test_crawler_push_and_export_data(tmp_path: Path) -> None: function test_crawler_export_data_additional_kwargs (line 746) | async def test_crawler_export_data_additional_kwargs(tmp_path: Path) -> ... function test_context_push_and_export_data (line 762) | async def test_context_push_and_export_data(tmp_path: Path) -> None: function test_context_update_kv_store (line 789) | async def test_context_update_kv_store() -> None: function test_context_use_state (line 803) | async def test_context_use_state() -> None: function test_crawler_use_state (line 818) | async def test_crawler_use_state() -> None: function test_context_use_state_crawlers_share_state (line 832) | async def test_context_use_state_crawlers_share_state() -> None: function test_crawlers_share_stats (line 851) | async def test_crawlers_share_stats() -> None: function test_context_use_state_crawlers_own_state (line 866) | async def test_context_use_state_crawlers_own_state() -> None: function test_context_handlers_use_state (line 883) | async def test_context_handlers_use_state(key_value_store: KeyValueStore... function test_max_requests_per_crawl (line 929) | async def test_max_requests_per_crawl(*, use_failed_requests: bool) -> N... function test_max_crawl_depth (line 960) | async def test_max_crawl_depth() -> None: function test_abort_on_error (line 999) | async def test_abort_on_error( function test_crawler_log (line 1027) | def test_crawler_log() -> None: function test_consecutive_runs_purge_request_queue (line 1033) | async def test_consecutive_runs_purge_request_queue() -> None: function test_logs_final_statistics (line 1061) | async def test_logs_final_statistics( function test_crawler_manual_stop (line 1131) | async def test_crawler_manual_stop() -> None: function test_crawler_multiple_stops_in_parallel (line 1158) | async def test_crawler_multiple_stops_in_parallel() -> None: function test_services_no_side_effect_on_crawler_init (line 1191) | async def test_services_no_side_effect_on_crawler_init() -> None: function test_crawler_uses_default_services (line 1207) | async def test_crawler_uses_default_services() -> None: function test_services_crawlers_can_use_different_services (line 1224) | async def test_services_crawlers_can_use_different_services() -> None: function test_crawler_uses_default_storages (line 1246) | async def test_crawler_uses_default_storages(tmp_path: Path) -> None: function test_crawler_can_use_other_storages (line 1264) | async def test_crawler_can_use_other_storages(tmp_path: Path) -> None: function test_crawler_can_use_other_storages_of_same_type (line 1282) | async def test_crawler_can_use_other_storages_of_same_type(tmp_path: Pat... function test_allows_storage_client_overwrite_before_run (line 1323) | async def test_allows_storage_client_overwrite_before_run(monkeypatch: p... function test_context_use_state_race_condition_in_handlers (line 1349) | async def test_context_use_state_race_condition_in_handlers(key_value_st... function test_timeout_in_handler (line 1389) | async def test_timeout_in_handler(sleep_type: str) -> None: function test_keep_alive (line 1447) | async def test_keep_alive( function test_session_retire_in_user_handler (line 1492) | async def test_session_retire_in_user_handler(*, retire: bool) -> None: function test_bound_session_to_request (line 1514) | async def test_bound_session_to_request() -> None: function test_bound_sessions_to_same_request (line 1536) | async def test_bound_sessions_to_same_request() -> None: function test_error_bound_session_to_request (line 1568) | async def test_error_bound_session_to_request() -> None: function test_handle_error_bound_session_to_request (line 1580) | async def test_handle_error_bound_session_to_request() -> None: function test_handles_session_error_in_failed_request_handler (line 1596) | async def test_handles_session_error_in_failed_request_handler() -> None: function test_lock_with_get_robots_txt_file_for_url (line 1615) | async def test_lock_with_get_robots_txt_file_for_url(server_url: URL) ->... function test_reduced_logs_from_timed_out_request_handler (line 1627) | async def test_reduced_logs_from_timed_out_request_handler(caplog: pytes... function test_reduced_logs_from_time_out_in_request_handler (line 1657) | async def test_reduced_logs_from_time_out_in_request_handler(caplog: pyt... function test_status_message_callback (line 1682) | async def test_status_message_callback() -> None: function test_status_message_emit (line 1721) | async def test_status_message_emit() -> None: function test_add_requests_with_rq_param (line 1748) | async def test_add_requests_with_rq_param(queue_name: str | None, queue_... function test_add_requests_error_with_multi_params (line 1790) | async def test_add_requests_error_with_multi_params( function test_crawler_purge_request_queue_uses_same_storage_client (line 1808) | async def test_crawler_purge_request_queue_uses_same_storage_client() ->... function _run_crawler (line 1831) | async def _run_crawler(crawler_id: int | None, requests: list[str], stor... class _CrawlerInput (line 1860) | class _CrawlerInput: function _process_run_crawlers (line 1865) | def _process_run_crawlers(crawler_inputs: list[_CrawlerInput], storage_d... function test_crawler_state_persistence (line 1872) | async def test_crawler_state_persistence(tmp_path: Path) -> None: function test_crawler_state_persistence_2_crawlers_with_migration (line 1921) | async def test_crawler_state_persistence_2_crawlers_with_migration(tmp_p... function test_crawler_intermediate_statistics (line 1967) | async def test_crawler_intermediate_statistics() -> None: function test_protect_request_in_run_handlers (line 1993) | async def test_protect_request_in_run_handlers() -> None: function test_new_request_error_handler (line 2016) | async def test_new_request_error_handler() -> None: FILE: tests/unit/crawlers/_basic/test_context_pipeline.py class EnhancedCrawlingContext (line 21) | class EnhancedCrawlingContext(BasicCrawlingContext): class MoreEnhancedCrawlingContext (line 26) | class MoreEnhancedCrawlingContext(EnhancedCrawlingContext): function test_calls_consumer_without_middleware (line 30) | async def test_calls_consumer_without_middleware() -> None: function test_calls_consumers_and_middlewares (line 51) | async def test_calls_consumers_and_middlewares() -> None: function test_wraps_consumer_errors (line 115) | async def test_wraps_consumer_errors() -> None: function test_handles_exceptions_in_middleware_initialization (line 135) | async def test_handles_exceptions_in_middleware_initialization() -> None: function test_handles_exceptions_in_middleware_finalization (line 167) | async def test_handles_exceptions_in_middleware_finalization() -> None: FILE: tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py function test_basic (line 21) | async def test_basic(server_url: URL, http_client: HttpClient) -> None: function test_enqueue_links (line 38) | async def test_enqueue_links(redirect_server_url: URL, server_url: URL, ... function test_enqueue_non_href_links (line 67) | async def test_enqueue_non_href_links(redirect_server_url: URL, server_u... function test_enqueue_links_selector (line 90) | async def test_enqueue_links_selector(server_url: URL, http_client: Http... function test_enqueue_links_with_max_crawl (line 108) | async def test_enqueue_links_with_max_crawl(server_url: URL, http_client... function test_enqueue_links_with_transform_request_function (line 132) | async def test_enqueue_links_with_transform_request_function(server_url:... function test_handle_blocked_request (line 173) | async def test_handle_blocked_request(server_url: URL, http_client: Http... function test_default_logger (line 179) | def test_default_logger() -> None: function test_respect_robots_txt (line 183) | async def test_respect_robots_txt(server_url: URL, http_client: HttpClie... function test_respect_robots_txt_with_problematic_links (line 203) | async def test_respect_robots_txt_with_problematic_links(server_url: URL... function test_on_skipped_request (line 239) | async def test_on_skipped_request(server_url: URL, http_client: HttpClie... function test_extract_links (line 262) | async def test_extract_links(server_url: URL, http_client: HttpClient) -... function test_extract_non_href_links (line 277) | async def test_extract_non_href_links(server_url: URL, http_client: Http... function test_enqueue_links_with_rq_param (line 300) | async def test_enqueue_links_with_rq_param( function test_enqueue_links_requests_with_rq_param (line 337) | async def test_enqueue_links_requests_with_rq_param( function test_enqueue_links_error_with_multi_params (line 383) | async def test_enqueue_links_error_with_multi_params( function test_navigation_timeout_on_slow_request (line 396) | async def test_navigation_timeout_on_slow_request(server_url: URL, http_... function test_navigation_timeout_applies_to_hooks (line 417) | async def test_navigation_timeout_applies_to_hooks(server_url: URL) -> N... function test_slow_navigation_does_not_count_toward_handler_timeout (line 435) | async def test_slow_navigation_does_not_count_toward_handler_timeout(ser... function test_enqueue_strategy_after_redirect (line 453) | async def test_enqueue_strategy_after_redirect(server_url: URL, redirect... function test_enqueue_links_with_limit (line 472) | async def test_enqueue_links_with_limit(server_url: URL, http_client: Ht... FILE: tests/unit/crawlers/_http/test_http_crawler.py function mock_request_handler (line 39) | async def mock_request_handler() -> Callable[[HttpCrawlingContext], Awai... function crawler (line 44) | async def crawler( function crawler_without_retries (line 51) | async def crawler_without_retries( function test_fetches_html (line 61) | async def test_fetches_html( function test_handles_redirects (line 73) | async def test_handles_redirects(crawler: HttpCrawler, mock_request_hand... function test_handles_client_errors (line 97) | async def test_handles_client_errors( function test_handles_session_block_errors (line 136) | async def test_handles_session_block_errors( function test_handles_server_error (line 166) | async def test_handles_server_error(crawler: HttpCrawler, mock_request_h... function test_stores_cookies (line 173) | async def test_stores_cookies(http_client: HttpClient, server_url: URL) ... function test_do_not_retry_on_client_errors (line 213) | async def test_do_not_retry_on_client_errors(crawler: HttpCrawler, serve... function test_http_status_statistics (line 223) | async def test_http_status_statistics(crawler: HttpCrawler, server_url: ... function test_sending_payload_as_raw_data (line 238) | async def test_sending_payload_as_raw_data(http_client: HttpClient, serv... function test_sending_payload_as_form_data (line 269) | async def test_sending_payload_as_form_data(http_client: HttpClient, ser... function test_sending_payload_as_json (line 295) | async def test_sending_payload_as_json(http_client: HttpClient, server_u... function test_sending_url_query_params (line 322) | async def test_sending_url_query_params(http_client: HttpClient, server_... function test_http_crawler_pre_navigation_hook_execution (line 344) | async def test_http_crawler_pre_navigation_hook_execution(server_url: UR... function test_http_crawler_post_navigation_hook_execution (line 361) | async def test_http_crawler_post_navigation_hook_execution(server_url: U... function test_http_crawler_navigation_hooks_order (line 378) | async def test_http_crawler_navigation_hooks_order(server_url: URL) -> N... function test_isolation_cookies (line 420) | async def test_isolation_cookies(http_client: HttpClient, server_url: UR... function test_store_complex_cookies (line 486) | async def test_store_complex_cookies(server_url: URL) -> None: function test_default_logger (line 585) | def test_default_logger() -> None: function test_get_snapshot (line 589) | async def test_get_snapshot(server_url: URL) -> None: function test_error_snapshot_through_statistics (line 606) | async def test_error_snapshot_through_statistics(server_url: URL) -> None: function test_request_state (line 633) | async def test_request_state(server_url: URL) -> None: FILE: tests/unit/crawlers/_parsel/test_parsel_crawler.py function test_basic (line 21) | async def test_basic(server_url: URL, http_client: HttpClient) -> None: function test_enqueue_links (line 38) | async def test_enqueue_links(redirect_server_url: URL, server_url: URL, ... function test_enqueue_non_href_links (line 68) | async def test_enqueue_non_href_links(redirect_server_url: URL, server_u... function test_enqueue_links_with_incompatible_kwargs_raises_error (line 91) | async def test_enqueue_links_with_incompatible_kwargs_raises_error(serve... function test_enqueue_links_selector (line 110) | async def test_enqueue_links_selector(server_url: URL, http_client: Http... function test_enqueue_links_with_max_crawl (line 128) | async def test_enqueue_links_with_max_crawl(server_url: URL, http_client... function test_enqueue_links_with_transform_request_function (line 152) | async def test_enqueue_links_with_transform_request_function(server_url:... function test_handle_blocked_request (line 192) | async def test_handle_blocked_request(server_url: URL, http_client: Http... function test_handle_blocked_status_code (line 199) | async def test_handle_blocked_status_code(server_url: URL, http_client: ... function test_import_error_handled (line 224) | def test_import_error_handled() -> None: function test_json (line 240) | async def test_json(server_url: URL, http_client: HttpClient) -> None: function test_xml (line 256) | async def test_xml(server_url: URL, http_client: HttpClient) -> None: function test_default_logger (line 272) | def test_default_logger() -> None: function test_respect_robots_txt (line 276) | async def test_respect_robots_txt(server_url: URL, http_client: HttpClie... function test_respect_robots_txt_with_problematic_links (line 296) | async def test_respect_robots_txt_with_problematic_links(server_url: URL... function test_on_skipped_request (line 332) | async def test_on_skipped_request(server_url: URL, http_client: HttpClie... function test_extract_links (line 355) | async def test_extract_links(server_url: URL, http_client: HttpClient) -... function test_extract_non_href_links (line 370) | async def test_extract_non_href_links(server_url: URL, http_client: Http... function test_enqueue_links_with_rq_param (line 393) | async def test_enqueue_links_with_rq_param( function test_enqueue_links_requests_with_rq_param (line 430) | async def test_enqueue_links_requests_with_rq_param( function test_enqueue_links_error_with_multi_params (line 476) | async def test_enqueue_links_error_with_multi_params( function test_enqueue_links_with_limit (line 489) | async def test_enqueue_links_with_limit(server_url: URL, http_client: Ht... FILE: tests/unit/crawlers/_playwright/test_playwright_crawler.py function test_basic_request (line 66) | async def test_basic_request(method: HttpMethod, path: str, payload: Htt... function test_enqueue_links (line 83) | async def test_enqueue_links(redirect_server_url: URL, server_url: URL) ... function test_enqueue_non_href_links (line 111) | async def test_enqueue_non_href_links(redirect_server_url: URL, server_u... function test_enqueue_links_with_incompatible_kwargs_raises_error (line 133) | async def test_enqueue_links_with_incompatible_kwargs_raises_error(serve... function test_enqueue_links_with_transform_request_function (line 156) | async def test_enqueue_links_with_transform_request_function(server_url:... function test_nonexistent_url_invokes_error_handler (line 185) | async def test_nonexistent_url_invokes_error_handler() -> None: function test_redirect_handling (line 199) | async def test_redirect_handling(server_url: URL, redirect_server_url: U... function test_chromium_headless_headers (line 233) | async def test_chromium_headless_headers( function test_firefox_headless_headers (line 266) | async def test_firefox_headless_headers(header_network: dict, server_url... function test_custom_headers (line 296) | async def test_custom_headers(server_url: URL) -> None: function test_pre_navigation_hook (line 315) | async def test_pre_navigation_hook() -> None: function test_proxy_set (line 329) | async def test_proxy_set() -> None: function test_isolation_cookies (line 362) | async def test_isolation_cookies(*, use_incognito_pages: bool, server_ur... function test_save_cookies_after_handler_processing (line 448) | async def test_save_cookies_after_handler_processing(server_url: URL) ->... function test_read_write_cookies (line 475) | async def test_read_write_cookies(server_url: URL) -> None: function test_custom_fingerprint_uses_generator_options (line 501) | async def test_custom_fingerprint_uses_generator_options(server_url: URL... function test_custom_fingerprint_matches_header_user_agent (line 536) | async def test_custom_fingerprint_matches_header_user_agent(server_url: ... function test_ignore_http_error_status_codes (line 556) | async def test_ignore_http_error_status_codes(server_url: URL) -> None: function test_additional_http_error_status_codes (line 571) | async def test_additional_http_error_status_codes(server_url: URL) -> None: function test_launch_with_user_data_dir (line 586) | async def test_launch_with_user_data_dir(tmp_path: Path, server_url: URL... function test_launch_with_user_data_dir_and_fingerprint (line 600) | async def test_launch_with_user_data_dir_and_fingerprint(tmp_path: Path,... function test_get_snapshot (line 626) | async def test_get_snapshot(server_url: URL) -> None: function test_error_snapshot_through_statistics (line 647) | async def test_error_snapshot_through_statistics(server_url: URL) -> None: function test_respect_robots_txt (line 697) | async def test_respect_robots_txt(server_url: URL) -> None: function test_respect_robots_txt_with_problematic_links (line 717) | async def test_respect_robots_txt_with_problematic_links(server_url: URL... function test_on_skipped_request (line 752) | async def test_on_skipped_request(server_url: URL) -> None: function test_send_request (line 775) | async def test_send_request(server_url: URL) -> None: function test_send_request_with_client (line 807) | async def test_send_request_with_client(server_url: URL) -> None: function test_passing_configuration (line 828) | async def test_passing_configuration() -> None: function test_extract_links (line 839) | async def test_extract_links(server_url: URL) -> None: function test_extract_non_href_links (line 854) | async def test_extract_non_href_links(server_url: URL) -> None: function test_reduced_logs_from_playwright_navigation_timeout (line 869) | async def test_reduced_logs_from_playwright_navigation_timeout(caplog: p... function test_enqueue_links_with_rq_param (line 902) | async def test_enqueue_links_with_rq_param( function test_enqueue_links_requests_with_rq_param (line 939) | async def test_enqueue_links_requests_with_rq_param( function test_enqueue_links_error_with_multi_params (line 985) | async def test_enqueue_links_error_with_multi_params( function test_navigation_timeout_on_slow_page_load (line 998) | async def test_navigation_timeout_on_slow_page_load(server_url: URL) -> ... function test_navigation_timeout_applies_to_hooks (line 1021) | async def test_navigation_timeout_applies_to_hooks(server_url: URL) -> N... function test_slow_navigation_does_not_count_toward_handler_timeout (line 1039) | async def test_slow_navigation_does_not_count_toward_handler_timeout(ser... function test_request_state (line 1056) | async def test_request_state(server_url: URL) -> None: function test_enqueue_links_with_limit (line 1110) | async def test_enqueue_links_with_limit(server_url: URL) -> None: function test_playwright_crawler_pre_navigation_hook_execution (line 1132) | async def test_playwright_crawler_pre_navigation_hook_execution(server_u... function test_playwright_crawler_post_navigation_hook_execution (line 1149) | async def test_playwright_crawler_post_navigation_hook_execution(server_... function test_playwright_navigation_hooks_order (line 1166) | async def test_playwright_navigation_hooks_order(server_url: URL) -> None: FILE: tests/unit/crawlers/_playwright/test_utils.py function test_infinite_scroll_on_dynamic_page (line 7) | async def test_infinite_scroll_on_dynamic_page(server_url: URL) -> None: function test_infinite_scroll_no_page_without_scroll (line 45) | async def test_infinite_scroll_no_page_without_scroll(server_url: URL) -... function test_double_call_infinite_scroll (line 62) | async def test_double_call_infinite_scroll(server_url: URL) -> None: function test_block_requests_default (line 81) | async def test_block_requests_default(server_url: URL) -> None: function test_block_requests_with_extra_patterns (line 116) | async def test_block_requests_with_extra_patterns(server_url: URL) -> None: function test_block_requests_with_custom_patterns (line 138) | async def test_block_requests_with_custom_patterns(server_url: URL) -> N... FILE: tests/unit/events/test_event_manager.py function event_manager (line 20) | async def event_manager() -> AsyncGenerator[EventManager, None]: function event_system_info_data (line 26) | def event_system_info_data() -> EventSystemInfoData: function async_listener (line 31) | def async_listener() -> AsyncMock: function sync_listener (line 41) | def sync_listener() -> MagicMock: function test_emit_invokes_registered_sync_listener (line 50) | async def test_emit_invokes_registered_sync_listener( function test_emit_invokes_both_sync_and_async_listeners (line 64) | async def test_emit_invokes_both_sync_and_async_listeners( function test_emit_event_with_no_listeners (line 83) | async def test_emit_event_with_no_listeners( function test_emit_invokes_parameterless_listener (line 99) | async def test_emit_invokes_parameterless_listener( function test_remove_nonexistent_listener_does_not_fail (line 123) | async def test_remove_nonexistent_listener_does_not_fail( function test_removed_listener_not_invoked_on_emit (line 133) | async def test_removed_listener_not_invoked_on_emit( function test_close_clears_listeners_and_tasks (line 146) | async def test_close_clears_listeners_and_tasks(async_listener: AsyncMoc... function test_close_after_emit_processes_event (line 155) | async def test_close_after_emit_processes_event( function test_wait_for_all_listeners_cancelled_error (line 171) | async def test_wait_for_all_listeners_cancelled_error( function test_methods_raise_error_when_not_active (line 191) | async def test_methods_raise_error_when_not_active(event_system_info_dat... function test_event_manager_in_context_persistence (line 213) | async def test_event_manager_in_context_persistence() -> None: FILE: tests/unit/events/test_local_event_manager.py function test_emit_system_info_event (line 12) | async def test_emit_system_info_event() -> None: FILE: tests/unit/fingerprint_suite/test_adapters.py function test_fingerprint_generator_has_default (line 15) | def test_fingerprint_generator_has_default() -> None: function test_fingerprint_generator_some_options_stress_test (line 20) | def test_fingerprint_generator_some_options_stress_test() -> None: function test_fingerprint_generator_all_options (line 35) | def test_fingerprint_generator_all_options() -> None: function test_patched_header_generator_generate (line 84) | def test_patched_header_generator_generate(browser: Iterable[str | Brows... FILE: tests/unit/fingerprint_suite/test_header_generator.py function test_get_common_headers (line 17) | def test_get_common_headers(header_network: dict) -> None: function test_get_random_user_agent_header (line 26) | def test_get_random_user_agent_header() -> None: function test_get_user_agent_header_stress_test (line 36) | def test_get_user_agent_header_stress_test(browser_type: SupportedBrowse... function test_get_user_agent_header_invalid_browser_type (line 49) | def test_get_user_agent_header_invalid_browser_type() -> None: function test_get_sec_ch_ua_headers_chromium (line 57) | def test_get_sec_ch_ua_headers_chromium(header_network: dict) -> None: function test_get_sec_ch_ua_headers_firefox (line 67) | def test_get_sec_ch_ua_headers_firefox() -> None: function test_get_sec_ch_ua_headers_invalid_browser_type (line 75) | def test_get_sec_ch_ua_headers_invalid_browser_type() -> None: FILE: tests/unit/http_clients/test_http_clients.py function custom_http_client (line 25) | async def custom_http_client(request: SubRequest) -> AsyncGenerator[Http... function test_http_1 (line 36) | async def test_http_1(http_client: HttpClient, server_url: URL) -> None: function test_http_2 (line 50) | async def test_http_2(custom_http_client: HttpClient) -> None: function test_crawl_with_proxy (line 56) | async def test_crawl_with_proxy( function test_crawl_with_proxy_disabled (line 71) | async def test_crawl_with_proxy_disabled( function test_send_request_with_proxy (line 84) | async def test_send_request_with_proxy( function test_send_request_with_proxy_disabled (line 96) | async def test_send_request_with_proxy_disabled( function test_crawl_allow_redirects_by_default (line 106) | async def test_crawl_allow_redirects_by_default(http_client: HttpClient,... function test_crawl_allow_redirects_false (line 125) | async def test_crawl_allow_redirects_false(custom_http_client: HttpClien... function test_send_request_allow_redirects_by_default (line 137) | async def test_send_request_allow_redirects_by_default(http_client: Http... function test_send_request_allow_redirects_false (line 155) | async def test_send_request_allow_redirects_false(custom_http_client: Ht... function test_stream (line 165) | async def test_stream(http_client: HttpClient, server_url: URL) -> None: function test_stream_error_double_read_stream (line 176) | async def test_stream_error_double_read_stream(http_client: HttpClient, ... function test_stream_error_for_read (line 189) | async def test_stream_error_for_read(http_client: HttpClient, server_url... function test_send_request_error_for_read_stream (line 197) | async def test_send_request_error_for_read_stream(http_client: HttpClien... function test_send_crawl_error_for_read_stream (line 205) | async def test_send_crawl_error_for_read_stream(http_client: HttpClient,... function test_reuse_context_manager (line 222) | async def test_reuse_context_manager(custom_http_client: HttpClient, ser... function test_work_after_cleanup (line 233) | async def test_work_after_cleanup(http_client: HttpClient, server_url: U... function test_compressed_chunked_stream (line 245) | async def test_compressed_chunked_stream(http_client: HttpClient, server... FILE: tests/unit/http_clients/test_httpx.py function http_client (line 21) | async def http_client() -> AsyncGenerator[HttpClient]: function test_common_headers_and_user_agent (line 26) | async def test_common_headers_and_user_agent(server_url: URL, header_net... FILE: tests/unit/otel/test_crawler_instrumentor.py function test_crawler_instrumentor_capability (line 18) | async def test_crawler_instrumentor_capability(server_url: URL) -> None: FILE: tests/unit/proxy_configuration/test_new_proxy_info.py function test_returns_proxy_info (line 11) | async def test_returns_proxy_info() -> None: function test_throws_on_invalid_new_url_function (line 27) | async def test_throws_on_invalid_new_url_function() -> None: function test_returns_proxy_info_with_new_url_function (line 36) | async def test_returns_proxy_info_with_new_url_function() -> None: function test_returns_proxy_info_with_new_url_function_async (line 54) | async def test_returns_proxy_info_with_new_url_function_async() -> None: function test_rotates_proxies (line 70) | async def test_rotates_proxies() -> None: function test_rotates_proxies_with_sessions (line 87) | async def test_rotates_proxies_with_sessions() -> None: function test_sets_port (line 169) | async def test_sets_port(url: str, expected_port: int) -> None: FILE: tests/unit/proxy_configuration/test_tiers.py function test_rotates_proxies_uniformly_with_no_request (line 7) | async def test_rotates_proxies_uniformly_with_no_request() -> None: function test_retrying_request_makes_tier_go_up (line 36) | async def test_retrying_request_makes_tier_go_up() -> None: function test_retrying_request_makes_tier_go_up_with_sessions (line 69) | async def test_retrying_request_makes_tier_go_up_with_sessions() -> None: function test_successful_request_makes_tier_go_down (line 105) | async def test_successful_request_makes_tier_go_down() -> None: function test_none_proxy_retrying_request_makes_tier_go_up (line 135) | async def test_none_proxy_retrying_request_makes_tier_go_up() -> None: function test_none_proxy_rotates_proxies_uniformly_with_no_request (line 158) | async def test_none_proxy_rotates_proxies_uniformly_with_no_request() ->... FILE: tests/unit/request_loaders/test_request_list.py function test_sync_traversal (line 7) | async def test_sync_traversal() -> None: function test_async_traversal (line 19) | async def test_async_traversal() -> None: function test_is_empty_does_not_depend_on_fetch_next_request (line 36) | async def test_is_empty_does_not_depend_on_fetch_next_request() -> None: function test_persist_requests_key_with_sync_iterable (line 62) | async def test_persist_requests_key_with_sync_iterable() -> None: function test_persist_requests_key_with_empty_iterator (line 81) | async def test_persist_requests_key_with_empty_iterator() -> None: function test_requests_restoration_without_state (line 98) | async def test_requests_restoration_without_state() -> None: function test_state_restoration (line 125) | async def test_state_restoration() -> None: function test_requests_and_state_restoration (line 170) | async def test_requests_and_state_restoration() -> None: function test_persist_requests_key_only_persists_once (line 216) | async def test_persist_requests_key_only_persists_once() -> None: FILE: tests/unit/request_loaders/test_sitemap_request_loader.py function compress_gzip (line 47) | def compress_gzip(data: str) -> bytes: function encode_base64 (line 52) | def encode_base64(data: bytes) -> str: function test_sitemap_traversal (line 57) | async def test_sitemap_traversal(server_url: URL, http_client: HttpClien... function test_is_empty_does_not_depend_on_fetch_next_request (line 73) | async def test_is_empty_does_not_depend_on_fetch_next_request(server_url... function test_abort_sitemap_loading (line 98) | async def test_abort_sitemap_loading(server_url: URL, http_client: HttpC... function test_create_persist_state_for_sitemap_loading (line 118) | async def test_create_persist_state_for_sitemap_loading( function test_data_persistence_for_sitemap_loading (line 134) | async def test_data_persistence_for_sitemap_loading( function test_recovery_data_persistence_for_sitemap_loading (line 158) | async def test_recovery_data_persistence_for_sitemap_loading( function test_transform_request_function (line 185) | async def test_transform_request_function(server_url: URL, http_client: ... FILE: tests/unit/server.py function get_headers_dict (line 40) | def get_headers_dict(scope: dict[str, Any]) -> dict[str, str]: function get_query_params (line 48) | def get_query_params(query_string: bytes) -> dict[str, str]: function get_cookies_from_headers (line 60) | def get_cookies_from_headers(headers: dict[str, Any]) -> dict[str, str]: function send_json_response (line 71) | async def send_json_response(send: Send, data: Any, status: int = 200) -... function send_html_response (line 83) | async def send_html_response(send: Send, html_content: bytes, status: in... function app (line 95) | async def app(scope: dict[str, Any], receive: Receive, send: Send) -> None: function get_cookies (line 145) | async def get_cookies(scope: dict[str, Any], _receive: Receive, send: Se... function set_cookies (line 152) | async def set_cookies(scope: dict[str, Any], _receive: Receive, send: Se... function hello_world (line 177) | async def hello_world(_scope: dict[str, Any], _receive: Receive, send: S... function hello_world_json (line 185) | async def hello_world_json(_scope: dict[str, Any], _receive: Receive, se... function hello_world_xml (line 193) | async def hello_world_xml(_scope: dict[str, Any], _receive: Receive, sen... function post_echo (line 202) | async def post_echo(scope: dict[str, Any], receive: Receive, send: Send)... function echo_status (line 252) | async def echo_status(scope: dict[str, Any], _receive: Receive, send: Se... function echo_headers (line 265) | async def echo_headers(scope: dict[str, Any], _receive: Receive, send: S... function start_enqueue_endpoint (line 271) | async def start_enqueue_endpoint(_scope: dict[str, Any], _receive: Recei... function secondary_index_endpoint (line 279) | async def secondary_index_endpoint(_scope: dict[str, Any], _receive: Rec... function incapsula_endpoint (line 287) | async def incapsula_endpoint(_scope: dict[str, Any], _receive: Receive, ... function generic_response_endpoint (line 295) | async def generic_response_endpoint(_scope: dict[str, Any], _receive: Re... function problematic_links_endpoint (line 303) | async def problematic_links_endpoint(_scope: dict[str, Any], _receive: R... function non_href_links_endpoint (line 311) | async def non_href_links_endpoint(_scope: dict[str, Any], _receive: Rece... function redirect_to_url (line 319) | async def redirect_to_url(scope: dict[str, Any], _receive: Receive, send... function echo_user_agent (line 339) | async def echo_user_agent(scope: dict[str, Any], _receive: Receive, send... function get_echo (line 346) | async def get_echo(scope: dict[str, Any], _receive: Receive, send: Send)... function set_complex_cookies (line 371) | async def set_complex_cookies(_scope: dict[str, Any], _receive: Receive,... function echo_content (line 394) | async def echo_content(scope: dict[str, Any], _receive: Receive, send: S... function robots_txt (line 415) | async def robots_txt(_scope: dict[str, Any], _receive: Receive, send: Se... function get_compressed (line 420) | async def get_compressed(_scope: dict[str, Any], _receive: Receive, send... function slow_response (line 433) | async def slow_response(scope: dict[str, Any], _receive: Receive, send: ... function infinite_scroll_endpoint (line 442) | async def infinite_scroll_endpoint(_scope: dict[str, Any], _receive: Rec... function resource_loading_endpoint (line 450) | async def resource_loading_endpoint(_scope: dict[str, Any], _receive: Re... function base_index_endpoint (line 458) | async def base_index_endpoint(_scope: dict[str, Any], _receive: Receive,... function start_enqueue_non_href_endpoint (line 468) | async def start_enqueue_non_href_endpoint(_scope: dict[str, Any], _recei... class TestServer (line 478) | class TestServer(Server): method url (line 482) | def url(self) -> URL: method serve (line 491) | async def serve(self, sockets: list[socket] | None = None) -> None: method restart (line 506) | async def restart(self) -> None: method watch_restarts (line 517) | async def watch_restarts(self) -> None: method run (line 532) | def run(self, sockets: list[socket] | None = None) -> None: function serve_in_thread (line 542) | def serve_in_thread(server: TestServer) -> Iterator[TestServer]: FILE: tests/unit/sessions/test_cookies.py function cookie_dict (line 9) | def cookie_dict() -> CookieParam: function session_cookies (line 25) | def session_cookies(cookie_dict: CookieParam) -> SessionCookies: function test_set_basic_cookie (line 31) | def test_set_basic_cookie() -> None: function test_set_cookie_with_all_attributes (line 46) | def test_set_cookie_with_all_attributes(session_cookies: SessionCookies,... function test_convert_cookie_to_dict (line 63) | def test_convert_cookie_to_dict(session_cookies: SessionCookies, cookie_... function test_convert_dict_format (line 74) | def test_convert_dict_format(session_cookies: SessionCookies) -> None: function test_get_cookies_as_browser_format (line 94) | def test_get_cookies_as_browser_format(session_cookies: SessionCookies, ... function test_get_cookies_as_dicts (line 106) | def test_get_cookies_as_dicts(session_cookies: SessionCookies, cookie_di... function test_store_cookie (line 113) | def test_store_cookie(session_cookies: SessionCookies) -> None: function test_store_multidomain_cookies (line 122) | def test_store_multidomain_cookies() -> None: FILE: tests/unit/sessions/test_models.py function session_direct (line 14) | def session_direct() -> SessionModel: function session_args_camel (line 32) | def session_args_camel() -> dict: function session_args_snake (line 50) | def session_args_snake() -> dict: function test_session_model (line 67) | def test_session_model( FILE: tests/unit/sessions/test_session.py function session (line 12) | def session() -> Session: function test_session_init (line 28) | def test_session_init(session: Session) -> None: function test_session_get_state (line 40) | def test_session_get_state(session: Session) -> None: function test_mark_good (line 52) | def test_mark_good(session: Session) -> None: function test_mark_bad (line 60) | def test_mark_bad(session: Session) -> None: function test_multiple_marks (line 67) | def test_multiple_marks(session: Session) -> None: function test_retire_method (line 83) | def test_retire_method(session: Session) -> None: function test_retire_on_blocked_status_code (line 90) | def test_retire_on_blocked_status_code(session: Session) -> None: function test_not_retire_on_not_block_status_code (line 97) | def test_not_retire_on_not_block_status_code(session: Session) -> None: function test_session_expiration (line 104) | def test_session_expiration() -> None: FILE: tests/unit/sessions/test_session_pool.py function kvs (line 25) | async def kvs() -> AsyncGenerator[KeyValueStore, None]: function event_manager (line 32) | async def event_manager() -> AsyncGenerator[EventManager, None]: function session_pool (line 38) | async def session_pool() -> AsyncGenerator[SessionPool, None]: function test_session_pool_init (line 43) | async def test_session_pool_init(session_pool: SessionPool) -> None: function test_add_session (line 50) | async def test_add_session(session_pool: SessionPool) -> None: function test_add_session_duplicate (line 61) | async def test_add_session_duplicate(caplog: pytest.LogCaptureFixture, s... function test_get_session (line 75) | async def test_get_session(session_pool: SessionPool) -> None: function test_get_session_no_usable (line 86) | async def test_get_session_no_usable(caplog: pytest.LogCaptureFixture, s... function test_create_session_function (line 102) | async def test_create_session_function() -> None: function test_session_pool_persist (line 116) | async def test_session_pool_persist(event_manager: EventManager, kvs_nam... function test_session_pool_persist_and_restore (line 147) | async def test_session_pool_persist_and_restore(event_manager: EventMana... function test_methods_raise_error_when_not_active (line 173) | async def test_methods_raise_error_when_not_active() -> None: FILE: tests/unit/storage_clients/_file_system/test_fs_dataset_client.py function configuration (line 21) | def configuration(tmp_path: Path) -> Configuration: function dataset_client (line 28) | async def dataset_client(configuration: Configuration) -> AsyncGenerator... function test_file_and_directory_creation (line 35) | async def test_file_and_directory_creation(configuration: Configuration)... function test_file_persistence_and_content_verification (line 54) | async def test_file_persistence_and_content_verification(dataset_client:... function test_drop_removes_files_from_disk (line 82) | async def test_drop_removes_files_from_disk(dataset_client: FileSystemDa... function test_metadata_file_updates (line 94) | async def test_metadata_file_updates(dataset_client: FileSystemDatasetCl... function test_data_persistence_across_reopens (line 134) | async def test_data_persistence_across_reopens() -> None: FILE: tests/unit/storage_clients/_file_system/test_fs_kvs_client.py function configuration (line 21) | def configuration(tmp_path: Path) -> Configuration: function kvs_client (line 28) | async def kvs_client(configuration: Configuration) -> AsyncGenerator[Fil... function test_file_and_directory_creation (line 35) | async def test_file_and_directory_creation(configuration: Configuration)... function test_value_file_creation_and_content (line 52) | async def test_value_file_creation_and_content(kvs_client: FileSystemKey... function test_binary_data_persistence (line 76) | async def test_binary_data_persistence(kvs_client: FileSystemKeyValueSto... function test_json_serialization_to_file (line 97) | async def test_json_serialization_to_file(kvs_client: FileSystemKeyValue... function test_file_deletion_on_value_delete (line 110) | async def test_file_deletion_on_value_delete(kvs_client: FileSystemKeyVa... function test_drop_removes_directory (line 132) | async def test_drop_removes_directory(kvs_client: FileSystemKeyValueStor... function test_metadata_file_updates (line 144) | async def test_metadata_file_updates(kvs_client: FileSystemKeyValueStore... function test_data_persistence_across_reopens (line 179) | async def test_data_persistence_across_reopens(configuration: Configurat... FILE: tests/unit/storage_clients/_file_system/test_fs_rq_client.py function configuration (line 21) | def configuration(tmp_path: Path) -> Configuration: function rq_client (line 28) | async def rq_client() -> AsyncGenerator[FileSystemRequestQueueClient, No... function test_file_and_directory_creation (line 37) | async def test_file_and_directory_creation() -> None: function test_request_file_persistence (line 54) | async def test_request_file_persistence(rq_client: FileSystemRequestQueu... function test_opening_rq_does_not_have_side_effect_on_service_locator (line 81) | async def test_opening_rq_does_not_have_side_effect_on_service_locator(c... function test_drop_removes_directory (line 89) | async def test_drop_removes_directory(rq_client: FileSystemRequestQueueC... function test_metadata_file_updates (line 102) | async def test_metadata_file_updates(rq_client: FileSystemRequestQueueCl... function test_data_persistence_across_reopens (line 142) | async def test_data_persistence_across_reopens() -> None: function test_get_request_does_not_mark_in_progress (line 178) | async def test_get_request_does_not_mark_in_progress(rq_client: FileSyst... FILE: tests/unit/storage_clients/_memory/test_memory_dataset_client.py function dataset_client (line 17) | async def dataset_client() -> AsyncGenerator[MemoryDatasetClient, None]: function test_memory_specific_purge_behavior (line 24) | async def test_memory_specific_purge_behavior() -> None: function test_memory_metadata_updates (line 46) | async def test_memory_metadata_updates(dataset_client: MemoryDatasetClie... FILE: tests/unit/storage_clients/_memory/test_memory_kvs_client.py function kvs_client (line 17) | async def kvs_client() -> AsyncGenerator[MemoryKeyValueStoreClient, None]: function test_memory_specific_purge_behavior (line 24) | async def test_memory_specific_purge_behavior() -> None: function test_memory_metadata_updates (line 48) | async def test_memory_metadata_updates(kvs_client: MemoryKeyValueStoreCl... FILE: tests/unit/storage_clients/_memory/test_memory_rq_client.py function rq_client (line 18) | async def rq_client() -> AsyncGenerator[MemoryRequestQueueClient, None]: function test_memory_specific_purge_behavior (line 25) | async def test_memory_specific_purge_behavior() -> None: function test_memory_metadata_updates (line 46) | async def test_memory_metadata_updates(rq_client: MemoryRequestQueueClie... FILE: tests/unit/storage_clients/_redis/test_redis_dataset_client.py function dataset_client (line 20) | async def dataset_client( function test_base_keys_creation (line 32) | async def test_base_keys_creation(dataset_client: RedisDatasetClient) ->... function test_record_and_content_verification (line 55) | async def test_record_and_content_verification(dataset_client: RedisData... function test_drop_removes_records (line 85) | async def test_drop_removes_records(dataset_client: RedisDatasetClient) ... function test_metadata_record_updates (line 114) | async def test_metadata_record_updates(dataset_client: RedisDatasetClien... FILE: tests/unit/storage_clients/_redis/test_redis_kvs_client.py function kvs_client (line 21) | async def kvs_client( function test_base_keys_creation (line 33) | async def test_base_keys_creation(kvs_client: RedisKeyValueStoreClient) ... function test_value_record_creation_and_content (line 52) | async def test_value_record_creation_and_content(kvs_client: RedisKeyVal... function test_binary_data_persistence (line 88) | async def test_binary_data_persistence(kvs_client: RedisKeyValueStoreCli... function test_json_serialization_to_record (line 113) | async def test_json_serialization_to_record(kvs_client: RedisKeyValueSto... function test_records_deletion_on_value_delete (line 127) | async def test_records_deletion_on_value_delete(kvs_client: RedisKeyValu... function test_drop_removes_keys (line 153) | async def test_drop_removes_keys(kvs_client: RedisKeyValueStoreClient) -... function test_metadata_record_updates (line 185) | async def test_metadata_record_updates(kvs_client: RedisKeyValueStoreCli... FILE: tests/unit/storage_clients/_redis/test_redis_rq_client.py function rq_client (line 22) | async def rq_client( function test_base_keys_creation (line 35) | async def test_base_keys_creation(rq_client: RedisRequestQueueClient) ->... function test_request_records_persistence (line 66) | async def test_request_records_persistence(rq_client: RedisRequestQueueC... function test_drop_removes_records (line 98) | async def test_drop_removes_records(rq_client: RedisRequestQueueClient) ... function test_metadata_file_updates (line 168) | async def test_metadata_file_updates(rq_client: RedisRequestQueueClient)... function test_get_request (line 203) | async def test_get_request(rq_client: RedisRequestQueueClient) -> None: function test_deduplication (line 225) | async def test_deduplication(rq_client: RedisRequestQueueClient) -> None: FILE: tests/unit/storage_clients/_sql/test_sql_dataset_client.py function configuration (line 24) | def configuration(tmp_path: Path) -> Configuration: function get_tables (line 32) | def get_tables(sync_conn: Connection) -> list[str]: function dataset_client (line 38) | async def dataset_client( function test_create_tables_with_connection_string (line 51) | async def test_create_tables_with_connection_string(configuration: Confi... function test_create_tables_with_engine (line 67) | async def test_create_tables_with_engine(configuration: Configuration, t... function test_tables_and_metadata_record (line 85) | async def test_tables_and_metadata_record(configuration: Configuration) ... function test_record_and_content_verification (line 112) | async def test_record_and_content_verification(dataset_client: SqlDatase... function test_drop_removes_records (line 143) | async def test_drop_removes_records(dataset_client: SqlDatasetClient) ->... function test_metadata_record_updates (line 167) | async def test_metadata_record_updates(dataset_client: SqlDatasetClient)... function test_data_persistence_across_reopens (line 211) | async def test_data_persistence_across_reopens(configuration: Configurat... FILE: tests/unit/storage_clients/_sql/test_sql_kvs_client.py function configuration (line 26) | def configuration(tmp_path: Path) -> Configuration: function kvs_client (line 34) | async def kvs_client( function get_tables (line 48) | def get_tables(sync_conn: Connection) -> list[str]: function test_create_tables_with_connection_string (line 53) | async def test_create_tables_with_connection_string(configuration: Confi... function test_create_tables_with_engine (line 69) | async def test_create_tables_with_engine(configuration: Configuration, t... function test_tables_and_metadata_record (line 87) | async def test_tables_and_metadata_record(configuration: Configuration) ... function test_value_record_creation (line 113) | async def test_value_record_creation(kvs_client: SqlKeyValueStoreClient)... function test_binary_data_persistence (line 129) | async def test_binary_data_persistence(kvs_client: SqlKeyValueStoreClien... function test_json_serialization_to_record (line 151) | async def test_json_serialization_to_record(kvs_client: SqlKeyValueStore... function test_record_deletion_on_value_delete (line 166) | async def test_record_deletion_on_value_delete(kvs_client: SqlKeyValueSt... function test_drop_removes_records (line 193) | async def test_drop_removes_records(kvs_client: SqlKeyValueStoreClient) ... function test_metadata_record_updates (line 217) | async def test_metadata_record_updates(kvs_client: SqlKeyValueStoreClien... function test_data_persistence_across_reopens (line 259) | async def test_data_persistence_across_reopens(configuration: Configurat... FILE: tests/unit/storage_clients/_sql/test_sql_rq_client.py function configuration (line 27) | def configuration(tmp_path: Path) -> Configuration: function rq_client (line 35) | async def rq_client( function get_tables (line 49) | def get_tables(sync_conn: Connection) -> list[str]: function test_create_tables_with_connection_string (line 54) | async def test_create_tables_with_connection_string(configuration: Confi... function test_create_tables_with_engine (line 71) | async def test_create_tables_with_engine(configuration: Configuration, t... function test_tables_and_metadata_record (line 90) | async def test_tables_and_metadata_record(configuration: Configuration) ... function test_request_records_persistence (line 117) | async def test_request_records_persistence(rq_client: SqlRequestQueueCli... function test_drop_removes_records (line 139) | async def test_drop_removes_records(rq_client: SqlRequestQueueClient) ->... function test_metadata_record_updates (line 160) | async def test_metadata_record_updates(rq_client: SqlRequestQueueClient)... function test_data_persistence_across_reopens (line 202) | async def test_data_persistence_across_reopens(configuration: Configurat... FILE: tests/unit/storages/conftest.py function storage_client (line 21) | def storage_client( FILE: tests/unit/storages/test_dataset.py function dataset (line 23) | async def dataset( function test_open_creates_new_dataset (line 35) | async def test_open_creates_new_dataset( function test_reopen_default (line 54) | async def test_reopen_default( function test_open_by_id (line 92) | async def test_open_by_id( function test_open_existing_dataset (line 126) | async def test_open_existing_dataset( function test_open_with_id_and_name (line 146) | async def test_open_with_id_and_name( function test_push_data_single_item (line 162) | async def test_push_data_single_item(dataset: Dataset) -> None: function test_push_data_multiple_items (line 173) | async def test_push_data_multiple_items(dataset: Dataset) -> None: function test_get_data_empty_dataset (line 188) | async def test_get_data_empty_dataset(dataset: Dataset) -> None: function test_get_data_with_pagination (line 197) | async def test_get_data_with_pagination(dataset: Dataset) -> None: function test_get_data_descending_order (line 224) | async def test_get_data_descending_order(dataset: Dataset) -> None: function test_get_data_skip_empty (line 238) | async def test_get_data_skip_empty(dataset: Dataset) -> None: function test_iterate_items (line 258) | async def test_iterate_items(dataset: Dataset) -> None: function test_iterate_items_with_options (line 272) | async def test_iterate_items_with_options(dataset: Dataset) -> None: function test_list_items (line 295) | async def test_list_items(dataset: Dataset) -> None: function test_list_items_with_options (line 309) | async def test_list_items_with_options(dataset: Dataset) -> None: function test_drop (line 368) | async def test_drop( function test_export_to_json (line 394) | async def test_export_to_json( function test_export_to_csv (line 432) | async def test_export_to_csv( function test_export_to_invalid_content_type (line 472) | async def test_export_to_invalid_content_type(dataset: Dataset) -> None: function test_export_with_multiple_kwargs (line 478) | async def test_export_with_multiple_kwargs(dataset: Dataset, tmp_path: P... function test_large_dataset (line 515) | async def test_large_dataset(dataset: Dataset) -> None: function test_purge (line 533) | async def test_purge( function test_open_with_alias (line 587) | async def test_open_with_alias( function test_alias_caching (line 624) | async def test_alias_caching( function test_alias_with_id_error (line 648) | async def test_alias_with_id_error( function test_alias_with_name_error (line 664) | async def test_alias_with_name_error( function test_alias_with_all_parameters_error (line 680) | async def test_alias_with_all_parameters_error( function test_alias_with_special_characters (line 697) | async def test_alias_with_special_characters( function test_named_vs_alias_conflict_detection (line 731) | async def test_named_vs_alias_conflict_detection( function test_alias_parameter (line 758) | async def test_alias_parameter( function test_alias_vs_named_isolation (line 781) | async def test_alias_vs_named_isolation( function test_default_vs_alias_default_equivalence (line 815) | async def test_default_vs_alias_default_equivalence( function test_multiple_alias_isolation (line 842) | async def test_multiple_alias_isolation( function test_purge_on_start_enabled (line 868) | async def test_purge_on_start_enabled(storage_client: StorageClient) -> ... function test_purge_on_start_disabled (line 954) | async def test_purge_on_start_disabled(storage_client: StorageClient) ->... function test_name_default_not_allowed (line 1042) | async def test_name_default_not_allowed(storage_client: StorageClient) -... function test_validate_name (line 1072) | async def test_validate_name(storage_client: StorageClient, name: str, *... function test_record_with_noascii_chars (line 1084) | async def test_record_with_noascii_chars(dataset: Dataset) -> None: FILE: tests/unit/storages/test_key_value_store.py function kvs (line 20) | async def kvs( function test_open_creates_new_kvs (line 32) | async def test_open_creates_new_kvs( function test_open_existing_kvs (line 48) | async def test_open_existing_kvs( function test_open_with_id_and_name (line 67) | async def test_open_with_id_and_name( function test_open_by_id (line 83) | async def test_open_by_id( function test_set_get_value (line 116) | async def test_set_get_value(kvs: KeyValueStore) -> None: function test_set_get_none (line 128) | async def test_set_get_none(kvs: KeyValueStore) -> None: function test_get_value_nonexistent (line 136) | async def test_get_value_nonexistent(kvs: KeyValueStore) -> None: function test_get_value_with_default (line 142) | async def test_get_value_with_default(kvs: KeyValueStore) -> None: function test_set_value_with_content_type (line 149) | async def test_set_value_with_content_type(kvs: KeyValueStore) -> None: function test_delete_value (line 160) | async def test_delete_value(kvs: KeyValueStore) -> None: function test_list_keys_empty_kvs (line 177) | async def test_list_keys_empty_kvs(kvs: KeyValueStore) -> None: function test_list_keys (line 183) | async def test_list_keys(kvs: KeyValueStore) -> None: function test_list_keys_with_limit (line 201) | async def test_list_keys_with_limit(kvs: KeyValueStore) -> None: function test_list_keys_with_exclusive_start_key (line 212) | async def test_list_keys_with_exclusive_start_key(kvs: KeyValueStore) ->... function test_iterate_keys (line 240) | async def test_iterate_keys(kvs: KeyValueStore) -> None: function test_iterate_keys_with_limit (line 257) | async def test_iterate_keys_with_limit(kvs: KeyValueStore) -> None: function test_drop (line 269) | async def test_drop( function test_reopen_default (line 296) | async def test_reopen_default( function test_complex_data_types (line 326) | async def test_complex_data_types(kvs: KeyValueStore) -> None: function test_string_data (line 349) | async def test_string_data(kvs: KeyValueStore) -> None: function test_key_with_special_characters (line 363) | async def test_key_with_special_characters(kvs: KeyValueStore) -> None: function test_data_persistence_on_reopen (line 387) | async def test_data_persistence_on_reopen() -> None: function test_purge (line 408) | async def test_purge( function test_record_exists_nonexistent (line 456) | async def test_record_exists_nonexistent(kvs: KeyValueStore) -> None: function test_record_exists_after_set (line 462) | async def test_record_exists_after_set(kvs: KeyValueStore) -> None: function test_record_exists_after_delete (line 477) | async def test_record_exists_after_delete(kvs: KeyValueStore) -> None: function test_record_exists_with_none_value (line 493) | async def test_record_exists_with_none_value(kvs: KeyValueStore) -> None: function test_record_exists_different_content_types (line 509) | async def test_record_exists_different_content_types(kvs: KeyValueStore)... function test_record_exists_multiple_keys (line 525) | async def test_record_exists_multiple_keys(kvs: KeyValueStore) -> None: function test_record_exists_after_purge (line 551) | async def test_record_exists_after_purge(kvs: KeyValueStore) -> None: function test_open_with_alias (line 569) | async def test_open_with_alias( function test_alias_caching (line 604) | async def test_alias_caching( function test_alias_with_id_error (line 628) | async def test_alias_with_id_error( function test_alias_with_name_error (line 644) | async def test_alias_with_name_error( function test_alias_with_special_characters (line 660) | async def test_alias_with_special_characters( function test_alias_key_operations (line 694) | async def test_alias_key_operations( function test_named_vs_alias_conflict_detection (line 743) | async def test_named_vs_alias_conflict_detection( function test_alias_parameter (line 781) | async def test_alias_parameter( function test_alias_vs_named_isolation (line 803) | async def test_alias_vs_named_isolation( function test_default_vs_alias_default_equivalence (line 837) | async def test_default_vs_alias_default_equivalence( function test_multiple_alias_isolation (line 864) | async def test_multiple_alias_isolation( function test_purge_on_start_enabled (line 894) | async def test_purge_on_start_enabled(storage_client: StorageClient) -> ... function test_purge_on_start_disabled (line 980) | async def test_purge_on_start_disabled(storage_client: StorageClient) ->... function test_name_default_not_allowed (line 1055) | async def test_name_default_not_allowed(storage_client: StorageClient) -... function test_validate_name (line 1085) | async def test_validate_name(storage_client: StorageClient, name: str, *... function test_get_auto_saved_value_various_global_clients (line 1113) | async def test_get_auto_saved_value_various_global_clients( function test_record_with_noascii_chars (line 1140) | async def test_record_with_noascii_chars(kvs: KeyValueStore) -> None: FILE: tests/unit/storages/test_request_manager_tandem.py class TestInput (line 14) | class TestInput: function test_basic_functionality (line 59) | async def test_basic_functionality(test_input: TestInput) -> None: FILE: tests/unit/storages/test_request_queue.py function rq (line 22) | async def rq( function test_open_creates_new_rq (line 34) | async def test_open_creates_new_rq( function test_open_existing_rq (line 54) | async def test_open_existing_rq( function test_open_with_id_and_name (line 73) | async def test_open_with_id_and_name( function test_open_by_id (line 89) | async def test_open_by_id( function test_add_request_string_url (line 121) | async def test_add_request_string_url(rq: RequestQueue) -> None: function test_add_request_object (line 139) | async def test_add_request_object(rq: RequestQueue) -> None: function test_add_duplicate_request (line 157) | async def test_add_duplicate_request(rq: RequestQueue) -> None: function test_add_requests_batch (line 179) | async def test_add_requests_batch(rq: RequestQueue) -> None: function test_add_requests_batch_with_forefront (line 200) | async def test_add_requests_batch_with_forefront(rq: RequestQueue) -> None: function test_add_requests_with_forefront (line 247) | async def test_add_requests_with_forefront(rq: RequestQueue) -> None: function test_add_requests_mixed_forefront (line 262) | async def test_add_requests_mixed_forefront(rq: RequestQueue) -> None: function test_fetch_next_request_and_mark_handled (line 313) | async def test_fetch_next_request_and_mark_handled(rq: RequestQueue) -> ... function test_get_request_by_id (line 348) | async def test_get_request_by_id(rq: RequestQueue) -> None: function test_handled_request_records_persistence (line 364) | async def test_handled_request_records_persistence(rq: RequestQueue) -> ... function test_get_non_existent_request (line 375) | async def test_get_non_existent_request(rq: RequestQueue) -> None: function test_reclaim_request (line 381) | async def test_reclaim_request(rq: RequestQueue) -> None: function test_reclaim_request_with_forefront (line 402) | async def test_reclaim_request_with_forefront(rq: RequestQueue) -> None: function test_is_empty (line 422) | async def test_is_empty(rq: RequestQueue) -> None: function test_add_requests_wait_for_all (line 448) | async def test_add_requests_wait_for_all( function test_is_finished (line 476) | async def test_is_finished(rq: RequestQueue) -> None: function test_mark_non_existent_request_as_handled (line 508) | async def test_mark_non_existent_request_as_handled(rq: RequestQueue) ->... function test_reclaim_non_existent_request (line 518) | async def test_reclaim_non_existent_request(rq: RequestQueue) -> None: function test_drop (line 528) | async def test_drop( function test_reopen_default (line 557) | async def test_reopen_default( function test_purge (line 614) | async def test_purge( function test_open_with_alias (line 667) | async def test_open_with_alias( function test_alias_caching (line 705) | async def test_alias_caching( function test_alias_with_id_error (line 729) | async def test_alias_with_id_error( function test_alias_with_name_error (line 745) | async def test_alias_with_name_error( function test_alias_with_special_characters (line 761) | async def test_alias_with_special_characters( function test_alias_request_operations (line 795) | async def test_alias_request_operations( function test_alias_forefront_operations (line 844) | async def test_alias_forefront_operations( function test_alias_batch_operations (line 874) | async def test_alias_batch_operations( function test_named_vs_alias_conflict_detection (line 903) | async def test_named_vs_alias_conflict_detection( function test_alias_parameter (line 944) | async def test_alias_parameter( function test_alias_vs_named_isolation (line 966) | async def test_alias_vs_named_isolation( function test_default_vs_alias_default_equivalence (line 1001) | async def test_default_vs_alias_default_equivalence( function test_multiple_alias_isolation (line 1028) | async def test_multiple_alias_isolation( function test_purge_on_start_enabled (line 1055) | async def test_purge_on_start_enabled(storage_client: StorageClient) -> ... function test_purge_on_start_disabled (line 1181) | async def test_purge_on_start_disabled(storage_client: StorageClient) ->... function test_name_default_not_allowed (line 1307) | async def test_name_default_not_allowed(storage_client: StorageClient) -... function test_validate_name (line 1337) | async def test_validate_name(storage_client: StorageClient, name: str, *... function test_reclaim_request_with_change_state (line 1349) | async def test_reclaim_request_with_change_state(rq: RequestQueue) -> None: function test_request_with_noascii_chars (line 1373) | async def test_request_with_noascii_chars(rq: RequestQueue) -> None: FILE: tests/unit/storages/test_storage_instance_manager.py function clean_storage_instance_manager (line 17) | def clean_storage_instance_manager() -> None: function storage_type (line 23) | def storage_type(request: pytest.FixtureRequest) -> type[Storage]: function test_unique_storage_by_storage_client (line 27) | async def test_unique_storage_by_storage_client(tmp_path: Path, storage_... function test_same_storage_when_different_client (line 35) | async def test_same_storage_when_different_client(tmp_path: Path, storag... function test_unique_storage_by_storage_type (line 43) | async def test_unique_storage_by_storage_type(tmp_path: Path) -> None: function test_unique_storage_by_name (line 52) | async def test_unique_storage_by_name(storage_type: type[Storage]) -> None: function test_unique_storage_by_unique_cache_key_different_path (line 61) | async def test_unique_storage_by_unique_cache_key_different_path(tmp_pat... function test_unique_storage_by_unique_cache_key_same_path (line 79) | async def test_unique_storage_by_unique_cache_key_same_path(tmp_path: Pa... function test_identical_storage_default_config (line 93) | async def test_identical_storage_default_config(storage_type: type[Stora... function test_identical_storage_default_storage (line 102) | async def test_identical_storage_default_storage(storage_type: type[Stor... function test_identical_storage_clear_cache (line 109) | async def test_identical_storage_clear_cache(storage_type: type[Storage]... function test_identical_storage_remove_from_cache (line 116) | async def test_identical_storage_remove_from_cache(storage_type: type[St... function test_preexisting_unnamed_storage_open_by_id (line 123) | async def test_preexisting_unnamed_storage_open_by_id(storage_type: type... function test_concurrent_open_datasets (line 137) | async def test_concurrent_open_datasets() -> None: function test_concurrent_open_datasets_with_same_name_and_alias (line 162) | async def test_concurrent_open_datasets_with_same_name_and_alias() -> None: FILE: tests/unit/test_cli.py function mock_cookiecutter (line 16) | def mock_cookiecutter(monkeypatch: pytest.MonkeyPatch) -> Mock: function test_create_interactive (line 23) | def test_create_interactive(mock_cookiecutter: Mock, monkeypatch: pytest... function test_create_interactive_non_default_template (line 56) | def test_create_interactive_non_default_template(mock_cookiecutter: Mock... function test_create_non_interactive (line 90) | def test_create_non_interactive(mock_cookiecutter: Mock) -> None: function test_create_existing_folder (line 124) | def test_create_existing_folder( function test_create_existing_folder_interactive (line 173) | def test_create_existing_folder_interactive( function test_create_existing_folder_interactive_multiple_attempts (line 213) | def test_create_existing_folder_interactive_multiple_attempts( FILE: tests/unit/test_configuration.py function test_global_configuration_works (line 20) | def test_global_configuration_works() -> None: function test_global_configuration_works_reversed (line 29) | def test_global_configuration_works_reversed() -> None: function test_storage_not_persisted_when_non_persistable_storage_used (line 38) | async def test_storage_not_persisted_when_non_persistable_storage_used(t... function test_storage_persisted_with_explicit_statistics_with_persistable_storage (line 54) | async def test_storage_persisted_with_explicit_statistics_with_persistab... function test_storage_persisted_when_enabled (line 79) | async def test_storage_persisted_when_enabled(tmp_path: Path, server_url... FILE: tests/unit/test_log_config.py function get_log_record (line 11) | def get_log_record(level: int, msg: str, exc_info: logging._SysExcInfoTy... function test_formatted_message (line 33) | def test_formatted_message(level: int, msg: str, expected: str) -> None: function test_formatting_with_exception (line 40) | def test_formatting_with_exception() -> None: function test_formatter_without_name (line 54) | def test_formatter_without_name() -> None: FILE: tests/unit/test_router.py class MockContext (line 14) | class MockContext(BasicCrawlingContext): method __init__ (line 15) | def __init__(self, *, label: str | None) -> None: function test_router_no_handlers (line 29) | async def test_router_no_handlers() -> None: function test_router_no_default_handler (line 36) | async def test_router_no_default_handler() -> None: function test_router_default_handler_invoked (line 50) | async def test_router_default_handler_invoked() -> None: function test_router_specific_handler_invoked (line 69) | async def test_router_specific_handler_invoked() -> None: function test_router_handler_not_nullified (line 94) | async def test_router_handler_not_nullified() -> None: function test_router_multi_labelled_handler (line 104) | async def test_router_multi_labelled_handler() -> None: FILE: tests/unit/test_service_locator.py function test_default_configuration (line 12) | def test_default_configuration() -> None: function test_custom_configuration (line 18) | def test_custom_configuration() -> None: function test_configuration_overwrite_not_possible (line 25) | def test_configuration_overwrite_not_possible() -> None: function test_configuration_conflict (line 34) | def test_configuration_conflict() -> None: function test_default_event_manager (line 42) | def test_default_event_manager() -> None: function test_custom_event_manager (line 47) | def test_custom_event_manager() -> None: function test_event_manager_overwrite_not_possible (line 54) | def test_event_manager_overwrite_not_possible() -> None: function test_event_manager_conflict (line 63) | def test_event_manager_conflict() -> None: function test_default_storage_client (line 71) | def test_default_storage_client() -> None: function test_custom_storage_client (line 76) | def test_custom_storage_client() -> None: function test_storage_client_overwrite_not_possible (line 83) | def test_storage_client_overwrite_not_possible() -> None: function test_storage_client_conflict (line 92) | def test_storage_client_conflict() -> None: FILE: website/docusaurus.config.js constant GROUP_ORDER (line 6) | const GROUP_ORDER = [ method configureWebpack (line 181) | configureWebpack() { method configureWebpack (line 196) | configureWebpack(config) { FILE: website/generate_module_shortcuts.py function get_module_shortcuts (line 15) | def get_module_shortcuts(module: ModuleType, parent_classes: list | None... function resolve_shortcuts (line 37) | def resolve_shortcuts(shortcuts: dict) -> None: FILE: website/roa-loader/index.js function hash (line 12) | function hash(source) { function getHash (line 16) | async function getHash(source) { function encodeAndSign (line 58) | async function encodeAndSign(source) { FILE: website/src/components/Button.jsx function Button (line 8) | function Button({ children, to, withIcon, type = 'primary', className, i... FILE: website/src/components/CopyButton.jsx function CopyButton (line 7) | function CopyButton({ copyText, compact = false, className }) { FILE: website/src/components/Gradients.jsx function Gradients (line 3) | function Gradients() { FILE: website/src/components/Highlights.jsx function Feature (line 77) | function Feature({ Svg, title, description }) { function Highlights (line 91) | function Highlights() { FILE: website/src/components/Homepage/HomepageCliExample.jsx function CliExample (line 8) | function CliExample() { FILE: website/src/components/Homepage/HomepageCtaSection.jsx function HomepageCtaSection (line 10) | function HomepageCtaSection() { FILE: website/src/components/Homepage/HomepageHeroSection.jsx function HomepageHeroSection (line 6) | function HomepageHeroSection() { FILE: website/src/components/Homepage/LanguageInfoWidget.jsx function LanguageInfoWidget (line 11) | function LanguageInfoWidget({ FILE: website/src/components/Homepage/LanguageSwitch.jsx function LanguageSwitch (line 5) | function LanguageSwitch({ FILE: website/src/components/Homepage/RiverSection.jsx function RiverSection (line 7) | function RiverSection({ title, description, content, reversed, to }) { FILE: website/src/components/Homepage/ThreeCardsWithIcon.jsx function ThreeCardsWithIcon (line 7) | function ThreeCardsWithIcon({ cards }) { FILE: website/src/components/LLMButtons.jsx constant DROPDOWN_OPTIONS (line 24) | const DROPDOWN_OPTIONS = [ constant CHAT_GPT_BASE (line 82) | const CHAT_GPT_BASE = 'https://chatgpt.com/?hints=search&q='; constant CLAUDE_BASE (line 83) | const CLAUDE_BASE = 'https://claude.ai/new?q='; constant PERPLEXITY_BASE (line 84) | const PERPLEXITY_BASE = 'https://www.perplexity.ai/search/new?q='; function getButtonText (line 289) | function getButtonText({ status }) { constant COPYING_STATUS_ICON (line 339) | const COPYING_STATUS_ICON = { function LLMButtons (line 416) | function LLMButtons() { FILE: website/src/components/RunnableCodeBlock.jsx constant PYTHON_ACTOR_RUNNER (line 7) | const PYTHON_ACTOR_RUNNER = 'HH9rhkFXiZbheuq1V' FILE: website/src/pages/home_page_example.py function main (line 6) | async def main() -> None: FILE: website/src/pages/index.js function GetStartedSection (line 22) | function GetStartedSection() { function CodeExampleSection (line 34) | function CodeExampleSection() { function BenefitsSection (line 65) | function BenefitsSection() { function OtherFeaturesSection (line 110) | function OtherFeaturesSection() { function DeployToCloudSection (line 212) | function DeployToCloudSection() { function BuildFastScrapersSection (line 269) | function BuildFastScrapersSection() { function JavascriptHomepage (line 337) | function JavascriptHomepage() { FILE: website/src/plugins/docusaurus-plugin-segment/index.js method getClientModules (line 9) | getClientModules() { method injectHtmlTags (line 13) | injectHtmlTags() { FILE: website/src/plugins/docusaurus-plugin-segment/segment.js method onRouteUpdate (line 4) | onRouteUpdate() { FILE: website/src/theme/ColorModeToggle/index.js function ColorModeToggle (line 10) | function ColorModeToggle({ FILE: website/src/theme/DocItem/Content/index.js function useSyntheticTitle (line 10) | function useSyntheticTitle() { function DocItemContent (line 21) | function DocItemContent({ children }) { FILE: website/src/theme/DocItem/Layout/index.js function useDocTOC (line 20) | function useDocTOC() { function DocItemLayout (line 39) | function DocItemLayout({ children }) { FILE: website/src/theme/Footer/LinkItem/index.js function FooterLinkItem (line 9) | function FooterLinkItem({ item }) { FILE: website/src/theme/Footer/index.js function FooterLinksColumn (line 12) | function FooterLinksColumn({ column }) { function Footer (line 27) | function Footer() { FILE: website/src/theme/MDXComponents/A.js function MDXA (line 6) | function MDXA(props) { FILE: website/src/theme/Navbar/Content/index.js function useNavbarItems (line 18) | function useNavbarItems() { function NavbarItems (line 22) | function NavbarItems({ items, className }) { function NavbarContentLayout (line 32) | function NavbarContentLayout({ left, right }) { constant VERSIONS_ITEM (line 41) | const VERSIONS_ITEM = { function NavbarContent (line 58) | function NavbarContent() { FILE: website/src/theme/Navbar/Logo/index.js function LogoWrapper (line 10) | function LogoWrapper(props) { FILE: website/src/theme/Navbar/MobileSidebar/Header/index.js function CloseButton (line 13) | function CloseButton() { function NavbarMobileSidebarHeader (line 29) | function NavbarMobileSidebarHeader() { FILE: website/src/theme/Navbar/MobileSidebar/Layout/index.js function NavbarMobileSidebarLayout (line 5) | function NavbarMobileSidebarLayout({ FILE: website/src/theme/Navbar/MobileSidebar/PrimaryMenu/index.js function useNavbarItems (line 6) | function useNavbarItems() { function NavbarMobilePrimaryMenu (line 10) | function NavbarMobilePrimaryMenu() { FILE: website/src/theme/Navbar/MobileSidebar/index.js function NavbarMobileSidebar (line 12) | function NavbarMobileSidebar() { FILE: website/src/theme/NavbarItem/ComponentTypes.js function DocNavbarItem (line 15) | function DocNavbarItem({ function ApiNavbarItem (line 38) | function ApiNavbarItem(ctx) { FILE: website/static/js/custom.js function load (line 1) | function load() { FILE: website/tools/utils/externalLink.js function isInternal (line 10) | function isInternal(href) { FILE: website/tools/website_gif/website_gif.mjs method requestHandler (line 21) | async requestHandler({ request }) {