[ { "path": ".editorconfig", "content": "# EditorConfig is awesome: http://EditorConfig.org\n\nroot = true\n\n[*]\ncharset = utf-8\nend_of_line = lf\nindent_style = space\nindent_size = 4\ninsert_final_newline = true\ntrim_trailing_whitespace = true\n\n[*.md]\ntrim_trailing_whitespace = false\n\n[*.yml]\nindent_size = 2\n\n[_cachefilecontent]\ninsert_final_newline = false\n" }, { "path": ".gitattributes", "content": ".github export-ignore\nbin/add-git-hooks export-ignore\ngit-hooks export-ignore\ntests export-ignore\n.editorconfig export-ignore\n.gitattributes export-ignore\n.gitignore export-ignore\n.php-cs-fixer.php export-ignore\nphpstan.neon export-ignore\nphpunit.xml export-ignore\n" }, { "path": ".github/workflows/ci.yml", "content": "name: CI\n\non: pull_request\n\njobs:\n tests:\n name: PestPHP Tests\n runs-on: ubuntu-latest\n strategy:\n matrix:\n php-versions: ['8.1', '8.2', '8.3', '8.4', '8.5']\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n\n - name: Install PHP\n uses: shivammathur/setup-php@v2\n with:\n php-version: ${{ matrix.php-versions }}\n\n - name: Install dependencies\n run: composer install --prefer-dist --no-progress\n\n - name: Run tests\n run: composer test\n\n - name: Run integration tests\n run: composer test-integration\n\n tests84:\n name: PestPHP Tests Running only on PHP >= 8.4\n runs-on: ubuntu-latest\n strategy:\n matrix:\n php-versions: ['8.4', '8.5']\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n\n - name: Install PHP\n uses: shivammathur/setup-php@v2\n with:\n php-version: ${{ matrix.php-versions }}\n\n - name: Install dependencies\n run: composer install --prefer-dist --no-progress\n\n - name: Run tests\n run: composer test-php84\n\n stanAndCs:\n name: Static Analysis (phpstan) and Code Style (PHP CS Fixer)\n runs-on: ubuntu-latest\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n\n - name: Install PHP\n uses: shivammathur/setup-php@v2\n with:\n php-version: '8.1'\n coverage: none\n\n - name: Install dependencies\n run: composer install --prefer-dist --no-progress\n\n - name: Run PHPStan\n run: composer stan\n\n - name: Run PHP CS Fixer\n run: composer cs\n" }, { "path": ".gitignore", "content": "composer.lock\nvendor\n.php_cs.cache\n.php-cs-fixer.cache\n.phpunit.result.cache\n.phpunit.cache\n/cachedir\n/storedir\n/tests/_Temp/_cachedir/*\n!/tests/_Temp/_cachedir/.gitkeep\n" }, { "path": ".php-cs-fixer.php", "content": "exclude(['tests/_Integration/_Server', '.github', 'bin', 'git-hooks'])\n ->in(__DIR__);\n\nreturn (new Config())\n ->setFinder($finder)\n ->setParallelConfig(ParallelConfigFactory::detect())\n ->setRules([\n '@PER-CS' => true,\n 'strict_param' => true,\n 'array_syntax' => ['syntax' => 'short'],\n 'no_unused_imports' => true,\n 'operator_linebreak' => ['only_booleans' => true, 'position' => 'end'],\n ])\n ->setRiskyAllowed(true)\n ->setUsingCache(true);\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\nAll notable changes to this project will be documented in this file.\n\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),\nand this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n\n## [Unreleased]\n\n## [3.5.6] - 2026-01-05\n### Fixed\n* Potential issues found with PHPStan 2 on level 8.\n\n## [3.5.5] - 2025-08-05\n### Fixed\n* Removed the overriding `validateAndSanitizeInput()` method from the `Paginate` HTTP step to ensure features like `staticUrl()` and `useInputKeyAsUrl()` work correctly.\n* The `Paginate` HTTP step now also supports receiving an array of URLs, initiating pagination separately for each one.\n\n### Deprecated\n* The `Crwlr\\Crawler\\Steps\\Loading\\Http\\Paginate` class. It shall be removed and its behavior implemented in the `Http` class directly, in the next major version.\n\n## [3.5.4] - 2025-07-28\n### Fixed\n* An issue in the `SimpleWebsitePaginator` when used with stop rules.\n\n## [3.5.3] - 2025-06-10\n### Fixed\n* Issues with passing cookies from the cookie jar to the headless browser when using the `useBrowser()` method on `Http` steps, in cases where the loader wasn’t globally configured to use the browser for all requests.\n\n## [3.5.2] - 2025-05-16\n### Fixed\n* The `Result::toArray()` method now converts all objects contained in the Result array (at any level of the array) to arrays. Also, if the only element in a result array has some autogenerated key containing \"unnamed\", but the value also is an associative array with string keys, the method only returns that child array.\n\n## [3.5.1] - 2025-04-23\n### Fixed\n* An issue that occurred, when a step uses the `PreStepInvocationLogger`. As refiners also use the logger, a newer logger (replacing the `PreStepInvocationLogger`) is now also passed to all registered refiners of a step.\n* Enable applying refiners to output properties with array value. E.g. if a step outputs an array of URLs (`['https://...', 'https://...']`), a `UrlRefiner` will be applied to all those URLs.\n\n## [3.5.0] - 2025-04-10\n### Added\n* Dynamically building request URLs from extracted data: `Http` steps now have a new `staticUrl()` method, and you can also use variables within that static URL - as well as in request headers and the body - like `https://www.example.com/foo/[crwl:some_extracted_property]`. These variables will be replaced with the corresponding properties from input data (also works with kept data).\n* New Refiners:\n * `DateTimeRefiner::reformat('Y-m-d H:i:s')` to reformat a date time string to a different format. Tries to automatically recognize the input format. If this does not work, you can provide an input format to use as the second argument.\n * `HtmlRefiner::remove('#foo')` to remove nodes matching the given selector from selected HTML.\n* Steps that produce multiple outputs per input can now group them per input by calling the new `Step::oneOutputPerInput()` method.\n\n## [3.4.5] - 2025-04-09\n### Fixed\n* When feeding an `Http` step with a string that is not a valid URL (e.g. `https://`), the exception when trying to parse it as a URL is caught, and an error logged.\n\n## [3.4.4] - 2025-04-04\n### Fixed\n* As sometimes, XML parsing errors occur because of characters that aren't valid within XML documents, the library now catches XML parsing errors, tries to find and replace invalid characters (with transliterates or HTML entities) and retries parsing the document. Works best when you additionally install the `voku/portable-ascii` composer package.\n\n## [3.4.3] - 2025-04-03\n### Fixed\n* When providing an empty base selector to an `Html` step (`Html::each('')`, `Html::first('')`, `Html::last('')`), it won't fail with an error, but instead log a warning, that it most likely doesn't make sense.\n* The `Step::keep()` methods now also work when applied to child steps within a group step.\n\n## [3.4.2] - 2025-03-08\n### Fixed\n* Issue when using `Http::get()->useBrowser()->postBrowserNavigateHook()`. Previously in this case, when the loader is configured to use the HTTP client, the post browser navigate hook was actually not set because of an issue with the order, things happened internally.\n\n## [3.4.1] - 2025-03-08\n### Fixed\n* Since, when using the Chrome browser for loading, we can only execute GET requests:\n * The loader now automatically switches to the HTTP client for POST, PUT, PATCH, and DELETE requests and logs a warning.\n * A warning is logged when attempting to use \"Post Browser Navigate Hooks\" with POST, PUT, PATCH, or DELETE requests.\n * Consequently, the `useBrowser()` method, introduced in v3.4.0, is also limited to GET requests.\n\n## [3.4.0] - 2025-03-06\n### Added\n* Two new methods to the base class of all `Http` steps:\n * `skipCache()` – Allows using the cache while skipping it for a specific loading step.\n * `useBrowser()` – Switches the loader to use a (headless) Chrome browser for loading calls in a specific step and then reverts the loader to its previous setting.\n* Introduced the new `BrowserAction::screenshot()` post browser navigate hook. It accepts an instance of the new `ScreenshotConfig` class, allowing you to configure various options (see the methods of `ScreenshotConfig`). If successful, the screenshot file paths are included in the `RespondedRequest` output object of the `Http` step.\n\n## [3.3.0] - 2025-03-02\n### Added\n* New `BrowserAction`s to use with the `postBrowserNavigateHook()` method: \n * `BrowserAction::clickInsideShadowDom()`\n * `BrowserAction::moveMouseToElement()`\n * `BrowserAction::moveMouseToPosition()`\n * `BrowserAction::scrollDown()`\n * `BrowserAction::scrollUp()`\n * `BrowserAction::typeText()`\n * `BrowserAction::waitForReload()`\n* A new method in `HeadlessBrowserLoaderHelper` to include the HTML content of shadow DOM elements in the returned HTML. Use it like this: `$crawler->getLoader()->browser()->includeShadowElementsInHtml()`.\n\n### Changed\n* The `BrowserAction::clickElement()` action, now automatically waits for an element matching the selector to be rendered, before performing the click. This means you don't need to put a `BrowserAction::waitUntilDocumentContainsElement()` before it. It works the same in the new `BrowserAction::clickInsideShadowDom()` and `BrowserAction::moveMouseToElement()` actions.\n\n### Deprecated\n* `BrowserAction::clickElementAndWaitForReload()` and `BrowserAction::evaluateAndWaitForReload()`. As a replacement, please use `BrowserAction::clickElement()` or `BrowserAction::evaluate()` and `BrowserAction::waitForReload()` separately.\n\n## [3.2.5] - 2025-02-26\n### Fixed\n* When a child step is nested in the `extract()` method of an `Html` or `Xml` step, and does not use `each()` as the base, the extracted value is an array with the keys defined in the `extract()` call, rather than an array of such arrays as it would be with `each()` as base.\n\n## [3.2.4] - 2025-02-25\n### Fixed\n* Trying to load a relative reference URI (no scheme and host/authority, only path) via the `HttpLoader` now immediately logs (or throws when `loadOrFail()` is used) an error instead of trying to actually load it.\n\n## [3.2.3] - 2025-01-28\n### Fixed\n* Fix deprecation warning triggered in the `DomQuery` class, when trying to get the value of an HTML/XML attribute that does not exist on the element.\n\n## [3.2.2] - 2025-01-17\n### Fixed\n* Warnings about loader hooks being called multiple times, when using a `BotUserAgent` and therefore loading and respecting the robots.txt file, or when using the `Http::stopOnErrorResponse()` method.\n\n## [3.2.1] - 2025-01-13\n### Fixed\n* Reuse previously opened page when using the (headless) Chrome browser, instead of opening a new page for each request.\n\n## [3.2.0] - 2025-01-12\n### Added\n* `RespondedRequest::isServedFromCache()` to determine whether a response was served from cache or actually loaded.\n\n## [3.1.5] - 2025-01-10\n### Fixed\n* Another improvement for getting XML source when using the browser, in cases where Chrome doesn't identify the response as an XML document (even though a Content-Type header is sent).\n\n## [3.1.4] - 2025-01-10\n### Fixed\n* `HttpLoader::dontUseCookies()` now also works when using the Chrome browser. Cookies are cleared before every request.\n\n## [3.1.3] - 2025-01-10\n### Fixed\n* Further improve getting the raw response body from non-HTML documents via Chrome browser.\n\n## [3.1.2] - 2025-01-08\n### Fixed\n* When loading a non-HTML document (e.g., XML) via the Chrome browser, the library now retrieves the original source. Previously, it returned the outerHTML of the rendered document, which wrapped the content in an HTML structure.\n\n## [3.1.1] - 2025-01-07\n### Fixed\n* When the `validateAndSanitize()` method of a step throws an `InvalidArgumentException`, the exception is now caught, logged and the step is not invoked with the invalid input. This improves fault tolerance. Feeding a step with one invalid input shouldn't cause the whole crawler run to fail. Exceptions other than `InvalidArgumentException` remain uncaught.\n\n## [3.1.0] - 2025-01-03\n### Added\n* New method `HeadlessBrowserLoaderHelper::setPageInitScript()` (`$crawler->getLoader()->browser()->setPageInitScript()`) to provide javascript code that is executed on every new browser page before navigating anywhere.\n* New method `HeadlessBrowserLoaderHelper::useNativeUserAgent()` (`$crawler->getLoader()->browser()->useNativeUserAgent()`) to allow using the native `User-Agent` that your Chrome browser sends by default.\n\n## [3.0.4] - 2024-12-18\n### Fixed\n* Minor improvement for the `DomQuery` (base for `Dom::cssSelector()` and `Dom::xPath()`): enable providing an empty string as selector, to simply get the node that the selector is applied to.\n\n## [3.0.3] - 2024-12-11\n### Fixed\n* Improved fix for non UTF-8 characters in HTML documents declared as UTF-8.\n\n## [3.0.2] - 2024-12-11\n### Fixed\n* When the new PHP 8.4 DOM API is used, and HTML declared as UTF-8 contains non UTF-8 compatible characters, it does not replace them with a � character, but instead removes it. This behaviour is consistent with the data returned by Symfony DomCrawler.\n\n## [3.0.1] - 2024-12-10\n### Undeprecated\n* Removed deprecations for all XPath functionality (`Dom::xPath()`, `XPathQuery` class and `Node::queryXPath()`), because it's still available with the net DOM API in PHP 8.4.\n\n## [3.0.0] - 2024-12-08\nThe primary change in version 3.0.0 is that the library now leverages PHP 8.4’s new DOM API when used in an environment with PHP >= 8.4. To maintain compatibility with PHP < 8.4, an abstraction layer has been implemented. This layer dynamically uses either the Symfony DomCrawler component or the new DOM API, depending on the PHP version.\n\nSince no direct interaction with an instance of the Symfony DomCrawler library was required at the step level provided by the library, it is highly likely that you won’t need to make any changes to your code to upgrade to v3. To ensure a smooth transition, please review the points under “Changed.”\n\n### Changed\n* __BREAKING__: The `DomQuery::innerText()` method (a.k.a. `Dom::cssSelector('...')->innerText()`) has been removed. `innerText` exists only in the Symfony DomCrawler component, and its usefulness is questionable. If you still require this variant of the DOM element text, please let us know or create a pull request yourself. Thank you!\n* __BREAKING__: The `DomQueryInterface` was removed. As the `DomQuery` class offers a lot more functionality than the interface defines, the purpose of the interface was questionable. Please use the abstract `DomQuery` class instead. This also means that some method signatures, type hinting the interface, have changed. Look for occurrences of `DomQueryInterface` and replace them.\n* __BREAKING__: The visibility of the `DomQuery::filter()` method was changed from public to protected. It is still needed in the `DomQuery` class, but outside of it, it is probably better and easier to directly use the new DOM abstraction (see the `src/Steps/Dom` directory). If you are extending the `DomQuery` class (which is not recommended), be aware that the argument now takes a `Node` (from the new DOM abstraction) instead of a Symfony `Crawler`.\n* __BREAKING__: The `Step::validateAndSanitizeToDomCrawlerInstance()` method was removed. Please use the `Step::validateAndSanitizeToHtmlDocumentInstance()` and `Step::validateAndSanitizeToXmlDocumentInstance()` methods instead.\n* __BREAKING__: The second argument in `Closure`s passed to the `Http::crawl()->customFilter()` has changed from an instance of Symfony `Crawler` class, to an `HtmlElement` instance from the new DOM abstraction (`Crwlr\\Crawler\\Steps\\Dom\\HtmlElement`).\n* __BREAKING__: The Filter class was split into `AbstractFilter` (base class for actual filter classes) and `Filter` only hosting the static function for easy instantiation, because otherwise each filter class also has all the static methods.\n* __BREAKING__: Further, the signatures of some methods that are mainly here for internal usage, have changed due to the new DOM abstraction:\n * The static `GetLink::isSpecialNonHttpLink()` method now needs an instance of `HtmlElement` instead of a Symfony `Crawler`.\n * `GetUrlsFromSitemap::fixUrlSetTag()` now takes an `XmlDocument` instead of a Symfony `Crawler`.\n * The `DomQuery::apply()` method now takes a `Node` instead of a Symfony `Crawler`.\n\n### Deprecated\n* `Dom::xPath()` method and\n* the `XPathQuery` class as well as\n* the new `Node::queryXPath()` method.\n\n### Added\n* New step output filter `Filter::arrayHasElement()`. When a step produces array output with a property being a numeric array, you can now filter outputs by checking if one element of that array property, matches certain filter criteria. Example: The outputs look like `['foo' => 'bar', 'baz' => ['one', 'two', 'three']]`. You can filter all outputs where `baz` contains `two` like: `Filter::arrayHasElement()->where('baz', Filter::equal('two'))`.\n\n## [2.1.3] - 2024-11-05\n### Fixed\n* Improvements for deprecations in PHP 8.4.\n\n## [2.1.2] - 2024-10-22\n### Fixed\n* Issue when converting cookie objects received from the chrome-php library.\n\n## [2.1.1] - 2024-10-21\n### Fixed\n* Also add cookies, set during headless browser usage, to the cookie jar. When switching back to the (guzzle) HTTP client the cookies should also be sent.\n* Don't call `Loader::afterLoad()` when `Loader::beforeLoad()` was not called before. This can potentially happen, when an exception is thrown before the call to the `beforeLoad` hook, but it is caught and the `afterLoader` hook method is called anyway. As this most likely won't make sense to users, the `afterLoad` hook callback functions will just not be called in this case.\n* The `Throttler` class now has protected methods `_internalTrackStartFor()`, `_requestToUrlWasStarted()` and `_internalTrackEndFor()`. When extending the `Throttler` class (be careful, actually that's not really recommended) they can be used to check if a request to a URL was actually started before.\n\n## [2.1.0] - 2024-10-19\n### Added\n* The new `postBrowserNavigateHook()` method in the `Http` step classes, which allows to define callback functions that are triggered after the headless browser navigated to the specified URL. They are called with the chrome-php `Page` object as argument, so you can interact with the page. Also, there is a new class `BrowserAction` providing some simple actions (like wait for element, click element,...) as Closures via static methods. You can use it like `Http::get()->postBrowserNavigateHook(BrowserAction::clickElement('#element'))`.\n\n## [2.0.1] - 2024-10-15\n### Fixed\n* Issue with the `afterLoad` hook of the `HttpLoader`, introduced in v2. Calling the hook was commented out, which slipped through because the test case was faulty.\n\n## [2.0.0] - 2024-10-15\n### Changed\n* __BREAKING__: Removed methods `BaseStep::addToResult()`, `BaseStep::addLaterToResult()`, `BaseStep::addsToOrCreatesResult()`, `BaseStep::createsResult()`, and `BaseStep::keepInputData()`. These methods were deprecated in v1.8.0 and should be replaced with `Step::keep()`, `Step::keepAs()`, `Step::keepFromInput()`, and `Step::keepInputAs()`.\n* __BREAKING__: Added the following keep methods to the `StepInterface`: `StepInterface::keep()`, `StepInterface::keepAs()`, `StepInterface::keepFromInput()`, `StepInterface::keepInputAs()`, as well as `StepInterface::keepsAnything()`, `StepInterface::keepsAnythingFromInputData()` and `StepInterface::keepsAnythingFromOutputData()`. If you have a class that implements this interface without extending `Step` (or `BaseStep`), you will need to implement these methods yourself. However, it is strongly recommended to extend `Step` instead.\n* __BREAKING__: With the removal of the `addToResult()` method, the library no longer uses `toArrayForAddToResult()` methods on output objects. Instead, please use `toArrayForResult()`. Consequently, `RespondedRequest::toArrayForAddToResult()` has been renamed to `RespondedRequest::toArrayForResult()`.\n* __BREAKING__: Removed the `result` and `addLaterToResult` properties from `Io` objects (`Input` and `Output`). These properties were part of the `addToResult` feature and are now removed. Instead, use the `keep` property where kept data is added.\n* __BREAKING__: The signature of the `Crawler::addStep()` method has changed. You can no longer provide a result key as the first parameter. Previously, this key was passed to the `Step::addToResult()` method internally. Now, please handle this call yourself.\n* __BREAKING__: The return type of the `Crawler::loader()` method no longer allows `array`. This means it's no longer possible to provide multiple loaders from the crawler. Instead, use the new functionality to directly provide a custom loader to a step described below. As part of this change, the `UnknownLoaderKeyException` was also removed as it is now obsolete. If you have any references to this class, please make sure to remove them.\n* __BREAKING__: Refactored the abstract `LoadingStep` class to a trait and removed the `LoadingStepInterface`. Loading steps should now extend the `Step` class and use the trait. As multiple loaders are no longer supported, the `addLoader` method was renamed to `setLoader`. Similarly, the methods `useLoader()` and `usesLoader()` for selecting loaders by key are removed. Now, you can directly provide a different loader to a single step using the trait's new `withLoader()` method (e.g., `Http::get()->withLoader($loader)`). The trait now also uses phpdoc template tags, for a generic loader type. You can define the loader type by putting `/** @use LoadingStep */` above `use LoadingStep;` in your step class. Then your IDE and static analysis (if supported) will know what type of loader, the trait methods return and accept.\n* __BREAKING__: Removed the `PaginatorInterface` to allow for better extensibility. The old `Crwlr\\Crawler\\Steps\\Loading\\Http\\Paginators\\AbstractPaginator` class has also been removed. Please use the newer, improved version `Crwlr\\Crawler\\Steps\\Loading\\Http\\AbstractPaginator`. This newer version has also changed: the first argument `UriInterface $url` is removed from the `processLoaded()` method, as the URL also is part of the request (`Psr\\Http\\Message\\RequestInterface`) which is now the first argument. Additionally, the default implementation of the `getNextRequest()` method is removed. Child implementations must define this method themselves. If your custom paginator still has a `getNextUrl()` method, note that it is no longer needed by the library and will not be called. The `getNextRequest()` method now fulfills its original purpose.\n* __BREAKING__: Removed methods from `HttpLoader`:\n * `$loader->setHeadlessBrowserOptions()` => use `$loader->browser()->setOptions()` instead\n * `$loader->addHeadlessBrowserOptions()` => use `$loader->browser()->addOptions()` instead\n * `$loader->setChromeExecutable()` => use `$loader->browser()->setExecutable()` instead\n * `$loader->browserHelper()` => use `$loader->browser()` instead\n* __BREAKING__: Removed method `RespondedRequest::cacheKeyFromRequest()`. Use `RequestKey::from()` instead.\n* __BREAKING__: The `HttpLoader::retryCachedErrorResponses()` method now returns an instance of the new `Crwlr\\Crawler\\Loader\\Http\\Cache\\RetryManager` class. This class provides the methods `only()` and `except()` to restrict retries to specific HTTP response status codes. Previously, this method returned the `HttpLoader` itself (`$this`), so if you're using it in a chain and calling other loader methods after it, you will need to refactor your code.\n* __BREAKING__: Removed the `Microseconds` class from this package. It has been moved to the `crwlr/utils` package, which you can use instead.\n\n### Added\n* New methods `FileCache::prolong()` and `FileCache::prolongAll()` to allow prolonging the time to live for cached responses.\n\n### Fixed\n* The `maxOutputs()` method is now also available and working on `Group` steps.\n* Improved warning messages for step validations that are happening before running a crawler.\n* A `PreRunValidationException` when the crawler finds a problem with the setup, before actually running, is not only logged as an error via the logger, but also rethrown to the user. This way the user won't get the impression, that the crawler ran successfully without looking at the log messages.\n\n## [1.10.0] - 2024-08-05\n### Added\n* URL refiners: `UrlRefiner::withScheme()`, `UrlRefiner::withHost()`, `UrlRefiner::withPort()`, `UrlRefiner::withoutPort()`, `UrlRefiner::withPath()`, `UrlRefiner::withQuery()`, `UrlRefiner::withoutQuery()`, `UrlRefiner::withFragment()` and `UrlRefiner::withoutFragment()`.\n* New paginator stop rules `PaginatorStopRules::contains()` and `PaginatorStopRules::notContains()`.\n* Static method `UserAgent::mozilla5CompatibleBrowser()` to get a `UserAgent` instance with the user agent string `Mozilla/5.0 (compatible)` and also the new method `withMozilla5CompatibleUserAgent` in the `AnonymousHttpCrawlerBuilder` that you can use like this: `HttpCrawler::make()->withMozilla5CompatibleUserAgent()`.\n\n## [1.9.5] - 2024-07-25\n### Fixed\n* Prevent PHP warnings when an HTTP response includes a `Content-Type: application/x-gzip` header, but the content is not actually compressed. This issue also occurred with cached responses, because compressed content is decoded during caching. Upon retrieval from the cache, the header indicated compression, but the content was already decoded.\n\n## [1.9.4] - 2024-07-24\n### Fixed\n* When using `HttpLoader::cacheOnlyWhereUrl()` to restrict caching, the filter rule is not only applied when adding newly loaded responses to the cache, but also for using cached responses. Example: a response for `https://www.example.com/foo` is already available in the cache, but `$loader->cacheOnlyWhereUrl(Filter::urlPathStartsWith('/bar/'))` was called, the cached response is not used.\n\n## [1.9.3] - 2024-07-05\n### Fixed\n* Add `HttpLoader::browser()` as a replacement for `HttpLoader::browserHelper()` and deprecate the `browserHelper()` method. It's an alias and just because it will read a little better: `$loader->browser()->xyz()` vs. `$loader->browserHelper()->xyz()`. `HttpLoader::browserHelper()` will be removed in v2.0.\n* Also deprecate `HttpLoader::setHeadlessBrowserOptions()`, `HttpLoader::addHeadlessBrowserOptions()` and `HttpLoader::setChromeExecutable()`. Use `$loader->browser()->setOptions()`, `$loader->browser()->addOptions()` and `$loader->browser()->setExecutable()` instead.\n\n## [1.9.2] - 2024-06-18\n### Fixed\n* Issue with setting the headless chrome executable, introduced in 1.9.0. \n\n## [1.9.1] - 2024-06-17\n### Added\n* Also add `HeadlessBrowserLoaderHelper::getTimeout()` to get the currently configured timeout value.\n\n## [1.9.0] - 2024-06-17\n### Added\n* New methods `HeadlessBrowserLoaderHelper::setTimeout()` and `HeadlessBrowserLoaderHelper::waitForNavigationEvent()` to allow defining the timeout for the headless chrome in milliseconds (default 30000 = 30 seconds) and the navigation event (`load` (default), `DOMContentLoaded`, `firstMeaningfulPaint`, `networkIdle`, etc.) to wait for when loading a URL.\n\n## [1.8.0] - 2024-06-05\n### Added\n* New methods `Step::keep()` and `Step::keepAs()`, as well as `Step::keepFromInput()` and `Step::keepInputAs()`, as alternatives to `Step::addToResult()` (or `Step::addLaterToResult()`). The `keep()` method can be called without any argument, to keep all from the output data. It can be called with a string, to keep a certain key or with an array to keep a list of keys. If the step yields scalar value outputs (not an associative array or object with keys) you need to use the `keepAs()` method with the key you want the output value to have in the kept data. The methods `keepFromInput()` and `keepInputAs()` work the same, but uses the input (not the output) that the step receives. Most likely only needed with a first step, to keep data from initial inputs (or in a sub crawler, see below). Kept properties can also be accessed with the `Step::useInputKey()` method, so you can easily reuse properties from multiple steps ago as input.\n* New method `Step::outputType()` with default implementation returning `StepOutputType::Mixed`. Please consider implementing this method yourself in all your custom steps, because it is going to be required in v2 of the library. It allows detecting (potential) problems in crawling procedures immediately when starting a run instead of failing after already running a while.\n* New method `Step::subCrawlerFor()`, allowing to fill output properties from an actual full child crawling procedure. As the first argument, you give it a key from the step's output, that the child crawler uses as input(s). As the second argument you need to provide a `Closure` that receives a clone of the current `Crawler` without steps and with initial inputs, set from the current output. In the `Closure` you then define the crawling procedure by adding steps as you're used to do it, and return it. This allows to achieve nested output data, scraped from different (sub-)pages, more flexible and less complicated as with the usual linear crawling procedure and `Step::addToResult()`.\n\n### Deprecated\n* The `Step::addToResult()`, `Step::addLaterToResult()` and `Step::keepInputData()` methods. Instead, please use the new keep methods. This can cause some migration work for v2, because especially the add to result methods are a pretty central functionality, but the new \"keep\" methodology (plus the new sub crawler feature) will make a lot of things easier, less complex and the library will most likely work more efficiently in v2.\n\n### Fixed\n* When a cache file was generated with compression, and you're trying to read it with a `FileCache` instance without compression enabled, it also works. When unserializing the file content fails it tries decoding the string first before unserializing it.\n\n## [1.7.2] - 2024-03-19\n### Fixed\n* When the `useInputKey()` method is used on a step and the defined key does not exist in input, it logs a warning and does not invoke the step instead of throwing an `Exception`.\n\n## [1.7.1] - 2024-03-11\n### Fixed\n* A PHP error that happened when the loader returns `null` for the initial request in the `Http::crawl()` step.\n\n## [1.7.0] - 2024-03-04\n### Added\n* Allow getting the whole decoded JSON as array with the new `Json::all()` and also allow to get the whole decoded JSON, when using `Json::get()`, inside a mapping using either empty string or `*` as target. Example: `Json::get(['all' => '*'])`. `*` only works, when there is no key `*` in the decoded data.\n\n### Fixed\n* Make it work with responses loaded by a headless browser. If decoding the input string fails, it now checks if it could be HTML. If that's the case, it extracts the text content of the `` and tries to decode this instead.\n\n## [1.6.2] - 2024-02-26\n### Fixed\n* When using `HttpLoader::cacheOnlyWhereUrl()` and a request was redirected (maybe even multiple times), previously all URLs in the chain had to match the filter rule. As this isn't really practicable, now only one of the URLs has to match the rule.\n\n## [1.6.1] - 2024-02-16\n### Changed\n* Make method `HttpLoader::addToCache()` public, so steps can update a cached response with an extended version.\n\n## [1.6.0] - 2024-02-13\n### Added\n* Enable dot notation in `Step::addToResult()`, so you can get data from nested output, like: `$step->addToResult(['url' => 'response.url', 'status' => 'response.status', 'foo' => 'bar'])`.\n* When a step adds output properties to the result, and the output contains objects, it tries to serialize those objects to arrays, by calling `__serialize()`. If you want an object to be serialized differently for that purpose, you can define a `toArrayForAddToResult()` method in that class. When that method exists, it's preferred to the `__serialize()` method.\n* Implemented above-mentioned `toArrayForAddToResult()` method in the `RespondedRequest` class, so on every step that somehow yields a `RespondedRequest` object, you can use the keys `url`, `uri`, `status`, `headers` and `body` with the `addToResult()` method. Previously this only worked for `Http` steps, because it defines output key aliases (`HttpBase::outputKeyAliases()`). Now, in combination with the ability to use dot notation when adding data to the result, if your custom step returns nested output like `['response' => RespondedRequest, 'foo' => 'bar']`, you can add response data to the result like this `$step->addToResult(['url' => 'response.url', 'body' => 'response.body'])`.\n\n### Fixed\n* Improvement regarding the timing when a store (`Store` class instance) is called by the crawler with a final crawling result. When a crawling step initiates a crawling result (so, `addToResult()` was called on the step instance), the crawler has to wait for all child outputs (resulting from one step-input) until it calls the store, because the child outputs can all add data to the same final result object. But previously this was not only the case for all child outputs starting from a step where `addToResult()` was called, but all children of one initial crawler input. So with this change, in a lot of cases, the store will earlier be called with finished `Result` objects and memory usage will be lowered.\n\n## [1.5.3] - 2024-02-07\n### Fixed\n* Merge `HttpBaseLoader` back to `HttpLoader`. It's probably not a good idea to have multiple loaders. At least not multiple loaders just for HTTP. It should be enough to publicly expose the `HeadlessBrowserLoaderHelper` via `HttpLoader::browserHelper()` for the extension steps. But keep the `HttpBase` step, to share the general HTTP functionality implemented there.\n\n## [1.5.2] - 2024-02-07\n### Fixed\n* Issue in `GetUrlsFromSitemap` (`Sitemap::getUrlsFromSitemap()`) step when XML content has no line breaks.\n\n## [1.5.1] - 2024-02-06\n### Fixed\n* For being more flexible to build a separate headless browser loader (in an extension package) extract the most basic HTTP loader functionality to a new `HttpBaseLoader` and important functionality for the headless browser loader to a new `HeadlessBrowserLoaderHelper`. Further, also share functionality from the `Http` steps via a new abstract `HttpBase` step. It's considered a fix, because there's no new functionality, just refactoring existing code for better extendability.\n\n## [1.5.0] - 2024-01-29\n### Added\n* The `DomQuery` class (parent of `CssSelector` (`Dom::cssSelector`) and `XPathQuery` (`Dom::xPath`)) has a new method `formattedText()` that uses the new crwlr/html-2-text package to convert the HTML to formatted plain text. You can also provide a customized instance of the `Html2Text` class to the `formattedText()` method.\n\n### Fixed\n* The `Http::crawl()` step won't yield a page again if a newly found URL responds with a redirect to a previously loaded URL.\n\n## [1.4.0] - 2024-01-14\n### Added\n* The `QueryParamsPaginator` can now also increase and decrease non first level query param values like `foo[bar][baz]=5` using dot notation: `QueryParamsPaginator::paramsInUrl()->increaseUsingDotNotation('foo.bar.baz', 5)`.\n\n## [1.3.5] - 2023-12-20\n### Fixed\n* The `FileCache` can now also read uncompressed cache files when compression is activated.\n\n## [1.3.4] - 2023-12-19\n### Fixed\n* Reset paginator state after finishing paginating for one base input, to enable paginating multiple listings of the same structure.\n\n## [1.3.3] - 2023-12-01\n### Fixed\n* Add forgotten getter method to get the DOM query that is attached to an `InvalidDomQueryException` instance.\n\n## [1.3.2] - 2023-12-01\n### Fixed\n* When creating a `CssSelector` or `XPathQuery` instance with invalid selector/query syntax, an `InvalidDomQueryException` is now immediately thrown. This change is considered to be not only non-breaking, but actually a fix, because the `CssSelector` would otherwise throw an exception later when the `apply()` method is called. The `XPathQuery` would silently return no result without notifying you of the invalid query and generate a PHP warning.\n\n## [1.3.1] - 2023-11-30\n### Fixed\n* Support usage with the new Symfony major version v7.\n\n## [1.3.0] - 2023-10-28\n### Added\n* New methods `HttpLoader::useProxy()` and `HttpLoader::useRotatingProxies([...])` to define proxies that the loader shall use. They can be used with a guzzle HTTP client instance (default) and when the loader uses the headless Chrome browser. Using them when providing some other PSR-18 implementation will throw an exception.\n* New `QueryParamsPaginator` to paginate by increasing and/or decreasing one or multiple query params, either in the URL or in the body of requests. Can be created via static method `Crwlr\\Crawler\\Steps\\Loading\\Http\\Paginator::queryParams()`.\n* New method `stopWhen` in the new `Crwlr\\Crawler\\Steps\\Loading\\Http\\AbstractPaginator` class (for more info see the deprecation below). You can pass implementations of the new `StopRule` interface or custom closures to that method and then, every time the Paginator receives a loaded response to process, those stop rules are called with the response. If any of the conditions of the stop rules is met, the Paginator stops paginating. Of course also added a few stop rules to use with that new method: `IsEmptyInHtml`, `IsEmptyInJson`, `IsEmptyInXml` and `IsEmptyResponse`, also available via static methods: `PaginatorStopRules::isEmptyInHtml()`, `PaginatorStopRules::isEmptyInJson()`, `PaginatorStopRules::isEmptyInXml()` and `PaginatorStopRules::isEmptyResponse()`.\n\n### Deprecated\n* Deprecated the `Crwlr\\Crawler\\Steps\\Loading\\Http\\PaginatorInterface` and the `Crwlr\\Crawler\\Steps\\Loading\\Http\\Paginators\\AbstractPaginator`. Instead, added a new version of the `AbstractPaginator` as `Crwlr\\Crawler\\Steps\\Loading\\Http\\AbstractPaginator` that can be used. Usually there shouldn't be a problem switching from the old to the new version. If you want to make your custom paginator implementation ready for v2 of the library, extend the new `AbstractPaginator` class, implement your own `getNextRequest` method (new requirement, with a default implementation in the abstract class, which will be removed in v2) and check if properties and methods of your existing class don't collide with the new properties and methods in the abstract class.\n\n### Fixed\n* The `HttpLoader::load()` implementation won't throw any exception, because it shouldn't kill a crawler run. When you want any loading error to end the whole crawler execution `HttpLoader::loadOrFail()` should be used. Also adapted the phpdoc in the `LoaderInterface`.\n\n## [1.2.2] - 2023-09-19\n### Fixed\n* Fix in `HttpCrawl` (`Http::crawl()`) step: when a page contains a broken link, that can't be resolved and throws an `Exception` from the URL library, ignore the link and log a warning message.\n* Minor fix for merging HTTP headers when an `Http` step gets both, statically defined headers and headers to use from array input.\n\n## [1.2.1] - 2023-08-21\n### Fixed\n* When a URL redirects, the `trackRequestEndFor()` method of the `HttpLoader`'s `Throttler` instance is called only once at the end and with the original request URL.\n\n## [1.2.0] - 2023-08-18\n### Added\n* New `onCacheHit` hook in the `Loader` class (in addition to `beforeLoad`, `onSuccess`, `onError` and `afterLoad`) that is called in the `HttpLoader` class when a response for a request was found in the cache.\n\n### Deprecated\n* Moved the `Microseconds` value object class to the crwlr/utils package, as it is a very useful and universal tool. The class in this package still exists, but just extends the class from the utils package and will be removed in v2. So, if you're using this class, please change to use the version from the utils package.\n\n## [1.1.6] - 2023-07-20\n### Fixed\n* Throttling now also works when using the headless browser.\n\n## [1.1.5] - 2023-07-14\n### Fixed\n* The `Http::crawl()` step, as well as the `Html::getLink()` and `Html::getLinks()` steps now ignore links, when the `href` attribute starts with `mailto:`, `tel:` or `javascript:`. For the crawl step it obviously makes no sense, but it's also considered a bugfix for the getLink(s) steps, because they are meant to deliver absolute HTTP URLs. If you want to get the values of such links, use the HTML data extraction step.\n\n## [1.1.4] - 2023-07-14\n### Fixed\n* The `Http::crawl()` step now also work with sitemaps as input URL, where the `` tag contains attributes that would cause the symfony DomCrawler to not find any elements.\n\n## [1.1.3] - 2023-06-29\n### Fixed\n* Improved `Json` step: if the target of the \"each\" (like `Json::each('target', [...])`) does not exist in the input JSON data, the step yields nothing and logs a warning.\n\n## [1.1.2] - 2023-05-28\n### Fixed\n* Using the `only()` method of the `MetaData` (`Html::metaData()`) step class, the `title` property was always contained in the output, even if not listed in the `only` properties. This is fixed now.\n\n## [1.1.1] - 2023-05-28\n### Fixed\n* There was an issue when adding multiple associative arrays with the same key to a `Result` object: let's say you're having a step producing array output like: `['bar' => 'something', 'baz' => 'something else']` and it (the whole array) shall be added to the result property `foo`. When the step produced multiple such array outputs, that led to a result like `['bar' => '...', 'baz' => '...', ['bar' => '...', 'baz' => '...'], ['bar' => '...', 'baz' => '...']`. Now it's fixed to result in `[['bar' => '...', 'baz' => '...'], ['bar' => '...', 'baz' => '...'], ['bar' => '...', 'baz' => '...']`.\n\n## [1.1.0] - 2023-05-21\n\n### Added\n* `Http` steps can now receive body and headers from input data (instead of statically defining them via argument like `Http::method(headers: ...)`) using the new methods `useInputKeyAsBody()` and `useInputKeyAsHeader(, )` or `useInputKeyAsHeaders()`. Further, when invoked with associative array input data, the step will by default use the value from `url` or `uri` for the request URL. If the input array contains the URL in a key with a different name, you can use the new `useInputKeyAsUrl()` method. That was basically already possible with the existing `useInputKey()` method, because the URL is the main input argument for the step. But if you want to use it in combination with the other new `useInputKeyAsXyz()` methods, you have to use `useInputKeyAsUrl()`, because using `useInputKey()` would invoke the whole step with that key only.\n* `Crawler::runAndDump()` as a simple way to just run a crawler and dump all results, each as an array.\n* `addToResult()` now also works with serializable objects.\n* If you know certain keys that the output of a step will contain, you can now also define aliases for those keys, to be used with `addToResult()`. The output of an `Http` step (`RespondedRequest`) contains the keys `requestUri` and `effectiveUri`. The aliases `url` and `uri` refer to `effectiveUri`, so `addToResult(['url'])` will add the `effectiveUri` as `url` to the result object.\n* The `GetLink` (`Html::getLink()`) and `GetLinks` (`Html::getLinks()`) steps, as well as the abstract `DomQuery` (parent of `CssSelector` (/`Dom::cssSelector`) and `XPathQuery` (/`Dom::xPath`)) now have a method `withoutFragment()` to get links respectively URLs without their fragment part.\n* The `HttpCrawl` step (`Http::crawl()`) has a new method `useCanonicalLinks()`. If you call it, the step will not yield responses if its canonical link URL was already yielded. And if it discovers a link, and some document pointing to that URL via canonical link was already loaded, it treats it as if it was already loaded. Further this feature also sets the canonical link URL as the `effectiveUri` of the response.\n* All filters can now be negated by calling the `negate()` method, so the `evaluate()` method will return the opposite bool value when called. The `negate()` method returns an instance of `NegatedFilter` that wraps the original filter.\n* New method `cacheOnlyWhereUrl()` in the `HttpLoader` class, that takes an instance of the `FilterInterface` as argument. If you define one or multiple filters using this method, the loader will cache only responses for URLs that match all the filters.\n\n### Fixed\n* The `HttpCrawl` step (`Http::crawl()`) by default now removes the fragment part of URLs to not load the same page multiple times, because in almost any case, servers won't respond with different content based on the fragment. That's why this change is considered non-breaking. For the rare cases when servers respond with different content based on the fragment, you can call the new `keepUrlFragment()` method of the step.\n* Although the `HttpCrawl` step (`Http::crawl()`) already respected the limit of outputs defined via the `maxOutputs()` method, it actually didn't stop loading pages. The limit had no effect on loading, only on passing on outputs (responses) to the next step. This is fixed in this version.\n* A so-called byte order mark at the beginning of a file (/string) can cause issues. So just remove it, when a step's input string starts with a UTF-8 BOM.\n* There seems to be an issue in guzzle when it gets a PSR-7 request object with a header with multiple string values (as array, like: `['accept-encoding' => ['gzip', 'deflate', 'br']]`). When testing it happened that it only sent the last part (in this case `br`). Therefore, the `HttpLoader` now prepares headers before sending (in this case to: `['accept-encoding' => ['gzip, deflate, br']]`).\n* You can now also use the output key aliases when filtering step outputs. You can even use keys that are only present in the serialized version of an output object.\n\n## [1.0.2] - 2023-03-20\n### Fixed\n* JSON step: another fix for JSON strings having keys without quotes with empty string value.\n\n## [1.0.1] - 2023-03-17\n### Fixed\n* JSON step: improve attempt to fix JSON string having keys without quotes.\n\n## [1.0.0] - 2023-02-08\n\n### Added\n* New method `Step::refineOutput()` to manually refine step output values. It takes either a `Closure` or an instance of the new `RefinerInterface` as argument. If the step produces array output, you can provide a key from the array output, to refine, as first argument and the refiner as second argument. You can call the method multiple times and all the refiners will be applied to the outputs in the order you add them. If you want to refine multiple output array keys with a `Closure`, you can skip providing a key and the `Closure` will receive the full output array for refinement. As mentioned you can provide an instance of the `RefinerInterface`. There are already a few implementations: `StringRefiner::afterFirst()`, `StringRefiner::afterLast()`, `StringRefiner::beforeFirst()`, `StringRefiner::beforeLast()`, `StringRefiner::betweenFirst()`, `StringRefiner::betweenLast()` and `StringRefiner::replace()`.\n* New method `Step::excludeFromGroupOutput()` to exclude a normal steps output from the combined output of a group that it's part of.\n* New method `HttpLoader::setMaxRedirects()` to customize the limit of redirects to follow. Works only when using the HTTP client.\n* New filters to filter by string length, with the same options as the comparison filters (equal, not equal, greater than,...).\n* New `Filter::custom()` that you can use with a Closure, so you're not limited to the available filters only.\n* New method `DomQuery::link()` as a shortcut for `DomQuery::attribute('href')->toAbsoluteUrl()`.\n* New static method `HttpCrawler::make()` returning an instance of the new class `AnonymousHttpCrawlerBuilder`. This makes it possible to create your own Crawler instance with a one-liner like: `HttpCrawler::make()->withBotUserAgent('MyCrawler')`. There's also a `withUserAgent()` method to create an instance with a normal (non bot) user agent.\n\n### Changed\n* __BREAKING__: The `FileCache` now also respects the `ttl` (time to live) argument and by default it is one hour (3600 seconds). If you're using the cache and expect the items to live (basically) forever, please provide a high enough value for default the time to live. When you try to get a cache item that is already expired, it (the file) is immediately deleted.\n* __BREAKING__: The `TooManyRequestsHandler` (and with that also the constructor argument in the `HttpLoader`) was renamed to `RetryErrorResponseHandler`. It now reacts the same to 503 (Service Unavailable) responses as to the 429 (Too Many Requests) responses. If you're actively passing your own instance to the `HttpLoader`, you need to update it.\n* You can now have multiple different loaders in a `Crawler`. To use this, return an array containing your loaders from the protected `Crawler::loader()` method with keys to name them. You can then selectively use them by calling the `Step::useLoader()` method on a loading step with the key of the loader it should use.\n\n### Removed\n* __BREAKING__: The loop feature. The only real world use case should be paginating listings and this should be solved with the Paginator feature.\n* __BREAKING__: `Step::dontCascade()` and `Step::cascades()` because with the change in v0.7, that groups can only produce combined output, there should be no use case for this anymore. If you want to exclude one steps output from the combined group output, you can use the new `Step::excludeFromGroupOutput()` method.\n\n## [0.7.0] - 2023-01-13\n\n### Added\n* New functionality to paginate: There is the new `Paginate` child class of the `Http` step class (easy access via `Http::get()->paginate()`). It takes an instance of the `PaginatorInterface` and uses it to iterate through pagination links. There is one implementation of that interface, the `SimpleWebsitePaginator`. The `Http::get()->paginate()` method uses it by default, when called just with a CSS selector to get pagination links. Paginators receive all loaded pages and implement the logic to find pagination links. The paginator class is also called before sending a request, with the request object that is about to be sent as an argument (`prepareRequest()`). This way, it should even be doable to implement more complex pagination functionality. For example when pagination is built using POST request with query strings in the request body.\n* New methods `stopOnErrorResponse()` and `yieldErrorResponses()` that can be used with `Http` steps. By calling `stopOnErrorResponse()` the step will throw a `LoadingException` when a response has a 4xx or 5xx status code. By calling the `yieldErrorResponse()` even error responses will be yielded and passed on to the next steps (this was default behaviour until this version. See the breaking change below).\n* The body of HTTP responses with a `Content-Type` header containing `application/x-gzip` are automatically decoded when `Http::getBodyString()` is used. Therefore, added `ext-zlib` to suggested in `composer.json`.\n* New methods `addToResult()` and `addLaterToResult()`. `addToResult()` is a single replacement for `setResultKey()` and `addKeysToResult()` (they are removed, see `Changed` below) that can be used for array and non array output. `addLaterToResult()` is a new method that does not create a Result object immediately, but instead adds the output of the current step to all the Results that will later be created originating from the current output.\n* New methods `outputKey()` and `keepInputData()` that can be used with any step. Using the `outputKey()` method, the step will convert non array output to an array and use the key provided as an argument to this method as array key for the output value. The `keepInputData()` method allows you to forward data from the step's input to the output. If the input is non array you can define a key using the method's argument. This is useful e.g. if you're having data in the initial inputs that you also want to add to the final crawling results.\n* New method `createsResult()` that can be used with any step, so you can differentiate if a step creates a Result object, or just keeps data to add to results later (new `addLaterToResult()` method). But primarily relevant for library internal use.\n* The `FileCache` class can compress the cache data now to save disk space. Use the `useCompression()` method to do so.\n* New method `retryCachedErrorResponses()` in `HttpLoader`. When called, the loader will only use successful responses (status code < 400) from the cache and therefore retry already cached error responses.\n* New method `writeOnlyCache()` in `HttpLoader` to only write to, but don't read from the response cache. Can be used to renew cached responses.\n* `Filter::urlPathMatches()` to filter URL paths using a regex.\n* Option to provide a chrome executable name to the `chrome-php/chrome` library via `HttpLoader::setChromeExecutable()`.\n\n### Changed\n* __BREAKING__: Group steps can now only produce combined outputs, as previously done when `combineToSingleOutput()` method was called. The method is removed. \n* __BREAKING__: `setResultKey()` and `addKeysToResult()` are removed. Calls to those methods can both be replaced with calls to the new `addToResult()` method.\n* __BREAKING__: `getResultKey()` is also removed with `setResultKey()`. It's removed without replacement, as it doesn't really make sense any longer.\n* __BREAKING__: Error responses (4xx as well as 5xx), by default, won't produce any step outputs any longer. If you want to receive error responses, use the new `yieldErrorResponses()` method.\n* __BREAKING__: Removed the `httpClient()` method in the `HttpCrawler` class. If you want to provide your own HTTP client, implement a custom `loader` method passing your client to the `HttpLoader` instead.\n* __Deprecated__ the loop feature (class `Loop` and `Crawler::loop()` method). Probably the only use case is iterating over paginated list pages, which can be done using the new Paginator functionality. It will be removed in v1.0.\n* In case of a 429 (Too Many Requests) response, the `HttpLoader` now automatically waits and retries. By default, it retries twice and waits 10 seconds for the first retry and a minute for the second one. In case the response also contains a `Retry-After` header with a value in seconds, it complies to that. Exception: by default it waits at max `60` seconds (you can set your own limit if you want), if the `Retry-After` value is higher, it will stop crawling. If all the retries also receive a `429` it also throws an Exception.\n* Removed logger from `Throttler` as it doesn't log anything.\n* Fail silently when `robots.txt` can't be parsed.\n* Default timeout configuration for the default guzzle HTTP client: `connect_timeout` is `10` seconds and `timeout` is `60` seconds.\n* The `validateAndSanitize...()` methods in the abstract `Step` class, when called with an array with one single element, automatically try to use that array element as input value.\n* With the `Html` and `Xml` data extraction steps you can now add layers to the data that is being extracted, by just adding further `Html`/`Xml` data extraction steps as values in the mapping array that you pass as argument to the `extract()` method.\n* The base `Http` step can now also be called with an array of URLs as a single input. Crawl and Paginate steps still require a single URL input.\n\n### Fixed\n* The `CookieJar` now also works with `localhost` or other hosts without a registered domain name.\n* Improve the `Sitemap::getUrlsFromSitemap()` step to also work when the `` tag contains attributes that would cause the symfony DomCrawler to not find any elements.\n* Fixed possibility of infinite redirects in `HttpLoader` by adding a redirects limit of 10.\n\n## [0.6.0] - 2022-10-03\n\n### Added\n* New step `Http::crawl()` (class `HttpCrawl` extending the normal `Http` step class) for conventional crawling. It loads all pages of a website (same host or domain) by following links. There's also a lot of options like depth, filtering by paths, and so on.\n* New steps `Sitemap::getSitemapsFromRobotsTxt()` (`GetSitemapsFromRobotsTxt`) and `Sitemap::getUrlsFromSitemap()` (`GetUrlsFromSitemap`) to get sitemap (URLs) from a robots.txt file and to get all the URLs from those sitemaps.\n* New step `Html::metaData()` to get data from meta tags (and title tag) in HTML documents.\n* New step `Html::schemaOrg()` (`SchemaOrg`) to get schema.org structured data in JSON-LD format from HTML documents.\n* The abstract `DomQuery` class (parent of the `CssSelector` and `XPathQuery` classes) now has some methods to narrow the selected matches further: `first()`, `last()`, `nth(n)`, `even()`, `odd()`.\n\n### Changed\n* __BREAKING__: Removed `PoliteHttpLoader` and traits `WaitPolitely` and `CheckRobotsTxt`. Converted the traits to classes `Throttler` and `RobotsTxtHandler` which are dependencies of the `HttpLoader`. The `HttpLoader` internally gets default instances of those classes. The `RobotsTxtHandler` will respect robots.txt rules by default if you use a `BotUserAgent` and it won't if you use a normal `UserAgent`. You can access the loader's `RobotsTxtHandler` via `HttpLoader::robotsTxt()`. You can pass your own instance of the `Throttler` to the loader and also access it via `HttpLoader::throttle()` to change settings.\n\n### Fixed\n* Getting absolute links via the `GetLink` and `GetLinks` steps and the `toAbsoluteUrl()` method of the `CssSelector` and `XPathQuery` classes, now also look for `` tags in HTML when resolving the URLs.\n* The `SimpleCsvFileStore` can now also save results with nested data (but only second level). It just concatenates the values separated with a ` | `.\n\n## [0.5.0] - 2022-09-03\n### Added\n* You can now call the new `useHeadlessBrowser` method on the `HttpLoader` class to use a headless Chrome browser to load pages. This is enough to get HTML after executing javascript in the browser. For more sophisticated tasks a separate Loader and/or Steps should better be created.\n* With the `maxOutputs()` method of the abstract `Step` class you can now limit how many outputs a certain step should yield at max. That's for example helpful during development, when you want to run the crawler only with a small subset of the data/requests it will actually have to process when you eventually remove the limits. When a step has reached its limit, it won't even call the `invoke()` method any longer until the step is reset after a run.\n* With the new `outputHook()` method of the abstract `Crawler` class you can set a closure that'll receive all the outputs from all the steps. Should be only for debugging reasons.\n* The `extract()` method of the `Html` and `Xml` (children of `Dom`) steps now also works with a single selector instead of an array with a mapping. Sometimes you'll want to just get a simple string output e.g. for a next step, instead of an array with mapped extracted data.\n* In addition to `uniqueOutputs()` there is now also `uniqueInputs()`. It works exactly the same as `uniqueOutputs()`, filtering duplicate input values instead. Optionally also by a key when expected input is an array or an object.\n* In order to be able to also get absolute links when using the `extract()` method of Dom steps, the abstract `DomQuery` class now has a method `toAbsoluteUrl()`. The Dom step will automatically provide the `DomQuery` instance with the base url, presumed that the input was an instance of the `RespondedRequest` class and resolve the selected value against that base url.\n\n### Changed\n* Remove some not so important log messages.\n* Improve behavior of group step's `combineToSingleOutput()`. When steps yield multiple outputs, don't combine all yielded outputs to one. Instead, combine the first output from the first step with the first output from the second step, and so on.\n* When results are not explicitly composed, but the outputs of the last step are arrays with string keys, it sets those keys on the Result object instead of setting a key `unnamed` with the whole array as value.\n\n### Fixed\n* The static methods `Html::getLink()` and `Html::getLinks()` now also work without argument, like the `GetLink` and `GetLinks` classes.\n* When a `DomQuery` (CSS selector or XPath query) doesn't match anything, its `apply()` method now returns `null` (instead of an empty string). When the `Html(/Xml)::extract()` method is used with a single, not matching selector/query, nothing is yielded. When it's used with an array with a mapping, it yields an array with null values. If the selector for one of the methods `Html(/Xml)::each()`, `Html(/Xml)::first()` or `Html(/Xml)::last()` doesn't match anything, that's not causing an error any longer, it just won't yield anything.\n* Removed the (unnecessary) second argument from the `Loop::withInput()` method because when `keepLoopingWithoutOutput()` is called and `withInput()` is called after that call, it resets the behavior.\n* Issue when date format for expires date in cookie doesn't have dashes in `d-M-Y` (so `d M Y`).\n\n## [0.4.1] - 2022-05-10\n### Fixed\n* The `Json` step now also works with Http responses as input.\n\n## [0.4.0] - 2022-05-06\n### Added\n* The `BaseStep` class now has `where()` and `orWhere()` methods to filter step outputs. You can set multiple filters that will be applied to all outputs. When setting a filter using `orWhere` it's linked to the previously added Filter with \"OR\". Outputs not matching one of the filters, are not yielded. The available filters can be accessed through static methods on the new `Filter` class. Currently available filters are comparison filters (equal, greater/less than,...), a few string filters (contains, starts/ends with) and url filters (scheme, domain, host,...).\n* The `GetLink` and `GetLinks` steps now have methods `onSameDomain()`, `notOnSameDomain()`, `onDomain()`, `onSameHost()`, `notOnSameHost()`, `onHost()` to restrict the which links to find.\n* Automatically add the crawler's logger to the `Store` so you can also log messages from there. This can be breaking as the `StoreInterface` now also requires the `addLogger` method. The new abstract `Store` class already implements it, so you can just extend it.\n\n### Changed\n* The `Csv` step can now also be used without defining a column mapping. In that case it will use the values from the first line (so this makes sense when there are column headlines) as output array keys.\n\n## [0.3.0] - 2022-04-27\n### Added\n* By calling `monitorMemoryUsage()` you can tell the Crawler to add log messages with the current memory usage after every step invocation. You can also set a limit in bytes when to start monitoring and below the limit it won't log memory usage.\n\n### Fixed\n* Previously the __use of Generators__ actually didn't make a lot of sense, because the outputs of one step were only iterated and passed on to the next step, after the current step was invoked with all its inputs. That makes steps with a lot of inputs bottlenecks and causes bigger memory consumption. So, changed the crawler to immediately pass on outputs of one step to the next step if there is one.\n\n## [0.2.0] - 2022-04-25\n### Added\n* `uniqueOutputs()` method to Steps to get only unique output values. If outputs are array or object, you can provide a key that will be used as identifier to check for uniqueness. Otherwise, the arrays or objects will be serialized for comparison which will probably be slower.\n* `runAndTraverse()` method to Crawler, so you don't need to manually traverse the Generator, if you don't need the results where you're calling the crawler.\n* Implement the behaviour for when a `Group` step should add something to the Result using `setResultKey()` or `addKeysToResult()`, which was still missing. For groups this will only work when using `combineToSingleOutput`.\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to this Package\n\nThat you're reading this must mean you consider contributing to\nthis package. So first off: Awesome! 👍🤘\n\n## Bugs\n\nIn case you encounter any bugs please\n[file an issue](https://github.com/crwlrsoft/crawler/issues/new).\nDescribe the issue as well as you can and provide an example to\nreproduce it. \nMaybe you're not 100 percent sure whether what you've discovered\nis a bug or the intended behavior. You can still file an issue\nand tell us which results you'd expect.\n\nIf you know how to fix the issue you're welcome to send a pull\nrequest. 💪\n\n## New Features\n\nIf you have ideas for new features you can tell us about it on\n[Twitter](https://twitter.com/crwlrsoft) or via\n[crwlr.software](https://www.crwlr.software/contact) or just\nsend a pull request. Please keep in mind that there is no\nguarantee that your feature will be merged.\n\n## Conventions\n\n### Coding Style\n\nThis package follows the\n[PSR-12](https://www.php-fig.org/psr/psr-12/) coding standard.\nYou can run PHP CS Fixer via `composer cs` for a dry run or\n`composer cs-fix` to automatically fix code style issues.\n\n### Code quality tools\n\nWhen you're making changes to this package please always run\ntests and linting. Commands: \n`composer test` \n`composer test-integration`\n`composer cs`\n`composer stan`\n\nIdeally you add the pre-commit git hook that is shipped with\nthis repo that will run tests and linting. Add it to your local\nclone by running: \n`composer add-git-hooks`\n\nThe integration tests start a simple PHP web server for the\ntesting purpose on port 8000. If you have anything else running\non that port, the integration tests won't work.\n\nAlso, please don't forget to add new test cases if necessary.\n\n### Documentation\n\nFor any code change that changes/adds something for users of\nthe package, please don't forget to add an entry to the\n`CHANGELOG.md` file.\n\n## Appreciation\n\nWhen your pull request is merged I will show some love and tweet\nabout it. Also, if you meet me in person I will be glad to buy you\na beer.\n" }, { "path": "LICENSE", "content": "Copyright (c) 2026 Christian Olear\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject\nto the following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "README.md", "content": "

\"crwlr.software

\n\n# Library for Rapid (Web) Crawler and Scraper Development\n\nThis library provides kind of a framework and a lot of ready to use, so-called __steps__, that you can use as building blocks, to build your own crawlers and scrapers with.\n\nTo give you an overview, here's a list of things that it helps you with:\n* [Crawler __Politeness__](https://www.crwlr.software/packages/crawler/the-crawler/politeness) 😇 (respecting robots.txt, throttling,...)\n* Load URLs using\n * [a __(PSR-18) HTTP client__](https://www.crwlr.software/packages/crawler/the-crawler/loaders) (default is of course Guzzle)\n * or a [__headless browser__](https://www.crwlr.software/packages/crawler/the-crawler/loaders#using-a-headless-browser) (chrome) to get source after Javascript execution\n* [Get __absolute links__ from HTML documents](https://www.crwlr.software/packages/crawler/included-steps/html#html-get-link) 🔗\n* [Get __sitemaps__ from robots.txt and get all URLs from those sitemaps](https://www.crwlr.software/packages/crawler/included-steps/sitemap)\n* [__Crawl__ (load) all pages of a website](https://www.crwlr.software/packages/crawler/included-steps/http#crawling) 🕷\n* [Use __cookies__ (or don't)](https://www.crwlr.software/packages/crawler/the-crawler/loaders#http-loader) 🍪\n* [Use any __HTTP methods__ (GET, POST,...) and send any headers or body](https://www.crwlr.software/packages/crawler/included-steps/http#http-requests)\n* [Easily iterate over __paginated__ list pages](https://www.crwlr.software/packages/crawler/included-steps/http#paginating) 🔁\n* Extract data from:\n * [__HTML__](https://www.crwlr.software/packages/crawler/included-steps/html#extracting-data) and also [__XML__](https://www.crwlr.software/packages/crawler/included-steps/xml) (using CSS selectors or XPath queries)\n * [__JSON__](https://www.crwlr.software/packages/crawler/included-steps/json) (using dot notation)\n * [__CSV__](https://www.crwlr.software/packages/crawler/included-steps/csv) (map columns)\n* [Extract __schema.org__ structured data](https://www.crwlr.software/packages/crawler/included-steps/html#schema-org) in __JSON-LD__ format from HTML documents\n* [Keep memory usage low](https://www.crwlr.software/packages/crawler/crawling-procedure#memory-usage) by using PHP __Generators__ 💪\n* [__Cache__ HTTP responses](https://www.crwlr.software/packages/crawler/response-cache) during development, so you don't have to load pages again and again after every code change\n* [Get __logs__](https://www.crwlr.software/packages/crawler/the-crawler#loggers) about what your crawler is doing (accepts any PSR-3 LoggerInterface)\n* And a lot more...\n\n## Documentation\n\nYou can find the documentation at [crwlr.software](https://www.crwlr.software/packages/crawler/getting-started).\n\n## Contributing\n\nIf you consider contributing something to this package, read the [contribution guide (CONTRIBUTING.md)](CONTRIBUTING.md).\n" }, { "path": "bin/add-git-hooks", "content": "#!/usr/bin/env php\n 0) {\n $lastLine = array_pop($output);\n\n if (trim($lastLine) !== '') {\n printLine($lastLine);\n return;\n }\n }\n}\n\nfunction printLine(string $string)\n{\n echo $string . PHP_EOL;\n}\n\nfunction printLines(array $lines)\n{\n echo implode(PHP_EOL, $lines) . PHP_EOL;\n}\n\nfunction printBlankLine()\n{\n printLine('');\n}\n\nfunction red(string $string): string\n{\n return color('0;31', $string);\n}\n\nfunction green(string $string): string\n{\n return color('0;32', $string);\n}\n\nfunction blue(string $string): string\n{\n return color('0;34', $string);\n}\n\nfunction color(string $colorCode, string $string): string\n{\n return \"\\e[\" . $colorCode . \"m\" . $string . \"\\e[0m\";\n}\n" }, { "path": "phpstan.neon", "content": "parameters:\n level: 8\n paths:\n - src\n - tests\n excludePaths:\n analyse:\n - tests/_Integration/_Server\n reportUnmatchedIgnoredErrors: false\n ignoreErrors:\n - \"#^Call to an undefined method Pest\\\\\\\\PendingCalls\\\\\\\\TestCall\\\\|Pest\\\\\\\\Support\\\\\\\\HigherOrderTapProxy\\\\:\\\\:(with|throws)\\\\(\\\\).$#\"\n - \"#^Access to an undefined property Spatie\\\\\\\\Invade\\\\\\\\Invader#\"\n - \"#^Call to an undefined method Spatie\\\\\\\\Invade\\\\\\\\Invader#\"\n - \"#^Call to protected method [a-zA-Z]{5,30}\\\\(\\\\) of class PHPUnit\\\\\\\\Framework\\\\\\\\TestCase.#\"\n - \"#^(?:Parameter|Method) .+ has invalid (return )?type Dom\\\\\\\\.+\\\\.#\"\n - \"#^Call to .+ on an unknown class Dom\\\\\\\\.+\\\\.#\"\n - \"#^Property .+ has unknown class Dom\\\\\\\\.+ as its type\\\\.#\"\n - \"#^Class Dom\\\\\\\\.+ not found.#\"\n - \"#^Access to property .+ on an unknown class Dom\\\\\\\\.+\\\\.#\"\n - \"#^PHPDoc tag .+ contains unknown class Dom\\\\\\\\.+\\\\.#\"\n - \"#^Call to an undefined (static )?method Dom\\\\\\\\.+::.+\\\\(\\\\)\\\\.#\"\n - \"#^Access to an undefined property Dom\\\\\\\\.+::\\\\$.+\\\\.#\"\n - \"#^Function .+ has invalid return type Dom\\\\\\\\.+\\\\.#\"\n - \"#^(?:Used )?(?:C|c)onstant DOM\\\\\\\\.+ not found\\\\.#\"\n - \"#^Instantiated class Dom\\\\\\\\.+ not found.#\"\n" }, { "path": "phpunit.xml", "content": "\n\n \n \n ./tests\n \n \n \n \n \n ./app\n ./src\n \n \n\n" }, { "path": "src/Cache/CacheItem.php", "content": "value) && method_exists($this->value, 'cacheKey')) {\n $this->key = $this->value->cacheKey();\n } else {\n $this->key = md5(serialize($this->value));\n }\n } else {\n $this->key = $key;\n }\n }\n\n public function key(): string\n {\n return $this->key;\n }\n\n public function value(): mixed\n {\n return $this->value;\n }\n\n /**\n * @throws Exception\n */\n public function isExpired(): bool\n {\n $ttl = $this->ttl instanceof DateInterval ? $this->ttl : new DateInterval('PT' . $this->ttl . 'S');\n\n return time() > $this->createdAt->add($ttl)->getTimestamp();\n }\n\n /**\n * Get a new instance with same data but a different time to live.\n */\n public function withTtl(DateInterval|int $ttl): CacheItem\n {\n return new CacheItem($this->value, $this->key, $ttl, $this->createdAt);\n }\n\n /**\n * @return mixed[]\n */\n public function __serialize(): array\n {\n return [\n 'value' => $this->value,\n 'key' => $this->key,\n 'ttl' => $this->ttl,\n 'createdAt' => $this->createdAt,\n ];\n }\n\n /**\n * @param mixed[] $data\n */\n public function __unserialize(array $data): void\n {\n $this->value = $data['value'];\n\n $this->key = $data['key'];\n\n $this->ttl = $data['ttl'];\n\n $this->createdAt = $data['createdAt'];\n }\n}\n" }, { "path": "src/Cache/Exceptions/MissingZlibExtensionException.php", "content": "useCompression = true;\n\n return $this;\n }\n\n public function ttl(DateInterval|int $ttl): static\n {\n $this->ttl = $ttl;\n\n return $this;\n }\n\n /**\n * @throws MissingZlibExtensionException|ReadingCacheFailedException|Exception|InvalidArgumentException\n */\n public function has(string $key): bool\n {\n if (file_exists($this->basePath . '/' . $key)) {\n $cacheItem = $this->getCacheItem($key);\n\n if (!$cacheItem->isExpired()) {\n return true;\n }\n\n $this->delete($key);\n }\n\n return false;\n }\n\n /**\n * @throws ReadingCacheFailedException|MissingZlibExtensionException|Exception|InvalidArgumentException\n */\n public function get(string $key, mixed $default = null): mixed\n {\n if (file_exists($this->basePath . '/' . $key)) {\n $cacheItem = $this->getCacheItem($key);\n\n if (!$cacheItem->isExpired()) {\n return $cacheItem->value();\n }\n\n $this->delete($key);\n }\n\n return $default;\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n public function set(string $key, mixed $value, DateInterval|int|null $ttl = null): bool\n {\n if (!$value instanceof CacheItem) {\n $value = new CacheItem($value, $key, $ttl ?? $this->ttl);\n } elseif ($value->key() !== $key) {\n $value = new CacheItem($value->value(), $key, $ttl ?? $value->ttl);\n }\n\n return $this->saveCacheItem($value);\n }\n\n public function delete(string $key): bool\n {\n return unlink($this->basePath . '/' . $key);\n }\n\n public function prolong(string $key, DateInterval|int $ttl): bool\n {\n try {\n $item = $this->getCacheItem($key);\n\n return $this->saveCacheItem($item->withTtl($ttl));\n } catch (Throwable) {\n return false;\n }\n }\n\n /**\n * @throws InvalidArgumentException\n */\n public function clear(): bool\n {\n $allFiles = scandir($this->basePath);\n\n if (is_array($allFiles)) {\n foreach ($allFiles as $file) {\n if ($file !== '.' && $file !== '..' && $file !== '.gitkeep' && !$this->delete($file)) {\n return false;\n }\n }\n }\n\n return true;\n }\n\n public function prolongAll(DateInterval|int $ttl): bool\n {\n $allFiles = scandir($this->basePath);\n\n if (is_array($allFiles)) {\n foreach ($allFiles as $file) {\n if ($file !== '.' && $file !== '..' && $file !== '.gitkeep' && !$this->prolong($file, $ttl)) {\n return false;\n }\n }\n }\n\n return true;\n }\n\n /**\n * @return iterable\n * @throws MissingZlibExtensionException|ReadingCacheFailedException|InvalidArgumentException\n */\n public function getMultiple(iterable $keys, mixed $default = null): iterable\n {\n $items = [];\n\n foreach ($keys as $key) {\n $items[$key] = $this->get($key, $default);\n }\n\n return $items;\n }\n\n /**\n * @param iterable $values\n * @throws MissingZlibExtensionException\n */\n public function setMultiple(iterable $values, DateInterval|int|null $ttl = null): bool\n {\n foreach ($values as $key => $value) {\n if (!$this->set($key, $value, $ttl)) {\n return false;\n }\n }\n\n return true;\n }\n\n public function deleteMultiple(iterable $keys): bool\n {\n foreach ($keys as $key) {\n if (!$this->delete($key)) {\n return false;\n }\n }\n\n return true;\n }\n\n /**\n * @throws MissingZlibExtensionException\n * @throws ReadingCacheFailedException\n */\n protected function getCacheItem(string $key): CacheItem\n {\n $fileContent = $this->getFileContents($key);\n\n if ($this->useCompression) {\n $fileContent = $this->decode($fileContent);\n }\n\n $unserialized = $this->unserialize($fileContent);\n\n if (!$unserialized instanceof CacheItem) {\n $unserialized = new CacheItem($unserialized, $key);\n }\n\n return $unserialized;\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function saveCacheItem(CacheItem $item): bool\n {\n $content = serialize($item);\n\n if ($this->useCompression) {\n $content = $this->encode($content);\n }\n\n return file_put_contents($this->basePath . '/' . $item->key(), $content) !== false;\n }\n\n protected function unserialize(string $content): mixed\n {\n // Temporarily set a new error handler, so unserializing a compressed string does not result in a PHP warning.\n set_error_handler(function ($errno, $errstr) {\n return $errno === E_WARNING && str_starts_with($errstr, 'unserialize(): Error at offset 0 of ');\n });\n\n $unserialized = unserialize($content);\n\n if ($unserialized === false) { // if unserializing fails, try if the string is compressed.\n try {\n $content = $this->decode($content);\n\n $unserialized = unserialize($content);\n } catch (Throwable) {\n }\n }\n\n restore_error_handler();\n\n return $unserialized;\n }\n\n /**\n * @throws ReadingCacheFailedException\n */\n protected function getFileContents(string $key): string\n {\n $fileContent = file_get_contents($this->basePath . '/' . $key);\n\n if ($fileContent === false) {\n throw new ReadingCacheFailedException('Failed to read cache file.');\n }\n\n return $fileContent;\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function encode(string $content): string\n {\n try {\n return Gzip::encode($content, true);\n } catch (MissingZlibExtensionException) {\n throw new MissingZlibExtensionException(\n 'Can\\'t compress response cache data. Compression needs PHP ext-zlib installed.',\n );\n }\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function decode(string $content): string\n {\n try {\n return Gzip::decode($content, true);\n } catch (MissingZlibExtensionException) {\n throw new MissingZlibExtensionException('FileCache compression needs PHP ext-zlib installed.');\n }\n }\n}\n" }, { "path": "src/Crawler.php", "content": "\n */\n protected array $steps = [];\n\n protected ?StoreInterface $store = null;\n\n protected bool|int $monitorMemoryUsage = false;\n\n protected ?Closure $outputHook = null;\n\n public function __construct()\n {\n $this->userAgent = $this->userAgent();\n\n $this->logger = $this->logger();\n\n $this->loader = $this->loader($this->userAgent, $this->logger);\n }\n\n public function __clone(): void\n {\n $this->inputs = [];\n\n $this->steps = [];\n\n $this->store = null;\n\n $this->outputHook = null;\n }\n\n abstract protected function userAgent(): UserAgentInterface;\n\n /**\n * @param UserAgentInterface $userAgent\n * @param LoggerInterface $logger\n * @return LoaderInterface\n */\n abstract protected function loader(UserAgentInterface $userAgent, LoggerInterface $logger): LoaderInterface;\n\n public static function group(): Group\n {\n return new Group();\n }\n\n public static function setMemoryLimit(string $memoryLimit): false|string\n {\n return ini_set('memory_limit', $memoryLimit);\n }\n\n public static function getMemoryLimit(): false|string\n {\n return ini_get('memory_limit');\n }\n\n public function getSubCrawler(): Crawler\n {\n return clone $this;\n }\n\n public function getUserAgent(): UserAgentInterface\n {\n return $this->userAgent;\n }\n\n public function setUserAgent(UserAgentInterface $userAgent): static\n {\n $this->userAgent = $userAgent;\n\n $this->loader = $this->loader($userAgent, $this->logger);\n\n return $this;\n }\n\n public function getLogger(): LoggerInterface\n {\n return $this->logger;\n }\n\n /**\n * @return LoaderInterface|array\n */\n public function getLoader(): LoaderInterface|array\n {\n return $this->loader;\n }\n\n public function setStore(StoreInterface $store): static\n {\n $store->addLogger($this->logger);\n\n $this->store = $store;\n\n return $this;\n }\n\n public function input(mixed $input): static\n {\n $this->inputs[] = $input;\n\n return $this;\n }\n\n /**\n * @param mixed[] $inputs\n */\n public function inputs(array $inputs): static\n {\n $this->inputs = array_merge($this->inputs, $inputs);\n\n return $this;\n }\n\n /**\n * @param StepInterface $step\n * @return $this\n * @throws InvalidArgumentException\n */\n public function addStep(StepInterface $step): static\n {\n $step->addLogger($this->logger);\n\n if (method_exists($step, 'setLoader')) {\n $step->setLoader($this->loader);\n }\n\n if ($step instanceof BaseStep) {\n $step->setParentCrawler($this);\n }\n\n $this->steps[] = $step;\n\n return $this;\n }\n\n /**\n * Run the crawler and traverse results\n *\n * When you've set a store, or you just don't need the results for any other reason (e.g. you use the crawler for\n * cache warming) where you're calling the crawler, use this method.\n *\n * @throws Exception\n */\n public function runAndTraverse(): void\n {\n foreach ($this->run() as $result) {\n }\n }\n\n /**\n * Easy way to just crawl and dump the results\n *\n * @throws Exception\n */\n public function runAndDump(): void\n {\n foreach ($this->run() as $result) {\n var_dump($result->toArray());\n }\n }\n\n /**\n * Run the Crawler\n *\n * Handles calling all the steps and cascading the data from step to step.\n * It returns a Generator, so when using this method directly, you need to traverse the Generator, otherwise nothing\n * happens. Alternatively you can use runAndTraverse().\n *\n * @return Generator\n * @throws Exception|PreRunValidationException\n */\n public function run(): Generator\n {\n $this->validateSteps();\n\n $inputs = $this->prepareInput();\n\n if ($this->firstStep()) {\n foreach ($inputs as $input) {\n $results = $this->invokeStepsRecursive($input, $this->firstStep(), 0);\n\n /** @var Generator $results */\n\n yield from $results;\n }\n }\n\n $this->reset();\n }\n\n /**\n * Use this method if you want the crawler to add log messages with the current memory usage after every step\n * invocation.\n *\n * @param int|null $ifAboveXBytes You can provide an int of bytes as a limit above which the crawler should log\n * the usage.\n */\n public function monitorMemoryUsage(?int $ifAboveXBytes = null): static\n {\n $this->monitorMemoryUsage = $ifAboveXBytes ?? true;\n\n return $this;\n }\n\n public function outputHook(Closure $callback): static\n {\n $this->outputHook = $callback;\n\n return $this;\n }\n\n protected function logger(): LoggerInterface\n {\n return new CliLogger();\n }\n\n /**\n * @return Generator\n */\n protected function invokeStepsRecursive(Input $input, StepInterface $step, int $stepIndex): Generator\n {\n $outputs = $step->invokeStep($input);\n\n $nextStep = $this->nextStep($stepIndex);\n\n if (!$nextStep) {\n yield from $this->storeAndReturnOutputsAsResults($outputs);\n\n return;\n }\n\n foreach ($outputs as $output) {\n if ($this->monitorMemoryUsage !== false) {\n $this->logMemoryUsage();\n }\n\n $this->outputHook?->call($this, $output, $stepIndex, $step);\n\n yield from $this->invokeStepsRecursive(\n new Input($output),\n $nextStep,\n $stepIndex + 1,\n );\n }\n }\n\n /**\n * @param Generator $outputs\n * @return Generator\n */\n protected function storeAndReturnOutputsAsResults(Generator $outputs): Generator\n {\n foreach ($outputs as $output) {\n $this->outputHook?->call($this, $output, count($this->steps) - 1, end($this->steps));\n\n $result = new Result();\n\n foreach ($output->keep as $key => $value) {\n $result->set($key, $value);\n }\n\n if (!$this->lastStep()?->keepsAnything()) {\n if ($output->isArrayWithStringKeys()) {\n foreach ($output->get() as $key => $value) {\n $result->set($key, $value);\n }\n } else {\n $result->set('unnamed', $output->get());\n }\n }\n\n $this->store?->store($result);\n\n yield $result;\n }\n }\n\n /**\n * @throws PreRunValidationException\n */\n protected function validateSteps(): void\n {\n $previousStep = null;\n\n foreach ($this->steps as $index => $step) {\n if ($index > 0) {\n $previousStep = $this->steps[$index - 1];\n }\n\n if (method_exists($step, 'validateBeforeRun')) {\n try {\n $step->validateBeforeRun($previousStep ?? $this->inputs);\n } catch (PreRunValidationException $exception) {\n $this->logger->error(\n 'Pre-Run validation error in step number ' . ($index + 1) . ': ' . $exception->getMessage(),\n );\n\n throw $exception;\n }\n }\n }\n }\n\n /**\n * @return Input[]\n * @throws Exception\n */\n protected function prepareInput(): array\n {\n return array_map(function ($input) {\n return new Input($input);\n }, $this->inputs);\n }\n\n protected function logMemoryUsage(): void\n {\n $memoryUsage = memory_get_usage();\n\n if (!is_int($this->monitorMemoryUsage) || $memoryUsage > $this->monitorMemoryUsage) {\n $this->logger->info('memory usage: ' . $memoryUsage);\n }\n }\n\n protected function firstStep(): ?StepInterface\n {\n return $this->steps[0] ?? null;\n }\n\n protected function lastStep(): ?BaseStep\n {\n $lastStep = end($this->steps);\n\n if (!$lastStep instanceof BaseStep) {\n return null;\n }\n\n return $lastStep;\n }\n\n protected function nextStep(int $afterIndex): ?StepInterface\n {\n return $this->steps[$afterIndex + 1] ?? null;\n }\n\n protected function reset(): void\n {\n $this->inputs = [];\n\n foreach ($this->steps as $step) {\n $step->resetAfterRun();\n }\n }\n}\n" }, { "path": "src/HttpCrawler/AnonymousHttpCrawlerBuilder.php", "content": "setUserAgent(new BotUserAgent($productToken));\n\n return $instance;\n }\n\n public function withUserAgent(string|UserAgentInterface $userAgent): HttpCrawler\n {\n $instance = new class extends HttpCrawler {\n protected function userAgent(): UserAgentInterface\n {\n return new UserAgent('temp');\n }\n };\n\n $userAgent = $userAgent instanceof UserAgentInterface ? $userAgent : new UserAgent($userAgent);\n\n $instance->setUserAgent($userAgent);\n\n return $instance;\n }\n\n public function withMozilla5CompatibleUserAgent(): HttpCrawler\n {\n return $this->withUserAgent(UserAgent::mozilla5CompatibleBrowser());\n }\n}\n" }, { "path": "src/HttpCrawler.php", "content": "value = $value->value;\n\n $this->keep = $value->keep;\n }\n }\n\n public function withValue(mixed $value): static\n {\n return new static($value, $this->keep);\n }\n\n public function withPropertyValue(string $key, mixed $value): static\n {\n if (!$this->isArrayWithStringKeys()) {\n return new static($this);\n }\n\n $newValue = $this->value;\n\n $newValue[$key] = $value;\n\n return $this->withValue($newValue);\n }\n\n public function get(): mixed\n {\n return $this->value;\n }\n\n public function getProperty(string $key, mixed $fallbackValue = null): mixed\n {\n if (is_array($this->value)) {\n return $this->value[$key] ?? $fallbackValue;\n } elseif (is_object($this->value)) {\n $array = OutputTypeHelper::objectToArray($this->value);\n\n return $array[$key] ?? $fallbackValue;\n }\n\n return $fallbackValue;\n }\n\n /**\n * Sets and returns a key to use as identifier\n *\n * To only get unique results from a step use the key this method creates for comparison.\n * In case the output values are arrays or objects and contain a unique identifier that can be used, provide that\n * key name, so it doesn't need to create a key from the whole array/object.\n */\n public function setKey(?string $useFromValue = null): string\n {\n if ($useFromValue && is_array($this->value) && array_key_exists($useFromValue, $this->value)) {\n $this->key = $this->valueToString($this->value[$useFromValue]);\n } elseif ($useFromValue && is_object($this->value) && property_exists($this->value, $useFromValue)) {\n $this->key = $this->valueToString($this->value->{$useFromValue});\n } else {\n $this->key = $this->valueToString($this->value);\n }\n\n return $this->key;\n }\n\n public function getKey(): string|int|float|bool|null\n {\n if ($this->key === null) {\n $this->setKey();\n }\n\n return $this->key;\n }\n\n /**\n * @param mixed[] $data\n */\n public function keep(array $data): static\n {\n $this->keep = array_merge_recursive($this->keep, $data);\n\n return $this;\n }\n\n public function isArrayWithStringKeys(): bool\n {\n if (!is_array($this->value)) {\n return false;\n }\n\n foreach ($this->value as $key => $value) {\n if (!is_string($key)) {\n return false;\n }\n }\n\n return true;\n }\n\n protected function valueToString(mixed $value): string\n {\n if (is_array($value) || is_object($value)) {\n return md5(serialize($this->value));\n } elseif (is_int($value) || is_float($value)) {\n return (string) $value;\n } elseif (is_bool($value)) {\n return $value ? 'true' : 'false';\n } elseif (is_null($value)) {\n return 'null';\n }\n\n return $value;\n }\n}\n" }, { "path": "src/Loader/Http/Browser/Screenshot.php", "content": "getCurrentUrl()) . '-' . Microseconds::now()->value . '.' . $this->fileType;\n\n return $this->storePath . (!str_ends_with($this->storePath, '/') ? '/' : '') . $filename;\n }\n\n public function setImageFileType(string $type): self\n {\n if (in_array($type, ['jpeg', 'png', 'webp'], true)) {\n $this->fileType = $type;\n\n if (in_array($type, ['jpeg', 'webp'], true) && $this->quality === null) {\n $this->quality = 80;\n } elseif ($type === 'png' && $this->quality !== null) {\n $this->quality = null;\n }\n }\n\n return $this;\n }\n\n public function setQuality(int $quality): self\n {\n if (in_array($this->fileType, ['jpeg', 'webp'], true) && $quality > 0 && $quality <= 100) {\n $this->quality = $quality;\n }\n\n return $this;\n }\n\n public function setFullPage(): self\n {\n $this->fullPage = true;\n\n return $this;\n }\n\n /**\n * @return array\n */\n public function toChromePhpScreenshotConfig(Page $page): array\n {\n $config = ['format' => $this->fileType];\n\n if ($this->quality && in_array($this->fileType, ['jpeg', 'webp'], true)) {\n $config['quality'] = $this->quality;\n }\n\n if ($this->fullPage) {\n $config['captureBeyondViewport'] = true;\n\n $config['clip'] = $page->getFullPageClip();\n }\n\n return $config;\n }\n}\n" }, { "path": "src/Loader/Http/Cache/RetryManager.php", "content": "only = $statusCodes;\n\n return $this;\n }\n\n /**\n * @param int|int[] $statusCodes\n */\n public function except(int|array $statusCodes): static\n {\n $statusCodes = is_array($statusCodes) ? $statusCodes : [$statusCodes];\n\n $this->except = $statusCodes;\n\n return $this;\n }\n\n public function shallBeRetried(int $statusCode): bool\n {\n return $statusCode >= 400 &&\n ($this->except === null || !in_array($statusCode, $this->except, true)) &&\n ($this->only === null || in_array($statusCode, $this->only, true));\n }\n}\n" }, { "path": "src/Loader/Http/Cookies/Cookie.php", "content": "receivedFromUrl = $receivedFromUrl instanceof Url ? $receivedFromUrl : Url::parse($receivedFromUrl);\n\n if (\n !is_string($this->receivedFromUrl->host()) ||\n empty($this->receivedFromUrl->host())\n ) {\n throw new InvalidCookieException('Url where cookie was received from has no host or domain');\n }\n\n $this->receivedFromHost = $this->receivedFromUrl->host();\n\n $this->setDomain($this->receivedFromUrl->domain() ?? $this->receivedFromUrl->host());\n\n $this->parseSetCookieHeader($this->setCookieHeader);\n }\n\n /**\n * @throws Exception\n */\n public function shouldBeSentTo(string|UriInterface|Url $url): bool\n {\n $url = $url instanceof Url ? $url : Url::parse($url);\n\n $urlHost = $url->host() ?? '';\n\n return\n str_contains($urlHost, $this->domain()) &&\n (!$this->hasHostPrefix() || $urlHost === $this->receivedFromHost) &&\n (!$this->secure() || $url->scheme() === 'https' || in_array($urlHost, ['localhost', '127.0.0.1'], true)) &&\n (!$this->path() || $this->pathMatches($url)) &&\n !$this->isExpired();\n }\n\n public function __toString(): string\n {\n return $this->name() . '=' . $this->value();\n }\n\n public function receivedFromUrl(): UriInterface\n {\n return new Uri($this->receivedFromUrl);\n }\n\n public function name(): string\n {\n return $this->cookieName;\n }\n\n public function value(): string\n {\n return $this->cookieValue;\n }\n\n public function expires(): ?Date\n {\n return $this->expires;\n }\n\n public function maxAge(): ?int\n {\n return $this->maxAge;\n }\n\n public function isExpired(): bool\n {\n if ($this->expires() === null && $this->maxAge() === null) {\n return false;\n }\n\n $nowTimestamp = time();\n\n if ($this->expires() instanceof Date && $nowTimestamp >= $this->expires()->dateTime()->getTimestamp()) {\n return true;\n }\n\n return $this->maxAge() !== null &&\n ($this->maxAge() <= 0 || $nowTimestamp > ($this->receivedAtTimestamp + $this->maxAge()));\n }\n\n public function domain(): string\n {\n return $this->domain;\n }\n\n public function path(): ?string\n {\n return $this->path;\n }\n\n public function secure(): bool\n {\n return $this->secure;\n }\n\n public function httpOnly(): bool\n {\n return $this->httpOnly;\n }\n\n public function sameSite(): string\n {\n return $this->sameSite;\n }\n\n /**\n * @throws Exception\n */\n public function isReceivedSecure(): bool\n {\n return $this->receivedFromUrl->scheme() === 'https';\n }\n\n public function hasSecurePrefix(): bool\n {\n return str_starts_with($this->cookieName, '__Secure-');\n }\n\n public function hasHostPrefix(): bool\n {\n return str_starts_with($this->cookieName, '__Host-');\n }\n\n /**\n * @throws InvalidCookieException\n */\n protected function parseSetCookieHeader(string $setCookieHeader): void\n {\n $splitAtSemicolon = explode(';', $setCookieHeader);\n\n $splitFirstPart = explode('=', trim(array_shift($splitAtSemicolon)), 2);\n\n if (count($splitFirstPart) !== 2) {\n throw new InvalidCookieException('Invalid cookie string');\n }\n\n [$this->cookieName, $this->cookieValue] = $splitFirstPart;\n\n foreach ($splitAtSemicolon as $attribute) {\n $this->parseAttribute($attribute);\n }\n\n $this->checkPrefixes();\n }\n\n /**\n * @throws InvalidCookieException\n */\n protected function parseAttribute(string $attribute): void\n {\n $splitAtEquals = explode('=', trim($attribute), 2);\n\n $attributeName = strtolower($splitAtEquals[0]);\n\n $attributeValue = $splitAtEquals[1] ?? '';\n\n if ($attributeName === 'expires') {\n $this->setExpires($attributeValue);\n } elseif ($attributeName === 'max-age') {\n $this->setMaxAge($attributeValue);\n } elseif ($attributeName === 'domain') {\n $this->setDomain($attributeValue, true);\n } elseif ($attributeName === 'path') {\n $this->setPath($attributeValue);\n } elseif ($attributeName === 'secure') {\n $this->setSecure();\n } elseif ($attributeName === 'httponly') {\n $this->httpOnly = true;\n } elseif ($attributeName === 'samesite') {\n $this->setSameSite($attributeValue);\n }\n }\n\n /**\n * @see https://datatracker.ietf.org/doc/html/draft-west-cookie-prefixes#section-3\n * @throws InvalidCookieException\n * @throws Exception\n */\n protected function checkPrefixes(): void\n {\n if ($this->hasSecurePrefix() || $this->hasHostPrefix()) {\n if (!$this->isReceivedSecure()) {\n throw new InvalidCookieException(\n 'Cookie is prefixed with __Secure- or __Host- but was not sent via https',\n );\n }\n\n if (!$this->secure()) {\n throw new InvalidCookieException(\n 'Cookie is prefixed with __Secure- or __Host- but Secure flag was not sent',\n );\n }\n }\n\n if ($this->hasHostPrefix()) {\n if ($this->domainSetViaAttribute) {\n throw new InvalidCookieException('Cookie with __Host- prefix must not contain a Domain attribute');\n }\n\n if ($this->path !== '/') {\n throw new InvalidCookieException('Cookie with __Host- prefix must have a Path attribute with value /');\n }\n }\n }\n\n protected function setExpires(string $value): void\n {\n $this->expires = new Date($value);\n }\n\n protected function setMaxAge(string $value): void\n {\n $this->maxAge = (int) $value;\n\n $this->receivedAtTimestamp = time();\n }\n\n /**\n * @throws InvalidCookieException\n * @throws Exception\n */\n protected function setDomain(string $value, bool $viaAttribute = false): void\n {\n if (str_starts_with($value, '.')) {\n $value = substr($value, 1);\n }\n\n if (!str_contains($this->receivedFromHost, $value)) {\n throw new InvalidCookieException(\n 'Setting cookie for ' . $value . ' from ' . $this->receivedFromUrl->host() . ' is not allowed.',\n );\n }\n\n $this->domain = $value;\n\n if ($viaAttribute) {\n $this->domainSetViaAttribute = true;\n }\n }\n\n protected function setPath(string $path): void\n {\n $this->path = $path;\n }\n\n /**\n * @throws InvalidCookieException\n * @throws Exception\n */\n protected function setSecure(): void\n {\n if (!$this->isReceivedSecure()) {\n throw new InvalidCookieException(\n 'Secure flag can\\'t be set when cookie was sent from non-https document url.',\n );\n }\n\n $this->secure = true;\n }\n\n /**\n * @throws InvalidCookieException\n */\n protected function setSameSite(string $value): void\n {\n $value = strtolower($value);\n\n if (!in_array(strtolower($value), ['strict', 'lax', 'none'], true)) {\n throw new InvalidCookieException('Invalid value for attribute SameSite');\n }\n\n $this->sameSite = ucfirst($value);\n }\n\n /**\n * @throws Exception\n */\n protected function pathMatches(Url $url): bool\n {\n $path = $this->path() ?? '';\n\n $urlPath = $url->path() ?? '';\n\n return str_starts_with($urlPath, $path) &&\n (\n $urlPath === $path ||\n $path === '/' ||\n str_starts_with($urlPath, $path . '/')\n );\n }\n}\n" }, { "path": "src/Loader/Http/Cookies/CookieJar.php", "content": "jar)) {\n return $this->jar[$domain];\n }\n\n return [];\n }\n\n public function flush(): void\n {\n $this->jar = [];\n }\n\n /**\n * @throws InvalidCookieException\n * @throws Exception\n */\n public function addFrom(string|UriInterface|Url $url, ResponseInterface|CookiesCollection $response): void\n {\n if ($response instanceof CookiesCollection) {\n $this->addFromBrowserCookieCollection($url, $response);\n } else {\n $cookieHeaders = $response->getHeader('set-cookie');\n\n if (!empty($cookieHeaders)) {\n $url = !$url instanceof Url ? Url::parse($url) : $url;\n\n $domain = $this->getForDomainFromUrl($url);\n\n if ($domain) {\n foreach ($cookieHeaders as $cookieHeader) {\n $cookie = new Cookie($url, $cookieHeader);\n\n $this->jar[$domain][$cookie->name()] = $cookie;\n }\n }\n }\n }\n }\n\n /**\n * @throws InvalidCookieException\n * @throws Exception\n */\n public function addFromBrowserCookieCollection(string|UriInterface|Url $url, CookiesCollection $collection): void\n {\n if ($collection->count() === 0) {\n return;\n }\n\n if (!$url instanceof Url) {\n $url = Url::parse($url);\n }\n\n $domain = $this->getForDomainFromUrl($url);\n\n if ($domain) {\n foreach ($collection as $cookie) {\n $setCookie = new Cookie($url, $this->buildSetCookieHeaderFromBrowserCookie($cookie));\n\n $this->jar[$domain][$setCookie->name()] = $setCookie;\n }\n }\n }\n\n /**\n * @return Cookie[]\n * @throws Exception\n */\n public function getFor(string|UriInterface $url): array\n {\n $forDomain = $this->getForDomainFromUrl($url);\n\n if (!$forDomain || !array_key_exists($forDomain, $this->jar)) {\n return [];\n }\n\n $cookiesToSend = [];\n\n foreach ($this->jar[$forDomain] as $cookie) {\n if ($cookie->shouldBeSentTo($url)) {\n $cookiesToSend[] = $cookie;\n }\n }\n\n return $cookiesToSend;\n }\n\n /**\n * @throws Exception\n */\n protected function getForDomainFromUrl(string|UriInterface|Url $url): ?string\n {\n if (!$url instanceof Url) {\n $url = Url::parse($url);\n }\n\n $forDomain = empty($url->domain()) ? $url->host() : $url->domain();\n\n if (!is_string($forDomain)) {\n return null;\n }\n\n return $forDomain;\n }\n\n protected function buildSetCookieHeaderFromBrowserCookie(BrowserCookie $cookie): string\n {\n $attributes = [\n 'domain' => 'Domain',\n 'expires' => 'Expires',\n 'max-age' => 'Max-Age',\n 'path' => 'Path',\n 'secure' => 'Secure',\n 'httpOnly' => 'HttpOnly',\n 'sameSite' => 'SameSite',\n ];\n\n $parts = [sprintf('%s=%s', $cookie->getName(), $cookie->getValue())];\n\n foreach ($attributes as $name => $setCookieName) {\n $setCookieValue = $cookie->offsetGet($name);\n\n if (empty($setCookieValue)) {\n continue;\n }\n\n // \"Expires\" attribute\n if ($name === 'expires') {\n if ($setCookieValue !== -1) {\n $parts[] = sprintf('%s=%s', $setCookieName, $this->formatExpiresValue($setCookieValue));\n }\n\n continue;\n }\n\n // Flag attributes\n if ($setCookieValue === true) {\n $parts[] = $setCookieName;\n\n continue;\n }\n\n $parts[] = sprintf('%s=%s', $setCookieName, $setCookieValue);\n }\n\n return implode('; ', $parts);\n }\n\n private function formatExpiresValue(mixed $value): string\n {\n if (is_numeric($value)) {\n $value = (string) $value;\n\n if (str_contains($value, '.')) {\n $expires = strlen(explode('.', $value, 2)[1]) <= 3 ?\n DateTime::createFromFormat('U.v', $value) :\n DateTime::createFromFormat('U.u', $value);\n } else {\n $expires = DateTime::createFromFormat('U', $value);\n }\n\n if ($expires !== false) {\n return $expires->format('l, d M Y H:i:s T');\n }\n }\n\n return (string) $value;\n }\n}\n" }, { "path": "src/Loader/Http/Cookies/Date.php", "content": "dateTime instanceof DateTime) {\n $dateTime = DateTime::createFromFormat(DateTimeInterface::COOKIE, $this->httpDateString);\n\n if (!$dateTime instanceof DateTime) {\n $dateTime = DateTime::createFromFormat('l, d M Y H:i:s T', $this->httpDateString);\n\n if (!$dateTime instanceof DateTime) {\n throw new InvalidArgumentException('Can\\'t parse date string ' . $this->httpDateString);\n }\n }\n\n $this->dateTime = $dateTime;\n }\n\n return $this->dateTime;\n }\n}\n" }, { "path": "src/Loader/Http/Cookies/Exceptions/InvalidCookieException.php", "content": "getMessage(),\n previous: $previousException,\n );\n }\n\n public static function make(string|UriInterface $uri, ?int $httpStatusCode = null): self\n {\n if ($uri instanceof UriInterface) {\n $uri = (string) $uri;\n }\n\n $message = 'Failed to load ' . $uri;\n\n if ($httpStatusCode !== null) {\n $message .= ' (' . $httpStatusCode . ').';\n } else {\n $message .= '.';\n }\n\n $instance = new self($message);\n\n if ($httpStatusCode !== null) {\n $instance->httpStatusCode = $httpStatusCode;\n }\n\n return $instance;\n }\n}\n" }, { "path": "src/Loader/Http/HeadlessBrowserLoaderHelper.php", "content": "\n */\n protected array $options = [\n 'windowSize' => [1920, 1000],\n ];\n\n protected bool $optionsDirty = false;\n\n protected ?Browser $browser = null;\n\n protected ?Page $page = null;\n\n protected ?string $proxy = null;\n\n protected ?string $waitForEvent = null;\n\n protected int $timeout = 30_000;\n\n protected ?string $pageInitScript = null;\n\n protected bool $useNativeUserAgent = false;\n\n protected bool $includeShadowElements = false;\n\n /**\n * @var Closure[]\n */\n protected array $tempPostNavigateHooks = [];\n\n public function __construct(\n private ?BrowserFactory $browserFactory = null,\n protected ?LoggerInterface $logger = null,\n ) {}\n\n /**\n * Set temporary post navigate hooks\n *\n * They will be executed after the next call to navigateToPageAndGetRespondedRequest()\n * and forgotten afterward.\n *\n * @param Closure[] $hooks\n */\n public function setTempPostNavigateHooks(array $hooks): static\n {\n $this->tempPostNavigateHooks = $hooks;\n\n return $this;\n }\n\n /**\n * @throws OperationTimedOut\n * @throws CommunicationException\n * @throws NoResponseAvailable\n * @throws NavigationExpired\n * @throws InvalidResponse\n * @throws CannotReadResponse\n * @throws ResponseHasError\n * @throws JavascriptException\n * @throws Exception\n */\n public function navigateToPageAndGetRespondedRequest(\n RequestInterface $request,\n Throttler $throttler,\n ?string $proxy = null,\n ?CookieJar $cookieJar = null,\n ): RespondedRequest {\n if (!$this->page || $this->shouldRenewBrowser($proxy)) {\n $this->page = $this->getBrowser($request, $proxy)->createPage();\n } else {\n try {\n $this->page->assertNotClosed();\n } catch (TargetDestroyed) {\n $this->page = $this->getBrowser($request, $proxy)->createPage();\n }\n }\n\n if ($cookieJar === null) {\n $this->page->getSession()->sendMessageSync(new Message('Network.clearBrowserCookies'));\n }\n\n $statusCode = 200;\n\n $responseHeaders = [];\n\n $requestId = null;\n\n $this->page->getSession()->once(\n \"method:Network.responseReceived\",\n function ($params) use (&$statusCode, &$responseHeaders, &$requestId) {\n $statusCode = $params['response']['status'];\n\n $responseHeaders = $this->sanitizeResponseHeaders($params['response']['headers']);\n\n $requestId = $params['requestId'] ?? null;\n },\n );\n\n $throttler->trackRequestStartFor($request->getUri());\n\n $this->navigate($request->getUri()->__toString());\n\n $throttler->trackRequestEndFor($request->getUri());\n\n $hookActionData = $this->callPostNavigateHooks();\n\n if (is_string($requestId) && $this->page && !$this->responseIsHtmlDocument($this->page)) {\n $html = $this->tryToGetRawResponseBody($this->page, $requestId) ?? $this->getHtmlFromPage();\n } else {\n $html = $this->getHtmlFromPage();\n }\n\n $this->addCookiesToJar($cookieJar, $request->getUri());\n\n return new RespondedRequest(\n $request,\n new Response($statusCode, $responseHeaders, $html),\n $hookActionData['screenshots'] ?? [],\n );\n }\n\n public function getOpenBrowser(): ?Browser\n {\n return $this->browser;\n }\n\n public function getOpenPage(): ?Page\n {\n return $this->page;\n }\n\n /**\n * @throws Exception\n */\n public function closeBrowser(): void\n {\n if ($this->browser) {\n if ($this->page) {\n $this->page->close();\n\n $this->page = null;\n }\n\n $this->browser->close();\n\n $this->browser = null;\n }\n }\n\n public function setExecutable(string $executable): static\n {\n $this->executable = $executable;\n\n return $this;\n }\n\n /**\n * @param array $options\n */\n public function setOptions(array $options): static\n {\n $this->options = $options;\n\n $this->optionsDirty = true;\n\n return $this;\n }\n\n /**\n * @param array $options\n */\n public function addOptions(array $options): static\n {\n foreach ($options as $key => $value) {\n $this->options[$key] = $value;\n }\n\n $this->optionsDirty = true;\n\n return $this;\n }\n\n public function waitForNavigationEvent(string $eventName): static\n {\n $this->waitForEvent = $eventName;\n\n return $this;\n }\n\n public function getTimeout(): int\n {\n return $this->timeout;\n }\n\n public function setTimeout(int $timeout): static\n {\n $this->timeout = $timeout;\n\n return $this;\n }\n\n /**\n * @param string[] $headers\n * @return string[]\n */\n public function sanitizeResponseHeaders(array $headers): array\n {\n foreach ($headers as $key => $value) {\n $headers[$key] = explode(PHP_EOL, $value)[0];\n }\n\n return $headers;\n }\n\n /**\n * @param string $scriptSource\n * @return $this\n */\n public function setPageInitScript(string $scriptSource): static\n {\n $this->pageInitScript = $scriptSource;\n\n return $this;\n }\n\n public function useNativeUserAgent(): static\n {\n $this->useNativeUserAgent = true;\n\n return $this;\n }\n\n public function includeShadowElementsInHtml(): static\n {\n $this->includeShadowElements = true;\n\n return $this;\n }\n\n /**\n * @throws OperationTimedOut\n * @throws CommunicationException\n * @throws NavigationExpired\n * @throws NoResponseAvailable\n * @throws InvalidResponse\n * @throws CannotReadResponse\n * @throws ResponseHasError\n */\n protected function navigate(string $url): void\n {\n if ($this->waitForEvent) {\n $this->page?->navigate($url)->waitForNavigation($this->waitForEvent, $this->timeout);\n } else {\n $this->page?->navigate($url)->waitForNavigation(timeout: $this->timeout);\n }\n }\n\n /**\n * @return array\n */\n protected function callPostNavigateHooks(): array\n {\n $returnData = [];\n\n if (!empty($this->tempPostNavigateHooks)) {\n foreach ($this->tempPostNavigateHooks as $hook) {\n $returnValue = $hook->call($this, $this->page, $this->logger);\n\n if ($returnValue instanceof Screenshot) {\n if (!array_key_exists('screenshots', $returnData)) {\n $returnData['screenshots'] = [$returnValue];\n } else {\n $returnData['screenshots'][] = $returnValue;\n }\n }\n }\n }\n\n $this->tempPostNavigateHooks = [];\n\n return $returnData;\n }\n\n /**\n * @throws CommunicationException\n * @throws OperationTimedOut\n * @throws NoResponseAvailable\n * @throws InvalidCookieException\n */\n protected function addCookiesToJar(?CookieJar $cookieJar, UriInterface $requestUrl): void\n {\n if (!$cookieJar) {\n return;\n }\n\n $cookies = $this->page?->getCookies();\n\n if ($cookies) {\n $cookieJar->addFrom($requestUrl, $cookies);\n }\n }\n\n /**\n * @throws Exception\n */\n protected function getBrowser(\n RequestInterface $request,\n ?string $proxy = null,\n ): Browser {\n if (!$this->browser || $this->shouldRenewBrowser($proxy)) {\n $this->closeBrowser();\n\n $options = $this->optionsFromRequest($request, $proxy);\n\n if (!$this->browserFactory) {\n $this->browserFactory = new BrowserFactory($this->executable);\n }\n\n $this->browser = $this->browserFactory->createBrowser($options);\n\n if ($this->pageInitScript) {\n $this->browser->setPagePreScript($this->pageInitScript);\n }\n\n $this->optionsDirty = false;\n }\n\n return $this->browser;\n }\n\n protected function shouldRenewBrowser(?string $proxy): bool\n {\n return $this->optionsDirty || ($proxy !== $this->proxy);\n }\n\n /**\n * @param RequestInterface $request\n * @return array\n */\n protected function optionsFromRequest(RequestInterface $request, ?string $proxy = null): array\n {\n $options = $this->options;\n\n if (isset($request->getHeader('User-Agent')[0]) && !$this->useNativeUserAgent) {\n $options['userAgent'] = $request->getHeader('User-Agent')[0];\n } elseif ($this->useNativeUserAgent && !empty($request->getHeader('User-Agent'))) {\n $request = $request->withoutHeader('User-Agent');\n }\n\n $options['headers'] = array_merge(\n $options['headers'] ?? [],\n $this->prepareRequestHeaders($request->getHeaders()),\n );\n\n if (!empty($proxy)) {\n $this->proxy = $options['proxyServer'] = $proxy;\n } else {\n $this->proxy = null;\n }\n\n return $options;\n }\n\n /**\n * @param mixed[] $headers\n * @return array\n */\n protected function prepareRequestHeaders(array $headers = []): array\n {\n $headers = $this->removeHeadersCausingErrorWithHeadlessBrowser($headers);\n\n return array_map(function ($headerValue) {\n return is_array($headerValue) ? implode(';', $headerValue) : $headerValue;\n }, $headers);\n }\n\n /**\n * @param mixed[] $headers\n * @return mixed[]\n */\n protected function removeHeadersCausingErrorWithHeadlessBrowser(array $headers = []): array\n {\n $removeHeaders = ['host'];\n\n foreach ($headers as $headerName => $headerValue) {\n if (in_array(strtolower($headerName), $removeHeaders, true)) {\n unset($headers[$headerName]);\n }\n }\n\n return $headers;\n }\n\n protected function responseIsHtmlDocument(?Page $page = null): bool\n {\n if (!$page) {\n return false;\n }\n\n try {\n return $page->evaluate(\n <<getReturnValue(3000);\n } catch (Throwable $e) {\n return true;\n }\n }\n\n /**\n * In production, retrieving the raw response body using the Network.getResponseBody message sometimes failed.\n * Waiting briefly before sending the message appeared to resolve the issue.\n * So, this method tries up to three times with a brief wait between each attempt.\n */\n protected function tryToGetRawResponseBody(Page $page, string $requestId): ?string\n {\n for ($i = 1; $i <= 3; $i++) {\n try {\n $message = $page->getSession()->sendMessageSync(new Message('Network.getResponseBody', [\n 'requestId' => $requestId,\n ]));\n\n if ($message->isSuccessful() && $message->getData()['result']['body']) {\n return $message->getData()['result']['body'];\n }\n } catch (Throwable) {\n }\n\n usleep($i * 100000);\n }\n\n return null;\n }\n\n /**\n * @throws CommunicationException\n * @throws JavascriptException\n */\n protected function getHtmlFromPage(): string\n {\n if ($this->page instanceof Page && $this->includeShadowElements) {\n try {\n // Found this script on\n // https://stackoverflow.com/questions/69867758/how-can-i-get-all-the-html-in-a-document-or-node-containing-shadowroot-elements\n return $this->page->evaluate(<<getReturnValue();\n } catch (Throwable) {\n return $this->page->getHtml();\n }\n }\n\n return $this->page?->getHtml() ?? '';\n }\n}\n" }, { "path": "src/Loader/Http/HttpLoader.php", "content": " 10,\n 'timeout' => 60,\n ];\n\n protected int $maxRedirects = 10;\n\n protected ?RetryManager $retryCachedErrorResponses = null;\n\n protected bool $writeOnlyCache = false;\n\n /**\n * @var array\n */\n protected array $cacheUrlFilters = [];\n\n protected bool $skipCacheForNextRequest = false;\n\n protected ?ProxyManager $proxies = null;\n\n /**\n * @param mixed[] $defaultGuzzleClientConfig\n */\n public function __construct(\n UserAgentInterface $userAgent,\n ?ClientInterface $httpClient = null,\n ?LoggerInterface $logger = null,\n ?Throttler $throttler = null,\n protected RetryErrorResponseHandler $retryErrorResponseHandler = new RetryErrorResponseHandler(),\n array $defaultGuzzleClientConfig = [],\n ) {\n parent::__construct($userAgent, $logger);\n\n $this->retryErrorResponseHandler->setLogger($this->logger);\n\n $this->httpClient = $httpClient ?? new Client($this->mergeClientConfigWithDefaults($defaultGuzzleClientConfig));\n\n $this->onSuccess(function (RequestInterface $request, ResponseInterface $response, LoggerInterface $logger) {\n $logger->info('Loaded ' . $request->getUri()->__toString());\n });\n\n $this->onError(function (RequestInterface $request, Exception|Error|ResponseInterface $exceptionOrResponse, $logger) {\n $logMessage = 'Failed to load ' . $request->getUri()->__toString() . ': ';\n\n if ($exceptionOrResponse instanceof ResponseInterface) {\n $logMessage .= 'got response ' . $exceptionOrResponse->getStatusCode() . ' - ' .\n $exceptionOrResponse->getReasonPhrase();\n } else {\n $logMessage .= $exceptionOrResponse->getMessage();\n }\n\n $logger->error($logMessage);\n });\n\n $this->cookieJar = new CookieJar();\n\n $this->throttler = $throttler ?? new Throttler();\n }\n\n /**\n * @param mixed $subject\n * @return RespondedRequest|null\n */\n public function load(mixed $subject): ?RespondedRequest\n {\n $this->_resetCalledHooks();\n\n try {\n $request = $this->validateSubjectType($subject);\n } catch (InvalidArgumentException|Exception $exception) {\n $url = $subject instanceof RequestInterface ? (string) $subject->getUri() : (string) $subject;\n\n $this->logger->error('Invalid input URL: ' . $url . ' - ' . $exception->getMessage());\n\n return null;\n }\n\n try {\n if (!$this->isAllowedToBeLoaded($request->getUri())) {\n return null;\n }\n\n $isFromCache = false;\n\n $respondedRequest = $this->tryLoading($request, $isFromCache);\n\n if ($respondedRequest->response->getStatusCode() < 400) {\n $this->callHook('onSuccess', $request, $respondedRequest->response);\n } else {\n $this->callHook('onError', $request, $respondedRequest->response);\n }\n\n if (!$isFromCache) {\n $this->addToCache($respondedRequest);\n }\n\n return $respondedRequest;\n } catch (Throwable $exception) {\n // Don't move to finally so hooks don't run before it.\n $this->throttler->trackRequestEndFor($request->getUri());\n\n $this->callHook('onError', $request, $exception);\n\n return null;\n } finally {\n $this->callHook('afterLoad', $request);\n\n $this->_resetCalledHooks();\n }\n }\n\n /**\n * @throws LoadingException|InvalidArgumentException|Exception\n */\n public function loadOrFail(mixed $subject): RespondedRequest\n {\n $this->_resetCalledHooks();\n\n $request = $this->validateSubjectType($subject);\n\n try {\n $this->isAllowedToBeLoaded($request->getUri(), true);\n\n $isFromCache = false;\n\n $respondedRequest = $this->tryLoading($request, $isFromCache);\n\n if ($respondedRequest->response->getStatusCode() >= 400) {\n throw LoadingException::make($request->getUri(), $respondedRequest->response->getStatusCode());\n }\n\n $this->callHook('onSuccess', $request, $respondedRequest->response);\n\n $this->callHook('afterLoad', $request);\n\n if (!$isFromCache) {\n $this->addToCache($respondedRequest);\n }\n\n return $respondedRequest;\n } catch (Throwable $exception) {\n $this->_resetCalledHooks();\n\n throw LoadingException::from($exception);\n }\n }\n\n public function dontUseCookies(): static\n {\n $this->useCookies = false;\n\n return $this;\n }\n\n public function flushCookies(): void\n {\n $this->cookieJar->flush();\n }\n\n public function useHeadlessBrowser(): static\n {\n $this->useHeadlessBrowser = true;\n\n return $this;\n }\n\n /**\n * @throws Exception\n */\n public function useHttpClient(): static\n {\n $this->useHeadlessBrowser = false;\n\n $this->browser()->closeBrowser();\n\n return $this;\n }\n\n public function usesHeadlessBrowser(): bool\n {\n return $this->useHeadlessBrowser;\n }\n\n public function setMaxRedirects(int $maxRedirects): static\n {\n $this->maxRedirects = $maxRedirects;\n\n return $this;\n }\n\n public function robotsTxt(): RobotsTxtHandler\n {\n if (!$this->robotsTxtHandler) {\n $this->robotsTxtHandler = new RobotsTxtHandler($this, $this->logger);\n }\n\n return $this->robotsTxtHandler;\n }\n\n public function throttle(): Throttler\n {\n return $this->throttler;\n }\n\n public function retryCachedErrorResponses(): RetryManager\n {\n $this->retryCachedErrorResponses = new RetryManager();\n\n return $this->retryCachedErrorResponses;\n }\n\n public function writeOnlyCache(): static\n {\n $this->writeOnlyCache = true;\n\n return $this;\n }\n\n public function cacheOnlyWhereUrl(FilterInterface $filter): static\n {\n $this->cacheUrlFilters[] = $filter;\n\n return $this;\n }\n\n /**\n * @throws Exception\n */\n public function useProxy(string $proxyUrl): void\n {\n $this->checkIfProxiesCanBeUsed();\n\n $this->proxies = new ProxyManager([$proxyUrl]);\n }\n\n /**\n * @param string[] $proxyUrls\n * @throws Exception\n */\n public function useRotatingProxies(array $proxyUrls): void\n {\n $this->checkIfProxiesCanBeUsed();\n\n $this->proxies = new ProxyManager($proxyUrls);\n }\n\n public function browser(): HeadlessBrowserLoaderHelper\n {\n if (!$this->browserHelper) {\n $this->browserHelper = new HeadlessBrowserLoaderHelper(logger: $this->logger);\n }\n\n return $this->browserHelper;\n }\n\n /**\n * @throws \\Psr\\SimpleCache\\InvalidArgumentException\n */\n public function addToCache(RespondedRequest $respondedRequest): void\n {\n if ($this->cache && $this->shouldResponseBeCached($respondedRequest)) {\n $this->cache->set($respondedRequest->cacheKey(), $respondedRequest);\n }\n }\n\n public function skipCacheForNextRequest(): static\n {\n $this->skipCacheForNextRequest = true;\n\n return $this;\n }\n\n /**\n * @throws LoadingException|Throwable|\\Psr\\SimpleCache\\InvalidArgumentException\n */\n protected function tryLoading(\n RequestInterface $request,\n bool &$isFromCache,\n ): RespondedRequest {\n $request = $this->prepareRequest($request);\n\n $this->callHook('beforeLoad', $request);\n\n $respondedRequest = $this->shouldRequestBeServedFromCache($request) ? $this->getFromCache($request) : null;\n\n if ($respondedRequest) {\n $isFromCache = true;\n\n $respondedRequest->setIsServedFromCache();\n\n $this->callHook('onCacheHit', $request, $respondedRequest->response);\n }\n\n $this->skipCacheForNextRequest = false;\n\n if (!$respondedRequest) {\n $respondedRequest = $this->waitForGoAndLoad($request);\n }\n\n return $respondedRequest;\n }\n\n /**\n * @throws ClientExceptionInterface\n * @throws GuzzleException\n * @throws LoadingException\n * @throws CommunicationException\n * @throws CannotReadResponse\n * @throws InvalidResponse\n * @throws ResponseHasError\n * @throws JavascriptException\n * @throws NavigationExpired\n * @throws NoResponseAvailable\n * @throws OperationTimedOut\n * @throws Exception\n */\n protected function waitForGoAndLoad(RequestInterface $request): RespondedRequest\n {\n $this->throttler->waitForGo($request->getUri());\n\n $respondedRequest = $this->loadViaClientOrHeadlessBrowser($request);\n\n if ($this->retryErrorResponseHandler->shouldWait($respondedRequest)) {\n $respondedRequest = $this->retryErrorResponseHandler->handleRetries(\n $respondedRequest,\n function () use ($request) {\n $request = $this->prepareRequest($request);\n\n return $this->loadViaClientOrHeadlessBrowser($request);\n },\n );\n }\n\n return $respondedRequest;\n }\n\n /**\n * @throws ClientExceptionInterface\n * @throws GuzzleException\n * @throws LoadingException\n * @throws CommunicationException\n * @throws CannotReadResponse\n * @throws InvalidResponse\n * @throws ResponseHasError\n * @throws JavascriptException\n * @throws NavigationExpired\n * @throws NoResponseAvailable\n * @throws OperationTimedOut\n */\n protected function loadViaClientOrHeadlessBrowser(RequestInterface $request): RespondedRequest\n {\n if ($this->useHeadlessBrowser) {\n $proxy = $this->proxies?->getProxy() ?? null;\n\n return $this->browser()->navigateToPageAndGetRespondedRequest(\n $request,\n $this->throttler,\n $proxy,\n $this->useCookies ? $this->cookieJar : null,\n );\n }\n\n return $this->handleRedirects($request);\n }\n\n /**\n * @throws ClientExceptionInterface\n * @throws LoadingException\n * @throws GuzzleException\n * @throws Exception\n */\n protected function handleRedirects(\n RequestInterface $request,\n ?RespondedRequest $respondedRequest = null,\n int $redirectNumber = 0,\n ): RespondedRequest {\n if ($redirectNumber >= $this->maxRedirects) {\n throw new LoadingException('Too many redirects.');\n }\n\n if (!$respondedRequest) {\n $this->throttler->trackRequestStartFor($request->getUri());\n }\n\n if ($this->proxies && $this->httpClient instanceof Client) {\n $response = $this->sendProxiedRequestUsingGuzzle($request, $this->httpClient);\n } else {\n $response = $this->httpClient->sendRequest($request);\n }\n\n if (!$respondedRequest) {\n $respondedRequest = new RespondedRequest($request, $response);\n } else {\n $respondedRequest->setResponse($response);\n }\n\n $this->addCookiesToJar($respondedRequest);\n\n if ($respondedRequest->isRedirect()) {\n $this->logger()->info('Load redirect to: ' . $respondedRequest->effectiveUri());\n\n $newRequest = $request->withUri(Url::parsePsr7($respondedRequest->effectiveUri()));\n\n $redirectNumber++;\n\n return $this->handleRedirects($newRequest, $respondedRequest, $redirectNumber);\n } else {\n $this->throttler->trackRequestEndFor($respondedRequest->request->getUri());\n }\n\n return $respondedRequest;\n }\n\n /**\n * @throws GuzzleException\n */\n protected function sendProxiedRequestUsingGuzzle(RequestInterface $request, Client $client): ResponseInterface\n {\n return $client->request(\n $request->getMethod(),\n $request->getUri(),\n [\n 'headers' => $request->getHeaders(),\n 'proxy' => $this->proxies?->getProxy(),\n 'version' => $request->getProtocolVersion(),\n 'body' => $request->getBody(),\n ],\n );\n }\n\n /**\n * @return void\n * @throws Exception\n */\n protected function checkIfProxiesCanBeUsed(): void\n {\n if (!$this->usesHeadlessBrowser() && !$this->httpClient instanceof Client) {\n throw new Exception(\n 'The included proxy feature can only be used when using a guzzle HTTP client or headless chrome ' .\n 'browser for loading.',\n );\n }\n }\n\n /**\n * @param mixed[] $config\n * @return mixed[]\n */\n protected function mergeClientConfigWithDefaults(array $config): array\n {\n $merged = $this->defaultGuzzleClientConfig;\n\n foreach ($config as $key => $value) {\n $merged[$key] = $value;\n }\n\n return $merged;\n }\n\n /**\n * @throws LoadingException\n * @throws Exception\n */\n protected function isAllowedToBeLoaded(UriInterface $uri, bool $throwsException = false): bool\n {\n if (!$this->robotsTxt()->isAllowed($uri)) {\n $message = 'Crawler is not allowed to load ' . $uri . ' according to robots.txt file.';\n\n $this->logger->warning($message);\n\n if ($throwsException) {\n throw new LoadingException($message);\n }\n\n return false;\n }\n\n return true;\n }\n\n /**\n * @throws \\Psr\\SimpleCache\\InvalidArgumentException\n * @throws Exception\n */\n protected function getFromCache(RequestInterface $request): ?RespondedRequest\n {\n if (!$this->cache || $this->writeOnlyCache) {\n return null;\n }\n\n $key = RequestKey::from($request);\n\n if ($this->cache->has($key)) {\n $this->logger->info('Found ' . $request->getUri()->__toString() . ' in cache.');\n\n $respondedRequest = $this->cache->get($key);\n\n // Previously, until v0.7 just used serialized arrays. Leave this for backwards compatibility.\n if (is_array($respondedRequest)) {\n $respondedRequest = RespondedRequest::fromArray($respondedRequest);\n }\n\n if ($this->retryCachedErrorResponses?->shallBeRetried($respondedRequest->response->getStatusCode())) {\n $this->logger->info('Cached response was an error response, retry.');\n\n return null;\n }\n\n return $respondedRequest;\n }\n\n return null;\n }\n\n protected function shouldResponseBeCached(RespondedRequest $respondedRequest): bool\n {\n if (!empty($this->cacheUrlFilters)) {\n foreach ($this->cacheUrlFilters as $filter) {\n $noUrlMatched = true;\n\n foreach ($respondedRequest->allUris() as $url) {\n if ($filter->evaluate($url)) {\n $noUrlMatched = false;\n }\n }\n\n if ($noUrlMatched) {\n return false;\n }\n }\n }\n\n return true;\n }\n\n protected function shouldRequestBeServedFromCache(RequestInterface $request): bool\n {\n if ($this->skipCacheForNextRequest === true) {\n return false;\n }\n\n if (!empty($this->cacheUrlFilters)) {\n foreach ($this->cacheUrlFilters as $filter) {\n if (!$filter->evaluate((string) $request->getUri())) {\n return false;\n }\n }\n }\n\n return true;\n }\n\n /**\n * @throws InvalidArgumentException|Exception\n */\n protected function validateSubjectType(RequestInterface|string $requestOrUri): RequestInterface\n {\n if (is_string($requestOrUri)) {\n try {\n $url = Url::parse($requestOrUri);\n\n if ($url->isRelativeReference()) {\n throw new InvalidArgumentException(\n 'The URI is a relative reference and therefore can\\'t be loaded.',\n );\n }\n\n return new Request('GET', $url->toPsr7());\n } catch (InvalidUrlException) {\n throw new InvalidArgumentException('Invalid URL.');\n }\n } elseif (\n empty(trim($requestOrUri->getUri()->getScheme())) &&\n Url::parse($requestOrUri->getUri())->isRelativeReference()\n ) {\n throw new InvalidArgumentException('The URI is a relative reference and therefore can\\'t be loaded.');\n }\n\n return $requestOrUri;\n }\n\n /**\n * @throws Exception\n */\n protected function prepareRequest(RequestInterface $request): RequestInterface\n {\n $request = $request->withHeader('User-Agent', $this->userAgent->__toString());\n\n // When writing tests I found that guzzle somehow messed up headers with multiple strings as value in the PSR-7\n // request object. It sent only the last part of the array, instead of concatenating the array of strings to a\n // comma separated string. Don't know if that happens with all handlers (curl, stream), will investigate\n // further. But until this is fixed, we just prepare the headers ourselves.\n foreach ($request->getHeaders() as $headerName => $headerValues) {\n $request = $request->withHeader($headerName, $request->getHeaderLine($headerName));\n }\n\n return $this->addCookiesToRequest($request);\n }\n\n protected function addCookiesToJar(RespondedRequest $respondedRequest): void\n {\n if ($this->useCookies) {\n try {\n $this->cookieJar->addFrom($respondedRequest->effectiveUri(), $respondedRequest->response);\n } catch (Exception $exception) {\n $this->logger->warning('Problem when adding cookies to the Jar: ' . $exception->getMessage());\n }\n }\n }\n\n /**\n * @throws Exception\n */\n protected function addCookiesToRequest(RequestInterface $request): RequestInterface\n {\n if (!$this->useCookies) {\n return $request;\n }\n\n foreach ($this->cookieJar->getFor($request->getUri()) as $cookie) {\n $request = $request->withAddedHeader('Cookie', $cookie->__toString());\n }\n\n return $request;\n }\n}\n" }, { "path": "src/Loader/Http/Messages/RespondedRequest.php", "content": "setResponse($this->response);\n }\n\n /**\n * @param mixed[] $data\n * @return RespondedRequest\n * @throws Exception\n */\n public static function fromArray(array $data): RespondedRequest\n {\n $respondedRequest = new RespondedRequest(\n self::requestFromArray($data),\n self::responseFromArray($data),\n self::screenshotsFromArray($data),\n );\n\n if ($data['effectiveUri'] && $data['effectiveUri'] !== $data['requestUri']) {\n $respondedRequest->addRedirectUri($data['effectiveUri']);\n }\n\n return $respondedRequest;\n }\n\n /**\n * @return mixed[]\n * @throws MissingZlibExtensionException\n */\n public function __serialize(): array\n {\n return [\n 'requestMethod' => $this->request->getMethod(),\n 'requestUri' => $this->request->getUri()->__toString(),\n 'requestHeaders' => $this->request->getHeaders(),\n 'requestBody' => Http::getBodyString($this->request),\n 'effectiveUri' => $this->effectiveUri(),\n 'responseStatusCode' => $this->response->getStatusCode(),\n 'responseHeaders' => $this->response->getHeaders(),\n 'responseBody' => Http::getBodyString($this->response),\n 'screenshots' => array_map(fn(Screenshot $screenshot) => $screenshot->path, $this->screenshots),\n ];\n }\n\n /**\n * @return mixed[]\n * @throws MissingZlibExtensionException\n */\n public function toArrayForResult(): array\n {\n $serialized = $this->__serialize();\n\n $mapping = [\n 'url' => 'effectiveUri',\n 'uri' => 'effectiveUri',\n 'status' => 'responseStatusCode',\n 'headers' => 'responseHeaders',\n 'body' => 'responseBody',\n ];\n\n foreach ($mapping as $newKey => $originalKey) {\n $serialized[$newKey] = $serialized[$originalKey];\n }\n\n return $serialized;\n }\n\n /**\n * @param mixed[] $data\n * @throws Exception\n */\n public function __unserialize(array $data): void\n {\n $this->request = self::requestFromArray($data);\n\n $this->response = self::responseFromArray($data);\n\n if ($data['effectiveUri'] && $data['effectiveUri'] !== $data['requestUri']) {\n $this->addRedirectUri($data['effectiveUri']);\n }\n\n $this->screenshots = self::screenshotsFromArray($data);\n }\n\n public function effectiveUri(): string\n {\n return empty($this->redirects) ? $this->requestedUri() : end($this->redirects);\n }\n\n public function requestedUri(): string\n {\n return $this->request->getUri();\n }\n\n /**\n * @return array\n */\n public function allUris(): array\n {\n $uris = [$this->requestedUri() => $this->requestedUri()];\n\n foreach ($this->redirects as $redirect) {\n $uris[$redirect] = $redirect;\n }\n\n return array_values($uris);\n }\n\n public function isRedirect(): bool\n {\n return $this->response->getStatusCode() >= 300 && $this->response->getStatusCode() < 400;\n }\n\n /**\n * @return string[]\n */\n public function redirects(): array\n {\n return $this->redirects;\n }\n\n /**\n * @throws Exception\n */\n public function setResponse(ResponseInterface $response): void\n {\n $this->response = $response;\n\n if ($this->isRedirect()) {\n $this->addRedirectUri();\n }\n }\n\n /**\n * @throws Exception\n */\n public function addRedirectUri(?string $redirectUri = null): void\n {\n $redirectUri = Url::parse($this->effectiveUri())\n ->resolve($redirectUri ?? $this->response->getHeaderLine('Location'))\n ->__toString();\n\n // Add it only if different from the previous one.\n if ($redirectUri !== end($this->redirects)) {\n $this->redirects[] = $redirectUri;\n }\n }\n\n public function cacheKey(): string\n {\n return RequestKey::from($this->request);\n }\n\n public function isServedFromCache(): bool\n {\n return $this->isServedFromCache;\n }\n\n public function setIsServedFromCache(bool $value = true): void\n {\n $this->isServedFromCache = $value;\n }\n\n /**\n * @param mixed[] $data\n */\n protected static function requestFromArray(array $data): Request\n {\n return new Request(\n $data['requestMethod'],\n $data['requestUri'],\n $data['requestHeaders'],\n $data['requestBody'],\n );\n }\n\n /**\n * @param mixed[] $data\n */\n protected static function responseFromArray(array $data): Response\n {\n return new Response(\n $data['responseStatusCode'],\n $data['responseHeaders'],\n $data['responseBody'],\n );\n }\n\n /**\n * @param mixed[] $data\n * @return Screenshot[]\n */\n protected static function screenshotsFromArray(array $data): array\n {\n $screenshots = [];\n\n if (array_key_exists('screenshots', $data)) {\n foreach ($data['screenshots'] as $screenshot) {\n if (file_exists($screenshot)) {\n $screenshots[] = new Screenshot($screenshot);\n }\n }\n }\n\n return $screenshots;\n }\n}\n" }, { "path": "src/Loader/Http/Politeness/RetryErrorResponseHandler.php", "content": "\n */\n protected array $waitErrors = [\n 429 => 'Too many Requests',\n 503 => 'Service Unavailable',\n ];\n\n /**\n * @param int[] $wait\n */\n public function __construct(\n protected int $retries = 2,\n protected array $wait = [10, 60],\n protected int $maxWait = 60,\n ) {}\n\n public function shouldWait(RespondedRequest $respondedRequest): bool\n {\n if (array_key_exists($respondedRequest->response->getStatusCode(), $this->waitErrors)) {\n return true;\n }\n\n return false;\n }\n\n public function setLogger(LoggerInterface $logger): void\n {\n $this->logger = $logger;\n }\n\n /**\n * @throws LoadingException\n */\n public function handleRetries(\n RespondedRequest $respondedRequest,\n Closure $retryCallback,\n ): RespondedRequest {\n $this->logReceivedErrorResponseMessage($respondedRequest);\n\n $retries = 0;\n\n $this->wait[0] = $this->getWaitTimeFromResponse($respondedRequest->response) ?? $this->wait[0];\n\n while ($retries < $this->retries) {\n $this->logWaitForRetryMessage($retries);\n\n sleep($this->wait[$retries]);\n\n $respondedRequest = $retryCallback();\n\n if ($respondedRequest instanceof RespondedRequest && !$this->shouldWait($respondedRequest)) {\n return $respondedRequest;\n } elseif ($respondedRequest) {\n $this->logRepeatedErrorMessage($respondedRequest);\n }\n\n $retries++;\n }\n\n $this->logger?->error('Stop crawling');\n\n throw new LoadingException('Stopped crawling because of repeated error responses.');\n }\n\n /**\n * @throws LoadingException\n */\n protected function getWaitTimeFromResponse(ResponseInterface $response): ?int\n {\n $retryAfterHeader = $response->getHeader('Retry-After');\n\n if (!empty($retryAfterHeader)) {\n $retryAfterHeader = reset($retryAfterHeader);\n\n if (is_numeric($retryAfterHeader)) {\n $waitFor = (int) $retryAfterHeader;\n\n if ($waitFor > $this->maxWait) {\n $this->retryAfterExceedsLimitMessage($response);\n }\n\n return (int) $retryAfterHeader;\n }\n }\n\n return null;\n }\n\n protected function getResponseCodeAndReasonPhrase(RespondedRequest|ResponseInterface $respondedRequest): string\n {\n $response = $respondedRequest instanceof RespondedRequest ? $respondedRequest->response : $respondedRequest;\n\n $statusCode = $response->getStatusCode();\n\n if (array_key_exists($statusCode, $this->waitErrors)) {\n return $statusCode . ' (' . $this->waitErrors[$statusCode] . ')';\n }\n\n return '?';\n }\n\n protected function logReceivedErrorResponseMessage(RespondedRequest $respondedRequest): void\n {\n $statusCodeAndReasonPhrase = $this->getResponseCodeAndReasonPhrase($respondedRequest);\n\n $this->logger?->warning(\n 'Request to ' . $respondedRequest->requestedUri() . ' returned ' . $statusCodeAndReasonPhrase,\n );\n }\n\n protected function logWaitForRetryMessage(int $retryNumber): void\n {\n $this->logger?->warning('Will wait for ' . $this->wait[$retryNumber] . ' seconds and then retry');\n }\n\n protected function logRepeatedErrorMessage(RespondedRequest $respondedRequest): void\n {\n $statusCodeAndReasonPhrase = $this->getResponseCodeAndReasonPhrase($respondedRequest);\n\n $this->logger?->warning('Retry again received an error response: ' . $statusCodeAndReasonPhrase);\n }\n\n /**\n * @throws LoadingException\n */\n protected function retryAfterExceedsLimitMessage(ResponseInterface $response): string\n {\n $statusCodeAndReasonPhrase = $this->getResponseCodeAndReasonPhrase($response);\n\n $message = 'Retry-After header in ' . $statusCodeAndReasonPhrase . ' response, requires to wait longer ' .\n 'than the defined max wait time for this case. If you want to increase this limit, set it ' .\n 'in the ErrorResponseHandler of your HttpLoader instance.';\n\n $this->logger?->error($message);\n\n throw new LoadingException($message);\n }\n}\n" }, { "path": "src/Loader/Http/Politeness/RobotsTxtHandler.php", "content": "\n */\n protected array $robotsTxts = [];\n\n protected bool $ignoreWildcardRules = false;\n\n public function __construct(\n protected Loader $loader,\n protected ?LoggerInterface $logger = null,\n ) {\n $this->userAgent = $this->loader->userAgent();\n }\n\n public function ignoreWildcardRules(): void\n {\n $this->ignoreWildcardRules = true;\n }\n\n /**\n * @throws Exception\n */\n public function isAllowed(string|UriInterface|Url $url): bool\n {\n if (!$this->userAgent instanceof BotUserAgent) {\n return true;\n }\n\n $url = $this->getUrlInstance($url);\n\n if ($url->path() === '/robots.txt') {\n return true;\n }\n\n $robotsTxt = $this->getRobotsTxtFor($url);\n\n if ($this->ignoreWildcardRules) {\n return !$robotsTxt->isExplicitlyNotAllowedFor($url, $this->userAgent->productToken());\n }\n\n return $robotsTxt->isAllowed($url, $this->userAgent->productToken());\n }\n\n /**\n * @return string[]\n * @throws InvalidRobotsTxtFileException\n */\n public function getSitemaps(string|UriInterface|Url $url): array\n {\n return $this->getRobotsTxtFor($url)->sitemaps();\n }\n\n /**\n * @throws InvalidRobotsTxtFileException|Exception\n */\n protected function getRobotsTxtFor(string|UriInterface|Url $url): RobotsTxt\n {\n $url = $this->getUrlInstance($url);\n\n $root = $url->root();\n\n if (isset($this->robotsTxts[$root])) {\n return $this->robotsTxts[$root];\n }\n\n $robotsTxtContent = $this->loadRobotsTxtContent($root . '/robots.txt');\n\n try {\n $this->robotsTxts[$root] = RobotsTxt::parse($robotsTxtContent);\n } catch (Exception $exception) {\n $this->logger?->warning('Failed to parse robots.txt: ' . $exception->getMessage());\n\n $this->robotsTxts[$root] = RobotsTxt::parse('');\n }\n\n return $this->robotsTxts[$root];\n }\n\n protected function loadRobotsTxtContent(string $robotsTxtUrl): string\n {\n $usedHeadlessBrowser = false;\n\n if ($this->loader instanceof HttpLoader) {\n // If loader is set to use headless browser, temporary switch to using PSR-18 HTTP Client.\n $usedHeadlessBrowser = $this->loader->usesHeadlessBrowser();\n\n $this->loader->useHttpClient();\n }\n\n $response = $this->loader->load($robotsTxtUrl);\n\n if ($this->loader instanceof HttpLoader && $usedHeadlessBrowser) {\n $this->loader->useHeadlessBrowser();\n }\n\n return $response ? Http::getBodyString($response) : '';\n }\n\n protected function getUrlInstance(string|UriInterface|Url $url): Url\n {\n if (is_string($url) || $url instanceof UriInterface) {\n return Url::parse($url);\n }\n\n return $url;\n }\n}\n" }, { "path": "src/Loader/Http/Politeness/Throttler.php", "content": "\n */\n protected array $latestRequestTimes = [];\n\n /**\n * @var array\n */\n protected array $latestResponseTimes = [];\n\n /**\n * @var array\n */\n protected array $latestDurations = [];\n\n protected Microseconds|MultipleOf $from;\n\n protected Microseconds|MultipleOf $to;\n\n protected Microseconds $min;\n\n /**\n * @var string[]\n */\n private array $_currentRequestUrls = [];\n\n /**\n * @throws InvalidArgumentException\n */\n public function __construct(\n Microseconds|MultipleOf|null $from = null,\n Microseconds|MultipleOf|null $to = null,\n ?Microseconds $min = null,\n protected ?Microseconds $max = null,\n ) {\n $this->from = $from ?? new MultipleOf(1.0);\n\n $this->to = $to ?? new MultipleOf(2.0);\n\n $this->validateFromAndTo();\n\n $this->min = $min ?? Microseconds::fromSeconds(0.25);\n }\n\n /**\n * @throws InvalidArgumentException\n */\n public function waitBetween(Microseconds|MultipleOf $from, Microseconds|MultipleOf $to): static\n {\n $this->from = $from;\n\n $this->to = $to;\n\n $this->validateFromAndTo();\n\n return $this;\n }\n\n public function waitAtLeast(Microseconds $seconds): static\n {\n $this->min = $seconds;\n\n return $this;\n }\n\n public function waitAtMax(Microseconds $seconds): static\n {\n $this->max = $seconds;\n\n return $this;\n }\n\n /**\n * @throws Exception\n */\n public function trackRequestStartFor(UriInterface $url): void\n {\n $domain = $this->getDomain($url);\n\n $this->latestRequestTimes[$domain] = $this->time();\n\n $this->_internalTrackStartFor($url);\n }\n\n /**\n * @throws Exception\n */\n public function trackRequestEndFor(UriInterface $url): void\n {\n if (!$this->_requestToUrlWasStarted($url)) {\n return;\n }\n\n $domain = $this->getDomain($url);\n\n if (!isset($this->latestRequestTimes[$domain])) {\n return;\n }\n\n $this->latestResponseTimes[$domain] = $responseTime = $this->time();\n\n $this->latestDurations[$domain] = $responseTime->subtract($this->latestRequestTimes[$domain]);\n\n unset($this->latestRequestTimes[$domain]);\n\n $this->_internalTrackEndFor($url);\n }\n\n /**\n * @throws Exception\n */\n public function waitForGo(UriInterface $url): void\n {\n $domain = $this->getDomain($url);\n\n if (!isset($this->latestDurations[$domain])) {\n return;\n }\n\n $waitUntil = $this->calcWaitUntil($this->latestDurations[$domain], $this->latestResponseTimes[$domain]);\n\n $now = $this->time();\n\n if ($now->isGreaterThanOrEqual($waitUntil)) {\n return;\n }\n\n $wait = $waitUntil->subtract($now);\n\n usleep($wait->value);\n }\n\n protected function time(): Microseconds\n {\n return Microseconds::fromSeconds(microtime(true));\n }\n\n /**\n * @throws Exception\n */\n protected function getDomain(UriInterface $url): string\n {\n $domain = Url::parse($url)->domain();\n\n if (!$domain) {\n $domain = $url->getHost();\n }\n\n if (!is_string($domain)) {\n $domain = '*';\n }\n\n return $domain;\n }\n\n protected function calcWaitUntil(\n Microseconds $latestResponseDuration,\n Microseconds $latestResponseTime,\n ): Microseconds {\n $from = $this->from instanceof MultipleOf ? $this->from->calc($latestResponseDuration) : $this->from;\n\n $to = $this->to instanceof MultipleOf ? $this->to->calc($latestResponseDuration) : $this->to;\n\n $waitValue = $this->getRandBetween($from, $to);\n\n if ($this->min->isGreaterThan($waitValue)) {\n $waitValue = $this->min;\n }\n\n if ($this->max && $this->max->isLessThan($waitValue)) {\n $waitValue = $this->max;\n }\n\n return $latestResponseTime->add($waitValue);\n }\n\n protected function getRandBetween(Microseconds $from, Microseconds $to): Microseconds\n {\n if ($from->equals($to)) {\n return $from;\n }\n\n return new Microseconds(rand($from->value, $to->value));\n }\n\n /**\n * @internal\n */\n protected function _internalTrackStartFor(UriInterface $url): void\n {\n $urlString = (string) $url;\n\n $this->_currentRequestUrls[$urlString] = $urlString;\n }\n\n /**\n * @internal\n */\n protected function _internalTrackEndFor(UriInterface $url): void\n {\n unset($this->_currentRequestUrls[(string) $url]);\n }\n\n protected function _requestToUrlWasStarted(UriInterface $url): bool\n {\n $urlString = (string) $url;\n\n if (array_key_exists($urlString, $this->_currentRequestUrls)) {\n return true;\n }\n\n return false;\n }\n\n protected function validateFromAndTo(): void\n {\n if (!$this->fromAndToAreOfSameType()) {\n throw new InvalidArgumentException('From and to values must be of the same type (Seconds or MultipleOf).');\n }\n\n if ($this->fromIsGreaterThanTo()) {\n throw new InvalidArgumentException('From value can\\'t be greater than to value.');\n }\n }\n\n protected function fromAndToAreOfSameType(): bool\n {\n return ($this->from instanceof Microseconds && $this->to instanceof Microseconds) ||\n ($this->from instanceof MultipleOf && $this->to instanceof MultipleOf);\n }\n\n protected function fromIsGreaterThanTo(): bool\n {\n if ($this->from instanceof Microseconds && $this->to instanceof Microseconds) {\n return $this->from->isGreaterThan($this->to);\n }\n\n if ($this->from instanceof MultipleOf && $this->to instanceof MultipleOf) {\n return $this->from->factorIsGreaterThan($this->to);\n }\n\n return false;\n }\n}\n" }, { "path": "src/Loader/Http/Politeness/TimingUnits/MultipleOf.php", "content": "factor, 2) * 100);\n\n $result = (int) round(($microseconds->value * $factorTwoDecimalsAsInt) / 100);\n\n return new Microseconds($result);\n }\n\n public function factorIsGreaterThan(MultipleOf $multipleOf): bool\n {\n return $this->factor > $multipleOf->factor;\n }\n}\n" }, { "path": "src/Loader/Http/ProxyManager.php", "content": "proxies = array_values($this->proxies);\n }\n\n public function singleProxy(): bool\n {\n return count($this->proxies) === 1;\n }\n\n public function hasOnlySingleProxy(): bool\n {\n return count($this->proxies) === 1;\n }\n\n public function hasMultipleProxies(): bool\n {\n return count($this->proxies) > 1;\n }\n\n public function getProxy(): string\n {\n if ($this->hasOnlySingleProxy()) {\n return $this->proxies[0];\n }\n\n if ($this->lastUsedProxy === null || !isset($this->proxies[$this->lastUsedProxy + 1])) {\n $this->lastUsedProxy = 0;\n } else {\n $this->lastUsedProxy += 1;\n }\n\n return $this->proxies[$this->lastUsedProxy];\n }\n}\n" }, { "path": "src/Loader/Loader.php", "content": "\n */\n protected array $hooks = [\n 'beforeLoad' => [],\n 'onCacheHit' => [],\n 'onSuccess' => [],\n 'onError' => [],\n 'afterLoad' => [],\n ];\n\n /**\n * @var array\n */\n private array $_hooksCalledInCurrentLoadCall = [];\n\n public function __construct(\n protected UserAgentInterface $userAgent,\n ?LoggerInterface $logger = null,\n ) {\n $this->logger = $logger ?? new CliLogger();\n }\n\n public function beforeLoad(callable $callback): void\n {\n $this->addHookCallback('beforeLoad', $callback);\n }\n\n public function onCacheHit(callable $callback): void\n {\n $this->addHookCallback('onCacheHit', $callback);\n }\n\n public function onSuccess(callable $callback): void\n {\n $this->addHookCallback('onSuccess', $callback);\n }\n\n public function onError(callable $callback): void\n {\n $this->addHookCallback('onError', $callback);\n }\n\n public function afterLoad(callable $callback): void\n {\n $this->addHookCallback('afterLoad', $callback);\n }\n\n public function setCache(CacheInterface $cache): static\n {\n $this->cache = $cache;\n\n return $this;\n }\n\n public function userAgent(): UserAgentInterface\n {\n return $this->userAgent;\n }\n\n /**\n * Can be implemented in a child class to check if it is allowed to load a certain uri (e.g. check robots.txt)\n * Throw a LoadingException when it's not allowed and $throwsException is set to true.\n */\n protected function isAllowedToBeLoaded(UriInterface $uri, bool $throwsException = false): bool\n {\n return true;\n }\n\n protected function callHook(string $hook, mixed ...$arguments): void\n {\n if (!array_key_exists($hook, $this->hooks)) {\n return;\n }\n\n if (array_key_exists($hook, $this->_hooksCalledInCurrentLoadCall)) {\n $this->logger->warning(\n $hook . ' was already called in this load call. Probably a problem in the loader implementation.',\n );\n }\n\n if (\n $hook === 'afterLoad' &&\n !empty($this->hooks[$hook]) &&\n !array_key_exists('beforeLoad', $this->_hooksCalledInCurrentLoadCall)\n ) {\n $this->logger->warning(\n 'The afterLoad hook was called without a preceding call to the beforeLoad hook. Therefore don\\'t ' .\n 'run the hook callbacks. Most likely an exception/error occurred before the beforeLoad hook call.',\n );\n\n return;\n }\n\n $arguments[] = $this->logger;\n\n foreach ($this->hooks[$hook] as $callback) {\n call_user_func($callback, ...$arguments);\n }\n\n $this->_hooksCalledInCurrentLoadCall[$hook] = true;\n }\n\n protected function logger(): LoggerInterface\n {\n return $this->logger;\n }\n\n protected function addHookCallback(string $hook, callable $callback): void\n {\n $this->hooks[$hook][] = $callback;\n }\n\n /**\n * @internal\n * @return void\n */\n protected function _resetCalledHooks(): void\n {\n $this->_hooksCalledInCurrentLoadCall = [];\n }\n}\n" }, { "path": "src/Loader/LoaderInterface.php", "content": "log('emergency', $message, $context);\n }\n\n public function alert(string|Stringable $message, array $context = []): void\n {\n $this->log('alert', $message, $context);\n }\n\n public function critical(string|Stringable $message, array $context = []): void\n {\n $this->log('critical', $message, $context);\n }\n\n public function error(string|Stringable $message, array $context = []): void\n {\n $this->log('error', $message, $context);\n }\n\n public function warning(string|Stringable $message, array $context = []): void\n {\n $this->log('warning', $message, $context);\n }\n\n public function notice(string|Stringable $message, array $context = []): void\n {\n $this->log('notice', $message, $context);\n }\n\n public function info(string|Stringable $message, array $context = []): void\n {\n $this->log('info', $message, $context);\n }\n\n public function debug(string|Stringable $message, array $context = []): void\n {\n $this->log('debug', $message, $context);\n }\n\n /**\n * @param mixed $level\n * @param mixed[] $context\n */\n public function log($level, string|Stringable $message, array $context = []): void\n {\n if (!is_string($level)) {\n throw new InvalidArgumentException('Level must be string.');\n }\n\n if (!in_array($level, ['emergency', 'alert', 'critical', 'error', 'warning', 'notice', 'info', 'debug'], true)) {\n throw new UnexpectedValueException('Unknown log level.');\n }\n\n $this->printTimeAndLevel($level);\n echo $message . \"\\n\";\n }\n\n protected function printTimeAndLevel(string $level): void\n {\n echo $this->time() . \" \\033[0;\" . $this->levelColor($level) . \"m[\" . strtoupper($level) . \"]\\033[0m \";\n }\n\n protected function time(): string\n {\n return (new DateTime())->format('H:i:s:u');\n }\n\n protected function levelColor(string $level): string\n {\n $levelColors = [\n 'emergency' => '91', // bright red\n 'alert' => '91',\n 'critical' => '91',\n 'error' => '31', // red\n 'warning' => '36', // cyan\n 'notice' => '34', // blue\n 'info' => '32', // green\n 'debug' => '33', // yellow\n ];\n\n return $levelColors[$level];\n }\n}\n" }, { "path": "src/Logger/PreStepInvocationLogger.php", "content": ">\n */\n public array $messages = [];\n\n public function emergency(string|Stringable $message, array $context = []): void\n {\n $this->log('emergency', $message, $context);\n }\n\n public function alert(string|Stringable $message, array $context = []): void\n {\n $this->log('alert', $message, $context);\n }\n\n public function critical(string|Stringable $message, array $context = []): void\n {\n $this->log('critical', $message, $context);\n }\n\n public function error(string|Stringable $message, array $context = []): void\n {\n $this->log('error', $message, $context);\n }\n\n public function warning(string|Stringable $message, array $context = []): void\n {\n $this->log('warning', $message, $context);\n }\n\n public function notice(string|Stringable $message, array $context = []): void\n {\n $this->log('notice', $message, $context);\n }\n\n public function info(string|Stringable $message, array $context = []): void\n {\n $this->log('info', $message, $context);\n }\n\n public function debug(string|Stringable $message, array $context = []): void\n {\n $this->log('debug', $message, $context);\n }\n\n /**\n * @param mixed $level\n * @param mixed[] $context\n */\n public function log($level, string|Stringable $message, array $context = []): void\n {\n if (!is_string($level)) {\n throw new InvalidArgumentException('Level must be string.');\n }\n\n if (!in_array($level, ['emergency', 'alert', 'critical', 'error', 'warning', 'notice', 'info', 'debug'], true)) {\n throw new UnexpectedValueException('Unknown log level.');\n }\n\n $this->messages[] = ['level' => $level, 'message' => $message];\n }\n\n public function passToOtherLogger(LoggerInterface $logger): void\n {\n foreach ($this->messages as $message) {\n $logger->{$message['level']}($message['message']);\n }\n }\n}\n" }, { "path": "src/Output.php", "content": "data = $result->data;\n }\n }\n\n public function set(string $key, mixed $value): self\n {\n if ($key === '') {\n $key = $this->getUnnamedKey();\n }\n\n if (array_key_exists($key, $this->data)) {\n if (!is_array($this->data[$key]) || $this->isAssociativeArray($this->data[$key])) {\n $this->data[$key] = [$this->data[$key], $value];\n } else {\n $this->data[$key][] = $value;\n }\n } else {\n $this->data[$key] = $value;\n }\n\n return $this;\n }\n\n public function has(string $key): bool\n {\n return array_key_exists($key, $this->data);\n }\n\n public function get(string $key, mixed $default = null): mixed\n {\n if ($this->has($key)) {\n return $this->data[$key];\n }\n\n return $default;\n }\n\n /**\n * @return mixed[]\n */\n public function toArray(): array\n {\n $data = OutputTypeHelper::recursiveChildObjectsToArray($this->data);\n\n if (\n count($data) === 1 &&\n str_contains('unnamed', array_key_first($data)) &&\n OutputTypeHelper::isAssociativeArray($data[array_key_first($data)])\n ) {\n return $data[array_key_first($data)];\n }\n\n return $data;\n }\n\n private function getUnnamedKey(): string\n {\n $i = 1;\n\n while ($this->get('unnamed' . $i) !== null) {\n $i++;\n }\n\n return 'unnamed' . $i;\n }\n\n /**\n * @param mixed[] $array\n */\n private function isAssociativeArray(array $array): bool\n {\n foreach ($array as $key => $value) {\n return is_string($key);\n }\n\n return false;\n }\n}\n" }, { "path": "src/Steps/BaseStep.php", "content": "\n */\n protected array $subCrawlers = [];\n\n protected ?LoggerInterface $logger = null;\n\n protected ?string $useInputKey = null;\n\n protected bool|string $uniqueInput = false;\n\n /**\n * @var array\n */\n protected array $uniqueInputKeys = [];\n\n protected bool|string $uniqueOutput = false;\n\n /**\n * @var array\n */\n protected array $uniqueOutputKeys = [];\n\n /**\n * @var array\n */\n protected array $refiners = [];\n\n protected ?string $outputKey = null;\n\n protected ?int $maxOutputs = null;\n\n protected int $currentOutputCount = 0;\n\n private ?Input $fullOriginalInput = null;\n\n /**\n * @param Input $input\n * @return Generator\n */\n abstract public function invokeStep(Input $input): Generator;\n\n public function addLogger(LoggerInterface $logger): static\n {\n if ($this->logger instanceof PreStepInvocationLogger) {\n $this->logger->passToOtherLogger($logger);\n }\n\n $this->logger = $logger;\n\n if (!empty($this->refiners)) {\n foreach ($this->refiners as $refiner) {\n if ($refiner instanceof RefinerInterface) {\n $refiner->addLogger($logger);\n } elseif (is_array($refiner) && $refiner['refiner'] instanceof RefinerInterface) {\n $refiner['refiner']->addLogger($logger);\n }\n }\n }\n\n return $this;\n }\n\n public function setParentCrawler(Crawler $crawler): static\n {\n $this->parentCrawler = $crawler;\n\n return $this;\n }\n\n /**\n * @param string|string[]|null $keys\n */\n public function keep(string|array|null $keys = null): static\n {\n if ($keys === null) {\n $this->keep = true;\n } else {\n $this->keep = $keys;\n }\n\n return $this;\n }\n\n public function keepAs(string $key): static\n {\n $this->keepAs = $key;\n\n return $this;\n }\n\n /**\n * @param string|string[]|null $keys\n */\n public function keepFromInput(string|array|null $keys = null): static\n {\n if ($keys === null) {\n $this->keepFromInput = true;\n } else {\n $this->keepFromInput = $keys;\n }\n\n return $this;\n }\n\n public function keepInputAs(string $key): static\n {\n $this->keepInputAs = $key;\n\n return $this;\n }\n\n public function keepsAnything(): bool\n {\n return $this->keepsAnythingFromOutputData() || $this->keepsAnythingFromInputData();\n }\n\n public function keepsAnythingFromInputData(): bool\n {\n return $this->keepFromInput !== false || $this->keepInputAs !== null;\n }\n\n public function keepsAnythingFromOutputData(): bool\n {\n return $this->keep !== false || $this->keepAs !== null;\n }\n\n public function useInputKey(string $key): static\n {\n $this->useInputKey = $key;\n\n return $this;\n }\n\n public function uniqueInputs(?string $key = null): static\n {\n $this->uniqueInput = $key ?? true;\n\n return $this;\n }\n\n public function uniqueOutputs(?string $key = null): static\n {\n $this->uniqueOutput = $key ?? true;\n\n return $this;\n }\n\n public function refineOutput(\n string|Closure|RefinerInterface $keyOrRefiner,\n Closure|RefinerInterface|null $refiner = null,\n ): static {\n if ($refiner instanceof RefinerInterface && $this->logger) {\n $refiner->addLogger($this->logger);\n } elseif ($keyOrRefiner instanceof RefinerInterface && $this->logger) {\n $keyOrRefiner->addLogger($this->logger);\n }\n\n if (is_string($keyOrRefiner) && $refiner === null) {\n throw new InvalidArgumentException(\n 'You have to provide a Refiner (Closure or instance of RefinerInterface)',\n );\n } elseif (is_string($keyOrRefiner)) {\n $this->refiners[] = ['key' => $keyOrRefiner, 'refiner' => $refiner];\n } else {\n $this->refiners[] = $keyOrRefiner;\n }\n\n return $this;\n }\n\n public function outputKey(string $key): static\n {\n $this->outputKey = $key;\n\n return $this;\n }\n\n public function maxOutputs(int $maxOutputs): static\n {\n $this->maxOutputs = $maxOutputs;\n\n return $this;\n }\n\n public function resetAfterRun(): void\n {\n $this->uniqueOutputKeys = $this->uniqueInputKeys = [];\n\n $this->currentOutputCount = 0;\n }\n\n /**\n * Define what type of outputs the step will yield\n *\n * Defining this in any step, helps to identify potential errors upfront when a crawler run is started.\n * If the step will only yield associative array (or object) outputs,\n * return StepOutputType::AssociativeArrayOrObject.\n * If it will only yield scalar (string, int, float, bool) outputs, return StepOutputType::Scalar.\n *\n * If it can potentially yield both types, but you can determine what it will yield, based on the state of the\n * class, please implement this. Only if it can't be defined upfront, because it depends on the input, return\n * StepOutputType::Mixed.\n *\n * @return StepOutputType\n */\n public function outputType(): StepOutputType\n {\n return StepOutputType::Mixed;\n }\n\n /**\n * @param BaseStep|mixed[] $previousStepOrInitialInputs\n * @throws PreRunValidationException\n */\n public function validateBeforeRun(BaseStep|array $previousStepOrInitialInputs): void\n {\n if (!$previousStepOrInitialInputs instanceof BaseStep) {\n $this->validateFirstStepBeforeRun($previousStepOrInitialInputs);\n }\n\n if ($this->keep !== false && $this->keepAs === null && $this->outputKey === null) {\n $outputType = $this->outputType();\n\n if ($outputType === StepOutputType::Scalar) {\n throw new PreRunValidationException(\n 'Keeping data from a step that yields scalar value outputs (= single string/int/bool/float with ' .\n 'no key like in an associative array or object) requires to define a key, by using keepAs() ' .\n 'instead of keep()',\n );\n } elseif ($outputType === StepOutputType::Mixed) {\n $this->logger?->warning(\n $this->getPreValidationRunMessageStartWithStepClassName() . ' potentially yields scalar value ' .\n 'outputs (= single string/int/bool/float with no key like in an associative array or object). ' .\n 'If it does (yield a scalar value output), it can not keep that output value, because it needs ' .\n 'a key for that. To avoid this, define a key for scalar outputs by using the keepAs() method.',\n );\n }\n }\n\n if (\n $this->keepFromInput !== false &&\n $previousStepOrInitialInputs instanceof BaseStep &&\n $this->keepInputAs === null\n ) {\n $previousStepOutputType = $previousStepOrInitialInputs->outputType();\n\n if ($previousStepOutputType === StepOutputType::Scalar) {\n throw new PreRunValidationException(\n 'You are trying to keep data from a step\\'s input with keepFromInput(), but the step before it ' .\n 'returns scalar value outputs (= single string/int/bool/float with no key like in an associative ' .\n 'array or object). Please define a key for the input data to keep, by using keepAs() instead.',\n );\n } elseif ($previousStepOutputType === StepOutputType::Mixed) {\n $this->logger?->warning(\n $this->getPreValidationRunMessageStartWithStepClassName($previousStepOrInitialInputs) .\n ' potentially yields scalar value outputs (= single string/int/bool/float with no key like in ' .\n 'an associative array or object). If it does (yield a scalar value output) the next step can not ' .\n 'keep it by using keepFromInput(). To avoid this, define a key for scalar inputs by using the ' .\n 'keepInputAs() method.',\n );\n }\n }\n }\n\n public function subCrawlerFor(string $for, Closure $crawlerBuilder): static\n {\n $this->subCrawlers[$for] = $crawlerBuilder;\n\n return $this;\n }\n\n /**\n * In case useInputKey() was used, use this method to store the original input so you can still\n * access it later.\n */\n protected function storeOriginalInput(Input $input): void\n {\n $this->fullOriginalInput = $input;\n }\n\n /**\n * In case useInputKey() was used, this method shall still provide access to the full input object,\n * that the step was last called with.\n */\n protected function getFullOriginalInput(): ?Input\n {\n return $this->fullOriginalInput;\n }\n\n protected function runSubCrawlersFor(Output $output): Output\n {\n if (empty($this->subCrawlers)) {\n return $output;\n }\n\n if (!$output->isArrayWithStringKeys()) {\n $this->logger?->error(\n 'The sub crawler feature works only with outputs that are associative arrays (arrays with ' .\n 'string keys). The feature was called with an output of type ' . gettype($output->get()) . '.',\n );\n\n return $output;\n }\n\n if (!$this->parentCrawler) {\n $this->logger?->error('Can\\'t make sub crawler, because the step has no reference to the parent crawler.');\n } else {\n foreach ($this->subCrawlers as $forKey => $crawlerBuilder) {\n $outputValue = $output->getProperty($forKey);\n\n if ($outputValue !== null) {\n $crawler = $crawlerBuilder($this->parentCrawler->getSubCrawler());\n\n is_array($outputValue) ? $crawler->inputs($outputValue) : $crawler->input($outputValue);\n\n $results = [];\n\n foreach ($crawler->run() as $result) {\n $results[] = $result;\n }\n\n $resultCount = count($results);\n\n if ($resultCount === 0) {\n $output = $output->withPropertyValue($forKey, null);\n } elseif ($resultCount === 1) {\n $output = $output->withPropertyValue($forKey, $results[0]->toArray());\n } else {\n $output = $output->withPropertyValue(\n $forKey,\n array_map(function (Result $result) {\n return $result->toArray();\n }, $results),\n );\n }\n }\n }\n }\n\n return $output;\n }\n\n /**\n * If you want to define aliases for certain output keys that can be used with keep(),\n * define this method in the child class and return the mappings.\n *\n * @return array alias => output key\n */\n protected function outputKeyAliases(): array\n {\n return [];\n }\n\n /**\n * @param mixed[] $initialInputs\n * @throws PreRunValidationException\n */\n protected function validateFirstStepBeforeRun(array $initialInputs): void\n {\n if ($initialInputs === []) {\n $this->logger?->error('You did not provide any initial inputs for your crawler.');\n\n return;\n }\n\n if ($this->keepFromInput !== false) {\n foreach ($initialInputs as $input) {\n if (!OutputTypeHelper::isAssociativeArrayOrObject($input)) {\n throw new PreRunValidationException(\n 'The initial inputs contain scalar values (without keys) and you are calling keepFromInput() ' .\n 'on the first step (if not the first step in your whole crawler, check sub crawlers). Please ' .\n 'use keepInputAs() instead with a key, that the input value should have in the kept data.',\n );\n }\n }\n }\n }\n\n protected function getPreValidationRunMessageStartWithStepClassName(?BaseStep $step = null): string\n {\n $stepClassName = $this->getStepClassName($step);\n\n if ($stepClassName) {\n return 'The ' . $stepClassName . ' step';\n } else {\n $stepClassName = $this->getParentStepClassName($step);\n\n if (\n $stepClassName &&\n $stepClassName !== 'Crwlr\\\\Crawler\\\\Steps\\\\Step' &&\n $stepClassName !== 'Crwlr\\\\Crawler\\\\Steps\\\\BaseStep'\n ) {\n return 'An anonymous class step, that is extending the ' . $stepClassName . ' step';\n } else {\n return 'An anonymous class step';\n }\n }\n }\n\n protected function getStepClassName(?BaseStep $step = null): ?string\n {\n $stepClassName = get_class($step ?? $this);\n\n if (str_contains($stepClassName, '@anonymous')) {\n return null;\n }\n\n return $stepClassName;\n }\n\n protected function getParentStepClassName(?BaseStep $step = null): ?string\n {\n $parents = class_parents($step ?? $this);\n\n $firstLevelParent = reset($parents);\n\n if ($firstLevelParent && !str_contains($firstLevelParent, '@anonymous')) {\n return $firstLevelParent;\n }\n\n return null;\n }\n\n protected function getInputKeyToUse(Input $input): ?Input\n {\n if ($this->useInputKey !== null) {\n $inputValue = $input->get();\n\n if (!is_array($inputValue) || !array_key_exists($this->useInputKey, $inputValue)) {\n if (!array_key_exists($this->useInputKey, $input->keep)) {\n $warningMessage = '';\n\n if (!is_array($inputValue)) {\n $warningMessage = 'Can\\'t get key from input, because input is of type ' .\n gettype($inputValue) . ' instead of array.';\n } elseif (!array_key_exists($this->useInputKey, $inputValue)) {\n $warningMessage = 'Can\\'t get key from input, because it does not exist.';\n }\n\n if (!empty($input->keep)) {\n $warningMessage .= ' Key also is not present in data kept from previous steps.';\n }\n\n $this->logger?->warning($warningMessage);\n\n return null;\n }\n\n $valueToUse = $input->keep[$this->useInputKey];\n } else {\n $valueToUse = $inputValue[$this->useInputKey];\n }\n\n $input = $input->withValue($valueToUse);\n }\n\n return $input;\n }\n\n protected function inputOrOutputIsUnique(Io $io): bool\n {\n $uniquenessSetting = $io instanceof Input ? $this->uniqueInput : $this->uniqueOutput;\n\n $uniqueKeys = $io instanceof Input ? $this->uniqueInputKeys : $this->uniqueOutputKeys;\n\n $key = is_string($uniquenessSetting) ? $io->setKey($uniquenessSetting) : $io->setKey();\n\n if (isset($uniqueKeys[$key])) {\n return false;\n }\n\n if ($io instanceof Input) {\n $this->uniqueInputKeys[$key] = true; // Don't keep value, just the key, to keep memory usage low.\n } else {\n $this->uniqueOutputKeys[$key] = true;\n }\n\n return true;\n }\n\n protected function applyRefiners(mixed $outputValue, mixed $inputValue): mixed\n {\n foreach ($this->refiners as $refiner) {\n $outputValueToRefine = $outputValue;\n\n if (is_array($refiner) && isset($outputValue[$refiner['key']])) {\n $outputValueToRefine = $outputValue[$refiner['key']];\n }\n\n if ($refiner instanceof Closure) {\n $refinedOutputValue = $refiner->call($this, $outputValueToRefine, $inputValue);\n } elseif ($refiner instanceof RefinerInterface) {\n $refinedOutputValue = $refiner->refine($outputValueToRefine);\n } else {\n if ($refiner['refiner'] instanceof Closure) {\n $refinedOutputValue = $refiner['refiner']->call($this, $outputValueToRefine, $inputValue);\n } else {\n $refinedOutputValue = $refiner['refiner']->refine($outputValueToRefine);\n }\n }\n\n if (is_array($refiner) && isset($outputValue[$refiner['key']])) {\n $outputValue[$refiner['key']] = $refinedOutputValue;\n } else {\n $outputValue = $refinedOutputValue;\n }\n }\n\n return $outputValue;\n }\n\n protected function makeOutput(mixed $outputData, Input $input): Output\n {\n $output = new Output(\n $outputData,\n $input->keep,\n );\n\n $output = $this->runSubCrawlersFor($output);\n\n $this->keepData($output, $input);\n\n return $output;\n }\n\n protected function keepData(Output $output, Input $input): void\n {\n if (!$this->keepsAnything()) {\n return;\n }\n\n if ($this->keepsAnythingFromInputData()) {\n $inputDataToKeep = $this->getInputDataToKeep($input, $output->keep);\n\n if (!empty($inputDataToKeep)) {\n $output->keep($inputDataToKeep);\n }\n }\n\n if ($this->keepsAnythingFromOutputData()) {\n $outputDataToKeep = $this->getOutputDataToKeep($output, $output->keep);\n\n if (!empty($outputDataToKeep)) {\n $output->keep($outputDataToKeep);\n }\n }\n }\n\n /**\n * @param array $alreadyKept\n * @return mixed[]|null\n */\n protected function getOutputDataToKeep(Output $output, array $alreadyKept): ?array\n {\n return $this->getInputOrOutputDataToKeep($output, $alreadyKept);\n }\n\n /**\n * @param array $alreadyKept\n * @return mixed[]|null\n */\n protected function getInputDataToKeep(Input $input, array $alreadyKept): ?array\n {\n return $this->getInputOrOutputDataToKeep($input, $alreadyKept);\n }\n\n /**\n * @param array $alreadyKept\n * @return mixed[]|null\n */\n protected function getInputOrOutputDataToKeep(Io $io, array $alreadyKept): ?array\n {\n $keepProperty = $io instanceof Output ? $this->keep : $this->keepFromInput;\n\n $keepAsProperty = $io instanceof Output ? $this->keepAs : $this->keepInputAs;\n\n $data = $io->get();\n\n $isScalarValue = OutputTypeHelper::isScalar($data);\n\n if ($keepAsProperty !== null && ($isScalarValue || $keepProperty === false)) {\n return [$keepAsProperty => $data];\n } elseif ($keepProperty !== false) {\n if ($isScalarValue) {\n $variableMessagePart = $io instanceof Output ? 'yielded an output' : 'received an input';\n\n $this->logger?->error(\n 'A ' . get_class($this) . ' step ' . $variableMessagePart . ' that is neither an associative ' .\n 'array, nor an object, so there is no key for the value to keep. Please define a key for the ' .\n 'output by using keepAs() instead of keep(). The value is now kept with an \\'unnamed\\' key.',\n );\n\n return [$this->nextUnnamedKey($alreadyKept) => $data];\n }\n\n $data = !is_array($data) ? OutputTypeHelper::objectToArray($data) : $data;\n\n if ($keepProperty === true) {\n return $data;\n } elseif (is_string($keepProperty)) {\n return [$keepProperty => $this->getOutputPropertyFromArray($keepProperty, $data)];\n }\n\n return $this->mapKeepProperties($data, $keepProperty);\n }\n\n return null;\n }\n\n /**\n * @param array $data\n * @return string\n */\n protected function nextUnnamedKey(array $data): string\n {\n $i = 1;\n\n while (isset($data['unnamed' . $i])) {\n $i++;\n }\n\n return 'unnamed' . $i;\n }\n\n /**\n * @param mixed[] $data\n * @param array $keep\n * @return mixed[]\n */\n protected function mapKeepProperties(array $data, array $keep): array\n {\n $keepData = [];\n\n foreach ($keep as $key => $value) {\n if (is_int($key)) {\n $keepData[$value] = $this->getOutputPropertyFromArray($value, $data);\n } elseif (is_string($key)) {\n $keepData[$key] = $this->getOutputPropertyFromArray($value, $data);\n }\n }\n\n return $keepData;\n }\n\n /**\n * @param mixed[] $data\n */\n protected function getOutputPropertyFromArray(string $key, array $data): mixed\n {\n if (array_key_exists($key, $data)) {\n return $data[$key];\n } elseif ($this->isOutputKeyAlias($key)) {\n return $data[$this->getOutputKeyAliasRealKey($key)];\n }\n\n $data = OutputTypeHelper::recursiveChildObjectsToArray($data);\n\n $dot = new Dot($data);\n\n return $dot->get($key);\n }\n\n protected function isOutputKeyAlias(string $key): bool\n {\n return array_key_exists($key, $this->outputKeyAliases());\n }\n\n protected function getOutputKeyAliasRealKey(string $key): string\n {\n $mapping = $this->outputKeyAliases();\n\n return $mapping[$key];\n }\n\n protected function maxOutputsExceeded(): bool\n {\n return $this->maxOutputs !== null && $this->currentOutputCount >= $this->maxOutputs;\n }\n\n protected function trackYieldedOutput(): void\n {\n if ($this->maxOutputs !== null) {\n $this->currentOutputCount += 1;\n }\n }\n}\n" }, { "path": "src/Steps/Csv.php", "content": " $columnMapping\n */\n public function __construct(protected array $columnMapping = [], protected bool $skipFirstLine = false) {}\n\n /**\n * @param array $columnMapping\n */\n public static function parseString(array $columnMapping = [], bool $skipFirstLine = false): self\n {\n return new self($columnMapping, $skipFirstLine);\n }\n\n /**\n * @param array $columnMapping\n */\n public static function parseFile(array $columnMapping = [], bool $skipFirstLine = false): self\n {\n $instance = new self($columnMapping, $skipFirstLine);\n\n $instance->method = 'file';\n\n return $instance;\n }\n\n public function skipFirstLine(): static\n {\n $this->skipFirstLine = true;\n\n return $this;\n }\n\n public function separator(string $separator): static\n {\n if (strlen($separator) > 1) {\n throw new InvalidArgumentException('CSV separator must be single character');\n }\n\n $this->separator = $separator;\n\n return $this;\n }\n\n public function enclosure(string $enclosure): static\n {\n $this->enclosure = $enclosure;\n\n return $this;\n }\n\n public function escape(string $escape): static\n {\n $this->escape = $escape;\n\n return $this;\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::AssociativeArrayOrObject;\n }\n\n protected function validateAndSanitizeInput(mixed $input): string\n {\n if ($this->method === 'string') {\n return $this->validateAndSanitizeStringOrHttpResponse($input);\n } elseif ($this->method === 'file') {\n return $this->validateAndSanitizeStringOrStringable($input);\n } else {\n throw new InvalidArgumentException('Parse CSV method must be string or file');\n }\n }\n\n /**\n * @param string $input\n * @throws Exception\n */\n protected function invoke(mixed $input): Generator\n {\n if ($this->method === 'file') {\n if (!file_exists($input)) {\n throw new Exception('CSV file not found');\n }\n\n yield from $this->readFile($input);\n } elseif ($this->method === 'string') {\n yield from $this->mapLines(explode(PHP_EOL, $input));\n }\n }\n\n protected function readFile(string $filePath): Generator\n {\n $handle = fopen($filePath, 'r');\n\n if ($handle === false) {\n return;\n }\n\n $isFirstLine = true;\n\n while (($row = fgetcsv($handle, 0, $this->separator, $this->enclosure, $this->escape)) !== false) {\n if ($isFirstLine) {\n if (empty($this->columnMapping)) {\n $this->columnMapping = $row;\n }\n\n $isFirstLine = false;\n\n if ($this->skipFirstLine) {\n continue;\n }\n }\n\n yield $this->mapRow($row);\n }\n\n fclose($handle);\n }\n\n /**\n * @param string[] $lines\n * @return Generator\n */\n protected function mapLines(array $lines): Generator\n {\n foreach ($lines as $key => $line) {\n if ($key === 0 && $this->skipFirstLine) {\n if (empty($this->columnMapping)) {\n $this->columnMapping = str_getcsv($line, $this->separator, $this->enclosure, $this->escape);\n }\n\n continue;\n }\n\n if (!empty($line)) {\n yield $this->mapRow(str_getcsv($line, $this->separator, $this->enclosure, $this->escape));\n }\n }\n }\n\n /**\n * @param mixed[] $row\n * @return mixed[]\n */\n protected function mapRow(array $row): array\n {\n $count = 0;\n $mapped = [];\n\n foreach ($row as $column) {\n if (isset($this->columnMapping[$count]) && !empty($this->columnMapping[$count])) {\n $mapped[$this->columnMapping[$count]] = $column;\n }\n\n $count++;\n }\n\n return $mapped;\n }\n}\n" }, { "path": "src/Steps/Dom/DomDocument.php", "content": "makeDocumentInstance($source)); // @phpstan-ignore-line\n }\n\n /**\n * @param string $source\n * @return Document|Crawler\n */\n abstract protected function makeDocumentInstance(string $source): object;\n}\n" }, { "path": "src/Steps/Dom/HtmlDocument.php", "content": " querySelectorAll(string $selector)\n * @method NodeList queryXPath(string $selector)\n */\n\nclass HtmlDocument extends DomDocument\n{\n /**\n * Gets the href attribute of a tag in the document\n *\n * In case there are multiple base elements in the document:\n * https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base\n * \"If multiple elements are used, only the first href and first target are obeyed...\"\n */\n public function getBaseHref(): ?string\n {\n $baseTag = $this->querySelector('base');\n\n return $baseTag?->getAttribute('href');\n }\n\n public function outerHtml(): string\n {\n return $this->outerSource();\n }\n\n /**\n * @param \\Dom\\Node|DOMNode|Crawler $node\n */\n protected function makeChildNodeInstance(object $node): Node\n {\n return new HtmlElement($node);\n }\n\n /**\n * @return \\Dom\\HTMLDocument|Crawler\n */\n protected function makeDocumentInstance(string $source): object\n {\n $source = $this->fixInvalidCharactersInSource($source);\n\n if (PhpVersion::isAtLeast(8, 4)) {\n return \\Dom\\HTMLDocument::createFromString($source, HTML_NO_DEFAULT_NS | LIBXML_NOERROR);\n }\n\n return new Crawler($source);\n }\n\n /**\n * Converts charset to HTML-entities to ensure valid parsing.\n */\n private function fixInvalidCharactersInSource(string $source): string\n {\n if (function_exists('iconv')) {\n $charset = preg_match('//u', $source) ? 'UTF-8' : 'ISO-8859-1';\n\n preg_match('/(charset *= *[\"\\']?)([a-zA-Z\\-0-9_:.]+)/i', $source, $matches);\n\n if ($matches && !empty($matches[2])) {\n $declaredCharset = strtoupper($matches[2]);\n } else {\n $declaredCharset = null;\n }\n\n if ($charset === 'ISO-8859-1' && $declaredCharset === 'UTF-8') {\n $fixedSource = iconv(\"ISO-8859-1\", \"UTF-8//TRANSLIT\", $source);\n\n if ($fixedSource !== false) {\n $source = $fixedSource;\n }\n }\n }\n\n return $source;\n }\n}\n" }, { "path": "src/Steps/Dom/HtmlElement.php", "content": " querySelectorAll(string $selector)\n * @method NodeList queryXPath(string $selector)\n */\n\nclass HtmlElement extends Node\n{\n public function outerHtml(): string\n {\n return $this->outerSource();\n }\n\n public function innerHtml(): string\n {\n return $this->innerSource();\n }\n\n public function html(): string\n {\n return $this->innerHtml();\n }\n\n /**\n * @param \\Dom\\Node|DOMNode|Crawler $node\n */\n protected function makeChildNodeInstance(object $node): Node\n {\n return new HtmlElement($node);\n }\n}\n" }, { "path": "src/Steps/Dom/Node.php", "content": "node = $node;\n }\n\n public function querySelector(string $selector): ?Node\n {\n if ($this->node instanceof Crawler) {\n $filtered = $this->node->filter($selector);\n\n return $filtered->count() > 0 ? $this->makeChildNodeInstance($filtered->first()) : null;\n }\n\n $result = $this->node->querySelector($selector);\n\n return $result !== null ? $this->makeChildNodeInstance($result) : null;\n }\n\n public function querySelectorAll(string $selector): NodeList\n {\n if ($this->node instanceof Crawler) {\n return $this->makeNodeListInstance($this->node->filter($selector));\n }\n\n return $this->makeNodeListInstance($this->node->querySelectorAll($selector));\n }\n\n public function queryXPath(string $query): NodeList\n {\n $node = $this->node;\n\n if (!$node instanceof Crawler) {\n $node = new Crawler($this->outerSource());\n }\n\n return $this->makeNodeListInstance($node->filterXPath($query));\n }\n\n public function removeNodesMatchingSelector(string $selector): void\n {\n foreach ($this->querySelectorAll($selector) as $node) {\n if ($node->node instanceof Crawler) {\n $node = $node->node->getNode(0);\n\n if ($node) {\n $node->parentNode?->removeChild($node);\n }\n } else {\n $node->node->parentNode?->removeChild($node->node);\n }\n }\n }\n\n public function removeNodesMatchingXPath(string $query): void\n {\n if ($this->node instanceof Crawler) {\n foreach ($this->node->filterXPath($query) as $node) {\n $node->parentNode?->removeChild($node);\n }\n } else {\n $node = $this->getParentDocumentOfNode($this->node);\n\n if ($node) {\n $xpath = new XPath($node);\n\n foreach ($xpath->query($query) as $node) {\n $node->parentNode?->removeChild($node);\n }\n }\n }\n }\n\n public function nodeName(): string\n {\n if ($this->node instanceof Crawler) {\n $nodeName = $this->node->nodeName();\n } else {\n $nodeName = $this->node->nodeName ?? '';\n }\n\n return strtolower($nodeName);\n }\n\n public function text(): string\n {\n if ($this->node instanceof Crawler) {\n $text = $this->node->text();\n } else {\n $text = is_string($this->node->textContent) ? $this->node->textContent : '';\n }\n\n return trim(\n preg_replace(\"/(?:[ \\n\\r\\t\\x0C]{2,}+|[\\n\\r\\t\\x0C])/\", ' ', $text) ?? $text,\n \" \\n\\r\\t\\x0C\",\n );\n }\n\n public function getAttribute(string $attributeName): ?string\n {\n if ($this->node instanceof Crawler) {\n return $this->node->attr($attributeName);\n }\n\n return $this->node->getAttribute($attributeName);\n }\n\n /**\n * @param \\Dom\\Node|DOMNode|Crawler $node\n */\n abstract protected function makeChildNodeInstance(object $node): Node;\n\n protected function outerSource(): string\n {\n if ($this->node instanceof Crawler) {\n return $this->node->count() > 0 ? $this->node->outerHtml() : '';\n }\n\n if ($this->node instanceof Document) {\n $node = $this->node->documentElement;\n\n if ($this->node instanceof \\Dom\\HTMLDocument) {\n return $this->node->saveHTML($node);\n } elseif ($this->node instanceof \\Dom\\XMLDocument) {\n $source = $this->node->saveXML($node);\n\n return $source !== false ? $source : '';\n }\n }\n\n $parentDocument = $this->getParentDocumentOfNode($this->node);\n\n if ($parentDocument) {\n if ($parentDocument instanceof \\Dom\\HTMLDocument) {\n return $parentDocument->saveHTML($this->node);\n } elseif ($parentDocument instanceof \\Dom\\XMLDocument) {\n $source = $parentDocument->saveXML($this->node);\n\n return $source !== false ? $source : '';\n }\n }\n\n return $this->node->innerHTML;\n }\n\n protected function innerSource(): string\n {\n if ($this->node instanceof Crawler) {\n return $this->node->html();\n }\n\n return $this->node->innerHTML;\n }\n\n /**\n * @param \\Dom\\NodeList<\\Dom\\Node>|Crawler $nodeList\n */\n protected function makeNodeListInstance(object $nodeList): NodeList\n {\n return new NodeList(\n $nodeList,\n function (object $node): Node {\n /** @var DOMNode|\\Dom\\Node $node */\n return $this->makeChildNodeInstance($node);\n },\n );\n }\n\n /**\n * @param \\Dom\\Node|Element $node\n * @return Document|null\n */\n private function getParentDocumentOfNode(object $node): ?object\n {\n if ($node instanceof Document) {\n return $node;\n }\n\n $parentDocument = $node->parentNode;\n\n while ($parentDocument && !$parentDocument instanceof Document) {\n $parentDocument = $parentDocument->parentNode;\n }\n\n if ($parentDocument instanceof Document) {\n return $parentDocument;\n }\n\n return null;\n }\n}\n" }, { "path": "src/Steps/Dom/NodeList.php", "content": "\n */\n\nclass NodeList implements IteratorAggregate, Countable\n{\n /**\n * @param \\Dom\\NodeList<\\Dom\\Node>|\\Dom\\NodeList|Crawler|array $nodeList\n */\n public function __construct(\n private readonly object|array $nodeList,\n private readonly ?Closure $makeNodeInstance = null,\n ) {}\n\n /**\n * @throws Exception\n */\n public function first(): ?Node\n {\n $iterator = $this->getIterator();\n\n $iterator->rewind();\n\n return $iterator->current();\n }\n\n /**\n * @throws Exception\n */\n public function last(): ?Node\n {\n $iterator = $this->getIterator();\n\n foreach ($iterator as $node) {\n }\n\n return $node ?? null;\n }\n\n /**\n * @throws Exception\n */\n public function nth(int $index): ?Node\n {\n $iterator = $this->getIterator();\n\n $i = 0;\n\n foreach ($iterator as $node) {\n if (($i + 1) === $index) {\n return $node;\n }\n\n $i++;\n }\n\n return null;\n }\n\n /**\n * @return mixed[]\n * @throws Exception\n */\n public function each(Closure $callback): array\n {\n $data = [];\n\n foreach ($this->getIterator() as $key => $node) {\n $data[] = $callback($node, $key);\n }\n\n return $data;\n }\n\n /**\n * @return int<0, max>\n */\n public function count(): int\n {\n if (is_array($this->nodeList)) {\n return count($this->nodeList);\n }\n\n return max(0, $this->nodeList->count());\n }\n\n public function getIterator(): Iterator\n {\n if (is_array($this->nodeList)) {\n return new ArrayIterator($this->nodeList);\n }\n\n $iterator = $this->nodeList->getIterator();\n\n /** @var Iterator $iterator */\n\n return new class ($iterator, $this->makeNodeInstance) implements Iterator {\n /**\n * @param Iterator $iterator\n */\n public function __construct(\n private readonly Iterator $iterator,\n private readonly ?Closure $makeNodeInstanceCallback = null,\n ) {}\n\n public function current(): ?Node\n {\n return $this->makeNodeInstance($this->iterator->current());\n }\n\n public function next(): void\n {\n $this->iterator->next();\n }\n\n public function key(): mixed\n {\n return $this->iterator->key();\n }\n\n public function valid(): bool\n {\n return $this->iterator->valid();\n }\n\n public function rewind(): void\n {\n $this->iterator->rewind();\n }\n\n /**\n * @param \\Dom\\Node|DOMNode|Crawler $node\n */\n private function makeNodeInstance(mixed $node): ?Node\n {\n if (!is_object($node)) { // @phpstan-ignore-line change when min. required PHP version is 8.4.\n return null;\n }\n\n return $this->makeNodeInstanceCallback?->__invoke($node) ?? null;\n }\n };\n }\n}\n" }, { "path": "src/Steps/Dom/XmlDocument.php", "content": " querySelectorAll(string $selector)\n * @method NodeList queryXPath(string $selector)\n */\n\nclass XmlDocument extends DomDocument\n{\n public function outerXml(): string\n {\n return $this->outerSource();\n }\n\n /**\n * @param \\Dom\\Node|DOMNode|Crawler $node\n */\n protected function makeChildNodeInstance(object $node): Node\n {\n return new XmlElement($node);\n }\n\n /**\n * @return \\Dom\\XMLDocument|Crawler\n */\n protected function makeDocumentInstance(string $source): object\n {\n if (PhpVersion::isAtLeast(8, 4)) {\n try {\n return \\Dom\\XMLDocument::createFromString($source, LIBXML_NOERROR | LIBXML_NONET);\n } catch (Throwable) {\n $source = $this->replaceInvalidXmlCharacters($source);\n\n try {\n return \\Dom\\XMLDocument::createFromString($source, LIBXML_NOERROR | LIBXML_NONET);\n } catch (Throwable) {\n } // If it fails again, try it with symfony DOM Crawler as fallback.\n }\n }\n\n $crawler = new Crawler($source);\n\n if ($crawler->count() === 0) {\n $source = $this->replaceInvalidXmlCharacters($source);\n\n $crawler = new Crawler($source);\n }\n\n return $crawler;\n }\n\n /**\n * Replace characters that aren't valid within XML documents\n *\n * Sometimes XML parsing errors occur because of characters that aren't valid within XML documents.\n * Therefore, this method finds and replaces them with valid alternatives or HTML entities.\n * For best results in those cases, please install the voku/portable-ascii composer package.\n *\n * @param string $value\n * @return string\n */\n private function replaceInvalidXmlCharacters(string $value): string\n {\n return preg_replace_callback('/[^\\x{9}\\x{A}\\x{D}\\x{20}-\\x{D7FF}\\x{E000}-\\x{FFFD}]/u', function ($match) {\n $replacement = class_exists('voku\\helper\\ASCII') ? ASCII::to_transliterate($match[0]) : '?';\n\n if ($replacement === '?') {\n return '&#' . mb_ord($match[0]) . ';';\n }\n\n return $replacement;\n }, $value) ?? $value;\n }\n}\n" }, { "path": "src/Steps/Dom/XmlElement.php", "content": " querySelectorAll(string $selector)\n * @method NodeList queryXPath(string $selector)\n */\n\nclass XmlElement extends Node\n{\n public function outerXml(): string\n {\n return $this->outerSource();\n }\n\n public function innerXml(): string\n {\n return $this->innerSource();\n }\n\n /**\n * @param \\Dom\\Node|DOMNode|Crawler $node\n */\n protected function makeChildNodeInstance(object $node): Node\n {\n return new XmlElement($node);\n }\n}\n" }, { "path": "src/Steps/Dom.php", "content": "\n */\n protected array $mapping = [];\n\n protected string|DomQuery|null $singleSelector = null;\n\n protected ?string $baseUrl = null;\n\n /**\n * @param string|DomQuery|array $selectorOrMapping\n */\n final public function __construct(string|DomQuery|array $selectorOrMapping = [])\n {\n $this->addLogger(new PreStepInvocationLogger());\n\n $this->extract($selectorOrMapping);\n }\n\n public static function root(): static\n {\n $instance = new static();\n\n $instance->root = true;\n\n return $instance;\n }\n\n public static function each(string|DomQuery $domQuery): static\n {\n $instance = new static();\n\n $instance->each = is_string($domQuery) ? $instance->makeDefaultDomQueryInstance($domQuery) : $domQuery;\n\n if (trim($instance->each->query) === '') {\n $instance->logger?->warning(\n 'The selector you provided for the ‘each’ option is empty. This option is intended to allow ' .\n 'extracting multiple output objects from a single page, so an empty selector most likely doesn’t ' .\n 'make sense, as it will definitely result in only one output object.',\n );\n }\n\n return $instance;\n }\n\n public static function first(string|DomQuery $domQuery): static\n {\n $instance = new static();\n\n $instance->first = is_string($domQuery) ? $instance->makeDefaultDomQueryInstance($domQuery) : $domQuery;\n\n if (trim($instance->first->query) === '') {\n $instance->logger?->warning(\n 'The selector you provided for the ‘first’ option is empty. This option is meant to restrict your ' .\n 'extraction to a specific parent element, so an empty selector most likely doesn’t make sense. ' .\n 'Either define the desired selector or use the root() method instead.',\n );\n }\n\n return $instance;\n }\n\n public static function last(string|DomQuery $domQuery): static\n {\n $instance = new static();\n\n $instance->last = is_string($domQuery) ? $instance->makeDefaultDomQueryInstance($domQuery) : $domQuery;\n\n if (trim($instance->last->query) === '') {\n $instance->logger?->warning(\n 'The selector you provided for the ‘last’ option is empty. This option is meant to restrict your ' .\n 'extraction to a specific parent element, so an empty selector most likely doesn’t make sense. ' .\n 'Either define the desired selector or use the root() method instead.',\n );\n }\n\n return $instance;\n }\n\n /**\n * @throws InvalidDomQueryException\n */\n public static function cssSelector(string $selector): CssSelector\n {\n return new CssSelector($selector);\n }\n\n /**\n * @throws InvalidDomQueryException\n */\n public static function xPath(string $query): XPathQuery\n {\n return new XPathQuery($query);\n }\n\n abstract protected function makeDefaultDomQueryInstance(string $query): DomQuery;\n\n /**\n * @param string|DomQuery|array $selectorOrMapping\n */\n public function extract(string|DomQuery|array $selectorOrMapping): static\n {\n if (is_array($selectorOrMapping)) {\n $this->mapping = $selectorOrMapping;\n } else {\n $this->singleSelector = $selectorOrMapping;\n }\n\n return $this;\n }\n\n public function outputType(): StepOutputType\n {\n return empty($this->mapping) && $this->singleSelector ?\n StepOutputType::Scalar :\n StepOutputType::AssociativeArrayOrObject;\n }\n\n /**\n * @param HtmlDocument|Node $input\n * @throws Exception\n */\n protected function invoke(mixed $input): Generator\n {\n $base = $this->getBase($input);\n\n if (!$base || ($base instanceof NodeList && $base->count() === 0)) {\n return;\n }\n\n if (empty($this->mapping) && $this->singleSelector) {\n yield from $this->singleSelector($base);\n } else {\n if ($this->each) {\n if ($base instanceof NodeList) {\n foreach ($base as $element) {\n yield $this->mapProperties($element);\n }\n }\n } elseif ($base instanceof Node) {\n yield $this->mapProperties($base);\n }\n }\n }\n\n\n /**\n * @throws InvalidArgumentException|MissingZlibExtensionException\n */\n protected function validateAndSanitizeInput(mixed $input): HtmlDocument|XmlDocument\n {\n if ($input instanceof RespondedRequest) {\n $this->baseUrl = $input->effectiveUri();\n }\n\n return new HtmlDocument($this->validateAndSanitizeStringOrHttpResponse($input));\n }\n\n /**\n * @throws InvalidHtmlException\n * @throws Exception\n */\n protected function singleSelector(Node|NodeList $nodeOrNodeList): Generator\n {\n if ($this->singleSelector === null) {\n return;\n }\n\n $domQuery = is_string($this->singleSelector) ?\n $this->makeDefaultDomQueryInstance($this->singleSelector) :\n $this->singleSelector;\n\n if ($this->baseUrl !== null) {\n $domQuery->setBaseUrl($this->baseUrl);\n }\n\n if ($nodeOrNodeList instanceof NodeList) {\n $outputs = [];\n\n foreach ($nodeOrNodeList as $node) {\n $outputs[] = $domQuery->apply($node);\n }\n } else {\n $outputs = $domQuery->apply($nodeOrNodeList);\n }\n\n if (is_array($outputs)) {\n foreach ($outputs as $output) {\n yield $output;\n }\n } elseif ($outputs !== null) {\n yield $outputs;\n }\n }\n\n /**\n * @return mixed[]\n * @throws Exception\n */\n protected function mapProperties(Node $node): array\n {\n $mappedProperties = [];\n\n foreach ($this->mapping as $key => $domQuery) {\n if ($domQuery instanceof Dom) {\n $domQuery->baseUrl = $this->baseUrl;\n\n $mappedProperties[$key] = $this->getDataFromChildDomStep($domQuery, $node);\n } else {\n if (is_string($domQuery)) {\n $domQuery = $this->makeDefaultDomQueryInstance($domQuery);\n }\n\n if ($this->baseUrl !== null) {\n $domQuery->setBaseUrl($this->baseUrl);\n }\n\n $mappedProperties[$key] = $domQuery->apply($node);\n }\n }\n\n return $mappedProperties;\n }\n\n /**\n * @throws Exception\n */\n protected function getBase(DomDocument|Node $document): Node|NodeList|null\n {\n if ($this->root) {\n return $document;\n } elseif ($this->each) {\n return $this->getBaseFromDomNode($document, $this->each, each: true);\n } elseif ($this->first) {\n return $this->getBaseFromDomNode($document, $this->first, first: true);\n } elseif ($this->last) {\n return $this->getBaseFromDomNode($document, $this->last, last: true);\n }\n\n throw new Exception('Invalid state: no base selector');\n }\n\n /**\n * @throws Exception\n */\n private function getBaseFromDomNode(\n DomDocument|Node $document,\n DomQuery $query,\n bool $each = false,\n bool $first = false,\n bool $last = false,\n ): Node|NodeList|null {\n if (trim($query->query) === '') {\n return $each ? new NodeList([$document]) : $document;\n }\n\n if ($each) {\n return $query instanceof CssSelector ?\n $document->querySelectorAll($query->query) :\n $document->queryXPath($query->query);\n } elseif ($first) {\n return $this->first instanceof CssSelector ?\n $document->querySelector($query->query) :\n $document->queryXPath($query->query)->first();\n } elseif ($last) {\n return $this->last instanceof CssSelector ?\n $document->querySelectorAll($query->query)->last() :\n $document->queryXPath($query->query)->last();\n }\n\n return $document;\n }\n\n /**\n * @return mixed[]\n * @throws Exception\n */\n protected function getDataFromChildDomStep(Dom $step, Node $node): array\n {\n $childValue = iterator_to_array($step->invoke($node));\n\n // When the child step was not used with each() as base and the result is an array with one\n // element (index/key \"0\") being an array, use that child array.\n if (!$step->each && count($childValue) === 1 && isset($childValue[0]) && is_array($childValue[0])) {\n return $childValue[0];\n }\n\n return $childValue;\n }\n}\n" }, { "path": "src/Steps/Exceptions/PreRunValidationException.php", "content": "useKey = $key;\n\n return $this;\n }\n\n /**\n * Step::orWhere() uses this method to link further Filters with OR to this filter.\n * The Step then takes care of checking if one of the ORs evaluates to true.\n */\n public function addOr(FilterInterface $filter): void\n {\n if ($this->or instanceof FilterInterface) {\n $or = $this->or;\n\n while ($or->getOr()) {\n $or = $or->getOr();\n }\n\n $or->addOr($filter);\n } else {\n $this->or = $filter;\n }\n }\n\n /**\n * Get the Filter linked to this Filter as OR.\n */\n public function getOr(): ?FilterInterface\n {\n return $this->or instanceof FilterInterface ? $this->or : null;\n }\n\n public function negate(): NegatedFilter\n {\n return new NegatedFilter($this);\n }\n\n /**\n * @throws Exception\n */\n protected function getKey(mixed $value): mixed\n {\n if ($this->useKey === null) {\n return $value;\n }\n\n if (!is_array($value) && !is_object($value)) {\n throw new InvalidArgumentException('Can only filter by key with array or object output.');\n }\n\n if (is_object($value) && !property_exists($value, $this->useKey) && method_exists($value, '__serialize')) {\n $serialized = $value->__serialize();\n\n if (array_key_exists($this->useKey, $serialized)) {\n $value = $serialized;\n }\n }\n\n if (\n (is_array($value) && !array_key_exists($this->useKey, $value)) ||\n (is_object($value) && !property_exists($value, $this->useKey))\n ) {\n throw new Exception('Key to filter by does not exist in output.');\n }\n\n return is_array($value) ? $value[$this->useKey] : $value->{$this->useKey};\n }\n}\n" }, { "path": "src/Steps/Filters/ArrayFilter.php", "content": "getKey($valueInQuestion);\n\n if (is_array($valueInQuestion) && !empty($valueInQuestion)) {\n foreach ($valueInQuestion as $value) {\n if ($this->passesAllFilters($value)) {\n return true;\n }\n }\n }\n\n return false;\n }\n}\n" }, { "path": "src/Steps/Filters/ClosureFilter.php", "content": "getKey($valueInQuestion);\n\n return $this->closure->call($this, $valueInQuestion);\n }\n}\n" }, { "path": "src/Steps/Filters/ComparisonFilter.php", "content": "filterRule->evaluate($this->getKey($valueInQuestion), $this->compareTo);\n }\n}\n" }, { "path": "src/Steps/Filters/Enums/ComparisonFilterRule.php", "content": " ($value === $compareTo),\n self::NotEqual => ($value !== $compareTo),\n self::GreaterThan => ($value > $compareTo),\n self::GreaterThanOrEqual => ($value >= $compareTo),\n self::LessThan => ($value < $compareTo),\n self::LessThanOrEqual => ($value <= $compareTo),\n };\n }\n}\n" }, { "path": "src/Steps/Filters/Enums/StringFilterRule.php", "content": " str_contains($haystack, $needle),\n self::StartsWith => str_starts_with($haystack, $needle),\n self::EndsWith => str_ends_with($haystack, $needle),\n };\n }\n}\n" }, { "path": "src/Steps/Filters/Enums/StringLengthFilterRule.php", "content": " ($actualStringLength === $compareTo),\n self::NotEqual => ($actualStringLength !== $compareTo),\n self::GreaterThan => ($actualStringLength > $compareTo),\n self::GreaterThanOrEqual => ($actualStringLength >= $compareTo),\n self::LessThan => ($actualStringLength < $compareTo),\n self::LessThanOrEqual => ($actualStringLength <= $compareTo),\n };\n }\n}\n" }, { "path": "src/Steps/Filters/Enums/UrlFilterRule.php", "content": " Url::parse($url)->scheme() === $needle,\n self::Host => Url::parse($url)->host() === $needle,\n self::Domain => Url::parse($url)->domain() === $needle,\n self::Path => Url::parse($url)->path() === $needle,\n self::PathStartsWith => str_starts_with(Url::parse($url)->path() ?? '', $needle),\n self::PathMatches => preg_match($this->prepareRegex($needle), Url::parse($url)->path() ?? '') === 1,\n };\n } catch (InvalidUrlException|Exception $exception) {\n return false;\n }\n }\n\n protected function prepareRegex(string $regex): string\n {\n return '~' . $regex . '~';\n }\n}\n" }, { "path": "src/Steps/Filters/Filter.php", "content": "isOutputKeyAlias($keyOrFilter)) {\n $keyOrFilter = $this->getOutputKeyAliasRealKey($keyOrFilter);\n }\n\n $filter->useKey($keyOrFilter);\n\n $this->filters[] = $filter;\n } else {\n $this->filters[] = $keyOrFilter;\n }\n\n return $this;\n }\n\n /**\n * @throws Exception\n */\n public function orWhere(string|FilterInterface $keyOrFilter, ?FilterInterface $filter = null): static\n {\n if (empty($this->filters)) {\n throw new Exception('No where before orWhere');\n } elseif (is_string($keyOrFilter) && $filter === null) {\n throw new InvalidArgumentException('You have to provide a Filter (instance of FilterInterface)');\n } elseif (is_string($keyOrFilter)) {\n $filter->useKey($keyOrFilter);\n } else {\n $filter = $keyOrFilter;\n }\n\n $lastFilter = end($this->filters);\n\n $lastFilter->addOr($filter);\n\n return $this;\n }\n\n protected function passesAllFilters(mixed $output): bool\n {\n foreach ($this->filters as $filter) {\n if (!$filter->evaluate($output)) {\n if ($filter->getOr()) {\n $orFilter = $filter->getOr();\n\n while ($orFilter) {\n if ($orFilter->evaluate($output)) {\n continue 2;\n }\n\n $orFilter = $orFilter->getOr();\n }\n }\n\n return false;\n }\n }\n\n return true;\n }\n}\n" }, { "path": "src/Steps/Filters/NegatedFilter.php", "content": "filter->useKey($key);\n\n return $this;\n }\n\n public function evaluate(mixed $valueInQuestion): bool\n {\n return !$this->filter->evaluate($valueInQuestion);\n }\n\n public function addOr(FilterInterface $filter): void\n {\n $this->filter->addOr($filter);\n }\n\n public function getOr(): ?FilterInterface\n {\n return $this->filter->getOr();\n }\n\n public function negate(): NegatedFilter\n {\n return new NegatedFilter($this);\n }\n}\n" }, { "path": "src/Steps/Filters/StringFilter.php", "content": "getKey($valueInQuestion);\n\n if (!is_string($valueInQuestion)) {\n return false;\n }\n\n return $this->filterRule->evaluate($valueInQuestion, $this->filterString);\n }\n}\n" }, { "path": "src/Steps/Filters/StringLengthFilter.php", "content": "getKey($valueInQuestion);\n\n if (!is_string($valueInQuestion)) {\n return false;\n }\n\n return $this->filterRule->evaluate($valueInQuestion, $this->compareToLength);\n }\n}\n" }, { "path": "src/Steps/Filters/UrlFilter.php", "content": "getKey($valueInQuestion);\n\n if (!is_string($valueInQuestion)) {\n return false;\n }\n\n return $this->filterRule->evaluate($valueInQuestion, $this->filterString);\n }\n}\n" }, { "path": "src/Steps/Group.php", "content": "\n * @throws Exception\n */\n public function invokeStep(Input $input): Generator\n {\n $combinedOutput = $combinedKeptData = [];\n\n if ($this->uniqueInput && !$this->inputOrOutputIsUnique($input)) {\n return;\n }\n\n $this->storeOriginalInput($input);\n\n // When input is array and useInputKey() was used, invoke the steps only with that input array element,\n // but keep the original input, because we want to use it e.g. for the keepInputData() functionality.\n $inputForStepInvocation = $this->getInputKeyToUse($input);\n\n if ($inputForStepInvocation) {\n foreach ($this->steps as $step) {\n foreach ($step->invokeStep($inputForStepInvocation) as $nthOutput => $output) {\n if (method_exists($step, 'callUpdateInputUsingOutput')) {\n $inputForStepInvocation = $step->callUpdateInputUsingOutput($inputForStepInvocation, $output);\n }\n\n if ($this->includeOutput($step)) {\n $combinedOutput = $this->addToCombinedOutputData(\n $output->get(),\n $combinedOutput,\n $nthOutput,\n );\n }\n\n // Also transfer data, kept in group child steps, to the kept data of the final group output.\n if ($output->keep !== $inputForStepInvocation->keep) {\n $keep = $this->getNewlyKeptData($output, $inputForStepInvocation);\n\n $combinedKeptData = $this->addToCombinedOutputData($keep, $combinedKeptData, $nthOutput);\n }\n }\n }\n\n yield from $this->prepareCombinedOutputs($combinedOutput, $combinedKeptData, $input);\n }\n }\n\n public function addStep(StepInterface $step): self\n {\n if ($this->logger instanceof LoggerInterface) {\n $step->addLogger($this->logger);\n }\n\n if (method_exists($step, 'setLoader') && $this->loader instanceof LoaderInterface) {\n $step->setLoader($this->loader);\n }\n\n if ($this->maxOutputs) {\n $step->maxOutputs($this->maxOutputs);\n }\n\n $this->steps[] = $step;\n\n return $this;\n }\n\n public function addLogger(LoggerInterface $logger): static\n {\n parent::addLogger($logger);\n\n foreach ($this->steps as $step) {\n $step->addLogger($logger);\n }\n\n return $this;\n }\n\n public function setLoader(LoaderInterface $loader): self\n {\n $this->loader = $loader;\n\n foreach ($this->steps as $step) {\n if (method_exists($step, 'setLoader')) {\n $step->setLoader($loader);\n }\n }\n\n return $this;\n }\n\n public function maxOutputs(int $maxOutputs): static\n {\n parent::maxOutputs($maxOutputs);\n\n foreach ($this->steps as $step) {\n $step->maxOutputs($maxOutputs);\n }\n\n return $this;\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::AssociativeArrayOrObject;\n }\n\n protected function includeOutput(StepInterface $step): bool\n {\n if (\n !method_exists($step, 'shouldOutputBeExcludedFromGroupOutput') ||\n $step->shouldOutputBeExcludedFromGroupOutput() === false\n ) {\n return true;\n }\n\n return false;\n }\n\n /**\n * @param mixed[] $combined\n * @return mixed[]\n */\n private function addToCombinedOutputData(mixed $add, array $combined, int $nthElement): array\n {\n if (is_array($add)) {\n foreach ($add as $key => $value) {\n $combined[$nthElement][$key][] = $value;\n }\n } else {\n $combined[$nthElement][][] = $add;\n }\n\n return $combined;\n }\n\n /**\n * @return mixed[]\n */\n private function getNewlyKeptData(Output $output, Input $input): array\n {\n return array_filter($output->keep, function ($key) use ($input) {\n return !array_key_exists($key, $input->keep);\n }, ARRAY_FILTER_USE_KEY);\n }\n\n /**\n * @param mixed[] $combinedOutputs\n * @param mixed[] $combinedKeptData\n * @param Input $input\n * @return Generator\n * @throws Exception\n */\n private function prepareCombinedOutputs(array $combinedOutputs, array $combinedKeptData, Input $input): Generator\n {\n foreach ($combinedOutputs as $key => $combinedOutput) {\n if ($this->maxOutputsExceeded()) {\n break;\n }\n\n $outputData = $this->normalizeCombinedOutputs($combinedOutput);\n\n $outputData = $this->applyRefiners($outputData, $input->get());\n\n if ($this->passesAllFilters($outputData)) {\n $output = $this->makeOutput($outputData, $input);\n\n if (array_key_exists($key, $combinedKeptData)) {\n $output->keep($this->normalizeCombinedOutputs($combinedKeptData[$key]));\n }\n\n if ($this->uniqueOutput !== false && !$this->inputOrOutputIsUnique($output)) {\n continue;\n }\n\n yield $output;\n\n $this->trackYieldedOutput();\n }\n }\n }\n\n /**\n * Normalize combined outputs\n *\n * When adding outputs to combined output during step invocation, it always adds as arrays.\n * Here it unwraps all array properties with just one element to have just that one element as value.\n *\n * @param mixed[] $combinedOutputs\n * @return mixed[]\n */\n private function normalizeCombinedOutputs(array $combinedOutputs): array\n {\n return array_map(function ($output) {\n return count($output) === 1 ? reset($output) : $output;\n }, $combinedOutputs);\n }\n}\n" }, { "path": "src/Steps/Html/CssSelector.php", "content": "toXPath($query);\n } catch (ExpressionErrorException|SyntaxErrorException $exception) {\n throw InvalidDomQueryException::fromSymfonyException($query, $exception);\n }\n } else {\n try {\n (new HtmlDocument(''))->querySelector($query);\n } catch (DOMException $exception) {\n throw InvalidDomQueryException::fromDomException($query, $exception);\n }\n }\n }\n\n parent::__construct($query);\n }\n\n protected function filter(Node $node): NodeList\n {\n if ($this->query === '') {\n return new NodeList([$node]);\n }\n\n return $node->querySelectorAll($this->query);\n }\n}\n" }, { "path": "src/Steps/Html/DomQuery.php", "content": "toAbsoluteUrl && $node instanceof HtmlDocument) {\n $baseHref = $node->getBaseHref();\n\n if ($baseHref) {\n $this->setBaseUrl($baseHref);\n }\n }\n\n $filtered = $this->filter($node);\n\n if ($this->filtersMatches()) {\n $filtered = $this->filterMatches($filtered);\n\n if ($filtered === null) {\n return null;\n }\n }\n\n if ($filtered->count() > 1) {\n return $filtered->each(function ($element) {\n return $this->getTarget($element);\n });\n } elseif ($filtered->count() === 1) {\n $node = $filtered->first();\n\n if ($node instanceof HtmlElement || $node instanceof XmlElement) {\n return $this->getTarget($node);\n }\n }\n\n return null;\n }\n\n public function first(): self\n {\n $this->onlyFirstMatch = true;\n\n return $this;\n }\n\n public function last(): self\n {\n $this->onlyLastMatch = true;\n\n return $this;\n }\n\n public function nth(int $n): self\n {\n if ($n < 1) {\n throw new InvalidArgumentException('Argument $n must be greater than 0');\n }\n\n $this->onlyNthMatch = $n;\n\n return $this;\n }\n\n public function even(): self\n {\n $this->onlyEvenMatches = true;\n\n return $this;\n }\n\n public function odd(): self\n {\n $this->onlyOddMatches = true;\n\n return $this;\n }\n\n public function text(): self\n {\n $this->target = SelectorTarget::Text;\n\n return $this;\n }\n\n public function formattedText(?Html2Text $converter = null): self\n {\n $this->target = SelectorTarget::FormattedText;\n\n if ($converter) {\n $this->html2TextConverter = $converter;\n }\n\n return $this;\n }\n\n public function html(): self\n {\n $this->target = SelectorTarget::Html;\n\n return $this;\n }\n\n public function attribute(string $attributeName): self\n {\n $this->target = SelectorTarget::Attribute;\n\n $this->attributeName = $attributeName;\n\n return $this;\n }\n\n public function outerHtml(): self\n {\n $this->target = SelectorTarget::OuterHtml;\n\n return $this;\n }\n\n public function link(): self\n {\n $this->target = SelectorTarget::Attribute;\n\n $this->attributeName = 'href';\n\n $this->toAbsoluteUrl = true;\n\n return $this;\n }\n\n public function withoutFragment(): self\n {\n $this->withFragment = false;\n\n return $this;\n }\n\n /**\n * Call this method and the selected value will be converted to an absolute url when apply() is called.\n *\n * @return $this\n */\n public function toAbsoluteUrl(): self\n {\n $this->toAbsoluteUrl = true;\n\n return $this;\n }\n\n /**\n * Automatically called when used in a Dom step.\n *\n * @throws Exception\n */\n public function setBaseUrl(string $baseUrl): static\n {\n if (!empty($this->baseUrl)) {\n $this->baseUrl = Url::parse($this->baseUrl)->resolve($baseUrl)->__toString();\n } else {\n $this->baseUrl = $baseUrl;\n }\n\n return $this;\n }\n\n abstract protected function filter(Node $node): NodeList;\n\n protected function filtersMatches(): bool\n {\n return $this->onlyFirstMatch ||\n $this->onlyLastMatch ||\n $this->onlyNthMatch !== false ||\n $this->onlyEvenMatches ||\n $this->onlyOddMatches;\n }\n\n /**\n * @return NodeList|null\n * @throws Exception\n */\n protected function filterMatches(NodeList $matches): ?NodeList\n {\n if (\n $matches->count() === 0 ||\n ($this->onlyNthMatch !== false && $matches->count() < $this->onlyNthMatch)\n ) {\n return null;\n }\n\n if ($this->onlyFirstMatch) {\n $node = $matches->first();\n\n return $node ? new NodeList([$node]) : new NodeList([]);\n } elseif ($this->onlyLastMatch) {\n $node = $matches->last();\n\n return $node ? new NodeList([$node]) : new NodeList([]);\n } elseif ($this->onlyNthMatch !== false) {\n $node = $matches->nth($this->onlyNthMatch);\n\n return $node ? new NodeList([$node]) : new NodeList([]);\n } elseif ($this->onlyEvenMatches || $this->onlyOddMatches) {\n return $this->filterEvenOrOdd($matches);\n }\n\n return null;\n }\n\n /**\n * @param NodeList $domCrawler\n * @return NodeList\n */\n protected function filterEvenOrOdd(NodeList $domCrawler): NodeList\n {\n $nodes = [];\n\n $i = 1;\n\n foreach ($domCrawler as $node) {\n if (\n ($this->onlyEvenMatches && $i % 2 === 0) ||\n ($this->onlyOddMatches && $i % 2 !== 0)\n ) {\n $nodes[] = $node;\n }\n\n $i++;\n }\n\n return new NodeList($nodes);\n }\n\n /**\n * @throws InvalidHtmlException\n * @throws Exception\n */\n protected function getTarget(HtmlElement|XmlElement $node): string\n {\n if ($this->target === SelectorTarget::FormattedText) {\n if (!$this->html2TextConverter) {\n $this->html2TextConverter = new Html2Text();\n }\n\n $target = $this->html2TextConverter->convertHtmlToText(\n $node instanceof HtmlElement ? $node->outerHtml() : $node->outerXml(),\n );\n } elseif ($this->target === SelectorTarget::Html) {\n $target = $node instanceof HtmlElement ? trim($node->innerHtml()) : trim($node->innerXml());\n } elseif ($this->target === SelectorTarget::OuterHtml) {\n $target = $node instanceof HtmlElement ? trim($node->outerHtml()) : trim($node->outerXml());\n } else {\n $target = trim(\n $this->attributeName ?\n ($node->getAttribute($this->attributeName) ?? '') :\n (\n method_exists($node, strtolower($this->target->name)) ?\n $node->{strtolower($this->target->name)}() :\n ''\n ),\n );\n }\n\n if ($this->toAbsoluteUrl && $this->baseUrl !== null) {\n $target = $this->handleUrlFragment(Url::parse($this->baseUrl)->resolve($target));\n }\n\n if (str_contains($target, '�')) {\n $target = str_replace('�', '', $target);\n }\n\n return $target;\n }\n\n /**\n * @throws Exception\n */\n protected function handleUrlFragment(Url $url): Url\n {\n if (!$this->withFragment) {\n $url->fragment('');\n }\n\n return $url;\n }\n}\n" }, { "path": "src/Steps/Html/Exceptions/InvalidDomQueryException.php", "content": "setDomQuery($domQuery);\n\n return $exception;\n }\n\n public static function fromSymfonyException(\n string $domQuery,\n ExpressionErrorException|SyntaxErrorException $originalException,\n ): self {\n $exception = new self(\n $originalException->getMessage(),\n $originalException->getCode(),\n $originalException,\n );\n\n $exception->setDomQuery($domQuery);\n\n return $exception;\n }\n\n public static function fromDomException(string $domQuery, DOMException $originalException): self\n {\n $exception = new self(\n $originalException->getMessage(),\n $originalException->getCode(),\n $originalException,\n );\n\n $exception->setDomQuery($domQuery);\n\n return $exception;\n }\n\n public function setDomQuery(string $domQuery): void\n {\n $this->query = $domQuery;\n }\n\n public function getDomQuery(): string\n {\n return $this->query;\n }\n}\n" }, { "path": "src/Steps/Html/GetLink.php", "content": "selector = is_string($selector) ? new CssSelector($selector) : $selector;\n }\n\n public static function isSpecialNonHttpLink(HtmlElement $linkElement): bool\n {\n $href = $linkElement->getAttribute('href') ?? '';\n\n return str_starts_with($href, 'mailto:') ||\n str_starts_with($href, 'tel:') ||\n str_starts_with($href, 'javascript:');\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Scalar;\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function validateAndSanitizeInput(mixed $input): HtmlDocument\n {\n if (!$input instanceof RespondedRequest) {\n throw new InvalidArgumentException('Input must be an instance of RespondedRequest.');\n }\n\n $this->baseUri = Url::parse($input->effectiveUri());\n\n return new HtmlDocument(Http::getBodyString($input));\n }\n\n /**\n * @param HtmlDocument $input\n * @return Generator\n * @throws Exception\n */\n protected function invoke(mixed $input): Generator\n {\n $this->getBaseFromDocument($input);\n\n $selector = $this->selector ?? 'a';\n\n if (is_string($selector)) {\n $selector = new CssSelector($selector);\n }\n\n foreach ($input->querySelectorAll($selector->query) as $link) {\n $linkUrl = $this->getLinkUrl($link);\n\n if ($linkUrl) {\n yield (string) $linkUrl;\n\n break;\n }\n }\n }\n\n public function onSameDomain(): static\n {\n $this->onSameDomain = true;\n\n return $this;\n }\n\n public function notOnSameDomain(): static\n {\n $this->onSameDomain = false;\n\n return $this;\n }\n\n /**\n * @param string|string[] $domains\n * @return $this\n */\n public function onDomain(string|array $domains): static\n {\n if (is_array($domains) && !$this->isArrayWithOnlyStrings($domains)) {\n throw new InvalidArgumentException('You can only set domains from string values');\n }\n\n $domains = is_string($domains) ? [$domains] : $domains;\n\n $this->onDomain = $this->onDomain ? array_merge($this->onDomain, $domains) : $domains;\n\n return $this;\n }\n\n public function onSameHost(): static\n {\n $this->onSameHost = true;\n\n return $this;\n }\n\n public function notOnSameHost(): static\n {\n $this->onSameHost = false;\n\n return $this;\n }\n\n /**\n * @param string|string[] $hosts\n */\n public function onHost(string|array $hosts): static\n {\n if (is_array($hosts) && !$this->isArrayWithOnlyStrings($hosts)) {\n throw new InvalidArgumentException('You can only set hosts from string values');\n }\n\n $hosts = is_string($hosts) ? [$hosts] : $hosts;\n\n $this->onHost = $this->onHost ? array_merge($this->onHost, $hosts) : $hosts;\n\n return $this;\n }\n\n public function withoutFragment(): static\n {\n $this->withFragment = false;\n\n return $this;\n }\n\n /**\n * @throws Exception\n */\n protected function getBaseFromDocument(HtmlDocument $document): void\n {\n $baseHref = $document->getBaseHref();\n\n if (!empty($baseHref)) {\n $this->baseUri = $this->baseUri->resolve($baseHref);\n }\n }\n\n /**\n * @throws Exception\n */\n protected function getLinkUrl(HtmlElement $link): ?Url\n {\n if ($link->nodeName() !== 'a') {\n $this->logger?->warning('Selector matched <' . $link->nodeName() . '> html element. Ignored it.');\n\n return null;\n }\n\n if (self::isSpecialNonHttpLink($link)) {\n return null;\n }\n\n $linkUrl = $this->handleUrlFragment(\n $this->baseUri->resolve($link->getAttribute('href') ?? ''),\n );\n\n if ($this->matchesAdditionalCriteria($linkUrl)) {\n return $linkUrl;\n }\n\n return null;\n }\n\n /**\n * @throws Exception\n */\n protected function matchesAdditionalCriteria(Url $link): bool\n {\n return ($this->onSameDomain === null || $this->isOnSameDomain($link)) &&\n ($this->onSameHost === null || $this->isOnSameHost($link)) &&\n ($this->onDomain === null || $this->isOnDomain($link)) &&\n ($this->onHost === null || $this->isOnHost($link));\n }\n\n protected function isOnSameDomain(Url $link): bool\n {\n return ($this->onSameDomain && $this->baseUri->isDomainEqualIn($link)) ||\n ($this->onSameDomain === false && !$this->baseUri->isDomainEqualIn($link));\n }\n\n protected function isOnSameHost(Url $link): bool\n {\n return ($this->onSameHost && $this->baseUri->isHostEqualIn($link)) ||\n ($this->onSameHost === false && !$this->baseUri->isHostEqualIn($link));\n }\n\n /**\n * @throws Exception\n */\n protected function isOnDomain(Url $link): bool\n {\n if (is_array($this->onDomain)) {\n foreach ($this->onDomain as $domain) {\n if ($link->domain() === $domain) {\n return true;\n }\n }\n }\n\n return false;\n }\n\n /**\n * @throws Exception\n */\n protected function isOnHost(Url $link): bool\n {\n if (is_array($this->onHost)) {\n foreach ($this->onHost as $host) {\n if ($link->host() === $host) {\n return true;\n }\n }\n }\n\n return false;\n }\n\n /**\n * @param mixed[] $array\n * @return bool\n */\n protected function isArrayWithOnlyStrings(array $array): bool\n {\n foreach ($array as $element) {\n if (!is_string($element)) {\n return false;\n }\n }\n\n return true;\n }\n\n /**\n * @throws Exception\n */\n protected function handleUrlFragment(Url $url): Url\n {\n if (!$this->withFragment) {\n $url->fragment('');\n }\n\n return $url;\n }\n}\n" }, { "path": "src/Steps/Html/GetLinks.php", "content": "\n * @throws Exception\n */\n protected function invoke(mixed $input): Generator\n {\n $this->getBaseFromDocument($input);\n\n $selector = $this->selector ?? 'a';\n\n if (is_string($selector)) {\n $selector = new CssSelector($selector);\n }\n\n foreach ($input->querySelectorAll($selector->query) as $link) {\n $linkUrl = $this->getLinkUrl($link);\n\n if ($linkUrl) {\n yield (string) $linkUrl;\n }\n }\n }\n}\n" }, { "path": "src/Steps/Html/MetaData.php", "content": "onlyKeys = $keys;\n\n return $this;\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::AssociativeArrayOrObject;\n }\n\n /**\n * @param HtmlDocument $input\n */\n protected function invoke(mixed $input): Generator\n {\n $data = $this->addToData([], 'title', $this->getTitle($input));\n\n foreach ($input->querySelectorAll('meta') as $metaElement) {\n $metaName = $metaElement->getAttribute('name');\n\n if (empty($metaName)) {\n $metaName = $metaElement->getAttribute('property');\n }\n\n if (!empty($metaName) && (empty($this->onlyKeys) || in_array($metaName, $this->onlyKeys, true))) {\n $data = $this->addToData($data, $metaName, $metaElement->getAttribute('content') ?? '');\n }\n }\n\n yield $data;\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function validateAndSanitizeInput(mixed $input): mixed\n {\n return $this->validateAndSanitizeToHtmlDocumentInstance($input);\n }\n\n protected function getTitle(HtmlDocument $document): string\n {\n $titleElement = $document->querySelector('title');\n\n if ($titleElement) {\n return $titleElement->text();\n }\n\n return '';\n }\n\n /**\n * @param array $data\n * @return array\n */\n protected function addToData(array $data, string $key, string $value): array\n {\n if (empty($this->onlyKeys) || in_array($key, $this->onlyKeys, true)) {\n $data[$key] = $value;\n }\n\n return $data;\n }\n}\n" }, { "path": "src/Steps/Html/SchemaOrg.php", "content": "\n */\n protected array $mapping = [];\n\n public function toArray(): static\n {\n $this->toArray = true;\n\n return $this;\n }\n\n public function onlyType(string $type = ''): static\n {\n $this->onlyType = $type;\n\n return $this;\n }\n\n /**\n * @param array $mapping\n */\n public function extract(array $mapping): static\n {\n $this->mapping = $mapping;\n\n return $this;\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::AssociativeArrayOrObject;\n }\n\n /**\n * @param string $input\n */\n protected function invoke(mixed $input): Generator\n {\n $data = \\Crwlr\\SchemaOrg\\SchemaOrg::fromHtml($input, $this->logger);\n\n foreach ($data as $schemaOrgObject) {\n if ($this->onlyType && $schemaOrgObject->getType() !== $this->onlyType) {\n yield from $this->scanChildrenForType($schemaOrgObject);\n\n continue;\n }\n\n yield $this->prepareReturnValue($schemaOrgObject);\n }\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function validateAndSanitizeInput(mixed $input): string\n {\n return $this->validateAndSanitizeStringOrHttpResponse($input);\n }\n\n protected function scanChildrenForType(BaseType $schemaOrgObject): Generator\n {\n foreach ($schemaOrgObject->getProperties() as $propertyName => $property) {\n $propertyValue = $schemaOrgObject->getProperty($propertyName);\n\n if ($propertyValue instanceof BaseType && $propertyValue->getType() === $this->onlyType) {\n yield $this->prepareReturnValue($propertyValue);\n } elseif ($propertyValue instanceof BaseType) {\n yield from $this->scanChildrenForType($propertyValue);\n }\n }\n }\n\n /**\n * @return BaseType|mixed[]\n */\n protected function prepareReturnValue(BaseType $object): BaseType|array\n {\n if ($this->toArray || !empty($this->mapping)) {\n if (empty($this->mapping)) {\n return $object->toArray();\n }\n\n return $this->applyMapping($object->toArray());\n }\n\n return $object;\n }\n\n /**\n * @param mixed[] $schemaOrgData\n * @return mixed[]\n */\n protected function applyMapping(array $schemaOrgData): array\n {\n $extractedData = [];\n\n $dot = new Dot($schemaOrgData);\n\n foreach ($this->mapping as $outputKey => $dotNotationKey) {\n if (is_int($outputKey)) {\n $outputKey = $dotNotationKey;\n }\n\n $extractedData[$outputKey] = $dot->get($dotNotationKey);\n }\n\n return $extractedData;\n }\n}\n" }, { "path": "src/Steps/Html/SelectorTarget.php", "content": "validateQuery($query);\n }\n\n parent::__construct(trim($query));\n }\n\n protected function filter(Node $node): NodeList\n {\n if ($this->query === '') {\n return new NodeList([$node]);\n }\n\n return $node->queryXPath($this->query);\n }\n\n /**\n * @throws InvalidDomQueryException\n */\n private function validateQuery(string $query): void\n {\n // Temporarily set a new error handler, so checking an invalid XPath query does not generate a PHP warning.\n set_error_handler(function ($errno, $errstr) {\n if ($errno === E_WARNING && $errstr === 'DOMXPath::evaluate(): Invalid expression') {\n return true;\n }\n\n return false;\n });\n\n $evaluation = (new DOMXPath(new DOMDocument()))->evaluate($query);\n\n restore_error_handler();\n\n if ($evaluation === false) {\n throw InvalidDomQueryException::make('Invalid XPath query', $query);\n }\n }\n}\n" }, { "path": "src/Steps/Html.php", "content": "baseUrl = $input->effectiveUri();\n }\n\n return $this->validateAndSanitizeToHtmlDocumentInstance($input);\n }\n\n /**\n * @throws InvalidDomQueryException\n */\n protected function makeDefaultDomQueryInstance(string $query): DomQuery\n {\n return new CssSelector($query);\n }\n}\n" }, { "path": "src/Steps/Json.php", "content": "validateAndSanitizeStringOrHttpResponse($input);\n }\n\n protected function invoke(mixed $input): Generator\n {\n $array = $this->inputStringToArray($input);\n\n if ($array === null || $this->propertyMapping === null) {\n if ($array === null) {\n $this->logger?->warning('Failed to decode JSON string.');\n } elseif ($this->propertyMapping === null) {\n yield $array;\n }\n\n return;\n }\n\n $dot = new Dot($array);\n\n if ($this->each === null) {\n yield $this->mapProperties($dot);\n } else {\n $each = $this->each === '' ? $dot->get() : $dot->get($this->each);\n\n if (!is_iterable($each)) {\n $this->logger?->warning('The target of \"each\" does not exist in the JSON data.');\n } else {\n foreach ($each as $item) {\n yield $this->mapProperties(new Dot($item));\n }\n }\n }\n }\n\n /**\n * @return mixed[]|null\n */\n protected function inputStringToArray(string $input): ?array\n {\n try {\n return JsonUtil::stringToArray($input);\n } catch (InvalidJsonException) {\n // If headless browser is used in loader, the JSON in the response body is wrapped in an HTML document.\n if (str_contains($input, 'querySelector('body')?->text() ?? '';\n\n return JsonUtil::stringToArray($bodyText);\n } catch (Throwable) {\n }\n }\n }\n\n return null;\n }\n\n /**\n * @param Dot $dot\n * @return mixed[]\n */\n protected function mapProperties(Dot $dot): array\n {\n if ($this->propertyMapping === null || $this->propertyMapping === []) {\n return [];\n }\n\n $mapped = [];\n\n foreach ($this->propertyMapping as $propertyKey => $dotNotation) {\n if (is_int($propertyKey)) {\n $propertyKey = $dotNotation;\n }\n\n if ($dotNotation === '' || ($dotNotation === '*' && $dot->get('*') === null)) {\n $mapped[$propertyKey] = $dot->all();\n } else {\n $mapped[$propertyKey] = $dot->get($dotNotation);\n }\n }\n\n return $mapped;\n }\n}\n" }, { "path": "src/Steps/Loading/GetSitemapsFromRobotsTxt.php", "content": "\n */\n use LoadingStep;\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Scalar;\n }\n\n /**\n * @throws InvalidRobotsTxtFileException\n */\n protected function invoke(mixed $input): Generator\n {\n $robotsTxtHandler = $this->getLoader()->robotsTxt();\n\n foreach ($robotsTxtHandler->getSitemaps($input) as $sitemapUrl) {\n yield $sitemapUrl;\n }\n }\n\n /**\n * @throws InvalidArgumentException\n */\n protected function validateAndSanitizeInput(mixed $input): UriInterface\n {\n return $this->validateAndSanitizeToUriInterface($input);\n }\n}\n" }, { "path": "src/Steps/Loading/Http/AbstractPaginator.php", "content": "\n */\n protected array $loaded = [];\n\n protected int $loadedCount = 0;\n\n protected ?RequestInterface $latestRequest;\n\n /**\n * @var array\n */\n protected array $stopRules = [];\n\n protected bool $hasFinished = false;\n\n public function __construct(protected int $maxPages = Paginator::MAX_PAGES_DEFAULT) {}\n\n public function processLoaded(\n RequestInterface $request,\n ?RespondedRequest $respondedRequest,\n ): void {\n $this->registerLoadedRequest($respondedRequest ?? $request);\n }\n\n public function hasFinished(): bool\n {\n return $this->hasFinished || $this->maxPagesReached();\n }\n\n /**\n * When a paginate step is called with multiple inputs, like:\n *\n * ['https://www.example.com/listing1', 'https://www.example.com/listing2', ...]\n *\n * it always has to start paginating again for each listing base URL.\n * Therefore, we reset the state after finishing paginating one base input.\n * Except for $this->found, because if it would be the case that the exact same pages are\n * discovered whilst paginating, we don't want to load the exact same pages again and again.\n */\n public function resetFinished(): void\n {\n $this->hasFinished = false;\n\n $this->loadedCount = 0;\n\n $this->latestRequest = null;\n }\n\n public function stopWhen(Closure|StopRule $callback): self\n {\n $this->stopRules[] = $callback;\n\n return $this;\n }\n\n public function logWhenFinished(LoggerInterface $logger): void\n {\n if ($this->maxPagesReached()) {\n $logger->notice('Max pages limit reached.');\n } else {\n $logger->info('Finished paginating.');\n }\n }\n\n abstract public function getNextRequest(): ?RequestInterface;\n\n protected function registerLoadedRequest(RequestInterface|RespondedRequest $request): void\n {\n $key = $request instanceof RespondedRequest ? RequestKey::from($request->request) : RequestKey::from($request);\n\n if (array_key_exists($key, $this->loaded)) {\n return;\n }\n\n $this->loaded[$key] = true;\n\n $this->loadedCount++;\n\n if ($request instanceof RespondedRequest) {\n foreach ($request->redirects() as $redirectUrl) {\n $this->loaded[RequestKey::from($request->request->withUri(Url::parsePsr7($redirectUrl)))] = true;\n }\n }\n\n $this->latestRequest = $request instanceof RespondedRequest ? $request->request : $request;\n\n $respondedRequest = $request instanceof RespondedRequest ? $request : null;\n\n $request = $request instanceof RequestInterface ? $request : $request->request;\n\n if ($this->shouldStop($request, $respondedRequest)) {\n $this->setFinished();\n }\n }\n\n protected function shouldStop(RequestInterface $request, ?RespondedRequest $respondedRequest): bool\n {\n if ($this->maxPagesReached()) {\n return true;\n }\n\n foreach ($this->stopRules as $stopRule) {\n if ($stopRule instanceof StopRule && $stopRule->shouldStop($request, $respondedRequest)) {\n return true;\n } elseif ($stopRule instanceof Closure && $stopRule->call($this, $request, $respondedRequest)) {\n return true;\n }\n }\n\n return false;\n }\n\n protected function maxPagesReached(): bool\n {\n return $this->loadedCount >= $this->maxPages;\n }\n\n protected function setFinished(): self\n {\n $this->hasFinished = true;\n\n return $this;\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Browser/BrowserAction.php", "content": "waitUntilContainsElement($cssSelector, $timeout);\n };\n }\n\n public static function clickElement(\n string $cssSelector,\n int $timeout = self::DEFAULT_TIMEOUT,\n ): Closure {\n return function (Page $page) use ($cssSelector, $timeout) {\n $page->waitUntilContainsElement($cssSelector, $timeout);\n\n $page->mouse()->find($cssSelector)->click();\n };\n }\n\n /**\n * Click an element that lives inside a shadow DOM within the document.\n *\n * For this purpose the action needs two selectors: the first one to select the shadow host element and the\n * second one to select the element that shall be clicked inside that shadow DOM.\n */\n public static function clickInsideShadowDom(\n string $shadowHostSelector,\n string $clickElementSelector,\n int $timeout = self::DEFAULT_TIMEOUT,\n ): Closure {\n return function (Page $page) use ($shadowHostSelector, $clickElementSelector, $timeout) {\n $page->evaluate(<< setTimeout(resolve, 25));\n shadowHostElement = document.querySelector('{$shadowHostSelector}');\n }\n\n if (shadowHostElement.shadowRoot) {\n let clickElement = shadowHostElement.shadowRoot.querySelector('{$clickElementSelector}');\n\n while (!clickElement) {\n await new Promise(resolve => setTimeout(resolve, 25));\n clickElement = shadowHostElement.shadowRoot.querySelector('{$clickElementSelector}');\n }\n\n clickElement.dispatchEvent(new MouseEvent(\"click\", { bubbles: true }));\n }\n })()\n JS)->waitForResponse($timeout);\n };\n }\n\n public static function moveMouseToElement(string $cssSelector, int $timeout = self::DEFAULT_TIMEOUT): Closure\n {\n return function (Page $page) use ($cssSelector, $timeout) {\n $page->waitUntilContainsElement($cssSelector, $timeout);\n\n $page->mouse()->find($cssSelector);\n };\n }\n\n public static function moveMouseToPosition(int $x, int $y, ?int $steps = null): Closure\n {\n return function (Page $page) use ($x, $y, $steps) {\n if ($steps !== null) {\n $page->mouse()->move($x, $y, ['steps' => $steps]);\n } else {\n $page->mouse()->move($x, $y);\n }\n };\n }\n\n public static function scrollDown(int $distance): Closure\n {\n return function (Page $page) use ($distance) {\n $page->mouse()->scrollDown($distance);\n };\n }\n\n public static function scrollUp(int $distance): Closure\n {\n return function (Page $page) use ($distance) {\n $page->mouse()->scrollUp($distance);\n };\n }\n\n public static function typeText(string $text, ?int $delay = null): Closure\n {\n return function (Page $page) use ($text, $delay) {\n if ($delay !== null) {\n $page->keyboard()->setKeyInterval($delay)->typeText($text);\n } else {\n $page->keyboard()->typeText($text);\n }\n };\n }\n\n public static function evaluate(string $jsCode): Closure\n {\n return function (Page $page) use ($jsCode) {\n $page->evaluate($jsCode);\n };\n }\n\n public static function waitForReload(int $timeout = self::DEFAULT_TIMEOUT): Closure\n {\n return function (Page $page) use ($timeout) {\n $page->waitForReload(timeout: $timeout);\n };\n }\n\n public static function wait(float $seconds): Closure\n {\n return function () use ($seconds) {\n usleep(Microseconds::fromSeconds($seconds)->value);\n };\n }\n\n public static function screenshot(ScreenshotConfig $config): Closure\n {\n return function (Page $page, ?LoggerInterface $logger) use ($config) {\n $fullFilePath = $config->getFullPath($page);\n\n try {\n $page->screenshot($config->toChromePhpScreenshotConfig($page))->saveToFile($fullFilePath);\n\n return new Screenshot($fullFilePath);\n } catch (Throwable $exception) {\n $logger?->error('Failed to take screenshot.');\n\n $logger?->debug($exception->getMessage());\n\n return null;\n }\n };\n }\n\n /**\n * @deprecated Use the two methods evaluate() and waitForReload() separately.\n */\n public static function evaluateAndWaitForReload(string $jsCode): Closure\n {\n return function (Page $page) use ($jsCode) {\n $page->evaluate($jsCode)->waitForPageReload();\n };\n }\n\n /**\n * @deprecated Use the two methods clickElement() and waitForReload() separately.\n */\n public static function clickElementAndWaitForReload(string $cssSelector): Closure\n {\n return function (Page $page) use ($cssSelector) {\n $page->waitUntilContainsElement($cssSelector);\n\n $page->mouse()->find($cssSelector)->click();\n\n $page->waitForReload();\n };\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Document.php", "content": "respondedRequest);\n\n $this->dom = new HtmlDocument($responseBody);\n\n $this->setBaseUrl();\n }\n\n public function dom(): HtmlDocument\n {\n return $this->dom;\n }\n\n public function url(): Url\n {\n return $this->url;\n }\n\n public function baseUrl(): Url\n {\n return $this->baseUrl;\n }\n\n public function canonicalUrl(): string\n {\n if ($this->canonicalUrl === null) {\n $canonicalLinkElement = $this->dom->querySelector('link[rel=canonical]');\n\n if ($canonicalLinkElement) {\n $canonicalHref = $canonicalLinkElement->getAttribute('href');\n\n if ($canonicalHref) {\n try {\n $this->canonicalUrl = $this->baseUrl->resolve($canonicalHref);\n } catch (Exception $exception) {\n $this->logger?->warning(\n 'Failed to resolve canonical link href value against the document base URL.',\n );\n }\n }\n }\n\n $this->canonicalUrl = $this->canonicalUrl ?? $this->url;\n }\n\n return $this->canonicalUrl;\n }\n\n private function setBaseUrl(): void\n {\n $this->url = Url::parse($this->respondedRequest->effectiveUri());\n\n $this->baseUrl = $this->url;\n\n $documentBaseHref = $this->dom->getBaseHref();\n\n if ($documentBaseHref) {\n try {\n $this->baseUrl = $this->baseUrl->resolve($documentBaseHref);\n } catch (Exception $exception) {\n $this->logger?->warning('Failed to resolve the document tag href against the document URL.');\n }\n }\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginate.php", "content": "paginateInputUrl($inputUrl);\n }\n } else {\n yield from $this->paginateInputUrl($input);\n }\n }\n\n /**\n * @throws LoadingException\n */\n private function paginateInputUrl(UriInterface $url): Generator\n {\n $request = $this->getRequestFromInputUri($url);\n\n $response = $this->getResponseFromRequest($request);\n\n if ($response) {\n yield $response;\n }\n\n $this->processLoaded($request, $response);\n\n while (!$this->paginator->hasFinished()) {\n $request = $this->paginator->getNextRequest();\n\n if (!$request) {\n break;\n }\n\n $response = $this->getResponseFromRequest($request);\n\n if ($response) {\n yield $response;\n }\n\n $this->processLoaded($request, $response);\n }\n\n $this->finish();\n }\n\n private function finish(): void\n {\n if ($this->logger) {\n $this->paginator->logWhenFinished($this->logger);\n\n $this->paginator->resetFinished();\n }\n }\n\n private function processLoaded(RequestInterface $request, ?RespondedRequest $response): void\n {\n try {\n $this->paginator->processLoaded($request, $response);\n } catch (Exception $exception) {\n $this->logger?->error('Paginate Error: ' . $exception->getMessage());\n }\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginator.php", "content": "has($this->queryParamName)) {\n return $query->get($this->queryParamName);\n }\n\n return $fallbackValue;\n }\n\n /**\n * @throws Exception\n */\n protected function getCurrentValueUsingDotNotation(Query $query, mixed $fallbackValue = null): mixed\n {\n $dot = new Dot($query->toArray());\n\n return $dot->get($this->queryParamName, $fallbackValue);\n }\n\n /**\n * @throws Exception\n */\n protected function getCurrentValueAsInt(Query $query): int\n {\n return (int) $this->getCurrentValue($query);\n }\n\n /**\n * @throws Exception\n */\n protected function getCurrentValueAsIntUsingDotNotation(Query $query): int\n {\n return (int) $this->getCurrentValueUsingDotNotation($query);\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/QueryParams/Decrementor.php", "content": "useDotNotation) {\n $dot = (new Dot($query->toArray()))->set(\n $this->queryParamName,\n (string) ($this->getCurrentValueAsIntUsingDotNotation($query) - $this->decrement),\n );\n\n return new Query($dot->all());\n }\n\n return $query->set(\n $this->queryParamName,\n (string) ($this->getCurrentValueAsInt($query) - $this->decrement),\n );\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/QueryParams/Incrementor.php", "content": "useDotNotation) {\n $dot = (new Dot($query->toArray()))->set(\n $this->queryParamName,\n (string) ($this->getCurrentValueAsIntUsingDotNotation($query) + $this->increment),\n );\n\n return new Query($dot->all());\n }\n\n return $query->set(\n $this->queryParamName,\n (string) ($this->getCurrentValueAsInt($query) + $this->increment),\n );\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/QueryParams/QueryParamManipulator.php", "content": "paramsInUrl = true;\n\n return $this;\n }\n\n public static function paramsInBody(int $maxPages = Paginator::MAX_PAGES_DEFAULT): self\n {\n $instance = new self($maxPages);\n\n $instance->paramsInUrl = false;\n\n return $instance;\n }\n\n public function inBody(): self\n {\n $this->paramsInUrl = false;\n\n return $this;\n }\n\n public function increase(string $queryParamName, int $by = 1, bool $useDotNotation = false): self\n {\n $this->manipulators[] = new Incrementor($queryParamName, $by, $useDotNotation);\n\n return $this;\n }\n\n public function increaseUsingDotNotation(string $queryParamName, int $by = 1): self\n {\n $this->manipulators[] = new Incrementor($queryParamName, $by, true);\n\n return $this;\n }\n\n public function decrease(string $queryParamName, int $by = 1, bool $useDotNotation = false): self\n {\n $this->manipulators[] = new Decrementor($queryParamName, $by, $useDotNotation);\n\n return $this;\n }\n\n public function decreaseUsingDotNotation(string $queryParamName, int $by = 1): self\n {\n $this->manipulators[] = new Decrementor($queryParamName, $by, true);\n\n return $this;\n }\n\n /**\n * @throws Exception\n */\n public function getNextRequest(): ?RequestInterface\n {\n if (!$this->latestRequest) {\n return null;\n }\n\n if ($this->paramsInUrl) {\n $url = Url::parse($this->latestRequest->getUri());\n\n $query = $url->queryString();\n } else {\n $query = Query::fromString(Http::getBodyString($this->latestRequest));\n }\n\n foreach ($this->manipulators as $manipulator) {\n $query = $manipulator->execute($query);\n }\n\n if ($this->paramsInUrl) {\n $request = $this->latestRequest->withUri($url->toPsr7());\n } else {\n $request = $this->latestRequest->withBody(Utils::streamFor($query->toString()));\n }\n\n return $request;\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/SimpleWebsitePaginator.php", "content": "\n */\n protected array $found = [];\n\n /**\n * @var array\n */\n protected array $loadedUrls = [];\n\n protected DomQuery $paginationLinksSelector;\n\n protected string $latestRequestKey = '';\n\n /**\n * @var array\n */\n protected array $parentRequests = [];\n\n /**\n * @throws InvalidDomQueryException\n */\n public function __construct(string|DomQuery $paginationLinksSelector, int $maxPages = 1000)\n {\n if (is_string($paginationLinksSelector)) {\n $this->paginationLinksSelector = Dom::cssSelector($paginationLinksSelector);\n } else {\n $this->paginationLinksSelector = $paginationLinksSelector;\n }\n\n parent::__construct($maxPages);\n }\n\n public function hasFinished(): bool\n {\n return $this->maxPagesReached() || empty($this->found) || $this->hasFinished;\n }\n\n public function getNextRequest(): ?RequestInterface\n {\n if (!$this->latestRequest) {\n return null;\n }\n\n $nextUrl = array_shift($this->found);\n\n if (!$nextUrl) {\n return null;\n }\n\n $request = $this->parentRequests[$nextUrl['foundOn']];\n\n $this->cleanUpParentRequests();\n\n return $request->withUri(Url::parsePsr7($nextUrl['url']));\n }\n\n /**\n * @throws Exception\n */\n public function processLoaded(\n RequestInterface $request,\n ?RespondedRequest $respondedRequest,\n ): void {\n $this->registerLoadedRequest($respondedRequest ?? $request);\n\n if ($this->latestRequest) {\n $this->latestRequestKey = RequestKey::from($this->latestRequest);\n }\n\n $this->loadedUrls[$request->getUri()->__toString()] = true;\n\n if ($respondedRequest) {\n foreach ($respondedRequest->redirects() as $redirectUrl) {\n $this->loadedUrls[$redirectUrl] = true;\n }\n\n $this->getPaginationLinksFromResponse($respondedRequest);\n }\n }\n\n public function logWhenFinished(LoggerInterface $logger): void\n {\n if ($this->maxPagesReached() && !empty($this->found)) {\n $logger->notice('Max pages limit reached');\n } else {\n $logger->info('All found pagination links loaded');\n }\n }\n\n /**\n * @throws Exception\n */\n protected function getPaginationLinksFromResponse(RespondedRequest $respondedRequest): void\n {\n $responseBody = Http::getBodyString($respondedRequest);\n\n $document = new Dom\\HtmlDocument($responseBody);\n\n $paginationLinksElements = $this->paginationLinksSelector instanceof CssSelector ?\n $document->querySelectorAll($this->paginationLinksSelector->query) :\n $document->queryXPath($this->paginationLinksSelector->query);\n\n foreach ($paginationLinksElements as $paginationLinksElement) {\n /** @var Dom\\HtmlElement $paginationLinksElement */\n $this->addFoundUrlFromLinkElement(\n $paginationLinksElement,\n $document,\n $respondedRequest->effectiveUri(),\n );\n\n foreach ($paginationLinksElement->querySelectorAll('a') as $linkInPaginationLinksElement) {\n $this->addFoundUrlFromLinkElement(\n $linkInPaginationLinksElement,\n $document,\n $respondedRequest->effectiveUri(),\n );\n }\n }\n }\n\n /**\n * @throws Exception\n */\n protected function addFoundUrlFromLinkElement(\n Dom\\HtmlElement $linkElement,\n Dom\\HtmlDocument $document,\n string $documentUrl,\n ): void {\n if ($this->isRelevantLinkElement($linkElement)) {\n $url = $this->getAbsoluteUrlFromLinkElement($linkElement, $document, $documentUrl);\n\n $this->addFoundUrl($url);\n }\n }\n\n /**\n * @throws Exception\n */\n protected function getAbsoluteUrlFromLinkElement(\n Dom\\HtmlElement $linkElement,\n Dom\\HtmlDocument $document,\n string $documentUrl,\n ): string {\n $baseUrl = Url::parse($documentUrl);\n\n $baseHref = $document->getBaseHref();\n\n if ($baseHref) {\n $baseUrl = $baseUrl->resolve($baseHref);\n }\n\n $linkHref = $linkElement->getAttribute('href') ?? '';\n\n return $baseUrl->resolve($linkHref)->__toString();\n }\n\n protected function isRelevantLinkElement(Dom\\HtmlElement $element): bool\n {\n if ($element->nodeName() !== 'a') {\n return false;\n }\n\n $href = $element->getAttribute('href');\n\n return !empty($href) && !str_starts_with($href, '#');\n }\n\n protected function addFoundUrl(string $url): void\n {\n if (!isset($this->found[$url]) && !isset($this->loadedUrls[$url])) {\n if ($this->latestRequest && !array_key_exists($this->latestRequestKey, $this->parentRequests)) {\n $this->parentRequests[$this->latestRequestKey] = $this->latestRequest;\n }\n\n $this->found[$url] = ['url' => $url, 'foundOn' => $this->latestRequestKey];\n }\n }\n\n /**\n * The parent requests for found links are stored, so the new requests are always created from the actual parent,\n * not the latest registered response. After getting the next request to load, always check for all parent\n * requests, if there are still children in the found URLs. If not, the parent request can be forgotten, so we\n * keep memory usage as low as possible.\n */\n protected function cleanUpParentRequests(): void\n {\n foreach ($this->parentRequests as $requestKey => $request) {\n foreach ($this->found as $found) {\n if ($found['foundOn'] === $requestKey) {\n continue 2;\n }\n }\n\n unset($this->parentRequests[$requestKey]);\n }\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/StopRules/Contains.php", "content": "response));\n\n return str_contains($content, $this->contains);\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/StopRules/IsEmptyInDom.php", "content": "response));\n\n try {\n $document = $this->makeDom($source);\n } catch (Throwable $exception) {\n return true;\n }\n\n $domQuery = $this->selector instanceof DomQuery ? $this->selector : new CssSelector($this->selector);\n\n $filtered = $domQuery instanceof CssSelector ?\n $document->querySelectorAll($domQuery->query) :\n $document->queryXPath($domQuery->query);\n\n if ($filtered->count() === 0) {\n return true;\n }\n\n foreach ($filtered as $element) {\n /** @var HtmlElement|XmlElement $element */\n if (!$this->nodeIsEmpty($element)) {\n return false;\n }\n }\n\n return true;\n }\n\n abstract protected function makeDom(string $source): DomDocument;\n\n private function nodeIsEmpty(HtmlElement|XmlElement $node): bool\n {\n return $node instanceof HtmlElement ? trim($node->innerHtml()) === '' : trim($node->innerXml()) === '';\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/StopRules/IsEmptyInHtml.php", "content": "response));\n\n $json = Json::stringToArray($content);\n\n $dot = new Dot($json);\n\n return empty($dot->get($this->dotNotationKey));\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/StopRules/IsEmptyInXml.php", "content": "response));\n\n return $content === '' || $content === '[]' || $content === '{}';\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/StopRules/NotContains.php", "content": "response));\n\n return !str_contains($content, $this->contains);\n }\n}\n" }, { "path": "src/Steps/Loading/Http/Paginators/StopRules/PaginatorStopRules.php", "content": "response : $message;\n\n $message->getBody()->rewind();\n\n $contents = $message->getBody()->getContents();\n\n $message->getBody()->rewind();\n\n if (in_array('application/x-gzip', $message->getHeader('Content-Type'), true)) {\n return Gzip::decode($contents);\n }\n\n return $contents;\n }\n\n /**\n * @throws InvalidDomQueryException\n */\n public function paginate(\n AbstractPaginator|string $paginator,\n int $defaultPaginatorMaxPages = Paginator::MAX_PAGES_DEFAULT,\n ): Paginate {\n if (is_string($paginator)) {\n $paginator = Paginator::simpleWebsite($paginator, $defaultPaginatorMaxPages);\n }\n\n return $this->transferSettingsToPaginateStep(\n new Paginate($paginator, $this->method, $this->headers, $this->body, $this->httpVersion),\n );\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::AssociativeArrayOrObject;\n }\n\n /**\n * @param UriInterface|UriInterface[] $input\n * @return Generator\n * @throws Exception\n */\n protected function invoke(mixed $input): Generator\n {\n $input = !is_array($input) ? [$input] : $input;\n\n foreach ($input as $uri) {\n $response = $this->getResponseFromInputUri($uri);\n\n if ($response) {\n yield $response;\n }\n }\n\n $this->resetInputRequestParams();\n }\n\n /**\n * Temporary fix to transfer settings that may have already been defined on the current instance,\n * to a new Paginate step instance. This shall be fixed in the next major version (v4) by removing\n * the Paginate class and implementing it in the Http class directly.\n */\n private function transferSettingsToPaginateStep(Paginate $step): Paginate\n {\n $step->stopOnErrorResponse = $this->stopOnErrorResponse;\n\n $step->yieldErrorResponses = $this->yieldErrorResponses;\n\n $step->useAsUrl = $this->useAsUrl;\n\n $step->useAsBody = $this->useAsBody;\n\n $step->useAsHeaders = $this->useAsHeaders;\n\n $step->useAsHeader = $this->useAsHeader;\n\n $step->staticUrl = $this->staticUrl;\n\n $step->postBrowserNavigateHooks = $this->postBrowserNavigateHooks;\n\n $step->skipCache = $this->skipCache;\n\n $step->forceBrowserUsage = $this->forceBrowserUsage;\n\n return $step;\n }\n}\n" }, { "path": "src/Steps/Loading/HttpBase.php", "content": "\n */\n use LoadingStep;\n\n protected bool $stopOnErrorResponse = false;\n\n protected bool $yieldErrorResponses = false;\n\n protected ?string $useAsUrl = null;\n\n protected ?string $useAsBody = null;\n\n protected ?string $inputBody = null;\n\n protected ?string $useAsHeaders = null;\n\n /**\n * @var null|array\n */\n protected ?array $useAsHeader = null;\n\n /**\n * @var null|array\n */\n protected ?array $inputHeaders = null;\n\n protected ?string $staticUrl = null;\n\n /**\n * @var Closure[]\n */\n protected array $postBrowserNavigateHooks = [];\n\n protected bool $skipCache = false;\n\n protected bool $forceBrowserUsage = false;\n\n /**\n * @param string $method\n * @param array $headers\n * @param string|StreamInterface|null $body\n * @param string $httpVersion\n */\n public function __construct(\n protected readonly string $method = 'GET',\n protected readonly array $headers = [],\n protected readonly string|StreamInterface|null $body = null,\n protected readonly string $httpVersion = '1.1',\n ) {}\n\n public function stopOnErrorResponse(): static\n {\n $this->stopOnErrorResponse = true;\n\n return $this;\n }\n\n public function yieldErrorResponses(): static\n {\n $this->yieldErrorResponses = true;\n\n return $this;\n }\n\n /**\n * Chose key from array input to use its value as request URL\n *\n * If input is an array with string keys, you can define which key from that array should be used as the URL for\n * the HTTP request.\n */\n public function useInputKeyAsUrl(string $key): static\n {\n $this->useAsUrl = $key;\n\n return $this;\n }\n\n /**\n * Chose key from array input to use its value as request body\n *\n * If input is an array with string keys, you can define which key from that array should be used as the body for\n * the HTTP request.\n */\n public function useInputKeyAsBody(string $key): static\n {\n $this->useAsBody = $key;\n\n return $this;\n }\n\n /**\n * Chose key from array input to use its value as a request header\n *\n * If input is an array with string keys, you can choose a key from that array and map it to an HTTP request header.\n */\n public function useInputKeyAsHeader(string $key, ?string $asHeader = null): static\n {\n $asHeader = $asHeader ?? $key;\n\n if ($this->useAsHeader === null) {\n $this->useAsHeader = [];\n }\n\n $this->useAsHeader[$key] = $asHeader;\n\n return $this;\n }\n\n /**\n * Chose key from array input to use its value as request headers\n *\n * If input is an array with string keys, you can choose a key from that array that will be used as headers for the\n * HTTP request. So, the value behind that array key, has to be an array with header names as keys. If you want to\n * map just one single HTTP header from input, use the `useInputKeyAsHeader()` method.\n */\n public function useInputKeyAsHeaders(string $key): static\n {\n $this->useAsHeaders = $key;\n\n return $this;\n }\n\n public function postBrowserNavigateHook(Closure $callback): static\n {\n if ($this->method !== 'GET') {\n $this->logger?->warning(\n 'A ' . $this->method . ' request cannot be executed using the (headless) browser, so post browser ' .\n 'navigate hooks can\\'t be defined for this step either.',\n );\n\n return $this;\n }\n\n $this->postBrowserNavigateHooks[] = $callback;\n\n return $this;\n }\n\n /**\n * Skip using the cache for this step\n *\n * If you're using a cache in your crawler's loader, but want to skip using the cache for one\n * particular step in the chain, use this method.\n *\n * Attention: this has no effect if you directly use the loader in a custom child step.\n * If you want to use this feature, please use getResponseFromInputUri() or getResponseFromRequest()\n * instead of the loader.\n */\n public function skipCache(): static\n {\n $this->skipCache = true;\n\n return $this;\n }\n\n /**\n * This allows the step to temporarily switch the loader to use the (headless) Chrome browser,\n * even if it is configured to use the (guzzle) HTTP client. When a request is finished,\n * it resets the loader setting.\n *\n * Attention: this has no effect if you directly use the loader in a custom child step.\n * If you want to use this feature, please use getResponseFromInputUri() or getResponseFromRequest()\n * instead of the loader.\n */\n public function useBrowser(): static\n {\n $this->forceBrowserUsage = true;\n\n return $this;\n }\n\n public function staticUrl(string $url): static\n {\n $this->staticUrl = $url;\n\n return $this;\n }\n\n /**\n * @return UriInterface|UriInterface[]\n * @throws InvalidArgumentException\n */\n protected function validateAndSanitizeInput(mixed $input): mixed\n {\n $this->getBodyFromArrayInput($input);\n\n $this->getHeadersFromArrayInput($input);\n\n $input = $this->staticUrl ? $this->resolveStaticUrl() : $this->getUrlFromArrayInput($input);\n\n if (is_array($input)) {\n foreach ($input as $key => $url) {\n $input[$key] = $this->validateAndSanitizeToUriInterface($url);\n }\n\n return $input;\n }\n\n return $this->validateAndSanitizeToUriInterface($input);\n }\n\n protected function outputKeyAliases(): array\n {\n return [\n 'url' => 'effectiveUri',\n 'uri' => 'effectiveUri',\n 'status' => 'responseStatusCode',\n 'headers' => 'responseHeaders',\n 'body' => 'responseBody',\n ];\n }\n\n /**\n * @throws LoadingException\n */\n protected function getResponseFromInputUri(UriInterface $input): ?RespondedRequest\n {\n $request = $this->getRequestFromInputUri($input);\n\n return $this->getResponseFromRequest($request);\n }\n\n protected function getRequestFromInputUri(UriInterface $uri): RequestInterface\n {\n $body = $this->inputBody ?? $this->body;\n\n $headers = $this->mergeHeaders();\n\n list($body, $headers) = $this->resolveVarsInRequestProperties($body, $headers);\n\n return new Request($this->method, $uri, $headers, $body, $this->httpVersion);\n }\n\n /**\n * @throws LoadingException\n * @throws Exception\n */\n protected function getResponseFromRequest(RequestInterface $request): ?RespondedRequest\n {\n $loader = $this->getLoader();\n\n $loaderResetConfig = $this->applyTempLoaderCustomizations();\n\n try {\n $response = $this->stopOnErrorResponse ? $loader->loadOrFail($request) : $loader->load($request);\n } finally {\n $this->resetTempLoaderCustomizations($loaderResetConfig);\n }\n\n if ($response !== null && ($response->response->getStatusCode() < 400 || $this->yieldErrorResponses)) {\n return $response;\n }\n\n return null;\n }\n\n /**\n * @return array\n * @throws Exception\n */\n private function applyTempLoaderCustomizations(): array\n {\n $loader = $this->getLoader();\n\n $resetConfig = ['resetToHttpClient' => false, 'resetToBrowser' => false];\n\n if ($this->skipCache) {\n $loader->skipCacheForNextRequest();\n }\n\n if ($this->method !== 'GET' && ($this->forceBrowserUsage || $loader->usesHeadlessBrowser())) {\n $this->logger?->warning(\n 'The (headless) browser can only be used for GET requests! Therefore this step will use the HTTP ' .\n 'client for loading.',\n );\n\n if ($loader->usesHeadlessBrowser()) {\n $loader->useHttpClient();\n\n $resetConfig['resetToBrowser'] = true;\n }\n } elseif ($this->forceBrowserUsage && !$loader->usesHeadlessBrowser()) {\n $resetConfig['resetToHttpClient'] = true;\n\n $loader->useHeadlessBrowser();\n }\n\n if (!empty($this->postBrowserNavigateHooks) && $loader->usesHeadlessBrowser()) {\n $loader->browser()->setTempPostNavigateHooks($this->postBrowserNavigateHooks);\n }\n\n return $resetConfig;\n }\n\n /**\n * @param array $resetConfig\n */\n private function resetTempLoaderCustomizations(array $resetConfig): void\n {\n $loader = $this->getLoader();\n\n if ($resetConfig['resetToHttpClient'] === true) {\n try {\n $loader->useHttpClient();\n } catch (Throwable) {\n }\n } elseif ($resetConfig['resetToBrowser']) {\n $loader->useHeadlessBrowser();\n }\n }\n\n /**\n * @return mixed\n */\n protected function getUrlFromArrayInput(mixed $input): mixed\n {\n if ($this->useAsUrl) {\n if (!is_array($input)) {\n $this->logger?->warning('Input is not array, therefore can\\'t get URL from input by key.');\n } elseif (array_key_exists($this->useAsUrl, $input)) {\n return [$input[$this->useAsUrl]];\n } else {\n $this->logger?->warning(\n 'Input key ' . $this->useAsUrl . ' that should be used as request URL isn\\'t present in input.',\n );\n }\n } elseif (is_array($input) && array_key_exists('url', $input)) {\n return $input['url'];\n } elseif (is_array($input) && array_key_exists('uri', $input)) {\n return $input['uri'];\n }\n\n return $input;\n }\n\n protected function getBodyFromArrayInput(mixed $input): void\n {\n if ($this->useAsBody) {\n if (!is_array($input)) {\n $this->logger?->warning('Input is not array, therefore can\\'t get body from input by key.');\n } elseif (array_key_exists($this->useAsBody, $input)) {\n $this->inputBody = $input[$this->useAsBody];\n } else {\n $this->logger?->warning(\n 'Input key ' . $this->useAsBody . ' that should be used as request body isn\\'t present in input.',\n );\n }\n }\n }\n\n protected function getHeadersFromArrayInput(mixed $input): void\n {\n if ($this->useAsHeaders) {\n if (!is_array($input)) {\n $this->logger?->warning('Input is not array, therefore can\\'t get headers from input by key.');\n } elseif (array_key_exists($this->useAsHeaders, $input)) {\n $this->inputHeaders = $input[$this->useAsHeaders];\n } else {\n $this->logger?->warning(\n 'Input key ' . $this->useAsHeaders . ' that should be used as request headers isn\\'t present in ' .\n 'input.',\n );\n }\n }\n\n if (is_array($this->useAsHeader)) {\n if (!is_array($input)) {\n $this->logger?->warning('Input is not array, therefore can\\'t get header from input by key.');\n } else {\n foreach ($this->useAsHeader as $inputKey => $headerName) {\n $this->addToInputHeadersFromInput($input, $inputKey, $headerName);\n }\n }\n }\n }\n\n protected function addToInputHeadersFromInput(mixed $input, string $inputKey, string $headerName): void\n {\n if (!is_array($this->inputHeaders)) {\n $this->inputHeaders = [];\n }\n\n if (!array_key_exists($inputKey, $input)) {\n $this->logger?->warning(\n 'Input key ' . $inputKey . ' that should be used as a request header, isn\\'t present in input.',\n );\n\n return;\n }\n\n $inputValue = $input[$inputKey];\n\n if (!array_key_exists($headerName, $this->inputHeaders)) {\n $this->inputHeaders[$headerName] = is_array($inputValue) ? $inputValue : [$inputValue];\n\n return;\n }\n\n $this->inputHeaders = HttpHeaders::addTo(HttpHeaders::normalize($this->inputHeaders), $headerName, $inputValue);\n }\n\n /**\n * @return array\n */\n protected function mergeHeaders(): array\n {\n $headers = HttpHeaders::normalize($this->headers);\n\n if (is_array($this->inputHeaders)) {\n $inputHeaders = HttpHeaders::normalize($this->inputHeaders);\n\n $headers = HttpHeaders::merge($headers, $inputHeaders);\n }\n\n return $headers;\n }\n\n protected function resetInputRequestParams(): void\n {\n $this->inputHeaders = null;\n\n $this->inputBody = null;\n }\n\n private function resolveStaticUrl(): string\n {\n $fullInput = $this->getFullOriginalInput();\n\n $inputValue = $fullInput?->get();\n\n if (!is_array($inputValue)) {\n $inputValue = [];\n }\n\n return TemplateString::resolve($this->staticUrl ?? '', $inputValue);\n }\n\n /**\n * @param StreamInterface|string|null $body\n * @param array $headers\n * @return array{ 0: string|StreamInterface|null, 1: array }\n */\n private function resolveVarsInRequestProperties(StreamInterface|string|null $body, array $headers): array\n {\n $fullInput = $this->getFullOriginalInput();\n\n if (!$fullInput) {\n return [$body, $headers];\n }\n\n $fullInputData = $fullInput->get();\n\n if (!is_array($fullInputData)) {\n return [$body, $headers];\n }\n\n return [\n is_string($body) ? TemplateString::resolve($body, $fullInputData) : $body,\n $this->resolveVarsInHeaders($headers, $fullInputData),\n ];\n }\n\n /**\n * @param array $headers\n * @param mixed[] $fullInputData\n * @return array\n */\n private function resolveVarsInHeaders(array $headers, array $fullInputData): array\n {\n foreach ($headers as $headerName => $headerValues) {\n foreach ($headerValues as $key => $headerValue) {\n $headers[$headerName][$key] = TemplateString::resolve($headerValue, $fullInputData);\n }\n }\n\n return $headers;\n }\n}\n" }, { "path": "src/Steps/Loading/HttpCrawl.php", "content": ">\n */\n protected array $urls = [];\n\n /**\n * @var array\n */\n protected array $loadedUrls = [];\n\n protected int $yieldedResponseCount = 0;\n\n public function __construct(array $headers = [], string $httpVersion = '1.1')\n {\n parent::__construct(headers: $headers, httpVersion: $httpVersion);\n }\n\n public function depth(int $depth): static\n {\n $this->depth = $depth;\n\n return $this;\n }\n\n public function sameHost(): static\n {\n $this->sameHost = true;\n\n $this->sameDomain = false;\n\n return $this;\n }\n\n public function sameDomain(): static\n {\n $this->sameDomain = true;\n\n $this->sameHost = false;\n\n return $this;\n }\n\n public function pathStartsWith(string $startsWith = ''): static\n {\n $this->pathStartsWith = $startsWith;\n\n return $this;\n }\n\n public function pathMatches(string $regexPattern = ''): static\n {\n $this->pathRegex = $regexPattern;\n\n return $this;\n }\n\n public function customFilter(Closure $closure): static\n {\n $this->customClosure = $closure;\n\n return $this;\n }\n\n public function inputIsSitemap(): static\n {\n $this->inputIsSitemap = true;\n\n return $this;\n }\n\n public function loadAllButYieldOnlyMatching(): static\n {\n $this->loadAll = true;\n\n return $this;\n }\n\n public function keepUrlFragment(): static\n {\n $this->keepUrlFragment = true;\n\n return $this;\n }\n\n public function useCanonicalLinks(): static\n {\n $this->useCanonicalLinks = true;\n\n return $this;\n }\n\n protected function validateAndSanitizeInput(mixed $input): mixed\n {\n return $this->validateAndSanitizeToUriInterface($input);\n }\n\n /**\n * @param UriInterface $input\n * @throws Exception\n */\n protected function invoke(mixed $input): Generator\n {\n $this->setHostOrDomain($input);\n\n $response = $this->getResponseFromInputUri($input);\n\n if (!$response) {\n return;\n }\n\n $initialResponseDocument = new Document($response);\n\n $this->setResponseCanonicalUrl($response, $initialResponseDocument);\n\n $this->addLoadedUrlsFromResponse($response);\n\n if (!$this->inputIsSitemap && $this->matchesAllCriteria(Url::parse($input))) {\n $this->yieldedResponseCount++;\n\n yield $response;\n }\n\n $this->urls = $this->getUrlsFromInitialResponse($response, $initialResponseDocument);\n\n $depth = 1;\n\n while (\n !$this->depthIsExceeded($depth) &&\n !empty($this->urls) &&\n (!$this->maxOutputs || $this->yieldedResponseCount < $this->maxOutputs)\n ) {\n yield from $this->loadUrls();\n\n $depth++;\n }\n }\n\n /**\n * @throws Exception\n */\n protected function setHostOrDomain(UriInterface $uri): void\n {\n if ($this->sameHost) {\n $this->host = $uri->getHost();\n } else {\n $domain = Url::parse($uri)->domain();\n\n if (!is_string($domain) || empty($domain)) {\n throw new Exception('No domain in input url');\n }\n\n $this->domain = $domain;\n }\n }\n\n /**\n * @throws Exception\n */\n protected function loadUrls(): Generator\n {\n $newUrls = [];\n\n foreach ($this->urls as $url => $yieldResponse) {\n $uri = Url::parsePsr7($url);\n\n $response = $this->getResponseFromInputUri($uri);\n\n if ($response !== null && !$this->wasAlreadyLoaded($response)) {\n $document = new Document($response, $this->logger);\n\n $this->setResponseCanonicalUrl($response, $document);\n\n $yieldResponse = $this->yieldResponse($document, $yieldResponse['yield']);\n\n $this->addLoadedUrlsFromResponse($response);\n\n $newUrls = array_merge($newUrls, $this->getUrlsFromHtmlDocument($document));\n\n if ($yieldResponse) {\n yield $response;\n\n $this->yieldedResponseCount++;\n\n if ($this->maxOutputs && $this->yieldedResponseCount >= $this->maxOutputs) {\n break;\n }\n }\n }\n }\n\n $this->urls = $newUrls;\n }\n\n /**\n * @return array>\n * @throws Exception\n */\n protected function getUrlsFromInitialResponse(RespondedRequest $respondedRequest, ?Document $document = null): array\n {\n if ($this->inputIsSitemap) {\n return $this->getUrlsFromSitemap($respondedRequest);\n } else {\n $document = $document ?? new Document($respondedRequest);\n\n return $this->getUrlsFromHtmlDocument($document);\n }\n }\n\n /**\n * @return array>\n * @throws Exception\n */\n protected function getUrlsFromSitemap(RespondedRequest $respondedRequest): array\n {\n $document = new XmlDocument(Http::getBodyString($respondedRequest));\n\n if (PhpVersion::isBelow(8, 4)) {\n $document = GetUrlsFromSitemap::fixUrlSetTag($document);\n }\n\n $urls = [];\n\n foreach ($document->querySelectorAll('urlset url loc') as $url) {\n $url = $this->handleUrlFragment(Url::parse($url->text()));\n\n if (!$this->isOnSameHostOrDomain($url)) {\n continue;\n }\n\n $matchesCriteria = $this->matchesCriteriaBesidesHostOrDomain($url);\n\n if (!$matchesCriteria && !$this->loadAll) {\n continue;\n }\n\n $url = $url->toString();\n\n if (!isset($urls[$url]) && !isset($this->urls[$url]) && !isset($this->loadedUrls[$url])) {\n $urls[$url] = ['yield' => $matchesCriteria];\n }\n }\n\n return $urls;\n }\n\n /**\n * @return array>\n * @throws Exception\n */\n protected function getUrlsFromHtmlDocument(Document $document): array\n {\n $this->addCanonicalUrlToLoadedUrls($document);\n\n $urls = [];\n\n foreach ($document->dom()->querySelectorAll('a') as $link) {\n if (GetLink::isSpecialNonHttpLink($link)) {\n continue;\n }\n\n try {\n $url = $this->handleUrlFragment($document->baseUrl()->resolve($link->getAttribute('href') ?? ''));\n } catch (Throwable) {\n $this->logger?->warning('Failed to resolve a link with href: ' . $link->getAttribute('href'));\n\n continue;\n }\n\n if (!$this->isOnSameHostOrDomain($url)) {\n continue;\n }\n\n $matchesCriteria = $this->matchesCriteriaBesidesHostOrDomain($url, $link);\n\n if (!$matchesCriteria && !$this->loadAll) {\n continue;\n }\n\n $url = $url->toString();\n\n if (!isset($urls[$url]) && !isset($this->urls[$url]) && !isset($this->loadedUrls[$url])) {\n $urls[$url] = ['yield' => $matchesCriteria];\n }\n }\n\n return $urls;\n }\n\n protected function addLoadedUrlsFromResponse(RespondedRequest $respondedRequest): void\n {\n $loadedUrls = [$respondedRequest->requestedUri() => true];\n\n foreach ($respondedRequest->redirects() as $redirectUrl) {\n $loadedUrls[$redirectUrl] = true;\n }\n\n foreach ($loadedUrls as $loadedUrl => $true) {\n if (!isset($this->loadedUrls[$loadedUrl])) {\n $this->loadedUrls[$loadedUrl] = true;\n }\n }\n }\n\n /**\n * If the loaded response had a redirect, it can be that it was a redirect to a page that was already loaded before.\n * In that case, don't yield that response again.\n *\n * @param RespondedRequest $respondedRequest\n * @return bool\n */\n protected function wasAlreadyLoaded(RespondedRequest $respondedRequest): bool\n {\n if (\n array_key_exists($respondedRequest->requestedUri(), $this->loadedUrls) ||\n array_key_exists($respondedRequest->effectiveUri(), $this->loadedUrls)\n ) {\n $this->logger?->info('Was already loaded before. Do not process this page again.');\n\n return true;\n }\n\n foreach ($respondedRequest->redirects() as $url) {\n if (array_key_exists($url, $this->loadedUrls)) {\n $this->logger?->info('Was already loaded before. Do not process this page again.');\n\n return true;\n }\n }\n\n return false;\n }\n\n protected function addCanonicalUrlToLoadedUrls(Document $document): void\n {\n if ($this->useCanonicalLinks && !isset($this->loadedUrls[$document->canonicalUrl()])) {\n $this->loadedUrls[$document->canonicalUrl()] = true;\n }\n }\n\n /**\n * Yield response only if the URL matches the defined criteria and if the canonical URL isn't already among the\n * loaded URLs (and of course, the user decided that canonical links shall be used, because this is optional).\n */\n protected function yieldResponse(Document $document, bool $urlMatchesCriteria): bool\n {\n if (!$urlMatchesCriteria) {\n return false;\n }\n\n return !$this->useCanonicalLinks || !array_key_exists($document->canonicalUrl(), $this->loadedUrls);\n }\n\n /**\n * @throws Exception\n */\n protected function setResponseCanonicalUrl(RespondedRequest $respondedRequest, Document $document): void\n {\n if ($this->useCanonicalLinks && $respondedRequest->effectiveUri() !== $document->canonicalUrl()) {\n $this->logger?->info('Canonical link URL of this document is: ' . $document->canonicalUrl());\n\n $respondedRequest->addRedirectUri($document->canonicalUrl());\n }\n }\n\n protected function depthIsExceeded(int $depth): bool\n {\n return $this->depth !== null && $depth > $this->depth;\n }\n\n /**\n * @throws Exception\n */\n protected function matchesAllCriteria(Url $url, ?HtmlElement $linkElement = null): bool\n {\n return $this->isOnSameHostOrDomain($url) && $this->matchesCriteriaBesidesHostOrDomain($url, $linkElement);\n }\n\n /**\n * @throws Exception\n */\n protected function matchesCriteriaBesidesHostOrDomain(Url $url, ?HtmlElement $linkElement = null): bool\n {\n return $this->matchesPathCriteria($url) &&\n $this->matchesCustomCriteria($url, $linkElement);\n }\n\n /**\n * @throws Exception\n */\n protected function isOnSameHostOrDomain(Url $url): bool\n {\n if ($this->sameHost) {\n return $this->host === $url->host();\n } else {\n return $this->domain === $url->domain();\n }\n }\n\n /**\n * @throws Exception\n */\n protected function matchesPathCriteria(Url $url): bool\n {\n if ($this->pathStartsWith === null && $this->pathRegex === null) {\n return true;\n }\n\n $path = $url->path() ?? '';\n\n return ($this->pathStartsWith === null || str_starts_with($path, $this->pathStartsWith)) &&\n ($this->pathRegex === null || preg_match($this->pathRegex, $path) === 1);\n }\n\n protected function matchesCustomCriteria(Url $url, ?HtmlElement $linkElement): bool\n {\n return $this->customClosure === null || $this->customClosure->call($this, $url, $linkElement);\n }\n\n /**\n * @throws Exception\n */\n protected function handleUrlFragment(Url $url): Url\n {\n if (!$this->keepUrlFragment) {\n $url->fragment('');\n }\n\n return $url;\n }\n}\n" }, { "path": "src/Steps/Loading/LoadingStep.php", "content": "loader = $loader;\n\n return $this;\n }\n\n /**\n * @param T $loader\n */\n public function withLoader(LoaderInterface $loader): static\n {\n $this->customLoader = $loader;\n\n return $this;\n }\n\n /**\n * @return T\n */\n protected function getLoader(): LoaderInterface\n {\n return $this->customLoader ?? $this->loader;\n }\n}\n" }, { "path": "src/Steps/Refiners/AbstractRefiner.php", "content": "logger = $logger;\n\n return $this;\n }\n\n protected function logTypeWarning(string $staticRefinerMethod, mixed $value): void\n {\n $this->logger?->warning(\n 'Refiner ' . $staticRefinerMethod . ' can\\'t be applied to value of type ' . gettype($value),\n );\n }\n}\n" }, { "path": "src/Steps/Refiners/DateTime/DateTimeFormat.php", "content": "apply($value, function ($value) {\n if ($this->originFormat) {\n $parsed = DateTime::createFromFormat($this->originFormat, $value);\n } else {\n $parsed = $this->parseFromUnknownFormat($value);\n }\n\n if ($parsed === null) {\n return $value;\n } elseif ($parsed === false) {\n $this->logger?->warning(\n 'Failed parsing date/time \"' . $value . '\", so can\\'t reformat it to requested format.',\n );\n\n return $value;\n }\n\n return $parsed->format($this->targetFormat);\n }, 'DateTimeRefiner::reformat()');\n }\n\n private function parseFromUnknownFormat(string $value): ?DateTime\n {\n $timestamp = strtotime($value);\n\n if ($timestamp === false || $timestamp === 0) {\n $this->logger?->warning(\n 'Failed to automatically (without known format) parse date/time \"' . $value . '\", so can\\'t reformat ' .\n 'it to requested format.',\n );\n\n return null;\n }\n\n return (new DateTime())->setTimestamp($timestamp);\n }\n}\n" }, { "path": "src/Steps/Refiners/DateTimeRefiner.php", "content": "query;\n\n if (trim($selectorString) === '') {\n $this->logger?->warning(\n 'Empty selector in remove HTML refiner. If you want HTML nodes to be removed, please define a ' .\n 'selector for those nodes.',\n );\n }\n\n if (is_string($selector)) {\n $selector = Dom::cssSelector($selector);\n }\n\n $this->selector = $selector;\n }\n\n public function refine(mixed $value): mixed\n {\n return $this->apply($value, function ($value) {\n try {\n $document = new HtmlDocument($value);\n } catch (Throwable $exception) {\n $this->logger?->warning(\n 'Failed parsing output as HTML in refiner to remove nodes from HTML: ' . $exception->getMessage(),\n );\n\n return $value;\n }\n\n if ($this->selector instanceof CssSelector) {\n $document->removeNodesMatchingSelector($this->selector->query);\n } else {\n $document->removeNodesMatchingXPath($this->selector->query);\n }\n\n if (str_contains($value, 'outerHtml();\n }\n\n return $document->querySelector('body')?->innerHtml() ?? $document->outerHtml();\n }, 'HtmlRefiner::remove()');\n }\n}\n" }, { "path": "src/Steps/Refiners/HtmlRefiner.php", "content": "logTypeWarning($staticRefinerMethod, $value);\n\n return $value;\n }\n\n if (is_array($value)) {\n foreach ($value as $key => $element) {\n if (is_string($element)) {\n $value[$key] = $refiner($element);\n }\n }\n } else {\n $value = $refiner($value);\n }\n\n return $value;\n }\n}\n" }, { "path": "src/Steps/Refiners/String/StrAfterFirst.php", "content": "apply($value, function ($value) {\n if ($this->first === '') {\n return $value;\n }\n\n $split = explode($this->first, $value, 2);\n\n $lastPart = end($split);\n\n return trim($lastPart);\n }, 'StringRefiner::afterFirst()');\n }\n}\n" }, { "path": "src/Steps/Refiners/String/StrAfterLast.php", "content": "apply($value, function ($value) {\n if ($this->last === '') {\n return '';\n }\n\n $split = explode($this->last, $value);\n\n $lastPart = end($split);\n\n return trim($lastPart);\n }, 'StringRefiner::afterLast()');\n }\n}\n" }, { "path": "src/Steps/Refiners/String/StrBeforeFirst.php", "content": "apply($value, function ($value) {\n if ($this->first === '') {\n return '';\n }\n\n return trim(explode($this->first, $value)[0]);\n }, 'StringRefiner::beforeFirst()');\n }\n}\n" }, { "path": "src/Steps/Refiners/String/StrBeforeLast.php", "content": "apply($value, function ($value) {\n if ($this->last === '') {\n return $value;\n }\n\n $split = explode($this->last, $value);\n\n if (count($split) === 1) {\n return $value;\n }\n\n array_pop($split);\n\n return trim(implode($this->last, $split));\n }, 'StringRefiner::beforeLast()');\n }\n}\n" }, { "path": "src/Steps/Refiners/String/StrBetweenFirst.php", "content": "apply($value, function ($value) {\n if ($this->start === '') {\n $splitAtStart = ['', $value];\n } else {\n $splitAtStart = explode($this->start, $value, 2);\n }\n\n if (count($splitAtStart) === 2) {\n if ($this->end === '') {\n return trim($splitAtStart[1]);\n }\n\n return trim(explode($this->end, $splitAtStart[1])[0]);\n }\n\n return '';\n }, 'StringRefiner::betweenFirst()');\n }\n}\n" }, { "path": "src/Steps/Refiners/String/StrBetweenLast.php", "content": "apply($value, function ($value) {\n if ($this->start === '') {\n $splitAtStart = ['', $value];\n } else {\n $splitAtStart = explode($this->start, $value);\n }\n\n $lastPart = end($splitAtStart);\n\n if ($this->end === '') {\n return trim($lastPart);\n }\n\n return trim(explode($this->end, $lastPart)[0]);\n }, 'StringRefiner::betweenLast()');\n }\n}\n" }, { "path": "src/Steps/Refiners/String/StrReplace.php", "content": "apply($value, function ($value) {\n $replaced = str_replace($this->search, $this->replace, $value);\n\n return trim($replaced);\n }, 'StringRefiner::replace()');\n\n // if (!is_string($value)) {\n // $this->logTypeWarning('StringRefiner::replace()', $value);\n //\n // return $value;\n // }\n //\n // $replaced = str_replace($this->search, $this->replace, $value);\n //\n // return trim($replaced);\n }\n}\n" }, { "path": "src/Steps/Refiners/StringRefiner.php", "content": " $url) {\n $value[$key] = $this->refine($url);\n }\n\n return $value;\n }\n\n if (!is_string($value) && !$value instanceof Url && !$value instanceof UriInterface) {\n $this->logTypeWarning($this->staticRefinerMethod(), $value);\n\n return $value;\n }\n\n if (!$value instanceof Url) {\n $value = Url::parse($value);\n }\n\n return $this->refineUrl($value);\n }\n\n abstract protected function staticRefinerMethod(): string;\n\n abstract protected function refineUrl(Url $url): string;\n}\n" }, { "path": "src/Steps/Refiners/Url/WithFragment.php", "content": "fragment($this->fragment);\n\n return (string) $url;\n }\n}\n" }, { "path": "src/Steps/Refiners/Url/WithHost.php", "content": "host($this->host);\n\n return (string) $url;\n }\n}\n" }, { "path": "src/Steps/Refiners/Url/WithPath.php", "content": "path($this->path);\n\n return (string) $url;\n }\n}\n" }, { "path": "src/Steps/Refiners/Url/WithPort.php", "content": "port($this->port);\n\n return (string) $url;\n }\n}\n" }, { "path": "src/Steps/Refiners/Url/WithQuery.php", "content": "query($this->query);\n\n return (string) $url;\n }\n}\n" }, { "path": "src/Steps/Refiners/Url/WithScheme.php", "content": "scheme($this->scheme);\n\n return (string) $url;\n }\n}\n" }, { "path": "src/Steps/Refiners/Url/WithoutPort.php", "content": "resetPort();\n\n return (string) $url;\n }\n}\n" }, { "path": "src/Steps/Refiners/UrlRefiner.php", "content": " tag\n *\n * Symfony's DomCrawler component has problems when a sitemap's tag contains certain attributes.\n * So, if the count of urls in the sitemap is zero, try to remove all attributes from the tag.\n */\n public static function fixUrlSetTag(XmlDocument $dom): XmlDocument\n {\n if ($dom->querySelectorAll('urlset url')->count() === 0) {\n return new XmlDocument(preg_replace('//', '', $dom->outerXml()) ?? $dom->outerXml());\n }\n\n return $dom;\n }\n\n public function withData(): static\n {\n $this->withData = true;\n\n return $this;\n }\n\n public function outputType(): StepOutputType\n {\n return $this->withData ? StepOutputType::AssociativeArrayOrObject : StepOutputType::Scalar;\n }\n\n /**\n * @param XmlDocument $input\n */\n protected function invoke(mixed $input): Generator\n {\n if (PhpVersion::isBelow(8, 4)) {\n $input = self::fixUrlSetTag($input);\n }\n\n foreach ($input->querySelectorAll('urlset url') as $urlNode) {\n if ($urlNode->querySelector('loc')) {\n if ($this->withData) {\n yield $this->getWithAdditionalData($urlNode);\n } else {\n yield $urlNode->querySelector('loc')->text();\n }\n }\n }\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function validateAndSanitizeInput(mixed $input): mixed\n {\n return $this->validateAndSanitizeToXmlDocumentInstance($input);\n }\n\n /**\n * @return string[]\n */\n protected function getWithAdditionalData(XmlElement $urlNode): array\n {\n $data = ['url' => $urlNode->querySelector('loc')?->text() ?? ''];\n\n $properties = ['lastmod', 'changefreq', 'priority'];\n\n foreach ($properties as $property) {\n $node = $urlNode->querySelector($property);\n\n if ($node) {\n $data[$property] = $node->text();\n }\n }\n\n return $data;\n }\n}\n" }, { "path": "src/Steps/Sitemap.php", "content": "\n */\n abstract protected function invoke(mixed $input): Generator;\n\n /**\n * Calls the validateAndSanitizeInput method and assures that the invoke method receives valid, sanitized input.\n *\n * @return Generator\n * @throws Exception\n */\n final public function invokeStep(Input $input): Generator\n {\n if ($this->maxOutputsExceeded()) {\n return;\n }\n\n $this->storeOriginalInput($input);\n\n $inputForStepInvocation = $this->getInputKeyToUse($input);\n\n if ($inputForStepInvocation) {\n try {\n $validInputValue = $this->validateAndSanitizeInput($inputForStepInvocation->get());\n } catch (InvalidArgumentException $exception) {\n $this->logInvalidInputException($exception, $inputForStepInvocation->get());\n\n return;\n }\n\n if ($this->uniqueInput === false || $this->inputOrOutputIsUnique(new Input($validInputValue))) {\n if (!$this->groupOutputsPerInput) {\n yield from $this->invokeAndYield($validInputValue, $input);\n } else {\n yield from $this->invokeAndYieldOneOutputPerInput($validInputValue, $input);\n }\n }\n }\n }\n\n /**\n * Callback that is called in a step group to adapt the input for further steps\n *\n * In groups all the steps are called with the same Input, but with this callback it's possible to adjust the input\n * for the following steps.\n */\n public function updateInputUsingOutput(Closure $closure): static\n {\n $this->updateInputUsingOutput = $closure;\n\n return $this;\n }\n\n public function excludeFromGroupOutput(): static\n {\n $this->excludeFromGroupOutput = true;\n\n return $this;\n }\n\n public function oneOutputPerInput(): static\n {\n $this->groupOutputsPerInput = true;\n\n return $this;\n }\n\n public function shouldOutputBeExcludedFromGroupOutput(): bool\n {\n return $this->excludeFromGroupOutput;\n }\n\n /**\n * If the user set a callback to update the input (see above) => call it.\n */\n public function callUpdateInputUsingOutput(Input $input, Output $output): Input\n {\n if ($this->updateInputUsingOutput instanceof Closure) {\n return $input->withValue(\n $this->updateInputUsingOutput->call($this, $input->get(), $output->get()),\n );\n }\n\n return $input;\n }\n\n /**\n * Validate and sanitize the incoming Input object\n *\n * In child classes you can add this method to validate and sanitize the incoming input. The method is called\n * automatically when the step is invoked within the Crawler and the invoke method receives the validated and\n * sanitized input. Also, you can just return any value from this method and in the invoke method it's again\n * incoming as an Input object.\n *\n * @throws InvalidArgumentException Throw this if the input value is invalid for this step.\n */\n protected function validateAndSanitizeInput(mixed $input): mixed\n {\n return $input;\n }\n\n /**\n * @throws InvalidArgumentException\n */\n protected function validateAndSanitizeStringOrStringable(\n mixed $inputValue,\n string $exceptionMessage = 'Input must be string or stringable',\n ): string {\n $inputValue = $this->getSingleElementFromArray($inputValue);\n\n if (is_object($inputValue) && method_exists($inputValue, '__toString')) {\n return $this->removeUtf8BomFromString($inputValue->__toString());\n }\n\n if (is_string($inputValue)) {\n return $this->removeUtf8BomFromString($inputValue);\n }\n\n throw new InvalidArgumentException($exceptionMessage);\n }\n\n /**\n * @throws InvalidArgumentException|MissingZlibExtensionException\n */\n protected function validateAndSanitizeStringOrHttpResponse(\n mixed $inputValue,\n string $exceptionMessage = 'Input must be string, stringable or HTTP response (RespondedRequest)',\n bool $allowOnlyRespondedRequest = false,\n ): string {\n if (is_array($inputValue) && count($inputValue) > 1 && array_key_exists('response', $inputValue)) {\n $inputValue = $inputValue['response'];\n }\n\n $inputValue = $this->getSingleElementFromArray($inputValue);\n\n if (\n $inputValue instanceof RespondedRequest ||\n ($inputValue instanceof ResponseInterface && !$allowOnlyRespondedRequest)\n ) {\n return $this->removeUtf8BomFromString(Http::getBodyString($inputValue));\n }\n\n return $this->validateAndSanitizeStringOrStringable($inputValue, $exceptionMessage);\n }\n\n /**\n * @throws InvalidArgumentException\n */\n protected function validateAndSanitizeToUriInterface(\n mixed $inputValue,\n string $exceptionMessage = 'Input must be string, stringable or an instance of UriInterface or Crwlr\\\\Url',\n ): UriInterface {\n $inputValue = $this->getSingleElementFromArray($inputValue);\n\n if ($inputValue instanceof UriInterface) {\n return $inputValue;\n }\n\n if (\n is_string($inputValue) ||\n $inputValue instanceof Url ||\n (is_object($inputValue) && method_exists($inputValue, '__toString'))\n ) {\n try {\n return Url::parsePsr7((string) $inputValue);\n } catch (InvalidUrlException $exception) {\n throw new InvalidArgumentException($exception->getMessage());\n }\n }\n\n throw new InvalidArgumentException($exceptionMessage);\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function validateAndSanitizeToHtmlDocumentInstance(\n mixed $inputValue,\n string $exceptionMessage = 'Input must be string, stringable or HTTP response (RespondedRequest)',\n ): HtmlDocument {\n return new HtmlDocument($this->validateAndSanitizeStringOrHttpResponse($inputValue, $exceptionMessage));\n }\n\n /**\n * @throws MissingZlibExtensionException\n */\n protected function validateAndSanitizeToXmlDocumentInstance(\n mixed $inputValue,\n string $exceptionMessage = 'Input must be string, stringable or HTTP response (RespondedRequest)',\n ): XmlDocument {\n return new XmlDocument($this->validateAndSanitizeStringOrHttpResponse($inputValue, $exceptionMessage));\n }\n\n protected function getSingleElementFromArray(mixed $inputValue): mixed\n {\n if (is_array($inputValue) && count($inputValue) === 1) {\n return reset($inputValue);\n }\n\n return $inputValue;\n }\n\n /**\n * @throws Exception\n */\n private function invokeAndYield(mixed $validInputValue, Input $input): Generator\n {\n foreach ($this->invoke($validInputValue) as $outputData) {\n $outputData = $this->applyRefiners($outputData, $input->get());\n\n if ($this->maxOutputsExceeded()) {\n break;\n } elseif (!$this->passesAllFilters($outputData)) {\n continue;\n }\n\n if (!is_array($outputData) && $this->outputKey) {\n $outputData = [$this->outputKey => $outputData];\n }\n\n $output = $this->makeOutput($outputData, $input);\n\n if ($this->uniqueOutput && !$this->inputOrOutputIsUnique($output)) {\n continue;\n }\n\n yield $output;\n\n $this->trackYieldedOutput();\n }\n }\n\n /**\n * Version of invokeAndYield() when oneOutputPerInput() was called.\n */\n private function invokeAndYieldOneOutputPerInput(mixed $validInputValue, Input $input): Generator\n {\n $outputDataArray = [];\n\n foreach ($this->invoke($validInputValue) as $outputData) {\n $outputData = $this->applyRefiners($outputData, $input->get());\n\n if (!$this->passesAllFilters($outputData)) {\n continue;\n }\n\n $outputDataArray[] = $outputData;\n }\n\n if ($this->outputKey) {\n $outputDataArray = [$this->outputKey => $outputDataArray];\n }\n\n $output = $this->makeOutput($outputDataArray, $input);\n\n if ($this->uniqueOutput && !$this->inputOrOutputIsUnique($output)) {\n return;\n }\n\n yield $output;\n\n $this->trackYieldedOutput();\n }\n\n /**\n * Sometimes there can be a so-called byte order mark character as first characters in a text file. See:\n * https://stackoverflow.com/questions/53303571/why-does-the-filereader-stream-read-239-187-191-from-a-textfile\n * 239, 187, 191 is the BOM for UTF-8. Remove it, as it is unnecessary and can cause issues when a string\n * needs to start with a certain character.\n *\n * @param string $string\n * @return string\n */\n private function removeUtf8BomFromString(string $string): string\n {\n if (substr($string, 0, 3) === (chr(239) . chr(187) . chr(191))) {\n return substr($string, 3);\n }\n\n return $string;\n }\n\n private function logInvalidInputException(InvalidArgumentException $exception, mixed $input): void\n {\n $exceptionMessage = $exception->getMessage();\n\n $stepClassName = $this->getStepClassName();\n\n $logMessage = ($stepClassName ? 'The ' . $stepClassName . ' step' : 'A step') . ' was called with input ' .\n 'that it can not work with: ' . $exceptionMessage;\n\n if (str_starts_with($exceptionMessage, 'Input must be string')) {\n $logMessage .= '. The invalid input is of type ' . gettype($input) . '.';\n }\n\n $this->logger?->error($logMessage);\n }\n}\n" }, { "path": "src/Steps/StepInterface.php", "content": "\n */\n public function invokeStep(Input $input): Generator;\n\n /**\n * @param string|string[]|null $keys\n */\n public function keep(string|array|null $keys = null): static;\n\n public function keepAs(string $key): static;\n\n /**\n * @param string|string[]|null $keys\n */\n public function keepFromInput(string|array|null $keys = null): static;\n\n public function keepInputAs(string $key): static;\n\n public function keepsAnything(): bool;\n\n public function keepsAnythingFromInputData(): bool;\n\n public function keepsAnythingFromOutputData(): bool;\n\n public function useInputKey(string $key): static;\n\n public function uniqueInputs(?string $key = null): static;\n\n public function uniqueOutputs(?string $key = null): static;\n\n public function where(string|FilterInterface $keyOrFilter, ?FilterInterface $filter = null): static;\n\n public function orWhere(string|FilterInterface $keyOrFilter, ?FilterInterface $filter = null): static;\n\n public function outputKey(string $key): static;\n\n public function maxOutputs(int $maxOutputs): static;\n\n public function resetAfterRun(): void;\n}\n" }, { "path": "src/Steps/StepOutputType.php", "content": "baseUrl = $input->effectiveUri();\n }\n\n return $this->validateAndSanitizeToXmlDocumentInstance($input);\n }\n}\n" }, { "path": "src/Stores/JsonFileStore.php", "content": "createTimestamp = time();\n\n touch($this->filePath());\n\n file_put_contents($this->filePath(), '[]');\n }\n\n /**\n * @throws Exception\n */\n public function store(Result $result): void\n {\n $currentResultsFileContent = file_get_contents($this->filePath());\n\n if (!$currentResultsFileContent) {\n $currentResultsFileContent = '[]';\n }\n\n $results = json_decode($currentResultsFileContent, true);\n\n $results[] = $result->toArray();\n\n file_put_contents($this->filePath(), json_encode($results));\n }\n\n public function filePath(): string\n {\n return $this->storePath . '/' .\n ($this->filePrefix ? $this->filePrefix . '-' : '') . $this->createTimestamp . '.json';\n }\n}\n" }, { "path": "src/Stores/SimpleCsvFileStore.php", "content": "createTimestamp = time();\n\n touch($this->filePath());\n }\n\n /**\n * @throws Exception\n */\n public function store(Result $result): void\n {\n $fileHandle = fopen($this->filePath(), 'a');\n\n if (!is_resource($fileHandle)) {\n throw new Exception('Failed to open file to store data');\n }\n\n if ($this->isFirstResult) {\n fputcsv($fileHandle, array_keys($result->toArray()), escape: '');\n\n $this->isFirstResult = false;\n }\n\n $resultArray = $result->toArray();\n\n if ($this->anyPropertyIsArray($result)) {\n $resultArray = $this->flattenResultArray($resultArray);\n }\n\n fputcsv($fileHandle, array_values($resultArray), escape: '');\n\n fclose($fileHandle);\n }\n\n public function filePath(): string\n {\n return $this->storePath . '/' .\n ($this->filePrefix ? $this->filePrefix . '-' : '') . $this->createTimestamp . '.csv';\n }\n\n protected function anyPropertyIsArray(Result $result): bool\n {\n foreach ($result->toArray() as $value) {\n if (is_array($value)) {\n return true;\n }\n }\n\n return false;\n }\n\n /**\n * @param mixed[] $result\n * @return array\n */\n protected function flattenResultArray(array $result): array\n {\n foreach ($result as $key => $value) {\n if (is_array($value)) {\n $result[$key] = implode(' | ', $value);\n }\n }\n\n return $result;\n }\n}\n" }, { "path": "src/Stores/Store.php", "content": "logger = $logger;\n\n return $this;\n }\n}\n" }, { "path": "src/Stores/StoreInterface.php", "content": "productToken;\n\n if ($this->version) {\n $botUserAgent .= '/' . $this->version;\n }\n\n if ($this->infoUri) {\n $botUserAgent .= '; +' . $this->infoUri;\n }\n\n return $botUserAgent . ')';\n }\n\n public function productToken(): string\n {\n return $this->productToken;\n }\n}\n" }, { "path": "src/UserAgents/BotUserAgentInterface.php", "content": "userAgent;\n }\n\n public static function mozilla5CompatibleBrowser(): self\n {\n return new self('Mozilla/5.0 (compatible)');\n }\n}\n" }, { "path": "src/UserAgents/UserAgentInterface.php", "content": " $headers\n * @return array\n */\n public static function normalize(array $headers): array\n {\n $normalized = [];\n\n foreach ($headers as $headerName => $value) {\n $normalized[$headerName] = is_array($value) ? $value : [$value];\n }\n\n return $normalized;\n }\n\n /**\n * @param array> $headers\n * @param array> $mergeHeaders\n * @return array>\n */\n public static function merge(array $headers, array $mergeHeaders): array\n {\n foreach ($mergeHeaders as $headerName => $value) {\n if (!array_key_exists($headerName, $headers)) {\n $headers[$headerName] = $value;\n } else {\n $headers = self::addTo($headers, $headerName, $value);\n }\n }\n\n return $headers;\n }\n\n /**\n * @param array> $headers\n * @param string $headerName\n * @param string|string[] $value\n * @return array>\n */\n public static function addTo(array $headers, string $headerName, string|array $value): array\n {\n if (!array_key_exists($headerName, $headers)) {\n $headers[$headerName] = is_array($value) ? $value : [$value];\n } elseif (is_array($value)) {\n foreach ($value as $valueItem) {\n if (!in_array($valueItem, $headers[$headerName], true)) {\n $headers[$headerName][] = $valueItem;\n }\n }\n } elseif (!in_array($value, $headers[$headerName], true)) {\n $headers[$headerName][] = $value;\n }\n\n return $headers;\n }\n}\n" }, { "path": "src/Utils/OutputTypeHelper.php", "content": "toArrayForResult();\n } elseif (method_exists($output, 'toArray')) {\n return $output->toArray();\n } elseif (method_exists($output, '__serialize')) {\n return $output->__serialize();\n }\n\n return (array) $output;\n }\n\n public static function isScalar(mixed $output): bool\n {\n return !self::isAssociativeArrayOrObject($output);\n }\n\n public static function isAssociativeArrayOrObject(mixed $output): bool\n {\n return self::isAssociativeArray($output) || is_object($output);\n }\n\n public static function isAssociativeArray(mixed $output): bool\n {\n if (!is_array($output)) {\n return false;\n }\n\n foreach ($output as $key => $value) {\n return is_string($key);\n }\n\n return false;\n }\n\n /**\n * @param mixed[] $data\n * @return mixed[]\n */\n public static function recursiveChildObjectsToArray(array $data): array\n {\n foreach ($data as $key => $value) {\n if (is_object($value)) {\n $data[$key] = self::recursiveChildObjectsToArray(self::objectToArray($value));\n } elseif (is_array($value)) {\n $data[$key] = self::recursiveChildObjectsToArray($value);\n }\n }\n\n return $data;\n }\n}\n" }, { "path": "src/Utils/RequestKey.php", "content": "request : $request;\n\n $data = [\n 'requestMethod' => $request->getMethod(),\n 'requestUri' => $request->getUri()->__toString(),\n 'requestHeaders' => $request->getHeaders(),\n 'requestBody' => Http::getBodyString($request),\n ];\n\n $data = self::removeIgnoreHeaders($data, $ignoreHeaders);\n\n $serialized = serialize($data);\n\n return md5($serialized);\n }\n\n /**\n * @param array $data\n * @param string[] $ignoreHeaders\n * @return array\n */\n private static function removeIgnoreHeaders(array $data, array $ignoreHeaders): array\n {\n foreach ($ignoreHeaders as $ignoreHeader) {\n if (isset($data['requestHeaders'][$ignoreHeader])) {\n unset($data['requestHeaders'][$ignoreHeader]);\n }\n\n $otherCase = strtolower($ignoreHeader);\n\n if ($otherCase === $ignoreHeader) {\n $otherCase = ucwords($ignoreHeader, '-');\n }\n\n $ignoreHeader = $otherCase;\n\n if (isset($data['requestHeaders'][$ignoreHeader])) {\n unset($data['requestHeaders'][$ignoreHeader]);\n }\n }\n\n return $data;\n }\n}\n" }, { "path": "src/Utils/TemplateString.php", "content": "get($varName);\n }\n\n return '';\n }, $string) ?? $string;\n }\n\n return $string;\n }\n\n private static function trimAndUnescapeQuotes(string $string): string\n {\n if (\n str_starts_with($string, '\\'') && str_ends_with($string, '\\'') ||\n str_starts_with($string, '\"') && str_ends_with($string, '\"')\n ) {\n $string = substr($string, 1, -1);\n }\n\n $string = str_replace([\"\\'\", '\\\"'], [\"'\", '\"'], $string);\n\n return $string;\n }\n}\n" }, { "path": "tests/Cache/CacheItemTest.php", "content": "value())->toBe('value');\n\n expect($unserialized->key())->toBe('key123');\n\n expect($unserialized->ttl)->toBe(123);\n\n expect($unserialized->createdAt->format('Y-m-d H:i:s'))->toBe('2023-01-10 12:10:00');\n});\n\nit('creates a key based on the value if you don\\'t provide a key manually', function () {\n $item = new CacheItem('foo');\n\n expect($item->key())->toBeString();\n\n expect(strlen($item->key()))->toBeGreaterThan(0);\n});\n\nit('tells if it is expired already', function () {\n $item = new CacheItem('v', 'k', 10);\n\n expect($item->isExpired())->toBeFalse();\n\n $item = new CacheItem('v', 'k', 10, (new DateTimeImmutable())->sub(new DateInterval('PT9S')));\n\n expect($item->isExpired())->toBeFalse();\n\n $item = new CacheItem('v', 'k', 10, (new DateTimeImmutable())->sub(new DateInterval('PT11S')));\n\n expect($item->isExpired())->toBeTrue();\n});\n" }, { "path": "tests/Cache/FileCacheTest.php", "content": "set($item->cacheKey(), $item);\n }\n}\n\nfunction helper_respondedRequestWithRequestUrl(string $requestUrl): RespondedRequest\n{\n return new RespondedRequest(new Request('GET', $requestUrl), new Response());\n}\n\n/**\n * Helper function to get the CacheItem instance, because FileCache::get() returns only\n * the value wrapped in the CacheItem object.\n */\nfunction helper_getCacheItemByKey(string $key): ?CacheItem\n{\n $cacheFileContent = file_get_contents(helper_cachedir() . '/' . $key);\n\n $cacheItem = unserialize($cacheFileContent !== false ? $cacheFileContent : 'a:0:{}');\n\n return $cacheItem instanceof CacheItem ? $cacheItem : null;\n}\n\nafterEach(function () {\n helper_resetCacheDir();\n});\n\n/** @var TestCase $this */\n\nit('caches a simple value', function () {\n $cache = new FileCache(helper_cachedir());\n\n $cache->set('user', 'otsch');\n\n expect($cache->get('user'))->toBe('otsch');\n});\n\nit('caches RespondedRequest objects', function () {\n $respondedRequest = new RespondedRequest(new Request('GET', '/'), new Response());\n\n $cache = new FileCache(helper_cachedir());\n\n expect($cache->set($respondedRequest->cacheKey(), $respondedRequest))->toBeTrue()\n ->and(file_exists(helper_cachedir() . '/' . $respondedRequest->cacheKey()))->toBeTrue()\n ->and($cache->get($respondedRequest->cacheKey()))->toBeInstanceOf(RespondedRequest::class);\n});\n\nit('checks if it has an item for a certain key', function () {\n $respondedRequest = new RespondedRequest(new Request('GET', '/'), new Response());\n\n $cache = new FileCache(helper_cachedir());\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n expect($cache->has($respondedRequest->cacheKey()))->toBeTrue()\n ->and($cache->has('otherKey'))->toBeFalse();\n});\n\nit('does not return expired items', function () {\n $respondedRequest = new RespondedRequest(new Request('GET', '/'), new Response());\n\n $cacheItem = new CacheItem(\n $respondedRequest,\n $respondedRequest->cacheKey(),\n 10,\n (new DateTimeImmutable())->sub(new DateInterval('PT11S')),\n );\n\n $cache = new FileCache(helper_cachedir());\n\n $cache->set($cacheItem->key(), $cacheItem);\n\n expect($cache->has($cacheItem->key()))->toBeFalse()\n ->and($cache->get($cacheItem->key()))->toBeNull();\n});\n\nit('deletes a cache item', function () {\n $respondedRequest = new RespondedRequest(new Request('GET', '/'), new Response());\n\n $cache = new FileCache(helper_cachedir());\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n expect($cache->has($respondedRequest->cacheKey()))->toBeTrue();\n\n $cache->delete($respondedRequest->cacheKey());\n\n expect($cache->has($respondedRequest->cacheKey()))->toBeFalse();\n});\n\nit('deletes an expired cache item when has() is called with its key', function () {\n $cacheItem = new CacheItem('bar', 'foo', 10, (new DateTimeImmutable())->sub(new DateInterval('PT11S')));\n\n $cache = new FileCache(helper_cachedir());\n\n $cache->set('foo', $cacheItem);\n\n expect(file_exists(helper_cachedir() . '/foo'))->toBeTrue()\n ->and($cache->has('foo'))->toBeFalse()\n ->and(file_exists(helper_cachedir() . '/foo'))->toBeFalse();\n});\n\nit('deletes an expired cache item when get() is called with its key', function () {\n $cacheItem = new CacheItem('bar', 'foo', 10, (new DateTimeImmutable())->sub(new DateInterval('PT11S')));\n\n $cache = new FileCache(helper_cachedir());\n\n $cache->set('foo', $cacheItem);\n\n expect(file_exists(helper_cachedir() . '/foo'))->toBeTrue()\n ->and($cache->get('foo', 'defaultValue'))->toBe('defaultValue')\n ->and(file_exists(helper_cachedir() . '/foo'))->toBeFalse();\n});\n\nit('clears the whole cache', function () {\n $cacheItem1 = helper_respondedRequestWithRequestUrl('/foo');\n\n $cacheItem2 = helper_respondedRequestWithRequestUrl('/bar');\n\n $cacheItem3 = helper_respondedRequestWithRequestUrl('/baz');\n\n $cache = new FileCache(helper_cachedir());\n\n helper_addMultipleItemsToCache([$cacheItem1, $cacheItem2, $cacheItem3], $cache);\n\n expect($cache->has($cacheItem1->cacheKey()))->toBeTrue()\n ->and($cache->has($cacheItem2->cacheKey()))->toBeTrue()\n ->and($cache->has($cacheItem3->cacheKey()))->toBeTrue();\n\n $cache->clear();\n\n expect($cache->has($cacheItem1->cacheKey()))->toBeFalse()\n ->and($cache->has($cacheItem2->cacheKey()))->toBeFalse()\n ->and($cache->has($cacheItem3->cacheKey()))->toBeFalse();\n});\n\nit('gets multiple items', function () {\n $cacheItem1 = helper_respondedRequestWithRequestUrl('/foo');\n\n $cacheItem2 = helper_respondedRequestWithRequestUrl('/bar');\n\n $cacheItem3 = helper_respondedRequestWithRequestUrl('/baz');\n\n $cache = new FileCache(helper_cachedir());\n\n helper_addMultipleItemsToCache([$cacheItem1, $cacheItem2, $cacheItem3], $cache);\n\n $items = $cache->getMultiple([$cacheItem1->cacheKey(), $cacheItem2->cacheKey(), $cacheItem3->cacheKey()]);\n\n expect(reset($items)->request->getUri()->__toString())->toBe('/foo')\n ->and(next($items)->request->getUri()->__toString())->toBe('/bar')\n ->and(next($items)->request->getUri()->__toString())->toBe('/baz');\n});\n\nit('sets multiple items', function () {\n $cacheItem1 = helper_respondedRequestWithRequestUrl('/foo');\n\n $cacheItem2 = helper_respondedRequestWithRequestUrl('/bar');\n\n $cacheItem3 = helper_respondedRequestWithRequestUrl('/baz');\n\n $cache = new FileCache(helper_cachedir());\n\n $cache->setMultiple([\n $cacheItem1->cacheKey() => $cacheItem1,\n $cacheItem2->cacheKey() => $cacheItem2,\n $cacheItem3->cacheKey() => $cacheItem3,\n ]);\n\n expect($cache->has($cacheItem1->cacheKey()))->toBeTrue()\n ->and($cache->has($cacheItem2->cacheKey()))->toBeTrue()\n ->and($cache->has($cacheItem3->cacheKey()))->toBeTrue();\n});\n\nit('deletes multiple items', function () {\n $cacheItem1 = helper_respondedRequestWithRequestUrl('/blog');\n\n $cacheItem2 = helper_respondedRequestWithRequestUrl('/contact');\n\n $cacheItem3 = helper_respondedRequestWithRequestUrl('/privacy');\n\n $cache = new FileCache(helper_cachedir());\n\n helper_addMultipleItemsToCache([$cacheItem1, $cacheItem2, $cacheItem3], $cache);\n\n $cache->deleteMultiple([$cacheItem1->cacheKey(), $cacheItem2->cacheKey(), $cacheItem3->cacheKey()]);\n\n expect($cache->has($cacheItem1->cacheKey()))->toBeFalse()\n ->and($cache->has($cacheItem2->cacheKey()))->toBeFalse()\n ->and($cache->has($cacheItem3->cacheKey()))->toBeFalse();\n});\n\nit('can still use legacy (pre CacheItem object) cache files', function () {\n $content = file_get_contents(__DIR__ . '/_cachefilecontent');\n\n file_put_contents(helper_cachedir() . '/foo', $content);\n\n $cache = new FileCache(helper_cachedir());\n\n expect($cache->has('foo'))->toBeTrue();\n\n $cacheItem = $cache->get('foo');\n\n expect($cacheItem)->toBeArray();\n\n $respondedRequest = RespondedRequest::fromArray($cacheItem);\n\n expect($respondedRequest)->toBeInstanceOf(RespondedRequest::class)\n ->and($respondedRequest->requestedUri())->toBe(\n 'https://www.crwlr.software/blog/dealing-with-http-url-query-strings-in-php',\n );\n});\n\nit('compresses cache data when useCompression() is used', function () {\n $data = <<set($respondedRequest->cacheKey(), $respondedRequest);\n\n $uncompressedFileSize = filesize(helper_cachedir() . '/' . $respondedRequest->cacheKey());\n\n expect($uncompressedFileSize)->not()->toBeFalse();\n\n if ($uncompressedFileSize === false) {\n throw new RuntimeException('Unable to determine cache file size.');\n }\n\n clearstatcache(); // Results of filesize() are cached. Clear that to get correct result for compressed file size.\n\n $cache->useCompression();\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $compressedFileSize = filesize(helper_cachedir() . '/' . $respondedRequest->cacheKey());\n\n expect($compressedFileSize)->not()->toBeFalse();\n\n if ($compressedFileSize === false) {\n throw new RuntimeException('Unable to determine cache file size.');\n }\n\n expect($compressedFileSize)->toBeLessThan($uncompressedFileSize)\n // Didn't want to check for exact numbers, because I guess they could be a bit different on different systems.\n // But thought the diff should at least be more than 30% for the test to succeed.\n ->and($uncompressedFileSize - $compressedFileSize)->toBeGreaterThan($uncompressedFileSize * 0.3);\n});\n\nit('gets compressed cache items', function () {\n $cache = new FileCache(helper_cachedir());\n\n $cache->useCompression();\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', '/compression'),\n new Response(body: Utils::streamFor('Hello World')),\n );\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $retrievedCacheItem = $cache->get($respondedRequest->cacheKey());\n\n expect($retrievedCacheItem)->toBeInstanceOf(RespondedRequest::class)\n ->and(Http::getBodyString($retrievedCacheItem))->toBe('Hello World');\n});\n\nit('is also able to decode uncompressed cache files when useCompression() is used', function () {\n $cache = new FileCache(helper_cachedir());\n\n $respondedRequest = new RespondedRequest(new Request('GET', '/yo'), new Response(body: Utils::streamFor('Yo')));\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $retrievedCacheItem = $cache->get($respondedRequest->cacheKey());\n\n expect($retrievedCacheItem)\n ->toBeInstanceOf(RespondedRequest::class)\n ->and(Http::getBodyString($retrievedCacheItem))\n ->toBe('Yo');\n\n $cache->useCompression();\n\n $retrievedCacheItem = $cache->get($respondedRequest->cacheKey());\n\n expect($retrievedCacheItem)\n ->toBeInstanceOf(RespondedRequest::class)\n ->and(Http::getBodyString($retrievedCacheItem))\n ->toBe('Yo');\n});\n\nit('can also read compressed cache files, when useCompression() is not used', function () {\n $cache = new FileCache(helper_cachedir());\n\n $cache->useCompression();\n\n $respondedRequest = new RespondedRequest(new Request('GET', '/no'), new Response(body: Utils::streamFor('No')));\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $cache = new FileCache(helper_cachedir());\n\n $retrievedCacheItem = $cache->get($respondedRequest->cacheKey());\n\n expect($retrievedCacheItem)\n ->toBeInstanceOf(RespondedRequest::class)\n ->and(Http::getBodyString($retrievedCacheItem))\n ->toBe('No');\n});\n\ntest('you can change the default ttl', function () {\n $cache = new FileCache(helper_cachedir());\n\n $cache->ttl(900);\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', '/foo'),\n new Response(body: Utils::streamFor('bar')),\n );\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $cacheItem = helper_getCacheItemByKey($respondedRequest->cacheKey());\n\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(900);\n});\n\nit('prolongs the time to live for a single item', function () {\n $cache = new FileCache(helper_cachedir());\n\n $cache->ttl(100);\n\n $respondedRequest = new RespondedRequest(new Request('GET', '/a'), new Response(body: Utils::streamFor('b')));\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $cacheItem = helper_getCacheItemByKey($respondedRequest->cacheKey());\n\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(100);\n\n /** @var CacheItem $cacheItem */\n\n $cache->prolong($cacheItem->key(), 200);\n\n $cacheItem = helper_getCacheItemByKey($cacheItem->key());\n\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(200);\n});\n\nit('prolongs the time to live for all items in the cache directory', function () {\n $cache = new FileCache(helper_cachedir());\n\n $respondedRequest = new RespondedRequest(new Request('GET', '/a'), new Response(body: Utils::streamFor('b')));\n\n $cache->set($key1 = $respondedRequest->cacheKey(), $respondedRequest, 100);\n\n $respondedRequest = new RespondedRequest(new Request('GET', '/c'), new Response(body: Utils::streamFor('d')));\n\n $cache->set($key2 = $respondedRequest->cacheKey(), $respondedRequest, 200);\n\n $respondedRequest = new RespondedRequest(new Request('GET', '/e'), new Response(body: Utils::streamFor('f')));\n\n $cache->set($key3 = $respondedRequest->cacheKey(), $respondedRequest, 300);\n\n $cacheItem = helper_getCacheItemByKey($key1);\n\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(100);\n\n $cacheItem = helper_getCacheItemByKey($key2);\n\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(200);\n\n $cacheItem = helper_getCacheItemByKey($key3);\n\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(300);\n\n $cache->prolongAll(250);\n\n $cacheItem = helper_getCacheItemByKey($key1);\n\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(250);\n\n $cacheItem = helper_getCacheItemByKey($key2);\n\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(250);\n\n $cacheItem = helper_getCacheItemByKey($key3);\n\n // Prolonging sets the provided value, no matter if an item's previous ttl value was\n // higher than the new one.\n expect($cacheItem)->toBeInstanceOf(CacheItem::class)\n ->and($cacheItem?->ttl)->toBe(250);\n});\n\ntest('the get() and has() methods delete an expired item, but prolong does not', function () {\n $cache = new FileCache(helper_cachedir());\n\n $resp = new RespondedRequest(new Request('GET', '/'), new Response());\n\n // with get()\n $cacheItem = new CacheItem($resp, $resp->cacheKey(), 10, (new DateTimeImmutable())->sub(new DateInterval('PT11S')));\n\n $cache->set($cacheItem->key(), $cacheItem);\n\n $cacheItem = $cache->get($cacheItem->key());\n\n expect($cacheItem)->toBeNull()\n ->and(file_exists(helper_cachedir($resp->cacheKey())))->toBeFalse();\n\n // with has()\n $cacheItem = new CacheItem($resp, $resp->cacheKey(), 10, (new DateTimeImmutable())->sub(new DateInterval('PT11S')));\n\n $cache->set($cacheItem->key(), $cacheItem);\n\n $cache->has($cacheItem->key());\n\n expect($cache->has($cacheItem->key()))->toBeFalse()\n ->and(file_exists(helper_cachedir($cacheItem->key())))->toBeFalse();\n\n // with prolong()\n $cache->set($cacheItem->key(), $cacheItem);\n\n $cache->prolong($cacheItem->key(), 20);\n\n expect($cache->has($cacheItem->key()))->toBeTrue()\n ->and(file_exists(helper_cachedir($cacheItem->key())))->toBeTrue();\n});\n" }, { "path": "tests/Cache/_cachefilecontent", "content": "a:8:{s:13:\"requestMethod\";s:3:\"GET\";s:10:\"requestUri\";s:74:\"https://www.crwlr.software/blog/dealing-with-http-url-query-strings-in-php\";s:14:\"requestHeaders\";a:3:{s:4:\"Host\";a:1:{i:0;s:18:\"www.crwlr.software\";}s:10:\"User-Agent\";a:1:{i:0;s:117:\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36\";}s:6:\"Cookie\";a:2:{i:0;s:20:\"XSRF-TOKEN=xsrftoken\";i:1;s:29:\"crwlrsoftware_session=session\";}}s:11:\"requestBody\";s:0:\"\";s:12:\"effectiveUri\";s:74:\"https://www.crwlr.software/blog/dealing-with-http-url-query-strings-in-php\";s:18:\"responseStatusCode\";i:200;s:15:\"responseHeaders\";a:12:{s:6:\"Server\";a:1:{i:0;s:12:\"nginx/1.21.4\";}s:12:\"Content-Type\";a:1:{i:0;s:24:\"text/html; charset=UTF-8\";}s:17:\"Transfer-Encoding\";a:1:{i:0;s:7:\"chunked\";}s:10:\"Connection\";a:1:{i:0;s:10:\"keep-alive\";}s:4:\"Vary\";a:1:{i:0;s:15:\"Accept-Encoding\";}s:12:\"X-Powered-By\";a:1:{i:0;s:9:\"PHP/8.1.1\";}s:13:\"Cache-Control\";a:1:{i:0;s:17:\"no-cache, private\";}s:4:\"Date\";a:1:{i:0;s:29:\"Tue, 03 Jan 2023 12:38:20 GMT\";}s:10:\"Set-Cookie\";a:2:{i:0;s:81:\"XSRF-TOKEN=xsrftoken; expires=Tue, 03-Jan-2023 14:38:20 GMT; Max-Age=7200; path=/\";i:1;s:100:\"crwlrsoftware_session=session; expires=Tue, 03-Jan-2023 14:38:20 GMT; Max-Age=7200; path=/; httponly\";}s:15:\"X-Frame-Options\";a:2:{i:0;s:10:\"SAMEORIGIN\";i:1;s:4:\"DENY\";}s:16:\"X-XSS-Protection\";a:1:{i:0;s:13:\"1; mode=block\";}s:22:\"X-Content-Type-Options\";a:2:{i:0;s:7:\"nosniff\";i:1;s:7:\"nosniff\";}}s:12:\"responseBody\";s:39078:\"\n\n\n\n\nDealing with HTTP (Url) Query Strings in PHP - crwlr.software Blog\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n\n\n \n\n\n \n \n \n \n \n \n \n \n\n\n\n\n\n \n \n \n \n \n \n\n\n \n\n\n\n
\n
\n

Dealing with HTTP (Url) Query Strings in PHP

\n
2022-06-02
\n

There is a new package in town called query-string. It allows to create, access and manipulate query strings for HTTP requests in a very convenient way. Here's a quick overview of what you can do with it and also how it can be used via the url package.

\n

The last months I started thinking about improving how you can change a URL's query string. It all started with this tweet as an answer to @heychazza's tweet about a nice way to build URLs in javascript.

\"Screenshot

\n

Then last week someone added this github issue for the url package, and it got me thinking more about this. I liked the suggested API to get and set query params, but I found that it's not enough for more complex query strings. As query strings are also used in POST requests and sent in the request body, I now finally added a separate query-string package and also implemented it in the url package.

\n

Implementation in the Url Package

\n

First off: I set the required PHP version for the new package to 8.0 as the last 7.x version (7.4) is already in the final \"security fixes only\" phase. The url package currently still requires only 7.2. As I probably plan another BC break for v2 of the url package, for now I just added the query-string package as suggestion to the composer.json. You can manually install it, when you're already on PHP 8.x and want to use the advanced query string functionality.

\n

When you've installed it via

\n
composer require crwlr/query-string
\n

the new queryString() method of the Url class returns an instance of the Query class shipped with the new package. Here's a quick usage example:

\n
$url = Url::parse('https://www.example.com/listing?page[number]=3&page[size]=25');\n\n$url->queryString()\n    ->get('page')\n    ->set('number', '4');\n\nvar_dump($url->__toString());\n\n// string(68) \"https://www.example.com/listing?page%5Bnumber%5D=4&page%5Bsize%5D=25\"
\n

Standalone Usage

\n

If you want to parse query strings standalone, not in the URL context, you can create an instance of the Query class from string or from array:

\n
$query = Query::fromString('foo=bar&baz=quz');\n\n$query = Query::fromArray(['foo' => 'bar', 'baz' => 'quz']);
\n

Access

\n

Here a quick example of different ways how to access query string params:

\n
$fooValue = Query::fromString('foo=bar&baz=quz')->get('foo'); // string(3) \"bar\"
\n

When the requested key is an array, the get() method returns another (child) Query instance that you can query further:

\n
$fooBazValue = Query::fromString('foo[bar]=1&foo[baz]=2&foo[quz]=3')\n    ->get('foo')\n    ->get('baz'); // string(1) \"2\"
\n

You can check if a certain key exists in the query:

\n
$query = Query::fromString('foo=1&bar=2');\n\n$query->has('bar'); // bool(true)\n\n$query->has('baz'); // bool(false)
\n

You can get the first or last element of an indexed array:

\n
$query = Query::fromString('foo[]=1&foo[]=2&foo[]=3');\n\n$query->first('foo'); // string(1) \"1\"\n\n$query->last('foo');  // string(1) \"3\"
\n

You can check if the value for a certain key is an array of a scalar value:

\n
$query = Query::fromString('foo[]=1&foo[]=2&bar=3');\n\n$query->isArray('foo'); // bool(true)\n\n$query->isScalar('foo'); // bool(false)\n\n$query->isArray('bar'); // bool(false)\n\n$query->isScalar('bar'); // bool(true)
\n

And of course you can then convert the query to a string or to an array again:

\n
$query = Query::fromString('foo=bar&baz=quz');\n\n$queryArray = $query->toArray();\n\n// array(2) {\n//   [\"foo\"]=>\n//   string(3) \"bar\"\n//   [\"baz\"]=>\n//   string(3) \"quz\"\n// }\n\n$query = Query::fromArray(['foo' => 'bar', 'baz' => 'quz']);\n\n$queryString = $query->toString(); // string(15) \"foo=bar&baz=quz\"
\n

Manipulation

\n

You can set a certain key:

\n
$query = Query::fromString('foo=bar')->set('baz', 'quz');\n\n// string(15) \"foo=bar&baz=quz\"
\n

Also to an array:

\n
$query = Query::fromString('foo=1&bar=2')\n    ->set('baz', ['3', '4']);\n\n// string(29) \"foo=1&bar=2&baz[0]=3&baz[1]=4\"
\n

You can also append values to an existing array:

\n
$query = Query::fromString('foo[]=1&foo[]=2')\n    ->appendTo('foo', '3');\n\n// string(26) \"foo[0]=1&foo[1]=2&foo[2]=3\"\n\n$query = Query::fromString('foo[bar]=1&foo[baz]=2')\n    ->appendTo('foo', ['quz' => '3']);\n\n// string(32) \"foo[bar]=1&foo[baz]=2&foo[quz]=3\"
\n

Remove keys or values from keys:

\n
$query = Query::fromString('foo[]=1&foo[]=2&bar=3&baz=4')\n    ->remove('foo');\n\n// string(11) \"bar=3&baz=4\"\n\n$query = Query::fromString('foo[]=1&foo[]=2&foo[]=3&foo[]=2')\n    ->removeValueFrom('foo', '2');\n\n// string(17) \"foo[0]=1&foo[1]=3\"
\n

And you can filter or map queries with callback functions:

\n
$query = Query::fromString('no1=12&no2=7&no3=23&no4=9&no5=10')\n    ->filter(function ($value, $key) {\n        return (int) $value >= 10;\n    });\n\n// string(20) \"no1=12&no3=23&no5=10\"\n\n$query = Query::fromString('foo=1&bar=2&baz=3&quz=4')\n    ->map(function ($value) {\n        return (int) $value + 3;\n    });\n\n// string(23) \"foo=4&bar=5&baz=6&quz=7\"
\n

For more details have a look at the documentation. If you're having any question or issues, don't be shy and reach out on twitter or github.

\n
\n\n
\n
\n\n\n\n\n\n\";}" }, { "path": "tests/CrawlerTest.php", "content": "addStep($step);\n\n return $crawler;\n}\n\n/** @var TestCase $this */\n\ntest(\n 'The methods to define UserAgent, Logger and Loader instances are called in construct and the getter methods ' .\n 'always return the same instance.',\n function () {\n $crawler = new DummyTwo();\n\n expect($crawler->getUserAgent()->testProperty)->toBe('foo')\n ->and($crawler->getLogger()->testProperty)->toBe('foo')\n ->and($crawler->getLoader()->testProperty)->toBe('foo')\n ->and($crawler->userAgentCalled)->toBe(1)\n ->and($crawler->loggerCalled)->toBe(1)\n ->and($crawler->loaderCalled)->toBe(1);\n\n $crawler->getUserAgent()->testProperty = 'bar';\n\n $crawler->getLogger()->testProperty = 'bar';\n\n $crawler->getLoader()->testProperty = 'bar';\n\n $crawler->addStep(Http::get()); // adding steps passes on logger and loader, should use the same instances\n\n expect($crawler->getUserAgent()->testProperty)->toBe('bar')\n ->and($crawler->getLogger()->testProperty)->toBe('bar')\n ->and($crawler->getLoader()->testProperty)->toBe('bar')\n ->and($crawler->userAgentCalled)->toBe(1)\n ->and($crawler->loggerCalled)->toBe(1)\n ->and($crawler->loaderCalled)->toBe(1);\n },\n);\n\nit('gives you the current memory limit', function () {\n expect(Crawler::getMemoryLimit())->toBeString();\n});\n\nit('changes the current memory limit when allowed', function () {\n $currentLimit = Crawler::getMemoryLimit();\n\n if ($currentLimit === '512M') {\n $newValue = '1G';\n } else {\n $newValue = '512M';\n }\n\n $setLimitReturnValue = Crawler::setMemoryLimit($newValue);\n\n if ($setLimitReturnValue === false) {\n expect(Crawler::getMemoryLimit())->toBe($currentLimit);\n } else {\n expect(Crawler::getMemoryLimit())->toBe($newValue);\n }\n});\n\ntest('You can set a single input for the first step using the input method', function () {\n $crawler = helper_getDummyCrawlerWithInputReturningStep();\n\n $crawler->input('https://www.example.com');\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->toArray()['unnamed'])->toBe('https://www.example.com');\n});\n\ntest('You can set multiple inputs by multiply calling the input method', function () {\n $crawler = helper_getDummyCrawlerWithInputReturningStep();\n\n $crawler->input('https://www.crwl.io');\n\n $crawler->input('https://www.otsch.codes');\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->toArray()['unnamed'])->toBe('https://www.crwl.io');\n\n expect($results[1]->toArray()['unnamed'])->toBe('https://www.otsch.codes');\n});\n\ntest('You can set multiple inputs using the inputs (plural) method', function () {\n $crawler = helper_getDummyCrawlerWithInputReturningStep();\n\n $crawler->inputs(['https://www.crwl.io', 'https://www.otsch.codes']);\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->toArray()['unnamed'])->toBe('https://www.crwl.io');\n\n expect($results[1]->toArray()['unnamed'])->toBe('https://www.otsch.codes');\n});\n\ntest('Initial inputs are reset after the crawler was run', function () {\n $crawler = helper_getDummyCrawlerWithInputReturningStep();\n\n $crawler->inputs(['https://www.crwl.io', 'https://www.otsch.codes']);\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(2);\n\n $crawler->input('https://fetzi.dev/');\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1);\n});\n\ntest('You can add steps and the Crawler class passes on its Logger and also its Loader if needed', function () {\n $step = Mockery::mock(StepInterface::class);\n\n $step->shouldReceive('addLogger')->once();\n\n $crawler = helper_getDummyCrawler();\n\n $crawler->addStep($step);\n\n $step = helper_getLoadingStep();\n\n $step = Mockery::mock($step)->makePartial();\n\n $step->shouldReceive('addLogger')->once();\n\n $step->shouldReceive('setLoader')->once();\n\n $step->shouldReceive('setParentCrawler')->once()->andReturnSelf();\n\n /** @var Step $step */\n\n $crawler->addStep($step);\n});\n\ntest('You can add steps and they are invoked when the Crawler is run', function () {\n $step1 = helper_getValueReturningStep('step1 output')->keepAs('step1');\n\n $step2 = helper_getValueReturningStep('step2 output')->keepAs('step2');\n\n $crawler = helper_getDummyCrawler()\n ->addStep($step1)\n ->addStep($step2);\n\n $crawler->input('randomInput');\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->toArray())->toBe(['step1' => 'step1 output', 'step2' => 'step2 output']);\n\n});\n\nit('resets the initial inputs and calls the resetAfterRun method of all its steps', function () {\n $step = helper_getInputReturningStep()->uniqueOutputs();\n\n $crawler = helper_getDummyCrawler()\n ->inputs(['input1', 'input1', 'input2'])\n ->addStep($step->keepAs('foo'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(2)\n ->and($results[0]->toArray())->toBe(['foo' => 'input1'])\n ->and($results[1]->toArray())->toBe(['foo' => 'input2']);\n\n $crawler->inputs(['input1', 'input3']);\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(2)\n ->and($results[0]->toArray())->toBe(['foo' => 'input1'])\n ->and($results[1]->toArray())->toBe(['foo' => 'input3']);\n\n});\n\ntest('You can add a step group as a step and all it\\'s steps are invoked when the Crawler is run', function () {\n $crawler = helper_getDummyCrawler();\n\n $step1 = Mockery::mock(StepInterface::class);\n\n $step1->shouldReceive('invokeStep')->andReturn(helper_arrayToGenerator(['foo']));\n\n $step1->shouldReceive('addLogger');\n\n $step2 = Mockery::mock(StepInterface::class);\n\n $step2->shouldReceive('invokeStep')->andReturn(helper_arrayToGenerator(['bar']));\n\n $step2->shouldReceive('addLogger');\n\n $step3 = Mockery::mock(StepInterface::class);\n\n $step3->shouldReceive('invokeStep')->andReturn(helper_arrayToGenerator(['baz']));\n\n $step3->shouldReceive('addLogger');\n\n $crawler->addStep(\n Crawler::group()\n ->addStep($step1)\n ->addStep($step2)\n ->addStep($step3),\n );\n\n expect(true)->toBeTrue(); // So pest doesn't complain that there is no assertion.\n});\n\n/* ----------------------------- keep() and keepAs() ----------------------------- */\n\ntest('when you call keep() or keepAs() on a step, it keeps its output data until the end', function () {\n $crawler = helper_getDummyCrawler();\n\n $crawler\n ->input('test')\n ->addStep(\n helper_getValueReturningStep(['father' => 'Karl', 'mother' => 'Ludmilla'])->keep(),\n )\n ->addStep(\n helper_getValueReturningStep([\n 'daughter1' => 'Elisabeth',\n 'son1' => 'Leon',\n 'son2' => 'Franz',\n 'daughter2' => 'Julia',\n 'daughter3' => 'Franziska',\n ])->keep(['daughter' => 'daughter2', 'son' => 'son2']),\n )\n ->addStep(helper_getValueReturningStep('Lea')->keepAs('cousin'))\n ->addStep(\n helper_getValueReturningStep([\n 'grandson1' => 'Jonah',\n 'granddaughter1' => 'Paula',\n 'granddaughter2' => 'Sophie',\n ]),\n );\n\n $results = iterator_to_array($crawler->run());\n\n expect($results[0]->toArray())->toBe([\n 'father' => 'Karl',\n 'mother' => 'Ludmilla',\n 'daughter' => 'Julia',\n 'son' => 'Franz',\n 'cousin' => 'Lea',\n 'grandson1' => 'Jonah',\n 'granddaughter1' => 'Paula',\n 'granddaughter2' => 'Sophie',\n ]);\n});\n\nit('immediately stops when keepAs() is not used with a scalar value output step', function () {\n $crawler = helper_getDummyCrawler();\n\n $step1 = new class extends Step {\n public bool $wasCalled = false;\n\n protected function invoke(mixed $input): Generator\n {\n $this->wasCalled = true;\n\n yield ['father' => 'Karl', 'mother' => 'Ludmilla'];\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::AssociativeArrayOrObject;\n }\n };\n\n $step2 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'foo';\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Scalar;\n }\n };\n\n $crawler\n ->input('test')\n ->addStep($step1->keep())\n ->addStep($step2->keep());\n\n try {\n $results = iterator_to_array($crawler->run());\n } catch (PreRunValidationException $exception) {\n }\n\n expect($results ?? null)->toBeEmpty()\n ->and($step1->wasCalled)->toBeFalse()\n ->and($this->getActualOutputForAssertion())->toContain('Pre-Run validation error in step number 2')\n ->and($exception ?? null)->toBeInstanceOf(PreRunValidationException::class);\n});\n\nit('sends all results to the Store when there is one and still yields the results', function () {\n $store = Mockery::mock(StoreInterface::class);\n\n $store->shouldReceive('addLogger');\n\n $store->shouldReceive('store')->times(3);\n\n $crawler = helper_getDummyCrawler();\n\n $crawler->input('gogogo');\n\n $crawler->setStore($store);\n\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'one';\n yield 'two';\n yield 'three';\n }\n };\n\n $crawler->addStep($step->keepAs('number'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(3)\n ->and($results[0]->toArray())->toBe(['number' => 'one'])\n ->and($results[1]->toArray())->toBe(['number' => 'two'])\n ->and($results[2]->toArray())->toBe(['number' => 'three']);\n});\n\nit(\n 'actually runs the crawler without the need to traverse results manually, when runAndTraverse is called',\n function () {\n $step = helper_getInputReturningStep();\n\n $store = Mockery::mock(StoreInterface::class);\n\n $store->shouldReceive('addLogger');\n\n $store->shouldNotReceive('store');\n\n $crawler = helper_getDummyCrawler()\n ->addStep($step)\n ->setStore($store)\n ->input('test');\n\n $crawler->run();\n\n $store = Mockery::mock(StoreInterface::class);\n\n $store->shouldReceive('store', 'addLogger')->once();\n\n $crawler = helper_getDummyCrawler()\n ->addStep($step)\n ->setStore($store)\n ->input('test');\n\n $crawler->runAndTraverse();\n },\n);\n\nit('yields only unique outputs from a step when uniqueOutput was called', function () {\n $crawler = helper_getDummyCrawler();\n\n $crawler->addStep(helper_getInputReturningStep()->uniqueOutputs());\n\n $crawler->inputs(['one', 'two', 'three', 'one', 'three', 'four', 'one', 'five', 'two']);\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(5);\n});\n\nit(\n 'cascades step outputs immediately and doesn\\'t wait for the current step being called with all the inputs',\n function () {\n $step1 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step1 called');\n\n yield $input . ' step1-1';\n\n yield $input . ' step1-2';\n }\n };\n\n $step2 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step2 called');\n\n yield $input . ' step2';\n }\n };\n\n $store = new class extends Store {\n public function store(Result $result): void\n {\n $this->logger?->info('Stored a result');\n }\n };\n\n $crawler = helper_getDummyCrawler()\n ->inputs(['input1', 'input2'])\n ->addStep($step1->keepAs('foo'))\n ->addStep($step2->keepAs('bar'))\n ->setStore($store);\n\n $crawler->runAndTraverse();\n\n $output = $this->getActualOutputForAssertion();\n\n $outputLines = explode(\"\\n\", $output);\n\n expect($outputLines[0])->toContain('step1 called')\n ->and($outputLines[1])->toContain('step2 called')\n ->and($outputLines[2])->toContain('Stored a result')\n ->and($outputLines[3])->toContain('step2 called')\n ->and($outputLines[4])->toContain('Stored a result')\n ->and($outputLines[5])->toContain('step1 called')\n ->and($outputLines[6])->toContain('step2 called')\n ->and($outputLines[7])->toContain('Stored a result')\n ->and($outputLines[8])->toContain('step2 called')\n ->and($outputLines[9])->toContain('Stored a result');\n },\n);\n\nit(\n 'immediately calls the store for each final output',\n function () {\n $step1 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step1 called');\n\n yield '1-1';\n\n yield '1-2';\n }\n };\n\n $step2 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step2 called: ' . $input);\n\n yield $input . ' 2-1';\n\n yield $input . ' 2-2';\n }\n };\n\n $step3 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step3 called: ' . $input);\n\n yield $input . ' 3-1';\n\n yield $input . ' 3-2';\n }\n };\n\n $step4 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step4 called: ' . $input);\n\n yield $input . ' 4-1';\n\n yield $input . ' 4-2';\n }\n };\n\n $store = new class extends Store {\n public function store(Result $result): void\n {\n $this->logger?->info('Stored a result: ' . $result->get('unnamed'));\n }\n };\n\n $crawler = helper_getDummyCrawler()\n ->input('input')\n ->addStep($step1)\n ->addStep($step2)\n ->addStep($step3)\n ->addStep($step4)\n ->setStore($store);\n\n $crawler->runAndTraverse();\n\n $output = $this->getActualOutputForAssertion();\n\n $outputLines = explode(\"\\n\", $output);\n\n expect($outputLines[0])\n ->toContain('step1 called')\n ->and($outputLines[1])->toContain('step2 called: 1-1')\n ->and($outputLines[2])->toContain('step3 called: 1-1 2-1')\n ->and($outputLines[3])->toContain('step4 called: 1-1 2-1 3-1')\n ->and($outputLines[4])->toContain('Stored a result: 1-1 2-1 3-1 4-1')\n ->and($outputLines[5])->toContain('Stored a result: 1-1 2-1 3-1 4-2')\n ->and($outputLines[6])->toContain('step4 called: 1-1 2-1 3-2')\n ->and($outputLines[7])->toContain('Stored a result: 1-1 2-1 3-2 4-1')\n ->and($outputLines[8])->toContain('Stored a result: 1-1 2-1 3-2 4-2')\n ->and($outputLines[9])->toContain('step3 called: 1-1 2-2')\n ->and($outputLines[10])->toContain('step4 called: 1-1 2-2 3-1')\n ->and($outputLines[11])->toContain('Stored a result: 1-1 2-2 3-1 4-1')\n ->and($outputLines[12])->toContain('Stored a result: 1-1 2-2 3-1 4-2')\n ->and($outputLines[13])->toContain('step4 called: 1-1 2-2 3-2')\n ->and($outputLines[14])->toContain('Stored a result: 1-1 2-2 3-2 4-1')\n ->and($outputLines[15])->toContain('Stored a result: 1-1 2-2 3-2 4-2')\n ->and($outputLines[16])->toContain('step2 called: 1-2')\n ->and($outputLines[17])->toContain('step3 called: 1-2 2-1')\n ->and($outputLines[18])->toContain('step4 called: 1-2 2-1 3-1')\n ->and($outputLines[19])->toContain('Stored a result: 1-2 2-1 3-1 4-1')\n ->and($outputLines[20])->toContain('Stored a result: 1-2 2-1 3-1 4-2')\n ->and($outputLines[21])->toContain('step4 called: 1-2 2-1 3-2')\n ->and($outputLines[22])->toContain('Stored a result: 1-2 2-1 3-2 4-1')\n ->and($outputLines[23])->toContain('Stored a result: 1-2 2-1 3-2 4-2')\n ->and($outputLines[24])->toContain('step3 called: 1-2 2-2')\n ->and($outputLines[25])->toContain('step4 called: 1-2 2-2 3-1')\n ->and($outputLines[26])->toContain('Stored a result: 1-2 2-2 3-1 4-1')\n ->and($outputLines[27])->toContain('Stored a result: 1-2 2-2 3-1 4-2')\n ->and($outputLines[28])->toContain('step4 called: 1-2 2-2 3-2')\n ->and($outputLines[29])->toContain('Stored a result: 1-2 2-2 3-2 4-1')\n ->and($outputLines[30])->toContain('Stored a result: 1-2 2-2 3-2 4-2');\n },\n);\n\nit(\n 'does not wait for all child outputs originating from an output of a step where keepAs() was called before ' .\n 'calling the store',\n function () {\n $step1 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step1 called');\n\n yield '1-1';\n\n yield '1-2';\n }\n };\n\n $step2 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step2 called: ' . $input);\n\n yield $input . ' 2-1';\n\n yield $input . ' 2-2';\n }\n };\n\n $step2->keepAs('foo');\n\n $step3 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step3 called: ' . $input);\n\n yield $input . ' 3-1';\n\n yield $input . ' 3-2';\n }\n };\n\n $step4 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('step4 called: ' . $input);\n\n yield $input . ' 4-1';\n\n yield $input . ' 4-2';\n }\n };\n\n $step4->keepAs('bar');\n\n $store = new class extends Store {\n public function store(Result $result): void\n {\n $this->logger?->info('Stored a result: ' . $result->get('bar'));\n }\n };\n\n $crawler = helper_getDummyCrawler()\n ->input('input')\n ->addStep($step1)\n ->addStep($step2)\n ->addStep($step3)\n ->addStep($step4)\n ->setStore($store);\n\n $crawler->runAndTraverse();\n\n $output = $this->getActualOutputForAssertion();\n\n $outputLines = explode(\"\\n\", $output);\n\n expect($outputLines[0])->toContain('step1 called')\n ->and($outputLines[1])->toContain('step2 called: 1-1')\n ->and($outputLines[2])->toContain('step3 called: 1-1 2-1')\n ->and($outputLines[3])->toContain('step4 called: 1-1 2-1 3-1')\n ->and($outputLines[4])->toContain('Stored a result: 1-1 2-1 3-1 4-1')\n ->and($outputLines[5])->toContain('Stored a result: 1-1 2-1 3-1 4-2')\n ->and($outputLines[6])->toContain('step4 called: 1-1 2-1 3-2')\n ->and($outputLines[7])->toContain('Stored a result: 1-1 2-1 3-2 4-1')\n ->and($outputLines[8])->toContain('Stored a result: 1-1 2-1 3-2 4-2')\n ->and($outputLines[9])->toContain('step3 called: 1-1 2-2')\n ->and($outputLines[10])->toContain('step4 called: 1-1 2-2 3-1')\n ->and($outputLines[11])->toContain('Stored a result: 1-1 2-2 3-1 4-1')\n ->and($outputLines[12])->toContain('Stored a result: 1-1 2-2 3-1 4-2')\n ->and($outputLines[13])->toContain('step4 called: 1-1 2-2 3-2')\n ->and($outputLines[14])->toContain('Stored a result: 1-1 2-2 3-2 4-1')\n ->and($outputLines[15])->toContain('Stored a result: 1-1 2-2 3-2 4-2')\n ->and($outputLines[16])->toContain('step2 called: 1-2')\n ->and($outputLines[17])->toContain('step3 called: 1-2 2-1')\n ->and($outputLines[18])->toContain('step4 called: 1-2 2-1 3-1')\n ->and($outputLines[19])->toContain('Stored a result: 1-2 2-1 3-1 4-1')\n ->and($outputLines[20])->toContain('Stored a result: 1-2 2-1 3-1 4-2')\n ->and($outputLines[21])->toContain('step4 called: 1-2 2-1 3-2')\n ->and($outputLines[22])->toContain('Stored a result: 1-2 2-1 3-2 4-1')\n ->and($outputLines[23])->toContain('Stored a result: 1-2 2-1 3-2 4-2')\n ->and($outputLines[24])->toContain('step3 called: 1-2 2-2')\n ->and($outputLines[25])->toContain('step4 called: 1-2 2-2 3-1')\n ->and($outputLines[26])->toContain('Stored a result: 1-2 2-2 3-1 4-1')\n ->and($outputLines[27])->toContain('Stored a result: 1-2 2-2 3-1 4-2')\n ->and($outputLines[28])->toContain('step4 called: 1-2 2-2 3-2')\n ->and($outputLines[29])->toContain('Stored a result: 1-2 2-2 3-2 4-1')\n ->and($outputLines[30])->toContain('Stored a result: 1-2 2-2 3-2 4-2');\n },\n);\n\nit('logs memory usage if you want it to', function () {\n $step1 = helper_getValueReturningStep('foo');\n\n $step2 = helper_getValueReturningStep('bar');\n\n $crawler = helper_getDummyCrawler()\n ->input('go')\n ->addStep($step1)\n ->addStep($step2)\n ->monitorMemoryUsage();\n\n $crawler->runAndTraverse();\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->toContain('memory usage: ');\n});\n\nit('sends all outputs to the outputHook when defined', function () {\n $outputs = [];\n\n $crawler = helper_getDummyCrawler()\n ->input(1)\n ->addStep(helper_getNumberIncrementingStep())\n ->addStep(helper_getNumberIncrementingStep())\n ->outputHook(function (Output $output, int $stepIndex, StepInterface $step) use (&$outputs) {\n $outputs[$stepIndex][] = $output->get();\n });\n\n $crawler->runAndTraverse();\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0])->toHaveCount(1)\n ->and($outputs[0][0])->toBe(2)\n ->and($outputs[1])->toHaveCount(1)\n ->and($outputs[1][0])->toBe(3);\n});\n\ntest(\n 'When result is not explicitly composed and last step produces array output with string keys, it uses those keys ' .\n 'for the result.',\n function () {\n $crawler = helper_getDummyCrawler()\n ->input('hello')\n ->addStep(helper_getValueReturningStep(['foo' => 'bar', 'baz' => 'quz']));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->toArray())->toBe(['foo' => 'bar', 'baz' => 'quz']);\n },\n);\n\nit('just runs the crawler and dumps all results as array when runAndDump() is called', function () {\n helper_getDummyCrawlerWithInputReturningStep()\n ->inputs([\n ['foo' => 'one', 'bar' => 'two'],\n ['baz' => 'three', 'quz' => 'four'],\n ])\n ->runAndDump();\n\n $actualOutput = $this->getActualOutputForAssertion();\n\n expect(explode('array(2)', $actualOutput))->toHaveCount(3)\n ->and($actualOutput)->toContain('[\"foo\"]=>')\n ->and($actualOutput)->toContain('string(3) \"one\"')\n ->and($actualOutput)->toContain('[\"bar\"]=>')\n ->and($actualOutput)->toContain('string(3) \"two\"')\n ->and($actualOutput)->toContain('[\"baz\"]=>')\n ->and($actualOutput)->toContain('string(5) \"three\"')\n ->and($actualOutput)->toContain('[\"quz\"]=>')\n ->and($actualOutput)->toContain('string(4) \"four\"');\n});\n" }, { "path": "tests/HttpCrawler/AnonymousHttpCrawlerBuilderTest.php", "content": "withBotUserAgent('YoloCrawler');\n\n expect($crawler)->toBeInstanceOf(HttpCrawler::class)\n ->and($crawler->getLoader())->toBeInstanceOf(HttpLoader::class);\n\n $loader = $crawler->getLoader();\n\n expect($loader->userAgent())->toBeInstanceOf(BotUserAgent::class);\n\n $userAgent = $loader->userAgent();\n\n /** @var BotUserAgent $userAgent */\n\n expect($userAgent->productToken())->toBe('YoloCrawler');\n});\n\nit('creates an HttpCrawler instance with a non bot user agent', function () {\n $crawler = HttpCrawler::make()\n ->withUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ...');\n\n expect($crawler)->toBeInstanceOf(HttpCrawler::class)\n ->and($crawler->getLoader())->toBeInstanceOf(HttpLoader::class);\n\n $loader = $crawler->getLoader();\n\n expect($loader->userAgent())->toBeInstanceOf(UserAgent::class);\n\n $userAgent = $loader->userAgent();\n\n /** @var UserAgent $userAgent */\n\n expect($userAgent->__toString())->toBe('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ...');\n});\n\nit('creates an HttpCrawler instance with a mozilla 5.0 compatible user agent', function () {\n $crawler = HttpCrawler::make()->withMozilla5CompatibleUserAgent();\n\n $userAgent = $crawler->getLoader()->userAgent();\n\n expect($userAgent->__toString())->toBe('Mozilla/5.0 (compatible)');\n});\n" }, { "path": "tests/IoTest.php", "content": "toBeInstanceOf(Io::class);\n});\n\ntest('you can add an array with data that should be kept (see Step::keep() functionality)', function () {\n $keep = ['foo' => 'bar', 'baz' => 'quz'];\n\n $io = helper_getIoInstance('test', keep: $keep);\n\n expect($io->keep)->toBe($keep);\n});\n\ntest('you can create it from another Io instance and it keeps the value of the original instance.', function () {\n $io1 = helper_getIoInstance('test');\n\n $io2 = helper_getIoInstance($io1);\n\n expect($io2->get())->toBe('test');\n});\n\ntest('when created from another Io instance it passes on the data to keep', function () {\n $io1 = helper_getIoInstance('test', keep: ['co' => 'derotsch']);\n\n $io2 = helper_getIoInstance($io1);\n\n expect($io2->keep)->toBe(['co' => 'derotsch']);\n});\n\ntest('the withValue() method creates a new instance with that value but keeps the keep data', function () {\n $io1 = helper_getIoInstance('hey', ['baz' => 'three']);\n\n $io2 = $io1->withValue('ho');\n\n expect($io2->get())->toBe('ho')\n ->and($io2->keep)->toBe(['baz' => 'three']);\n});\n\ntest(\n 'the withPropertyValue() method creates a new instance and replaces a certain property in its array value',\n function () {\n $io1 = helper_getIoInstance(['a' => '1', 'b' => '2', 'c' => '3'], ['baz' => 'three']);\n\n $io2 = $io1->withPropertyValue('c', '4');\n\n expect($io2->get())->toBe(['a' => '1', 'b' => '2', 'c' => '4'])\n ->and($io2->keep)->toBe(['baz' => 'three']);\n },\n);\n\ntest('if the property does not exist, it is added, when withPropertyValue() is used', function () {\n $io1 = helper_getIoInstance(['a' => '1', 'b' => '2']);\n\n $io2 = $io1->withPropertyValue('c', '3');\n\n expect($io2->get())->toBe(['a' => '1', 'b' => '2', 'c' => '3']);\n});\n\nit('gets a particular property by key from array output', function () {\n $io = helper_getIoInstance(['foo' => 'so', 'bar' => 'lala', 'baz' => 'bla']);\n\n expect($io->getProperty('bar'))->toBe('lala');\n});\n\nit('when the property does not exist, getProperty() returns the defined fallback value (default null)', function () {\n $io = helper_getIoInstance(['foo' => 'so', 'bar' => 'lala', 'baz' => 'bla']);\n\n expect($io->getProperty('quz'))->toBeNull()\n ->and($io->getProperty('quz', 123))->toBe(123);\n});\n\nit('sets a simple value key', function ($value, $key) {\n $io = helper_getIoInstance($value);\n\n expect($io->setKey())->toBe($key)\n ->and($io->getKey())->toBe($key);\n})->with([\n ['foo', 'foo'],\n [123, '123'],\n [123.1234, '123.1234'],\n [true, 'true'],\n [false, 'false'],\n [null, 'null'],\n]);\n\nit('sets a key from array output', function () {\n $io = helper_getIoInstance(['foo' => 'bar', 'yo' => 123.45]);\n\n expect($io->setKey('yo'))->toBe('123.45')\n ->and($io->getKey())->toBe('123.45');\n});\n\nit('sets a key from object output', function () {\n $value = helper_getStdClassWithData(['foo' => 'bar', 'yo' => 123.45]);\n\n $io = helper_getIoInstance($value);\n\n expect($io->setKey('yo'))->toBe('123.45')\n ->and($io->getKey())->toBe('123.45');\n});\n\nit('creates a string key for array output when not providing a key name', function () {\n $io = helper_getIoInstance(['one', 'two', 'three']);\n\n expect($io->setKey())->toBe('6975f1fd65cae4b21e32f4f47bf153a8')\n ->and($io->getKey())->toBe('6975f1fd65cae4b21e32f4f47bf153a8');\n});\n\nit('creates a string key for object output when not providing a key name', function () {\n $object = helper_getStdClassWithData(['one', 'two', 'three']);\n\n $io = helper_getIoInstance($object);\n\n expect($io->setKey())->toBe('bb8dd69ea029ca1379df3994721f5fa9')\n ->and($io->getKey())->toBe('bb8dd69ea029ca1379df3994721f5fa9');\n});\n\nit('creates a string key for array output when provided key name doesn\\'t exist in output array', function () {\n $io = helper_getIoInstance(['one', 'two', 'three']);\n\n expect($io->setKey('four'))->toBe('6975f1fd65cae4b21e32f4f47bf153a8')\n ->and($io->getKey())->toBe('6975f1fd65cae4b21e32f4f47bf153a8');\n});\n\nit('creates a string key for array output when provided key name doesn\\'t exist in output object', function () {\n $object = helper_getstdClassWithData(['one', 'two', 'three']);\n\n $io = helper_getIoInstance($object);\n\n expect($io->setKey('four'))->toBe('bb8dd69ea029ca1379df3994721f5fa9')\n ->and($io->getKey())->toBe('bb8dd69ea029ca1379df3994721f5fa9');\n});\n\ntest('getKey returns a key when setKey was not called yet', function () {\n $io = helper_getIoInstance('test');\n\n expect($io->getKey())->toBe('test');\n});\n\ntest('isArrayWithStringKeys returns true when the value is an array with string keys', function () {\n $io = helper_getIoInstance(['foo' => 'one', 'bar' => 'two', 'baz' => 'three']);\n\n expect($io->isArrayWithStringKeys())->toBeTrue();\n});\n\ntest('isArrayWithStringKeys returns false when the value is not an array with string keys', function ($value) {\n $io = helper_getIoInstance($value);\n\n expect($io->isArrayWithStringKeys())->toBeFalse();\n})->with([\n 123,\n true,\n ['foo', 'bar'],\n helper_getStdClassWithData(['foo' => 'bar']),\n]);\n\nit('adds data to keep when calling keep() and makes already existing keys an array', function () {\n $io = helper_getIoInstance('value', keep: ['foo' => 'one', 'bar' => 'two']);\n\n $io->keep(['bar' => 'three', 'baz' => 'four']);\n\n expect($io->keep)->toBe(['foo' => 'one', 'bar' => ['two', 'three'], 'baz' => 'four']);\n});\n" }, { "path": "tests/Loader/Http/Browser/ScreenshotConfigTest.php", "content": "storePath)->toBe('/some/path')\n ->and($instance->fileType)->toBe('png')\n ->and($instance->quality)->toBeNull()\n ->and($instance->fullPage)->toBeFalse();\n});\n\nit('can be constructed via the static make() method', function () {\n $instance = ScreenshotConfig::make('/some/different/path');\n\n expect($instance->storePath)->toBe('/some/different/path')\n ->and($instance->fileType)->toBe('png')\n ->and($instance->quality)->toBeNull()\n ->and($instance->fullPage)->toBeFalse();\n});\n\ntest('the image file type can be changed to jpeg via the setImageFileType() method', function () {\n $instance = ScreenshotConfig::make('/some/path')->setImageFileType('jpeg');\n\n expect($instance->fileType)->toBe('jpeg')\n ->and($instance->quality)->toBe(80);\n});\n\ntest('the image file type can be changed to webp via the setImageFileType() method', function () {\n $instance = ScreenshotConfig::make('/some/path')->setImageFileType('webp');\n\n expect($instance->fileType)->toBe('webp')\n ->and($instance->quality)->toBe(80);\n});\n\ntest('the image file type can be changed to png via the setImageFileType() method', function () {\n $instance = ScreenshotConfig::make('/some/path')->setImageFileType('jpeg');\n\n $instance->setImageFileType('png');\n\n expect($instance->fileType)->toBe('png')\n ->and($instance->quality)->toBeNull();\n});\n\ntest('setting the image file type to something different than png, jpeg or webp does not work', function () {\n $instance = ScreenshotConfig::make('/some/path')->setImageFileType('gif');\n\n expect($instance->fileType)->toBe('png');\n});\n\ntest('the image quality can be changed via setQuality()', function () {\n $instance = ScreenshotConfig::make('/some/path')->setImageFileType('jpeg')->setQuality(65);\n\n expect($instance->quality)->toBe(65);\n});\n\ntest('the image quality can not be changed via setQuality() when the file type is png', function () {\n $instance = ScreenshotConfig::make('/some/path')->setQuality(65);\n\n expect($instance->quality)->toBeNull();\n});\n\ntest('the full page param can be set to true via setFullPage()', function () {\n $instance = ScreenshotConfig::make('/some/path')->setFullPage();\n\n expect($instance->fullPage)->toBeTrue();\n});\n\nit('creates a config array for the chrome-php library', function () {\n $pageMock = Mockery::mock(Page::class);\n\n $instance = ScreenshotConfig::make('/some/path');\n\n expect($instance->toChromePhpScreenshotConfig($pageMock))->toBe(['format' => 'png']);\n});\n\ntest('the config array for the chrome-php library contains the image quality', function () {\n $pageMock = Mockery::mock(Page::class);\n\n $instance = ScreenshotConfig::make('/some/path')->setImageFileType('webp')->setQuality(75);\n\n expect($instance->toChromePhpScreenshotConfig($pageMock))->toBe(['format' => 'webp', 'quality' => 75]);\n});\n\ntest('the config array has the necessary properties when fullPage is set to true', function () {\n $pageMock = Mockery::mock(Page::class);\n\n $pageMock->shouldReceive('getFullPageClip')->andReturn(Mockery::mock(Clip::class));\n\n $instance = ScreenshotConfig::make('/some/path')->setFullPage();\n\n $configArray = $instance->toChromePhpScreenshotConfig($pageMock);\n\n expect($configArray['format'])->toBe('png')\n ->and($configArray['captureBeyondViewport'])->toBeTrue()\n ->and($configArray['clip'])->toBeInstanceOf(Clip::class);\n});\n" }, { "path": "tests/Loader/Http/Cache/RetryManagerTest.php", "content": "= 400 when nothing else was defined', function (int $statusCode) {\n expect((new RetryManager())->shallBeRetried($statusCode))->toBeTrue();\n})->with([[403], [404], [500], [503]]);\n\nit('returns false for status codes below 400 when nothing else was defined', function (int $statusCode) {\n expect((new RetryManager())->shallBeRetried($statusCode))->toBeFalse();\n})->with([[100], [200], [302], [308]]);\n\nit(\n 'returns true for only one error status code when only() was used with an int',\n function (int $statusCode, bool $expected) {\n $retryManager = new RetryManager();\n\n $retryManager->only(404);\n\n expect($retryManager->shallBeRetried($statusCode))->toBe($expected);\n },\n)->with([\n [401, false],\n [403, false],\n [404, true],\n [405, false],\n [500, false],\n [503, false],\n]);\n\nit(\n 'returns true for only a set of error status codes when only() was used with an array',\n function (int $statusCode, bool $expected) {\n $retryManager = new RetryManager();\n\n $retryManager->only([404, 503]);\n\n expect($retryManager->shallBeRetried($statusCode))->toBe($expected);\n },\n)->with([\n [401, false],\n [403, false],\n [404, true],\n [405, false],\n [500, false],\n [503, true],\n]);\n\nit(\n 'returns true for all error status codes except one, when except() was used with an int',\n function (int $statusCode, bool $expected) {\n $retryManager = new RetryManager();\n\n $retryManager->except(404);\n\n expect($retryManager->shallBeRetried($statusCode))->toBe($expected);\n },\n)->with([\n [401, true],\n [403, true],\n [404, false],\n [405, true],\n [500, true],\n [503, true],\n]);\n\nit(\n 'returns true except for a set of error status codes, when except() was used with an array',\n function (int $statusCode, bool $expected) {\n $retryManager = new RetryManager();\n\n $retryManager->except([403, 410, 500]);\n\n expect($retryManager->shallBeRetried($statusCode))->toBe($expected);\n },\n)->with([\n [401, true],\n [403, false],\n [404, true],\n [405, true],\n [410, false],\n [500, false],\n [503, true],\n]);\n" }, { "path": "tests/Loader/Http/Cookies/CookieJarTest.php", "content": "addFrom('https://www.crwl.io', new Response(200, [\n 'Set-Cookie' => ['cook13=v4lu3; Secure'],\n ]));\n\n $allCookiesForDomain = $jar->allByDomain('crwl.io');\n\n expect($allCookiesForDomain)->toHaveCount(1);\n});\n\ntest('addFrom works with an instance of UriInterface', function () {\n $jar = new CookieJar();\n\n $jar->addFrom(Url::parsePsr7('https://www.crwl.io'), new Response(200, [\n 'Set-Cookie' => ['cook13=v4lu3; Secure'],\n ]));\n\n $allCookiesForDomain = $jar->allByDomain('crwl.io');\n\n expect($allCookiesForDomain)->toHaveCount(1);\n});\n\ntest('addFrom works with an instance of Url', function () {\n $jar = new CookieJar();\n\n $jar->addFrom(Url::parse('https://www.crwl.io'), new Response(200, [\n 'Set-Cookie' => ['cook13=v4lu3; Secure'],\n ]));\n\n $allCookiesForDomain = $jar->allByDomain('crwl.io');\n\n expect($allCookiesForDomain)->toHaveCount(1);\n});\n\ntest('addFrom() works with a CookieCollection from the chrome-php lib', function () {\n $jar = new CookieJar();\n\n $jar->addFrom(Url::parse('https://www.crwl.io'), new CookiesCollection([\n new Cookie([\n 'name' => 'foo',\n 'value' => 'one',\n 'domain' => '.www.crwl.io',\n 'expires' => '1745068860',\n 'max-age' => '86400',\n 'secure' => true,\n 'httpOnly' => true,\n 'sameSite' => 'Strict',\n ]),\n new Cookie([\n 'name' => 'bar',\n 'value' => 'two',\n 'domain' => '.www.crwl.io',\n 'expires' => '1729603260.5272',\n 'path' => '/bar',\n ]),\n new Cookie([\n 'name' => 'baz',\n 'value' => 'three',\n 'domain' => '.www.crwl.io',\n 'expires' => '1764076860.878',\n ]),\n ]));\n\n $allCookiesForDomain = $jar->allByDomain('crwl.io');\n\n expect($allCookiesForDomain)->toHaveCount(3)\n ->and($allCookiesForDomain['foo']->expires()?->dateTime()->format('Y-m-d H:i'))->toBe('2025-04-19 13:21')\n ->and($allCookiesForDomain['foo']->name())->toBe('foo')\n ->and($allCookiesForDomain['foo']->value())->toBe('one')\n ->and($allCookiesForDomain['foo']->domain())->toBe('www.crwl.io')\n ->and($allCookiesForDomain['foo']->maxAge())->toBe(86400)\n ->and($allCookiesForDomain['foo']->path())->toBeNull()\n ->and($allCookiesForDomain['foo']->secure())->toBeTrue()\n ->and($allCookiesForDomain['foo']->httpOnly())->toBeTrue()\n ->and($allCookiesForDomain['foo']->sameSite())->toBe('Strict')\n ->and($allCookiesForDomain['bar']->expires()?->dateTime()->format('Y-m-d H:i'))->toBe('2024-10-22 13:21')\n ->and($allCookiesForDomain['bar']->name())->toBe('bar')\n ->and($allCookiesForDomain['bar']->value())->toBe('two')\n ->and($allCookiesForDomain['bar']->domain())->toBe('www.crwl.io')\n ->and($allCookiesForDomain['bar']->maxAge())->toBeNull()\n ->and($allCookiesForDomain['bar']->path())->toBe('/bar')\n ->and($allCookiesForDomain['bar']->secure())->toBeFalse()\n ->and($allCookiesForDomain['bar']->httpOnly())->toBeFalse()\n ->and($allCookiesForDomain['bar']->sameSite())->toBe('Lax')\n ->and($allCookiesForDomain['baz']->expires()?->dateTime()->format('Y-m-d H:i'))->toBe('2025-11-25 13:21');\n});\n\nit('adds all cookies from a response', function () {\n $jar = new CookieJar();\n\n $jar->addFrom(Url::parse('https://www.otsch.codes'), new Response(200, [\n 'Set-Cookie' => ['cook13=v4lu3; Secure', 'anotherCookie=andItsValue', 'oneMoreCookie=dough'],\n ]));\n\n $allCookiesForDomain = $jar->allByDomain('otsch.codes');\n\n expect($allCookiesForDomain)->toHaveCount(3);\n});\n\nit('returns all cookies that should be sent to a url', function () {\n $jar = new CookieJar();\n\n $jar->addFrom(Url::parse('https://www.otsch.codes/blog'), new Response(200, [\n 'Set-Cookie' => [\n 'cook13=v4lu3; Secure',\n '__Host-anotherCookie=andItsValue; Secure; Path=/',\n 'oneMoreCookie=dough',\n ],\n ]));\n\n expect($jar->getFor('https://www.otsch.codes/contact'))->toHaveCount(3)\n ->and($jar->getFor('https://jobs.otsch.codes/index'))->toHaveCount(2)\n ->and($jar->getFor('http://games.otsch.codes'))->toHaveCount(1);\n});\n" }, { "path": "tests/Loader/Http/Cookies/CookieTest.php", "content": "toBeInstanceOf(Cookie::class);\n});\n\ntest('It can be created with received from url as Url object', function () {\n $cookie = new Cookie(Url::parse('https://www.crwlr.software/packages'), 'cookieName=cookieValue');\n expect($cookie)->toBeInstanceOf(Cookie::class);\n});\n\ntest('It provides the received from url as PSR-7 Uri object', function () {\n $cookie = new Cookie('https://www.crwlr.software/contact', 'cookieName=cookieValue');\n expect($cookie->receivedFromUrl())->toBeInstanceOf(UriInterface::class);\n});\n\ntest('It must at least have a name and value', function () {\n new Cookie(Url::parse('https://www.crwlr.software/packages'), 'cookieNameWithoutValueIsInvalid');\n})->throws(InvalidCookieException::class);\n\ntest('It parses the name and value of the cookie', function () {\n $cookie = new Cookie('https://www.crwlr.software/blog', 'crwlrsoftware_session=foobar');\n expect($cookie->name())->toBe('crwlrsoftware_session');\n expect($cookie->value())->toBe('foobar');\n});\n\ntest('The __toString() method returns name=value (only)', function () {\n $cookie = new Cookie('https://www.crwl.io', '__Secure-cook13N4m3=c00k1eV4lu3; Secure; Path=/');\n expect($cookie->__toString())->toBe('__Secure-cook13N4m3=c00k1eV4lu3');\n});\n\ntest('It automatically sets the domain based on the received from url when no attribute is included', function () {\n // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie\n // If omitted, this attribute defaults to the host of the current document URL, not including subdomains.\n $cookie = new Cookie('https://www.otsch.codes/blog', 'otschcodes_session=cook13');\n expect($cookie->domain())->toBe('otsch.codes');\n});\n\ntest('It parses an expires attribute when included', function () {\n $cookie = new Cookie(\n 'https://www.otsch.codes/blog',\n 'otschcodes_session=cook13; Expires=Wed, 23-Feb-2022 10:13:41 GMT',\n );\n expect($cookie->expires())->toBeInstanceOf(Date::class);\n expect($cookie->expires()->dateTime()->format('Y-m-d H:i'))->toBe('2022-02-23 10:13'); // @phpstan-ignore-line\n});\n\ntest('It parses a maxAge attribute when included', function () {\n $cookie = new Cookie('https://www.otsch.codes/blog', 'otschcodes_session=cook13; Max-Age=600');\n expect($cookie->maxAge())->toBeInt();\n expect($cookie->maxAge())->toBe(600);\n});\n\ntest('It parses a domain attribute when included', function () {\n $cookie = new Cookie('https://sub.domain.example.com/foobar', 'fookie=cook13; domain=domain.example.com');\n expect($cookie->domain())->toBe('domain.example.com');\n});\n\ntest('It\\'s not allowed to set a different domain than the one of the document url it was received from', function () {\n new Cookie('https://sub.domain.example.com/foobar', 'fookie=cook13; domain=crwl.io');\n})->throws(InvalidCookieException::class);\n\ntest('It\\'s not allowed to set a subdomain that is not included in the document url it was received from', function () {\n new Cookie('https://sub.domain.example.com/foobar', 'fookie=cook13; domain=foo.example.com');\n})->throws(InvalidCookieException::class);\n\ntest('When domain attribute is defined with leading dot, it\\'s ignored', function () {\n $cookie = new Cookie('https://sub.domain.example.com/', 'fookie=cook13; domain=.domain.example.com');\n expect($cookie->domain())->toBe('domain.example.com');\n});\n\ntest('It parses a path attribute when included', function () {\n $cookie = new Cookie('https://sub.domain.example.com/foobar', 'co=asdf2345; path=/foobar');\n expect($cookie->path())->toBe('/foobar');\n});\n\ntest('It parses a secure attribute when included', function () {\n $cookie = new Cookie('https://sub.domain.example.com/foobar', 'co=asdf2345; Secure');\n expect($cookie->secure())->toBeTrue();\n});\n\ntest(\n 'It throws an exception when secure attribute is sent but url where it was received from is not on https',\n function () {\n new Cookie('http://www.example.io/foobar', 'eggs=ample; Secure');\n },\n)->throws(InvalidCookieException::class);\n\ntest('It parses a SameSite attribute when included', function ($value) {\n $cookie = new Cookie('https://www.example.io/foobar', 'eggs=ample; SameSite=' . $value);\n expect($cookie->sameSite())->toBe($value);\n})->with(['Strict', 'Lax', 'None']);\n\ntest('It throws an error when an unknown value is sent for the SameSite attribute', function () {\n new Cookie('https://www.example.io/foobar', 'eggs=ample; SameSite=Foo');\n})->throws(InvalidCookieException::class);\n\ntest('It parses an HttpOnly attribute when included', function () {\n $cookie = new Cookie('https://jobs.foo.bar/', 'csrf=asdfjkloe123; HttpOnly');\n expect($cookie->httpOnly())->toBeTrue();\n});\n\ntest('It\\'s possible to set multiple attributes', function () {\n $cookie = new Cookie(\n 'https://www.crwl.io',\n '__Secure-cook13N4m3=c00k1eV4lu3; Expires=Wed, 23-Feb-2022 10:13:41 GMT; Secure; Path=/foo',\n );\n expect($cookie->secure())->toBeTrue();\n expect($cookie->expires()?->dateTime()->format('d.m.Y H:i'))->toBe('23.02.2022 10:13');\n expect($cookie->path())->toBe('/foo');\n});\n\ntest(\n 'It throws an Exception when cookie name is prefixed with __Secure- or __Host- and not sent via https',\n function ($prefix) {\n new Cookie('http://example.com', $prefix . 'Abc=defg123; Secure');\n },\n)->with(['__Secure-', '__Host-'])->throws(InvalidCookieException::class);\n\ntest(\n 'It throws an Exception when cookie name is prefixed with __Secure- or __Host- and Secure flag is not included',\n function ($prefix) {\n new Cookie('https://example.com', $prefix . 'Abc=defg123;');\n },\n)->with(['__Secure-', '__Host-'])->throws(InvalidCookieException::class);\n\ntest('Using __Secure- prefix works when received via https and Secure flag is included', function () {\n $cookie = new Cookie('https://www.crwl.io', '__Secure-Foo=bar123; Secure');\n expect($cookie->hasSecurePrefix())->toBeTrue();\n});\n\ntest('It throws an Exception when __Host- prefix used and Domain attribute included', function () {\n new Cookie('https://www.crwlr.software/', '__Host-Foo=bar123; Secure; Domain=www.crwlr.software; Path=/');\n})->throws(InvalidCookieException::class);\n\ntest('It throws an Exception when __Host- prefix used and Path attribute is not included', function () {\n new Cookie('https://www.crwlr.software/', '__Host-Foo=bar123; Secure;');\n})->throws(InvalidCookieException::class);\n\ntest('It throws an Exception when __Host- prefix used and Path attribute is not \"/\"', function () {\n new Cookie('https://www.crwlr.software/', '__Host-Foo=bar123; Secure; Path=/foo');\n})->throws(InvalidCookieException::class);\n\ntest('Using __Host- works when everything is valid', function () {\n $cookie = new Cookie('https://www.crwlr.software/', '__Host-Foo=bar123; Secure; Path=/');\n expect($cookie->hasHostPrefix())->toBeTrue();\n});\n\ntest(\n 'It should not be sent to a url when the domain doesn\\'t match',\n function ($receivedFrom, $domainAttribute, $shouldBeSentTo) {\n $cookie = new Cookie($receivedFrom, 'cookie=value' . ($domainAttribute ? '; Domain=' . $domainAttribute : ''));\n expect($cookie->shouldBeSentTo($shouldBeSentTo))->toBeFalse();\n },\n)->with([\n ['https://www.crwlr.software', null, 'https://www.otsch.codes'],\n ['https://www.crwlr.software', 'www.crwlr.software', 'https://jobs.crwlr.software'],\n ['https://www.crwlr.software', 'www.crwlr.software', 'https://crwlr.software'],\n ['https://sub.domain.crwlr.software', 'sub.domain.crwlr.software', 'https://sab.domain.crwlr.software'],\n ['https://sub.domain.crwlr.software', 'sub.domain.crwlr.software', 'https://domain.crwlr.software'],\n]);\n\ntest('It should be sent to a url when the domain matches', function ($receivedFrom, $domainAttribute, $shouldBeSentTo) {\n $cookie = new Cookie($receivedFrom, 'cookie=value' . ($domainAttribute ? '; Domain=' . $domainAttribute : ''));\n expect($cookie->shouldBeSentTo($shouldBeSentTo))->toBeTrue();\n})->with([\n ['https://www.crwlr.software', null, 'https://www.crwlr.software'],\n ['https://www.crwlr.software', null, 'https://crwlr.software'],\n ['https://www.crwlr.software', null, 'https://anything.crwlr.software'],\n ['https://sub.domain.crwlr.software', 'domain.crwlr.software', 'https://domain.crwlr.software'],\n ['https://sub.domain.crwlr.software', 'domain.crwlr.software', 'https://sab.domain.crwlr.software'],\n]);\n\ntest(\n 'It should not be sent to a url when it has a __Host- prefix and hosts don\\'t match exactly',\n function ($receivedFrom, $shouldBeSentTo) {\n $cookie = new Cookie($receivedFrom, '__Host-cookie=value; Secure; Path=/');\n expect($cookie->shouldBeSentTo($shouldBeSentTo))->toBeFalse();\n },\n)->with([\n ['https://www.crwlr.software', 'https://jobs.crwlr.software'],\n ['https://sub.domain.crwlr.software', 'https://domain.crwlr.software'],\n ['https://subdomain.crwlr.software', 'https://sabdomain.crwlr.software'],\n]);\n\ntest('It should not be sent to non https url when secure flag is included', function () {\n $cookie = new Cookie('https://www.crwl.io', 'cookie=value; Secure');\n expect($cookie->shouldBeSentTo('http://www.crwl.io'))->toBeFalse();\n});\n\ntest('It should be sent to https url when secure flag is included', function () {\n $cookie = new Cookie('https://www.crwl.io', 'cookie=value; Secure');\n expect($cookie->shouldBeSentTo('https://www.crwl.io'))->toBeTrue();\n});\n\ntest('It should be sent to non https url when secure flag is included but host is localhost', function ($host) {\n $cookie = new Cookie('https://' . $host, 'cookie=value; Secure');\n expect($cookie->shouldBeSentTo('http://' . $host))->toBeTrue();\n})->with(['localhost', '127.0.0.1']);\n\ntest(\n 'It should not be sent to urls where the path doesn\\'t match the sent path attribute',\n function ($path, $shouldBeSentTo) {\n $cookie = new Cookie('https://www.crwlr.software', 'cookie=value; Path=' . $path);\n expect($cookie->shouldBeSentTo('https://www.crwlr.software' . $shouldBeSentTo))->toBeFalse();\n },\n)->with([\n ['/foo', '/bar'],\n ['/foo', '/foobar'],\n ['/foo', '/'],\n ['/foo', '/bar/foo'],\n]);\n\ntest(\n 'It should be sent to urls where the path does match the sent path attribute',\n function ($path, $shouldBeSentTo) {\n $cookie = new Cookie('https://www.crwlr.software', 'cookie=value; Path=' . $path);\n expect($cookie->shouldBeSentTo('https://www.crwlr.software' . $shouldBeSentTo))->toBeTrue();\n },\n)->with([\n ['/', '/anything'],\n ['/foo', '/foo'],\n ['/foo', '/foo/something'],\n ['/foo', '/foo/some/thing'],\n]);\n\ntest('It should not be sent when already expired', function () {\n $now = new DateTime('now', new DateTimeZone('GMT'));\n $now = $now->sub(new DateInterval('PT1S'));\n $cookie = new Cookie(\n 'https://www.crwlr.software',\n 'cookie=value; Expires=' . $now->format(DateTimeInterface::COOKIE),\n );\n expect($cookie->shouldBeSentTo('https://www.crwlr.software'))->toBeFalse();\n});\n\ntest('It should be sent when date of expires attribute is not reached', function () {\n $now = new DateTime('now', new DateTimeZone('GMT'));\n $now = $now->add(new DateInterval('PT5S'));\n $cookie = new Cookie(\n 'https://www.crwlr.software',\n 'cookie=value; Expires=' . $now->format(DateTimeInterface::COOKIE),\n );\n expect($cookie->shouldBeSentTo('https://www.crwlr.software'))->toBeTrue();\n});\n\ntest('It should not be sent when maxAge attribute is already reached', function () {\n $cookie = new Cookie('https://www.crwlr.software', 'cookie=value; Max-Age=1');\n\n expect($cookie->shouldBeSentTo('https://www.crwlr.software'))->toBeTrue();\n\n invade($cookie)->receivedAtTimestamp -= 2; // instead of sleep, manipulate the timestamp when it was received.\n\n expect($cookie->shouldBeSentTo('https://www.crwlr.software'))->toBeFalse();\n});\n\ntest('It is immediately expired when the max-age attribute is zero or negative', function ($maxAgeValue) {\n $cookie = new Cookie('https://www.crwlr.software', 'cookie=value; Max-Age=' . $maxAgeValue);\n expect($cookie->shouldBeSentTo('https://www.crwlr.software'))->toBeFalse();\n})->with([0, -1, -5, -1000]);\n\ntest('It should be sent when maxAge attribute is not yet reached', function () {\n $cookie = new Cookie('https://www.crwlr.software', 'cookie=value; Max-Age=1');\n expect($cookie->shouldBeSentTo('https://www.crwlr.software'))->toBeTrue();\n});\n" }, { "path": "tests/Loader/Http/Cookies/DateTest.php", "content": "toBeInstanceOf(Date::class);\n\n expect($date->dateTime()->format('Y-m-d H:i:s'))->toBe('2022-02-22 16:04:55');\n});\n\ntest('It gets the timezone right', function () {\n $date = new Date('Tue, 22-Feb-2022 20:04:29 GMT');\n\n expect(\n $date->dateTime()->setTimezone(new DateTimeZone('Europe/Vienna'))->format('d.m.Y H:i:s'),\n )->toBe('22.02.2022 21:04:29');\n});\n\ntest('It also works without the dashes between d-M-Y in the format', function () {\n $date = new Date('Wed, 05 Jul 2023 15:19:55 GMT');\n\n expect(\n $date->dateTime()->setTimezone(new DateTimeZone('Europe/Vienna'))->format('d.m.Y H:i:s'),\n )->toBe('05.07.2023 17:19:55');\n});\n" }, { "path": "tests/Loader/Http/HeadlessBrowserLoaderHelperTest.php", "content": "shouldReceive('createBrowser');\n\n if ($createBrowserArgsExpectationCallback) {\n $createBrowserExpectation->withArgs($createBrowserArgsExpectationCallback);\n }\n\n $createBrowserExpectation->andReturn($browserMock);\n\n $pageMock = Mockery::mock(Page::class);\n\n $browserMock->shouldReceive('createPage')->andReturn($pageMock);\n\n if ($browserMockCallback) {\n $browserMockCallback($browserMock);\n }\n\n $sessionMock = Mockery::mock(Session::class);\n\n $pageMock->shouldReceive('getSession')->andReturn($sessionMock);\n\n if ($pageSessionMockCallback) {\n $pageSessionMockCallback($sessionMock);\n }\n\n $pageMock->shouldReceive('getCookies')->andReturn(new CookiesCollection([]));\n\n $sessionMock->shouldReceive('once');\n\n $pageNavigationMock = Mockery::mock(PageNavigation::class);\n\n $pageMock->shouldReceive('navigate')->andReturn($pageNavigationMock);\n\n $pageMock->shouldReceive('getHtml')->andReturn('Hello World!');\n\n if ($pageMockCallback) {\n $pageMockCallback($pageMock);\n }\n\n $waitForNavigationCall = $pageNavigationMock->shouldReceive('waitForNavigation');\n\n if ($pageNavigationArgsClosure) {\n $waitForNavigationCall->withArgs($pageNavigationArgsClosure);\n }\n\n return $browserFactoryMock;\n}\n\nit('uses the configured timeout', function () {\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(function (string $event, int $timeout) {\n return $event === Page::LOAD && $timeout === 45_000;\n });\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);\n\n $helper->setTimeout(45_000);\n\n $response = $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n helper_getMinThrottler(),\n cookieJar: new CookieJar(),\n );\n\n expect(Http::getBodyString($response))->toBe('Hello World!');\n});\n\nit('returns the configured timeout', function () {\n $helper = new HeadlessBrowserLoaderHelper();\n\n expect($helper->getTimeout())->toBe(30_000);\n\n $helper->setTimeout(75_000);\n\n expect($helper->getTimeout())->toBe(75_000);\n});\n\nit('waits for the configured browser navigation event', function () {\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(function (string $event, int $timeout) {\n return $event === Page::FIRST_MEANINGFUL_PAINT && $timeout === 57_000;\n });\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);\n\n $helper\n ->waitForNavigationEvent(Page::FIRST_MEANINGFUL_PAINT)\n ->setTimeout(57_000);\n\n $response = $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n helper_getMinThrottler(),\n cookieJar: new CookieJar(),\n );\n\n expect(Http::getBodyString($response))->toBe('Hello World!');\n});\n\nit('uses the correct executable', function () {\n $helper = new HeadlessBrowserLoaderHelper();\n\n $helper->setExecutable('somethingthatdefinitelyisntachromeexecutable');\n\n $invadedHelper = invade($helper);\n\n $exception = null;\n\n try {\n $invadedHelper->getBrowser(new Request('GET', 'https://www.example.com/foo'));\n } catch (Exception $exception) {\n }\n\n expect($exception)->not->toBeNull();\n\n $chromeExecutable = (new AutoDiscover())->guessChromeBinaryPath();\n\n $helper = new HeadlessBrowserLoaderHelper();\n\n $helper->setExecutable($chromeExecutable);\n\n $invadedHelper = invade($helper);\n\n $invadedHelper->getBrowser(new Request('GET', 'https://www.example.com/foo'));\n\n $browserFactory = $invadedHelper->browserFactory;\n\n expect($browserFactory)->toBeInstanceOf(BrowserFactory::class);\n\n /** @var BrowserFactory $browserFactory */\n\n $invadedBrowserFactory = invade($browserFactory);\n\n expect($invadedBrowserFactory->chromeBinary)->toBe($chromeExecutable);\n});\n\nit('calls the temporary post navigate hooks once', function () {\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(\n pageMockCallback: function (Mockery\\MockInterface $pageMock) {\n $pageMock->shouldReceive('assertNotClosed')->once();\n },\n );\n\n $logger = new DummyLogger();\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock, $logger);\n\n $helper->setTempPostNavigateHooks([\n function (Page $page, LoggerInterface $logger) {\n $logger->info('hook 1 called');\n },\n function (Page $page, LoggerInterface $logger) {\n $logger->info('hook 2 called');\n },\n function (Page $page, LoggerInterface $logger) {\n $logger->info('hook 3 called');\n },\n ]);\n\n $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n helper_getMinThrottler(),\n cookieJar: new CookieJar(),\n );\n\n expect($logger->messages)->toHaveCount(3)\n ->and($logger->messages[0]['message'])->toBe('hook 1 called')\n ->and($logger->messages[1]['message'])->toBe('hook 2 called')\n ->and($logger->messages[2]['message'])->toBe('hook 3 called');\n\n $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n helper_getMinThrottler(),\n cookieJar: new CookieJar(),\n );\n\n expect($logger->messages)->toHaveCount(3);\n});\n\nit(\n 'passes the script source provided via the setPageInitScript() method, to the ' .\n 'ProcessAwareBrowser::setPagePreScript() method',\n function () {\n $script = 'console.log(\\'hey\\');';\n\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(\n browserMockCallback: function (Mockery\\MockInterface $browser) use ($script) {\n $browser\n ->shouldReceive('setPagePreScript')\n ->once()\n ->with($script);\n },\n );\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);\n\n $helper->setPageInitScript($script);\n\n $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/bar'),\n helper_getMinThrottler(),\n cookieJar: new CookieJar(),\n );\n },\n);\n\nit('does not call the ProcessAwareBrowser::setPagePreScript() when no page init script was defined', function () {\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(\n browserMockCallback: function (Mockery\\MockInterface $browser) {\n $browser->shouldNotReceive('setPagePreScript');\n },\n );\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);\n\n $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/bar'),\n helper_getMinThrottler(),\n cookieJar: new CookieJar(),\n );\n});\n\nit(\n 'passes the userAgent option when Request contains a user-agent header and useNativeUserAgent() was not called',\n function () {\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(\n createBrowserArgsExpectationCallback: function ($options) {\n return array_key_exists('userAgent', $options) && $options['userAgent'] === 'MyBot';\n },\n );\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);\n\n $response = $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/bar', ['user-agent' => ['MyBot']]),\n helper_getMinThrottler(),\n cookieJar: new CookieJar(),\n );\n\n expect(Http::getBodyString($response))->toBe('Hello World!');\n },\n);\n\nit(\n 'does not pass the userAgent option when Request contains a user-agent header and useNativeUserAgent() was called',\n function () {\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(\n createBrowserArgsExpectationCallback: function ($options) {\n return !array_key_exists('userAgent', $options);\n },\n );\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);\n\n $helper->useNativeUserAgent();\n\n $response = $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/bar', ['user-agent' => ['MyBot']]),\n helper_getMinThrottler(),\n cookieJar: new CookieJar(),\n );\n\n expect(Http::getBodyString($response))->toBe('Hello World!');\n },\n);\n\nit('clears the browsers cookies when no cookie jar is provided', function () {\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(\n pageSessionMockCallback: function (Mockery\\MockInterface $mock) {\n $mock\n ->shouldReceive('sendMessageSync')\n ->once()\n ->withArgs(function (Message $message) {\n return $message->getMethod() === 'Network.clearBrowserCookies';\n });\n },\n );\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);\n\n $response = $helper->navigateToPageAndGetRespondedRequest(\n new Request('GET', 'https://www.example.com/yolo', ['user-agent' => ['MyBot']]),\n helper_getMinThrottler(),\n );\n\n expect(Http::getBodyString($response))->toBe('Hello World!');\n});\n\nit('reuses a previously opened page', function () {\n $browserFactoryMock = helper_setUpHeadlessChromeMocks(\n pageMockCallback: function (Mockery\\MockInterface $pageMock) {\n $pageMock->shouldReceive('assertNotClosed')->twice();\n },\n );\n\n $helper = new HeadlessBrowserLoaderHelper($browserFactoryMock);\n\n $t = helper_getMinThrottler();\n\n $c = new CookieJar();\n\n $helper->navigateToPageAndGetRespondedRequest(new Request('GET', 'https://www.example.com/foo'), $t, null, $c);\n\n $helper->navigateToPageAndGetRespondedRequest(new Request('GET', 'https://www.example.com/bar'), $t, null, $c);\n\n $helper->navigateToPageAndGetRespondedRequest(new Request('GET', 'https://www.example.com/baz'), $t, null, $c);\n});\n" }, { "path": "tests/Loader/Http/HttpLoaderPolitenessTest.php", "content": "shouldReceive('sendRequest')->once()->andReturnUsing(function (RequestInterface $request) {\n $response = new Response(200, [], $request->getUri()->__toString() . ' response');\n\n helper_wait300ms();\n\n return $response;\n });\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response(200));\n\n $loader = new HttpLoader(new UserAgent('SomeUserAgent'), $httpClient);\n\n $loader->{$loadingMethod}('https://www.example.com/foo');\n\n $firstResponse = microtime(true);\n\n $loader->{$loadingMethod}('https://www.example.com/bar');\n\n $secondResponse = microtime(true);\n\n $diff = $secondResponse - $firstResponse;\n\n expect($diff)->toBeGreaterThan(0.3)\n ->and($diff)->toBeLessThan(0.62);\n})->with(['load', 'loadOrFail']);\n\nit('also throttles requests using the headless browser', function ($loadingMethod) {\n $browserMock = Mockery::mock(Browser::class);\n\n $pageMock = Mockery::mock(Page::class);\n\n $sessionMock = Mockery::mock(Session::class);\n\n $sessionMock->shouldReceive('once');\n\n $pageMock->shouldReceive('assertNotClosed')->once();\n\n $pageMock->shouldReceive('getSession')->andReturn($sessionMock);\n\n $pageNavigationMock = Mockery::mock(PageNavigation::class);\n\n $pageNavigationMock->shouldReceive('waitForNavigation');\n\n $pageMock\n ->shouldReceive('navigate')\n ->once()\n ->andReturnUsing(function (string $url) use ($pageNavigationMock) {\n helper_wait300ms();\n\n return $pageNavigationMock;\n });\n\n $pageMock->shouldReceive('getCookies')->andReturn(new CookiesCollection());\n\n $pageMock->shouldReceive('getHtml')->andReturn('foo');\n\n $browserMock->shouldReceive('createPage')->andReturn($pageMock);\n\n $browserHelperMock = Mockery::mock(HeadlessBrowserLoaderHelper::class)->makePartial();\n\n $browserHelperMock\n ->shouldAllowMockingProtectedMethods()\n ->shouldReceive('getBrowser')\n ->andReturn($browserMock);\n\n $loader = new HttpLoader(new UserAgent('SomeUserAgent'));\n\n invade($loader)->browserHelper = $browserHelperMock;\n\n $loader->useHeadlessBrowser();\n\n $loader->{$loadingMethod}('https://www.example.com/foo');\n\n $pageMock->shouldReceive('navigate')->andReturn($pageNavigationMock);\n\n $pageMock->shouldReceive('getCookies')->andReturn(new CookiesCollection());\n\n $firstResponse = microtime(true);\n\n $loader->{$loadingMethod}('https://www.example.com/bar');\n\n $secondResponse = microtime(true);\n\n $diff = $secondResponse - $firstResponse;\n\n expect($diff)->toBeGreaterThan(0.3)\n ->and($diff)->toBeLessThan(0.62);\n})->with(['load', 'loadOrFail']);\n\nit('does not throttle requests to different domains', function ($loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturnUsing(function (RequestInterface $request) {\n $response = new Response(200, [], $request->getUri()->__toString() . ' response');\n\n helper_wait300ms();\n\n return $response;\n });\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response(200));\n\n $loader = new HttpLoader(new UserAgent('SomeUserAgent'), $httpClient);\n\n $loader->{$loadingMethod}('https://www.example.com/foo');\n\n $firstResponse = microtime(true);\n\n $loader->{$loadingMethod}('https://www.example.org/bar');\n\n $secondResponse = microtime(true);\n\n $diff = $secondResponse - $firstResponse;\n\n expect($diff)->toBeLessThan(0.001);\n})->with(['load', 'loadOrFail']);\n\nit('respects rules from robots.txt from load method', function () {\n $client = Mockery::mock(ClientInterface::class);\n\n $client->shouldReceive('sendRequest')->once()->andReturn(helper_getDummyRobotsTxtResponse());\n\n $loader = new HttpLoader(new BotUserAgent('FooBot'), $client);\n\n $response = $loader->load('https://www.crwlr.software/secret');\n\n expect($response)->toBeNull();\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->toContain('Loaded https://www.crwlr.software/robots.txt');\n\n expect($output)->toContain('Crawler is not allowed to load https://www.crwlr.software/secret');\n});\n\nit('respects rules from robots.txt from loadOrFail method', function () {\n $client = Mockery::mock(ClientInterface::class);\n\n $client->shouldReceive('sendRequest')->once()->andReturn(helper_getDummyRobotsTxtResponse());\n\n $loader = new HttpLoader(new BotUserAgent('FooBot'), $client);\n\n $loader->loadOrFail('https://www.crwlr.software/secret');\n})->throws(LoadingException::class);\n\nit('does not respect rules from robots.txt when user agent isn\\'t instance of BotUserAgent', function () {\n $client = Mockery::mock(ClientInterface::class);\n\n $client->shouldReceive('sendRequest')->once()->andReturn(helper_getDummyRobotsTxtResponse());\n\n $loader = new HttpLoader(new UserAgent('FooBot'), $client);\n\n $response = $loader->load('https://www.crwlr.software/secret');\n\n expect($response)->toBeInstanceOf(RespondedRequest::class);\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->not()->toContain('Loaded https://www.crwlr.software/robots.txt');\n\n expect($output)->not()->toContain('Crawler is not allowed to load https://www.crwlr.software/secret');\n});\n" }, { "path": "tests/Loader/Http/HttpLoaderTest.php", "content": "shouldReceive('sendRequest')->twice()->andReturn(new Response());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->load('https://www.crwlr.software');\n\n $httpLoader->loadOrFail('https://www.crwlr.software');\n});\n\nit('fails and logs an error when invoked with a relative reference URI', function () {\n $logger = new DummyLogger();\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);\n\n $httpLoader->load('/foo');\n\n expect($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toBe(\n 'Invalid input URL: /foo - The URI is a relative reference and therefore can\\'t be loaded.',\n );\n});\n\nit('fails and throws an exception when loadOrFail() is called with a relative reference URI', function () {\n $logger = new DummyLogger();\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);\n\n $httpLoader->loadOrFail('/foo');\n})->throws(InvalidArgumentException::class);\n\nit('accepts RequestInterface as argument to load', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->twice()->andReturn(new Response());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->load(new Request('GET', 'https://www.crwlr.software'));\n\n $httpLoader->loadOrFail(new Request('GET', 'https://www.crwlr.software'));\n});\n\nit('fails and logs an error when invoked with a RequestInterface object having a relative reference URI', function () {\n $logger = new DummyLogger();\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);\n\n $httpLoader->load(new Request('GET', '/foo'));\n\n expect($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toBe(\n 'Invalid input URL: /foo - The URI is a relative reference and therefore can\\'t be loaded.',\n );\n});\n\nit(\n 'fails and throws an exception when loadOrFail() is called with a RequestInterface object having a relative ' .\n 'reference URI',\n function () {\n $logger = new DummyLogger();\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);\n\n $httpLoader->loadOrFail(new Request('GET', '/foo'));\n },\n)->throws(InvalidArgumentException::class);\n\nit(\n 'calls the before and after load hooks regardless whether the response was successful or not',\n function ($responseStatusCode) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n if ($responseStatusCode === 300) {\n $httpClient->shouldReceive('sendRequest')\n ->twice()\n ->andReturn(new Response($responseStatusCode), new Response(200));\n } else {\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response($responseStatusCode));\n }\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $beforeLoadWasCalled = false;\n\n $httpLoader->beforeLoad(function () use (&$beforeLoadWasCalled) {\n $beforeLoadWasCalled = true;\n });\n\n $afterLoadWasCalled = false;\n\n $httpLoader->afterLoad(function () use (&$afterLoadWasCalled) {\n $afterLoadWasCalled = true;\n });\n\n $httpLoader->load('https://www.otsch.codes');\n\n expect($beforeLoadWasCalled)->toBeTrue()\n ->and($afterLoadWasCalled)->toBeTrue();\n },\n)->with([\n [100],\n [200],\n [300],\n [400],\n [500],\n]);\n\nit('calls the onSuccess hook on a successful response', function ($responseStatusCode) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->twice()->andReturn(new Response($responseStatusCode));\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $onSuccessWasCalled = false;\n\n $httpLoader->onSuccess(function () use (&$onSuccessWasCalled) {\n $onSuccessWasCalled = true;\n });\n\n $httpLoader->load('https://www.otsch.codes');\n\n expect($onSuccessWasCalled)->toBeTrue();\n\n $onSuccessWasCalled = false;\n\n $httpLoader->loadOrFail('https://www.otsch.codes');\n\n expect($onSuccessWasCalled)->toBeTrue();\n})->with([\n [200],\n [201],\n [202],\n]);\n\nit('calls the onError hook on a failed request', function ($responseStatusCode) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response($responseStatusCode));\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $onErrorWasCalled = false;\n\n $httpLoader->onError(function () use (&$onErrorWasCalled) {\n $onErrorWasCalled = true;\n });\n\n $httpLoader->load('https://www.otsch.codes');\n\n expect($onErrorWasCalled)->toBeTrue();\n})->with([\n [400],\n [404],\n [422],\n [500],\n]);\n\nit('calls the onCacheHit hook when a response for the request was found in the cache', function (string $loadMethod) {\n $cache = new FileCache(helper_cachedir());\n\n $userAgent = helper_nonBotUserAgent();\n\n $respondedRequest = new RespondedRequest(\n new Request(\n 'GET',\n 'https://www.example.com/foo',\n ['Host' => ['www.example.com'], 'User-Agent' => [(string) $userAgent]],\n ),\n new Response(body: 'Hello World!'),\n );\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $httpLoader = new HttpLoader($userAgent);\n\n $httpLoader->setCache($cache);\n\n $onCacheHitWasCalled = false;\n\n $httpLoader->onCacheHit(function () use (&$onCacheHitWasCalled) {\n $onCacheHitWasCalled = true;\n });\n\n $response = $httpLoader->{$loadMethod}('https://www.example.com/foo');\n\n /** @var RespondedRequest $response */\n\n expect($onCacheHitWasCalled)->toBeTrue()\n ->and($response->isServedFromCache())->toBeTrue();\n})->with(['load', 'loadOrFail']);\n\nit('throws an Exception when request fails in loadOrFail method', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response(400));\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $onErrorWasCalled = false;\n\n $httpLoader->onError(function () use (&$onErrorWasCalled) {\n $onErrorWasCalled = true;\n });\n\n try {\n $httpLoader->loadOrFail('https://www.otsch.codes');\n } catch (LoadingException $exception) {\n expect($exception)->toBeInstanceOf(LoadingException::class);\n }\n\n expect($onErrorWasCalled)->toBeFalse();\n});\n\ntest('You can implement logic to disallow certain request', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response());\n\n $httpLoader = new class (new BotUserAgent('Foo'), $httpClient) extends HttpLoader {\n public function isAllowedToBeLoaded(UriInterface $uri, bool $throwsException = false): bool\n {\n return $uri->__toString() === 'https://www.example.com/foo';\n }\n };\n\n $response = $httpLoader->load('https://www.example.com/foo');\n\n expect($response)->toBeInstanceOf(RespondedRequest::class);\n\n $response = $httpLoader->load('https://www.example.com/bar');\n\n expect($response)->toBeNull();\n});\n\ntest(\n 'The isAllowedToBeLoaded method is called with argument throwsException true when called from loadOrFail',\n function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response());\n\n $httpLoader = new class (new BotUserAgent('Foo'), $httpClient) extends HttpLoader {\n public function isAllowedToBeLoaded(UriInterface $uri, bool $throwsException = false): bool\n {\n if ($throwsException) {\n throw new LoadingException('Fail to load ' . $uri->__toString());\n }\n\n return $uri->__toString() === 'https://www.example.com';\n }\n };\n\n $httpLoader->load('https://www.example.com');\n\n try {\n $httpLoader->loadOrFail('https://www.example.com');\n } catch (LoadingException $exception) {\n expect($exception)->toBeInstanceOf(LoadingException::class);\n }\n },\n);\n\nit('automatically handles redirects', function (string $loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')\n ->twice()\n ->andReturn(\n new Response(301, ['Location' => 'https://www.redirect.com']),\n new Response(200, [], 'YES'),\n );\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.crwlr.software/packages');\n\n /** @var RespondedRequest $respondedRequest */\n expect($respondedRequest->requestedUri())->toBe('https://www.crwlr.software/packages')\n ->and($respondedRequest->effectiveUri())->toBe('https://www.redirect.com')\n ->and($respondedRequest->response->getBody()->getContents())->toBe('YES');\n})->with(['load', 'loadOrFail']);\n\nit('calls request start and end tracking methods', function (string $loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response(200));\n\n $throttler = new class extends Throttler {\n public function trackRequestStartFor(UriInterface $url): void\n {\n echo 'Track request start ' . $url . PHP_EOL;\n\n parent::trackRequestStartFor($url);\n }\n\n public function trackRequestEndFor(UriInterface $url): void\n {\n echo 'Track request end ' . $url . PHP_EOL;\n\n parent::trackRequestEndFor($url);\n }\n };\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient, throttler: $throttler);\n\n $httpLoader->{$loadingMethod}('https://www.twitter.com');\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->toContain('Track request start https://www.twitter.com')\n ->and($output)->toContain('Track request end https://www.twitter.com');\n})->with(['load', 'loadOrFail']);\n\nit(\n 'calls trackRequestEndFor only once and with the original request URL when there is a redirect',\n function (string $loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->once()\n ->withArgs(function (Request $request) {\n return (string) $request->getUri() === 'https://www.example.com/foo';\n })\n ->andReturn(new Response(301, ['Location' => 'https://www.example.com/bar']));\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->once()\n ->withArgs(function (Request $request) {\n return (string) $request->getUri() === 'https://www.example.com/bar';\n })\n ->andReturn(new Response(200));\n\n $throttler = new class extends Throttler {\n public function trackRequestEndFor(UriInterface $url): void\n {\n echo 'Track request end ' . $url . PHP_EOL;\n\n parent::trackRequestEndFor($url);\n }\n };\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient, throttler: $throttler);\n\n $httpLoader->{$loadingMethod}('https://www.example.com/foo');\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->toContain('Track request end https://www.example.com/foo')\n ->and(count(explode('Track request end', $output)))->toBe(2);\n },\n)->with(['load', 'loadOrFail']);\n\nit('automatically logs loading success message', function ($loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->{$loadingMethod}(new Request('GET', 'https://phpstan.org/'));\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->toContain('Loaded https://phpstan.org/');\n})->with(['load', 'loadOrFail']);\n\nit('automatically logs loading error message in normal load method', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response(500));\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->load(new Request('GET', 'https://phpstan.org/'));\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->toContain('Failed to load https://phpstan.org/');\n});\n\nit('automatically adds the User-Agent header before sending', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')\n ->once()\n ->withArgs(function ($request) {\n return str_contains($request->getHeaderLine('User-Agent'), 'FooBot');\n })\n ->andReturn(new Response());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->load('https://www.facebook.com');\n});\n\nit('tries to get responses from cache', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldNotReceive('sendRequest');\n\n $cache = Mockery::mock(CacheInterface::class);\n\n $cache->shouldReceive('has')->once()->andReturn(true);\n\n $cache->shouldReceive('get')\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', '/'), new Response()));\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader->load('https://www.facebook.com');\n});\n\ntest(\n 'when a response is served from cache, the RespondedRequest::isServedFromCache() method returns true,',\n function (string $loadMethod) {\n $cache = new FileCache(helper_cachedir());\n\n $userAgent = helper_nonBotUserAgent();\n\n $respondedRequest = new RespondedRequest(\n new Request(\n 'GET',\n 'https://www.example.com/bar',\n ['Host' => ['www.example.com'], 'User-Agent' => [(string) $userAgent]],\n ),\n new Response(body: 'Hi!'),\n );\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $clientMock = Mockery::mock(Client::class);\n\n $clientMock\n ->shouldReceive('sendRequest')\n ->once()\n ->withArgs(function (Request $request) {\n return (string) $request->getUri() === 'https://www.example.com/foo';\n })\n ->andReturn(new Response(body: 'Hi!'));\n\n $httpLoader = (new HttpLoader($userAgent, $clientMock))->setCache($cache);\n\n $response = $httpLoader->{$loadMethod}('https://www.example.com/foo');\n\n /** @var RespondedRequest $response */\n\n expect($response->isServedFromCache())->toBeFalse();\n\n $response = $httpLoader->{$loadMethod}('https://www.example.com/bar');\n\n /** @var RespondedRequest $response */\n\n expect($response->isServedFromCache())->toBeTrue();\n },\n)->with(['load', 'loadOrFail']);\n\nit(\n 'does not serve a request from the cache, when skipCacheForNextRequest() was called',\n function (string $loadMethod) {\n $cache = new FileCache(helper_cachedir());\n\n $userAgent = helper_nonBotUserAgent();\n\n $respondedRequest = new RespondedRequest(\n new Request(\n 'GET',\n 'https://www.example.com/blog/posts',\n ['Host' => ['www.example.com'], 'User-Agent' => [(string) $userAgent]],\n ),\n new Response(body: 'previously cached blog posts'),\n );\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $clientMock = Mockery::mock(Client::class);\n\n $clientMock\n ->shouldReceive('sendRequest')\n ->once()\n ->withArgs(function (Request $request) {\n return (string) $request->getUri() === 'https://www.example.com/blog/posts';\n })\n ->andReturn(new Response(body: 'loaded blog posts'));\n\n $httpLoader = (new HttpLoader($userAgent, $clientMock))\n ->setCache($cache)\n ->skipCacheForNextRequest();\n\n $response = $httpLoader->{$loadMethod}('https://www.example.com/blog/posts');\n\n /** @var RespondedRequest $response */\n\n expect($response->isServedFromCache())->toBeFalse()\n ->and(Http::getBodyString($response))->toBe('loaded blog posts');\n\n // Skipping the cache is only effective for loading. It still adds the loaded response to the cache.\n // So on the next request, when not again calling the skip cache method, the cache will return that\n // previously loaded response.\n $response = $httpLoader->{$loadMethod}('https://www.example.com/blog/posts');\n\n expect($response->isServedFromCache())->toBeTrue()\n ->and(Http::getBodyString($response))->toBe('loaded blog posts');\n },\n)->with(['load', 'loadOrFail']);\n\nit('still handles legacy (until v0.7) cached responses', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldNotReceive('sendRequest');\n\n $cache = Mockery::mock(CacheInterface::class);\n\n $cache->shouldReceive('has')->once()->andReturn(true);\n\n $cache->shouldReceive('get')\n ->once()\n ->andReturn([\n 'requestMethod' => 'GET',\n 'requestUri' => 'https://www.example.com/index',\n 'requestHeaders' => ['foo' => ['bar']],\n 'requestBody' => 'requestbody',\n 'effectiveUri' => 'https://www.example.com/home',\n 'responseStatusCode' => 201,\n 'responseHeaders' => ['baz' => ['quz']],\n 'responseBody' => 'responsebody',\n ]);\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $respondedRequest = $httpLoader->load('https://www.example.com/index');\n\n expect($respondedRequest)->toBeInstanceOf(RespondedRequest::class)\n ->and($respondedRequest?->request->getMethod())->toBe('GET')\n ->and($respondedRequest?->requestedUri())->toBe('https://www.example.com/index')\n ->and($respondedRequest?->request->getHeaders())->toHaveKey('foo')\n ->and($respondedRequest?->request->getBody()->getContents())->toBe('requestbody')\n ->and($respondedRequest?->effectiveUri())->toBe('https://www.example.com/home')\n ->and($respondedRequest?->response->getStatusCode())->toBe(201)\n ->and($respondedRequest?->response->getHeaders())->toHaveKey('baz')\n ->and($respondedRequest?->response->getBody()->getContents())->toBe('responsebody');\n});\n\nit('fails when it gets a failed response from cache', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $cache = Mockery::mock(CacheInterface::class);\n\n $cache->shouldReceive('has')->once()->andReturn(true);\n\n $cache->shouldReceive('get')\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', '/'), new Response(404)));\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $onErrorWasCalled = false;\n\n $httpLoader->onError(function () use (&$onErrorWasCalled) {\n $onErrorWasCalled = true;\n });\n\n $httpLoader->load('https://www.facebook.com');\n\n expect($onErrorWasCalled)->toBeTrue();\n});\n\nit('fails when it gets a failed response from cache in loadOrFail', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $cache = Mockery::mock(CacheInterface::class);\n\n $cache->shouldReceive('has')->once()->andReturn(true);\n\n $cache->shouldReceive('get')\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'facebook'), new Response(404)));\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader->loadOrFail('https://www.facebook.com');\n})->throws(LoadingException::class);\n\nit('adds loaded responses to the cache when it has a cache', function ($loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->once()->andReturn(new Response());\n\n $cache = Mockery::mock(CacheInterface::class);\n\n $cache->shouldReceive('has')->once()->andReturn(false);\n\n $cache->shouldReceive('set')->once();\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader->{$loadingMethod}('https://laravel.com/');\n})->with(['load', 'loadOrFail']);\n\ntest(\n 'when a cached response was an error response it retries to load it when retryCachedErrorResponses() was called',\n function (string $loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->twice()\n ->andReturn(new Response(404), new Response(200));\n\n $cache = new FileCache(helper_cachedir());\n\n $httpLoader = helper_getFastLoader(httpClient: $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader->retryCachedErrorResponses();\n\n try {\n $httpLoader->{$loadingMethod}('https://www.example.com/articles/123');\n } catch (Throwable $exception) {\n }\n\n try {\n $httpLoader->{$loadingMethod}('https://www.example.com/articles/123');\n } catch (Throwable $exception) {\n }\n },\n)->with(['load', 'loadOrFail']);\n\ntest('retrying cached error responses can be restricted to only certain response status codes', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->twice()\n ->andReturn(new Response(404), new Response(400));\n\n $cache = new FileCache(helper_cachedir());\n\n $httpLoader = helper_getFastLoader(httpClient: $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader\n ->retryCachedErrorResponses()\n ->only([404, 503]);\n\n $respondedRequest = $httpLoader->load('https://www.example.com/foo');\n\n expect($respondedRequest?->response->getStatusCode())->toBe(404);\n\n $respondedRequest = $httpLoader->load('https://www.example.com/foo');\n\n expect($respondedRequest?->response->getStatusCode())->toBe(400);\n\n $respondedRequest = $httpLoader->load('https://www.example.com/foo');\n\n expect($respondedRequest?->response->getStatusCode())->toBe(400);\n});\n\ntest('certain error status codes can be excluded from being retried', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->twice()\n ->andReturn(new Response(404), new Response(500));\n\n $cache = new FileCache(helper_cachedir());\n\n $httpLoader = helper_getFastLoader(httpClient: $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader\n ->retryCachedErrorResponses()\n ->except([410, 500]);\n\n $respondedRequest = $httpLoader->load('https://www.example.com/foo');\n\n expect($respondedRequest?->response->getStatusCode())->toBe(404);\n\n $respondedRequest = $httpLoader->load('https://www.example.com/foo');\n\n expect($respondedRequest?->response->getStatusCode())->toBe(500);\n\n $respondedRequest = $httpLoader->load('https://www.example.com/foo');\n\n expect($respondedRequest?->response->getStatusCode())->toBe(500);\n});\n\nit(\n 'adds responses to the cache but doesn\\'t try to get them from the cache, when writeOnlyCache() was called',\n function ($loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->twice()->andReturn(new Response());\n\n $cache = new FileCache(helper_cachedir());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader->writeOnlyCache();\n\n try {\n $httpLoader->{$loadingMethod}('https://www.example.com/articles/123');\n } catch (Throwable $exception) {\n }\n\n try {\n $httpLoader->{$loadingMethod}('https://www.example.com/articles/123');\n } catch (Throwable $exception) {\n }\n },\n)->with(['load', 'loadOrFail']);\n\ntest(\n 'When cache filters are defined via the cacheOnlyWhereUrl() method it caches only responses for matching URLs',\n function (string $loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->twice()\n ->andReturnUsing(function (Request $request) {\n return new Response(200, body: $request->getUri() . ' response');\n });\n\n $cache = new FileCache(helper_cachedir());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader->cacheOnlyWhereUrl(Filter::urlPathStartsWith('/bar/'));\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.example.com/foo/something');\n\n expect($cache->get($respondedRequest->cacheKey()))->toBeNull();\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.example.com/bar/something');\n\n expect($cache->get($respondedRequest->cacheKey()))->toBeInstanceOf(RespondedRequest::class);\n },\n)->with(['load', 'loadOrFail']);\n\ntest(\n 'When multiple cache filters are defined via the cacheOnlyWhereUrl() method, all of them are used',\n function (string $loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->times(3)\n ->andReturnUsing(function (Request $request) {\n return new Response(200, body: $request->getUri() . ' response');\n });\n\n $cache = new FileCache(helper_cachedir());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader\n ->cacheOnlyWhereUrl(Filter::urlPathStartsWith('/bar/'))\n ->cacheOnlyWhereUrl(Filter::urlHost('www.example.com'));\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.example.com/foo/something');\n\n expect($cache->get($respondedRequest->cacheKey()))->toBeNull();\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.crwlr.software/bar/something');\n\n expect($cache->get($respondedRequest->cacheKey()))->toBeNull();\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.example.com/bar/something');\n\n expect($cache->get($respondedRequest->cacheKey()))->toBeInstanceOf(RespondedRequest::class);\n },\n)->with(['load', 'loadOrFail']);\n\ntest(\n 'when a request was redirected, only one of the URLs has to match the filters defined via cacheOnlyWhereUrl()',\n function (string $loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->andReturnUsing(function (Request $request) {\n $url = (string) $request->getUri();\n\n $redirectUrl = null;\n\n if ($url === 'https://www.example.com/foo/something') {\n $redirectUrl = 'https://www.example.com/bar/something';\n } elseif ($url === 'https://www.example.com/bar/something') {\n $redirectUrl = 'https://www.example.com/baz/something';\n }\n\n if ($redirectUrl) {\n return new Response(301, ['Location' => $redirectUrl]);\n }\n\n return new Response(200, body: $request->getUri() . ' response');\n });\n\n $cache = new FileCache(helper_cachedir());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader->cacheOnlyWhereUrl(Filter::urlPathStartsWith('/bar/'));\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.example.com/foo/something');\n\n expect($cache->get($respondedRequest->cacheKey()))->toBeInstanceOf(RespondedRequest::class);\n\n $cache->clear();\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.example.com/bar/something');\n\n expect($cache->get($respondedRequest->cacheKey()))->toBeInstanceOf(RespondedRequest::class);\n\n $cache->clear();\n\n $respondedRequest = $httpLoader->{$loadingMethod}('https://www.example.com/baz/something');\n\n expect($cache->get($respondedRequest->cacheKey()))->toBeNull();\n },\n)->with(['load', 'loadOrFail']);\n\nit('uses the cache only for requests that meet the filter criteria', function (string $loadingMethod) {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->once()\n ->andReturnUsing(function (Request $request) {\n return new Response(200, body: $request->getUri() . ' response');\n });\n\n $userAgent = helper_nonBotUserAgent();\n\n $cache = new FileCache(helper_cachedir());\n\n $cachedResponse = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo/test', headers: ['User-Agent' => $userAgent->__toString()]),\n new Response(),\n );\n\n $cache->set($cachedResponse->cacheKey(), $cachedResponse);\n\n $cachedResponse = new RespondedRequest(\n new Request('GET', 'https://www.example.com/bar/test', headers: ['User-Agent' => $userAgent->__toString()]),\n new Response(),\n );\n\n $cache->set($cachedResponse->cacheKey(), $cachedResponse);\n\n $httpLoader = new HttpLoader($userAgent, $httpClient);\n\n $httpLoader->setCache($cache);\n\n $httpLoader->cacheOnlyWhereUrl(Filter::urlPathStartsWith('/bar/'));\n\n $httpLoader->{$loadingMethod}('https://www.example.com/foo/test');\n\n $httpLoader->{$loadingMethod}('https://www.example.com/bar/test');\n})->with(['load', 'loadOrFail']);\n\nit('updates an existing cached response', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient\n ->shouldReceive('sendRequest')\n ->once()\n ->andReturn(new Response(body: 'hello'));\n\n $cache = new FileCache(helper_cachedir());\n\n $cache->clear();\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $response = $httpLoader->load('https://www.example.com/idontknow');\n\n if (!$response) {\n throw new Exception('failed to get response');\n }\n\n $extendedResponse = RespondedRequestChild::fromRespondedRequest($response);\n\n $httpLoader->addToCache($extendedResponse);\n\n $response = $httpLoader->load('https://www.example.com/idontknow');\n\n /** @var RespondedRequestChild $response */\n\n expect($response)->toBeInstanceOf(RespondedRequestChild::class)\n ->and($response->itseme())->toBe('mario');\n});\n\nit('does not add cookies to the cookie jar when a response was served from the cache', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldNotReceive('sendRequest');\n\n $cache = new FileCache(helper_cachedir());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->setCache($cache);\n\n $respondedRequest = new RespondedRequest(\n new Request(\n 'GET',\n 'https://www.example.com/wtf',\n ['Host' => ['www.example.com'], 'User-Agent' => [(string) helper_nonBotUserAgent()]],\n ),\n new Response(headers: ['Set-Cookie' => 'foo=bar'], body: 'Wtf!'),\n );\n\n $cache->set($respondedRequest->cacheKey(), $respondedRequest);\n\n $httpLoader->load('https://www.example.com/wtf');\n\n $cookieJar = invade($httpLoader)->cookieJar;\n\n /** @var CookieJar $cookieJar */\n\n $cookies = $cookieJar->allByDomain('example.com');\n\n expect($cookies)->toHaveCount(0);\n});\n\ntest('By default it uses the cookie jar and passes on cookies', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n return $request->getUri()->__toString() === 'https://www.crwlr.software/';\n })->andReturn(new Response(200, ['Set-Cookie' => ['cookie1=foo']]));\n\n $httpClient->shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n $cookiesHeader = $request->getHeader('Cookie');\n\n return $request->getUri()->__toString() === 'https://www.crwlr.software/blog' &&\n $cookiesHeader === ['cookie1=foo'];\n })->andReturn(new Response(200, ['Set-Cookie' => ['cookie1=foo', 'cookie2=bar']]));\n\n $httpClient->shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n $cookiesHeader = $request->getHeader('Cookie');\n\n return $request->getUri()->__toString() === 'https://www.crwlr.software/contact' &&\n $cookiesHeader === ['cookie1=foo', 'cookie2=bar'];\n })->andReturn(new Response(200, ['Set-Cookie' => ['cookie1=foo2', 'cookie2=bar2', 'cookie3=baz']]));\n\n $httpClient->shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n $cookiesHeader = $request->getHeader('Cookie');\n\n return $request->getUri()->__toString() === 'https://www.crwlr.software/packages' &&\n $cookiesHeader === ['cookie1=foo2', 'cookie2=bar2', 'cookie3=baz'];\n })->andReturn(new Response());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->load('https://www.crwlr.software/');\n\n $httpLoader->load('https://www.crwlr.software/blog');\n\n $httpLoader->loadOrFail('https://www.crwlr.software/contact');\n\n $httpLoader->loadOrFail('https://www.crwlr.software/packages');\n\n expect(true)->toBeTrue(); // Just here so pest doesn't complain that there is no assertion.\n});\n\ntest('You can turn off using the cookie jar', function () {\n $httpClient = Mockery::mock(ClientInterface::class);\n\n $httpClient->shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n return $request->getUri()->__toString() === 'https://www.crwlr.software/';\n })->andReturn(new Response(200, ['Set-Cookie' => ['cookie1=foo']]));\n\n $httpClient->shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n $cookiesHeader = $request->getHeader('Cookie');\n\n return $request->getUri()->__toString() === 'https://www.crwlr.software/blog' && $cookiesHeader === [];\n })->andReturn(new Response(200, ['Set-Cookie' => ['cookie1=foo', 'cookie2=bar']]));\n\n $httpClient->shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n $cookiesHeader = $request->getHeader('Cookie');\n\n return $request->getUri()->__toString() === 'https://www.crwlr.software/contact' && $cookiesHeader === [];\n })->andReturn(new Response(200, ['Set-Cookie' => ['cookie1=foo2', 'cookie2=bar2', 'cookie3=baz']]));\n\n $httpClient->shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n $cookiesHeader = $request->getHeader('Cookie');\n\n return $request->getUri()->__toString() === 'https://www.crwlr.software/packages' && $cookiesHeader === [];\n })->andReturn(new Response());\n\n $httpLoader = new HttpLoader(helper_nonBotUserAgent(), $httpClient);\n\n $httpLoader->dontUseCookies();\n\n $httpLoader->load('https://www.crwlr.software/');\n\n $httpLoader->load('https://www.crwlr.software/blog');\n\n $httpLoader->loadOrFail('https://www.crwlr.software/contact');\n\n $httpLoader->loadOrFail('https://www.crwlr.software/packages');\n\n expect(true)->toBeTrue(); // Just here so pest doesn't complain that there is no assertion.\n});\n" }, { "path": "tests/Loader/Http/Messages/RespondedRequestTest.php", "content": "toBeInstanceOf(RespondedRequest::class);\n});\n\ntest('creating with a redirect response adds a redirect uri.', function ($statusCode) {\n $request = new Request('GET', '/');\n\n $response = new Response($statusCode);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n expect($respondedRequest->redirects())->toHaveCount(1);\n})->with([300, 301, 302, 303, 304, 305, 307, 308]);\n\ntest('creating with non redirect responses doesn\\'t add a redirect uri.', function ($statusCode) {\n $request = new Request('GET', '/');\n\n $response = new Response($statusCode);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n expect($respondedRequest->redirects())->toHaveCount(0);\n})->with([101, 200, 404, 500]);\n\ntest('isRedirect returns false when the response is not a redirect', function () {\n $request = new Request('GET', '/');\n\n $response = new Response(200);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n expect($respondedRequest->isRedirect())->toBeFalse();\n});\n\ntest('isRedirect returns true when the response is a redirect', function () {\n $request = new Request('GET', '/');\n\n $response = new Response(301);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n expect($respondedRequest->isRedirect())->toBeTrue();\n});\n\ntest('isRedirect returns true when the last response is a redirect', function () {\n $request = new Request('GET', '/');\n\n $response = new Response(301);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n $respondedRequest->setResponse(new Response(302));\n\n expect($respondedRequest->isRedirect())->toBeTrue();\n});\n\ntest('isRedirect returns false when the last response is not a redirect', function () {\n $request = new Request('GET', '/');\n\n $response = new Response(301);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n $respondedRequest->setResponse(new Response(200));\n\n expect($respondedRequest->isRedirect())->toBeFalse();\n});\n\ntest('the requested uri remains the same when the request was redirected.', function () {\n $request = new Request('GET', '/request-uri');\n\n $response = new Response(301, ['Location' => '/redirect-uri']);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n $respondedRequest->setResponse(new Response(200));\n\n expect($respondedRequest->requestedUri())->toBe('/request-uri');\n});\n\ntest('when request was not redirected the effective uri equals the requested uri', function () {\n $request = new Request('GET', '/request-uri');\n\n $response = new Response(200);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n expect($respondedRequest->effectiveUri())->toBe('/request-uri');\n});\n\ntest('when request was redirected the effective uri is the redirect uri', function () {\n $request = new Request('GET', '/request-uri');\n\n $response = new Response(301, ['Location' => '/redirect-uri']);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n $respondedRequest->setResponse(new Response(200));\n\n expect($respondedRequest->effectiveUri())->toBe('/redirect-uri');\n});\n\ntest('the allUris() method returns all unique URIs', function () {\n $request = new Request('GET', '/request-uri');\n\n $response = new Response(301, ['Location' => '/redirect-uri']);\n\n $respondedRequest = new RespondedRequest($request, $response);\n\n $respondedRequest->setResponse(new Response(301, ['Location' => '/request-uri']));\n\n $respondedRequest->setResponse(new Response(301, ['Location' => '/another-redirect-uri']));\n\n $respondedRequest->setResponse(new Response(200));\n\n expect($respondedRequest->allUris())->toBe([\n '/request-uri',\n '/redirect-uri',\n '/another-redirect-uri',\n ]);\n});\n\nit('can be serialized', function () {\n $respondedRequest = new RespondedRequest(\n new Request('POST', '/home', ['key' => 'val'], 'bod'),\n new Response(201, ['k' => 'v'], 'res'),\n [new Screenshot('/path/to/screenshot.png'), new Screenshot('/another/path/to/screenshot.webp')],\n );\n\n $respondedRequest->addRedirectUri('/index');\n\n $serialized = serialize($respondedRequest);\n\n expect($serialized)->toBe(\n 'O:51:\"Crwlr\\\\Crawler\\\\Loader\\\\Http\\\\Messages\\\\RespondedRequest\":9:{s:13:\"requestMethod\";s:4:\"POST\";s:10:' .\n '\"requestUri\";s:5:\"/home\";s:14:\"requestHeaders\";a:1:{s:3:\"key\";a:1:{i:0;s:3:\"val\";}}s:11:\"requestBody\";' .\n 's:3:\"bod\";s:12:\"effectiveUri\";s:6:\"/index\";s:18:\"responseStatusCode\";i:201;s:15:\"responseHeaders\";a:1:{' .\n 's:1:\"k\";a:1:{i:0;s:1:\"v\";}}s:12:\"responseBody\";s:3:\"res\";s:11:\"screenshots\";a:2:{i:0;' .\n 's:23:\"/path/to/screenshot.png\";i:1;s:32:\"/another/path/to/screenshot.webp\";}}',\n );\n});\n\ntest('an old serialized instance without screenshots array can be unserialized', function () {\n $serialized = 'O:51:\"Crwlr\\Crawler\\Loader\\Http\\Messages\\RespondedRequest\":8:{s:13:\"requestMethod\";s:4:\"POST\";' .\n 's:10:\"requestUri\";s:5:\"/home\";s:14:\"requestHeaders\";a:1:{s:3:\"key\";a:1:{i:0;s:3:\"val\";}}s:11:\"requestBody\";' .\n 's:3:\"bod\";s:12:\"effectiveUri\";s:6:\"/index\";s:18:\"responseStatusCode\";i:201;s:15:\"responseHeaders\";a:1:{' .\n 's:1:\"k\";a:1:{i:0;s:1:\"v\";}}s:12:\"responseBody\";s:3:\"res\";}';\n\n $respondedRequest = unserialize($serialized);\n\n /** @var RespondedRequest $respondedRequest */\n\n expect($respondedRequest)->toBeInstanceOf(RespondedRequest::class)\n ->and($respondedRequest->request->getMethod())->toBe('POST')\n ->and($respondedRequest->request->getUri()->__toString())->toBe('/home')\n ->and($respondedRequest->request->getHeaders())->toBe(['key' => ['val']])\n ->and($respondedRequest->request->getBody()->getContents())->toBe('bod')\n ->and($respondedRequest->effectiveUri())->toBe('/index')\n ->and($respondedRequest->response->getStatusCode())->toBe(201)\n ->and($respondedRequest->response->getHeaders())->toBe(['k' => ['v']])\n ->and($respondedRequest->response->getBody()->getContents())->toBe('res');\n});\n\ntest('a serialized instance can be unserialized', function () {\n // We need actual existing file paths for screenshots\n $screenshot1 = helper_testfilesdir('screenshot1.png');\n\n $screenshot2 = helper_testfilesdir('screenshot2.jpeg');\n\n $serialized = 'O:51:\"Crwlr\\Crawler\\Loader\\Http\\Messages\\RespondedRequest\":9:{s:13:\"requestMethod\";s:4:\"POST\";' .\n 's:10:\"requestUri\";s:5:\"/home\";s:14:\"requestHeaders\";a:1:{s:3:\"key\";a:1:{i:0;s:3:\"val\";}}s:11:\"requestBody\";' .\n 's:3:\"bod\";s:12:\"effectiveUri\";s:6:\"/index\";s:18:\"responseStatusCode\";i:201;s:15:\"responseHeaders\";a:1:{' .\n 's:1:\"k\";a:1:{i:0;s:1:\"v\";}}s:12:\"responseBody\";s:3:\"res\";s:11:\"screenshots\";a:2:{i:0;' .\n 's:' . strlen($screenshot1) . ':\"' . $screenshot1 . '\";i:1;' .\n 's:' . strlen($screenshot2) . ':\"' . $screenshot2 . '\";}}';\n\n $respondedRequest = unserialize($serialized);\n\n /** @var RespondedRequest $respondedRequest */\n\n expect($respondedRequest)->toBeInstanceOf(RespondedRequest::class)\n ->and($respondedRequest->request->getMethod())->toBe('POST')\n ->and($respondedRequest->request->getUri()->__toString())->toBe('/home')\n ->and($respondedRequest->request->getHeaders())->toBe(['key' => ['val']])\n ->and($respondedRequest->request->getBody()->getContents())->toBe('bod')\n ->and($respondedRequest->effectiveUri())->toBe('/index')\n ->and($respondedRequest->response->getStatusCode())->toBe(201)\n ->and($respondedRequest->response->getHeaders())->toBe(['k' => ['v']])\n ->and($respondedRequest->response->getBody()->getContents())->toBe('res')\n ->and($respondedRequest->screenshots[0]->path)->toBe($screenshot1)\n ->and($respondedRequest->screenshots[1]->path)->toBe($screenshot2);\n});\n\nit('can be created from an old serialized array that was not containing the screenshots array', function () {\n $serialized = 'a:8:{s:13:\"requestMethod\";s:3:\"GET\";s:10:\"requestUri\";s:4:\"/foo\";s:14:\"requestHeaders\";a:0:{}s:11:' .\n '\"requestBody\";s:0:\"\";s:12:\"effectiveUri\";s:4:\"/bar\";s:18:\"responseStatusCode\";i:200;s:15:\"responseHeaders\";' .\n 'a:0:{}s:12:\"responseBody\";s:0:\"\";}';\n\n $respondedRequest = RespondedRequest::fromArray(unserialize($serialized));\n\n expect($respondedRequest)->toBeInstanceOf(RespondedRequest::class)\n ->and($respondedRequest->request->getUri()->__toString())->toBe('/foo')\n ->and($respondedRequest->effectiveUri())->toBe('/bar');\n});\n\nit('can be created from a serialized array that is containing the screenshots array', function () {\n // We need actual existing file paths\n $screenshot1 = helper_testfilesdir('screenshot1.png');\n\n $screenshot2 = helper_testfilesdir('screenshot2.jpeg');\n\n $serialized = 'a:9:{s:13:\"requestMethod\";s:3:\"GET\";s:10:\"requestUri\";s:4:\"/foo\";s:14:\"requestHeaders\";a:0:{}s:11:' .\n '\"requestBody\";s:0:\"\";s:12:\"effectiveUri\";s:4:\"/bar\";s:18:\"responseStatusCode\";i:200;s:15:\"responseHeaders\";' .\n 'a:0:{}s:12:\"responseBody\";s:0:\"\";s:11:\"screenshots\";a:2:{i:0;' .\n 's:' . strlen($screenshot1) . ':\"' . $screenshot1 . '\";i:1;' .\n 's:' . strlen($screenshot2) . ':\"' . $screenshot2 . '\";}}';\n\n $respondedRequest = RespondedRequest::fromArray(unserialize($serialized));\n\n expect($respondedRequest)->toBeInstanceOf(RespondedRequest::class)\n ->and($respondedRequest->request->getUri()->__toString())->toBe('/foo')\n ->and($respondedRequest->effectiveUri())->toBe('/bar')\n ->and($respondedRequest->screenshots[0]->path)->toBe($screenshot1)\n ->and($respondedRequest->screenshots[1]->path)->toBe($screenshot2);\n});\n\ntest(\n 'when creating from a serialized array, it checks screenshot paths for existence and throws away screenshots ' .\n 'when the files don\\'t exist',\n function () {\n $serialized = 'a:9:{s:13:\"requestMethod\";s:3:\"GET\";s:10:\"requestUri\";s:4:\"/foo\";s:14:\"requestHeaders\";' .\n 'a:0:{}s:11:\"requestBody\";s:0:\"\";s:12:\"effectiveUri\";s:4:\"/bar\";s:18:\"responseStatusCode\";i:200;' .\n 's:15:\"responseHeaders\";a:0:{}s:12:\"responseBody\";s:0:\"\";s:11:\"screenshots\";a:2:{i:0;' .\n 's:24:\"/path/to/screenshot1.png\";i:1;s:25:\"/path/to/screenshot2.jpeg\";}}';\n\n $respondedRequest = RespondedRequest::fromArray(unserialize($serialized));\n\n expect($respondedRequest)->toBeInstanceOf(RespondedRequest::class)\n ->and($respondedRequest->request->getUri()->__toString())->toBe('/foo')\n ->and($respondedRequest->effectiveUri())->toBe('/bar')\n ->and($respondedRequest->screenshots)->toHaveCount(0);\n },\n);\n\nit('has a toArrayForResult() method', function () {\n $respondedRequest = new RespondedRequest(\n new Request('POST', '/home', ['key' => 'val'], 'bod'),\n new Response(201, ['k' => 'v'], 'res'),\n [new Screenshot('/path/to/screenshot.jpg')],\n );\n\n expect($respondedRequest->toArrayForResult())->toBe([\n 'requestMethod' => 'POST',\n 'requestUri' => '/home',\n 'requestHeaders' => ['key' => ['val']],\n 'requestBody' => 'bod',\n 'effectiveUri' => '/home',\n 'responseStatusCode' => 201,\n 'responseHeaders' => ['k' => ['v']],\n 'responseBody' => 'res',\n 'screenshots' => ['/path/to/screenshot.jpg'],\n 'url' => '/home',\n 'uri' => '/home',\n 'status' => 201,\n 'headers' => ['k' => ['v']],\n 'body' => 'res',\n ]);\n});\n\nit('generates a cache key for an instance', function () {\n $respondedRequest = new RespondedRequest(new Request('GET', '/foo/bar'), new Response());\n\n expect($respondedRequest->cacheKey())->toBe('27ca75942fb28ed0d8fb3f9b077dd582');\n});\n" }, { "path": "tests/Loader/Http/Politeness/RobotsTxtHandlerTest.php", "content": "shouldReceive('sendRequest')->withArgs(function (RequestInterface $request) {\n return str_ends_with($request->getUri()->__toString(), '/robots.txt');\n })->andReturn(new Response(200, [], Utils::streamFor($robotsTxtContent)));\n }\n\n return new HttpLoader($userAgent, $httpClient);\n}\n\n/** @var TestCase $this */\n\ntest('route is disallowed when it\\'s disallowed for my user agent', function () {\n $robotsTxt = <<isAllowed('https://www.example.com/foo/bar'))->toBeFalse();\n});\n\ntest('route is disallowed when it\\'s disallowed for all user agents', function () {\n $robotsTxt = <<isAllowed('https://www.example.com/foo/bar'))->toBeFalse();\n});\n\ntest(\n 'route is not disallowed when it\\'s disallowed for all user agents but my user agent is not a BotUserAgent',\n function () {\n $robotsTxt = <<isAllowed('https://www.example.com/foo/bar'))->toBeTrue();\n },\n);\n\ntest(\n 'route is not disallowed when it\\'s disallowed for all user agent but I want to ignore wildcard rules',\n function () {\n $robotsTxt = <<ignoreWildcardRules();\n\n expect($robotsTxt->isAllowed('https://www.example.com/foo/bar'))->toBeTrue();\n },\n);\n\nit('gets all the sitemap URLs from robots.txt', function () {\n $robotsTxt = <<getSitemaps('https://www.example.com/home'))->toBe([\n 'https://www.example.com/sitemap.xml',\n 'https://www.example.com/sitemap2.xml',\n 'https://www.example.com/sitemap3.xml',\n ]);\n});\n\nit('fails silently when parsing fails', function () {\n $robotsTxt = <<isAllowed('https://www.example.com/anything'))->toBeTrue();\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)->toContain('Failed to parse robots.txt');\n});\n" }, { "path": "tests/Loader/Http/Politeness/ThrottlerTest.php", "content": "waitAtLeast(Microseconds::fromSeconds(0.001));\n\n $throttler->trackRequestStartFor($url);\n\n usleep(Microseconds::fromSeconds(0.1)->value);\n\n $throttler->trackRequestEndFor($url);\n\n $requestEndTime = Microseconds::fromSeconds(microtime(true));\n\n $throttler->waitForGo($url);\n\n $readyForNextRequest = Microseconds::fromSeconds(microtime(true));\n\n $diff = $readyForNextRequest->subtract($requestEndTime);\n\n expect($diff->value)->toBeGreaterThan(100000)\n ->and($diff->value)->toBeLessThan(220000); // A bit more than * 2.0 because other things happening also take time.\n});\n\nit('waits min 0.25s by default', function () {\n $url = Url::parsePsr7('https://www.example.com');\n\n $throttler = new Throttler();\n\n $throttler->trackRequestStartFor($url);\n\n $throttler->trackRequestEndFor($url);\n\n $requestEndTime = Microseconds::fromSeconds(microtime(true));\n\n $throttler->waitForGo($url);\n\n $readyForNextRequest = Microseconds::fromSeconds(microtime(true));\n\n $diff = $readyForNextRequest->subtract($requestEndTime);\n\n expect($diff->value)->toBeGreaterThan(250000);\n});\n\nit('respects the max wait time you set', function () {\n $url = Url::parsePsr7('https://www.example.com');\n\n $throttler = new Throttler();\n\n $throttler\n ->waitBetween(new MultipleOf(10), new MultipleOf(20))\n ->waitAtMax(Microseconds::fromSeconds(0.1));\n\n $throttler->trackRequestStartFor($url);\n\n usleep(Microseconds::fromSeconds(0.1)->value);\n\n $throttler->trackRequestEndFor($url);\n\n $requestEndTime = Microseconds::fromSeconds(microtime(true));\n\n $throttler->waitForGo($url);\n\n $readyForNextRequest = Microseconds::fromSeconds(microtime(true));\n\n $diff = $readyForNextRequest->subtract($requestEndTime);\n\n expect($diff->value)->toBeLessThan(110000); // A bit more than * 1.0 because other things happening also take time.\n});\n\nit('waits only if there was already a request to the same domain', function () {\n $url = Url::parsePsr7('https://www.example.com');\n\n $throttler = new Throttler();\n\n $throttler\n ->waitBetween(new MultipleOf(10), new MultipleOf(20))\n ->waitAtMax(Microseconds::fromSeconds(0.1));\n\n $throttler->trackRequestStartFor($url);\n\n usleep(Microseconds::fromSeconds(0.01)->value);\n\n $throttler->trackRequestEndFor($url);\n\n $requestEndTime = Microseconds::fromSeconds(microtime(true));\n\n $throttler->waitForGo(Url::parsePsr7('https://www.crwlr.software'));\n\n $readyForNextRequest = Microseconds::fromSeconds(microtime(true));\n\n $diff = $readyForNextRequest->subtract($requestEndTime);\n\n expect($diff->value)->toBeLessThan(1000);\n});\n\nit('throws an exception if you try to set different types for from and to', function () {\n new Throttler(Microseconds::fromSeconds(0.1), new MultipleOf(0.5));\n})->throws(InvalidArgumentException::class);\n\nit('throws an exception if you try to set the from value bigger than the to value with Microseconds', function () {\n new Throttler(Microseconds::fromSeconds(2.0), Microseconds::fromSeconds(1.0));\n})->throws(InvalidArgumentException::class);\n\nit('throws an exception if you try to set the from value bigger than the to value with MultipleOf', function () {\n new Throttler(new MultipleOf(1.0), new MultipleOf(0.9));\n})->throws(InvalidArgumentException::class);\n\nit('does not throw an exception when from and to values are equal', function () {\n new Throttler(Microseconds::fromSeconds(2.0), Microseconds::fromSeconds(2.0));\n\n new Throttler(new MultipleOf(1.0), new MultipleOf(1.0));\n\n expect(true)->toBeTrue();\n});\n\ntest('internal _requestToUrlWasStarted returns false when _internalTrackStartFor was not called', function () {\n $url = Url::parsePsr7('https://www.example.com');\n\n $throttler = new Throttler();\n\n $throttler\n ->waitBetween(Microseconds::fromSeconds(0.001), Microseconds::fromSeconds(0.002))\n ->waitAtMax(Microseconds::fromSeconds(0.002));\n\n expect(invade($throttler)->_requestToUrlWasStarted($url))->toBeFalse();\n\n $throttler->trackRequestEndFor($url); // To check if no error/exception occurs when start was not called before.\n});\n\ntest('internal _requestToUrlWasStarted returns true when _internalTrackStartFor was called', function () {\n $url = Url::parsePsr7('https://www.example.com');\n\n $throttler = new Throttler();\n\n $throttler\n ->waitBetween(Microseconds::fromSeconds(0.001), Microseconds::fromSeconds(0.002))\n ->waitAtMax(Microseconds::fromSeconds(0.002));\n\n $throttler->trackRequestStartFor($url);\n\n $invadedThrottler = invade($throttler);\n\n expect($invadedThrottler->_requestToUrlWasStarted($url))->toBeTrue();\n\n // And after end of the request is tracked, it should return false again.\n $throttler->trackRequestEndFor($url);\n\n expect($invadedThrottler->_requestToUrlWasStarted($url))->toBeFalse();\n});\n" }, { "path": "tests/Loader/Http/Politeness/TimingUnits/MultipleOfTest.php", "content": "calc(Microseconds::fromSeconds(1.23))\n ->toSeconds(),\n )->toBe(9.7047);\n});\n" }, { "path": "tests/Loader/Http/ProxyManagerTest.php", "content": "hasOnlySingleProxy())\n ->toBeTrue()\n ->and($manager->hasMultipleProxies())\n ->toBeFalse();\n\n $manager = new ProxyManager(['http://127.0.0.1:8001', 'http://127.0.0.1:8002']);\n\n expect($manager->hasOnlySingleProxy())\n ->toBeFalse()\n ->and($manager->hasMultipleProxies())\n ->toBeTrue();\n});\n\nit('returns the proxy when only one is defined', function () {\n $manager = new ProxyManager(['http://127.0.0.1:8003']);\n\n expect($manager->getProxy())\n ->toBe('http://127.0.0.1:8003')\n ->and($manager->getProxy())\n ->toBe('http://127.0.0.1:8003');\n});\n\nit('rotates the proxies when multiple are defined', function () {\n $manager = new ProxyManager(['http://127.0.0.1:8001', 'http://127.0.0.1:8002', 'http://127.0.0.1:8003']);\n\n expect($manager->getProxy())\n ->toBe('http://127.0.0.1:8001')\n ->and($manager->getProxy())\n ->toBe('http://127.0.0.1:8002')\n ->and($manager->getProxy())\n ->toBe('http://127.0.0.1:8003')\n ->and($manager->getProxy())\n ->toBe('http://127.0.0.1:8001');\n});\n" }, { "path": "tests/Loader/LoaderTest.php", "content": "hookName === 'afterLoad') {\n $this->callHook('beforeLoad'); // Loader won't run afterLoad when beforeLoad wasn't called.\n }\n\n $this->callHook($this->hookName);\n\n return 'something';\n }\n\n public function loadOrFail(mixed $subject): mixed\n {\n return 'something';\n }\n };\n $callback1Called = false;\n $loader->{$hookName}(function () use (&$callback1Called) {\n $callback1Called = true;\n });\n $callback2Called = false;\n $loader->{$hookName}(function () use (&$callback2Called) {\n $callback2Called = true;\n });\n $callback3Called = false;\n $loader->{$hookName}(function () use (&$callback3Called) {\n $callback3Called = true;\n });\n\n $loader->load('something');\n\n expect($callback1Called)->toBeTrue()\n ->and($callback2Called)->toBeTrue()\n ->and($callback3Called)->toBeTrue();\n})->with([\n 'beforeLoad',\n 'onCacheHit',\n 'onSuccess',\n 'onError',\n 'afterLoad',\n]);\n\nit('does not call the afterLoad hook when beforeLoad was not called before it', function () {\n $logger = new DummyLogger();\n\n $loader = new class (new BotUserAgent('FooBot'), $logger) extends Loader {\n public function load(mixed $subject): mixed\n {\n $this->callHook('afterLoad');\n\n return 'something';\n }\n\n public function loadOrFail(mixed $subject): mixed\n {\n return 'something';\n }\n };\n\n $callbackCalled = false;\n\n $loader->afterLoad(function () use (&$callbackCalled) {\n $callbackCalled = true;\n });\n\n $loader->load('something');\n\n expect($callbackCalled)->toBeFalse()\n ->and($logger->messages[0]['message'])->toStartWith(\n 'The afterLoad hook was called without a preceding call to the beforeLoad hook.',\n );\n});\n\nit('calls the afterLoad hook when beforeLoad was called before it', function () {\n $logger = new DummyLogger();\n\n $loader = new class (new BotUserAgent('FooBot'), $logger) extends Loader {\n public function load(mixed $subject): mixed\n {\n $this->callHook('beforeLoad');\n\n $this->callHook('afterLoad');\n\n return 'something';\n }\n\n public function loadOrFail(mixed $subject): mixed\n {\n return 'something';\n }\n };\n\n $callbackCalled = false;\n\n $loader->afterLoad(function () use (&$callbackCalled) {\n $callbackCalled = true;\n });\n\n $loader->load('something');\n\n expect($callbackCalled)->toBeTrue()\n ->and($logger->messages)->toHaveCount(0);\n});\n\ntest('You can set a cache and use it in the load function', function () {\n $loader = new class (new BotUserAgent('FooBot')) extends Loader {\n public function load(mixed $subject): string\n {\n $this->cache?->get('foo');\n\n return 'something';\n }\n public function loadOrFail(mixed $subject): mixed\n {\n return 'something';\n }\n };\n\n $cache = Mockery::mock(CacheInterface::class);\n\n $cache->shouldReceive('get')->with('foo')->once();\n\n $loader->setCache($cache);\n\n $loader->load('something');\n});\n" }, { "path": "tests/Logger/CliLoggerTest.php", "content": "log('info', 'Some log message.');\n $output = $this->getActualOutputForAssertion();\n expect($output)->toContain('Some log message.');\n});\n\ntest('It prints the log level', function () {\n $logger = new CliLogger();\n $logger->log('alert', 'Everybody panic!');\n $output = $this->getActualOutputForAssertion();\n expect($output)->toContain('[ALERT]');\n});\n\ntest('It starts with printing the time', function () {\n $logger = new CliLogger();\n $logger->log('warning', 'Warn about something.');\n $this->expectOutputRegex('/^\\d\\d:\\d\\d:\\d\\d:\\d\\d\\d\\d\\d\\d/');\n});\n\ntest('It has methods for all the log levels', function ($logLevel) {\n $logger = new CliLogger();\n $logger->{$logLevel}('Some message');\n $output = $this->getActualOutputForAssertion();\n expect($output)->toContain('Some message');\n expect($output)->toContain('[' . strtoupper($logLevel) . ']');\n})->with([\n 'emergency',\n 'alert',\n 'critical',\n 'error',\n 'warning',\n 'notice',\n 'info',\n 'debug',\n]);\n" }, { "path": "tests/Logger/PreStepInvocationLoggerTest.php", "content": "info('test');\n\n $logger->warning('foo');\n\n $logger->error('some error');\n\n expect($logger->messages)->toHaveCount(3)\n ->and($logger->messages[0]['level'])->toBe('info')\n ->and($logger->messages[0]['message'])->toBe('test')\n ->and($logger->messages[1]['level'])->toBe('warning')\n ->and($logger->messages[1]['message'])->toBe('foo')\n ->and($logger->messages[2]['level'])->toBe('error')\n ->and($logger->messages[2]['message'])->toBe('some error');\n});\n\nit('passes log messages to another logger', function () {\n $logger = new PreStepInvocationLogger();\n\n $logger->info('test');\n\n $logger->warning('foo');\n\n $logger->error('some error');\n\n $anotherLogger = new DummyLogger();\n\n $logger->passToOtherLogger($anotherLogger);\n\n expect($anotherLogger->messages)->toHaveCount(3)\n ->and($anotherLogger->messages[0]['level'])->toBe('info')\n ->and($anotherLogger->messages[0]['message'])->toBe('test')\n ->and($anotherLogger->messages[1]['level'])->toBe('warning')\n ->and($anotherLogger->messages[1]['message'])->toBe('foo')\n ->and($anotherLogger->messages[2]['level'])->toBe('error')\n ->and($anotherLogger->messages[2]['message'])->toBe('some error');\n});\n" }, { "path": "tests/Pest.php", "content": "group('integration')\n ->beforeEach(function () {\n if (!isset(TestServerProcess::$process)) {\n TestServerProcess::$process = Process::fromShellCommandline(\n 'php -S localhost:8000 ' . __DIR__ . '/_Integration/Server.php',\n );\n\n TestServerProcess::$process->start();\n\n usleep(100000);\n }\n })\n ->afterAll(function () {\n TestServerProcess::$process?->stop(3, SIGINT);\n\n TestServerProcess::$process = null;\n })\n ->in('_Integration');\n\nfunction helper_dump(mixed $var): void\n{\n error_log(var_export($var, true));\n}\n\nfunction helper_dieDump(mixed $var): void\n{\n var_dump($var);\n ob_end_flush();\n exit;\n}\n\nfunction helper_getValueReturningStep(mixed $value): Step\n{\n return new class ($value) extends Step {\n public function __construct(private mixed $value) {}\n\n protected function invoke(mixed $input): Generator\n {\n yield $this->value;\n }\n\n public function outputType(): StepOutputType\n {\n return OutputTypeHelper::isAssociativeArrayOrObject($this->value) ?\n StepOutputType::AssociativeArrayOrObject :\n StepOutputType::Scalar;\n }\n };\n}\n\nfunction helper_getInputReturningStep(): Step\n{\n return new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n}\n\nfunction helper_getNumberIncrementingStep(): Step\n{\n return new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield $input + 1;\n }\n };\n}\n\nfunction helper_getStepYieldingMultipleNumbers(): Step\n{\n return new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n foreach (['one', 'two', 'two', 'three', 'four', 'three', 'five', 'three'] as $number) {\n yield $number;\n }\n }\n };\n}\n\nfunction helper_getStepYieldingMultipleArraysWithNumber(): Step\n{\n return new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n foreach (['one', 'two', 'two', 'three', 'four', 'three', 'five', 'three'] as $key => $number) {\n yield ['number' => $number, 'foo' => 'bar' . ($input === true ? ' ' . $key : '')];\n }\n }\n };\n}\n\nfunction helper_getStepYieldingObjectWithNumber(int $number): Step\n{\n return new class ($number) extends Step {\n public function __construct(private int $number) {}\n\n protected function invoke(mixed $input): Generator\n {\n yield helper_getStdClassWithData(\n ['number' => $this->number, 'foo' => 'bar' . (is_int($input) ? ' ' . $input : '')],\n );\n }\n };\n}\n\nfunction helper_getStepYieldingMultipleObjectsWithNumber(): Step\n{\n return new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n foreach (['one', 'two', 'two', 'three', 'four', 'three', 'five', 'three'] as $key => $number) {\n yield helper_getStdClassWithData(\n ['number' => $number, 'foo' => 'bar' . ($input === true ? ' ' . $key : '')],\n );\n }\n }\n };\n}\n\nfunction helper_getStepYieldingInputArrayAsSeparateOutputs(): Step\n{\n return new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n foreach ($input as $output) {\n yield $output;\n }\n }\n };\n}\n\nfunction helper_getLoadingStep(): Step\n{\n return new class extends Step {\n /**\n * @use LoadingStep\n */\n use LoadingStep;\n\n protected function invoke(mixed $input): Generator\n {\n yield 'yo';\n }\n };\n}\n\nfunction helper_getDummyRobotsTxtResponse(?string $forDomain = null): Response\n{\n return new Response(\n 200,\n [],\n \"User-agent: FooBot\\n\" .\n \"Disallow: \" . ($forDomain ? '/' . $forDomain . '/secret' : 'secret'),\n );\n}\n\n/**\n * @param iterable $iterable\n * @return void\n */\nfunction helper_traverseIterable(iterable $iterable): void\n{\n foreach ($iterable as $key => $value) {\n // just traverse\n }\n}\n\n/**\n * @param mixed[] $array\n * @return Generator\n */\nfunction helper_arrayToGenerator(array $array): Generator\n{\n foreach ($array as $element) {\n yield $element;\n }\n}\n\n/**\n * @param Generator $generator\n * @return mixed[]\n */\nfunction helper_generatorToArray(Generator $generator): array\n{\n $array = [];\n\n foreach ($generator as $value) {\n $array[] = $value;\n }\n\n return $array;\n}\n\n/**\n * @return Output[]\n */\nfunction helper_invokeStepWithInput(StepInterface $step, mixed $input = null): array\n{\n return helper_generatorToArray($step->invokeStep(new Input($input ?? 'anything')));\n}\n\nfunction helper_getStepFilesContent(string $filePathInFilesFolder): string\n{\n $content = file_get_contents(__DIR__ . '/Steps/_Files/' . $filePathInFilesFolder);\n\n if ($content === false) {\n return '';\n }\n\n return $content;\n}\n\n/**\n * @param mixed[] $data\n */\nfunction helper_getStdClassWithData(array $data): stdClass\n{\n $object = new stdClass();\n\n foreach ($data as $key => $value) {\n $object->{$key} = $value;\n }\n\n return $object;\n}\n\nfunction helper_getSimpleListHtml(): string\n{\n return <<\n
  • one
    • \n
  • two
    • \n
  • three
    • \n
  • four
    • \n \n HTML;\n}\n\nfunction helper_getFastLoader(\n ?UserAgentInterface $userAgent = null,\n ?LoggerInterface $logger = null,\n ?ClientInterface $httpClient = null,\n): HttpLoader {\n $loader = new HttpLoader($userAgent ?? UserAgent::mozilla5CompatibleBrowser(), $httpClient, $logger);\n\n $loader->throttle()\n ->waitBetween(new MultipleOf(0.0001), new MultipleOf(0.0002))\n ->waitAtLeast(Microseconds::fromSeconds(0.0001));\n\n return $loader;\n}\n\nfunction helper_getFastCrawler(): HttpCrawler\n{\n return new class extends HttpCrawler {\n protected function userAgent(): UserAgentInterface\n {\n return new UserAgent('TestBot');\n }\n\n protected function loader(UserAgentInterface $userAgent, LoggerInterface $logger): LoaderInterface\n {\n return helper_getFastLoader($userAgent, $logger);\n }\n };\n}\n\nfunction helper_nonBotUserAgent(): UserAgent\n{\n return new UserAgent('Mozilla/5.0 (compatible; FooBot)');\n}\n\nfunction helper_getMinThrottler(): Throttler\n{\n return new Throttler(new MultipleOf(0.0001), new MultipleOf(0.0002), Microseconds::fromSeconds(0.0001));\n}\n\n/**\n * @param array $requestHeaders\n * @param array $responseHeaders\n */\nfunction helper_getRespondedRequest(\n string $method = 'GET',\n string $url = 'https://www.example.com/foo',\n array $requestHeaders = [],\n ?string $requestBody = null,\n int $statusCode = 200,\n array $responseHeaders = [],\n ?string $responseBody = null,\n): RespondedRequest {\n if ($requestBody !== null) {\n $request = new Request($method, $url, $requestHeaders, Utils::streamFor($requestBody));\n } else {\n $request = new Request($method, $url, $requestHeaders);\n }\n\n if ($responseBody !== null) {\n $response = new Response($statusCode, $responseHeaders, body: Utils::streamFor($responseBody));\n } else {\n $response = new Response($statusCode, $responseHeaders);\n }\n\n return new RespondedRequest($request, $response);\n}\n\nfunction helper_cachedir(?string $inDir = null): string\n{\n $path = __DIR__ . '/_Temp/_cachedir';\n\n if ($inDir !== null) {\n return $path . (str_starts_with($inDir, '/') ? $inDir : '/' . $inDir);\n }\n\n return $path;\n}\n\nfunction helper_resetCacheDir(): void\n{\n helper_resetTempDir(helper_cachedir());\n}\n\nfunction helper_storagedir(?string $inDir = null): string\n{\n $path = __DIR__ . '/_Temp/_storagedir';\n\n if ($inDir !== null) {\n return $path . (str_starts_with($inDir, '/') ? $inDir : '/' . $inDir);\n }\n\n return $path;\n}\n\nfunction helper_resetStorageDir(): void\n{\n helper_resetTempDir(helper_storagedir());\n}\n\nfunction helper_resetTempDir(string $dirPath): void\n{\n $files = scandir($dirPath);\n\n if (is_array($files)) {\n foreach ($files as $file) {\n if ($file === '.' || $file === '..' || $file === '.gitkeep') {\n continue;\n }\n\n @unlink($dirPath . '/' . $file);\n }\n }\n}\n\nfunction helper_testfilesdir(?string $inDir = null): string\n{\n $path = __DIR__ . '/_Temp/_testfilesdir';\n\n if ($inDir !== null) {\n return $path . (str_starts_with($inDir, '/') ? $inDir : '/' . $inDir);\n }\n\n return $path;\n}\n" }, { "path": "tests/ResultTest.php", "content": "set('title', 'PHP Web Developer');\n\n expect($result->get('title'))->toBe('PHP Web Developer');\n});\n\ntest('You can set multiple values for a property', function () {\n $result = new Result();\n\n $result->set('location', 'Linz');\n\n expect($result->get('location'))->toBe('Linz');\n\n $result->set('location', 'Wien');\n\n expect($result->get('location'))->toBe(['Linz', 'Wien']);\n});\n\ntest('The get method has a default value that you can set yourself', function () {\n $result = new Result();\n\n expect($result->get('foo'))->toBeNull()\n ->and($result->get('foo', '123'))->toBe('123');\n});\n\ntest('You can convert it to a plain array', function () {\n $result = new Result();\n\n $result->set('title', 'PHP Web Developer (w/m/x)');\n\n $result->set('location', 'Linz');\n\n $result->set('location', 'Wien');\n\n expect($result->toArray())->toBe([\n 'title' => 'PHP Web Developer (w/m/x)',\n 'location' => ['Linz', 'Wien'],\n ]);\n});\n\ntest('Converting to an array, also converts all objects at any level in the array to arrays', function () {\n $result = new Result();\n\n $result->set('foo', 'one');\n\n $result->set(\n 'bar',\n helper_getStdClassWithData([\n 'a' => 'b',\n 'c' => helper_getStdClassWithData(['d' => 'e', 'f' => 'g']),\n ]),\n );\n\n $resultArray = $result->toArray();\n\n expect($resultArray)->toBe([\n 'foo' => 'one',\n 'bar' => [\n 'a' => 'b',\n 'c' => ['d' => 'e', 'f' => 'g'],\n ],\n ]);\n});\n\ntest(\n 'when the only element of the output array is some unnamed property, but the value is an array with keys, ' .\n 'it returns only that child array',\n function () {\n $result = new Result();\n\n $result->set('unnamed', new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(200, [], 'Hello World!'),\n [new Screenshot('/path/to/screenshot.png')],\n ));\n\n $resultArray = $result->toArray();\n\n expect($resultArray)->toBeArray()\n ->and(count($resultArray))->toBeGreaterThanOrEqual(14)\n ->and($resultArray['url'])->toBe('https://www.example.com/foo')\n ->and($resultArray['status'])->toBe(200)\n ->and($resultArray['body'])->toBe('Hello World!')\n ->and($resultArray['screenshots'][0])->toBe('/path/to/screenshot.png');\n },\n);\n\ntest(\n 'when the only element of the output array is an unnamed property, with a scalar value, it returns the unnamed key',\n function () {\n $result = new Result();\n\n $result->set('unnamed', 'foo');\n\n $resultArray = $result->toArray();\n\n expect($resultArray)->toBe(['unnamed' => 'foo']);\n },\n);\n\ntest('when you add something with empty string as key it creates a name with incrementing number', function () {\n $result = new Result();\n\n $result->set('', 'foo');\n\n expect($result->get('unnamed1'))->toBe('foo');\n\n $result->set('', 'bar');\n\n expect($result->get('unnamed2'))->toBe('bar');\n\n $result->set('', 'baz');\n\n expect($result->get('unnamed3'))->toBe('baz');\n});\n\ntest('you can create a new instance from another instance', function () {\n $instance1 = new Result();\n\n $instance1->set('foo', 'bar');\n\n $instance2 = new Result($instance1);\n\n expect($instance1->get('foo'))->toBe('bar')\n ->and($instance2->get('foo'))->toBe('bar');\n\n $instance2->set('baz', 'quz');\n\n expect($instance1->get('baz'))->toBeNull()\n ->and($instance2->get('baz'))->toBe('quz');\n});\n\ntest('it makes a proper array of arrays if you repeatedly add (associative) arrays with the same key', function () {\n $result = new Result();\n\n $result->set('foo', ['bar' => 'one', 'baz' => 'two']);\n\n expect($result->get('foo'))->toBe(['bar' => 'one', 'baz' => 'two']);\n\n $result->set('foo', ['bar' => 'three', 'baz' => 'four']);\n\n expect($result->get('foo'))->toBe([\n ['bar' => 'one', 'baz' => 'two'],\n ['bar' => 'three', 'baz' => 'four'],\n ]);\n});\n" }, { "path": "tests/Steps/BaseStepTest.php", "content": "passesAllFilters = $this->passesAllFilters($input->get());\n\n yield new Output('yo');\n }\n}\n\n/** @var TestCase $this */\n\ntest('You can set a filter and passesAllFilters() tells if an output value passes that filter', function () {\n $step = new TestStep();\n\n $step->where(Filter::equal('hello'));\n\n helper_invokeStepWithInput($step, new Input('hello'));\n\n expect($step->passesAllFilters)->toBeTrue();\n\n helper_invokeStepWithInput($step, new Input('hola'));\n\n expect($step->passesAllFilters)->toBeFalse();\n});\n\ntest('You can set multiple filters and passesAllFilters() tells if an output value passes that filters', function () {\n $step = new TestStep();\n\n $step->where(Filter::stringContains('foo'))\n ->where(Filter::equal('boo foo too'))\n ->where(Filter::notEqual('pew foo tew'));\n\n helper_invokeStepWithInput($step, new Input('boo foo too'));\n\n expect($step->passesAllFilters)->toBeTrue();\n\n helper_invokeStepWithInput($step, new Input('foo something'));\n\n expect($step->passesAllFilters)->toBeFalse();\n\n helper_invokeStepWithInput($step, new Input('pew foo tew'));\n\n expect($step->passesAllFilters)->toBeFalse();\n});\n\ntest(\n 'you can link filters using orWhere and passesAllFilters() is true when one of those filters evaluates to true',\n function () {\n $step = new TestStep();\n\n $step->where(Filter::stringStartsWith('foo'))\n ->orWhere(Filter::stringStartsWith('bar'))\n ->orWhere(Filter::stringEndsWith('foo'));\n\n helper_invokeStepWithInput($step, new Input('foo bar baz'));\n\n expect($step->passesAllFilters)->toBeTrue();\n\n helper_invokeStepWithInput($step, new Input('bar foo baz'));\n\n expect($step->passesAllFilters)->toBeTrue();\n\n helper_invokeStepWithInput($step, new Input('bar baz foo'));\n\n expect($step->passesAllFilters)->toBeTrue();\n\n helper_invokeStepWithInput($step, new Input('funky town'));\n\n expect($step->passesAllFilters)->toBeFalse();\n },\n);\n\nit('uses a key from an array when providing a key to the filter() method', function () {\n $step = new TestStep();\n\n $step->where('vendor', Filter::equal('crwlr'));\n\n helper_invokeStepWithInput($step, new Input(['vendor' => 'crwlr', 'package' => 'url']));\n\n expect($step->passesAllFilters)->toBeTrue();\n\n helper_invokeStepWithInput($step, new Input(['vendor' => 'illuminate', 'package' => 'support']));\n\n expect($step->passesAllFilters)->toBeFalse();\n});\n\nit('uses a key from an object when providing a key to the filter() method', function () {\n $step = new TestStep();\n\n $step->where('vendor', Filter::equal('crwlr'));\n\n helper_invokeStepWithInput($step, new Input(\n helper_getStdClassWithData(['vendor' => 'crwlr', 'package' => 'url']),\n ));\n\n expect($step->passesAllFilters)->toBeTrue();\n\n helper_invokeStepWithInput($step, new Input(\n helper_getStdClassWithData(['vendor' => 'illuminate', 'package' => 'support']),\n ));\n\n expect($step->passesAllFilters)->toBeFalse();\n});\n\nit('filters using a custom Closure filter', function () {\n $step = new TestStep();\n\n $step->where('bar', Filter::custom(function (mixed $value) {\n return in_array($value, ['one', 'two', 'three'], true);\n }));\n\n helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two']);\n\n expect($step->passesAllFilters)->toBeTrue();\n\n helper_invokeStepWithInput($step, ['foo' => 'three', 'bar' => 'four']);\n\n expect($step->passesAllFilters)->toBeFalse();\n});\n\nit('throws an exception when you provide a string as first argument to filter but no second argument', function () {\n $step = new TestStep();\n\n $step->where('test');\n})->throws(InvalidArgumentException::class);\n\nit('removes an UTF-8 byte order mark from the beginning of a string', function () {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n\n protected function validateAndSanitizeInput(mixed $input): mixed\n {\n return parent::validateAndSanitizeStringOrHttpResponse($input);\n }\n };\n\n $stringWithBom = helper_getStepFilesContent('Xml/rss-with-bom.xml');\n\n $response = new RespondedRequest(\n new Request('GET', 'https://www.example.com/rss'),\n new Response(body: $stringWithBom),\n );\n\n $outputs = helper_invokeStepWithInput($step, $response);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBeString()\n ->and(substr($outputs[0]->get(), 0, 5))->toBe('toHaveCount(1)\n ->and($outputs[0]->get())->toBeString()\n ->and(substr($outputs[0]->get(), 0, 5))->toBe('addLogger(new PreStepInvocationLogger());\n\n $this->logger?->info('test');\n\n $this->logger?->warning('foo');\n }\n\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n\n $crawlerLogger = new DummyLogger();\n\n $step->addLogger($crawlerLogger);\n\n expect($crawlerLogger->messages)->toHaveCount(2)\n ->and($crawlerLogger->messages[0]['level'])->toBe('info')\n ->and($crawlerLogger->messages[0]['message'])->toBe('test')\n ->and($crawlerLogger->messages[1]['level'])->toBe('warning')\n ->and($crawlerLogger->messages[1]['message'])->toBe('foo');\n },\n);\n\nit(\n 'when using a PreStepInvocationLogger, the later created logger is also passed to refiners, so its log messages ' .\n 'won\\'t be lost',\n function () {\n $step = new class extends Step {\n public function __construct()\n {\n $this->addLogger(new PreStepInvocationLogger());\n\n $this->logger?->info('test');\n }\n\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n\n $step->refineOutput('foo', StringRefiner::replace('foo', 'bar'));\n\n $logger = new DummyLogger();\n\n $step->addLogger($logger);\n\n helper_invokeStepWithInput($step, ['foo' => 1.2]);\n\n expect($logger->messages)->toHaveCount(2)\n ->and($logger->messages[1]['message'])->toBe(\n 'Refiner StringRefiner::replace() can\\'t be applied to value of type double',\n );\n },\n);\n\n/* ----------------------------- validateBeforeRun() ----------------------------- */\n\nit(\n 'throws an exception in validateBeforeRun() when output type is scalar and keep() was used but not keepAs()',\n function () {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Scalar;\n }\n };\n\n $step->keep()->validateBeforeRun(Http::get());\n },\n)->throws(PreRunValidationException::class);\n\nit(\n 'logs a warning in validateBeforeRun() when output type is mixed and keep() was used but not keepAs()',\n function () {\n class SomeDemoStep extends Step\n {\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n }\n\n $step = new SomeDemoStep();\n\n $step->addLogger(new CliLogger())->keep()->validateBeforeRun(Http::get());\n\n expect($this->getActualOutputForAssertion())\n ->toContain('The tests\\Steps\\SomeDemoStep step potentially yields scalar value outputs');\n },\n);\n\ntest(\n 'the warning message, when output type is mixed and keep() was used but not keepAs() with an anonymous step ' .\n 'class, extending a step that isn\\'t one of the abstract classes Step or BaseStep, contains the parent step ' .\n 'class',\n function () {\n class ParentStepClass extends Step\n {\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n }\n\n $step = new class extends ParentStepClass {};\n\n $step->addLogger(new CliLogger())->keep()->validateBeforeRun(Http::get());\n\n expect($this->getActualOutputForAssertion())\n ->toContain(\n 'An anonymous class step, that is extending the tests\\\\Steps\\\\ParentStepClass step potentially ' .\n 'yields scalar value outputs',\n );\n },\n);\n\ntest(\n 'the warning message, when output type is mixed and keep() was used but not keepAs() with an anonymous step ' .\n 'class, extending one of the abstract classes Step or BaseStep, only mentions that it is an anonymous step class',\n function (string $extendClass) {\n $step = null;\n\n if ($extendClass === Step::class) {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n } elseif ($extendClass === BaseStep::class) {\n $step = new class extends BaseStep {\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n\n public function invokeStep(Input $input): Generator\n {\n yield from $this->invoke($input);\n }\n };\n }\n\n if ($step === null) {\n throw new Exception('Invalid $extendClass parameter');\n }\n\n $step->addLogger(new CliLogger())->keep()->validateBeforeRun(Http::get());\n\n expect($this->getActualOutputForAssertion())\n ->toContain(\n 'An anonymous class step potentially yields scalar value outputs',\n );\n },\n)->with([\n [Step::class],\n [BaseStep::class],\n]);\n\nit('does not throw an exception or log a warning when output type is scalar and keepAs() was called', function () {\n helper_getInputReturningStep()->addLogger(new CliLogger())->keepAs('foo')->validateBeforeRun(Http::get());\n\n expect($this->getActualOutputForAssertion())\n ->not()\n ->toContain('The tests\\Steps\\SomeDemoStep step potentially yields scalar value outputs');\n});\n\nit('does not throw an exception or log a warning when output type is scalar and outputKey() was called', function () {\n helper_getInputReturningStep()->addLogger(new CliLogger())->outputKey('foo')->validateBeforeRun(Http::get());\n\n expect($this->getActualOutputForAssertion())\n ->not()\n ->toContain('The tests\\Steps\\SomeDemoStep step potentially yields scalar value outputs');\n});\n\nit('throws an exception when keepFromInput() was called and initial inputs contain a scalar value', function () {\n Http::get()\n ->keepFromInput()\n ->validateBeforeRun([\n ['foo' => 'bar', 'baz' => 'quz'],\n 'scalar',\n ]);\n})->throws(PreRunValidationException::class);\n\nit('does not throw an exception when keepFromInput() was called and initial inputs are associative array', function () {\n Http::get()\n ->keepFromInput()\n ->validateBeforeRun([\n ['foo' => 'one'],\n ['foo' => 'two'],\n ]);\n})->throwsNoExceptions();\n\nit('logs an error when initial inputs are empty', function () {\n Http::get()\n ->addLogger(new CliLogger())\n ->validateBeforeRun([]);\n\n expect($this->getActualOutputForAssertion())\n ->toContain('You did not provide any initial inputs for your crawler.');\n});\n\nit('throws an exception when keepFromInput() was called and previous step yields scalar outputs', function () {\n Http::get()\n ->keepFromInput()\n ->validateBeforeRun(Html::getLink('.link'));\n})->throws(PreRunValidationException::class);\n\nit('does not throw an exception when keepInputAs() was called and previous step yields scalar outputs', function () {\n Http::get()\n ->keepInputAs('link')\n ->validateBeforeRun(Html::getLink('.link'));\n})->throwsNoExceptions();\n\nit('logs a warning, when keepFromInput() was called and previous step yields mixed outputs', function () {\n $stepWithMixedOutputType = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'yo';\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Mixed;\n }\n };\n\n Http::get()\n ->keepFromInput()\n ->addLogger(new CliLogger())\n ->validateBeforeRun($stepWithMixedOutputType);\n\n expect($this->getActualOutputForAssertion())\n ->toContain('potentially yields scalar value outputs ')\n ->toContain('the next step can not keep it by using keepFromInput()');\n});\n\ntest(\n 'the warning message, when keepFromInput() was called and previous step yields mixed outputs with an anonymous ' .\n 'step class, extending a step that isn\\'t one of the abstract classes Step or BaseStep, contains the parent step ' .\n 'class',\n function () {\n class ParentStepClassTwo extends Step\n {\n protected function invoke(mixed $input): Generator\n {\n yield 'yo';\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Mixed;\n }\n }\n\n $stepWithMixedOutputType = new class extends ParentStepClassTwo {};\n\n Http::get()\n ->keepFromInput()\n ->addLogger(new CliLogger())\n ->validateBeforeRun($stepWithMixedOutputType);\n\n expect($this->getActualOutputForAssertion())\n ->toContain(\n 'An anonymous class step, that is extending the tests\\\\Steps\\\\ParentStepClassTwo step potentially ' .\n 'yields scalar value outputs',\n );\n },\n);\n\ntest(\n 'the warning message, when keepFromInput() was called and previous step yields mixed outputs with an anonymous ' .\n 'step class, extending one of the abstract classes Step or BaseStep, only mentions that it is an anonymous step ' .\n 'class',\n function (string $extendClass) {\n $stepWithMixedOutputType = null;\n\n if ($extendClass === Step::class) {\n $stepWithMixedOutputType = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'yo';\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Mixed;\n }\n };\n } elseif ($extendClass === BaseStep::class) {\n $stepWithMixedOutputType = new class extends BaseStep {\n protected function invoke(mixed $input): Generator\n {\n yield 'yo';\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Mixed;\n }\n\n public function invokeStep(Input $input): Generator\n {\n yield from $this->invoke($input);\n }\n };\n }\n\n if ($stepWithMixedOutputType === null) {\n throw new Exception('Invalid $extendClass parameter');\n }\n\n Http::get()\n ->keepFromInput()\n ->addLogger(new CliLogger())\n ->validateBeforeRun($stepWithMixedOutputType);\n\n expect($this->getActualOutputForAssertion())\n ->toContain('An anonymous class step potentially yields scalar value outputs');\n },\n)->with([\n [Step::class],\n [BaseStep::class],\n]);\n\n/* ----------------------------- keep() ----------------------------- */\n\nit('adds all from array output to the keep array in the output object, when keep() is called', function () {\n $step = helper_getInputReturningStep()->keep();\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two']);\n\n expect($outputs[0]->keep)->toBe(['foo' => 'one', 'bar' => 'two']);\n});\n\nit('adds all from object output to the keep array in the output object, when keep() is called', function () {\n $step = helper_getInputReturningStep()->keep();\n\n $outputObject = new class {\n /**\n * @return array\n */\n public function toArray(): array\n {\n return ['key' => 'value', 'key2' => 'value2'];\n }\n };\n\n $outputs = helper_invokeStepWithInput($step, $outputObject);\n\n expect($outputs[0]->keep)->toBe(['key' => 'value', 'key2' => 'value2']);\n});\n\nit('adds a key from array output to the keep array in the output, when keep() was called with a string', function () {\n $step = helper_getInputReturningStep()->keep('bar');\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two']);\n\n expect($outputs[0]->keep)->toBe(['bar' => 'two']);\n});\n\nit('adds multiple keys to the keep array in the output, when keep() was called with an array', function () {\n $step = helper_getInputReturningStep()->keep(['foo', 'baz']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two', 'baz' => 'three']);\n\n expect($outputs[0]->keep)->toBe(['foo' => 'one', 'baz' => 'three']);\n});\n\nit('maps output data to the keep array in the output, when keep() was called with an associative array', function () {\n $step = helper_getInputReturningStep()->keep(['foo', 'mappedKey' => 'baz']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two', 'baz' => 'three']);\n\n expect($outputs[0]->keep)->toBe(['foo' => 'one', 'mappedKey' => 'three']);\n});\n\nit('logs an error when output is scalar value and keep was used, and adds the value with an unnamed key', function () {\n $step = helper_getInputReturningStep()\n ->addLogger(new CliLogger())\n ->keep();\n\n $outputs = helper_invokeStepWithInput($step, 'hello');\n\n expect($outputs[0]->keep)->toBe(['unnamed1' => 'hello'])\n ->and($this->getActualOutputForAssertion())\n ->toContain('yielded an output that is neither an associative array, nor an object');\n});\n\nit('repeatedly adds properties with unnamed keys with increasing numbers', function () {\n $step = helper_getValueReturningStep('world')\n ->keepFromInput()\n ->keep();\n\n $outputs = helper_invokeStepWithInput($step, new Input('hello', keep: ['unnamed1' => 'servus']));\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->keep)->toBe(['unnamed1' => 'servus', 'unnamed2' => 'hello', 'unnamed3' => 'world']);\n});\n\n/* ----------------------------- keepAs() ----------------------------- */\n\nit('adds scalar value output with the defined key to keep output data, when keepAs() was used', function () {\n $step = helper_getInputReturningStep()\n ->keepAs('greeting');\n\n $outputs = helper_invokeStepWithInput($step, 'hello');\n\n expect($outputs[0]->keep)->toBe(['greeting' => 'hello']);\n});\n\nit('adds array output with the defined key to keep output data, when keepAs() was used', function () {\n $step = helper_getInputReturningStep()\n ->keepAs('test');\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'bar']);\n\n expect($outputs[0]->keep)->toBe(['test' => ['foo' => 'bar']]);\n});\n\n/* ----------------------------- keepFromInput() ----------------------------- */\n\nit('adds all from array input to the keep array in the output object, when keepFromInput() is called', function () {\n $step = helper_getValueReturningStep('foo')->keepFromInput();\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two']);\n\n expect($outputs[0]->keep)->toBe(['foo' => 'one', 'bar' => 'two']);\n});\n\nit('adds all from object input to the keep array in the output object, when keepFromInput() is called', function () {\n $step = helper_getValueReturningStep('foo')->keepFromInput();\n\n $inputObject = new class {\n /**\n * @return array\n */\n public function toArray(): array\n {\n return ['key' => 'value', 'key2' => 'value2'];\n }\n };\n\n $outputs = helper_invokeStepWithInput($step, $inputObject);\n\n expect($outputs[0]->keep)->toBe(['key' => 'value', 'key2' => 'value2']);\n});\n\nit(\n 'adds a key from array input to the keep array in the output, when keepFromInput() was called with a string',\n function () {\n $step = helper_getValueReturningStep('foo')->keepFromInput('bar');\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two']);\n\n expect($outputs[0]->keep)->toBe(['bar' => 'two']);\n },\n);\n\nit(\n 'adds multiple keys from the input to the keep array in the output, when keepFromInput() was called with an array',\n function () {\n $step = helper_getValueReturningStep('foo')->keepFromInput(['foo', 'baz']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two', 'baz' => 'three']);\n\n expect($outputs[0]->keep)->toBe(['foo' => 'one', 'baz' => 'three']);\n },\n);\n\nit(\n 'maps input data to the keep array in the output, when keepFromInput() was called with an associative array',\n function () {\n $step = helper_getValueReturningStep('foo')->keepFromInput(['foo', 'mappedKey' => 'baz']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two', 'baz' => 'three']);\n\n expect($outputs[0]->keep)->toBe(['foo' => 'one', 'mappedKey' => 'three']);\n },\n);\n\nit('logs an error when input is scalar value and keep was used, and adds the value with an unnamed key', function () {\n $step = helper_getValueReturningStep('foo')\n ->addLogger(new CliLogger())\n ->keepFromInput();\n\n $outputs = helper_invokeStepWithInput($step, 'hey');\n\n expect($outputs[0]->keep)->toBe(['unnamed1' => 'hey'])\n ->and($this->getActualOutputForAssertion())\n ->toContain('received an input that is neither an associative array, nor an object');\n});\n\n/* ----------------------------- keepInputAs() ----------------------------- */\n\nit('adds scalar value input with the defined key to keep output data, when keepInputAs() was used', function () {\n $step = helper_getValueReturningStep('yo')\n ->keepInputAs('greeting');\n\n $outputs = helper_invokeStepWithInput($step, 'hello');\n\n expect($outputs[0]->keep)->toBe(['greeting' => 'hello']);\n});\n\nit('adds array input with the defined key to keep output data, when keepAs() was used', function () {\n $step = helper_getValueReturningStep('yay')\n ->keepInputAs('test');\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'bar']);\n\n expect($outputs[0]->keep)->toBe(['test' => ['foo' => 'bar']]);\n});\n\n/* ------------------------ combinations of keep calls ------------------------ */\n\nit('makes an array of values when the same key should be kept from input and output', function () {\n $step = helper_getValueReturningStep(['foo' => 'one', 'bar' => 'two'])\n ->keepFromInput('foo')\n ->keep(['foo', 'bar']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'bar']);\n\n expect($outputs[0]->keep)->toBe(['foo' => ['bar', 'one'], 'bar' => 'two']);\n});\n\ntest('same key in input and output, but they are mapped to different keys for keep data', function () {\n $step = helper_getValueReturningStep(['foo' => 'one', 'bar' => 'two'])\n ->keepFromInput(['inputFoo' => 'foo'])\n ->keep(['foo', 'bar']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'bar']);\n\n expect($outputs[0]->keep)->toBe(['inputFoo' => 'bar', 'foo' => 'one', 'bar' => 'two']);\n});\n\nit('merges data for the same key recursively', function () {\n $step = helper_getValueReturningStep(['foo' => ['one', 'two'], 'bar' => 'two'])\n ->keepFromInput('foo')\n ->keep(['foo', 'bar']);\n\n $outputs = helper_invokeStepWithInput(\n $step,\n new Input(['foo' => ['bar', 'baz']], keep: ['foo' => 'test']),\n );\n\n expect($outputs[0]->keep)->toBe(['foo' => ['test', 'bar', 'baz', 'one', 'two'], 'bar' => 'two']);\n});\n\n/* ----------------------------- keepsAnything() ----------------------------- */\n\ntest(\n 'keepsAnything() returns true when one of keep(), keepAs(), keepFromInput() or keepInputAs() was called',\n function (bool $callKeep, bool $callKeepAs, bool $callKeepFromInput, bool $callKeepInputAs, bool $expected) {\n $step = helper_getInputReturningStep();\n\n if ($callKeep) {\n $step->keep();\n }\n\n if ($callKeepAs) {\n $step->keepAs('foo');\n }\n\n if ($callKeepFromInput) {\n $step->keepFromInput();\n }\n\n if ($callKeepInputAs) {\n $step->keepInputAs('bar');\n }\n\n expect($step->keepsAnything())->toBe($expected);\n },\n)->with([\n [false, false, false, false, false],\n [true, false, false, false, true],\n [false, true, false, false, true],\n [false, false, true, false, true],\n [false, false, false, true, true],\n]);\n\ntest(\n 'keepsAnythingFromInputData() returns true when one of keepFromInput() or keepInputAs() was called',\n function (bool $callKeep, bool $callKeepAs, bool $callKeepFromInput, bool $callKeepInputAs, bool $expected) {\n $step = helper_getInputReturningStep();\n\n if ($callKeep) {\n $step->keep();\n }\n\n if ($callKeepAs) {\n $step->keepAs('foo');\n }\n\n if ($callKeepFromInput) {\n $step->keepFromInput();\n }\n\n if ($callKeepInputAs) {\n $step->keepInputAs('bar');\n }\n\n expect($step->keepsAnythingFromInputData())->toBe($expected);\n },\n)->with([\n [false, false, false, false, false],\n [true, false, false, false, false],\n [false, true, false, false, false],\n [false, false, true, false, true],\n [false, false, false, true, true],\n]);\n\ntest(\n 'keepsAnythingFromOutputData() returns true when one of keep() or keepAs() was called',\n function (bool $callKeep, bool $callKeepAs, bool $callKeepFromInput, bool $callKeepInputAs, bool $expected) {\n $step = helper_getInputReturningStep();\n\n if ($callKeep) {\n $step->keep();\n }\n\n if ($callKeepAs) {\n $step->keepAs('foo');\n }\n\n if ($callKeepFromInput) {\n $step->keepFromInput();\n }\n\n if ($callKeepInputAs) {\n $step->keepInputAs('bar');\n }\n\n expect($step->keepsAnythingFromOutputData())->toBe($expected);\n },\n)->with([\n [false, false, false, false, false],\n [true, false, false, false, true],\n [false, true, false, false, true],\n [false, false, true, false, false],\n [false, false, false, true, false],\n]);\n\n/* ----------------------------- sub crawlers ----------------------------- */\n\nit('logs an error message when a sub crawler is defined and step has no reference to a parent crawler', function () {\n $step = helper_getInputReturningStep()->addLogger(new CliLogger());\n\n $step->subCrawlerFor('bar', function (Crawler $crawler) {\n return $crawler->addStep(Http::get());\n });\n\n helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => ['https://www.example.com']]);\n\n expect($this->getActualOutputForAssertion())->toContain(\n 'Can\\'t make sub crawler, because the step has no reference to the parent crawler.',\n );\n});\n\nit('logs an error message when a sub crawler is defined and output is scalar value', function () {\n $step = helper_getInputReturningStep()->addLogger(new CliLogger());\n\n $step->setParentCrawler(HttpCrawler::make()->withUserAgent('Test'));\n\n $step->subCrawlerFor('bar', function (Crawler $crawler) {\n return $crawler->addStep(Http::get());\n });\n\n helper_invokeStepWithInput($step, 'foo');\n\n expect($this->getActualOutputForAssertion())\n ->toContain('The sub crawler feature works only with outputs that are associative arrays');\n});\n\nit('runs a sub crawler for a certain output property', function () {\n $step = helper_getInputReturningStep()->addLogger(new CliLogger());\n\n $step->setParentCrawler(HttpCrawler::make()->withUserAgent('Test'));\n\n $step->subCrawlerFor('bar', function (Crawler $crawler) {\n return $crawler->addStep(Html::root()->extract(['title' => 'h1']));\n });\n\n $results = helper_invokeStepWithInput($step, [\n 'foo' => 'hey',\n 'bar' => '

    Hello World!

    ',\n ]);\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get())->toBe(['foo' => 'hey', 'bar' => ['title' => 'Hello World!']]);\n});\n\ntest('when a sub crawler returns multiple results, they are an array in the parent output', function () {\n $step = helper_getInputReturningStep()->addLogger(new CliLogger());\n\n $step->setParentCrawler(HttpCrawler::make()->withUserAgent('Test'));\n\n $step->subCrawlerFor('bar', function (Crawler $crawler) {\n return $crawler->addStep(Html::each('.item')->extract(['title' => 'h3']));\n });\n\n $html = <<\n \n \n \n

    one

    \n

    two

    \n

    three

    \n \n HTML;\n\n $results = helper_invokeStepWithInput($step, ['foo' => 'hey', 'bar' => $html, 'baz' => 'yo']);\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get())\n ->toBe([\n 'foo' => 'hey',\n 'bar' => [\n ['title' => 'one'],\n ['title' => 'two'],\n ['title' => 'three'],\n ],\n 'baz' => 'yo',\n ]);\n});\n\nit('runs a sub crawler with multiple inputs, when defined property is array', function () {\n $step = helper_getInputReturningStep()->addLogger(new CliLogger());\n\n $step->setParentCrawler(HttpCrawler::make()->withUserAgent('Test'));\n\n $step->subCrawlerFor('bar', function (Crawler $crawler) {\n return $crawler->addStep(Html::root()->extract(['title' => 'h1']));\n });\n\n $results = helper_invokeStepWithInput($step, [\n 'foo' => 'hey',\n 'bar' => [\n '

    No. 1

    ',\n '

    No. 2

    ',\n '

    No. 3

    ',\n ],\n 'baz' => 'yo',\n ]);\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get())\n ->toBe([\n 'foo' => 'hey',\n 'bar' => [\n ['title' => 'No. 1'],\n ['title' => 'No. 2'],\n ['title' => 'No. 3'],\n ],\n 'baz' => 'yo',\n ]);\n});\n\nit('does not run a sub crawler, when output does not contain the defined key', function () {\n $step = helper_getInputReturningStep()->addLogger(new CliLogger());\n\n $step->setParentCrawler(HttpCrawler::make()->withUserAgent('Test'));\n\n $step->subCrawlerFor('bar', function (Crawler $crawler) {\n return $crawler->addStep(Html::root()->extract(['title' => 'h1']));\n });\n\n $results = helper_invokeStepWithInput($step, ['foo' => 'hey', 'baz' => 'ho']);\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get())->toBe(['foo' => 'hey', 'baz' => 'ho']);\n});\n" }, { "path": "tests/Steps/CsvTest.php", "content": "toHaveCount(4)\n ->and($outputs[0]->get())->toBe(['id' => '123', 'domain' => 'crwl.io', 'url' => 'https://www.crwl.io'])\n ->and($outputs[1]->get())->toBe(['id' => '234', 'domain' => 'example.com', 'url' => 'https://www.example.com'])\n ->and($outputs[2]->get())->toBe(['id' => '345', 'domain' => 'otsch.codes', 'url' => 'https://www.otsch.codes'])\n ->and($outputs[3]->get())->toBe(\n ['id' => '456', 'domain' => 'crwlr.software', 'url' => 'https://www.crwlr.software'],\n );\n});\n\nit('maps a file', function () {\n $outputs = helper_invokeStepWithInput(Csv::parseFile(['id', 'name', 'homepage']), helper_csvFilePath('basic.csv'));\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['id' => '123', 'name' => 'Otsch', 'homepage' => 'https://www.otsch.codes'])\n ->and($outputs[1]->get())->toBe(['id' => '234', 'name' => 'John Doe', 'homepage' => 'https://www.john.doe'])\n ->and($outputs[2]->get())->toBe(['id' => '345', 'name' => 'Jane Doe', 'homepage' => 'https://www.jane.doe']);\n});\n\nit('works with a RespondedRequest as input', function () {\n $body = <<toHaveCount(2)\n ->and($outputs[0]->get())->toBe(['id' => '123', 'name' => 'John Doe', 'phone' => '+431234567'])\n ->and($outputs[1]->get())->toBe(['id' => '234', 'name' => 'Jane Doe', 'phone' => '+432345678']);\n});\n\nit('works with an object having a __toString method', function () {\n $object = new class {\n public function __toString(): string\n {\n return <<toHaveCount(2)\n ->and($outputs[0]->get())->toBe(['id' => '123', 'name' => 'Max Mustermann', 'phone' => '+431234567'])\n ->and($outputs[1]->get())->toBe(['id' => '234', 'name' => 'Julia Musterfrau', 'phone' => '+432345678']);\n});\n\nit('logs an error message for other inputs', function (string $method, mixed $input) {\n $logger = new DummyLogger();\n\n $step = ($method === 'string' ? Csv::parseString(['column']) : Csv::parseFile(['column']))->addLogger($logger);\n\n helper_traverseIterable($step->invokeStep(new Input($input)));\n\n $logMessages = $logger->messages;\n\n expect($logMessages)->not->toBeEmpty()\n ->and($logMessages[0]['message'])->toStartWith(\n 'The Crwlr\\\\Crawler\\\\Steps\\\\Csv step was called with input that it can not work with: ',\n )\n ->and($logMessages[0]['message'])->toEndWith('. The invalid input is of type ' . gettype($input) . '.');\n})->with([\n ['string', 123],\n ['string', new stdClass()],\n ['string', 12.345],\n ['string', true],\n ['string', null],\n ['file', 123],\n ['file', new stdClass()],\n ['file', 12.345],\n ['file', true],\n ['file', null],\n]);\n\nit('can map columns using numerical array keys for the columns', function () {\n $string = << 'domain', 3 => 'description']), $string);\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe([\n 'domain' => 'crwlr.software', 'description' => 'PHP Web Crawling and Scraping Library',\n ])\n ->and($outputs[1]->get())->toBe(['domain' => 'otsch.codes', 'description' => 'I am Otsch, I code']);\n});\n\nit('can map columns using numerical array keys for the columns when parsing file', function () {\n $outputs = helper_invokeStepWithInput(\n Csv::parseFile([1 => 'name', 2 => 'homepage']),\n helper_csvFilePath('basic.csv'),\n );\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['name' => 'Otsch', 'homepage' => 'https://www.otsch.codes'])\n ->and($outputs[1]->get())->toBe(['name' => 'John Doe', 'homepage' => 'https://www.john.doe'])\n ->and($outputs[2]->get())->toBe(['name' => 'Jane Doe', 'homepage' => 'https://www.jane.doe']);\n});\n\nit('can map columns using null for columns to skip', function () {\n $string = <<toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['make' => 'Ford', 'price' => '3000.00'])\n ->and($outputs[1]->get())->toBe(['make' => 'Chevy', 'price' => '4900.00'])\n ->and($outputs[2]->get())->toBe(['make' => 'Chevy', 'price' => '5000.00']);\n});\n\nit('can map columns using null for columns to skip when parsing file', function () {\n $outputs = helper_invokeStepWithInput(Csv::parseFile(['id', null, 'homepage']), helper_csvFilePath('basic.csv'));\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['id' => '123', 'homepage' => 'https://www.otsch.codes'])\n ->and($outputs[1]->get())->toBe(['id' => '234', 'homepage' => 'https://www.john.doe'])\n ->and($outputs[2]->get())->toBe(['id' => '345', 'homepage' => 'https://www.jane.doe']);\n});\n\nit('uses the values from the first line as output keys when no column mapping defined', function () {\n $string = <<skipFirstLine(), $string);\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['id' => '1', 'title' => 'Raspberry Pi Zero 2 W', 'price' => '16.99']);\n});\n\nit('uses the values from the first line as output keys when no column mapping defined when parsing file', function () {\n $outputs = helper_invokeStepWithInput(\n Csv::parseFile()->skipFirstLine(),\n helper_csvFilePath('with-column-headlines.csv'),\n );\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe([\n 'Stunde' => '1',\n 'Montag' => 'Mathematik',\n 'Dienstag' => 'Deutsch',\n 'Mittwoch' => 'Englisch',\n 'Donnerstag' => 'Erdkunde',\n 'Freitag' => 'Politik',\n ]);\n});\n\nit('skips the first line when defined via method call to skipFirstLine method', function () {\n $string = <<skipFirstLine();\n\n $outputs = helper_invokeStepWithInput($step, $string);\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['make' => 'Ford', 'price' => '3000.00']);\n});\n\nit('skips the first line when parsing file when defined via method call to skipFirstLine method', function () {\n $step = Csv::parseFile([1 => 'fach-erste', 2 => 'fach-zweite'])\n ->skipFirstLine();\n\n $outputs = helper_invokeStepWithInput($step, helper_csvFilePath('with-column-headlines.csv'));\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['fach-erste' => 'Mathematik', 'fach-zweite' => 'Deutsch'])\n ->and($outputs[1]->get())->toBe(['fach-erste' => 'Sport', 'fach-zweite' => 'Deutsch'])\n ->and($outputs[2]->get())->toBe(['fach-erste' => 'Sport', 'fach-zweite' => 'Religion (ev., kath.)']);\n});\n\nit('skips the first line when defined via constructor param', function () {\n $string = <<toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['make' => 'Ford', 'price' => '3000.00']);\n});\n\nit('skips the first line when parsing file when defined via constructor param', function () {\n $outputs = helper_invokeStepWithInput(\n Csv::parseFile([1 => 'fach-erste', 3 => 'fach-dritte'], true),\n helper_csvFilePath('with-column-headlines.csv'),\n );\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['fach-erste' => 'Mathematik', 'fach-dritte' => 'Englisch'])\n ->and($outputs[1]->get())->toBe(['fach-erste' => 'Sport', 'fach-dritte' => 'Englisch'])\n ->and($outputs[2]->get())->toBe(['fach-erste' => 'Sport', 'fach-dritte' => 'Kunst']);\n});\n\nit('uses a different separator when you set one', function () {\n $string = << 'username', 2 => 'firstname', 3 => 'surname'])\n ->separator('|');\n\n $outputs = helper_invokeStepWithInput($step, $string);\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['username' => 'CoDerOtsch', 'firstname' => 'Christian', 'surname' => 'Olear'])\n ->and($outputs[1]->get())->toBe(['username' => 'g3n1u5', 'firstname' => 'Albert', 'surname' => 'Einstein'])\n ->and($outputs[2]->get())->toBe(['username' => 'sWiFtY', 'firstname' => 'Taylor', 'surname' => 'Swift']);\n});\n\nit('uses a different separator when you set one, when parsing a file', function () {\n $step = Csv::parseFile([1 => 'username', 4 => 'age'])\n ->separator('*');\n\n $outputs = helper_invokeStepWithInput($step, helper_csvFilePath('separator.csv'));\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['username' => 'CoDerOtsch', 'age' => '35'])\n ->and($outputs[1]->get())->toBe(['username' => 'g3n1u5', 'age' => '143'])\n ->and($outputs[2]->get())->toBe(['username' => 'sWiFtY', 'age' => '32']);\n});\n\nit('throws an InvalidArgumentException when you try to set a multi character separator', function () {\n Csv::parseString([])->separator('***');\n})->throws(InvalidArgumentException::class);\n\nit('uses a different enclosure when you set one', function () {\n $string = << 'meal', 2 => 'price'])\n ->enclosure('/');\n\n $outputs = helper_invokeStepWithInput($step, $string);\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['meal' => 'Fritattensuppe', 'price' => '3.9'])\n ->and($outputs[1]->get())->toBe(['meal' => 'Wiener Schnitzel vom Schwein', 'price' => '12.7'])\n ->and($outputs[2]->get())->toBe(['meal' => 'Semmelknödel mit Schwammerlsauce', 'price' => '9.5']);\n});\n\nit('uses a different enclosure when you set one, when parsing a file', function () {\n $step = Csv::parseFile([1 => 'meal', 2 => 'price'])\n ->enclosure('?');\n\n $outputs = helper_invokeStepWithInput($step, helper_csvFilePath('enclosure.csv'));\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['meal' => 'Kräftige Rindsuppe', 'price' => '4.5'])\n ->and($outputs[1]->get())->toBe(['meal' => 'Crispy Chicken Burger', 'price' => '12'])\n ->and($outputs[2]->get())->toBe(['meal' => 'Duett von Saibling und Forelle', 'price' => '21']);\n});\n\nit('uses a different escape character when you set one', function () {\n $string = << 'escaped'])\n ->escape('&');\n\n $outputs = helper_invokeStepWithInput($step, $string);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['escaped' => 'test &\"escape&\" test']);\n});\n\nit('uses a different escape character when you set one, when parsing a file', function () {\n $step = Csv::parseFile([1 => 'escaped'])\n ->escape('%');\n\n $outputs = helper_invokeStepWithInput($step, helper_csvFilePath('escape.csv'));\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe(['escaped' => 'test %\"escape%\" test'])\n ->and($outputs[1]->get())->toBe(['escaped' => 'foo %\"escape%\" bar %\"baz%\" lorem']);\n});\n\nit('filters rows', function () {\n $string = << 'isPremium'])\n ->skipFirstLine()\n ->where('isPremium', Filter::equal('1'));\n\n $outputs = helper_invokeStepWithInput($step, $string);\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe(['id' => '123', 'isPremium' => '1'])\n ->and($outputs[1]->get())->toBe(['id' => '124', 'isPremium' => '1']);\n});\n\nit('filters rows when parsing a file', function () {\n $step = Csv::parseFile(['Stunde', 'Fach'])\n ->skipFirstLine()\n ->where('Fach', Filter::equal('Sport'));\n\n $outputs = helper_invokeStepWithInput($step, helper_csvFilePath('with-column-headlines.csv'));\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe(['Stunde' => '2', 'Fach' => 'Sport'])\n ->and($outputs[1]->get())->toBe(['Stunde' => '3', 'Fach' => 'Sport']);\n});\n\nit('filters rows by multiple filters', function () {\n $string = << 'isVip', 4 => 'isQueenBandMember'])\n ->skipFirstLine()\n ->where('isVip', Filter::equal('1'))\n ->where('isQueenBandMember', Filter::equal('1'));\n\n $outputs = helper_invokeStepWithInput($step, $string);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['id' => '123', 'isVip' => '1', 'isQueenBandMember' => '1']);\n});\n\nit('filters rows by multiple filters when parsing a file', function () {\n $step = Csv::parseFile(['Stunde', 'Montag', 'Dienstag', 'Mittwoch', 'Donnerstag', 'Freitag'])\n ->skipFirstLine()\n ->where('Montag', Filter::equal('Sport'))\n ->where('Donnerstag', Filter::equal('Sport'));\n\n $outputs = helper_invokeStepWithInput($step, helper_csvFilePath('with-column-headlines.csv'));\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe([\n 'Stunde' => '2',\n 'Montag' => 'Sport',\n 'Dienstag' => 'Deutsch',\n 'Mittwoch' => 'Englisch',\n 'Donnerstag' => 'Sport',\n 'Freitag' => 'Geschichte',\n ]);\n});\n\nit('filters rows with a StringCheck filter', function () {\n $string = <<skipFirstLine()\n ->where('firstname', Filter::stringContains('Christian'));\n\n $outputs = helper_invokeStepWithInput($step, $string);\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe(['id' => '123', 'firstname' => 'Christian'])\n ->and($outputs[1]->get())->toBe(['id' => '124', 'firstname' => 'Christian Anton'])\n ->and($outputs[2]->get())->toBe(['id' => '125', 'firstname' => 'Another Christian']);\n});\n" }, { "path": "tests/Steps/Dom/HtmlDocumentTest.php", "content": "foohello';\n\n $document = new HtmlDocument($html);\n\n expect($document->getBaseHref())->toBe('/foo/bar');\n});\n\nit('gets the href of the first base tag in the document', function () {\n $html = <<\n \n \n foo\n \n \n \n hey\n \n HTML;\n\n $document = new HtmlDocument($html);\n\n expect($document->getBaseHref())->toBe('/foo');\n});\n\ntest('getBaseHref() returns null if the document does not contain a base tag', function () {\n $html = 'foohey';\n\n $document = new HtmlDocument($html);\n\n expect($document->getBaseHref())->toBeNull();\n});\n\ntest('the querySelector() method returns an HtmlElement object', function () {\n $html = 'foo
    hello
    ';\n\n $document = new HtmlDocument($html);\n\n expect($document->querySelector('.element'))->toBeInstanceOf(HtmlElement::class);\n});\n\ntest('the querySelectorAll() method returns a NodeList of HtmlElement objects', function () {\n $html = 'foo
    • foo
    • bar
    ';\n\n $document = new HtmlDocument($html);\n\n $nodeList = $document->querySelectorAll('ul li');\n\n expect($nodeList)->toBeInstanceOf(NodeList::class);\n\n $anyNodesChecked = false;\n\n foreach ($nodeList as $node) {\n expect($node)->toBeInstanceOf(HtmlElement::class);\n\n $anyNodesChecked = true;\n }\n\n expect($anyNodesChecked)->toBeTrue();\n});\n\ntest('the queryXPath() method returns a NodeList of HtmlElement objects', function () {\n $html = 'foo
    • foo
    • bar
    ';\n\n $document = new HtmlDocument($html);\n\n $nodeList = $document->queryXPath('//ul/li');\n\n expect($nodeList)->toBeInstanceOf(NodeList::class);\n\n $anyNodesChecked = false;\n\n foreach ($nodeList as $node) {\n expect($node)->toBeInstanceOf(HtmlElement::class);\n\n $anyNodesChecked = true;\n }\n\n expect($anyNodesChecked)->toBeTrue();\n});\n" }, { "path": "tests/Steps/Dom/HtmlElementTest.php", "content": "\n \n \n \n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $wrapperElement = $document->querySelector('#wrapper');\n\n expect($wrapperElement)->toBeInstanceOf(HtmlElement::class)\n ->and($wrapperElement?->querySelector('.element'))->toBeInstanceOf(HtmlElement::class);\n});\n\ntest('child nodes selected via querySelectorAll() are HtmlElement instances', function () {\n $html = <<\n \n \n \n
    \n
    foo
    \n
    bar
    \n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $wrapperElement = $document->querySelector('#wrapper');\n\n expect($wrapperElement)->toBeInstanceOf(HtmlElement::class);\n\n $childNodeList = $wrapperElement?->querySelectorAll('.element');\n\n expect($childNodeList)->toBeInstanceOf(NodeList::class)\n ->and($childNodeList?->count())->toBe(2)\n ->and($childNodeList?->first())->toBeInstanceOf(HtmlElement::class)\n ->and($childNodeList?->last())->toBeInstanceOf(HtmlElement::class);\n});\n\ntest('child nodes selected via queryXPath() are HtmlElement instances', function () {\n $html = <<\n \n \n \n
    \n
    foo
    \n
    bar
    \n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $wrapperElement = $document->queryXPath('//*[@id=\"wrapper\"]')->first();\n\n expect($wrapperElement)->toBeInstanceOf(HtmlElement::class);\n\n $childNodeList = $wrapperElement?->queryXPath('//*[contains(@class, \"element\")]');\n\n expect($childNodeList)->toBeInstanceOf(NodeList::class)\n ->and($childNodeList?->count())->toBe(2)\n ->and($childNodeList?->first())->toBeInstanceOf(HtmlElement::class)\n ->and($childNodeList?->first()?->text())->toBe('foo')\n ->and($childNodeList?->last())->toBeInstanceOf(HtmlElement::class)\n ->and($childNodeList?->last()?->text())->toBe('bar');\n});\n\nit('gets the node name', function () {\n $html = <<\n \n \n \n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $node = $document->querySelector('.element');\n\n expect($node?->nodeName())->toBe('div')\n ->and($node?->querySelector('.child')?->nodeName())->toBe('span');\n});\n\nit('gets the text of a node', function () {\n $html = <<\n \n \n \n
    \n bli bla blub\n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $node = $document->querySelector('.element');\n\n expect($node?->text())->toBe('bli bla blub');\n});\n\nit('gets the outer HTML of a node', function () {\n $html = <<\n \n \n \n
    \n bli bla blub\n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $node = $document->querySelector('.element');\n\n expect($node?->outerHtml())->toBe(\n '
    ' . PHP_EOL .\n ' bli bla blub' . PHP_EOL .\n '
    ',\n );\n});\n\nit('gets the inner HTML of a node', function () {\n $html = <<\n \n \n \n
    \n bli bla blub\n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $node = $document->querySelector('.element');\n\n expect($node?->innerHtml())->toBe(\n PHP_EOL .\n ' bli bla blub' . PHP_EOL,\n );\n});\n\nit('gets an attribute from a node', function () {\n $html = <<\n \n \n \n Link\n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $node = $document->querySelector('.element');\n\n expect($node?->getAttribute('href'))->toBe('/foo/bar');\n});\n" }, { "path": "tests/Steps/Dom/NodeListTest.php", "content": "\n \n \n \n
    • foo
    • bar
    • baz
    \n \n \n HTML;\n\n $crawler = new Crawler($html);\n\n $filtered = $crawler->filter('ul li');\n\n $nodeList = new NodeList(\n $filtered,\n function (object $node): HtmlElement {\n /** @var \\Dom\\Node|DOMNode|Crawler $node */\n return new HtmlElement($node);\n },\n );\n\n expect($nodeList->count())->toBe(3)\n ->and($nodeList->first()?->text())->toBe('foo')\n ->and($nodeList->nth(2)?->text())->toBe('bar')\n ->and($nodeList->last()?->text())->toBe('baz')\n ->and($nodeList->each(fn($node) => $node->text()))->toBe(['foo', 'bar', 'baz']);\n});\n\nit('can be constructed from a \\Dom\\NodeList instance', function () {\n $html = <<\n \n \n \n
    • foo
    • bar
    • baz
    \n \n \n HTML;\n\n $document = \\Dom\\HTMLDocument::createFromString($html, LIBXML_NOERROR);\n\n $nodeList = new NodeList(\n $document->querySelectorAll('ul li'),\n function (object $node): HtmlElement {\n /** @var \\Dom\\Node|DOMNode|Crawler $node */\n return new HtmlElement($node);\n },\n );\n\n expect($nodeList->count())->toBe(3)\n ->and($nodeList->first()?->text())->toBe('foo')\n ->and($nodeList->nth(2)?->text())->toBe('bar')\n ->and($nodeList->last()?->text())->toBe('baz')\n ->and($nodeList->each(fn($node) => $node->text()))->toBe(['foo', 'bar', 'baz']);\n})->group('php84');\n\nit('can be instantiated from an array of Nodes (object instances from this library)', function () {\n $html = <<\n \n \n \n
    \n
    foo
    bar
    baz
    \n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $array = [];\n\n foreach ($document->querySelectorAll('.list .element') as $node) {\n $array[] = $node;\n }\n\n $newNodeList = new NodeList($array);\n\n expect($newNodeList->count())->toBe(3)\n ->and($newNodeList->first()?->text())->toBe('foo')\n ->and($newNodeList->last()?->text())->toBe('baz')\n ->and($newNodeList->nth(2)?->text())->toBe('bar');\n});\n\nit('gets the count of the node list', function () {\n $html = <<\n \n \n Foo\n \n \n
    • foo
    • bar
    • baz
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n expect($document->querySelectorAll('ul li')->count())->toBe(3);\n});\n\nit('can be iterated and the elements are instances of Crwlr\\Crawler\\Steps\\Dom\\Node', function () {\n $html = <<\n \n \n Foo\n \n \n
    • foo
    • bar
    • baz
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $iteratesAnyNodes = false;\n\n foreach ($document->querySelectorAll('ul li') as $node) {\n expect($node)->toBeInstanceOf(Node::class);\n\n $iteratesAnyNodes = true;\n }\n\n expect($iteratesAnyNodes)->toBeTrue();\n});\n\nit(\n 'can be iterated with the each() method and return values are returned as an array from the each() call',\n function () {\n $html = <<\n \n \n \n
    \n
    foo
    \n
    bar
    \n
    baz
    \n
    quz
    \n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $result = $document->querySelectorAll('.list .element')->each(function ($node) {\n return $node->text() . ' check';\n });\n\n expect($result)->toBe([\n 'foo check',\n 'bar check',\n 'baz check',\n 'quz check',\n ]);\n },\n);\n\ntest('an empty NodeList can be iterated', function () {\n $html = <<\n \n \n Foo\n \n \n
    • foo
    • bar
    • baz
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $iteratesAnyNodes = false;\n\n foreach ($document->querySelectorAll('ul lulu') as $node) {\n $iteratesAnyNodes = true;\n }\n\n expect($iteratesAnyNodes)->toBeFalse();\n});\n\nit('returns the first, last and nth element of the NodeList', function () {\n $html = <<\n \n \n \n
    \n
    foo
    \n
    bar
    \n
    baz
    \n
    quz
    \n
    \n \n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $list = $document->querySelectorAll('.list .element');\n\n expect($list->first())->toBeInstanceOf(HtmlElement::class)\n ->and($list->first()?->text())->toBe('foo')\n ->and($list->nth(2))->toBeInstanceOf(HtmlElement::class)\n ->and($list->nth(2)?->text())->toBe('bar')\n ->and($list->nth(3))->toBeInstanceOf(HtmlElement::class)\n ->and($list->nth(3)?->text())->toBe('baz')\n ->and($list->last())->toBeInstanceOf(HtmlElement::class)\n ->and($list->last()?->text())->toBe('quz');\n});\n" }, { "path": "tests/Steps/Dom/NodeTest.php", "content": "filter($selectNode)->first();\n}\n\n/**\n * @throws Exception\n */\nfunction helper_getLegacyDomNodeInstanceFromSource(string $source, string $selectNode = 'body'): DOMNode\n{\n $node = (new Crawler($source))->filter($selectNode)->first()->getNode(0);\n\n if (!$node) {\n throw new Exception('Can\\'t get legacy node');\n }\n\n return $node;\n}\n\nfunction helper_getPhp84HtmlDomNodeInstanceFromSource(string $source, string $selectNode = 'body'): Element\n{\n $node = HTMLDocument::createFromString($source, HTML_NO_DEFAULT_NS | LIBXML_NOERROR)->querySelector($selectNode);\n\n /** @var Element $node */\n\n return $node;\n}\n\nfunction helper_getPhp84XmlDomNodeInstanceFromSource(string $source, string $selectNode = 'body'): Element\n{\n $node = XMLDocument::createFromString($source, LIBXML_NOERROR)->querySelector($selectNode);\n\n /** @var Element $node */\n\n return $node;\n}\n\n/**\n * @param \\Dom\\Node|Element|DOMNode|Crawler $originalNode\n */\nfunction helper_getAbstractNodeInstance(object $originalNode, bool $html = true): HtmlNodeStub|XmlNodeStub\n{\n if ($html) {\n return new HtmlNodeStub($originalNode);\n }\n\n return new XmlNodeStub($originalNode);\n}\n\n/* ----------------------------- Instantiation ----------------------------- */\n\nit('can be created from a \\DOM\\Node instance', function () {\n $xml = <<\n \n 1Foo\n \n XML;\n\n $domNode = helper_getPhp84XmlDomNodeInstanceFromSource($xml, 'items item');\n\n expect($domNode)->toBeInstanceOf(\\Dom\\Node::class);\n\n $node = new class ($domNode) extends Node {\n protected function makeChildNodeInstance(object $node): Node\n {\n return new XmlElement($node);\n }\n };\n\n expect($node)->toBeInstanceOf(Node::class)\n ->and($node->text())->toBe('1Foo');\n})->group('php84');\n\nit('can be instantiated from a symfony Crawler instance', function () {\n $xml = <<\n \n 1Foo\n \n XML;\n\n $crawler = helper_getSymfonyCrawlerInstanceFromSource($xml, 'items item');\n\n expect($crawler)->toBeInstanceOf(Crawler::class);\n\n $node = new class ($crawler) extends Node {\n protected function makeChildNodeInstance(object $node): Node\n {\n return new XmlElement($node);\n }\n };\n\n expect($node)->toBeInstanceOf(Node::class)\n ->and($node->text())->toBe('1Foo');\n});\n\nit('can be instantiated from a DOMNode instance', function () {\n $xml = <<\n \n 1Foo\n \n XML;\n\n $domNode = helper_getLegacyDomNodeInstanceFromSource($xml, 'items item');\n\n expect($domNode)->toBeInstanceOf(DOMNode::class);\n\n $node = new class ($domNode) extends Node {\n protected function makeChildNodeInstance(object $node): Node\n {\n return new XmlElement($node);\n }\n };\n\n expect($node)->toBeInstanceOf(Node::class)\n ->and($node->text())->toBe('1Foo');\n});\n\n/* ----------------------------- querySelector(All)() ----------------------------- */\n\n$html = <<\n \n \n Foo\n \n \n
    \n

    Title

    \n
    \n \n \n HTML;\n\nit('selects an element within a node via querySelector()', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selectedNode = $node->querySelector('.foo h1');\n\n expect($selectedNode)->toBeInstanceOf(Node::class)\n ->and($selectedNode?->text())->toBe('Title');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html)],\n [helper_getLegacyDomNodeInstanceFromSource($html)],\n]);\n\nit('selects an element within a node via querySelector() in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selectedNode = $node->querySelector('.foo h1');\n\n expect($selectedNode)->toBeInstanceOf(Node::class)\n ->and($selectedNode?->text())->toBe('Title');\n})->group('php84');\n\n$html = <<\n \n Bar\n \n
    \n

    Foo

    \n
    \n
    \n

    Bar

    \n
    \n \n \n HTML;\n\ntest(\n 'querySelector() selects the first element within a node, when multiple nodes match a selector',\n function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selectedNode = $node->querySelector('.foo h2');\n\n expect($selectedNode)->toBeInstanceOf(Node::class)\n ->and($selectedNode?->text())->toBe('Foo');\n },\n)->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html)],\n [helper_getLegacyDomNodeInstanceFromSource($html)],\n]);\n\nit(\n 'selects the first element within a node using querySelector(), when multiple nodes match a selector in PHP >= 8.4',\n function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selectedNode = $node->querySelector('.foo h2');\n\n expect($selectedNode)->toBeInstanceOf(Node::class)\n ->and($selectedNode?->text())->toBe('Foo');\n },\n)->group('php84');\n\n$html = <<\n \n Foo\n \n yo\n \n \n HTML;\n\nit('returns null when the selector passed to querySelector() matches nothing', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selectedNode = $node->querySelector('.foo h2');\n\n expect($selectedNode)->toBeNull();\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html)],\n [helper_getLegacyDomNodeInstanceFromSource($html)],\n]);\n\nit('returns null when the selector passed to querySelector() matches nothing in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selectedNode = $node->querySelector('.foo h2');\n\n expect($selectedNode)->toBeNull();\n})->group('php84');\n\n$xml = <<\n \n \n 1Foo\n 2Bar\n 3Baz\n \n \n XML;\n\nit('selects all elements within a node, matching a selector using querySelectorAll()', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->querySelectorAll('items item title');\n\n expect($selected)->toBeInstanceOf(NodeList::class)\n ->and($selected->count())->toBe(3)\n ->and($selected->first()?->text())->toBe('Foo')\n ->and($selected->nth(2)?->text())->toBe('Bar')\n ->and($selected->last()?->text())->toBe('Baz');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($xml, 'feed')],\n [helper_getLegacyDomNodeInstanceFromSource($xml, 'feed')],\n]);\n\nit(\n 'selects all elements within a node, matching a selector using querySelectorAll() in PHP >= 8.4',\n function () use ($xml) {\n $originalNode = helper_getPhp84XmlDomNodeInstanceFromSource($xml, 'feed');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->querySelectorAll('items item title');\n\n expect($selected)->toBeInstanceOf(NodeList::class)\n ->and($selected->count())->toBe(3)\n ->and($selected->first()?->text())->toBe('Foo')\n ->and($selected->nth(2)?->text())->toBe('Bar')\n ->and($selected->last()?->text())->toBe('Baz');\n },\n)->group('php84');\n\n$xml = <<\n \n 123\n \n XML;\n\nit(\n 'gets an empty NodeList when nothing matches the selector passed to querySelectorAll()',\n function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->querySelectorAll('items item author');\n\n expect($selected)->toBeInstanceOf(NodeList::class)\n ->and($selected->count())->toBe(0);\n },\n)->with([\n [helper_getSymfonyCrawlerInstanceFromSource($xml, 'feed')],\n [helper_getLegacyDomNodeInstanceFromSource($xml, 'feed')],\n]);\n\nit(\n 'gets an empty NodeList when nothing matches the selector passed to querySelectorAll() in PHP >= 8.4',\n function () use ($xml) {\n $originalNode = helper_getPhp84XmlDomNodeInstanceFromSource($xml, 'feed');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->querySelectorAll('items item author');\n\n expect($selected)->toBeInstanceOf(NodeList::class)\n ->and($selected->count())->toBe(0);\n },\n)->group('php84');\n\n$html = <<\n \n Lorem Ipsum\n \n
    • hip
    • hop
    • hooray
    \n \n \n HTML;\n\n/* ----------------------------- queryXPath() ----------------------------- */\n\nit(\n 'selects all elements within a node, matching an XPath query using queryXPath()',\n function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->queryXPath('//ul/li');\n\n expect($selected)->toBeInstanceOf(NodeList::class)\n ->and($selected->count())->toBe(3)\n ->and($selected->first()?->text())->toBe('hip')\n ->and($selected->nth(2)?->text())->toBe('hop')\n ->and($selected->last()?->text())->toBe('hooray');\n },\n)->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html)],\n [helper_getLegacyDomNodeInstanceFromSource($html)],\n]);\n\nit(\n 'selects all elements within a node, matching an XPath query using queryXPath() in PHP >= 8.4',\n function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->queryXPath('//ul/li');\n\n expect($selected)->toBeInstanceOf(NodeList::class)\n ->and($selected->count())->toBe(3)\n ->and($selected->first()?->text())->toBe('hip')\n ->and($selected->nth(2)?->text())->toBe('hop')\n ->and($selected->last()?->text())->toBe('hooray');\n },\n)->group('php84');\n\nit('gets an empty NodeList when nothing matches the selector passed to queryXPath()', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->queryXPath('//ul/li/strong');\n\n expect($selected)->toBeInstanceOf(NodeList::class)\n ->and($selected->count())->toBe(0);\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html)],\n [helper_getLegacyDomNodeInstanceFromSource($html)],\n]);\n\nit(\n 'gets an empty NodeList when nothing matches the selector passed to queryXPath() in PHP => 8.4',\n function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->queryXPath('//ul/li/strong');\n\n expect($selected)->toBeInstanceOf(NodeList::class)\n ->and($selected->count())->toBe(0);\n },\n)->group('php84');\n\n/* ----------------------------- removeNodesMatchingSelector() ----------------------------- */\n\n$html = <<\n \n \n \n
      \n
    • foo
      • \n
    • bar
      • \n
    • baz
      • \n
    • quz
      • \n
    • lorem
      • \n
    \n \n \n HTML;\n\nit('removes all nodes that match a given CSS selector', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $node->removeNodesMatchingSelector('#list .remove');\n\n $sourceAfterRemoval = $node->outer();\n\n expect($sourceAfterRemoval)->toContain('
  • bar
    • ')\n ->toContain('
  • baz
    • ')\n ->not()->toContain('
  • ')\n ->not()->toContain('foo')\n ->not()->toContain('quz');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html)],\n [helper_getLegacyDomNodeInstanceFromSource($html)],\n]);\n\nit('removes all nodes that match a given CSS selector in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $node->removeNodesMatchingSelector('#list .remove');\n\n $sourceAfterRemoval = $node->outer();\n\n expect($sourceAfterRemoval)->toContain('
  • bar
    • ')\n ->toContain('
  • baz
    • ')\n ->not()->toContain('
  • ')\n ->not()->toContain('foo')\n ->not()->toContain('quz');\n})->group('php84');\n\n$xml = <<\n \n \n \n 1\n foo\n lorem\n \n \n 2\n bar\n ipsum\n \n \n 3\n baz\n dolor\n \n \n \n XML;\n\nit('removes all nodes that match a given CSS selector from XML', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode, false);\n\n $node->removeNodesMatchingSelector('feed items item title');\n\n $sourceAfterRemoval = $node->outer();\n\n expect($sourceAfterRemoval)->toContain('')\n ->toContain('')\n ->not()->toContain('');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($xml, 'feed')],\n]);\n\nit('removes all nodes that match a given CSS selector from XML in PHP >= 8.4', function () use ($xml) {\n $originalNode = helper_getPhp84XmlDomNodeInstanceFromSource($xml, 'feed');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $node->removeNodesMatchingSelector('feed items item title');\n\n $sourceAfterRemoval = $node->outer();\n\n expect($sourceAfterRemoval)->toContain('<id>')\n ->toContain('<description>')\n ->not()->toContain('<title>');\n})->group('php84');\n\n/* ----------------------------- removeNodesMatchingXPath() ----------------------------- */\n\n$html = <<<HTML\n <!doctype html>\n <html>\n <head></head>\n <body>\n <ul id=\"list\">\n <li class=\"remove\">foo</li>\n <li>bar</li>\n <li>baz</li>\n <li class=\"remove\">quz</li>\n <li>lorem</li>\n </ul>\n </body>\n </html>\n HTML;\n\nit('removes all nodes that match a given XPath query', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $node->removeNodesMatchingXPath('//li[contains(@class, \\'remove\\')]');\n\n $sourceAfterRemoval = $node->outer();\n\n expect($sourceAfterRemoval)->toContain('<li>bar</li>')\n ->toContain('<li>baz</li>')\n ->not()->toContain('<li class=\"remove\">')\n ->not()->toContain('foo')\n ->not()->toContain('quz');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html)],\n [helper_getLegacyDomNodeInstanceFromSource($html)],\n]);\n\nit('removes all nodes that match a given XPath query in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $node->removeNodesMatchingXPath('//li[contains(@class, \\'remove\\')]');\n\n $sourceAfterRemoval = $node->outer();\n\n expect($sourceAfterRemoval)->toContain('<li>bar</li>')\n ->toContain('<li>baz</li>')\n ->not()->toContain('<li class=\"remove\">')\n ->not()->toContain('foo')\n ->not()->toContain('quz');\n})->group('php84');\n\n$xml = <<<XML\n <?xml version=\"1.0\" encoding=\"utf-8\"?>\n <feed>\n <items>\n <item>\n <id>1</id>\n <title>foo\n lorem\n \n \n 2\n bar\n ipsum\n \n \n 3\n baz\n dolor\n \n \n \n XML;\n\nit('removes all nodes that match a given XPath query from XML', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $node->removeNodesMatchingXPath('//feed/items/item/title');\n\n $sourceAfterRemoval = $node->outer();\n\n expect($sourceAfterRemoval)->toContain('')\n ->toContain('')\n ->not()->toContain('');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($xml, 'feed')],\n]);\n\nit('removes all nodes that match a given XPath query from XML in PHP >= 8.4', function () use ($xml) {\n $originalNode = helper_getPhp84XmlDomNodeInstanceFromSource($xml, 'feed');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $node->removeNodesMatchingXPath('//feed/items/item/title');\n\n $sourceAfterRemoval = $node->outer();\n\n expect($sourceAfterRemoval)->toContain('<id>')\n ->toContain('<description>')\n ->not()->toContain('<title>');\n})->group('php84');\n\n/* ----------------------------- getAttribute() ----------------------------- */\n\n$html = <<<HTML\n <!doctype html>\n <html>\n <head><title>Foo\n \n
    \n \n \n HTML;\n\nit('gets the value of an attribute', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->getAttribute('data-test'))->toBe('hi');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html, '.element')],\n [helper_getLegacyDomNodeInstanceFromSource($html, '.element')],\n]);\n\nit('gets the value of an attribute in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html, '.element');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->getAttribute('data-test'))->toBe('hi');\n})->group('php84');\n\n$html = <<\n \n Foo\n
    \n \n HTML;\n\nit('returns null when an attribute does not exist', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->getAttribute('data-test'))->toBeNull();\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html, '.element')],\n [helper_getLegacyDomNodeInstanceFromSource($html, '.element')],\n]);\n\nit('returns null when an attribute does not exist in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html, '.element');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->getAttribute('data-test'))->toBeNull();\n})->group('php84');\n\n/* ----------------------------- nodeName() ----------------------------- */\n\nit('gets the name of a node', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->nodeName())->toBe('div');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html, '.element')],\n [helper_getLegacyDomNodeInstanceFromSource($html, '.element')],\n]);\n\nit('gets the name of a node in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html, '.element');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->nodeName())->toBe('div');\n})->group('php84');\n\n$html = <<\n \n Bar\n \n

    Title

    Lorem ipsum.

    \n \n \n HTML;\n\n/* ----------------------------- text() ----------------------------- */\n\nit('gets the text content of an HTML node', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->text())->toBe('Title Lorem ipsum.');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html, 'article')],\n [helper_getLegacyDomNodeInstanceFromSource($html, 'article')],\n]);\n\nit('gets the text content of an HTML node in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html, 'article');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->text())->toBe('Title Lorem ipsum.');\n})->group('php84');\n\n/* ----------------------------- innerSource() ----------------------------- */\n\nit('gets the inner source of an HTML node', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->inner())->toBe('

    Title

    Lorem ipsum.

    ');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html, 'article')],\n [helper_getLegacyDomNodeInstanceFromSource($html, 'article')],\n]);\n\nit('gets the inner source of an HTML node in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html, 'article');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->inner())->toBe('

    Title

    Lorem ipsum.

    ');\n})->group('php84');\n\n/* ----------------------------- outerSource () ----------------------------- */\n\nit('gets the outer source of an HTML node', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->outer())->toBe('

    Title

    Lorem ipsum.

    ');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($html, 'article')],\n [helper_getLegacyDomNodeInstanceFromSource($html, 'article')],\n]);\n\nit('gets the outer source of an HTML node in PHP >= 8.4', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html, 'article');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->outer())->toBe('

    Title

    Lorem ipsum.

    ');\n})->group('php84');\n\n$xml = <<\n 1 Lorem Ipsum \n XML;\n\n/* ----------------------------- text() ----------------------------- */\n\nit('gets the text content of an XML node', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->text())->toBe('1 Lorem Ipsum');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($xml, 'items item')],\n [helper_getLegacyDomNodeInstanceFromSource($xml, 'items item')],\n]);\n\nit('gets the text content of an XML node in PHP >= 8.4', function () use ($xml) {\n $originalNode = helper_getPhp84XmlDomNodeInstanceFromSource($xml, 'items item');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->text())->toBe('1 Lorem Ipsum');\n})->group('php84');\n\n/* ----------------------------- innerSource() XML ----------------------------- */\n\nit('gets the inner source of an XML node', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->inner())->toBe(' 1 Lorem Ipsum ');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($xml, 'items item')],\n [helper_getLegacyDomNodeInstanceFromSource($xml, 'items item')],\n]);\n\nit('gets the inner source of an XML node in PHP >= 8.4', function () use ($xml) {\n $originalNode = helper_getPhp84XmlDomNodeInstanceFromSource($xml, 'items item');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->inner())->toBe(' 1 Lorem Ipsum ');\n})->group('php84');\n\n/* ----------------------------- outerSource() XML ----------------------------- */\n\nit('gets the outer source of an XML node', function (object $originalNode) {\n /** @var Crawler|DOMNode $originalNode */\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->outer())->toBe(' 1 Lorem Ipsum ');\n})->with([\n [helper_getSymfonyCrawlerInstanceFromSource($xml, 'items item')],\n [helper_getLegacyDomNodeInstanceFromSource($xml, 'items item')],\n]);\n\nit('gets the outer source of an XML node in PHP >= 8.4', function () use ($xml) {\n $originalNode = helper_getPhp84XmlDomNodeInstanceFromSource($xml, 'items item');\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n expect($node->outer())->toBe(' 1 Lorem Ipsum ');\n})->group('php84');\n\n$html = <<\n \n Bar\n \n
    • one
    \n\n
    • foo
    \n \n \n HTML;\n\n/* ------------ :has() :not() CSS pseudo class selectors in PHP 8.4 ------------- */\n\nit('selects elements using a CSS selector containing the :has() pseudo class', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->querySelector('ul:has(.foo)');\n\n expect($selected)->toBeInstanceOf(HtmlElement::class)\n ->and($selected?->text())->toBe('one');\n})->group('php84');\n\nit('selects elements using a CSS selector containing the :not() pseudo class', function () use ($html) {\n $originalNode = helper_getPhp84HtmlDomNodeInstanceFromSource($html);\n\n $node = helper_getAbstractNodeInstance($originalNode);\n\n $selected = $node->querySelector('ul:not(:has(.foo))');\n\n expect($selected)->toBeInstanceOf(HtmlElement::class)\n ->and($selected?->text())->toBe('foo');\n})->group('php84');\n" }, { "path": "tests/Steps/Dom/XmlDocumentTest.php", "content": "\n \n 1\n \n XML;\n\n $document = new XmlDocument($xml);\n\n expect($document->querySelector('feed items item'))->toBeInstanceOf(XmlElement::class);\n});\n\ntest('the querySelectorAll() method returns a NodeList of XmlElement objects', function () {\n $xml = <<\n \n 123\n \n XML;\n\n $document = new XmlDocument($xml);\n\n $nodeList = $document->querySelectorAll('feed items item');\n\n expect($nodeList)->toBeInstanceOf(NodeList::class);\n\n $anyNodesChecked = false;\n\n foreach ($nodeList as $node) {\n expect($node)->toBeInstanceOf(XmlElement::class);\n\n $anyNodesChecked = true;\n }\n\n expect($anyNodesChecked)->toBeTrue();\n});\n\ntest('the queryXPath() method returns a NodeList of XmlElement objects', function () {\n $xml = <<\n \n 123\n \n XML;\n\n $document = new XmlDocument($xml);\n\n $nodeList = $document->queryXPath('//feed/items/item');\n\n expect($nodeList)->toBeInstanceOf(NodeList::class);\n\n $anyNodesChecked = false;\n\n foreach ($nodeList as $node) {\n expect($node)->toBeInstanceOf(XmlElement::class);\n\n $anyNodesChecked = true;\n }\n\n expect($anyNodesChecked)->toBeTrue();\n});\n\nit('is able to parse documents containing characters that aren\\'t valid within XML documents', function (string $char) {\n $xml = <<\n \n \n \n \n <![CDATA[foo - {$char} - bar]]>\n \n \n \n \n XML;\n\n $document = new XmlDocument($xml);\n\n $titles = $document->querySelectorAll('channel item title');\n\n expect($titles)->toBeInstanceOf(NodeList::class)\n ->and($titles->count())->toBe(1)\n ->and($titles->first()?->text())->toStartWith('foo - ')\n ->and($titles->first()?->text())->toEndWith(' - bar');\n})->with([\n [mb_chr(0)],\n [mb_chr(6)],\n [mb_chr(12)],\n [mb_chr(20)],\n [mb_chr(31)],\n [mb_chr(128)],\n [mb_chr(157)],\n [mb_chr(195)],\n [mb_chr(253)],\n]);\n" }, { "path": "tests/Steps/Dom/XmlElementTest.php", "content": "\n \n foo\n foo\n \n \n abc-123\n 2024-11-07T11:00:31Z\n Foo bar baz!\n https://www.example.com/item-1?utm_source=foo&utm_medium=feed-xml\n test \n \n \n abc-124\n 2024-12-04T22:43:14Z\n Lorem Ipsum!\n https://www.example.com/item-2?utm_source=foo&utm_medium=feed-xml\n heyho\n \n \n \n XML;\n\ntest('child nodes selected via querySelector() are HtmlElement instances', function () use ($xml) {\n $document = new XmlDocument($xml);\n\n $wrapperElement = $document->querySelector('feed');\n\n expect($wrapperElement)->toBeInstanceOf(XmlElement::class)\n ->and($wrapperElement?->querySelector('items item'))->toBeInstanceOf(XmlElement::class);\n});\n\ntest('child nodes selected via querySelectorAll() are HtmlElement instances', function () use ($xml) {\n $document = new XmlDocument($xml);\n\n $wrapperElement = $document->querySelector('feed');\n\n expect($wrapperElement)->toBeInstanceOf(XmlElement::class);\n\n $childNodeList = $wrapperElement?->querySelectorAll('items item');\n\n expect($childNodeList)->toBeInstanceOf(NodeList::class)\n ->and($childNodeList?->count())->toBe(2)\n ->and($childNodeList?->first())->toBeInstanceOf(XmlElement::class)\n ->and($childNodeList?->last())->toBeInstanceOf(XmlElement::class);\n});\n\nit('gets the node name', function () use ($xml) {\n $document = new XmlDocument($xml);\n\n $node = $document->querySelector('feed');\n\n expect($node?->nodeName())->toBe('feed')\n ->and($node?->querySelector('items item')?->nodeName())->toBe('item');\n});\n\nit('gets the text of a node', function () use ($xml) {\n $document = new XmlDocument($xml);\n\n $node = $document->querySelector('feed items item:nth-child(2) foo');\n\n expect($node?->text())->toBe('heyho');\n});\n\nit('gets the outer XML of a node', function () use ($xml) {\n $document = new XmlDocument($xml);\n\n $node = $document->querySelector('feed items item foo baRbaz');\n\n expect($node?->outerXml())->toBe('test');\n});\n\nit('gets the inner XML of a node', function () use ($xml) {\n $document = new XmlDocument($xml);\n\n $node = $document->querySelector('feed items item foo');\n\n expect($node?->innerXml())->toBe(' test ');\n});\n\nit('gets an attribute from a node', function () use ($xml) {\n $document = new XmlDocument($xml);\n\n $node = $document->querySelector('feed items item:first-child title');\n\n expect($node?->getAttribute('lang'))->toBe('en');\n});\n" }, { "path": "tests/Steps/Dom/_Stubs/HtmlNodeStub.php", "content": "innerSource();\n }\n\n public function outer(): string\n {\n return $this->outerSource();\n }\n\n protected function makeChildNodeInstance(object $node): Node\n {\n return new HtmlElement($node);\n }\n}\n" }, { "path": "tests/Steps/Dom/_Stubs/XmlNodeStub.php", "content": "innerSource();\n }\n\n public function outer(): string\n {\n return $this->outerSource();\n }\n\n protected function makeChildNodeInstance(object $node): Node\n {\n return new XmlElement($node);\n }\n}\n" }, { "path": "tests/Steps/DomTest.php", "content": "

    Überschrift

    ';\n\n $output = helper_invokeStepWithInput(helper_getDomStepInstance()::root(), $html);\n\n expect($output[0]->get())->toBe([]);\n});\n\ntest('ResponseInterface is a valid input', function () {\n $output = helper_invokeStepWithInput(helper_getDomStepInstance()::root(), new Response());\n\n expect($output[0]->get())->toBe([]);\n});\n\ntest('RespondedRequest is a valid input', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root(),\n new RespondedRequest(new Request('GET', '/'), new Response()),\n );\n\n expect($output[0]->get())->toBe([]);\n});\n\ntest('For other inputs an error message is logged', function (mixed $input) {\n $logger = new DummyLogger();\n\n helper_traverseIterable(helper_getDomStepInstance()::root()->addLogger($logger)->invokeStep(new Input($input)));\n\n expect($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toStartWith('A step was called with input that it can not work with: ')\n ->and($logger->messages[0]['message'])->toEndWith('. The invalid input is of type ' . gettype($input) . '.');\n})->with([\n [123],\n [123.456],\n [new stdClass()],\n]);\n\nit('outputs a single string when argument for extract is a selector string matching only one element', function () {\n $outputs = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root()->extract('.list .item:first-child .match'),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe('match 2');\n});\n\nit('outputs multiple strings when argument for extract is a selector string matching multiple elements', function () {\n $outputs = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root()->extract('.match'),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe('match 1')\n ->and($outputs[2]->get())->toBe('match 3');\n});\n\nit('also takes a DomQuery instance as argument for extract', function () {\n $outputs = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root()->extract(Dom::cssSelector('.list .item:first-child .match')),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe('match 2');\n});\n\ntest('Extracting with single selector also works with each', function () {\n $outputs = helper_invokeStepWithInput(\n helper_getDomStepInstance()::each('.list .item')->extract('.match'),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe('match 2')\n ->and($outputs[1]->get())->toBe('match 3');\n});\n\ntest('Extracting with single selector also works with first', function () {\n $outputs = helper_invokeStepWithInput(\n helper_getDomStepInstance()::first('.list .item')->extract('.match'),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe('match 2');\n});\n\ntest('Extracting with single selector also works with last', function () {\n $outputs = helper_invokeStepWithInput(\n helper_getDomStepInstance()::last('.list .item')->extract('.match'),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe('match 3');\n});\n\ntest('Extracting with single selector that doesn\\'t match anything doesn\\'t yield any output', function () {\n $outputs = helper_invokeStepWithInput(\n helper_getDomStepInstance()::last('.list .item')->extract('.m\\ätch'),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($outputs)->toHaveCount(0);\n});\n\nit('extracts one result from the root node when the root method is used', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root()->extract(['matches' => '.match']),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['matches' => ['match 1', 'match 2', 'match 3']]);\n});\n\nit('extracts each matching result when the each method is used', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::each('.list .item')->extract(['match' => '.match']),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($output)->toHaveCount(2)\n ->and($output[0]->get())->toBe(['match' => 'match 2'])\n ->and($output[1]->get())->toBe(['match' => 'match 3']);\n});\n\nit('logs a warning, when the each() method is used with an empty selector', function (string|DomQuery $selector) {\n $logger = new DummyLogger();\n\n $step = helper_getDomStepInstance()::each($selector)->extract(['match' => '.match']);\n\n $step->addLogger($logger);\n\n $outputs = helper_invokeStepWithInput($step, helper_getStepFilesContent('Html/basic.html'));\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['match' => ['match 1', 'match 2', 'match 3']])\n ->and($logger->messages[0]['level'])->toBe('warning')\n ->and($logger->messages[0]['message'])\n ->toStartWith('The selector you provided for the ‘each’ option is empty.');\n})->with([\n [''],\n [Dom::cssSelector('')],\n [Dom::xPath('')],\n]);\n\nit('extracts the first matching result when the first method is used', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::first('.list .item')->extract(['match' => '.match']),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['match' => 'match 2']);\n});\n\nit('logs a warning, when the first() method is used with an empty selector', function (string|DomQuery $selector) {\n $logger = new DummyLogger();\n\n $step = helper_getDomStepInstance()::first($selector)->extract(['match' => '.match']);\n\n $step->addLogger($logger);\n\n $outputs = helper_invokeStepWithInput($step, helper_getStepFilesContent('Html/basic.html'));\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['match' => ['match 1', 'match 2', 'match 3']])\n ->and($logger->messages[0]['level'])->toBe('warning')\n ->and($logger->messages[0]['message'])\n ->toStartWith('The selector you provided for the ‘first’ option is empty.');\n})->with([\n [''],\n [Dom::cssSelector('')],\n [Dom::xPath('')],\n]);\n\nit('extracts the last matching result when the last method is used', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::last('.list .item')->extract(['match' => '.match']),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['match' => 'match 3']);\n});\n\nit('logs a warning, when the last() method is used with an empty selector', function (string|DomQuery $selector) {\n $logger = new DummyLogger();\n\n $step = helper_getDomStepInstance()::last($selector)->extract(['match' => '.match']);\n\n $step->addLogger($logger);\n\n $outputs = helper_invokeStepWithInput($step, helper_getStepFilesContent('Html/basic.html'));\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['match' => ['match 1', 'match 2', 'match 3']])\n ->and($logger->messages[0]['level'])->toBe('warning')\n ->and($logger->messages[0]['message'])\n ->toStartWith('The selector you provided for the ‘last’ option is empty.');\n})->with([\n [''],\n [Dom::cssSelector('')],\n [Dom::xPath('')],\n]);\n\nit('doesn\\'t yield any output when the each selector doesn\\'t match anything', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::each('.list .ytem')->extract(['match' => '.match']),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($output)->toHaveCount(0);\n});\n\nit('doesn\\'t yield any output when the first selector doesn\\'t match anything', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::first('.list .ytem')->extract(['match' => '.match']),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($output)->toHaveCount(0);\n});\n\nit('doesn\\'t yield any output when the last selector doesn\\'t match anything', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::last('.list .otem')->extract(['match' => '.match']),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($output)->toHaveCount(0);\n});\n\nit('returns an array with null values when selectors in an extract array mapping don\\'t match anything', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::last('.list .item')->extract(['match' => '.match', 'noMatch' => '.doesntMatch']),\n helper_getStepFilesContent('Html/basic.html'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['match' => 'match 3', 'noMatch' => null]);\n});\n\ntest('The static cssSelector method returns an instance of CssSelector using the provided selector', function () {\n $cssSelector = Dom::cssSelector('.item');\n\n expect($cssSelector)->toBeInstanceOf(CssSelector::class);\n\n $itemContent = $cssSelector->apply(new Dom\\HtmlDocument('yes'));\n\n expect($itemContent)->toBe('yes');\n});\n\ntest('The static xPath method returns an instance of XPathQuery using the provided query', function () {\n $xPathQuery = Dom::xPath('//item');\n\n expect($xPathQuery)->toBeInstanceOf(XPathQuery::class);\n\n $itemContent = $xPathQuery->apply(new Dom\\XmlDocument('yes'));\n\n expect($itemContent)->toBe('yes');\n});\n\nit('uses the keys of the provided mapping as keys in the returned output', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root()->extract(['foo' => '.foo', 'notBar' => '.bar', '.baz']),\n '

    foo content

    bar content

    baz content

    ',\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'foo content', 'notBar' => 'bar content', 0 => 'baz content']);\n});\n\nit('trims the extracted data', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root()->extract(['foo' => '.foo']),\n \"

    \\n foo content \\n \\n

    \",\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'foo content']);\n});\n\nit('automatically passes on the base url to dom query instances when the input is a RespondedRequest', function () {\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root()->extract([\n 'one' => Dom::cssSelector('#one')->attribute('href')->toAbsoluteUrl(),\n 'two' => Dom::cssSelector('#two')->link(),\n ]),\n new RespondedRequest(\n new Request('GET', 'https://www.example.com/home'),\n new Response(body: '

    foo bar yolo

    '),\n ),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe([\n 'one' => 'https://www.example.com/foo/bar',\n 'two' => 'https://www.example.com/yo/lo',\n ]);\n});\n\nit('removes the fragment part from URLs when the withoutFragment method is called on a DomQuery instance', function () {\n $body = <<\n one
    \n two
    \n three
    \n four
    \n

    \n HTML;\n\n $output = helper_invokeStepWithInput(\n helper_getDomStepInstance()::root()->extract([\n 'one' => Dom::cssSelector('#one')->link(),\n 'two' => Dom::xPath('//a[@id=\\'two\\']')->link(),\n 'three' => Dom::cssSelector('#three')->link()->withoutFragment(),\n 'four' => Dom::xPath('//a[@id=\\'four\\']')->link()->withoutFragment(),\n ]),\n new RespondedRequest(\n new Request('GET', 'https://www.example.com/home'),\n new Response(body: $body),\n ),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe([\n 'one' => 'https://www.example.com/foo#foo',\n 'two' => 'https://www.example.com/bar#bar',\n 'three' => 'https://www.example.com/baz',\n 'four' => 'https://www.example.com/quz',\n ]);\n});\n" }, { "path": "tests/Steps/Filters/ArrayFilterTest.php", "content": "where(Filter::equal('foo'));\n\n expect($filter->evaluate($values))->toBe($evaluationResult);\n})->with([\n [['foo', 'bar', 'baz'], true],\n [['bar', 'baz', 'quz'], false],\n]);\n\nit('filters a multi-level array by a key of the array elements (which are also arrays)', function () {\n $values = [\n ['foo' => 'one', 'bar' => 'two'],\n ['foo' => 'two', 'bar' => 'three'],\n ['foo' => 'three', 'bar' => 'four'],\n ];\n\n $filter = Filter::arrayHasElement()->where('foo', Filter::equal('four'));\n\n expect($filter->evaluate($values))->toBeFalse();\n\n $filter = Filter::arrayHasElement()->where('foo', Filter::equal('two'));\n\n expect($filter->evaluate($values))->toBeTrue();\n});\n\nit('applies multiple complex filters on a multi-level array', function () {\n $values = [\n [\n 'id' => '123',\n 'name' => 'abc',\n 'tags' => [\n ['type' => 'companyId', 'value' => '123'],\n ['type' => 'type', 'value' => 'job-ad'],\n ['type' => 'companyId', 'value' => '125'],\n ],\n ],\n [\n 'id' => '124',\n 'name' => 'abd',\n 'tags' => [\n ['type' => 'companyId', 'value' => '123'],\n ['type' => 'type', 'value' => 'blog-post'],\n ['type' => 'author', 'value' => 'John Doe'],\n ],\n ],\n [\n 'id' => '125',\n 'name' => 'abf',\n 'tags' => [\n ['type' => 'companyId', 'value' => '123'],\n ['type' => 'companyId', 'value' => '124'],\n ['type' => 'type', 'value' => 'job-ad'],\n ['type' => 'companyId', 'value' => '125'],\n ],\n ],\n ];\n\n $filter = Filter::arrayHasElement()\n ->where(\n 'tags',\n Filter::arrayHasElement()\n ->where('type', Filter::equal('companyId'))\n ->where('value', Filter::equal('123')),\n )\n ->where(\n 'tags',\n Filter::arrayHasElement()\n ->where('type', Filter::equal('companyId'))\n ->where('value', Filter::equal('124'))\n ->negate(),\n )\n ->where(\n 'tags',\n Filter::arrayHasElement()\n ->where('type', Filter::equal('type'))\n ->where('value', Filter::equal('job-ad')),\n );\n\n expect($filter->evaluate($values))->toBeTrue();\n\n $filter = Filter::arrayHasElement()\n ->where(\n 'tags',\n Filter::arrayHasElement()\n ->where('type', Filter::equal('companyId'))\n ->where('value', Filter::equal('123')),\n )\n ->where(\n 'tags',\n Filter::arrayHasElement()\n ->where('type', Filter::equal('companyId'))\n ->where('value', Filter::equal('125'))\n ->negate(),\n )\n ->where(\n 'tags',\n Filter::arrayHasElement()\n ->where('type', Filter::equal('type'))\n ->where('value', Filter::equal('job-ad')),\n );\n\n expect($filter->evaluate($values))->toBeFalse();\n});\n" }, { "path": "tests/Steps/Filters/ClosureFilterTest.php", "content": "evaluate('one'))->toBeTrue();\n\n expect($closure->evaluate('four'))->toBeFalse();\n});\n\nit('evaluates with a value from an array by key', function () {\n $closure = new ClosureFilter(function (mixed $value) {\n return in_array($value, ['one', 'two', 'three'], true);\n });\n\n $closure->useKey('bar');\n\n expect($closure->evaluate(['foo' => 'one', 'bar' => 'two']))->toBeTrue();\n\n expect($closure->evaluate(['foo' => 'three', 'bar' => 'four']))->toBeFalse();\n});\n\nit('compares a value from an object by key', function () {\n $closure = new ClosureFilter(function (mixed $value) {\n return in_array($value, ['one', 'two', 'three'], true);\n });\n\n $closure->useKey('bar');\n\n expect($closure->evaluate(helper_getStdClassWithData(['foo' => 'one', 'bar' => 'two'])))->toBeTrue();\n\n expect($closure->evaluate(helper_getStdClassWithData(['foo' => 'three', 'bar' => 'four'])))->toBeFalse();\n});\n" }, { "path": "tests/Steps/Filters/ComparisonFilterTest.php", "content": "evaluate(4))->toBeTrue()\n ->and($comparison->evaluate(2))->toBeFalse();\n});\n\nit('compares a value from an array by key', function () {\n $comparison = new ComparisonFilter(ComparisonFilterRule::NotEqual, 'barValue');\n\n $comparison->useKey('bar');\n\n expect($comparison->evaluate(['foo' => 'fooValue', 'bar' => 'barValue']))->toBeFalse()\n ->and($comparison->evaluate(['foo' => 'fooValue', 'bar' => 'barzValue']))->toBeTrue();\n});\n\nit('compares a value from an object by key', function () {\n $comparison = new ComparisonFilter(ComparisonFilterRule::NotEqual, 'barValue');\n\n $comparison->useKey('bar');\n\n expect($comparison->evaluate(helper_getStdClassWithData(['foo' => 'fooValue', 'bar' => 'barValue'])))->toBeFalse()\n ->and($comparison->evaluate(helper_getStdClassWithData(['foo' => 'fooValue', 'bar' => 'barzValue'])))->toBeTrue();\n});\n" }, { "path": "tests/Steps/Filters/Enums/ComparisonFilterRuleTest.php", "content": "evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 1, 1],\n [true, 'one', 'one'],\n [true, 1.12, 1.12],\n [false, 1, 2],\n [false, 1, '1'],\n [false, 'one', 'two'],\n [false, 1.12, 1.122],\n]);\n\nit('correctly applies not equal operator', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = ComparisonFilterRule::NotEqual;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [false, 1, 1],\n [false, 'one', 'one'],\n [false, 1.12, 1.12],\n [true, 1, 2],\n [true, 1, '1'],\n [true, 'one', 'two'],\n [true, 1.12, 1.122],\n]);\n\nit('correctly applies greater than operator', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = ComparisonFilterRule::GreaterThan;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 1, 0],\n [true, 12, 3],\n [true, 1.12, 1.11],\n [false, 11, 11],\n [false, 0, 1],\n [false, 3.59, 3.591],\n [true, '123', '122'],\n [true, '123', 122],\n [true, 123, '122'],\n [false, '123', '124'],\n [false, '123', 124],\n [false, 123, '124'],\n [true, '123.45', '123.44'],\n [true, '123.45', 123.44],\n [true, 123.45, '123.44'],\n [false, '123.45', '123.46'],\n [false, '123.45', 123.46],\n [false, 123.45, '123.46'],\n]);\n\nit('correctly applies greater than or equal operator', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = ComparisonFilterRule::GreaterThanOrEqual;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 1, 0],\n [true, 12, 3],\n [true, 1.12, 1.11],\n [true, 11, 11],\n [false, 0, 1],\n [false, 3.59, 3.591],\n [true, '123', '122'],\n [true, '123', 122],\n [true, 123, '123'],\n [false, '123', '124'],\n [false, '123', 124],\n [false, 123, '124'],\n [true, '123.45', '123.44'],\n [true, '123.44', 123.44],\n [true, 123.45, '123.44'],\n [false, '123.45', '123.46'],\n [false, '123.45', 123.46],\n [false, 123.45, '123.46'],\n]);\n\nit('correctly applies less than operator', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = ComparisonFilterRule::LessThan;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 0, 1],\n [true, 4, 5],\n [true, 5.79, 5.7901],\n [false, 11, 11],\n [false, 1, 0],\n [false, 9.2901, 9.29],\n [true, '123', '124'],\n [true, '123', 124],\n [true, 123, '124'],\n [false, '123', '122'],\n [false, '123', 122],\n [false, 123, '122'],\n [true, '123.45', '123.46'],\n [true, '123.45', 123.46],\n [true, 123.45, '123.46'],\n [false, '123.45', '123.44'],\n [false, '123.45', 123.44],\n [false, 123.45, '123.44'],\n]);\n\nit('correctly applies less than or equal operator', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = ComparisonFilterRule::LessThanOrEqual;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 0, 1],\n [true, 4, 5],\n [true, 5.79, 5.7901],\n [true, 11, 11],\n [false, 1, 0],\n [false, 9.2901, 9.29],\n [true, '123', '124'],\n [true, '123', 124],\n [true, 123, '123'],\n [false, '123', '122'],\n [false, '123', 122],\n [false, 123, '122'],\n [true, '123.45', '123.46'],\n [true, '123.45', 123.45],\n [true, 123.45, '123.46'],\n [false, '123.45', '123.44'],\n [false, '123.45', 123.44],\n [false, 123.45, '123.44'],\n]);\n" }, { "path": "tests/Steps/Filters/Enums/StringFilterRuleTest.php", "content": "evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'foobarbaz', 'foo'],\n [true, 'foo bar baz', 'foo'],\n [true, 'foo bar baz', 'bar'],\n [true, 'foo bar baz', 'baz'],\n [false, 'foo bar baz', 'Foo'],\n]);\n\nit('checks if a string starts with another string', function (\n bool $expectedResult,\n mixed $haystack,\n mixed $needle,\n) {\n $stringFilterRule = StringFilterRule::StartsWith;\n\n expect($stringFilterRule->evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'foobarbaz', 'foo'],\n [true, 'foo bar baz', 'foo'],\n [true, 'foo bar baz', 'foo bar'],\n [false, 'foo bar baz', 'bar'],\n [false, 'foo bar baz', 'baz'],\n [false, 'foo bar baz', 'Foo'],\n]);\n\nit('checks if a string ends with another string', function (\n bool $expectedResult,\n mixed $haystack,\n mixed $needle,\n) {\n $stringFilterRule = StringFilterRule::EndsWith;\n\n expect($stringFilterRule->evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'foobarbaz', 'baz'],\n [true, 'foo bar baz', 'baz'],\n [true, 'foo bar baz', 'bar baz'],\n [false, 'foo bar baz', 'bar'],\n [false, 'foo bar baz', 'foo'],\n [false, 'foo bar baz', 'Baz'],\n]);\n" }, { "path": "tests/Steps/Filters/Enums/StringLengthFilterRuleTest.php", "content": "evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 'foo', 3],\n [true, 'lorem', 5],\n [true, 'foo bar', 7],\n [false, 'bar', 4],\n [false, 'baz quz', 6],\n]);\n\nit('correctly applies not equal rule', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = StringLengthFilterRule::NotEqual;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 'foo', 2],\n [true, 'foo bar', 8],\n [false, 'foo', 3],\n [false, 'lorem ipsum', 11],\n]);\n\nit('correctly applies greater than rule', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = StringLengthFilterRule::GreaterThan;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 'foo', 2],\n [true, 'foo bar', 6],\n [false, 'foo', 3],\n [false, 'foo bar', 7],\n]);\n\nit('correctly applies greater than or equal operator', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = StringLengthFilterRule::GreaterThanOrEqual;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 'foo', 2],\n [true, 'foo', 3],\n [true, 'foo bar', 6],\n [true, 'foo bar', 7],\n [false, 'foo', 4],\n [false, 'foo bar', 8],\n]);\n\nit('correctly applies less than operator', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = StringLengthFilterRule::LessThan;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 'foo', 4],\n [true, 'foo bar', 8],\n [false, 'foo', 3],\n [false, 'foo bar', 7],\n]);\n\nit('correctly applies less than or equal operator', function (bool $expectedResult, mixed $value1, mixed $value2) {\n $comparisonFilterRule = StringLengthFilterRule::LessThanOrEqual;\n\n expect($comparisonFilterRule->evaluate($value1, $value2))->toBe($expectedResult);\n})->with([\n [true, 'foo', 4],\n [true, 'foo', 3],\n [true, 'foo bar', 8],\n [true, 'foo bar', 7],\n [false, 'foo', 2],\n [false, 'foo bar', 6],\n]);\n" }, { "path": "tests/Steps/Filters/Enums/UrlFilterRuleTest.php", "content": "evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'https://www.example.com', 'https'],\n [true, 'http://www.example.com', 'http'],\n [true, 'ftp://user:password@example.com:21/path', 'ftp'],\n [false, 'https://www.example.com', 'http'],\n]);\n\nit('checks if a URL has a certain host', function (bool $expectedResult, mixed $haystack, mixed $needle) {\n $urlFilterRule = UrlFilterRule::Host;\n\n expect($urlFilterRule->evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'https://www.example.com', 'www.example.com'],\n [true, 'https://jobs.example.com', 'jobs.example.com'],\n [true, 'https://pew.pew.pew.example.com:8080/pew', 'pew.pew.pew.example.com'],\n [false, 'https://jobs.example.com', 'www.example.com'],\n]);\n\nit('checks if a URL has a certain domain', function (bool $expectedResult, mixed $haystack, mixed $needle) {\n $urlFilterRule = UrlFilterRule::Domain;\n\n expect($urlFilterRule->evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'https://www.example.com', 'example.com'],\n [true, 'https://jobs.example.com', 'example.com'],\n [true, 'https://pew.pew.pew.example.com:8080/pew', 'example.com'],\n [false, 'https://www.example.com', 'yolo.com'],\n [false, 'https://www.example.com', 'www.example.com'],\n]);\n\nit('checks if a URL has a certain path', function (bool $expectedResult, mixed $haystack, mixed $needle) {\n $urlFilterRule = UrlFilterRule::Path;\n\n expect($urlFilterRule->evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'https://www.example.com/foo/bar', '/foo/bar'],\n [false, 'https://www.example.com/foo/bar/baz', '/foo/bar'],\n]);\n\nit('checks if a URL path starts with a certain path', function (bool $expectedResult, mixed $haystack, mixed $needle) {\n $urlFilterRule = UrlFilterRule::PathStartsWith;\n\n expect($urlFilterRule->evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'https://www.example.com/foo/bar', '/foo/bar'],\n [true, 'https://www.example.com/foo/bar', '/foo'],\n [false, 'https://www.example.com/foo/bar', '/bar'],\n]);\n\nit('checks if a URL path matches a regex pattern', function (bool $expectedResult, mixed $haystack, mixed $needle) {\n $urlFilterRule = UrlFilterRule::PathMatches;\n\n expect($urlFilterRule->evaluate($haystack, $needle))->toBe($expectedResult);\n})->with([\n [true, 'https://www.example.com/foo/bar', '^/foo/'],\n [true, 'https://www.example.com/56/something/foo', '^/\\d{1,5}/[a-z]{1,20}'],\n [false, 'https://www.example.com/56/some-thing/foo', '^/\\d{1,5}/[a-z]{1,20}/'],\n]);\n" }, { "path": "tests/Steps/Filters/FilterTest.php", "content": "value = $this->getKey($valueInQuestion);\n\n return true;\n }\n}\n\nit('gets a key from an array', function () {\n $filter = new TestFilter();\n\n $filter->useKey('foo');\n\n $filter->evaluate(['foo' => 'fooValue', 'bar' => 'barValue']);\n\n expect($filter->value)->toBe('fooValue');\n});\n\nit('gets a key from an object', function () {\n $filter = new TestFilter();\n\n $filter->useKey('foo');\n\n $filter->evaluate(helper_getStdClassWithData(['foo' => 'fooValue', 'bar' => 'barValue']));\n\n expect($filter->value)->toBe('fooValue');\n});\n\nit('throws an exception when the value in question is not array or object when a key to use was defined', function () {\n $filter = new TestFilter();\n\n $filter->useKey('foo');\n\n $filter->evaluate('foo');\n})->throws(InvalidArgumentException::class);\n\nit('throws an exception when the key to use is not contained in an array', function () {\n $filter = new TestFilter();\n\n $filter->useKey('foo');\n\n $filter->evaluate(['bar' => 'barValue', 'baz' => 'bazValue']);\n})->throws(Exception::class);\n\nit('throws an exception when the key to use is not contained in an object', function () {\n $filter = new TestFilter();\n\n $filter->useKey('foo');\n\n $filter->evaluate(helper_getStdClassWithData(['bar' => 'barValue', 'baz' => 'bazValue']));\n})->throws(Exception::class);\n" }, { "path": "tests/Steps/Filters/NegatedFilterTest.php", "content": "evaluate('foo'))->toBeTrue();\n\n expect($negatedFilter->evaluate('foo'))->toBeFalse();\n\n expect($filter->evaluate('bar'))->toBeFalse();\n\n expect($negatedFilter->evaluate('bar'))->toBeTrue();\n});\n" }, { "path": "tests/Steps/Filters/StringFilterTest.php", "content": "evaluate('foo bar baz'))->toBeTrue();\n\n expect($stringCheck->evaluate('lorem ipsum'))->toBeFalse();\n});\n\nit('checks a string from an array using a key', function () {\n $stringCheck = new StringFilter(StringFilterRule::StartsWith, 'waldo');\n\n $stringCheck->useKey('bar');\n\n expect($stringCheck->evaluate(['foo' => 'something', 'bar' => 'waldo check', 'baz' => 'test']))->toBeTrue();\n\n expect($stringCheck->evaluate(['foo' => 'something', 'bar' => 'check waldo', 'baz' => 'test']))->toBeFalse();\n});\n\nit('checks a string from an object using a key', function () {\n $stringCheck = new StringFilter(StringFilterRule::EndsWith, 'waldo');\n\n $stringCheck->useKey('bar');\n\n $object = helper_getStdClassWithData(['foo' => 'something', 'bar' => 'check waldo', 'baz' => 'test']);\n\n expect($stringCheck->evaluate($object))->toBeTrue();\n\n $object = helper_getStdClassWithData(['foo' => 'something', 'bar' => 'waldo check', 'baz' => 'test']);\n\n expect($stringCheck->evaluate($object))->toBeFalse();\n});\n" }, { "path": "tests/Steps/Filters/StringLengthFilterTest.php", "content": "evaluate('foo'))->toBeFalse();\n\n expect($stringCheck->evaluate('lorem ipsum'))->toBeTrue();\n});\n\nit('checks a string from an array using a key', function () {\n $stringCheck = new StringLengthFilter(StringLengthFilterRule::GreaterThan, 10);\n\n $stringCheck->useKey('bar');\n\n expect($stringCheck->evaluate(['foo' => 'one', 'bar' => 'two', 'baz' => 'three']))->toBeFalse();\n\n expect($stringCheck->evaluate(['foo' => 'one', 'bar' => 'lorem ipsum', 'baz' => 'three']))->toBeTrue();\n});\n\nit('checks a string from an object using a key', function () {\n $stringCheck = new StringLengthFilter(StringLengthFilterRule::GreaterThan, 10);\n\n $stringCheck->useKey('bar');\n\n $object = helper_getStdClassWithData(['foo' => 'one', 'bar' => 'two', 'baz' => 'three']);\n\n expect($stringCheck->evaluate($object))->toBeFalse();\n\n $object = helper_getStdClassWithData(['foo' => 'one', 'bar' => 'lorem ipsum', 'baz' => 'three']);\n\n expect($stringCheck->evaluate($object))->toBeTrue();\n});\n" }, { "path": "tests/Steps/Filters/UrlFilterTest.php", "content": "evaluate('https://www.crwlr.software/packages'))->toBeTrue();\n\n expect($urlFilter->evaluate('https://www.example.com/something'))->toBeFalse();\n});\n\nit('evaluates an url from an array using a key', function () {\n $urlFilter = (new UrlFilter(UrlFilterRule::Scheme, 'https'))->useKey('bar');\n\n expect($urlFilter->evaluate(['foo' => 'yo', 'bar' => 'https://www.example.com']))->toBeTrue();\n\n expect($urlFilter->evaluate(['foo' => 'yo', 'bar' => 'http://www.example.com']))->toBeFalse();\n});\n\nit('evaluates a string from an object using a key', function () {\n $urlFilter = (new UrlFilter(UrlFilterRule::PathStartsWith, '/foo'))->useKey('bar');\n\n expect($urlFilter->evaluate(\n helper_getStdClassWithData(['foo' => 'yo', 'bar' => 'https://www.example.com/foo/bar/baz']),\n ))->toBeTrue();\n\n expect($urlFilter->evaluate(\n helper_getStdClassWithData(['foo' => 'yo', 'bar' => 'https://www.example.com/articles/1']),\n ))->toBeFalse();\n});\n\nit('doesnt throw an exception when value is not a valid url', function () {\n $urlFilter = new UrlFilter(UrlFilterRule::Host, 'invalid');\n\n expect($urlFilter->evaluate('https*://invalid'))->toBeFalse();\n});\n" }, { "path": "tests/Steps/GroupTest.php", "content": "addStep($step);\n }\n\n return $group;\n}\n\nfunction helper_addUpdateInputUsingOutputCallbackToSteps(Closure $callback, Step ...$steps): void\n{\n foreach ($steps as $step) {\n $step->updateInputUsingOutput($callback);\n }\n}\n\nfunction helper_getStepThatRemembersIfItWasCalled(): Step\n{\n return new class extends Step {\n public bool $called = false;\n\n protected function invoke(mixed $input): Generator\n {\n $this->called = true;\n\n yield 'test';\n }\n };\n}\n\ntest('You can add a step and it passes on the logger', function () {\n $step = Mockery::mock(StepInterface::class);\n\n $step->shouldReceive('addLogger')->once();\n\n $step->shouldNotReceive('setLoader');\n\n $group = new Group();\n\n $group->addLogger(new CliLogger());\n\n $group->addStep($step);\n});\n\nit('also passes on a new logger to all steps when the logger is added after the steps', function () {\n $step1 = Mockery::mock(StepInterface::class);\n\n $step1->shouldReceive('addLogger')->once();\n\n $step2 = Mockery::mock(StepInterface::class);\n\n $step2->shouldReceive('addLogger')->once();\n\n $group = new Group();\n\n $group->addStep($step1);\n\n $group->addStep($step2);\n\n $group->addLogger(new CliLogger());\n});\n\nit('also passes on the loader to the step when setLoader method exists in step', function () {\n $step = Mockery::mock(helper_getLoadingStep());\n\n $step->shouldReceive('addLogger')->once();\n\n $step->shouldReceive('setLoader')->once();\n\n $group = new Group();\n\n $group->addLogger(new CliLogger());\n\n $group->setLoader(new HttpLoader(new BotUserAgent('MyBot')));\n\n /** @var Step $step */\n\n $group->addStep($step);\n});\n\nit('also passes on a new loader to all steps when it is added after the steps', function () {\n $step1 = Mockery::mock(helper_getLoadingStep());\n\n $step1->shouldReceive('setLoader')->once();\n\n $step2 = Mockery::mock(helper_getLoadingStep());\n\n $step2->shouldReceive('setLoader')->once();\n\n $group = new Group();\n\n /** @var Step $step1 */\n\n $group->addStep($step1);\n\n /** @var Step $step2 */\n\n $group->addStep($step2);\n\n $group->setLoader(new HttpLoader(new BotUserAgent('MyBot')));\n});\n\ntest('The factory method returns a Group object instance', function () {\n expect(Crawler::group())->toBeInstanceOf(Group::class);\n});\n\ntest('You can add multiple steps and invokeStep calls all of them', function () {\n $step1 = helper_getStepThatRemembersIfItWasCalled();\n\n $step2 = helper_getStepThatRemembersIfItWasCalled();\n\n $step3 = helper_getStepThatRemembersIfItWasCalled();\n\n $group = new Group();\n\n $group->addStep($step1)->addStep($step2)->addStep($step3);\n\n helper_invokeStepWithInput($group);\n\n expect($step1->called)->toBeTrue() // @phpstan-ignore-line\n ->and($step2->called)->toBeTrue() // @phpstan-ignore-line\n ->and($step3->called)->toBeTrue(); // @phpstan-ignore-line\n});\n\nit('combines the outputs of all it\\'s steps into one output containing an array', function () {\n $step1 = helper_getValueReturningStep('lorem');\n\n $step2 = helper_getValueReturningStep('ipsum');\n\n $step3 = helper_getValueReturningStep('dolor');\n\n $group = new Group();\n\n $group->addStep($step1)->addStep($step2)->addStep($step3);\n\n $output = helper_invokeStepWithInput($group, 'gogogo');\n\n expect($output)->toHaveCount(1)\n ->and($output[0])->toBeInstanceOf(Output::class)\n ->and($output[0]->get())->toBe(['lorem', 'ipsum', 'dolor']);\n});\n\ntest(\n 'When defining keys for the steps via $step->outputKey(), the combined output array has those keys',\n function () {\n $step1 = helper_getValueReturningStep('ich');\n\n $step2 = helper_getValueReturningStep('bin');\n\n $step3 = helper_getValueReturningStep('ein berliner');\n\n $group = (new Group())\n ->addStep($step1->outputKey('foo'))\n ->addStep($step2->outputKey('bar'))\n ->addStep($step3->outputKey('baz'));\n\n $output = helper_invokeStepWithInput($group, 'https://www.gogo.go');\n\n expect($output)->toHaveCount(1)\n ->and($output[0])->toBeInstanceOf(Output::class);\n\n $expectedOutputAndResultArray = ['foo' => 'ich', 'bar' => 'bin', 'baz' => 'ein berliner'];\n\n expect($output[0]->get())->toBe($expectedOutputAndResultArray);\n },\n);\n\nit('merges array outputs with string keys to one array', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'fooValue', 'bar' => 'barValue']);\n\n $step2 = helper_getValueReturningStep(['baz' => 'bazValue', 'yo' => 'lo']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2);\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe([\n 'foo' => 'fooValue',\n 'bar' => 'barValue',\n 'baz' => 'bazValue',\n 'yo' => 'lo',\n ]);\n});\n\nit('doesn\\'t invoke twice with duplicate inputs when uniqueInput was called', function () {\n $step1 = helper_getValueReturningStep('one');\n\n $step2 = helper_getValueReturningStep('two');\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2);\n\n $outputs = helper_invokeStepWithInput($group, 'foo');\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, 'foo');\n\n expect($outputs)->toHaveCount(1);\n\n $group->resetAfterRun();\n\n $group->uniqueInputs();\n\n $outputs = helper_invokeStepWithInput($group, 'foo');\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, 'foo');\n\n expect($outputs)->toHaveCount(0);\n});\n\nit(\n 'doesn\\'t invoke twice with array inputs with duplicate keys when uniqueInput was called with that key',\n function () {\n $step1 = helper_getValueReturningStep('one');\n\n $step2 = helper_getValueReturningStep('two');\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2);\n\n $group->uniqueInputs();\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'bar', 'bttfc' => 'marty']);\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'bar', 'bttfc' => 'doc']);\n\n expect($outputs)->toHaveCount(1);\n\n $group->resetAfterRun();\n\n $group->uniqueInputs('foo');\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'bar', 'bttfc' => 'marty']);\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'bar', 'bttfc' => 'doc']);\n\n expect($outputs)->toHaveCount(0);\n },\n);\n\nit(\n 'doesn\\'t invoke twice with object inputs with duplicate keys when uniqueInput was called with that key',\n function () {\n $step1 = helper_getValueReturningStep('one');\n\n $step2 = helper_getValueReturningStep('two');\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2);\n\n $group->uniqueInputs();\n\n $outputs = helper_invokeStepWithInput($group, helper_getStdClassWithData(['foo' => 'bar', 'bttfc' => 'marty']));\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, helper_getStdClassWithData(['foo' => 'bar', 'bttfc' => 'doc']));\n\n expect($outputs)->toHaveCount(1);\n\n $group->resetAfterRun();\n\n $group->uniqueInputs('foo');\n\n $outputs = helper_invokeStepWithInput($group, helper_getStdClassWithData(['foo' => 'bar', 'bttfc' => 'marty']));\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, helper_getStdClassWithData(['foo' => 'bar', 'bttfc' => 'doc']));\n\n expect($outputs)->toHaveCount(0);\n },\n);\n\nit('returns only unique outputs when uniqueOutput was called', function () {\n $step1 = helper_getInputReturningStep();\n\n $step2 = helper_getValueReturningStep('test');\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2)->uniqueOutputs();\n\n $outputs = helper_invokeStepWithInput($group, 'foo');\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, 'bar');\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, 'foo');\n\n expect($outputs)->toHaveCount(0);\n});\n\nit('returns only unique outputs when outputs are arrays and uniqueOutput was called', function () {\n $step1 = helper_getInputReturningStep();\n\n $step2 = helper_getValueReturningStep(['lorem' => 'ipsum']);\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2)->uniqueOutputs();\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'bar']);\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, ['baz' => 'quz']);\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'bar']);\n\n expect($outputs)->toHaveCount(0);\n});\n\nit(\n 'returns only unique outputs when outputs are arrays and uniqueOutput was called with a key from the output arrays',\n function () {\n $step1 = helper_getInputReturningStep();\n\n $step2 = helper_getValueReturningStep(['lorem' => 'ipsum']);\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2)->uniqueOutputs('foo');\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'bar']);\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'baz']);\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($group, ['foo' => 'bar', 'something' => 'else']);\n\n expect($outputs)->toHaveCount(0);\n },\n);\n\nit('returns only unique outputs when outputs are objects and uniqueOutput was called', function () {\n $step1 = helper_getStepYieldingObjectWithNumber(10);\n\n $step2 = helper_getStepYieldingObjectWithNumber(11);\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2);\n\n expect(helper_invokeStepWithInput($group))->toHaveCount(1);\n\n $group->uniqueOutputs();\n\n expect(helper_invokeStepWithInput($group))->toHaveCount(1)\n ->and(helper_invokeStepWithInput($group))->toHaveCount(0);\n\n $incrementNumberCallback = function (mixed $input) {\n return $input + 1;\n };\n\n helper_addUpdateInputUsingOutputCallbackToSteps($incrementNumberCallback, $step1, $step2);\n\n expect(helper_invokeStepWithInput($group, new Input(1)))->toHaveCount(1);\n});\n\nit(\n 'returns only unique outputs when outputs are objects and uniqueOutput was called with a property name from the ' .\n 'output objects',\n function () {\n $step1 = helper_getStepYieldingObjectWithNumber(21);\n\n $step2 = helper_getStepYieldingObjectWithNumber(23);\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2);\n\n expect(helper_invokeStepWithInput($group))->toHaveCount(1);\n\n $group->resetAfterRun();\n\n $group->uniqueOutputs('number');\n\n expect(helper_invokeStepWithInput($group))->toHaveCount(1)\n ->and(helper_invokeStepWithInput($group))->toHaveCount(0);\n\n $group->resetAfterRun();\n\n $incrementNumberCallback = function (mixed $input) {\n return $input + 1;\n };\n\n helper_addUpdateInputUsingOutputCallbackToSteps($incrementNumberCallback, $step1, $step2);\n\n expect(helper_invokeStepWithInput($group, new Input(1)))->toHaveCount(1);\n },\n);\n\nit(\n 'excludes the output of a step from the combined group output, when the excludeFromGroupOutput() method was called',\n function () {\n $step1 = helper_getValueReturningStep(['foo' => 'one']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two'])->excludeFromGroupOutput();\n\n $step3 = helper_getValueReturningStep(['baz' => 'three']);\n\n $group = helper_addStepsToGroup(new Group(), $step1, $step2, $step3);\n\n $outputs = helper_invokeStepWithInput($group);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['foo' => 'one', 'baz' => 'three']);\n },\n);\n\ntest('You can update the input for further steps with the output of a step that is before those steps', function () {\n $step1 = helper_getValueReturningStep(' rocks')\n ->updateInputUsingOutput(function (mixed $input, mixed $output) {\n return $input . $output['foo'];\n });\n\n $step2 = helper_getInputReturningStep();\n\n $group = (new Group())\n ->addStep($step1->outputKey('foo'))\n ->addStep($step2->outputKey('bar'));\n\n $outputs = helper_invokeStepWithInput($group, 'crwlr.software');\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['foo' => ' rocks', 'bar' => 'crwlr.software rocks']);\n});\n\nit('uses a key from array input when defined', function () {\n $step = helper_getInputReturningStep();\n\n $group = (new Group())\n ->addStep($step->outputKey('test'))\n ->useInputKey('bar');\n\n $outputs = helper_invokeStepWithInput($group, new Input(\n ['foo' => 'fooValue', 'bar' => 'barValue', 'baz' => 'bazValue'],\n ));\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['test' => 'barValue']);\n});\n\nit('keeps the combined output with a certain key when keepAs() is used', function () {\n $step1 = helper_getValueReturningStep('foo');\n\n $step2 = helper_getValueReturningStep('bar');\n\n $group = (new Group())\n ->addStep($step1->outputKey('key1'))\n ->addStep($step2->outputKey('key2'))\n ->keepAs('test');\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->keep)->toBe(['test' => ['key1' => 'foo', 'key2' => 'bar']]);\n});\n\nit('keeps all keys from a combined array output when keep() was called without argument', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'fooValue', 'bar' => 'barValue']);\n\n $step2 = helper_getValueReturningStep(['baz' => 'bazValue', 'yo' => 'lo']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->keep();\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->keep)->toBe([\n 'foo' => 'fooValue',\n 'bar' => 'barValue',\n 'baz' => 'bazValue',\n 'yo' => 'lo',\n ]);\n});\n\nit('keeps all defined keys from a combined array output when keep() was called with keys', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'fooValue', 'bar' => 'barValue']);\n\n $step2 = helper_getValueReturningStep(['baz' => 'bazValue', 'yo' => 'lo']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->keep(['foo', 'baz', 'yo']);\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->keep)->toBe([\n 'foo' => 'fooValue',\n 'baz' => 'bazValue',\n 'yo' => 'lo',\n ]);\n});\n\nit('keeps data, when keep() is called on child steps', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'fooValue', 'bar' => 'barValue']);\n\n $step2 = helper_getValueReturningStep(['baz' => 'bazValue', 'quz' => 'quzValue']);\n\n $group = (new Group())\n ->addStep($step1->keep('foo'))\n ->addStep($step2->keep(['baz', 'quz']));\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->keep)->toBe([\n 'foo' => 'fooValue',\n 'baz' => 'bazValue',\n 'quz' => 'quzValue',\n ]);\n});\n\nit('keeps data, when keepAs() is called on child steps', function () {\n $step1 = helper_getValueReturningStep('fooValue');\n\n $step2 = helper_getValueReturningStep(['bar' => 'barValue', 'baz' => 'bazValue']);\n\n $group = (new Group())\n ->addStep($step1->keepAs('foo'))\n ->addStep($step2->keepAs('quz'));\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->keep)->toBe([\n 'foo' => 'fooValue',\n 'quz' => [\n 'bar' => 'barValue',\n 'baz' => 'bazValue',\n ],\n ]);\n});\n\ntest(\n 'when steps yield multiple outputs it combines the first output from first step with first output from second ' .\n 'step and so on.',\n function () {\n $step1 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield ['one' => 'foo'];\n\n yield ['two' => 'bar'];\n }\n };\n\n $step2 = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield ['three' => 'baz'];\n\n yield ['four' => 'quz'];\n }\n };\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2);\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(2)\n ->and($output[0]->get())->toBe(['one' => 'foo', 'three' => 'baz'])\n ->and($output[1]->get())->toBe(['two' => 'bar', 'four' => 'quz']);\n },\n);\n\nit('ignores the key set via outputKey because group step output is always an array', function () {\n $step1 = helper_getValueReturningStep(['one' => 'foo']);\n\n $step2 = helper_getValueReturningStep(['two' => 'bar']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->outputKey('baz');\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['one' => 'foo', 'two' => 'bar']);\n});\n\nit(\n 'keeps input data when keepFromInput() was called when outputs are combined',\n function () {\n $step1 = helper_getValueReturningStep(['foo' => 'one']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->keepFromInput();\n\n $output = helper_invokeStepWithInput($group, new Input(['baz' => 'three']));\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'one', 'bar' => 'two'])\n ->and($output[0]->keep)->toBe(['baz' => 'three']);\n },\n);\n\nit('keeps non array input data in array output with key', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'one']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->keepInputAs('baz');\n\n $output = helper_invokeStepWithInput($group, new Input('three'));\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'one', 'bar' => 'two'])\n ->and($output[0]->keep)->toBe(['baz' => 'three']);\n});\n\nit('keeps a value with unnamed key, when non array input should be kept but no key is defined', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'one']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->keepFromInput();\n\n $output = helper_invokeStepWithInput($group, new Input('three'));\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'one', 'bar' => 'two'])\n ->and($output[0]->keep)->toBe(['unnamed1' => 'three']);\n});\n\nit('contains an element with a numeric key when it contains a step that yields non array output', function () {\n $step1 = helper_getValueReturningStep('one');\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2);\n\n $output = helper_invokeStepWithInput($group);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe([0 => 'one', 'bar' => 'two']);\n});\n\nit('keeps array input data when some output is non array but converted to array using outputKey()', function () {\n $step1 = helper_getValueReturningStep('one');\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1->outputKey('foo'))\n ->addStep($step2)\n ->keepFromInput();\n\n $output = helper_invokeStepWithInput($group, new Input(['baz' => 'three']));\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'one', 'bar' => 'two'])\n ->and($output[0]->keep)->toBe(['baz' => 'three']);\n});\n\nit(\n 'keeps an input value with an unnamed key, when it is a non array value and no key is defined (via keepInputAs())',\n function () {\n $step1 = helper_getValueReturningStep('one');\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->keepFromInput();\n\n $output = helper_invokeStepWithInput($group, new Input('three'));\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe([0 => 'one', 'bar' => 'two'])\n ->and($output[0]->keep)->toBe(['unnamed1' => 'three']);\n },\n);\n\nit('keeps the original input data when useInputKey() is used', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'one']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->useInputKey('baz')\n ->keepFromInput();\n\n $output = helper_invokeStepWithInput($group, new Input(['baz' => 'three', 'quz' => 'four']));\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'one', 'bar' => 'two'])\n ->and($output[0]->keep)->toBe(['baz' => 'three', 'quz' => 'four']);\n});\n\nit('applies a Closure refiner to the steps output', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'one']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->refineOutput(function (mixed $outputValue) {\n $outputValue['baz'] = 'three';\n\n $outputValue['bar'] .= ' refined';\n\n return $outputValue;\n });\n\n $outputs = helper_invokeStepWithInput($group);\n\n expect($outputs[0]->get())->toBe(['foo' => 'one', 'bar' => 'two refined', 'baz' => 'three']);\n});\n\nit('applies an instance of the RefinerInterface to the steps output', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'lorem ipsum dolor']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->refineOutput('foo', StringRefiner::betweenFirst('lorem', 'dolor'));\n\n $outputs = helper_invokeStepWithInput($group);\n\n expect($outputs[0]->get())->toBe(['foo' => 'ipsum', 'bar' => 'two']);\n});\n\nit('applies multiple refiners to the steps output in the order they\\'re added', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'lorem ipsum dolor']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->refineOutput('foo', StringRefiner::betweenFirst('lorem', 'dolor'))\n ->refineOutput('bar', fn(mixed $outputValue) => $outputValue . ' refined');\n\n $outputs = helper_invokeStepWithInput($group);\n\n expect($outputs[0]->get())->toBe(['foo' => 'ipsum', 'bar' => 'two refined']);\n});\n\ntest('you can apply multiple refiners to the same output array key', function () {\n $step1 = helper_getValueReturningStep(['foo' => 'lorem ipsum dolor']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->refineOutput('foo', StringRefiner::betweenFirst('lorem', 'dolor'))\n ->refineOutput('foo', fn(mixed $outputValue) => $outputValue . ' refined');\n\n $outputs = helper_invokeStepWithInput($group);\n\n expect($outputs[0]->get())->toBe(['foo' => 'ipsum refined', 'bar' => 'two']);\n});\n\nit(\n 'uses the original input value when applying a refiner, not only the value of an input array key chosen via ' .\n 'useInputKey()',\n function () {\n $step1 = helper_getValueReturningStep(['foo' => 'one']);\n\n $step2 = helper_getValueReturningStep(['bar' => 'two']);\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->refineOutput(fn(mixed $outputValue, mixed $originalInputValue) => $originalInputValue);\n\n $outputs = helper_invokeStepWithInput($group, ['yo' => 'lo']);\n\n expect($outputs[0]->get())->toBe(['yo' => 'lo']);\n },\n);\n\nit('stops calling its steps and producing outputs when maxOutputs is reached', function () {\n $step1 = new class extends Step {\n public int $called = 0;\n\n protected function invoke(mixed $input): Generator\n {\n yield ['foo' => 'one'];\n\n $this->called++;\n }\n };\n\n $step2 = new class extends Step {\n public int $called = 0;\n\n protected function invoke(mixed $input): Generator\n {\n yield ['bar' => 'two'];\n\n $this->called++;\n }\n };\n\n $group = (new Group())\n ->addStep($step1)\n ->addStep($step2)\n ->maxOutputs(2);\n\n expect(helper_invokeStepWithInput($group, 'hey'))->toHaveCount(1)\n ->and(helper_invokeStepWithInput($group, 'ho'))->toHaveCount(1)\n ->and(helper_invokeStepWithInput($group, 'hey'))->toHaveCount(0)\n ->and($step1->called)->toBe(2)\n ->and($step2->called)->toBe(2);\n});\n\nit(\n 'also stops creating outputs when maxOutputs is reached, when maxOutputs() was called before addStep()',\n function () {\n $step1 = new class extends Step {\n public int $called = 0;\n\n protected function invoke(mixed $input): Generator\n {\n yield ['foo' => 'one'];\n\n $this->called++;\n }\n };\n\n $step2 = new class extends Step {\n public int $called = 0;\n\n protected function invoke(mixed $input): Generator\n {\n yield ['bar' => 'two'];\n\n $this->called++;\n }\n };\n\n $group = (new Group())\n ->maxOutputs(2)\n ->addStep($step1)\n ->addStep($step2);\n\n expect(helper_invokeStepWithInput($group, 'hey'))->toHaveCount(1)\n ->and(helper_invokeStepWithInput($group, 'ho'))->toHaveCount(1)\n ->and(helper_invokeStepWithInput($group, 'hey'))->toHaveCount(0)\n ->and($step1->called)->toBe(2)\n ->and($step2->called)->toBe(2);\n },\n);\n" }, { "path": "tests/Steps/Html/CssSelectorTest.php", "content": "throws(InvalidDomQueryException::class)->with(['.foo;', '.foo:before']);\n\ntest('The apply method returns a string for a single match', function () {\n $html = '
    test
    ';\n\n expect((new CssSelector('.item'))->apply(new HtmlDocument($html)))->toBe('test');\n});\n\ntest('The apply method returns an array of strings for multiple matches', function () {\n $html = '
    test
    test 2 sub
    test 3
    ';\n\n expect((new CssSelector('.item'))->apply(new HtmlDocument($html)))->toBe(['test', 'test 2 sub', 'test 3']);\n});\n\ntest('The apply method returns null if nothing matches', function () {\n $html = '
    test
    ';\n\n expect((new CssSelector('.aitem'))->apply(new HtmlDocument($html)))->toBeNull();\n});\n\nit('trims whitespace', function () {\n $html = <<\n test\n \n HTML;\n\n expect((new CssSelector('.item'))->apply(new HtmlDocument($html)))->toBe('test');\n});\n\nit('contains inner tags when the html method is called', function () {\n $html = '
    test sub
    ';\n\n expect((new CssSelector('.item'))->html()->apply(new HtmlDocument($html)))->toBe('test sub');\n});\n\nit('contains also the outer tag when the outerHtml method is called', function () {\n $html = '
    test sub
    ';\n\n expect((new CssSelector('.item'))->outerHtml()->apply(new HtmlDocument($html)))\n ->toBe('
    test sub
    ');\n});\n\nit('returns formatted text when formattedText() is called', function () {\n $html = '

    headline

    paragraph

    • item 1
    • item 2
    ';\n\n expect((new CssSelector('#a'))->formattedText()->apply(new HtmlDocument($html)))\n ->toBe(<<removeConverter('ul');\n\n expect((new CssSelector('#a'))->formattedText($converter)->apply(new HtmlDocument($html)))\n ->toBe(<<attribute('data-attr')->apply(new HtmlDocument($html)))->toBe('content');\n});\n\ntest('getting an attribute value returns an empty string when the attribute does not exist', function () {\n $html = '
    test
    ';\n\n expect((new CssSelector('.item'))->attribute('foo')->apply(new HtmlDocument($html)))->toBe('');\n});\n\nit('turns the value into an absolute url when toAbsoluteUrl() is called', function () {\n $html = 'getting started';\n\n $document = new HtmlDocument($html);\n\n $selector = new CssSelector('a');\n\n $selector->setBaseUrl('https://www.crwlr.software/')\n ->attribute('href');\n\n expect($selector->apply($document))->toBe('/packages/crawler/v0.4/getting-started');\n\n $selector->toAbsoluteUrl();\n\n expect($selector->apply($document))->toBe('https://www.crwlr.software/packages/crawler/v0.4/getting-started');\n});\n\nit(\n 'turns the value into the correct absolute url when toAbsoluteUrl() is called and the HTML contains a base tag',\n function () {\n $html = <<\n \n \n \n \n link\n \n HTML;\n\n $document = new HtmlDocument($html);\n\n $selector = new CssSelector('a');\n\n $selector->setBaseUrl('https://www.example.com/a/b')\n ->attribute('href');\n\n expect($selector->apply($document))->toBe('e');\n\n $selector->toAbsoluteUrl();\n\n expect($selector->apply($document))->toBe('https://www.example.com/c/e');\n },\n);\n\nit('gets an absolute link from the href attribute of a link element, when the link() method is called', function () {\n $html = '';\n\n $document = new HtmlDocument($html);\n\n $selector = new CssSelector('#foo .bar');\n\n $selector->setBaseUrl('https://www.example.com/');\n\n expect($selector->apply($document))->toBe('Foo');\n\n $selector->link();\n\n expect($selector->apply($document))->toBe('https://www.example.com/foo/bar');\n});\n\nit('gets only the first matching element when the first() method is called', function () {\n $selector = (new CssSelector('#list .item'))->first();\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe('one');\n});\n\nit('gets only the last matching element when the last() method is called', function () {\n $selector = (new CssSelector('#list .item'))->last();\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe('four');\n});\n\nit('gets only the nth matching element when the nth() method is called', function () {\n $selector = (new CssSelector('#list .item'))->nth(3);\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe('three');\n});\n\nit('returns null when no nth matching element exists', function () {\n $selector = (new CssSelector('#list .item'))->nth(5);\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBeNull();\n});\n\nit('gets only even matching elements when the even() method is called', function () {\n $selector = (new CssSelector('#list .item'))->even();\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe(['two', 'four']);\n});\n\nit('gets only odd matching elements when the odd() method is called', function () {\n $selector = (new CssSelector('#list .item'))->odd();\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe(['one', 'three']);\n});\n" }, { "path": "tests/Steps/Html/Exceptions/InvalidDomQueryExceptionTest.php", "content": "getDomQuery())\n ->toBe('.foo:before')\n ->and($exception->getMessage())\n ->toBe('error');\n});\n\nit('can be created from a symfony SyntaxErrorException', function () {\n $exception = InvalidDomQueryException::fromSymfonyException('.foo;', new SyntaxErrorException('error message'));\n\n expect($exception->getDomQuery())\n ->toBe('.foo;')\n ->and($exception->getMessage())\n ->toBe('error message');\n});\n\nit('can be created from a message and a query', function () {\n $exception = InvalidDomQueryException::make('message', '.foo > .bar;');\n\n expect($exception->getDomQuery())\n ->toBe('.foo > .bar;')\n ->and($exception->getMessage())\n ->toBe('message');\n});\n" }, { "path": "tests/Steps/Html/GetLinkTest.php", "content": "link'),\n ));\n\n expect($link)->toHaveCount(1)\n ->and($link[0]->get())->toBe('https://www.crwl.io/blog');\n});\n\nit('logs an error message when fed with invalid input', function () {\n $logger = new DummyLogger();\n\n $step = (new GetLink())->addLogger($logger);\n\n helper_traverseIterable($step->invokeStep(new Input(new Response())));\n\n expect($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toBe(\n 'The Crwlr\\Crawler\\Steps\\Html\\GetLink step was called with input that it can not work with: Input must ' .\n 'be an instance of RespondedRequest.',\n );\n});\n\ntest('When called without selector it just returns the first link', function () {\n $step = (new GetLink());\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.crwlr.software/packages/url/'),\n new Response(\n 200,\n [],\n '',\n ),\n ));\n\n expect($link[0]->get())->toBe('https://www.crwlr.software/packages/url/v0.1');\n});\n\ntest('When passing a CSS selector it selects the first matching link', function () {\n $step = (new GetLink('.matchingLink'));\n\n $responseHtml = <<\n Jobs\n Numbers\n Products\n \n HTML;\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.foo.bar/company/about'),\n new Response(200, [], $responseHtml),\n ));\n\n expect($link[0]->get())->toBe('https://www.foo.bar/company/jobs');\n});\n\ntest('When selector matches on a non-link element it\\'s ignored', function () {\n $step = (new GetLink('.link'));\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], 'not a linklink'),\n ));\n\n expect($link)->toHaveCount(1)\n ->and($link[0]->get())->toBe('https://www.otsch.codes/foo');\n});\n\nit('finds only links on the same domain when onSameDomain() was called', function () {\n $html = <<link1\n link2\n HTML;\n\n $step = (new GetLink())->onSameDomain();\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($link)->toHaveCount(1)\n ->and($link[0]->get())->toBe('https://blog.otsch.codes/articles');\n});\n\nit('doesn\\'t find a link on the same domain when notOnSameDomain() was called', function () {\n $html = <<link1\n link2\n HTML;\n\n $step = (new GetLink())->notOnSameDomain();\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($link)->toHaveCount(1)\n ->and($link[0]->get())->toBe('https://www.crwlr.software/packages');\n});\n\nit('finds only links from domains the onDomain() method was called with', function () {\n $html = <<link1\n link2\n link3\n link4\n HTML;\n\n $step = (new GetLink())->onDomain('example.com');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.example.com');\n});\n\ntest('onDomain() also takes an array of domains', function () {\n $html = <<link1\n link2\n HTML;\n\n $step = (new GetLink())->onDomain(['otsch.codes', 'example.com']);\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.otsch.codes/contact');\n\n $html = <<link1\n link2\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.example.com/foo');\n});\n\ntest('onDomain() can be called multiple times and merges all domains it was called with', function () {\n $html = <<link1\n HTML;\n\n $step = (new GetLink())->onDomain('crwl.io');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(0);\n\n $step->onDomain(['otsch.codes', 'crwlr.software']);\n\n $html = <<link1\n link2\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.crwl.io');\n\n $html = <<link1\n link2\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.otsch.codes/contact');\n});\n\nit('finds only links on the same host when onSameHost() was called', function () {\n $html = <<link1\n link2\n link3\n HTML;\n\n $step = (new GetLink())->onSameHost();\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($link)->toHaveCount(1)\n ->and($link[0]->get())->toBe('https://www.otsch.codes/contact');\n});\n\nit('doesn\\'t find a link on the same host when notOnSameHost() was called', function () {\n $html = <<link1\n link2\n HTML;\n\n $step = (new GetLink())->notOnSameHost();\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($link)->toHaveCount(1)\n ->and($link[0]->get())->toBe('https://jobs.otsch.codes');\n});\n\nit('finds only links from hosts the onHost() method was called with', function () {\n $html = <<link1\n link2\n link3\n link4\n HTML;\n\n $step = (new GetLink())->onHost('www.example.com');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.example.com');\n});\n\ntest('onHost() also takes an array of hosts', function () {\n $html = <<link1\n link2\n HTML;\n\n $step = (new GetLink())->onHost(['www.otsch.codes', 'blog.example.com']);\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.otsch.codes/contact');\n\n $html = <<link1\n link2\n link3\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://blog.example.com/articles/1');\n});\n\ntest('onHost() can be called multiple times and merges all hosts it was called with', function () {\n $html = <<link1\n HTML;\n\n $step = (new GetLink())->onHost('www.crwl.io');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(0);\n\n $step->onHost(['www.otsch.codes', 'www.crwlr.software']);\n\n $html = <<link1\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.crwl.io');\n\n $html = <<link1\n link2\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.otsch.codes/blog');\n});\n\nit('works correctly when HTML contains a base tag', function () {\n $html = <<\n \n \n \n \n link\n \n HTML;\n\n $step = (new GetLink());\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.example.com/a/b'),\n new Response(200, [], $html),\n ));\n\n expect($links[0]->get())->toBe('https://www.example.com/c/e');\n});\n\nit('throws away the URL fragment part when withoutFragment() was called', function () {\n $html = <<\n \n \n link\n \n HTML;\n\n $step = (new GetLink());\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo/baz'),\n new Response(200, [], $html),\n );\n\n $links = helper_invokeStepWithInput($step, $respondedRequest);\n\n expect($links[0]->get())->toBe('https://www.example.com/foo/bar#fragment');\n\n $step->withoutFragment();\n\n $links = helper_invokeStepWithInput($step, $respondedRequest);\n\n expect($links[0]->get())->toBe('https://www.example.com/foo/bar');\n});\n\nit('ignores special non HTTP links', function () {\n $html = <<\n \n \n \n mailto link\n javascript link\n phone link\n link\n \n \n HTML;\n\n $step = (new GetLink());\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/home'),\n new Response(200, [], $html),\n );\n\n $links = helper_invokeStepWithInput($step, $respondedRequest);\n\n expect($links[0]->get())->toBe('https://www.example.com/foo/bar');\n});\n" }, { "path": "tests/Steps/Html/GetLinksTest.php", "content": "link'),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.example.com/blog');\n});\n\nit('logs an error message when fed with invalid input', function () {\n $logger = new DummyLogger();\n\n $step = (new GetLinks())->addLogger($logger);\n\n helper_traverseIterable($step->invokeStep(new Input(new stdClass())));\n\n expect($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toBe(\n 'The Crwlr\\Crawler\\Steps\\Html\\GetLinks step was called with input that it can not work with: Input must ' .\n 'be an instance of RespondedRequest.',\n );\n});\n\ntest('When called without selector it just gets all links', function () {\n $step = (new GetLinks());\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.crwlr.software/packages/url/'),\n new Response(\n 200,\n [],\n '',\n ),\n ));\n\n expect($links[0]->get())->toBe('https://www.crwlr.software/packages/url/v0.1')\n ->and($links[1]->get())->toBe('https://www.crwlr.software/packages/url/v1.0')\n ->and($links[2]->get())->toBe('https://www.crwlr.software/packages/url/v1.1');\n});\n\ntest('When passing a CSS selector it only selects matching links', function () {\n $step = (new GetLinks('.matchingLink'));\n\n $responseHtml = <<\n Jobs\n Numbers\n Products\n Team\n \n HTML;\n\n $outputs = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.example.com/company/about'),\n new Response(200, [], $responseHtml),\n ));\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe('https://www.example.com/company/jobs')\n ->and($outputs[1]->get())->toBe('https://www.example.com/company/numbers')\n ->and($outputs[2]->get())->toBe('https://www.example.com/team');\n});\n\ntest('When selector matches on a non-link element it\\'s ignored', function () {\n $step = (new GetLinks('.link'));\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], 'FooBar'),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.otsch.codes/foo');\n});\n\nit('finds only links on the same domain when onSameDomain() was called', function () {\n $html = <<link1\n link2\n link3\n HTML;\n\n $step = (new GetLinks())->onSameDomain();\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($link)->toHaveCount(2)\n ->and($link[0]->get())->toBe('https://blog.otsch.codes/articles')\n ->and($link[1]->get())->toBe('https://www.otsch.codes/blog');\n});\n\nit('doesn\\'t find links on the same domain when notOnSameDomain() was called', function () {\n $html = <<link1\n link2\n link3\n HTML;\n\n $step = (new GetLinks())->notOnSameDomain();\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($link)->toHaveCount(2)\n ->and($link[0]->get())->toBe('https://www.crwlr.software/packages')\n ->and($link[1]->get())->toBe('https://www.example.com/foo');\n});\n\nit('finds only links from domains the onDomain() method was called with', function () {\n $html = <<link1\n link2\n link3\n link4\n HTML;\n\n $step = (new GetLinks())->onDomain('crwlr.software');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(2)\n ->and($links[0]->get())->toBe('https://www.crwlr.software/packages')\n ->and($links[1]->get())->toBe('https://www.crwlr.software/blog');\n});\n\ntest('onDomain() also takes an array of domains', function () {\n $html = <<link1\n link2\n link3\n HTML;\n\n $step = (new GetLinks())->onDomain(['otsch.codes', 'crwlr.software']);\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(2)\n ->and($links[0]->get())->toBe('https://www.otsch.codes/contact')\n ->and($links[1]->get())->toBe('https://www.crwlr.software/packages');\n});\n\ntest('onDomain() can be called multiple times and merges all domains it was called with', function () {\n $html = <<link1\n link2\n link3\n HTML;\n\n $step = (new GetLinks())->onDomain('crwl.io');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(0);\n\n $step->onDomain(['otsch.codes', 'crwlr.software']);\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(2);\n\n $step->onDomain('example.com');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(3);\n});\n\nit('finds only links on the same host when onSameHost() was called', function () {\n $html = <<link1\n link2\n link3\n link4\n HTML;\n\n $step = (new GetLinks())->onSameHost();\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($link)->toHaveCount(2)\n ->and($link[0]->get())->toBe('https://www.otsch.codes/contact')\n ->and($link[1]->get())->toBe('https://www.otsch.codes/blog');\n});\n\nit('doesn\\'t find links on the same host when notOnSameHost() was called', function () {\n $html = <<link1\n link2\n link3\n HTML;\n\n $step = (new GetLinks())->notOnSameHost();\n\n $link = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($link)->toHaveCount(2)\n ->and($link[0]->get())->toBe('https://jobs.otsch.codes')\n ->and($link[1]->get())->toBe('https://www.crwlr.software/packages');\n});\n\nit('finds only links from hosts the onHost() method was called with', function () {\n $html = <<link1\n link2\n link3\n link4\n HTML;\n\n $step = (new GetLinks())->onHost('www.crwlr.software');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(2)\n ->and($links[0]->get())->toBe('https://www.crwlr.software/packages')\n ->and($links[1]->get())->toBe('https://www.crwlr.software/packages/crawler/v0.4/getting-started');\n});\n\ntest('onHost() also takes an array of hosts', function () {\n $html = <<link1\n link2\n HTML;\n\n $step = (new GetLinks())->onHost(['www.otsch.codes', 'blog.example.com']);\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.otsch.codes/contact');\n\n $html = <<link1\n link2\n link3\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(2)\n ->and($links[1]->get())->toBe('https://blog.example.com/articles/1');\n});\n\ntest('onHost() can be called multiple times and merges all hosts it was called with', function () {\n $html = <<link1\n HTML;\n\n $step = (new GetLinks())->onHost('www.crwl.io');\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(0);\n\n $step->onHost(['www.otsch.codes', 'www.crwlr.software']);\n\n $html = <<link1\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(1)\n ->and($links[0]->get())->toBe('https://www.crwl.io');\n\n $html = <<link1\n link2\n HTML;\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.otsch.codes'),\n new Response(200, [], $html),\n ));\n\n expect($links)->toHaveCount(2)\n ->and($links[0]->get())->toBe('https://www.otsch.codes/blog')\n ->and($links[1]->get())->toBe('https://www.crwl.io');\n});\n\nit('works correctly when HTML contains a base tag', function () {\n $html = <<\n \n \n \n \n \n link\n link2\n link3\n \n \n HTML;\n\n $step = (new GetLinks());\n\n $links = helper_invokeStepWithInput($step, new RespondedRequest(\n new Request('GET', 'https://www.example.com/a/b'),\n new Response(200, [], $html),\n ));\n\n expect($links[0]->get())->toBe('https://www.example.com/c/e')\n ->and($links[1]->get())->toBe('https://www.example.com/f/g')\n ->and($links[2]->get())->toBe('https://www.example.com/c/h');\n});\n\nit('throws away the URL fragment part when withoutFragment() was called', function () {\n $html = <<\n \n \n \n link
    \n another link
    \n \n \n HTML;\n\n $step = (new GetLinks());\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo/baz'),\n new Response(200, [], $html),\n );\n\n $links = helper_invokeStepWithInput($step, $respondedRequest);\n\n expect($links[0]->get())->toBe('https://www.example.com/foo/bar#fragment')\n ->and($links[1]->get())->toBe('https://www.example.com/baz#quz-fragment');\n\n $step->withoutFragment();\n\n $links = helper_invokeStepWithInput($step, $respondedRequest);\n\n expect($links[0]->get())->toBe('https://www.example.com/foo/bar')\n ->and($links[1]->get())->toBe('https://www.example.com/baz');\n});\n\nit('ignores special non HTTP links', function () {\n $html = <<\n \n \n \n mailto link\n link one\n javascript link\n link two\n phone link\n link three\n \n \n HTML;\n\n $step = (new GetLinks());\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/home'),\n new Response(200, [], $html),\n );\n\n $links = helper_invokeStepWithInput($step, $respondedRequest);\n\n expect($links)->toHaveCount(3)\n ->and($links[0]->get())->toBe('https://www.example.com/one')\n ->and($links[1]->get())->toBe('https://www.example.com/two')\n ->and($links[2]->get())->toBe('https://www.example.com/three');\n});\n" }, { "path": "tests/Steps/Html/MetaDataTest.php", "content": "\n \n \n \n Hello World!\n \n HTML;\n\n $outputs = helper_invokeStepWithInput(new MetaData(), $html);\n\n expect($outputs[0]->get())->toBe(['title' => '']);\n});\n\nit('returns an array with the title and all meta tags having a name or property attribute', function () {\n $html = <<\n \n \n \n \n Hello World!\n \n \n \n \n \n \n Hello World!\n \n HTML;\n\n $outputs = helper_invokeStepWithInput(new MetaData(), $html);\n\n expect($outputs[0]->get())->toBe([\n 'title' => 'Hello World!',\n 'description' => 'This is a page saying: Hello World!',\n 'keywords' => 'lorem, ipsum, hello, world',\n 'og:title' => 'Hello World!',\n 'og:type' => 'website',\n ]);\n});\n\nit('returns only the meta tags defined via the only() method', function () {\n $html = <<\n \n \n \n \n Hello World!\n \n \n \n \n \n \n Hello World!\n \n HTML;\n\n $outputs = helper_invokeStepWithInput(Html::metaData()->only(['description', 'og:title']), $html);\n\n expect($outputs[0]->get())->toBe([\n 'description' => 'This is a page saying: Hello World!',\n 'og:title' => 'Hello World!',\n ]);\n});\n" }, { "path": "tests/Steps/Html/SchemaOrgTest.php", "content": "\n \n Foo Bar\n \n \n

    Baz

    Other content

    \n \n \n HTML;\n}\n\nfunction helper_schemaOrgExampleMultipleObjects(): string\n{\n return <<\n \n \n Foo Bar\n \n \n \n \n \n \n

    Some Article

    \n

    This is some article about something.

    \n \n \n \n HTML;\n}\n\nit('extracts schema.org data in JSON-LD format from an HTML document', function () {\n $html = helper_schemaOrgExampleOneJobPostingInBody();\n\n $outputs = helper_invokeStepWithInput(Html::schemaOrg(), $html);\n\n expect($outputs)->toHaveCount(1);\n\n expect($outputs[0]->get())->toBeInstanceOf(JobPosting::class);\n});\n\nit('converts the spatie schema.org objects to arrays when calling the toArray() method', function () {\n $html = helper_schemaOrgExampleOneJobPostingInBody();\n\n $outputs = helper_invokeStepWithInput(Html::schemaOrg()->toArray(), $html);\n\n expect($outputs)->toHaveCount(1);\n\n expect($outputs[0]->get())->toBeArray();\n\n expect($outputs[0]->get()['hiringOrganization'])->toBeArray();\n\n expect($outputs[0]->get()['hiringOrganization'])->toHaveKey('name');\n\n expect($outputs[0]->get()['hiringOrganization']['name'])->toBe('Foo Ltd.');\n});\n\nit('gets all the schema.org objects contained in a document', function () {\n $html = helper_schemaOrgExampleMultipleObjects();\n\n $outputs = helper_invokeStepWithInput(Html::schemaOrg(), $html);\n\n expect($outputs)->toHaveCount(3);\n});\n\nit('gets only schema.org objects of a certain type if you use the onlyType method', function () {\n $html = helper_schemaOrgExampleMultipleObjects();\n\n $outputs = helper_invokeStepWithInput(\n Html::schemaOrg()->onlyType('Article'),\n $html,\n );\n\n expect($outputs)->toHaveCount(1);\n\n expect($outputs[0]->get())->toBeInstanceOf(Article::class);\n});\n\nit('also finds schema.org objects of a certain type in children of another schema.org object', function () {\n $html = helper_schemaOrgExampleMultipleObjects();\n\n $outputs = helper_invokeStepWithInput(\n Html::schemaOrg()->onlyType('Organization'),\n $html,\n );\n\n expect($outputs)->toHaveCount(2);\n\n expect($outputs[0]->get()->getProperty('name'))->toBe('Example Company');\n\n expect($outputs[1]->get()->getProperty('name'))->toBe('Some Organization, Inc.');\n});\n\nit('extracts certain data from schema.org objects when using the extract() method', function () {\n $html = helper_schemaOrgExampleMultipleObjects();\n\n $outputs = helper_invokeStepWithInput(\n Html::schemaOrg()->onlyType('Article')->extract(['url', 'headline', 'publisher' => 'publisher.name']),\n $html,\n );\n\n expect($outputs)->toHaveCount(1);\n\n expect($outputs[0]->get())->toBe([\n 'url' => 'https://de.example.org/articles/some',\n 'headline' => 'This is some article about something.',\n 'publisher' => 'Some Organization, Inc.',\n ]);\n});\n\ntest('If an object doesn\\'t contain a property from the extract mapping, it\\'s just null in the output', function () {\n $html = helper_schemaOrgExampleMultipleObjects();\n\n $outputs = helper_invokeStepWithInput(\n Html::schemaOrg()->onlyType('Article')->extract(['url', 'headline', 'alternativeHeadline']),\n $html,\n );\n\n expect($outputs)->toHaveCount(1);\n\n expect($outputs[0]->get())->toBe([\n 'url' => 'https://de.example.org/articles/some',\n 'headline' => 'This is some article about something.',\n 'alternativeHeadline' => null,\n ]);\n});\n" }, { "path": "tests/Steps/Html/XPathQueryTest.php", "content": "throws(InvalidDomQueryException::class);\n\ntest('The apply method returns a string for a single match', function () {\n $xml = 'test';\n\n expect((new XPathQuery('//item'))->apply(new XmlDocument($xml)))->toBe('test');\n});\n\ntest('The apply method returns an array of strings for multiple matches', function () {\n $html = 'testtest 2 subtest 3';\n\n expect((new XPathQuery('//item'))->apply(new HtmlDocument($html)))->toBe(['test', 'test 2 sub', 'test 3']);\n});\n\ntest('The apply method returns null if nothing matches', function () {\n $xml = 'test';\n\n expect((new XPathQuery('//aitem'))->apply(new XmlDocument($xml)))->toBeNull();\n});\n\nit('trims whitespace', function () {\n $xml = <<\n test\n \n XML;\n\n expect((new XPathQuery('//item'))->apply(new XmlDocument($xml)))->toBe('test');\n});\n\nit('contains inner tags when the html method is called', function () {\n $xml = 'test sub';\n\n expect((new XPathQuery('//item'))->html()->apply(new XmlDocument($xml)))->toBe('test sub');\n});\n\nit('contains also the outer tag when the outerHtml method is called', function () {\n $xml = 'test sub';\n\n expect((new XPathQuery('//item'))->outerHtml()->apply(new XmlDocument($xml)))->toBe('test sub');\n});\n\nit('gets the contents of an attribute using the attribute method', function () {\n $xml = 'test';\n\n expect((new XPathQuery('//item'))->attribute('attr')->apply(new XmlDocument($xml)))->toBe('content');\n});\n\ntest('getting an attribute value returns an empty string when the attribute does not exist', function () {\n $xml = 'test';\n\n expect((new XPathQuery('//item'))->attribute('attr')->apply(new XmlDocument($xml)))->toBe('');\n});\n\nit('turns the value into an absolute url when toAbsoluteUrl() is called', function () {\n $xml = '/foo/bar';\n\n $document = new XmlDocument($xml);\n\n $query = (new XPathQuery('//item'))\n ->setBaseUrl('https://www.example.com');\n\n expect($query->apply($document))->toBe('/foo/bar');\n\n $query->toAbsoluteUrl();\n\n expect($query->apply($document))->toBe('https://www.example.com/foo/bar');\n});\n\nit('gets an absolute link from the href attribute of a link element, when the link() method is called', function () {\n $html = '';\n\n $document = new HtmlDocument($html);\n\n $selector = (new XPathQuery('//*[@id=\\'foo\\']/a[@class=\\'bar\\']'))\n ->setBaseUrl('https://www.example.com/');\n\n expect($selector->apply($document))->toBe('Foo');\n\n $selector->link();\n\n expect($selector->apply($document))->toBe('https://www.example.com/foo/bar');\n});\n\nit('gets only the first matching element when the first() method is called', function () {\n $selector = (new XPathQuery(\"//*[@id = 'list']/*[contains(@class, 'item')]\"))->first();\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe('one');\n});\n\nit('gets only the last matching element when the last() method is called', function () {\n $selector = (new XPathQuery(\"//*[@id = 'list']/*[contains(@class, 'item')]\"))->last();\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe('four');\n});\n\nit('gets only the nth matching element when the nth() method is called', function () {\n $selector = (new XPathQuery(\"//*[@id = 'list']/*[contains(@class, 'item')]\"))->nth(3);\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe('three');\n});\n\nit('returns null when no nth matching element exists', function () {\n $selector = (new XPathQuery(\"//*[@id = 'list']/*[contains(@class, 'item')]\"))->nth(5);\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBeNull();\n});\n\nit('gets only even matching elements when the even() method is called', function () {\n $selector = (new XPathQuery(\"//*[@id = 'list']/*[contains(@class, 'item')]\"))->even();\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe(['two', 'four']);\n});\n\nit('gets only odd matching elements when the odd() method is called', function () {\n $selector = (new XPathQuery(\"//*[@id = 'list']/*[contains(@class, 'item')]\"))->odd();\n\n expect($selector->apply(new HtmlDocument(helper_getSimpleListHtml())))->toBe(['one', 'three']);\n});\n" }, { "path": "tests/Steps/HtmlTest.php", "content": "extract('.title'),\n helper_getHtmlContent('bookstore.html'),\n );\n\n expect($output)->toHaveCount(4)\n ->and($output[0]->get())->toBe('Everyday Italian')\n ->and($output[3]->get())->toBe('Learning XML');\n});\n\nit('extracts data from an HTML document with CSS selectors by default', function () {\n $output = helper_invokeStepWithInput(\n Html::each('#bookstore .book')->extract(['title' => '.title', 'author' => '.author', 'year' => '.year']),\n helper_getHtmlContent('bookstore.html'),\n );\n\n expect($output)->toHaveCount(4)\n ->and($output[0]->get())->toBe(\n ['title' => 'Everyday Italian', 'author' => 'Giada De Laurentiis', 'year' => '2005'],\n )\n ->and($output[1]->get())->toBe(['title' => 'Harry Potter', 'author' => 'J K. Rowling', 'year' => '2005'])\n ->and($output[2]->get())->toBe(\n [\n 'title' => 'XQuery Kick Start',\n 'author' => ['James McGovern', 'Per Bothner', 'Kurt Cagle', 'James Linn', 'Vaidyanathan Nagarajan'],\n 'year' => '2003',\n ],\n )\n ->and($output[3]->get())->toBe(['title' => 'Learning XML', 'author' => 'Erik T. Ray', 'year' => '2003']);\n});\n\nit('can also extract data using XPath queries', function () {\n $output = helper_invokeStepWithInput(\n Html::each(Dom::xPath('//div[@id=\\'bookstore\\']/div[@class=\\'book\\']'))->extract([\n 'title' => Dom::xPath('//h3[@class=\\'title\\']'),\n 'author' => Dom::xPath('//*[@class=\\'author\\']'),\n 'year' => Dom::xPath('//span[@class=\\'year\\']'),\n ]),\n helper_getHtmlContent('bookstore.html'),\n );\n\n expect($output)->toHaveCount(4)\n ->and($output[2]->get())->toBe(\n [\n 'title' => 'XQuery Kick Start',\n 'author' => ['James McGovern', 'Per Bothner', 'Kurt Cagle', 'James Linn', 'Vaidyanathan Nagarajan'],\n 'year' => '2003',\n ],\n );\n});\n\nit('returns only one (compound) output when the root method is used', function () {\n $output = helper_invokeStepWithInput(\n Html::root()->extract(['title' => '.title', 'author' => '.author', 'year' => '.year',]),\n helper_getHtmlContent('bookstore.html'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get()['title'])->toBe(['Everyday Italian', 'Harry Potter', 'XQuery Kick Start', 'Learning XML']);\n});\n\nit('extracts the data of the first matching element when the first method is used', function () {\n $output = helper_invokeStepWithInput(\n Html::first('#bookstore .book')->extract(['title' => '.title', 'author' => '.author', 'year' => '.year']),\n helper_getHtmlContent('bookstore.html'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(\n ['title' => 'Everyday Italian', 'author' => 'Giada De Laurentiis', 'year' => '2005'],\n );\n});\n\nit('extracts the data of the last matching element when the last method is used', function () {\n $output = helper_invokeStepWithInput(\n Html::last('#bookstore .book')->extract(['title' => '.title', 'author' => '.author', 'year' => '.year']),\n helper_getHtmlContent('bookstore.html'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['title' => 'Learning XML', 'author' => 'Erik T. Ray', 'year' => '2003']);\n});\n\ntest(\n 'you can extract data in a second level to the output array using another Html step as an element in the mapping ' .\n 'array',\n function () {\n $response = new RespondedRequest(\n new Request('GET', 'https://www.example.com/meetups/some-meetup/'),\n new Response(body: helper_getHtmlContent('event.html')),\n );\n\n $output = helper_invokeStepWithInput(\n Html::root()->extract([\n 'title' => '#event h1',\n 'location' => '#event .location',\n 'date' => '#event .date',\n 'talks' => Html::each('#event .talks .talk')->extract([\n 'title' => '.title',\n 'speaker' => '.speaker',\n 'slides' => Dom::cssSelector('.slidesLink')->attribute('href')->toAbsoluteUrl(),\n ]),\n ]),\n $response,\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe([\n 'title' => 'Some Meetup',\n 'location' => 'Somewhere',\n 'date' => '2023-01-14 21:00',\n 'talks' => [\n [\n 'title' => 'Sophisticated talk title',\n 'speaker' => 'Super Mario',\n 'slides' => 'https://www.example.com/meetups/some-meetup/slides/talk1.pdf',\n ],\n [\n 'title' => 'Simple beginner talk',\n 'speaker' => 'Luigi',\n 'slides' => 'https://www.example.com/meetups/some-meetup/slides/talk2.pdf',\n ],\n [\n 'title' => 'Fun talk',\n 'speaker' => 'Princess Peach',\n 'slides' => 'https://www.example.com/meetups/some-meetup/slides/talk3.pdf',\n ],\n ],\n ]);\n },\n);\n\ntest(\n 'When a child step is nested in the extraction and does not use each(), the extracted value is an array with ' .\n 'the keys defined in extract(), rather than an array of such arrays as it would be with each().',\n function () {\n $xml = <<\n \n something\n \n
    \n
    ABCDEFGmbH
    \n
    1984
    \n
    \n Germany, Frankfurt\n
    \n
    \n
    \n
    Saubär GmbH
    \n
    2014
    \n
    \n Austria, Klagenfurt\n
    \n
    \n \n \n HTML;\n\n $expectedCompany1 = [\n 'name' => 'ABCDEFGmbH',\n 'founded' => '1984',\n 'location' => ['country' => 'Germany', 'city' => 'Frankfurt'],\n ];\n\n $expectedCompany2 = [\n 'name' => 'Saubär GmbH',\n 'founded' => '2014',\n 'location' => ['country' => 'Austria', 'city' => 'Klagenfurt'],\n ];\n\n // With base root()\n $step = Html::each('.company')->extract([\n 'name' => '.name',\n 'founded' => '.founded',\n 'location' => Html::root()->extract(['country' => '.location .country', 'city' => '.location .city']),\n ]);\n\n $outputs = helper_invokeStepWithInput($step, $xml);\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe($expectedCompany1)\n ->and($outputs[1]->get())->toBe($expectedCompany2);\n\n // With base first()\n $step = Html::each('.company')->extract([\n 'name' => '.name',\n 'founded' => '.founded',\n 'location' => Html::first('.location')->extract(['country' => '.country', 'city' => '.city']),\n ]);\n\n $outputs = helper_invokeStepWithInput($step, $xml);\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe($expectedCompany1)\n ->and($outputs[1]->get())->toBe($expectedCompany2);\n\n // With base last()\n $step = Html::each('.company')->extract([\n 'name' => '.name',\n 'founded' => '.founded',\n 'location' => Html::last('.location')->extract(['country' => '.country', 'city' => '.city']),\n ]);\n\n $outputs = helper_invokeStepWithInput($step, $xml);\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe($expectedCompany1)\n ->and($outputs[1]->get())->toBe($expectedCompany2);\n },\n);\n\ntest(\n 'when selecting elements with each(), you can reference the element already selected within the each() selector ' .\n 'itself, in sub selectors',\n function () {\n $html = <<\n \n \n Bookstore Example in HTML :)\n \n \n
    \n
    \n direct element child\n
    \n sub child\n
    \n
    \n
    \n \n \n HTML;\n\n $response = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(body: $html),\n );\n\n $output = helper_invokeStepWithInput(\n Html::each('#list .element')->extract([\n // This is what this test is about. The element already selected in each (.element) can be\n // referenced in these child selectors.\n 'link' => Dom::cssSelector('.element > a')->link(),\n 'attribute' => Dom::cssSelector('')->attribute('data-attr'),\n ]),\n $response,\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe([\n 'link' => 'https://www.example.com/bar',\n 'attribute' => 'yo',\n ]);\n },\n);\n\ntest('the static getLink method works without argument', function () {\n expect(Html::getLink())->toBeInstanceOf(GetLink::class);\n});\n\ntest('the static getLinks method works without argument', function () {\n expect(Html::getLinks())->toBeInstanceOf(GetLinks::class);\n});\n" }, { "path": "tests/Steps/JsonTest.php", "content": " 'data.foo']), $respondedRequest);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'bar']);\n});\n\nit('accepts PSR-7 Response as input', function () {\n $json = '{ \"data\": { \"foo\": \"bar\" } }';\n\n $response = new Response(body: Utils::streamFor($json));\n\n $output = helper_invokeStepWithInput(Json::get(['foo' => 'data.foo']), $response);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'bar']);\n});\n\nit('extracts data defined using dot notation', function () {\n $json = << 'data.target.foo', 'baz' => 'data.target.baz']), $json);\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['foo' => 'bar', 'baz' => 'yo']);\n});\n\nit('uses the array values in the mapping as output key when no string keys defined in the mapping array', function () {\n $jsonString = << 'data.target.baz']), $jsonString);\n\n expect($output[0]->get())->toBe(['data.target.foo' => 'bar', 'baz' => 'yo']);\n});\n\nit('can get items from a json array using a numeric key', function () {\n $jsonString = << 'data.target.array.1.name']), $jsonString);\n\n expect($output[0]->get())->toBe(['name' => 'Eve']);\n});\n\ntest('Using the each method you can iterate over a json array and yield multiple results', function () {\n $json = << 'name', 'age' => 'age.years']), $json);\n\n expect($output)->toHaveCount(3)\n ->and($output[0]->get())->toBe(['name' => 'Peter', 'age' => 19])\n ->and($output[1]->get())->toBe(['name' => 'Paul', 'age' => 22])\n ->and($output[2]->get())->toBe(['name' => 'Mary', 'age' => 20]);\n});\n\ntest('When the root element is an array you can use each with empty string as param', function () {\n $jsonString = <<toHaveCount(4)\n ->and($output[0]->get())->toBe(['nickname' => 'Axel'])\n ->and($output[1]->get())->toBe(['nickname' => 'Lilo'])\n ->and($output[2]->get())->toBe(['nickname' => 'Poppi'])\n ->and($output[3]->get())->toBe(['nickname' => 'Dominik']);\n\n});\n\nit('yields no results and logs a warning when the target for \"each\" does not exist', function () {\n $jsonString = '{ \"foo\": { \"bar\": [{ \"number\": \"one\" }, { \"number\": \"two\" }] } }';\n\n $step = Json::each('boo.bar', ['number']);\n\n $step->addLogger(new CliLogger());\n\n $output = helper_invokeStepWithInput($step, $jsonString);\n\n expect($output)->toHaveCount(0);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)->toContain('The target of \"each\" does not exist in the JSON data.');\n});\n\nit('also works with JS style JSON objects without quotes around keys', function () {\n $jsonString = <<toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['foo' => 'one', 'bar' => 'two', 'baz' => 'three']);\n});\n\nit('also correctly fixes keys without quotes, even when values contain colons', function () {\n $jsonString = <<toHaveCount(1)\n ->and($outputs[0]->get())\n ->toBe([\n 'foo' => 'https://www.example.com',\n 'bar' => 2,\n 'baz' => 'some: thing',\n ]);\n});\n\nit('also correctly fixes keys without quotes, when the value is an empty string', function () {\n $jsonString = <<toHaveCount(1)\n ->and($outputs[0]->get())\n ->toBe([\n 'foo' => '',\n 'bar' => 'baz',\n ]);\n});\n\nit('works with a string that is an HTML document and inside the body there\\'s a JSON object', function () {\n $jsonString = <<\n \n \n JSON\n \n \n { \"foo\": \"Hello World!\", \"bar\": \"baz\" }\n \n HTML;\n\n $outputs = helper_invokeStepWithInput(Json::get(['title' => 'foo']), $jsonString);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())\n ->toBe(['title' => 'Hello World!']);\n});\n\nit('gets the whole JSON object as array, when using the all() method', function () {\n $jsonString = <<toHaveCount(1)\n ->and($outputs[0]->get())\n ->toBe([\n 'foo' => 'one',\n 'bar' => 'two',\n 'array' => ['one', 'two', 'three'],\n ]);\n});\n\nit('can also map the whole decoded data array to a output property', function () {\n $jsonString = << '*']), $jsonString);\n\n expect($outputs)\n ->toHaveCount(1)\n ->and($outputs[0]->get())\n ->toBe([\n 'all' => [\n 'foo' => 'one',\n 'bar' => 'two',\n 'array' => ['one', 'two', 'three'],\n ],\n ]);\n});\n\ntest('when there is a key * in the object, the * gets that key, not the whole decoded data', function () {\n $jsonString = << '*']), $jsonString);\n\n expect($outputs)\n ->toHaveCount(1)\n ->and($outputs[0]->get())\n ->toBe(['shouldBeYes' => 'yes']);\n});\n\nit('can also get the whole decoded data in the each() context', function () {\n $jsonString = << '*']), $jsonString);\n\n expect($outputs)\n ->toHaveCount(3)\n ->and($outputs[0]->get())\n ->toBe(['full' => ['name' => 'foo', 'value' => 'one']])\n ->and($outputs[1]->get())\n ->toBe(['full' => ['name' => 'bar', 'value' => 'two']])\n ->and($outputs[2]->get())\n ->toBe(['full' => ['name' => 'baz', 'value' => 'three']]);\n});\n\ntest('in the each() context, when there is a key *, it gets that, not the whole decoded data', function () {\n $jsonString = << '*']), $jsonString);\n\n expect($outputs)\n ->toHaveCount(2)\n ->and($outputs[0]->get())\n ->toBe(['full' => 'yo'])\n ->and($outputs[1]->get())\n ->toBe(['full' => ['name' => 'bar', 'value' => 'two']]);\n});\n" }, { "path": "tests/Steps/Loading/GetSitemapsFromRobotsTxtTest.php", "content": "shouldReceive('sendRequest')\n ->once()\n ->withArgs(function (RequestInterface $request) {\n return $request->getUri()->__toString() === 'https://www.crwlr.software/robots.txt';\n })\n ->andReturn(new Response(200, body: Utils::streamFor($robotsTxt)));\n\n $loader = new HttpLoader(new UserAgent('SomeUserAgent'), $httpClient);\n\n $step = Sitemap::getSitemapsFromRobotsTxt()->setLoader($loader);\n\n $outputs = helper_invokeStepWithInput($step, new Input('https://www.crwlr.software/packages'));\n\n expect($outputs)->toHaveCount(3);\n});\n" }, { "path": "tests/Steps/Loading/Http/DocumentTest.php", "content": "foohello';\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(200, body: $body),\n );\n\n $document = new Document($respondedRequest);\n\n expect($document->dom())->toBeInstanceOf(HtmlDocument::class)\n ->and($document->dom()->outerHtml())->toBe(\n 'foohello',\n );\n});\n\nit('returns the effectiveUri as url()', function () {\n $body = 'foohello';\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(301, ['Location' => 'https://www.example.com/bar'], $body),\n );\n\n $respondedRequest->addRedirectUri('https://www.example.com/bar');\n\n $document = new Document($respondedRequest);\n\n expect((string) $document->url())->toBe('https://www.example.com/bar');\n});\n\nit('returns the effectiveUri as baseUrl() if no base tag in HTML', function () {\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(301, ['Location' => 'https://www.example.com/bar']),\n );\n\n $respondedRequest->addRedirectUri('https://www.example.com/bar');\n\n $document = new Document($respondedRequest);\n\n expect((string) $document->baseUrl())->toBe('https://www.example.com/bar');\n});\n\nit('returns the URL referenced in base tag as baseUrl()', function () {\n $body = 'foohello';\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(301, ['Location' => 'https://www.example.com/bar'], $body),\n );\n\n $respondedRequest->addRedirectUri('https://www.example.com/bar');\n\n $document = new Document($respondedRequest);\n\n expect((string) $document->baseUrl())->toBe('https://www.example.com/baz');\n});\n\nit('returns the effectiveUri as canonicalUrl() if no canonical link in HTML', function () {\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(301, ['Location' => 'https://www.example.com/bar']),\n );\n\n $respondedRequest->addRedirectUri('https://www.example.com/bar');\n\n $document = new Document($respondedRequest);\n\n expect($document->canonicalUrl())->toBe('https://www.example.com/bar');\n});\n\nit('returns the URL referenced in canonical link as canonicalUrl()', function () {\n $body = 'foohello';\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(301, ['Location' => 'https://www.example.com/bar'], $body),\n );\n\n $respondedRequest->addRedirectUri('https://www.example.com/bar');\n\n $document = new Document($respondedRequest);\n\n expect($document->canonicalUrl())->toBe('https://www.example.com/quz');\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/AbstractPaginatorTest.php", "content": "processLoaded($respondedRequest1->request, $respondedRequest1);\n\n expect($paginator->getLoaded())\n ->toBe(['f2be1fcc5667a8f4ee2fd7f48c69c909' => true])\n ->and($paginator->getLoadedCount())\n ->toBe(1)\n ->and($paginator->getLatestRequest())\n ->toBe($respondedRequest1->request);\n\n $respondedRequest2 = helper_getRespondedRequest('GET', 'https://www.example.com/bar', [], 'Yo');\n\n $paginator->processLoaded($respondedRequest2->request, $respondedRequest2);\n\n expect($paginator->getLoaded())->toBe([\n 'f2be1fcc5667a8f4ee2fd7f48c69c909' => true,\n 'd9e0c3987944f190782f5af9506eb478' => true,\n ])\n ->and($paginator->getLoadedCount())\n ->toBe(2)\n ->and($paginator->getLatestRequest())\n ->toBe($respondedRequest2->request);\n});\n\nit('registers loaded requests from RespondedRequest objects', function () {\n $paginator = new AbstractTestPaginator(nextUrl: 'https://www.example.com/bar');\n\n $requestOne = new Request('GET', Url::parsePsr7('https://www.example.com/foo'), [], 'Hi');\n\n $requestTwo = new Request('GET', Url::parsePsr7('https://www.example.com/bar'), [], 'Yo');\n\n $paginator->processLoaded($requestOne, new RespondedRequest($requestTwo, new Response()));\n\n expect($paginator->getLoaded())\n ->toBe(['d9e0c3987944f190782f5af9506eb478' => true])\n ->and($paginator->getLoadedCount())\n ->toBe(1)\n ->and($paginator->getLatestRequest())\n ->toBe($requestTwo);\n});\n\nit('knows when the max pages to load limit is reached', function () {\n $paginator = new AbstractTestPaginator(3);\n\n $respondedRequest = helper_getRespondedRequest(url: 'https://www.example.com/foo');\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->limitReached())->toBeFalse();\n\n $respondedRequest = helper_getRespondedRequest(url: 'https://www.example.com/bar');\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->limitReached())->toBeFalse();\n\n $respondedRequest = helper_getRespondedRequest(url: 'https://www.example.com/baz');\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->limitReached())->toBeTrue();\n\n expect($paginator->hasFinished())->toBeTrue();\n});\n\ntest('the same request is not registered twice', function () {\n $paginator = new AbstractTestPaginator();\n\n $respondedRequest = helper_getRespondedRequest();\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->getLoadedCount())->toBe(1);\n\n $respondedRequest = helper_getRespondedRequest();\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->getLoadedCount())->toBe(1);\n});\n\nit('logs a message when the max pages limit was reached', function () {\n $paginator = new AbstractTestPaginator(2);\n\n $respondedRequest = helper_getRespondedRequest(url: 'https://www.example.com/foo');\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $logger = new DummyLogger();\n\n $paginator->logWhenFinished($logger);\n\n expect($logger->messages[0])->toBe([\n 'level' => 'info',\n 'message' => 'Finished paginating.',\n ]);\n\n $respondedRequest = helper_getRespondedRequest(url: 'https://www.example.com/bar');\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $paginator->logWhenFinished($logger);\n\n expect($logger->messages[1])->toBe([\n 'level' => 'notice',\n 'message' => 'Max pages limit reached.',\n ]);\n});\n\nit('logs a message when it finished paginating', function () {\n $paginator = new AbstractTestPaginator();\n\n $paginator->stopWhen(PaginatorStopRules::isEmptyResponse());\n\n $respondedRequest = helper_getRespondedRequest();\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $logger = new DummyLogger();\n\n $paginator->logWhenFinished($logger);\n\n expect($logger->messages[0])->toBe([\n 'level' => 'info',\n 'message' => 'Finished paginating.',\n ]);\n});\n\nit('stops paginating when a stop condition is met', function () {\n $paginator = new AbstractTestPaginator();\n\n $paginator\n ->stopWhen(PaginatorStopRules::isEmptyResponse())\n ->stopWhen(PaginatorStopRules::isEmptyInJson('items'));\n\n $respondedRequest = helper_getRespondedRequest(\n url: 'https://www.example.com/list?page=1',\n responseBody: '{ \"items\": [\"foo\"] }',\n );\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeFalse();\n\n $respondedRequest = helper_getRespondedRequest(url: 'https://www.example.com/list?page=2', responseBody: '{}');\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeTrue();\n\n $paginator = new AbstractTestPaginator();\n\n $paginator\n ->stopWhen(PaginatorStopRules::isEmptyResponse())\n ->stopWhen(PaginatorStopRules::isEmptyInJson('items'));\n\n $respondedRequest = helper_getRespondedRequest(\n url: 'https://www.example.com/list?page=1',\n responseBody: '{ \"items\": [] }',\n );\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeTrue();\n});\n\ntest('after calling the setFinished() method, the hasFinished() method returns true', function () {\n $paginator = new AbstractTestPaginator();\n\n expect($paginator->hasFinished())->toBeFalse();\n\n $paginator->setFinished();\n\n expect($paginator->hasFinished())->toBeTrue();\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/QueryParams/AbstractQueryParamManipulatorTest.php", "content": "currentParamValue = $this->getCurrentValue($query);\n\n return $query;\n }\n };\n\n $manipulator->execute(Query::fromString('foo=bar'));\n\n expect($manipulator->currentParamValue)->toBe('bar');\n});\n\nit('gets the current value of a query param as integer', function () {\n $manipulator = new class ('foo') extends AbstractQueryParamManipulator {\n public int $currentParamValue = 0;\n\n public function execute(Query $query): Query\n {\n $this->currentParamValue = $this->getCurrentValueAsInt($query);\n\n return $query;\n }\n };\n\n $manipulator->execute(Query::fromString('foo=123'));\n\n expect($manipulator->currentParamValue)->toBe(123);\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/QueryParams/DecrementorTest.php", "content": "get('foo'))->toBe('20');\n\n $decrementor->execute($query);\n\n expect($query->get('foo'))->toBe('10');\n\n $decrementor->execute($query);\n\n expect($query->get('foo'))->toBe('0');\n\n $decrementor->execute($query);\n\n expect($query->get('foo'))->toBe('-10');\n});\n\nit('reduces a non first level query param value by a certain number', function () {\n $decrementor = new Decrementor('foo.bar.baz', 7, true);\n\n $query = Query::fromString('foo[bar][baz]=10');\n\n expect($decrementor->execute($query)->toString())->toBe('foo%5Bbar%5D%5Bbaz%5D=3');\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/QueryParams/IncrementorTest.php", "content": "get('foo'))->toBe('-10');\n\n $incrementor->execute($query);\n\n expect($query->get('foo'))->toBe('0');\n\n $incrementor->execute($query);\n\n expect($query->get('foo'))->toBe('10');\n\n $incrementor->execute($query);\n\n expect($query->get('foo'))->toBe('20');\n});\n\nit('increments a non first level query param value by a certain number', function () {\n $incrementor = new Incrementor('foo.bar.baz', 7, true);\n\n $query = Query::fromString('foo[bar][baz]=3');\n\n expect($incrementor->execute($query)->toString())->toBe('foo%5Bbar%5D%5Bbaz%5D=10');\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/QueryParamsPaginatorTest.php", "content": "increase('page')\n ->increase('offset', 20)\n ->decrease('foo', 10)\n ->decrease('bar', 20);\n\n $request = new Request('GET', 'https://www.example.com/list?page=1&offset=20&foo=40&bar=10');\n\n $respondedRequest = new RespondedRequest($request, new Response());\n\n $paginator->processLoaded($request, $respondedRequest);\n\n $nextRequest = $paginator->getNextRequest();\n\n expect($nextRequest?->getUri()->__toString())->toBe('https://www.example.com/list?page=2&offset=40&foo=30&bar=-10');\n});\n\nit('increases and decreases values in query params in the body', function () {\n $paginator = QueryParamsPaginator::paramsInBody()\n ->increase('page')\n ->increase('offset', 20)\n ->decrease('foo', 10)\n ->decrease('bar', 20);\n\n $request = new Request('POST', 'https://www.example.com/list', body: 'page=1&offset=20&foo=40&bar=10');\n\n $respondedRequest = new RespondedRequest($request, new Response());\n\n $paginator->processLoaded($request, $respondedRequest);\n\n $nextRequest = $paginator->getNextRequest();\n\n expect($nextRequest?->getMethod())\n ->toBe('POST')\n ->and($nextRequest?->getUri()->__toString())\n ->toBe('https://www.example.com/list')\n ->and($nextRequest?->getBody()->getContents())\n ->toBe('page=2&offset=40&foo=30&bar=-10');\n});\n\nit('increases and decreases non first level (of query array) parameters using dot notation', function () {\n $paginator = QueryParamsPaginator::paramsInBody()\n ->increaseUsingDotNotation('pagination.page')\n ->increase('pagination.size', 5, true)\n ->decreaseUsingDotNotation('pagination2.page')\n ->decrease('pagination2.size', 5, true);\n\n $request = new Request(\n 'POST',\n 'https://www.example.com/list',\n body: 'pagination[page]=1&pagination[size]=25&pagination2[page]=1&pagination2[size]=25&foo=bar',\n );\n\n $respondedRequest = new RespondedRequest($request, new Response());\n\n $paginator->processLoaded($request, $respondedRequest);\n\n $nextRequest = $paginator->getNextRequest();\n\n expect($nextRequest?->getBody()->getContents())\n ->toBe(\n 'pagination%5Bpage%5D=2&pagination%5Bsize%5D=30&pagination2%5Bpage%5D=0&pagination2%5Bsize%5D=20&foo=bar',\n );\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/SimpleWebsitePaginatorTest.php", "content": " $links\n */\nfunction helper_createResponseBodyWithPaginationLinks(array $links): string\n{\n $body = '
    ';\n\n foreach ($links as $url => $text) {\n $body .= '' . $text . ' ' . PHP_EOL;\n }\n\n return $body . '
    ';\n}\n\n/** @var TestCase $this */\n\nit('says it has finished when no initial response was provided yet', function () {\n $paginator = new SimpleWebsitePaginator('.pagination');\n\n expect($paginator->hasFinished())->toBeTrue();\n});\n\nit('says it has finished when a response is provided, but it has no pagination links', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing', '
    ');\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeTrue();\n});\n\nit('says it has not finished when an initial response with pagination links is provided', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $responseBody = helper_createResponseBodyWithPaginationLinks([\n '/listing?page=1' => 'First page',\n '/listing?page=2' => 'Next page',\n '/listing?page12' => 'Last page',\n ]);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeFalse();\n});\n\nit('has finished when the loaded pages count exceeds the max pages limit', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $responseBody = helper_createResponseBodyWithPaginationLinks([\n '/listing?page=1' => 'First page',\n '/listing?page=2' => 'Next page',\n '/listing?page12' => 'Last page',\n ]);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeFalse();\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=1', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeFalse();\n\n $responseBody = helper_createResponseBodyWithPaginationLinks([\n '/listing?page=1' => 'First page',\n '/listing?page=3' => 'Next page',\n '/listing?page12' => 'Last page',\n ]);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=2', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeTrue();\n});\n\nit('says it has finished when there are no more found pagination links, that haven\\'t been loaded yet', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $responseBody = helper_createResponseBodyWithPaginationLinks(['/listing?page=2' => 'Page Two']);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=1', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeFalse();\n\n $paginator->getNextRequest();\n\n $responseBody = helper_createResponseBodyWithPaginationLinks(['/listing?page=2' => 'Page Two']);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=2', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeTrue();\n});\n\nit('finds pagination links when the selector matches the link itself', function () {\n $paginator = new SimpleWebsitePaginator('.nextPageLink', 3);\n\n $responseBody = 'Next Page';\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=1', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->getNextRequest()?->getUri()->__toString())->toBe('https://www.example.com/listing?page=2');\n});\n\nit('finds pagination links when the selected element is a wrapper for pagination links', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $responseBody = '
    Next Page
    ';\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=1', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->getNextRequest()?->getUri()->__toString())->toBe('https://www.example.com/listing?page=2');\n});\n\nit('finds all pagination links, when multiple elements match the pagination links selector', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $responseBody = <<Next Page\n
    Last Page
    \n HTML;\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=1', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->getNextRequest()?->getUri()->__toString())->toBe('https://www.example.com/listing?page=2')\n ->and($paginator->getNextRequest()?->getUri()->__toString())->toBe('https://www.example.com/listing?page=12');\n\n});\n\nit('logs that max pages limit was reached when it was reached', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $responseBody = <<\n Page One\n Page Two\n Page Three\n Page Four\n \n HTML;\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=1', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=2', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=3', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeTrue();\n\n $paginator->logWhenFinished(new CliLogger());\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->toContain('Max pages limit reached');\n});\n\nit('logs that all found pagination links have been loaded when max pages limit was not reached', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $responseBody = <<\n Page One\n Page Two\n Page Three\n \n HTML;\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=1', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $paginator->getNextRequest();\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=2', $responseBody);\n\n $paginator->logWhenFinished(new CliLogger());\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $paginator->logWhenFinished(new CliLogger());\n\n $paginator->getNextRequest();\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing?page=3', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeTrue();\n\n $paginator->logWhenFinished(new CliLogger());\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)\n ->not()->toContain('Max pages limit reached')\n ->and($output)\n ->toContain('All found pagination links loaded');\n});\n\nit(\n 'always creates upcoming requests from the parent request, where a link was found (which does not have to be ' .\n 'the latest processed response)',\n function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $responseBody = <<\n Page One\n Page Two\n Page Three\n \n HTML;\n\n $respondedRequest = helper_getRespondedRequest(\n 'GET',\n 'https://www.example.com/list?page=1',\n ['foo' => 'bar'],\n responseBody: $responseBody,\n );\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $responseBody = <<\n Page One\n Page Two\n Page Three\n \n HTML;\n\n $respondedRequest = helper_getRespondedRequest(\n 'GET',\n 'https://www.example.com/list?page=2',\n ['foo' => 'baz'],\n responseBody: $responseBody,\n );\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n $nextRequest = $paginator->getNextRequest();\n\n expect($nextRequest?->getHeader('foo'))->toBe(['bar']);\n },\n);\n\nit('cleans up the stored parent requests always when getting the next request to load', function () {\n $paginator = new class ('.pagination') extends SimpleWebsitePaginator {\n /**\n * @return array\n */\n public function parentRequests(): array\n {\n return $this->parentRequests;\n }\n };\n\n $responseBody = <<\n Page Two\n Page Three\n \n HTML;\n\n $respondedRequest = helper_getRespondedRequest(\n 'GET',\n 'https://www.example.com/list?page=1',\n ['foo' => 'bar'],\n responseBody: $responseBody,\n );\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect(count($paginator->parentRequests()))->toBe(1);\n\n $nextRequest = $paginator->getNextRequest();\n\n if (!$nextRequest) {\n $this->fail('failed to get next request');\n }\n\n $respondedRequest = new RespondedRequest($nextRequest, new Response());\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect(count($paginator->parentRequests()))->toBe(1);\n\n $nextRequest = $paginator->getNextRequest();\n\n if (!$nextRequest) {\n $this->fail('failed to get next request');\n }\n\n $respondedRequest = new RespondedRequest($nextRequest, new Response());\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect(count($paginator->parentRequests()))->toBe(0);\n});\n\nit('does not stop, when a response does not meet the stop rule criterion', function () {\n $paginator = new SimpleWebsitePaginator('.pagination', 3);\n\n $paginator->stopWhen(PaginatorStopRules::contains('hello world'));\n\n $responseBody = helper_createResponseBodyWithPaginationLinks(['/listing?page=2' => 'Next page']);\n\n $respondedRequest = helper_getRespondedRequestWithResponseBody('/listing', $responseBody);\n\n $paginator->processLoaded($respondedRequest->request, $respondedRequest);\n\n expect($paginator->hasFinished())->toBeFalse();\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/StopRules/ContainsTest.php", "content": "shouldStop(new Request('GET', 'https://www.example.com/foo'), null))->toBeTrue();\n});\n\nit('stops when the string is contained in the response body', function () {\n $rule = PaginatorStopRules::contains('foo');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: 'This string contains foo'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('does not stop when the string is not contained in the response body', function () {\n $rule = PaginatorStopRules::contains('foo');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: 'This does not contain the string'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeFalse();\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/StopRules/IsEmptyInHtmlTest.php", "content": "shouldStop(new Request('GET', 'https://www.crwl.io/'), null))->toBeTrue();\n});\n\nit('should stop, when response is not HTML', function () {\n $rule = PaginatorStopRules::isEmptyInHtml('#list .item');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '{ \"foo\": \"bar\" }'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should stop, when the selector target does not exist in the HTML response', function () {\n $rule = PaginatorStopRules::isEmptyInHtml('#list');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '
    '),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should stop, when the selector target is empty in the response', function () {\n $rule = PaginatorStopRules::isEmptyInHtml('#list');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '
    '),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should not stop, when the selector target is not empty in the response', function () {\n $rule = PaginatorStopRules::isEmptyInHtml('#list');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '
    a
    '),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeFalse();\n\n // Also if the content is only child elements.\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '
    '),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeFalse();\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/StopRules/IsEmptyInJsonTest.php", "content": "shouldStop(new Request('GET', 'https://www.crwl.io/'), null))->toBeTrue();\n});\n\nit('throws an exception when response is not valid JSON', function () {\n $rule = PaginatorStopRules::isEmptyInJson('data.items');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: ''),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n})->throws(InvalidJsonException::class);\n\nit('should stop, when the dot notation key does not exist in the response', function () {\n $rule = PaginatorStopRules::isEmptyInJson('data.items');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '{ \"data\": { \"foo\": \"bar\" } }'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should stop, when the dot notation key is empty in the response', function () {\n $rule = PaginatorStopRules::isEmptyInJson('data.items');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '{ \"data\": { \"items\": [] } }'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should not stop, when the dot notation key is not empty in the response', function () {\n $rule = PaginatorStopRules::isEmptyInJson('data.items');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '{ \"data\": { \"items\": [\"foo\", \"bar\"] } }'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeFalse();\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/StopRules/IsEmptyInXmlTest.php", "content": "shouldStop(new Request('GET', 'https://www.crwl.io/'), null))->toBeTrue();\n});\n\nit('should stop, when response is not XML', function () {\n $rule = PaginatorStopRules::isEmptyInXml('channel item');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: '{}'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should stop, when the selector target does not exist in the XML response', function () {\n $rule = PaginatorStopRules::isEmptyInXml('channel item');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: ''),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should stop, when the selector target is empty in the response', function () {\n $rule = PaginatorStopRules::isEmptyInXml('channel item');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(\n body: ' ',\n ),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should not stop, when the selector target is not empty in the response', function () {\n $rule = PaginatorStopRules::isEmptyInXml('channel item');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(\n body: 'a',\n ),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeFalse();\n\n // Also if the content is only child elements.\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(\n body: '',\n ),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeFalse();\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/StopRules/IsEmptyResponseTest.php", "content": "shouldStop(new Request('GET', 'https://www.crwl.io/'), null))->toBeTrue();\n});\n\nit('should stop, when the response body is empty', function () {\n $rule = PaginatorStopRules::isEmptyResponse();\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: ''),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should stop, when the response body is only spaces', function () {\n $rule = PaginatorStopRules::isEmptyResponse();\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/'),\n new Response(body: \" \\n\\r\\t \"),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should stop, when the response body is an empty JSON array', function () {\n $rule = PaginatorStopRules::isEmptyResponse();\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwlr.software/packages'),\n new Response(body: \" [] \"),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('should stop, when the response body is an empty JSON object', function () {\n $rule = PaginatorStopRules::isEmptyResponse();\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/en/home'),\n new Response(body: \"{}\"),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n" }, { "path": "tests/Steps/Loading/Http/Paginators/StopRules/NotContainsTest.php", "content": "shouldStop(new Request('GET', 'https://www.example.com/foo'), null))->toBeTrue();\n});\n\nit('stops when the string is not contained in the response body', function () {\n $rule = PaginatorStopRules::notContains('foo');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: 'This does not contain the string'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeTrue();\n});\n\nit('does not stop when the string is contained in the response body', function () {\n $rule = PaginatorStopRules::notContains('foo');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/'),\n new Response(body: 'This contains the string foo'),\n );\n\n expect($rule->shouldStop($respondedRequest->request, $respondedRequest))->toBeFalse();\n});\n" }, { "path": "tests/Steps/Loading/HttpTest.php", "content": "shouldReceive('load')->once();\n\n $step = (new Http('GET'))->setLoader($loader);\n\n helper_traverseIterable($step->invokeStep(new Input('https://www.foo.bar/baz')));\n});\n\nit('can be invoked with a PSR-7 Uri object as input', function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('load')->once();\n\n $step = (new Http('GET'))->setLoader($loader);\n\n helper_traverseIterable($step->invokeStep(new Input(Url::parsePsr7('https://www.linkedin.com/'))));\n});\n\nit('logs an error message when invoked with something else as input', function () {\n $logger = new DummyLogger();\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $step = (new Http('GET'))->setLoader($loader)->addLogger($logger);\n\n helper_traverseIterable($step->invokeStep(new Input(new stdClass())));\n\n expect($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toStartWith(\n 'The Crwlr\\Crawler\\Steps\\Loading\\Http step was called with input that it can not work with:',\n )\n ->and($logger->messages[0]['message'])->toEndWith('. The invalid input is of type object.');\n});\n\nit('logs an error message when invoked with a relative reference URI', function () {\n $logger = new DummyLogger();\n\n $loader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);\n\n $step = (new Http('GET'))->setLoader($loader)->addLogger($logger);\n\n helper_invokeStepWithInput($step, '/foo/bar');\n\n expect($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toBe(\n 'Invalid input URL: /foo/bar - The URI is a relative reference and therefore can\\'t be loaded.',\n );\n});\n\nit('catches the exception and logs an error when feeded with an invalid URL', function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $logger = new DummyLogger();\n\n $step = (new Http('GET'))->setLoader($loader);\n\n $step->addLogger($logger);\n\n helper_traverseIterable($step->invokeStep(new Input('https://')));\n\n expect($logger->messages)->toHaveCount(1)\n ->and($logger->messages[0]['level'])->toBe('error')\n ->and($logger->messages[0]['message'])->toBe(\n 'The Crwlr\\\\Crawler\\\\Steps\\\\Loading\\\\Http step was called with input that it can not work with: https:// ' .\n 'is not a valid URL.',\n );\n});\n\nit('throws an exception when invoked with a relative reference URI and stopOnErrorResponse() was called', function () {\n $logger = new DummyLogger();\n\n $loader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);\n\n $step = (new Http('GET'))->setLoader($loader)->addLogger($logger);\n\n $step->stopOnErrorResponse();\n\n helper_invokeStepWithInput($step, '/foo/bar');\n})->throws(InvalidArgumentException::class);\n\ntest('You can set the request method via constructor', function (string $httpMethod) {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('load')->withArgs(function (RequestInterface $request) use ($httpMethod) {\n return $request->getMethod() === $httpMethod;\n })->once();\n\n if ($httpMethod !== 'GET') {\n $loader->shouldReceive('usesHeadlessBrowser')->andReturnFalse();\n }\n\n $step = (new Http($httpMethod))->setLoader($loader);\n\n helper_traverseIterable($step->invokeStep(new Input('https://www.foo.bar/baz')));\n})->with(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']);\n\ntest('You can set request headers via constructor', function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $headers = [\n 'Accept' => [\n 'text/html',\n 'application/xhtml+xml',\n 'application/xml;q=0.9',\n 'image/avif',\n 'image/webp',\n 'image/apng',\n '*/*;q=0.8',\n 'application/signed-exchange;v=b3;q=0.9',\n ],\n 'Accept-Encoding' => ['gzip', 'deflate', 'br'],\n 'Accept-Language' => ['de-DE', 'de;q=0.9', 'en-US;q=0.8', 'en;q=0.7'],\n ];\n\n $loader->shouldReceive('load')->withArgs(function (RequestInterface $request) use ($headers) {\n foreach ($headers as $headerName => $values) {\n if (!$request->getHeader($headerName) || $request->getHeader($headerName) !== $values) {\n return false;\n }\n }\n\n return true;\n })->once();\n\n $step = (new Http('GET', $headers))->setLoader($loader);\n\n helper_traverseIterable($step->invokeStep(new Input('https://www.crwlr.software/packages/url')));\n});\n\ntest('You can set request body via constructor', function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $body = 'This is the request body';\n\n $loader->shouldReceive('load')->withArgs(function (RequestInterface $request) use ($body) {\n return $request->getBody()->getContents() === $body;\n })->once();\n\n $loader->shouldReceive('usesHeadlessBrowser')->andReturnFalse();\n\n $step = (new Http('PATCH', [], $body))->setLoader($loader);\n\n helper_traverseIterable($step->invokeStep(new Input('https://github.com/')));\n});\n\ntest('You can set the http version for the request via constructor', function (string $httpVersion) {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('load')->withArgs(function (RequestInterface $request) use ($httpVersion) {\n return $request->getProtocolVersion() === $httpVersion;\n })->once();\n\n $loader->shouldReceive('usesHeadlessBrowser')->andReturnFalse();\n\n $step = (new Http('PATCH', [], 'body', $httpVersion))->setLoader($loader);\n\n helper_traverseIterable($step->invokeStep(new Input('https://packagist.org/packages/crwlr/url')));\n})->with(['1.0', '1.1', '2.0']);\n\nit('has static methods to create instances with all the different http methods', function (string $httpMethod) {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('load')->withArgs(function (RequestInterface $request) use ($httpMethod) {\n return $request->getMethod() === $httpMethod;\n })->once();\n\n if ($httpMethod !== 'GET') {\n $loader->shouldReceive('usesHeadlessBrowser')->andReturnFalse();\n }\n\n $step = (Http::{strtolower($httpMethod)}())->setLoader($loader);\n\n helper_traverseIterable($step->invokeStep(new Input('https://dev.to/otsch')));\n})->with(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']);\n\nit(\n 'calls the loadOrFail() loader method when the stopOnErrorResponse() method was called',\n function (string $httpMethod) {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('loadOrFail')->withArgs(function (RequestInterface $request) use ($httpMethod) {\n return $request->getMethod() === $httpMethod;\n })->once()->andReturn(new RespondedRequest(new Request('GET', '/foo'), new Response(200)));\n\n if ($httpMethod !== 'GET') {\n $loader->shouldReceive('usesHeadlessBrowser')->andReturnFalse();\n }\n\n $step = (Http::{strtolower($httpMethod)}())\n ->setLoader($loader)\n ->stopOnErrorResponse();\n\n helper_traverseIterable($step->invokeStep(new Input('https://example.com/otsch')));\n },\n)->with(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']);\n\ntest('you can keep response properties with their aliases', function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('load')->once()->andReturn(\n new RespondedRequest(\n new Request('GET', 'https://www.example.com/testresponse'),\n new Response(202, ['foo' => 'bar'], Utils::streamFor('testbody')),\n ),\n );\n\n $step = Http::get()\n ->setLoader($loader)\n ->keep(['url', 'status', 'headers', 'body']);\n\n $outputs = helper_invokeStepWithInput($step);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->keep)->toBe([\n 'url' => 'https://www.example.com/testresponse',\n 'status' => 202,\n 'headers' => ['foo' => ['bar']],\n 'body' => 'testbody',\n ]);\n\n});\n\ntest(\n 'the value behind url and uri is the effectiveUri',\n function (string $outputKey) {\n $loader = Mockery::mock(HttpLoader::class);\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/testresponse'),\n new Response(202, ['foo' => 'bar'], Utils::streamFor('testbody')),\n );\n\n $respondedRequest->addRedirectUri('https://www.example.com/testresponseredirect');\n\n $loader->shouldReceive('load')->once()->andReturn($respondedRequest);\n\n $step = Http::get()\n ->setLoader($loader)\n ->keep([$outputKey]);\n\n $outputs = helper_invokeStepWithInput($step);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->keep)->toBe([$outputKey => 'https://www.example.com/testresponseredirect']);\n },\n)->with(['url', 'uri']);\n\nit('gets the URL for the request from an input array when useInputKeyAsUrl() was called', function () {\n $inputArray = [\n 'foo' => 'bar',\n 'someUrl' => 'https://www.example.com/baz',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('load')->withArgs(function (RequestInterface $request) use ($inputArray) {\n return $request->getUri()->__toString() === $inputArray['someUrl'];\n })->once()->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/baz'), new Response(200)));\n\n $step = Http::get()\n ->setLoader($loader)\n ->useInputKeyAsUrl('someUrl');\n\n helper_invokeStepWithInput($step, $inputArray);\n});\n\nit(\n 'automatically gets the URL for the request from an input array when it contains an url or uri key',\n function ($key) {\n $inputArray = [\n 'foo' => 'bar',\n $key => 'https://www.example.com/baz',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('load')->withArgs(function (RequestInterface $request) use ($inputArray, $key) {\n return $request->getUri()->__toString() === $inputArray[$key];\n })->once()->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/baz'), new Response(200)));\n\n $step = Http::get()\n ->setLoader($loader);\n\n helper_invokeStepWithInput($step, $inputArray);\n },\n)->with(['url', 'uri']);\n\nit('gets the body for the request from an input array when useInputKeyAsBody() was called', function () {\n $inputArray = [\n 'foo' => 'bar',\n 'someUrl' => 'https://www.example.com/baz',\n 'someBodyThatIUsedToKnow' => 'foo=bar&baz=quz',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) use ($inputArray) {\n return $request->getBody()->getContents() === $inputArray['someBodyThatIUsedToKnow'];\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/baz'), new Response(200)));\n\n $step = Http::get()\n ->setLoader($loader)\n ->useInputKeyAsUrl('someUrl')\n ->useInputKeyAsBody('someBodyThatIUsedToKnow');\n\n helper_invokeStepWithInput($step, $inputArray);\n});\n\nit('gets as single header for the request from an input array when useInputKeyAsHeader() was called', function () {\n $inputArray = [\n 'foo' => 'bar',\n 'someUrl' => 'https://www.example.com/baz',\n 'someHeader' => 'someHeaderValue',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) use ($inputArray) {\n return $request->getHeader('header-name-x') === [$inputArray['someHeader']];\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/baz'), new Response(200)));\n\n $step = Http::get()\n ->setLoader($loader)\n ->useInputKeyAsUrl('someUrl')\n ->useInputKeyAsHeader('someHeader', 'header-name-x');\n\n helper_invokeStepWithInput($step, $inputArray);\n});\n\nit('uses the input key as header name if no header name defined as argument', function () {\n $inputArray = [\n 'foo' => 'bar',\n 'url' => 'https://www.example.com/baz',\n 'header-name' => 'someHeaderValue',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) use ($inputArray) {\n return $request->getHeader('header-name') === [$inputArray['header-name']];\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/baz'), new Response(200)));\n\n $step = Http::get()\n ->setLoader($loader)\n ->useInputKeyAsHeader('header-name');\n\n helper_invokeStepWithInput($step, $inputArray);\n});\n\nit('merges header values if you provide a static header value and use an input value as header', function () {\n $inputArray = [\n 'foo' => 'bar',\n 'someUrl' => 'https://www.example.com/baz',\n 'someHeader' => 'someHeaderValue',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) use ($inputArray) {\n return $request->getHeader('header-name-x') === ['foo', $inputArray['someHeader']];\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/baz'), new Response(200)));\n\n $step = Http::get(['header-name-x' => 'foo'])\n ->setLoader($loader)\n ->useInputKeyAsUrl('someUrl')\n ->useInputKeyAsHeader('someHeader', 'header-name-x');\n\n helper_invokeStepWithInput($step, $inputArray);\n});\n\ntest('you can use useInputKeyAsHeader() multiple times', function () {\n $inputArray = [\n 'foo' => 'bar',\n 'someUrl' => 'https://www.example.com/baz',\n 'someHeader' => 'someHeaderValue',\n 'anotherHeader' => 'anotherHeaderValue',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) use ($inputArray) {\n return $request->getHeader('header-name-x') === [$inputArray['someHeader']] &&\n $request->getHeader('header-name-y') === [$inputArray['anotherHeader']];\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/baz'), new Response(200)));\n\n $step = Http::get()\n ->setLoader($loader)\n ->useInputKeyAsUrl('someUrl')\n ->useInputKeyAsHeader('someHeader', 'header-name-x')\n ->useInputKeyAsHeader('anotherHeader', 'header-name-y');\n\n helper_invokeStepWithInput($step, $inputArray);\n});\n\nit('gets multiple headers from an input array using useInputKeyAsHeaders()', function () {\n $inputArray = [\n 'foo' => 'bar',\n 'someUrl' => 'https://www.example.com/baz',\n 'customHeaders' => [\n 'header-name-x' => 'foo',\n 'header-name-y' => ['bar', 'baz'],\n ],\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) use ($inputArray) {\n $customHeaders = $inputArray['customHeaders'];\n\n $yHeaderExpectedValue = array_merge(['quz'], $customHeaders['header-name-y']);\n\n return $request->getHeader('header-name-x') === [$customHeaders['header-name-x']] &&\n $request->getHeader('header-name-y') === $yHeaderExpectedValue;\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/baz'), new Response(200)));\n\n $step = Http::get(['header-name-y' => 'quz'])\n ->setLoader($loader)\n ->useInputKeyAsUrl('someUrl')\n ->useInputKeyAsHeaders('customHeaders');\n\n helper_invokeStepWithInput($step, $inputArray);\n});\n\nit('uses a static URL when defined', function () {\n $input = 'foo';\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) {\n return $request->getUri()->__toString() === 'https://www.example.com/servus';\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/servus'), new Response(200)));\n\n $step = Http::get()\n ->setLoader($loader)\n ->staticUrl('https://www.example.com/servus');\n\n helper_invokeStepWithInput($step, $input);\n});\n\nit('resolves variables in a static URL from input data', function () {\n $input = ['one' => 'foo', 'two' => 'bar'];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->andReturn(false);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) {\n return $request->getUri()->__toString() === 'https://www.example.com/foo/bar/baz';\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/foo/bar/baz'), new Response(200)));\n\n $step = Http::get()\n ->setLoader($loader)\n ->staticUrl('https://www.example.com/[crwl:\\'one\\']/[crwl:two]/baz');\n\n helper_invokeStepWithInput($step, $input);\n});\n\nit('resolves variables in the request body from input data', function () {\n $input = [\n 'url' => 'https://www.example.com/foo',\n 'hey' => 'ho',\n 'yo' => 'lo',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->andReturn(false);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) {\n $bodyString = Http::getBodyString($request);\n\n return $bodyString === 'Ho ho ho and lo asdf';\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/foo'), new Response(200)));\n\n $step = Http::post(body: 'Ho ho [crwl:hey] and [crwl:yo] asdf')\n ->setLoader($loader);\n\n helper_invokeStepWithInput($step, $input);\n});\n\nit('resolves variables in request headers from input data', function () {\n $input = [\n 'url' => 'https://www.example.com/foo',\n 'encoding' => 'deflate, br',\n 'language' => 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',\n ];\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader\n ->shouldReceive('load')\n ->withArgs(function (RequestInterface $request) {\n return $request->getHeaderLine('Accept-Encoding') === 'gzip, deflate, br, zstd' &&\n $request->getHeaderLine('Accept-Language') === 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7';\n })\n ->once()\n ->andReturn(new RespondedRequest(new Request('GET', 'https://www.example.com/foo'), new Response(200)));\n\n $step = Http::get([\n 'Accept-Encoding' => 'gzip, [crwl:\"encoding\"], zstd',\n 'Accept-Language' => '[crwl:language]',\n ])\n ->setLoader($loader);\n\n helper_invokeStepWithInput($step, $input);\n});\n\ntest(\n 'the getBodyString() method does not generate a warning, when the response contains a ' .\n 'Content-Type: application/x-gzip header, but the content actually isn\\'t compressed',\n function () {\n $warnings = [];\n\n set_error_handler(function ($errno, $errstr) use (&$warnings) {\n if ($errno === E_WARNING) {\n $warnings[] = $errstr;\n }\n\n return false;\n });\n\n $response = helper_getRespondedRequest(\n url: 'https://example.com/yolo',\n responseHeaders: ['Content-Type' => 'application/x-gzip'],\n responseBody: 'Servas!',\n );\n\n $string = Http::getBodyString($response);\n\n restore_error_handler();\n\n expect($warnings)->toBeEmpty()\n ->and($string)->toBe('Servas!');\n },\n);\n\nit('rejects post browser navigate hooks, when the HTTP method is not GET', function (string $httpMethod) {\n $logger = new DummyLogger();\n\n $step = (new Http($httpMethod))->addLogger($logger)->postBrowserNavigateHook(BrowserAction::wait(1.0));\n\n expect($logger->messages)->toHaveCount(1)\n ->and($logger->messages[0]['message'])->toBe(\n 'A ' . $httpMethod . ' request cannot be executed using the (headless) browser, so post browser ' .\n 'navigate hooks can\\'t be defined for this step either.',\n )\n ->and(invade($step)->postBrowserNavigateHooks)->toBe([]);\n})->with(['POST', 'PUT', 'PATCH', 'DELETE']);\n\nit(\n 'calls the HttpLoader::skipCacheForNextRequest() method before calling load when the skipCache() method was called',\n function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/blog/posts'),\n new Response(200, body: Utils::streamFor('blog posts')),\n );\n\n $loader->shouldReceive('skipCacheForNextRequest')->once();\n\n $loader->shouldReceive('load')->once()->andReturn($respondedRequest);\n\n $step = Http::get()->setLoader($loader)->skipCache();\n\n helper_invokeStepWithInput($step);\n },\n);\n\nit(\n 'calls the HttpLoader::skipCacheForNextRequest() method before calling loadOrFail() when the skipCache() method ' .\n 'was called',\n function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/blog/posts'),\n new Response(200, body: Utils::streamFor('blog posts')),\n );\n\n $loader->shouldReceive('skipCacheForNextRequest')->once();\n\n $loader->shouldReceive('loadOrFail')->once()->andReturn($respondedRequest);\n\n $step = Http::get()->setLoader($loader)->skipCache()->stopOnErrorResponse();\n\n helper_invokeStepWithInput($step);\n },\n);\n\nit(\n 'switches the loader to use the browser, when useBrowser() was called and the loader is configured to use the ' .\n 'HTTP client',\n function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->once()->andReturn(false);\n\n $loader->shouldReceive('useHeadlessBrowser')->once();\n\n $loader->shouldReceive('useHttpClient')->once();\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/hello/world'),\n new Response(200, body: Utils::streamFor('Hello World!')),\n );\n\n $loader->shouldReceive('load')->once()->andReturn($respondedRequest);\n\n $step = Http::get()->setLoader($loader)->useBrowser();\n\n helper_invokeStepWithInput($step);\n },\n);\n\nit(\n 'switches the loader to use the browser, when stopOnErrorResponse() and useBrowser() was called and the loader ' .\n 'is configured to use the HTTP client',\n function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->once()->andReturn(false);\n\n $loader->shouldReceive('useHeadlessBrowser')->once();\n\n $loader->shouldReceive('useHttpClient')->once();\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/hello/world'),\n new Response(200, body: Utils::streamFor('Hello World!')),\n );\n\n $loader->shouldReceive('loadOrFail')->once()->andReturn($respondedRequest);\n\n $step = Http::get()->setLoader($loader)->stopOnErrorResponse()->useBrowser();\n\n helper_invokeStepWithInput($step);\n },\n);\n\nit(\n 'does not switch the loader to use the browser, when useBrowser() was called, the loader is configured to use ' .\n 'the HTTP client, but the request method is not GET',\n function (string $httpMethod) {\n $logger = new DummyLogger();\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->once()->andReturn(false);\n\n $loader->shouldNotReceive('useHeadlessBrowser');\n\n $respondedRequest = new RespondedRequest(\n new Request($httpMethod, 'https://www.example.com/something'),\n new Response(200, body: Utils::streamFor('Something!')),\n );\n\n $loader->shouldReceive('load')->once()->andReturn($respondedRequest);\n\n $step = Http::{$httpMethod}()->setLoader($loader)->addLogger($logger)->useBrowser();\n\n helper_invokeStepWithInput($step);\n\n expect($logger->messages)->toHaveCount(1)\n ->and($logger->messages[0]['message'])->toBe(\n 'The (headless) browser can only be used for GET requests! Therefore this step will use the HTTP ' .\n 'client for loading.',\n );\n },\n)->with(['post', 'put', 'patch', 'delete']);\n\nit(\n 'automatically switches the loader to use the HTTP client, when the HTTP method is not GET and the loader is ' .\n 'configured to use the browser',\n function (string $httpMethod) {\n $logger = new DummyLogger();\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->once()->andReturn(true);\n\n $loader->shouldReceive('useHttpClient')->once();\n\n $loader->shouldReceive('useHeadlessBrowser')->once();\n\n $respondedRequest = new RespondedRequest(\n new Request($httpMethod, 'https://www.example.com/something'),\n new Response(200, body: Utils::streamFor('Something!')),\n );\n\n $loader->shouldReceive('load')->once()->andReturn($respondedRequest);\n\n $step = Http::{$httpMethod}()->setLoader($loader)->addLogger($logger)->useBrowser();\n\n helper_invokeStepWithInput($step);\n\n expect($logger->messages)->toHaveCount(1)\n ->and($logger->messages[0]['message'])->toBe(\n 'The (headless) browser can only be used for GET requests! Therefore this step will use the HTTP ' .\n 'client for loading.',\n );\n },\n)->with(['post', 'put', 'patch', 'delete']);\n\nit(\n 'switches back the loader to use the HTTP client, when stopOnErrorResponse() and useBrowser() was called and ' .\n 'loading throws an exception',\n function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->once()->andReturn(false);\n\n $loader->shouldReceive('useHeadlessBrowser')->once();\n\n $loader->shouldReceive('useHttpClient')->once();\n\n $loader->shouldReceive('loadOrFail')->once()->andThrow(new LoadingException('error message'));\n\n $step = Http::get()->setLoader($loader)->stopOnErrorResponse()->useBrowser();\n\n try {\n helper_invokeStepWithInput($step);\n } catch (Throwable $exception) {\n }\n },\n);\n\nit(\n 'does not call the useHeadlessBrowser() method of the loader, when useBrowser() was called and the loader is ' .\n 'already configured to use the browser',\n function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->once()->andReturn(true);\n\n $loader->shouldNotReceive('useHeadlessBrowser');\n\n $loader->shouldNotReceive('useHttpClient');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/hello/world'),\n new Response(200, body: Utils::streamFor('Hello World!')),\n );\n\n $loader->shouldReceive('load')->once()->andReturn($respondedRequest);\n\n $step = Http::get()->setLoader($loader)->useBrowser();\n\n helper_invokeStepWithInput($step);\n },\n);\n\nit(\n 'does not call the useHeadlessBrowser() method of the loader, when stopOnErrorResponse() and useBrowser() was ' .\n 'called and the loader is already configured to use the browser',\n function () {\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('usesHeadlessBrowser')->once()->andReturn(true);\n\n $loader->shouldNotReceive('useHeadlessBrowser');\n\n $loader->shouldNotReceive('useHttpClient');\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/hello/world'),\n new Response(200, body: Utils::streamFor('Hello World!')),\n );\n\n $loader->shouldReceive('loadOrFail')->once()->andReturn($respondedRequest);\n\n $step = Http::get()->setLoader($loader)->stopOnErrorResponse()->useBrowser();\n\n helper_invokeStepWithInput($step);\n },\n);\n\nit(\n 'sets post browser navigate hooks, when useBrowser() was called and the loader is configured to use the HTTP ' .\n 'client',\n function () {\n $loader = Mockery::mock(HttpLoader::class)->makePartial();\n\n $browserHelperMock = Mockery::mock(HeadlessBrowserLoaderHelper::class);\n\n $loader->shouldReceive('browser')->andReturn($browserHelperMock);\n\n $browserHelperMock\n ->shouldReceive('setTempPostNavigateHooks')\n ->once()\n ->withArgs(function (array $hooks) {\n return $hooks[0] instanceof Closure;\n });\n\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.example.com/woop'),\n new Response(200, body: Utils::streamFor('Woop')),\n );\n\n $loader->shouldReceive('load')->once()->andReturn($respondedRequest);\n\n $step = Http::get()->setLoader($loader)->useBrowser()->postBrowserNavigateHook(BrowserAction::wait(1.0));\n\n helper_invokeStepWithInput($step);\n },\n);\n" }, { "path": "tests/Steps/Loading/LoadingStepTest.php", "content": "\n */\n use LoadingStep;\n\n protected function invoke(mixed $input): Generator\n {\n $this->getLoader()->load($input);\n\n yield [];\n }\n };\n\n $loader = Mockery::mock(HttpLoader::class);\n\n $loader->shouldReceive('load')->once();\n\n $step->setLoader($loader);\n\n helper_traverseIterable($step->invokeStep(new Input('https://www.digitalocean.com/blog')));\n});\n\ntest(\n 'you can provide a custom loader to a step via the withLoader() method, and it will be preferred to the loader ' .\n 'provided via setLoader()',\n function () {\n $loaderOne = Mockery::mock(Loader::class);\n\n $loaderOne->shouldNotReceive('load');\n\n $loaderTwo = Mockery::mock(Loader::class);\n\n $loaderTwo->shouldReceive('load')->once()->andReturn('Hi');\n\n $step = new class extends Step {\n /**\n * @use LoadingStep\n */\n use LoadingStep;\n\n protected function invoke(mixed $input): Generator\n {\n yield $this->getLoader()->load($input);\n }\n };\n\n $step->withLoader($loaderTwo);\n\n // The crawler will call the setLoader() method of the step after the step was added to the crawler.\n // So, the call to withLoader() will happen before that.\n // Nevertheless, the loader passed to withLoader() should be preferred.\n $step->setLoader($loaderOne);\n\n helper_invokeStepWithInput($step);\n },\n);\n" }, { "path": "tests/Steps/Refiners/AbstractRefinerTest.php", "content": "logger?->info('logging works');\n\n return $value;\n }\n\n public function testLogTypeWarning(): void\n {\n $this->logTypeWarning('Some::staticMethodName()', 'foo');\n }\n}\n\n/** @var TestCase $this */\n\nit('takes a logger that can be used in the Refiner', function () {\n $refiner = new SomeRefiner();\n\n $refiner->addLogger(new CliLogger());\n\n $refiner->refine('foo');\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)->toContain('logging works');\n});\n\nit('provides a method for children to log a warning if the type of the incoming value is wrong', function () {\n (new SomeRefiner())->addLogger(new CliLogger())->testLogTypeWarning();\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)->toContain('Refiner Some::staticMethodName() can\\'t be applied to value of type string');\n});\n" }, { "path": "tests/Steps/Refiners/DateTime/DateTimeFormatTest.php", "content": "refine($from);\n\n expect($refinedValue)->toBe($to);\n})->with([\n ['2024-09-21T13:55:41Z', '2024-09-21 13:55:41'],\n ['2024-09-21T13:55:41.000Z', '2024-09-21 13:55:41'],\n ['2024-09-21', '2024-09-21 00:00:00'],\n ['2024-09-21, 13:55:41', '2024-09-21 13:55:41'],\n ['21 September 2024, 13:55:41', '2024-09-21 13:55:41'],\n ['21. September 2024, 13:55:41', '2024-09-21 13:55:41'],\n ['21 September 2024', '2024-09-21 00:00:00'],\n ['21. September 2024', '2024-09-21 00:00:00'],\n ['21.09.2024', '2024-09-21 00:00:00'],\n ['21.09.2024 13:55', '2024-09-21 13:55:00'],\n ['21.09.2024 13:55:41', '2024-09-21 13:55:41'],\n ['Sat, 21 September 2024 13:55:41 +0000', '2024-09-21 13:55:41'],\n ['Sat Sep 21 2024 16:55:41 GMT+0100', '2024-09-21 15:55:41'],\n]);\n\nit('reformats a format that PHP\\'s strtotime() does not know, when the origin format is provided', function () {\n $refinedValue = DateTimeRefiner::reformat('Y-m-d H:i:s', 'd. F Y \\u\\m H:i:s')\n ->refine('21. September 2024 um 13:55:41');\n\n expect($refinedValue)->toBe('2024-09-21 13:55:41');\n});\n\nit('logs a warning message (and keeps original input) when it wasn\\'t able to auto-convert a date time string', function () {\n $refiner = DateTimeRefiner::reformat('Y-m-d H:i:s');\n\n $logger = new DummyLogger();\n\n $refiner->addLogger($logger);\n\n $refinedValue = $refiner->refine('21. September 2024 um 13:55:41');\n\n expect($logger->messages)->toHaveCount(1)\n ->and($logger->messages[0]['level'])->toBe('warning')\n ->and($logger->messages[0]['message'])->toStartWith('Failed to automatically (without known format) parse')\n ->and($refinedValue)->toBe('21. September 2024 um 13:55:41');\n});\n\nit(\n 'logs a warning message (and keeps original input) when it wasn\\'t able to convert a date time string with the ' .\n 'given origin format',\n function () {\n $refiner = DateTimeRefiner::reformat('Y-m-d H:i:s', 'd. F Y um H:i:s');\n\n $logger = new DummyLogger();\n\n $refiner->addLogger($logger);\n\n $refinedValue = $refiner->refine('21. September 2024 um 13:55:41');\n\n expect($logger->messages)->toHaveCount(1)\n ->and($logger->messages[0]['level'])->toBe('warning')\n ->and($logger->messages[0]['message'])->toStartWith('Failed parsing date/time ')\n ->and($refinedValue)->toBe('21. September 2024 um 13:55:41');\n },\n);\n\nit('reformats an array of date time strings', function () {\n $refinedValue = DateTimeRefiner::reformat('Y-m-d H:i:s')->refine([\n '2024-09-21T13:55:41Z',\n '2024-09-21T13:55:41.000Z',\n '2024-09-21',\n ]);\n\n expect($refinedValue)->toBe([\n '2024-09-21 13:55:41',\n '2024-09-21 13:55:41',\n '2024-09-21 00:00:00',\n ]);\n});\n" }, { "path": "tests/Steps/Refiners/Html/RemoveFromHtmlTest.php", "content": "\n \n \n \n

    Hi!

    \n
    remove this!
    \n \n \n HTML;\n\n $refinedValue = HtmlRefiner::remove('#foo')->refine($html);\n\n expect($refinedValue)->not()->toContain('remove this!')\n ->and($refinedValue)->toContain('

    Hi!

    ');\n});\n\nit('removes a certain node from an HTML snippet by selector', function () {\n $html = <<\n

    Hi!

    \n

    remove this!

    \n \n HTML;\n\n $refinedValue = HtmlRefiner::remove('#foo')->refine($html);\n\n expect($refinedValue)->not()->toContain('remove this!')\n ->and($refinedValue)->toContain('

    Hi!

    ')\n ->and($refinedValue)->not()->toContain('');\n});\n\nit('removes multiple nodes from an HTML snippet by selector', function () {\n $html = <<\n
      \n
    • foo
      • \n
    • bar
      • \n
    • baz
      • \n
    • quz
      • \n
    \n \n HTML;\n\n $refinedValue = HtmlRefiner::remove('#list .remove')->refine($html);\n\n expect($refinedValue)->not()->toContain('bar')\n ->and($refinedValue)->not()->toContain('quz')\n ->and($refinedValue)->toContain('
  • foo
    • ')\n ->and($refinedValue)->toContain('
  • baz
    • ')\n ->and($refinedValue)->not()->toContain('');\n});\n\nit('removes multiple nodes from HTML by xpath query', function () {\n $html = <<\n
      \n
    • foo
      • \n
    • bar
      • \n
    • baz
      • \n
    • quz
      • \n
    \n \n HTML;\n\n $refinedValue = HtmlRefiner::remove(Dom::xPath('//li[contains(@class, \\'remove\\')]'))->refine($html);\n\n expect($refinedValue)->not()->toContain('bar')\n ->and($refinedValue)->not()->toContain('quz')\n ->and($refinedValue)->toContain('
  • foo
    • ')\n ->and($refinedValue)->toContain('
  • baz
    • ')\n ->and($refinedValue)->not()->toContain('');\n});\n\nit('removes node from an array of HTML snippets', function () {\n $html = [\n <<\n
  • foo
    • \n
  • bar
    • \n
  • baz
    • \n
  • quz
    • \n \n HTML,\n <<\n
  • lorem
    • \n
  • ipsum
    • \n
  • dolor
    • \n
  • sit
    • \n \n HTML,\n ];\n\n $refinedValue = HtmlRefiner::remove('.remove')->refine($html);\n\n expect($refinedValue[0])->not()->toContain('bar')\n ->and($refinedValue[0])->not()->toContain('quz')\n ->and($refinedValue[1])->not()->toContain('ipsum')\n ->and($refinedValue[1])->not()->toContain('sit');\n});\n" }, { "path": "tests/Steps/Refiners/String/AfterFirstTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner StringRefiner::afterFirst() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n})->with([\n [123],\n [12.3],\n [true],\n]);\n\nit('works with an array of strings as value', function () {\n $refinedValue = StringRefiner::afterFirst('a')\n ->addLogger(new CliLogger())\n ->refine(['foo a bar a baz', 'lorem a ipsum a dolor']);\n\n expect($refinedValue)->toBe(['bar a baz', 'ipsum a dolor']);\n});\n\nit('returns the string after first occurrence of another string', function () {\n expect(StringRefiner::afterFirst('foo')->refine('yo lo foo boo choo foo gnu'))->toBe('boo choo foo gnu');\n});\n\nit('returns the full string if the string to look for is empty', function () {\n expect(StringRefiner::afterFirst('')->refine('yo lo foo boo choo'))->toBe('yo lo foo boo choo');\n});\n\nit('returns the full string when the string to look for is not contained', function () {\n expect(StringRefiner::afterFirst('moo')->refine('yo lo foo boo choo'))->toBe('yo lo foo boo choo');\n});\n" }, { "path": "tests/Steps/Refiners/String/AfterLastTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner StringRefiner::afterLast() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n})->with([\n [123],\n [12.3],\n [true],\n]);\n\nit('works with an array of strings as value', function () {\n $refinedValue = StringRefiner::afterLast('a')\n ->addLogger(new CliLogger())\n ->refine(['foo a bar a baz', 'lorem a ipsum a dolor']);\n\n expect($refinedValue)->toBe(['z', 'dolor']);\n});\n\nit('returns the string after last occurrence of another string', function () {\n expect(StringRefiner::afterLast('foo')->refine('yo lo foo boo choo foo gnu'))->toBe('gnu');\n});\n\nit('returns an empty string if the string to look for is empty', function () {\n expect(StringRefiner::afterLast('')->refine('yo lo foo boo choo'))->toBe('');\n});\n\nit('returns the full string when the string to look for is not contained', function () {\n expect(StringRefiner::afterLast('moo')->refine('yo lo foo boo choo'))->toBe('yo lo foo boo choo');\n});\n" }, { "path": "tests/Steps/Refiners/String/BeforeFirstTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner StringRefiner::beforeFirst() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n})->with([\n [123],\n [12.3],\n [true],\n]);\n\nit('works with an array of strings as value', function () {\n $refinedValue = StringRefiner::beforeFirst('a')\n ->addLogger(new CliLogger())\n ->refine(['foo a bar a baz', 'lorem a ipsum a dolor']);\n\n expect($refinedValue)->toBe(['foo', 'lorem']);\n});\n\nit('returns the string before the first occurrence of another string', function () {\n expect(StringRefiner::beforeFirst('foo')->refine('yo lo foo boo choo foo gnu'))->toBe('yo lo');\n});\n\nit('returns an empty string if the string to look for is empty', function () {\n expect(StringRefiner::beforeFirst('')->refine('yo lo foo boo choo'))->toBe('');\n});\n\nit('returns the full string when the string to look for is not contained', function () {\n expect(StringRefiner::beforeFirst('moo')->refine('yo lo foo boo choo'))->toBe('yo lo foo boo choo');\n});\n" }, { "path": "tests/Steps/Refiners/String/BeforeLastTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner StringRefiner::beforeLast() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n})->with([\n [123],\n [12.3],\n [true],\n]);\n\nit('works with an array of strings as value', function () {\n $refinedValue = StringRefiner::beforeLast('a')\n ->addLogger(new CliLogger())\n ->refine(['foo a bar a baz', 'lorem a ipsum a dolor']);\n\n expect($refinedValue)->toBe(['foo a bar a b', 'lorem a ipsum']);\n});\n\nit('returns the string before the last occurrence of another string', function () {\n expect(StringRefiner::beforeLast('foo')->refine('yo lo foo boo choo foo gnu'))->toBe('yo lo foo boo choo');\n});\n\nit('returns the full string if the string to look for is empty', function () {\n expect(StringRefiner::beforeLast('')->refine('yo lo foo boo choo'))->toBe('yo lo foo boo choo');\n});\n\nit('returns the full string when the string to look for is not contained', function () {\n expect(StringRefiner::beforeLast('moo')->refine('yo lo foo boo choo'))->toBe('yo lo foo boo choo');\n});\n" }, { "path": "tests/Steps/Refiners/String/BetweenFirstTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner StringRefiner::betweenFirst() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n})->with([\n [123],\n [12.3],\n [true],\n]);\n\nit('works with an array of strings as value', function () {\n $refinedValue = StringRefiner::betweenFirst('foo', 'bar')\n ->addLogger(new CliLogger())\n ->refine(['one foo two bar three foo four bar five', 'six foo seven bar eight foo nine bar ten']);\n\n expect($refinedValue)->toBe(['two', 'seven']);\n});\n\nit('gets the (trimmed) string between the first occurrence of start and the next occurrence of end', function () {\n $refiner = StringRefiner::betweenFirst('foo', 'bar');\n\n $refinedValue = $refiner->refine('bla foo bli bar blu foo bar asdf foo bar');\n\n expect($refinedValue)->toBe('bli');\n});\n\ntest('if start is an empty string, start from the beginning', function () {\n $refiner = StringRefiner::betweenFirst('', 'bar');\n\n $refinedValue = $refiner->refine('bla foo bli bar blu foo bar asdf foo bar');\n\n expect($refinedValue)->toBe('bla foo bli');\n});\n\ntest('if end is an empty string, it takes the rest of the string until the end', function () {\n $refiner = StringRefiner::betweenFirst('blu', '');\n\n $refinedValue = $refiner->refine('bla foo bli bar blu foo bar asdf foo bar');\n\n expect($refinedValue)->toBe('foo bar asdf foo bar');\n});\n\nit('returns an empty string if start is not contained in the string', function () {\n $refiner = StringRefiner::betweenFirst('not contained', '');\n\n $refinedValue = $refiner->refine('bla foo bli bar blu foo bar asdf foo bar');\n\n expect($refinedValue)->toBe('');\n});\n" }, { "path": "tests/Steps/Refiners/String/BetweenLastTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner StringRefiner::betweenLast() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n})->with([\n [123],\n [12.3],\n [true],\n]);\n\nit('works with an array of strings as value', function () {\n $refinedValue = StringRefiner::betweenLast('foo', 'bar')\n ->addLogger(new CliLogger())\n ->refine(['one foo two bar three foo four bar five', 'six foo seven bar eight foo nine bar ten']);\n\n expect($refinedValue)->toBe(['four', 'nine']);\n});\n\nit('gets the (trimmed) string between the last occurrence of start and the next occurrence of end', function () {\n $refiner = StringRefiner::betweenLast('foo', 'bar');\n\n $refinedValue = $refiner->refine('bla foo bli bar blu foo ble foo blo bar blö bar blä');\n\n expect($refinedValue)->toBe('blo');\n});\n\ntest('if start is an empty string, start from the beginning', function () {\n $refiner = StringRefiner::betweenLast('', 'blu');\n\n $refinedValue = $refiner->refine('bla foo bli bar blu foo ble foo blo bar blö bar blä');\n\n expect($refinedValue)->toBe('bla foo bli bar');\n});\n\ntest('if end is an empty string, it takes the rest of the string until the end', function () {\n $refiner = StringRefiner::betweenLast('blo', '');\n\n $refinedValue = $refiner->refine('bla foo bli bar blu foo ble foo blo bar blö bar blä');\n\n expect($refinedValue)->toBe('bar blö bar blä');\n});\n\nit('returns an empty string if start is not contained in the string', function () {\n $refiner = StringRefiner::betweenFirst('not contained', '');\n\n $refinedValue = $refiner->refine('bla foo bli bar blu foo bar asdf foo bar');\n\n expect($refinedValue)->toBe('');\n});\n" }, { "path": "tests/Steps/Refiners/String/ReplaceTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner StringRefiner::replace() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n})->with([\n [123],\n [12.3],\n [true],\n]);\n\nit('works when the value is an array of strings', function () {\n $refinedValue = StringRefiner::replace('foo', 'bar')\n ->addLogger(new CliLogger())\n ->refine(['foo boo', 'who foo', 'yo lo']);\n\n expect($refinedValue)->toBe(['bar boo', 'who bar', 'yo lo']);\n});\n\nit('replaces occurrences of a string with another string', function () {\n expect(StringRefiner::replace('foo', 'bar')->refine('foo, test lorem foo yolo'))->toBe('bar, test lorem bar yolo');\n});\n\nit('replaces occurrences of an array of strings with another array of strings', function () {\n expect(StringRefiner::replace(['foo', 'bar'], ['yo', 'lo'])->refine('foo bar baz'))->toBe('yo lo baz');\n});\n\nit('replaces occurrences of an array of strings with some single string', function () {\n expect(StringRefiner::replace(['foo', 'bar'], '-')->refine('foo bar baz'))->toBe('- - baz');\n});\n" }, { "path": "tests/Steps/Refiners/Url/WithFragmentTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner UrlRefiner::withFragment() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n },\n)->with([\n [123],\n [true],\n [new stdClass()],\n]);\n\nit('replaces the query in a URL', function (mixed $value, string $expected) {\n expect(UrlRefiner::withFragment('#lorem')->refine($value))->toBe($expected);\n})->with([\n ['https://www.example.com/path#foo', 'https://www.example.com/path#lorem'],\n ['https://www.example.com/path', 'https://www.example.com/path#lorem'],\n [Url::parse('https://www.crwlr.software/some/path#abc'), 'https://www.crwlr.software/some/path#lorem'],\n [Url::parsePsr7('https://www.crwl.io/quz#'), 'https://www.crwl.io/quz#lorem'],\n]);\n\nit('resets any query', function (mixed $value, string $expected) {\n expect(UrlRefiner::withoutFragment()->refine($value))->toBe($expected);\n})->with([\n ['https://www.example.com/foo#bar', 'https://www.example.com/foo'],\n ['https://www.crwlr.software/#', 'https://www.crwlr.software/'],\n]);\n\nit('refines an array of URLs', function () {\n expect(\n UrlRefiner::withFragment('#lorem')\n ->refine([\n 'https://www.example.com/path#foo',\n 'https://www.example.com/path#bar',\n ]),\n )->toBe(['https://www.example.com/path#lorem', 'https://www.example.com/path#lorem']);\n});\n" }, { "path": "tests/Steps/Refiners/Url/WithHostTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner UrlRefiner::withHost() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n },\n)->with([\n [123],\n [true],\n [new stdClass()],\n]);\n\nit('replaces the host in a URL', function (mixed $value, string $expected) {\n expect(UrlRefiner::withHost('www.crwlr.software')->refine($value))->toBe($expected);\n})->with([\n ['https://www.example.com/foo', 'https://www.crwlr.software/foo'],\n ['https://www.crwl.io/bar', 'https://www.crwlr.software/bar'],\n [Url::parse('https://www.crwlr.software/baz'), 'https://www.crwlr.software/baz'],\n [Url::parsePsr7('https://crwl.io/quz'), 'https://www.crwlr.software/quz'],\n]);\n\nit('refines an array of URLs', function () {\n expect(\n UrlRefiner::withHost('crwl.io')\n ->refine([\n 'https://www.example.com/foo',\n 'https://www.example.com/bar',\n ]),\n )->toBe(['https://crwl.io/foo', 'https://crwl.io/bar']);\n});\n" }, { "path": "tests/Steps/Refiners/Url/WithPathTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner UrlRefiner::withPath() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n },\n)->with([\n [123],\n [true],\n [new stdClass()],\n]);\n\nit('replaces the path in a URL', function (mixed $value, string $expected) {\n expect(UrlRefiner::withPath('/some/path/123')->refine($value))->toBe($expected);\n})->with([\n ['https://www.example.com/foo', 'https://www.example.com/some/path/123'],\n ['https://localhost/yo', 'https://localhost/some/path/123'],\n [Url::parse('https://www.crwlr.software/packages'), 'https://www.crwlr.software/some/path/123'],\n [Url::parsePsr7('https://www.crwl.io/'), 'https://www.crwl.io/some/path/123'],\n]);\n\nit('refines an array of URLs', function () {\n expect(\n UrlRefiner::withPath('/hawedere')\n ->refine([\n 'https://www.example.com/foo',\n 'https://www.example.com/bar',\n ]),\n )->toBe(['https://www.example.com/hawedere', 'https://www.example.com/hawedere']);\n});\n" }, { "path": "tests/Steps/Refiners/Url/WithPortTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner UrlRefiner::withPort() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n },\n)->with([\n [123],\n [true],\n [new stdClass()],\n]);\n\nit('replaces the port in a URL', function (mixed $value, string $expected) {\n expect(UrlRefiner::withPort(1234)->refine($value))->toBe($expected);\n})->with([\n ['https://www.example.com:8000/foo', 'https://www.example.com:1234/foo'],\n ['https://localhost:8080/yo', 'https://localhost:1234/yo'],\n [Url::parse('https://www.crwlr.software:5678/bar'), 'https://www.crwlr.software:1234/bar'],\n [Url::parsePsr7('https://crwl.io/quz'), 'https://crwl.io:1234/quz'],\n]);\n\nit('refines an array of URLs', function () {\n expect(\n UrlRefiner::withPort(1234)\n ->refine([\n 'https://www.example.com/foo',\n 'https://www.example.com/bar',\n ]),\n )->toBe(['https://www.example.com:1234/foo', 'https://www.example.com:1234/bar']);\n});\n" }, { "path": "tests/Steps/Refiners/Url/WithQueryTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner UrlRefiner::withQuery() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n },\n)->with([\n [123],\n [true],\n [new stdClass()],\n]);\n\nit('replaces the query in a URL', function (mixed $value, string $expected) {\n expect(UrlRefiner::withQuery('a=b&c=d')->refine($value))->toBe($expected);\n})->with([\n ['https://www.example.com/foo?one=two', 'https://www.example.com/foo?a=b&c=d'],\n ['https://www.example.com/bar', 'https://www.example.com/bar?a=b&c=d'],\n [Url::parse('https://www.crwlr.software/?'), 'https://www.crwlr.software/?a=b&c=d'],\n [Url::parsePsr7('https://www.crwl.io/quz?a=c&b=d'), 'https://www.crwl.io/quz?a=b&c=d'],\n]);\n\nit('resets any query', function (mixed $value, string $expected) {\n expect(UrlRefiner::withoutQuery()->refine($value))->toBe($expected);\n})->with([\n ['https://www.example.com/foo?one=two', 'https://www.example.com/foo'],\n ['https://www.crwlr.software/?', 'https://www.crwlr.software/'],\n]);\n\nit('refines an array of URLs', function () {\n expect(\n UrlRefiner::withoutQuery()\n ->refine([\n 'https://www.example.com/foo?one=two',\n 'https://www.example.com/bar?three=four',\n ]),\n )->toBe(['https://www.example.com/foo', 'https://www.example.com/bar']);\n});\n" }, { "path": "tests/Steps/Refiners/Url/WithSchemeTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner UrlRefiner::withScheme() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n },\n)->with([\n [123],\n [true],\n [new stdClass()],\n]);\n\nit('replaces the scheme in a URL', function (mixed $value, string $expected) {\n expect(UrlRefiner::withScheme('https')->refine($value))->toBe($expected);\n})->with([\n ['http://www.example.com/foo', 'https://www.example.com/foo'],\n ['https://www.example.com/foo', 'https://www.example.com/foo'],\n [Url::parse('ftp://www.example.com/bar'), 'https://www.example.com/bar'],\n [Url::parsePsr7('http://www.example.com/baz'), 'https://www.example.com/baz'],\n]);\n\nit('refines an array of URLs', function () {\n expect(\n UrlRefiner::withScheme('https')\n ->refine([\n 'http://www.example.com/foo',\n 'https://www.example.com/bar',\n ]),\n )->toBe(['https://www.example.com/foo', 'https://www.example.com/bar']);\n});\n" }, { "path": "tests/Steps/Refiners/Url/WithoutPortTest.php", "content": "addLogger(new CliLogger())\n ->refine($value);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)\n ->toContain('Refiner UrlRefiner::withoutPort() can\\'t be applied to value of type ' . gettype($value))\n ->and($refinedValue)->toBe($value);\n },\n)->with([\n [123],\n [true],\n [new stdClass()],\n]);\n\nit('resets the port to null in a URL', function (mixed $value, string $expected) {\n expect(UrlRefiner::withoutPort()->refine($value))->toBe($expected);\n})->with([\n ['https://www.example.com:8000/foo', 'https://www.example.com/foo'],\n ['http://localhost:8080/yo', 'http://localhost/yo'],\n [Url::parse('https://www.crwlr.software:5678/bar'), 'https://www.crwlr.software/bar'],\n [Url::parsePsr7('https://crwl.io/quz'), 'https://crwl.io/quz'],\n]);\n\nit('refines an array of URLs', function () {\n expect(\n UrlRefiner::withoutPort()\n ->refine([\n 'https://www.example.com:8000/foo',\n 'https://www.example.com:8080/bar',\n ]),\n )->toBe(['https://www.example.com/foo', 'https://www.example.com/bar']);\n});\n" }, { "path": "tests/Steps/Sitemap/GetUrlsFromSitemapTest.php", "content": "\n \n https://www.crwlr.software/0.5\n https://www.crwlr.software/packages0.7\n https://www.crwlr.software/blog0.7\n https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-512022-09-03\n https://www.crwlr.software/blog/dealing-with-http-url-query-strings-in-php12022-06-02\n https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-412022-05-10\n https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-2-and-v0-312022-04-30\n https://www.crwlr.software/blog/release-of-crwlr-crawler-v-0-1-012022-04-18\n https://www.crwlr.software/blog/prevent-homograph-attacks-in-user-input-urls12022-01-19\n \n XML;\n\n $outputs = helper_invokeStepWithInput(Sitemap::getUrlsFromSitemap(), $xml);\n\n expect($outputs)->toHaveCount(9)\n ->and($outputs[0]->get())->toBe('https://www.crwlr.software/')\n ->and($outputs[8]->get())->toBe('https://www.crwlr.software/blog/prevent-homograph-attacks-in-user-input-urls');\n});\n\nit('gets all urls with additional data when the withData() method is used', function () {\n $xml = <<\n \n https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-512022-09-03\n https://www.crwlr.software/blog/dealing-with-http-url-query-strings-in-php12022-06-02\n https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-40.72022-05-10\n \n XML;\n\n $outputs = helper_invokeStepWithInput(Sitemap::getUrlsFromSitemap()->withData(), $xml);\n\n expect($outputs)->toHaveCount(3)\n ->and($outputs[0]->get())->toBe([\n 'url' => 'https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-5',\n 'lastmod' => '2022-09-03',\n 'priority' => '1',\n ])\n ->and($outputs[1]->get())->toBe([\n 'url' => 'https://www.crwlr.software/blog/dealing-with-http-url-query-strings-in-php',\n 'lastmod' => '2022-06-02',\n 'priority' => '1',\n ])\n ->and($outputs[2]->get())->toBe([\n 'url' => 'https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-4',\n 'lastmod' => '2022-05-10',\n 'priority' => '0.7',\n ]);\n});\n\nit('doesn\\'t fail when sitemap is empty', function () {\n $xml = <<\n \n \n XML;\n\n $outputs = helper_invokeStepWithInput(Sitemap::getUrlsFromSitemap()->withData(), $xml);\n\n expect($outputs)->toHaveCount(0);\n});\n\nit(\n 'doesn\\'t fail when the urlset tag contains attributes, that would cause the symfony DomCrawler to not find the ' .\n 'elements',\n function () {\n $xml = <<\n \n https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-5\n https://www.crwlr.software/blog/dealing-with-http-url-query-strings-in-php\n https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-4\n \n XML;\n\n $outputs = helper_invokeStepWithInput(Sitemap::getUrlsFromSitemap(), $xml);\n\n expect($outputs)->toHaveCount(3);\n },\n);\n\nit(\n 'doesn\\'t fail when the urlset tag contains attributes, that would cause the symfony DomCrawler to not find the ' .\n 'elements, when the XML content has no line breaks',\n function () {\n $xml = <<https://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-5https://www.crwlr.software/blog/dealing-with-http-url-query-strings-in-phphttps://www.crwlr.software/blog/whats-new-in-crwlr-crawler-v0-4\n XML;\n\n $outputs = helper_invokeStepWithInput(Sitemap::getUrlsFromSitemap(), $xml);\n\n expect($outputs)->toHaveCount(3);\n },\n);\n" }, { "path": "tests/Steps/StepTest.php", "content": "\n */\n protected function invoke(mixed $input): Generator\n {\n $this->logger?->info('logging works');\n\n yield 'something';\n }\n };\n\n $step->addLogger(new CliLogger());\n\n helper_traverseIterable($step->invokeStep(new Input('test')));\n\n $output = $this->getActualOutputForAssertion();\n\n expect($output)->toContain('logging works');\n});\n\ntest('The invokeStep method wraps the values returned by invoke in Output objects', function () {\n $step = helper_getValueReturningStep('returnValue');\n\n $output = helper_invokeStepWithInput($step);\n\n expect($output)->toHaveCount(1)\n ->and($output[0])->toBeInstanceOf(Output::class)\n ->and($output[0]->get())->toBe('returnValue');\n});\n\n/* ------------------------------- keep() ------------------------------- */\n\ntest('keep() can pick keys from nested (array) output using dot notation', function () {\n $step = helper_getValueReturningStep([\n 'users' => [\n ['user' => 'otsch', 'firstname' => 'Christian', 'surname' => 'Olear'],\n ['user' => 'juerx', 'firstname' => 'Jürgen', 'surname' => 'Müller'],\n ['user' => 'sandy', 'firstname' => 'Sandra', 'surname' => 'Mayr'],\n ],\n 'foo' => 'bar',\n ])\n ->keep(['nickname' => 'users.0.user', 'foo']);\n\n $output = helper_invokeStepWithInput($step);\n\n expect($output[0]->keep)->toBe(['nickname' => 'otsch', 'foo' => 'bar']);\n});\n\ntest('keep() picks keys from nested output including a RespondedRequest object', function () {\n $step = helper_getValueReturningStep([\n 'response' => new RespondedRequest(\n new Request('GET', 'https://www.example.com/something'),\n new Response(200, body: 'Hi :)'),\n ),\n 'foo' => 'bar',\n ])\n ->keep(['content' => 'response.body']);\n\n $output = helper_invokeStepWithInput($step);\n\n expect($output[0]->keep)->toBe(['content' => 'Hi :)']);\n});\n\nit('maps output keys to different keys when defined in the array passed to keep()', function () {\n $step = helper_getValueReturningStep(['user' => 'otsch', 'firstname' => 'Christian', 'surname' => 'Olear'])\n ->keep(['foo' => 'firstname', 'bar' => 'surname']);\n\n $output = helper_invokeStepWithInput($step);\n\n expect($output[0]->keep)->toBe(['foo' => 'Christian', 'bar' => 'Olear']);\n});\n\n/* ------------------------------- useInputKey() ------------------------------- */\n\nit('uses a key from array input when defined', function () {\n $step = helper_getInputReturningStep()->useInputKey('bar');\n\n $output = helper_invokeStepWithInput($step, new Input(\n ['foo' => 'fooValue', 'bar' => 'barValue', 'baz' => 'bazValue'],\n ));\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe('barValue');\n});\n\nit('logs a warning message when the input key to use does not exist in input array', function () {\n $step = helper_getInputReturningStep()->useInputKey('baz');\n\n $step->addLogger(new CliLogger());\n\n $output = helper_invokeStepWithInput($step, new Input(['foo' => 'one', 'bar' => 'two']));\n\n expect($output)->toHaveCount(0)\n ->and($this->getActualOutputForAssertion())\n ->toContain('Can\\'t get key from input, because it does not exist.');\n});\n\nit(\n 'logs a warning message when useInputKey() was called but the input value is not an array',\n function (mixed $inputValue) {\n $step = helper_getInputReturningStep()->useInputKey('baz');\n\n $step->addLogger(new CliLogger());\n\n $output = helper_invokeStepWithInput($step, new Input($inputValue));\n\n expect($output)->toHaveCount(0)\n ->and($this->getActualOutputForAssertion())\n ->toContain(\n 'Can\\'t get key from input, because input is of type ' . gettype($inputValue) . ' instead of array.',\n );\n },\n)->with([\n ['string'],\n [0],\n [new stdClass()],\n]);\n\nit('does not lose previously kept data, when it uses the useInputKey() method', function () {\n $step = helper_getValueReturningStep(['test' => 'test'])->useInputKey('foo');\n\n $outputs = helper_invokeStepWithInput($step, new Input(['foo' => 'test'], ['some' => 'thing']));\n\n expect($outputs[0]->keep)->toBe(['some' => 'thing']);\n});\n\nit('keeps the original input data when useInputKey() is used', function () {\n $step = helper_getValueReturningStep(['baz' => 'three'])\n ->keepFromInput()\n ->useInputKey('bar');\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'one', 'bar' => 'two']);\n\n expect($outputs[0]->get())->toBe(['baz' => 'three'])\n ->and($outputs[0]->keep)->toBe(['foo' => 'one', 'bar' => 'two']);\n});\n\ntest('useInputKey() can be used to get data that was kept from a previous step with keep() or keepAs()', function () {\n $step = helper_getInputReturningStep();\n\n $step->useInputKey('bar');\n\n $outputs = helper_invokeStepWithInput($step, new Input('value', keep: ['bar' => 'baz']));\n\n expect($outputs[0]->get())->toBe('baz');\n});\n\nit(\n 'also passes on kept data through further steps when they don\\'t define any further data to keep',\n function () {\n $step = helper_getValueReturningStep('returnValue');\n\n $output = helper_invokeStepWithInput($step, new Input('inputValue', ['prevProperty' => 'foobar']));\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->keep)->toBe(['prevProperty' => 'foobar']);\n },\n);\n\n/* ------------------------------- uniqueInputs() ------------------------------- */\n\nit('doesn\\'t invoke twice with duplicate inputs when uniqueInput was called', function () {\n $step = helper_getInputReturningStep();\n\n $outputs = helper_invokeStepWithInput($step, 'foo');\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($step, 'foo');\n\n expect($outputs)->toHaveCount(1);\n\n $step->uniqueInputs();\n\n $outputs = helper_invokeStepWithInput($step, 'foo');\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($step, 'foo');\n\n expect($outputs)->toHaveCount(0);\n});\n\nit(\n 'doesn\\'t invoke twice with inputs with the same value in an array key when uniqueInput was called with that key',\n function () {\n $step = helper_getInputReturningStep();\n\n $step->uniqueInputs();\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'bar', 'number' => 1]);\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'bar', 'number' => 2]);\n\n expect($outputs)->toHaveCount(1);\n\n $step->resetAfterRun();\n\n $step->uniqueInputs('foo');\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'bar', 'number' => 1]);\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($step, ['foo' => 'bar', 'number' => 2]);\n\n expect($outputs)->toHaveCount(0);\n },\n);\n\nit(\n 'doesn\\'t invoke twice with inputs with the same value in an object key when uniqueInput was called with that key',\n function () {\n $step = helper_getInputReturningStep();\n\n $step->uniqueInputs();\n\n $outputs = helper_invokeStepWithInput($step, helper_getStdClassWithData(['foo' => 'bar', 'number' => 1]));\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($step, helper_getStdClassWithData(['foo' => 'bar', 'number' => 2]));\n\n expect($outputs)->toHaveCount(1);\n\n $step->resetAfterRun();\n\n $step->uniqueInputs('foo');\n\n $outputs = helper_invokeStepWithInput($step, helper_getStdClassWithData(['foo' => 'bar', 'number' => 1]));\n\n expect($outputs)->toHaveCount(1);\n\n $outputs = helper_invokeStepWithInput($step, helper_getStdClassWithData(['foo' => 'bar', 'number' => 2]));\n\n expect($outputs)->toHaveCount(0);\n },\n);\n\n/* ------------------------------- uniqueOutputs() ------------------------------- */\n\nit('makes outputs unique when uniqueOutput was called', function () {\n $step = helper_getStepYieldingMultipleNumbers();\n\n $step->uniqueOutputs();\n\n $output = helper_invokeStepWithInput($step, new Input('anything'));\n\n expect($output)->toHaveCount(5)\n ->and($output[0]->get())->toBe('one')\n ->and($output[1]->get())->toBe('two')\n ->and($output[2]->get())->toBe('three')\n ->and($output[3]->get())->toBe('four')\n ->and($output[4]->get())->toBe('five');\n});\n\nit('makes outputs unique when providing a key name to uniqueOutput to use from array output', function () {\n $step = helper_getStepYieldingMultipleArraysWithNumber();\n\n $step->uniqueOutputs('number');\n\n $output = helper_invokeStepWithInput($step, new Input('anything'));\n\n expect($output)->toHaveCount(5);\n});\n\nit('makes outputs unique when providing a key name to uniqueOutput to use from object output', function () {\n $step = helper_getStepYieldingMultipleObjectsWithNumber();\n\n $step->uniqueOutputs('number');\n\n $output = helper_invokeStepWithInput($step, new Input('anything'));\n\n expect($output)->toHaveCount(5);\n});\n\nit('makes array outputs unique when providing no key name to uniqueOutput', function () {\n $step = helper_getStepYieldingMultipleArraysWithNumber();\n\n $step->uniqueOutputs();\n\n $output = helper_invokeStepWithInput($step, new Input(false));\n\n expect($output)->toHaveCount(5);\n\n $output = helper_invokeStepWithInput($step, new Input(true));\n\n expect($output)->toHaveCount(8);\n});\n\nit('makes object outputs unique when providing no key name to uniqueOutput', function () {\n $step = helper_getStepYieldingMultipleArraysWithNumber();\n\n $step->uniqueOutputs();\n\n $output = helper_invokeStepWithInput($step, new Input(false));\n\n expect($output)->toHaveCount(5);\n\n $output = helper_invokeStepWithInput($step, new Input(true));\n\n expect($output)->toHaveCount(8);\n});\n\n/* ----------------------------- oneOutputPerInput() ----------------------------- */\n\ntest(\n 'when a step yields multiple outputs per input and the oneOutputPerInput() method was called, the step yields it ' .\n 'as a single output with an array of all the single output values',\n function () {\n $step = helper_getStepYieldingInputArrayAsSeparateOutputs();\n\n $step->oneOutputPerInput();\n\n $outputs = helper_invokeStepWithInput($step, ['foo', 'bar', 'baz']);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['foo', 'bar', 'baz']);\n },\n);\n\ntest('when using oneOutputPerInput(), the combined output counts as one output for the max outputs limit', function () {\n $step = helper_getStepYieldingInputArrayAsSeparateOutputs();\n\n $step->oneOutputPerInput()->maxOutputs(2);\n\n $outputs = helper_invokeStepWithInput($step, ['foo', 'bar', 'baz']);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['foo', 'bar', 'baz']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo', 'bar', 'baz']);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['foo', 'bar', 'baz']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo', 'bar', 'baz']);\n\n expect($outputs)->toHaveCount(0);\n});\n\ntest('when using oneOutputPerInput(), refiners are applied to the single elements of the combined output', function () {\n $step = helper_getStepYieldingInputArrayAsSeparateOutputs();\n\n $step->oneOutputPerInput()->refineOutput('title', fn(mixed $outputValue) => $outputValue . '-hey');\n\n $outputs = helper_invokeStepWithInput($step, [\n ['title' => 'foo'],\n ['title' => 'bar'],\n ['title' => 'baz'],\n ]);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe([\n ['title' => 'foo-hey'],\n ['title' => 'bar-hey'],\n ['title' => 'baz-hey'],\n ]);\n});\n\ntest('when using oneOutputPerInput(), filters are applied to the single elements of the combined output', function () {\n $step = helper_getStepYieldingInputArrayAsSeparateOutputs();\n\n $step->where('id', Filter::greaterThan(109))->oneOutputPerInput();\n\n $outputs = helper_invokeStepWithInput($step, [\n ['title' => 'foo', 'id' => 109],\n ['title' => 'bar', 'id' => 110],\n ['title' => 'baz', 'id' => 111],\n ]);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe([\n ['title' => 'bar', 'id' => 110],\n ['title' => 'baz', 'id' => 111],\n ]);\n});\n\ntest(\n 'when using oneOutputPerInput() in combination with outputKey(), the whole combined output is returned in an ' .\n 'array with the defined key',\n function () {\n $step = helper_getStepYieldingInputArrayAsSeparateOutputs();\n\n $step->outputKey('test')->oneOutputPerInput();\n\n $outputs = helper_invokeStepWithInput($step, ['foo', 'bar', 'baz']);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['test' => ['foo', 'bar', 'baz']]);\n },\n);\n\ntest(\n 'when using oneOutputPerInput() in combination with uniqueOutputs(), the whole combined output is compared',\n function () {\n $step = helper_getStepYieldingInputArrayAsSeparateOutputs();\n\n $step->oneOutputPerInput()->uniqueOutputs();\n\n $outputs = helper_invokeStepWithInput($step, ['foo', 'bar', 'baz']);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['foo', 'bar', 'baz']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo', 'bar', 'quz']);\n\n expect($outputs)->toHaveCount(1)\n ->and($outputs[0]->get())->toBe(['foo', 'bar', 'quz']);\n\n $outputs = helper_invokeStepWithInput($step, ['foo', 'bar', 'baz']);\n\n expect($outputs)->toHaveCount(0);\n },\n);\n\n/* -------------------------- validateAndSanitizeInput() -------------------------- */\n\nit('calls the validateAndSanitizeInput method', function () {\n $step = new class extends Step {\n protected function validateAndSanitizeInput(mixed $input): string\n {\n return $input . ' validated and sanitized';\n }\n\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n\n $output = helper_invokeStepWithInput($step, 'inputValue');\n\n expect($output[0]->get())->toBe('inputValue validated and sanitized');\n});\n\ntest(\n 'when calling validateAndSanitizeStringOrStringable() and the input is array with a single element it tries to ' .\n 'use that element as input value',\n function () {\n $step = new class extends Step {\n protected function validateAndSanitizeInput(mixed $input): string\n {\n return $this->validateAndSanitizeStringOrStringable($input);\n }\n\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n\n $output = helper_invokeStepWithInput($step, ['inputValue']);\n\n expect($output[0]->get())->toBe('inputValue');\n },\n);\n\ntest(\n 'when calling validateAndSanitizeStringOrStringable() and the input is array with multiple elements it logs ' .\n 'an error message',\n function () {\n $logger = new DummyLogger();\n\n $step = new class extends Step {\n protected function validateAndSanitizeInput(mixed $input): string\n {\n return $this->validateAndSanitizeStringOrStringable($input);\n }\n\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n\n $step->addLogger($logger);\n\n helper_invokeStepWithInput($step, ['inputValue', 'foo' => 'bar']);\n\n expect($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toStartWith(\n 'A step was called with input that it can not work with:',\n )\n ->and($logger->messages[0]['message'])->toEndWith('. The invalid input is of type array.');\n },\n);\n\ntest(\n 'when throwing an InvalidArgumentException from the validateAndSanitizeInput() it is caught and logged as an error',\n function () {\n $logger = new DummyLogger();\n\n $step = new class extends Step {\n protected function validateAndSanitizeInput(mixed $input): string\n {\n throw new InvalidArgumentException('hey :)');\n }\n\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n\n $step->addLogger($logger);\n\n $outputs = helper_invokeStepWithInput($step, 'anything');\n\n expect($outputs)->toBeEmpty()\n ->and($logger->messages)->not->toBeEmpty()\n ->and($logger->messages[0]['message'])->toBe(\n 'A step was called with input that it can not work with: hey :)',\n );\n },\n);\n\ntest(\n 'when throwing an Exception that is not an InvalidArgumentException, from the validateAndSanitizeInput() it is ' .\n 'not caught',\n function () {\n $logger = new DummyLogger();\n\n $step = new class extends Step {\n protected function validateAndSanitizeInput(mixed $input): string\n {\n throw new Exception('hey :)');\n }\n\n protected function invoke(mixed $input): Generator\n {\n yield $input;\n }\n };\n\n $step->addLogger($logger);\n\n helper_invokeStepWithInput($step, 'anything');\n },\n)->throws(Exception::class);\n\nit('is possible that a step does not produce any output at all', function () {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n if ($input === 'foo') {\n yield 'bar';\n }\n }\n };\n\n $output = helper_invokeStepWithInput($step, 'lol');\n\n expect($output)->toHaveCount(0);\n\n $output = helper_invokeStepWithInput($step, 'foo');\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe('bar');\n});\n\n/* --------------------------- updateInputUsingOutput() --------------------------- */\n\ntest('You can add and call an updateInputUsingOutput callback', function () {\n $step = helper_getValueReturningStep('something');\n\n $step->updateInputUsingOutput(function (mixed $input, mixed $output) {\n return $input . ' ' . $output;\n });\n\n $updatedInput = $step->callUpdateInputUsingOutput(new Input('Boo'), new Output('Yah!'));\n\n expect($updatedInput)->toBeInstanceOf(Input::class)\n ->and($updatedInput->get())->toBe('Boo Yah!');\n});\n\nit('does not lose previously kept data, when updateInputUsingOutput() is called', function () {\n $step = helper_getValueReturningStep('something');\n\n $step->updateInputUsingOutput(function (mixed $input, mixed $output) {\n return $input . ' ' . $output;\n });\n\n $updatedInput = $step->callUpdateInputUsingOutput(\n new Input('Some', ['foo' => 'bar']),\n new Output('thing'),\n );\n\n expect($updatedInput->keep)->toBe(['foo' => 'bar']);\n});\n\n/* -------------------------------- maxOutputs() -------------------------------- */\n\nit('does not yield more outputs than defined via maxOutputs() method', function () {\n $step = helper_getValueReturningStep('yolo')->maxOutputs(3);\n\n for ($i = 1; $i <= 5; $i++) {\n $outputs = helper_invokeStepWithInput($step, new Input('asdf'));\n\n if ($i <= 3) {\n expect($outputs)->toHaveCount(1);\n } else {\n expect($outputs)->toHaveCount(0);\n }\n }\n});\n\nit(\n 'does not yield more outputs than defined via maxOutputs() when step yields multiple outputs per input and the ' .\n 'limit is reached in the middle of the outputs resulting from one input',\n function () {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'one';\n\n yield 'two';\n\n yield 'three';\n }\n };\n\n $step->maxOutputs(7);\n\n $outputs = helper_invokeStepWithInput($step, new Input('a'));\n\n expect($outputs)->toHaveCount(3);\n\n $outputs = helper_invokeStepWithInput($step, new Input('b'));\n\n expect($outputs)->toHaveCount(3);\n\n $outputs = helper_invokeStepWithInput($step, new Input('c'));\n\n expect($outputs)->toHaveCount(1);\n },\n);\n\ntest('When a step has max outputs defined, it won\\'t call the invoke method after the limit was reached', function () {\n $step = new class extends Step {\n public int $_invokeCallCount = 0;\n\n protected function invoke(mixed $input): Generator\n {\n $this->_invokeCallCount += 1;\n\n yield 'something';\n }\n };\n\n $step->maxOutputs(2);\n\n helper_invokeStepWithInput($step, new Input('one'));\n\n helper_invokeStepWithInput($step, new Input('two'));\n\n helper_invokeStepWithInput($step, new Input('three'));\n\n helper_invokeStepWithInput($step, new Input('four'));\n\n expect($step->_invokeCallCount)->toBe(2);\n});\n\nit('resets outputs count for maxOutputs rule when resetAfterRun() is called', function () {\n $step = helper_getValueReturningStep('gogogo')->maxOutputs(2);\n\n helper_invokeStepWithInput($step, new Input('one'));\n\n helper_invokeStepWithInput($step, new Input('two'));\n\n $step->resetAfterRun();\n\n expect(helper_invokeStepWithInput($step, new Input('three')))->toHaveCount(1);\n});\n\n/* -------------------------------- outputKey() -------------------------------- */\n\nit('converts non array output to array with a certain key using the outputKey() method', function () {\n $step = helper_getValueReturningStep('bar')->outputKey('foo');\n\n $outputs = helper_invokeStepWithInput($step);\n\n expect($outputs[0]->get())->toBe(['foo' => 'bar']);\n});\n\ntest('keeping a scalar output value with keep() also works when outputKey() was used', function () {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'hey';\n }\n\n public function outputType(): StepOutputType\n {\n return StepOutputType::Scalar;\n }\n };\n\n $step\n ->outputKey('greeting')\n ->keep();\n\n $step->validateBeforeRun(Http::get());\n\n $outputs = helper_invokeStepWithInput($step, 'guten tag');\n\n expect($outputs[0]->get())->toBe(['greeting' => 'hey']);\n});\n\n/* -------------------------------- refineOutput() -------------------------------- */\n\nit('applies a Closure refiner to the steps output', function () {\n $step = helper_getValueReturningStep('output');\n\n $step->refineOutput(function (mixed $outputValue) {\n return $outputValue . ' refined';\n });\n\n $outputs = helper_invokeStepWithInput($step);\n\n expect($outputs[0]->get())->toBe('output refined');\n});\n\nit('applies an instance of the RefinerInterface to the steps output', function () {\n $step = helper_getInputReturningStep();\n\n $step->refineOutput(StringRefiner::betweenFirst('foo', 'baz'));\n\n $outputs = helper_invokeStepWithInput($step, 'foo bar baz');\n\n expect($outputs[0]->get())->toBe('bar');\n});\n\nit('applies multiple refiners to the steps output in the order they\\'re added', function () {\n $step = helper_getInputReturningStep();\n\n $step\n ->refineOutput(StringRefiner::betweenFirst('foo', 'baz'))\n ->refineOutput(function (mixed $outputValue) {\n return $outputValue . ' refined';\n })\n ->refineOutput(function (mixed $outputValue) {\n return $outputValue . ', and refined further';\n });\n\n $outputs = helper_invokeStepWithInput($step, 'foo bar baz');\n\n expect($outputs[0]->get())->toBe('bar refined, and refined further');\n});\n\nit('applies refiners to certain keys from array output when the key is provided', function () {\n $step = helper_getInputReturningStep();\n\n $step\n ->refineOutput('foo', StringRefiner::betweenFirst('lorem', 'dolor'))\n ->refineOutput('baz', function (mixed $outputValue) {\n return 'refined ' . $outputValue;\n });\n\n $outputs = helper_invokeStepWithInput(\n $step,\n ['foo' => 'lorem ipsum dolor', 'bar' => 'bla', 'baz' => 'quz'],\n );\n\n expect($outputs[0]->get())->toBe([\n 'foo' => 'ipsum',\n 'bar' => 'bla',\n 'baz' => 'refined quz',\n ]);\n});\n\ntest('you can apply multiple refiners to the same output array key', function () {\n $step = helper_getInputReturningStep();\n\n $step\n ->refineOutput('foo', StringRefiner::betweenFirst('lorem', 'dolor'))\n ->refineOutput('foo', function (mixed $outputValue) {\n return $outputValue . ' yolo';\n });\n\n $outputs = helper_invokeStepWithInput(\n $step,\n ['foo' => 'lorem ipsum dolor', 'bar' => 'bla'],\n );\n\n expect($outputs[0]->get())->toBe([\n 'foo' => 'ipsum yolo',\n 'bar' => 'bla',\n ]);\n});\n\nit(\n 'uses the original input value when applying a refiner, not only the value of an input array key chosen via ' .\n 'useInputKey()',\n function () {\n $step = helper_getInputReturningStep();\n\n $step\n ->useInputKey('bar')\n ->refineOutput(function (mixed $outputValue, mixed $originalInputValue) {\n return $originalInputValue;\n });\n\n $outputs = helper_invokeStepWithInput(\n $step,\n ['foo' => 'one', 'bar' => 'two'],\n );\n\n expect($outputs[0]->get())->toBe(['foo' => 'one', 'bar' => 'two']);\n },\n);\n\n/* ------------------------------- outputKeyAliases() ------------------------------- */\n\ntest('you can define aliases for output keys and they are considered when using keep()', function () {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield [\n 'foo' => 'one',\n 'bar' => 'two',\n 'baz' => 'three',\n ];\n }\n\n protected function outputKeyAliases(): array\n {\n return [\n 'woo' => 'foo',\n 'war' => 'bar',\n 'waz' => 'baz',\n ];\n }\n };\n\n $step->keep(['woo', 'far' => 'war', 'waz']);\n\n $outputs = helper_invokeStepWithInput($step);\n\n expect($outputs[0]->keep)->toBe([\n 'woo' => 'one',\n 'far' => 'two',\n 'waz' => 'three',\n ]);\n});\n\ntest('you can filter outputs using an output key alias', function () {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield [\n 'foo' => 'one',\n 'bar' => 'two',\n ];\n }\n\n protected function outputKeyAliases(): array\n {\n return [\n 'baz' => 'bar',\n ];\n }\n };\n\n $step->where('baz', Filter::equal('two'));\n\n $outputs = helper_invokeStepWithInput($step);\n\n expect($outputs[0])->toBeInstanceOf(Output::class);\n});\n\nit('can filter by a key that only exists in the serialized version of an output object', function () {\n $step = new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield new class {\n public string $foo = 'one';\n\n public string $bar = 'two';\n\n /**\n * @return string[]\n */\n public function __serialize(): array\n {\n return [\n 'foo' => $this->foo,\n 'bar' => $this->bar,\n 'baz' => $this->bar,\n ];\n }\n };\n }\n\n protected function outputKeyAliases(): array\n {\n return [\n 'quz' => 'baz',\n ];\n }\n };\n\n $step->where('quz', Filter::equal('two'));\n\n $outputs = helper_invokeStepWithInput($step);\n\n expect($outputs[0])->toBeInstanceOf(Output::class);\n});\n" }, { "path": "tests/Steps/XmlTest.php", "content": "extract('title'),\n helper_getStepFilesContent('Xml/bookstore.xml'),\n );\n\n expect($output)->toHaveCount(4)\n ->and($output[0]->get())->toBe('Everyday Italian')\n ->and($output[3]->get())->toBe('Learning XML');\n});\n\nit('extracts data from an XML document with XPath queries per default', function () {\n $output = helper_invokeStepWithInput(\n Xml::each('bookstore book')->extract([\n 'title' => 'title',\n 'author' => 'author',\n 'year' => 'year',\n ]),\n helper_getStepFilesContent('Xml/bookstore.xml'),\n );\n\n expect($output)->toHaveCount(4)\n ->and($output[0]->get())->toBe(\n ['title' => 'Everyday Italian', 'author' => 'Giada De Laurentiis', 'year' => '2005'],\n )\n ->and($output[1]->get())->toBe(['title' => 'Harry Potter', 'author' => 'J K. Rowling', 'year' => '2005'])\n ->and($output[2]->get())->toBe(\n [\n 'title' => 'XQuery Kick Start',\n 'author' => ['James McGovern', 'Per Bothner', 'Kurt Cagle', 'James Linn', 'Vaidyanathan Nagarajan'],\n 'year' => '2003',\n ],\n )\n ->and($output[3]->get())->toBe(['title' => 'Learning XML', 'author' => 'Erik T. Ray', 'year' => '2003']);\n});\n\nit('can also extract data using XPath queries', function () {\n $output = helper_invokeStepWithInput(\n Xml::each(Dom::xPath('//bookstore/book'))->extract([\n 'title' => Dom::xPath('//title'),\n 'author' => Dom::xPath('//author'),\n 'year' => Dom::xPath('//year'),\n ]),\n helper_getStepFilesContent('Xml/bookstore.xml'),\n );\n\n expect($output)->toHaveCount(4)\n ->and($output[2]->get())->toBe(\n [\n 'title' => 'XQuery Kick Start',\n 'author' => ['James McGovern', 'Per Bothner', 'Kurt Cagle', 'James Linn', 'Vaidyanathan Nagarajan'],\n 'year' => '2003',\n ],\n );\n});\n\nit('returns only one (compound) output when the root method is used', function () {\n $output = helper_invokeStepWithInput(\n Xml::root()->extract(['title' => 'title', 'author' => 'author', 'year' => 'year']),\n helper_getStepFilesContent('Xml/bookstore.xml'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get()['title'])->toBe(['Everyday Italian', 'Harry Potter', 'XQuery Kick Start', 'Learning XML']);\n});\n\nit('extracts the data of the first matching element when the first method is used', function () {\n $output = helper_invokeStepWithInput(\n Xml::first('bookstore book')->extract(['title' => 'title', 'author' => 'author', 'year' => 'year']),\n helper_getStepFilesContent('Xml/bookstore.xml'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(\n ['title' => 'Everyday Italian', 'author' => 'Giada De Laurentiis', 'year' => '2005'],\n );\n});\n\nit('extracts the data of the last matching element when the last method is used', function () {\n $output = helper_invokeStepWithInput(\n Xml::last('bookstore book')->extract(['title' => 'title', 'author' => 'author', 'year' => 'year']),\n helper_getStepFilesContent('Xml/bookstore.xml'),\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['title' => 'Learning XML', 'author' => 'Erik T. Ray', 'year' => '2003']);\n});\n\ntest(\n 'you can extract data in a second level to the output array using another Xml step as an element in the mapping ' .\n 'array',\n function () {\n $response = new RespondedRequest(\n new Request('GET', 'https://www.example.com/events.xml'),\n new Response(body: helper_getStepFilesContent('Xml/events.xml')),\n );\n\n $outputs = helper_invokeStepWithInput(\n Xml::each('events event')->extract([\n 'title' => 'name',\n 'location' => 'location',\n 'date' => 'date',\n 'talks' => Xml::each('talks talk')->extract([\n 'title' => 'title',\n 'speaker' => 'speaker',\n ]),\n ]),\n $response,\n );\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe([\n 'title' => 'Some Meetup',\n 'location' => 'Somewhere',\n 'date' => '2023-01-14 20:00',\n 'talks' => [\n [\n 'title' => 'Sophisticated talk title',\n 'speaker' => 'Super Mario',\n ],\n [\n 'title' => 'Fun talk',\n 'speaker' => 'Princess Peach',\n ],\n ],\n ])\n ->and($outputs[1]->get())->toBe([\n 'title' => 'Another Meetup',\n 'location' => 'Somewhere else',\n 'date' => '2023-01-21 19:00',\n 'talks' => [\n [\n 'title' => 'Join the dark side',\n 'speaker' => 'Wario',\n ],\n [\n 'title' => 'Let\\'s go',\n 'speaker' => 'Yoshi',\n ],\n ],\n ]);\n\n },\n);\n\ntest(\n 'When a child step is nested in the extraction and does not use each(), the extracted value is an array with ' .\n 'the keys defined in extract(), rather than an array of such arrays as it would be with each().',\n function () {\n $xml = <<\n \n \n ABCDEFGmbH\n foo\n \n Germany\n Frankfurt\n \n \n \n Saubär GmbH\n bar\n \n Austria\n Klagenfurt\n \n \n \n XML;\n\n $expectedCompany1 = [\n 'name' => 'ABCDEFGmbH',\n 'founded' => '1984',\n 'location' => ['country' => 'Germany', 'city' => 'Frankfurt'],\n ];\n\n $expectedCompany2 = [\n 'name' => 'Saubär GmbH',\n 'founded' => '2014',\n 'location' => ['country' => 'Austria', 'city' => 'Klagenfurt'],\n ];\n\n // With base root()\n $step = Xml::each(Dom::xPath('//companies/company'))->extract([\n 'name' => Dom::cssSelector('name')->text(),\n 'founded' => Dom::xPath('//founded')->attribute('year'),\n 'location' => Xml::root()->extract([\n 'country' => Dom::xPath('//location/country')->text(),\n 'city' => Dom::cssSelector('location city')->text(),\n ]),\n ]);\n\n $outputs = helper_invokeStepWithInput($step, $xml);\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe($expectedCompany1)\n ->and($outputs[1]->get())->toBe($expectedCompany2);\n\n // With base first()\n $step = Xml::each(Dom::xPath('//companies/company'))->extract([\n 'name' => Dom::cssSelector('name')->text(),\n 'founded' => Dom::xPath('//founded')->attribute('year'),\n 'location' => Xml::first(Dom::cssSelector('location'))->extract([\n 'country' => Dom::xPath('//country')->text(),\n 'city' => Dom::cssSelector('city')->text(),\n ]),\n ]);\n\n $outputs = helper_invokeStepWithInput($step, $xml);\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe($expectedCompany1)\n ->and($outputs[1]->get())->toBe($expectedCompany2);\n\n // With base last()\n $step = Xml::each(Dom::xPath('//companies/company'))->extract([\n 'name' => Dom::cssSelector('name')->text(),\n 'founded' => Dom::xPath('//founded')->attribute('year'),\n 'location' => Xml::last(Dom::cssSelector('location'))->extract([\n 'country' => Dom::xPath('//country')->text(),\n 'city' => Dom::cssSelector('city')->text(),\n ]),\n ]);\n\n $outputs = helper_invokeStepWithInput($step, $xml);\n\n expect($outputs)->toHaveCount(2)\n ->and($outputs[0]->get())->toBe($expectedCompany1)\n ->and($outputs[1]->get())->toBe($expectedCompany2);\n },\n);\n\nit('works when the response string starts with an UTF-8 byte order mark character', function () {\n $response = new RespondedRequest(\n new Request('GET', 'https://www.example.com/rss'),\n new Response(body: helper_getStepFilesContent('Xml/rss-with-bom.xml')),\n );\n\n $outputs = helper_invokeStepWithInput(\n Xml::each('channel item')->extract([\n 'url' => 'link',\n 'title' => 'title',\n ]),\n $response,\n );\n\n expect($outputs[0]->get())->toBe([\n 'url' => 'https://www.example.com/story/1234567/foo-bar-baz?ref=rss',\n 'title' => 'Some title',\n ]);\n});\n\ntest(\n 'when selecting elements with each(), you can reference the element already selected within the each() selector ' .\n 'itself, in sub selectors',\n function () {\n $xml = <<\n \n \n \n 123\n \n \n 456\n \n \n \n \n \n XML;\n\n $response = new RespondedRequest(\n new Request('GET', 'https://www.example.com/foo'),\n new Response(body: $xml),\n );\n\n $output = helper_invokeStepWithInput(\n Xml::each('data items item')->extract([\n // This is what this test is about. The element already selected in each (item) can be\n // referenced in these child selectors.\n 'id' => Dom::cssSelector('item > id'),\n 'attribute' => Dom::cssSelector('')->attribute('attr'),\n ]),\n $response,\n );\n\n expect($output)->toHaveCount(1)\n ->and($output[0]->get())->toBe(['id' => '123', 'attribute' => 'abc']);\n },\n);\n\nit('works with tags with camelCase names', function () {\n $xml = <<\n \n foo\n foo\n \n \n abc-123\n 2024-11-07T11:00:31Z\n Foo bar baz!\n https://www.example.com/item-1?utm_source=foo&utm_medium=feed-xml\n \n test\n \n \n \n \n XML;\n\n $response = new RespondedRequest(\n new Request('GET', 'https://www.example.com/xml-feed'),\n new Response(body: $xml),\n );\n\n $outputs = helper_invokeStepWithInput(\n Xml::each(Dom::cssSelector('feed items item'))->extract([\n 'title' => 'title',\n 'some-url' => 'someUrl',\n 'foo-bar-baz' => 'foo baRbaz',\n ]),\n $response,\n );\n\n expect($outputs[0]->get())->toBe([\n 'title' => 'Foo bar baz!',\n 'some-url' => 'https://www.example.com/item-1?utm_source=foo&utm_medium=feed-xml',\n 'foo-bar-baz' => 'test',\n ]);\n})->group('php84');\n" }, { "path": "tests/Steps/_Files/Csv/basic.csv", "content": "123,\"Otsch\",\"https://www.otsch.codes\"\n234,\"John Doe\",\"https://www.john.doe\"\n345,\"Jane Doe\",\"https://www.jane.doe\"\n" }, { "path": "tests/Steps/_Files/Csv/enclosure.csv", "content": "123,?Kräftige Rindsuppe?,4.5\n234,?Crispy Chicken Burger?,12\n345,?Duett von Saibling und Forelle?,21\n" }, { "path": "tests/Steps/_Files/Csv/escape.csv", "content": "123,\"test %\"escape%\" test\",test\n123,\"foo %\"escape%\" bar %\"baz%\" lorem\",test\n" }, { "path": "tests/Steps/_Files/Csv/separator.csv", "content": "123*\"CoDerOtsch\"*Christian*Olear*35\n234*\"g3n1u5\"*Albert*Einstein*143\n345*\"sWiFtY\"*Taylor*Swift*32\n" }, { "path": "tests/Steps/_Files/Csv/with-column-headlines.csv", "content": "Stunde,Montag,Dienstag,Mittwoch,Donnerstag,Freitag\n1,Mathematik,Deutsch,Englisch,Erdkunde,Politik\n2,Sport,Deutsch,Englisch,Sport,Geschichte\n3,Sport,\"Religion (ev., kath.)\",Kunst,,Kunst\n" }, { "path": "tests/Steps/_Files/Html/basic.html", "content": "\n\n\n\n

    match 1

    \n\n
    \n
    \n

    match 2

    \n
    \n
    \n

    match 3

    \n
    \n
    \n\n\n" }, { "path": "tests/Steps/_Files/Html/bookstore.html", "content": "\n\n \n Bookstore Example in HTML :)\n \n \n
    \n\n
    \n

    Everyday Italian

    \n
    Giada De Laurentiis
    \n 2005 - 30.00\n
    \n\n
    \n

    Harry Potter

    \n
    J K. Rowling
    \n 2005 - 29.99\n
    \n\n
    \n

    XQuery Kick Start

    \n James McGovern,\n Per Bothner,\n Kurt Cagle,\n James Linn,\n Vaidyanathan Nagarajan\n 2003 - 49.99\n
    \n\n
    \n

    Learning XML

    \n
    Erik T. Ray
    \n 2003 - 39.95\n
    \n\n
    \n \n\n" }, { "path": "tests/Steps/_Files/Html/event.html", "content": "\n\n\n Bookstore Example in HTML :)\n\n\n
    \n

    Some Meetup

    \n
    Somewhere
    \n
    2023-01-14 21:00
    \n\n
    \n
    \n

    Sophisticated talk title

    \n
    Super Mario
    \n Slides\n
    \n
    \n

    Simple beginner talk

    \n
    Luigi
    \n Slides\n
    \n
    \n

    Fun talk

    \n
    Princess Peach
    \n Slides\n
    \n
    \n
    \n\n\n" }, { "path": "tests/Steps/_Files/Xml/bookstore.xml", "content": "\n\n\n \n Everyday Italian\n Giada De Laurentiis\n 2005\n 30.00\n \n\n \n Harry Potter\n J K. Rowling\n 2005\n 29.99\n \n\n \n XQuery Kick Start\n James McGovern\n Per Bothner\n Kurt Cagle\n James Linn\n Vaidyanathan Nagarajan\n 2003\n 49.99\n \n\n \n Learning XML\n Erik T. Ray\n 2003\n 39.95\n \n\n\n" }, { "path": "tests/Steps/_Files/Xml/events.xml", "content": "\n\n\n \n Some Meetup\n Somewhere\n 2023-01-14 20:00\n \n \n Sophisticated talk title\n Super Mario\n \n \n Fun talk\n Princess Peach\n \n \n \n\n \n Another Meetup\n Somewhere else\n 2023-01-21 19:00\n \n \n Join the dark side\n Wario\n \n \n Let's go\n Yoshi\n \n \n \n\n\n" }, { "path": "tests/Steps/_Files/Xml/rss-with-bom.xml", "content": "Foo - Barhttps://www.example.com/somethinglorem ipsum dolor sitde-dehttps://www.example.com/story/1234567/foo-bar-baz?ref=rsshttps://www.example.com/story/1234567/foo-bar-baz?ref=rssFooBarSome titlelorem ipsum dolor sit ametMon, 08 May 2023 14:08:21 ZFoto: Foo/BarFoto: Foo/Bar\n" }, { "path": "tests/Stores/JsonFileStoreTest.php", "content": " $value) {\n $result->set($key, $value);\n }\n\n return $result;\n}\n\nit('saves Results to a JSON file', function () {\n $result1 = helper_getResultWithJsonData(['user' => 'otsch', 'firstname' => 'Christian', 'surname' => 'Olear']);\n\n $store = new JsonFileStore(__DIR__ . '/_files', 'test');\n\n $store->store($result1);\n\n expect(file_get_contents($store->filePath()))->toBe('[{\"user\":\"otsch\",\"firstname\":\"Christian\",\"surname\":\"Olear\"}]');\n\n $result2 = helper_getResultWithJsonData(['user' => 'hader', 'firstname' => 'Josef', 'surname' => 'Hader']);\n\n $store->store($result2);\n\n expect(file_get_contents($store->filePath()))->toBe(\n '[{\"user\":\"otsch\",\"firstname\":\"Christian\",\"surname\":\"Olear\"},' .\n '{\"user\":\"hader\",\"firstname\":\"Josef\",\"surname\":\"Hader\"}]',\n );\n\n $result3 = helper_getResultWithJsonData(['user' => 'evamm', 'firstname' => 'Eva Maria', 'surname' => 'Maier']);\n\n $store->store($result3);\n\n expect(file_get_contents($store->filePath()))->toBe(\n '[{\"user\":\"otsch\",\"firstname\":\"Christian\",\"surname\":\"Olear\"},' .\n '{\"user\":\"hader\",\"firstname\":\"Josef\",\"surname\":\"Hader\"},' .\n '{\"user\":\"evamm\",\"firstname\":\"Eva Maria\",\"surname\":\"Maier\"}]',\n );\n});\n\nafterAll(function () {\n $dir = __DIR__ . '/_files';\n\n if (file_exists($dir)) {\n $files = scandir($dir);\n\n if (is_array($files)) {\n foreach ($files as $file) {\n if ($file === '.' || $file === '..' || !str_ends_with($file, '.json')) {\n continue;\n }\n\n @unlink($dir . '/' . $file);\n }\n }\n }\n});\n" }, { "path": "tests/Stores/SimpleCsvFileStoreTest.php", "content": " $value) {\n $result->set($key, $value);\n }\n\n return $result;\n}\n\nit('saves Results to a csv file', function () {\n $result1 = helper_getResultWithData(['user' => 'otsch', 'firstname' => 'Christian', 'surname' => 'Olear']);\n\n $store = new SimpleCsvFileStore(__DIR__ . '/_files', 'test');\n\n $store->store($result1);\n\n expect(file_get_contents($store->filePath()))->toBe(\"user,firstname,surname\\notsch,Christian,Olear\\n\");\n\n $result2 = helper_getResultWithData(['user' => 'hader', 'firstname' => 'Josef', 'surname' => 'Hader']);\n\n $store->store($result2);\n\n expect(file_get_contents($store->filePath()))->toBe(\n \"user,firstname,surname\\notsch,Christian,Olear\\nhader,Josef,Hader\\n\",\n );\n\n $result3 = helper_getResultWithData(['user' => 'evamm', 'firstname' => 'Eva Maria', 'surname' => 'Maier']);\n\n $store->store($result3);\n\n expect(file_get_contents($store->filePath()))->toBe(\n \"user,firstname,surname\\notsch,Christian,Olear\\nhader,Josef,Hader\\nevamm,\\\"Eva Maria\\\",Maier\\n\",\n );\n});\n\ntest('if the value of a result property is an array, it concatenates the values separated with a pipe', function () {\n $result1 = helper_getResultWithData(['col1' => 'foo', 'col2' => ['bar', 'baz', 'quz']]);\n\n $store = new SimpleCsvFileStore(__DIR__ . '/_files', 'test2');\n\n $store->store($result1);\n\n expect(file_get_contents($store->filePath()))->toBe(\"col1,col2\\nfoo,\\\"bar | baz | quz\\\"\\n\");\n\n $result2 = helper_getResultWithData(['col1' => 'Donald', 'col2' => ['Tick', 'Trick', 'Track']]);\n\n $store->store($result2);\n\n expect(file_get_contents($store->filePath()))->toBe(\n \"col1,col2\\nfoo,\\\"bar | baz | quz\\\"\\nDonald,\\\"Tick | Trick | Track\\\"\\n\",\n );\n});\n\nafterAll(function () {\n $dir = __DIR__ . '/_files';\n\n if (file_exists($dir)) {\n $files = scandir($dir);\n\n if (is_array($files)) {\n foreach ($files as $file) {\n if ($file === '.' || $file === '..' || !str_ends_with($file, '.csv')) {\n continue;\n }\n\n unlink($dir . '/' . $file);\n }\n }\n }\n});\n" }, { "path": "tests/Stores/_files/.gitkeep", "content": "" }, { "path": "tests/UserAgents/BotUserAgentTest.php", "content": "assertStringContainsString('SomeBot', $userAgent);\n});\n\ntest('Create UserAgent instance via static make method', function () {\n $userAgent = BotUserAgent::make('CrwlrBot');\n $this->assertStringContainsString('CrwlrBot', $userAgent);\n});\n\ntest('Create instance with info uri', function () {\n $userAgent = new BotUserAgent('SomeBot', 'https://www.example.com/somebot');\n $this->assertStringContainsString('SomeBot; +https://www.example.com/somebot', $userAgent);\n});\n\ntest('Create instance with info uri and version', function () {\n $userAgent = new BotUserAgent('SomeBot', 'https://www.example.com/somebot', '1.3');\n $this->assertStringContainsString('SomeBot/1.3; +https://www.example.com/somebot', $userAgent);\n});\n\ntest('Create instance with version but without info uri', function () {\n $userAgent = new BotUserAgent('SomeBot', version: '1.3');\n $this->assertStringContainsString('SomeBot/1.3)', $userAgent);\n});\n\ntest('User agent string starts with Mozilla/5.0', function () {\n $userAgent = new BotUserAgent('ExampleBot', 'https://www.example.com/bot', '2.0');\n expect($userAgent->__toString())->toStartWith('Mozilla/5.0');\n});\n" }, { "path": "tests/UserAgents/UserAgentTest.php", "content": "__toString())->toBe($string);\n },\n)->with([\n '',\n 'Foo',\n 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 ' .\n 'Safari/537.36',\n '%$§$!\")(=aäöüäö?ßß``2304980=)(§$/&!\"=)=',\n]);\n" }, { "path": "tests/Utils/GzipTest.php", "content": "not->toBe($string)\n ->and(strlen($compressed))->toBeLessThan(strlen($string));\n});\n\nit('decodes a string', function () {\n $encoded = Gzip::encode('Hello World!');\n\n expect($encoded)->not->toBe('Hello World!')\n ->and(Gzip::decode($encoded))->toBe('Hello World!');\n});\n\nit('does not generate a warning, when string to decode actually isn\\'t encoded', function () {\n $warnings = [];\n\n set_error_handler(function ($errno, $errstr) use (&$warnings) {\n if ($errno === E_WARNING) {\n $warnings[] = $errstr;\n }\n\n return false;\n });\n\n $decoded = Gzip::decode('Hello World!');\n\n restore_error_handler();\n\n expect($decoded)->toBe('Hello World!')\n ->and($warnings)->toBeEmpty();\n});\n" }, { "path": "tests/Utils/HttpHeadersTest.php", "content": " 'de',\n 'Accept-Encoding' => ['gzip', 'deflate', 'br'],\n ]))->toBe([\n 'Accept-Language' => ['de'],\n 'Accept-Encoding' => ['gzip', 'deflate', 'br'],\n ]);\n});\n\nit('merges two header arrays', function () {\n $headers = [\n 'Accept-Language' => ['de'],\n 'Accept-Encoding' => ['gzip', 'deflate', 'br'],\n ];\n\n $merge = [\n 'Accept' => ['text/html', 'application/xhtml+xml', 'application/xml'],\n 'Accept-Language' => ['de', 'en'],\n ];\n\n expect(HttpHeaders::merge($headers, $merge))->toBe([\n 'Accept-Language' => ['de', 'en'],\n 'Accept-Encoding' => ['gzip', 'deflate', 'br'],\n 'Accept' => ['text/html', 'application/xhtml+xml', 'application/xml'],\n ]);\n});\n\nit('adds a single value to a certain header in a headers array', function () {\n $headers = ['Accept-Language' => ['de']];\n\n expect(HttpHeaders::addTo($headers, 'Accept-Language', 'en'))->toBe(['Accept-Language' => ['de', 'en']]);\n});\n\nit('adds an array of values to a certain header in a headers array', function () {\n $headers = ['Accept-Language' => ['de']];\n\n expect(\n HttpHeaders::addTo($headers, 'Accept-Language', ['en-US', 'en']),\n )->toBe(['Accept-Language' => ['de', 'en-US', 'en']]);\n});\n\nit('adds the header when calling addTo() with a header name that the array does not contain yet', function () {\n $headers = ['Accept-Encoding' => ['gzip', 'deflate', 'br']];\n\n expect(\n HttpHeaders::addTo($headers, 'Accept-Language', ['de', 'en']),\n )->toBe([\n 'Accept-Encoding' => ['gzip', 'deflate', 'br'],\n 'Accept-Language' => ['de', 'en'],\n ]);\n});\n" }, { "path": "tests/Utils/OutputTypeHelperTest.php", "content": " 'bar', 'baz'];\n }\n };\n\n expect(OutputTypeHelper::objectToArray($object))->toBe(['foo' => 'bar', 'baz']);\n});\n\nit('converts an object with a toArray() method to an array', function () {\n $object = new class {\n /**\n * @return string[]\n */\n public function toArray(): array\n {\n return ['foo' => 'bar'];\n }\n };\n\n expect(OutputTypeHelper::objectToArray($object))->toBe(['foo' => 'bar']);\n});\n\nit('converts an object with a __serialize() method to an array', function () {\n $object = new class {\n public function __serialize(): array\n {\n return ['winnie' => 'the pooh'];\n }\n };\n\n expect(OutputTypeHelper::objectToArray($object))->toBe(['winnie' => 'the pooh']);\n});\n\nit('converts an object to an array by just casting it', function () {\n $object = new class {\n public string $foo = 'one';\n\n public string $bar = 'two';\n };\n\n expect(OutputTypeHelper::objectToArray($object))->toBe(['foo' => 'one', 'bar' => 'two']);\n});\n\nit('checks if a value is a scalar value', function (mixed $value, bool $expectedResult) {\n expect(OutputTypeHelper::isScalar($value))->toBe($expectedResult);\n})->with([\n ['foo', true],\n [123, true],\n [true, true],\n [false, true],\n [1.23, true],\n [['foo', 'bar'], true], // only associative array counts as non scalar for the output types\n [['foo' => 'bar'], false],\n [new stdClass(), false],\n]);\n\nit('checks if a value is an associative array', function (mixed $value, bool $expectedResult) {\n expect(OutputTypeHelper::isAssociativeArray($value))->toBe($expectedResult);\n})->with([\n ['foo', false],\n [['foo', 'bar'], false],\n [['foo' => 'bar'], true],\n [new stdClass(), false],\n]);\n\nit(\n 'checks if a value is an associative array or object (a.k.a. non-scalar)',\n function (mixed $value, bool $expectedResult) {\n expect(OutputTypeHelper::isAssociativeArrayOrObject($value))->toBe($expectedResult);\n },\n)->with([\n ['foo', false],\n [['foo', 'bar'], false],\n [['foo' => 'bar'], true],\n [new stdClass(), true],\n]);\n" }, { "path": "tests/Utils/RequestKeyTest.php", "content": " 'gzip, deflate, br']);\n\n expect(RequestKey::from($request))->toBe('fc2a9e78c97e68674201853cea4a3d74');\n\n $request = $request->withAddedHeader('accept-language', 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7');\n\n expect(RequestKey::from($request))->not()->toBe('fc2a9e78c97e68674201853cea4a3d74');\n});\n\nit('makes a cache key from a RespondedRequest object', function () {\n $respondedRequest = new RespondedRequest(\n new Request('GET', 'https://www.crwl.io/en/home', ['accept-encoding' => 'gzip, deflate, br']),\n new Response(),\n );\n\n expect(RequestKey::from($respondedRequest))->toBe('08bcc643c9fb21af5e4f3361243e2220');\n});\n\ntest('when creating the key it ignores cookies in the sent headers by default', function () {\n $request = new Request('GET', 'https://www.crwlr.software/packages', ['accept-encoding' => 'gzip, deflate, br']);\n\n $keyWithoutCookie = RequestKey::from($request);\n\n $request = new Request('GET', 'https://www.crwlr.software/packages', [\n 'accept-encoding' => 'gzip, deflate, br',\n 'Cookie' => 'cookieName=v4lu3',\n ]);\n\n expect(RequestKey::from($request))->toBe($keyWithoutCookie);\n});\n\nit('also ignores other headers when provided in second parameter', function () {\n $request = new Request('GET', 'https://www.example.com', ['accept-encoding' => 'gzip, deflate, br']);\n\n $keyWithAcceptEncodingHeader = RequestKey::from($request);\n\n $keyWithoutAcceptEncodingHeader = RequestKey::from($request, ['accept-encoding']);\n\n expect($keyWithAcceptEncodingHeader)->not()->toBe($keyWithoutAcceptEncodingHeader);\n\n $request = new Request('GET', 'https://www.example.com', ['Accept-Encoding' => 'gzip']);\n\n $anotherKeyWithoutAcceptEncodingHeader = RequestKey::from($request, ['accept-encoding']);\n\n expect($keyWithoutAcceptEncodingHeader)->toBe($anotherKeyWithoutAcceptEncodingHeader);\n});\n" }, { "path": "tests/Utils/TemplateStringTest.php", "content": " 'foo',\n 'asdf' => 'asdf',\n 'var' => 'yolo',\n 'asdf\\'asdf' => 'replace',\n 'qu\"z' => 'double',\n ]);\n\n expect($replaced)->toBe(\n << 'bonjour', 'two' => 'ciao'],\n ),\n )->toBe('hi bonjour/ciao bye');\n});\n" }, { "path": "tests/_Integration/GroupTest.php", "content": "input('http://localhost:8000/blog-post-with-json-ld');\n\n $crawler\n ->addStep(Http::get())\n ->addStep(\n Crawler::group()\n ->addStep(\n Html::first('#content article.blog-post')\n ->extract(['title' => 'h1', 'date' => '.date']),\n )\n ->addStep(\n Html::schemaOrg()\n ->onlyType('BlogPosting')\n ->extract([\n 'author' => 'author.name',\n 'keywords',\n ]),\n )\n ->keep(),\n );\n\n $result = helper_generatorToArray($crawler->run());\n\n expect($result[0]->toArray())->toBe([\n 'title' => 'Prevent Homograph Attacks using the crwlr/url Package',\n 'date' => '2022-01-19',\n 'author' => 'Christian Olear',\n 'keywords' => 'homograph, attack, security, idn, internationalized domain names, prevention, url, uri',\n ]);\n },\n);\n" }, { "path": "tests/_Integration/Http/CharsetTest.php", "content": "input('http://localhost:8000/non-utf-8-charset')\n ->addStep(Http::get())\n ->addStep(Html::root()->extract(['foo' => '.element']));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->toArray())->toBe(['foo' => '0 l/m²']);\n});\n" }, { "path": "tests/_Integration/Http/CrawlingTest.php", "content": "robotsTxtHandler = new class ($this, $this->logger) extends RobotsTxtHandler {\n public function isAllowed(UriInterface|Url|string $url): bool\n {\n if (is_string($url)) {\n $url = Url::parse($url);\n } elseif ($url instanceof UriInterface) {\n $url = Url::parse($url);\n }\n\n if ($url->path() === '/not-allowed') {\n return false;\n }\n\n return parent::isAllowed($url);\n }\n };\n }\n\n public function load(mixed $subject): ?RespondedRequest\n {\n $request = $this->validateSubjectType($subject);\n\n $this->loadedUrls[] = $request->getUri()->__toString();\n\n return parent::load($subject);\n }\n}\n\n/**\n * To check if the Crawler stays on the same host or same domain when crawling, the PSR-18 HTTP ClientInterface\n * of this Crawler's Loader, replaces the host in the request URI just before sending the Request. The Loader thinks\n * it actually loaded the page from the incoming URI and the returned RespondedRequest object also has that original URI\n * as effectiveUri (except if the requested page redirects).\n */\n\nclass Crawler extends HttpCrawler\n{\n public function loader(UserAgentInterface $userAgent, LoggerInterface $logger): TestLoader\n {\n $client = new class implements ClientInterface {\n private Client $guzzleClient;\n\n public function __construct()\n {\n $this->guzzleClient = new Client();\n }\n\n public function sendRequest(RequestInterface $request): ResponseInterface\n {\n $request = $request->withUri($request->getUri()->withHost('localhost')->withPort(8000));\n\n return $this->guzzleClient->sendRequest($request);\n }\n };\n\n $loader = new TestLoader($userAgent, $client, $logger);\n\n // To not slow down tests unnecessarily\n $loader->throttle()\n ->waitBetween(new MultipleOf(0.0001), new MultipleOf(0.0002))\n ->waitAtLeast(Microseconds::fromSeconds(0.0001));\n\n return $loader;\n }\n\n protected function userAgent(): UserAgentInterface\n {\n return new UserAgent('SomeUserAgent');\n }\n\n /**\n * This method is here for the return type, so phpstan doesn't complain.\n */\n public function getLoader(): TestLoader\n {\n return parent::getLoader(); // @phpstan-ignore-line\n }\n}\n\n/** @var TestCase $this */\n\nit('stays on the same host by default', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(Http::crawl());\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->not()->toContain('http://foo.example.com/crawling/main-on-subdomain');\n});\n\nit('stays on the same domain when method sameDomain() is called', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(Http::crawl()->sameDomain());\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toContain('http://foo.example.com/crawling/main-on-subdomain')\n ->and($crawler->getLoader()->loadedUrls)->not()->toContain('https://www.crwlr.software/packages/crawler');\n});\n\nit('stays on the same host when method sameHost() is called', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(\n Http::crawl()\n ->sameDomain()\n ->sameHost(),\n );\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->not()->toContain('http://foo.example.com/crawling/main-on-subdomain');\n});\n\nit('crawls every page of a website that is linked somewhere', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(Http::crawl());\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(6)\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/main')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub1')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub1/sub1')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub2')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub2/sub1')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub2/sub1/sub1');\n});\n\nit('crawls only to a certain depth when the crawl depth is defined', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(Http::crawl()->depth(1));\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(3);\n\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(Http::crawl()->depth(2));\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(5);\n});\n\nit('extracts URLs from a sitemap if you call method inputIsSitemap()', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap.xml')\n ->addStep(Http::crawl()->inputIsSitemap());\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(7);\n});\n\nit('fails to extract URLs if you provide a sitemap as input and don\\'t call inputIsSitemap()', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap.xml')\n ->addStep(Http::crawl());\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(1);\n});\n\nit(\n 'extracts URLs from a sitemap where the tag contains attributes that cause symfony DomCrawler to fail',\n function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap2.xml')\n ->addStep(Http::crawl()->inputIsSitemap());\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(7);\n },\n);\n\nit('loads only pages where the path starts with a certain string when method pathStartsWith() is called', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap.xml')\n ->addStep(\n Http::crawl()\n ->inputIsSitemap()\n ->pathStartsWith('/crawling/sub1'),\n );\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(3)\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sitemap.xml')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub1')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub1/sub1');\n});\n\nit('loads only URLs where the path matches a regex when method pathMatches() is used', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap.xml')\n ->addStep(\n Http::crawl()\n ->inputIsSitemap()\n ->pathMatches('/^\\/crawling\\/sub[12]$/'),\n );\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(3);\n});\n\nit('loads only URLs where the Closure passed to method customFilter() returns true', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap.xml')\n ->addStep(\n Http::crawl()\n ->inputIsSitemap()\n ->customFilter(function (Url $url) {\n return in_array($url->path(), [\n '/crawling/main',\n '/crawling/sub1/sub1',\n '/crawling/sub2/sub1/sub1',\n ], true);\n }),\n );\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(4)\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/main')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub1/sub1')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub2/sub1/sub1');\n});\n\nit(\n 'receives the link element where the URL was found, as second param in the Closure passed to method ' .\n 'customFilter() when it was found in an HTML document',\n function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(\n Http::crawl()\n ->customFilter(function (Url $url, ?HtmlElement $linkElement) {\n return $linkElement && str_contains($linkElement->text(), 'Subpage 2');\n }),\n );\n\n $crawler->runAndTraverse();\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(4)\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/main')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub2')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub2/sub1')\n ->and($crawler->getLoader()->loadedUrls)->toContain('http://www.example.com/crawling/sub2/sub1/sub1');\n },\n);\n\nit(\n 'loads all pages, but yields only responses where the URL path starts with a certain string, when methods ' .\n 'pathStartsWith() and loadAllButYieldOnlyMatching() are called',\n function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap.xml')\n ->addStep(\n Http::crawl()\n ->inputIsSitemap()\n ->pathStartsWith('/crawling/sub2')\n ->loadAllButYieldOnlyMatching(),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(7)\n ->and($results)->toHaveCount(3);\n },\n);\n\nit(\n 'loads all URLs, but yields only responses where the URL path matches a regex, when methods pathMatches() and ' .\n 'loadAllButYieldOnlyMatching() are called',\n function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap.xml')\n ->addStep(\n Http::crawl()\n ->inputIsSitemap()\n ->pathMatches('/^\\/crawling\\/sub[12]$/')\n ->loadAllButYieldOnlyMatching(),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(7)\n ->and($results)->toHaveCount(2);\n },\n);\n\nit(\n 'loads all URLs but yields only responses where the Closure passed to method customFilter() returns true, when ' .\n 'methods customFilter() and loadAllButYieldOnlyMatching() are called',\n function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/sitemap.xml')\n ->addStep(\n Http::crawl()\n ->inputIsSitemap()\n ->customFilter(function (Url $url) {\n return in_array($url->path(), [\n '/crawling/main',\n '/crawling/sub1/sub1',\n '/crawling/sub2/sub1/sub1',\n ], true);\n })\n ->loadAllButYieldOnlyMatching(),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($crawler->getLoader()->loadedUrls)->toHaveCount(7)\n ->and($results)->toHaveCount(3);\n },\n);\n\nit(\n 'keeps the fragment parts in URLs and treats the same URL with a different fragment part as separate URLs when ' .\n 'keepUrlFragment() was called',\n function () {\n // Explanation: in almost all cases URLs with a fragment part at the end (#something) will respond with the\n // same content. So, to avoid loading the same page multiple times, the step throws away the fragment part of\n // discovered URLs by default.\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(Http::crawl()->keepUrlFragment()->keep(['url']));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(8);\n\n $urls = [];\n\n foreach ($results as $result) {\n $urls[] = $result->get('url');\n }\n\n expect($urls)->toContain('http://www.example.com/crawling/sub2')\n ->and($urls)->toContain('http://www.example.com/crawling/sub2#fragment1')\n ->and($urls)->toContain('http://www.example.com/crawling/sub2#fragment2');\n },\n);\n\nit('stops crawling when maxOutputs is reached', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(\n Http::crawl()\n ->keepUrlFragment()\n ->maxOutputs(4),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(4)\n ->and($crawler->getLoader()->loadedUrls)->toHaveCount(4);\n});\n\nit('uses canonical links when useCanonicalLinks() is called', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/main')\n ->addStep(\n Http::crawl()\n ->useCanonicalLinks()\n ->keep(['url']),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n $resultUrls = array_map(function (Result $result) {\n return $result->get('url');\n }, $results);\n\n expect($resultUrls)\n ->toBe([\n 'http://www.example.com/crawling/main',\n 'http://www.example.com/crawling/sub1/sub1', // actual loaded url was sub1, but canonical is sub1/sub1\n 'http://www.example.com/crawling/sub2',\n 'http://www.example.com/crawling/sub2/sub1/sub1',\n ])\n ->and($crawler->getLoader()->loadedUrls)\n ->toBe([\n 'http://www.example.com/crawling/main',\n 'http://www.example.com/crawling/sub1', // => /crawling/sub1/sub1 => this URL wasn't loaded yet,\n 'http://www.example.com/crawling/sub2', // so when the link is discovered it won't load it.\n 'http://www.example.com/crawling/sub2/sub1', // => /crawling/sub1/sub1 => this URL was already loaded,\n 'http://www.example.com/crawling/sub2/sub1/sub1', // so the response is not yielded as a separate result.\n ]);\n});\n\nit('does not yield the same page twice when a URL was redirected to an already loaded page', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/crawling/redirect')\n ->addStep(Http::crawl()->keep(['url']));\n\n $results = helper_generatorToArray($crawler->run());\n\n $resultUrls = array_map(function (Result $result) {\n return $result->get('url');\n }, $results);\n\n expect($resultUrls)\n ->toContain('http://www.example.com/crawling/main')\n ->and($resultUrls)\n ->not()\n ->toContain('http://www.example.com/crawling/redirect')\n ->and($this->getActualOutputForAssertion())\n ->toContain('Was already loaded before. Do not process this page again.');\n});\n\nit('does not produce a fatal error when the initial request fails', function () {\n $crawler = (new Crawler())\n ->input('http://www.example.com/not-allowed')\n ->addStep(Http::crawl()->keep(['url']));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(0);\n});\n" }, { "path": "tests/_Integration/Http/ErrorResponsesTest.php", "content": "inputs(['http://localhost:8000/client-error-response'])\n ->addStep(Http::{$method}()->keepAs('response'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toBeEmpty();\n})->with(['get', 'post', 'put', 'patch', 'delete']);\n\nit('does not yield server error responses by default', function (string $method) {\n $crawler = new ErrorCrawler();\n\n $crawler->inputs(['http://localhost:8000/server-error-response'])\n ->addStep(Http::{$method}()->keepAs('response'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toBeEmpty();\n})->with(['get', 'post', 'put', 'patch', 'delete']);\n\nit('yields client error responses when yieldErrorResponses() was called', function (string $method) {\n $crawler = new ErrorCrawler();\n\n $crawler->inputs(['http://localhost:8000/client-error-response'])\n ->addStep(Http::{$method}()->yieldErrorResponses()->keepAs('response'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1);\n})->with(['get', 'post', 'put', 'patch', 'delete']);\n\nit('yields server error responses when yieldErrorResponses() was called', function (string $method) {\n $crawler = new ErrorCrawler();\n\n $crawler->inputs(['http://localhost:8000/server-error-response'])\n ->addStep(Http::{$method}()->yieldErrorResponses()->keepAs('response'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1);\n})->with(['get', 'post', 'put', 'patch', 'delete']);\n\nit(\n 'goes on crawling after a client error response when stopOnErrorResponse() wasn\\'t called',\n function (string $method) {\n $crawler = new ErrorCrawler();\n\n $crawler->inputs(['http://localhost:8000/client-error-response', 'http://localhost:8000/simple-listing'])\n ->addStep(Http::{$method}()->keepAs('response'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1);\n },\n)->with(['get', 'post', 'put', 'patch', 'delete']);\n\nit(\n 'goes on crawling after a server error response when stopOnErrorResponse() wasn\\'t called',\n function (string $method) {\n $crawler = new ErrorCrawler();\n\n $crawler->inputs(['http://localhost:8000/server-error-response', 'http://localhost:8000/simple-listing'])\n ->addStep(Http::{$method}()->keepAs('response'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1);\n },\n)->with(['get', 'post', 'put', 'patch', 'delete']);\n\nit(\n 'stops crawling (throws exception) after a client error response when the stopOnErrorResponse() method was called',\n function (string $method) {\n $crawler = new ErrorCrawler();\n\n $crawler->inputs(['http://localhost:8000/client-error-response', 'http://localhost:8000/simple-listing'])\n ->addStep(Http::{$method}()->stopOnErrorResponse());\n\n $crawler->runAndTraverse();\n },\n)->with(['get', 'post', 'put', 'patch', 'delete'])->throws(LoadingException::class);\n\nit(\n 'stops crawling (throws exception) after a server error response when the stopOnErrorResponse() method was called',\n function (string $method) {\n $crawler = new ErrorCrawler();\n\n $crawler->inputs(['http://localhost:8000/client-error-response', 'http://localhost:8000/simple-listing'])\n ->addStep(\n Http::{$method}()\n ->stopOnErrorResponse(),\n );\n\n $crawler->runAndTraverse();\n },\n)->with(['get', 'post', 'put', 'patch', 'delete'])->throws(LoadingException::class);\n\nit('does not log warnings about multiple loader hook calls when stopOnErrorResponse() is used', function () {\n $crawler = new ErrorCrawler();\n\n $crawler->inputs(['http://localhost:8000/hello-world', 'http://localhost:8000/simple-listing'])\n ->addStep(Http::get()->stopOnErrorResponse());\n\n $crawler->runAndTraverse();\n\n foreach ($crawler->getLogger()->messages as $message) {\n expect($message['message'])->not->toContain(' was already called in this load call.');\n }\n});\n" }, { "path": "tests/_Integration/Http/GzipTest.php", "content": "input('http://localhost:8000/gzip')\n ->addStep(Http::get()->keepAs('response'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0])->toBeInstanceOf(Result::class)\n ->and($results[0]->get('response'))->toBeInstanceOf(RespondedRequest::class)\n ->and(Http::getBodyString($results[0]->get('response')))->toBe('This is a gzip compressed string');\n});\n" }, { "path": "tests/_Integration/Http/HeadlessBrowserTest.php", "content": "useHeadlessBrowser();\n\n return $loader;\n }\n}\n\nclass GetJsonFromResponseHtmlBody extends Step\n{\n protected function invoke(mixed $input): Generator\n {\n $html = Http::getBodyString($input->response);\n\n $jsonString = (new HtmlDocument($html))->querySelector('body pre')?->text() ?? '';\n\n yield json_decode($jsonString, true);\n }\n}\n\nclass GetStringFromResponseHtmlBody extends Step\n{\n protected function invoke(mixed $input): Generator\n {\n $html = Http::getBodyString($input->response);\n\n yield (new HtmlDocument($html))->querySelector('body')?->text() ?? '';\n }\n}\n\n/**\n * @return Cookie[]\n */\nfunction helper_getCookiesByDomainFromLoader(HttpLoader $loader, string $domain): array\n{\n $cookieJar = invade($loader)->cookieJar;\n\n /** @var CookieJar $cookieJar */\n\n return $cookieJar->allByDomain($domain);\n}\n\nit('automatically uses the Loader\\'s user agent', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler->input('http://localhost:8000/print-headers')\n ->addStep(Http::get())\n ->addStep((new GetJsonFromResponseHtmlBody())->keepAs('responseBody'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get('responseBody'))->toBeArray()\n ->and($results[0]->get('responseBody'))->toHaveKey('User-Agent')\n ->and($results[0]->get('responseBody')['User-Agent'])->toBe('HeadlessBrowserBot');\n});\n\nit(\n 'does not use the user-agent defined in the crawler, when useNativeUserAgent() was called on the browser loader ' .\n 'helper',\n function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->getLoader()\n ->browser()\n ->useNativeUserAgent();\n\n $crawler->input('http://localhost:8000/print-headers')\n ->addStep(Http::get())\n ->addStep((new GetJsonFromResponseHtmlBody())->keepAs('responseBody'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get('responseBody'))->toBeArray()\n ->and($results[0]->get('responseBody'))->toHaveKey('User-Agent')\n ->and($results[0]->get('responseBody')['User-Agent'])->toStartWith('Mozilla/5.0 (');\n },\n);\n\nit('uses cookies', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/set-cookie')\n ->addStep(Http::get())\n ->addStep(new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'http://localhost:8000/print-cookie';\n }\n })\n ->addStep(Http::get())\n ->addStep((new GetStringFromResponseHtmlBody())->keepAs('printed-cookie'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get('printed-cookie'))->toBeString()\n ->and($results[0]->get('printed-cookie'))->toBe('foo123');\n});\n\nit('does not use cookies when HttpLoader::dontUseCookies() was called', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler->getLoader()->dontUseCookies();\n\n $crawler\n ->input('http://localhost:8000/set-cookie')\n ->addStep(Http::get())\n ->addStep(new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'http://localhost:8000/print-cookie';\n }\n })\n ->addStep(Http::get())\n ->addStep((new GetStringFromResponseHtmlBody())->keepAs('printed-cookie'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get('printed-cookie'))->toBeEmpty();\n});\n\nit('renders javascript', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler->input('http://localhost:8000/js-rendering')\n ->addStep(Http::get())\n ->addStep(\n Html::root()\n ->extract(['content' => '#content p']),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->toArray())->toBe([\n 'content' => 'This was added through javascript',\n ]);\n});\n\nit('gets cookies that are set via javascript', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $cache = new FileCache(helper_cachedir());\n\n $cache->clear();\n\n $crawler->getLoader()->setCache($cache);\n\n $crawler\n ->input('http://localhost:8000/set-js-cookie')\n ->addStep(Http::get());\n\n helper_generatorToArray($crawler->run());\n\n $cookiesInJar = helper_getCookiesByDomainFromLoader($crawler->getLoader(), 'localhost');\n\n $testCookie = $cookiesInJar['testcookie'] ?? null;\n\n expect($cookiesInJar)->toHaveCount(1)\n ->and($testCookie?->name())->toBe('testcookie')\n ->and($testCookie?->value())->toBe('javascriptcookie');\n\n // Check that cookie is not added to the cookiejar when the response was served from cache.\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler->getLoader()->setCache($cache);\n\n $crawler\n ->input('http://localhost:8000/set-js-cookie')\n ->addStep(Http::get());\n\n helper_generatorToArray($crawler->run());\n\n $cookiesInJar = helper_getCookiesByDomainFromLoader($crawler->getLoader(), 'localhost');\n\n expect($cookiesInJar)->toHaveCount(0);\n});\n\nit('gets a cookie that is set via a click, executed via post browser navigate hook', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/set-delayed-js-cookie')\n ->addStep(\n Http::get()\n ->postBrowserNavigateHook(BrowserAction::clickElement('#consent_btn')),\n )\n ->addStep(new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'http://localhost:8000/print-cookie';\n }\n })\n ->addStep(Http::get())\n ->addStep((new GetStringFromResponseHtmlBody())->keepAs('printed-cookie'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get('printed-cookie'))->toBeString()\n ->and($results[0]->get('printed-cookie'))->toBe('javascriptcookie');\n\n $cookiesInJar = helper_getCookiesByDomainFromLoader($crawler->getLoader(), 'localhost');\n\n $testCookie = $cookiesInJar['testcookie'] ?? null;\n\n expect($cookiesInJar)->toHaveCount(1)\n ->and($testCookie?->name())->toBe('testcookie')\n ->and($testCookie?->value())->toBe('javascriptcookie');\n});\n\nit(\n 'sending cookies works correctly when the loader is not configured to use the browser but two steps use the ' .\n 'browser by calling the useBrowser() method of Http steps',\n function () {\n $crawler = HttpCrawler::make()->withMozilla5CompatibleUserAgent();\n\n $crawler\n ->input('http://localhost:8000/set-multiple-js-cookies')\n ->addStep(Http::get()->useBrowser())\n ->addStep(new class extends Step {\n protected function invoke(mixed $input): Generator\n {\n yield 'http://localhost:8000/print-cookies';\n }\n })\n ->addStep(Http::get()->useBrowser())\n ->addStep((new GetStringFromResponseHtmlBody())->keepAs('printed-cookies'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get('printed-cookies'))->toBeString()\n ->and($results[0]->get('printed-cookies'))\n ->toBe('cookie3=cookie3value;cookie2=cookie2value;cookie1=cookie1value');\n\n $cookiesInJar = helper_getCookiesByDomainFromLoader($crawler->getLoader(), 'localhost');\n\n expect($cookiesInJar)->toHaveCount(3)\n ->and($cookiesInJar['cookie1']->value())->toBe('cookie1value')\n ->and($cookiesInJar['cookie2']->value())->toBe('cookie2value')\n ->and($cookiesInJar['cookie3']->value())->toBe('cookie3value');\n },\n);\n\ntest(\n 'BrowserAction::clickElement(), clickInsideShadowDom(), evaluate(), moveMouseToElement(), ' .\n 'moveMouseToPosition(), scrollDown(), scrollUp() and typeText() work as expected',\n function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->getLoader()\n ->browser()\n ->includeShadowElementsInHtml();\n\n $crawler\n ->input('http://localhost:8000/browser-actions')\n ->addStep(\n Http::get()\n // Inserting the #click_element is delayed in the page, so this also tests, that the\n // BrowserAction::clickElement() action automatically waits for an element matching the selector\n // to be present.\n ->postBrowserNavigateHook(BrowserAction::clickElement('#click_element'))\n ->postBrowserNavigateHook(BrowserAction::screenshot(ScreenshotConfig::make(helper_storagedir())))\n ->postBrowserNavigateHook(BrowserAction::clickInsideShadowDom('#shadow_host', '#shadow_click_div'))\n ->postBrowserNavigateHook(\n BrowserAction::evaluate(\n 'document.getElementById(\\'evaluation_container\\').innerHTML = \\'evaluated\\'',\n ),\n )\n ->postBrowserNavigateHook(BrowserAction::moveMouseToElement('#mouseover_check_1'))\n ->postBrowserNavigateHook(BrowserAction::moveMouseToPosition(305, 405))\n ->postBrowserNavigateHook(BrowserAction::scrollDown(4000))\n ->postBrowserNavigateHook(\n BrowserAction::screenshot(\n ScreenshotConfig::make(helper_storagedir())\n ->setImageFileType('jpeg')\n ->setQuality(20)\n ->setFullPage(),\n ),\n )\n ->postBrowserNavigateHook(BrowserAction::scrollUp(2000))\n ->postBrowserNavigateHook(BrowserAction::scrollUp(2000))\n ->postBrowserNavigateHook(BrowserAction::clickElement('#input'))\n ->postBrowserNavigateHook(BrowserAction::typeText('typing text works'))\n ->keep(['body', 'screenshots']),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n $body = $results[0]->get('body');\n\n $screenshots = $results[0]->get('screenshots');\n\n expect($body)->toContain('
    yes
    ')\n // This also tests the `HeadlessBrowserLoaderHelper::includeShadowElementsInHtml()` method,\n // because even if the click worked, with the normal way of getting HTML this wouldn't be\n // included in the returned HTML.\n ->and($body)->toContain('
    clicked
    ')\n ->and($body)->toContain('
    evaluated
    ')\n ->and($body)->toContain('
    mouse was here
    ')\n ->and($body)->toContain('
    mouse was here
    ')\n ->and($body)->toContain('
    scrolled down
    ')\n ->and($body)->toContain('
    scrolled up
    ')\n ->and($body)->toContain('
    typing text works
    ')\n ->and($screenshots)->toHaveCount(2)\n ->and($screenshots[0])->toEndWith('.png')\n ->and($screenshots[1])->toEndWith('.jpeg');\n\n if (function_exists('getimagesize')) {\n $screenshot1Size = getimagesize($screenshots[0]);\n\n $screenshot2Size = getimagesize($screenshots[1]);\n\n if (is_array($screenshot1Size) && is_array($screenshot2Size)) {\n expect($screenshot1Size[1])->toBeLessThan(2100)\n ->and($screenshot2Size[1])->toBeGreaterThan(4000);\n }\n }\n\n helper_resetStorageDir();\n },\n);\n\ntest('BrowserAction::waitUntilDocumentContainsElement() works as expected', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/browser-actions/wait')\n ->addStep(\n Http::get()\n ->postBrowserNavigateHook(\n BrowserAction::waitUntilDocumentContainsElement('#delayed_container'),\n )\n ->keep('body'),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n $body = $results[0]->get('body');\n\n expect($body)->toContain('
    hooray
    ');\n});\n\ntest('BrowserAction::clickElementAndWaitForReload() works as expected', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/browser-actions/click-and-wait-for-reload')\n ->addStep(\n Http::get()\n ->postBrowserNavigateHook(BrowserAction::clickElementAndWaitForReload('#click'))\n ->keep('body'),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n $body = $results[0]->get('body');\n\n expect($body)->toContain('
    yes
    ');\n});\n\ntest(\n 'when on the click and wait for reload page, and the element is only clicked but we don\\'t wait for reload, ' .\n 'we don\\'t get the reloaded page content',\n function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/browser-actions/click-and-wait-for-reload')\n ->addStep(\n Http::get()\n ->postBrowserNavigateHook(BrowserAction::clickElement('#click'))\n ->keep('body'),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n $body = $results[0]->get('body');\n\n expect($body)->not()->toContain('
    yes
    ');\n },\n);\n\ntest(\n 'when on the click and wait for reload page, and the element is clicked and we also wait for reload, we get the ' .\n 'reloaded page content',\n function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/browser-actions/click-and-wait-for-reload')\n ->addStep(\n Http::get()\n ->postBrowserNavigateHook(BrowserAction::clickElement('#click'))\n ->postBrowserNavigateHook(BrowserAction::waitForReload())\n ->keep('body'),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n $body = $results[0]->get('body');\n\n expect($body)->toContain('
    yes
    ');\n },\n);\n\ntest('BrowserAction::evaluateAndWaitForReload() works as expected', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/browser-actions/evaluate-and-wait-for-reload')\n ->addStep(\n Http::get()\n ->postBrowserNavigateHook(\n BrowserAction::evaluateAndWaitForReload(\n 'window.location.href = \\'http://localhost:8000/browser-actions/' .\n 'evaluate-and-wait-for-reload-reloaded\\'',\n ),\n )\n ->keep('body'),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n $body = $results[0]->get('body');\n\n expect($body)->toContain('
    yay
    ');\n});\n\ntest('BrowserAction::wait() works as expected', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/browser-actions/wait')\n ->addStep(\n Http::get()\n ->postBrowserNavigateHook(BrowserAction::wait(0.3))\n ->keep('body'),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n $body = $results[0]->get('body');\n\n expect($body)->toContain('
    hooray
    ');\n});\n\nit('executes the javascript code provided via HeadlessBrowserLoaderHelper::setPageInitScript()', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->getLoader()\n ->browser()\n ->setPageInitScript('window._secret_content = \\'secret content\\'');\n\n $crawler\n ->input('http://localhost:8000/page-init-script')\n ->addStep(Http::get())\n ->addStep(Html::root()->extract(['content' => '#content']));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->get('content'))->toBe('secret content');\n});\n\nit('gets the source of an XML response without being wrapped in an HTML document', function () {\n $crawler = new HeadlessBrowserCrawler();\n\n $crawler\n ->input('http://localhost:8000/rss-feed')\n ->addStep(Http::get()->keep(['body']));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->get('body'))->toStartWith('' . PHP_EOL . 'input('http://localhost:8000/broken-mime-type-rss')\n ->addStep(Http::get()->keep(['body']));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->get('body'))->toStartWith('');\n },\n);\n" }, { "path": "tests/_Integration/Http/Html/PaginatedListingTest.php", "content": "input('http://localhost:8000/paginated-listing');\n\n $crawler\n ->addStep(Http::get()->paginate('#nextPage'))\n ->addStep(Html::getLinks('#listing .item a')->keepAs('url'))\n ->addStep(Http::get())\n ->addStep(\n Html::first('article')\n ->extract(['title' => 'h1', 'number' => '.someNumber'])\n ->keep(),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(10)\n ->and($results[0]->toArray())->toBe([\n 'url' => 'http://localhost:8000/paginated-listing/items/1',\n 'title' => 'Some Item 1',\n 'number' => '10',\n ])\n ->and($results[9]->toArray())->toBe([\n 'url' => 'http://localhost:8000/paginated-listing/items/10',\n 'title' => 'Some Item 10',\n 'number' => '100',\n ]);\n});\n" }, { "path": "tests/_Integration/Http/Html/SimpleListingTest.php", "content": "input('http://localhost:8000/simple-listing');\n\n $crawler->addStep(Http::get())\n ->addStep(Html::getLinks('.listingItem a'))\n ->addStep(Http::get())\n ->addStep(\n Html::first('article')\n ->extract([\n 'title' => 'h1',\n 'date' => '.date',\n 'author' => '.articleAuthor',\n ])\n ->keep(),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(3)\n ->and($results[0]->toArray())->toBe([\n 'title' => 'Some Article 1',\n 'date' => '2022-04-13',\n 'author' => 'Christian Olear',\n ])\n ->and($results[1]->toArray())->toBe([\n 'title' => 'Some Article 2',\n 'date' => '2022-04-14',\n 'author' => 'Christian Olear',\n ])\n ->and($results[2]->toArray())->toBe([\n 'title' => 'Some Article 3',\n 'date' => '2022-04-15',\n 'author' => 'Christian Olear',\n ]);\n});\n" }, { "path": "tests/_Integration/Http/PaginationTest.php", "content": "input('http://localhost:8000/paginated-listing')\n ->addStep(Http::get()->paginate('#pagination'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(5);\n});\n\nit('only iterates pagination until max pages limit is reached', function () {\n $crawler = new PaginationCrawler();\n\n $crawler->input('http://localhost:8000/paginated-listing')\n ->addStep(Http::get()->paginate('#pagination', 2));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(2)\n ->and($this->getActualOutputForAssertion())->toContain('Max pages limit reached');\n});\n\nit('resets the finished paginating state after each processed (/paginated) input', function () {\n $crawler = new PaginationCrawler();\n\n $crawler\n ->inputs(['http://localhost:8000/paginated-listing', 'http://localhost:8000/paginated-listing?foo=bar'])\n ->addStep(Http::get()->paginate('#pagination', 2)->outputKey('response'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(4);\n});\n" }, { "path": "tests/_Integration/Http/ProxyingTest.php", "content": "\n */\n public static array $processes = [8001 => null, 8002 => null, 8003 => null];\n}\n\nbeforeEach(function () {\n $startedProcesses = false;\n\n foreach (ProxyServerProcesses::PORTS as $port) {\n if (!ProxyServerProcesses::$processes[$port]) {\n ProxyServerProcesses::$processes[$port] = Process::fromShellCommandline(\n 'php -S localhost:' . $port . ' ' . __DIR__ . '/../ProxyServer.php',\n );\n\n ProxyServerProcesses::$processes[$port]->start();\n\n $startedProcesses = true;\n }\n }\n\n if ($startedProcesses) {\n usleep(100_000);\n }\n});\n\nafterAll(function () {\n foreach (ProxyServerProcesses::PORTS as $port) {\n ProxyServerProcesses::$processes[$port]?->stop(3, SIGINT);\n\n ProxyServerProcesses::$processes[$port] = null;\n }\n});\n\nit('uses a proxy when the useProxy() method of the loader was called', function () {\n $crawler = helper_getFastCrawler();\n\n $crawler->getLoader()->useProxy('http://localhost:8001');\n\n $crawler\n ->input('http://www.crwlr.software/packages')\n ->addStep(Http::get()->keep(['body']));\n\n $results = iterator_to_array($crawler->run());\n\n expect($results[0])\n ->toBeInstanceOf(Result::class)\n ->and($results[0]->get('body'))\n ->toContain('Proxy Server Response for http://www.crwlr.software/packages');\n});\n\nit('uses correct method, headers and HTTP version in the proxied request', function () {\n $crawler = helper_getFastCrawler();\n\n $crawler->getLoader()->useProxy('http://localhost:8001');\n\n $crawler\n ->input('http://www.crwlr.software/packages')\n ->addStep(\n Http::put(['Accept-Encoding' => 'gzip, deflate, br'], 'Hello World', '1.0')\n ->keep(['body']),\n );\n\n $results = iterator_to_array($crawler->run());\n\n expect($results[0])\n ->toBeInstanceOf(Result::class)\n ->and($results[0]->get('body'))\n ->toContain('Protocol Version: HTTP/1.0')\n ->toContain('Request Method: PUT')\n ->toContain('Request Body: Hello World')\n ->toContain('[\"Accept-Encoding\"]=>' . PHP_EOL . ' string(17) \"gzip, deflate, br\"');\n});\n\nit('uses rotating proxies when the useRotatingProxies() method of the loader was called', function () {\n $crawler = helper_getFastCrawler();\n\n $crawler->getLoader()->useRotatingProxies([\n 'http://localhost:8001',\n 'http://localhost:8002',\n 'http://localhost:8003',\n ]);\n\n $crawler\n ->input([\n 'http://www.crwlr.software/packages/crawler/v1.1/getting-started',\n 'http://www.crwlr.software/packages/url/v2.0/getting-started',\n 'http://www.crwlr.software/packages/query-string/v1.0/getting-started',\n 'http://www.crwlr.software/packages/robots-txt/v1.1/getting-started',\n ])\n ->addStep(Http::get()->keep(['body']));\n\n $results = iterator_to_array($crawler->run());\n\n expect($results)->toHaveCount(4)\n ->and($results[0])\n ->toBeInstanceOf(Result::class)\n ->and($results[0]->get('body'))\n ->toContain('Port: 8001') // First request with first proxy\n ->and($results[1])\n ->toBeInstanceOf(Result::class)\n ->and($results[1]->get('body'))\n ->toContain('Port: 8002') // Second request with second proxy\n ->and($results[2])\n ->toBeInstanceOf(Result::class)\n ->and($results[2]->get('body'))\n ->toContain('Port: 8003') // Third request with third proxy\n ->and($results[3])\n ->toBeInstanceOf(Result::class)\n ->and($results[3]->get('body'))\n ->toContain('Port: 8001'); // And finally the fourth request with the first proxy again.\n});\n\nit('can also use a proxy when using the headless browser', function () {\n $crawler = helper_getFastCrawler();\n\n $crawler\n ->getLoader()\n ->useHeadlessBrowser()\n ->useProxy('http://localhost:8001');\n\n $crawler\n ->input('http://www.crwlr.software/blog')\n ->addStep(\n Http::get(['Accept-Language' => 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7'])\n ->keep(['body']),\n );\n\n $results = iterator_to_array($crawler->run());\n\n expect($results[0])\n ->toBeInstanceOf(Result::class)\n ->and($results[0]->get('body'))\n ->toContain('[\"Accept-Language\"]=>' . PHP_EOL . ' string(35) \"de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7\"');\n});\n\nit('can also use rotating proxies when using the headless browser', function () {\n $crawler = helper_getFastCrawler();\n\n $crawler\n ->getLoader()\n ->useHeadlessBrowser()\n ->useRotatingProxies([\n 'http://localhost:8001',\n 'http://localhost:8002',\n ]);\n\n $crawler\n ->input([\n 'http://www.crwlr.software/packages/crawler/v1.1',\n 'http://www.crwlr.software/packages/url/v2.0',\n 'http://www.crwlr.software/packages/query-string/v1.0',\n ])\n ->addStep(Http::get()->keep(['body']));\n\n $results = iterator_to_array($crawler->run());\n\n expect($results)->toHaveCount(3)\n ->and($results[0])\n ->toBeInstanceOf(Result::class)\n ->and($results[0]->get('body'))\n ->toContain('Port: 8001') // First request with first proxy\n ->and($results[1])\n ->toBeInstanceOf(Result::class)\n ->and($results[1]->get('body'))\n ->toContain('Port: 8002') // Second request with second proxy\n ->and($results[2])\n ->toBeInstanceOf(Result::class)\n ->and($results[2]->get('body'))\n ->toContain('Port: 8001'); // And finally the third request with the first proxy again.\n});\n" }, { "path": "tests/_Integration/Http/PublisherExampleTest.php", "content": "input('http://localhost:8000/publisher/authors')\n ->addStep(Http::get())\n ->addStep(Html::getLinks('#authors a'))\n ->addStep(Http::get())\n ->addStep(\n Html::root()\n ->extract([\n 'author' => 'h1',\n 'bookUrls' => Dom::cssSelector('#author-data .books a.book')->attribute('href')->toAbsoluteUrl(),\n ])\n ->keep(['author']),\n )\n ->addStep(Http::get()->useInputKey('bookUrls'))\n ->addStep(\n Html::root()\n ->extract(['book' => 'h1'])\n ->keep(),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(5)\n ->and($results[0]->toArray())->toBe([\n 'author' => 'John Example',\n 'book' => 'Some novel',\n ])\n ->and($results[1]->toArray())->toBe([\n 'author' => 'John Example',\n 'book' => 'Another novel',\n ])\n ->and($results[2]->toArray())->toBe([\n 'author' => 'Susan Example',\n 'book' => 'Poems #1',\n ])\n ->and($results[3]->toArray())->toBe([\n 'author' => 'Susan Example',\n 'book' => 'Poems #2',\n ])\n ->and($results[4]->toArray())->toBe([\n 'author' => 'Susan Example',\n 'book' => 'Poems #3',\n ]);\n});\n\nit('turns an array of URLs to nested extracted data from those child pages using sub crawlers', function () {\n $crawlerBuilder = new class {\n public function build(): \\Crwlr\\Crawler\\Crawler\n {\n $crawler = new PublisherExampleCrawler();\n\n return $crawler\n ->input('http://localhost:8000/publisher/authors')\n ->addStep(Http::get())\n ->addStep(Html::getLinks('#authors a'))\n ->addStep(Http::get())\n ->addStep($this->extractAuthorData());\n }\n\n private function extractAuthorData(): Html\n {\n return Html::root()\n ->extract([\n 'name' => 'h1',\n 'age' => '#author-data .age',\n 'bornIn' => '#author-data .born-in',\n 'books' => Dom::cssSelector('#author-data .books a.book')->link(),\n ])\n ->subCrawlerFor('books', function (\\Crwlr\\Crawler\\Crawler $crawler) {\n return $crawler\n ->addStep(Http::get())\n ->addStep(\n $this->extractBookData(),\n );\n });\n }\n\n private function extractBookData(): Html\n {\n return Html::root()\n ->extract(['title' => 'h1', 'editions' => Dom::cssSelector('#editions a')->link()])\n ->subCrawlerFor('editions', function (\\Crwlr\\Crawler\\Crawler $crawler) {\n return $crawler\n ->addStep(Http::get())\n ->addStep($this->extractEditionData());\n });\n }\n\n private function extractEditionData(): Html\n {\n return Html::root()\n ->extract(['year' => '.year', 'publisher' => '.publishingCompany']);\n }\n };\n\n $results = helper_generatorToArray($crawlerBuilder->build()->run());\n\n expect($results)->toHaveCount(2)\n ->and($results[0]->toArray())->toBe([\n 'name' => 'John Example',\n 'age' => '51',\n 'bornIn' => 'Lisbon',\n 'books' => [\n [\n 'title' => 'Some novel',\n 'editions' => [\n ['year' => '1996', 'publisher' => 'Foo'],\n ['year' => '2005', 'publisher' => 'Foo'],\n ],\n ],\n [\n 'title' => 'Another novel',\n 'editions' => [\n ['year' => '2001', 'publisher' => 'Foo'],\n ['year' => '2009', 'publisher' => 'Bar'],\n ['year' => '2017', 'publisher' => 'Bar'],\n ],\n ],\n ],\n ])\n ->and($results[1]->toArray())->toBe([\n 'name' => 'Susan Example',\n 'age' => '49',\n 'bornIn' => 'Athens',\n 'books' => [\n [\n 'title' => 'Poems #1',\n 'editions' => [\n ['year' => '2008', 'publisher' => 'Poems'],\n ['year' => '2009', 'publisher' => 'Poems'],\n ],\n ],\n [\n 'title' => 'Poems #2',\n 'editions' => [\n ['year' => '2011', 'publisher' => 'Poems'],\n ['year' => '2014', 'publisher' => 'New Poems'],\n ],\n ],\n [\n 'title' => 'Poems #3',\n 'editions' => [\n ['year' => '2013', 'publisher' => 'Poems'],\n ['year' => '2017', 'publisher' => 'New Poems'],\n ],\n ],\n ],\n ]);\n});\n\ntest('it can also keep the URLs, provided to the sub crawler', function () {\n $crawlerBuilder = new class {\n public function build(): \\Crwlr\\Crawler\\Crawler\n {\n $crawler = new PublisherExampleCrawler();\n\n return $crawler\n ->input('http://localhost:8000/publisher/authors')\n ->addStep(Http::get())\n ->addStep(Html::getLinks('#authors a'))\n ->addStep(Http::get())\n ->addStep($this->extractAuthorData());\n }\n\n private function extractAuthorData(): Html\n {\n return Html::root()\n ->extract([\n 'name' => 'h1',\n 'age' => '#author-data .age',\n 'bornIn' => '#author-data .born-in',\n 'books' => Dom::cssSelector('#author-data .books a.book')->link(),\n ])\n ->subCrawlerFor('books', function (\\Crwlr\\Crawler\\Crawler $crawler) {\n return $crawler\n ->addStep(Http::get()->keepInputAs('url'))\n ->addStep($this->extractBookData());\n });\n }\n\n private function extractBookData(): Html\n {\n return Html::root()\n ->extract(['title' => 'h1', 'editions' => Dom::cssSelector('#editions a')->link()])\n ->subCrawlerFor('editions', function (\\Crwlr\\Crawler\\Crawler $crawler) {\n return $crawler\n ->addStep(Http::get()->keepInputAs('url'))\n ->addStep($this->extractEditionData());\n });\n }\n\n private function extractEditionData(): Html\n {\n return Html::root()\n ->extract(['year' => '.year', 'publisher' => '.publishingCompany']);\n }\n };\n\n $results = helper_generatorToArray($crawlerBuilder->build()->run());\n\n expect($results)->toHaveCount(2)\n ->and($results[0]->toArray())->toBe([\n 'name' => 'John Example',\n 'age' => '51',\n 'bornIn' => 'Lisbon',\n 'books' => [\n [\n 'url' => 'http://localhost:8000/publisher/books/1',\n 'title' => 'Some novel',\n 'editions' => [\n [\n 'url' => 'http://localhost:8000/publisher/books/1/edition/1',\n 'year' => '1996',\n 'publisher' => 'Foo',\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/1/edition/2',\n 'year' => '2005',\n 'publisher' => 'Foo',\n ],\n ],\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/2',\n 'title' => 'Another novel',\n 'editions' => [\n [\n 'url' => 'http://localhost:8000/publisher/books/2/edition/1',\n 'year' => '2001',\n 'publisher' => 'Foo',\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/2/edition/2',\n 'year' => '2009',\n 'publisher' => 'Bar',\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/2/edition/3',\n 'year' => '2017',\n 'publisher' => 'Bar',\n ],\n ],\n ],\n ],\n ])\n ->and($results[1]->toArray())->toBe([\n 'name' => 'Susan Example',\n 'age' => '49',\n 'bornIn' => 'Athens',\n 'books' => [\n [\n 'url' => 'http://localhost:8000/publisher/books/3',\n 'title' => 'Poems #1',\n 'editions' => [\n [\n 'url' => 'http://localhost:8000/publisher/books/3/edition/1',\n 'year' => '2008',\n 'publisher' => 'Poems',\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/3/edition/2',\n 'year' => '2009',\n 'publisher' => 'Poems',\n ],\n ],\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/4',\n 'title' => 'Poems #2',\n 'editions' => [\n [\n 'url' => 'http://localhost:8000/publisher/books/4/edition/1',\n 'year' => '2011',\n 'publisher' => 'Poems',\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/4/edition/2',\n 'year' => '2014',\n 'publisher' => 'New Poems',\n ],\n ],\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/5',\n 'title' => 'Poems #3',\n 'editions' => [\n [\n 'url' => 'http://localhost:8000/publisher/books/5/edition/1',\n 'year' => '2013',\n 'publisher' => 'Poems',\n ],\n [\n 'url' => 'http://localhost:8000/publisher/books/5/edition/2',\n 'year' => '2017',\n 'publisher' => 'New Poems',\n ],\n ],\n ],\n ],\n ]);\n});\n" }, { "path": "tests/_Integration/Http/QueryParamPaginationTest.php", "content": "input('http://localhost:8000/query-param-pagination')\n ->addStep(\n Http::post(body: 'page=1')\n ->paginate(\n Paginator::queryParams(5)\n ->inBody()\n ->increase('page')\n ->stopWhen(PaginatorStopRules::isEmptyInJson('data.items')),\n )->keep(['body']),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(4);\n});\n\nit('also paginates using query params sent in the request body, when used in combination with static URL', function () {\n $crawler = new QueryParamPaginationCrawler();\n\n $crawler\n ->input('foo')\n ->addStep(\n Http::post(body: 'page=1')\n ->staticUrl('http://localhost:8000/query-param-pagination')\n ->paginate(\n Paginator::queryParams(3)\n ->inBody()\n ->increase('page'),\n )->keep(['body']),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(3);\n});\n\nit('paginates using URL query params', function () {\n $crawler = new QueryParamPaginationCrawler();\n\n $crawler\n ->input('http://localhost:8000/query-param-pagination?page=1')\n ->addStep(\n Http::get()\n ->paginate(\n Paginator::queryParams(5)\n ->inUrl()\n ->increase('page')\n ->stopWhen(PaginatorStopRules::isEmptyInJson('data.items')),\n )->keep(['body']),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(4);\n});\n\nit('paginates only until the max pages limit', function () {\n $crawler = new QueryParamPaginationCrawler();\n\n $crawler\n ->input('http://localhost:8000/query-param-pagination?page=1')\n ->addStep(\n Http::get()\n ->paginate(\n QueryParamsPaginator::paramsInUrl(2)\n ->increase('page')\n ->stopWhen(PaginatorStopRules::isEmptyInJson('data.items')),\n )->keep(['body']),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(2);\n});\n\nit('resets the finished paginating state after each processed (/paginated) input', function () {\n $crawler = new QueryParamPaginationCrawler();\n\n $crawler\n ->inputs([\n 'http://localhost:8000/query-param-pagination?page=1',\n 'http://localhost:8000/query-param-pagination?page=1&foo=bar',\n ])\n ->addStep(\n Http::get()\n ->paginate(\n QueryParamsPaginator::paramsInUrl(2)\n ->increase('page')\n ->stopWhen(PaginatorStopRules::isEmptyInJson('data.items')),\n )->keep(['body']),\n );\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(4);\n});\n" }, { "path": "tests/_Integration/Http/RedirectTest.php", "content": "input('http://localhost:8000/redirect?stopAt=5')\n ->addStep(Http::get())\n ->addStep((new GetResponseBodyAsString())->keepAs('body'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get('body'))->toBe('success after 5 redirects');\n});\n\nit('stops at 10 redirects by default', function () {\n $crawler = new RedirectTestCrawler();\n\n $crawler\n ->input('http://localhost:8000/redirect?stopAt=11')\n ->addStep(Http::get())\n ->addStep((new GetResponseBodyAsString())->keepAs('body'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(0);\n\n $logOutput = $this->getActualOutputForAssertion();\n\n expect($logOutput)->toContain('Failed to load http://localhost:8000/redirect?stopAt=11: Too many redirects.');\n});\n\ntest('you can set your own max redirects limit', function () {\n $crawler = new class extends HttpCrawler {\n protected function userAgent(): UserAgentInterface\n {\n return new UserAgent('RedirectBot');\n }\n\n protected function loader(UserAgentInterface $userAgent, LoggerInterface $logger): LoaderInterface\n {\n $loader = parent::loader($userAgent, $logger);\n\n if ($loader instanceof HttpLoader) {\n $loader->setMaxRedirects(15);\n }\n\n return $loader;\n }\n };\n\n $crawler\n ->input('http://localhost:8000/redirect?stopAt=11')\n ->addStep(Http::get())\n ->addStep((new GetResponseBodyAsString())->keepAs('body'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1)\n ->and($results[0]->get('body'))->toBe('success after 11 redirects');\n});\n" }, { "path": "tests/_Integration/Http/RequestParamsFromInputTest.php", "content": " 'http://localhost:8000/print-headers',\n 'body' => 'test',\n 'headers' => [\n 'header-x' => 'foo',\n 'header-y' => ['bar'],\n ],\n 'header-y' => 'baz',\n 'header-z' => ['quz'],\n ];\n }\n };\n\n $crawler = helper_getFastCrawler();\n\n $crawler\n ->input('anything')\n ->addStep($paramsStep)\n ->addStep(\n Http::get()\n ->useInputKeyAsBody('body')\n ->useInputKeyAsHeaders('headers')\n ->useInputKeyAsHeader('header-y', 'header-y')\n ->useInputKeyAsHeader('header-z', 'header-z'),\n )\n ->addStep(Json::all());\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results)->toHaveCount(1);\n\n $result = $results[0]->toArray();\n\n expect($result['Content-Length'])->toBe('4');\n\n expect($result['header-x'])->toBe('foo');\n\n expect($result['header-y'])->toBe('bar, baz');\n\n expect($result['header-z'])->toBe('quz');\n});\n" }, { "path": "tests/_Integration/Http/RetryErrorResponsesTest.php", "content": "input('http://localhost:8000' . $path)\n ->addStep(Http::get());\n\n $start = microtime(true);\n\n helper_generatorToArray($crawler->run());\n\n $end = microtime(true);\n\n $diff = $end - $start;\n\n expect($diff)->toBeGreaterThan(3.0);\n\n expect($diff)->toBeLessThan(3.5);\n})->with(['/too-many-requests', '/service-unavailable']);\n\nit(\n 'starts the first retry after the number of seconds returned in the Retry-After HTTP header',\n function (string $path) {\n $crawler = new RetryErrorResponsesCrawler();\n\n $crawler\n ->input('http://localhost:8000' . $path . '/retry-after')\n ->addStep(Http::get());\n\n $start = microtime(true);\n\n helper_generatorToArray($crawler->run());\n\n $end = microtime(true);\n\n $diff = $end - $start;\n\n expect($diff)->toBeGreaterThan(4.0);\n\n expect($diff)->toBeLessThan(4.5);\n },\n)->with(['/too-many-requests', '/service-unavailable']);\n\nit('goes on crawling when a retry receives a successful response', function (string $path) {\n $crawler = new RetryErrorResponsesCrawler();\n\n $crawler->input('http://localhost:8000' . $path . '/succeed-on-second-attempt')\n ->addStep(Http::get());\n\n $start = microtime(true);\n\n $results = helper_generatorToArray($crawler->run());\n\n $end = microtime(true);\n\n $diff = $end - $start;\n\n expect($results)->toHaveCount(1);\n\n expect($diff)->toBeGreaterThan(1.0);\n\n expect($diff)->toBeLessThan(1.5);\n})->with(['/too-many-requests', '/service-unavailable']);\n" }, { "path": "tests/_Integration/Http/RobotsTxtTest.php", "content": "input('http://localhost:8000/hello-world')\n ->addStep(Http::get())\n ->addStep(Html::root()->extract('body')->keepAs('body'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->get('body'))->toBe('Hello World!');\n\n $logger = $crawler->getLogger();\n\n foreach ($logger->messages as $message) {\n expect($message['message'])->not->toContain(' was already called in this load call.');\n }\n});\n\nit('also does not warn about loader hooks being called multiple times when loadOrFail() is used', function () {\n // See comment in the test above.\n $crawler = new RobotsTxtCrawler();\n\n $crawler\n ->input('http://localhost:8000/hello-world')\n ->addStep(Http::get()->stopOnErrorResponse())\n ->addStep(Html::root()->extract('body')->keepAs('body'));\n\n $results = helper_generatorToArray($crawler->run());\n\n expect($results[0]->get('body'))->toBe('Hello World!');\n\n $logger = $crawler->getLogger();\n\n foreach ($logger->messages as $message) {\n expect($message['message'])->not->toContain(' was already called in this load call.');\n }\n});\n" }, { "path": "tests/_Integration/Http/TimeoutTest.php", "content": " 1,\n 'timeout' => 1,\n ]);\n }\n };\n\n $crawler->input('http://localhost:8000/sleep')\n ->addStep(Http::get());\n\n $crawler->runAndTraverse();\n\n expect($this->getActualOutputForAssertion())->toContain('Operation timed out');\n});\n" }, { "path": "tests/_Integration/ProxyServer.php", "content": "= $stopAt) {\n echo 'success after ' . $redirectNo . ' redirects';\n\n return;\n } else {\n $stopAt = '&stopAt=' . $stopAt;\n }\n } else {\n $stopAt = '';\n }\n\n header('Location: http://localhost:8000/redirect?no=' . ($redirectNo + 1) . $stopAt);\n}\n\nif (str_starts_with($route, '/non-utf-8-charset')) {\n return include(__DIR__ . '/_Server/NonUtf8.php');\n}\n\nif (str_starts_with($route, '/page-init-script')) {\n return include(__DIR__ . '/_Server/PageInitScript.php');\n}\n\nif ($route === '/rss-feed') {\n header('Content-Type: text/xml; charset=utf-8');\n\n return include(__DIR__ . '/_Server/RssFeed.php');\n}\n\nif ($route === '/broken-mime-type-rss') {\n header('Content-Type: application/rss+xml; charset=UTF-8');\n\n return include(__DIR__ . '/_Server/BrokenMimeTypeRss.php');\n}\n\nif ($route === '/robots.txt') {\n return <<\n\n\n \n Prevent Homograph Attacks using the crwlr/url Package - crwlr.software\n\n\n\n
    \n
    \n
    \n

    Prevent Homograph Attacks using the crwlr/url Package

    \n
    2022-01-19
    \n

    This post is not crawling/scraping related, but about another\n valuable use case for the url package, to prevent so-called\n homograph attacks.

    \n

    About the attack

    \n

    Homograph attacks are using internationalized domain names (IDN) for\n malicious links including domains that look like trusted organizations.\n You might know attacks where they want to trick you with typos\n like faecbook or things like zeros instead of Os (g00gle).\n Using internationalized domain names this kind of attack is even\n harder to spot because they are using characters that almost exactly\n look like other characters (also depending on the font they're\n displayed with).

    \n

    Can you see the difference between those two As?

    \n

    a а

    \n

    No? But in fact they aren't the same. The second one is a Cyrillic\n character.
    \n You can check it e.g. by using PHP's ord function.

    \n 
    var_dump(ord('a')); // int(97)\nvar_dump(ord('а')); // int(208)
    \n

    Browsers already implemented mechanisms to warn users that a page\n they're visiting might not be as legitimate as they thought.

    \n

    But still: if on your website, you are linking to urls originating\n from user input, it'd be a good idea to have an eye on urls\n containing internationalized domain names.

    \n

    How to identify IDN urls using the Url class

    \n

    The Url class has the handy hasIdn method:

    \n 
    $legitUrl = Url::parse('https://www.apple.com');\n$seemsLegitUrl = Url::parse('https://www.аpple.com');\n\nvar_dump($legitUrl->hasIdn());              // bool(false)\nvar_dump($seemsLegitUrl->hasIdn());         // bool(true)\n\nvar_dump($legitUrl->__toString());          // string(21) \"https://www.apple.com\"\nvar_dump($seemsLegitUrl->__toString());     // string(28) \"https://www.xn--pple-43d.com\"
    \n

    So you see, it's very easy to identify IDN urls with it. Of course\n there are many legitimate IDN domains, so you might not want to\n automatically block all of them. I'd suggest you could put some kind\n of monitoring in place that notifies you about users posting links\n to IDNs.

    \n

    Maybe you're operating in a country where IDNs are very common. Maybe\n in that case you can find a way to automatically sort out legitimate\n uses from your area.

    \n
    \n \n
    \n
    \n\n\n\n" }, { "path": "tests/_Integration/_Server/BrokenMimeTypeRss.php", "content": "\n\n \n Lorem ipsum\n \n https://www.example.com/\n Lorem ipsum dolor sit amet\n Fri, 10 Jan 2025 10:48:01 +0000\n en\n \n hourly\t\n \n 1\t\n\n \n Foo\n https://www.example.com/some-article\n https://www.example.com/some-article#comments\n\n \n Fri, 10 Jan 2025 10:48:01 +0000\n \n \n \n https://www.example.com/?a=123\n\n Lorem ipsum dolor

    sit amet

    ]]>    \n\n\n \t    \n     \n    \n" }, { "path": "tests/_Integration/_Server/BrowserActions/ClickAndWaitForReload.php", "content": "\n\n\n \n Hello World\n\n\n
    \n
    Click here
    \n\n \n\n \n
    yes
    \n \n
    \n\n\n" }, { "path": "tests/_Integration/_Server/BrowserActions/EvaluateAndWaitForReload.php", "content": "\n\nHello World\n\n\n" }, { "path": "tests/_Integration/_Server/BrowserActions/EvaluateAndWaitForReloadReloaded.php", "content": "\n\nHello World\n\n
    yay
    \n\n\n" }, { "path": "tests/_Integration/_Server/BrowserActions/Main.php", "content": "\n\n\n \n Hello World\n \n\n\n
    \n
    \n
    \n
    \n
    \n
    \n \n
    \n
    mouse wasn't here yet
    \n
    mouse wasn't here yet
    \n
    not scrolled up yet
    \n
    not scrolled down yet
    \n\n \n
    \n\n\n" }, { "path": "tests/_Integration/_Server/BrowserActions/Wait.php", "content": "\n\n\n \n Hello World\n\n\n
    \n
    \n\n \n
    \n\n\n" }, { "path": "tests/_Integration/_Server/Crawling.php", "content": " /crawling/sub1\n * => /crawling/sub1/sub1\n * => /crawling/sub2\n * => /crawling/sub2/sub1\n * => /crawling/sub2/sub1/sub1\n */\n\nif ($route === '/crawling/sitemap.xml') {\n echo <<\n\nhttp://www.example.com/crawling/main\nhttp://www.example.com/crawling/sub1\nhttp://www.example.com/crawling/sub1/sub1\nhttp://www.example.com/crawling/sub2\nhttp://www.example.com/crawling/sub2/sub1\nhttp://www.example.com/crawling/sub2/sub1/sub1\n\nXML;\n}\n\nif ($route === '/crawling/sitemap2.xml') {\n echo <<\n\nhttp://www.example.com/crawling/main\nhttp://www.example.com/crawling/sub1\nhttp://www.example.com/crawling/sub1/sub1\nhttp://www.example.com/crawling/sub2\nhttp://www.example.com/crawling/sub2/sub1\nhttp://www.example.com/crawling/sub2/sub1/sub1\n\nXML;\n}\n\nif ($route === '/crawling' || $route === '/crawling/redirect') {\n header('Location: http://www.example.com/crawling/main?redirect=1', true, 301);\n\n return '';\n}\n\nif ($route === '/crawling/main' || $route === '/crawling/main?redirect=1') {\n $showRedirectLinkHtml = '';\n\n if (!empty($_GET['redirect'] ?? null)) {\n $showRedirectLinkHtml = PHP_EOL . 'link';\n }\n\n echo <<\n \n \n {$showRedirectLinkHtml}\n\n Subpage 1
    \n Subpage 2
    \n Subpage 2 - Fragment 1
    \n Subpage 2 - Fragment 2
    \n\n External link\n\n mailto link\n javascript link\n phone link\n\n broken link\n \n \n HTML;\n}\n\nif ($route === '/crawling/sub1') {\n echo <<\n \n \n foo\n \n \n \n \n Subpage 1 of Subpage 1
    \n\n External link\n\n Link to subdomain\n \n \n HTML;\n}\n\nif ($route === '/crawling/sub1/sub1') {\n echo <<\n \n \n

    Final level of sub1

    \n

    Subpage 1 of Subpage 1

    \n Back to main\n \n \n HTML;\n}\n\nif ($route === '/crawling/sub2') {\n echo <<\n \n \n Subpage 1 of Subpage 2\n \n \n HTML;\n}\n\nif ($route === '/crawling/sub2/sub1') {\n echo <<\n \n \n foo\n \n \n \n Subpage 1 of Subpage 1 of Subpage 2\n \n \n HTML;\n}\n\nif ($route === '/crawling/sub2/sub1/sub1') {\n echo <<\n \n \n

    Final level of sub2

    \n

    Subpage 1 of Subpage 1 of Subpage 2

    \n Back to Subpage 2\n \n \n HTML;\n}\n\nif ($route === '/crawling/main-on-subdomain') {\n echo <<\n \n \n

    Main page on subdomain

    \n \n \n HTML;\n}\n" }, { "path": "tests/_Integration/_Server/HelloWorld.php", "content": "\n\n\n Hello World!\n\n\nHello World!\n\n\n" }, { "path": "tests/_Integration/_Server/JsGeneratedContent.php", "content": "\n\n\n JS Generated Content\n\n\n
    \n
    \n\n\n\n" }, { "path": "tests/_Integration/_Server/NonUtf8.php", "content": "\n\n\n \n \n Non UTF-8 charset page\n\n\n
    \n\n
    \n\n\n" }, { "path": "tests/_Integration/_Server/PageInitScript.php", "content": "\n\n\n\n\n
    \n\n\n\n" }, { "path": "tests/_Integration/_Server/PaginatedListing/Detail.php", "content": "\n\n\n Paginated listing item detail\n\n\n
    \n

    Some Item

    \n \n
    \n\n\n" }, { "path": "tests/_Integration/_Server/PaginatedListing.php", "content": "\n\n\n Paginated Listing\n\n\n
    \n \n
    \n \">Item \n

    asdlfkj asdlfka jsdlfk ajsdflk

    \n
    \n \n \n
    \n \">Item \n

    asdflk jasdlfk asdlfk asldfk

    \n
    \n \n\n
    \n 1) { ?>\n \"><<\n \n\n \n \">>>\n \n
    \n
    \n\n\n" }, { "path": "tests/_Integration/_Server/PrintCookie.php", "content": " $value) {\n echo $key . '=' . $value . ($key !== $lastKey ? ';' : '');\n }\n}\n" }, { "path": "tests/_Integration/_Server/PrintHeaders.php", "content": "\n\n\n \n <?=$author?>\n\n\n

    \n \n

    \n\n
    \n
    \n \n
    \n
    \n \n
    \n
    \n \n \n \n \n \n \n \n \n
    \n
    \n\n\n" }, { "path": "tests/_Integration/_Server/Publisher/AuthorsListPage.php", "content": "\n\n\n \n Example Publishing Authors\n\n\n

    Our authors

    \n\n \n\n\n" }, { "path": "tests/_Integration/_Server/Publisher/BookDetailPage.php", "content": "\n\n\n \n Book\n\n\n

    \n \n

    \n\n
    \nFirst Edition ' .\n 'Second Edition';\n } elseif ($bookNo === 2) {\n // Another Novel\n echo 'First Edition ' .\n 'Second Edition ' .\n 'Third Edition';\n }\n?>\n
    \n\n\n" }, { "path": "tests/_Integration/_Server/Publisher/EditionDetailPage.php", "content": "\n\n\n \n Book Edition\n\n\n1996 Foo';\n } elseif ($edition === 2) {\n echo '2005 Foo';\n }\n } elseif ($bookNo === 2) {\n // Another Novel\n if ($edition === 1) {\n echo '2001 Foo';\n } elseif ($edition === 2) {\n echo '2009 Bar';\n } elseif ($edition === 3) {\n echo '2017 Bar';\n }\n } elseif ($bookNo === 3) {\n // Poems #1\n if ($edition === 1) {\n echo '2008 Poems';\n } elseif ($edition === 2) {\n echo '2009 Poems';\n }\n } elseif ($bookNo === 4) {\n // Poems #2\n if ($edition === 1) {\n echo '2011 Poems';\n } elseif ($edition === 2) {\n echo '2014 New Poems';\n }\n } elseif ($bookNo === 5) {\n // Poems #3\n if ($edition === 1) {\n echo '2013 Poems';\n } elseif ($edition === 2) {\n echo '2017 New Poems';\n }\n }\n?>\n\n\n" }, { "path": "tests/_Integration/_Server/QueryParamPagination.php", "content": "\n\n \n Example\n https://www.example.com\n Public RSS feed\n en\n \n \n <![CDATA[Foo, bar, baz]]>\n \n \n Wed, 08 Jan 2025 12:14:47 GMT\n Christian Olear\n \n \n \n \n\n" }, { "path": "tests/_Integration/_Server/ServiceUnavailable.php", "content": "\n\nyo\n\n
    {$cookies}
    \n\n\n\n" }, { "path": "tests/_Integration/_Server/SetDelayedCookieJs.php", "content": "\n\n\n Hey\n \n\n\n
    \n \n
    \n\n\n" }, { "path": "tests/_Integration/_Server/SetMultipleCookiesJs.php", "content": "\n\nyo\n\n\n\n\n" }, { "path": "tests/_Integration/_Server/SimpleListing/Detail.php", "content": "\n\n\n Simple listing article detail\n\n\n
    \n

    Some Article

    \n 2022-04-\n Christian Olear\n
    \n\n\n" }, { "path": "tests/_Integration/_Server/SimpleListing.php", "content": "\n\n\n Simple article listing\n\n\n
    \n
    \n Article 1\n

    asdfa sdlfka sdflkja sdflkj

    \n
    \n
    \n Article 2\n

    asldfkj aldfk jaslfk asdjflkajsdlf

    \n
    \n
    \n Article 3\n

    asldfk aslfdkjasd flkajsdfl kajsdflakjsdlf

    \n
    \n
    \n\n\n" }, { "path": "tests/_Integration/_Server/TooManyRequests.php", "content": "nextUrl);\n }\n\n /**\n * @return array\n */\n public function getLoaded(): array\n {\n return $this->loaded;\n }\n\n public function getLoadedCount(): int\n {\n return $this->loadedCount;\n }\n\n public function getLatestRequest(): ?RequestInterface\n {\n return $this->latestRequest;\n }\n\n public function limitReached(): bool\n {\n return $this->maxPagesReached();\n }\n\n public function setFinished(): AbstractPaginator\n {\n return parent::setFinished();\n }\n}\n" }, { "path": "tests/_Stubs/Crawlers/DummyOne.php", "content": "userAgentCalled += 1;\n\n return new DummyTwoUserAgent('FooBot');\n }\n\n /**\n * @return DummyTwoLogger\n */\n protected function logger(): LoggerInterface\n {\n $this->loggerCalled += 1;\n\n return new DummyTwoLogger();\n }\n\n /**\n * @return DummyTwoLoader\n */\n protected function loader(UserAgentInterface $userAgent, LoggerInterface $logger): LoaderInterface\n {\n $this->loaderCalled += 1;\n\n return new DummyTwoLoader($userAgent, null, $logger);\n }\n}\n" }, { "path": "tests/_Stubs/DummyLogger.php", "content": ">\n */\n public array $messages = [];\n\n public function emergency(string|Stringable $message, array $context = []): void\n {\n $this->log('emergency', $message, $context);\n }\n\n public function alert(string|Stringable $message, array $context = []): void\n {\n $this->log('alert', $message, $context);\n }\n\n public function critical(string|Stringable $message, array $context = []): void\n {\n $this->log('critical', $message, $context);\n }\n\n public function error(string|Stringable $message, array $context = []): void\n {\n $this->log('error', $message, $context);\n }\n\n public function warning(string|Stringable $message, array $context = []): void\n {\n $this->log('warning', $message, $context);\n }\n\n public function notice(string|Stringable $message, array $context = []): void\n {\n $this->log('notice', $message, $context);\n }\n\n public function info(string|Stringable $message, array $context = []): void\n {\n $this->log('info', $message, $context);\n }\n\n public function debug(string|Stringable $message, array $context = []): void\n {\n $this->log('debug', $message, $context);\n }\n\n /**\n * @param mixed $level\n * @param mixed[] $context\n */\n public function log($level, string|Stringable $message, array $context = []): void\n {\n if (!is_string($level)) {\n throw new InvalidArgumentException('Level must be string.');\n }\n\n if (!in_array($level, ['emergency', 'alert', 'critical', 'error', 'warning', 'notice', 'info', 'debug'], true)) {\n throw new UnexpectedValueException('Unknown log level.');\n }\n\n $this->messages[] = ['level' => $level, 'message' => $message];\n }\n}\n" }, { "path": "tests/_Stubs/PhantasyLoader.php", "content": "request, $respondedRequest->response);\n }\n\n public static function fromArray(array $data): RespondedRequestChild\n {\n $respondedRequest = parent::fromArray($data);\n\n return self::fromRespondedRequest($respondedRequest);\n }\n\n public function itseme(): string\n {\n return 'mario';\n }\n}\n" }, { "path": "tests/_Temp/_cachedir/.gitkeep", "content": "" }, { "path": "tests/_Temp/_storagedir/.gitkeep", "content": "" } ]