Showing preview only (774K chars total). Download the full file or copy to clipboard to get everything.
Repository: berstend/puppeteer-extra
Branch: master
Commit: 39248f1f5dee
Files: 267
Total size: 703.5 KB
Directory structure:
gitextract_u4zlbb78/
├── .editorconfig
├── .eslintrc
├── .gitattributes
├── .github/
│ ├── FUNDING.yml
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug.md
│ │ ├── config.yml
│ │ └── feature_request.md
│ ├── labeler.yml
│ └── workflows/
│ ├── extract-stealth.yml
│ ├── label.yml
│ └── test.yml
├── .gitignore
├── .prettierrc.js
├── .travis.yml
├── LICENSE
├── README.md
├── lerna.json
├── package.json
└── packages/
├── extract-stealth-evasions/
│ ├── .gitignore
│ ├── index.js
│ ├── package.json
│ └── readme.md
├── playwright-extra/
│ ├── .prettierrc.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── extra.ts
│ │ ├── helper/
│ │ │ └── loader.ts
│ │ ├── index.ts
│ │ ├── plugins.ts
│ │ ├── puppeteer-compatiblity-shim/
│ │ │ ├── index.ts
│ │ │ └── playwright-shim.d.ts
│ │ └── types/
│ │ └── index.ts
│ ├── test/
│ │ ├── exports.spec.ts
│ │ ├── fixtures/
│ │ │ ├── dummyplugin.ts
│ │ │ └── extra.ts
│ │ ├── playwright.config.ts
│ │ ├── plugin-events.spec.ts
│ │ └── puppeteer-plugins/
│ │ ├── anonymize-ua.spec.ts
│ │ ├── recaptcha.spec.ts
│ │ └── stealth.spec.ts
│ └── tsconfig.json
├── plugin-proxy-router/
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── index.ts
│ │ ├── plugin.ts
│ │ ├── router.ts
│ │ ├── stats.ts
│ │ └── utils/
│ │ └── port.ts
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra/
│ ├── ava.config-ts.js
│ ├── ava.config.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── ambient.d.ts
│ │ ├── index.ts
│ │ └── puppeteer-legacy.d.ts
│ ├── test/
│ │ ├── addExtra.ts
│ │ ├── basic.ts
│ │ ├── connect.js
│ │ ├── events.js
│ │ ├── options.js
│ │ └── plugin-support.js
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra-plugin/
│ ├── ava.config-ts.js
│ ├── ava.config.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── ambient.d.ts
│ │ ├── index.test.ts
│ │ ├── index.ts
│ │ └── puppeteer.ts
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra-plugin-adblocker/
│ ├── ava.config-ts.js
│ ├── ava.config.js
│ ├── build_version_check.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── ambient.d.ts
│ │ ├── index.test.ts
│ │ └── index.ts
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra-plugin-anonymize-ua/
│ ├── index.d.ts
│ ├── index.js
│ ├── index.test.js
│ ├── package.json
│ ├── readme.md
│ └── test/
│ ├── headless.js
│ ├── headless_off.js
│ ├── popup.js
│ └── stresstest.js
├── puppeteer-extra-plugin-block-resources/
│ ├── example.js
│ ├── index.d.ts
│ ├── index.js
│ ├── index.test.js
│ ├── package.json
│ └── readme.md
├── puppeteer-extra-plugin-click-and-wait/
│ ├── example.js
│ ├── index.js
│ ├── package.json
│ └── readme.md
├── puppeteer-extra-plugin-devtools/
│ ├── index.js
│ ├── index.test.js
│ ├── lib/
│ │ ├── RemoteDevTools.js
│ │ └── RemoteDevTools.test.js
│ ├── package.json
│ ├── readme.md
│ └── test/
│ └── headless.js
├── puppeteer-extra-plugin-flash/
│ ├── example.js
│ ├── index.js
│ ├── package.json
│ └── readme.md
├── puppeteer-extra-plugin-font-size/
│ ├── index.js
│ ├── package.json
│ └── readme.md
├── puppeteer-extra-plugin-recaptcha/
│ ├── ava.config-ts.js
│ ├── ava.config.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── ambient.d.ts
│ │ ├── content-hcaptcha.ts
│ │ ├── content.ts
│ │ ├── detection.test.ts
│ │ ├── index.test.ts
│ │ ├── index.ts
│ │ ├── playwright-mods.d.ts
│ │ ├── provider/
│ │ │ ├── 2captcha-api.ts
│ │ │ └── 2captcha.ts
│ │ ├── puppeteer-mods.d.ts
│ │ ├── solve.test.ts
│ │ └── types.ts
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra-plugin-repl/
│ ├── index.d.ts
│ ├── index.js
│ ├── index.test.js
│ ├── lib/
│ │ ├── REPLSession.js
│ │ ├── REPLSession.test.js
│ │ ├── super-readline.js
│ │ └── super-readline.test.js
│ ├── package.json
│ ├── readme.md
│ └── test/
│ └── headless.js
├── puppeteer-extra-plugin-stealth/
│ ├── .npmignore
│ ├── evasions/
│ │ ├── _template/
│ │ │ ├── index.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── _utils/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── readme.md
│ │ │ └── withUtils.js
│ │ ├── chrome.app/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── chrome.csi/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── chrome.loadTimes/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── chrome.runtime/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ ├── readme.md
│ │ │ └── staticData.json
│ │ ├── defaultArgs/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── iframe.contentWindow/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── media.codecs/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.hardwareConcurrency/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.languages/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.permissions/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.plugins/
│ │ │ ├── data.json
│ │ │ ├── functionMocks.js
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── magicArray.js
│ │ │ ├── mimeTypes.js
│ │ │ ├── mimeTypes.test.js
│ │ │ ├── package.json
│ │ │ ├── plugins.js
│ │ │ ├── plugins.test.js
│ │ │ └── readme.md
│ │ ├── navigator.vendor/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.webdriver/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── readme.md
│ │ ├── sourceurl/
│ │ │ ├── _fixtures/
│ │ │ │ └── test.html
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── user-agent-override/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── webgl.vendor/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ └── window.outerdimensions/
│ │ ├── index.js
│ │ ├── package.json
│ │ └── readme.md
│ ├── examples/
│ │ ├── detect-headless.js
│ │ ├── test1.js
│ │ └── test2.js
│ ├── index.d.ts
│ ├── index.js
│ ├── index.test.js
│ ├── package.json
│ ├── readme.md
│ ├── runall_stealthtests.sh
│ ├── stealthtests/
│ │ ├── headful-chrome-stealth.js
│ │ ├── headful-chrome-vanilla.js
│ │ ├── headful-chromium-stealth.js
│ │ ├── headful-chromium-vanilla.js
│ │ ├── headless-chrome-stealth.js
│ │ ├── headless-chrome-vanilla.js
│ │ ├── headless-chromium-stealth.js
│ │ └── headless-chromium-vanilla.js
│ └── test/
│ ├── cat-and-mouse.test.js
│ ├── fixtures/
│ │ ├── dummy-with-service-worker.html
│ │ ├── dummy.html
│ │ └── sw.js
│ ├── fpscanner.test.js
│ ├── service-worker.test.js
│ └── util.js
├── puppeteer-extra-plugin-user-data-dir/
│ ├── index.js
│ ├── package.json
│ └── readme.md
└── puppeteer-extra-plugin-user-preferences/
├── index.js
├── package.json
└── readme.md
================================================
FILE CONTENTS
================================================
================================================
FILE: .editorconfig
================================================
# EditorConfig helps developers define and maintain
# consistent coding styles between different editors and IDEs.
root = true
[*]
charset = utf-8
indent_style = space
indent_size = 2
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[*.md]
trim_trailing_whitespace = false
================================================
FILE: .eslintrc
================================================
{
"extends": ["prettier-standard"],
"rules": {
"lines-between-class-members": "off"
}
}
================================================
FILE: .gitattributes
================================================
* text=auto eol=lf
================================================
FILE: .github/FUNDING.yml
================================================
custom: https://www.buymeacoffee.com/brstnd
================================================
FILE: .github/ISSUE_TEMPLATE/bug.md
================================================
---
name: Bug report
about: Create a bug report
title: '[Bug] '
labels: 'issue: bug report, needs triage'
---
<!--
Thanks for filing an issue!
Kindly search the issue tracker before posting a new issue.
Due to bad experiences we have to unfortunately mention it:
- Be respectful with our time, this is an open-source project run by volunteers in their free time.
- Don't come across as entitled, unfriendly or angry or nobody will help you (general rule in life).
- Reporting a "bug" without ways to reproduce it is not "contributing to open-source".
- NOTE: "SiteA.com stopped working" is not a valid stealth-plugin bug report, unless used
as an example to show a technical issue while providing specific example code of the problem.
For general purpose usage discussions please use the community discord: https://extra.community
-->
**Describe the bug**
<!--
- What you were trying to accomplish when the bug occurred
- A description of what you expected and what actually happened
- How to reproduce the issue
-->
**Code Snippet**
<!--
Help us help you! Put down a short code snippet that illustrates your bug and
that we can run and debug locally. If possible remove everything
that is not related and make it as short as possible. For example:
-->
```javascript
const puppeteer = require('puppeteer-extra')
;(async () => {
const browser = await puppeteer.launch()
// ...
})()
```
**Versions**
<!--
Run the following command in your project directory, and paste its results here:
npx envinfo@latest --system --binaries --npmPackages '*(puppeteer*|playwright*|automation-extra*|@extra*)'
-->
================================================
FILE: .github/ISSUE_TEMPLATE/config.yml
================================================
contact_links:
- name: Questions and Help
url: https://github.com/berstend/puppeteer-extra/wiki/Scraping-Chat
about: This issue tracker is not for support questions. You can join our Discord server and ask the community for help.
================================================
FILE: .github/ISSUE_TEMPLATE/feature_request.md
================================================
---
name: Feature request
about: Suggest an idea or feature
title: '[Feature] '
labels: 'issue: proposal'
---
**Feature request**
<!--
Let us know what functionality you'd like to see and what is your use case.
Do you think others might benefit from this as well?
Please provide details about:
- What you're trying to do
- Why you can't use the project for this
- And maybe how you think the project could handle this
-->
================================================
FILE: .github/labeler.yml
================================================
# This is used with the label workflow which
# will triage pull requests and apply a label based on the
# paths that are modified in the pull request.
#
# For more information, see:
# https://github.com/actions/labeler
'package: core':
- packages/automation-extra/**/*
- packages/playright-extra/**/*
- packages/puppeteer-extra/**/*
- packages/automation-extra-plugin/**/*
- packages/puppeteer-extra-plugin/**/*
'plugin: automation-extra':
- packages/plugin-*/**/*
'plugin: puppeteer-extra':
- packages/puppeteer-extra-plugin-*/**/*
'plugin: recaptcha 🏴':
- packages/*recaptcha*/**/*
'plugin: stealth ㊙️':
- packages/*stealth*/**/*
================================================
FILE: .github/workflows/extract-stealth.yml
================================================
name: Extract stealth.min.js
on:
push:
branches:
- master
paths:
- 'packages/puppeteer-extra-plugin-stealth/**'
- 'packages/extract-stealth-evasions/**'
- '.github/workflows/extract-stealth.yml'
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Sleep for 190 seconds
uses: jakejarvis/wait-action@master
with:
time: '190s'
- name: Checkout
uses: actions/checkout@v2
- name: 'Fix for: error fsevents@2.1.2: The platform "linux" is incompatible with this module.'
run: npx json -I -f package.json -e 'this.resolutions={}'
- name: Build packages
run: |
yarn install
yarn bootstrap
yarn build
- name: Extract stealth.min.js
run: |
cd packages/extract-stealth-evasions
node index.js
cp stealth.min.js ../../
- name: Commit stealth.min.js
uses: EndBug/add-and-commit@v4
with:
add: 'stealth.min.js'
force: true
ref: 'stealth-js'
message: 'Auto-updated stealth.min.js with newest evasions'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .github/workflows/label.yml
================================================
# This workflow will triage pull requests and apply a label based on the
# paths that are modified in the pull request.
#
# To use this workflow, you will need to set up a .github/labeler.yml
# file with configuration. For more information, see:
# https://github.com/actions/labeler
name: "Pull Request Labeler"
on:
- pull_request_target
jobs:
triage:
runs-on: ubuntu-latest
steps:
- uses: actions/labeler@main
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
sync-labels: true
================================================
FILE: .github/workflows/test.yml
================================================
name: Test
on:
pull_request:
branches:
- '*'
types:
- opened
- synchronize
- reopened
push:
branches:
- master
- 'test/*'
workflow_dispatch:
branches:
- '*'
env:
CI: 'true'
FORCE_COLOR: 'true'
jobs:
test:
name: node v${{ matrix.node }}, pptr ${{ matrix.puppeteer_version }}, ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
node:
# - 16
- 14
puppeteer_version:
# - 15.5.0
- 14.2.0 # Chromium 103.0.5059.0 # requires >=14.1.0
# - 10.2.0 # Chromium 93.0.4577.0
# - 7.0.0 # Chromium 90.0.4403.0, Feb 3, 2021
# - 5.5.0 # Chromium 88.0.4298.0
# - 5.0.0 # Chromium 83.0.4103.0, Jul 2, 2020
# - 2.1.1 # Chromium 79.0.3942.0, Oct 24 2019
# - 2.0.0 # Chromium 79.0.3942.0, Oct 24 2019
# - 1.20.0 # Chromium 78.0.3882.0, Sep 13 2019
# - 1.15.0 # Chromium 75.0.3765.0, Apr 26 2019
# - 1.9.0 # Chromium 71.0.3563.0, Oct 4, 2018
# - 1.6.2 # Chromium 69.0.3494.0, Aug 1, 2018
os:
- ubuntu-latest
# - macOS-latest
# - windows-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Node
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node }}
- name: 'Fix for: error fsevents@2.1.2: The platform "linux" is incompatible with this module.'
run: npx json -I -f package.json -e 'this.resolutions={}'
- name: yarn install
uses: bahmutov/npm-install@v1
- name: yarn bootstrap
run: yarn bootstrap
- name: install puppeteer@${{ matrix.puppeteer_version }}
run: yarn lerna add --dev puppeteer@${{ matrix.puppeteer_version }}
- name: lerna link
run: yarn lerna link
- name: lerna build
run: yarn lerna run build --concurrency 1
- name: debug
run: |
yarn list --pattern "puppeteer|puppeteer-extra|playwright"
file node_modules/puppeteer-extra/dist/index.cjs.js
- uses: microsoft/playwright-github-action@v1
- name: playwright install
run: cd packages/playwright-extra && yarn playwright install
- name: test
uses: GabrielBB/xvfb-action@v1
env:
DISPLAY: ':99.0'
with:
run: yarn test-ci
================================================
FILE: .gitignore
================================================
# See https://help.github.com/ignore-files/ for more about ignoring files.
# dependencies
node_modules
# builds
build
dist
# misc
.DS_Store
.env
.env.local
.env.development.local
.env.test.local
.env.production.local
.cache
.rpt2_cache
lerna-debug.log*
lerna-error.log*
npm-debug.log*
yarn-debug.log*
yarn-error.log*
TODO.md
packages/testing/
packages/plugin-stealth/
packages/puppeteer-extra2/
packages/puppeteer-extra-old/
packages/test-*
packages/internal-testing-*
testing/
*.tgz*
================================================
FILE: .prettierrc.js
================================================
module.exports = {
...require('prettier-config-standard'),
// override for Windows
endOfLine: 'lf',
}
================================================
FILE: .travis.yml
================================================
language: node_js
dist: trusty
addons:
apt:
packages:
# This is required to run new chrome on old trusty
- libnss3
language: node_js
# allow headful tests
before_install:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
# test against multiple node versions
node_js:
- '13'
- '10'
# Fix for: error fsevents@2.1.2: The platform "linux" is incompatible with this module.
install: skip
# Prevent potential issues
cache:
npm: false
yarn: false
# test against multiple puppeteer versions
env:
- PUPPETEER_VERSION=5.0.0
- PUPPETEER_VERSION=2.1.1 # Chromium 79.0.3942.0, Oct 24 2019
# - PUPPETEER_VERSION=2.0.0 # Chromium 79.0.3942.0, Oct 24 2019
# - PUPPETEER_VERSION=1.20.0 # Chromium 78.0.3882.0, Sep 13 2019
# - PUPPETEER_VERSION=1.15.0 # Chromium 75.0.3765.0, Apr 26 2019
# - PUPPETEER_VERSION=1.9.0 # Chromium 71.0.3563.0, Oct 4, 2018
# - PUPPETEER_VERSION=1.6.2 # Chromium 69.0.3494.0, Aug 1, 2018
script:
# Make sure to use latest @next package
# https://github.com/yarnpkg/yarn/issues/4731
# Fix for: error fsevents@2.1.2: The platform "linux" is incompatible with this module.
- npx json -I -f package.json -e 'this.resolutions={}'
# - npx json -I -f package.json -e 'this.resolutions={"**/puppeteer":"'${PUPPETEER_VERSION}'"}'
# Install older version when required
- rm -rf ./node_modules/
# - 'yarn; echo 0'
# - 'yarn lerna exec "rm -f yarn.lock; rm -rf node_modules; echo 0"'
# - 'rm -rf yarn.lock && yarn cache clean && rm -rf ./node_modules/puppeteer && yarn lerna add puppeteer@${PUPPETEER_VERSION}'
# - "yarn lerna exec --concurrency 1 'yarn set resolution --save puppeteer@* 5.0.0; echo 0'"
- 'yarn'
- 'yarn bootstrap'
- yarn lerna add puppeteer@${PUPPETEER_VERSION}
- 'yarn lerna link'
- 'yarn lerna run build --concurrency 1'
# For debugging
- yarn list puppeteer
- yarn list puppeteer-extra
- file node_modules/puppeteer-extra/dist/index.cjs.js
# Run tests
- yarn test-ci
================================================
FILE: LICENSE
================================================
The MIT License (MIT)
Copyright (c) 2019 berstend <github@berstend.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# puppeteer-extra [](https://github.com/berstend/puppeteer-extra/)
This is the monorepo for [`puppeteer-extra`](./packages/puppeteer-extra), a modular plugin framework for [`puppeteer`](https://github.com/puppeteer/puppeteer). :-)
🌟 **For the main documentation, please head over to the [`puppeteer-extra`](./packages/puppeteer-extra) package.**
We've also recently introduced support for Playwright, if you're interested in that head over to [`playwright-extra`](./packages/playwright-extra).
## Monorepo
<details>
<summary><strong>Contributing</strong></summary>
### Contributing
PRs and new plugins are welcome! The plugin API for `puppeteer-extra` is clean and fun to use. Have a look the [`PuppeteerExtraPlugin`](./packages/puppeteer-extra-plugin) base class documentation to get going and check out the [existing plugins](./packages/) (minimal example is the [anonymize-ua](./packages/puppeteer-extra-plugin-anonymize-ua/index.js) plugin) for reference.
We use a [monorepo](https://github.com/berstend/puppeteer-extra) powered by [Lerna](https://github.com/lerna/lerna#--use-workspaces) (and yarn workspaces), [ava](https://github.com/avajs/ava) for testing, the [standard](https://standardjs.com/) style for linting and [JSDoc](http://usejsdoc.org/about-getting-started.html) heavily to auto-generate markdown [documentation](https://github.com/documentationjs/documentation) based on code. :-)
</details>
<details>
<summary><strong>Lerna</strong></summary>
### Lerna
This monorepo is powered by [Lerna](https://github.com/lerna/lerna) and yarn workspaces.
#### Initial setup
```bash
# Install deps
yarn
# Bootstrap the packages in the current Lerna repo.
# Installs all of their dependencies and links any cross-dependencies.
yarn bootstrap
# Build all TypeScript sources
yarn build
```
#### Development flow
```bash
# Install debug in all packages
yarn lerna add debug
# Install fs-extra to puppeteer-extra-plugin-user-data-dir
yarn lerna add fs-extra --scope=puppeteer-extra-plugin-user-data-dir
# Remove dependency
# https://github.com/lerna/lerna/issues/833
yarn lerna exec --concurrency 1 'yarn remove fs-extra; echo 0'
# Run test in all packages
yarn test
# Update JSDoc based documentation in markdown files
yarn docs
# Upgrade project wide deps like puppeteer
# (We keep the devDependency version blurry)
rm -rf node_modules
rm -rf yarn.lock
yarn
yarn lerna bootstrap
# Update deps within packages (interactive)
yarn lernaupdate
# If in doubt :-(
yarn lerna exec "rm -f yarn.lock; rm -rf node_modules; echo 0"
rm -f yarn.lock && rm -rf node_modules && yarn cache clean
# Run tests of specific package
cd packages/puppeteer-extra-plugin-stealth
yarn test
# Run tests of specific stealth evasion
cd packages/puppeteer-extra-plugin-stealth
yarn ava -v ./evasions/user-agent-override/index.test.js
# Test a local monorepo package in an outside folder as it would've been installed from the registry
# Change PACKAGE_DIR to the path of this monorepo and PACKAGE to the package you wish to install
PACKAGE=puppeteer-extra PACKAGE_DIR=/Users/foo/puppeteer-extra/packages && yarn remove $(echo $PACKAGE); true && rm -f $(pwd)/$(echo $PACKAGE)-latest.tgz && yarn --cwd $(echo $PACKAGE_DIR)/$(echo $PACKAGE) pack --filename $(pwd)/$(echo $PACKAGE)-latest.tgz && YARN_CACHE_FOLDER=/tmp/yarn yarn add file:$(pwd)/$(echo $PACKAGE)-latest.tgz && rm -rf /tmp/yarn
```
#### Publishing
```bash
# make sure you're signed into npm before publishing
# yarn publishing is broken so lerna uses npm
npm whoami
# ensure everything is up2date and peachy
yarn
yarn bootstrap
yarn lerna link
yarn build
yarn test
# Phew, let's publish these packages!
# - Will publish all changed packages
# - Will ask for new pkg version per package
# - Will updated inter-package dependency versions automatically
yarn lerna publish
# Fix new dependency version symlinks
yarn bootstrap && yarn lerna link
```
</details>
<br>
<p align="center">
<img src="https://i.imgur.com/EuqiF5F.png" height="240" />
</p>
================================================
FILE: lerna.json
================================================
{
"packages": ["packages/*"],
"version": "independent",
"useWorkspaces": true,
"npmClient": "yarn"
}
================================================
FILE: package.json
================================================
{
"private": true,
"description": "Modular framework to teach Puppeteer new tricks.",
"repository": "berstend/puppeteer-extra",
"author": "berstend",
"license": "MIT",
"main": "packages/puppeteer-extra/index.js",
"engines": {
"node": ">=8"
},
"scripts": {
"bootstrap": "lerna bootstrap",
"build": "yarn lerna exec --concurrency 1 'yarn build; echo 0'",
"docs": "lerna run docs",
"test": "lerna run test --concurrency 1 --stream",
"test-ci": "lerna run test-ci --concurrency 1 --stream",
"prepare": "lerna run prepare",
"release": "lerna publish --npm-client npm"
},
"workspaces": {
"packages": [
"packages/*"
],
"nohoist": [
"**/@types",
"**/@types/**",
"**/typescript",
"**/typescript/**"
]
},
"devDependencies": {
"eslint": "^6.7.1",
"eslint-config-prettier": "^6.7.0",
"eslint-config-prettier-standard": "^3.0.1",
"eslint-config-standard": "^14.1.0",
"eslint-plugin-import": "^2.18.2",
"eslint-plugin-node": "^10.0.0",
"eslint-plugin-prettier": "^3.1.1",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^4.0.1",
"lerna": "^3.19.0",
"lerna-update-wizard": "^0.17.5",
"prettier": "^1.19.1",
"prettier-config-standard": "^1.0.1"
},
"optionalDependencies": {
"fsevents": "^2.1.2"
},
"resolutions": {
"**/fsevents": "^2.1.2"
},
"dependencies": {},
"version": "0.0.0"
}
================================================
FILE: packages/extract-stealth-evasions/.gitignore
================================================
stealth.min.js
stealth.js
================================================
FILE: packages/extract-stealth-evasions/index.js
================================================
#!/usr/bin/env node
const puppeteer = require('puppeteer-extra')
const stealth = require('puppeteer-extra-plugin-stealth')()
const { minify } = require('terser')
const argv = require('yargs')
.usage('Usage: $0 [options]')
.alias('e', 'exclude')
.describe('e', 'Exclude evasion (repeat for multiple)')
.alias('i', 'include')
.describe('i', 'Include evasion (repeat for multiple)')
.alias('l', 'list')
.describe('l', 'List available evasions')
.alias('m', 'minify')
.describe('minify', 'Minify the output')
.boolean('m')
.default('m', true)
.help('h')
.alias('h', 'help').argv
const fs = require('fs')
const file = 'stealth' + (argv.minify === true ? '.min' : '') + '.js'
if (argv.exclude) {
if (typeof argv.exclude === 'string') {
stealth.enabledEvasions.delete(argv.exclude)
} else {
argv.exclude.forEach(e => {
stealth.enabledEvasions.delete(e)
})
}
} else if (argv.include) {
if (typeof argv.include === 'string') {
stealth.enabledEvasions = [argv.include]
} else {
stealth.enabledEvasions = []
argv.include.forEach(e => {
stealth.enabledEvasions.push(e)
})
}
} else if (argv.list) {
console.log('Available evasions:', [...stealth.availableEvasions].join(', '))
process.exit(0)
}
let scripts = ''
puppeteer
.use(stealth)
.launch({
headless: true
})
.then(async browser => {
// Patch evaluateOnNewDocument()
const page = (await browser.pages()).find(Boolean)
page.__proto__.evaluateOnNewDocument = patchEval // eslint-disable-line no-proto
page.__proto__.evaluate = patchEval // eslint-disable-line no-proto
await (await browser.newPage()).goto('about:blank')
await browser.close()
fs.writeFile(
file,
`/*!
* Note: Auto-generated, do not update manually.
* Generated by: https://github.com/berstend/puppeteer-extra/tree/master/packages/extract-stealth-evasions
* Generated on: ${new Date().toUTCString()}
* License: MIT
*/
` +
(argv.minify === true
? (await minify(scripts, { toplevel: true })).code
: scripts),
err => {
if (err) throw err
console.log(`File ${file} written!`)
console.log(
'Included evasions: ',
[...stealth.enabledEvasions].join(', ')
)
}
)
})
function patchEval(f, args) {
// Check if there are options supplied
if (typeof args !== 'undefined') {
scripts += '(' + f.toString() + ')(' + JSON.stringify(args) + ');\n'
} else {
scripts += '(' + f.toString() + ')();\n'
}
}
================================================
FILE: packages/extract-stealth-evasions/package.json
================================================
{
"name": "extract-stealth-evasions",
"version": "2.7.3",
"description": "Extract stealth evasions from puppeteer-extra-plugin-stealth",
"main": "index.js",
"bin": {
"extract-stealth-evasions": "./index.js"
},
"homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/extract-stealth-evasions#readme",
"repository": "berstend/puppeteer-extra",
"author": "berstend",
"license": "MIT",
"engines": {
"node": ">=8"
},
"keywords": [
"puppeteer",
"puppeteer-extra",
"puppeteer-extra-plugin",
"chrome",
"headless",
"pupeteer"
],
"dependencies": {
"puppeteer": "*",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"terser": "^5.1.0",
"yargs": "^15.4.1"
}
}
================================================
FILE: packages/extract-stealth-evasions/readme.md
================================================
# extract-stealth-evasions
This script offers a quick way to extract the latest stealth evasions from [puppeteer-extra-stealth](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth) to (minified) JavaScript. The resulting JS file can be used in pure [CDP](https://chromedevtools.github.io/devtools-protocol/tot/) implementations or to test the evasions in your devtools.
#### Usage with `npx`
You don't need to install anything, `npx` runs wherever NodeJS is installed. :-)
```bash
npx extract-stealth-evasions
```
Will create a `stealth.min.js` file in the current folder.
#### Using the CDN version
You can also fetch the latest version from [gitCDN](https://gitcdn.xyz/repo/berstend/puppeteer-extra/stealth-js/stealth.min.js). For example, paste this one-liner in your browser devtools console:
```js
document.body.appendChild(Object.assign(document.createElement('script'), {src: 'https://gitcdn.xyz/repo/berstend/puppeteer-extra/stealth-js/stealth.min.js'}))
```
#### How to use locally
```bash
yarn install
node index.js
```
Use the resulting `stealth.min.js` file however you like.
#### Options
```bash
$ npx extract-stealth-evasions -h
Usage: extract-stealth-evasions [options]
Options:
--version Show version number [boolean]
-e, --exclude Exclude evasion (repeat for multiple)
-i, --include Include evasion (repeat for multiple)
-l, --list List available evasions
-h, --help Show help [boolean]
-m, --minify Minify the output [boolean] [default: true]
```
================================================
FILE: packages/playwright-extra/.prettierrc.js
================================================
module.exports = 'prettier-config-standard'
================================================
FILE: packages/playwright-extra/package.json
================================================
{
"name": "playwright-extra",
"version": "4.3.6",
"description": "Teach playwright new tricks through plugins.",
"repository": "berstend/puppeteer-extra",
"homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/playwright-extra#readme",
"author": "berstend",
"license": "MIT",
"typings": "dist/index.d.ts",
"main": "dist/index.cjs.js",
"module": "dist/index.esm.js",
"files": [
"dist"
],
"scripts": {
"clean": "rimraf dist/*",
"prebuild": "run-s clean",
"build": "run-s build:tsc build:rollup ambient-dts",
"build:tsc": "tsc --module commonjs",
"build:rollup": "rollup -c rollup.config.ts",
"docs": "echo \"No docs\"",
"test": "yarn playwright test --config test/playwright.config.ts",
"test-ci": "run-s test",
"ambient-dts": "run-s ambient-dts-copy ambient-dts-fix-path",
"ambient-dts-copy": "copyfiles -u 1 \"src/**/*.d.ts\" dist",
"ambient-dts-fix-path": "replace-in-files --string='/// <reference path=\"../src/' --replacement='/// <reference path=\"../dist/' 'dist/**/*.d.ts'"
},
"keywords": [
"playwright",
"playwright-extra",
"stealth",
"recaptcha",
"user-preferences",
"chrome",
"headless",
"pupeteer"
],
"engines": {
"node": ">=12"
},
"devDependencies": {
"@playwright/test": "^1.23.1",
"@types/debug": "^4.1.7",
"@types/node": "^18.0.0",
"esbuild": "^0.14.47",
"esbuild-register": "^3.3.3",
"npm-run-all": "^4.1.5",
"playwright": "1.24.2",
"prettier": "^2.7.1",
"puppeteer-extra-plugin": "^3.2.3",
"puppeteer-extra-plugin-anonymize-ua": "^2.4.5",
"rimraf": "^3.0.0",
"rollup": "^1.27.5",
"rollup-plugin-commonjs": "^10.1.0",
"rollup-plugin-node-resolve": "^5.2.0",
"rollup-plugin-sourcemaps": "^0.4.2",
"rollup-plugin-typescript2": "^0.25.2",
"typescript": "4.4.3"
},
"dependencies": {
"debug": "^4.3.4"
},
"peerDependencies": {
"playwright": "*",
"playwright-core": "*"
},
"peerDependenciesMeta": {
"playwright": {
"optional": true
},
"playwright-core": {
"optional": true
}
}
}
================================================
FILE: packages/playwright-extra/readme.md
================================================
# playwright-extra [](https://github.com/berstend/puppeteer-extra/actions) [](https://extra.community) [](https://www.npmjs.com/package/playwright-extra)
> A modular plugin framework for [playwright](https://github.com/microsoft/playwright) to enable cool [plugins](#plugins) through a clean interface.
## Installation
```bash
yarn add playwright playwright-extra
# - or -
npm install playwright playwright-extra
```
<details>
<summary>Changelog</summary>
> Please check the `announcements` channel in our [discord server](https://extra.community) until we've automated readme updates. :)
- **v4.3**
- Rerelease due to versioning issues with previous beta packages
- **v3.3**
- Initial public release
</details>
## Quickstart
```js
// playwright-extra is a drop-in replacement for playwright,
// it augments the installed playwright with plugin functionality
const { chromium } = require('playwright-extra')
// Load the stealth plugin and use defaults (all tricks to hide playwright usage)
// Note: playwright-extra is compatible with most puppeteer-extra plugins
const stealth = require('puppeteer-extra-plugin-stealth')()
// Add the plugin to playwright (any number of plugins can be added)
chromium.use(stealth)
// That's it, the rest is playwright usage as normal 😊
chromium.launch({ headless: true }).then(async browser => {
const page = await browser.newPage()
console.log('Testing the stealth plugin..')
await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle' })
await page.screenshot({ path: 'stealth.png', fullPage: true })
console.log('All done, check the screenshot. ✨')
await browser.close()
})
```
The above example uses the compatible [`stealth`](/packages/puppeteer-extra-plugin-stealth) plugin from puppeteer-extra, that plugin needs to be installed as well:
```bash
yarn add puppeteer-extra-plugin-stealth
# - or -
npm install puppeteer-extra-plugin-stealth
```
If you'd like to see debug output just run your script like so:
```bash
# macOS/Linux (Bash)
DEBUG=playwright-extra*,puppeteer-extra* node myscript.js
# Windows (Powershell)
$env:DEBUG='playwright-extra*,puppeteer-extra*';node myscript.js
```
### More examples
<details>
<summary><strong>TypeScript & ESM usage</strong></summary><br/>
`playwright-extra` and most plugins are written in TS, so you get perfect type support out of the box. :)
```ts
// playwright-extra is a drop-in replacement for playwright,
// it augments the installed playwright with plugin functionality
import { chromium } from 'playwright-extra'
// Load the stealth plugin and use defaults (all tricks to hide playwright usage)
// Note: playwright-extra is compatible with most puppeteer-extra plugins
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
// Add the plugin to playwright (any number of plugins can be added)
chromium.use(StealthPlugin())
// ...(the rest of the quickstart code example is the same)
chromium.launch({ headless: true }).then(async browser => {
const page = await browser.newPage()
console.log('Testing the stealth plugin..')
await page.goto('https://bot.sannysoft.com', { waitUntil: 'networkidle' })
await page.screenshot({ path: 'stealth.png', fullPage: true })
console.log('All done, check the screenshot. ✨')
await browser.close()
})
```
New to Typescript? Here it is in 30 seconds or less 😄:
```bash
# Optional: If you don't have yarn yet
npm i --global yarn
# Optional: Create new package.json if it's a new project
yarn init -y
# Add basic typescript dependencies
yarn add --dev typescript @types/node esbuild esbuild-register
# Bootstrap a tsconfig.json
yarn tsc --init --target ES2020 --lib ES2020 --module commonjs --rootDir src --outDir dist
# Add dependencies used in the quick start example
yarn add playwright playwright-extra puppeteer-extra-plugin-stealth
# Create source folder for the .ts files
mkdir src
# Now place the example code above in `src/index.ts`
# Run the typescript code without the need of compiling it first
node -r esbuild-register src/index.ts
# You can now add Typescript to your CV 🎉
```
</details>
<details>
<summary><strong>Using different browsers</strong></summary><br/>
```ts
// Any browser supported by playwright can be used with plugins
import { chromium, firefox, webkit } from 'playwright-extra'
chromium.use(plugin)
firefox.use(plugin)
webkit.use(plugin)
```
</details>
<details>
<summary><strong>Multiple instances with different plugins</strong></summary><br/>
Node.js imports are cached, therefore the default `chromium`, `firefox`, `webkit` export from `playwright-extra` will always return the same playwright instance.
```ts
// Use `addExtra` to create a fresh and independent instance
import playwright from 'playwright'
import { addExtra } from 'playwright-extra'
const chromium1 = addExtra(playwright.chromium)
const chromium2 = addExtra(playwright.chromium)
chromium1.use(onePlugin)
chromium2.use(anotherPlugin)
// chromium1 and chromium2 are independent
```
</details>
---
## Plugins
We're currently in the process of making the existing [puppeteer-extra](/packages/puppeteer-extra) plugins compatible with playwright-extra, the following plugins have been successfully tested already:
### 🔥 [`puppeteer-extra-plugin-stealth`](/packages/puppeteer-extra-plugin-stealth)
- Applies various evasion techniques to make detection of an automated browser harder
- Compatible with Puppeteer & Playwright and chromium based browsers
<details>
<summary> Example: Using stealth in Playwright with custom options</summary>
```js
// The stealth plugin is optimized for chromium based browsers currently
import { chromium } from 'playwright-extra'
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
chromium.use(StealthPlugin())
// New way to overwrite the default options of stealth evasion plugins
// https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth/evasions
chromium.plugins.setDependencyDefaults('stealth/evasions/webgl.vendor', {
vendor: 'Bob',
renderer: 'Alice'
})
// That's it, the rest is playwright usage as normal 😊
chromium.launch({ headless: true }).then(async browser => {
const page = await browser.newPage()
console.log('Testing the webgl spoofing feature of the stealth plugin..')
await page.goto('https://webglreport.com', { waitUntil: 'networkidle' })
await page.screenshot({ path: 'webgl.png', fullPage: true })
console.log('All done, check the screenshot. ✨')
await browser.close()
})
```
</details>
### 🏴 [`puppeteer-extra-plugin-recaptcha`](/packages/puppeteer-extra-plugin-recaptcha)
- Solves reCAPTCHAs and hCaptchas automatically, using a single line of code: `page.solveRecaptchas()`
- Compatible with Puppeteer & Playwright and all browsers (chromium, firefox, webkit)
<details>
<summary> Example: Solving captchas in Playwright & Firefox</summary>
```js
// Any browser (chromium, webkit, firefox) can be used
import { firefox } from 'playwright-extra'
import RecaptchaPlugin from 'puppeteer-extra-plugin-recaptcha'
firefox.use(
RecaptchaPlugin({
provider: {
id: '2captcha',
token: process.env.TWOCAPTCHA_TOKEN || 'YOUR_API_KEY'
}
})
)
// Works in headless as well, just so you can see it in action
firefox.launch({ headless: false }).then(async browser => {
const context = await browser.newContext()
const page = await context.newPage()
const url = 'https://www.google.com/recaptcha/api2/demo'
await page.goto(url, { waitUntil: 'networkidle' })
console.log('Solving captchas..')
await page.solveRecaptchas()
await Promise.all([
page.waitForNavigation({ waitUntil: 'networkidle' }),
page.click(`#recaptcha-demo-submit`)
])
const content = await page.content()
const isSuccess = content.includes('Verification Success')
console.log('Done', { isSuccess })
await browser.close()
})
```
</details>
### 🆕 [`plugin-proxy-router`](/packages/plugin-proxy-router)
- Use multiple proxies dynamically with flexible per-host routing and more
- Compatible with Puppeteer & Playwright and all browsers (chromium, firefox, webkit)
**Notes**
- If you're in need of adblocking use [this package](https://www.npmjs.com/package/@cliqz/adblocker-playwright) or [block resources natively](https://github.com/berstend/puppeteer-extra/wiki/Block-resources-without-request-interception)
- We're focussing on compatiblity with existing plugins at the moment, more documentation on how to write your own playwright-extra plugins will follow
---
## Contributors
<a href="https://github.com/berstend/puppeteer-extra/graphs/contributors">
<img src="https://contributors-img.firebaseapp.com/image?repo=berstend/puppeteer-extra" />
</a>
---
## License
Copyright © 2018 - 2023, [berstend̡̲̫̹̠̖͚͓̔̄̓̐̄͛̀͘](https://github.com/berstend). Released under the MIT License.
<!--
Reference links
-->
[playwright-extra]: https://github.com/berstend/puppeteer-extra/tree/master/packages/playwright-extra
[puppeteer-extra]: https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra
[`puppeteer-extra`]: https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra
================================================
FILE: packages/playwright-extra/rollup.config.ts
================================================
import commonjs from 'rollup-plugin-commonjs'
import resolve from 'rollup-plugin-node-resolve'
import sourceMaps from 'rollup-plugin-sourcemaps'
import typescript from 'rollup-plugin-typescript2'
const pkg = require('./package.json')
const entryFile = 'index'
const banner = `
/*!
* ${pkg.name} v${pkg.version} by ${pkg.author}
* ${pkg.homepage || `https://github.com/${pkg.repository}`}
* @license ${pkg.license}
*/
`.trim()
const defaultExportOutro = `
module.exports = exports.default || {}
Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value })
`
export default {
input: `src/${entryFile}.ts`,
output: [
{
file: pkg.main,
format: 'cjs',
sourcemap: true,
exports: 'named',
outro: defaultExportOutro,
banner
},
{
file: pkg.module,
format: 'es',
sourcemap: true,
exports: 'named',
banner
}
],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [
...Object.keys(pkg.dependencies || {}),
...Object.keys(pkg.peerDependencies || {})
],
watch: {
include: 'src/**'
},
plugins: [
// Compile TypeScript files
typescript({ useTsconfigDeclarationDir: true }),
// Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
commonjs(),
// Allow node_modules resolution, so you can use 'external' to control
// which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve(),
// Resolve source maps to the original source
sourceMaps()
]
}
================================================
FILE: packages/playwright-extra/src/extra.ts
================================================
import Debug from 'debug'
const debug = Debug('playwright-extra')
import type * as pw from 'playwright-core'
import type { CompatiblePlugin, Plugin } from './types'
import { PluginList } from './plugins'
import { playwrightLoader } from './helper/loader'
type PlaywrightBrowserLauncher = pw.BrowserType
/**
* The Playwright browser launcher APIs we're augmenting
* @private
*/
interface AugmentedLauncherAPIs
extends Pick<
PlaywrightBrowserLauncher,
'launch' | 'launchPersistentContext' | 'connect' | 'connectOverCDP'
> {}
/**
* Modular plugin framework to teach `playwright` new tricks.
*/
export class PlaywrightExtraClass implements AugmentedLauncherAPIs {
/** Plugin manager */
public readonly plugins: PluginList
constructor(private _launcher?: Partial<PlaywrightBrowserLauncher>) {
this.plugins = new PluginList()
}
/**
* The **main interface** to register plugins.
*
* Can be called multiple times to enable multiple plugins.
*
* Plugins derived from `PuppeteerExtraPlugin` will be used with a compatiblity layer.
*
* @example
* chromium.use(plugin1).use(plugin2)
* firefox.use(plugin1).use(plugin2)
*
* @see [PuppeteerExtraPlugin]
*
* @return The same `PlaywrightExtra` instance (for optional chaining)
*/
public use(plugin: CompatiblePlugin): this {
const isValid = plugin && 'name' in plugin
if (!isValid) {
throw new Error('A plugin must be provided to .use()')
}
if (this.plugins.add(plugin as Plugin)) {
debug('Plugin registered', plugin.name)
}
return this
}
/**
* In order to support a default export which will require vanilla playwright automatically,
* as well as `addExtra` to patch a provided launcher, we need to so some gymnastics here.
*
* Otherwise this would throw immediately, even when only using the `addExtra` export with an arbitrary compatible launcher.
*
* The solution is to make the vanilla launcher optional and only throw once we try to effectively use and can't find it.
*
* @internal
*/
public get launcher(): Partial<PlaywrightBrowserLauncher> {
if (!this._launcher) {
throw playwrightLoader.requireError
}
return this._launcher
}
public async launch(
...args: Parameters<PlaywrightBrowserLauncher['launch']>
): ReturnType<PlaywrightBrowserLauncher['launch']> {
if (!this.launcher.launch) {
throw new Error('Launcher does not support "launch"')
}
let [options] = args
options = { args: [], ...(options || {}) } // Initialize args array
debug('launch', options)
this.plugins.prepare()
// Give plugins the chance to modify the options before continuing
options =
(await this.plugins.dispatchBlocking('beforeLaunch', options)) || options
debug('launch with options', options)
if ('userDataDir' in options) {
debug(
"A plugin defined userDataDir during .launch, which isn't supported by playwright - ignoring"
)
delete (options as any).userDataDir
}
const browser = await this.launcher['launch'](options)
await this.plugins.dispatchBlocking('onBrowser', browser)
await this._bindBrowserEvents(browser)
await this.plugins.dispatchBlocking('afterLaunch', browser)
return browser
}
public async launchPersistentContext(
...args: Parameters<PlaywrightBrowserLauncher['launchPersistentContext']>
): ReturnType<PlaywrightBrowserLauncher['launchPersistentContext']> {
if (!this.launcher.launchPersistentContext) {
throw new Error('Launcher does not support "launchPersistentContext"')
}
let [userDataDir, options] = args
options = { args: [], ...(options || {}) } // Initialize args array
debug('launchPersistentContext', options)
this.plugins.prepare()
// Give plugins the chance to modify the options before continuing
options =
(await this.plugins.dispatchBlocking('beforeLaunch', options)) || options
const context = await this.launcher['launchPersistentContext'](
userDataDir,
options
)
await this.plugins.dispatchBlocking('afterLaunch', context)
this._bindBrowserContextEvents(context)
return context
}
async connect(
wsEndpointOrOptions: string | (pw.ConnectOptions & { wsEndpoint?: string }),
wsOptions: pw.ConnectOptions = {}
): ReturnType<PlaywrightBrowserLauncher['connect']> {
if (!this.launcher.connect) {
throw new Error('Launcher does not support "connect"')
}
this.plugins.prepare()
// Playwright currently supports two function signatures for .connect
let options: pw.ConnectOptions & { wsEndpoint?: string } = {}
let wsEndpointAsString = false
if (typeof wsEndpointOrOptions === 'object') {
options = { ...wsEndpointOrOptions, ...wsOptions }
} else {
wsEndpointAsString = true
options = { wsEndpoint: wsEndpointOrOptions, ...wsOptions }
}
debug('connect', options)
// Give plugins the chance to modify the options before launch/connect
options =
(await this.plugins.dispatchBlocking('beforeConnect', options)) || options
// Follow call signature of end user
const args: any[] = []
const wsEndpoint = options.wsEndpoint
if (wsEndpointAsString) {
delete options.wsEndpoint
args.push(wsEndpoint, options)
} else {
args.push(options)
}
const browser = (await (this.launcher['connect'] as any)(
...args
)) as pw.Browser
await this.plugins.dispatchBlocking('onBrowser', browser)
await this._bindBrowserEvents(browser)
await this.plugins.dispatchBlocking('afterConnect', browser)
return browser
}
async connectOverCDP(
wsEndpointOrOptions:
| string
| (pw.ConnectOverCDPOptions & { endpointURL?: string }),
wsOptions: pw.ConnectOverCDPOptions = {}
): ReturnType<PlaywrightBrowserLauncher['connectOverCDP']> {
if (!this.launcher.connectOverCDP) {
throw new Error(`Launcher does not implement 'connectOverCDP'`)
}
this.plugins.prepare()
// Playwright currently supports two function signatures for .connectOverCDP
let options: pw.ConnectOverCDPOptions & { endpointURL?: string } = {}
let wsEndpointAsString = false
if (typeof wsEndpointOrOptions === 'object') {
options = { ...wsEndpointOrOptions, ...wsOptions }
} else {
wsEndpointAsString = true
options = { endpointURL: wsEndpointOrOptions, ...wsOptions }
}
debug('connectOverCDP'), options
// Give plugins the chance to modify the options before launch/connect
options =
(await this.plugins.dispatchBlocking('beforeConnect', options)) || options
// Follow call signature of end user
const args: any[] = []
const endpointURL = options.endpointURL
if (wsEndpointAsString) {
delete options.endpointURL
args.push(endpointURL, options)
} else {
args.push(options)
}
const browser = (await (this.launcher['connectOverCDP'] as any)(
...args
)) as pw.Browser
await this.plugins.dispatchBlocking('onBrowser', browser)
await this._bindBrowserEvents(browser)
await this.plugins.dispatchBlocking('afterConnect', browser)
return browser
}
protected async _bindBrowserContextEvents(
context: pw.BrowserContext,
contextOptions?: pw.BrowserContextOptions
) {
debug('_bindBrowserContextEvents')
this.plugins.dispatch('onContextCreated', context, contextOptions)
// Make sure things like `addInitScript` show an effect on the very first page as well
context.newPage = ((originalMethod, ctx) => {
return async () => {
const page = await originalMethod.call(ctx)
await page.goto('about:blank')
return page
}
})(context.newPage, context)
context.on('close', () => {
// When using `launchPersistentContext` context closing is the same as browser closing
if (!context.browser()) {
this.plugins.dispatch('onDisconnected')
}
})
context.on('page', page => {
this.plugins.dispatch('onPageCreated', page)
page.on('close', () => {
this.plugins.dispatch('onPageClose', page)
})
})
}
protected async _bindBrowserEvents(browser: pw.Browser) {
debug('_bindPlaywrightBrowserEvents')
browser.on('disconnected', () => {
this.plugins.dispatch('onDisconnected', browser)
})
// Note: `browser.newPage` will implicitly call `browser.newContext` as well
browser.newContext = ((originalMethod, ctx) => {
return async (options: pw.BrowserContextOptions = {}) => {
const contextOptions: pw.BrowserContextOptions =
(await this.plugins.dispatchBlocking(
'beforeContext',
options,
browser
)) || options
const context = await originalMethod.call(ctx, contextOptions)
this._bindBrowserContextEvents(context, contextOptions)
return context
}
})(browser.newContext, browser)
}
}
/**
* PlaywrightExtra class with additional launcher methods.
*
* Augments the class with an instance proxy to pass on methods that are not augmented to the original target.
*
*/
export const PlaywrightExtra = new Proxy(PlaywrightExtraClass, {
construct(classTarget, args) {
debug(`create instance of ${classTarget.name}`)
const result = Reflect.construct(classTarget, args) as PlaywrightExtraClass
return new Proxy(result, {
get(target, prop) {
if (prop in target) {
return Reflect.get(target, prop)
}
debug('proxying property to original launcher: ', prop)
return Reflect.get(target.launcher, prop)
}
})
}
})
================================================
FILE: packages/playwright-extra/src/helper/loader.ts
================================================
import type * as pw from 'playwright-core'
/** Node.js module loader helper */
export class Loader<TargetModule> {
constructor(public moduleName: string, public packageNames: string[]) {}
/**
* Lazy load a top level export from another module by wrapping it in a JS proxy.
*
* This allows us to re-export e.g. `devices` from `playwright` while redirecting direct calls
* to it to the module version the user has installed, rather than shipping with a hardcoded version.
*
* If we don't do this and the user doesn't have the target module installed we'd throw immediately when our code is imported.
*
* We use a "super" Proxy defining all traps, so calls like `Object.keys(playwright.devices).length` will return the correct value.
*/
public lazyloadExportOrDie<T extends keyof TargetModule>(exportName: T) {
const that = this
const trapHandler = Object.fromEntries(
Object.getOwnPropertyNames(Reflect).map((name: any) => [
name,
function (target: any, ...args: any[]) {
const moduleExport = that.loadModuleOrDie()[exportName]
const customTarget = moduleExport as any
const result = ((Reflect as any)[name] as any)(
customTarget || target,
...args
)
return result
}
])
)
return new Proxy({}, trapHandler) as TargetModule[T]
}
/** Load the module if possible */
public loadModule() {
return requirePackages<TargetModule>(this.packageNames)
}
/** Load the module if possible or throw */
public loadModuleOrDie(): TargetModule {
const module = requirePackages<TargetModule>(this.packageNames)
if (module) {
return module
}
throw this.requireError
}
public get requireError() {
const moduleNamePretty =
this.moduleName.charAt(0).toUpperCase() + this.moduleName.slice(1)
return new Error(`
${moduleNamePretty} is missing. :-)
I've tried loading ${this.packageNames
.map(p => `"${p}"`)
.join(', ')} - no luck.
Make sure you install one of those packages or use the named 'addExtra' export,
to patch a specific (and maybe non-standard) implementation of ${moduleNamePretty}.
To get the latest stable version of ${moduleNamePretty} run:
'yarn add ${this.moduleName}' or 'npm i ${this.moduleName}'
`)
}
}
export function requirePackages<TargetModule = any>(packageNames: string[]) {
for (const name of packageNames) {
try {
return require(name) as TargetModule
} catch (_) {
continue // noop
}
}
return
}
/** Playwright specific module loader */
export const playwrightLoader = new Loader<typeof pw>('playwright', [
'playwright-core',
'playwright'
])
================================================
FILE: packages/playwright-extra/src/index.ts
================================================
import type * as pw from 'playwright-core'
import { PlaywrightExtra, PlaywrightExtraClass } from './extra'
import { PluginList } from './plugins'
import { playwrightLoader as loader } from './helper/loader'
export { PlaywrightExtra, PlaywrightExtraClass } from './extra'
export { PluginList } from './plugins'
/** A playwright browser launcher */
export type PlaywrightBrowserLauncher = pw.BrowserType<{}>
/** A playwright browser launcher with plugin functionality */
export type AugmentedBrowserLauncher = PlaywrightExtraClass &
PlaywrightBrowserLauncher
/**
* The minimum shape we expect from a playwright compatible launcher object.
* We intentionally keep this not strict so other custom or compatible launchers can be used.
*/
export interface PlaywrightCompatibleLauncher {
connect(...args: any[]): Promise<any>
launch(...args: any[]): Promise<any>
}
/** Our custom module exports */
interface ExtraModuleExports {
PlaywrightExtra: typeof PlaywrightExtra
PlaywrightExtraClass: typeof PlaywrightExtraClass
PluginList: typeof PluginList
addExtra: typeof addExtra
chromium: AugmentedBrowserLauncher
firefox: AugmentedBrowserLauncher
webkit: AugmentedBrowserLauncher
}
/** Vanilla playwright module exports */
type PlaywrightModuleExports = typeof pw
/**
* Augment the provided Playwright browser launcher with plugin functionality.
*
* Using `addExtra` will always create a fresh PlaywrightExtra instance.
*
* @example
* import playwright from 'playwright'
* import { addExtra } from 'playwright-extra'
*
* const chromium = addExtra(playwright.chromium)
* chromium.use(plugin)
*
* @param launcher - Playwright (or compatible) browser launcher
*/
export const addExtra = <Launcher extends PlaywrightCompatibleLauncher>(
launcher?: Launcher
) => new PlaywrightExtra(launcher) as PlaywrightExtraClass & Launcher
/**
* This object can be used to launch or connect to Chromium with plugin functionality.
*
* This default export will behave exactly the same as the regular playwright
* (just with extra plugin functionality) and can be used as a drop-in replacement.
*
* Behind the scenes it will try to require either the `playwright-core`
* or `playwright` module from the installed dependencies.
*
* @note
* Due to Node.js import caching this will result in a single
* PlaywrightExtra instance, even when used in different files. If you need multiple
* instances with different plugins please use `addExtra`.
*
* @example
* // javascript import
* const { chromium } = require('playwright-extra')
*
* // typescript/es6 module import
* import { chromium } from 'playwright-extra'
*
* // Add plugins
* chromium.use(...)
*/
export const chromium = addExtra((loader.loadModule() || {}).chromium)
/**
* This object can be used to launch or connect to Firefox with plugin functionality
* @note This export will always return the same instance, if you wish to use multiple instances with different plugins use `addExtra`
*/
export const firefox = addExtra((loader.loadModule() || {}).firefox)
/**
* This object can be used to launch or connect to Webkit with plugin functionality
* @note This export will always return the same instance, if you wish to use multiple instances with different plugins use `addExtra`
*/
export const webkit = addExtra((loader.loadModule() || {}).webkit)
// Other playwright module exports we simply re-export with lazy loading
export const _android = loader.lazyloadExportOrDie('_android')
export const _electron = loader.lazyloadExportOrDie('_electron')
export const request = loader.lazyloadExportOrDie('request')
export const selectors = loader.lazyloadExportOrDie('selectors')
export const devices = loader.lazyloadExportOrDie('devices')
export const errors = loader.lazyloadExportOrDie('errors')
/** Playwright with plugin functionality */
const moduleExports: ExtraModuleExports & PlaywrightModuleExports = {
// custom exports
PlaywrightExtra,
PlaywrightExtraClass,
PluginList,
addExtra,
chromium,
firefox,
webkit,
// vanilla exports
_android,
_electron,
request,
selectors,
devices,
errors
}
export default moduleExports
================================================
FILE: packages/playwright-extra/src/plugins.ts
================================================
import Debug from 'debug'
const debug = Debug('playwright-extra:plugins')
import {
Plugin,
PluginMethodName,
PluginMethodFn,
PluginModule,
CompatiblePluginModule
} from './types'
import { requirePackages } from './helper/loader'
import { addPuppeteerCompat } from './puppeteer-compatiblity-shim'
export class PluginList {
private readonly _plugins: Plugin[] = []
private readonly _dependencyDefaults: Map<string, any> = new Map()
private readonly _dependencyResolution: Map<string, CompatiblePluginModule> =
new Map()
constructor() {}
/**
* Get a list of all registered plugins.
*/
public get list() {
return this._plugins
}
/**
* Get the names of all registered plugins.
*/
public get names() {
return this._plugins.map(p => p.name)
}
/**
* Add a new plugin to the list (after checking if it's well-formed).
*
* @param plugin
* @internal
*/
public add(plugin: Plugin) {
if (!this.isValidPluginInstance(plugin)) {
return false
}
if (!!plugin.onPluginRegistered) {
plugin.onPluginRegistered({ framework: 'playwright' })
}
// PuppeteerExtraPlugin: Populate `_childClassMembers` list containing methods defined by the plugin
if (!!plugin._registerChildClassMembers) {
plugin._registerChildClassMembers(Object.getPrototypeOf(plugin))
}
if (plugin.requirements?.has('dataFromPlugins')) {
plugin.getDataFromPlugins = this.getData.bind(this)
}
this._plugins.push(plugin)
return true
}
/** Check if the shape of a plugin is correct or warn */
protected isValidPluginInstance(plugin: Plugin) {
if (
!plugin ||
typeof plugin !== 'object' ||
!plugin._isPuppeteerExtraPlugin
) {
console.error(
`Warning: Plugin is not derived from PuppeteerExtraPlugin, ignoring.`,
plugin
)
return false
}
if (!plugin.name) {
console.error(
`Warning: Plugin with no name registering, ignoring.`,
plugin
)
return false
}
return true
}
/** Error callback in case calling a plugin method throws an error. Can be overwritten. */
public onPluginError(plugin: Plugin, method: PluginMethodName, err: Error) {
console.warn(
`An error occured while executing "${method}" in plugin "${plugin.name}":`,
err
)
}
/**
* Define default values for plugins implicitly required through the `dependencies` plugin stanza.
*
* @param dependencyPath - The string by which the dependency is listed (not the plugin name)
*
* @example
* chromium.use(stealth)
* chromium.plugins.setDependencyDefaults('stealth/evasions/webgl.vendor', { vendor: 'Bob', renderer: 'Alice' })
*/
public setDependencyDefaults(dependencyPath: string, opts: any) {
this._dependencyDefaults.set(dependencyPath, opts)
return this
}
/**
* Define custom plugin modules for plugins implicitly required through the `dependencies` plugin stanza.
*
* Using this will prevent dynamic imports from being used, which JS bundlers often have issues with.
*
* @example
* chromium.use(stealth)
* chromium.plugins.setDependencyResolution('stealth/evasions/webgl.vendor', VendorPlugin)
*/
public setDependencyResolution(
dependencyPath: string,
pluginModule: CompatiblePluginModule
) {
this._dependencyResolution.set(dependencyPath, pluginModule)
return this
}
/**
* Prepare plugins to be used (resolve dependencies, ordering)
* @internal
*/
public prepare() {
this.resolveDependencies()
this.order()
}
/** Return all plugins using the supplied method */
protected filterByMethod(methodName: PluginMethodName) {
return this._plugins.filter(plugin => {
// PuppeteerExtraPlugin: The base class will already define all methods, hence we need to do a different check
if (
!!plugin._childClassMembers &&
Array.isArray(plugin._childClassMembers)
) {
return plugin._childClassMembers.includes(methodName)
}
return methodName in plugin
})
}
/** Conditionally add puppeteer compatibility to values provided to the plugins */
protected _addPuppeteerCompatIfNeeded<TMethod extends PluginMethodName>(
plugin: Plugin,
method: TMethod,
args: Parameters<PluginMethodFn<TMethod>>
) {
const canUseShim = plugin._isPuppeteerExtraPlugin && !plugin.noPuppeteerShim
const methodWhitelist: PluginMethodName[] = [
'onBrowser',
'onPageCreated',
'onPageClose',
'afterConnect',
'afterLaunch'
]
const shouldUseShim = methodWhitelist.includes(method)
if (!canUseShim || !shouldUseShim) {
return args
}
debug('add puppeteer compatibility', plugin.name, method)
return [...args.map(arg => addPuppeteerCompat(arg as any))] as Parameters<
PluginMethodFn<TMethod>
>
}
/**
* Dispatch plugin lifecycle events in a typesafe way.
* Only Plugins that expose the supplied property will be called.
*
* Will not await results to dispatch events as fast as possible to all plugins.
*
* @param method - The lifecycle method name
* @param args - Optional: Any arguments to be supplied to the plugin methods
* @internal
*/
public dispatch<TMethod extends PluginMethodName>(
method: TMethod,
...args: Parameters<PluginMethodFn<TMethod>>
): void {
const plugins = this.filterByMethod(method)
debug('dispatch', method, {
all: this._plugins.length,
filteredByMethod: plugins.length
})
for (const plugin of plugins) {
try {
args = this._addPuppeteerCompatIfNeeded.bind(this)(plugin, method, args)
const fnType = plugin[method]?.constructor?.name
debug('dispatch to plugin', {
plugin: plugin.name,
method,
fnType
})
if (fnType === 'AsyncFunction') {
;(plugin[method] as any)(...args).catch((err: any) =>
this.onPluginError(plugin, method, err)
)
} else {
;(plugin[method] as any)(...args)
}
} catch (err) {
this.onPluginError(plugin, method, err as any)
}
}
}
/**
* Dispatch plugin lifecycle events in a typesafe way.
* Only Plugins that expose the supplied property will be called.
*
* Can also be used to get a definite return value after passing it to plugins:
* Calls plugins sequentially and passes on a value (waterfall style).
*
* The plugins can either modify the value or return an updated one.
* Will return the latest, updated value which ran through all plugins.
*
* By convention only the first argument will be used as the updated value.
*
* @param method - The lifecycle method name
* @param args - Optional: Any arguments to be supplied to the plugin methods
* @internal
*/
public async dispatchBlocking<TMethod extends PluginMethodName>(
method: TMethod,
...args: Parameters<PluginMethodFn<TMethod>>
): Promise<ReturnType<PluginMethodFn<TMethod>>> {
const plugins = this.filterByMethod(method)
debug('dispatchBlocking', method, {
all: this._plugins.length,
filteredByMethod: plugins.length
})
let retValue: any = null
for (const plugin of plugins) {
try {
args = this._addPuppeteerCompatIfNeeded.bind(this)(plugin, method, args)
retValue = await (plugin[method] as any)(...args)
// In case we got a return value use that as new first argument for followup function calls
if (retValue !== undefined) {
args[0] = retValue
}
} catch (err) {
this.onPluginError(plugin, method, err as any)
return retValue
}
}
return retValue
}
/**
* Order plugins that have expressed a special placement requirement.
*
* This is useful/necessary for e.g. plugins that depend on the data from other plugins.
*
* @private
*/
protected order() {
debug('order:before', this.names)
const runLast = this._plugins
.filter(p => p.requirements?.has('runLast'))
.map(p => p.name)
for (const name of runLast) {
const index = this._plugins.findIndex(p => p.name === name)
this._plugins.push(this._plugins.splice(index, 1)[0])
}
debug('order:after', this.names)
}
/**
* Collects the exposed `data` property of all registered plugins.
* Will be reduced/flattened to a single array.
*
* Can be accessed by plugins that listed the `dataFromPlugins` requirement.
*
* Implemented mainly for plugins that need data from other plugins (e.g. `user-preferences`).
*
* @see [PuppeteerExtraPlugin]/data
* @param name - Filter data by optional name
*
* @private
*/
protected getData(name?: string) {
const data = this._plugins
.filter((p: any) => !!p.data)
.map((p: any) => (Array.isArray(p.data) ? p.data : [p.data]))
.reduce((acc, arr) => [...acc, ...arr], [])
return name ? data.filter((d: any) => d.name === name) : data
}
/**
* Handle `plugins` stanza (already instantiated plugins that don't require dynamic imports)
*/
protected resolvePluginsStanza() {
debug('resolvePluginsStanza')
const pluginNames = new Set(this.names)
this._plugins
.filter(p => !!p.plugins && p.plugins.length)
.filter(p => !pluginNames.has(p.name)) // TBD: Do we want to filter out existing?
.forEach(parent => {
;(parent.plugins || []).forEach(p => {
debug(parent.name, 'adding missing plugin', p.name)
this.add(p as Plugin)
})
})
}
/**
* Handle `dependencies` stanza (which requires dynamic imports)
*
* Plugins can define `dependencies` as a Set or Array of dependency paths, or a Map with additional opts
*
* @note
* - The default opts for implicit dependencies can be defined using `setDependencyDefaults()`
* - Dynamic imports can be avoided by providing plugin modules with `setDependencyResolution()`
*/
protected resolveDependenciesStanza() {
debug('resolveDependenciesStanza')
/** Attempt to dynamically require a plugin module */
const requireDependencyOrDie = (
parentName: string,
dependencyPath: string
) => {
// If the user provided the plugin module already we use that
if (this._dependencyResolution.has(dependencyPath)) {
return this._dependencyResolution.get(dependencyPath) as PluginModule
}
const possiblePrefixes = ['puppeteer-extra-plugin-'] // could be extended later
const isAlreadyPrefixed = possiblePrefixes.some(prefix =>
dependencyPath.startsWith(prefix)
)
const packagePaths: string[] = []
// If the dependency is not already prefixed we attempt to require all possible combinations to find one that works
if (!isAlreadyPrefixed) {
packagePaths.push(
...possiblePrefixes.map(prefix => prefix + dependencyPath)
)
}
// We always attempt to require the path verbatim (as a last resort)
packagePaths.push(dependencyPath)
const pluginModule = requirePackages<PluginModule>(packagePaths)
if (pluginModule) {
return pluginModule
}
const explanation = `
The plugin '${parentName}' listed '${dependencyPath}' as dependency,
which could not be found. Please install it:
${packagePaths
.map(packagePath => `yarn add ${packagePath.split('/')[0]}`)
.join(`\n or:\n`)}
Note: You don't need to require the plugin yourself,
unless you want to modify it's default settings.
If your bundler has issues with dynamic imports take a look at '.plugins.setDependencyResolution()'.
`
console.warn(explanation)
throw new Error('Plugin dependency not found')
}
const existingPluginNames = new Set(this.names)
const recursivelyLoadMissingDependencies = ({
name: parentName,
dependencies
}: Plugin): any => {
if (!dependencies) {
return
}
const processDependency = (dependencyPath: string, opts?: any) => {
const pluginModule = requireDependencyOrDie(parentName, dependencyPath)
opts = opts || this._dependencyDefaults.get(dependencyPath) || {}
const plugin = pluginModule(opts)
if (existingPluginNames.has(plugin.name)) {
debug(parentName, '=> dependency already exists:', plugin.name)
return
}
existingPluginNames.add(plugin.name)
debug(parentName, '=> adding new dependency:', plugin.name, opts)
this.add(plugin)
return recursivelyLoadMissingDependencies(plugin)
}
if (dependencies instanceof Set || Array.isArray(dependencies)) {
return [...dependencies].forEach(dependencyPath =>
processDependency(dependencyPath)
)
}
if (dependencies instanceof Map) {
// Note: `k,v => v,k` (Map + forEach will reverse the order)
return dependencies.forEach((v, k) => processDependency(k, v))
}
}
this.list.forEach(recursivelyLoadMissingDependencies)
}
/**
* Lightweight plugin dependency management to require plugins and code mods on demand.
* @private
*/
protected resolveDependencies() {
debug('resolveDependencies')
this.resolvePluginsStanza()
this.resolveDependenciesStanza()
}
}
================================================
FILE: packages/playwright-extra/src/puppeteer-compatiblity-shim/index.ts
================================================
import Debug from 'debug'
const debug = Debug('playwright-extra:puppeteer-compat')
import type * as pw from 'playwright-core'
export type PlaywrightObject = pw.Page | pw.Frame | pw.Browser
export interface PuppeteerBrowserShim {
isCompatShim?: boolean
isPlaywright?: boolean
pages?: pw.BrowserContext['pages']
userAgent: () => Promise<'string'>
}
export interface PuppeteerPageShim {
isCompatShim?: boolean
isPlaywright?: boolean
browser?: () => pw.Browser
evaluateOnNewDocument?: pw.Page['addInitScript']
_client: () => pw.CDPSession
}
export const isPlaywrightPage = (obj: unknown): obj is pw.Page => {
return 'unroute' in (obj as pw.Page)
}
export const isPlaywrightFrame = (obj: unknown): obj is pw.Frame => {
return ['parentFrame', 'frameLocator'].every(x => x in (obj as pw.Frame))
}
export const isPlaywrightBrowser = (obj: unknown): obj is pw.Browser => {
return 'newContext' in (obj as pw.Browser)
}
export const isPuppeteerCompat = (obj?: unknown): obj is PlaywrightObject => {
return !!obj && typeof obj === 'object' && !!(obj as any).isCompatShim
}
const cache = {
objectToShim: new Map<PlaywrightObject, PlaywrightObject>(),
cdpSession: {
page: new Map<pw.Page | pw.Frame, pw.CDPSession>(),
browser: new Map<pw.Browser, pw.CDPSession>()
}
}
/** Augment a Playwright object with compatibility with certain Puppeteer methods */
export function addPuppeteerCompat<
Input extends pw.Page | pw.Frame | pw.Browser | null
>(object: Input): Input {
if (!object || typeof object !== 'object') {
return object
}
if (cache.objectToShim.has(object)) {
return cache.objectToShim.get(object) as Input
}
if (isPuppeteerCompat(object)) {
return object
}
debug('addPuppeteerCompat', cache.objectToShim.size)
if (isPlaywrightPage(object) || isPlaywrightFrame(object)) {
const shim = createPageShim(object)
cache.objectToShim.set(object, shim)
return shim as Input
}
if (isPlaywrightBrowser(object)) {
const shim = createBrowserShim(object)
cache.objectToShim.set(object, shim)
return shim as Input
}
debug('Received unknown object:', Reflect.ownKeys(object))
return object
}
// Only chromium browsers support CDP
const dummyCDPClient = {
send: async (...args: any[]) => {
debug('dummy CDP client called', 'send', args)
},
on: (...args: any[]) => {
debug('dummy CDP client called', 'on', args)
}
} as pw.CDPSession
export async function getPageCDPSession(page: pw.Page | pw.Frame) {
let session = cache.cdpSession.page.get(page)
if (session) {
debug('getPageCDPSession: use existing')
return session
}
debug('getPageCDPSession: use new')
const context = isPlaywrightFrame(page)
? page.page().context()
: page.context()
try {
session = await context.newCDPSession(page)
cache.cdpSession.page.set(page, session)
return session
} catch (err: any) {
debug('getPageCDPSession: error while creating session:', err.message)
debug(
'getPageCDPSession: Unable create CDP session (most likely a different browser than chromium) - returning a dummy'
)
}
return dummyCDPClient
}
export async function getBrowserCDPSession(browser: pw.Browser) {
let session = cache.cdpSession.browser.get(browser)
if (session) {
debug('getBrowserCDPSession: use existing')
return session
}
debug('getBrowserCDPSession: use new')
try {
session = await browser.newBrowserCDPSession()
cache.cdpSession.browser.set(browser, session)
return session
} catch (err: any) {
debug('getBrowserCDPSession: error while creating session:', err.message)
debug(
'getBrowserCDPSession: Unable create CDP session (most likely a different browser than chromium) - returning a dummy'
)
}
return dummyCDPClient
}
export function createPageShim(page: pw.Page | pw.Frame) {
const objId = Math.random().toString(36).substring(2, 7)
const shim = new Proxy(page, {
get(target, prop) {
if (prop === 'isCompatShim' || prop === 'isPlaywright') {
return true
}
debug('page - get', objId, prop)
if (prop === '_client') {
return () => ({
send: async (method: string, params: any) => {
const session = await getPageCDPSession(page)
return await session.send(method as any, params)
},
on: (event: string, listener: any) => {
getPageCDPSession(page).then(session => {
session.on(event as any, listener)
})
}
})
}
if (prop === 'setBypassCSP') {
return async (enabled: boolean) => {
const session = await getPageCDPSession(page)
return await session.send('Page.setBypassCSP', {
enabled
})
}
}
if (prop === 'setUserAgent') {
return async (userAgent: string, userAgentMetadata?: any) => {
const session = await getPageCDPSession(page)
return await session.send('Emulation.setUserAgentOverride', {
userAgent,
userAgentMetadata
})
}
}
if (prop === 'browser') {
if (isPlaywrightPage(page)) {
return () => {
let browser = page.context().browser()
if (!browser) {
debug(
'page.browser() - not available, most likely due to launchPersistentContext'
)
// Use a page shim as quick drop-in (so browser.userAgent() still works)
browser = page as any
}
return addPuppeteerCompat(browser)
}
}
}
if (prop === 'evaluateOnNewDocument') {
if (isPlaywrightPage(page)) {
return async function (pageFunction: any | string, ...args: any[]) {
return await page.addInitScript(pageFunction, args[0])
}
}
}
// Only relevant when page is being used a pseudo stand-in for the browser object (launchPersistentContext)
if (prop === 'userAgent') {
return async (enabled: boolean) => {
const session = await getPageCDPSession(page)
const data = await session.send('Browser.getVersion')
return data.userAgent
}
}
return Reflect.get(target, prop)
}
})
return shim
}
export function createBrowserShim(browser: pw.Browser) {
const objId = Math.random().toString(36).substring(2, 7)
const shim = new Proxy(browser, {
get(target, prop) {
if (prop === 'isCompatShim' || prop === 'isPlaywright') {
return true
}
debug('browser - get', objId, prop)
if (prop === 'pages') {
return () =>
browser
.contexts()
.flatMap(c => c.pages().map(page => addPuppeteerCompat(page)))
}
if (prop === 'userAgent') {
return async () => {
const session = await getBrowserCDPSession(browser)
const data = await session.send('Browser.getVersion')
return data.userAgent
}
}
return Reflect.get(target, prop)
}
})
return shim
}
================================================
FILE: packages/playwright-extra/src/puppeteer-compatiblity-shim/playwright-shim.d.ts
================================================
// Playwright objects extended with puppeteer compatiblity shims
import type {} from 'playwright-core'
import type { PuppeteerPageShim, PuppeteerBrowserShim } from '.'
declare module 'playwright-core' {
interface Page extends PuppeteerPageShim {}
interface Frame extends PuppeteerPageShim {}
interface Browser extends PuppeteerBrowserShim {}
}
================================================
FILE: packages/playwright-extra/src/types/index.ts
================================================
import type * as pw from 'playwright-core'
type PropType<TObj, TProp extends keyof TObj> = TObj[TProp]
type PluginEnv = { framework: 'playwright' }
/** Strongly typed plugin lifecycle events for internal use */
export abstract class PluginLifecycleMethods {
async onPluginRegistered(env?: PluginEnv): Promise<void> {}
async beforeLaunch(
options: pw.LaunchOptions
): Promise<pw.LaunchOptions | void> {}
async afterLaunch(browserOrContext?: pw.Browser | pw.BrowserContext) {}
async beforeConnect(
options: pw.ConnectOptions
): Promise<pw.ConnectOptions | void> {}
async afterConnect(browser: pw.Browser) {}
async onBrowser(browser: pw.Browser) {}
async onPageCreated(page: pw.Page) {}
async onPageClose(page: pw.Page) {}
async onDisconnected(browser?: pw.Browser) {}
// Playwright only at the moment
async beforeContext(
options?: pw.BrowserContextOptions,
browser?: pw.Browser
): Promise<pw.BrowserContextOptions | void> {}
async onContextCreated(
context?: pw.BrowserContext,
options?: pw.BrowserContextOptions
) {}
}
/** A valid plugin method name */
export type PluginMethodName = keyof PluginLifecycleMethods
/** A valid plugin method function */
export type PluginMethodFn<TName extends PluginMethodName> = PropType<
PluginLifecycleMethods,
TName
>
type PluginRequirements = Set<
'launch' | 'headful' | 'dataFromPlugins' | 'runLast'
>
// PuppeteerExtraPlugin only supports Set, the others are future proofing
type PluginDependencies = Set<string> | Map<string, any> | string[]
interface PluginData {
name:
| string
// below is compat with a previously incorrect typing
| {
[key: string]: any
}
value: {
[key: string]: any
}
}
export interface CompatiblePluginLifecycleMethods {
onPluginRegistered(...any: any[]): Promise<any> | any
beforeLaunch(...any: any[]): Promise<any> | any
afterLaunch(...any: any[]): Promise<any> | any
beforeConnect(...any: any[]): Promise<any> | any
afterConnect(...any: any[]): Promise<any> | any
onBrowser(...any: any[]): Promise<any> | any
onPageCreated(...any: any[]): Promise<any> | any
onPageClose(...any: any[]): Promise<any> | any
onDisconnected(...any: any[]): Promise<any> | any
// Playwright only at the moment
beforeContext(...any: any[]): Promise<any> | any
onContextCreated(...any: any[]): Promise<any> | any
}
/**
* PuppeteerExtraPlugin interface, strongly typed for internal use
* @private
*/
export interface PuppeteerExtraPlugin extends Partial<PluginLifecycleMethods> {
_isPuppeteerExtraPlugin: boolean
name: string
/** Disable the puppeteer compatibility shim for this plugin */
noPuppeteerShim?: boolean
requirements?: PluginRequirements
dependencies?: PluginDependencies
data?: PluginData[]
getDataFromPlugins?(name?: string): void
_registerChildClassMembers?(prototype: any): void
_childClassMembers?: string[]
plugins?: CompatiblePlugin[]
// [propName: string]: any
}
/**
* Minimal compatible PuppeteerExtraPlugin interface
* @private
*/
export interface CompatiblePuppeteerPlugin
extends Partial<CompatiblePluginLifecycleMethods> {
_isPuppeteerExtraPlugin: boolean
name?: string
}
// Future proofing
export interface CompatiblePlaywrightPlugin
extends Partial<CompatiblePluginLifecycleMethods> {
_isPlaywrightExtraPlugin: boolean
name?: string
}
// Future proofing
export interface CompatibleExtraPlugin
extends Partial<CompatiblePluginLifecycleMethods> {
_isExtraPlugin: boolean
name?: string
}
/**
* A compatible plugin
*/
export type CompatiblePlugin =
| CompatiblePuppeteerPlugin
| CompatiblePlaywrightPlugin
| CompatibleExtraPlugin
export type CompatiblePluginModule = (...args: any[]) => CompatiblePlugin
export type Plugin = PuppeteerExtraPlugin
export type PluginModule = (...args: any[]) => Plugin
================================================
FILE: packages/playwright-extra/test/exports.spec.ts
================================================
import { test, expect } from './fixtures/extra'
test('should export the basic functionality', async ({ playwrightExtra }) => {
expect(playwrightExtra.addExtra).toBeDefined()
expect(playwrightExtra.chromium).toBeDefined()
expect(playwrightExtra.chromium.use).toBeDefined()
expect(playwrightExtra.chromium.plugins).toBeDefined()
expect(playwrightExtra.chromium.plugins.list).toBeDefined()
expect(playwrightExtra.chromium.plugins.names).toBeDefined()
expect(playwrightExtra.chromium.plugins.onPluginError).toBeDefined()
expect(playwrightExtra.chromium.launch).toBeDefined()
expect(playwrightExtra.chromium.launchPersistentContext).toBeDefined()
expect(playwrightExtra.chromium.connect).toBeDefined()
expect(playwrightExtra.chromium.connectOverCDP).toBeDefined()
expect(playwrightExtra.firefox).toBeDefined()
expect(playwrightExtra.firefox.use).toBeDefined()
expect(playwrightExtra.firefox.launch).toBeDefined()
expect(playwrightExtra.firefox.connect).toBeDefined()
expect(playwrightExtra.webkit).toBeDefined()
expect(playwrightExtra.webkit.use).toBeDefined()
expect(playwrightExtra.webkit.launch).toBeDefined()
expect(playwrightExtra.webkit.connect).toBeDefined()
expect((playwrightExtra as any).nonexistent).toBeUndefined()
})
test('chromium export should be well formed', async ({ playwrightExtra }) => {
const { chromium } = playwrightExtra
expect(typeof chromium).toBe('object')
expect(typeof chromium.use).toBe('function')
expect(typeof chromium.launch).toBe('function')
expect(typeof chromium.connect).toBe('function')
expect(typeof chromium.name).toBe('function')
expect(typeof chromium.name()).toBe('string')
expect(chromium.constructor.name).toBe('PlaywrightExtraClass')
})
test('addExtra export should be well formed', async ({ playwrightExtra }) => {
const { addExtra } = playwrightExtra
expect(typeof addExtra).toBe('function')
const launcher = addExtra()
expect(typeof launcher).toBe('object')
expect(launcher.constructor.name).toBe('PlaywrightExtraClass')
})
test('should re-export the same additional exports verbatim', async ({
playwrightExtra,
playwrightVanilla
}) => {
expect(playwrightExtra.errors).toStrictEqual(playwrightVanilla.errors)
expect(playwrightExtra.devices).toStrictEqual(playwrightVanilla.devices)
expect(playwrightExtra.selectors).toStrictEqual(playwrightVanilla.selectors)
expect(playwrightExtra.request).toStrictEqual(playwrightVanilla.request)
})
================================================
FILE: packages/playwright-extra/test/fixtures/dummyplugin.ts
================================================
import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin'
export class DummyPlugin extends PuppeteerExtraPlugin {
public pluginEventList: string[] = []
public pluginEventMap: Map<string, any> = new Map()
constructor(opts = {}) {
super(opts)
}
get name() {
return 'dummy'
}
async onPluginRegistered(...args: any[]) {
this.pluginEventList.push('onPluginRegistered')
}
async beforeLaunch(...args: any[]) {
this.pluginEventList.push('beforeLaunch')
}
async afterLaunch(...args: any[]) {
this.pluginEventList.push('afterLaunch')
}
async beforeConnect(...args: any[]) {
this.pluginEventList.push('beforeConnect')
}
async afterConnect(...args: any[]) {
this.pluginEventList.push('afterConnect')
}
async onBrowser(...args: any[]) {
this.pluginEventList.push('onBrowser')
}
async onTargetCreated(...args: any[]) {
this.pluginEventList.push('onTargetCreated')
}
async onPageCreated(...args: any[]) {
this.pluginEventList.push('onPageCreated')
}
async onTargetChanged(...args: any[]) {
this.pluginEventList.push('onTargetChanged')
}
async onTargetDestroyed(...args: any[]) {
this.pluginEventList.push('onTargetDestroyed')
}
async onDisconnected(...args: any[]) {
this.pluginEventList.push('onDisconnected')
}
async onClose(...args: any[]) {
this.pluginEventList.push('onClose')
}
// playwright only at the moment
async beforeContext(...args: any[]) {
this.pluginEventList.push('beforeContext')
}
async onContextCreated(...args: any[]) {
this.pluginEventList.push('onContextCreated')
}
}
================================================
FILE: packages/playwright-extra/test/fixtures/extra.ts
================================================
// Playwrights test runner is great, originally based on folio (which unfortunately isn't maintained anymore): https://github.com/microsoft/folio
import { test as base } from '@playwright/test'
import * as pwTest from '@playwright/test'
import * as pwExtraModule from '../../src'
import * as pwVanillaModule from 'playwright-core'
type PluginModuleWithOptions = { module: any; opts?: Record<string, any> }
export type ExtraOptions = {}
export type ExtraFixtures = {
/** playwright-extra module */
playwrightExtra: typeof pwExtraModule
/** playwright-core module */
playwrightVanilla: typeof pwVanillaModule
/** Augmented launcher */
extraLauncher: pwExtraModule.AugmentedBrowserLauncher
}
type WorkerFixtures = {
_connectedBrowser: pwTest.Browser | undefined
_browserOptions: pwTest.LaunchOptions
_artifactsDir: () => string
_snapshotSuffix: string
plugins: PluginModuleWithOptions[]
}
export const worker = base.extend<{}, WorkerFixtures>({
plugins: [[], { option: true, scope: 'worker' as any }],
browser: async (
{ playwright, browserName, _connectedBrowser, plugins },
use
) => {
if (_connectedBrowser) {
await use(_connectedBrowser)
return
}
if (!['chromium', 'firefox', 'webkit'].includes(browserName))
throw new Error(
`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`
)
const launcher = pwExtraModule.addExtra(playwright[browserName])
plugins.forEach(({ module: pluginModule, opts }) => {
launcher.use(pluginModule(opts))
})
const browser = await launcher.launch()
;(browser as any)._launcher = launcher
await use(browser as any)
await browser.close()
}
})
// Extend base test by providing "todoPage" and "settingsPage".
// This new "test" can be used in multiple test files, and each of them will get the fixtures.
export const test = worker.extend<ExtraOptions & ExtraFixtures>({
extraLauncher: async (
{ plugins, playwrightExtra, playwrightVanilla, browserName },
use
) => {
const launcher = playwrightExtra.addExtra(playwrightVanilla[browserName])
plugins.forEach(({ module: pluginModule, opts }) => {
launcher.use(pluginModule(opts))
})
await use(launcher)
},
playwrightExtra: async ({}, use) => {
await use(pwExtraModule)
},
playwrightVanilla: async ({}, use) => {
await use(pwVanillaModule)
}
})
export { expect } from '@playwright/test'
================================================
FILE: packages/playwright-extra/test/playwright.config.ts
================================================
import { type PlaywrightTestConfig } from '@playwright/test'
const config: PlaywrightTestConfig = {
retries: 3,
workers: 3,
use: {
browserName: 'chromium'
},
projects: [
{
name: 'chromium',
use: {
browserName: 'chromium',
launchOptions: {
chromiumSandbox: process.env.CI ? false : true,
args: process.env.CI
? ['--no-sandbox', '--disable-setuid-sandbox']
: []
}
}
},
{
name: 'firefox',
use: {
browserName: 'firefox'
}
},
{
name: 'webkit',
use: {
browserName: 'webkit'
// Note: webkit doesn't support --no-sandbox
}
}
]
}
export default config
================================================
FILE: packages/playwright-extra/test/plugin-events.spec.ts
================================================
import { test, expect } from './fixtures/extra'
import { DummyPlugin } from './fixtures/dummyplugin'
test.use({ plugins: [{ module: (opts: any) => new DummyPlugin(opts) }] })
test('emits correct events for launch', async ({ extraLauncher }) => {
const browser = await extraLauncher.launch()
const context = await browser.newContext()
const page = await context.newPage()
await page.close()
await browser.close()
const plugin = extraLauncher.plugins.list[0] as unknown as DummyPlugin
expect(plugin.pluginEventList).toStrictEqual([
'onPluginRegistered',
'beforeLaunch',
'onBrowser',
'afterLaunch',
'beforeContext',
'onContextCreated',
'onPageCreated',
'onDisconnected'
])
})
test('emits correct events for launch without .newContext()', async ({
extraLauncher
}) => {
const browser = await extraLauncher.launch()
const page = await browser.newPage()
await page.close()
await browser.close()
const plugin = extraLauncher.plugins.list[0] as unknown as DummyPlugin
expect(plugin.pluginEventList).toStrictEqual([
'onPluginRegistered',
'beforeLaunch',
'onBrowser',
'afterLaunch',
'beforeContext',
'onContextCreated',
'onPageCreated',
'onDisconnected'
])
})
test('emits correct events for launchPersistentContext', async ({
extraLauncher
}) => {
const context = await extraLauncher.launchPersistentContext('')
const page = await context.newPage()
await page.close()
await context.close()
const plugin = extraLauncher.plugins.list[0] as unknown as DummyPlugin
expect(plugin.pluginEventList).toStrictEqual([
'onPluginRegistered',
'beforeLaunch',
'afterLaunch',
'onContextCreated',
'onPageCreated',
'onDisconnected'
])
})
test('emits correct events for connect', async ({ extraLauncher }) => {
const server = await extraLauncher.launchServer()
const browser = await extraLauncher.connect(server.wsEndpoint())
const context = await browser.newContext()
const page = await context.newPage()
await page.close()
await browser.close()
await server.close()
const plugin = extraLauncher.plugins.list[0] as unknown as DummyPlugin
expect(plugin.pluginEventList).toStrictEqual([
'onPluginRegistered',
'beforeConnect',
'onBrowser',
'afterConnect',
'beforeContext',
'onContextCreated',
'onPageCreated',
'onDisconnected'
])
})
test('emits correct events for connectOverCDP', async ({
extraLauncher,
browserName
}) => {
test.skip(browserName !== 'chromium', 'Chromium only')
const server = await extraLauncher.launchServer({
args: ['--remote-debugging-port=9333']
})
const browser = await extraLauncher.connectOverCDP('http://localhost:9333')
const context = await browser.newContext()
const page = await context.newPage()
await page.close()
await browser.close()
await server.close()
const plugin = extraLauncher.plugins.list[0] as unknown as DummyPlugin
expect(plugin.pluginEventList).toStrictEqual([
'onPluginRegistered',
'beforeConnect',
'onBrowser',
'afterConnect',
'beforeContext',
'onContextCreated',
'onPageCreated',
'onDisconnected'
])
})
================================================
FILE: packages/playwright-extra/test/puppeteer-plugins/anonymize-ua.spec.ts
================================================
import { test, expect } from '../fixtures/extra'
import AnonymizeUAPlugin from 'puppeteer-extra-plugin-anonymize-ua'
test('puppeteer-extra-plugin-anonymize-ua will remove headless', async ({
browserName,
extraLauncher,
_browserOptions
}) => {
test.skip(browserName !== 'chromium', 'Chromium only')
const pluginErrors = []
extraLauncher.plugins.onPluginError = (plugin, method, err) => {
pluginErrors.push(err)
}
extraLauncher.use(AnonymizeUAPlugin())
expect(extraLauncher.plugins.list.length).toEqual(1)
expect(extraLauncher.plugins.list[0].name).toEqual('anonymize-ua')
const browser = await extraLauncher.launch(_browserOptions)
const context = await browser.newContext()
const page = await context.newPage()
await page.goto('https://example.com')
const ua = await page.evaluate(() => navigator.userAgent)
expect(ua.includes('Headless')).toBeFalsy()
expect(pluginErrors).toStrictEqual([])
await browser.close()
})
test('puppeteer-extra-plugin-anonymize-ua will allow a custom UA', async ({
browserName,
extraLauncher,
_browserOptions
}) => {
test.skip(browserName !== 'chromium', 'Chromium only')
const pluginErrors = []
extraLauncher.plugins.onPluginError = (plugin, method, err) => {
pluginErrors.push(err)
}
extraLauncher.use(
AnonymizeUAPlugin({
customFn: ua => 'MyCoolUserAgent'
})
)
expect(extraLauncher.plugins.list.length).toEqual(1)
expect(extraLauncher.plugins.list[0].name).toEqual('anonymize-ua')
const browser = await extraLauncher.launch(_browserOptions)
const context = await browser.newContext()
const page = await context.newPage()
await page.goto('https://example.com')
const ua = await page.evaluate(() => navigator.userAgent)
expect(ua).toBe('MyCoolUserAgent')
expect(pluginErrors).toStrictEqual([])
await browser.close()
})
================================================
FILE: packages/playwright-extra/test/puppeteer-plugins/recaptcha.spec.ts
================================================
import { test, expect } from '../fixtures/extra'
import RecaptchaPlugin from 'puppeteer-extra-plugin-recaptcha'
// Supports all browsers
test('puppeteer-extra-plugin-recaptcha will detect captchas', async ({
extraLauncher,
_browserOptions
}) => {
const pluginErrors = []
extraLauncher.plugins.onPluginError = (plugin, method, err) => {
pluginErrors.push(err)
}
const instance = RecaptchaPlugin()
extraLauncher.use(instance)
expect(extraLauncher.plugins.list.length).toEqual(1)
expect(extraLauncher.plugins.list[0].name).toEqual(instance.name)
const url =
'https://berstend.github.io/static/recaptcha/v2-checkbox-auto.html'
const browser = await extraLauncher.launch(_browserOptions)
const context = await browser.newContext()
const page = await context.newPage()
await page.goto(url)
const { captchas, error } = await (page as any).findRecaptchas()
expect(error).toBeFalsy()
expect(captchas).toBeTruthy()
expect(captchas.length).toBe(1)
const captcha = captchas[0]
expect(captcha._vendor).toBe('recaptcha')
expect(captcha._type).toBe('checkbox')
expect(captcha.url).toBe(url)
expect(captcha.id).toBeTruthy()
expect(captcha.sitekey).toBeTruthy()
expect(pluginErrors).toStrictEqual([])
await browser.close()
})
test('puppeteer-extra-plugin-recaptcha will solve captchas', async ({
extraLauncher,
_browserOptions
}) => {
test.skip(!process.env.TWOCAPTCHA_TOKEN, 'TWOCAPTCHA_TOKEN not set')
test.slow()
const pluginErrors = []
extraLauncher.plugins.onPluginError = (plugin, method, err) => {
pluginErrors.push(err)
}
const instance = RecaptchaPlugin({
provider: {
id: '2captcha',
token: process.env.TWOCAPTCHA_TOKEN
}
})
extraLauncher.use(instance)
expect(extraLauncher.plugins.list.length).toEqual(1)
expect(extraLauncher.plugins.list[0].name).toEqual(instance.name)
const url = 'https://www.google.com/recaptcha/api2/demo'
const browser = await extraLauncher.launch(_browserOptions)
const context = await browser.newContext()
const page = await context.newPage()
await page.goto(url, { waitUntil: 'networkidle' })
const { solved, error } = await (page as any).solveRecaptchas()
expect(error).toBeFalsy()
expect(solved).toBeTruthy()
expect(solved.length).toBe(1)
await Promise.all([
page.waitForNavigation({ waitUntil: 'networkidle' }),
page.click(`#recaptcha-demo-submit`)
])
const content = await page.content()
expect(content).toMatch('Verification Success... Hooray!')
expect(pluginErrors).toStrictEqual([])
await browser.close()
})
================================================
FILE: packages/playwright-extra/test/puppeteer-plugins/stealth.spec.ts
================================================
import { test, expect } from '../fixtures/extra'
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
test('puppeteer-extra-plugin-stealth will work', async ({
browserName,
extraLauncher,
_browserOptions
}) => {
test.skip(browserName !== 'chromium', 'Chromium only')
const pluginErrors = []
extraLauncher.plugins.onPluginError = (plugin, method, err) => {
pluginErrors.push(err)
}
extraLauncher.use(StealthPlugin())
expect(extraLauncher.plugins.list.length).toEqual(1)
expect(extraLauncher.plugins.list[0].name).toEqual('stealth')
extraLauncher.plugins.setDependencyDefaults('stealth/evasions/webgl.vendor', {
vendor: 'Bob',
renderer: 'Alice'
})
const browser = await extraLauncher.launch(_browserOptions)
const context = await browser.newContext()
const page = await context.newPage()
await page.goto('https://example.com')
const webgl = await page.evaluate(getWebglUnmasked)
expect(webgl).toStrictEqual({ renderer: 'Alice', vendor: 'Bob' })
expect(pluginErrors).toStrictEqual([])
await browser.close()
})
function getWebglUnmasked() {
const gl = document.createElement('canvas').getContext('webgl') as any
if (!gl) {
return {
error: 'no webgl'
}
}
const debugInfo = gl.getExtension('WEBGL_debug_renderer_info')
if (debugInfo) {
return {
vendor: gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL),
renderer: gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL)
}
}
return {
error: 'no WEBGL_debug_renderer_info'
}
}
================================================
FILE: packages/playwright-extra/tsconfig.json
================================================
{
"compilerOptions": {
"outDir": "./dist",
"target": "es2017",
"module": "es2015",
"moduleResolution": "node",
"lib": ["es2015", "es2016", "es2017", "dom"],
"sourceMap": true,
"declaration": true,
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"strict": true,
"noFallthroughCasesInSwitch": true,
"noImplicitReturns": false,
"noUnusedLocals": true,
"noUnusedParameters": false,
"pretty": true,
"stripInternal": true,
"types": ["node"]
},
"include": ["./src/**/*.tsx", "./src/**/*.ts"],
"exclude": ["node_modules", "dist", "./test/**/*.spec.ts"]
}
================================================
FILE: packages/plugin-proxy-router/package.json
================================================
{
"name": "@extra/proxy-router",
"version": "3.1.6",
"description": "A plugin for playwright & puppeteer to route proxies dynamically.",
"repository": "berstend/puppeteer-extra",
"homepage": "https://github.com/berstend/puppeteer-extra/tree/master/packages/plugin-proxy-router",
"author": "berstend",
"license": "MIT",
"main": "dist/index.cjs.js",
"module": "dist/index.esm.js",
"typings": "dist/index.d.ts",
"files": [
"dist"
],
"publishConfig": {
"access": "public"
},
"scripts": {
"clean": "rimraf dist/*",
"tscheck": "tsc --pretty --noEmit",
"prebuild": "run-s clean",
"build": "run-s build:tsc build:rollup",
"build:tsc": "tsc --project tsconfig.json --module commonjs",
"build:rollup": "rollup -c rollup.config.ts",
"docs": "node -e 0",
"test": "run-s build",
"pretest-ci": "run-s build",
"test-ci": "run-s build"
},
"engines": {
"node": ">=14"
},
"prettier": {
"printWidth": 80,
"semi": false,
"singleQuote": true
},
"keywords": [
"puppeteer",
"playwright",
"puppeteer-extra",
"playwright-extra",
"proxy",
"proxy-router",
"headless",
"luminati"
],
"devDependencies": {
"@types/debug": "^4.1.5",
"@types/node": "14.17.6",
"@types/puppeteer": "*",
"ava": "2.4.0",
"copyfiles": "^2.1.1",
"npm-run-all": "^4.1.5",
"playwright-core": "1.24.2",
"prettier": "^2.7.1",
"puppeteer": "^15.5.0",
"puppeteer-extra": "^3.3.6",
"replace-in-files-cli": "^0.3.1",
"rimraf": "^3.0.0",
"rollup-plugin-commonjs": "^10.1.0",
"rollup-plugin-node-resolve": "^5.2.0",
"rollup-plugin-sourcemaps": "^0.4.2",
"rollup-plugin-typescript2": "^0.25.2",
"ts-node": "^8.5.4",
"typescript": "^4.7.4"
},
"dependencies": {
"debug": "^4.1.1",
"merge-deep": "^3.0.2",
"proxy-chain": "^2.0.6",
"puppeteer-extra-plugin": "^3.2.3"
},
"peerDependencies": {
"playwright-extra": "*",
"puppeteer-extra": "*"
},
"peerDependenciesMeta": {
"puppeteer-extra": {
"optional": true
},
"playwright-extra": {
"optional": true
}
}
}
================================================
FILE: packages/plugin-proxy-router/readme.md
================================================
# @extra/proxy-router [](https://github.com/berstend/puppeteer-extra/actions) [](https://extra.community) [](https://www.npmjs.com/package/@extra/proxy-router)
> A plugin for [playwright-extra] and [puppeteer-extra] to route proxies dynamically.
## Install
```bash
yarn add @extra/proxy-router
# - or -
npm install @extra/proxy-router
```
<details>
<summary>Playwright</summary>
If this is your first [playwright-extra] plugin here's everything you need:
```bash
yarn add playwright playwright-extra @extra/proxy-router
# - or -
npm install playwright playwright-extra @extra/proxy-router
```
</details>
<details>
<summary>Puppeteer</summary>
If this is your first [puppeteer-extra] plugin here's everything you need:
```bash
yarn add puppeteer puppeteer-extra @extra/proxy-router
# - or -
npm install puppeteer puppeteer-extra @extra/proxy-router
```
</details>
### Compatibility
| 💫 | [<img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/chromium/chromium.png" alt="Chrome" width="24px" height="24px" />](#)<br/>Chromium | [<img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/chrome/chrome_48x48.png" alt="Chrome" width="24px" height="24px" />](#)<br/>Chrome | [<img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/firefox/firefox_48x48.png" alt="Firefox" width="24px" height="24px" />](#)<br/>Firefox | [<img src="https://raw.githubusercontent.com/alrra/browser-logos/master/src/safari/safari_48x48.png" alt="Webkit" width="24px" height="24px" />](#)<br/>Webkit |
| :--------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------: |
| **[playwright-extra]** | ✅ | ✅ | ✅ | ✅ |
| **[puppeteer-extra]** | ✅ | ✅ | [🕒](https://github.com/berstend/puppeteer-extra/wiki/Is-Puppeteer-Firefox-ready-yet%3F) | - |
| Headless | Headful | Launch | Connect |
| :------: | :-----: | :----: | :------------------------------: |
| ✅ | ✅ | ✅ | ✅ <sup><sub>(local)</sub></sup> |
### Features
The plugin makes using proxies in the browser a lot more convenient:
- Handles proxy authentication
- Multiple proxies can be used
- Flexible proxy routing using the host/domain
- Change proxies dynamically after browser launch
- Collect traffic stats per proxy or host
- Uses native browser features, no performance loss
## Usage
> Using puppeteer? To use the following playwright examples simply change your [imports](#imports)
### Simple
A single proxy for all browser connections
```js
// playwright-extra is a drop-in replacement for playwright,
// it augments the installed playwright with plugin functionality
// Note: Instead of chromium you can use firefox and webkit as well.
const { chromium } = require('playwright-extra')
// Configure and add the proxy router plugin with a default proxy
const ProxyRouter = require('@extra/proxy-router')
chromium.use(
ProxyRouter({
proxies: { DEFAULT: 'http://user:pass@proxyhost:port' },
})
)
// That's it, the default proxy will be used and proxy authentication handled automatically
chromium.launch({ headless: false }).then(async (browser) => {
const page = await browser.newPage()
await page.goto('https://canhazip.com', { waitUntil: 'domcontentloaded' })
const ip = await page.evaluate('document.body.innerText')
console.log('Outbound IP:', ip)
await browser.close()
})
```
### Dynamic routing
Use multiple proxies and route connections flexibly
```js
// playwright-extra is a drop-in replacement for playwright,
// it augments the installed playwright with plugin functionality
// Note: Instead of chromium you can use firefox and webkit as well.
const { chromium } = require('playwright-extra')
// Configure the proxy router plugin
const ProxyRouter = require('@extra/proxy-router')
const proxyRouter = ProxyRouter({
// define the available proxies (replace this with your proxies)
proxies: {
// the default browser proxy, can be `null` as well for direct connections
DEFAULT: 'http://user:pass@proxyhost:port',
// optionally define more proxies you can use in `routeByHost`
// you can use whatever names you'd like for them
DATACENTER: 'http://user:pass@proxyhost2:port',
RESIDENTIAL_US: 'http://user:pass@proxyhost3:port',
},
// optional function for flexible proxy routing
// if this is not specified the `DEFAULT` proxy will be used for all connections
routeByHost: async ({ host }) => {
if (['pagead2.googlesyndication.com', 'fonts.gstatic.com'].includes(host)) {
return 'ABORT' // block connection to certain hosts
}
if (host.includes('google')) {
return 'DIRECT' // use a direct connection for all google domains
}
if (host.endsWith('.tile.openstreetmap.org')) {
return 'DATACENTER' // route heavy images through datacenter proxy
}
if (host === 'canhazip.com') {
return 'RESIDENTIAL_US' // special proxy for this domain
}
// everything else will use `DEFAULT` proxy
},
})
// Add the plugin
chromium.use(proxyRouter)
// Launch a browser and run some IP checks
chromium.launch({ headless: true }).then(async (browser) => {
const page = await browser.newPage()
await page.goto('https://showmyip.com/', { waitUntil: 'domcontentloaded' })
const ip1 = await page.evaluate("document.querySelector('#ipv4').innerText")
console.log('Outbound IP #1:', ip1)
// => 77.191.128.0 (the DEFAULT proxy)
await page.goto('https://canhazip.com', { waitUntil: 'domcontentloaded' })
const ip2 = await page.evaluate('document.body.innerText')
console.log('Outbound IP #2:', ip2)
// => 104.179.129.27 (the RESIDENTIAL_US proxy)
console.log(proxyRouter.stats.connectionLog) // list of connections (host => proxy name)
// { id: 0, proxy: 'DIRECT', host: 'accounts.google.com' },
// { id: 1, proxy: 'DEFAULT', host: 'www.showmyip.com' },
// { id: 2, proxy: 'ABORT', host: 'pagead2.googlesyndication.com' },
// { id: 3, proxy: 'DEFAULT', host: 'unpkg.com' },
// ...
console.log(proxyRouter.stats.byProxy) // bytes used by proxy
// {
// DATACENTER: 441734,
// DEFAULT: 125823,
// DIRECT: 100457,
// RESIDENTIAL_US: 4764,
// ABORT: 0
// }
console.log(proxyRouter.stats.byHost) // bytes used by host
// {
// 'a.tile.openstreetmap.org': 150685,
// 'c.tile.openstreetmap.org': 147054,
// 'b.tile.openstreetmap.org': 143995,
// 'unpkg.com': 57621,
// 'www.googletagmanager.com': 49572,
// 'www.showmyip.com': 40408,
// ...
await browser.close()
})
```
### Imports
<details>
<summary>Usage with Puppeteer</summary><br/>
> The code is essentially the same as the playwright example above. :-)
Just change the import and package name:
```diff
- const { chromium } = require('playwright-extra')
+ const puppeteer = require('puppeteer-extra')
// ...
- chromium.use(proxyRouter)
+ puppeteer.use(proxyRouter)
// ...
- chromium.launch()
+ puppeteer.launch()
// ...
```
</details>
<details>
<summary>Typescript & ESM</summary>
<br/>
> The plugin is written in Typescript and ships with types.
**Playwright:**
```js
// You can use any browser: chromium, firefox, webkit
import { firefox } from 'playwright-extra'
import ProxyRouter from '@extra/proxy-router'
// ...
firefox.use(proxyRouter)
```
**Puppeteer:**
```js
import puppeteer from 'puppeteer-extra'
import ProxyRouter from '@extra/proxy-router'
// ...
puppeteer.use(proxyRouter)
```
</details>
### Debug logs
If you'd like to see debug output just run your script like so:
```bash
# macOS/Linux (Bash)
DEBUG=*proxy-router* node myscript.js
# Windows (Powershell)
$env:DEBUG='*proxy-router*';node myscript.js
```
## How it works
The proxy router will launch a local proxy server and instruct the browser to use it.
That local proxy server will in turn connect to the configured upstream proxy servers and relay connections depending on the optional user-defined routing function, while handling upstream proxy authentication and a few other things.
## API
### Options
```ts
export interface ProxyRouterOpts {
/**
* A dictionary of proxies to be made available to the browser and router.
*
* An optional entry named `DEFAULT` will be used for all requests, unless overriden by `routeByHost`.
* If the `DEFAULT` entry is omitted no proxy will be used by default.
*
* The value of an entry can be a string (format: `http://user:pass@proxyhost:port`) or `null` (direct connection).
* Proxy authentication is handled automatically by the router.
*
* @example
* proxies: {
* DEFAULT: "http://user:pass@proxyhost:port", // use this proxy by default
* RESIDENTIAL_US: "http://user:pass@proxyhost2:port" // use this for specific hosts with `routeByHost`
* }
*/
proxies?: {
/**
* The default proxy for the browser (format: `http://user:pass@proxyhost:port`),
* if omitted or `null` no proxy will be used by default
*/
DEFAULT?: string | null
/**
* Any other custom proxy names which can be used for routing later
* (e.g. `'DATACENTER_US': 'http://user:pass@proxyhost:port'`)
*/
[key: string]: string | null
}
/**
* An optional function to allow proxy routing based on the target host of the request.
*
* A return value of nothing, `null` or `DEFAULT` will result in the DEFAULT proxy being used as configured.
* A return value of `DIRECT` will result in no proxy being used.
* A return value of `ABORT` will cancel/block this request.
*
* Any other string as return value is assumed to be a reference to the configured `proxies` dict.
*
* @note The browser will most often establish only a single proxy connection per host.
*
* @example
* routeByHost: async ({ host }) => {
* if (host.includes('google')) { return "DIRECT" }
* return 'RESIDENTIAL_US'
* }
*
*/
routeByHost?: RouteByHostFn
/** Collect traffic and connection stats, default: true */
collectStats?: boolean
/** Don't print any proxy connection errors to stderr, default: false */
muteProxyErrors?: boolean
/** Suppress proxy errors for specific hosts */
muteProxyErrorsForHost?: string[]
/** Options for the local proxy-chain server */
proxyServerOpts?: ProxyServerOpts
/**
* Optionally exempt hosts from going through a proxy, even our internal routing proxy.
*
* Examples:
* `.com` or `chromium.org` or `.domain.com`
*
* @see
* https://chromium.googlesource.com/chromium/src/+/HEAD/net/docs/proxy.md#proxy-bypass-rules
* https://www-archive.mozilla.org/quality/networking/docs/aboutno_proxy_for.html
*/
proxyBypassList?: string[]
}
```
## Alternatives
### Proxy.pac files <sup><sub>[Reference](https://developer.mozilla.org/en-US/docs/Web/HTTP/Proxy_servers_and_tunneling/Proxy_Auto-Configuration_PAC_file)</sub></sup>
- Only supported in chromium in headful mode
- Despite the name (`FindProxyForURL`) can only route by host
- Firefox supports PAC files and including the path through a pref
- Only loaded once at browser launch, no dynamic proxies possible
- Does not handle authentication
### Various "per-page proxy" plugins for puppeteer
- Advantage: Route proxies by page not host
- They rely on a massive hack: Using Node.js to send the requests instead of the browser
- Will change the TLS fingerprint, error prone
- Uses CDP request interception which is chromium only
- Increased latency and resource overhead
## License
Copyright © 2018 - 2023, [berstend̡̲̫̹̠̖͚͓̔̄̓̐̄͛̀͘](https://github.com/berstend). Released under the MIT License.
<!--
Reference links
-->
[playwright-extra]: https://github.com/berstend/puppeteer-extra/tree/master/packages/playwright-extra
[puppeteer-extra]: https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra
================================================
FILE: packages/plugin-proxy-router/rollup.config.ts
================================================
import resolve from 'rollup-plugin-node-resolve'
import sourceMaps from 'rollup-plugin-sourcemaps'
import typescript from 'rollup-plugin-typescript2'
const pkg = require('./package.json')
const entryFile = 'index'
const banner = `
/*!
* ${pkg.name} v${pkg.version} by ${pkg.author}
* ${pkg.homepage || `https://github.com/${pkg.repository}`}
* @license ${pkg.license}
*/
`.trim()
const defaultExportOutro = `
module.exports = exports.default || {}
Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value })
`
export default {
input: `src/${entryFile}.ts`,
output: [
{
file: pkg.main,
format: 'cjs',
sourcemap: true,
exports: 'named',
outro: defaultExportOutro,
banner
},
{
file: pkg.module,
format: 'es',
sourcemap: true,
exports: 'named',
banner
}
],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [
...Object.keys(pkg.dependencies || {}),
...Object.keys(pkg.peerDependencies || {})
],
watch: {
include: 'src/**'
},
plugins: [
// Compile TypeScript files
typescript({ useTsconfigDeclarationDir: true }),
// Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
// commonjs(),
// Allow node_modules resolution, so you can use 'external' to control
// which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve(),
// Resolve source maps to the original source
sourceMaps()
]
}
================================================
FILE: packages/plugin-proxy-router/src/index.ts
================================================
import { ExtraPluginProxyRouter, ExtraPluginProxyRouterOptions } from './plugin'
export * from './plugin'
export * from './router'
export * from './stats'
/** Default export, ExtraPluginProxyRouter */
const defaultExport = (options?: Partial<ExtraPluginProxyRouterOptions>) => {
return new ExtraPluginProxyRouter(options || {})
}
export default defaultExport
================================================
FILE: packages/plugin-proxy-router/src/plugin.ts
================================================
import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin'
import { ProxyRouter, ProxyRouterOpts } from './router'
export type ExtraPluginProxyRouterOptions = ProxyRouterOpts & {
/**
* Optionally exempt hosts from going through a proxy, even our internal routing proxy.
*
* Examples:
* `.com` or `chromium.org` or `.domain.com`
*
* @see
* https://chromium.googlesource.com/chromium/src/+/HEAD/net/docs/proxy.md#proxy-bypass-rules
* https://www-archive.mozilla.org/quality/networking/docs/aboutno_proxy_for.html
*/
proxyBypassList?: string[]
}
export class ExtraPluginProxyRouter extends PuppeteerExtraPlugin {
/** The underlying proxy router instance */
public router: ProxyRouter
/** The name of the automation framework used */
public framework: 'playwright' | 'puppeteer' | null = null
// Disable the puppeteer compat shim when used with playwright-extra
public noPuppeteerShim = true
constructor(opts: Partial<ExtraPluginProxyRouterOptions>) {
super(opts)
this.debug('Initialized', this.opts)
this.router = new ProxyRouter(this.opts)
}
get name() {
return 'proxy-router'
}
get defaults(): ExtraPluginProxyRouterOptions {
return {
collectStats: true,
proxyServerOpts: {
port: 2800,
},
}
}
// Make accessing router methods shorter
/** Get or set proxies at runtime */
public get proxies() {
return this.router.proxies
}
public set proxies(proxies) {
this.router.proxies = proxies
}
/** Retrieve traffic statistics */
public get stats() {
return this.router.stats
}
/** Get or set the `routeByHost` function at runtime */
public get routeByHost() {
return this.router.routeByHost
}
public set routeByHost(fn) {
this.router.routeByHost = fn
}
private get proxyBypassListString() {
return (this.opts.proxyBypassList || []).join(',') || undefined
}
async onPluginRegistered(args?: { framework: 'playwright' }): Promise<void> {
this.framework =
args?.framework === 'playwright' ? 'playwright' : 'puppeteer'
this.debug('plugin registered', this.framework)
}
async beforeLaunch(options: unknown = {}): Promise<void> {
this.debug('beforeLaunch - before', options)
await this.router.listen()
const proxyUrl = this.router.proxyServerUrl
if (!proxyUrl) {
throw new Error('No local proxy server available')
}
if (this.framework === 'playwright') {
const pwOptions = options as PlaywrightLaunchOptions
pwOptions.proxy = {
server: proxyUrl,
bypass: this.proxyBypassListString,
}
} else if (this.framework === 'puppeteer') {
const pptrOptions = options as PuppeteerLaunchOptions
pptrOptions.args = pptrOptions.args || []
pptrOptions.args.push(`--proxy-server=${proxyUrl}`)
if (this.proxyBypassListString) {
pptrOptions.args.push(
`--proxy-bypass-list=${this.proxyBypassListString}`
)
}
} else {
this.debug('Unsupported framework, not setting proxy')
}
this.debug('beforeLaunch - after', options)
}
async onDisconnected(): Promise<void> {
await this.router.close().catch(this.debug)
}
}
interface PuppeteerLaunchOptions {
args?: string[]
}
interface PlaywrightLaunchOptions {
proxy?: {
/**
* Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or
* `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy.
*/
server: string
/**
* Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`.
*/
bypass?: string
}
}
================================================
FILE: packages/plugin-proxy-router/src/router.ts
================================================
import { Server as ProxyServer, RequestError, redactUrl } from 'proxy-chain'
import type * as ProxyChain from 'proxy-chain'
import getPort from './utils/port'
import { ProxyRouterStats } from './stats'
import Debug from 'debug'
const debug = Debug('puppeteer-extra:proxy-router')
const debugVerbose = debug.extend('verbose')
const warn = console.warn.bind(console, `\n[proxy-router] %s`) // Preserves line numbers
type ProxyServerOpts = ConstructorParameters<typeof ProxyServer>[0]
export interface Proxies {
/** The default proxy for the browser (format: `http://user:pass@proxyhost:port`), if omitted or `null` no proxy will be used by default */
DEFAULT?: string | null
/** Any other custom proxy names which can be used for routing later (e.g. `'DATACENTER_US': 'http://user:pass@proxyhost:port'`) */
[key: string]: string | null
}
export type ProxyName = 'DIRECT' | 'DEFAULT' | 'ABORT' | string
/** Data available to the `routeByHost` function */
export interface RouteByHostArgs {
/** Request URL host */
host: string
/** Whether the request is http or not */
isHttp: boolean
/** Request port (typically 443 or 80) */
port: number
}
export type RouteByHostResponse = ProxyName | void
export type RouteByHostFn = (
args: RouteByHostArgs
) => Promise<RouteByHostResponse>
export interface ProxyRouterOpts {
/**
* A dictionary of proxies to be made available to the browser and router.
*
* An optional entry named `DEFAULT` will be used for all requests, unless overriden by `routeByHost`.
* If the `DEFAULT` entry is omitted no proxy will be used by default.
*
* The value of an entry can be a string (format: `http://user:pass@proxyhost:port`) or `null` (direct connection).
* Proxy authentication is handled automatically by the router.
*
* @example
* proxies: {
* DEFAULT: "http://user:pass@proxyhost:port", // use this proxy by default
* RESIDENTIAL_US: "http://user:pass@proxyhost2:port" // use this for specific hosts with `routeByHost`
* }
*/
proxies?: Proxies
/**
* An optional function to allow proxy routing based on the target host of the request.
*
* A return value of nothing, `null` or `DEFAULT` will result in the DEFAULT proxy being used as configured.
* A return value of `DIRECT` will result in no proxy being used.
* A return value of `ABORT` will cancel/block this request.
*
* Any other string as return value is assumed to be a reference to the configured `proxies` dict.
*
* @note The browser will most often establish only a single proxy connection per host.
*
* @example
* routeByHost: async ({ host }) => {
* if (host.includes('google')) { return "DIRECT" }
* return 'RESIDENTIAL_US'
* }
*
*/
routeByHost?: RouteByHostFn
/** Collect traffic and connection stats, default: true */
collectStats?: boolean
/** Don't print any proxy connection errors to stderr, default: false */
muteProxyErrors?: boolean
/** Suppress proxy errors for specific hosts */
muteProxyErrorsForHost?: string[]
/** Options for the local proxy-chain server */
proxyServerOpts?: ProxyServerOpts
}
export class ProxyRouter {
/** The underlying local proxy server used for routing to upstream proxies */
public proxyServer: ProxyChain.Server
/** An optional function to route hosts */
public routeByHost: RouteByHostFn | null
/**
* The dictionary of proxies made available (format: `FOOBAR: 'http://user:pass@proxyhost:port'`).
* Can be modified at runtime.
*/
public proxies: Proxies
/** Traffic stats collector */
public readonly stats: ProxyRouterStats
public isListening: boolean = false
protected serverStartPromise: Promise<number> | null
protected collectStats: boolean
protected muteProxyErrors: boolean
protected muteProxyErrorsForHost: string[]
/** Internal list of failed connections to only print the same connection issue once */
protected failedConnections: { host: string; proxy: string }[] = []
constructor(opts: ProxyRouterOpts = {}) {
const proxyServerOpts: ProxyServerOpts = {
...opts.proxyServerOpts,
prepareRequestFunction: this.handleProxyServerRequest.bind(this),
}
proxyServerOpts.port = proxyServerOpts.port || 2800
this.proxies = opts.proxies || {}
this.routeByHost = opts.routeByHost || null
this.proxyServer = new ProxyServer(proxyServerOpts)
this.collectStats = opts.collectStats ?? true
this.stats = new ProxyRouterStats(this.proxyServer)
this.muteProxyErrors = opts.muteProxyErrors ?? false
this.muteProxyErrorsForHost = opts.muteProxyErrorsForHost || []
debug('initialized', opts)
// Emitted when HTTP connection is closed
this.proxyServer.on('connectionClosed', ({ connectionId, stats }) => {
if (stats && this.collectStats) {
this.stats.addStats(connectionId as number, stats)
}
debugVerbose(`Connection ${connectionId} closed`)
})
// Emitted when a HTTP request fails
this.proxyServer.on('requestFailed', ({ request, error }) => {
if (!this.muteProxyErrors) {
warn('Request failed:', request.url, error)
}
})
// Emitted in case of a upstream proxy error (which can mean various things)
this.proxyServer.on(
'proxyAuthenticationFailed',
({
connectionId,
str: errorStr,
}: {
connectionId: unknown
str: string
}) => {
// resolve the affected host and proxy
const { host, proxy } =
this.stats.connectionLog.find(({ id }) => id === connectionId) || {}
const proxyUrl = !!proxy ? this.getProxyForName(proxy) : null
const info: string[] = [errorStr]
info.push(
"This error can be thrown if a resource on a site simply can't be accessed (often temporarily), in this case this can be ignored.",
` - To not have errors like this printed to the console you can set 'muteProxyErrors: true' ${
!!host ? `or 'muteProxyErrorsForHost: ["${host}"]'` : ''
}`,
'It can also indicate incorrect proxy credentials or that the target host is blocked by the proxy.',
' - Make sure the provided proxy string and credentials are correct and the site is not blocked by the proxy (or vice versa).',
" - In case the site is blocked by the proxy: Use 'routeByHost' to route the host through a different proxy or as 'DIRECT' or 'ABORT'."
)
if (host && proxy) {
info.push(
'',
`Affected target host: "${host}"`,
`Affected proxy name: "${proxy}"`
)
}
if (proxyUrl) {
info.push(`Affected proxy URL: "${proxyUrl}"`)
info.push(
'',
`To test the proxy with curl: curl -v --proxy '${proxyUrl}' 'https://${host}'`,
''
)
if (!`${proxyUrl}`.includes('http://')) {
info.push('PS: Did you forget to prefix the proxy with "http://"?')
}
}
const probablyNoise =
errorStr.includes('authenticate') && errorStr.includes('522')
const isMuted =
this.muteProxyErrors || this.muteProxyErrorsForHost.includes(host)
const alreadySeen = !!this.failedConnections.find(
(entry) => entry.host === host && entry.proxy === proxy
)
const logger = probablyNoise || isMuted || alreadySeen ? debug : warn
logger(info.join('\n'))
if (host && proxy) {
this.failedConnections.push({ host, proxy })
}
}
)
// Resurface some errors that proxy-chain seems to swallow
this.proxyServer.log = (function (originalMethod, context) {
return function (connectionId: unknown, str: string) {
if (`${str}`.includes('Failed to authenticate upstream proxy')) {
context.emit('proxyAuthenticationFailed', {
connectionId,
str,
})
}
if (`${str}`.includes('Error: Invalid "upstreamProxyUrl" provided')) {
context.emit('proxyAuthenticationFailed', {
connectionId,
str,
})
}
if (`${str}`.includes('Failed to connect to upstream proxy')) {
context.emit('proxyAuthenticationFailed', {
connectionId,
str,
})
}
originalMethod.apply(context, [connectionId, str])
}
})(this.proxyServer.log, this.proxyServer)
}
/** Proxy server URL of the local proxy server used for routing */
public get proxyServerUrl() {
const port = this.proxyServer?.port
if (!port || !this.isListening) {
return
}
return `http://localhost:${port}`
}
public get effectiveProxies() {
return {
DIRECT: null,
...(this.proxies || {}),
}
}
/** Start the local proxy server and accept connections */
public async listen(): Promise<number> {
debug('starting server..')
if (this.serverStartPromise) {
debug('server start promise exists already')
return this.serverStartPromise
}
this.serverStartPromise = new Promise(async (resolve) => {
if (this.isListening) {
debug('server listening already')
return resolve(this.proxyServer.port)
}
const desiredPort = this.proxyServer.port
debug('finding available port', { desiredPort })
const availablePort = await getPort({ port: desiredPort })
debug('availablePort:', availablePort)
this.proxyServer.port = availablePort
this.proxyServer.listen((err) => {
if (err === null) {
debug(`server listening on port ${this.proxyServer.port}`)
this.isListening = true
return resolve(this.proxyServer.port)
}
warn('Unable to start local server:', err)
})
})
return this.serverStartPromise
}
/** Stop the local proxy server */
public async close(): Promise<NodeJS.ErrnoException | null> {
debug('closing..')
return new Promise((resolve) => {
this.proxyServer.close(true, (err) => {
if (err === null) {
debug('closed without error')
return resolve(null)
}
debug('closed with error', err)
return resolve(err)
})
})
}
public getProxyForName(name: ProxyName): string | null {
return this.effectiveProxies[name]
}
/** Handle requests to the proxy server */
protected async handleProxyServerRequest({
request,
hostname: host,
port,
connectionId,
isHttp,
}: ProxyChain.PrepareRequestFunctionOpts): Promise<void | ProxyChain.PrepareRequestFunctionResult> {
let proxyName = 'DEFAULT'
if (!!this.routeByHost) {
const fnResult = await this.routeByHost({ host, isHttp, port })
if (typeof fnResult === 'string' && !!fnResult) {
proxyName = fnResult
}
}
if (this.collectStats) {
this.stats.addConnection(connectionId, proxyName, host)
}
let proxyUrl = this.getProxyForName(proxyName)
debugVerbose(
'handleProxyServerRequest',
host,
proxyName,
redactProxyUrl(proxyUrl)
)
if (proxyName === 'ABORT') {
throw new RequestError('Request aborted', 400)
}
if (!proxyUrl && proxyUrl !== null) {
warn(
`No proxy configured for proxy name "${proxyName}" - configuration error?`
)
proxyUrl = null
}
return {
upstreamProxyUrl: proxyUrl,
}
}
}
function redactProxyUrl(input: unknown) {
if (!input || typeof input !== 'string') {
return `${input}`
}
try {
return redactUrl(input)
} catch (err) {
return `${input}`
}
}
/** Standalone proxy router not requiring plugin events */
export const ProxyRouterStandalone = ProxyRouter
================================================
FILE: packages/plugin-proxy-router/src/stats.ts
================================================
import type { Server as ProxyServer } from 'proxy-chain'
export interface ConnectionLogEntry {
/** Connection Id */
id: number
/** Proxy name */
proxy: string
/** Host */
host: string
}
export interface ConnectionStats {
srcTxBytes: number
srcRxBytes: number
trgTxBytes: number
trgRxBytes: number
}
export class ProxyRouterStats {
/** Log of all connections (id, proxyName, host) */
public connectionLog: ConnectionLogEntry[] = []
protected connectionStats: Map<number, ConnectionStats> = new Map()
constructor(private proxyServer: ProxyServer) {}
/** @internal */
public addConnection(id: number, proxy: string, host: string) {
this.connectionLog.push({ id, proxy, host })
}
/** @internal */
public addStats(connectionId: number, stats: ConnectionStats) {
this.connectionStats.set(connectionId as number, stats)
}
/** Get bytes transferred by proxy */
public get byProxy() {
this.getStatsFromActiveConnections()
// Get unique proxy names from our actual connection logs
const proxyNames = Array.from(
new Set(this.connectionLog.map(({ proxy }) => proxy))
)
const getConnectionIdsForProxy = (proxyName: string) =>
this.connectionLog
.filter(({ proxy }) => proxy === proxyName)
.map(({ id }) => id)
const trafficByProxy = Object.fromEntries(
proxyNames
.map((proxyName) => {
const ids = getConnectionIdsForProxy(proxyName)
const stats = ids.map((id) => this.connectionStats.get(id))
const totalBytes = stats
.map((stat) => this.calculateProxyBytes(stat))
.reduce((a, b) => a + b)
return [proxyName, totalBytes]
})
// Sort by most bytes on top
.sort((a, b) => (b[1] as number) - (a[1] as number))
)
return trafficByProxy
}
/** Get bytes transferred by host */
public get byHost() {
this.getStatsFromActiveConnections()
// Get unique proxy names from our actual connection logs
const hostNames = Array.from(
new Set(this.connectionLog.map(({ host }) => host))
)
const getConnectionIdsForHost = (hostName: string) =>
this.connectionLog
.filter(({ host }) => host === hostName)
.map(({ id }) => id)
const trafficByHost = Object.fromEntries(
hostNames
.map((hostName) => {
const ids = getConnectionIdsForHost(hostName)
const stats = ids.map((id) => this.connectionStats.get(id))
const totalBytes = stats
.map((stat) => this.calculateProxyBytes(stat))
.reduce((a, b) => a + b)
return [hostName, totalBytes]
})
// Sort by most bytes on top
.sort((a, b) => (b[1] as number) - (a[1] as number))
)
return trafficByHost
}
protected getStatsFromActiveConnections() {
// collect stats for active connections
this.proxyServer.getConnectionIds().forEach((connectionId) => {
const stats = this.proxyServer.getConnectionStats(connectionId)
if (stats) {
this.connectionStats.set(connectionId as number, stats)
}
})
}
protected calculateProxyBytes(stats?: Partial<ConnectionStats>) {
if (!stats) {
return 0
}
return (stats.trgRxBytes || 0) + (stats.trgTxBytes || 0)
}
}
================================================
FILE: packages/plugin-proxy-router/src/utils/port.ts
================================================
import net from 'net'
export interface Options {
/**
* A preferred port or an array of preferred ports to use.
*/
port?: number | ReadonlyArray<number>
/**
* The host on which port resolution should be performed. Can be either an IPv4 or IPv6 address.
*/
host?: string
}
const isAvailable = (options: Options): Promise<number> =>
new Promise((resolve, reject) => {
const server = net.createServer()
server.unref()
server.on('error', reject)
server.listen(options, () => {
const { port } = server.address() as any
server.close(() => {
resolve(port as number)
})
})
})
const getPort = (options: Options) => {
options = Object.assign({}, options)
if (typeof options.port === 'number') {
options.port = [options.port]
}
return (options.port || []).reduce(
(seq, port) =>
seq.catch(() => isAvailable(Object.assign({}, options, { port }))),
Promise.reject()
)
}
export default (options?: Options) =>
options
? getPort(options).catch(() => getPort(Object.assign(options, { port: 0 })))
: getPort({ port: 0 })
================================================
FILE: packages/plugin-proxy-router/tsconfig.json
================================================
{
"compilerOptions": {
"outDir": "./dist",
"target": "es2017",
"module": "es2015",
"moduleResolution": "node",
"lib": ["es2015", "es2016", "es2017", "es2019", "dom"],
// "noResolve": true, // Important: Otherwise TS would rewrite our ambient d.ts file locations (see: yarn copy-dts) :(
"sourceMap": true,
"declaration": true,
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"strict": false,
"noFallthroughCasesInSwitch": true,
"noImplicitReturns": false,
"noUnusedLocals": true,
"noUnusedParameters": false,
"pretty": true,
"stripInternal": true,
"types": ["node"]
},
"include": [
"./src/**/*.tsx",
"./src/**/*.ts",
"./src/**/*.d.ts",
"./src/**/*.test.ts",
"./test/**/*.ts"
],
"exclude": ["node_modules", "dist", "./test/**/*.spec.ts"]
}
================================================
FILE: packages/plugin-proxy-router/tslint.json
================================================
{
"extends": ["tslint-config-standard", "tslint-config-prettier"],
"rules": {
"ordered-imports": true
}
}
================================================
FILE: packages/puppeteer-extra/ava.config-ts.js
================================================
export default {
compileEnhancements: false,
environmentVariables: {
TS_NODE_COMPILER_OPTIONS: '{"module":"commonjs"}'
},
files: ['test/*.ts'],
extensions: ['ts'],
require: ['ts-node/register']
}
================================================
FILE: packages/puppeteer-extra/ava.config.js
================================================
export default {
files: ['test/*.js']
}
================================================
FILE: packages/puppeteer-extra/package.json
================================================
{
"name": "puppeteer-extra",
"version": "3.3.6",
"description": "Teach puppeteer new tricks through plugins.",
"repository": "berstend/puppeteer-extra",
"author": "berstend",
"license": "MIT",
"typings": "dist/index.d.ts",
"main": "dist/index.cjs.js",
"module": "dist/index.esm.js",
"files": [
"dist"
],
"scripts": {
"clean": "rimraf dist/*",
"prebuild": "run-s clean",
"build": "run-s build:tsc build:rollup ambient-dts",
"build:tsc": "tsc --module commonjs",
"build:rollup": "rollup -c rollup.config.ts",
"docs": "documentation readme --quiet --shallow --github --markdown-theme transitivebs --readme-file readme.md --section API ./src/index.ts",
"postdocs": "npx prettier --write readme.md",
"test:ts": "ava -v --config ava.config-ts.js",
"test:js": "ava -v --serial --concurrency 1 --fail-fast",
"test": "run-p test:js test:ts",
"test-ci": "run-s test",
"ambient-dts": "run-s ambient-dts-copy ambient-dts-fix-path",
"ambient-dts-copy": "copyfiles -u 1 \"src/**/*.d.ts\" dist",
"ambient-dts-fix-path": "replace-in-files --string='/// <reference path=\"../src/' --replacement='/// <reference path=\"../dist/' 'dist/**/*.d.ts'"
},
"keywords": [
"puppeteer",
"puppeteer-extra",
"flash",
"stealth",
"prefs",
"user-preferences",
"chrome",
"headless",
"pupeteer"
],
"engines": {
"node": ">=8"
},
"devDependencies": {
"@types/node": "^18.0.0",
"@types/puppeteer": "*",
"ava": "^2.4.0",
"documentation-markdown-themes": "^12.1.5",
"npm-run-all": "^4.1.5",
"puppeteer": "^10.2.0",
"puppeteer-extra-plugin": "^3.2.3",
"puppeteer-extra-plugin-anonymize-ua": "^2.4.6",
"rimraf": "^3.0.0",
"rollup": "^1.27.5",
"rollup-plugin-commonjs": "^10.1.0",
"rollup-plugin-node-resolve": "^5.2.0",
"rollup-plugin-sourcemaps": "^0.4.2",
"rollup-plugin-typescript2": "^0.25.2",
"ts-node": "^8.5.4",
"tslint": "^5.20.1",
"tslint-config-prettier": "^1.18.0",
"tslint-config-standard": "^9.0.0",
"typescript": "4.4.3"
},
"dependencies": {
"@types/debug": "^4.1.0",
"debug": "^4.1.1",
"deepmerge": "^4.2.2"
},
"peerDependencies": {
"@types/puppeteer": "*",
"puppeteer": "*",
"puppeteer-core": "*"
},
"peerDependenciesMeta": {
"puppeteer": {
"optional": true
},
"puppeteer-core": {
"optional": true
},
"@types/puppeteer": {
"optional": true
}
},
"gitHead": "babb041828cab50c525e0b9aab02d58f73416ef3"
}
================================================
FILE: packages/puppeteer-extra/readme.md
================================================
# puppeteer-extra [](https://github.com/berstend/puppeteer-extra/actions) [](https://extra.community) [](https://www.npmjs.com/package/puppeteer-extra) [](https://www.npmjs.com/package/puppeteer-extra) [](https://www.npmjs.com/package/puppeteer-extra)
> A light-weight wrapper around [`puppeteer`](https://github.com/GoogleChrome/puppeteer) and [friends](#more-examples) to enable cool [plugins](#plugins) through a clean interface.
<a href="https://github.com/berstend/puppeteer-extra"><img src="https://i.imgur.com/qtlnoQL.png" width="279px" height="187px" align="right" /></a>
## Installation
```bash
yarn add puppeteer puppeteer-extra
# - or -
npm install puppeteer puppeteer-extra
# puppeteer-extra works with any puppeteer version:
yarn add puppeteer@2.0.0 puppeteer-extra
```
## Quickstart
```js
// puppeteer-extra is a drop-in replacement for puppeteer,
// it augments the installed puppeteer with plugin functionality.
// Any number of plugins can be added through `puppeteer.use()`
const puppeteer = require('puppeteer-extra')
// Add stealth plugin and use defaults (all tricks to hide puppeteer usage)
const StealthPlugin = require('puppeteer-extra-plugin-stealth')
puppeteer.use(StealthPlugin())
// Add adblocker plugin to block all ads and trackers (saves bandwidth)
const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker')
puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
// That's it, the rest is puppeteer usage as normal 😊
puppeteer.launch({ headless: true }).then(async browser => {
const page = await browser.newPage()
await page.setViewport({ width: 800, height: 600 })
console.log(`Testing adblocker plugin..`)
await page.goto('https://www.vanityfair.com')
await page.waitForTimeout(1000)
await page.screenshot({ path: 'adblocker.png', fullPage: true })
console.log(`Testing the stealth plugin..`)
await page.goto('https://bot.sannysoft.com')
await page.waitForTimeout(5000)
await page.screenshot({ path: 'stealth.png', fullPage: true })
console.log(`All done, check the screenshots. ✨`)
await browser.close()
})
```
The above example uses the [`stealth`](/packages/puppeteer-extra-plugin-stealth) and [`adblocker`](/packages/puppeteer-extra-plugin-adblocker) plugin, which need to be installed as well:
```bash
yarn add puppeteer-extra-plugin-stealth puppeteer-extra-plugin-adblocker
# - or -
npm install puppeteer-extra-plugin-stealth puppeteer-extra-plugin-adblocker
```
If you'd like to see debug output just run your script like so:
```bash
DEBUG=puppeteer-extra,puppeteer-extra-plugin:* node myscript.js
```
### More examples
<details>
<summary><strong>TypeScript usage</strong></summary><br/>
> `puppeteer-extra` and most plugins are written in TS,
> so you get perfect type support out of the box. :)
```ts
import puppeteer from 'puppeteer-extra'
import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
puppeteer.use(AdblockerPlugin()).use(StealthPlugin())
puppeteer
.launch({ headless: false, defaultViewport: null })
.then(async browser => {
const page = await browser.newPage()
await page.goto('https://bot.sannysoft.com')
await page.waitForTimeout(5000)
await page.screenshot({ path: 'stealth.png', fullPage: true })
await browser.close()
})
```
> Please check this [wiki](https://github.com/berstend/puppeteer-extra/wiki/TypeScript-usage) entry in case you have TypeScript related import issues.
</details>
<details>
<summary><strong>Playwright usage</strong></summary><br/>
[`playright-extra`](/packages/playwright-extra) with plugin support is available as well.
</details>
<details>
<summary><strong>Multiple puppeteers with different plugins</strong></summary><br/>
```js
const vanillaPuppeteer = require('puppeteer')
const { addExtra } = require('puppeteer-extra')
const AnonymizeUA = require('puppeteer-extra-plugin-anonymize-ua')
async function main() {
const pptr1 = addExtra(vanillaPuppeteer)
pptr1.use(
AnonymizeUA({
customFn: ua => 'Hello1/' + ua.replace('Chrome', 'Beer')
})
)
const pptr2 = addExtra(vanillaPuppeteer)
pptr2.use(
AnonymizeUA({
customFn: ua => 'Hello2/' + ua.replace('Chrome', 'Beer')
})
)
await checkUserAgent(pptr1)
await checkUserAgent(pptr2)
}
main()
async function checkUserAgent(pptr) {
const browser = await pptr.launch({ headless: true })
const page = await browser.newPage()
await page.goto('https://httpbin.org/headers', {
waitUntil: 'domcontentloaded'
})
const content = await page.content()
console.log(content)
await browser.close()
}
```
</details>
<details>
<summary><strong>Using with <code>puppeteer-cluster</code></strong></summary><br/>
> [puppeteer-cluster](https://github.com/thomasdondorf/puppeteer-cluster) allows you to create a cluster of puppeteer workers and plays well together with `puppeteer-extra`.
```js
const { Cluster } = require('puppeteer-cluster')
const vanillaPuppeteer = require('puppeteer')
const { addExtra } = require('puppeteer-extra')
const Stealth = require('puppeteer-extra-plugin-stealth')
const Recaptcha = require('puppeteer-extra-plugin-recaptcha')
async function main() {
// Create a custom puppeteer-extra instance using `addExtra`,
// so we could create additional ones with different plugin config.
const puppeteer = addExtra(vanillaPuppeteer)
puppeteer.use(Stealth())
puppeteer.use(Recaptcha())
// Launch cluster with puppeteer-extra
const cluster = await Cluster.launch({
puppeteer,
maxConcurrency: 2,
concurrency: Cluster.CONCURRENCY_CONTEXT
})
// Define task handler
await cluster.task(async ({ page, data: url }) => {
await page.goto(url)
const { hostname } = new URL(url)
const { captchas } = await page.findRecaptchas()
console.log(`Found ${captchas.length} captcha on ${hostname}`)
await page.screenshot({ path: `${hostname}.png`, fullPage: true })
})
// Queue any number of tasks
cluster.queue('https://bot.sannysoft.com')
cluster.queue('https://www.google.com/recaptcha/api2/demo')
cluster.queue('http://www.wikipedia.org/')
await cluster.idle()
await cluster.close()
console.log(`All done, check the screenshots. ✨`)
}
// Let's go
main().catch(console.warn)
```
For using with TypeScript, just change your imports to:
```ts
import { Cluster } from 'puppeteer-cluster'
import vanillaPuppeteer from 'puppeteer'
import { addExtra } from 'puppeteer-extra'
import Stealth from 'puppeteer-extra-plugin-stealth'
import Recaptcha from 'puppeteer-extra-plugin-recaptcha'
```
</details>
<details>
<summary><strong>Using with <code>chrome-aws-lambda</code></strong></summary><br/>
> If you plan to use [chrome-aws-lambda](https://github.com/alixaxel/chrome-aws-lambda) with the [`stealth`](/packages/puppeteer-extra-plugin-stealth) plugin, you'll need to modify the default args to remove the
> `--disable-notifications` flag to pass all the tests.
```js
const chromium = require('chrome-aws-lambda')
const { addExtra } = require('puppeteer-extra')
const puppeteerExtra = addExtra(chromium.puppeteer)
const launch = async () => {
puppeteerExtra
.launch({
args: chromium.args,
defaultViewport: chromium.defaultViewport,
executablePath: await chromium.executablePath,
headless: chromium.headless
})
.then(async browser => {
const page = await browser.newPage()
await page.goto('https://www.spacejam.com/archive/spacejam/movie/jam.htm')
await page.waitForTimeout(10 * 1000)
await browser.close()
})
}
launch() // Launch Browser
```
</details>
<details>
<summary><strong>Using with <code>Kikobeats/browserless</code></strong></summary><br/>
> [Kikobeats/browserless](https://github.com/Kikobeats/browserless) is a puppeteer-like Node.js library for interacting with Headless production scenarios.
```js
const puppeteer = require('puppeteer-extra')
const StealthPlugin = require('puppeteer-extra-plugin-stealth')
puppeteer.use(StealthPlugin())
const browserless = require('browserless')({ puppeteer })
const saveBufferToFile = (buffer, fileName) => {
const wstream = require('fs').createWriteStream(fileName)
wstream.write(buffer)
wstream.end()
}
browserless
.screenshot('https://bot.sannysoft.com', { device: 'iPhone 6' })
.then(buffer => {
const fileName = 'screenshot.png'
saveBufferToFile(buffer, fileName)
console.log(`your screenshot is here: `, fileName)
})
```
</details>
---
## Plugins
#### 🔥 [`puppeteer-extra-plugin-stealth`](/packages/puppeteer-extra-plugin-stealth)
- Applies various evasion techniques to make detection of puppeteer harder.
#### 🏴 [`puppeteer-extra-plugin-recaptcha`](/packages/puppeteer-extra-plugin-recaptcha)
- Solves reCAPTCHAs and hCaptchas automatically, using a single line of code: `page.solveRecaptchas()`.
#### [`puppeteer-extra-plugin-adblocker`](/packages/puppeteer-extra-plugin-adblocker)
- Very fast & efficient blocker for ads and trackers. Reduces bandwidth & load times.
#### [`puppeteer-extra-plugin-devtools`](/packages/puppeteer-extra-plugin-devtools)
- Makes puppeteer browser debugging possible from anywhere.
- Creates a secure tunnel to make the devtools frontend (**incl. screencasting**) accessible from the public internet
#### [`puppeteer-extra-plugin-repl`](/packages/puppeteer-extra-plugin-repl)
- Makes quick puppeteer debugging and exploration fun with an interactive REPL.
#### [`puppeteer-extra-plugin-block-resources`](/packages/puppeteer-extra-plugin-block-resources)
- Blocks resources (images, media, css, etc.) in puppeteer.
- Supports all resource types, blocking can be toggled dynamically.
#### [`puppeteer-extra-plugin-flash`](/packages/puppeteer-extra-plugin-flash)
- Allows flash content to run on all sites without user interaction.
#### [`puppeteer-extra-plugin-anonymize-ua`](/packages/puppeteer-extra-plugin-anonymize-ua)
- Anonymizes the user-agent on all pages.
- Supports dynamic replacing, so the browser version stays intact and recent.
#### [`puppeteer-extra-plugin-user-preferences`](/packages/puppeteer-extra-plugin-user-preferences)
- Allows setting custom Chrome/Chromium user preferences.
- Has itself a plugin interface which is used by e.g. [`puppeteer-extra-plugin-font-size`](/packages/puppeteer-extra-plugin-font-size).
> Check out the [packages folder](/packages/) for more plugins.
### Community Plugins
_These plugins have been generously contributed by members of the community._
_Please note that they're hosted outside the main project and not under our control or supervision._
#### [`puppeteer-extra-plugin-minmax`](https://github.com/Stillerman/puppeteer-extra-minmax)
- Minimize and maximize puppeteer in real time.
- Great for manually solving captchas.
#### [`puppeteer-extra-plugin-portal`](https://github.com/claabs/puppeteer-extra-plugin-portal)
- Use the Chromium screencast API to remotely view and interact with puppeteer sessions.
- Great for remotely intervening when an automated task gets stuck, like captchas.
> Please check the `Contributing` section below if you're interested in creating a plugin as well.
---
## Contributors
<a href="https://github.com/berstend/puppeteer-extra/graphs/contributors">
<img src="https://contributors-img.firebaseapp.com/image?repo=berstend/puppeteer-extra" />
</a>
## Further info
<details>
<summary><strong>Contributing</strong></summary><br/>
PRs and new plugins are welcome! 🎉 The plugin API for `puppeteer-extra` is clean and fun to use. Have a look the [PuppeteerExtraPlugin](/packages/puppeteer-extra-plugin) base class documentation to get going and check out the [existing plugins](./packages/) (minimal example is the [anonymize-ua](/packages/puppeteer-extra-plugin-anonymize-ua/index.js) plugin) for reference.
We use a [monorepo](/) powered by [Lerna](https://github.com/lerna/lerna#--use-workspaces) (and yarn workspaces), [ava](https://github.com/avajs/ava) for testing, TypeScript for the core, the [standard](https://standardjs.com/) style for linting and [JSDoc](http://usejsdoc.org/about-getting-started.html) heavily to auto-generate markdown [documentation](https://github.com/documentationjs/documentation) based on code. :-)
</details>
<details>
<summary><strong>Kudos</strong></summary><br/>
- Thanks to [skyiea](https://github.com/skyiea) for [this PR](https://github.com/GoogleChrome/puppeteer/pull/1806) that started the project idea.
- Thanks to [transitive-bullshit](https://github.com/transitive-bullshit) for [suggesting](https://github.com/berstend/puppeteer-extra/issues/2) a modular plugin design, which was fun to implement.
</details>
<details>
<summary><strong>Compatibility</strong></summary><br/>
`puppeteer-extra` and all plugins are [tested continously](https://github.com/berstend/puppeteer-extra/actions) in a matrix of current (stable & LTS) NodeJS and puppeteer versions.
We never broke compatibility and still support puppeteer down to very early versions from 2018.
A few plugins won't work in headless mode (it's noted if that's the case) due to Chrome limitations (e.g. the [`user-preferences`](/packages/puppeteer-extra-plugin-user-preferences) plugin), look into `xvfb-run` if you still require a headless experience in these circumstances.
</details>
## Changelog
<details>
<summary><code>2.1.6 ➠ 3.1.1</code></summary>
### `2.1.6` ➠ `3.1.1`
Big refactor, the core is now **written in TypeScript** 🎉
That means out of the box type safety for fellow TS users and nice auto-completion in VSCode for JS users. Also:
- A new [`addExtra`](#addextrapuppeteer) export, to **patch any puppeteer compatible library with plugin functionality** (`chrome-aws-lambda`, etc). This also allows for multiple puppeteer instances with different plugins.
The API is backwards compatible, I bumped the major version just in case I missed something. Please report any issues you might find with the new release. :)
</details>
---
## API
<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
#### Table of Contents
- [class: PuppeteerExtra](#class-puppeteerextra)
- [.use(plugin)](#useplugin)
- [.launch(options?)](#launchoptions)
- [.connect(options?)](#connectoptions)
- [.defaultArgs(options?)](#defaultargsoptions)
- [.executablePath()](#executablepath)
- [.createBrowserFetcher(options?)](#createbrowserfetcheroptions)
- [.plugins](#plugins)
- [.getPluginData(name?)](#getplugindataname)
- [defaultExport()](#defaultexport)
- [addExtra(puppeteer)](#addextrapuppeteer)
### class: [PuppeteerExtra](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L67-L474)
Modular plugin framework to teach `puppeteer` new tricks.
This module acts as a drop-in replacement for `puppeteer`.
Allows PuppeteerExtraPlugin's to register themselves and
to extend puppeteer with additional functionality.
Example:
```javascript
const puppeteer = require('puppeteer-extra')
puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')())
puppeteer.use(
require('puppeteer-extra-plugin-font-size')({ defaultFontSize: 18 })
)
;(async () => {
const browser = await puppeteer.launch({ headless: false })
const page = await browser.newPage()
await page.goto('http://example.com', { waitUntil: 'domcontentloaded' })
await browser.close()
})()
```
---
#### .[use(plugin)](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L85-L107)
- `plugin` **PuppeteerExtraPlugin**
Returns: **this** The same `PuppeteerExtra` instance (for optional chaining)
The **main interface** to register `puppeteer-extra` plugins.
Example:
```javascript
puppeteer.use(plugin1).use(plugin2)
```
- **See: [PuppeteerExtraPlugin]**
---
#### .[launch(options?)](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L153-L177)
- `options` **Puppeteer.LaunchOptions?** See [puppeteer docs](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions).
Returns: **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<Puppeteer.Browser>**
The method launches a browser instance with given arguments. The browser will be closed when the parent node.js process is closed.
Augments the original `puppeteer.launch` method with plugin lifecycle methods.
All registered plugins that have a `beforeLaunch` method will be called
in sequence to potentially update the `options` Object before launching the browser.
Example:
```javascript
const browser = await puppeteer.launch({
headless: false,
defaultViewport: null
})
```
---
#### .[connect(options?)](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L189-L208)
- `options` **Puppeteer.ConnectOptions?** See [puppeteer docs](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerconnectoptions).
Returns: **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)<Puppeteer.Browser>**
Attach Puppeteer to an existing Chromium instance.
Augments the original `puppeteer.connect` method with plugin lifecycle methods.
All registered plugins that have a `beforeConnect` method will be called
in sequence to potentially update the `options` Object before launching the browser.
---
#### .[defaultArgs(options?)](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L215-L217)
- `options` **Puppeteer.ChromeArgOptions?** See [puppeteer docs](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerdefaultargsoptions).
Returns: **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>**
The default flags that Chromium will be launched with.
---
#### .[executablePath()](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L220-L222)
Returns: **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**
Path where Puppeteer expects to find bundled Chromium.
---
#### .[createBrowserFetcher(options?)](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L229-L233)
- `options` **Puppeteer.FetcherOptions?** See [puppeteer docs](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteercreatebrowserfetcheroptions).
Returns: **Puppeteer.BrowserFetcher**
This methods attaches Puppeteer to an existing Chromium instance.
---
#### .[plugins](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L283-L285)
Type: **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<PuppeteerExtraPlugin>**
Get a list of all registered plugins.
---
#### .[getPluginData(name?)](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L310-L315)
- `name` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** Filter data by optional plugin name
Collects the exposed `data` property of all registered plugins.
Will be reduced/flattened to a single array.
Can be accessed by plugins that listed the `dataFromPlugins` requirement.
Implemented mainly for plugins that need data from other plugins (e.g. `user-preferences`).
- **See: [PuppeteerExtraPlugin]/data**
---
### [defaultExport()](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L494-L496)
Type: **[PuppeteerExtra](#puppeteerextra)**
The **default export** will behave exactly the same as the regular puppeteer
(just with extra plugin functionality) and can be used as a drop-in replacement.
Behind the scenes it will try to require either `puppeteer`
or [`puppeteer-core`](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteer-vs-puppeteer-core)
from the installed dependencies.
Example:
```javascript
// javascript import
const puppeteer = require('puppeteer-extra')
// typescript/es6 module import
import puppeteer from 'puppeteer-extra'
// Add plugins
puppeteer.use(...)
```
---
### [addExtra(puppeteer)](https://github.com/berstend/puppeteer-extra/blob/dc8b90260a927c0c66c4585c5a56092ea9c35049/packages/puppeteer-extra/src/index.ts#L519-L520)
- `puppeteer` **VanillaPuppeteer** Any puppeteer API-compatible puppeteer implementation or version.
Returns: **[PuppeteerExtra](#puppeteerextra)** A fresh PuppeteerExtra instance using the provided puppeteer
An **alternative way** to use `puppeteer-extra`: Augments the provided puppeteer with extra plugin functionality.
This is useful in case you need multiple puppeteer instances with different plugins or to add plugins to a non-standard puppeteer package.
Example:
```javascript
// js import
const puppeteerVanilla = require('puppeteer')
const { addExtra } = require('puppeteer-extra')
// ts/es6 import
import puppeteerVanilla from 'puppeteer'
import { addExtra } from 'puppeteer-extra'
// Patch provided puppeteer and add plugins
const puppeteer = addExtra(puppeteerVanilla)
puppeteer.use(...)
```
---
## License
Copyright © 2018 - 2023, [berstend̡̲̫̹̠̖͚͓̔̄̓̐̄͛̀͘](mailto:github@berstend.com?subject=[GitHub]%20PuppeteerExtra). Released under the MIT License.
<!-- Markdown footnotes (for links) -->
[puppeteerextraplugin]: https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin 'PuppeteerExtraPlugin Documentation'
================================================
FILE: packages/puppeteer-extra/rollup.config.ts
================================================
import commonjs from 'rollup-plugin-commonjs'
import resolve from 'rollup-plugin-node-resolve'
import sourceMaps from 'rollup-plugin-sourcemaps'
import typescript from 'rollup-plugin-typescript2'
const pkg = require('./package.json')
const entryFile = 'index'
const banner = `
/*!
* ${pkg.name} v${pkg.version} by ${pkg.author}
* ${pkg.homepage || `https://github.com/${pkg.repository}`}
* @license ${pkg.license}
*/
`.trim()
const defaultExportOutro = `
module.exports = exports.default || {}
Object.entries(exports).forEach(([key, value]) => { module.exports[key] = value })
`
export default {
input: `src/${entryFile}.ts`,
output: [
{
file: pkg.main,
format: 'cjs',
sourcemap: true,
exports: 'named',
outro: defaultExportOutro,
banner
},
{
file: pkg.module,
format: 'es',
sourcemap: true,
exports: 'named',
banner
}
],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [
...Object.keys(pkg.dependencies || {}),
...Object.keys(pkg.peerDependencies || {})
],
watch: {
include: 'src/**'
},
plugins: [
// Compile TypeScript files
typescript({ useTsconfigDeclarationDir: true }),
// Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
commonjs(),
// Allow node_modules resolution, so you can use 'external' to control
// which external modules to include in the bundle
// https://github.com/rollup/rollup-plugin-node-resolve#usage
resolve(),
// Resolve source maps to the original source
sourceMaps()
]
}
================================================
FILE: packages/puppeteer-extra/src/ambient.d.ts
================================================
export {}
// https://github.com/sindresorhus/type-fest/issues/19
declare global {
interface SymbolConstructor {
readonly observable: symbol
}
}
================================================
FILE: packages/puppeteer-extra/src/index.ts
================================================
/// <reference path="./puppeteer-legacy.d.ts" />
import { PuppeteerNode, Browser, Page } from 'puppeteer'
import Debug from 'debug'
const debug = Debug('puppeteer-extra')
import merge from 'deepmerge'
/**
* Original Puppeteer API
* @private
*/
export interface VanillaPuppeteer
extends Pick<
PuppeteerNode,
| 'connect'
| 'defaultArgs'
| 'executablePath'
| 'launch'
| 'createBrowserFetcher'
> {}
/**
* Minimal plugin interface
* @private
*/
export interface PuppeteerExtraPlugin {
_isPuppeteerExtraPlugin: boolean
[propName: string]: any
}
/**
* We need to hook into non-public APIs in rare occasions to fix puppeteer bugs. :(
* @private
*/
interface BrowserInternals extends Browser {
_createPageInContext(contextId?: string): Promise<Page>
}
/**
* Modular plugin framework to teach `puppeteer` new tricks.
*
* This module acts as a drop-in replacement for `puppeteer`.
*
* Allows PuppeteerExtraPlugin's to register themselves and
* to extend puppeteer with additional functionality.
*
* @class PuppeteerExtra
* @implements {VanillaPuppeteer}
*
* @example
* const puppeteer = require('puppeteer-extra')
* puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')())
* puppeteer.use(require('puppeteer-extra-plugin-font-size')({defaultFontSize: 18}))
*
* ;(async () => {
* const browser = await puppeteer.launch({headless: false})
* const page = await browser.newPage()
* await page.goto('http://example.com', {waitUntil: 'domcontentloaded'})
* await browser.close()
* })()
*/
export class PuppeteerExtra implements VanillaPuppeteer {
private _plugins: PuppeteerExtraPlugin[] = []
constructor(
private _pptr?: VanillaPuppeteer,
private _requireError?: Error
) {}
/**
* The **main interface** to register `puppeteer-extra` plugins.
*
* @example
* puppeteer.use(plugin1).use(plugin2)
*
* @see [PuppeteerExtraPlugin]
*
* @return The same `PuppeteerExtra` instance (for optional chaining)
*/
use(plugin: PuppeteerExtraPlugin): this {
if (typeof plugin !== 'object' || !plugin._isPuppeteerExtraPlugin) {
console.error(
`Warning: Plugin is not derived from PuppeteerExtraPlugin, ignoring.`,
plugin
)
return this
}
if (!plugin.name) {
console.error(
`Warning: Plugin with no name registering, ignoring.`,
plugin
)
return this
}
if (plugin.requirements.has('dataFromPlugins')) {
plugin.getDataFromPlugins = this.getPluginData.bind(this)
}
plugin._register(Object.getPrototypeOf(plugin))
this._plugins.push(plugin)
debug('plugin registered', plugin.name)
return this
}
/**
* To stay backwards compatible with puppeteer's (and our) default export after adding `addExtra`
* we need to defer the check if we have a puppeteer instance to work with.
* Otherwise we would throw even if the user intends to use their non-standard puppeteer implementation.
*
* @private
*/
get pptr(): VanillaPuppeteer {
if (this._pptr) {
return this._pptr
}
// Whoopsie
console.warn(`
Puppeteer is missing. :-)
Note: puppeteer is a peer dependency of puppeteer-extra,
which means you can install your own preferred version.
- To get the latest stable version run: 'yarn add puppeteer' or 'npm i puppeteer'
Alternatively:
- To get puppeteer without the bundled Chromium browser install 'puppeteer-core'
`)
throw this._requireError || new Error('No puppeteer instance provided.')
}
/**
* The method launches a browser instance with given arguments. The browser will be closed when the parent node.js process is closed.
*
* Augments the original `puppeteer.launch` method with plugin lifecycle methods.
*
* All registered plugins that have a `beforeLaunch` method will be called
* in sequence to potentially update the `options` Object before launching the browser.
*
* @example
* const browser = await puppeteer.launch({
* headless: false,
* defaultViewport: null
* })
*
* @param options - See [puppeteer docs](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerlaunchoptions).
*/
async launch(
options?: Parameters<VanillaPuppeteer['launch']>[0]
): ReturnType<VanillaPuppeteer['launch']> {
// Ensure there are certain properties (e.g. the `options.args` array)
const defaultLaunchOptions = { args: [] }
options = merge(defaultLaunchOptions, options || {})
this.resolvePluginDependencies()
this.orderPlugins()
// Give plugins the chance to modify the options before launch
options = await this.callPluginsWithValue('beforeLaunch', options)
const opts = {
context: 'launch',
options,
defaultArgs: this.defaultArgs
}
// Let's check requirements after plugin had the chance to modify the options
this.checkPluginRequirements(opts)
const browser = await this.pptr.launch(options)
this._patchPageCreationMethods(browser as BrowserInternals)
await this.callPlugins('_bindBrowserEvents', browser, opts)
return browser
}
/**
* Attach Puppeteer to an existing Chromium instance.
*
* Augments the original `puppeteer.connect` method with plugin lifecycle methods.
*
* All registered plugins that have a `beforeConnect` method will be called
* in sequence to potentially update the `options` Object before launching the browser.
*
* @param options - See [puppeteer docs](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerconnectoptions).
*/
async connect(
options: Parameters<VanillaPuppeteer['connect']>[0]
): ReturnType<VanillaPuppeteer['connect']> {
this.resolvePluginDependencies()
this.orderPlugins()
// Give plugins the chance to modify the options before connect
options = await this.callPluginsWithValue('beforeConnect', options)
const opts = { context: 'connect', options }
// Let's check requirements after plugin had the chance to modify the options
this.checkPluginRequirements(opts)
const browser = await this.pptr.connect(options)
this._patchPageCreationMethods(browser as BrowserInternals)
await this.callPlugins('_bindBrowserEvents', browser, opts)
return browser
}
/**
* The default flags that Chromium will be launched with.
*
* @param options - See [puppeteer docs](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteerdefaultargsoptions).
*/
defaultArgs(
options?: Parameters<VanillaPuppeteer['defaultArgs']>[0]
): ReturnType<VanillaPuppeteer['defaultArgs']> {
return this.pptr.defaultArgs(options)
}
/** Path where Puppeteer expects to find bundled Chromium. */
executablePath(): string {
return this.pptr.executablePath()
}
/**
* This methods attaches Puppeteer to an existing Chromium instance.
*
* @param options - See [puppeteer docs](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteercreatebrowserfetcheroptions).
*/
createBrowserFetcher(
options: Parameters<VanillaPuppeteer['createBrowserFetcher']>[0]
): ReturnType<VanillaPuppeteer['createBrowserFetcher']> {
return this.pptr.createBrowserFetcher(options)
}
/**
* Patch page creation methods (both regular and incognito contexts).
*
* Unfortunately it's possible that the `targetcreated` events are not triggered
* early enough for listeners (e.g. plugins using `onPageCreated`) to be able to
* modify the page instance (e.g. user-agent) before the browser request occurs.
*
* This only affects the first request of a newly created page target.
*
* As a workaround I've noticed that navigating to `about:blank` (again),
* right after a page has been created reliably fixes this issue and adds
* no noticable delay or side-effects.
*
* This problem is not specific to `puppeteer-extra` but default Puppeteer behaviour.
*
* Note: This patch only fixes explicitly created pages, implicitly created ones
* (e.g. through `window.open`) are still subject to this issue. I didn't find a
* reliable mitigation for implicitly created pages yet.
*
* Puppeteer issues:
* https://github.com/GoogleChrome/puppeteer/issues/2669
* https://github.com/puppeteer/puppeteer/issues/3667
* https://github.com/GoogleChrome/puppeteer/issues/386#issuecomment-343059315
* https://github.com/GoogleChrome/puppeteer/issues/1378#issue-273733905
*
* @private
*/
private _patchPageCreationMethods(browser: BrowserInternals) {
if (!browser._createPageInContext) {
debug(
'warning: _patchPageCreationMethods failed (no browser._createPageInContext)'
)
return
}
browser._createPageInContext = (function(originalMethod, context) {
return async function() {
const page = await originalMethod.apply(context, arguments as any)
await page.goto('about:blank')
return page
}
})(browser._createPageInContext, browser)
}
/**
* Get a list of all registered plugins.
*
* @member {Array<PuppeteerExtraPlugin>}
*/
get plugins() {
return this._plugins
}
/**
* Get the names of all registered plugins.
*
* @member {Array<string>}
* @private
*/
get pluginNames() {
return this._plugins.map(p => p.name)
}
/**
* Collects the exposed `data` property of all registered plugins.
* Will be reduced/flattened to a single array.
*
* Can be accessed by plugins that listed the `dataFromPlugins` requirement.
*
* Implemented mainly for plugins that need data from other plugins (e.g. `user-preferences`).
*
* @see [PuppeteerExtraPlugin]/data
* @param name - Filter data by optional plugin name
*
* @private
*/
public getPluginData(name?: string) {
const data = this._plugins
.map(p => (Array.isArray(p.data) ? p.data : [p.data]))
.reduce((acc, arr) => [...acc, ...arr], [])
return name ? data.filter((d: any) => d.name === name) : data
}
/**
* Get all plugins that feature a given property/class method.
*
* @private
*/
private getPluginsByProp(prop: string): PuppeteerExtraPlugin[] {
return this._plugins.filter(plugin => prop in plugin)
}
/**
* Lightweight plugin dependency management to require plugins and code mods on demand.
*
* This uses the `dependencies` stanza (a `Set`) exposed by `puppeteer-extra` plugins.
*
* @todo Allow objects as depdencies that contains opts for the requested plugin.
*
* @private
*/
private resolvePluginDependencies() {
// Request missing dependencies from all plugins and flatten to a single Set
const missingPlugins = this._plugins
.map(p => p._getMissingDependencies(this._plugins))
.reduce((combined, list) => {
return new Set([...combined, ...list])
}, new Set())
if (!missingPlugins.size) {
debug('no dependencies are missing')
return
}
debug('dependencies missing', missingPlugins)
// Loop through all dependencies declared missing by plugins
for (let name of [...missingPlugins]) {
// Check if the dependency hasn't been registered as plugin already.
// This might happen when multiple plugins have nested dependencies.
if (this.pluginNames.includes(name)) {
debug(`ignoring dependency '${name}', which has been required already.`)
continue
}
// We follow a plugin naming convention, but let's rather enforce it <3
name = name.startsWith('puppeteer-extra-plugin')
? name
: `puppeteer-extra-plugin-${name}`
// In case a module sub resource is requested print out the main package name
// e.g. puppeteer-extra-plugin-stealth/evasions/console.debug => puppeteer-extra-plugin-stealth
const packageName = name.split('/')[0]
let dep = null
try {
// Try to require and instantiate the stated dependency
dep = require(name)()
// Register it with `puppeteer-extra` as plugin
this.use(dep)
} catch (err) {
console.warn(`
A plugin listed '${name}' as dependency,
which is currently missing. Please install it:
yarn add ${packageName}
Note: You don't need to require the plugin yourself,
unless you want to modify it's default settings.
`)
throw err
}
// Handle nested dependencies :D
if (dep.dependencies.size) {
this.resolvePluginDependencies()
}
}
}
/**
* Order plugins that have expressed a special placement requirement.
*
* This is useful/necessary for e.g. plugins that depend on the data from other plugins.
*
* @todo Support more than 'runLast'.
* @todo If there are multiple plugins defining 'runLast', sort them depending on who depends on whom. :D
*
* @private
*/
private orderPlugins() {
debug('orderPlugins:before', this.pluginNames)
const runLast = this._plugins
.filter(p => p.requirements.has('runLast'))
.map(p => p.name)
for (const name of runLast) {
const index = this._plugins.findIndex(p => p.name === name)
this._plugins.push(this._plugins.splice(index, 1)[0])
}
debug('orderPlugins:after', this.pluginNames)
}
/**
* Lightweight plugin requirement checking.
*
* The main intent is to notify the user when a plugin won't work as expected.
*
* @todo This could be improved, e.g. be evaluated by the plugin base class.
*
* @private
*/
private checkPluginRequirements(opts = {} as any) {
for (const plugin of this._plugins) {
for (const requirement of plugin.requirements) {
if (
opts.context === 'launch' &&
requirement === 'headful' &&
opts.options.headless
) {
console.warn(
`Warning: Plugin '${plugin.name}' is not supported in headless mode.`
)
}
if (opts.context === 'connect' && requirement === 'launch') {
console.warn(
`Warning: Plugin '${plugin.name}' doesn't support puppeteer.connect().`
)
}
}
}
}
/**
* Call plugins sequentially with the same values.
* Plugins that expose the supplied property will be called.
*
* @param prop - The plugin property to call
* @param values - Any number of values
* @private
*/
private async callPlugins(prop: string, ...values: any[]) {
for (const plugin of this.getPluginsByProp(prop)) {
await plugin[prop].apply(plugin, values)
}
}
/**
* Call plugins sequentially and pass on a value (waterfall style).
* Plugins that expose the supplied property will be called.
*
* The plugins can either modify the value or return an updated one.
* Will return the latest, updated value which ran through all plugins.
*
* @param prop - The plugin property to call
* @param value - Any value
* @return The new updated value
* @private
*/
private async callPluginsWithValue(prop: string, value: any) {
for (const plugin of this.getPluginsByProp(prop)) {
const newValue = await plugin[prop](value)
if (newValue) {
value = newValue
}
}
return value
}
}
/**
* The **default export** will behave exactly the same as the regular puppeteer
* (just with extra plugin functionality) and can be used as a drop-in replacement.
*
* Behind the scenes it will try to require either `puppeteer`
* or [`puppeteer-core`](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#puppeteer-vs-puppeteer-core)
* from the installed dependencies.
*
* @example
* // javascript import
* const puppeteer = require('puppeteer-extra')
*
* // typescript/es6 module import
* import puppeteer from 'puppeteer-extra'
*
* // Add plugins
* puppeteer.use(...)
*/
const defaultExport: PuppeteerExtra = (() => {
return new PuppeteerExtra(...requireVanillaPuppeteer())
})()
export default defaultExport
/**
* An **alternative way** to use `puppeteer-extra`: Augments the provided puppeteer with extra plugin functionality.
*
* This is useful in case you need multiple puppeteer instances with different plugins or to add plugins to a non-standard puppeteer package.
*
* @example
* // js import
* const { addExtra } = require('puppeteer-extra')
*
* // ts/es6 import
* import { addExtra } from 'puppeteer-extra'
*
* // Patch e.g. puppeteer-firefox and add plugins
* const puppeteer = addExtra(require('puppeteer-firefox'))
* puppeteer.use(...)
*
* @param puppeteer Any puppeteer API-compatible puppeteer implementation or version.
* @return A fresh PuppeteerExtra instance using the provided puppeteer
*/
export const addExtra = (puppeteer: VanillaPuppeteer): PuppeteerExtra =>
new PuppeteerExtra(puppeteer)
/**
* Attempt to require puppeteer or puppeteer-core from dependencies.
* To stay backwards compatible with the existing default export we have to do some gymnastics here.
*
* @return Either a Puppeteer instance or an Error, which we'll throw later if need be.
* @private
*/
function requireVanillaPuppeteer(): [VanillaPuppeteer?, Error?] {
try {
return [require('puppeteer'), undefined]
} catch (_) {
// noop
}
try {
return [require('puppeteer-core'), undefined]
} catch (err) {
return [undefined, err as Error]
}
}
================================================
FILE: packages/puppeteer-extra/src/puppeteer-legacy.d.ts
================================================
// @ts-nocheck
// NOTE: The above comment is crucial for all this to work
// The puppeteer project caused a type breaking shift in v6 while switching from @types/puppeteer to built-in types
// This type definition file is only relevant when puppeteer < v6 is being used,
// if we don't instruct TS to skip checking this file it would cause errors when pptr >= v6 is used (e.g. ChromeArgOptions is missing)
import {} from 'puppeteer'
import { Browser, ConnectOptions, ChromeArgOptions, LaunchOptions, FetcherOptions, BrowserFetcher} from "puppeteer"
// Make puppeteer-extra typings backwards compatible with puppeteer < v6
// In pptr >= v6 they switched to built-in types and the `@types/puppeteer` package is not needed anymore.
// This is essentially a shim for `PuppeteerNode`, which is found in pptr >= v6 and missing in `@types/puppeteer`.
// Requires the `@types/puppeteer` package to be installed when using pptr < v6, `@types/puppeteer` will be ignored by TS when built-in types are available.
interface VanillaPuppeteer {
/** Attaches Puppeteer to an existing Chromium instance */
connect(options?: ConnectOptions): Promise<Browser>
/** The default flags that Chromium will be launched with */
defaultArgs(options?: ChromeArgOptions): string[]
/** Path where Puppeteer expects to find bundled Chromium */
executablePath(): string
/** The method launches a browser instance with given arguments. The browser will be closed when the parent node.js process is closed. */
launch(options?: LaunchOptions): Promise<Browser>
/** This methods attaches Puppeteer to an existing Chromium instance. */
createBrowserFetcher(
options?: FetcherOptions
): BrowserFetcher
}
declare module 'puppeteer' {
interface PuppeteerNode extends VanillaPuppeteer {}
}
declare module 'puppeteer-core' {
interface PuppeteerNode extends VanillaPuppeteer {}
}
================================================
FILE: packages/puppeteer-extra/test/addExtra.ts
================================================
import test from 'ava'
import { addExtra } from '../src/index'
test('is a function', async t => {
t.is(typeof addExtra, 'function')
})
test('is an instance of Function', async t => {
t.is(addExtra.constructor.name, 'Function')
})
test('returns an object', async t => {
t.is(typeof addExtra(null as any), 'object')
})
test('returns an instance of PuppeteerExtra', async t => {
t.is(addExtra(null as any).constructor.name, 'PuppeteerExtra')
})
test('will throw without puppeteer', async t => {
const pptr = addExtra(null as any)
t.throws(() => pptr.pptr, null, 'No puppeteer instance provided.')
})
================================================
FILE: packages/puppeteer-extra/test/basic.ts
================================================
import test from 'ava'
import puppeteer from '../src/index'
test('is an object', async t => {
t.is(typeof puppeteer, 'object')
})
test('is an instance of PuppeteerExtra', async t => {
t.is(puppeteer.constructor.name, 'PuppeteerExtra')
})
test('should have the public class members', async t => {
t.true(puppeteer.use instanceof Function)
t.true(puppeteer.plugins instanceof Array)
t.true(puppeteer.pluginNames instanceof Array)
t.true(puppeteer.getPluginData instanceof Function)
})
test('should have the internal class members', async t => {
t.true('getPluginsByProp' in puppeteer)
t.true('resolvePluginDependencies' in puppeteer)
t.true('orderPlugins' in puppeteer)
t.true('checkPluginRequirements' in puppeteer)
t.true('callPlugins' in puppeteer)
t.true('callPluginsWithValue' in puppeteer)
})
test('should have the orginal puppeteer public class members', async t => {
t.true(puppeteer.launch instanceof Function)
t.true(puppeteer.connect instanceof Function)
t.true(puppeteer.executablePath instanceof Function)
t.true(puppeteer.defaultArgs instanceof Function)
t.true(puppeteer.createBrowserFetcher instanceof Function)
})
================================================
FILE: packages/puppeteer-extra/test/connect.js
================================================
'use strict'
const test = require('ava')
const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
test.beforeEach(t => {
// Make sure we work with pristine modules
try {
delete require.cache[require.resolve('puppeteer-extra')]
// delete require.cache[require.resolve('puppeteer-extra-plugin')]
} catch (error) {
console.log(error)
}
})
test('will remove headless from remote browser', async t => {
// Mitigate CI quirks
try {
// Launch vanilla puppeteer browser with no plugins
const puppeteerVanilla = require('puppeteer')
const browserVanilla = await puppeteerVanilla.launch({
args: PUPPETEER_ARGS
})
const browserWSEndpoint = browserVanilla.wsEndpoint()
// Use puppeteer-extra with plugin to conntect to existing browser
const puppeteer = require('puppeteer-extra')
puppeteer.use(require('puppeteer-extra-plugin-anonymize-ua')())
const browser = await puppeteer.connect({ browserWSEndpoint })
// Let's ensure we've anonymized the user-agent, despite not using .launch
const page = await browser.newPage()
const ua = await page.evaluate(() => window.navigator.userAgent)
t.true(!ua.includes('HeadlessChrome'))
await browser.close()
t.true(true)
} catch (err) {
console.log(`Caught error:`, err)
if (
err.message &&
err.message.includes(
'Session closed. Most likely the page has been closed'
)
) {
t.true(true) // ignore this error
} else {
throw err
}
}
})
================================================
FILE: packages/puppeteer-extra/test/events.js
================================================
'use strict'
const test = require('ava')
const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
const puppeteerVanilla = require('puppeteer')
const { addExtra } = require('puppeteer-extra')
test.beforeEach(t => {
// Make sure we work with pristine modules
try {
delete require.cache[require.resolve('puppeteer-extra')]
delete require.cache[require.resolve('puppeteer-extra-plugin')]
} catch (error) {
console.log(error)
}
})
test('will bind launched browser events to plugins', async t => {
const PLUGIN_EVENTS = []
const puppeteer = addExtra(puppeteerVanilla)
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const pluginName = 'hello-world'
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return pluginName
}
onPluginRegistered() {
PLUGIN_EVENTS.push('onPluginRegistered')
}
beforeLaunch() {
PLUGIN_EVENTS.push('beforeLaunch')
}
afterLaunch() {
PLUGIN_EVENTS.push('afterLaunch')
}
beforeConnect() {
PLUGIN_EVENTS.push('beforeConnect')
}
afterConnect() {
PLUGIN_EVENTS.push('afterConnect')
}
onBrowser() {
PLUGIN_EVENTS.push('onBrowser')
}
onTargetCreated() {
PLUGIN_EVENTS.push('onTargetCreated')
}
onPageCreated() {
PLUGIN_EVENTS.push('onPageCreated')
}
onTargetChanged() {
PLUGIN_EVENTS.push('onTargetChanged')
}
onTargetDestroyed() {
PLUGIN_EVENTS.push('onTargetDestroyed')
}
onDisconnected() {
PLUGIN_EVENTS.push('onDisconnected')
}
onClose() {
PLUGIN_EVENTS.push('onClose')
}
}
const instance = new Plugin()
puppeteer.use(instance)
t.true(PLUGIN_EVENTS.includes('onPluginRegistered'))
const browser = await puppeteer.launch({ args: PUPPETEER_ARGS })
t.true(PLUGIN_EVENTS.includes('beforeLaunch'))
t.true(PLUGIN_EVENTS.includes('afterLaunch'))
// t.true(!PLUGIN_EVENTS.includes('beforeConnect'))
// t.true(!PLUGIN_EVENTS.includes('afterConnect'))
t.true(PLUGIN_EVENTS.includes('onBrowser'))
const page = await browser.newPage().catch(console.log)
t.true(PLUGIN_EVENTS.includes('onTargetCreated'))
t.true(PLUGIN_EVENTS.includes('onPageCreated'))
await page.goto('about:blank#foo').catch(console.log)
t.true(PLUGIN_EVENTS.includes('onTargetChanged'))
await page.close().catch(console.log)
t.true(PLUGIN_EVENTS.includes('onTargetDestroyed'))
await browser.close().catch(console.log)
t.true(PLUGIN_EVENTS.includes('onDisconnected'))
t.true(PLUGIN_EVENTS.includes('onClose'))
})
test('will bind connected browser events to plugins', async t => {
const PLUGIN_EVENTS = []
// Launch vanilla puppeteer browser with no plugins
const pptr1 = addExtra(puppeteerVanilla)
const browserVanilla = await pptr1.launch({
args: PUPPETEER_ARGS
})
const browserWSEndpoint = browserVanilla.wsEndpoint()
const puppeteer = addExtra(puppeteerVanilla)
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const pluginName = 'hello-world'
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return pluginName
}
onPluginRegistered() {
PLUGIN_EVENTS.push('onPluginRegistered')
}
beforeLaunch() {
PLUGIN_EVENTS.push('beforeLaunch')
}
afterLaunch() {
PLUGIN_EVENTS.push('afterLaunch')
}
beforeConnect() {
PLUGIN_EVENTS.push('beforeConnect')
}
afterConnect() {
PLUGIN_EVENTS.push('afterConnect')
}
onBrowser() {
PLUGIN_EVENTS.push('onBrowser')
}
onTargetCreated() {
PLUGIN_EVENTS.push('onTargetCreated')
}
onPageCreated() {
PLUGIN_EVENTS.push('onPageCreated')
}
onTargetChanged() {
PLUGIN_EVENTS.push('onTargetChanged')
}
onTargetDestroyed() {
PLUGIN_EVENTS.push('onTargetDestroyed')
}
onDisconnected() {
PLUGIN_EVENTS.push('onDisconnected')
}
onClose() {
PLUGIN_EVENTS.push('onClose')
}
}
const instance = new Plugin()
puppeteer.use(instance)
t.true(PLUGIN_EVENTS.includes('onPluginRegistered'))
const browser = await puppeteer
.connect({ browserWSEndpoint })
.catch(console.log)
t.true(!PLUGIN_EVENTS.includes('beforeLaunch'))
t.true(!PLUGIN_EVENTS.includes('afterLaunch'))
t.true(PLUGIN_EVENTS.includes('beforeConnect'))
t.true(PLUGIN_EVENTS.includes('afterConnect'))
t.true(PLUGIN_EVENTS.includes('onBrowser'))
const page = await browser.newPage()
t.true(PLUGIN_EVENTS.includes('onTargetCreated'))
t.true(PLUGIN_EVENTS.includes('onPageCreated'))
await page.goto('about:blank#foo').catch(console.log)
t.true(PLUGIN_EVENTS.includes('onTargetChanged'))
await page.close().catch(console.log)
t.true(PLUGIN_EVENTS.includes('onTargetDestroyed'))
await browser.close().catch(console.log)
t.true(PLUGIN_EVENTS.includes('onDisconnected'))
t.true(!PLUGIN_EVENTS.includes('onClose'))
})
================================================
FILE: packages/puppeteer-extra/test/options.js
================================================
'use strict'
import test, { beforeEach } from 'ava'
const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
beforeEach(t => {
// Make sure we work with pristine modules
try {
delete require.cache[require.resolve('puppeteer-extra')]
delete require.cache[require.resolve('puppeteer-extra-plugin')]
} catch (error) {
console.log(error)
}
})
test('will modify puppeteer launch options through plugins', async t => {
let FINAL_OPTIONS = null
const puppeteer = require('puppeteer-extra')
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const pluginName = 'hello-world'
const pluginData = [{ name: 'foo', value: 'bar' }]
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return pluginName
}
get data() {
return pluginData
}
beforeLaunch(options) {
options.args.push('--foobar=true')
options.timeout = 60 * 1000
options.headless = true
}
afterLaunch(browser, opts) {
FINAL_OPTIONS = opts.options
}
}
const instance = new Plugin()
puppeteer.use(instance)
const browser = await puppeteer.launch({
args: PUPPETEER_ARGS,
headless: false
})
t.deepEqual(FINAL_OPTIONS, {
headless: true,
timeout: 60000,
args: [].concat(PUPPETEER_ARGS, ['--foobar=true'])
})
await browser.close()
t.true(true)
})
test('will modify puppeteer connect options through plugins', async t => {
let FINAL_OPTIONS = null
// Launch vanilla puppeteer browser with no plugins
const puppeteerVanilla = require('puppeteer')
const browserVanilla = await puppeteerVanilla.launch({
args: PUPPETEER_ARGS
})
const browserWSEndpoint = browserVanilla.wsEndpoint()
const puppeteer = require('puppeteer-extra')
const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')
const pluginName = 'hello-world'
const pluginData = [{ name: 'foo', value: 'bar' }]
class Plugin extends PuppeteerExtraPlugin {
constructor(opts = {}) {
super(opts)
}
get name() {
return pluginName
}
get data() {
return pluginData
}
beforeConnect(options) {
options.foo1 = 60 * 1000
options.foo2 = true
}
afterConnect(browser, opts) {
FINAL_OPTIONS = opts.options
}
}
const instance = new Plugin()
puppeteer.use(instance)
const browser = await puppeteer.connect({ browserWSEndpoint })
t.deepEqual(FINAL_OPTIONS, {
foo1: 60 * 1000,
foo2: true,
browserWSEndpoint
})
await browser.close()
t.true(true)
})
================================================
FILE: packages/puppeteer-extra/test/plugin-support.js
================================================
'use strict'
const test = require('ava')
const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
const PAGE_TIMEOUT = 60 * 1000 // 60s
test.beforeEach(t => {
// Make sure we work wit
gitextract_u4zlbb78/
├── .editorconfig
├── .eslintrc
├── .gitattributes
├── .github/
│ ├── FUNDING.yml
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug.md
│ │ ├── config.yml
│ │ └── feature_request.md
│ ├── labeler.yml
│ └── workflows/
│ ├── extract-stealth.yml
│ ├── label.yml
│ └── test.yml
├── .gitignore
├── .prettierrc.js
├── .travis.yml
├── LICENSE
├── README.md
├── lerna.json
├── package.json
└── packages/
├── extract-stealth-evasions/
│ ├── .gitignore
│ ├── index.js
│ ├── package.json
│ └── readme.md
├── playwright-extra/
│ ├── .prettierrc.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── extra.ts
│ │ ├── helper/
│ │ │ └── loader.ts
│ │ ├── index.ts
│ │ ├── plugins.ts
│ │ ├── puppeteer-compatiblity-shim/
│ │ │ ├── index.ts
│ │ │ └── playwright-shim.d.ts
│ │ └── types/
│ │ └── index.ts
│ ├── test/
│ │ ├── exports.spec.ts
│ │ ├── fixtures/
│ │ │ ├── dummyplugin.ts
│ │ │ └── extra.ts
│ │ ├── playwright.config.ts
│ │ ├── plugin-events.spec.ts
│ │ └── puppeteer-plugins/
│ │ ├── anonymize-ua.spec.ts
│ │ ├── recaptcha.spec.ts
│ │ └── stealth.spec.ts
│ └── tsconfig.json
├── plugin-proxy-router/
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── index.ts
│ │ ├── plugin.ts
│ │ ├── router.ts
│ │ ├── stats.ts
│ │ └── utils/
│ │ └── port.ts
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra/
│ ├── ava.config-ts.js
│ ├── ava.config.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── ambient.d.ts
│ │ ├── index.ts
│ │ └── puppeteer-legacy.d.ts
│ ├── test/
│ │ ├── addExtra.ts
│ │ ├── basic.ts
│ │ ├── connect.js
│ │ ├── events.js
│ │ ├── options.js
│ │ └── plugin-support.js
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra-plugin/
│ ├── ava.config-ts.js
│ ├── ava.config.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── ambient.d.ts
│ │ ├── index.test.ts
│ │ ├── index.ts
│ │ └── puppeteer.ts
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra-plugin-adblocker/
│ ├── ava.config-ts.js
│ ├── ava.config.js
│ ├── build_version_check.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── ambient.d.ts
│ │ ├── index.test.ts
│ │ └── index.ts
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra-plugin-anonymize-ua/
│ ├── index.d.ts
│ ├── index.js
│ ├── index.test.js
│ ├── package.json
│ ├── readme.md
│ └── test/
│ ├── headless.js
│ ├── headless_off.js
│ ├── popup.js
│ └── stresstest.js
├── puppeteer-extra-plugin-block-resources/
│ ├── example.js
│ ├── index.d.ts
│ ├── index.js
│ ├── index.test.js
│ ├── package.json
│ └── readme.md
├── puppeteer-extra-plugin-click-and-wait/
│ ├── example.js
│ ├── index.js
│ ├── package.json
│ └── readme.md
├── puppeteer-extra-plugin-devtools/
│ ├── index.js
│ ├── index.test.js
│ ├── lib/
│ │ ├── RemoteDevTools.js
│ │ └── RemoteDevTools.test.js
│ ├── package.json
│ ├── readme.md
│ └── test/
│ └── headless.js
├── puppeteer-extra-plugin-flash/
│ ├── example.js
│ ├── index.js
│ ├── package.json
│ └── readme.md
├── puppeteer-extra-plugin-font-size/
│ ├── index.js
│ ├── package.json
│ └── readme.md
├── puppeteer-extra-plugin-recaptcha/
│ ├── ava.config-ts.js
│ ├── ava.config.js
│ ├── package.json
│ ├── readme.md
│ ├── rollup.config.ts
│ ├── src/
│ │ ├── ambient.d.ts
│ │ ├── content-hcaptcha.ts
│ │ ├── content.ts
│ │ ├── detection.test.ts
│ │ ├── index.test.ts
│ │ ├── index.ts
│ │ ├── playwright-mods.d.ts
│ │ ├── provider/
│ │ │ ├── 2captcha-api.ts
│ │ │ └── 2captcha.ts
│ │ ├── puppeteer-mods.d.ts
│ │ ├── solve.test.ts
│ │ └── types.ts
│ ├── tsconfig.json
│ └── tslint.json
├── puppeteer-extra-plugin-repl/
│ ├── index.d.ts
│ ├── index.js
│ ├── index.test.js
│ ├── lib/
│ │ ├── REPLSession.js
│ │ ├── REPLSession.test.js
│ │ ├── super-readline.js
│ │ └── super-readline.test.js
│ ├── package.json
│ ├── readme.md
│ └── test/
│ └── headless.js
├── puppeteer-extra-plugin-stealth/
│ ├── .npmignore
│ ├── evasions/
│ │ ├── _template/
│ │ │ ├── index.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── _utils/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── readme.md
│ │ │ └── withUtils.js
│ │ ├── chrome.app/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── chrome.csi/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── chrome.loadTimes/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── chrome.runtime/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ ├── readme.md
│ │ │ └── staticData.json
│ │ ├── defaultArgs/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── iframe.contentWindow/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── media.codecs/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.hardwareConcurrency/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.languages/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.permissions/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.plugins/
│ │ │ ├── data.json
│ │ │ ├── functionMocks.js
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── magicArray.js
│ │ │ ├── mimeTypes.js
│ │ │ ├── mimeTypes.test.js
│ │ │ ├── package.json
│ │ │ ├── plugins.js
│ │ │ ├── plugins.test.js
│ │ │ └── readme.md
│ │ ├── navigator.vendor/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── navigator.webdriver/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── readme.md
│ │ ├── sourceurl/
│ │ │ ├── _fixtures/
│ │ │ │ └── test.html
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── user-agent-override/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ ├── webgl.vendor/
│ │ │ ├── index.js
│ │ │ ├── index.test.js
│ │ │ ├── package.json
│ │ │ └── readme.md
│ │ └── window.outerdimensions/
│ │ ├── index.js
│ │ ├── package.json
│ │ └── readme.md
│ ├── examples/
│ │ ├── detect-headless.js
│ │ ├── test1.js
│ │ └── test2.js
│ ├── index.d.ts
│ ├── index.js
│ ├── index.test.js
│ ├── package.json
│ ├── readme.md
│ ├── runall_stealthtests.sh
│ ├── stealthtests/
│ │ ├── headful-chrome-stealth.js
│ │ ├── headful-chrome-vanilla.js
│ │ ├── headful-chromium-stealth.js
│ │ ├── headful-chromium-vanilla.js
│ │ ├── headless-chrome-stealth.js
│ │ ├── headless-chrome-vanilla.js
│ │ ├── headless-chromium-stealth.js
│ │ └── headless-chromium-vanilla.js
│ └── test/
│ ├── cat-and-mouse.test.js
│ ├── fixtures/
│ │ ├── dummy-with-service-worker.html
│ │ ├── dummy.html
│ │ └── sw.js
│ ├── fpscanner.test.js
│ ├── service-worker.test.js
│ └── util.js
├── puppeteer-extra-plugin-user-data-dir/
│ ├── index.js
│ ├── package.json
│ └── readme.md
└── puppeteer-extra-plugin-user-preferences/
├── index.js
├── package.json
└── readme.md
SYMBOL INDEX (638 symbols across 105 files)
FILE: packages/extract-stealth-evasions/index.js
function patchEval (line 85) | function patchEval(f, args) {
FILE: packages/playwright-extra/src/extra.ts
type PlaywrightBrowserLauncher (line 10) | type PlaywrightBrowserLauncher = pw.BrowserType
type AugmentedLauncherAPIs (line 16) | interface AugmentedLauncherAPIs
class PlaywrightExtraClass (line 25) | class PlaywrightExtraClass implements AugmentedLauncherAPIs {
method constructor (line 29) | constructor(private _launcher?: Partial<PlaywrightBrowserLauncher>) {
method use (line 48) | public use(plugin: CompatiblePlugin): this {
method launcher (line 69) | public get launcher(): Partial<PlaywrightBrowserLauncher> {
method launch (line 76) | public async launch(
method launchPersistentContext (line 106) | public async launchPersistentContext(
method connect (line 131) | async connect(
method connectOverCDP (line 175) | async connectOverCDP(
method _bindBrowserContextEvents (line 221) | protected async _bindBrowserContextEvents(
method _bindBrowserEvents (line 251) | protected async _bindBrowserEvents(browser: pw.Browser) {
method construct (line 282) | construct(classTarget, args) {
FILE: packages/playwright-extra/src/helper/loader.ts
class Loader (line 4) | class Loader<TargetModule> {
method constructor (line 5) | constructor(public moduleName: string, public packageNames: string[]) {}
method lazyloadExportOrDie (line 17) | public lazyloadExportOrDie<T extends keyof TargetModule>(exportName: T) {
method loadModule (line 37) | public loadModule() {
method loadModuleOrDie (line 42) | public loadModuleOrDie(): TargetModule {
method requireError (line 50) | public get requireError() {
function requirePackages (line 69) | function requirePackages<TargetModule = any>(packageNames: string[]) {
FILE: packages/playwright-extra/src/index.ts
type PlaywrightBrowserLauncher (line 11) | type PlaywrightBrowserLauncher = pw.BrowserType<{}>
type AugmentedBrowserLauncher (line 13) | type AugmentedBrowserLauncher = PlaywrightExtraClass &
type PlaywrightCompatibleLauncher (line 20) | interface PlaywrightCompatibleLauncher {
type ExtraModuleExports (line 26) | interface ExtraModuleExports {
type PlaywrightModuleExports (line 37) | type PlaywrightModuleExports = typeof pw
FILE: packages/playwright-extra/src/plugins.ts
class PluginList (line 15) | class PluginList {
method constructor (line 21) | constructor() {}
method list (line 26) | public get list() {
method names (line 33) | public get names() {
method add (line 43) | public add(plugin: Plugin) {
method isValidPluginInstance (line 62) | protected isValidPluginInstance(plugin: Plugin) {
method onPluginError (line 85) | public onPluginError(plugin: Plugin, method: PluginMethodName, err: Er...
method setDependencyDefaults (line 101) | public setDependencyDefaults(dependencyPath: string, opts: any) {
method setDependencyResolution (line 115) | public setDependencyResolution(
method prepare (line 127) | public prepare() {
method filterByMethod (line 133) | protected filterByMethod(methodName: PluginMethodName) {
method _addPuppeteerCompatIfNeeded (line 147) | protected _addPuppeteerCompatIfNeeded<TMethod extends PluginMethodName>(
method dispatch (line 180) | public dispatch<TMethod extends PluginMethodName>(
method dispatchBlocking (line 227) | public async dispatchBlocking<TMethod extends PluginMethodName>(
method order (line 261) | protected order() {
method getData (line 286) | protected getData(name?: string) {
method resolvePluginsStanza (line 297) | protected resolvePluginsStanza() {
method resolveDependenciesStanza (line 320) | protected resolveDependenciesStanza() {
method resolveDependencies (line 407) | protected resolveDependencies() {
FILE: packages/playwright-extra/src/puppeteer-compatiblity-shim/index.ts
type PlaywrightObject (line 6) | type PlaywrightObject = pw.Page | pw.Frame | pw.Browser
type PuppeteerBrowserShim (line 8) | interface PuppeteerBrowserShim {
type PuppeteerPageShim (line 15) | interface PuppeteerPageShim {
function addPuppeteerCompat (line 45) | function addPuppeteerCompat<
function getPageCDPSession (line 82) | async function getPageCDPSession(page: pw.Page | pw.Frame) {
function getBrowserCDPSession (line 105) | async function getBrowserCDPSession(browser: pw.Browser) {
function createPageShim (line 125) | function createPageShim(page: pw.Page | pw.Frame) {
function createBrowserShim (line 199) | function createBrowserShim(browser: pw.Browser) {
FILE: packages/playwright-extra/src/puppeteer-compatiblity-shim/playwright-shim.d.ts
type Page (line 8) | interface Page extends PuppeteerPageShim {}
type Frame (line 9) | interface Frame extends PuppeteerPageShim {}
type Browser (line 10) | interface Browser extends PuppeteerBrowserShim {}
FILE: packages/playwright-extra/src/types/index.ts
type PropType (line 3) | type PropType<TObj, TProp extends keyof TObj> = TObj[TProp]
type PluginEnv (line 4) | type PluginEnv = { framework: 'playwright' }
method onPluginRegistered (line 8) | async onPluginRegistered(env?: PluginEnv): Promise<void> {}
method beforeLaunch (line 9) | async beforeLaunch(
method afterLaunch (line 12) | async afterLaunch(browserOrContext?: pw.Browser | pw.BrowserContext) {}
method beforeConnect (line 13) | async beforeConnect(
method afterConnect (line 16) | async afterConnect(browser: pw.Browser) {}
method onBrowser (line 17) | async onBrowser(browser: pw.Browser) {}
method onPageCreated (line 18) | async onPageCreated(page: pw.Page) {}
method onPageClose (line 19) | async onPageClose(page: pw.Page) {}
method onDisconnected (line 20) | async onDisconnected(browser?: pw.Browser) {}
method beforeContext (line 22) | async beforeContext(
method onContextCreated (line 26) | async onContextCreated(
type PluginMethodName (line 33) | type PluginMethodName = keyof PluginLifecycleMethods
type PluginMethodFn (line 35) | type PluginMethodFn<TName extends PluginMethodName> = PropType<
type PluginRequirements (line 40) | type PluginRequirements = Set<
type PluginDependencies (line 45) | type PluginDependencies = Set<string> | Map<string, any> | string[]
type PluginData (line 47) | interface PluginData {
type CompatiblePluginLifecycleMethods (line 59) | interface CompatiblePluginLifecycleMethods {
type PuppeteerExtraPlugin (line 78) | interface PuppeteerExtraPlugin extends Partial<PluginLifecycleMethods> {
type CompatiblePuppeteerPlugin (line 97) | interface CompatiblePuppeteerPlugin
type CompatiblePlaywrightPlugin (line 103) | interface CompatiblePlaywrightPlugin
type CompatibleExtraPlugin (line 109) | interface CompatibleExtraPlugin
type CompatiblePlugin (line 118) | type CompatiblePlugin =
type CompatiblePluginModule (line 122) | type CompatiblePluginModule = (...args: any[]) => CompatiblePlugin
type Plugin (line 124) | type Plugin = PuppeteerExtraPlugin
type PluginModule (line 125) | type PluginModule = (...args: any[]) => Plugin
FILE: packages/playwright-extra/test/fixtures/dummyplugin.ts
class DummyPlugin (line 3) | class DummyPlugin extends PuppeteerExtraPlugin {
method constructor (line 7) | constructor(opts = {}) {
method name (line 10) | get name() {
method onPluginRegistered (line 14) | async onPluginRegistered(...args: any[]) {
method beforeLaunch (line 17) | async beforeLaunch(...args: any[]) {
method afterLaunch (line 20) | async afterLaunch(...args: any[]) {
method beforeConnect (line 23) | async beforeConnect(...args: any[]) {
method afterConnect (line 26) | async afterConnect(...args: any[]) {
method onBrowser (line 29) | async onBrowser(...args: any[]) {
method onTargetCreated (line 32) | async onTargetCreated(...args: any[]) {
method onPageCreated (line 35) | async onPageCreated(...args: any[]) {
method onTargetChanged (line 38) | async onTargetChanged(...args: any[]) {
method onTargetDestroyed (line 41) | async onTargetDestroyed(...args: any[]) {
method onDisconnected (line 44) | async onDisconnected(...args: any[]) {
method onClose (line 47) | async onClose(...args: any[]) {
method beforeContext (line 52) | async beforeContext(...args: any[]) {
method onContextCreated (line 55) | async onContextCreated(...args: any[]) {
FILE: packages/playwright-extra/test/fixtures/extra.ts
type PluginModuleWithOptions (line 8) | type PluginModuleWithOptions = { module: any; opts?: Record<string, any> }
type ExtraOptions (line 10) | type ExtraOptions = {}
type ExtraFixtures (line 12) | type ExtraFixtures = {
type WorkerFixtures (line 21) | type WorkerFixtures = {
FILE: packages/playwright-extra/test/puppeteer-plugins/stealth.spec.ts
function getWebglUnmasked (line 39) | function getWebglUnmasked() {
FILE: packages/plugin-proxy-router/src/plugin.ts
type ExtraPluginProxyRouterOptions (line 4) | type ExtraPluginProxyRouterOptions = ProxyRouterOpts & {
class ExtraPluginProxyRouter (line 18) | class ExtraPluginProxyRouter extends PuppeteerExtraPlugin {
method constructor (line 26) | constructor(opts: Partial<ExtraPluginProxyRouterOptions>) {
method name (line 32) | get name() {
method defaults (line 36) | get defaults(): ExtraPluginProxyRouterOptions {
method proxies (line 47) | public get proxies() {
method proxies (line 50) | public set proxies(proxies) {
method stats (line 55) | public get stats() {
method routeByHost (line 60) | public get routeByHost() {
method routeByHost (line 63) | public set routeByHost(fn) {
method proxyBypassListString (line 67) | private get proxyBypassListString() {
method onPluginRegistered (line 71) | async onPluginRegistered(args?: { framework: 'playwright' }): Promise<...
method beforeLaunch (line 77) | async beforeLaunch(options: unknown = {}): Promise<void> {
method onDisconnected (line 107) | async onDisconnected(): Promise<void> {
type PuppeteerLaunchOptions (line 112) | interface PuppeteerLaunchOptions {
type PlaywrightLaunchOptions (line 116) | interface PlaywrightLaunchOptions {
FILE: packages/plugin-proxy-router/src/router.ts
type ProxyServerOpts (line 14) | type ProxyServerOpts = ConstructorParameters<typeof ProxyServer>[0]
type Proxies (line 16) | interface Proxies {
type ProxyName (line 23) | type ProxyName = 'DIRECT' | 'DEFAULT' | 'ABORT' | string
type RouteByHostArgs (line 26) | interface RouteByHostArgs {
type RouteByHostResponse (line 34) | type RouteByHostResponse = ProxyName | void
type RouteByHostFn (line 35) | type RouteByHostFn = (
type ProxyRouterOpts (line 39) | interface ProxyRouterOpts {
class ProxyRouter (line 86) | class ProxyRouter {
method constructor (line 107) | constructor(opts: ProxyRouterOpts = {}) {
method proxyServerUrl (line 226) | public get proxyServerUrl() {
method effectiveProxies (line 234) | public get effectiveProxies() {
method listen (line 242) | public async listen(): Promise<number> {
method close (line 271) | public async close(): Promise<NodeJS.ErrnoException | null> {
method getProxyForName (line 285) | public getProxyForName(name: ProxyName): string | null {
method handleProxyServerRequest (line 290) | protected async handleProxyServerRequest({
function redactProxyUrl (line 329) | function redactProxyUrl(input: unknown) {
FILE: packages/plugin-proxy-router/src/stats.ts
type ConnectionLogEntry (line 3) | interface ConnectionLogEntry {
type ConnectionStats (line 12) | interface ConnectionStats {
class ProxyRouterStats (line 19) | class ProxyRouterStats {
method constructor (line 24) | constructor(private proxyServer: ProxyServer) {}
method addConnection (line 27) | public addConnection(id: number, proxy: string, host: string) {
method addStats (line 31) | public addStats(connectionId: number, stats: ConnectionStats) {
method byProxy (line 36) | public get byProxy() {
method byHost (line 63) | public get byHost() {
method getStatsFromActiveConnections (line 89) | protected getStatsFromActiveConnections() {
method calculateProxyBytes (line 99) | protected calculateProxyBytes(stats?: Partial<ConnectionStats>) {
FILE: packages/plugin-proxy-router/src/utils/port.ts
type Options (line 3) | interface Options {
FILE: packages/puppeteer-extra-plugin-adblocker/src/ambient.d.ts
type SymbolConstructor (line 5) | interface SymbolConstructor {
FILE: packages/puppeteer-extra-plugin-adblocker/src/index.test.ts
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra-plugin-adblocker/src/index.ts
type PluginOptions (line 13) | interface PluginOptions {
class PuppeteerExtraPluginAdblocker (line 30) | class PuppeteerExtraPluginAdblocker extends PuppeteerExtraPlugin {
method constructor (line 33) | constructor(opts: Partial<PluginOptions>) {
method name (line 38) | get name() {
method defaults (line 42) | get defaults(): PluginOptions {
method engineCacheFile (line 52) | get engineCacheFile() {
method persistToCache (line 62) | private async persistToCache(blocker: PuppeteerBlocker): Promise<void> {
method loadFromCache (line 74) | private async loadFromCache(): Promise<PuppeteerBlocker> {
method loadFromRemote (line 90) | private async loadFromRemote(): Promise<PuppeteerBlocker> {
method getBlocker (line 109) | async getBlocker(): Promise<PuppeteerBlocker> {
method setRequestInterceptionPriority (line 127) | private setRequestInterceptionPriority(): void {
method beforeLaunch (line 134) | async beforeLaunch() {
method beforeConnect (line 142) | async beforeConnect() {
method onPageCreated (line 150) | async onPageCreated(page: any) {
FILE: packages/puppeteer-extra-plugin-anonymize-ua/index.d.ts
type CustomFn (line 3) | type CustomFn = ((ua: string) => string | null) | null;
class Plugin (line 4) | class Plugin extends PuppeteerExtraPlugin {
FILE: packages/puppeteer-extra-plugin-anonymize-ua/index.js
class Plugin (line 24) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 25) | constructor(opts = {}) {
method name (line 29) | get name() {
method defaults (line 33) | get defaults() {
method onPageCreated (line 41) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-anonymize-ua/index.test.js
constant PLUGIN_NAME (line 3) | const PLUGIN_NAME = 'anonymize-ua'
FILE: packages/puppeteer-extra-plugin-anonymize-ua/test/headless.js
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra-plugin-anonymize-ua/test/headless_off.js
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra-plugin-anonymize-ua/test/popup.js
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra-plugin-anonymize-ua/test/stresstest.js
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra-plugin-block-resources/index.d.ts
type PluginOptions (line 4) | interface PluginOptions {
class Plugin (line 10) | class Plugin extends PuppeteerExtraPlugin {
FILE: packages/puppeteer-extra-plugin-block-resources/index.js
class Plugin (line 44) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 45) | constructor(opts = {}) {
method name (line 49) | get name() {
method defaults (line 53) | get defaults() {
method availableTypes (line 83) | get availableTypes() {
method blockedTypes (line 94) | get blockedTypes() {
method interceptResolutionPriority (line 105) | get interceptResolutionPriority() {
method onRequest (line 112) | onRequest(request) {
method onPageCreated (line 147) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-block-resources/index.test.js
constant PLUGIN_NAME (line 3) | const PLUGIN_NAME = 'block-resources'
FILE: packages/puppeteer-extra-plugin-click-and-wait/index.js
class Plugin (line 24) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 25) | constructor(opts = {}) {
method name (line 29) | get name() {
method clickAndWaitForNavigation (line 33) | async clickAndWaitForNavigation(selector, clickOptions, waitOptions) {
method onPageCreated (line 42) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-devtools/index.js
class Plugin (line 34) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 35) | constructor(opts = {}) {
method name (line 42) | get name() {
method defaults (line 46) | get defaults() {
method createTunnel (line 86) | async createTunnel(browser) {
method setAuthCredentials (line 125) | setAuthCredentials(user, pass) {
method getLocalDevToolsUrl (line 149) | getLocalDevToolsUrl(browser) {
method _printGeneratedPasswordWhenNotOverridden (line 164) | _printGeneratedPasswordWhenNotOverridden(url) {
class Tunnel (line 188) | class Tunnel extends RemoteDevTools.DevToolsTunnel {
method constructor (line 189) | constructor(wsEndpoint, opts = {}) {
method url (line 203) | get url() {
method getUrlForPage (line 219) | getUrlForPage(page) {
method close (line 234) | close() {
FILE: packages/puppeteer-extra-plugin-devtools/index.test.js
constant PLUGIN_NAME (line 3) | const PLUGIN_NAME = 'devtools'
FILE: packages/puppeteer-extra-plugin-devtools/lib/RemoteDevTools.js
class DevToolsCommon (line 22) | class DevToolsCommon {
method constructor (line 23) | constructor(webSocketDebuggerUrl, opts = {}) {
method fetchVersion (line 36) | async fetchVersion() {
method fetchList (line 46) | async fetchList() {
class DevToolsLocal (line 60) | class DevToolsLocal extends DevToolsCommon {
method constructor (line 61) | constructor(webSocketDebuggerUrl, opts = {}) {
method url (line 65) | get url() {
method getUrlForPageId (line 69) | getUrlForPageId(pageId) {
class DevToolsTunnel (line 88) | class DevToolsTunnel extends DevToolsCommon {
method constructor (line 89) | constructor(webSocketDebuggerUrl, opts = {}) {
method defaults (line 99) | get defaults() {
method url (line 108) | get url() {
method getUrlForPageId (line 112) | getUrlForPageId(pageId) {
method create (line 116) | async create() {
method close (line 146) | close() {
method _generateSubdomain (line 154) | _generateSubdomain(prefix) {
method _createBasicAuth (line 163) | _createBasicAuth(user, pass) {
method _modifyFetchToIncludeCredentials (line 186) | _modifyFetchToIncludeCredentials(body) {
method _modifyJSONResponse (line 203) | _modifyJSONResponse(body) {
method _createProxyServer (line 214) | _createProxyServer(targetHost = 'localhost', targetPort) {
method _createServer (line 247) | async _createServer(port, auth = null) {
method _createTunnel (line 259) | async _createTunnel(options) {
FILE: packages/puppeteer-extra-plugin-flash/index.js
class Plugin (line 31) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 32) | constructor(opts = {}) {
method name (line 36) | get name() {
method defaults (line 40) | get defaults() {
method requirements (line 48) | get requirements() {
method dependencies (line 52) | get dependencies() {
method beforeLaunch (line 56) | async beforeLaunch(options) {
method data (line 69) | get data() {
FILE: packages/puppeteer-extra-plugin-font-size/index.js
class Plugin (line 18) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 19) | constructor(opts = {}) {
method name (line 23) | get name() {
method defaults (line 27) | get defaults() {
method requirements (line 31) | get requirements() {
method dependencies (line 35) | get dependencies() {
method data (line 39) | get data() {
FILE: packages/puppeteer-extra-plugin-recaptcha/src/ambient.d.ts
type SymbolConstructor (line 5) | interface SymbolConstructor {
FILE: packages/puppeteer-extra-plugin-recaptcha/src/content-hcaptcha.ts
class HcaptchaContentScript (line 15) | class HcaptchaContentScript {
method constructor (line 24) | constructor(
method _waitUntilDocumentReady (line 39) | private async _waitUntilDocumentReady() {
method _paintCaptchaBusy (line 56) | private _paintCaptchaBusy($iframe: HTMLIFrameElement) {
method _findRegularCheckboxes (line 68) | private _findRegularCheckboxes() {
method _findActiveChallenges (line 76) | private _findActiveChallenges() {
method _extractInfoFromIframes (line 83) | private _extractInfoFromIframes(iframes: HTMLIFrameElement[]) {
method findRecaptchas (line 101) | public async findRecaptchas() {
method enterRecaptchaSolutions (line 126) | public async enterRecaptchaSolutions() {
FILE: packages/puppeteer-extra-plugin-recaptcha/src/content.ts
type FrameSources (line 12) | interface FrameSources {
class RecaptchaContentScript (line 21) | class RecaptchaContentScript {
method constructor (line 26) | constructor(
method _isInViewport (line 64) | private _isInViewport(elem: any) {
method _flattenObject (line 79) | private _flattenObject(item: any, levels = 2, ignoreHTML = true) {
method _getKeyByValue (line 104) | private _getKeyByValue(object: any, value: any) {
method _waitUntilDocumentReady (line 108) | private async _waitUntilDocumentReady() {
method _paintCaptchaBusy (line 129) | private _paintCaptchaBusy($iframe: HTMLIFrameElement) {
method _paintCaptchaSolved (line 140) | private _paintCaptchaSolved($iframe: HTMLIFrameElement) {
method _findVisibleIframeNodes (line 151) | private _findVisibleIframeNodes() {
method _findVisibleIframeNodeById (line 158) | private _findVisibleIframeNodeById(id?: string) {
method _hideChallengeWindowIfPresent (line 164) | private _hideChallengeWindowIfPresent(id: string = '') {
method _generateFrameSources (line 185) | private _generateFrameSources(): FrameSources {
method getFrameSelectorForId (line 210) | private getFrameSelectorForId(type: 'anchor' | 'bframe' = 'anchor', id...
method getClients (line 217) | private getClients() {
method getVisibleIframesIds (line 227) | private getVisibleIframesIds() {
method getInvisibleIframesIds (line 243) | private getInvisibleIframesIds() {
method getIframesIds (line 261) | private getIframesIds() {
method isEnterpriseCaptcha (line 274) | private isEnterpriseCaptcha(id?: string) {
method isInvisible (line 283) | private isInvisible(id?: string) {
method hasActiveChallengePopup (line 290) | private hasActiveChallengePopup(id?: string) {
method hasChallengeFrame (line 301) | private hasChallengeFrame(id?: string) {
method isInViewport (line 309) | private isInViewport(id?: string) {
method getResponseInputById (line 321) | private getResponseInputById(id?: string) {
method getClientById (line 336) | private getClientById(id?: string) {
method extractInfoFromClient (line 359) | private extractInfoFromClient(client?: any) {
method findRecaptchas (line 386) | public async findRecaptchas() {
method enterRecaptchaSolutions (line 437) | public async enterRecaptchaSolutions() {
FILE: packages/puppeteer-extra-plugin-recaptcha/src/detection.test.ts
constant PUPPETEER_ARGS (line 7) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra-plugin-recaptcha/src/index.test.ts
constant PUPPETEER_ARGS (line 10) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra-plugin-recaptcha/src/index.ts
class PuppeteerExtraPluginRecaptcha (line 22) | class PuppeteerExtraPluginRecaptcha extends PuppeteerExtraPlugin {
method constructor (line 25) | constructor(opts: Partial<types.PluginOptions>) {
method name (line 32) | get name() {
method defaults (line 36) | get defaults(): types.PluginOptions {
method opts (line 46) | get opts(): types.PluginOptions {
method contentScriptOpts (line 50) | get contentScriptOpts(): types.ContentScriptOpts {
method _generateContentScript (line 63) | private _generateContentScript(
method _filterRecaptchas (line 91) | private _filterRecaptchas(recaptchas: types.CaptchaInfo[] = []) {
method findRecaptchas (line 128) | async findRecaptchas(page: Page | Frame) {
method getRecaptchaSolutions (line 201) | async getRecaptchaSolutions(
method enterRecaptchaSolutions (line 248) | async enterRecaptchaSolutions(
method solveRecaptchas (line 283) | async solveRecaptchas(
method _addCustomMethods (line 330) | private _addCustomMethods(prop: Page | Frame) {
method onPageCreated (line 342) | async onPageCreated(page: Page) {
method onBrowser (line 358) | async onBrowser(browser: Browser) {
FILE: packages/puppeteer-extra-plugin-recaptcha/src/playwright-mods.d.ts
type Page (line 7) | interface Page extends RecaptchaPluginPageAdditions {}
type Frame (line 8) | interface Frame extends RecaptchaPluginPageAdditions {}
FILE: packages/puppeteer-extra-plugin-recaptcha/src/provider/2captcha-api.ts
function pollCaptcha (line 19) | function pollCaptcha(captchaId, options, invalid, callback) {
FILE: packages/puppeteer-extra-plugin-recaptcha/src/provider/2captcha.ts
constant PROVIDER_ID (line 1) | const PROVIDER_ID = '2captcha'
type DecodeRecaptchaAsyncResult (line 13) | interface DecodeRecaptchaAsyncResult {
type TwoCaptchaProviderOpts (line 19) | interface TwoCaptchaProviderOpts {
function decodeRecaptchaAsync (line 29) | async function decodeRecaptchaAsync(
function getSolutions (line 54) | async function getSolutions(
function getSolution (line 66) | async function getSolution(
FILE: packages/puppeteer-extra-plugin-recaptcha/src/puppeteer-mods.d.ts
type Page (line 14) | interface Page extends RecaptchaPluginPageAdditions {}
type Frame (line 15) | interface Frame extends RecaptchaPluginPageAdditions {}
type Page (line 19) | interface Page extends RecaptchaPluginPageAdditions {}
type Frame (line 20) | interface Frame extends RecaptchaPluginPageAdditions {}
FILE: packages/puppeteer-extra-plugin-recaptcha/src/solve.test.ts
constant PUPPETEER_ARGS (line 7) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra-plugin-recaptcha/src/types.ts
type Window (line 9) | interface Window {
type RecaptchaPluginPageAdditions (line 17) | type RecaptchaPluginPageAdditions = {
type SolutionProvider (line 34) | interface SolutionProvider<TOpts = any> {
type FindRecaptchasResult (line 41) | interface FindRecaptchasResult {
type EnterRecaptchaSolutionsResult (line 46) | interface EnterRecaptchaSolutionsResult {
type GetSolutionsResult (line 50) | interface GetSolutionsResult {
type SolveRecaptchasResult (line 55) | type SolveRecaptchasResult = FindRecaptchasResult &
type CaptchaVendor (line 59) | type CaptchaVendor = 'recaptcha' | 'hcaptcha'
type CaptchaType (line 61) | type CaptchaType = 'checkbox' | 'invisible' | 'score'
type CaptchaInfo (line 63) | interface CaptchaInfo {
type FilteredCaptcha (line 92) | type FilteredCaptcha = CaptchaInfo & {
type CaptchaSolution (line 100) | interface CaptchaSolution {
type CaptchaSolved (line 113) | interface CaptchaSolved {
type PluginOptions (line 123) | interface PluginOptions {
type ContentScriptOpts (line 139) | interface ContentScriptOpts {
type ContentScriptData (line 144) | interface ContentScriptData {
FILE: packages/puppeteer-extra-plugin-repl/index.d.ts
type Page (line 10) | interface Page extends EventEmitter, FrameBase {
type Browser (line 14) | interface Browser extends EventEmitter, TargetAwaiter {
type Options (line 27) | interface Options extends DefaultOptions, PluginOptions {}
type DefaultOptions (line 29) | interface DefaultOptions {
class Plugin (line 37) | class Plugin extends PuppeteerExtraPlugin {
FILE: packages/puppeteer-extra-plugin-repl/index.js
class Plugin (line 38) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 39) | constructor(opts = {}) {
method name (line 43) | get name() {
method defaults (line 47) | get defaults() {
method requirements (line 56) | get requirements() {
method repl (line 78) | async repl(obj) {
method onPageCreated (line 87) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-repl/index.test.js
constant PLUGIN_NAME (line 3) | const PLUGIN_NAME = 'repl'
FILE: packages/puppeteer-extra-plugin-repl/lib/REPLSession.js
class REPLSession (line 4) | class REPLSession {
method constructor (line 5) | constructor(opts) {
method extraMethods (line 19) | get extraMethods() {
method start (line 23) | async start() {
method _createInterface (line 30) | _createInterface() {
method _showIntro (line 47) | _showIntro() {
method _onLineInput (line 59) | async _onLineInput(line) {
method _evalAsync (line 72) | async _evalAsync(cmd) {
FILE: packages/puppeteer-extra-plugin-repl/lib/super-readline.js
class SuperInterface (line 41) | class SuperInterface extends Interface {
method constructor (line 42) | constructor(options) {
method _tabComplete (line 48) | _tabComplete(lastKeypressWasTab) {
method showTabCompletions (line 54) | showTabCompletions() {
method _writeToOutput (line 58) | _writeToOutput(stringToWrite) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/_template/index.js
class Plugin (line 10) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 11) | constructor(opts = {}) {
method name (line 15) | get name() {
method onPageCreated (line 19) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js
function recurse (line 467) | function recurse(obj) {
function fromEntries (line 501) | function fromEntries(iterable) {
method apply (line 538) | apply(target, ctx, args) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.test.js
method get (line 34) | get(target, param) {
method apply (line 40) | apply() {
method get (line 66) | get(target, param) {
method apply (line 72) | apply() {
function toStringTest (line 258) | function toStringTest(obj) {
method get (line 438) | get() {
method apply (line 441) | apply() {
method get (line 503) | get() {
method get (line 515) | get() {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/index.js
class Plugin (line 10) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 11) | constructor(opts = {}) {
method name (line 15) | get name() {
method onPageCreated (line 19) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/index.js
class Plugin (line 24) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 25) | constructor(opts = {}) {
method name (line 29) | get name() {
method onPageCreated (line 33) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/index.js
class Plugin (line 22) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 23) | constructor(opts = {}) {
method name (line 27) | get name() {
method onPageCreated (line 31) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js
constant STATIC_DATA (line 7) | const STATIC_DATA = require('./staticData.json')
class Plugin (line 12) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 13) | constructor(opts = {}) {
method name (line 17) | get name() {
method defaults (line 21) | get defaults() {
method onPageCreated (line 25) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.test.js
constant STATIC_DATA (line 12) | const STATIC_DATA = require('./staticData.json')
FILE: packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/index.js
class Plugin (line 15) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 16) | constructor(opts = {}) {
method name (line 20) | get name() {
method requirements (line 24) | get requirements() {
method beforeLaunch (line 28) | async beforeLaunch(options = {}) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/index.js
class Plugin (line 13) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 14) | constructor(opts = {}) {
method name (line 18) | get name() {
method requirements (line 22) | get requirements() {
method onPageCreated (line 27) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/index.js
class Plugin (line 11) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 12) | constructor(opts = {}) {
method name (line 16) | get name() {
method onPageCreated (line 20) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js
class Plugin (line 16) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 17) | constructor(opts = {}) {
method name (line 21) | get name() {
method defaults (line 25) | get defaults() {
method onPageCreated (line 31) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js
class Plugin (line 12) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 13) | constructor(opts = {}) {
method name (line 17) | get name() {
method defaults (line 21) | get defaults() {
method onPageCreated (line 27) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js
class Plugin (line 13) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 14) | constructor(opts = {}) {
method name (line 18) | get name() {
method onPageCreated (line 23) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.test.js
function getNotificationPermission (line 28) | async function getNotificationPermission() {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/functionMocks.js
method apply (line 13) | apply(target, ctx, args) {
method apply (line 31) | apply(target, ctx, args) {
method apply (line 45) | apply(target, ctx, args) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js
class Plugin (line 26) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 27) | constructor(opts = {}) {
method name (line 31) | get name() {
method onPageCreated (line 35) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/magicArray.js
method ownKeys (line 60) | ownKeys(target) {
method getOwnPropertyDescriptor (line 63) | getOwnPropertyDescriptor(target, prop) {
method get (line 109) | get(target, key = '') {
method ownKeys (line 123) | ownKeys(target) {
method getOwnPropertyDescriptor (line 135) | getOwnPropertyDescriptor(target, prop) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.vendor/index.js
class Plugin (line 30) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 31) | constructor(opts = {}) {
method name (line 35) | get name() {
method defaults (line 39) | get defaults() {
method onPageCreated (line 45) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/index.js
class Plugin (line 9) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 10) | constructor(opts = {}) {
method name (line 14) | get name() {
method onPageCreated (line 18) | async onPageCreated(page) {
method beforeLaunch (line 34) | async beforeLaunch(options) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/navigator.webdriver/index.test.js
function getExpectedValue (line 6) | function getExpectedValue(looseVersionString) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/index.js
class Plugin (line 9) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 10) | constructor(opts = {}) {
method name (line 14) | get name() {
method onPageCreated (line 18) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/sourceurl/index.test.js
constant TEST_HTML_FILE (line 6) | const TEST_HTML_FILE = require('path').join(__dirname, './_fixtures/test...
FILE: packages/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.js
class Plugin (line 42) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 43) | constructor(opts = {}) {
method name (line 49) | get name() {
method dependencies (line 53) | get dependencies() {
method defaults (line 57) | get defaults() {
method onPageCreated (line 65) | async onPageCreated(page) {
method beforeLaunch (line 185) | async beforeLaunch(options) {
method beforeConnect (line 190) | async beforeConnect() {
method data (line 195) | get data() {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/user-agent-override/index.test.js
function userAgentData (line 264) | async function userAgentData() {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/webgl.vendor/index.js
class Plugin (line 16) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 17) | constructor(opts = {}) {
method name (line 21) | get name() {
method onPageCreated (line 26) | async onPageCreated(page) {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/webgl.vendor/index.test.js
function extendedTests (line 35) | async function extendedTests() {
function getVideoCardInfo (line 122) | function getVideoCardInfo(context = 'webgl') {
FILE: packages/puppeteer-extra-plugin-stealth/evasions/window.outerdimensions/index.js
class Plugin (line 9) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 10) | constructor(opts = {}) {
method name (line 14) | get name() {
method onPageCreated (line 18) | async onPageCreated(page) {
method beforeLaunch (line 32) | async beforeLaunch(options) {
FILE: packages/puppeteer-extra-plugin-stealth/examples/detect-headless.js
function test (line 10) | async function test(name, fn) {
FILE: packages/puppeteer-extra-plugin-stealth/index.d.ts
class StealthPlugin (line 73) | class StealthPlugin extends StealthPlugin_base {
FILE: packages/puppeteer-extra-plugin-stealth/index.js
class StealthPlugin (line 72) | class StealthPlugin extends PuppeteerExtraPlugin {
method constructor (line 73) | constructor(opts = {}) {
method name (line 77) | get name() {
method defaults (line 81) | get defaults() {
method dependencies (line 112) | get dependencies() {
method availableEvasions (line 130) | get availableEvasions() {
method enabledEvasions (line 147) | get enabledEvasions() {
method enabledEvasions (line 154) | set enabledEvasions(evasions) {
method onBrowser (line 158) | async onBrowser(browser) {
FILE: packages/puppeteer-extra-plugin-stealth/index.test.js
constant PLUGIN_NAME (line 3) | const PLUGIN_NAME = 'stealth'
FILE: packages/puppeteer-extra-plugin-stealth/stealthtests/headful-chrome-stealth.js
function main (line 9) | async function main() {
FILE: packages/puppeteer-extra-plugin-stealth/stealthtests/headful-chrome-vanilla.js
function main (line 7) | async function main() {
FILE: packages/puppeteer-extra-plugin-stealth/stealthtests/headful-chromium-stealth.js
function main (line 8) | async function main() {
FILE: packages/puppeteer-extra-plugin-stealth/stealthtests/headful-chromium-vanilla.js
function main (line 7) | async function main() {
FILE: packages/puppeteer-extra-plugin-stealth/stealthtests/headless-chrome-stealth.js
function main (line 9) | async function main() {
FILE: packages/puppeteer-extra-plugin-stealth/stealthtests/headless-chrome-vanilla.js
function main (line 7) | async function main() {
FILE: packages/puppeteer-extra-plugin-stealth/stealthtests/headless-chromium-stealth.js
function main (line 9) | async function main() {
FILE: packages/puppeteer-extra-plugin-stealth/stealthtests/headless-chromium-vanilla.js
function main (line 7) | async function main() {
FILE: packages/puppeteer-extra-plugin-stealth/test/cat-and-mouse.test.js
function detectHeadless (line 36) | async function detectHeadless() {
FILE: packages/puppeteer-extra-plugin-stealth/test/service-worker.test.js
function detectFingerprint (line 84) | function detectFingerprint() {
FILE: packages/puppeteer-extra-plugin-user-data-dir/index.js
class Plugin (line 18) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 19) | constructor(opts = {}) {
method name (line 38) | get name() {
method requirements (line 42) | get requirements() {
method shouldDeleteDirectory (line 46) | get shouldDeleteDirectory() {
method temporaryDirectoryPath (line 53) | get temporaryDirectoryPath() {
method defaultProfilePath (line 57) | get defaultProfilePath() {
method makeTemporaryDirectory (line 61) | async makeTemporaryDirectory() {
method deleteUserDataDir (line 66) | deleteUserDataDir() {
method writeFilesToProfile (line 87) | async writeFilesToProfile() {
method beforeLaunch (line 110) | async beforeLaunch(options) {
method onDisconnected (line 120) | async onDisconnected() {
FILE: packages/puppeteer-extra-plugin-user-preferences/index.js
class Plugin (line 30) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 31) | constructor(opts = {}) {
method name (line 42) | get name() {
method requirements (line 46) | get requirements() {
method dependencies (line 50) | get dependencies() {
method data (line 54) | get data() {
method combinedPrefs (line 67) | get combinedPrefs() {
method beforeLaunch (line 71) | async beforeLaunch(options) {
FILE: packages/puppeteer-extra-plugin/src/ambient.d.ts
type SymbolConstructor (line 5) | interface SymbolConstructor {
FILE: packages/puppeteer-extra-plugin/src/index.test.ts
class Derived (line 10) | class Derived extends PuppeteerExtraPlugin {}
class Plugin (line 17) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 18) | constructor(opts = {}) {
method name (line 21) | get name() {
method constructor (line 41) | constructor(opts = {}) {
method name (line 44) | get name() {
method constructor (line 66) | constructor(opts = {}) {
method name (line 69) | get name() {
method constructor (line 89) | constructor(opts = {}) {
method name (line 92) | get name() {
method defaults (line 95) | get defaults() {
method constructor (line 113) | constructor(opts = {}) {
method name (line 116) | get name() {
class Plugin (line 40) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 18) | constructor(opts = {}) {
method name (line 21) | get name() {
method constructor (line 41) | constructor(opts = {}) {
method name (line 44) | get name() {
method constructor (line 66) | constructor(opts = {}) {
method name (line 69) | get name() {
method constructor (line 89) | constructor(opts = {}) {
method name (line 92) | get name() {
method defaults (line 95) | get defaults() {
method constructor (line 113) | constructor(opts = {}) {
method name (line 116) | get name() {
class Plugin (line 65) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 18) | constructor(opts = {}) {
method name (line 21) | get name() {
method constructor (line 41) | constructor(opts = {}) {
method name (line 44) | get name() {
method constructor (line 66) | constructor(opts = {}) {
method name (line 69) | get name() {
method constructor (line 89) | constructor(opts = {}) {
method name (line 92) | get name() {
method defaults (line 95) | get defaults() {
method constructor (line 113) | constructor(opts = {}) {
method name (line 116) | get name() {
class Plugin (line 88) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 18) | constructor(opts = {}) {
method name (line 21) | get name() {
method constructor (line 41) | constructor(opts = {}) {
method name (line 44) | get name() {
method constructor (line 66) | constructor(opts = {}) {
method name (line 69) | get name() {
method constructor (line 89) | constructor(opts = {}) {
method name (line 92) | get name() {
method defaults (line 95) | get defaults() {
method constructor (line 113) | constructor(opts = {}) {
method name (line 116) | get name() {
class Plugin (line 112) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 18) | constructor(opts = {}) {
method name (line 21) | get name() {
method constructor (line 41) | constructor(opts = {}) {
method name (line 44) | get name() {
method constructor (line 66) | constructor(opts = {}) {
method name (line 69) | get name() {
method constructor (line 89) | constructor(opts = {}) {
method name (line 92) | get name() {
method defaults (line 95) | get defaults() {
method constructor (line 113) | constructor(opts = {}) {
method name (line 116) | get name() {
FILE: packages/puppeteer-extra-plugin/src/index.ts
type PluginOptions (line 7) | interface PluginOptions {
type PluginData (line 10) | interface PluginData {
type PluginDependencies (line 19) | type PluginDependencies = Set<string>
type PluginRequirements (line 20) | type PluginRequirements = Set<
method constructor (line 73) | constructor(opts?: PluginOptions) {
method name (line 92) | get name(): string {
method defaults (line 117) | get defaults(): PluginOptions {
method requirements (line 145) | get requirements(): PluginRequirements {
method dependencies (line 160) | get dependencies(): PluginDependencies {
method data (line 196) | get data(): PluginData[] {
method opts (line 215) | get opts(): PluginOptions {
method debug (line 235) | get debug(): Debugger {
method beforeLaunch (line 256) | async beforeLaunch(options: any) {
method afterLaunch (line 287) | async afterLaunch(
method beforeConnect (line 305) | async beforeConnect(options: Puppeteer.ConnectOptions) {
method afterConnect (line 319) | async afterConnect(browser: Puppeteer.Browser, opts = {}) {
method onBrowser (line 335) | public async onBrowser(browser: Puppeteer.Browser, opts: any): Promise<v...
method onTargetCreated (line 348) | async onTargetCreated(target: Puppeteer.Target) {
method onPageCreated (line 371) | async onPageCreated(page: Puppeteer.Page) {
method onTargetChanged (line 384) | async onTargetChanged(target: Puppeteer.Target) {
method onTargetDestroyed (line 397) | async onTargetDestroyed(target: Puppeteer.Target) {
method onDisconnected (line 408) | async onDisconnected() {
method onClose (line 424) | async onClose() {
method onPluginRegistered (line 433) | async onPluginRegistered() {
method getDataFromPlugins (line 448) | getDataFromPlugins(name?: string): PluginData[] {
method _getMissingDependencies (line 461) | _getMissingDependencies(plugins: any) {
method _bindBrowserEvents (line 484) | async _bindBrowserEvents(browser: Puppeteer.Browser, opts: any = {}) {
method _onTargetCreated (line 533) | async _onTargetCreated(target: Puppeteer.Target) {
method _register (line 555) | _register(prototype: any) {
method _registerChildClassMembers (line 563) | _registerChildClassMembers(prototype: any) {
method _hasChildClassMember (line 570) | _hasChildClassMember(name: string) {
method _isPuppeteerExtraPlugin (line 577) | get _isPuppeteerExtraPlugin() {
FILE: packages/puppeteer-extra/src/ambient.d.ts
type SymbolConstructor (line 5) | interface SymbolConstructor {
FILE: packages/puppeteer-extra/src/index.ts
type VanillaPuppeteer (line 13) | interface VanillaPuppeteer
type PuppeteerExtraPlugin (line 27) | interface PuppeteerExtraPlugin {
type BrowserInternals (line 36) | interface BrowserInternals extends Browser {
class PuppeteerExtra (line 63) | class PuppeteerExtra implements VanillaPuppeteer {
method constructor (line 66) | constructor(
method use (line 81) | use(plugin: PuppeteerExtraPlugin): this {
method pptr (line 112) | get pptr(): VanillaPuppeteer {
method launch (line 148) | async launch(
method connect (line 186) | async connect(
method defaultArgs (line 212) | defaultArgs(
method executablePath (line 219) | executablePath(): string {
method createBrowserFetcher (line 228) | createBrowserFetcher(
method _patchPageCreationMethods (line 261) | private _patchPageCreationMethods(browser: BrowserInternals) {
method plugins (line 282) | get plugins() {
method pluginNames (line 292) | get pluginNames() {
method getPluginData (line 309) | public getPluginData(name?: string) {
method getPluginsByProp (line 321) | private getPluginsByProp(prop: string): PuppeteerExtraPlugin[] {
method resolvePluginDependencies (line 334) | private resolvePluginDependencies() {
method orderPlugins (line 396) | private orderPlugins() {
method checkPluginRequirements (line 417) | private checkPluginRequirements(opts = {} as any) {
method callPlugins (line 446) | private async callPlugins(prop: string, ...values: any[]) {
method callPluginsWithValue (line 464) | private async callPluginsWithValue(prop: string, value: any) {
function requireVanillaPuppeteer (line 528) | function requireVanillaPuppeteer(): [VanillaPuppeteer?, Error?] {
FILE: packages/puppeteer-extra/src/puppeteer-legacy.d.ts
type VanillaPuppeteer (line 13) | interface VanillaPuppeteer {
type PuppeteerNode (line 29) | interface PuppeteerNode extends VanillaPuppeteer {}
type PuppeteerNode (line 32) | interface PuppeteerNode extends VanillaPuppeteer {}
FILE: packages/puppeteer-extra/test/connect.js
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
FILE: packages/puppeteer-extra/test/events.js
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
class Plugin (line 26) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 27) | constructor(opts = {}) {
method name (line 30) | get name() {
method onPluginRegistered (line 34) | onPluginRegistered() {
method beforeLaunch (line 37) | beforeLaunch() {
method afterLaunch (line 40) | afterLaunch() {
method beforeConnect (line 43) | beforeConnect() {
method afterConnect (line 46) | afterConnect() {
method onBrowser (line 49) | onBrowser() {
method onTargetCreated (line 52) | onTargetCreated() {
method onPageCreated (line 55) | onPageCreated() {
method onTargetChanged (line 58) | onTargetChanged() {
method onTargetDestroyed (line 61) | onTargetDestroyed() {
method onDisconnected (line 64) | onDisconnected() {
method onClose (line 67) | onClose() {
method constructor (line 109) | constructor(opts = {}) {
method name (line 112) | get name() {
method onPluginRegistered (line 116) | onPluginRegistered() {
method beforeLaunch (line 119) | beforeLaunch() {
method afterLaunch (line 122) | afterLaunch() {
method beforeConnect (line 125) | beforeConnect() {
method afterConnect (line 128) | afterConnect() {
method onBrowser (line 131) | onBrowser() {
method onTargetCreated (line 134) | onTargetCreated() {
method onPageCreated (line 137) | onPageCreated() {
method onTargetChanged (line 140) | onTargetChanged() {
method onTargetDestroyed (line 143) | onTargetDestroyed() {
method onDisconnected (line 146) | onDisconnected() {
method onClose (line 149) | onClose() {
class Plugin (line 108) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 27) | constructor(opts = {}) {
method name (line 30) | get name() {
method onPluginRegistered (line 34) | onPluginRegistered() {
method beforeLaunch (line 37) | beforeLaunch() {
method afterLaunch (line 40) | afterLaunch() {
method beforeConnect (line 43) | beforeConnect() {
method afterConnect (line 46) | afterConnect() {
method onBrowser (line 49) | onBrowser() {
method onTargetCreated (line 52) | onTargetCreated() {
method onPageCreated (line 55) | onPageCreated() {
method onTargetChanged (line 58) | onTargetChanged() {
method onTargetDestroyed (line 61) | onTargetDestroyed() {
method onDisconnected (line 64) | onDisconnected() {
method onClose (line 67) | onClose() {
method constructor (line 109) | constructor(opts = {}) {
method name (line 112) | get name() {
method onPluginRegistered (line 116) | onPluginRegistered() {
method beforeLaunch (line 119) | beforeLaunch() {
method afterLaunch (line 122) | afterLaunch() {
method beforeConnect (line 125) | beforeConnect() {
method afterConnect (line 128) | afterConnect() {
method onBrowser (line 131) | onBrowser() {
method onTargetCreated (line 134) | onTargetCreated() {
method onPageCreated (line 137) | onPageCreated() {
method onTargetChanged (line 140) | onTargetChanged() {
method onTargetDestroyed (line 143) | onTargetDestroyed() {
method onDisconnected (line 146) | onDisconnected() {
method onClose (line 149) | onClose() {
FILE: packages/puppeteer-extra/test/options.js
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
class Plugin (line 24) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 25) | constructor(opts = {}) {
method name (line 28) | get name() {
method data (line 31) | get data() {
method beforeLaunch (line 34) | beforeLaunch(options) {
method afterLaunch (line 39) | afterLaunch(browser, opts) {
method constructor (line 75) | constructor(opts = {}) {
method name (line 78) | get name() {
method data (line 81) | get data() {
method beforeConnect (line 84) | beforeConnect(options) {
method afterConnect (line 88) | afterConnect(browser, opts) {
class Plugin (line 74) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 25) | constructor(opts = {}) {
method name (line 28) | get name() {
method data (line 31) | get data() {
method beforeLaunch (line 34) | beforeLaunch(options) {
method afterLaunch (line 39) | afterLaunch(browser, opts) {
method constructor (line 75) | constructor(opts = {}) {
method name (line 78) | get name() {
method data (line 81) | get data() {
method beforeConnect (line 84) | beforeConnect(options) {
method afterConnect (line 88) | afterConnect(browser, opts) {
FILE: packages/puppeteer-extra/test/plugin-support.js
constant PUPPETEER_ARGS (line 5) | const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']
constant PAGE_TIMEOUT (line 6) | const PAGE_TIMEOUT = 60 * 1000 // 60s
class Plugin (line 35) | class Plugin extends PuppeteerExtraPlugin {
method constructor (line 36) | constructor(opts = {}) {
method name (line 39) | get name() {
method data (line 42) | get data() {
Condensed preview — 267 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (771K chars).
[
{
"path": ".editorconfig",
"chars": 304,
"preview": "# EditorConfig helps developers define and maintain\n# consistent coding styles between different editors and IDEs.\n\nroot"
},
{
"path": ".eslintrc",
"chars": 98,
"preview": "{\n \"extends\": [\"prettier-standard\"],\n \"rules\": {\n \"lines-between-class-members\": \"off\"\n }\n}\n"
},
{
"path": ".gitattributes",
"chars": 19,
"preview": "* text=auto eol=lf\n"
},
{
"path": ".github/FUNDING.yml",
"chars": 44,
"preview": "custom: https://www.buymeacoffee.com/brstnd\n"
},
{
"path": ".github/ISSUE_TEMPLATE/bug.md",
"chars": 1645,
"preview": "---\nname: Bug report\nabout: Create a bug report\ntitle: '[Bug] '\nlabels: 'issue: bug report, needs triage'\n---\n\n<!--\n Th"
},
{
"path": ".github/ISSUE_TEMPLATE/config.yml",
"chars": 240,
"preview": "contact_links:\n - name: Questions and Help\n url: https://github.com/berstend/puppeteer-extra/wiki/Scraping-Chat\n "
},
{
"path": ".github/ISSUE_TEMPLATE/feature_request.md",
"chars": 428,
"preview": "---\nname: Feature request\nabout: Suggest an idea or feature\ntitle: '[Feature] '\nlabels: 'issue: proposal'\n---\n\n**Feature"
},
{
"path": ".github/labeler.yml",
"chars": 655,
"preview": "# This is used with the label workflow which\n# will triage pull requests and apply a label based on the\n# paths that are"
},
{
"path": ".github/workflows/extract-stealth.yml",
"chars": 1208,
"preview": "name: Extract stealth.min.js\n\non:\n push:\n branches:\n - master\n paths:\n - 'packages/puppeteer-extra-plug"
},
{
"path": ".github/workflows/label.yml",
"chars": 515,
"preview": "# This workflow will triage pull requests and apply a label based on the\n# paths that are modified in the pull request.\n"
},
{
"path": ".github/workflows/test.yml",
"chars": 2433,
"preview": "name: Test\n\non:\n pull_request:\n branches:\n - '*'\n types:\n - opened\n - synchronize\n - reopened"
},
{
"path": ".gitignore",
"chars": 492,
"preview": "# See https://help.github.com/ignore-files/ for more about ignoring files.\n\n# dependencies\nnode_modules\n\n# builds\nbuild\n"
},
{
"path": ".prettierrc.js",
"chars": 109,
"preview": "module.exports = {\n ...require('prettier-config-standard'),\n\n // override for Windows\n endOfLine: 'lf',\n}\n"
},
{
"path": ".travis.yml",
"chars": 1992,
"preview": "language: node_js\ndist: trusty\naddons:\n apt:\n packages:\n # This is required to run new chrome on old trusty\n "
},
{
"path": "LICENSE",
"chars": 1097,
"preview": "The MIT License (MIT)\n\nCopyright (c) 2019 berstend <github@berstend.com>\n\nPermission is hereby granted, free of charge, "
},
{
"path": "README.md",
"chars": 4288,
"preview": "# puppeteer-extra [![Downloads](https://img.shields.io/endpoint?style=social&url=https://runkit.io/fezvrasta/combined-np"
},
{
"path": "lerna.json",
"chars": 109,
"preview": "{\n \"packages\": [\"packages/*\"],\n \"version\": \"independent\",\n \"useWorkspaces\": true,\n \"npmClient\": \"yarn\"\n}\n"
},
{
"path": "package.json",
"chars": 1459,
"preview": "{\n \"private\": true,\n \"description\": \"Modular framework to teach Puppeteer new tricks.\",\n \"repository\": \"berstend/pupp"
},
{
"path": "packages/extract-stealth-evasions/.gitignore",
"chars": 25,
"preview": "stealth.min.js\nstealth.js"
},
{
"path": "packages/extract-stealth-evasions/index.js",
"chars": 2551,
"preview": "#!/usr/bin/env node\n\nconst puppeteer = require('puppeteer-extra')\nconst stealth = require('puppeteer-extra-plugin-stealt"
},
{
"path": "packages/extract-stealth-evasions/package.json",
"chars": 779,
"preview": "{\n \"name\": \"extract-stealth-evasions\",\n \"version\": \"2.7.3\",\n \"description\": \"Extract stealth evasions from puppeteer-"
},
{
"path": "packages/extract-stealth-evasions/readme.md",
"chars": 1649,
"preview": "# extract-stealth-evasions\n\nThis script offers a quick way to extract the latest stealth evasions from [puppeteer-extra-"
},
{
"path": "packages/playwright-extra/.prettierrc.js",
"chars": 44,
"preview": "module.exports = 'prettier-config-standard'\n"
},
{
"path": "packages/playwright-extra/package.json",
"chars": 2156,
"preview": "{\n \"name\": \"playwright-extra\",\n \"version\": \"4.3.6\",\n \"description\": \"Teach playwright new tricks through plugins.\",\n "
},
{
"path": "packages/playwright-extra/readme.md",
"chars": 9463,
"preview": "# playwright-extra [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/berstend/puppeteer-e"
},
{
"path": "packages/playwright-extra/rollup.config.ts",
"chars": 1642,
"preview": "import commonjs from 'rollup-plugin-commonjs'\nimport resolve from 'rollup-plugin-node-resolve'\nimport sourceMaps from 'r"
},
{
"path": "packages/playwright-extra/src/extra.ts",
"chars": 9717,
"preview": "import Debug from 'debug'\nconst debug = Debug('playwright-extra')\n\nimport type * as pw from 'playwright-core'\nimport typ"
},
{
"path": "packages/playwright-extra/src/helper/loader.ts",
"chars": 2724,
"preview": "import type * as pw from 'playwright-core'\n\n/** Node.js module loader helper */\nexport class Loader<TargetModule> {\n co"
},
{
"path": "packages/playwright-extra/src/index.ts",
"chars": 4160,
"preview": "import type * as pw from 'playwright-core'\n\nimport { PlaywrightExtra, PlaywrightExtraClass } from './extra'\nimport { Plu"
},
{
"path": "packages/playwright-extra/src/plugins.ts",
"chars": 13385,
"preview": "import Debug from 'debug'\nconst debug = Debug('playwright-extra:plugins')\n\nimport {\n Plugin,\n PluginMethodName,\n Plug"
},
{
"path": "packages/playwright-extra/src/puppeteer-compatiblity-shim/index.ts",
"chars": 7123,
"preview": "import Debug from 'debug'\nconst debug = Debug('playwright-extra:puppeteer-compat')\n\nimport type * as pw from 'playwright"
},
{
"path": "packages/playwright-extra/src/puppeteer-compatiblity-shim/playwright-shim.d.ts",
"chars": 353,
"preview": "// Playwright objects extended with puppeteer compatiblity shims\n\nimport type {} from 'playwright-core'\n\nimport type { P"
},
{
"path": "packages/playwright-extra/src/types/index.ts",
"chars": 3852,
"preview": "import type * as pw from 'playwright-core'\n\ntype PropType<TObj, TProp extends keyof TObj> = TObj[TProp]\ntype PluginEnv ="
},
{
"path": "packages/playwright-extra/test/exports.spec.ts",
"chars": 2464,
"preview": "import { test, expect } from './fixtures/extra'\n\ntest('should export the basic functionality', async ({ playwrightExtra "
},
{
"path": "packages/playwright-extra/test/fixtures/dummyplugin.ts",
"chars": 1618,
"preview": "import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin'\n\nexport class DummyPlugin extends PuppeteerExtraPlugin {\n "
},
{
"path": "packages/playwright-extra/test/fixtures/extra.ts",
"chars": 2476,
"preview": "// Playwrights test runner is great, originally based on folio (which unfortunately isn't maintained anymore): https://g"
},
{
"path": "packages/playwright-extra/test/playwright.config.ts",
"chars": 733,
"preview": "import { type PlaywrightTestConfig } from '@playwright/test'\n\nconst config: PlaywrightTestConfig = {\n retries: 3,\n wor"
},
{
"path": "packages/playwright-extra/test/plugin-events.spec.ts",
"chars": 3190,
"preview": "import { test, expect } from './fixtures/extra'\n\nimport { DummyPlugin } from './fixtures/dummyplugin'\n\ntest.use({ plugin"
},
{
"path": "packages/playwright-extra/test/puppeteer-plugins/anonymize-ua.spec.ts",
"chars": 1855,
"preview": "import { test, expect } from '../fixtures/extra'\n\nimport AnonymizeUAPlugin from 'puppeteer-extra-plugin-anonymize-ua'\n\nt"
},
{
"path": "packages/playwright-extra/test/puppeteer-plugins/recaptcha.spec.ts",
"chars": 2594,
"preview": "import { test, expect } from '../fixtures/extra'\n\nimport RecaptchaPlugin from 'puppeteer-extra-plugin-recaptcha'\n\n// Sup"
},
{
"path": "packages/playwright-extra/test/puppeteer-plugins/stealth.spec.ts",
"chars": 1535,
"preview": "import { test, expect } from '../fixtures/extra'\n\nimport StealthPlugin from 'puppeteer-extra-plugin-stealth'\n\ntest('pupp"
},
{
"path": "packages/playwright-extra/tsconfig.json",
"chars": 709,
"preview": "{\n \"compilerOptions\": {\n \"outDir\": \"./dist\",\n \"target\": \"es2017\",\n \"module\": \"es2015\",\n \"moduleResolution\":"
},
{
"path": "packages/plugin-proxy-router/package.json",
"chars": 2166,
"preview": "{\n \"name\": \"@extra/proxy-router\",\n \"version\": \"3.1.6\",\n \"description\": \"A plugin for playwright & puppeteer to route "
},
{
"path": "packages/plugin-proxy-router/readme.md",
"chars": 13831,
"preview": "# @extra/proxy-router [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/berstend/puppetee"
},
{
"path": "packages/plugin-proxy-router/rollup.config.ts",
"chars": 1599,
"preview": "import resolve from 'rollup-plugin-node-resolve'\nimport sourceMaps from 'rollup-plugin-sourcemaps'\nimport typescript fro"
},
{
"path": "packages/plugin-proxy-router/src/index.ts",
"chars": 365,
"preview": "import { ExtraPluginProxyRouter, ExtraPluginProxyRouterOptions } from './plugin'\n\nexport * from './plugin'\nexport * from"
},
{
"path": "packages/plugin-proxy-router/src/plugin.ts",
"chars": 3728,
"preview": "import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin'\nimport { ProxyRouter, ProxyRouterOpts } from './router'\n\ne"
},
{
"path": "packages/plugin-proxy-router/src/router.ts",
"chars": 11792,
"preview": "import { Server as ProxyServer, RequestError, redactUrl } from 'proxy-chain'\nimport type * as ProxyChain from 'proxy-cha"
},
{
"path": "packages/plugin-proxy-router/src/stats.ts",
"chars": 3307,
"preview": "import type { Server as ProxyServer } from 'proxy-chain'\n\nexport interface ConnectionLogEntry {\n /** Connection Id */\n "
},
{
"path": "packages/plugin-proxy-router/src/utils/port.ts",
"chars": 1117,
"preview": "import net from 'net'\n\nexport interface Options {\n /**\n * A preferred port or an array of preferred ports to use.\n "
},
{
"path": "packages/plugin-proxy-router/tsconfig.json",
"chars": 926,
"preview": "{\n \"compilerOptions\": {\n \"outDir\": \"./dist\",\n \"target\": \"es2017\",\n \"module\": \"es2015\",\n \"moduleResolution\":"
},
{
"path": "packages/plugin-proxy-router/tslint.json",
"chars": 116,
"preview": "{\n \"extends\": [\"tslint-config-standard\", \"tslint-config-prettier\"],\n \"rules\": {\n \"ordered-imports\": true\n }\n}\n"
},
{
"path": "packages/puppeteer-extra/ava.config-ts.js",
"chars": 212,
"preview": "export default {\n compileEnhancements: false,\n environmentVariables: {\n TS_NODE_COMPILER_OPTIONS: '{\"module\":\"commo"
},
{
"path": "packages/puppeteer-extra/ava.config.js",
"chars": 42,
"preview": "export default {\n files: ['test/*.js']\n}\n"
},
{
"path": "packages/puppeteer-extra/package.json",
"chars": 2569,
"preview": "{\n \"name\": \"puppeteer-extra\",\n \"version\": \"3.3.6\",\n \"description\": \"Teach puppeteer new tricks through plugins.\",\n \""
},
{
"path": "packages/puppeteer-extra/readme.md",
"chars": 22462,
"preview": "# puppeteer-extra [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/berstend/puppeteer-ex"
},
{
"path": "packages/puppeteer-extra/rollup.config.ts",
"chars": 1642,
"preview": "import commonjs from 'rollup-plugin-commonjs'\nimport resolve from 'rollup-plugin-node-resolve'\nimport sourceMaps from 'r"
},
{
"path": "packages/puppeteer-extra/src/ambient.d.ts",
"chars": 153,
"preview": "export {}\n\n// https://github.com/sindresorhus/type-fest/issues/19\ndeclare global {\n interface SymbolConstructor {\n r"
},
{
"path": "packages/puppeteer-extra/src/index.ts",
"chars": 17471,
"preview": "/// <reference path=\"./puppeteer-legacy.d.ts\" />\nimport { PuppeteerNode, Browser, Page } from 'puppeteer'\n\nimport Debug "
},
{
"path": "packages/puppeteer-extra/src/puppeteer-legacy.d.ts",
"chars": 1869,
"preview": "// @ts-nocheck\n// NOTE: The above comment is crucial for all this to work\n// The puppeteer project caused a type breakin"
},
{
"path": "packages/puppeteer-extra/test/addExtra.ts",
"chars": 615,
"preview": "import test from 'ava'\n\nimport { addExtra } from '../src/index'\n\ntest('is a function', async t => {\n t.is(typeof addExt"
},
{
"path": "packages/puppeteer-extra/test/basic.ts",
"chars": 1169,
"preview": "import test from 'ava'\n\nimport puppeteer from '../src/index'\n\ntest('is an object', async t => {\n t.is(typeof puppeteer,"
},
{
"path": "packages/puppeteer-extra/test/connect.js",
"chars": 1530,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\n\ntest.bef"
},
{
"path": "packages/puppeteer-extra/test/events.js",
"chars": 5040,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\n\nconst pu"
},
{
"path": "packages/puppeteer-extra/test/options.js",
"chars": 2597,
"preview": "'use strict'\n\nimport test, { beforeEach } from 'ava'\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'"
},
{
"path": "packages/puppeteer-extra/test/plugin-support.js",
"chars": 1936,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\nconst PAG"
},
{
"path": "packages/puppeteer-extra/tsconfig.json",
"chars": 709,
"preview": "{\n \"compilerOptions\": {\n \"outDir\": \"./dist\",\n \"target\": \"es2017\",\n \"module\": \"es2015\",\n \"moduleResolution\":"
},
{
"path": "packages/puppeteer-extra/tslint.json",
"chars": 116,
"preview": "{\n \"extends\": [\"tslint-config-standard\", \"tslint-config-prettier\"],\n \"rules\": {\n \"ordered-imports\": true\n }\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin/ava.config-ts.js",
"chars": 219,
"preview": "export default {\n compileEnhancements: false,\n environmentVariables: {\n TS_NODE_COMPILER_OPTIONS: '{\"module\":\"commo"
},
{
"path": "packages/puppeteer-extra-plugin/ava.config.js",
"chars": 47,
"preview": "export default {\n files: ['dist/*.test.js']\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin/package.json",
"chars": 2237,
"preview": "{\n \"name\": \"puppeteer-extra-plugin\",\n \"version\": \"3.2.3\",\n \"description\": \"Base class for puppeteer-extra plugins.\",\n"
},
{
"path": "packages/puppeteer-extra-plugin/readme.md",
"chars": 16710,
"preview": "# puppeteer-extra-plugin [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/berstend/puppe"
},
{
"path": "packages/puppeteer-extra-plugin/rollup.config.ts",
"chars": 1403,
"preview": "import commonjs from 'rollup-plugin-commonjs'\nimport resolve from 'rollup-plugin-node-resolve'\nimport sourceMaps from 'r"
},
{
"path": "packages/puppeteer-extra-plugin/src/ambient.d.ts",
"chars": 153,
"preview": "export {}\n\n// https://github.com/sindresorhus/type-fest/issues/19\ndeclare global {\n interface SymbolConstructor {\n r"
},
{
"path": "packages/puppeteer-extra-plugin/src/index.test.ts",
"chars": 3634,
"preview": "import test from 'ava'\n\nimport { PuppeteerExtraPlugin } from '.'\n\ntest('is a function', async t => {\n t.is(typeof Puppe"
},
{
"path": "packages/puppeteer-extra-plugin/src/index.ts",
"chars": 16977,
"preview": "import debug, { Debugger } from 'debug'\nimport * as Puppeteer from './puppeteer'\n\n/** @private */\nconst merge = require("
},
{
"path": "packages/puppeteer-extra-plugin/src/puppeteer.ts",
"chars": 434,
"preview": "// A wildcard import would result in a `require(\"puppeteer\")` statement\n// at the top of the transpiled js file, not wha"
},
{
"path": "packages/puppeteer-extra-plugin/tsconfig.json",
"chars": 769,
"preview": "{\n \"compilerOptions\": {\n \"outDir\": \"./dist\",\n \"target\": \"es2017\",\n \"module\": \"es2015\",\n \"moduleResolution\":"
},
{
"path": "packages/puppeteer-extra-plugin/tslint.json",
"chars": 85,
"preview": "{\n \"extends\": [\"tslint-config-standard\", \"tslint-config-prettier\"],\n \"rules\": {}\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/ava.config-ts.js",
"chars": 217,
"preview": "export default {\n compileEnhancements: false,\n environmentVariables: {\n TS_NODE_COMPILER_OPTIONS: '{\"module\":\"commo"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/ava.config.js",
"chars": 47,
"preview": "export default {\n files: ['dist/*.test.js']\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/build_version_check.js",
"chars": 449,
"preview": "const pkg = require('./package.json')\n\nconst isIncompatiblePuppeteerVersion = () => {\n const version = pkg.devDependenc"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/package.json",
"chars": 2303,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-adblocker\",\n \"version\": \"2.13.6\",\n \"description\": \"A puppeteer-extra plugin to blo"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/readme.md",
"chars": 3980,
"preview": "# puppeteer-extra-plugin-adblocker [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/bers"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/rollup.config.ts",
"chars": 1665,
"preview": "import resolve from 'rollup-plugin-node-resolve'\nimport sourceMaps from 'rollup-plugin-sourcemaps'\nimport typescript fro"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/src/ambient.d.ts",
"chars": 153,
"preview": "export {}\n\n// https://github.com/sindresorhus/type-fest/issues/19\ndeclare global {\n interface SymbolConstructor {\n r"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/src/index.test.ts",
"chars": 887,
"preview": "import test from 'ava'\n\nimport AdblockerPlugin from './index'\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/src/index.ts",
"chars": 4993,
"preview": "import { promises as fs } from 'fs'\nimport os from 'os'\nimport path from 'path'\n\nimport { PuppeteerBlocker } from '@cliq"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/tsconfig.json",
"chars": 769,
"preview": "{\n \"compilerOptions\": {\n \"outDir\": \"./dist\",\n \"target\": \"es2017\",\n \"module\": \"es2015\",\n \"moduleResolution\":"
},
{
"path": "packages/puppeteer-extra-plugin-adblocker/tslint.json",
"chars": 116,
"preview": "{\n \"extends\": [\"tslint-config-standard\", \"tslint-config-prettier\"],\n \"rules\": {\n \"ordered-imports\": true\n }\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/index.d.ts",
"chars": 537,
"preview": "declare const PuppeteerExtraPlugin: typeof import(\"puppeteer-extra-plugin\").PuppeteerExtraPlugin;\ndeclare const Page: ty"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/index.js",
"chars": 1586,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\n/**\n * Anonymize the User-Agent on all"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/index.test.js",
"chars": 1028,
"preview": "'use strict'\n\nconst PLUGIN_NAME = 'anonymize-ua'\n\nconst test = require('ava')\n\nconst Plugin = require('.')\n\ntest('is a f"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/package.json",
"chars": 1226,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-anonymize-ua\",\n \"version\": \"2.4.6\",\n \"description\": \"Anonymize User-Agent in puppe"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/readme.md",
"chars": 1727,
"preview": "# puppeteer-extra-plugin-anonymize-ua\n\n> A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra).\n\n#"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/test/headless.js",
"chars": 2363,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\n\ntest.bef"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/test/headless_off.js",
"chars": 1022,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\n\ntest.bef"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/test/popup.js",
"chars": 1880,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\n\nconst wa"
},
{
"path": "packages/puppeteer-extra-plugin-anonymize-ua/test/stresstest.js",
"chars": 2928,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\n\ntest.bef"
},
{
"path": "packages/puppeteer-extra-plugin-block-resources/example.js",
"chars": 1699,
"preview": "'use strict'\n\n//\n// With debug logs:\n// DEBUG=puppeteer-extra,puppeteer-extra-plugin,puppeteer-extra-plugin:* node examp"
},
{
"path": "packages/puppeteer-extra-plugin-block-resources/index.d.ts",
"chars": 656,
"preview": "import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin'\nimport { ResourceType } from 'puppeteer'\n\ndeclare interfac"
},
{
"path": "packages/puppeteer-extra-plugin-block-resources/index.js",
"chars": 4414,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\n/**\n * Block resources (images, media,"
},
{
"path": "packages/puppeteer-extra-plugin-block-resources/index.test.js",
"chars": 1296,
"preview": "'use strict'\n\nconst PLUGIN_NAME = 'block-resources'\n\nconst test = require('ava')\n\nconst Plugin = require('.')\n\ntest('is "
},
{
"path": "packages/puppeteer-extra-plugin-block-resources/package.json",
"chars": 1054,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-block-resources\",\n \"version\": \"2.4.3\",\n \"description\": \"Block resources (images, m"
},
{
"path": "packages/puppeteer-extra-plugin-block-resources/readme.md",
"chars": 3496,
"preview": "# puppeteer-extra-plugin-block-resources\n\n> A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra)."
},
{
"path": "packages/puppeteer-extra-plugin-click-and-wait/example.js",
"chars": 434,
"preview": "'use strict'\n\nconst puppeteer = require('puppeteer-extra')\npuppeteer.use(require('puppeteer-extra-plugin-click-and-wait'"
},
{
"path": "packages/puppeteer-extra-plugin-click-and-wait/index.js",
"chars": 1255,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\n/**\n * Convenience function to wait fo"
},
{
"path": "packages/puppeteer-extra-plugin-click-and-wait/package.json",
"chars": 1107,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-click-and-wait\",\n \"version\": \"2.3.3\",\n \"description\": \"Convenience function to wai"
},
{
"path": "packages/puppeteer-extra-plugin-click-and-wait/readme.md",
"chars": 1186,
"preview": "# puppeteer-extra-plugin-click-and-wait\n\n> A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra).\n"
},
{
"path": "packages/puppeteer-extra-plugin-devtools/index.js",
"chars": 7011,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\nconst RemoteDevTools = require('./lib/R"
},
{
"path": "packages/puppeteer-extra-plugin-devtools/index.test.js",
"chars": 1097,
"preview": "'use strict'\n\nconst PLUGIN_NAME = 'devtools'\n\nconst test = require('ava')\n\nconst Plugin = require('.')\n\ntest('is a funct"
},
{
"path": "packages/puppeteer-extra-plugin-devtools/lib/RemoteDevTools.js",
"chars": 7535,
"preview": "'use strict'\n\nconst debug = require('debug')('remote-devtools')\nconst ow = require('ow')\n\nconst got = require('got')\ncon"
},
{
"path": "packages/puppeteer-extra-plugin-devtools/lib/RemoteDevTools.test.js",
"chars": 1861,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst {\n DevToolsCommon,\n DevToolsLocal,\n DevToolsTunnel\n} = require('./Re"
},
{
"path": "packages/puppeteer-extra-plugin-devtools/package.json",
"chars": 1501,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-devtools\",\n \"version\": \"2.4.6\",\n \"description\": \"Make puppeteer browser debugging "
},
{
"path": "packages/puppeteer-extra-plugin-devtools/readme.md",
"chars": 7832,
"preview": "# puppeteer-extra-plugin-devtools\n\n> A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra).\n\n## In"
},
{
"path": "packages/puppeteer-extra-plugin-devtools/test/headless.js",
"chars": 1527,
"preview": "'use strict'\n\nconst test = require('ava')\n\n// const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\n\n// te"
},
{
"path": "packages/puppeteer-extra-plugin-flash/example.js",
"chars": 811,
"preview": "'use strict'\n\nconst puppeteer = require('puppeteer-extra')\n\n// This might not be the flashPath you're looking for. ;-)\nc"
},
{
"path": "packages/puppeteer-extra-plugin-flash/index.js",
"chars": 2362,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\n/**\n * Allow flash on all sites withou"
},
{
"path": "packages/puppeteer-extra-plugin-flash/package.json",
"chars": 1062,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-flash\",\n \"version\": \"2.3.3\",\n \"description\": \"Allow flash on all sites without use"
},
{
"path": "packages/puppeteer-extra-plugin-flash/readme.md",
"chars": 2242,
"preview": "# puppeteer-extra-plugin-flash\n\n> A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra).\n\n## Insta"
},
{
"path": "packages/puppeteer-extra-plugin-font-size/index.js",
"chars": 1197,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\n/**\n * Modify/increase the default fon"
},
{
"path": "packages/puppeteer-extra-plugin-font-size/package.json",
"chars": 1015,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-font-size\",\n \"version\": \"2.3.3\",\n \"description\": \"Adjust font sizes in puppeteer.\""
},
{
"path": "packages/puppeteer-extra-plugin-font-size/readme.md",
"chars": 1201,
"preview": "# puppeteer-extra-plugin-font-size\n\n> A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra).\n\n### "
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/ava.config-ts.js",
"chars": 217,
"preview": "export default {\n compileEnhancements: false,\n environmentVariables: {\n TS_NODE_COMPILER_OPTIONS: '{\"module\":\"commo"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/ava.config.js",
"chars": 47,
"preview": "export default {\n files: ['dist/*.test.js']\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/package.json",
"chars": 2762,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-recaptcha\",\n \"version\": \"3.6.8\",\n \"description\": \"A puppeteer-extra plugin to solv"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/readme.md",
"chars": 15021,
"preview": "# puppeteer-extra-plugin-recaptcha [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/bers"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/rollup.config.ts",
"chars": 1599,
"preview": "import resolve from 'rollup-plugin-node-resolve'\nimport sourceMaps from 'rollup-plugin-sourcemaps'\nimport typescript fro"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/ambient.d.ts",
"chars": 153,
"preview": "export {}\n\n// https://github.com/sindresorhus/type-fest/issues/19\ndeclare global {\n interface SymbolConstructor {\n r"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/content-hcaptcha.ts",
"chars": 4758,
"preview": "import * as types from './types'\n\nexport const ContentScriptDefaultOpts: types.ContentScriptOpts = {\n visualFeedback: t"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/content.ts",
"chars": 17788,
"preview": "import * as types from './types'\n\nexport const ContentScriptDefaultOpts: types.ContentScriptOpts = {\n visualFeedback: t"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/detection.test.ts",
"chars": 8737,
"preview": "import test from 'ava'\n\nimport RecaptchaPlugin from './index'\n\nimport { addExtra } from 'puppeteer-extra'\n\nconst PUPPETE"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/index.test.ts",
"chars": 3545,
"preview": "import test from 'ava'\n\nimport RecaptchaPlugin from './index'\n// import * as types from './types'\n\n// import { Puppeteer"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/index.ts",
"chars": 11847,
"preview": "import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin'\n\nimport { Browser, Frame, Page } from 'puppeteer'\n\nimport "
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/playwright-mods.d.ts",
"chars": 305,
"preview": "// Extend Playwright interfaces transparently to the end user.\nimport {} from 'playwright-core'\n\nimport { RecaptchaPlugi"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/provider/2captcha-api.ts",
"chars": 7362,
"preview": "// https://github.com/bochkarev-artem/2captcha/blob/master/index.js\n// TODO: Create our own API wrapper\n\nvar https = req"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/provider/2captcha.ts",
"chars": 3593,
"preview": "export const PROVIDER_ID = '2captcha'\nimport * as types from '../types'\n\nimport Debug from 'debug'\nconst debug = Debug(`"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/puppeteer-mods.d.ts",
"chars": 1008,
"preview": "// Extend Puppeteer interfaces transparently to the end user.\n\n// Note, we need to manually copy this file into the buil"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/solve.test.ts",
"chars": 6414,
"preview": "import test from 'ava'\n\nimport RecaptchaPlugin from './index'\n\nimport { addExtra } from 'puppeteer-extra'\n\nconst PUPPETE"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/src/types.ts",
"chars": 3904,
"preview": "/// <reference path=\"./puppeteer-mods.d.ts\" />\n/// <reference path=\"./playwright-mods.d.ts\" />\n// Warn: The above is EXT"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/tsconfig.json",
"chars": 962,
"preview": "{\n \"compilerOptions\": {\n \"outDir\": \"./dist\",\n \"target\": \"es2017\",\n \"module\": \"es2015\",\n \"moduleResolution\":"
},
{
"path": "packages/puppeteer-extra-plugin-recaptcha/tslint.json",
"chars": 116,
"preview": "{\n \"extends\": [\"tslint-config-standard\", \"tslint-config-prettier\"],\n \"rules\": {\n \"ordered-imports\": true\n }\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-repl/index.d.ts",
"chars": 1332,
"preview": "/// <reference types=\"node\" />\n\nimport { EventEmitter } from 'events';\nimport 'puppeteer';\nimport { PuppeteerExtraPlugin"
},
{
"path": "packages/puppeteer-extra-plugin-repl/index.js",
"chars": 2842,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\nconst REPLSession = require('./lib/REPL"
},
{
"path": "packages/puppeteer-extra-plugin-repl/index.test.js",
"chars": 1064,
"preview": "'use strict'\n\nconst PLUGIN_NAME = 'repl'\n\nconst test = require('ava')\n\nconst Plugin = require('.')\n\ntest('is a function'"
},
{
"path": "packages/puppeteer-extra-plugin-repl/lib/REPLSession.js",
"chars": 2056,
"preview": "const ow = require('ow')\nconst readline = require('./super-readline')\n\nclass REPLSession {\n constructor(opts) {\n ow("
},
{
"path": "packages/puppeteer-extra-plugin-repl/lib/REPLSession.test.js",
"chars": 1419,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst REPLSession = require('./REPLSession')\n\ntest('is a function', async t ="
},
{
"path": "packages/puppeteer-extra-plugin-repl/lib/super-readline.js",
"chars": 2531,
"preview": "const chalk = require('chalk')\n\nconst {\n Interface,\n clearLine,\n clearScreenDown,\n cursorTo,\n emitKeypressEvents,\n "
},
{
"path": "packages/puppeteer-extra-plugin-repl/lib/super-readline.test.js",
"chars": 1823,
"preview": "'use strict'\n\nconst test = require('ava')\n\nconst readline = require('./super-readline')\n\ntest('is an object', async t =>"
},
{
"path": "packages/puppeteer-extra-plugin-repl/package.json",
"chars": 1137,
"preview": "{\n \"name\": \"puppeteer-extra-plugin-repl\",\n \"version\": \"2.3.3\",\n \"description\": \"Start an interactive REPL in your pup"
},
{
"path": "packages/puppeteer-extra-plugin-repl/readme.md",
"chars": 4564,
"preview": "# puppeteer-extra-plugin-repl\n\n> A plugin for [puppeteer-extra](https://github.com/berstend/puppeteer-extra).\n\n## Instal"
},
{
"path": "packages/puppeteer-extra-plugin-repl/test/headless.js",
"chars": 1420,
"preview": "'use strict'\n\nconst test = require('ava')\n\n// const PUPPETEER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox']\n\ntest."
},
{
"path": "packages/puppeteer-extra-plugin-stealth/.npmignore",
"chars": 36,
"preview": "stealthtests/\nrunall_stealthtests.sh"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/_template/index.js",
"chars": 591,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\n/**\n * Minimal stealth plugin template"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/_template/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/_template/readme.md",
"chars": 564,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.js",
"chars": 21689,
"preview": "/**\n * A set of shared utility functions specifically for the purpose of modifying native browser APIs without leaving t"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/_utils/index.test.js",
"chars": 23712,
"preview": "const test = require('ava')\n\nconst { vanillaPuppeteer } = require('../../test/util')\n\nconst utils = require('.')\nconst w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/_utils/readme.md",
"chars": 14880,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/_utils/withUtils.js",
"chars": 1657,
"preview": "const utils = require('./index')\n\n/**\n * Wrap a page with utilities.\n *\n * @param {Puppeteer.Page} page\n */\nmodule.expor"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/index.js",
"chars": 2620,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst withUtils = require('../_utils/w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/index.test.js",
"chars": 1920,
"preview": "const test = require('ava')\n\nconst { vanillaPuppeteer, addExtra } = require('../../test/util')\n\nconst Plugin = require('"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.app/readme.md",
"chars": 502,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/index.js",
"chars": 2463,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst withUtils = require('../_utils/w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/index.test.js",
"chars": 1161,
"preview": "const test = require('ava')\n\nconst { vanillaPuppeteer, addExtra } = require('../../test/util')\n\nconst Plugin = require('"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.csi/readme.md",
"chars": 1353,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/index.js",
"chars": 5997,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst withUtils = require('../_utils/w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/index.test.js",
"chars": 2025,
"preview": "const test = require('ava')\n\nconst { vanillaPuppeteer, addExtra } = require('../../test/util')\n\nconst Plugin = require('"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes/readme.md",
"chars": 1249,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.js",
"chars": 9415,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst withUtils = require('../_utils/w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/index.test.js",
"chars": 9864,
"preview": "const test = require('ava')\n\nconst {\n getVanillaFingerPrint,\n getStealthFingerPrint\n} = require('../../test/util')\n\nco"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/readme.md",
"chars": 1162,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/chrome.runtime/staticData.json",
"chars": 883,
"preview": "{\n \"OnInstalledReason\": {\n \"CHROME_UPDATE\": \"chrome_update\",\n \"INSTALL\": \"install\",\n \"SHARED_MODULE_UPDATE\": \""
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/index.js",
"chars": 1215,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst argsToIgnore = [\n '--disable-ex"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/index.test.js",
"chars": 1202,
"preview": "const test = require('ava')\n\nconst { vanillaPuppeteer, addExtra } = require('../../test/util')\nconst Plugin = require('."
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/defaultArgs/readme.md",
"chars": 608,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/index.js",
"chars": 4543,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst withUtils = require('../_utils/w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/index.test.js",
"chars": 14026,
"preview": "const test = require('ava')\n\nconst {\n getVanillaFingerPrint,\n getStealthFingerPrint,\n dummyHTMLPath,\n vanillaPuppete"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow/readme.md",
"chars": 677,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/index.js",
"chars": 2530,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst withUtils = require('../_utils/w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/index.test.js",
"chars": 3397,
"preview": "const test = require('ava')\n\nconst {\n getVanillaFingerPrint,\n getStealthFingerPrint\n} = require('../../test/util')\ncon"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/media.codecs/readme.md",
"chars": 1189,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js",
"chars": 1164,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst withUtils = require('../_utils/w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.test.js",
"chars": 1693,
"preview": "const test = require('ava')\nconst os = require('os')\n\nconst { vanillaPuppeteer, addExtra } = require('../../test/util')\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/readme.md",
"chars": 928,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js",
"chars": 1181,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\nconst withUtils = require('../_utils/wi"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.test.js",
"chars": 3051,
"preview": "const test = require('ava')\n\nconst {\n getVanillaFingerPrint,\n getStealthFingerPrint\n} = require('../../test/util')\ncon"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/package.json",
"chars": 44,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}\n"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/readme.md",
"chars": 860,
"preview": "## API\n\n<!-- Generated by documentation.js. Update this documentation by updating the source code. -->\n\n#### Table of Co"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js",
"chars": 1924,
"preview": "'use strict'\n\nconst { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin')\n\nconst withUtils = require('../_utils/w"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.test.js",
"chars": 2816,
"preview": "/* global Notification */\nconst test = require('ava')\n\nconst {\n getVanillaFingerPrint,\n getStealthFingerPrint\n} = requ"
},
{
"path": "packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/package.json",
"chars": 43,
"preview": "{\n \"private\": true,\n \"main\": \"index.js\"\n}"
}
]
// ... and 67 more files (download for full content)
About this extraction
This page contains the full source code of the berstend/puppeteer-extra GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 267 files (703.5 KB), approximately 189.8k tokens, and a symbol index with 638 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.