[ { "path": ".github/workflows/.keep", "content": "" }, { "path": ".github/workflows/cd.yml", "content": "run-name: Build push and deploy (CD)\non:\n push:\n branches:\n - main\n - ci-debug\n - dev\n tags:\n - '*'\n\njobs:\n build-and-push-to-gcr:\n runs-on: ubuntu-latest\n concurrency:\n group: ${{ github.ref_type == 'branch' && github.ref }}\n cancel-in-progress: true\n permissions:\n contents: read\n steps:\n - uses: actions/checkout@v4\n with:\n lfs: true\n submodules: true\n token: ${{ secrets.THINAPPS_SHARED_READ_TOKEN }}\n - uses: 'google-github-actions/auth@v2'\n with:\n credentials_json: '${{ secrets.GCLOUD_SERVICE_ACCOUNT_SECRET_JSON }}'\n - name: 'Set up Cloud SDK'\n uses: 'google-github-actions/setup-gcloud@v2'\n with:\n install_components: beta\n - name: \"Docker auth\"\n run: |-\n gcloud auth configure-docker us-docker.pkg.dev --quiet\n - name: Set controller release version\n run: echo \"RELEASE_VERSION=${GITHUB_REF#refs/*/}\" >> $GITHUB_ENV\n - name: Set up Node.js\n uses: actions/setup-node@v4\n with:\n node-version: 22.12.0\n cache: npm\n\n - name: npm install\n run: npm ci\n - name: get maxmind mmdb\n run: mkdir -p licensed && curl -o licensed/GeoLite2-City.mmdb https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb\n - name: get source han sans font\n run: curl -o licensed/SourceHanSansSC-Regular.otf https://raw.githubusercontent.com/adobe-fonts/source-han-sans/refs/heads/release/OTF/SimplifiedChinese/SourceHanSansSC-Regular.otf\n - name: build application\n run: npm run build\n - name: Set package version\n run: npm version --no-git-tag-version ${{ env.RELEASE_VERSION }}\n if: github.ref_type == 'tag'\n - name: Docker meta\n id: meta\n uses: docker/metadata-action@v5\n with:\n images: |\n us-docker.pkg.dev/reader-6b7dc/jina-reader/reader\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v3\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n - name: Build and push\n id: container\n uses: docker/build-push-action@v6\n with:\n context: .\n push: true\n tags: ${{ steps.meta.outputs.tags }}\n labels: ${{ steps.meta.outputs.labels }}\n - name: Deploy CRAWL with Tag\n run: |\n gcloud beta run deploy crawl --image us-docker.pkg.dev/reader-6b7dc/jina-reader/reader@${{steps.container.outputs.imageid}} --tag ${{ env.RELEASE_VERSION }} --command '' --args build/stand-alone/crawl.js --region us-central1 --async --min-instances 0 --deploy-health-check --use-http2\n - name: Deploy SEARCH with Tag\n run: |\n gcloud beta run deploy search --image us-docker.pkg.dev/reader-6b7dc/jina-reader/reader@${{steps.container.outputs.imageid}} --tag ${{ env.RELEASE_VERSION }} --command '' --args build/stand-alone/search.js --region us-central1 --async --min-instances 0 --deploy-health-check --use-http2\n - name: Deploy SERP with Tag\n run: |\n gcloud beta run deploy serp --image us-docker.pkg.dev/reader-6b7dc/jina-reader/reader@${{steps.container.outputs.imageid}} --tag ${{ env.RELEASE_VERSION }} --command '' --args build/stand-alone/serp.js --region us-central1 --async --min-instances 0 --deploy-health-check --use-http2\n - name: Deploy CRAWL-EU with Tag\n run: |\n gcloud beta run deploy crawl-eu --image us-docker.pkg.dev/reader-6b7dc/jina-reader/reader@${{steps.container.outputs.imageid}} --tag ${{ env.RELEASE_VERSION }} --command '' --args build/stand-alone/crawl.js --region europe-west1 --async --min-instances 0 --deploy-health-check --use-http2\n - name: Deploy SEARCH-EU with Tag\n run: |\n gcloud beta run deploy search-eu --image us-docker.pkg.dev/reader-6b7dc/jina-reader/reader@${{steps.container.outputs.imageid}} --tag ${{ env.RELEASE_VERSION }} --command '' --args build/stand-alone/search.js --region europe-west1 --async --min-instances 0 --deploy-health-check --use-http2\n - name: Deploy SERP-HK with Tag\n run: |\n gcloud beta run deploy serp-hk --image us-docker.pkg.dev/reader-6b7dc/jina-reader/reader@${{steps.container.outputs.imageid}} --tag ${{ env.RELEASE_VERSION }} --command '' --args build/stand-alone/serp.js --region asia-east2 --async --min-instances 0 --deploy-health-check --use-http2" }, { "path": ".gitignore", "content": "# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\nfirebase-debug.log*\nfirebase-debug.*.log*\n\n# Firebase cache\n.firebase/\n\n# Firebase config\n\n# Uncomment this if you'd like others to create their own Firebase project.\n# For a team working on the same Firebase project(s), it is recommended to leave\n# it commented so all members can deploy to the same project(s) in .firebaserc.\n# .firebaserc\n\n# Runtime data\npids\n*.pid\n*.seed\n*.pid.lock\n\n# Directory for instrumented libs generated by jscoverage/JSCover\nlib-cov\n\n# Coverage directory used by tools like istanbul\ncoverage\n\n# nyc test coverage\n.nyc_output\n\n# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)\n.grunt\n\n# Bower dependency directory (https://bower.io/)\nbower_components\n\n# node-waf configuration\n.lock-wscript\n\n# Compiled binary addons (http://nodejs.org/api/addons.html)\nbuild/Release\n\n# Dependency directories\nnode_modules/\n\n# Optional npm cache directory\n.npm\n\n# Optional eslint cache\n.eslintcache\n\n# Optional REPL history\n.node_repl_history\n\n# Output of 'npm pack'\n*.tgz\n\n# Yarn Integrity file\n.yarn-integrity\n\n# dotenv environment variables file\n.env\n.secret.local\n\ntoy*.ts\n\n.DS_Store\nbuild/\n.firebase-emu/\n*.log\n.DS_Store\n\n*.local\n.secret.*\nlicensed/" }, { "path": ".gitmodules", "content": "[submodule \"thinapps-shared\"]\n\tpath = thinapps-shared\n\turl = git@github.com:jina-ai/thinapps-shared.git\n" }, { "path": ".vscode/exensions.json", "content": "{\n \"recommendations\": [\n \"editorconfig.editorconfig\",\n \"octref.vetur\",\n \"redhat.vscode-yaml\",\n \"dbaeumer.vscode-eslint\",\n \"esbenp.prettier-vscode\",\n \"streetsidesoftware.code-spell-checker\"\n ]\n}" }, { "path": ".vscode/launch.json", "content": "{\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"Attach\",\n \"port\": 9229,\n \"request\": \"attach\",\n \"skipFiles\": [\n \"/**\"\n ],\n \"type\": \"node\"\n },\n {\n \"name\": \"Attach by Process ID\",\n \"processId\": \"${command:PickProcess}\",\n \"request\": \"attach\",\n \"skipFiles\": [\n \"/**\"\n ],\n \"type\": \"node\"\n },\n {\n \"name\": \"Debug Stand Alone Crawl\",\n \"request\": \"launch\",\n \"runtimeArgs\": [\n \"--env-file=.secret.local\",\n ],\n \"env\": {\n \"GCLOUD_PROJECT\": \"reader-6b7dc\",\n \"LD_PRELOAD\": \"/usr/local/lib/libcurl-impersonate-chrome.dylib\"\n },\n \"cwd\": \"${workspaceFolder}\",\n \"program\": \"build/stand-alone/crawl.js\",\n \"skipFiles\": [\n \"/**\"\n ],\n \"type\": \"node\",\n \"outputCapture\": \"std\",\n \"preLaunchTask\": \"Backend:build:watch\",\n \"killBehavior\": \"forceful\"\n },\n {\n \"name\": \"Debug Stand Alone Crawl + Browser\",\n \"request\": \"launch\",\n \"runtimeArgs\": [\n \"--env-file=.secret.local\",\n ],\n \"env\": {\n \"GCLOUD_PROJECT\": \"reader-6b7dc\",\n \"DEBUG_BROWSER\": \"true\",\n \"LD_PRELOAD\": \"/usr/local/lib/libcurl-impersonate-chrome.dylib\"\n },\n \"cwd\": \"${workspaceFolder}\",\n \"program\": \"build/stand-alone/crawl.js\",\n \"skipFiles\": [\n \"/**\"\n ],\n \"type\": \"node\",\n \"outputCapture\": \"std\",\n \"preLaunchTask\": \"Backend:build:watch\",\n \"killBehavior\": \"forceful\"\n },\n {\n \"name\": \"Debug Stand Alone Crawl - EU\",\n \"request\": \"launch\",\n \"runtimeArgs\": [\n \"--env-file=.secret.local\",\n ],\n \"env\": {\n \"GCLOUD_PROJECT\": \"reader-6b7dc\",\n \"FIRESTORE_DATABASE\": \"reader-eu\",\n \"GCP_STORAGE_BUCKET\": \"reader-eu\",\n \"LD_PRELOAD\": \"/usr/local/lib/libcurl-impersonate-chrome.dylib\"\n },\n \"cwd\": \"${workspaceFolder}\",\n \"program\": \"build/stand-alone/crawl.js\",\n \"skipFiles\": [\n \"/**\"\n ],\n \"type\": \"node\",\n \"outputCapture\": \"std\",\n \"preLaunchTask\": \"Backend:build:watch\",\n \"killBehavior\": \"forceful\"\n },\n {\n \"name\": \"Debug Stand Alone Search\",\n \"request\": \"launch\",\n \"runtimeArgs\": [\n \"--env-file=.secret.local\",\n ],\n \"env\": {\n \"GCLOUD_PROJECT\": \"reader-6b7dc\",\n \"LD_PRELOAD\": \"/usr/local/lib/libcurl-impersonate-chrome.dylib\"\n },\n \"cwd\": \"${workspaceFolder}\",\n \"program\": \"build/stand-alone/search.js\",\n \"skipFiles\": [\n \"/**\"\n ],\n \"type\": \"node\",\n \"outputCapture\": \"std\",\n \"preLaunchTask\": \"Backend:build:watch\",\n \"killBehavior\": \"forceful\"\n },\n {\n \"name\": \"Debug Stand Alone SERP\",\n \"request\": \"launch\",\n \"runtimeArgs\": [\n \"--env-file=.secret.local\",\n ],\n \"env\": {\n \"GCLOUD_PROJECT\": \"reader-6b7dc\",\n \"PREFERRED_PROXY_COUNTRY\": \"hk\",\n \"OVERRIDE_GOOGLE_DOMAIN\": \"www.google.com.hk\",\n \"LD_PRELOAD\": \"/usr/local/lib/libcurl-impersonate-chrome.dylib\"\n },\n \"cwd\": \"${workspaceFolder}\",\n \"program\": \"build/stand-alone/serp.js\",\n \"skipFiles\": [\n \"/**\"\n ],\n \"type\": \"node\",\n \"outputCapture\": \"std\",\n \"preLaunchTask\": \"Backend:build:watch\",\n \"killBehavior\": \"forceful\"\n },\n ]\n}" }, { "path": ".vscode/settings.json", "content": "{\n \"editor.wordWrap\": \"on\",\n \"editor.wordWrapColumn\": 120,\n \"files.trimTrailingWhitespace\": true,\n \"files.trimFinalNewlines\": true,\n \"[javascript]\": {\n \"editor.defaultFormatter\": \"vscode.typescript-language-features\"\n },\n \"[jsonc]\": {\n \"editor.defaultFormatter\": \"vscode.json-language-features\"\n },\n \"[typescript]\": {\n \"editor.defaultFormatter\": \"vscode.typescript-language-features\"\n },\n \"[json]\": {\n \"editor.defaultFormatter\": \"vscode.json-language-features\"\n },\n \"[yaml]\": {\n \"editor.defaultFormatter\": \"redhat.vscode-yaml\"\n },\n \"[markdown]\": {\n \"files.trimTrailingWhitespace\": false\n },\n \"typescript.tsdk\": \"node_modules/typescript/lib\",\n \"typescript.preferences.quoteStyle\": \"single\",\n \"typescript.format.semicolons\": \"insert\",\n \"typescript.preferences.importModuleSpecifier\": \"project-relative\",\n \"typescript.locale\": \"en\",\n \"cSpell.enabled\": true,\n \"cSpell.words\": [\n ],\n}" }, { "path": ".vscode/tasks.json", "content": "{\n \"version\": \"2.0.0\",\n \"tasks\": [\n {\n \"type\": \"npm\",\n \"script\": \"build\",\n \"group\": \"build\",\n \"options\": {\n \"cwd\": \"${workspaceFolder}\"\n },\n \"problemMatcher\": [],\n \"label\": \"Backend:rebuild\",\n \"detail\": \"Backend:rebuild\"\n },\n {\n \"type\": \"typescript\",\n \"options\": {\n \"cwd\": \"${workspaceFolder}\"\n },\n \"tsconfig\": \"tsconfig.json\",\n \"option\": \"watch\",\n \"isBackground\": true,\n \"problemMatcher\": [\n \"$tsc-watch\"\n ],\n \"group\": \"build\",\n \"label\": \"Backend:build:watch\"\n }\n ]\n}" }, { "path": "Dockerfile", "content": "# syntax=docker/dockerfile:1\nFROM lwthiker/curl-impersonate:0.6-chrome-slim-bullseye\n\nFROM node:22\n\nRUN apt-get update \\\n && apt-get install -y wget gnupg \\\n && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \\\n && sh -c 'echo \"deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main\" >> /etc/apt/sources.list.d/google.list' \\\n && apt-get update \\\n && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd \\\n --no-install-recommends \\\n && rm -rf /var/lib/apt/lists/*\n\nCOPY --from=0 /usr/local/lib/libcurl-impersonate.so /usr/local/lib/libcurl-impersonate.so\n\nRUN groupadd -r jina\nRUN useradd -g jina -G audio,video -m jina\nUSER jina\n\nWORKDIR /app\n\nCOPY package.json package-lock.json ./\nRUN npm ci\n\nCOPY build ./build\nCOPY public ./public\nCOPY licensed ./licensed\n\nRUN rm -rf ~/.config/chromium && mkdir -p ~/.config/chromium\n\nRUN NODE_COMPILE_CACHE=node_modules npm run dry-run\n\nENV OVERRIDE_CHROME_EXECUTABLE_PATH=/usr/bin/google-chrome-stable\nENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate.so CURL_IMPERSONATE=chrome116 CURL_IMPERSONATE_HEADERS=no\nENV NODE_COMPILE_CACHE=node_modules\nENV PORT=8080\n\nEXPOSE 3000 3001 8080 8081\nENTRYPOINT [\"node\"]\nCMD [ \"build/stand-alone/crawl.js\" ]\n" }, { "path": "LICENSE", "content": "Copyright 2020-2024 Jina AI Limited. All rights reserved.\n\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n Copyright 2020-2021 Jina AI Limited\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "# Reader\n\nYour LLMs deserve better input.\n\nReader does two things:\n- **Read**: It converts any URL to an **LLM-friendly** input with `https://r.jina.ai/https://your.url`. Get improved output for your agent and RAG systems at no cost.\n- **Search**: It searches the web for a given query with `https://s.jina.ai/your+query`. This allows your LLMs to access the latest world knowledge from the web.\n\nCheck out [the live demo](https://jina.ai/reader#demo)\n\nOr just visit these URLs (**Read**) https://r.jina.ai/https://github.com/jina-ai/reader, (**Search**) https://s.jina.ai/Who%20will%20win%202024%20US%20presidential%20election%3F and see yourself.\n\n> Feel free to use Reader API in production. It is free, stable and scalable. We are maintaining it actively as one of the core products of Jina AI. [Check out rate limit](https://jina.ai/reader#pricing)\n\n\"image\"\n\"image\"\n\n\n## Updates\n\n- **2024-07-15**: To restrict the results of `s.jina.ai` to certain domain/website, you can set e.g. `site=jina.ai` in the query parameters, which enables in-site search. For more options, [try our updated live-demo](https://jina.ai/reader/#apiform).\n- **2024-05-30**: Reader can now read abitrary PDF from any URL! Check out [this PDF result from NASA.gov](https://r.jina.ai/https://www.nasa.gov/wp-content/uploads/2023/01/55583main_vision_space_exploration2.pdf) vs [the original](https://www.nasa.gov/wp-content/uploads/2023/01/55583main_vision_space_exploration2.pdf).\n- **2024-05-15**: We introduced a new endpoint `s.jina.ai` that searches on the web and return top-5 results, each in a LLM-friendly format. [Read more about this new feature here](https://jina.ai/news/jina-reader-for-search-grounding-to-improve-factuality-of-llms).\n- **2024-05-08**: Image caption is off by default for better latency. To turn it on, set `x-with-generated-alt: true` in the request header.\n- **2024-04-24**: You now have more fine-grained control over Reader API [using headers](#using-request-headers), e.g. forwarding cookies, using HTTP proxy.\n- **2024-04-15**: Reader now supports image reading! It captions all images at the specified URL and adds `Image [idx]: [caption]` as an alt tag (if they initially lack one). This enables downstream LLMs to interact with the images in reasoning, summarizing etc. [See example here](https://x.com/JinaAI_/status/1780094402071023926).\n\n## Usage\n\n### Using `r.jina.ai` for single URL fetching\nSimply prepend `https://r.jina.ai/` to any URL. For example, to convert the URL `https://en.wikipedia.org/wiki/Artificial_intelligence` to an LLM-friendly input, use the following URL:\n\n[https://r.jina.ai/https://en.wikipedia.org/wiki/Artificial_intelligence](https://r.jina.ai/https://en.wikipedia.org/wiki/Artificial_intelligence)\n\n### [Using `r.jina.ai` for a full website fetching (Google Colab)](https://colab.research.google.com/drive/1uoBy6_7BhxqpFQ45vuhgDDDGwstaCt4P#scrollTo=5LQjzJiT9ewT)\n\n### Using `s.jina.ai` for web search\nSimply prepend `https://s.jina.ai/` to your search query. Note that if you are using this in the code, make sure to encode your search query first, e.g. if your query is `Who will win 2024 US presidential election?` then your url should look like:\n\n[https://s.jina.ai/Who%20will%20win%202024%20US%20presidential%20election%3F](https://s.jina.ai/Who%20will%20win%202024%20US%20presidential%20election%3F)\n\nBehind the scenes, Reader searches the web, fetches the top 5 results, visits each URL, and applies `r.jina.ai` to it. This is different from many `web search function-calling` in agent/RAG frameworks, which often return only the title, URL, and description provided by the search engine API. If you want to read one result more deeply, you have to fetch the content yourself from that URL. With Reader, `http://s.jina.ai` automatically fetches the content from the top 5 search result URLs for you (reusing the tech stack behind `http://r.jina.ai`). This means you don't have to handle browser rendering, blocking, or any issues related to JavaScript and CSS yourself.\n\n### Using `s.jina.ai` for in-site search\nSimply specify `site` in the query parameters such as:\n\n```bash\ncurl 'https://s.jina.ai/When%20was%20Jina%20AI%20founded%3F?site=jina.ai&site=github.com'\n```\n\n### [Interactive Code Snippet Builder](https://jina.ai/reader#apiform)\n\nWe highly recommend using the code builder to explore different parameter combinations of the Reader API.\n\n\"image\"\n\n\n### Using request headers\n\nAs you have already seen above, one can control the behavior of the Reader API using request headers. Here is a complete list of supported headers.\n\n- You can enable the image caption feature via the `x-with-generated-alt: true` header.\n- You can ask the Reader API to forward cookies settings via the `x-set-cookie` header.\n - Note that requests with cookies will not be cached.\n- You can bypass `readability` filtering via the `x-respond-with` header, specifically:\n - `x-respond-with: markdown` returns markdown *without* going through `reability`\n - `x-respond-with: html` returns `documentElement.outerHTML`\n - `x-respond-with: text` returns `document.body.innerText`\n - `x-respond-with: screenshot` returns the URL of the webpage's screenshot\n- You can specify a proxy server via the `x-proxy-url` header.\n- You can customize cache tolerance via the `x-cache-tolerance` header (integer in seconds).\n- You can bypass the cached page (lifetime 3600s) via the `x-no-cache: true` header (equivalent of `x-cache-tolerance: 0`).\n- If you already know the HTML structure of your target page, you may specify `x-target-selector` or `x-wait-for-selector` to direct the Reader API to focus on a specific part of the page.\n - By setting `x-target-selector` header to a CSS selector, the Reader API return the content within the matched element, instead of the full HTML. Setting this header is useful when the automatic content extraction fails to capture the desired content and you can manually select the correct target.\n - By setting `x-wait-for-selector` header to a CSS selector, the Reader API will wait until the matched element is rendered before returning the content. If you already specified `x-wait-for-selector`, this header can be omitted if you plan to wait for the same element.\n\n### Using `r.jina.ai` for single page application (SPA) fetching\nMany websites nowadays rely on JavaScript frameworks and client-side rendering. Usually known as Single Page Application (SPA). Thanks to [Puppeteer](https://github.com/puppeteer/puppeteer) and headless Chrome browser, Reader natively supports fetching these websites. However, due to specific approach some SPA are developed, there may be some extra precautions to take. \n\n#### SPAs with hash-based routing\nBy definition of the web standards, content come after `#` in a URL is not sent to the server. To mitigate this issue, use `POST` method with `url` parameter in body.\n\n```bash\ncurl -X POST 'https://r.jina.ai/' -d 'url=https://example.com/#/route' \n```\n\n#### SPAs with preloading contents\nSome SPAs, or even some websites that are not strictly SPAs, may show preload contents before later loading the main content dynamically. In this case, Reader may be capturing the preload content instead of the main content. To mitigate this issue, here are some possible solutions:\n\n##### Specifying `x-timeout` \nWhen timeout is explicitly specified, Reader will not attempt to return early and will wait for network idle until the timeout is reached. This is useful when the target website will eventually come to a network idle. \n\n```bash\ncurl 'https://example.com/' -H 'x-timeout: 30'\n```\n\n##### Specifying `x-wait-for-selector` \nWhen wait-for-selector is explicitly specified, Reader will wait for the appearance of the specified CSS selector until timeout is reached. This is useful when you know exactly what element to wait for. \n\n```bash\ncurl 'https://example.com/' -H 'x-wait-for-selector: #content'\n```\n\n### Streaming mode\n\nStreaming mode is useful when you find that the standard mode provides an incomplete result. This is because the Reader will wait a bit longer until the page is *stablely* rendered. Use the accept-header to toggle the streaming mode:\n\n```bash\ncurl -H \"Accept: text/event-stream\" https://r.jina.ai/https://en.m.wikipedia.org/wiki/Main_Page\n```\n\nThe data comes in a stream; each subsequent chunk contains more complete information. **The last chunk should provide the most complete and final result.** If you come from LLMs, please note that it is a different behavior than the LLMs' text-generation streaming.\n\nFor example, compare these two curl commands below. You can see streaming one gives you complete information at last, whereas standard mode does not. This is because the content loading on this particular site is triggered by some js *after* the page is fully loaded, and standard mode returns the page \"too soon\".\n```bash\ncurl -H 'x-no-cache: true' https://access.redhat.com/security/cve/CVE-2023-45853\ncurl -H \"Accept: text/event-stream\" -H 'x-no-cache: true' https://r.jina.ai/https://access.redhat.com/security/cve/CVE-2023-45853\n```\n\n> Note: `-H 'x-no-cache: true'` is used only for demonstration purposes to bypass the cache.\n\nStreaming mode is also useful if your downstream LLM/agent system requires immediate content delivery or needs to process data in chunks to interleave I/O and LLM processing times. This allows for quicker access and more efficient data handling:\n\n```text\nReader API: streamContent1 ----> streamContent2 ----> streamContent3 ---> ... \n | | |\n v | |\nYour LLM: LLM(streamContent1) | |\n v |\n LLM(streamContent2) |\n v\n LLM(streamContent3)\n```\n\nNote that in terms of completeness: `... > streamContent3 > streamContent2 > streamContent1`, each subsequent chunk contains more complete information.\n\n### JSON mode\n\nThis is still very early and the result is not really a \"useful\" JSON. It contains three fields `url`, `title` and `content` only. Nonetheless, you can use accept-header to control the output format:\n```bash\ncurl -H \"Accept: application/json\" https://r.jina.ai/https://en.m.wikipedia.org/wiki/Main_Page\n```\n\nJSON mode is probably more useful in `s.jina.ai` than `r.jina.ai`. For `s.jina.ai` with JSON mode, it returns 5 results in a list, each in the structure of `{'title', 'content', 'url'}`.\n\n### Generated alt\n\nAll images in that page that lack `alt` tag can be auto-captioned by a VLM (vision langauge model) and formatted as `!(Image [idx]: [VLM_caption])[img_URL]`. This should give your downstream text-only LLM *just enough* hints to include those images into reasoning, selecting, and summarization. Use the x-with-generated-alt header to toggle the streaming mode:\n\n```bash\ncurl -H \"X-With-Generated-Alt: true\" https://r.jina.ai/https://en.m.wikipedia.org/wiki/Main_Page\n```\n\n## How it works\n[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/jina-ai/reader)\n\n## What is `thinapps-shared` submodule?\n\nYou might notice a reference to `thinapps-shared` submodule, an internal package we use to share code across our products. While it’s not open-sourced and isn't integral to the Reader's functions, it mainly helps with decorators, logging, secrets management, etc. Feel free to ignore it for now.\n\nThat said, this is *the single codebase* behind `https://r.jina.ai`, so everytime we commit here, we will deploy the new version to the `https://r.jina.ai`.\n\n## Having trouble on some websites?\nPlease raise an issue with the URL you are having trouble with. We will look into it and try to fix it.\n\n## License\nReader is backed by [Jina AI](https://jina.ai) and licensed under [Apache-2.0](./LICENSE).\n" }, { "path": "integrity-check.cjs", "content": "#!/usr/bin/env node\n\nconst fs = require('fs');\nconst path = require('path');\n\nconst file = path.resolve(__dirname, 'licensed/GeoLite2-City.mmdb');\n\nif (!fs.existsSync(file)) {\n console.error(`Integrity check failed: ${file} does not exist.`);\n process.exit(1);\n}\n" }, { "path": "package.json", "content": "{\n \"name\": \"reader\",\n \"scripts\": {\n \"lint\": \"eslint --ext .js,.ts .\",\n \"build\": \"node ./integrity-check.cjs && tsc -p .\",\n \"build:watch\": \"tsc --watch\",\n \"build:clean\": \"rm -rf ./build\",\n \"serve\": \"npm run build && npm run start\",\n \"debug\": \"npm run build && npm run dev\",\n \"start\": \"node ./build/stand-alone/crawl.js\",\n \"dry-run\": \"NODE_ENV=dry-run node ./build/stand-alone/search.js\"\n },\n \"engines\": {\n \"node\": \">=18\"\n },\n \"main\": \"build/index.js\",\n \"dependencies\": {\n \"@esm2cjs/normalize-url\": \"^8.0.0\",\n \"@google-cloud/translate\": \"^8.2.0\",\n \"@koa/bodyparser\": \"^5.1.1\",\n \"@mozilla/readability\": \"^0.6.0\",\n \"@napi-rs/canvas\": \"^0.1.68\",\n \"@types/turndown\": \"^5.0.4\",\n \"@xmldom/xmldom\": \"^0.9.3\",\n \"archiver\": \"^6.0.1\",\n \"axios\": \"^1.3.3\",\n \"bcrypt\": \"^5.1.0\",\n \"busboy\": \"^1.6.0\",\n \"civkit\": \"^0.9.0-2570394\",\n \"cors\": \"^2.8.5\",\n \"dayjs\": \"^1.11.9\",\n \"express\": \"^4.19.2\",\n \"firebase-admin\": \"^12.1.0\",\n \"firebase-functions\": \"^6.1.1\",\n \"htmlparser2\": \"^9.0.0\",\n \"jose\": \"^5.1.0\",\n \"koa\": \"^2.16.0\",\n \"koa-compress\": \"^5.1.1\",\n \"langdetect\": \"^0.2.1\",\n \"linkedom\": \"^0.18.4\",\n \"lru-cache\": \"^11.0.2\",\n \"maxmind\": \"^4.3.18\",\n \"minio\": \"^7.1.3\",\n \"node-libcurl\": \"^4.1.0\",\n \"openai\": \"^4.20.0\",\n \"pdfjs-dist\": \"^4.10.38\",\n \"puppeteer\": \"^23.3.0\",\n \"puppeteer-extra\": \"^3.3.6\",\n \"puppeteer-extra-plugin-block-resources\": \"^2.4.3\",\n \"robots-parser\": \"^3.0.1\",\n \"set-cookie-parser\": \"^2.6.0\",\n \"simple-zstd\": \"^1.4.2\",\n \"stripe\": \"^11.11.0\",\n \"svg2png-wasm\": \"^1.4.1\",\n \"tiktoken\": \"^1.0.16\",\n \"tld-extract\": \"^2.1.0\",\n \"turndown\": \"^7.1.3\",\n \"turndown-plugin-gfm\": \"^1.0.2\",\n \"undici\": \"^7.8.0\"\n },\n \"devDependencies\": {\n \"@types/archiver\": \"^5.3.4\",\n \"@types/bcrypt\": \"^5.0.0\",\n \"@types/busboy\": \"^1.5.4\",\n \"@types/cors\": \"^2.8.17\",\n \"@types/koa\": \"^2.15.0\",\n \"@types/koa-compress\": \"^4.0.6\",\n \"@types/node\": \"^20.14.13\",\n \"@types/set-cookie-parser\": \"^2.4.7\",\n \"@types/xmldom\": \"^0.1.34\",\n \"@typescript-eslint/eslint-plugin\": \"^5.12.0\",\n \"@typescript-eslint/parser\": \"^5.12.0\",\n \"eslint\": \"^8.9.0\",\n \"eslint-config-google\": \"^0.14.0\",\n \"eslint-plugin-import\": \"^2.25.4\",\n \"firebase-functions-test\": \"^3.0.0\",\n \"pino-pretty\": \"^13.0.0\",\n \"replicate\": \"^0.16.1\",\n \"typescript\": \"^5.5.4\"\n },\n \"private\": true,\n \"exports\": {\n \".\": \"./build/index.js\"\n }\n}\n" }, { "path": "public/robots.txt", "content": "User-Agent: *\nDisallow: /\n" }, { "path": "src/api/crawler.ts", "content": "import { singleton } from 'tsyringe';\nimport { pathToFileURL } from 'url';\nimport { randomUUID } from 'crypto';\nimport _ from 'lodash';\n\nimport {\n assignTransferProtocolMeta, RPCHost, RPCReflection,\n AssertionFailureError, ParamValidationError,\n RawString,\n ApplicationError,\n DataStreamBrokenError,\n assignMeta,\n} from 'civkit/civ-rpc';\nimport { marshalErrorLike } from 'civkit/lang';\nimport { Defer } from 'civkit/defer';\nimport { retryWith } from 'civkit/decorators';\nimport { FancyFile } from 'civkit/fancy-file';\n\nimport { CONTENT_FORMAT, CrawlerOptions, CrawlerOptionsHeaderOnly, ENGINE_TYPE, RESPOND_TIMING } from '../dto/crawler-options';\n\nimport { Crawled } from '../db/crawled';\nimport { DomainBlockade } from '../db/domain-blockade';\nimport { OutputServerEventStream } from '../lib/transform-server-event-stream';\n\nimport { PageSnapshot, PuppeteerControl, ScrappingOptions } from '../services/puppeteer';\nimport { JSDomControl } from '../services/jsdom';\nimport { FormattedPage, md5Hasher, SnapshotFormatter } from '../services/snapshot-formatter';\nimport { CurlControl } from '../services/curl';\nimport { LmControl } from '../services/lm';\nimport { tryDecodeURIComponent } from '../utils/misc';\nimport { CFBrowserRendering } from '../services/cf-browser-rendering';\n\nimport { GlobalLogger } from '../services/logger';\nimport { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit';\nimport { AsyncLocalContext } from '../services/async-context';\nimport { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';\nimport {\n BudgetExceededError, InsufficientBalanceError,\n SecurityCompromiseError, ServiceBadApproachError, ServiceBadAttemptError,\n ServiceNodeResourceDrainError\n} from '../services/errors';\n\nimport { countGPTToken as estimateToken } from '../shared/utils/openai';\nimport { ProxyProviderService } from '../shared/services/proxy-provider';\nimport { FirebaseStorageBucketControl } from '../shared/services/firebase-storage-bucket';\nimport { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';\nimport { RobotsTxtService } from '../services/robots-text';\nimport { TempFileManager } from '../services/temp-file';\nimport { MiscService } from '../services/misc';\nimport { HTTPServiceError } from 'civkit/http';\nimport { GeoIPService } from '../services/geoip';\n\nexport interface ExtraScrappingOptions extends ScrappingOptions {\n withIframe?: boolean | 'quoted';\n withShadowDom?: boolean;\n targetSelector?: string | string[];\n removeSelector?: string | string[];\n keepImgDataUrl?: boolean;\n engine?: string;\n allocProxy?: string;\n private?: boolean;\n countryHint?: string;\n}\n\nconst indexProto = {\n toString: function (): string {\n return _(this)\n .toPairs()\n .map(([k, v]) => k ? `[${_.upperFirst(_.lowerCase(k))}] ${v}` : '')\n .value()\n .join('\\n') + '\\n';\n }\n};\n\n@singleton()\nexport class CrawlerHost extends RPCHost {\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n cacheRetentionMs = 1000 * 3600 * 24 * 7;\n cacheValidMs = 1000 * 3600;\n urlValidMs = 1000 * 3600 * 4;\n abuseBlockMs = 1000 * 3600;\n domainProfileRetentionMs = 1000 * 3600 * 24 * 30;\n\n batchedCaches: Crawled[] = [];\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected puppeteerControl: PuppeteerControl,\n protected curlControl: CurlControl,\n protected cfBrowserRendering: CFBrowserRendering,\n protected proxyProvider: ProxyProviderService,\n protected lmControl: LmControl,\n protected jsdomControl: JSDomControl,\n protected snapshotFormatter: SnapshotFormatter,\n protected firebaseObjectStorage: FirebaseStorageBucketControl,\n protected rateLimitControl: RateLimitControl,\n protected threadLocal: AsyncLocalContext,\n protected robotsTxtService: RobotsTxtService,\n protected tempFileManager: TempFileManager,\n protected geoIpService: GeoIPService,\n protected miscService: MiscService,\n ) {\n super(...arguments);\n\n puppeteerControl.on('crawled', async (snapshot: PageSnapshot, options: ExtraScrappingOptions & { url: URL; }) => {\n if (!snapshot.title?.trim() && !snapshot.pdfs?.length) {\n return;\n }\n if (options.cookies?.length || options.private) {\n // Potential privacy issue, dont cache if cookies are used\n return;\n }\n if (options.injectFrameScripts?.length || options.injectPageScripts?.length || options.viewport) {\n // Potentially mangeled content, dont cache if scripts are injected\n return;\n }\n if (snapshot.isIntermediate) {\n return;\n }\n if (!snapshot.lastMutationIdle) {\n // Never reached mutationIdle, presumably too short timeout\n return;\n }\n if (options.locale) {\n Reflect.set(snapshot, 'locale', options.locale);\n }\n\n const analyzed = await this.jsdomControl.analyzeHTMLTextLite(snapshot.html);\n if (analyzed.tokens < 200) {\n // Does not contain enough content\n if (snapshot.status !== 200) {\n return;\n }\n if (snapshot.html.includes('captcha') || snapshot.html.includes('cf-turnstile')) {\n return;\n }\n }\n\n await this.setToCache(options.url, snapshot);\n });\n\n puppeteerControl.on('abuse', async (abuseEvent: { url: URL; reason: string, sn: number; }) => {\n this.logger.warn(`Abuse detected on ${abuseEvent.url}, blocking ${abuseEvent.url.hostname}`, { reason: abuseEvent.reason, sn: abuseEvent.sn });\n\n await DomainBlockade.save(DomainBlockade.from({\n domain: abuseEvent.url.hostname.toLowerCase(),\n triggerReason: `${abuseEvent.reason}`,\n triggerUrl: abuseEvent.url.toString(),\n createdAt: new Date(),\n expireAt: new Date(Date.now() + this.abuseBlockMs),\n })).catch((err) => {\n this.logger.warn(`Failed to save domain blockade for ${abuseEvent.url.hostname}`, { err: marshalErrorLike(err) });\n });\n\n });\n\n setInterval(() => {\n const thisBatch = this.batchedCaches;\n this.batchedCaches = [];\n if (!thisBatch.length) {\n return;\n }\n const batch = Crawled.DB.batch();\n\n for (const x of thisBatch) {\n batch.set(Crawled.COLLECTION.doc(x._id), x.degradeForFireStore(), { merge: true });\n }\n\n batch.commit()\n .then(() => {\n this.logger.debug(`Saved ${thisBatch.length} caches by batch`);\n })\n .catch((err) => {\n this.logger.warn(`Failed to save cache in batch`, { err });\n });\n }, 1000 * 10 + Math.round(1000 * Math.random())).unref();\n }\n\n override async init() {\n await this.dependencyReady();\n\n if (this.puppeteerControl.effectiveUA) {\n this.curlControl.impersonateChrome(this.puppeteerControl.effectiveUA);\n }\n\n this.emit('ready');\n }\n\n async getIndex(auth?: JinaEmbeddingsAuthDTO) {\n const indexObject: Record = Object.create(indexProto);\n Object.assign(indexObject, {\n usage1: 'https://r.jina.ai/YOUR_URL',\n usage2: 'https://s.jina.ai/YOUR_SEARCH_QUERY',\n homepage: 'https://jina.ai/reader',\n });\n\n await auth?.solveUID();\n if (auth && auth.user) {\n indexObject[''] = undefined;\n indexObject.authenticatedAs = `${auth.user.user_id} (${auth.user.full_name})`;\n indexObject.balanceLeft = auth.user.wallet.total_balance;\n }\n\n return indexObject;\n }\n\n @Method({\n name: 'getIndex',\n description: 'Index of the service',\n proto: {\n http: {\n action: 'get',\n path: '/',\n }\n },\n tags: ['misc', 'crawl'],\n returnType: [String, Object],\n })\n async getIndexCtrl(@Ctx() ctx: Context, @Param({ required: false }) auth?: JinaEmbeddingsAuthDTO) {\n const indexObject = await this.getIndex(auth);\n\n if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {\n return indexObject;\n }\n\n return assignTransferProtocolMeta(`${indexObject}`,\n { contentType: 'text/plain; charset=utf-8', envelope: null }\n );\n }\n\n\n @Method({\n name: 'crawlByPostingToIndex',\n description: 'Crawl any url into markdown',\n proto: {\n http: {\n action: 'POST',\n path: '/',\n }\n },\n tags: ['crawl'],\n returnType: [String, OutputServerEventStream],\n })\n @Method({\n description: 'Crawl any url into markdown',\n proto: {\n http: {\n action: ['GET', 'POST'],\n path: '::url',\n }\n },\n tags: ['crawl'],\n returnType: [String, OutputServerEventStream, RawString],\n })\n async crawl(\n @RPCReflect() rpcReflect: RPCReflection,\n @Ctx() ctx: Context,\n auth: JinaEmbeddingsAuthDTO,\n crawlerOptionsHeaderOnly: CrawlerOptionsHeaderOnly,\n crawlerOptionsParamsAllowed: CrawlerOptions,\n ) {\n const uid = await auth.solveUID();\n let chargeAmount = 0;\n const crawlerOptions = ctx.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed;\n const tierPolicy = await this.saasAssertTierPolicy(crawlerOptions, auth);\n\n // Use koa ctx.URL, a standard URL object to avoid node.js framework prop naming confusion\n const targetUrl = await this.getTargetUrl(tryDecodeURIComponent(`${ctx.URL.pathname}${ctx.URL.search}`), crawlerOptions);\n if (!targetUrl) {\n return await this.getIndex(auth);\n }\n\n // Prevent circular crawling\n this.puppeteerControl.circuitBreakerHosts.add(\n ctx.hostname.toLowerCase()\n );\n\n if (uid) {\n const user = await auth.assertUser();\n if (!(user.wallet.total_balance > 0)) {\n throw new InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`);\n }\n\n const rateLimitPolicy = auth.getRateLimits('CRAWL') || [\n parseInt(user.metadata?.speed_level) >= 2 ?\n RateLimitDesc.from({\n occurrence: 5000,\n periodSeconds: 60\n }) :\n RateLimitDesc.from({\n occurrence: 500,\n periodSeconds: 60\n })\n ];\n\n const apiRoll = await this.rateLimitControl.simpleRPCUidBasedLimit(\n rpcReflect, uid, ['CRAWL'],\n ...rateLimitPolicy\n );\n\n rpcReflect.finally(() => {\n if (crawlerOptions.tokenBudget && chargeAmount > crawlerOptions.tokenBudget) {\n return;\n }\n if (chargeAmount) {\n auth.reportUsage(chargeAmount, `reader-crawl`).catch((err) => {\n this.logger.warn(`Unable to report usage for ${uid}`, { err: marshalErrorLike(err) });\n });\n apiRoll.chargeAmount = chargeAmount;\n }\n });\n } else if (ctx.ip) {\n const apiRoll = await this.rateLimitControl.simpleRpcIPBasedLimit(rpcReflect, ctx.ip, ['CRAWL'],\n [\n // 20 requests per minute\n new Date(Date.now() - 60 * 1000), 20\n ]\n );\n\n rpcReflect.finally(() => {\n if (crawlerOptions.tokenBudget && chargeAmount > crawlerOptions.tokenBudget) {\n return;\n }\n apiRoll.chargeAmount = chargeAmount;\n });\n }\n\n if (!uid) {\n // Enforce no proxy is allocated for anonymous users due to abuse.\n crawlerOptions.proxy = 'none';\n const blockade = (await DomainBlockade.fromFirestoreQuery(\n DomainBlockade.COLLECTION\n .where('domain', '==', targetUrl.hostname.toLowerCase())\n .where('expireAt', '>=', new Date())\n .limit(1)\n ))[0];\n if (blockade) {\n throw new SecurityCompromiseError(`Domain ${targetUrl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`);\n }\n }\n const crawlOpts = await this.configure(crawlerOptions);\n this.logger.info(`Accepting request from ${uid || ctx.ip}`, { opts: crawlerOptions });\n if (crawlerOptions.robotsTxt) {\n await this.robotsTxtService.assertAccessAllowed(targetUrl, crawlerOptions.robotsTxt);\n }\n if (rpcReflect.signal.aborted) {\n return;\n }\n if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) {\n const sseStream = new OutputServerEventStream();\n rpcReflect.return(sseStream);\n\n try {\n for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) {\n if (!scrapped) {\n continue;\n }\n if (rpcReflect.signal.aborted) {\n break;\n }\n\n const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts);\n chargeAmount = this.assignChargeAmount(formatted, tierPolicy);\n sseStream.write({\n event: 'data',\n data: formatted,\n });\n if (chargeAmount && scrapped.pdfs?.length) {\n break;\n }\n }\n } catch (err: any) {\n this.logger.error(`Failed to crawl ${targetUrl}`, { err: marshalErrorLike(err) });\n sseStream.write({\n event: 'error',\n data: marshalErrorLike(err),\n });\n }\n\n sseStream.end();\n\n return sseStream;\n }\n\n let lastScrapped;\n if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {\n try {\n for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) {\n lastScrapped = scrapped;\n if (rpcReflect.signal.aborted) {\n break;\n }\n if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) {\n continue;\n }\n if (!scrapped.title) {\n continue;\n }\n\n const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts);\n chargeAmount = this.assignChargeAmount(formatted, tierPolicy);\n\n if (scrapped?.pdfs?.length && !chargeAmount) {\n continue;\n }\n\n return formatted;\n }\n } catch (err) {\n if (!lastScrapped) {\n throw err;\n }\n }\n\n if (!lastScrapped) {\n if (crawlOpts.targetSelector) {\n throw new AssertionFailureError(`No content available for URL ${targetUrl} with target selector ${Array.isArray(crawlOpts.targetSelector) ? crawlOpts.targetSelector.join(', ') : crawlOpts.targetSelector}`);\n }\n throw new AssertionFailureError(`No content available for URL ${targetUrl}`);\n }\n\n const formatted = await this.formatSnapshot(crawlerOptions, lastScrapped, targetUrl, this.urlValidMs, crawlOpts);\n chargeAmount = this.assignChargeAmount(formatted, tierPolicy);\n\n return formatted;\n }\n\n if (crawlerOptions.isRequestingCompoundContentFormat()) {\n throw new ParamValidationError({\n path: 'respondWith',\n message: `You are requesting compound content format, please explicitly accept 'text/event-stream' or 'application/json' in header.`\n });\n }\n\n try {\n for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) {\n lastScrapped = scrapped;\n if (rpcReflect.signal.aborted) {\n break;\n }\n if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) {\n continue;\n }\n if (!scrapped.title) {\n continue;\n }\n\n const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts);\n chargeAmount = this.assignChargeAmount(formatted, tierPolicy);\n\n if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {\n return assignTransferProtocolMeta(`${formatted.textRepresentation}`,\n { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } }\n );\n }\n if (crawlerOptions.respondWith === 'pageshot' && Reflect.get(formatted, 'pageshotUrl')) {\n return assignTransferProtocolMeta(`${formatted.textRepresentation}`,\n { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'pageshotUrl') } }\n );\n }\n\n return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { contentType: 'text/plain; charset=utf-8', envelope: null });\n }\n } catch (err) {\n if (!lastScrapped) {\n throw err;\n }\n }\n\n if (!lastScrapped) {\n if (crawlOpts.targetSelector) {\n throw new AssertionFailureError(`No content available for URL ${targetUrl} with target selector ${Array.isArray(crawlOpts.targetSelector) ? crawlOpts.targetSelector.join(', ') : crawlOpts.targetSelector}`);\n }\n throw new AssertionFailureError(`No content available for URL ${targetUrl}`);\n }\n const formatted = await this.formatSnapshot(crawlerOptions, lastScrapped, targetUrl, this.urlValidMs, crawlOpts);\n chargeAmount = this.assignChargeAmount(formatted, tierPolicy);\n\n if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {\n\n return assignTransferProtocolMeta(`${formatted.textRepresentation}`,\n { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } }\n );\n }\n if (crawlerOptions.respondWith === 'pageshot' && Reflect.get(formatted, 'pageshotUrl')) {\n\n return assignTransferProtocolMeta(`${formatted.textRepresentation}`,\n { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'pageshotUrl') } }\n );\n }\n\n return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { contentType: 'text/plain; charset=utf-8', envelope: null });\n\n }\n\n async getTargetUrl(originPath: string, crawlerOptions: CrawlerOptions) {\n let url: string = '';\n\n const targetUrlFromGet = originPath.slice(1);\n if (crawlerOptions.pdf) {\n const pdfFile = crawlerOptions.pdf;\n const identifier = pdfFile instanceof FancyFile ? (await pdfFile.sha256Sum) : randomUUID();\n url = `blob://pdf/${identifier}`;\n crawlerOptions.url ??= url;\n } else if (targetUrlFromGet) {\n url = targetUrlFromGet.trim();\n } else if (crawlerOptions.url) {\n url = crawlerOptions.url.trim();\n }\n\n if (!url) {\n throw new ParamValidationError({\n message: 'No URL provided',\n path: 'url'\n });\n }\n\n const { url: safeURL, ips } = await this.miscService.assertNormalizedUrl(url);\n if (this.puppeteerControl.circuitBreakerHosts.has(safeURL.hostname.toLowerCase())) {\n throw new SecurityCompromiseError({\n message: `Circular hostname: ${safeURL.protocol}`,\n path: 'url'\n });\n }\n crawlerOptions._hintIps = ips;\n\n return safeURL;\n }\n\n getUrlDigest(urlToCrawl: URL) {\n const normalizedURL = new URL(urlToCrawl);\n if (!normalizedURL.hash.startsWith('#/')) {\n normalizedURL.hash = '';\n }\n const normalizedUrl = normalizedURL.toString().toLowerCase();\n const digest = md5Hasher.hash(normalizedUrl.toString());\n\n return digest;\n }\n\n async *queryCache(urlToCrawl: URL, cacheTolerance: number) {\n const digest = this.getUrlDigest(urlToCrawl);\n\n const cache = (\n await\n (Crawled.fromFirestoreQuery(\n Crawled.COLLECTION.where('urlPathDigest', '==', digest).orderBy('createdAt', 'desc').limit(1)\n ).catch((err) => {\n this.logger.warn(`Failed to query cache, unknown issue`, { err });\n // https://github.com/grpc/grpc-node/issues/2647\n // https://github.com/googleapis/nodejs-firestore/issues/1023\n // https://github.com/googleapis/nodejs-firestore/issues/1023\n\n return undefined;\n }))\n )?.[0];\n\n yield cache;\n\n if (!cache) {\n return;\n }\n\n const age = Date.now() - cache.createdAt.valueOf();\n const stale = cache.createdAt.valueOf() < (Date.now() - cacheTolerance);\n this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for ${urlToCrawl}, normalized digest: ${digest}, ${age}ms old, tolerance ${cacheTolerance}ms`, {\n url: urlToCrawl, digest, age, stale, cacheTolerance\n });\n\n let snapshot: PageSnapshot | undefined;\n let screenshotUrl: string | undefined;\n let pageshotUrl: string | undefined;\n const preparations = [\n this.firebaseObjectStorage.downloadFile(`snapshots/${cache._id}`).then((r) => {\n snapshot = JSON.parse(r.toString('utf-8'));\n }),\n cache.screenshotAvailable ?\n this.firebaseObjectStorage.signDownloadUrl(`screenshots/${cache._id}`, Date.now() + this.urlValidMs).then((r) => {\n screenshotUrl = r;\n }) :\n Promise.resolve(undefined),\n cache.pageshotAvailable ?\n this.firebaseObjectStorage.signDownloadUrl(`pageshots/${cache._id}`, Date.now() + this.urlValidMs).then((r) => {\n pageshotUrl = r;\n }) :\n Promise.resolve(undefined)\n ];\n try {\n await Promise.all(preparations);\n } catch (_err) {\n // Swallow cache errors.\n return undefined;\n }\n\n yield {\n isFresh: !stale,\n ...cache,\n snapshot: {\n ...snapshot,\n screenshot: undefined,\n pageshot: undefined,\n screenshotUrl,\n pageshotUrl,\n } as PageSnapshot & { screenshotUrl?: string; pageshotUrl?: string; }\n };\n }\n\n async setToCache(urlToCrawl: URL, snapshot: PageSnapshot) {\n const digest = this.getUrlDigest(urlToCrawl);\n\n this.logger.info(`Caching snapshot of ${urlToCrawl}...`, { url: urlToCrawl, digest, title: snapshot?.title, href: snapshot?.href });\n const nowDate = new Date();\n\n const cache = Crawled.from({\n _id: randomUUID(),\n url: urlToCrawl.toString(),\n createdAt: nowDate,\n expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs),\n htmlSignificantlyModifiedByJs: snapshot.htmlSignificantlyModifiedByJs,\n urlPathDigest: digest,\n });\n\n const savingOfSnapshot = this.firebaseObjectStorage.saveFile(`snapshots/${cache._id}`,\n Buffer.from(\n JSON.stringify({\n ...snapshot,\n screenshot: undefined,\n pageshot: undefined,\n }),\n 'utf-8'\n ),\n {\n metadata: {\n contentType: 'application/json',\n }\n }\n ).then((r) => {\n cache.snapshotAvailable = true;\n return r;\n });\n\n if (snapshot.screenshot) {\n await this.firebaseObjectStorage.saveFile(`screenshots/${cache._id}`, snapshot.screenshot, {\n metadata: {\n contentType: 'image/png',\n }\n });\n cache.screenshotAvailable = true;\n }\n if (snapshot.pageshot) {\n await this.firebaseObjectStorage.saveFile(`pageshots/${cache._id}`, snapshot.pageshot, {\n metadata: {\n contentType: 'image/png',\n }\n });\n cache.pageshotAvailable = true;\n }\n await savingOfSnapshot;\n this.batchedCaches.push(cache);\n // const r = await Crawled.save(cache.degradeForFireStore()).catch((err) => {\n // this.logger.error(`Failed to save cache for ${urlToCrawl}`, { err: marshalErrorLike(err) });\n\n // return undefined;\n // });\n\n return cache;\n }\n\n async *iterSnapshots(urlToCrawl: URL, crawlOpts?: ExtraScrappingOptions, crawlerOpts?: CrawlerOptions) {\n // if (crawlerOpts?.respondWith.includes(CONTENT_FORMAT.VLM)) {\n // const finalBrowserSnapshot = await this.getFinalSnapshot(urlToCrawl, {\n // ...crawlOpts, engine: ENGINE_TYPE.BROWSER\n // }, crawlerOpts);\n\n // yield* this.lmControl.geminiFromBrowserSnapshot(finalBrowserSnapshot);\n\n // return;\n // }\n\n if (crawlerOpts?.respondWith.includes(CONTENT_FORMAT.READER_LM)) {\n const finalAutoSnapshot = await this.getFinalSnapshot(urlToCrawl, {\n ...crawlOpts,\n engine: crawlOpts?.engine || ENGINE_TYPE.AUTO,\n }, CrawlerOptions.from({\n ...crawlerOpts,\n respondWith: 'html',\n }));\n\n if (!finalAutoSnapshot?.html) {\n throw new AssertionFailureError(`Unexpected non HTML content for ReaderLM: ${urlToCrawl}`);\n }\n\n if (crawlerOpts?.instruction || crawlerOpts?.jsonSchema) {\n const jsonSchema = crawlerOpts.jsonSchema ? JSON.stringify(crawlerOpts.jsonSchema, undefined, 2) : undefined;\n yield* this.lmControl.readerLMFromSnapshot(crawlerOpts.instruction, jsonSchema, finalAutoSnapshot);\n\n return;\n }\n\n try {\n yield* this.lmControl.readerLMMarkdownFromSnapshot(finalAutoSnapshot);\n } catch (err) {\n if (err instanceof HTTPServiceError && err.status === 429) {\n throw new ServiceNodeResourceDrainError(`Reader LM is at capacity, please try again later.`);\n }\n throw err;\n }\n\n return;\n }\n\n yield* this.cachedScrap(urlToCrawl, crawlOpts, crawlerOpts);\n }\n\n async *cachedScrap(urlToCrawl: URL, crawlOpts?: ExtraScrappingOptions, crawlerOpts?: CrawlerOptions) {\n if (crawlerOpts?.html) {\n const snapshot = {\n href: urlToCrawl.toString(),\n html: crawlerOpts.html,\n title: '',\n text: '',\n } as PageSnapshot;\n yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts);\n\n return;\n }\n\n if (crawlerOpts?.pdf) {\n const pdfFile = crawlerOpts.pdf instanceof FancyFile ? crawlerOpts.pdf : this.tempFileManager.cacheBuffer(Buffer.from(crawlerOpts.pdf, 'base64'));\n const pdfLocalPath = pathToFileURL((await pdfFile.filePath));\n const snapshot = {\n href: urlToCrawl.toString(),\n html: ``,\n title: '',\n text: '',\n pdfs: [pdfLocalPath.href],\n } as PageSnapshot;\n\n yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts);\n\n return;\n }\n\n if (\n crawlOpts?.engine === ENGINE_TYPE.CURL ||\n // deprecated name\n crawlOpts?.engine === 'direct'\n ) {\n let sideLoaded;\n try {\n sideLoaded = (crawlOpts?.allocProxy && !crawlOpts?.proxyUrl) ?\n await this.sideLoadWithAllocatedProxy(urlToCrawl, crawlOpts) :\n await this.curlControl.sideLoad(urlToCrawl, crawlOpts);\n\n } catch (err) {\n if (err instanceof ServiceBadAttemptError) {\n throw new AssertionFailureError(err.message);\n }\n throw err;\n }\n if (!sideLoaded?.file) {\n throw new AssertionFailureError(`Remote server did not return a body: ${urlToCrawl}`);\n }\n const draftSnapshot = await this.snapshotFormatter.createSnapshotFromFile(urlToCrawl, sideLoaded.file, sideLoaded.contentType, sideLoaded.fileName);\n draftSnapshot.status = sideLoaded.status;\n draftSnapshot.statusText = sideLoaded.statusText;\n yield this.jsdomControl.narrowSnapshot(draftSnapshot, crawlOpts);\n return;\n }\n if (crawlOpts?.engine === ENGINE_TYPE.CF_BROWSER_RENDERING) {\n const html = await this.cfBrowserRendering.fetchContent(urlToCrawl.href);\n const snapshot = {\n href: urlToCrawl.toString(),\n html,\n title: '',\n text: '',\n } as PageSnapshot;\n yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts);\n return;\n }\n\n const cacheTolerance = crawlerOpts?.cacheTolerance ?? this.cacheValidMs;\n const cacheIt = this.queryCache(urlToCrawl, cacheTolerance);\n\n let cache = (await cacheIt.next()).value;\n if (cache?.htmlSignificantlyModifiedByJs === false) {\n if (crawlerOpts && crawlerOpts.timeout === undefined) {\n crawlerOpts.respondTiming ??= RESPOND_TIMING.HTML;\n }\n }\n\n if (!crawlerOpts || crawlerOpts.isCacheQueryApplicable()) {\n cache = (await cacheIt.next()).value;\n }\n cacheIt.return(undefined);\n\n if (cache?.isFresh &&\n (!crawlOpts?.favorScreenshot || (crawlOpts?.favorScreenshot && (cache.screenshotAvailable && cache.pageshotAvailable))) &&\n (_.get(cache.snapshot, 'locale') === crawlOpts?.locale)\n ) {\n if (cache.snapshot) {\n cache.snapshot.isFromCache = true;\n }\n yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts);\n\n return;\n }\n\n if (crawlOpts?.engine !== ENGINE_TYPE.BROWSER && !this.knownUrlThatSideLoadingWouldCrashTheBrowser(urlToCrawl)) {\n const sideLoadSnapshotPermitted = crawlerOpts?.browserIsNotRequired() &&\n [RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes(crawlerOpts.presumedRespondTiming);\n try {\n const altOpts = { ...crawlOpts };\n let sideLoaded = (crawlOpts?.allocProxy && !crawlOpts?.proxyUrl) ?\n await this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts) :\n await this.curlControl.sideLoad(urlToCrawl, altOpts).catch((err) => {\n this.logger.warn(`Failed to side load ${urlToCrawl.origin}`, { err: marshalErrorLike(err), href: urlToCrawl.href });\n\n if (err instanceof ApplicationError && !(err instanceof ServiceBadAttemptError)) {\n return Promise.reject(err);\n }\n\n return this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts);\n });\n if (!sideLoaded.file) {\n throw new ServiceBadAttemptError(`Remote server did not return a body: ${urlToCrawl}`);\n }\n const draftSnapshot = await this.snapshotFormatter.createSnapshotFromFile(\n urlToCrawl, sideLoaded.file, sideLoaded.contentType, sideLoaded.fileName\n ).catch((err) => {\n if (err instanceof ApplicationError) {\n return Promise.reject(new ServiceBadAttemptError(err.message));\n }\n return Promise.reject(err);\n });\n draftSnapshot.status = sideLoaded.status;\n draftSnapshot.statusText = sideLoaded.statusText;\n if (sideLoaded.status == 200 && !sideLoaded.contentType.startsWith('text/html')) {\n yield draftSnapshot;\n return;\n }\n\n let analyzed = await this.jsdomControl.analyzeHTMLTextLite(draftSnapshot.html);\n draftSnapshot.title ??= analyzed.title;\n draftSnapshot.isIntermediate = true;\n if (sideLoadSnapshotPermitted) {\n yield this.jsdomControl.narrowSnapshot(draftSnapshot, crawlOpts);\n }\n let fallbackProxyIsUsed = false;\n if (\n ((!crawlOpts?.allocProxy || crawlOpts.allocProxy !== 'none') && !crawlOpts?.proxyUrl) &&\n (analyzed.tokens < 42 || sideLoaded.status !== 200)\n ) {\n const proxyLoaded = await this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts);\n if (!proxyLoaded.file) {\n throw new ServiceBadAttemptError(`Remote server did not return a body: ${urlToCrawl}`);\n }\n const proxySnapshot = await this.snapshotFormatter.createSnapshotFromFile(\n urlToCrawl, proxyLoaded.file, proxyLoaded.contentType, proxyLoaded.fileName\n ).catch((err) => {\n if (err instanceof ApplicationError) {\n return Promise.reject(new ServiceBadAttemptError(err.message));\n }\n return Promise.reject(err);\n });\n proxySnapshot.status = proxyLoaded.status;\n proxySnapshot.statusText = proxyLoaded.statusText;\n if (proxyLoaded.status === 200 && crawlerOpts?.browserIsNotRequired()) {\n }\n analyzed = await this.jsdomControl.analyzeHTMLTextLite(proxySnapshot.html);\n if (proxyLoaded.status === 200 || analyzed.tokens >= 200) {\n proxySnapshot.isIntermediate = true;\n if (sideLoadSnapshotPermitted) {\n yield this.jsdomControl.narrowSnapshot(proxySnapshot, crawlOpts);\n }\n sideLoaded = proxyLoaded;\n fallbackProxyIsUsed = true;\n }\n }\n\n if (crawlOpts && (sideLoaded.status === 200 || analyzed.tokens >= 200 || crawlOpts.allocProxy)) {\n this.logger.info(`Side load seems to work, applying to crawler.`, { url: urlToCrawl.href });\n crawlOpts.sideLoad ??= sideLoaded.sideLoadOpts;\n if (fallbackProxyIsUsed) {\n this.logger.info(`Proxy seems to salvage the page`, { url: urlToCrawl.href });\n }\n }\n } catch (err: any) {\n this.logger.warn(`Failed to side load ${urlToCrawl.origin}`, { err: marshalErrorLike(err), href: urlToCrawl.href });\n if (err instanceof ApplicationError &&\n !(err instanceof ServiceBadAttemptError) &&\n !(err instanceof DataStreamBrokenError)\n ) {\n throw err;\n }\n }\n } else if (crawlOpts?.allocProxy && crawlOpts.allocProxy !== 'none' && !crawlOpts.proxyUrl) {\n const proxyUrl = await this.proxyProvider.alloc(this.figureOutBestProxyCountry(crawlOpts));\n crawlOpts.proxyUrl = proxyUrl.href;\n }\n\n try {\n if (crawlOpts?.targetSelector || crawlOpts?.removeSelector || crawlOpts?.withIframe || crawlOpts?.withShadowDom) {\n for await (const x of this.puppeteerControl.scrap(urlToCrawl, crawlOpts)) {\n yield this.jsdomControl.narrowSnapshot(x, crawlOpts);\n }\n\n return;\n }\n\n yield* this.puppeteerControl.scrap(urlToCrawl, crawlOpts);\n } catch (err: any) {\n if (cache && !(err instanceof SecurityCompromiseError)) {\n this.logger.warn(`Failed to scrap ${urlToCrawl}, but a stale cache is available. Falling back to cache`, { err: marshalErrorLike(err) });\n yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts);\n return;\n }\n throw err;\n }\n }\n\n assignChargeAmount(formatted: FormattedPage, saasTierPolicy?: Parameters[0]) {\n if (!formatted) {\n return 0;\n }\n\n let amount = 0;\n if (formatted.content) {\n amount = estimateToken(formatted.content);\n } else if (formatted.description) {\n amount += estimateToken(formatted.description);\n }\n\n if (formatted.text) {\n amount += estimateToken(formatted.text);\n }\n\n if (formatted.html) {\n amount += estimateToken(formatted.html);\n }\n if (formatted.screenshotUrl || formatted.screenshot) {\n // OpenAI image token count for 1024x1024 image\n amount += 765;\n }\n\n if (saasTierPolicy) {\n amount = this.saasApplyTierPolicy(saasTierPolicy, amount);\n }\n\n Object.assign(formatted, { usage: { tokens: amount } });\n assignMeta(formatted, { usage: { tokens: amount } });\n\n return amount;\n }\n\n\n async *scrapMany(urls: URL[], options?: ExtraScrappingOptions, crawlerOpts?: CrawlerOptions) {\n const iterators = urls.map((url) => this.cachedScrap(url, options, crawlerOpts));\n\n const results: (PageSnapshot | undefined)[] = iterators.map((_x) => undefined);\n\n let nextDeferred = Defer();\n let concluded = false;\n\n const handler = async (it: AsyncGenerator, idx: number) => {\n try {\n for await (const x of it) {\n results[idx] = x;\n\n if (x) {\n nextDeferred.resolve();\n nextDeferred = Defer();\n }\n\n }\n } catch (err: any) {\n this.logger.warn(`Failed to scrap ${urls[idx]}`, { err: marshalErrorLike(err) });\n }\n };\n\n Promise.allSettled(\n iterators.map((it, idx) => handler(it, idx))\n ).finally(() => {\n concluded = true;\n nextDeferred.resolve();\n });\n\n yield results;\n\n try {\n while (!concluded) {\n await nextDeferred.promise;\n\n yield results;\n }\n yield results;\n } finally {\n for (const x of iterators) {\n x.return();\n }\n }\n }\n\n async configure(opts: CrawlerOptions) {\n\n this.threadLocal.set('withGeneratedAlt', opts.withGeneratedAlt);\n this.threadLocal.set('withLinksSummary', opts.withLinksSummary);\n this.threadLocal.set('withImagesSummary', opts.withImagesSummary);\n this.threadLocal.set('keepImgDataUrl', opts.keepImgDataUrl);\n this.threadLocal.set('cacheTolerance', opts.cacheTolerance);\n this.threadLocal.set('withIframe', opts.withIframe);\n this.threadLocal.set('withShadowDom', opts.withShadowDom);\n this.threadLocal.set('userAgent', opts.userAgent);\n if (opts.timeout) {\n this.threadLocal.set('timeout', opts.timeout * 1000);\n }\n this.threadLocal.set('retainImages', opts.retainImages);\n this.threadLocal.set('noGfm', opts.noGfm);\n this.threadLocal.set('DNT', Boolean(opts.doNotTrack));\n if (opts.markdown) {\n this.threadLocal.set('turndownOpts', opts.markdown);\n }\n\n const crawlOpts: ExtraScrappingOptions = {\n proxyUrl: opts.proxyUrl,\n cookies: opts.setCookies,\n favorScreenshot: ['screenshot', 'pageshot'].some((x) => opts.respondWith.includes(x)),\n removeSelector: opts.removeSelector,\n targetSelector: opts.targetSelector,\n waitForSelector: opts.waitForSelector,\n overrideUserAgent: opts.userAgent,\n timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined,\n withIframe: opts.withIframe,\n withShadowDom: opts.withShadowDom,\n locale: opts.locale,\n referer: opts.referer,\n viewport: opts.viewport,\n engine: opts.engine,\n allocProxy: opts.proxy?.endsWith('+') ? opts.proxy.slice(0, -1) : opts.proxy,\n proxyResources: (opts.proxyUrl || opts.proxy?.endsWith('+')) ? true : false,\n private: Boolean(opts.doNotTrack),\n };\n\n if (crawlOpts.targetSelector?.length) {\n if (typeof crawlOpts.targetSelector === 'string') {\n crawlOpts.targetSelector = [crawlOpts.targetSelector];\n }\n for (const s of crawlOpts.targetSelector) {\n for (const e of s.split(',').map((x) => x.trim())) {\n if (e.startsWith('*') || e.startsWith(':') || e.includes('*:')) {\n throw new ParamValidationError({\n message: `Unacceptable selector: '${e}'. We cannot accept match-all selector for performance reasons. Sorry.`,\n path: 'targetSelector'\n });\n }\n }\n }\n }\n\n if (opts._hintIps?.length) {\n const hints = await this.geoIpService.lookupCities(opts._hintIps);\n const board: Record = {};\n for (const x of hints) {\n if (x.country?.code) {\n board[x.country.code] = (board[x.country.code] || 0) + 1;\n }\n }\n const hintCountry = _.maxBy(Array.from(Object.entries(board)), 1)?.[0];\n crawlOpts.countryHint = hintCountry?.toLowerCase();\n }\n\n if (opts.locale) {\n crawlOpts.extraHeaders ??= {};\n crawlOpts.extraHeaders['Accept-Language'] = opts.locale;\n }\n\n if (opts.respondWith.includes(CONTENT_FORMAT.VLM)) {\n crawlOpts.favorScreenshot = true;\n }\n\n if (opts.injectFrameScript?.length) {\n crawlOpts.injectFrameScripts = (await Promise.all(\n opts.injectFrameScript.map((x) => {\n if (URL.canParse(x)) {\n return fetch(x).then((r) => r.text());\n }\n\n return x;\n })\n )).filter(Boolean);\n }\n\n if (opts.injectPageScript?.length) {\n crawlOpts.injectPageScripts = (await Promise.all(\n opts.injectPageScript.map((x) => {\n if (URL.canParse(x)) {\n return fetch(x).then((r) => r.text());\n }\n\n return x;\n })\n )).filter(Boolean);\n }\n\n return crawlOpts;\n }\n\n protected async formatSnapshot(\n crawlerOptions: CrawlerOptions,\n snapshot: PageSnapshot & {\n screenshotUrl?: string;\n pageshotUrl?: string;\n },\n nominalUrl?: URL,\n urlValidMs?: number,\n scrappingOptions?: ScrappingOptions\n ) {\n const presumedURL = crawlerOptions.base === 'final' ? new URL(snapshot.href) : nominalUrl;\n\n const respondWith = crawlerOptions.respondWith;\n if (respondWith === CONTENT_FORMAT.READER_LM || respondWith === CONTENT_FORMAT.VLM) {\n const output: FormattedPage = {\n title: snapshot.title,\n content: snapshot.parsed?.textContent,\n url: presumedURL?.href || snapshot.href,\n };\n\n Object.defineProperty(output, 'textRepresentation', {\n value: snapshot.parsed?.textContent,\n enumerable: false,\n });\n\n return output;\n }\n\n return this.formatSnapshotWithPDFSideLoad(respondWith, snapshot, presumedURL, urlValidMs, scrappingOptions);\n }\n\n async formatSnapshotWithPDFSideLoad(mode: string, snapshot: PageSnapshot, nominalUrl?: URL, urlValidMs?: number, scrappingOptions?: ScrappingOptions) {\n const snapshotCopy = _.cloneDeep(snapshot);\n\n if (snapshotCopy.pdfs?.length) {\n const pdfUrl = snapshotCopy.pdfs[0];\n if (pdfUrl.startsWith('http')) {\n const sideLoaded = scrappingOptions?.sideLoad?.impersonate[pdfUrl];\n if (sideLoaded?.status === 200 && sideLoaded.body) {\n snapshotCopy.pdfs[0] = pathToFileURL(await sideLoaded?.body.filePath).href;\n return this.snapshotFormatter.formatSnapshot(mode, snapshotCopy, nominalUrl, urlValidMs);\n }\n\n const r = await this.curlControl.sideLoad(new URL(pdfUrl), scrappingOptions).catch((err) => {\n if (err instanceof ServiceBadAttemptError) {\n return Promise.reject(new AssertionFailureError(`Failed to load PDF(${pdfUrl}): ${err.message}`));\n }\n\n return Promise.reject(err);\n });\n if (r.status !== 200) {\n throw new AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server responded status ${r.status}`);\n }\n if (!r.contentType.includes('application/pdf')) {\n throw new AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server responded with wrong content type ${r.contentType}`);\n }\n if (!r.file) {\n throw new AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server did not return a body`);\n }\n snapshotCopy.pdfs[0] = pathToFileURL(await r.file.filePath).href;\n }\n }\n\n return this.snapshotFormatter.formatSnapshot(mode, snapshotCopy, nominalUrl, urlValidMs);\n }\n\n async getFinalSnapshot(url: URL, opts?: ExtraScrappingOptions, crawlerOptions?: CrawlerOptions): Promise {\n const it = this.cachedScrap(url, opts, crawlerOptions);\n\n let lastSnapshot;\n let lastError;\n try {\n for await (const x of it) {\n lastSnapshot = x;\n }\n } catch (err) {\n lastError = err;\n }\n\n if (!lastSnapshot && lastError) {\n throw lastError;\n }\n\n if (!lastSnapshot) {\n throw new AssertionFailureError(`No content available`);\n }\n\n return lastSnapshot;\n }\n\n async simpleCrawl(mode: string, url: URL, opts?: ExtraScrappingOptions) {\n const it = this.iterSnapshots(url, { ...opts, minIntervalMs: 500 });\n\n let lastSnapshot;\n let goodEnough = false;\n try {\n for await (const x of it) {\n lastSnapshot = x;\n\n if (goodEnough) {\n break;\n }\n\n if (lastSnapshot?.parsed?.content) {\n // After it's good enough, wait for next snapshot;\n goodEnough = true;\n }\n }\n\n } catch (err) {\n if (lastSnapshot) {\n return this.snapshotFormatter.formatSnapshot(mode, lastSnapshot, url, this.urlValidMs);\n }\n\n throw err;\n }\n\n if (!lastSnapshot) {\n throw new AssertionFailureError(`No content available`);\n }\n\n return this.snapshotFormatter.formatSnapshot(mode, lastSnapshot, url, this.urlValidMs);\n }\n\n getDomainProfileUrlDigest(url: URL) {\n const pathname = url.pathname;\n const pathVec = pathname.split('/');\n const parentPath = pathVec.slice(0, -1).join('/');\n\n const finalPath = parentPath || pathname;\n\n const key = url.origin.toLocaleLowerCase() + finalPath;\n\n return {\n digest: md5Hasher.hash(key),\n path: finalPath,\n };\n }\n\n proxyIterMap = new WeakMap>();\n @retryWith((err) => {\n if (err instanceof ServiceBadApproachError) {\n return false;\n }\n if (err instanceof ServiceBadAttemptError) {\n // Keep trying\n return true;\n }\n if (err instanceof ApplicationError) {\n // Quit with this error\n return false;\n }\n return undefined;\n }, 3)\n async sideLoadWithAllocatedProxy(url: URL, opts?: ExtraScrappingOptions) {\n if (opts?.allocProxy === 'none') {\n return this.curlControl.sideLoad(url, opts);\n }\n let proxy;\n if (opts) {\n let it = this.proxyIterMap.get(opts);\n if (!it) {\n it = this.proxyProvider.iterAlloc(this.figureOutBestProxyCountry(opts));\n this.proxyIterMap.set(opts, it);\n }\n proxy = (await it.next()).value;\n }\n\n proxy ??= await this.proxyProvider.alloc(this.figureOutBestProxyCountry(opts));\n this.logger.debug(`Proxy allocated`, { proxy: proxy.href });\n const r = await this.curlControl.sideLoad(url, {\n ...opts,\n proxyUrl: proxy.href,\n });\n\n if (opts && opts.allocProxy) {\n opts.proxyUrl ??= proxy.href;\n }\n\n return { ...r, proxy };\n }\n\n protected figureOutBestProxyCountry(opts?: ExtraScrappingOptions) {\n if (!opts) {\n return 'auto';\n }\n\n let draft;\n\n if (opts.allocProxy) {\n if (this.proxyProvider.supports(opts.allocProxy)) {\n draft = opts.allocProxy;\n } else if (opts.allocProxy === 'none') {\n return 'none';\n }\n }\n\n if (opts.countryHint) {\n if (this.proxyProvider.supports(opts.countryHint)) {\n draft ??= opts.countryHint;\n }\n }\n\n draft ??= opts.allocProxy || 'auto';\n\n return draft;\n }\n\n knownUrlThatSideLoadingWouldCrashTheBrowser(url: URL) {\n if (url.hostname === 'chromewebstore.google.com') {\n return true;\n }\n\n return false;\n }\n\n async saasAssertTierPolicy(opts: CrawlerOptions, auth: JinaEmbeddingsAuthDTO) {\n let chargeScalar = 1;\n let minimalCharge = 0;\n\n if (opts.withGeneratedAlt) {\n await auth.assertTier(0, 'Alt text generation');\n minimalCharge = 765;\n }\n\n if (opts.injectPageScript || opts.injectFrameScript) {\n await auth.assertTier(0, 'Script injection');\n minimalCharge = 4_000;\n }\n\n if (opts.withIframe) {\n await auth.assertTier(0, 'Iframe');\n }\n\n if (opts.engine === ENGINE_TYPE.CF_BROWSER_RENDERING) {\n await auth.assertTier(0, 'Cloudflare browser rendering');\n minimalCharge = 4_000;\n }\n\n if (opts.respondWith.includes('lm') || opts.engine?.includes('lm')) {\n await auth.assertTier(0, 'Language model');\n minimalCharge = 4_000;\n chargeScalar = 3;\n }\n\n if (opts.proxy && opts.proxy !== 'none') {\n await auth.assertTier(['auto', 'any'].includes(opts.proxy) ? 0 : 2, 'Proxy allocation');\n chargeScalar = 5;\n }\n\n return {\n budget: opts.tokenBudget || 0,\n chargeScalar,\n minimalCharge,\n };\n }\n\n saasApplyTierPolicy(policy: Awaited>, chargeAmount: number) {\n const effectiveChargeAmount = policy.chargeScalar * Math.max(chargeAmount, policy.minimalCharge);\n if (policy.budget && policy.budget < effectiveChargeAmount) {\n throw new BudgetExceededError(`Token budget (${policy.budget}) exceeded, intended charge amount ${effectiveChargeAmount}`);\n }\n\n return effectiveChargeAmount;\n }\n}\n" }, { "path": "src/api/searcher.ts", "content": "import { singleton } from 'tsyringe';\nimport {\n assignTransferProtocolMeta, RPCHost, RPCReflection, AssertionFailureError, assignMeta, RawString,\n} from 'civkit/civ-rpc';\nimport { marshalErrorLike } from 'civkit/lang';\nimport { objHashMd5B64Of } from 'civkit/hash';\nimport _ from 'lodash';\n\nimport { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit';\n\nimport { CrawlerHost, ExtraScrappingOptions } from './crawler';\nimport { CrawlerOptions, RESPOND_TIMING } from '../dto/crawler-options';\nimport { SnapshotFormatter, FormattedPage as RealFormattedPage } from '../services/snapshot-formatter';\nimport { GoogleSearchExplicitOperatorsDto } from '../services/serper-search';\n\nimport { GlobalLogger } from '../services/logger';\nimport { AsyncLocalContext } from '../services/async-context';\nimport { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';\nimport { OutputServerEventStream } from '../lib/transform-server-event-stream';\nimport { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';\nimport { InsufficientBalanceError } from '../services/errors';\n\nimport { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper';\nimport { toAsyncGenerator } from '../utils/misc';\nimport type { JinaEmbeddingsTokenAccount } from '../shared/db/jina-embeddings-token-account';\nimport { LRUCache } from 'lru-cache';\nimport { API_CALL_STATUS } from '../shared/db/api-roll';\nimport { SERPResult } from '../db/searched';\nimport { SerperSearchQueryParams, WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search';\nimport { InternalJinaSerpService } from '../services/serp/internal';\nimport { WebSearchEntry } from '../services/serp/compat';\n\nconst WORLD_COUNTRY_CODES = Object.keys(WORLD_COUNTRIES).map((x) => x.toLowerCase());\n\ninterface FormattedPage extends RealFormattedPage {\n favicon?: string;\n date?: string;\n}\n\ntype RateLimitCache = {\n blockedUntil?: Date;\n user?: JinaEmbeddingsTokenAccount;\n};\n\n@singleton()\nexport class SearcherHost extends RPCHost {\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n cacheRetentionMs = 1000 * 3600 * 24 * 7;\n cacheValidMs = 1000 * 3600;\n pageCacheToleranceMs = 1000 * 3600 * 24;\n\n reasonableDelayMs = 15_000;\n\n targetResultCount = 5;\n\n highFreqKeyCache = new LRUCache({\n max: 256,\n ttl: 60 * 60 * 1000,\n updateAgeOnGet: false,\n updateAgeOnHas: false,\n });\n\n batchedCaches: SERPResult[] = [];\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected rateLimitControl: RateLimitControl,\n protected threadLocal: AsyncLocalContext,\n protected crawler: CrawlerHost,\n protected snapshotFormatter: SnapshotFormatter,\n protected serperGoogle: SerperGoogleSearchService,\n protected serperBing: SerperBingSearchService,\n protected jinaSerp: InternalJinaSerpService,\n ) {\n super(...arguments);\n\n setInterval(() => {\n const thisBatch = this.batchedCaches;\n this.batchedCaches = [];\n if (!thisBatch.length) {\n return;\n }\n const batch = SERPResult.DB.batch();\n\n for (const x of thisBatch) {\n batch.set(SERPResult.COLLECTION.doc(), x.degradeForFireStore());\n }\n batch.commit()\n .then(() => {\n this.logger.debug(`Saved ${thisBatch.length} caches by batch`);\n })\n .catch((err) => {\n this.logger.warn(`Failed to cache search result in batch`, { err });\n });\n }, 1000 * 10 + Math.round(1000 * Math.random())).unref();\n }\n\n override async init() {\n await this.dependencyReady();\n\n this.emit('ready');\n }\n\n @Method({\n name: 'searchIndex',\n ext: {\n http: {\n action: ['get', 'post'],\n path: '/search'\n }\n },\n tags: ['search'],\n returnType: [String, OutputServerEventStream],\n })\n @Method({\n ext: {\n http: {\n action: ['get', 'post'],\n path: '::q'\n }\n },\n tags: ['search'],\n returnType: [String, OutputServerEventStream, RawString],\n })\n async search(\n @RPCReflect() rpcReflect: RPCReflection,\n @Ctx() ctx: Context,\n auth: JinaEmbeddingsAuthDTO,\n crawlerOptions: CrawlerOptions,\n searchExplicitOperators: GoogleSearchExplicitOperatorsDto,\n @Param('count', { validate: (v: number) => v >= 0 && v <= 20 })\n count: number,\n @Param('type', { type: new Set(['web', 'images', 'news']), default: 'web' })\n variant: 'web' | 'images' | 'news',\n @Param('provider', { type: new Set(['google', 'bing']), default: 'google' })\n searchEngine: 'google' | 'bing',\n @Param('num', { validate: (v: number) => v >= 0 && v <= 20 })\n num?: number,\n @Param('gl', { validate: (v: string) => WORLD_COUNTRY_CODES.includes(v?.toLowerCase()) }) gl?: string,\n @Param('hl', { validate: (v: string) => WORLD_LANGUAGES.some(l => l.code === v) }) hl?: string,\n @Param('location') location?: string,\n @Param('page') page?: number,\n @Param('fallback', { type: Boolean, default: true }) fallback?: boolean,\n @Param('q') q?: string,\n ) {\n // We want to make our search API follow SERP schema, so we need to expose 'num' parameter.\n // Since we used 'count' as 'num' previously, we need to keep 'count' for old users.\n // Here we combine 'count' and 'num' to 'count' for the rest of the function.\n count = (num !== undefined ? num : count) ?? 10;\n\n const authToken = auth.bearerToken;\n let highFreqKey: RateLimitCache | undefined;\n if (authToken && this.highFreqKeyCache.has(authToken)) {\n highFreqKey = this.highFreqKeyCache.get(authToken)!;\n auth.user = highFreqKey.user;\n auth.uid = highFreqKey.user?.user_id;\n }\n\n const uid = await auth.solveUID();\n // Return content by default\n const crawlWithoutContent = crawlerOptions.respondWith.includes('no-content');\n const withFavicon = Boolean(ctx.get('X-With-Favicons'));\n this.threadLocal.set('collect-favicon', withFavicon);\n crawlerOptions.respondTiming ??= RESPOND_TIMING.VISIBLE_CONTENT;\n\n let chargeAmount = 0;\n const noSlashPath = decodeURIComponent(ctx.path).slice(1);\n if (!noSlashPath && !q) {\n const index = await this.crawler.getIndex(auth);\n if (!uid) {\n index.note = 'Authentication is required to use this endpoint. Please provide a valid API key via Authorization header.';\n }\n if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {\n\n return index;\n }\n\n return assignTransferProtocolMeta(`${index}`,\n { contentType: 'text/plain', envelope: null }\n );\n }\n\n const user = await auth.assertUser();\n if (!(user.wallet.total_balance > 0)) {\n throw new InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`);\n }\n\n if (highFreqKey?.blockedUntil) {\n const now = new Date();\n const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf());\n if (blockedTimeRemaining > 0) {\n throw RateLimitTriggeredError.from({\n message: `Per UID rate limit exceeded (async)`,\n retryAfter: Math.ceil(blockedTimeRemaining / 1000),\n });\n }\n }\n\n const rateLimitPolicy = auth.getRateLimits(rpcReflect.name.toUpperCase()) || [\n parseInt(user.metadata?.speed_level) >= 2 ?\n RateLimitDesc.from({\n occurrence: 1000,\n periodSeconds: 60\n }) :\n RateLimitDesc.from({\n occurrence: 100,\n periodSeconds: 60\n })\n ];\n\n const apiRollPromise = this.rateLimitControl.simpleRPCUidBasedLimit(\n rpcReflect, uid!, [rpcReflect.name.toUpperCase()],\n ...rateLimitPolicy\n );\n\n if (!highFreqKey) {\n // Normal path\n await apiRollPromise;\n\n if (rateLimitPolicy.some(\n (x) => {\n const rpm = x.occurrence / (x.periodSeconds / 60);\n if (rpm >= 400) {\n return true;\n }\n\n return false;\n })\n ) {\n this.highFreqKeyCache.set(auth.bearerToken!, {\n user,\n });\n }\n\n } else {\n // High freq key path\n apiRollPromise.then(\n // Rate limit not triggered, make sure not blocking.\n () => {\n delete highFreqKey.blockedUntil;\n },\n // Rate limit triggered\n (err) => {\n if (!(err instanceof RateLimitTriggeredError)) {\n return;\n }\n const now = Date.now();\n let tgtDate;\n if (err.retryAfterDate) {\n tgtDate = err.retryAfterDate;\n } else if (err.retryAfter) {\n tgtDate = new Date(now + err.retryAfter * 1000);\n }\n\n if (tgtDate) {\n const dt = tgtDate.valueOf() - now;\n highFreqKey.blockedUntil = tgtDate;\n setTimeout(() => {\n if (highFreqKey.blockedUntil === tgtDate) {\n delete highFreqKey.blockedUntil;\n }\n }, dt).unref();\n }\n }\n ).finally(async () => {\n // Always asynchronously update user(wallet);\n const user = await auth.getBrief().catch(() => undefined);\n if (user) {\n highFreqKey.user = user;\n }\n });\n }\n\n rpcReflect.finally(async () => {\n if (chargeAmount) {\n auth.reportUsage(chargeAmount, `reader-${rpcReflect.name}`).catch((err) => {\n this.logger.warn(`Unable to report usage for ${uid}`, { err: marshalErrorLike(err) });\n });\n try {\n const apiRoll = await apiRollPromise;\n apiRoll.chargeAmount = chargeAmount;\n\n } catch (err) {\n await this.rateLimitControl.record({\n uid,\n tags: [rpcReflect.name.toUpperCase()],\n status: API_CALL_STATUS.SUCCESS,\n chargeAmount,\n }).save().catch((err) => {\n this.logger.warn(`Failed to save rate limit record`, { err: marshalErrorLike(err) });\n });\n }\n }\n });\n\n delete crawlerOptions.html;\n\n const crawlOpts = await this.crawler.configure(crawlerOptions);\n const searchQuery = searchExplicitOperators.addTo(q || noSlashPath);\n\n let fetchNum = count;\n if ((page ?? 1) === 1) {\n fetchNum = count > 10 ? 30 : 20;\n }\n\n let fallbackQuery: string | undefined;\n let chargeAmountScaler = 1;\n if (searchEngine === 'bing') {\n this.threadLocal.set('bing-preferred', true);\n chargeAmountScaler = 3;\n }\n\n if (variant !== 'web') {\n chargeAmountScaler = 5;\n }\n\n // Search with fallback logic if enabled\n const searchParams = {\n variant,\n provider: searchEngine,\n q: searchQuery,\n num: fetchNum,\n gl,\n hl,\n location,\n page,\n };\n\n const { results, query: successQuery, tryTimes } = await this.searchWithFallback(\n searchParams, fallback, crawlerOptions.noCache\n );\n chargeAmountScaler *= tryTimes;\n\n fallbackQuery = successQuery !== searchQuery ? successQuery : undefined;\n\n if (!results.length) {\n throw new AssertionFailureError(`No search results available for query ${searchQuery}`);\n }\n\n if (crawlOpts.timeoutMs && crawlOpts.timeoutMs < 30_000) {\n delete crawlOpts.timeoutMs;\n }\n\n\n let lastScrapped: any[] | undefined;\n const targetResultCount = crawlWithoutContent ? count : count + 2;\n const trimmedResults: any[] = results.filter((x) => Boolean(x.link)).slice(0, targetResultCount).map((x) => this.mapToFinalResults(x));\n trimmedResults.toString = function () {\n let r = this.map((x, i) => x ? Reflect.apply(x.toString, x, [i]) : '').join('\\n\\n').trimEnd() + '\\n';\n if (fallbackQuery) {\n r = `Fallback query: ${fallbackQuery}\\n\\n${r}`;\n }\n return r;\n };\n if (!crawlerOptions.respondWith.includes('no-content') &&\n ['html', 'text', 'shot', 'markdown', 'content'].some((x) => crawlerOptions.respondWith.includes(x))\n ) {\n for (const x of trimmedResults) {\n x.content ??= '';\n }\n }\n const assigningOfGeneralMixins = Promise.allSettled(\n trimmedResults.map((x) => this.assignGeneralMixin(x))\n );\n\n let it;\n\n if (crawlWithoutContent || count === 0) {\n it = toAsyncGenerator(trimmedResults);\n await assigningOfGeneralMixins;\n } else {\n it = this.fetchSearchResults(crawlerOptions.respondWith, trimmedResults, crawlOpts,\n CrawlerOptions.from({ ...crawlerOptions, cacheTolerance: crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs }),\n count,\n );\n }\n\n if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) {\n const sseStream = new OutputServerEventStream();\n rpcReflect.return(sseStream);\n try {\n for await (const scrapped of it) {\n if (!scrapped) {\n continue;\n }\n if (rpcReflect.signal.aborted) {\n break;\n }\n\n chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);\n lastScrapped = scrapped;\n\n if (fallbackQuery) {\n sseStream.write({\n event: 'meta',\n data: { fallback: fallbackQuery },\n });\n }\n\n sseStream.write({\n event: 'data',\n data: scrapped,\n });\n }\n } catch (err: any) {\n this.logger.error(`Failed to collect search result for query ${searchQuery}`,\n { err: marshalErrorLike(err) }\n );\n sseStream.write({\n event: 'error',\n data: marshalErrorLike(err),\n });\n }\n\n sseStream.end();\n\n return sseStream;\n }\n\n let earlyReturn = false;\n if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {\n let earlyReturnTimer: ReturnType | undefined;\n const setEarlyReturnTimer = () => {\n if (earlyReturnTimer) {\n return;\n }\n earlyReturnTimer = setTimeout(async () => {\n if (!lastScrapped) {\n return;\n }\n await assigningOfGeneralMixins;\n chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);\n\n rpcReflect.return(lastScrapped);\n earlyReturn = true;\n }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);\n };\n\n for await (const scrapped of it) {\n lastScrapped = scrapped;\n if (rpcReflect.signal.aborted || earlyReturn) {\n break;\n }\n if (_.some(scrapped, (x) => this.pageQualified(x))) {\n setEarlyReturnTimer();\n }\n if (!this.searchResultsQualified(scrapped, count)) {\n continue;\n }\n if (earlyReturnTimer) {\n clearTimeout(earlyReturnTimer);\n }\n await assigningOfGeneralMixins;\n chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);\n\n return scrapped;\n }\n\n if (earlyReturnTimer) {\n clearTimeout(earlyReturnTimer);\n }\n\n if (!lastScrapped) {\n throw new AssertionFailureError(`No content available for query ${searchQuery}`);\n }\n\n if (!earlyReturn) {\n await assigningOfGeneralMixins;\n chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);\n }\n\n return lastScrapped;\n }\n\n let earlyReturnTimer: ReturnType | undefined;\n const setEarlyReturnTimer = () => {\n if (earlyReturnTimer) {\n return;\n }\n earlyReturnTimer = setTimeout(async () => {\n if (!lastScrapped) {\n return;\n }\n await assigningOfGeneralMixins;\n chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);\n\n rpcReflect.return(assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null }));\n earlyReturn = true;\n }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);\n };\n\n for await (const scrapped of it) {\n lastScrapped = scrapped;\n if (rpcReflect.signal.aborted || earlyReturn) {\n break;\n }\n if (_.some(scrapped, (x) => this.pageQualified(x))) {\n setEarlyReturnTimer();\n }\n\n if (!this.searchResultsQualified(scrapped, count)) {\n continue;\n }\n\n if (earlyReturnTimer) {\n clearTimeout(earlyReturnTimer);\n }\n await assigningOfGeneralMixins;\n chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);\n\n return assignTransferProtocolMeta(`${scrapped}`, { contentType: 'text/plain', envelope: null });\n }\n\n if (earlyReturnTimer) {\n clearTimeout(earlyReturnTimer);\n }\n\n if (!lastScrapped) {\n throw new AssertionFailureError(`No content available for query ${searchQuery}`);\n }\n\n if (!earlyReturn) {\n await assigningOfGeneralMixins;\n chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);\n }\n\n return assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null });\n }\n\n /**\n * Search with fallback to progressively shorter queries if no results found\n * @param params Search parameters\n * @param useFallback Whether to use the fallback mechanism\n * @param noCache Whether to bypass cache\n * @returns Search response and the successful query\n */\n async searchWithFallback(\n params: SerperSearchQueryParams & { variant: 'web' | 'images' | 'news'; provider?: string; },\n useFallback: boolean = false,\n noCache: boolean = false\n ) {\n // Try original query first\n const originalQuery = params.q;\n const containsRTL = /[\\u0600-\\u06FF\\u0750-\\u077F\\u08A0-\\u08FF\\uFB50-\\uFDFF\\uFE70-\\uFEFF\\u0590-\\u05FF\\uFB1D-\\uFB4F\\u0700-\\u074F\\u0780-\\u07BF\\u07C0-\\u07FF]/.test(originalQuery);\n\n // Extract results based on variant\n let tryTimes = 1;\n const results = await this.cachedSearch(params.variant, params, noCache);\n if (results.length || !useFallback) {\n return { results, query: params.q, tryTimes };\n }\n\n let queryTerms = originalQuery.split(/\\s+/);\n const lastResort = containsRTL ? queryTerms.slice(queryTerms.length - 2) : queryTerms.slice(0, 2);\n\n this.logger.info(`No results for \"${originalQuery}\", trying fallback queries`);\n\n let terms: string[] = [];\n // fallback n times\n const n = 4;\n\n while (tryTimes < n) {\n const delta = Math.ceil(queryTerms.length / n) * tryTimes;\n terms = containsRTL ? queryTerms.slice(delta) : queryTerms.slice(0, queryTerms.length - delta);\n const query = terms.join(' ');\n if (!query) {\n break;\n }\n tryTimes += 1;\n this.logger.info(`Retrying search with fallback query: \"${query}\"`);\n const fallbackParams = { ...params, q: query };\n const fallbackResults = await this.cachedSearch(params.variant, fallbackParams, noCache);\n if (fallbackResults.length > 0) {\n return { results: fallbackResults, query: fallbackParams.q, tryTimes };\n }\n }\n\n if (terms.length > lastResort.length) {\n const query = lastResort.join(' ');\n this.logger.info(`Retrying search with fallback query: \"${query}\"`);\n const fallbackParams = { ...params, q: query };\n tryTimes += 1;\n const fallbackResults = await this.cachedSearch(params.variant, fallbackParams, noCache);\n\n if (fallbackResults.length > 0) {\n return { results: fallbackResults, query, tryTimes };\n }\n }\n\n return { results, query: originalQuery, tryTimes };\n }\n\n async *fetchSearchResults(\n mode: string | 'markdown' | 'html' | 'text' | 'screenshot' | 'favicon' | 'content',\n searchResults?: FormattedPage[],\n options?: ExtraScrappingOptions,\n crawlerOptions?: CrawlerOptions,\n count?: number,\n ) {\n if (!searchResults) {\n return;\n }\n const urls = searchResults.map((x) => new URL(x.url!));\n const snapshotMap = new WeakMap();\n for await (const scrapped of this.crawler.scrapMany(urls, options, crawlerOptions)) {\n const mapped = scrapped.map((x, i) => {\n if (!x) {\n return {};\n }\n if (snapshotMap.has(x)) {\n return snapshotMap.get(x);\n }\n return this.crawler.formatSnapshotWithPDFSideLoad(mode, x, urls[i], undefined, options).then((r) => {\n snapshotMap.set(x, r);\n\n return r;\n }).catch((err) => {\n this.logger.error(`Failed to format snapshot for ${urls[i].href}`, { err: marshalErrorLike(err) });\n\n return {};\n });\n });\n\n const resultArray = await Promise.all(mapped) as FormattedPage[];\n for (const [i, v] of resultArray.entries()) {\n if (v) {\n Object.assign(searchResults[i], v);\n }\n }\n\n yield this.reOrganizeSearchResults(searchResults, count);\n }\n }\n\n reOrganizeSearchResults(searchResults: FormattedPage[], count?: number) {\n const targetResultCount = count || this.targetResultCount;\n const [qualifiedPages, unqualifiedPages] = _.partition(searchResults, (x) => this.pageQualified(x));\n const acceptSet = new Set(qualifiedPages);\n\n const n = targetResultCount - qualifiedPages.length;\n for (const x of unqualifiedPages.slice(0, n >= 0 ? n : 0)) {\n acceptSet.add(x);\n }\n\n const filtered = searchResults.filter((x) => acceptSet.has(x)).slice(0, targetResultCount);\n\n const resultArray = filtered;\n\n resultArray.toString = searchResults.toString;\n\n return resultArray;\n }\n\n assignChargeAmount(formatted: FormattedPage[], num: number, scaler: number, fallbackQuery?: string) {\n let contentCharge = 0;\n for (const x of formatted) {\n const itemAmount = this.crawler.assignChargeAmount(x) || 0;\n\n if (!itemAmount) {\n continue;\n }\n\n contentCharge += itemAmount;\n }\n\n const numCharge = Math.ceil(formatted.length / 10) * 10000 * scaler;\n\n const final = Math.max(contentCharge, numCharge);\n\n if (final === numCharge) {\n for (const x of formatted) {\n x.usage = { tokens: Math.ceil(numCharge / formatted.length) };\n }\n }\n\n const metadata: Record = { usage: { tokens: final } };\n if (fallbackQuery) {\n metadata.fallback = fallbackQuery;\n }\n\n assignMeta(formatted, metadata);\n\n return final;\n }\n\n pageQualified(formattedPage: FormattedPage) {\n return formattedPage.title &&\n formattedPage.content ||\n formattedPage.screenshotUrl ||\n formattedPage.pageshotUrl ||\n formattedPage.text ||\n formattedPage.html;\n }\n\n searchResultsQualified(results: FormattedPage[], targetResultCount = this.targetResultCount) {\n return _.every(results, (x) => this.pageQualified(x)) && results.length >= targetResultCount;\n }\n\n async getFavicon(domain: string) {\n const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`;\n\n try {\n const response = await fetch(url);\n if (!response.ok) {\n return '';\n }\n const ab = await response.arrayBuffer();\n const buffer = Buffer.from(ab);\n const base64 = buffer.toString('base64');\n return `data:image/png;base64,${base64}`;\n } catch (error: any) {\n this.logger.warn(`Failed to get favicon base64 string`, { err: marshalErrorLike(error) });\n return '';\n }\n }\n\n *iterProviders(preference?: string, variant?: string) {\n if (preference === 'bing') {\n yield this.serperBing;\n yield variant === 'web' ? this.jinaSerp : this.serperGoogle;\n yield this.serperGoogle;\n\n return;\n }\n\n if (preference === 'google') {\n yield variant === 'web' ? this.jinaSerp : this.serperGoogle;\n yield this.serperGoogle;\n yield this.serperGoogle;\n\n return;\n }\n\n yield variant === 'web' ? this.jinaSerp : this.serperGoogle;\n yield this.serperGoogle;\n yield this.serperGoogle;\n }\n\n async cachedSearch(variant: 'web' | 'news' | 'images', query: Record, noCache?: boolean): Promise {\n const queryDigest = objHashMd5B64Of({ ...query, variant });\n const provider = query.provider;\n Reflect.deleteProperty(query, 'provider');\n let cache;\n if (!noCache) {\n cache = (await SERPResult.fromFirestoreQuery(\n SERPResult.COLLECTION.where('queryDigest', '==', queryDigest)\n .orderBy('createdAt', 'desc')\n .limit(1)\n ))[0];\n if (cache) {\n const age = Date.now() - cache.createdAt.valueOf();\n const stale = cache.createdAt.valueOf() < (Date.now() - this.cacheValidMs);\n this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for search query \"${query.q}\", normalized digest: ${queryDigest}, ${age}ms old`, {\n query, digest: queryDigest, age, stale\n });\n\n if (!stale) {\n return cache.response as any;\n }\n }\n }\n\n try {\n let r: any[] | undefined;\n let lastError;\n outerLoop:\n for (const client of this.iterProviders(provider, variant)) {\n const t0 = Date.now();\n try {\n switch (variant) {\n case 'images': {\n r = await Reflect.apply(client.imageSearch, client, [query]);\n break;\n }\n case 'news': {\n r = await Reflect.apply(client.newsSearch, client, [query]);\n break;\n }\n case 'web':\n default: {\n r = await Reflect.apply(client.webSearch, client, [query]);\n break;\n }\n }\n const dt = Date.now() - t0;\n this.logger.info(`Search took ${dt}ms, ${client.constructor.name}(${variant})`, { searchDt: dt, variant, client: client.constructor.name });\n break outerLoop;\n } catch (err) {\n lastError = err;\n const dt = Date.now() - t0;\n this.logger.warn(`Failed to do ${variant} search using ${client.constructor.name}`, { err, variant, searchDt: dt, });\n }\n }\n\n if (r?.length) {\n const nowDate = new Date();\n const record = SERPResult.from({\n query,\n queryDigest,\n response: r,\n createdAt: nowDate,\n expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs)\n });\n\n this.batchedCaches.push(record);\n } else if (lastError) {\n throw lastError;\n }\n\n return r as WebSearchEntry[];\n } catch (err: any) {\n if (cache) {\n this.logger.warn(`Failed to fetch search result, but a stale cache is available. falling back to stale cache`, { err: marshalErrorLike(err) });\n\n return cache.response as any;\n }\n\n throw err;\n }\n }\n\n mapToFinalResults(input: WebSearchEntry) {\n const whitelistedProps = [\n 'imageUrl', 'imageWidth', 'imageHeight', 'source', 'date', 'siteLinks'\n ];\n const result = {\n title: input.title,\n url: input.link,\n description: Reflect.get(input, 'snippet'),\n ..._.pick(input, whitelistedProps),\n };\n\n return result;\n }\n\n async assignGeneralMixin(result: FormattedPage) {\n const collectFavicon = this.threadLocal.get('collect-favicon');\n\n if (collectFavicon && result.url) {\n const url = new URL(result.url);\n Reflect.set(result, 'favicon', await this.getFavicon(url.origin));\n }\n\n Object.setPrototypeOf(result, searchResultProto);\n }\n}\n\nconst dataItems = [\n { key: 'title', label: 'Title' },\n { key: 'source', label: 'Source' },\n { key: 'url', label: 'URL Source' },\n { key: 'imageUrl', label: 'Image URL' },\n { key: 'description', label: 'Description' },\n { key: 'publishedTime', label: 'Published Time' },\n { key: 'imageWidth', label: 'Image Width' },\n { key: 'imageHeight', label: 'Image Height' },\n { key: 'date', label: 'Date' },\n { key: 'favicon', label: 'Favicon' },\n];\n\nconst searchResultProto = {\n toString(this: FormattedPage, i?: number) {\n const chunks = [];\n for (const item of dataItems) {\n const v = Reflect.get(this, item.key);\n if (typeof v !== 'undefined') {\n if (i === undefined) {\n chunks.push(`[${item.label}]: ${v}`);\n } else {\n chunks.push(`[${i + 1}] ${item.label}: ${v}`);\n }\n }\n }\n\n if (this.content) {\n chunks.push(`\\n${this.content}`);\n }\n\n if (this.images) {\n const imageSummaryChunks = [`${i === undefined ? '' : `[${i + 1}] `}Images:`];\n for (const [k, v] of Object.entries(this.images)) {\n imageSummaryChunks.push(`- ![${k}](${v})`);\n }\n if (imageSummaryChunks.length === 1) {\n imageSummaryChunks.push('This page does not seem to contain any images.');\n }\n chunks.push(imageSummaryChunks.join('\\n'));\n }\n if (this.links) {\n const linkSummaryChunks = [`${i === undefined ? '' : `[${i + 1}] `}Links/Buttons:`];\n if (Array.isArray(this.links)) {\n for (const [k, v] of this.links) {\n linkSummaryChunks.push(`- [${k}](${v})`);\n }\n } else {\n for (const [k, v] of Object.entries(this.links)) {\n linkSummaryChunks.push(`- [${k}](${v})`);\n }\n }\n if (linkSummaryChunks.length === 1) {\n linkSummaryChunks.push('This page does not seem to contain any buttons/links.');\n }\n chunks.push(linkSummaryChunks.join('\\n'));\n }\n\n return chunks.join('\\n');\n }\n};\n" }, { "path": "src/api/serp.ts", "content": "import { singleton } from 'tsyringe';\nimport {\n RPCHost, RPCReflection, assignMeta, RawString,\n ParamValidationError,\n assignTransferProtocolMeta,\n} from 'civkit/civ-rpc';\nimport { marshalErrorLike } from 'civkit/lang';\nimport _ from 'lodash';\n\nimport { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit';\n\nimport { GlobalLogger } from '../services/logger';\nimport { AsyncLocalContext } from '../services/async-context';\nimport { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';\nimport { OutputServerEventStream } from '../lib/transform-server-event-stream';\nimport { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';\nimport { InsufficientBalanceError } from '../services/errors';\nimport { WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search';\nimport { GoogleSERP } from '../services/serp/google';\nimport { WebSearchEntry } from '../services/serp/compat';\nimport { CrawlerOptions } from '../dto/crawler-options';\nimport { ScrappingOptions } from '../services/serp/puppeteer';\nimport { objHashMd5B64Of } from 'civkit/hash';\nimport { SERPResult } from '../db/searched';\nimport { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper';\nimport type { JinaEmbeddingsTokenAccount } from '../shared/db/jina-embeddings-token-account';\nimport { LRUCache } from 'lru-cache';\nimport { API_CALL_STATUS } from '../shared/db/api-roll';\nimport { InternalJinaSerpService } from '../services/serp/internal';\n\nconst WORLD_COUNTRY_CODES = Object.keys(WORLD_COUNTRIES).map((x) => x.toLowerCase());\n\ntype RateLimitCache = {\n blockedUntil?: Date;\n user?: JinaEmbeddingsTokenAccount;\n};\n\nconst indexProto = {\n toString: function (): string {\n return _(this)\n .toPairs()\n .map(([k, v]) => k ? `[${_.upperFirst(_.lowerCase(k))}] ${v}` : '')\n .value()\n .join('\\n') + '\\n';\n }\n};\n\n@singleton()\nexport class SerpHost extends RPCHost {\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n cacheRetentionMs = 1000 * 3600 * 24 * 7;\n cacheValidMs = 1000 * 3600;\n pageCacheToleranceMs = 1000 * 3600 * 24;\n\n reasonableDelayMs = 15_000;\n\n targetResultCount = 5;\n\n highFreqKeyCache = new LRUCache({\n max: 256,\n ttl: 60 * 60 * 1000,\n updateAgeOnGet: false,\n updateAgeOnHas: false,\n });\n\n batchedCaches: SERPResult[] = [];\n\n async getIndex(ctx: Context, auth?: JinaEmbeddingsAuthDTO) {\n const indexObject: Record = Object.create(indexProto);\n Object.assign(indexObject, {\n usage1: 'https://r.jina.ai/YOUR_URL',\n usage2: 'https://s.jina.ai/YOUR_SEARCH_QUERY',\n usage3: `${ctx.origin}/?q=YOUR_SEARCH_QUERY`,\n homepage: 'https://jina.ai/reader',\n });\n\n if (auth && auth.user) {\n indexObject[''] = undefined;\n indexObject.authenticatedAs = `${auth.user.user_id} (${auth.user.full_name})`;\n indexObject.balanceLeft = auth.user.wallet.total_balance;\n } else {\n indexObject.note = 'Authentication is required to use this endpoint. Please provide a valid API key via Authorization header.';\n }\n\n return indexObject;\n }\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected rateLimitControl: RateLimitControl,\n protected threadLocal: AsyncLocalContext,\n protected googleSerp: GoogleSERP,\n protected serperGoogle: SerperGoogleSearchService,\n protected serperBing: SerperBingSearchService,\n protected jinaSerp: InternalJinaSerpService,\n ) {\n super(...arguments);\n\n setInterval(() => {\n const thisBatch = this.batchedCaches;\n this.batchedCaches = [];\n if (!thisBatch.length) {\n return;\n }\n const batch = SERPResult.DB.batch();\n\n for (const x of thisBatch) {\n batch.set(SERPResult.COLLECTION.doc(), x.degradeForFireStore());\n }\n batch.commit()\n .then(() => {\n this.logger.debug(`Saved ${thisBatch.length} caches by batch`);\n })\n .catch((err) => {\n this.logger.warn(`Failed to cache search result in batch`, { err });\n });\n }, 1000 * 10 + Math.round(1000 * Math.random())).unref();\n }\n\n override async init() {\n await this.dependencyReady();\n\n this.emit('ready');\n }\n\n @Method({\n name: 'searchIndex',\n ext: {\n http: {\n action: ['get', 'post'],\n path: '/'\n }\n },\n tags: ['search'],\n returnType: [String, OutputServerEventStream, RawString],\n })\n @Method({\n ext: {\n http: {\n action: ['get', 'post'],\n }\n },\n tags: ['search'],\n returnType: [String, OutputServerEventStream, RawString],\n })\n async search(\n @RPCReflect() rpcReflect: RPCReflection,\n @Ctx() ctx: Context,\n crawlerOptions: CrawlerOptions,\n auth: JinaEmbeddingsAuthDTO,\n @Param('type', { type: new Set(['web', 'images', 'news']), default: 'web' })\n variant: 'web' | 'images' | 'news',\n @Param('q') q?: string,\n @Param('provider', { type: new Set(['google', 'bing']) })\n searchEngine?: 'google' | 'bing',\n @Param('num', { validate: (v: number) => v >= 0 && v <= 20 })\n num?: number,\n @Param('gl', { validate: (v: string) => WORLD_COUNTRY_CODES.includes(v?.toLowerCase()) }) gl?: string,\n @Param('hl', { validate: (v: string) => WORLD_LANGUAGES.some(l => l.code === v) }) _hl?: string,\n @Param('location') location?: string,\n @Param('page') page?: number,\n @Param('fallback') fallback?: boolean,\n ) {\n const authToken = auth.bearerToken;\n let highFreqKey: RateLimitCache | undefined;\n if (authToken && this.highFreqKeyCache.has(authToken)) {\n highFreqKey = this.highFreqKeyCache.get(authToken)!;\n auth.user = highFreqKey.user;\n auth.uid = highFreqKey.user?.user_id;\n }\n\n const uid = await auth.solveUID();\n if (!q) {\n if (ctx.path === '/') {\n const indexObject = await this.getIndex(ctx, auth);\n if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {\n return indexObject;\n }\n\n return assignTransferProtocolMeta(`${indexObject}`,\n { contentType: 'text/plain; charset=utf-8', envelope: null }\n );\n }\n throw new ParamValidationError({\n path: 'q',\n message: `Required but not provided`\n });\n }\n // Return content by default\n const user = await auth.assertUser();\n if (!(user.wallet.total_balance > 0)) {\n throw new InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`);\n }\n\n if (highFreqKey?.blockedUntil) {\n const now = new Date();\n const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf());\n if (blockedTimeRemaining > 0) {\n this.logger.warn(`Rate limit triggered for ${uid}, this request should have been blocked`);\n // throw RateLimitTriggeredError.from({\n // message: `Per UID rate limit exceeded (async)`,\n // retryAfter: Math.ceil(blockedTimeRemaining / 1000),\n // });\n }\n }\n\n const PREMIUM_KEY_LIMIT = 400;\n const rateLimitPolicy = auth.getRateLimits('SEARCH') || [\n parseInt(user.metadata?.speed_level) >= 2 ?\n RateLimitDesc.from({\n occurrence: PREMIUM_KEY_LIMIT,\n periodSeconds: 60\n }) :\n RateLimitDesc.from({\n occurrence: 40,\n periodSeconds: 60\n })\n ];\n\n const apiRollPromise = this.rateLimitControl.simpleRPCUidBasedLimit(\n rpcReflect, uid!, ['SEARCH'],\n ...rateLimitPolicy\n );\n\n if (!highFreqKey) {\n // Normal path\n await apiRollPromise;\n\n if (rateLimitPolicy.some(\n (x) => {\n const rpm = x.occurrence / (x.periodSeconds / 60);\n if (rpm >= PREMIUM_KEY_LIMIT) {\n return true;\n }\n\n return false;\n })\n ) {\n this.highFreqKeyCache.set(auth.bearerToken!, {\n user,\n });\n }\n } else {\n // High freq key path\n apiRollPromise.then(\n // Rate limit not triggered, make sure not blocking.\n () => {\n delete highFreqKey.blockedUntil;\n },\n // Rate limit triggered\n (err) => {\n if (!(err instanceof RateLimitTriggeredError)) {\n return;\n }\n const now = Date.now();\n let tgtDate;\n if (err.retryAfterDate) {\n tgtDate = err.retryAfterDate;\n } else if (err.retryAfter) {\n tgtDate = new Date(now + err.retryAfter * 1000);\n }\n\n if (tgtDate) {\n const dt = tgtDate.valueOf() - now;\n highFreqKey.blockedUntil = tgtDate;\n setTimeout(() => {\n if (highFreqKey.blockedUntil === tgtDate) {\n delete highFreqKey.blockedUntil;\n }\n }, dt).unref();\n }\n }\n ).finally(async () => {\n // Always asynchronously update user(wallet);\n const user = await auth.getBrief().catch(() => undefined);\n if (user) {\n highFreqKey.user = user;\n }\n });\n }\n\n let chargeAmount = 0;\n rpcReflect.finally(async () => {\n if (chargeAmount) {\n auth.reportUsage(chargeAmount, `reader-search`).catch((err) => {\n this.logger.warn(`Unable to report usage for ${uid}`, { err: marshalErrorLike(err) });\n });\n try {\n const apiRoll = await apiRollPromise;\n apiRoll.chargeAmount = chargeAmount;\n } catch (err) {\n await this.rateLimitControl.record({\n uid,\n tags: [rpcReflect.name.toUpperCase()],\n status: API_CALL_STATUS.SUCCESS,\n chargeAmount,\n }).save().catch((err) => {\n this.logger.warn(`Failed to save rate limit record`, { err: marshalErrorLike(err) });\n });\n }\n }\n });\n\n let chargeAmountScaler = 1;\n if (searchEngine === 'bing') {\n chargeAmountScaler = 3;\n }\n if (variant !== 'web') {\n chargeAmountScaler = 5;\n }\n\n let realQuery = q;\n let queryTerms = q.split(/\\s+/g).filter((x) => !!x);\n\n let results = await this.cachedSearch(variant, {\n provider: searchEngine,\n q,\n num,\n gl,\n // hl,\n location,\n page,\n }, crawlerOptions);\n\n\n if (fallback && !results?.length && (!page || page === 1)) {\n let tryTimes = 1;\n const containsRTL = /[\\u0600-\\u06FF\\u0750-\\u077F\\u08A0-\\u08FF\\uFB50-\\uFDFF\\uFE70-\\uFEFF\\u0590-\\u05FF\\uFB1D-\\uFB4F\\u0700-\\u074F\\u0780-\\u07BF\\u07C0-\\u07FF]/.test(q);\n const lastResort = (containsRTL ? queryTerms.slice(queryTerms.length - 2) : queryTerms.slice(0, 2)).join(' ');\n const n = 4;\n let terms: string[] = [];\n while (tryTimes < n) {\n const delta = Math.ceil(queryTerms.length / n) * tryTimes;\n terms = containsRTL ? queryTerms.slice(delta) : queryTerms.slice(0, queryTerms.length - delta);\n const query = terms.join(' ');\n if (!query) {\n break;\n }\n if (realQuery === query) {\n continue;\n }\n tryTimes += 1;\n realQuery = query;\n this.logger.info(`Retrying search with fallback query: \"${realQuery}\"`);\n results = await this.cachedSearch(variant, {\n provider: searchEngine,\n q: realQuery,\n num,\n gl,\n // hl,\n location,\n }, crawlerOptions);\n if (results?.length) {\n break;\n }\n }\n\n if (!results?.length && realQuery.length > lastResort.length) {\n realQuery = lastResort;\n this.logger.info(`Retrying search with fallback query: \"${realQuery}\"`);\n tryTimes += 1;\n results = await this.cachedSearch(variant, {\n provider: searchEngine,\n q: realQuery,\n num,\n gl,\n // hl,\n location,\n }, crawlerOptions);\n }\n\n chargeAmountScaler *= tryTimes;\n }\n\n if (!results?.length) {\n results = [];\n }\n\n const finalResults = results.map((x: any) => this.mapToFinalResults(x));\n\n await Promise.all(finalResults.map((x: any) => this.assignGeneralMixin(x)));\n\n chargeAmount = this.assignChargeAmount(finalResults, chargeAmountScaler);\n assignMeta(finalResults, {\n query: realQuery,\n fallback: realQuery === q ? undefined : realQuery,\n });\n\n return finalResults;\n }\n\n\n assignChargeAmount(items: unknown[], scaler: number) {\n const numCharge = Math.ceil(items.length / 10) * 10000 * scaler;\n assignMeta(items, { usage: { tokens: numCharge } });\n\n return numCharge;\n }\n\n async getFavicon(domain: string) {\n const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`;\n\n try {\n const response = await fetch(url);\n if (!response.ok) {\n return '';\n }\n const ab = await response.arrayBuffer();\n const buffer = Buffer.from(ab);\n const base64 = buffer.toString('base64');\n return `data:image/png;base64,${base64}`;\n } catch (error: any) {\n this.logger.warn(`Failed to get favicon base64 string`, { err: marshalErrorLike(error) });\n return '';\n }\n }\n\n async configure(opts: CrawlerOptions) {\n const crawlOpts: ScrappingOptions = {\n proxyUrl: opts.proxyUrl,\n cookies: opts.setCookies,\n overrideUserAgent: opts.userAgent,\n timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined,\n locale: opts.locale,\n referer: opts.referer,\n viewport: opts.viewport,\n proxyResources: (opts.proxyUrl || opts.proxy?.endsWith('+')) ? true : false,\n allocProxy: opts.proxy?.endsWith('+') ? opts.proxy.slice(0, -1) : opts.proxy,\n };\n\n if (opts.locale) {\n crawlOpts.extraHeaders ??= {};\n crawlOpts.extraHeaders['Accept-Language'] = opts.locale;\n }\n\n return crawlOpts;\n }\n\n mapToFinalResults(input: WebSearchEntry) {\n const whitelistedProps = [\n 'imageUrl', 'imageWidth', 'imageHeight', 'source', 'date', 'siteLinks'\n ];\n const result = {\n title: input.title,\n url: input.link,\n description: Reflect.get(input, 'snippet'),\n ..._.pick(input, whitelistedProps),\n };\n\n return result;\n }\n\n *iterProviders(preference?: string, variant?: string) {\n if (preference === 'bing') {\n yield this.serperBing;\n yield this.serperGoogle;\n yield this.googleSerp;\n\n return;\n }\n\n if (preference === 'google') {\n yield this.googleSerp;\n yield this.googleSerp;\n yield this.serperGoogle;\n\n return;\n }\n\n // yield variant === 'web' ? this.jinaSerp : this.serperGoogle;\n yield this.serperGoogle\n yield this.serperGoogle;\n yield this.googleSerp;\n }\n\n async cachedSearch(variant: 'web' | 'news' | 'images', query: Record, opts: CrawlerOptions) {\n const queryDigest = objHashMd5B64Of({ ...query, variant });\n const provider = query.provider;\n Reflect.deleteProperty(query, 'provider');\n const noCache = opts.noCache;\n let cache;\n if (!noCache) {\n cache = (await SERPResult.fromFirestoreQuery(\n SERPResult.COLLECTION.where('queryDigest', '==', queryDigest)\n .orderBy('createdAt', 'desc')\n .limit(1)\n ))[0];\n if (cache) {\n const age = Date.now() - cache.createdAt.valueOf();\n const stale = cache.createdAt.valueOf() < (Date.now() - this.cacheValidMs);\n this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for search query \"${query.q}\", normalized digest: ${queryDigest}, ${age}ms old`, {\n query, digest: queryDigest, age, stale\n });\n\n if (!stale) {\n return cache.response as any;\n }\n }\n }\n const scrappingOptions = await this.configure(opts);\n\n try {\n let r: any[] | undefined;\n let lastError;\n outerLoop:\n for (const client of this.iterProviders(provider, variant)) {\n const t0 = Date.now();\n try {\n switch (variant) {\n case 'images': {\n r = await Reflect.apply(client.imageSearch, client, [query, scrappingOptions]);\n break;\n }\n case 'news': {\n r = await Reflect.apply(client.newsSearch, client, [query, scrappingOptions]);\n break;\n }\n case 'web':\n default: {\n r = await Reflect.apply(client.webSearch, client, [query, scrappingOptions]);\n break;\n }\n }\n const dt = Date.now() - t0;\n this.logger.info(`Search took ${dt}ms, ${client.constructor.name}(${variant})`, { searchDt: dt, variant, client: client.constructor.name });\n break outerLoop;\n } catch (err) {\n lastError = err;\n const dt = Date.now() - t0;\n this.logger.warn(`Failed to do ${variant} search using ${client.constructor.name}`, { err, variant, searchDt: dt, });\n }\n }\n\n if (r?.length) {\n const nowDate = new Date();\n const record = SERPResult.from({\n query,\n queryDigest,\n response: r,\n createdAt: nowDate,\n expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs)\n });\n this.batchedCaches.push(record);\n } else if (lastError) {\n throw lastError;\n }\n\n return r;\n } catch (err: any) {\n if (cache) {\n this.logger.warn(`Failed to fetch search result, but a stale cache is available. falling back to stale cache`, { err: marshalErrorLike(err) });\n\n return cache.response as any;\n }\n\n throw err;\n }\n }\n\n async assignGeneralMixin(result: Partial) {\n const collectFavicon = this.threadLocal.get('collect-favicon');\n\n if (collectFavicon && result.link) {\n const url = new URL(result.link);\n Reflect.set(result, 'favicon', await this.getFavicon(url.origin));\n }\n }\n}\n" }, { "path": "src/cloud-functions/adaptive-crawler.ts", "content": "import {\n AssertionFailureError,\n assignTransferProtocolMeta,\n HashManager,\n ParamValidationError,\n RPCHost, RPCReflection,\n} from 'civkit';\nimport { singleton } from 'tsyringe';\nimport { CloudHTTPv2, CloudTaskV2, Ctx, FirebaseStorageBucketControl, Logger, Param, RPCReflect } from '../shared';\nimport _ from 'lodash';\nimport { Request, Response } from 'express';\nimport { JinaEmbeddingsAuthDTO } from '../shared/dto/jina-embeddings-auth';\nimport robotsParser from 'robots-parser';\nimport { DOMParser } from '@xmldom/xmldom';\n\nimport { AdaptiveCrawlerOptions } from '../dto/adaptive-crawler-options';\nimport { CrawlerOptions } from '../dto/crawler-options';\nimport { JinaEmbeddingsTokenAccount } from '../shared/db/jina-embeddings-token-account';\nimport { AdaptiveCrawlTask, AdaptiveCrawlTaskStatus } from '../db/adaptive-crawl-task';\nimport { getFunctions } from 'firebase-admin/functions';\nimport { getFunctionUrl } from '../utils/get-function-url';\nimport { Timestamp } from 'firebase-admin/firestore';\n\nconst md5Hasher = new HashManager('md5', 'hex');\nconst removeURLHash = (url: string) => {\n try {\n const o = new URL(url);\n o.hash = '';\n return o.toString();\n } catch (e) {\n return url;\n }\n}\n\n@singleton()\nexport class AdaptiveCrawlerHost extends RPCHost {\n logger = this.globalLogger.child({ service: this.constructor.name });\n // Actual cache storage (gcp buckets) exists for 7 days, so here we need to select a time < 7 days.\n cacheExpiry = 3 * 1000 * 60 * 60 * 24;\n\n static readonly __singleCrawlQueueName = 'singleCrawlQueue';\n\n constructor(\n protected globalLogger: Logger,\n protected firebaseObjectStorage: FirebaseStorageBucketControl,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n\n this.emit('ready');\n }\n\n @CloudHTTPv2({\n runtime: {\n memory: '1GiB',\n timeoutSeconds: 300,\n concurrency: 22,\n },\n tags: ['Crawler'],\n httpMethod: ['post', 'get'],\n returnType: [String],\n })\n async adaptiveCrawl(\n @RPCReflect() rpcReflect: RPCReflection,\n @Ctx() ctx: {\n req: Request,\n res: Response,\n },\n auth: JinaEmbeddingsAuthDTO,\n crawlerOptions: CrawlerOptions,\n adaptiveCrawlerOptions: AdaptiveCrawlerOptions,\n ) {\n this.logger.debug({\n adaptiveCrawlerOptions,\n crawlerOptions,\n });\n\n\n const uid = await auth.solveUID();\n const { useSitemap, maxPages } = adaptiveCrawlerOptions;\n\n let tmpUrl = ctx.req.url.slice(1)?.trim();\n if (!tmpUrl) {\n tmpUrl = crawlerOptions.url?.trim() ?? '';\n }\n const targetUrl = new URL(tmpUrl);\n\n if (!targetUrl) {\n const latestUser = uid ? await auth.assertUser() : undefined;\n if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {\n return this.getIndex(latestUser);\n }\n\n return assignTransferProtocolMeta(`${this.getIndex(latestUser)}`,\n { contentType: 'text/plain', envelope: null }\n );\n }\n\n const meta = {\n targetUrl: targetUrl.toString(),\n useSitemap,\n maxPages,\n };\n\n const digest = md5Hasher.hash(JSON.stringify(meta));\n const shortDigest = Buffer.from(digest, 'hex').toString('base64url');\n const existing = await AdaptiveCrawlTask.fromFirestore(shortDigest);\n\n if (existing?.createdAt) {\n if (existing.createdAt.getTime() > Date.now() - this.cacheExpiry) {\n this.logger.info(`Cache hit for ${shortDigest}, created at ${existing.createdAt.toDateString()}`);\n return { taskId: shortDigest };\n } else {\n this.logger.info(`Cache expired for ${shortDigest}, created at ${existing.createdAt.toDateString()}`);\n }\n }\n\n await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).set({\n _id: shortDigest,\n status: AdaptiveCrawlTaskStatus.PENDING,\n statusText: 'Pending',\n meta,\n createdAt: new Date(),\n urls: [],\n processed: {},\n failed: {},\n });\n\n let urls: string[] = [];\n if (useSitemap) {\n urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages);\n }\n\n if (urls.length > 0) {\n await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({\n status: AdaptiveCrawlTaskStatus.PROCESSING,\n statusText: `Processing 0/${urls.length}`,\n urls,\n });\n\n const promises = [];\n for (const url of urls) {\n promises.push(getFunctions().taskQueue(AdaptiveCrawlerHost.__singleCrawlQueueName).enqueue({\n shortDigest, url, token: auth.bearerToken, meta\n }, {\n dispatchDeadlineSeconds: 1800,\n uri: await getFunctionUrl(AdaptiveCrawlerHost.__singleCrawlQueueName),\n }));\n };\n\n await Promise.all(promises);\n } else {\n meta.useSitemap = false;\n\n await AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({\n urls: [targetUrl.toString()],\n });\n\n await getFunctions().taskQueue(AdaptiveCrawlerHost.__singleCrawlQueueName).enqueue({\n shortDigest, url: targetUrl.toString(), token: auth.bearerToken, meta\n }, {\n dispatchDeadlineSeconds: 1800,\n uri: await getFunctionUrl(AdaptiveCrawlerHost.__singleCrawlQueueName),\n })\n }\n\n return { taskId: shortDigest };\n }\n\n @CloudHTTPv2({\n runtime: {\n memory: '1GiB',\n timeoutSeconds: 300,\n concurrency: 22,\n },\n tags: ['Crawler'],\n httpMethod: ['post', 'get'],\n returnType: AdaptiveCrawlTask,\n })\n async adaptiveCrawlStatus(\n @RPCReflect() rpcReflect: RPCReflection,\n @Ctx() ctx: {\n req: Request,\n res: Response,\n },\n auth: JinaEmbeddingsAuthDTO,\n @Param('taskId') taskId: string,\n @Param('urls') urls: string[] = [],\n ) {\n if (!taskId) {\n throw new ParamValidationError('taskId is required');\n }\n\n const state = await AdaptiveCrawlTask.fromFirestore(taskId);\n\n if (!state) {\n throw new AssertionFailureError('The task does not exist');\n }\n\n if (state?.createdAt && state.createdAt.getTime() < Date.now() - this.cacheExpiry) {\n throw new AssertionFailureError('The task has expired');\n }\n\n if (urls.length) {\n const promises = Object.entries(state?.processed ?? {}).map(async ([url, cachePath]) => {\n if (urls.includes(url)) {\n const raw = await this.firebaseObjectStorage.downloadFile(cachePath);\n state!.processed[url] = JSON.parse(raw.toString('utf-8'));\n }\n });\n\n await Promise.all(promises);\n }\n\n\n return state;\n }\n\n @CloudTaskV2({\n name: AdaptiveCrawlerHost.__singleCrawlQueueName,\n runtime: {\n cpu: 1,\n memory: '1GiB',\n timeoutSeconds: 3600,\n concurrency: 2,\n maxInstances: 200,\n retryConfig: {\n maxAttempts: 3,\n minBackoffSeconds: 60,\n },\n rateLimits: {\n maxConcurrentDispatches: 150,\n maxDispatchesPerSecond: 5,\n },\n }\n })\n async singleCrawlQueue(\n @Param('shortDigest') shortDigest: string,\n @Param('url') url: string,\n @Param('token') token: string,\n @Param('meta') meta: AdaptiveCrawlTask['meta'],\n ) {\n const error = {\n reason: ''\n };\n\n const state = await AdaptiveCrawlTask.fromFirestore(shortDigest);\n if (state?.status === AdaptiveCrawlTaskStatus.COMPLETED) {\n return;\n }\n\n try {\n url = removeURLHash(url);\n } catch(e) {\n error.reason = `Failed to parse url: ${url}`;\n }\n\n this.logger.debug(shortDigest, url, meta);\n const cachePath = `adaptive-crawl-task/${shortDigest}/${md5Hasher.hash(url)}`;\n\n if (!error.reason) {\n const result = meta.useSitemap\n ? await this.handleSingleCrawl(shortDigest, url, token, cachePath)\n : await this.handleSingleCrawlRecursively(shortDigest, url, token, meta, cachePath);\n\n if (!result) {\n return;\n }\n\n error.reason = result.error.reason;\n }\n\n await AdaptiveCrawlTask.DB.runTransaction(async (transaction) => {\n const ref = AdaptiveCrawlTask.COLLECTION.doc(shortDigest);\n const state = await transaction.get(ref);\n const data = state.data() as AdaptiveCrawlTask & { createdAt: Timestamp };\n\n if (error.reason) {\n data.failed[url] = error;\n } else {\n data.processed[url] = cachePath;\n }\n\n const status = Object.keys(data.processed).length + Object.keys(data.failed).length >= data.urls.length\n ? AdaptiveCrawlTaskStatus.COMPLETED : AdaptiveCrawlTaskStatus.PROCESSING;\n const statusText = Object.keys(data.processed).length + Object.keys(data.failed).length >= data.urls.length\n ? `Completed ${Object.keys(data.processed).length} Succeeded, ${Object.keys(data.failed).length} Failed`\n : `Processing ${Object.keys(data.processed).length + Object.keys(data.failed).length}/${data.urls.length}`;\n\n const payload: Partial = {\n status,\n statusText,\n processed: data.processed,\n failed: data.failed,\n };\n\n if (status === AdaptiveCrawlTaskStatus.COMPLETED) {\n payload.finishedAt = new Date();\n payload.duration = new Date().getTime() - data.createdAt.toDate().getTime();\n }\n\n transaction.update(ref, payload);\n });\n }\n\n async handleSingleCrawl(shortDigest: string, url: string, token: string, cachePath: string) {\n const error = {\n reason: ''\n }\n\n const response = await fetch('https://r.jina.ai', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': `Bearer ${token}`,\n 'Accept': 'application/json',\n },\n body: JSON.stringify({ url })\n })\n\n if (!response.ok) {\n error.reason = `Failed to crawl ${url}, ${response.statusText}`;\n } else {\n const json = await response.json();\n\n await this.firebaseObjectStorage.saveFile(cachePath,\n Buffer.from(\n JSON.stringify(json),\n 'utf-8'\n ),\n {\n metadata: {\n contentType: 'application/json',\n }\n }\n )\n }\n\n return {\n error,\n }\n }\n\n async handleSingleCrawlRecursively(\n shortDigest: string, url: string, token: string, meta: AdaptiveCrawlTask['meta'], cachePath: string\n ) {\n const error = {\n reason: ''\n }\n const response = await fetch('https://r.jina.ai', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': `Bearer ${token}`,\n 'Accept': 'application/json',\n 'X-With-Links-Summary': 'true',\n },\n body: JSON.stringify({ url })\n });\n\n if (!response.ok) {\n error.reason = `Failed to crawl ${url}, ${response.statusText}`;\n } else {\n const json = await response.json();\n await this.firebaseObjectStorage.saveFile(cachePath,\n Buffer.from(\n JSON.stringify(json),\n 'utf-8'\n ),\n {\n metadata: {\n contentType: 'application/json',\n }\n }\n )\n\n const title = json.data.title;\n const description = json.data.description;\n const links = json.data.links as Record;\n\n const relevantUrls = await this.getRelevantUrls(token, { title, description, links });\n this.logger.debug(`Total urls: ${Object.keys(links).length}, relevant urls: ${relevantUrls.length}`);\n\n for (const url of relevantUrls) {\n let abortContinue = false;\n let abortBreak = false;\n await AdaptiveCrawlTask.DB.runTransaction(async (transaction) => {\n const ref = AdaptiveCrawlTask.COLLECTION.doc(shortDigest);\n const state = await transaction.get(ref);\n const data = state.data() as AdaptiveCrawlTask & { createdAt: Timestamp };\n\n if (data.urls.includes(url)) {\n this.logger.debug('Recursive CONTINUE', data);\n abortContinue = true;\n return;\n }\n\n const urls = [\n ...data.urls,\n url\n ];\n\n if (urls.length > meta.maxPages || data.status === AdaptiveCrawlTaskStatus.COMPLETED) {\n this.logger.debug('Recursive BREAK', data);\n abortBreak = true;\n return;\n }\n\n transaction.update(ref, { urls });\n });\n\n if (abortContinue) {\n continue;\n }\n if (abortBreak) {\n break;\n }\n\n await getFunctions().taskQueue(AdaptiveCrawlerHost.__singleCrawlQueueName).enqueue({\n shortDigest, url, token, meta\n }, {\n dispatchDeadlineSeconds: 1800,\n uri: await getFunctionUrl(AdaptiveCrawlerHost.__singleCrawlQueueName),\n });\n };\n }\n\n return {\n error,\n }\n }\n\n async getRelevantUrls(token: string, {\n title, description, links\n }: {\n title: string;\n description: string;\n links: Record;\n }) {\n const invalidSuffix = [\n '.zip',\n '.docx',\n '.pptx',\n '.xlsx',\n ];\n\n const validLinks = Object.entries(links)\n .map(([title, link]) => link)\n .filter(link => link.startsWith('http') && !invalidSuffix.some(suffix => link.endsWith(suffix)));\n\n let query = '';\n if (!description) {\n query += title;\n } else {\n query += `TITLE: ${title}; DESCRIPTION: ${description}`;\n }\n\n const data = {\n model: 'jina-reranker-v2-base-multilingual',\n query,\n top_n: 15,\n documents: validLinks,\n };\n\n const response = await fetch('https://api.jina.ai/v1/rerank', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Authorization': `Bearer ${token}`\n },\n body: JSON.stringify(data)\n });\n\n const json = (await response.json()) as {\n results: {\n index: number;\n document: {\n text: string;\n };\n relevance_score: number;\n }[];\n };\n\n const highestRelevanceScore = json.results[0]?.relevance_score ?? 0;\n return json.results.filter(r => r.relevance_score > Math.max(highestRelevanceScore * 0.6, 0.1)).map(r => removeURLHash(r.document.text));\n }\n\n getIndex(user?: JinaEmbeddingsTokenAccount) {\n // TODO: 需要更新使用方式\n // const indexObject: Record = Object.create(indexProto);\n\n // Object.assign(indexObject, {\n // usage1: 'https://r.jina.ai/YOUR_URL',\n // usage2: 'https://s.jina.ai/YOUR_SEARCH_QUERY',\n // homepage: 'https://jina.ai/reader',\n // sourceCode: 'https://github.com/jina-ai/reader',\n // });\n\n // if (user) {\n // indexObject[''] = undefined;\n // indexObject.authenticatedAs = `${user.user_id} (${user.full_name})`;\n // indexObject.balanceLeft = user.wallet.total_balance;\n // }\n\n // return indexObject;\n }\n\n async crawlUrlsFromSitemap(url: URL, maxPages: number) {\n const sitemapsFromRobotsTxt = await this.getSitemapsFromRobotsTxt(url);\n\n const initialSitemaps: string[] = [];\n if (sitemapsFromRobotsTxt === null) {\n initialSitemaps.push(`${url.origin}/sitemap.xml`);\n } else {\n initialSitemaps.push(...sitemapsFromRobotsTxt);\n }\n\n\n const allUrls: Set = new Set();\n const processedSitemaps: Set = new Set();\n\n const fetchSitemapUrls = async (sitemapUrl: string) => {\n sitemapUrl = sitemapUrl.trim();\n\n if (processedSitemaps.has(sitemapUrl)) {\n return;\n }\n\n processedSitemaps.add(sitemapUrl);\n\n try {\n const response = await fetch(sitemapUrl);\n const sitemapContent = await response.text();\n const parser = new DOMParser();\n const xmlDoc = parser.parseFromString(sitemapContent, 'text/xml');\n\n // handle normal sitemap\n const urlElements = xmlDoc.getElementsByTagName('url');\n for (let i = 0; i < urlElements.length; i++) {\n const locElement = urlElements[i].getElementsByTagName('loc')[0];\n if (locElement) {\n const loc = locElement.textContent?.trim() || '';\n if (loc.startsWith(url.origin) && !loc.endsWith('.xml')) {\n allUrls.add(removeURLHash(loc));\n }\n if (allUrls.size >= maxPages) {\n return;\n }\n }\n }\n\n // handle sitemap index\n const sitemapElements = xmlDoc.getElementsByTagName('sitemap');\n for (let i = 0; i < sitemapElements.length; i++) {\n const locElement = sitemapElements[i].getElementsByTagName('loc')[0];\n if (locElement) {\n await fetchSitemapUrls(locElement.textContent?.trim() || '');\n if (allUrls.size >= maxPages) {\n return;\n }\n }\n }\n } catch (error) {\n this.logger.error(`Error fetching sitemap ${sitemapUrl}:`, error);\n }\n };\n\n for (const sitemapUrl of initialSitemaps) {\n await fetchSitemapUrls(sitemapUrl);\n if (allUrls.size >= maxPages) {\n break;\n }\n }\n\n const urlsToProcess = Array.from(allUrls).slice(0, maxPages);\n\n return urlsToProcess;\n }\n\n async getSitemapsFromRobotsTxt(url: URL) {\n const hostname = url.origin;\n const robotsUrl = `${hostname}/robots.txt`;\n const response = await fetch(robotsUrl);\n if (response.status === 404) {\n return null;\n }\n const robotsTxt = await response.text();\n if (robotsTxt.length) {\n const robot = robotsParser(robotsUrl, robotsTxt);\n return robot.getSitemaps();\n }\n\n return null;\n }\n}\n" }, { "path": "src/cloud-functions/data-crunching.ts", "content": "import {\n Defer,\n PromiseThrottle,\n RPCHost,\n} from 'civkit';\nimport { singleton } from 'tsyringe';\nimport {\n // CloudScheduleV2, CloudTaskV2,\n FirebaseStorageBucketControl, Logger, Param, TempFileManager\n} from '../shared';\nimport _ from 'lodash';\nimport { CrawlerHost } from '../api/crawler';\n\nimport { Crawled } from '../db/crawled';\nimport dayjs from 'dayjs';\nimport { createReadStream } from 'fs';\nimport { appendFile } from 'fs/promises';\nimport { createGzip } from 'zlib';\nimport { getFunctions } from 'firebase-admin/functions';\nimport { SnapshotFormatter } from '../services/snapshot-formatter';\nimport { getFunctionUrl } from '../utils/get-function-url';\n\ndayjs.extend(require('dayjs/plugin/utc'));\n\n@singleton()\nexport class DataCrunchingHost extends RPCHost {\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n pageCacheCrunchingPrefix = 'crunched-pages';\n pageCacheCrunchingBatchSize = 5000;\n pageCacheCrunchingTMinus = 6 * 24 * 60 * 60 * 1000;\n rev = 7;\n\n constructor(\n protected globalLogger: Logger,\n\n protected crawler: CrawlerHost,\n protected snapshotFormatter: SnapshotFormatter,\n protected tempFileManager: TempFileManager,\n protected firebaseObjectStorage: FirebaseStorageBucketControl,\n ) {\n super(..._.without(arguments, crawler));\n }\n\n override async init() {\n await this.dependencyReady();\n\n this.emit('ready');\n }\n\n // @CloudTaskV2({\n // runtime: {\n // cpu: 2,\n // memory: '4GiB',\n // timeoutSeconds: 3600,\n // concurrency: 2,\n // maxInstances: 200,\n // retryConfig: {\n // maxAttempts: 3,\n // minBackoffSeconds: 60,\n // },\n // rateLimits: {\n // maxConcurrentDispatches: 150,\n // maxDispatchesPerSecond: 2,\n // },\n // },\n // tags: ['DataCrunching'],\n // })\n async crunchPageCacheWorker(\n @Param('date') date: string,\n @Param('offset', { default: 0 }) offset: number\n ) {\n this.logger.info(`Crunching page cache @${date}+${offset}...`);\n for await (const { fileName, records } of this.iterPageCacheRecords(date, offset)) {\n this.logger.info(`Crunching ${fileName}...`);\n const fileOnDrive = await this.crunchCacheRecords(records);\n const fstream = createReadStream(fileOnDrive.path);\n const gzipStream = createGzip();\n fstream.pipe(gzipStream, { end: true });\n await this.firebaseObjectStorage.bucket.file(fileName).save(gzipStream, {\n contentType: 'application/jsonl+gzip',\n });\n }\n\n this.logger.info(`Crunching page cache @${date}+${offset} done.`);\n\n return true;\n }\n\n // @CloudScheduleV2('2 0 * * *', {\n // name: 'crunchPageCacheEveryday',\n // runtime: {\n // cpu: 2,\n // memory: '4GiB',\n // timeoutSeconds: 1800,\n // timeZone: 'UTC',\n // retryCount: 3,\n // minBackoffSeconds: 60,\n // },\n // tags: ['DataCrunching'],\n // })\n async dispatchPageCacheCrunching() {\n for await (const { fileName, date, offset } of this.iterPageCacheChunks()) {\n this.logger.info(`Dispatching ${fileName}...`);\n // sse.write({ data: `Dispatching ${fileName}...` });\n\n await getFunctions().taskQueue('crunchPageCacheWorker').enqueue({ date, offset }, {\n dispatchDeadlineSeconds: 1800,\n uri: await getFunctionUrl('crunchPageCacheWorker'),\n });\n }\n\n return true;\n }\n\n // @CloudHTTPv2({\n // runtime: {\n // cpu: 2,\n // memory: '4GiB',\n // timeoutSeconds: 3600,\n // concurrency: 2,\n // maxInstances: 200,\n // },\n // tags: ['DataCrunching'],\n // })\n // async dispatchPageCacheCrunching(\n // @RPCReflect() rpcReflect: RPCReflection\n // ) {\n // const sse = new OutputServerEventStream({ highWaterMark: 4096 });\n // rpcReflect.return(sse);\n // rpcReflect.catch((err) => {\n // sse.end({ data: `Error: ${err.message}` });\n // });\n // for await (const { fileName, date, offset } of this.iterPageCacheChunks()) {\n // this.logger.info(`Dispatching ${fileName}...`);\n // sse.write({ data: `Dispatching ${fileName}...` });\n\n // await getFunctions().taskQueue('crunchPageCacheWorker').enqueue({ date, offset }, {\n // dispatchDeadlineSeconds: 1800,\n // uri: await getFunctionUrl('crunchPageCacheWorker'),\n // });\n // }\n\n // sse.end({ data: 'done' });\n\n // return true;\n // }\n\n async* iterPageCacheRecords(date?: string, inputOffset?: number | string) {\n const startOfToday = dayjs().utc().startOf('day');\n const startingPoint = dayjs().utc().subtract(this.pageCacheCrunchingTMinus, 'ms').startOf('day');\n let theDay = startingPoint;\n\n if (date) {\n theDay = dayjs(date).utc().startOf('day');\n }\n\n let counter = 0;\n if (inputOffset) {\n counter = parseInt(inputOffset as string, 10);\n }\n\n while (theDay.isBefore(startOfToday)) {\n const fileName = `${this.pageCacheCrunchingPrefix}/r${this.rev}/${theDay.format('YYYY-MM-DD')}/${counter}.jsonl.gz`;\n const offset = counter;\n counter += this.pageCacheCrunchingBatchSize;\n const fileExists = (await this.firebaseObjectStorage.bucket.file(fileName).exists())[0];\n if (fileExists) {\n continue;\n }\n\n const records = await Crawled.fromFirestoreQuery(Crawled.COLLECTION\n .where('createdAt', '>=', theDay.toDate())\n .where('createdAt', '<', theDay.add(1, 'day').toDate())\n .orderBy('createdAt', 'asc')\n .offset(offset)\n .limit(this.pageCacheCrunchingBatchSize)\n );\n\n this.logger.info(`Found ${records.length} records for ${theDay.format('YYYY-MM-DD')} at offset ${offset}`, { fileName, counter });\n\n if (!records.length) {\n if (date) {\n break;\n }\n theDay = theDay.add(1, 'day');\n counter = 0;\n continue;\n }\n\n yield { fileName, records };\n\n if (offset) {\n break;\n }\n }\n }\n\n async* iterPageCacheChunks() {\n const startOfToday = dayjs().utc().startOf('day');\n const startingPoint = dayjs().utc().subtract(this.pageCacheCrunchingTMinus, 'ms').startOf('day');\n let theDay = startingPoint;\n\n let counter = 0;\n\n while (theDay.isBefore(startOfToday)) {\n const fileName = `${this.pageCacheCrunchingPrefix}/r${this.rev}/${theDay.format('YYYY-MM-DD')}/${counter}.jsonl.gz`;\n const offset = counter;\n counter += this.pageCacheCrunchingBatchSize;\n const fileExists = (await this.firebaseObjectStorage.bucket.file(fileName).exists())[0];\n if (fileExists) {\n continue;\n }\n\n const nRecords = (await Crawled.COLLECTION\n .where('createdAt', '>=', theDay.toDate())\n .where('createdAt', '<', theDay.add(1, 'day').toDate())\n .orderBy('createdAt', 'asc')\n .offset(offset)\n .limit(this.pageCacheCrunchingBatchSize)\n .count().get()).data().count;\n\n this.logger.info(`Found ${nRecords} records for ${theDay.format('YYYY-MM-DD')} at offset ${offset}`, { fileName, counter });\n if (nRecords < this.pageCacheCrunchingBatchSize) {\n theDay = theDay.add(1, 'day');\n counter = 0;\n }\n if (nRecords) {\n yield { fileName, date: theDay.toISOString(), offset };\n }\n }\n }\n\n async crunchCacheRecords(records: Crawled[]) {\n const throttle = new PromiseThrottle(30);\n const localFilePath = this.tempFileManager.alloc();\n let nextDrainDeferred = Defer();\n nextDrainDeferred.resolve();\n\n for (const record of records) {\n await throttle.acquire();\n this.firebaseObjectStorage.downloadFile(`snapshots/${record._id}`)\n .then(async (snapshotTxt) => {\n try {\n const snapshot = JSON.parse(snapshotTxt.toString('utf-8'));\n\n let formatted = await this.snapshotFormatter.formatSnapshot('default', snapshot);\n if (!formatted.content) {\n formatted = await this.snapshotFormatter.formatSnapshot('markdown', snapshot);\n }\n\n await nextDrainDeferred.promise;\n await appendFile(localFilePath, JSON.stringify({\n url: snapshot.href,\n title: snapshot.title || '',\n html: snapshot.html || '',\n text: snapshot.text || '',\n content: formatted.content || '',\n }) + '\\n', { encoding: 'utf-8' });\n\n } catch (err) {\n this.logger.warn(`Failed to parse snapshot for ${record._id}`, { err });\n }\n })\n .finally(() => {\n throttle.release();\n });\n }\n\n await throttle.nextDrain();\n\n\n const ro = {\n path: localFilePath\n };\n\n this.tempFileManager.bindPathTo(ro, localFilePath);\n\n return ro;\n }\n}\n" }, { "path": "src/db/adaptive-crawl-task.ts", "content": "import { Also, Prop, parseJSONText } from 'civkit';\nimport { FirestoreRecord } from '../shared/lib/firestore';\nimport _ from 'lodash';\n\nexport enum AdaptiveCrawlTaskStatus {\n PENDING = 'pending',\n PROCESSING = 'processing',\n COMPLETED = 'completed',\n FAILED = 'failed',\n}\n\n@Also({\n dictOf: Object\n})\nexport class AdaptiveCrawlTask extends FirestoreRecord {\n static override collectionName = 'adaptiveCrawlTasks';\n\n override _id!: string;\n\n @Prop({\n required: true\n })\n status!: AdaptiveCrawlTaskStatus;\n\n @Prop({\n required: true\n })\n statusText!: string;\n\n @Prop()\n meta!: {\n useSitemap: boolean;\n maxPages: number;\n targetUrl: string;\n };\n\n @Prop()\n urls!: string[];\n\n @Prop()\n processed!: {\n [url: string]: string;\n };\n\n @Prop()\n failed!: {\n [url: string]: any;\n };\n\n @Prop()\n createdAt!: Date;\n\n @Prop()\n finishedAt?: Date;\n\n @Prop()\n duration?: number;\n\n static patchedFields = [\n 'meta',\n ];\n\n static override from(input: any) {\n for (const field of this.patchedFields) {\n if (typeof input[field] === 'string') {\n input[field] = parseJSONText(input[field]);\n }\n }\n\n return super.from(input) as AdaptiveCrawlTask;\n }\n\n override degradeForFireStore() {\n const copy: any = { ...this };\n\n for (const field of (this.constructor as typeof AdaptiveCrawlTask).patchedFields) {\n if (typeof copy[field] === 'object') {\n copy[field] = JSON.stringify(copy[field]) as any;\n }\n }\n\n return copy;\n }\n\n [k: string]: any;\n}\n" }, { "path": "src/db/crawled.ts", "content": "import { Also, parseJSONText, Prop } from 'civkit';\nimport { FirestoreRecord } from '../shared/lib/firestore';\nimport _ from 'lodash';\nimport type { PageSnapshot } from '../services/puppeteer';\n\n@Also({\n dictOf: Object\n})\nexport class Crawled extends FirestoreRecord {\n static override collectionName = 'crawled';\n\n override _id!: string;\n\n @Prop({\n required: true\n })\n url!: string;\n\n @Prop({\n required: true\n })\n urlPathDigest!: string;\n\n @Prop()\n htmlSignificantlyModifiedByJs?: boolean;\n\n @Prop()\n snapshot?: PageSnapshot & { screenshot: never; pageshot: never; };\n\n @Prop()\n screenshotAvailable?: boolean;\n\n @Prop()\n pageshotAvailable?: boolean;\n\n @Prop()\n snapshotAvailable?: boolean;\n\n @Prop()\n createdAt!: Date;\n\n @Prop()\n expireAt!: Date;\n\n static patchedFields = [\n 'snapshot'\n ];\n\n static override from(input: any) {\n for (const field of this.patchedFields) {\n if (typeof input[field] === 'string') {\n input[field] = parseJSONText(input[field]);\n }\n }\n\n return super.from(input) as Crawled;\n }\n\n override degradeForFireStore() {\n const copy: any = { ...this };\n\n for (const field of (this.constructor as typeof Crawled).patchedFields) {\n if (typeof copy[field] === 'object') {\n copy[field] = JSON.stringify(copy[field]) as any;\n }\n }\n\n return copy;\n }\n\n [k: string]: any;\n}\n" }, { "path": "src/db/domain-blockade.ts", "content": "import { Also, Prop } from 'civkit';\nimport { FirestoreRecord } from '../shared/lib/firestore';\n\n@Also({\n dictOf: Object\n})\nexport class DomainBlockade extends FirestoreRecord {\n static override collectionName = 'domainBlockades';\n\n override _id!: string;\n\n @Prop({\n required: true\n })\n domain!: string;\n\n @Prop({ required: true })\n triggerReason!: string;\n\n @Prop()\n triggerUrl?: string;\n\n @Prop()\n createdAt!: Date;\n\n @Prop()\n expireAt?: Date;\n\n [k: string]: any;\n}\n" }, { "path": "src/db/domain-profile.ts", "content": "import { Also, Prop } from 'civkit';\nimport { FirestoreRecord } from '../shared/lib/firestore';\nimport { ENGINE_TYPE } from '../dto/crawler-options';\n\n@Also({\n dictOf: Object\n})\nexport class DomainProfile extends FirestoreRecord {\n static override collectionName = 'domainProfiles';\n\n override _id!: string;\n\n @Prop({\n required: true\n })\n path!: string;\n\n @Prop()\n triggerUrl?: string;\n\n @Prop({ required: true, type: ENGINE_TYPE })\n engine!: string;\n\n @Prop()\n createdAt!: Date;\n\n @Prop()\n expireAt?: Date;\n\n [k: string]: any;\n}\n" }, { "path": "src/db/img-alt.ts", "content": "import { Also, Prop } from 'civkit';\nimport { FirestoreRecord } from '../shared/lib/firestore';\nimport _ from 'lodash';\n\n@Also({\n dictOf: Object\n})\nexport class ImgAlt extends FirestoreRecord {\n static override collectionName = 'imgAlts';\n\n override _id!: string;\n\n @Prop({\n required: true\n })\n src!: string;\n\n @Prop({\n required: true\n })\n urlDigest!: string;\n\n @Prop()\n width?: number;\n\n @Prop()\n height?: number;\n\n @Prop()\n generatedAlt?: string;\n\n @Prop()\n originalAlt?: string;\n\n @Prop()\n createdAt!: Date;\n\n @Prop()\n expireAt?: Date;\n\n [k: string]: any;\n}\n" }, { "path": "src/db/pdf.ts", "content": "import { Also, Prop, parseJSONText } from 'civkit';\nimport { FirestoreRecord } from '../shared/lib/firestore';\nimport _ from 'lodash';\n\n@Also({\n dictOf: Object\n})\nexport class PDFContent extends FirestoreRecord {\n static override collectionName = 'pdfs';\n\n override _id!: string;\n\n @Prop({\n required: true\n })\n src!: string;\n\n @Prop({\n required: true\n })\n urlDigest!: string;\n\n @Prop()\n meta?: { [k: string]: any; };\n\n @Prop()\n text?: string;\n\n @Prop()\n content?: string;\n\n @Prop()\n createdAt!: Date;\n\n @Prop()\n expireAt?: Date;\n\n static patchedFields = [\n 'meta'\n ];\n\n static override from(input: any) {\n for (const field of this.patchedFields) {\n if (typeof input[field] === 'string') {\n input[field] = parseJSONText(input[field]);\n }\n }\n\n return super.from(input) as PDFContent;\n }\n\n override degradeForFireStore() {\n const copy: any = { ...this };\n\n for (const field of (this.constructor as typeof PDFContent).patchedFields) {\n if (typeof copy[field] === 'object') {\n copy[field] = JSON.stringify(copy[field]) as any;\n }\n }\n\n return copy;\n }\n\n [k: string]: any;\n}\n" }, { "path": "src/db/searched.ts", "content": "import { Also, parseJSONText, Prop } from 'civkit';\nimport { FirestoreRecord } from '../shared/lib/firestore';\nimport _ from 'lodash';\n\n@Also({\n dictOf: Object\n})\nexport class SearchResult extends FirestoreRecord {\n static override collectionName = 'searchResults';\n\n override _id!: string;\n\n @Prop({\n required: true\n })\n query!: any;\n\n @Prop({\n required: true\n })\n queryDigest!: string;\n\n @Prop()\n response?: any;\n\n @Prop()\n createdAt!: Date;\n\n @Prop()\n expireAt?: Date;\n\n [k: string]: any;\n\n static patchedFields = [\n 'query',\n 'response',\n ];\n\n static override from(input: any) {\n for (const field of this.patchedFields) {\n if (typeof input[field] === 'string') {\n input[field] = parseJSONText(input[field]);\n }\n }\n\n return super.from(input) as SearchResult;\n }\n\n override degradeForFireStore() {\n const copy: any = { ...this };\n\n for (const field of (this.constructor as typeof SearchResult).patchedFields) {\n if (typeof copy[field] === 'object') {\n copy[field] = JSON.stringify(copy[field]) as any;\n }\n }\n\n return copy;\n }\n}\n\nexport class SerperSearchResult extends SearchResult {\n static override collectionName = 'serperSearchResults';\n}\n\nexport class SERPResult extends SearchResult {\n static override collectionName = 'SERPResults';\n}" }, { "path": "src/dto/adaptive-crawler-options.ts", "content": "import { Also, AutoCastable, Prop, RPC_CALL_ENVIRONMENT } from 'civkit';\nimport type { Request, Response } from 'express';\n\n\n@Also({\n openapi: {\n operation: {\n parameters: {\n 'X-Use-Sitemap': {\n description: 'Use sitemap to crawl the website.',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Max-Depth': {\n description: 'Max deep level to crawl.',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Max-Pages': {\n description: 'Max number of pages to crawl.',\n in: 'header',\n schema: { type: 'string' }\n },\n }\n }\n }\n})\nexport class AdaptiveCrawlerOptions extends AutoCastable {\n @Prop({\n default: true,\n desc: 'Use sitemap to crawl the website.',\n })\n useSitemap!: boolean;\n\n @Prop({\n default: 10,\n desc: 'Max number of pages to crawl.',\n validate: (v: number) => v >= 1 && v <= 100,\n })\n maxPages!: number;\n\n static override from(input: any) {\n const instance = super.from(input) as AdaptiveCrawlerOptions;\n const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as {\n req: Request,\n res: Response,\n } | undefined;\n\n let maxPages = parseInt(ctx?.req.get('x-max-pages') || '');\n if (!isNaN(maxPages) && maxPages > 0) {\n instance.maxPages = maxPages <= 100 ? maxPages : 100;\n }\n\n const useSitemap = ctx?.req.get('x-use-sitemap');\n if (useSitemap !== undefined) {\n instance.useSitemap = Boolean(useSitemap);\n }\n\n return instance;\n }\n}\n" }, { "path": "src/dto/crawler-options.ts", "content": "import { Also, AutoCastable, ParamValidationError, Prop, RPC_CALL_ENVIRONMENT } from 'civkit/civ-rpc';\nimport { FancyFile } from 'civkit/fancy-file';\nimport { Cookie, parseString as parseSetCookieString } from 'set-cookie-parser';\nimport { Context } from '../services/registry';\nimport { TurnDownTweakableOptions } from './turndown-tweakable-options';\nimport type { PageSnapshot } from '../services/puppeteer';\n\nexport enum CONTENT_FORMAT {\n CONTENT = 'content',\n MARKDOWN = 'markdown',\n HTML = 'html',\n TEXT = 'text',\n PAGESHOT = 'pageshot',\n SCREENSHOT = 'screenshot',\n VLM = 'vlm',\n READER_LM = 'readerlm-v2',\n}\n\nexport enum ENGINE_TYPE {\n AUTO = 'auto',\n BROWSER = 'browser',\n CURL = 'curl',\n CF_BROWSER_RENDERING = 'cf-browser-rendering',\n}\n\nexport enum RESPOND_TIMING {\n HTML = 'html',\n VISIBLE_CONTENT = 'visible-content',\n MUTATION_IDLE = 'mutation-idle',\n RESOURCE_IDLE = 'resource-idle',\n MEDIA_IDLE = 'media-idle',\n NETWORK_IDLE = 'network-idle',\n}\n\nconst CONTENT_FORMAT_VALUES = new Set(Object.values(CONTENT_FORMAT));\n\nexport const IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'] as const;\nconst IMAGE_RETENTION_MODE_VALUES = new Set(IMAGE_RETENTION_MODES);\nexport const BASE_URL_MODES = ['initial', 'final'] as const;\nconst BASE_URL_MODE_VALUES = new Set(BASE_URL_MODES);\n\nclass Viewport extends AutoCastable {\n @Prop({\n default: 1024\n })\n width!: number;\n @Prop({\n default: 1024\n })\n height!: number;\n @Prop()\n deviceScaleFactor?: number;\n @Prop()\n isMobile?: boolean;\n @Prop()\n isLandscape?: boolean;\n @Prop()\n hasTouch?: boolean;\n}\n\n@Also({\n openapi: {\n operation: {\n parameters: {\n 'Accept': {\n description: `Specifies your preference for the response format.\\n\\n` +\n `Supported formats: \\n` +\n `- text/event-stream\\n` +\n `- application/json or text/json\\n` +\n `- text/plain`\n ,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Cache-Tolerance': {\n description: `Sets internal cache tolerance in seconds if this header is specified with a integer.`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-No-Cache': {\n description: `Ignores internal cache if this header is specified with a value.\\n\\nEquivalent to X-Cache-Tolerance: 0`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Respond-With': {\n description: `Specifies the (non-default) form of the crawled data you prefer.\\n\\n` +\n `Supported formats: \\n` +\n `- markdown\\n` +\n `- html\\n` +\n `- text\\n` +\n `- pageshot\\n` +\n `- screenshot\\n` +\n `- content\\n` +\n `- any combination of the above\\n` +\n `- readerlm-v2\\n` +\n `- vlm\\n\\n` +\n `Default: content\\n`\n ,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Wait-For-Selector': {\n description: `Specifies a CSS selector to wait for the appearance of such an element before returning.\\n\\n` +\n 'Example: `X-Wait-For-Selector: .content-block`\\n'\n ,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Target-Selector': {\n description: `Specifies a CSS selector for return target instead of the full html.\\n\\n` +\n 'Implies `X-Wait-For-Selector: (same selector)`'\n ,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Remove-Selector': {\n description: `Specifies a CSS selector to remove elements from the full html.\\n\\n` +\n 'Example `X-Remove-Selector: nav`'\n ,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Keep-Img-Data-Url': {\n description: `Keep data-url as it instead of transforming them to object-url. (Only applicable when targeting markdown format)\\n\\n` +\n 'Example `X-Keep-Img-Data-Url: true`'\n ,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Proxy-Url': {\n description: `Specifies your custom proxy if you prefer to use one.\\n\\n` +\n `Supported protocols: \\n` +\n `- http\\n` +\n `- https\\n` +\n `- socks4\\n` +\n `- socks5\\n\\n` +\n `For authentication, https://user:pass@host:port`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Proxy': {\n description: `Use a proxy server provided by us.\\n\\nOptionally specify two-letter country code.`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Robots-Txt': {\n description: `Load and conform to the respective robot.txt on the target origin.\\n\\nOptionally specify a bot UA to check against.\\n\\n`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'DNT': {\n description: `When set to 1, prevent the result of this request to be cached in the system.\\n\\n`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Set-Cookie': {\n description: `Sets cookie(s) to the headless browser for your request. \\n\\n` +\n `Syntax is the same with standard Set-Cookie`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-With-Generated-Alt': {\n description: `Enable automatic alt-text generating for images without an meaningful alt-text.\\n\\n` +\n `Note: Does not work when \\`X-Respond-With\\` is specified`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-With-Images-Summary': {\n description: `Enable dedicated summary section for images on the page.`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-With-links-Summary': {\n description: `Enable dedicated summary section for hyper links on the page.`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Retain-Images': {\n description: `Image retention modes.\\n\\n` +\n `Supported modes: \\n` +\n `- all: all images\\n` +\n `- none: no images\\n` +\n `- alt: only alt text\\n` +\n `- all_p: all images and with generated alt text\\n` +\n `- alt_p: only alt text and with generated alt\\n\\n`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-With-Iframe': {\n description: `Enable filling iframe contents into main. (violates standards)`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-With-Shadow-Dom': {\n description: `Enable filling shadow dom contents into main. (violates standards)`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-User-Agent': {\n description: `Override User-Agent.`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Timeout': {\n description: `Specify timeout in seconds. Max 180.`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Locale': {\n description: 'Specify browser locale for the page.',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Referer': {\n description: 'Specify referer for the page.',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Token-Budget': {\n description: 'Specify a budget in tokens.\\n\\nIf the resulting token cost exceeds the budget, the request is rejected.',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Respond-Timing': {\n description: `Explicitly specify the respond timing. One of the following:\\n\\n` +\n `- html: directly return unrendered HTML\\n` +\n `- visible-content: return immediately when any content becomes available\\n` +\n `- mutation-idle: wait for DOM mutations to settle and remain unchanged for at least 0.2s\\n` +\n `- resource-idle: wait for no additional resources that would affect page logic and content has SUCCEEDED loading in 0.5s\\n` +\n `- media-idle: wait for no additional resources, including media resources, has SUCCEEDED loading in 0.5s\\n` +\n `- network-idle: wait for full load of webpage, also known as networkidle0.\\n\\n`,\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Engine': {\n description: 'Specify the engine to use for crawling.\\n\\nSupported: browser, direct, cf-browser-rendering',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Base': {\n description: 'Select base modes of relative URLs.\\n\\nSupported: initial, final',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Md-Heading-Style': {\n description: 'Heading style of the generated markdown.\\n\\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\\n\\nSupported: setext, atx',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Md-Hr': {\n description: 'Hr text of the generated markdown.\\n\\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Md-Bullet-List-Marker': {\n description: 'Bullet list marker of the generated markdown.\\n\\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\\n\\nSupported: -, +, *',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Md-Em-Delimiter': {\n description: 'Em delimiter of the generated markdown.\\n\\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\\n\\nSupported: _, *',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Md-Strong-Delimiter': {\n description: 'Strong delimiter of the generated markdown.\\n\\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\\n\\nSupported: **, __',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Md-Link-Style': {\n description: 'Link style of the generated markdown.\\n\\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\\n\\nSupported: inlined, referenced, discarded',\n in: 'header',\n schema: { type: 'string' }\n },\n 'X-Md-Link-Reference-Style': {\n description: 'Link reference style of the generated markdown.\\n\\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\\n\\nSupported: full, collapsed, shortcut, discarded',\n in: 'header',\n schema: { type: 'string' }\n },\n }\n }\n }\n})\nexport class CrawlerOptions extends AutoCastable {\n\n @Prop()\n url?: string;\n\n @Prop()\n html?: string;\n\n @Prop({\n type: BASE_URL_MODE_VALUES,\n default: 'initial',\n })\n base?: typeof BASE_URL_MODES[number];\n\n @Prop({\n desc: 'Base64 encoded PDF.',\n type: [FancyFile, String]\n })\n pdf?: FancyFile | string;\n\n @Prop({\n default: CONTENT_FORMAT.CONTENT,\n type: [CONTENT_FORMAT, String]\n })\n respondWith!: string;\n\n @Prop({\n default: false,\n })\n withGeneratedAlt!: boolean;\n\n @Prop({ default: 'all', type: IMAGE_RETENTION_MODE_VALUES })\n retainImages?: typeof IMAGE_RETENTION_MODES[number];\n\n @Prop({\n default: false,\n })\n withLinksSummary!: boolean | string;\n\n @Prop({\n default: false,\n })\n withImagesSummary!: boolean;\n\n @Prop({\n default: false,\n })\n noCache!: boolean;\n\n @Prop({\n default: false,\n })\n noGfm!: string | boolean;\n\n @Prop()\n cacheTolerance?: number;\n\n @Prop({ arrayOf: String })\n targetSelector?: string | string[];\n\n @Prop({ arrayOf: String })\n waitForSelector?: string | string[];\n\n @Prop({ arrayOf: String })\n removeSelector?: string | string[];\n\n @Prop({\n default: false,\n })\n keepImgDataUrl!: boolean;\n\n @Prop({\n default: false,\n type: [String, Boolean]\n })\n withIframe!: boolean | 'quoted';\n\n @Prop({\n default: false,\n })\n withShadowDom!: boolean;\n\n @Prop({\n arrayOf: String,\n })\n setCookies?: Cookie[];\n\n @Prop()\n proxyUrl?: string;\n\n @Prop()\n proxy?: string;\n\n @Prop()\n userAgent?: string;\n\n @Prop()\n engine?: string;\n\n @Prop({\n arrayOf: String,\n })\n injectPageScript?: string[];\n\n @Prop({\n arrayOf: String,\n })\n injectFrameScript?: string[];\n\n @Prop({\n validate: (v: number) => v > 0 && v <= 180,\n type: Number,\n nullable: true,\n })\n timeout?: number | null;\n\n @Prop()\n locale?: string;\n\n @Prop()\n referer?: string;\n\n @Prop()\n tokenBudget?: number;\n\n @Prop()\n viewport?: Viewport;\n\n @Prop()\n instruction?: string;\n\n @Prop()\n jsonSchema?: object;\n\n @Prop()\n robotsTxt?: string;\n\n @Prop()\n doNotTrack?: number | null;\n\n @Prop()\n markdown?: TurnDownTweakableOptions;\n\n @Prop({\n type: RESPOND_TIMING,\n })\n respondTiming?: RESPOND_TIMING;\n\n _hintIps?: string[];\n\n static override from(input: any) {\n const instance = super.from(input) as CrawlerOptions;\n const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined;\n\n const customMode = ctx?.get('x-respond-with') || ctx?.get('x-return-format');\n if (customMode) {\n instance.respondWith = customMode;\n }\n if (instance.respondWith) {\n instance.respondWith = instance.respondWith.toLowerCase();\n }\n if (instance.respondWith?.includes('lm')) {\n if (instance.respondWith.includes('content') || instance.respondWith.includes('markdown')) {\n throw new ParamValidationError({\n path: 'respondWith',\n message: `LM formats conflicts with content/markdown.`,\n });\n }\n }\n\n const locale = ctx?.get('x-locale');\n if (locale) {\n instance.locale = locale;\n }\n\n const referer = ctx?.get('x-referer');\n if (referer) {\n instance.referer = referer;\n }\n\n const withGeneratedAlt = ctx?.get('x-with-generated-alt');\n if (withGeneratedAlt) {\n instance.withGeneratedAlt = Boolean(withGeneratedAlt);\n }\n const withLinksSummary = ctx?.get('x-with-links-summary');\n if (withLinksSummary) {\n if (withLinksSummary === 'all') {\n instance.withLinksSummary = withLinksSummary;\n } else {\n instance.withLinksSummary = Boolean(withLinksSummary);\n }\n }\n const withImagesSummary = ctx?.get('x-with-images-summary');\n if (withImagesSummary) {\n instance.withImagesSummary = Boolean(withImagesSummary);\n }\n const retainImages = ctx?.get('x-retain-images');\n if (retainImages && IMAGE_RETENTION_MODE_VALUES.has(retainImages)) {\n instance.retainImages = retainImages as any;\n }\n if (instance.withGeneratedAlt) {\n instance.retainImages = 'all_p';\n }\n const noCache = ctx?.get('x-no-cache');\n if (noCache) {\n instance.noCache = Boolean(noCache);\n }\n if (instance.noCache && instance.cacheTolerance === undefined) {\n instance.cacheTolerance = 0;\n }\n let cacheTolerance = parseInt(ctx?.get('x-cache-tolerance') || '');\n if (!isNaN(cacheTolerance)) {\n instance.cacheTolerance = cacheTolerance;\n }\n\n const noGfm = ctx?.get('x-no-gfm');\n if (noGfm) {\n instance.noGfm = noGfm === 'table' ? noGfm : Boolean(noGfm);\n }\n\n let timeoutSeconds = parseInt(ctx?.get('x-timeout') || '');\n if (!isNaN(timeoutSeconds) && timeoutSeconds > 0) {\n instance.timeout = timeoutSeconds <= 180 ? timeoutSeconds : 180;\n } else if (ctx?.get('x-timeout')) {\n instance.timeout = null;\n }\n\n const removeSelector = ctx?.get('x-remove-selector')?.split(', ').filter(Boolean);\n instance.removeSelector ??= removeSelector?.length ? removeSelector : undefined;\n const targetSelector = ctx?.get('x-target-selector')?.split(', ').filter(Boolean);\n instance.targetSelector ??= targetSelector?.length ? targetSelector : undefined;\n const waitForSelector = ctx?.get('x-wait-for-selector')?.split(', ').filter(Boolean);\n instance.waitForSelector ??= (waitForSelector?.length ? waitForSelector : undefined) || instance.targetSelector;\n const overrideUserAgent = ctx?.get('x-user-agent') || undefined;\n instance.userAgent ??= overrideUserAgent;\n\n const engine = ctx?.get('x-engine');\n if (engine) {\n instance.engine = engine;\n }\n if (instance.engine) {\n instance.engine = instance.engine.toLowerCase();\n }\n if (instance.engine === 'vlm') {\n instance.engine = ENGINE_TYPE.BROWSER;\n instance.respondWith = CONTENT_FORMAT.VLM;\n } else if (instance.engine === 'readerlm-v2') {\n instance.engine = ENGINE_TYPE.AUTO;\n instance.respondWith = CONTENT_FORMAT.READER_LM;\n }\n\n const keepImgDataUrl = ctx?.get('x-keep-img-data-url');\n if (keepImgDataUrl) {\n instance.keepImgDataUrl = Boolean(keepImgDataUrl);\n }\n const withIframe = ctx?.get('x-with-iframe');\n if (withIframe) {\n instance.withIframe = withIframe.toLowerCase() === 'quoted' ? 'quoted' : Boolean(withIframe);\n }\n if (instance.withIframe) {\n instance.timeout ??= null;\n }\n const withShadowDom = ctx?.get('x-with-shadow-dom');\n if (withShadowDom) {\n instance.withShadowDom = Boolean(withShadowDom);\n }\n if (instance.withShadowDom) {\n instance.timeout ??= null;\n }\n\n const cookies: Cookie[] = [];\n const setCookieHeaders = (ctx?.get('x-set-cookie')?.split(', ') || (instance.setCookies as any as string[])).filter(Boolean);\n if (Array.isArray(setCookieHeaders)) {\n for (const setCookie of setCookieHeaders) {\n cookies.push({\n ...parseSetCookieString(setCookie, { decodeValues: true }),\n });\n }\n } else if (setCookieHeaders && typeof setCookieHeaders === 'string') {\n cookies.push({\n ...parseSetCookieString(setCookieHeaders, { decodeValues: true }),\n });\n }\n instance.setCookies = cookies;\n\n const proxyUrl = ctx?.get('x-proxy-url');\n instance.proxyUrl ??= proxyUrl || undefined;\n const proxy = ctx?.get('x-proxy');\n instance.proxy ??= proxy || undefined;\n const robotsTxt = ctx?.get('x-robots-txt');\n instance.robotsTxt ??= robotsTxt || undefined;\n\n const tokenBudget = ctx?.get('x-token-budget');\n instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined;\n\n const baseMode = ctx?.get('x-base');\n if (baseMode) {\n instance.base = baseMode as any;\n }\n\n const dnt = ctx?.get('dnt');\n instance.doNotTrack ??= (parseInt(dnt || '') || null);\n\n const respondTiming = ctx?.get('x-respond-timing');\n if (respondTiming) {\n instance.respondTiming ??= respondTiming as RESPOND_TIMING;\n }\n\n if (instance.cacheTolerance) {\n instance.cacheTolerance = instance.cacheTolerance * 1000;\n }\n\n if (ctx) {\n instance.markdown ??= TurnDownTweakableOptions.fromCtx(ctx);\n }\n\n return instance;\n }\n\n get presumedRespondTiming() {\n if (this.respondTiming) {\n return this.respondTiming;\n }\n if (this.timeout && this.timeout >= 20) {\n return RESPOND_TIMING.NETWORK_IDLE;\n }\n if (this.respondWith.includes('shot') || this.respondWith.includes('vlm')) {\n return RESPOND_TIMING.MEDIA_IDLE;\n }\n\n return RESPOND_TIMING.RESOURCE_IDLE;\n }\n\n isSnapshotAcceptableForEarlyResponse(snapshot: PageSnapshot) {\n if (this.waitForSelector?.length) {\n return false;\n }\n const presumedTiming = this.presumedRespondTiming;\n if (presumedTiming === RESPOND_TIMING.MEDIA_IDLE && snapshot.lastMediaResourceLoaded && snapshot.lastMutationIdle) {\n const now = Date.now();\n if ((Math.max(snapshot.lastMediaResourceLoaded, snapshot.lastContentResourceLoaded || 0) + 500) < now) {\n return true;\n }\n }\n if ((this.respondWith.includes('vlm') || this.respondWith.includes('pageshot')) && !snapshot.pageshot) {\n return false;\n }\n if ((this.respondWith.includes('vlm') || this.respondWith.includes('screenshot')) && !snapshot.screenshot) {\n return false;\n }\n if (presumedTiming === RESPOND_TIMING.RESOURCE_IDLE && snapshot.lastContentResourceLoaded && snapshot.lastMutationIdle) {\n const now = Date.now();\n if ((snapshot.lastContentResourceLoaded + 500) < now) {\n return true;\n }\n }\n if (this.injectFrameScript?.length || this.injectPageScript?.length) {\n return false;\n }\n if (presumedTiming === RESPOND_TIMING.VISIBLE_CONTENT && snapshot.parsed?.content) {\n return true;\n }\n if (presumedTiming === RESPOND_TIMING.HTML && snapshot.html) {\n return true;\n }\n if (presumedTiming === RESPOND_TIMING.NETWORK_IDLE) {\n return false;\n }\n if (presumedTiming === RESPOND_TIMING.MUTATION_IDLE && snapshot.lastMutationIdle) {\n return true;\n }\n if (this.respondWith.includes('lm')) {\n return false;\n }\n if (this.withIframe) {\n return false;\n }\n\n return !snapshot.isIntermediate;\n }\n\n isCacheQueryApplicable() {\n if (this.noCache) {\n return false;\n }\n if (this.cacheTolerance === 0) {\n return false;\n }\n if (this.setCookies?.length) {\n return false;\n }\n if (this.injectFrameScript?.length || this.injectPageScript?.length) {\n return false;\n }\n if (this.viewport) {\n return false;\n }\n\n return true;\n }\n\n isRequestingCompoundContentFormat() {\n return !CONTENT_FORMAT_VALUES.has(this.respondWith);\n }\n\n browserIsNotRequired() {\n if (this.respondTiming && ![RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes(this.respondTiming)) {\n return false;\n }\n if (this.respondWith.includes(CONTENT_FORMAT.PAGESHOT) || this.respondWith.includes(CONTENT_FORMAT.SCREENSHOT)) {\n return false;\n }\n if (this.injectFrameScript?.length || this.injectPageScript?.length) {\n return false;\n }\n if (this.waitForSelector?.length) {\n return false;\n }\n if (this.withIframe || this.withShadowDom) {\n return false;\n }\n if (this.viewport) {\n return false;\n }\n if (this.pdf) {\n return false;\n }\n if (this.html) {\n return false;\n }\n\n return true;\n }\n}\n\nexport class CrawlerOptionsHeaderOnly extends CrawlerOptions {\n static override from(input: any) {\n const instance = super.from({\n [RPC_CALL_ENVIRONMENT]: Reflect.get(input, RPC_CALL_ENVIRONMENT),\n }) as CrawlerOptionsHeaderOnly;\n\n return instance;\n }\n}" }, { "path": "src/dto/jina-embeddings-auth.ts", "content": "import _ from 'lodash';\nimport {\n Also, AuthenticationFailedError, AuthenticationRequiredError,\n RPC_CALL_ENVIRONMENT,\n AutoCastable,\n DownstreamServiceError,\n} from 'civkit/civ-rpc';\nimport { htmlEscape } from 'civkit/escape';\nimport { marshalErrorLike } from 'civkit/lang';\n\nimport type { Context } from 'koa';\n\nimport logger from '../services/logger';\nimport { InjectProperty } from '../services/registry';\nimport { AsyncLocalContext } from '../services/async-context';\n\nimport envConfig from '../shared/services/secrets';\nimport { JinaEmbeddingsDashboardHTTP } from '../shared/3rd-party/jina-embeddings';\nimport { JinaEmbeddingsTokenAccount } from '../shared/db/jina-embeddings-token-account';\nimport { TierFeatureConstraintError } from '../services/errors';\n\nconst authDtoLogger = logger.child({ service: 'JinaAuthDTO' });\n\n\nconst THE_VERY_SAME_JINA_EMBEDDINGS_CLIENT = new JinaEmbeddingsDashboardHTTP(envConfig.JINA_EMBEDDINGS_DASHBOARD_API_KEY);\n\n@Also({\n openapi: {\n operation: {\n parameters: {\n 'Authorization': {\n description: htmlEscape`Jina Token for authentication.\\n\\n` +\n htmlEscape`- Member of \\n\\n` +\n `- Authorization: Bearer {YOUR_JINA_TOKEN}`\n ,\n in: 'header',\n schema: {\n anyOf: [\n { type: 'string', format: 'token' }\n ]\n }\n }\n }\n }\n }\n})\nexport class JinaEmbeddingsAuthDTO extends AutoCastable {\n uid?: string;\n bearerToken?: string;\n user?: JinaEmbeddingsTokenAccount;\n\n @InjectProperty(AsyncLocalContext)\n ctxMgr!: AsyncLocalContext;\n\n jinaEmbeddingsDashboard = THE_VERY_SAME_JINA_EMBEDDINGS_CLIENT;\n\n static override from(input: any) {\n const instance = super.from(input) as JinaEmbeddingsAuthDTO;\n\n const ctx = input[RPC_CALL_ENVIRONMENT] as Context;\n\n if (ctx) {\n const authorization = ctx.get('authorization');\n\n if (authorization) {\n const authToken = authorization.split(' ')[1] || authorization;\n instance.bearerToken = authToken;\n }\n\n }\n\n if (!instance.bearerToken && input._token) {\n instance.bearerToken = input._token;\n }\n\n return instance;\n }\n\n async getBrief(ignoreCache?: boolean | string) {\n if (!this.bearerToken) {\n throw new AuthenticationRequiredError({\n message: 'Jina API key is required to authenticate. Please get one from https://jina.ai'\n });\n }\n\n let firestoreDegradation = false;\n let account;\n try {\n account = await JinaEmbeddingsTokenAccount.fromFirestore(this.bearerToken);\n } catch (err) {\n // FireStore would not accept any string as input and may throw if not happy with it\n firestoreDegradation = true;\n logger.warn(`Firestore issue`, { err });\n }\n\n\n const age = account?.lastSyncedAt ? Date.now() - account.lastSyncedAt.valueOf() : Infinity;\n const jitter = Math.ceil(Math.random() * 30 * 1000);\n\n if (account && !ignoreCache) {\n if ((age < (180_000 - jitter)) && (account.wallet?.total_balance > 0)) {\n this.user = account;\n this.uid = this.user?.user_id;\n\n return account;\n }\n }\n\n if (firestoreDegradation) {\n logger.debug(`Using remote UC cached user`);\n let r;\n try {\n r = await this.jinaEmbeddingsDashboard.authorization(this.bearerToken);\n } catch (err: any) {\n if (err?.status === 401) {\n throw new AuthenticationFailedError({\n message: 'Invalid API key, please get a new one from https://jina.ai'\n });\n }\n logger.warn(`Failed load remote cached user: ${err}`, { err });\n throw new DownstreamServiceError(`Failed to authenticate: ${err}`);\n }\n const brief = r?.data;\n const draftAccount = JinaEmbeddingsTokenAccount.from({\n ...account, ...brief, _id: this.bearerToken,\n lastSyncedAt: new Date()\n });\n this.user = draftAccount;\n this.uid = this.user?.user_id;\n\n return draftAccount;\n }\n\n try {\n // TODO: go back using validateToken after performance issue fixed\n const r = ((account?.wallet?.total_balance || 0) > 0) ?\n await this.jinaEmbeddingsDashboard.authorization(this.bearerToken) :\n await this.jinaEmbeddingsDashboard.validateToken(this.bearerToken);\n const brief = r.data;\n const draftAccount = JinaEmbeddingsTokenAccount.from({\n ...account, ...brief, _id: this.bearerToken,\n lastSyncedAt: new Date()\n });\n await JinaEmbeddingsTokenAccount.save(draftAccount.degradeForFireStore(), undefined, { merge: true });\n\n this.user = draftAccount;\n this.uid = this.user?.user_id;\n\n return draftAccount;\n } catch (err: any) {\n authDtoLogger.warn(`Failed to get user brief: ${err}`, { err: marshalErrorLike(err) });\n\n if (err?.status === 401) {\n throw new AuthenticationFailedError({\n message: 'Invalid API key, please get a new one from https://jina.ai'\n });\n }\n\n if (account) {\n this.user = account;\n this.uid = this.user?.user_id;\n\n return account;\n }\n\n\n throw new DownstreamServiceError(`Failed to authenticate: ${err}`);\n }\n }\n\n async reportUsage(tokenCount: number, mdl: string, endpoint: string = '/encode') {\n const user = await this.assertUser();\n const uid = user.user_id;\n user.wallet.total_balance -= tokenCount;\n\n return this.jinaEmbeddingsDashboard.reportUsage(this.bearerToken!, {\n model_name: mdl,\n api_endpoint: endpoint,\n consumer: {\n id: uid,\n user_id: uid,\n },\n usage: {\n total_tokens: tokenCount\n },\n labels: {\n model_name: mdl\n }\n }).then((r) => {\n JinaEmbeddingsTokenAccount.COLLECTION.doc(this.bearerToken!)\n .update({ 'wallet.total_balance': JinaEmbeddingsTokenAccount.OPS.increment(-tokenCount) })\n .catch((err) => {\n authDtoLogger.warn(`Failed to update cache for ${uid}: ${err}`, { err: marshalErrorLike(err) });\n });\n\n return r;\n }).catch((err) => {\n user.wallet.total_balance += tokenCount;\n authDtoLogger.warn(`Failed to report usage for ${uid}: ${err}`, { err: marshalErrorLike(err) });\n });\n }\n\n async solveUID() {\n if (this.uid) {\n this.ctxMgr.set('uid', this.uid);\n\n return this.uid;\n }\n\n if (this.bearerToken) {\n await this.getBrief();\n this.ctxMgr.set('uid', this.uid);\n\n return this.uid;\n }\n\n return undefined;\n }\n\n async assertUID() {\n const uid = await this.solveUID();\n\n if (!uid) {\n throw new AuthenticationRequiredError('Authentication failed');\n }\n\n return uid;\n }\n\n async assertUser() {\n if (this.user) {\n return this.user;\n }\n\n await this.getBrief();\n\n return this.user!;\n }\n\n async assertTier(n: number, feature?: string) {\n let user;\n try {\n user = await this.assertUser();\n } catch (err) {\n if (err instanceof AuthenticationRequiredError) {\n throw new AuthenticationRequiredError({\n message: `Authentication is required to use this feature${feature ? ` (${feature})` : ''}. Please provide a valid API key.`\n });\n }\n\n throw err;\n }\n\n const tier = parseInt(user.metadata?.speed_level);\n if (isNaN(tier) || tier < n) {\n throw new TierFeatureConstraintError({\n message: `Your current plan does not support this feature${feature ? ` (${feature})` : ''}. Please upgrade your plan.`\n });\n }\n\n return true;\n }\n\n getRateLimits(...tags: string[]) {\n const descs = tags.map((x) => this.user?.customRateLimits?.[x] || []).flat().filter((x) => x.isEffective());\n\n if (descs.length) {\n return descs;\n }\n\n return undefined;\n }\n}\n" }, { "path": "src/dto/turndown-tweakable-options.ts", "content": "import { AutoCastable, Prop } from 'civkit/civ-rpc';\nimport {Context} from '../services/registry';\nimport _ from 'lodash';\n\n\nexport class TurnDownTweakableOptions extends AutoCastable {\n @Prop({\n desc: 'Turndown options > headingStyle',\n type: new Set(['setext', 'atx']),\n })\n headingStyle?: 'setext' | 'atx';\n\n @Prop({\n desc: 'Turndown options > hr',\n validate: (v: string) => v.length > 0 && v.length <= 128\n })\n hr?: string;\n\n @Prop({\n desc: 'Turndown options > bulletListMarker',\n type: new Set(['-', '+', '*']),\n })\n bulletListMarker?: '-' | '+' | '*';\n\n @Prop({\n desc: 'Turndown options > emDelimiter',\n type: new Set(['_', '*']),\n })\n emDelimiter?: '_' | '*';\n\n @Prop({\n desc: 'Turndown options > strongDelimiter',\n type: new Set(['__', '**']),\n })\n strongDelimiter?: '__' | '**';\n\n @Prop({\n desc: 'Turndown options > linkStyle',\n type: new Set(['inlined', 'referenced', 'discarded']),\n })\n linkStyle?: 'inlined' | 'referenced' | 'discarded';\n\n @Prop({\n desc: 'Turndown options > linkReferenceStyle',\n type: new Set(['full', 'collapsed', 'shortcut', 'discarded']),\n })\n linkReferenceStyle?: 'full' | 'collapsed' | 'shortcut' | 'discarded';\n\n static fromCtx(ctx: Context, prefix= 'x-md-') {\n const draft: Record = {};\n for (const [k, v] of Object.entries(ctx.headers)) {\n if (k.startsWith(prefix)) {\n const prop = k.slice(prefix.length);\n const sk = _.camelCase(prop);\n draft[sk] = v as string;\n }\n }\n\n return this.from(draft);\n }\n}\n" }, { "path": "src/fetch.d.ts", "content": "declare global {\n export const {\n fetch,\n FormData,\n Headers,\n Request,\n Response,\n File,\n }: typeof import('undici');\n export type { FormData, Headers, Request, RequestInit, Response, RequestInit, File } from 'undici';\n}\n\nexport { };\n" }, { "path": "src/lib/transform-server-event-stream.ts", "content": "import { TPM, parseJSONText } from 'civkit';\nimport { Transform, TransformCallback, TransformOptions } from 'stream';\n\nexport class InputServerEventStream extends Transform {\n cache: string[] = [];\n\n constructor(options?: TransformOptions) {\n super({\n ...options,\n readableObjectMode: true\n });\n }\n\n decodeRoutine() {\n if (!this.cache.length) {\n return;\n }\n\n const vecs = this.cache.join('').split(/\\r?\\n\\r?\\n/);\n this.cache.length = 0;\n const lastVec = vecs.pop();\n if (lastVec) {\n this.cache.push(lastVec);\n }\n\n for (const x of vecs) {\n const lines: string[] = x.split(/\\r?\\n/);\n\n const event: {\n event?: string;\n data?: string;\n id?: string;\n retry?: number;\n } = {};\n\n for (const l of lines) {\n const columnPos = l.indexOf(':');\n if (columnPos <= 0) {\n continue;\n }\n const key = l.substring(0, columnPos);\n const rawValue = l.substring(columnPos + 1);\n const value = rawValue.startsWith(' ') ? rawValue.slice(1) : rawValue;\n if (key === 'data') {\n if (event.data) {\n event.data += value || '\\n';\n } else if (event.data === '') {\n event.data += '\\n';\n event.data += value || '\\n';\n } else {\n event.data = value;\n }\n } else if (key === 'retry') {\n event.retry = parseInt(value, 10);\n } else {\n Reflect.set(event, key, value);\n }\n }\n\n if (event.data) {\n const parsed = parseJSONText(event.data);\n if (parsed && typeof parsed === 'object') {\n event.data = parsed;\n }\n }\n\n if (Object.keys(event).length) {\n this.push(event);\n }\n }\n }\n\n override _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void {\n if (chunk === null) {\n this.push(null);\n }\n\n this.cache.push(chunk.toString());\n this.decodeRoutine();\n\n callback();\n }\n\n override _final(callback: (error?: Error | null | undefined) => void): void {\n this.decodeRoutine();\n callback();\n }\n}\n\n@TPM({\n contentType: 'text/event-stream',\n})\nexport class OutputServerEventStream extends Transform {\n n: number = 0;\n\n constructor(options?: TransformOptions) {\n super({\n ...options, writableObjectMode: true, encoding: 'utf-8'\n });\n }\n\n encodeRoutine(chunk: {\n event?: string;\n data?: any;\n id?: string;\n retry?: number;\n } | string) {\n if (typeof chunk === 'object') {\n const lines: string[] = [];\n\n if (chunk.event) {\n lines.push(`event: ${chunk.event}`);\n }\n if (chunk.data) {\n if (typeof chunk.data === 'string') {\n for (const x of chunk.data.split(/\\r?\\n/)) {\n lines.push(`data: ${x}`);\n }\n } else {\n lines.push(`data: ${JSON.stringify(chunk.data)}`);\n }\n }\n if (chunk.id) {\n lines.push(`id: ${chunk.id}`);\n }\n if (chunk.retry) {\n lines.push(`retry: ${chunk.retry}`);\n }\n if (!lines.length) {\n lines.push(`data: ${JSON.stringify(chunk)}`);\n }\n this.push(lines.join('\\n'));\n this.push('\\n\\n');\n this.n++;\n\n return;\n } else if (typeof chunk === 'string') {\n const lines: string[] = [];\n for (const x of chunk.split(/\\r?\\n/)) {\n lines.push(`data: ${x}`);\n }\n\n this.push(lines.join('\\n'));\n this.push('\\n\\n');\n this.n++;\n }\n }\n\n override _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void {\n if (chunk === null) {\n this.push(null);\n }\n\n this.encodeRoutine(chunk);\n\n callback();\n }\n}\n\nexport interface OutputServerEventStream extends Transform {\n write(chunk: string | {\n event?: string;\n data?: any;\n id?: string;\n retry?: number;\n }, callback?: (error: Error | null | undefined) => void): boolean;\n write(chunk: any, callback?: (error: Error | null | undefined) => void): boolean;\n write(chunk: any, encoding: BufferEncoding, callback?: (error: Error | null | undefined) => void): boolean;\n}\n" }, { "path": "src/services/alt-text.ts", "content": "import { AssertionFailureError, AsyncService, HashManager } from 'civkit';\nimport { singleton } from 'tsyringe';\nimport { GlobalLogger } from './logger';\nimport { CanvasService } from './canvas';\nimport { ImageInterrogationManager } from '../shared/services/common-iminterrogate';\nimport { ImgBrief } from './puppeteer';\nimport { ImgAlt } from '../db/img-alt';\nimport { AsyncLocalContext } from './async-context';\n\nconst md5Hasher = new HashManager('md5', 'hex');\n\n@singleton()\nexport class AltTextService extends AsyncService {\n\n altsToIgnore = 'image,img,photo,picture,pic,alt,figure,fig'.split(',');\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected imageInterrogator: ImageInterrogationManager,\n protected canvasService: CanvasService,\n protected asyncLocalContext: AsyncLocalContext\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n }\n\n async caption(url: string) {\n try {\n const img = await this.canvasService.loadImage(url);\n const contentTypeHint = Reflect.get(img, 'contentType');\n if (Math.min(img.naturalHeight, img.naturalWidth) <= 1) {\n return `A ${img.naturalWidth}x${img.naturalHeight} image, likely be a tacker probe`;\n }\n if (Math.min(img.naturalHeight, img.naturalWidth) < 64) {\n return `A ${img.naturalWidth}x${img.naturalHeight} small image, likely a logo, icon or avatar`;\n }\n const resized = this.canvasService.fitImageToSquareBox(img, 1024);\n const exported = await this.canvasService.canvasToBuffer(resized, 'image/png');\n\n const svgHint = contentTypeHint.includes('svg') ? `Beware this image is a SVG rendered on a gray background, the gray background is not part of the image.\\n\\n` : '';\n const svgSystemHint = contentTypeHint.includes('svg') ? ` Sometimes the system renders SVG on a gray background. When this happens, you must not include the gray background in the description.` : '';\n\n const r = await this.imageInterrogator.interrogate('vertex-gemini-2.0-flash', {\n image: exported,\n prompt: `${svgHint}Give a concise image caption descriptive sentence in third person. Start directly with the description.`,\n system: `You are BLIP2, an image caption model. You will generate Alt Text (in web pages) for any image for a11y purposes. You must not start with \"This image is sth...\", instead, start direly with \"sth...\"${svgSystemHint}`,\n });\n\n return r.replaceAll(/[\\n\\\"]|(\\.\\s*$)/g, '').trim();\n } catch (err) {\n throw new AssertionFailureError({ message: `Could not generate alt text for url ${url}`, cause: err });\n }\n }\n\n async getAltText(imgBrief: ImgBrief) {\n if (!imgBrief.src) {\n return undefined;\n }\n if (imgBrief.alt && !this.altsToIgnore.includes(imgBrief.alt.trim().toLowerCase())) {\n return imgBrief.alt;\n }\n const digest = md5Hasher.hash(imgBrief.src);\n const shortDigest = Buffer.from(digest, 'hex').toString('base64url');\n let dims: number[] = [];\n do {\n if (imgBrief.loaded) {\n if (imgBrief.naturalWidth && imgBrief.naturalHeight) {\n if (Math.min(imgBrief.naturalWidth, imgBrief.naturalHeight) < 64) {\n dims = [imgBrief.naturalWidth, imgBrief.naturalHeight];\n break;\n }\n }\n }\n\n if (imgBrief.width && imgBrief.height) {\n if (Math.min(imgBrief.width, imgBrief.height) < 64) {\n dims = [imgBrief.width, imgBrief.height];\n break;\n }\n }\n\n } while (false);\n\n if (Math.min(...dims) <= 1) {\n return `A ${dims[0]}x${dims[1]} image, likely be a tacker probe`;\n }\n if (Math.min(...dims) < 64) {\n return `A ${dims[0]}x${dims[1]} small image, likely a logo, icon or avatar`;\n }\n\n const existing = await ImgAlt.fromFirestore(shortDigest);\n\n if (existing) {\n return existing.generatedAlt || existing.originalAlt || '';\n }\n\n let generatedCaption = '';\n\n try {\n generatedCaption = await this.caption(imgBrief.src);\n } catch (err) {\n this.logger.warn(`Unable to generate alt text for ${imgBrief.src}`, { err });\n }\n\n if (this.asyncLocalContext.ctx.DNT) {\n // Don't cache alt text if DNT is set\n return generatedCaption;\n }\n\n // Don't try again until the next day\n const expireMixin = generatedCaption ? {} : { expireAt: new Date(Date.now() + 1000 * 3600 * 24) };\n\n await ImgAlt.COLLECTION.doc(shortDigest).set(\n {\n _id: shortDigest,\n src: imgBrief.src || '',\n width: imgBrief.naturalWidth || 0,\n height: imgBrief.naturalHeight || 0,\n urlDigest: digest,\n originalAlt: imgBrief.alt || '',\n generatedAlt: generatedCaption || '',\n createdAt: new Date(),\n ...expireMixin\n }, { merge: true }\n );\n\n return generatedCaption;\n }\n};\n" }, { "path": "src/services/async-context.ts", "content": "import { GlobalAsyncContext } from 'civkit/async-context';\nimport { container, singleton } from 'tsyringe';\n\n@singleton()\nexport class AsyncLocalContext extends GlobalAsyncContext { }\n\nconst instance = container.resolve(AsyncLocalContext);\nReflect.set(process, 'asyncLocalContext', instance);\n\nexport default instance;\n" }, { "path": "src/services/blackhole-detector.ts", "content": "import { singleton } from 'tsyringe';\nimport { AsyncService } from 'civkit/async-service';\nimport { GlobalLogger } from './logger';\nimport { delay } from 'civkit/timeout';\n\n\n@singleton()\nexport class BlackHoleDetector extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n lastWorkedTs?: number;\n lastDoneRequestTs?: number;\n lastIncomingRequestTs?: number;\n\n maxDelay = 1000 * 30;\n concurrentRequests = 0;\n\n strikes = 0;\n\n constructor(protected globalLogger: GlobalLogger) {\n super(...arguments);\n\n if (process.env.NODE_ENV?.startsWith('prod')) {\n setInterval(() => {\n this.routine();\n }, 1000 * 30).unref();\n }\n }\n\n override async init() {\n await this.dependencyReady();\n this.logger.debug('BlackHoleDetector started');\n this.emit('ready');\n }\n\n async routine() {\n // We give routine a 3s grace period for potentially paused CPU to spin up and process some requests\n await delay(3000);\n const now = Date.now();\n const lastWorked = this.lastWorkedTs;\n if (!lastWorked) {\n return;\n }\n const dt = (now - lastWorked);\n if (this.concurrentRequests > 1 &&\n this.lastIncomingRequestTs && lastWorked &&\n this.lastIncomingRequestTs >= lastWorked &&\n (dt > (this.maxDelay * (this.strikes + 1)))\n ) {\n this.logger.warn(`BlackHole detected, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`);\n this.strikes += 1;\n }\n\n if (this.strikes >= 3) {\n this.logger.error(`BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`);\n process.nextTick(() => {\n this.emit('error', new Error(`BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`));\n // process.exit(1);\n });\n }\n }\n\n incomingRequest() {\n this.lastIncomingRequestTs = Date.now();\n this.lastWorkedTs ??= Date.now();\n this.concurrentRequests++;\n }\n doneWithRequest() {\n this.concurrentRequests--;\n this.lastDoneRequestTs = Date.now();\n }\n\n itWorked() {\n this.lastWorkedTs = Date.now();\n this.strikes = 0;\n }\n\n};\n" }, { "path": "src/services/brave-search.ts", "content": "import { AsyncService, AutoCastable, DownstreamServiceFailureError, Prop, RPC_CALL_ENVIRONMENT, delay, marshalErrorLike } from 'civkit';\nimport { singleton } from 'tsyringe';\nimport { GlobalLogger } from './logger';\nimport { SecretExposer } from '../shared/services/secrets';\nimport { BraveSearchHTTP, WebSearchQueryParams } from '../shared/3rd-party/brave-search';\nimport { GEOIP_SUPPORTED_LANGUAGES, GeoIPService } from './geoip';\nimport { AsyncLocalContext } from './async-context';\nimport { WebSearchOptionalHeaderOptions } from '../shared/3rd-party/brave-types';\nimport type { Request, Response } from 'express';\nimport { BlackHoleDetector } from './blackhole-detector';\n\n@singleton()\nexport class BraveSearchService extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n braveSearchHTTP!: BraveSearchHTTP;\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected secretExposer: SecretExposer,\n protected geoipControl: GeoIPService,\n protected threadLocal: AsyncLocalContext,\n protected blackHoleDetector: BlackHoleDetector,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n\n this.braveSearchHTTP = new BraveSearchHTTP(this.secretExposer.BRAVE_SEARCH_API_KEY);\n }\n\n async webSearch(query: WebSearchQueryParams) {\n const ip = this.threadLocal.get('ip');\n const extraHeaders: WebSearchOptionalHeaderOptions = {};\n if (ip) {\n const geoip = await this.geoipControl.lookupCity(ip, GEOIP_SUPPORTED_LANGUAGES.EN);\n\n if (geoip?.city) {\n extraHeaders['X-Loc-City'] = encodeURIComponent(geoip.city);\n }\n if (geoip?.country) {\n extraHeaders['X-Loc-Country'] = geoip.country.code;\n }\n if (geoip?.timezone) {\n extraHeaders['X-Loc-Timezone'] = geoip.timezone;\n }\n if (geoip?.coordinates) {\n extraHeaders['X-Loc-Lat'] = `${geoip.coordinates[0]}`;\n extraHeaders['X-Loc-Long'] = `${geoip.coordinates[1]}`;\n }\n if (geoip?.subdivisions?.length) {\n extraHeaders['X-Loc-State'] = encodeURIComponent(`${geoip.subdivisions[0].code}`);\n extraHeaders['X-Loc-State-Name'] = encodeURIComponent(`${geoip.subdivisions[0].name}`);\n }\n }\n if (this.threadLocal.get('userAgent')) {\n extraHeaders['User-Agent'] = this.threadLocal.get('userAgent');\n }\n\n const encoded = { ...query };\n if (encoded.q) {\n encoded.q = (Buffer.from(encoded.q).toString('ascii') === encoded.q) ? encoded.q : encodeURIComponent(encoded.q);\n }\n\n let maxTries = 11;\n\n while (maxTries--) {\n try {\n const r = await this.braveSearchHTTP.webSearch(encoded, { headers: extraHeaders as Record });\n this.blackHoleDetector.itWorked();\n\n return r.parsed;\n } catch (err: any) {\n this.logger.error(`Web search failed: ${err?.message}`, { err: marshalErrorLike(err) });\n if (err?.status === 429) {\n await delay(500 + 1000 * Math.random());\n continue;\n }\n\n throw new DownstreamServiceFailureError({ message: `Search failed` });\n }\n }\n\n throw new DownstreamServiceFailureError({ message: `Search failed` });\n }\n\n}\n\n\nexport class BraveSearchExplicitOperatorsDto extends AutoCastable {\n @Prop({\n arrayOf: String,\n desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.`\n })\n ext?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.`\n })\n filetype?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages containing the specified term in the body of the page. Example: to find information about the Nvidia GeForce GTX 1080 Ti, making sure the page contains the keywords “founders edition” in the body, type “nvidia 1080 ti inbody:“founders edition””.`\n })\n inbody?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.`\n })\n intitle?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns webpages containing the specified term either in the title or in the body of the page. Example: to find pages about the 2024 Oscars containing the keywords “best costume design” in the page, type “oscars 2024 inpage:“best costume design””.`\n })\n inpage?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`\n })\n lang?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`\n })\n loc?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.`\n })\n site?: string | string[];\n\n addTo(searchTerm: string) {\n const chunks = [];\n for (const [key, value] of Object.entries(this)) {\n if (value) {\n const values = Array.isArray(value) ? value : [value];\n const textValue = values.map((v) => `${key}:${v}`).join(' OR ');\n if (textValue) {\n chunks.push(textValue);\n }\n }\n }\n const opPart = chunks.length > 1 ? chunks.map((x) => `(${x})`).join(' AND ') : chunks;\n\n if (opPart.length) {\n return [searchTerm, opPart].join(' ');\n }\n\n return searchTerm;\n }\n\n static override from(input: any) {\n const instance = super.from(input) as BraveSearchExplicitOperatorsDto;\n const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as {\n req: Request,\n res: Response,\n } | undefined;\n\n const params = ['ext', 'filetype', 'inbody', 'intitle', 'inpage', 'lang', 'loc', 'site'];\n\n for (const p of params) {\n const customValue = ctx?.req.get(`x-${p}`) || ctx?.req.get(`${p}`);\n if (!customValue) {\n continue;\n }\n\n const filtered = customValue.split(', ').filter(Boolean);\n if (filtered.length) {\n Reflect.set(instance, p, filtered);\n }\n }\n\n return instance;\n }\n}\n" }, { "path": "src/services/canvas.ts", "content": "import { singleton, container } from 'tsyringe';\nimport { AsyncService, mimeOf, ParamValidationError, SubmittedDataMalformedError, /* downloadFile */ } from 'civkit';\nimport { readFile } from 'fs/promises';\n\nimport type canvas from '@napi-rs/canvas';\nexport type { Canvas, Image } from '@napi-rs/canvas';\n\nimport { GlobalLogger } from './logger';\nimport { TempFileManager } from './temp-file';\n\nimport { isMainThread } from 'worker_threads';\nimport type { svg2png } from 'svg2png-wasm' with { 'resolution-mode': 'import' };\nimport path from 'path';\nimport { Threaded } from './threaded';\n\nconst downloadFile = async (uri: string) => {\n const resp = await fetch(uri);\n if (!(resp.ok && resp.body)) {\n throw new Error(`Unexpected response ${resp.statusText}`);\n }\n const contentLength = parseInt(resp.headers.get('content-length') || '0');\n if (contentLength > 1024 * 1024 * 100) {\n throw new Error('File too large');\n }\n const buff = await resp.arrayBuffer();\n\n return { buff, contentType: resp.headers.get('content-type') };\n};\n\n@singleton()\nexport class CanvasService extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n svg2png!: typeof svg2png;\n canvas!: typeof canvas;\n\n constructor(\n protected temp: TempFileManager,\n protected globalLogger: GlobalLogger,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n if (!isMainThread) {\n const { createSvg2png, initialize } = require('svg2png-wasm');\n const wasmBuff = await readFile(path.resolve(path.dirname(require.resolve('svg2png-wasm')), '../svg2png_wasm_bg.wasm'));\n const fontBuff = await readFile(path.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf'));\n await initialize(wasmBuff);\n this.svg2png = createSvg2png({\n fonts: [Uint8Array.from(fontBuff)],\n defaultFontFamily: {\n serifFamily: 'Source Han Sans SC',\n sansSerifFamily: 'Source Han Sans SC',\n cursiveFamily: 'Source Han Sans SC',\n fantasyFamily: 'Source Han Sans SC',\n monospaceFamily: 'Source Han Sans SC',\n }\n });\n }\n this.canvas = require('@napi-rs/canvas');\n\n this.emit('ready');\n }\n\n @Threaded()\n async renderSvgToPng(svgContent: string,) {\n return this.svg2png(svgContent, { backgroundColor: '#D3D3D3' });\n }\n\n protected async _loadImage(input: string | Buffer) {\n let buff;\n let contentType;\n do {\n if (typeof input === 'string') {\n if (input.startsWith('data:')) {\n const firstComma = input.indexOf(',');\n const header = input.slice(0, firstComma);\n const data = input.slice(firstComma + 1);\n const encoding = header.split(';')[1];\n contentType = header.split(';')[0].split(':')[1];\n if (encoding?.startsWith('base64')) {\n buff = Buffer.from(data, 'base64');\n } else {\n buff = Buffer.from(decodeURIComponent(data), 'utf-8');\n }\n break;\n }\n if (input.startsWith('http')) {\n const r = await downloadFile(input);\n buff = Buffer.from(r.buff);\n contentType = r.contentType;\n break;\n }\n }\n if (Buffer.isBuffer(input)) {\n buff = input;\n const mime = await mimeOf(buff);\n contentType = `${mime.mediaType}/${mime.subType}`;\n break;\n }\n throw new ParamValidationError('Invalid input');\n } while (false);\n\n if (!buff) {\n throw new ParamValidationError('Invalid input');\n }\n\n if (contentType?.includes('svg')) {\n buff = await this.renderSvgToPng(buff.toString('utf-8'));\n }\n\n const img = await this.canvas.loadImage(buff);\n Reflect.set(img, 'contentType', contentType);\n\n return img;\n }\n\n async loadImage(uri: string | Buffer) {\n const t0 = Date.now();\n try {\n const theImage = await this._loadImage(uri);\n const t1 = Date.now();\n this.logger.debug(`Image loaded in ${t1 - t0}ms`);\n\n return theImage;\n } catch (err: any) {\n if (err?.message?.includes('Unsupported image type') || err?.message?.includes('unsupported')) {\n this.logger.warn(`Failed to load image ${uri.slice(0, 128)}`, { err });\n throw new SubmittedDataMalformedError(`Unknown image format for ${uri.slice(0, 128)}`);\n }\n throw err;\n }\n }\n\n fitImageToSquareBox(image: canvas.Image | canvas.Canvas, size: number = 1024) {\n // this.logger.debug(`Fitting image(${ image.width }x${ image.height }) to ${ size } box`);\n // const t0 = Date.now();\n if (image.width <= size && image.height <= size) {\n if (image instanceof this.canvas.Canvas) {\n return image;\n }\n const canvasInstance = this.canvas.createCanvas(image.width, image.height);\n const ctx = canvasInstance.getContext('2d');\n ctx.drawImage(image, 0, 0, image.width, image.height, 0, 0, canvasInstance.width, canvasInstance.height);\n // this.logger.debug(`No need to resize, copied to canvas in ${ Date.now() - t0 } ms`);\n\n return canvasInstance;\n }\n\n const aspectRatio = image.width / image.height;\n\n const resizedWidth = Math.round(aspectRatio > 1 ? size : size * aspectRatio);\n const resizedHeight = Math.round(aspectRatio > 1 ? size / aspectRatio : size);\n\n const canvasInstance = this.canvas.createCanvas(resizedWidth, resizedHeight);\n const ctx = canvasInstance.getContext('2d');\n ctx.drawImage(image, 0, 0, image.width, image.height, 0, 0, resizedWidth, resizedHeight);\n // this.logger.debug(`Resized to ${ resizedWidth }x${ resizedHeight } in ${ Date.now() - t0 } ms`);\n\n return canvasInstance;\n }\n\n corpImage(image: canvas.Image | canvas.Canvas, x: number, y: number, w: number, h: number) {\n // this.logger.debug(`Cropping image(${ image.width }x${ image.height }) to ${ w }x${ h } at ${ x },${ y } `);\n // const t0 = Date.now();\n const canvasInstance = this.canvas.createCanvas(w, h);\n const ctx = canvasInstance.getContext('2d');\n ctx.drawImage(image, x, y, w, h, 0, 0, w, h);\n // this.logger.debug(`Crop complete in ${ Date.now() - t0 } ms`);\n\n return canvasInstance;\n }\n\n canvasToDataUrl(canvas: canvas.Canvas, mimeType?: 'image/png' | 'image/jpeg') {\n // this.logger.debug(`Exporting canvas(${ canvas.width }x${ canvas.height })`);\n // const t0 = Date.now();\n return canvas.toDataURLAsync((mimeType || 'image/png') as 'image/png');\n }\n\n async canvasToBuffer(canvas: canvas.Canvas, mimeType?: 'image/png' | 'image/jpeg') {\n // this.logger.debug(`Exporting canvas(${ canvas.width }x${ canvas.height })`);\n // const t0 = Date.now();\n return canvas.toBuffer((mimeType || 'image/png') as 'image/png');\n }\n\n}\n\nconst instance = container.resolve(CanvasService);\nexport default instance;\n" }, { "path": "src/services/cf-browser-rendering.ts", "content": "import { container, singleton } from 'tsyringe';\nimport { AsyncService } from 'civkit/async-service';\nimport { SecretExposer } from '../shared/services/secrets';\nimport { GlobalLogger } from './logger';\nimport { CloudFlareHTTP } from '../shared/3rd-party/cloud-flare';\nimport { HTTPServiceError } from 'civkit/http';\nimport { ServiceNodeResourceDrainError } from './errors';\n\n@singleton()\nexport class CFBrowserRendering extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n client!: CloudFlareHTTP;\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected secretExposer: SecretExposer,\n ) {\n super(...arguments);\n }\n\n\n override async init() {\n await this.dependencyReady();\n const [account, key] = this.secretExposer.CLOUD_FLARE_API_KEY?.split(':');\n this.client = new CloudFlareHTTP(account, key);\n\n this.emit('ready');\n }\n\n async fetchContent(url: string) {\n try {\n const r = await this.client.fetchBrowserRenderedHTML({ url });\n\n return r.parsed.result;\n } catch (err) {\n if (err instanceof HTTPServiceError) {\n if (err.status === 429) {\n // Rate limit exceeded, return empty result\n this.logger.warn('Cloudflare browser rendering rate limit exceeded', { url });\n\n throw new ServiceNodeResourceDrainError(`Cloudflare browser rendering (our account) is at capacity, please try again later or switch to another engine.`,);\n }\n }\n\n throw err;\n }\n }\n\n}\n\nconst instance = container.resolve(CFBrowserRendering);\n\nexport default instance;\n" }, { "path": "src/services/curl.ts", "content": "import { AsyncService } from 'civkit/async-service';\nimport { singleton } from 'tsyringe';\n\nimport { Curl, CurlCode, CurlFeature, HeaderInfo } from 'node-libcurl';\nimport { parseString as parseSetCookieString } from 'set-cookie-parser';\n\nimport { ScrappingOptions } from './puppeteer';\nimport { GlobalLogger } from './logger';\nimport { AssertionFailureError, FancyFile } from 'civkit';\nimport { ServiceBadAttemptError, ServiceBadApproachError } from './errors';\nimport { TempFileManager } from '../services/temp-file';\nimport { createBrotliDecompress, createInflate, createGunzip } from 'zlib';\nimport { ZSTDDecompress } from 'simple-zstd';\nimport _ from 'lodash';\nimport { Readable } from 'stream';\nimport { AsyncLocalContext } from './async-context';\nimport { BlackHoleDetector } from './blackhole-detector';\n\nexport interface CURLScrappingOptions extends ScrappingOptions {\n method?: string;\n body?: string | Buffer;\n}\n\n@singleton()\nexport class CurlControl extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n chromeVersion: string = `132`;\n safariVersion: string = `537.36`;\n platform: string = `Linux`;\n ua: string = `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/${this.safariVersion} (KHTML, like Gecko) Chrome/${this.chromeVersion}.0.0.0 Safari/${this.safariVersion}`;\n\n lifeCycleTrack = new WeakMap();\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected tempFileManager: TempFileManager,\n protected asyncLocalContext: AsyncLocalContext,\n protected blackHoleDetector: BlackHoleDetector,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n\n if (process.platform === 'darwin') {\n this.platform = `macOS`;\n } else if (process.platform === 'win32') {\n this.platform = `Windows`;\n }\n\n this.emit('ready');\n }\n\n impersonateChrome(ua: string) {\n this.chromeVersion = ua.match(/Chrome\\/(\\d+)/)![1];\n this.safariVersion = ua.match(/AppleWebKit\\/([\\d\\.]+)/)![1];\n this.ua = ua;\n }\n\n curlImpersonateHeader(curl: Curl, headers?: object) {\n let uaPlatform = this.platform;\n if (this.ua.includes('Windows')) {\n uaPlatform = 'Windows';\n } else if (this.ua.includes('Android')) {\n uaPlatform = 'Android';\n } else if (this.ua.includes('iPhone') || this.ua.includes('iPad') || this.ua.includes('iPod')) {\n uaPlatform = 'iOS';\n } else if (this.ua.includes('CrOS')) {\n uaPlatform = 'Chrome OS';\n } else if (this.ua.includes('Macintosh')) {\n uaPlatform = 'macOS';\n }\n\n const mixinHeaders: Record = {\n 'Sec-Ch-Ua': `Not A(Brand\";v=\"8\", \"Chromium\";v=\"${this.chromeVersion}\", \"Google Chrome\";v=\"${this.chromeVersion}\"`,\n 'Sec-Ch-Ua-Mobile': '?0',\n 'Sec-Ch-Ua-Platform': `\"${uaPlatform}\"`,\n 'Upgrade-Insecure-Requests': '1',\n 'User-Agent': this.ua,\n 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',\n 'Sec-Fetch-Site': 'none',\n 'Sec-Fetch-Mode': 'navigate',\n 'Sec-Fetch-User': '?1',\n 'Sec-Fetch-Dest': 'document',\n 'Accept-Encoding': 'gzip, deflate, br, zstd',\n 'Accept-Language': 'en-US,en;q=0.9',\n };\n const headersCopy: Record = { ...headers };\n for (const k of Object.keys(mixinHeaders)) {\n const lowerK = k.toLowerCase();\n if (headersCopy[lowerK]) {\n mixinHeaders[k] = headersCopy[lowerK];\n delete headersCopy[lowerK];\n }\n }\n Object.assign(mixinHeaders, headersCopy);\n\n curl.setOpt(Curl.option.HTTPHEADER, Object.entries(mixinHeaders).flatMap(([k, v]) => {\n if (Array.isArray(v) && v.length) {\n return v.map((v2) => `${k}: ${v2}`);\n }\n return [`${k}: ${v}`];\n }));\n\n return curl;\n }\n\n urlToFile1Shot(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) {\n return new Promise<{\n statusCode: number,\n statusText?: string,\n data?: FancyFile,\n headers: HeaderInfo[],\n }>((resolve, reject) => {\n let contentType = '';\n const curl = new Curl();\n curl.enable(CurlFeature.StreamResponse);\n curl.setOpt('URL', urlToCrawl.toString());\n curl.setOpt(Curl.option.FOLLOWLOCATION, false);\n curl.setOpt(Curl.option.SSL_VERIFYPEER, false);\n curl.setOpt(Curl.option.TIMEOUT_MS, crawlOpts?.timeoutMs || 30_000);\n curl.setOpt(Curl.option.CONNECTTIMEOUT_MS, 3_000);\n curl.setOpt(Curl.option.LOW_SPEED_LIMIT, 32768);\n curl.setOpt(Curl.option.LOW_SPEED_TIME, 5_000);\n if (crawlOpts?.method) {\n curl.setOpt(Curl.option.CUSTOMREQUEST, crawlOpts.method.toUpperCase());\n }\n if (crawlOpts?.body) {\n curl.setOpt(Curl.option.POSTFIELDS, crawlOpts.body.toString());\n }\n\n const headersToSet = { ...crawlOpts?.extraHeaders };\n if (crawlOpts?.cookies?.length) {\n const cookieKv: Record = {};\n for (const cookie of crawlOpts.cookies) {\n cookieKv[cookie.name] = cookie.value;\n }\n for (const cookie of crawlOpts.cookies) {\n if (cookie.maxAge && cookie.maxAge < 0) {\n delete cookieKv[cookie.name];\n continue;\n }\n if (cookie.expires && cookie.expires < new Date()) {\n delete cookieKv[cookie.name];\n continue;\n }\n if (cookie.secure && urlToCrawl.protocol !== 'https:') {\n delete cookieKv[cookie.name];\n continue;\n }\n if (cookie.domain && !urlToCrawl.hostname.endsWith(cookie.domain)) {\n delete cookieKv[cookie.name];\n continue;\n }\n if (cookie.path && !urlToCrawl.pathname.startsWith(cookie.path)) {\n delete cookieKv[cookie.name];\n continue;\n }\n }\n const cookieChunks = Object.entries(cookieKv).map(([k, v]) => `${k}=${encodeURIComponent(v)}`);\n headersToSet.cookie ??= cookieChunks.join('; ');\n }\n if (crawlOpts?.referer) {\n headersToSet.referer ??= crawlOpts.referer;\n }\n if (crawlOpts?.overrideUserAgent) {\n headersToSet['user-agent'] ??= crawlOpts.overrideUserAgent;\n }\n\n this.curlImpersonateHeader(curl, headersToSet);\n\n if (crawlOpts?.proxyUrl) {\n const proxyUrlCopy = new URL(crawlOpts.proxyUrl);\n curl.setOpt(Curl.option.PROXY, proxyUrlCopy.href);\n }\n\n let curlStream: Readable | undefined;\n curl.on('error', (err, errCode) => {\n curl.close();\n this.logger.warn(`Curl ${urlToCrawl.origin}: ${err}`, { err, urlToCrawl });\n const err2 = this.digestCurlCode(errCode, err.message) ||\n new AssertionFailureError(`Failed to access ${urlToCrawl.origin}: ${err.message}`);\n err2.cause ??= err;\n if (curlStream) {\n // For some reason, manually emitting error event is required for curlStream.\n curlStream.emit('error', err2);\n curlStream.destroy(err2);\n }\n reject(err2);\n });\n curl.setOpt(Curl.option.MAXFILESIZE, 4 * 1024 * 1024 * 1024); // 4GB\n let status = -1;\n let statusText: string|undefined;\n let contentEncoding = '';\n curl.once('end', () => {\n if (curlStream) {\n curlStream.once('end', () => curl.close());\n return;\n }\n curl.close();\n });\n curl.on('stream', (stream, statusCode, headers) => {\n this.logger.debug(`CURL: [${statusCode}] ${urlToCrawl.origin}`, { statusCode });\n status = statusCode;\n curlStream = stream;\n for (const headerSet of (headers as HeaderInfo[])) {\n for (const [k, v] of Object.entries(headerSet)) {\n if (k.trim().endsWith(':')) {\n Reflect.set(headerSet, k.slice(0, k.indexOf(':')), v || '');\n Reflect.deleteProperty(headerSet, k);\n continue;\n }\n if (v === undefined) {\n Reflect.set(headerSet, k, '');\n continue;\n }\n if (k.toLowerCase() === 'content-type' && typeof v === 'string') {\n contentType = v.toLowerCase();\n }\n }\n }\n const lastResHeaders = headers[headers.length - 1];\n statusText = (lastResHeaders as HeaderInfo).result?.reason;\n for (const [k, v] of Object.entries(lastResHeaders)) {\n const kl = k.toLowerCase();\n if (kl === 'content-type') {\n contentType = (v || '').toLowerCase();\n }\n if (kl === 'content-encoding') {\n contentEncoding = (v || '').toLowerCase();\n }\n if (contentType && contentEncoding) {\n break;\n }\n }\n\n if ([301, 302, 303, 307, 308].includes(statusCode)) {\n if (stream) {\n stream.resume();\n }\n resolve({\n statusCode: status,\n statusText,\n data: undefined,\n headers: headers as HeaderInfo[],\n });\n return;\n }\n\n if (!stream) {\n resolve({\n statusCode: status,\n statusText,\n data: undefined,\n headers: headers as HeaderInfo[],\n });\n return;\n }\n\n switch (contentEncoding) {\n case 'gzip': {\n const decompressed = createGunzip();\n stream.pipe(decompressed);\n stream.once('error', (err) => {\n decompressed.destroy(err);\n });\n stream = decompressed;\n break;\n }\n case 'deflate': {\n const decompressed = createInflate();\n stream.pipe(decompressed);\n stream.once('error', (err) => {\n decompressed.destroy(err);\n });\n stream = decompressed;\n break;\n }\n case 'br': {\n const decompressed = createBrotliDecompress();\n stream.pipe(decompressed);\n stream.once('error', (err) => {\n decompressed.destroy(err);\n });\n stream = decompressed;\n break;\n }\n case 'zstd': {\n const decompressed = ZSTDDecompress();\n stream.pipe(decompressed);\n stream.once('error', (err) => {\n decompressed.destroy(err);\n });\n stream = decompressed;\n break;\n }\n default: {\n break;\n }\n }\n\n const fpath = this.tempFileManager.alloc();\n const fancyFile = FancyFile.auto(stream, fpath);\n this.tempFileManager.bindPathTo(fancyFile, fpath);\n resolve({\n statusCode: status,\n statusText,\n data: fancyFile,\n headers: headers as HeaderInfo[],\n });\n });\n\n curl.perform();\n });\n }\n\n async urlToFile(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) {\n let leftRedirection = 6;\n let cookieRedirects = 0;\n let opts = { ...crawlOpts };\n let nextHopUrl = urlToCrawl;\n const fakeHeaderInfos: HeaderInfo[] = [];\n do {\n const r = await this.urlToFile1Shot(nextHopUrl, opts);\n\n if ([301, 302, 303, 307, 308].includes(r.statusCode)) {\n fakeHeaderInfos.push(...r.headers);\n const headers = r.headers[r.headers.length - 1];\n const location: string | undefined = headers.Location || headers.location;\n\n const setCookieHeader = headers['Set-Cookie'] || headers['set-cookie'];\n if (setCookieHeader) {\n const cookieAssignments = Array.isArray(setCookieHeader) ? setCookieHeader : [setCookieHeader];\n const parsed = cookieAssignments.filter(Boolean).map((x) => parseSetCookieString(x, { decodeValues: true }));\n if (parsed.length) {\n opts.cookies = [...(opts.cookies || []), ...parsed];\n }\n if (!location) {\n cookieRedirects += 1;\n }\n }\n\n if (!location && !setCookieHeader) {\n // Follow curl behavior\n return {\n statusCode: r.statusCode,\n data: r.data,\n headers: fakeHeaderInfos.concat(r.headers),\n };\n }\n if (!location && cookieRedirects > 1) {\n throw new ServiceBadApproachError(`Failed to access ${urlToCrawl}: Browser required to solve complex cookie preconditions.`);\n }\n\n nextHopUrl = new URL(location || '', nextHopUrl);\n leftRedirection -= 1;\n continue;\n }\n\n return {\n statusCode: r.statusCode,\n statusText: r.statusText,\n data: r.data,\n headers: fakeHeaderInfos.concat(r.headers),\n };\n } while (leftRedirection > 0);\n\n throw new ServiceBadAttemptError(`Failed to access ${urlToCrawl}: Too many redirections.`);\n }\n\n async sideLoad(targetUrl: URL, crawlOpts?: CURLScrappingOptions) {\n const curlResult = await this.urlToFile(targetUrl, crawlOpts);\n this.blackHoleDetector.itWorked();\n let finalURL = targetUrl;\n const sideLoadOpts: CURLScrappingOptions['sideLoad'] = {\n impersonate: {},\n proxyOrigin: {},\n };\n for (const headers of curlResult.headers) {\n sideLoadOpts.impersonate[finalURL.href] = {\n status: headers.result?.code || -1,\n headers: _.omit(headers, 'result'),\n contentType: headers['Content-Type'] || headers['content-type'],\n };\n if (crawlOpts?.proxyUrl) {\n sideLoadOpts.proxyOrigin[finalURL.origin] = crawlOpts.proxyUrl;\n }\n if (headers.result?.code && [301, 302, 307, 308].includes(headers.result.code)) {\n const location = headers.Location || headers.location;\n if (location) {\n finalURL = new URL(location, finalURL);\n }\n }\n }\n const lastHeaders = curlResult.headers[curlResult.headers.length - 1];\n const contentType = (lastHeaders['Content-Type'] || lastHeaders['content-type'])?.toLowerCase() || (await curlResult.data?.mimeType) || 'application/octet-stream';\n const contentDisposition = lastHeaders['Content-Disposition'] || lastHeaders['content-disposition'];\n const fileName = contentDisposition?.match(/filename=\"([^\"]+)\"/i)?.[1] || finalURL.pathname.split('/').pop();\n\n if (sideLoadOpts.impersonate[finalURL.href] && (await curlResult.data?.size)) {\n sideLoadOpts.impersonate[finalURL.href].body = curlResult.data;\n }\n\n // This should keep the file from being garbage collected and deleted until this asyncContext/request is done.\n this.lifeCycleTrack.set(this.asyncLocalContext.ctx, curlResult.data);\n\n return {\n finalURL,\n sideLoadOpts,\n chain: curlResult.headers,\n status: curlResult.statusCode,\n statusText: curlResult.statusText,\n headers: lastHeaders,\n contentType,\n contentDisposition,\n fileName,\n file: curlResult.data\n };\n }\n\n digestCurlCode(code: CurlCode, msg: string) {\n switch (code) {\n // 400 User errors\n case CurlCode.CURLE_COULDNT_RESOLVE_HOST: {\n return new AssertionFailureError(msg);\n }\n\n // Maybe retry but dont retry with curl again\n case CurlCode.CURLE_OPERATION_TIMEDOUT:\n case CurlCode.CURLE_UNSUPPORTED_PROTOCOL:\n case CurlCode.CURLE_PEER_FAILED_VERIFICATION: {\n return new ServiceBadApproachError(msg);\n }\n\n // Retryable errors\n case CurlCode.CURLE_REMOTE_ACCESS_DENIED:\n case CurlCode.CURLE_SEND_ERROR:\n case CurlCode.CURLE_RECV_ERROR:\n case CurlCode.CURLE_GOT_NOTHING:\n case CurlCode.CURLE_SSL_CONNECT_ERROR:\n case CurlCode.CURLE_QUIC_CONNECT_ERROR:\n case CurlCode.CURLE_COULDNT_RESOLVE_PROXY:\n case CurlCode.CURLE_COULDNT_CONNECT:\n case CurlCode.CURLE_PARTIAL_FILE: {\n return new ServiceBadAttemptError(msg);\n }\n\n default: {\n return undefined;\n }\n }\n }\n}\n" }, { "path": "src/services/errors.ts", "content": "import { ApplicationError, StatusCode } from 'civkit/civ-rpc';\nimport _ from 'lodash';\nimport dayjs from 'dayjs';\nimport utc from 'dayjs/plugin/utc';\n\ndayjs.extend(utc);\n\n@StatusCode(50301)\nexport class ServiceDisabledError extends ApplicationError { }\n\n@StatusCode(50302)\nexport class ServiceCrashedError extends ApplicationError { }\n\n@StatusCode(50303)\nexport class ServiceNodeResourceDrainError extends ApplicationError { }\n\n@StatusCode(50304)\nexport class ServiceBadAttemptError extends ApplicationError { }\n\n@StatusCode(50305)\nexport class ServiceBadApproachError extends ServiceBadAttemptError { }\n\n@StatusCode(40104)\nexport class EmailUnverifiedError extends ApplicationError { }\n\n@StatusCode(40201)\nexport class InsufficientCreditsError extends ApplicationError { }\n\n@StatusCode(40202)\nexport class TierFeatureConstraintError extends ApplicationError { }\n\n@StatusCode(40203)\nexport class InsufficientBalanceError extends ApplicationError { }\n\n@StatusCode(40903)\nexport class LockConflictError extends ApplicationError { }\n\n@StatusCode(40904)\nexport class BudgetExceededError extends ApplicationError { }\n\n@StatusCode(45101)\nexport class HarmfulContentError extends ApplicationError { }\n\n@StatusCode(45102)\nexport class SecurityCompromiseError extends ApplicationError { }\n\n@StatusCode(41201)\nexport class BatchSizeTooLargeError extends ApplicationError { }\n" }, { "path": "src/services/finalizer.ts", "content": "import { AbstractFinalizerService } from 'civkit/finalizer';\nimport { container, singleton } from 'tsyringe';\nimport { isMainThread } from 'worker_threads';\nimport { GlobalLogger } from './logger';\n\nconst realProcessExit = process.exit;\nprocess.exit = ((code?: number) => {\n if (isMainThread) {\n return;\n }\n return realProcessExit(code);\n}) as typeof process.exit;\n\n@singleton()\nexport class FinalizerService extends AbstractFinalizerService {\n\n container = container;\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n override quitProcess(code?: string | number | null | undefined): never {\n return realProcessExit(code);\n }\n\n constructor(protected globalLogger: GlobalLogger) {\n super(...arguments);\n }\n\n override onUnhandledRejection(err: unknown, _triggeringPromise: Promise): void {\n this.logger.warn(`Unhandled promise rejection in pid ${process.pid}`, { err });\n }\n}\n\nconst instance = container.resolve(FinalizerService);\nexport const { Finalizer } = instance.decorators();\nexport default instance;\n\nif (isMainThread) {\n instance.serviceReady();\n}\n" }, { "path": "src/services/geoip.ts", "content": "import { container, singleton } from 'tsyringe';\nimport fsp from 'fs/promises';\nimport { CityResponse, Reader } from 'maxmind';\nimport { AsyncService, AutoCastable, Prop, runOnce } from 'civkit';\nimport { GlobalLogger } from './logger';\nimport path from 'path';\nimport { Threaded } from './threaded';\n\nexport enum GEOIP_SUPPORTED_LANGUAGES {\n EN = 'en',\n ZH_CN = 'zh-CN',\n JA = 'ja',\n DE = 'de',\n FR = 'fr',\n ES = 'es',\n PT_BR = 'pt-BR',\n RU = 'ru',\n}\n\nexport class GeoIPInfo extends AutoCastable {\n @Prop()\n code?: string;\n\n @Prop()\n name?: string;\n}\n\nexport class GeoIPCountryInfo extends GeoIPInfo {\n @Prop()\n eu?: boolean;\n}\n\nexport class GeoIPCityResponse extends AutoCastable {\n @Prop()\n continent?: GeoIPInfo;\n\n @Prop()\n country?: GeoIPCountryInfo;\n\n @Prop({\n arrayOf: GeoIPInfo\n })\n subdivisions?: GeoIPInfo[];\n\n @Prop()\n city?: string;\n\n @Prop({\n arrayOf: Number\n })\n coordinates?: [number, number, number];\n\n @Prop()\n timezone?: string;\n}\n\n@singleton()\nexport class GeoIPService extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n mmdbCity!: Reader;\n\n constructor(\n protected globalLogger: GlobalLogger,\n ) {\n super(...arguments);\n }\n\n\n override async init() {\n await this.dependencyReady();\n\n this.emit('ready');\n }\n\n @runOnce()\n async _lazyload() {\n const mmdpPath = path.resolve(__dirname, '..', '..', 'licensed', 'GeoLite2-City.mmdb');\n\n const dbBuff = await fsp.readFile(mmdpPath, { flag: 'r', encoding: null });\n\n this.mmdbCity = new Reader(dbBuff);\n\n this.logger.info(`Loaded GeoIP database, ${dbBuff.byteLength} bytes`);\n }\n\n\n @Threaded()\n async lookupCity(ip: string, lang: GEOIP_SUPPORTED_LANGUAGES = GEOIP_SUPPORTED_LANGUAGES.EN) {\n await this._lazyload();\n\n const r = this.mmdbCity.get(ip);\n\n if (!r) {\n return undefined;\n }\n\n return GeoIPCityResponse.from({\n continent: r.continent ? {\n code: r.continent?.code,\n name: r.continent?.names?.[lang] || r.continent?.names?.en,\n } : undefined,\n country: r.country ? {\n code: r.country?.iso_code,\n name: r.country?.names?.[lang] || r.country?.names.en,\n eu: r.country?.is_in_european_union,\n } : undefined,\n city: r.city?.names?.[lang] || r.city?.names?.en,\n subdivisions: r.subdivisions?.map((x) => ({\n code: x.iso_code,\n name: x.names?.[lang] || x.names?.en,\n })),\n coordinates: r.location ? [\n r.location.latitude, r.location.longitude, r.location.accuracy_radius\n ] : undefined,\n timezone: r.location?.time_zone,\n });\n }\n\n @Threaded()\n async lookupCities(ips: string[], lang: GEOIP_SUPPORTED_LANGUAGES = GEOIP_SUPPORTED_LANGUAGES.EN) {\n const r = (await Promise.all(ips.map((ip) => this.lookupCity(ip, lang)))).filter(Boolean) as GeoIPCityResponse[];\n\n return r;\n }\n\n}\n\nconst instance = container.resolve(GeoIPService);\n\nexport default instance;\n" }, { "path": "src/services/jsdom.ts", "content": "import { container, singleton } from 'tsyringe';\nimport { GlobalLogger } from './logger';\nimport { ExtendedSnapshot, ImgBrief, PageSnapshot } from './puppeteer';\nimport { Readability } from '@mozilla/readability';\nimport TurndownService from 'turndown';\nimport { Threaded } from '../services/threaded';\nimport type { ExtraScrappingOptions } from '../api/crawler';\nimport { tailwindClasses } from '../utils/tailwind-classes';\nimport { countGPTToken } from '../shared/utils/openai';\nimport { AsyncService } from 'civkit/async-service';\nimport { ApplicationError, AssertionFailureError } from 'civkit/civ-rpc';\n\nconst pLinkedom = import('linkedom');\n\n@singleton()\nexport class JSDomControl extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n linkedom!: Awaited;\n\n constructor(\n protected globalLogger: GlobalLogger,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.linkedom = await pLinkedom;\n this.emit('ready');\n }\n\n async narrowSnapshot(snapshot: PageSnapshot | undefined, options?: ExtraScrappingOptions) {\n if (snapshot?.parsed && !options?.targetSelector && !options?.removeSelector && !options?.withIframe && !options?.withShadowDom) {\n return snapshot;\n }\n if (!snapshot?.html) {\n return snapshot;\n }\n\n try {\n // SideLoad contains native objects that cannot go through thread boundaries.\n return await this.actualNarrowSnapshot(snapshot, { ...options, sideLoad: undefined });\n } catch (err: any) {\n this.logger.warn(`Error narrowing snapshot`, { err });\n if (err instanceof ApplicationError) {\n throw err;\n }\n\n throw new AssertionFailureError(`Failed to process the page: ${err?.message}`);\n }\n }\n\n @Threaded()\n async actualNarrowSnapshot(snapshot: PageSnapshot, options?: ExtraScrappingOptions): Promise {\n const t0 = Date.now();\n let sourceHTML = snapshot.html;\n if (options?.withShadowDom && snapshot.shadowExpanded) {\n sourceHTML = snapshot.shadowExpanded;\n }\n let jsdom = this.linkedom.parseHTML(sourceHTML);\n if (!jsdom.window.document.documentElement) {\n jsdom = this.linkedom.parseHTML(`${sourceHTML}`);\n }\n const allNodes: Node[] = [];\n jsdom.window.document.querySelectorAll('svg').forEach((x) => x.innerHTML = '');\n if (options?.withIframe) {\n jsdom.window.document.querySelectorAll('iframe[src],frame[src]').forEach((x) => {\n const src = x.getAttribute('src');\n const thisSnapshot = snapshot.childFrames?.find((f) => f.href === src);\n if (options?.withIframe === 'quoted') {\n const blockquoteElem = jsdom.window.document.createElement('blockquote');\n const preElem = jsdom.window.document.createElement('pre');\n preElem.innerHTML = thisSnapshot?.text || '';\n blockquoteElem.appendChild(preElem);\n x.replaceWith(blockquoteElem);\n } else if (thisSnapshot?.html) {\n x.innerHTML = thisSnapshot.html;\n x.querySelectorAll('script, style').forEach((s) => s.remove());\n if (src) {\n x.querySelectorAll('[src]').forEach((el) => {\n const imgSrc = el.getAttribute('src')!;\n if (URL.canParse(imgSrc, src!)) {\n el.setAttribute('src', new URL(imgSrc, src!).toString());\n }\n });\n x.querySelectorAll('[href]').forEach((el) => {\n const linkHref = el.getAttribute('href')!;\n if (URL.canParse(linkHref, src!)) {\n el.setAttribute('href', new URL(linkHref, src!).toString());\n }\n });\n }\n }\n });\n }\n\n if (Array.isArray(options?.removeSelector)) {\n for (const rl of options!.removeSelector) {\n jsdom.window.document.querySelectorAll(rl).forEach((x) => x.remove());\n }\n } else if (options?.removeSelector) {\n jsdom.window.document.querySelectorAll(options.removeSelector).forEach((x) => x.remove());\n }\n\n let bewareTargetContentDoesNotExist = false;\n if (Array.isArray(options?.targetSelector)) {\n bewareTargetContentDoesNotExist = true;\n for (const x of options!.targetSelector.map((x) => jsdom.window.document.querySelectorAll(x))) {\n x.forEach((el) => {\n if (!allNodes.includes(el)) {\n allNodes.push(el);\n }\n });\n }\n } else if (options?.targetSelector) {\n bewareTargetContentDoesNotExist = true;\n jsdom.window.document.querySelectorAll(options.targetSelector).forEach((el) => {\n if (!allNodes.includes(el)) {\n allNodes.push(el);\n }\n });\n } else {\n allNodes.push(jsdom.window.document);\n }\n\n if (!allNodes.length) {\n\n if (bewareTargetContentDoesNotExist) {\n return undefined;\n }\n\n return snapshot;\n }\n const textNodes: HTMLElement[] = [];\n let rootDoc: Document;\n if (allNodes.length === 1 && allNodes[0].nodeName === '#document' && (allNodes[0] as any).documentElement) {\n rootDoc = allNodes[0] as any;\n if (rootDoc.body?.innerText) {\n textNodes.push(rootDoc.body);\n }\n } else {\n rootDoc = this.linkedom.parseHTML('').window.document;\n for (const n of allNodes) {\n rootDoc.body.appendChild(n);\n rootDoc.body.appendChild(rootDoc.createTextNode('\\n\\n'));\n if ((n as HTMLElement).innerText) {\n textNodes.push(n as HTMLElement);\n }\n }\n }\n const textChunks = textNodes.map((x) => {\n const clone = x.cloneNode(true) as HTMLElement;\n clone.querySelectorAll('script,style,link,svg').forEach((s) => s.remove());\n\n return clone.innerText;\n });\n\n let parsed;\n try {\n parsed = new Readability(rootDoc.cloneNode(true) as any).parse();\n } catch (err: any) {\n this.logger.warn(`Failed to parse selected element`, { err });\n }\n\n const imgSet = new Set();\n const rebuiltImgs: ImgBrief[] = [];\n Array.from(rootDoc.querySelectorAll('img[src],img[data-src]'))\n .map((x: any) => [x.getAttribute('src'), x.getAttribute('data-src'), x.getAttribute('alt')])\n .forEach(([u1, u2, alt]) => {\n let absUrl: string | undefined;\n if (u1) {\n try {\n const u1Txt = new URL(u1, snapshot.rebase || snapshot.href).toString();\n imgSet.add(u1Txt);\n absUrl = u1Txt;\n } catch (err) {\n // void 0;\n }\n }\n if (u2) {\n try {\n const u2Txt = new URL(u2, snapshot.rebase || snapshot.href).toString();\n imgSet.add(u2Txt);\n absUrl = u2Txt;\n } catch (err) {\n // void 0;\n }\n }\n if (absUrl) {\n rebuiltImgs.push({\n src: absUrl,\n alt\n });\n }\n });\n\n const r = {\n ...snapshot,\n title: snapshot.title || jsdom.window.document.title,\n description: snapshot.description ||\n (jsdom.window.document.head?.querySelector('meta[name=\"description\"]')?.getAttribute('content') ?? ''),\n parsed,\n html: rootDoc.documentElement.outerHTML,\n text: textChunks.join('\\n'),\n imgs: (snapshot.imgs || rebuiltImgs)?.filter((x) => imgSet.has(x.src)) || [],\n } as PageSnapshot;\n\n const dt = Date.now() - t0;\n if (dt > 1000) {\n this.logger.warn(`Performance issue: Narrowing snapshot took ${dt}ms`, { url: snapshot.href, dt });\n }\n\n return r;\n }\n\n @Threaded()\n async inferSnapshot(snapshot: PageSnapshot) {\n const t0 = Date.now();\n const extendedSnapshot = { ...snapshot } as ExtendedSnapshot;\n try {\n const jsdom = this.linkedom.parseHTML(snapshot.html);\n\n jsdom.window.document.querySelectorAll('svg').forEach((x) => x.innerHTML = '');\n const links = Array.from(jsdom.window.document.querySelectorAll('a[href]'))\n .map((x: any) => [x.textContent.replace(/\\s+/g, ' ').trim(), x.getAttribute('href'),])\n .map(([text, href]) => {\n if (!href) {\n return undefined;\n }\n try {\n const parsed = new URL(href, snapshot.rebase || snapshot.href);\n\n return [text, parsed.toString()] as const;\n } catch (err) {\n return undefined;\n }\n })\n .filter(Boolean) as [string, string][];\n\n extendedSnapshot.links = links;\n\n const imgs = Array.from(jsdom.window.document.querySelectorAll('img[src],img[data-src]'))\n .map((x: any) => {\n let linkPreferredSrc = x.getAttribute('src') || '';\n if (linkPreferredSrc.startsWith('data:')) {\n const dataSrc = x.getAttribute('data-src') || '';\n if (dataSrc && !dataSrc.startsWith('data:')) {\n linkPreferredSrc = dataSrc;\n }\n }\n\n return {\n src: new URL(linkPreferredSrc, snapshot.rebase || snapshot.href).toString(),\n width: parseInt(x.getAttribute('width') || '0'),\n height: parseInt(x.getAttribute('height') || '0'),\n alt: x.getAttribute('alt') || x.getAttribute('title'),\n };\n });\n\n extendedSnapshot.imgs = imgs as any;\n } catch (_err) {\n void 0;\n }\n\n const dt = Date.now() - t0;\n if (dt > 1000) {\n this.logger.warn(`Performance issue: Inferring snapshot took ${dt}ms`, { url: snapshot.href, dt });\n }\n\n return extendedSnapshot;\n }\n\n cleanRedundantEmptyLines(text: string) {\n const lines = text.split(/\\r?\\n/g);\n const mappedFlag = lines.map((line) => Boolean(line.trim()));\n\n return lines.filter((_line, i) => mappedFlag[i] || mappedFlag[i - 1]).join('\\n');\n }\n\n @Threaded()\n async cleanHTMLforLMs(sourceHTML: string, ...discardSelectors: string[]): Promise {\n const t0 = Date.now();\n let jsdom = this.linkedom.parseHTML(sourceHTML);\n if (!jsdom.window.document.documentElement) {\n jsdom = this.linkedom.parseHTML(`${sourceHTML}`);\n }\n\n for (const rl of discardSelectors) {\n jsdom.window.document.querySelectorAll(rl).forEach((x) => x.remove());\n }\n\n jsdom.window.document.querySelectorAll('img[src],img[data-src]').forEach((x) => {\n const src = x.getAttribute('src') || x.getAttribute('data-src');\n if (src?.startsWith('data:')) {\n x.setAttribute('src', 'blob:opaque');\n }\n x.removeAttribute('data-src');\n x.removeAttribute('srcset');\n });\n\n jsdom.window.document.querySelectorAll('[class]').forEach((x) => {\n const classes = x.getAttribute('class')?.split(/\\s+/g) || [];\n const newClasses = classes.filter((c) => !tailwindClasses.has(c));\n x.setAttribute('class', newClasses.join(' '));\n });\n jsdom.window.document.querySelectorAll('[style]').forEach((x) => {\n const style = x.getAttribute('style')?.toLocaleLowerCase() || '';\n if (style.startsWith('display: none')) {\n return;\n }\n x.removeAttribute('style');\n });\n const treeWalker = jsdom.window.document.createTreeWalker(\n jsdom.window.document, // Start from the root document\n 0x80 // Only show comment nodes\n );\n\n let currentNode;\n while ((currentNode = treeWalker.nextNode())) {\n currentNode.parentNode?.removeChild(currentNode); // Remove each comment node\n }\n\n jsdom.window.document.querySelectorAll('*').forEach((x) => {\n const attrs = x.getAttributeNames();\n for (const attr of attrs) {\n if (attr.startsWith('data-') || attr.startsWith('aria-')) {\n x.removeAttribute(attr);\n }\n }\n });\n\n const final = this.cleanRedundantEmptyLines(jsdom.window.document.documentElement.outerHTML);\n\n const dt = Date.now() - t0;\n if (dt > 1000) {\n this.logger.warn(`Performance issue: Cleaning HTML for LMs took ${dt}ms`, { dt });\n }\n\n return final;\n }\n\n snippetToElement(snippet?: string, url?: string) {\n const parsed = this.linkedom.parseHTML(snippet || '');\n\n // Hack for turndown gfm table plugin.\n parsed.window.document.querySelectorAll('table').forEach((x) => {\n Object.defineProperty(x, 'rows', { value: Array.from(x.querySelectorAll('tr')), enumerable: true });\n });\n Object.defineProperty(parsed.window.document.documentElement, 'cloneNode', {\n value: function () { return this; },\n });\n\n return parsed.window.document.documentElement;\n }\n\n runTurndown(turndownService: TurndownService, html: TurndownService.Node | string) {\n const t0 = Date.now();\n\n try {\n return turndownService.turndown(html);\n } finally {\n const dt = Date.now() - t0;\n if (dt > 1000) {\n this.logger.warn(`Performance issue: Turndown took ${dt}ms`, { dt });\n }\n }\n }\n\n @Threaded()\n async analyzeHTMLTextLite(sourceHTML: string) {\n let jsdom = this.linkedom.parseHTML(sourceHTML);\n if (!jsdom.window.document.documentElement) {\n jsdom = this.linkedom.parseHTML(`${sourceHTML}`);\n }\n jsdom.window.document.querySelectorAll('script,style,link,svg').forEach((s) => s.remove());\n const text = jsdom.window.document.body.innerText || '';\n\n return {\n title: jsdom.window.document.title,\n text,\n tokens: countGPTToken(text.replaceAll(/[\\s\\r\\n\\t]+/g, ' ')),\n };\n }\n}\n\nconst jsdomControl = container.resolve(JSDomControl);\n\nexport default jsdomControl;\n" }, { "path": "src/services/lm.ts", "content": "import { AsyncService } from 'civkit/async-service';\nimport { singleton } from 'tsyringe';\n\nimport { PageSnapshot } from './puppeteer';\nimport { GlobalLogger } from './logger';\nimport _ from 'lodash';\nimport { AssertionFailureError } from 'civkit';\nimport { LLMManager } from '../shared/services/common-llm';\nimport { JSDomControl } from './jsdom';\n\nconst tripleBackTick = '```';\n\n@singleton()\nexport class LmControl extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected commonLLM: LLMManager,\n protected jsdomControl: JSDomControl,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n\n this.emit('ready');\n }\n\n async* geminiFromBrowserSnapshot(snapshot?: PageSnapshot & {\n pageshotUrl?: string,\n }) {\n const pageshot = snapshot?.pageshotUrl || snapshot?.pageshot;\n\n if (!pageshot) {\n throw new AssertionFailureError('Screenshot of the page is not available');\n }\n\n const html = await this.jsdomControl.cleanHTMLforLMs(snapshot.html, 'script,link,style,textarea,select>option,svg');\n\n const it = this.commonLLM.iterRun('vertex-gemini-1.5-flash-002', {\n prompt: [\n `HTML: \\n${html}\\n\\nSCREENSHOT: \\n`,\n typeof pageshot === 'string' ? new URL(pageshot) : pageshot,\n `Convert this webpage into a markdown source file that does not contain HTML tags, retaining the page language and visual structures.`,\n ],\n\n options: {\n system: 'You are ReaderLM-v7, a model that generates Markdown source files only. No HTML, notes and chit-chats allowed',\n stream: true\n }\n });\n\n const chunks: string[] = [];\n for await (const txt of it) {\n chunks.push(txt);\n const output: PageSnapshot = {\n ...snapshot,\n parsed: {\n ...snapshot?.parsed,\n textContent: chunks.join(''),\n }\n };\n yield output;\n }\n\n return;\n }\n\n async* readerLMMarkdownFromSnapshot(snapshot?: PageSnapshot) {\n if (!snapshot) {\n throw new AssertionFailureError('Snapshot of the page is not available');\n }\n\n const html = await this.jsdomControl.cleanHTMLforLMs(snapshot.html, 'script,link,style,textarea,select>option,svg');\n\n const it = this.commonLLM.iterRun('readerlm-v2', {\n prompt: `Extract the main content from the given HTML and convert it to Markdown format.\\n\\n${tripleBackTick}html\\n${html}\\n${tripleBackTick}\\n`,\n\n options: {\n // system: 'You are an AI assistant developed by VENDOR_NAME',\n stream: true,\n modelSpecific: {\n top_k: 1,\n temperature: 0,\n repetition_penalty: 1.13,\n presence_penalty: 0.25,\n frequency_penalty: 0.25,\n max_tokens: 8192,\n }\n },\n maxTry: 1,\n });\n\n const chunks: string[] = [];\n for await (const txt of it) {\n chunks.push(txt);\n const output: PageSnapshot = {\n ...snapshot,\n parsed: {\n ...snapshot?.parsed,\n textContent: chunks.join(''),\n }\n };\n yield output;\n }\n\n return;\n }\n\n async* readerLMFromSnapshot(schema?: string, instruction: string = 'Infer useful information from the HTML and present it in a structured JSON object.', snapshot?: PageSnapshot) {\n if (!snapshot) {\n throw new AssertionFailureError('Snapshot of the page is not available');\n }\n\n const html = await this.jsdomControl.cleanHTMLforLMs(snapshot.html, 'script,link,style,textarea,select>option,svg');\n\n const it = this.commonLLM.iterRun('readerlm-v2', {\n prompt: `${instruction}\\n\\n${tripleBackTick}html\\n${html}\\n${tripleBackTick}\\n${schema ? `The JSON schema:\\n${tripleBackTick}json\\n${schema}\\n${tripleBackTick}\\n` : ''}`,\n options: {\n // system: 'You are an AI assistant developed by VENDOR_NAME',\n stream: true,\n modelSpecific: {\n top_k: 1,\n temperature: 0,\n repetition_penalty: 1.13,\n presence_penalty: 0.25,\n frequency_penalty: 0.25,\n max_tokens: 8192,\n }\n },\n maxTry: 1,\n });\n\n const chunks: string[] = [];\n for await (const txt of it) {\n chunks.push(txt);\n const output: PageSnapshot = {\n ...snapshot,\n parsed: {\n ...snapshot?.parsed,\n textContent: chunks.join(''),\n }\n };\n yield output;\n }\n\n return;\n }\n}\n" }, { "path": "src/services/logger.ts", "content": "import { AbstractPinoLogger } from 'civkit/pino-logger';\nimport { singleton, container } from 'tsyringe';\nimport { threadId } from 'node:worker_threads';\nimport { getTraceCtx } from 'civkit/async-context';\n\n\nconst levelToSeverityMap: { [k: string]: string | undefined; } = {\n trace: 'DEFAULT',\n debug: 'DEBUG',\n info: 'INFO',\n warn: 'WARNING',\n error: 'ERROR',\n fatal: 'CRITICAL',\n};\n\n@singleton()\nexport class GlobalLogger extends AbstractPinoLogger {\n loggerOptions = {\n level: 'debug',\n base: {\n tid: threadId,\n }\n };\n\n override init(): void {\n if (process.env['NODE_ENV']?.startsWith('prod')) {\n super.init(process.stdout);\n } else {\n const PinoPretty = require('pino-pretty').PinoPretty;\n super.init(PinoPretty({\n singleLine: true,\n colorize: true,\n messageFormat(log: any, messageKey: any) {\n return `${log['tid'] ? `[${log['tid']}]` : ''}[${log['service'] || 'ROOT'}] ${log[messageKey]}`;\n },\n }));\n }\n\n\n this.emit('ready');\n }\n\n override log(...args: any[]) {\n const [levelObj, ...rest] = args;\n const severity = levelToSeverityMap[levelObj?.level];\n const traceCtx = getTraceCtx();\n const patched: any= { ...levelObj, severity };\n const traceId = traceCtx?.googleTraceId || traceCtx?.traceId;\n if (traceId && process.env['GCLOUD_PROJECT']) {\n patched['logging.googleapis.com/trace'] = `projects/${process.env['GCLOUD_PROJECT']}/traces/${traceId}`;\n }\n return super.log(patched, ...rest);\n }\n}\n\nconst instance = container.resolve(GlobalLogger);\nexport default instance;\n" }, { "path": "src/services/minimal-stealth.js", "content": "\n\nexport function minimalStealth() {\n /**\n * A set of shared utility functions specifically for the purpose of modifying native browser APIs without leaving traces.\n *\n * Meant to be passed down in puppeteer and used in the context of the page (everything in here runs in NodeJS as well as a browser).\n *\n * Note: If for whatever reason you need to use this outside of `puppeteer-extra`:\n * Just remove the `module.exports` statement at the very bottom, the rest can be copy pasted into any browser context.\n *\n * Alternatively take a look at the `extract-stealth-evasions` package to create a finished bundle which includes these utilities.\n *\n */\n const utils = {};\n\n // const toStringRedirects = new WeakMap();\n\n utils.init = () => {\n utils.preloadCache();\n // const handler = {\n // apply: function (target, ctx) {\n // // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + \"\"`\n // if (ctx === Function.prototype.toString) {\n // return utils.makeNativeString('toString');\n // }\n\n // const originalObj = toStringRedirects.get(ctx);\n\n // // `toString` targeted at our proxied Object detected\n // if (originalObj) {\n // const fallback = () =>\n // originalObj && originalObj.name\n // ? utils.makeNativeString(originalObj.name)\n // : utils.makeNativeString(ctx.name);\n\n // // Return the toString representation of our original object if possible\n // return originalObj + '' || fallback();\n // }\n\n // if (typeof ctx === 'undefined' || ctx === null) {\n // return target.call(ctx);\n // }\n\n // // Check if the toString protype of the context is the same as the global prototype,\n // // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case\n // const hasSameProto = Object.getPrototypeOf(\n // Function.prototype.toString\n // ).isPrototypeOf(ctx.toString); // eslint-disable-line no-prototype-builtins\n // if (!hasSameProto) {\n // // Pass the call on to the local Function.prototype.toString instead\n // return ctx.toString();\n // }\n\n // return target.call(ctx);\n // }\n // };\n\n // const toStringProxy = new Proxy(\n // Function.prototype.toString,\n // utils.stripProxyFromErrors(handler)\n // );\n // utils.replaceProperty(Function.prototype, 'toString', {\n // value: toStringProxy\n // });\n };\n\n /**\n * Wraps a JS Proxy Handler and strips it's presence from error stacks, in case the traps throw.\n *\n * The presence of a JS Proxy can be revealed as it shows up in error stack traces.\n *\n * @param {object} handler - The JS Proxy handler to wrap\n */\n utils.stripProxyFromErrors = (handler = {}) => {\n const newHandler = {\n setPrototypeOf: function (target, proto) {\n if (proto === null)\n throw new TypeError('Cannot convert object to primitive value');\n if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {\n throw new TypeError('Cyclic __proto__ value');\n }\n return Reflect.setPrototypeOf(target, proto);\n }\n };\n // We wrap each trap in the handler in a try/catch and modify the error stack if they throw\n const traps = Object.getOwnPropertyNames(handler);\n traps.forEach(trap => {\n newHandler[trap] = function () {\n try {\n // Forward the call to the defined proxy handler\n return handler[trap].call(this, ...(arguments || []));\n } catch (err) {\n // Stack traces differ per browser, we only support chromium based ones currently\n if (!err || !err.stack || !err.stack.includes(`at `)) {\n throw err;\n }\n\n // When something throws within one of our traps the Proxy will show up in error stacks\n // An earlier implementation of this code would simply strip lines with a blacklist,\n // but it makes sense to be more surgical here and only remove lines related to our Proxy.\n // We try to use a known \"anchor\" line for that and strip it with everything above it.\n // If the anchor line cannot be found for some reason we fall back to our blacklist approach.\n\n const stripWithBlacklist = (stack, stripFirstLine = true) => {\n const blacklist = [\n `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply\n `at Object.${trap} `, // e.g. Object.get or Object.apply\n `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-)\n ];\n return (\n err.stack\n .split('\\n')\n // Always remove the first (file) line in the stack (guaranteed to be our proxy)\n .filter((line, index) => !(index === 1 && stripFirstLine))\n // Check if the line starts with one of our blacklisted strings\n .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))\n .join('\\n')\n );\n };\n\n const stripWithAnchor = (stack, anchor) => {\n const stackArr = stack.split('\\n');\n anchor = anchor || `at Object.newHandler. [as ${trap}] `; // Known first Proxy line in chromium\n const anchorIndex = stackArr.findIndex(line =>\n line.trim().startsWith(anchor)\n );\n if (anchorIndex === -1) {\n return false; // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex);\n return stackArr.join('\\n');\n };\n\n // Special cases due to our nested toString proxies\n err.stack = err.stack.replace(\n 'at Object.toString (',\n 'at Function.toString ('\n );\n if ((err.stack || '').includes('at Function.toString (')) {\n err.stack = stripWithBlacklist(err.stack, false);\n throw err;\n }\n\n // Try using the anchor method, fallback to blacklist if necessary\n err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack);\n\n throw err; // Re-throw our now sanitized error\n }\n };\n });\n return newHandler;\n };\n\n /**\n * Strip error lines from stack traces until (and including) a known line the stack.\n *\n * @param {object} err - The error to sanitize\n * @param {string} anchor - The string the anchor line starts with\n */\n utils.stripErrorWithAnchor = (err, anchor) => {\n const stackArr = err.stack.split('\\n');\n const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor));\n if (anchorIndex === -1) {\n return err; // 404, anchor not found\n }\n // Strip everything from the top until we reach the anchor line (remove anchor line as well)\n // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`)\n stackArr.splice(1, anchorIndex);\n err.stack = stackArr.join('\\n');\n return err;\n };\n\n /**\n * Replace the property of an object in a stealthy way.\n *\n * Note: You also want to work on the prototype of an object most often,\n * as you'd otherwise leave traces (e.g. showing up in Object.getOwnPropertyNames(obj)).\n *\n * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty\n *\n * @example\n * replaceProperty(WebGLRenderingContext.prototype, 'getParameter', { value: \"alice\" })\n * // or\n * replaceProperty(Object.getPrototypeOf(navigator), 'languages', { get: () => ['en-US', 'en'] })\n *\n * @param {object} obj - The object which has the property to replace\n * @param {string} propName - The property name to replace\n * @param {object} descriptorOverrides - e.g. { value: \"alice\" }\n */\n utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => {\n return Object.defineProperty(obj, propName, {\n // Copy over the existing descriptors (writable, enumerable, configurable, etc)\n ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),\n // Add our overrides (e.g. value, get())\n ...descriptorOverrides\n });\n };\n\n /**\n * Preload a cache of function copies and data.\n *\n * For a determined enough observer it would be possible to overwrite and sniff usage of functions\n * we use in our internal Proxies, to combat that we use a cached copy of those functions.\n *\n * Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before,\n * by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups).\n *\n * This is evaluated once per execution context (e.g. window)\n */\n utils.preloadCache = () => {\n if (utils.cache) {\n return;\n }\n utils.cache = {\n // Used in our proxies\n Reflect: {\n get: Reflect.get.bind(Reflect),\n apply: Reflect.apply.bind(Reflect)\n },\n // Used in `makeNativeString`\n nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }`\n };\n };\n\n /**\n * Utility function to generate a cross-browser `toString` result representing native code.\n *\n * There's small differences: Chromium uses a single line, whereas FF & Webkit uses multiline strings.\n * To future-proof this we use an existing native toString result as the basis.\n *\n * The only advantage we have over the other team is that our JS runs first, hence we cache the result\n * of the native toString result once, so they cannot spoof it afterwards and reveal that we're using it.\n *\n * @example\n * makeNativeString('foobar') // => `function foobar() { [native code] }`\n *\n * @param {string} [name] - Optional function name\n */\n utils.makeNativeString = (name = '') => {\n return utils.cache.nativeToStringStr.replace('toString', name || '');\n };\n\n /**\n * Helper function to modify the `toString()` result of the provided object.\n *\n * Note: Use `utils.redirectToString` instead when possible.\n *\n * There's a quirk in JS Proxies that will cause the `toString()` result to differ from the vanilla Object.\n * If no string is provided we will generate a `[native code]` thing based on the name of the property object.\n *\n * @example\n * patchToString(WebGLRenderingContext.prototype.getParameter, 'function getParameter() { [native code] }')\n *\n * @param {object} obj - The object for which to modify the `toString()` representation\n * @param {string} str - Optional string used as a return value\n */\n utils.patchToString = (obj, str = '') => {\n // toStringRedirects.set(obj, str);\n Object.defineProperty(obj, 'toString', {\n value: ()=> str,\n enumerable: false,\n writable: true,\n configurable: true,\n });\n };\n\n /**\n * Make all nested functions of an object native.\n *\n * @param {object} obj\n */\n utils.patchToStringNested = (obj = {}) => {\n return utils.execRecursively(obj, ['function'], utils.patchToString);\n };\n\n /**\n * Redirect toString requests from one object to another.\n *\n * @param {object} proxyObj - The object that toString will be called on\n * @param {object} originalObj - The object which toString result we wan to return\n */\n utils.redirectToString = (proxyObj, originalObj) => {\n // toStringRedirects.set(proxyObj, originalObj);\n Object.defineProperty(proxyObj, 'toString', {\n value: ()=> originalObj.toString(),\n enumerable: false,\n writable: true,\n configurable: true,\n });\n };\n\n\n /**\n * All-in-one method to replace a property with a JS Proxy using the provided Proxy handler with traps.\n *\n * Will stealthify these aspects (strip error stack traces, redirect toString, etc).\n * Note: This is meant to modify native Browser APIs and works best with prototype objects.\n *\n * @example\n * replaceWithProxy(WebGLRenderingContext.prototype, 'getParameter', proxyHandler)\n *\n * @param {object} obj - The object which has the property to replace\n * @param {string} propName - The name of the property to replace\n * @param {object} handler - The JS Proxy handler to use\n */\n utils.replaceWithProxy = (obj, propName, handler) => {\n const originalObj = obj[propName];\n const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler));\n\n utils.replaceProperty(obj, propName, { value: proxyObj });\n utils.redirectToString(proxyObj, originalObj);\n\n return true;\n };\n /**\n * All-in-one method to replace a getter with a JS Proxy using the provided Proxy handler with traps.\n *\n * @example\n * replaceGetterWithProxy(Object.getPrototypeOf(navigator), 'vendor', proxyHandler)\n *\n * @param {object} obj - The object which has the property to replace\n * @param {string} propName - The name of the property to replace\n * @param {object} handler - The JS Proxy handler to use\n */\n utils.replaceGetterWithProxy = (obj, propName, handler) => {\n const fn = Object.getOwnPropertyDescriptor(obj, propName).get;\n const fnStr = fn.toString(); // special getter function string\n const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler));\n\n utils.replaceProperty(obj, propName, { get: proxyObj });\n utils.patchToString(proxyObj, fnStr);\n\n return true;\n };\n\n /**\n * All-in-one method to replace a getter and/or setter. Functions get and set\n * of handler have one more argument that contains the native function.\n *\n * @example\n * replaceGetterSetter(HTMLIFrameElement.prototype, 'contentWindow', handler)\n *\n * @param {object} obj - The object which has the property to replace\n * @param {string} propName - The name of the property to replace\n * @param {object} handlerGetterSetter - The handler with get and/or set\n * functions\n * @see https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty#description\n */\n utils.replaceGetterSetter = (obj, propName, handlerGetterSetter) => {\n const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName);\n const handler = { ...ownPropertyDescriptor };\n\n if (handlerGetterSetter.get !== undefined) {\n const nativeFn = ownPropertyDescriptor.get;\n handler.get = function () {\n return handlerGetterSetter.get.call(this, nativeFn.bind(this));\n };\n utils.redirectToString(handler.get, nativeFn);\n }\n\n if (handlerGetterSetter.set !== undefined) {\n const nativeFn = ownPropertyDescriptor.set;\n handler.set = function (newValue) {\n handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this));\n };\n utils.redirectToString(handler.set, nativeFn);\n }\n\n Object.defineProperty(obj, propName, handler);\n };\n\n /**\n * All-in-one method to mock a non-existing property with a JS Proxy using the provided Proxy handler with traps.\n *\n * Will stealthify these aspects (strip error stack traces, redirect toString, etc).\n *\n * @example\n * mockWithProxy(chrome.runtime, 'sendMessage', function sendMessage() {}, proxyHandler)\n *\n * @param {object} obj - The object which has the property to replace\n * @param {string} propName - The name of the property to replace or create\n * @param {object} pseudoTarget - The JS Proxy target to use as a basis\n * @param {object} handler - The JS Proxy handler to use\n */\n utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler));\n\n utils.replaceProperty(obj, propName, { value: proxyObj });\n utils.patchToString(proxyObj);\n\n return true;\n };\n\n /**\n * All-in-one method to create a new JS Proxy with stealth tweaks.\n *\n * This is meant to be used whenever we need a JS Proxy but don't want to replace or mock an existing known property.\n *\n * Will stealthify certain aspects of the Proxy (strip error stack traces, redirect toString, etc).\n *\n * @example\n * createProxy(navigator.mimeTypes.__proto__.namedItem, proxyHandler) // => Proxy\n *\n * @param {object} pseudoTarget - The JS Proxy target to use as a basis\n * @param {object} handler - The JS Proxy handler to use\n */\n utils.createProxy = (pseudoTarget, handler) => {\n const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler));\n utils.patchToString(proxyObj);\n\n return proxyObj;\n };\n\n /**\n * Helper function to split a full path to an Object into the first part and property.\n *\n * @example\n * splitObjPath(`HTMLMediaElement.prototype.canPlayType`)\n * // => {objName: \"HTMLMediaElement.prototype\", propName: \"canPlayType\"}\n *\n * @param {string} objPath - The full path to an object as dot notation string\n */\n utils.splitObjPath = objPath => ({\n // Remove last dot entry (property) ==> `HTMLMediaElement.prototype`\n objName: objPath.split('.').slice(0, -1).join('.'),\n // Extract last dot entry ==> `canPlayType`\n propName: objPath.split('.').slice(-1)[0]\n });\n\n /**\n * Convenience method to replace a property with a JS Proxy using the provided objPath.\n *\n * Supports a full path (dot notation) to the object as string here, in case that makes it easier.\n *\n * @example\n * replaceObjPathWithProxy('WebGLRenderingContext.prototype.getParameter', proxyHandler)\n *\n * @param {string} objPath - The full path to an object (dot notation string) to replace\n * @param {object} handler - The JS Proxy handler to use\n */\n utils.replaceObjPathWithProxy = (objPath, handler) => {\n const { objName, propName } = utils.splitObjPath(objPath);\n const obj = eval(objName); // eslint-disable-line no-eval\n return utils.replaceWithProxy(obj, propName, handler);\n };\n\n /**\n * Traverse nested properties of an object recursively and apply the given function on a whitelist of value types.\n *\n * @param {object} obj\n * @param {array} typeFilter - e.g. `['function']`\n * @param {Function} fn - e.g. `utils.patchToString`\n */\n utils.execRecursively = (obj = {}, typeFilter = [], fn) => {\n function recurse(obj) {\n for (const key in obj) {\n if (obj[key] === undefined) {\n continue;\n }\n if (obj[key] && typeof obj[key] === 'object') {\n recurse(obj[key]);\n } else {\n if (obj[key] && typeFilter.includes(typeof obj[key])) {\n fn.call(this, obj[key]);\n }\n }\n }\n }\n recurse(obj);\n return obj;\n };\n\n /**\n * Everything we run through e.g. `page.evaluate` runs in the browser context, not the NodeJS one.\n * That means we cannot just use reference variables and functions from outside code, we need to pass everything as a parameter.\n *\n * Unfortunately the data we can pass is only allowed to be of primitive types, regular functions don't survive the built-in serialization process.\n * This utility function will take an object with functions and stringify them, so we can pass them down unharmed as strings.\n *\n * We use this to pass down our utility functions as well as any other functions (to be able to split up code better).\n *\n * @see utils.materializeFns\n *\n * @param {object} fnObj - An object containing functions as properties\n */\n utils.stringifyFns = (fnObj = { hello: () => 'world' }) => {\n // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine\n // https://github.com/feross/fromentries\n function fromEntries(iterable) {\n return [...iterable].reduce((obj, [key, val]) => {\n obj[key] = val;\n return obj;\n }, {});\n }\n return (Object.fromEntries || fromEntries)(\n Object.entries(fnObj)\n .filter(([key, value]) => typeof value === 'function')\n .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval\n );\n };\n\n /**\n * Utility function to reverse the process of `utils.stringifyFns`.\n * Will materialize an object with stringified functions (supports classic and fat arrow functions).\n *\n * @param {object} fnStrObj - An object containing stringified functions as properties\n */\n utils.materializeFns = (fnStrObj = { hello: \"() => 'world'\" }) => {\n return Object.fromEntries(\n Object.entries(fnStrObj).map(([key, value]) => {\n if (value.startsWith('function')) {\n // some trickery is needed to make oldschool functions work :-)\n return [key, eval(`() => ${value}`)()]; // eslint-disable-line no-eval\n } else {\n // arrow functions just work\n return [key, eval(value)]; // eslint-disable-line no-eval\n }\n })\n );\n };\n\n // Proxy handler templates for re-usability\n utils.makeHandler = () => ({\n // Used by simple `navigator` getter evasions\n getterValue: value => ({\n apply(target, ctx, args) {\n // Let's fetch the value first, to trigger and escalate potential errors\n // Illegal invocations like `navigator.__proto__.vendor` will throw here\n utils.cache.Reflect.apply(...arguments);\n return value;\n }\n })\n });\n\n /**\n * Compare two arrays.\n *\n * @param {array} array1 - First array\n * @param {array} array2 - Second array\n */\n utils.arrayEquals = (array1, array2) => {\n if (array1.length !== array2.length) {\n return false;\n }\n for (let i = 0; i < array1.length; ++i) {\n if (array1[i] !== array2[i]) {\n return false;\n }\n }\n return true;\n };\n\n /**\n * Cache the method return according to its arguments.\n *\n * @param {Function} fn - A function that will be cached\n */\n utils.memoize = fn => {\n const cache = [];\n return function (...args) {\n if (!cache.some(c => utils.arrayEquals(c.key, args))) {\n cache.push({ key: args, value: fn.apply(this, args) });\n }\n return cache.find(c => utils.arrayEquals(c.key, args)).value;\n };\n };\n\n utils.init();\n\n const getParameterProxyHandler = {\n apply: function (target, ctx, args) {\n const param = (args || [])[0]\n const result = utils.cache.Reflect.apply(target, ctx, args)\n // UNMASKED_VENDOR_WEBGL\n if (param === 37445) {\n return 'Intel Inc.' // default in headless: Google Inc.\n }\n // UNMASKED_RENDERER_WEBGL\n if (param === 37446) {\n return 'Intel Iris OpenGL Engine' // default in headless: Google SwiftShader\n }\n return result\n }\n }\n\n // There's more than one WebGL rendering context\n // https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility\n // To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter)\n const addProxy = (obj, propName) => {\n utils.replaceWithProxy(obj, propName, getParameterProxyHandler)\n }\n // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing:\n addProxy(WebGLRenderingContext.prototype, 'getParameter')\n addProxy(WebGL2RenderingContext.prototype, 'getParameter')\n\n}" }, { "path": "src/services/misc.ts", "content": "import { singleton } from 'tsyringe';\nimport { AsyncService } from 'civkit/async-service';\nimport { ParamValidationError } from 'civkit/civ-rpc';\nimport { SecurityCompromiseError } from '../shared/lib/errors';\nimport { isIP } from 'node:net';\nimport { isIPInNonPublicRange } from '../utils/ip';\nimport { GlobalLogger } from './logger';\nimport { lookup } from 'node:dns/promises';\nimport { Threaded } from './threaded';\n\nconst normalizeUrl = require('@esm2cjs/normalize-url').default;\n\n@singleton()\nexport class MiscService extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n constructor(\n protected globalLogger: GlobalLogger,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n\n this.emit('ready');\n }\n\n @Threaded()\n async assertNormalizedUrl(input: string) {\n let result: URL;\n try {\n result = new URL(\n normalizeUrl(\n input,\n {\n stripWWW: false,\n removeTrailingSlash: false,\n removeSingleSlash: false,\n sortQueryParameters: false,\n }\n )\n );\n } catch (err) {\n throw new ParamValidationError({\n message: `${err}`,\n path: 'url'\n });\n }\n\n if (!['http:', 'https:', 'blob:'].includes(result.protocol)) {\n throw new ParamValidationError({\n message: `Invalid protocol ${result.protocol}`,\n path: 'url'\n });\n }\n\n const normalizedHostname = result.hostname.startsWith('[') ? result.hostname.slice(1, -1) : result.hostname;\n let ips: string[] = [];\n const isIp = isIP(normalizedHostname);\n if (isIp) {\n ips.push(normalizedHostname);\n }\n if (\n (result.hostname === 'localhost') ||\n (isIp && isIPInNonPublicRange(normalizedHostname))\n ) {\n this.logger.warn(`Suspicious action: Request to localhost or non-public IP: ${normalizedHostname}`, { href: result.href });\n throw new SecurityCompromiseError({\n message: `Suspicious action: Request to localhost or non-public IP: ${normalizedHostname}`,\n path: 'url'\n });\n }\n if (!isIp && result.protocol !== 'blob:') {\n const resolved = await lookup(result.hostname, { all: true }).catch((err) => {\n if (err.code === 'ENOTFOUND') {\n return Promise.reject(new ParamValidationError({\n message: `Domain '${result.hostname}' could not be resolved`,\n path: 'url'\n }));\n }\n\n return;\n });\n if (resolved) {\n for (const x of resolved) {\n if (isIPInNonPublicRange(x.address)) {\n this.logger.warn(`Suspicious action: Domain resolved to non-public IP: ${result.hostname} => ${x.address}`, { href: result.href, ip: x.address });\n throw new SecurityCompromiseError({\n message: `Suspicious action: Domain resolved to non-public IP: ${x.address}`,\n path: 'url'\n });\n }\n ips.push(x.address);\n }\n\n }\n }\n\n return {\n url: result,\n ips\n };\n }\n\n}" }, { "path": "src/services/pdf-extract.ts", "content": "import { singleton } from 'tsyringe';\nimport _ from 'lodash';\nimport { TextItem } from 'pdfjs-dist/types/src/display/api';\nimport { AssertionFailureError, AsyncService, HashManager } from 'civkit';\nimport { GlobalLogger } from './logger';\nimport { PDFContent } from '../db/pdf';\nimport dayjs from 'dayjs';\nimport { FirebaseStorageBucketControl } from '../shared/services/firebase-storage-bucket';\nimport { randomUUID } from 'crypto';\nimport type { PDFDocumentLoadingTask } from 'pdfjs-dist';\nimport path from 'path';\nimport { AsyncLocalContext } from './async-context';\nconst utc = require('dayjs/plugin/utc'); // Import the UTC plugin\ndayjs.extend(utc); // Extend dayjs with the UTC plugin\nconst timezone = require('dayjs/plugin/timezone');\ndayjs.extend(timezone);\n\nconst pPdfjs = import('pdfjs-dist/legacy/build/pdf.mjs');\nconst nodeCmapUrl = path.resolve(require.resolve('pdfjs-dist'), '../../cmaps') + '/';\n\nconst md5Hasher = new HashManager('md5', 'hex');\n\nfunction stdDev(numbers: number[]) {\n const mean = _.mean(numbers);\n const squareDiffs = numbers.map((num) => Math.pow(num - mean, 2));\n const avgSquareDiff = _.mean(squareDiffs);\n return Math.sqrt(avgSquareDiff);\n}\n\nfunction isRotatedByAtLeast35Degrees(transform?: [number, number, number, number, number, number]): boolean {\n if (!transform) {\n return false;\n }\n const [a, b, c, d, _e, _f] = transform;\n\n // Calculate the rotation angles using arctan(b/a) and arctan(-c/d)\n const angle1 = Math.atan2(b, a) * (180 / Math.PI); // from a, b\n const angle2 = Math.atan2(-c, d) * (180 / Math.PI); // from c, d\n\n // Either angle1 or angle2 can be used to determine the rotation, they should be equivalent\n const rotationAngle1 = Math.abs(angle1);\n const rotationAngle2 = Math.abs(angle2);\n\n // Check if the absolute rotation angle is greater than or equal to 35 degrees\n return rotationAngle1 >= 35 || rotationAngle2 >= 35;\n}\n\n@singleton()\nexport class PDFExtractor extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n pdfjs!: Awaited;\n\n cacheRetentionMs = 1000 * 3600 * 24 * 7;\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected firebaseObjectStorage: FirebaseStorageBucketControl,\n protected asyncLocalContext: AsyncLocalContext,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.pdfjs = await pPdfjs;\n\n this.emit('ready');\n }\n\n isDataUrl(url: string) {\n return url.startsWith('data:');\n }\n\n parseDataUrl(url: string) {\n const protocol = url.slice(0, url.indexOf(':'));\n const contentType = url.slice(url.indexOf(':') + 1, url.indexOf(';'));\n const data = url.slice(url.indexOf(',') + 1);\n if (protocol !== 'data' || !data) {\n throw new Error('Invalid data URL');\n }\n\n if (contentType !== 'application/pdf') {\n throw new Error('Invalid data URL type');\n }\n\n return {\n type: contentType,\n data: data\n };\n }\n\n async extract(url: string | URL) {\n let loadingTask: PDFDocumentLoadingTask;\n\n if (typeof url === 'string' && this.isDataUrl(url)) {\n const { data } = this.parseDataUrl(url);\n const binary = Uint8Array.from(Buffer.from(data, 'base64'));\n loadingTask = this.pdfjs.getDocument({\n data: binary,\n disableFontFace: true,\n verbosity: 0,\n cMapUrl: nodeCmapUrl,\n });\n } else {\n loadingTask = this.pdfjs.getDocument({\n url,\n disableFontFace: true,\n verbosity: 0,\n cMapUrl: nodeCmapUrl,\n });\n }\n\n\n const doc = await loadingTask.promise;\n const meta = await doc.getMetadata();\n\n const textItems: TextItem[][] = [];\n\n for (const pg of _.range(0, doc.numPages)) {\n const page = await doc.getPage(pg + 1);\n const textContent = await page.getTextContent({ includeMarkedContent: true });\n textItems.push((textContent.items as TextItem[]));\n }\n\n const articleCharHeights: number[] = [];\n for (const textItem of textItems.flat()) {\n if (textItem.height) {\n articleCharHeights.push(...Array(textItem.str.length).fill(textItem.height));\n }\n }\n const articleAvgHeight = _.mean(articleCharHeights);\n const articleStdDevHeight = stdDev(articleCharHeights);\n // const articleMedianHeight = articleCharHeights.sort()[Math.floor(articleCharHeights.length / 2)];\n const mdOps: Array<{\n text: string;\n op?: 'new' | 'append';\n mode: 'h1' | 'h2' | 'p' | 'appendix' | 'space';\n }> = [];\n\n const rawChunks: string[] = [];\n\n let op: 'append' | 'new' = 'new';\n let mode: 'h1' | 'h2' | 'p' | 'space' | 'appendix' = 'p';\n for (const pageTextItems of textItems) {\n const charHeights = [];\n for (const textItem of pageTextItems as TextItem[]) {\n if (textItem.height) {\n charHeights.push(...Array(textItem.str.length).fill(textItem.height));\n }\n rawChunks.push(`${textItem.str}${textItem.hasEOL ? '\\n' : ''}`);\n }\n\n const avgHeight = _.mean(charHeights);\n const stdDevHeight = stdDev(charHeights);\n // const medianHeight = charHeights.sort()[Math.floor(charHeights.length / 2)];\n\n for (const textItem of pageTextItems) {\n if (textItem.height > articleAvgHeight + 3 * articleStdDevHeight) {\n mode = 'h1';\n } else if (textItem.height > articleAvgHeight + 2 * articleStdDevHeight) {\n mode = 'h2';\n } else if (textItem.height && textItem.height < avgHeight - stdDevHeight) {\n mode = 'appendix';\n } else if (textItem.height) {\n mode = 'p';\n } else {\n mode = 'space';\n }\n\n if (isRotatedByAtLeast35Degrees(textItem.transform as any)) {\n mode = 'appendix';\n }\n\n mdOps.push({\n op,\n mode,\n text: textItem.str\n });\n\n if (textItem.hasEOL && !textItem.str) {\n op = 'new';\n } else {\n op = 'append';\n }\n }\n }\n\n const mdChunks = [];\n const appendixChunks = [];\n mode = 'space';\n for (const x of mdOps) {\n const previousMode: string = mode;\n const changeToMdChunks = [];\n\n const isNewStart = x.mode !== 'space' && (x.op === 'new' || (previousMode === 'appendix' && x.mode !== previousMode));\n\n if (isNewStart) {\n switch (x.mode) {\n case 'h1': {\n changeToMdChunks.push(`\\n\\n# `);\n mode = x.mode;\n break;\n }\n\n case 'h2': {\n changeToMdChunks.push(`\\n\\n## `);\n mode = x.mode;\n break;\n }\n\n case 'p': {\n changeToMdChunks.push(`\\n\\n`);\n mode = x.mode;\n break;\n }\n\n case 'appendix': {\n mode = x.mode;\n appendixChunks.push(`\\n\\n`);\n break;\n }\n\n default: {\n break;\n }\n }\n } else {\n if (x.mode === 'appendix' && appendixChunks.length) {\n const lastChunk = appendixChunks[appendixChunks.length - 1];\n if (!lastChunk.match(/(\\s+|-)$/) && lastChunk.length !== 1) {\n appendixChunks.push(' ');\n }\n } else if (mdChunks.length) {\n const lastChunk = mdChunks[mdChunks.length - 1];\n if (!lastChunk.match(/(\\s+|-)$/) && lastChunk.length !== 1) {\n changeToMdChunks.push(' ');\n }\n }\n }\n\n if (x.text) {\n if (x.mode == 'appendix') {\n if (appendixChunks.length || isNewStart) {\n appendixChunks.push(x.text);\n } else {\n changeToMdChunks.push(x.text);\n }\n } else {\n changeToMdChunks.push(x.text);\n }\n }\n\n if (isNewStart && x.mode !== 'appendix' && appendixChunks.length) {\n const appendix = appendixChunks.join('').split(/\\r?\\n/).map((x) => x.trim()).filter(Boolean).map((x) => `> ${x}`).join('\\n');\n changeToMdChunks.unshift(appendix);\n changeToMdChunks.unshift(`\\n\\n`);\n appendixChunks.length = 0;\n }\n\n if (x.mode === 'space' && changeToMdChunks.length) {\n changeToMdChunks.length = 1;\n }\n if (changeToMdChunks.length) {\n mdChunks.push(...changeToMdChunks);\n }\n }\n\n if (mdChunks.length) {\n mdChunks[0] = mdChunks[0].trimStart();\n }\n\n return { meta: meta.info as Record, content: mdChunks.join(''), text: rawChunks.join('') };\n }\n\n async cachedExtract(url: string, cacheTolerance: number = 1000 * 3600 * 24, alternativeUrl?: string) {\n if (!url) {\n return undefined;\n }\n let nameUrl = alternativeUrl || url;\n const digest = md5Hasher.hash(nameUrl);\n\n if (this.isDataUrl(url)) {\n nameUrl = `blob://pdf:${digest}`;\n }\n\n const cache: PDFContent | undefined = nameUrl.startsWith('blob:') ? undefined :\n (await PDFContent.fromFirestoreQuery(PDFContent.COLLECTION.where('urlDigest', '==', digest).orderBy('createdAt', 'desc').limit(1)))?.[0];\n\n if (cache) {\n const age = Date.now() - cache?.createdAt.valueOf();\n const stale = cache.createdAt.valueOf() < (Date.now() - cacheTolerance);\n this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for PDF ${nameUrl}, normalized digest: ${digest}, ${age}ms old, tolerance ${cacheTolerance}ms`, {\n data: url, url: nameUrl, digest, age, stale, cacheTolerance\n });\n\n if (!stale) {\n if (cache.content && cache.text) {\n return {\n meta: cache.meta,\n content: cache.content,\n text: cache.text\n };\n }\n\n try {\n const r = await this.firebaseObjectStorage.downloadFile(`pdfs/${cache._id}`);\n let cached = JSON.parse(r.toString('utf-8'));\n\n return {\n meta: cached.meta,\n content: cached.content,\n text: cached.text\n };\n } catch (err) {\n this.logger.warn(`Unable to load cached content for ${nameUrl}`, { err });\n\n return undefined;\n }\n }\n }\n\n let extracted;\n\n try {\n extracted = await this.extract(url);\n } catch (err: any) {\n this.logger.warn(`Unable to extract from pdf ${nameUrl}`, { err, url, nameUrl });\n throw new AssertionFailureError(`Unable to process ${nameUrl} as pdf: ${err?.message}`);\n }\n\n if (!this.asyncLocalContext.ctx.DNT && !nameUrl.startsWith('blob:')) {\n const theID = randomUUID();\n await this.firebaseObjectStorage.saveFile(`pdfs/${theID}`,\n Buffer.from(JSON.stringify(extracted), 'utf-8'), { contentType: 'application/json' });\n PDFContent.save(\n PDFContent.from({\n _id: theID,\n src: nameUrl,\n meta: extracted?.meta || {},\n urlDigest: digest,\n createdAt: new Date(),\n expireAt: new Date(Date.now() + this.cacheRetentionMs)\n }).degradeForFireStore()\n ).catch((r) => {\n this.logger.warn(`Unable to cache PDF content for ${nameUrl}`, { err: r });\n });\n }\n\n return extracted;\n }\n\n parsePdfDate(pdfDate: string | undefined) {\n if (!pdfDate) {\n return undefined;\n }\n // Remove the 'D:' prefix\n const cleanedDate = pdfDate.slice(2);\n\n // Define the format without the timezone part first\n const dateTimePart = cleanedDate.slice(0, 14);\n const timezonePart = cleanedDate.slice(14);\n\n // Construct the full date string in a standard format\n const formattedDate = `${dateTimePart}${timezonePart.replace(\"'\", \"\").replace(\"'\", \"\")}`;\n\n // Parse the date with timezone\n const parsedDate = dayjs(formattedDate, \"YYYYMMDDHHmmssZ\");\n\n const date = parsedDate.toDate();\n\n if (!date.valueOf()) {\n return undefined;\n }\n\n return date;\n }\n}\n" }, { "path": "src/services/pseudo-transfer.ts", "content": "import { marshalErrorLike } from 'civkit';\nimport { AbstractPseudoTransfer, SYM_PSEUDO_TRANSFERABLE } from 'civkit/pseudo-transfer';\nimport { container, singleton } from 'tsyringe';\n\n\n@singleton()\nexport class PseudoTransfer extends AbstractPseudoTransfer {\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n }\n\n}\n\nconst instance = container.resolve(PseudoTransfer);\n\nObject.defineProperty(Error.prototype, SYM_PSEUDO_TRANSFERABLE, {\n value: function () {\n const prototype = this;\n return {\n copyOwnProperty: 'all',\n marshall: (input: Error) => marshalErrorLike(input),\n unMarshall: (input: object) => {\n Object.setPrototypeOf(input, prototype);\n return input;\n },\n };\n },\n enumerable: false,\n});\ninstance.expectPseudoTransferableType(Error);\nfor (const x of [...Object.values(require('./errors')), ...Object.values(require('civkit/civ-rpc'))]) {\n if (typeof x === 'function' && x.prototype instanceof Error) {\n instance.expectPseudoTransferableType(x as any);\n }\n}\n\n\nObject.defineProperty(URL.prototype, SYM_PSEUDO_TRANSFERABLE, {\n value: function () {\n return {\n copyOwnProperty: 'none',\n marshall: (input: URL) => ({ href: input.href }),\n unMarshall: (input: { href: string; }) => new URL(input.href),\n };\n },\n enumerable: false,\n});\ninstance.expectPseudoTransferableType(URL);\n\nObject.defineProperty(Buffer.prototype, SYM_PSEUDO_TRANSFERABLE, {\n value: function () {\n return {\n copyOwnProperty: 'none',\n unMarshall: (input: Uint8Array | Buffer) => Buffer.isBuffer(input) ? input : Buffer.from(input),\n marshall: (input: Uint8Array | Buffer) => input,\n };\n },\n enumerable: false,\n});\ninstance.expectPseudoTransferableType(Buffer);\n\n\nexport default instance;\n" }, { "path": "src/services/puppeteer.ts", "content": "import _ from 'lodash';\nimport { isIP } from 'net';\nimport { readFile } from 'fs/promises';\nimport fs from 'fs';\nimport { container, singleton } from 'tsyringe';\n\nimport type { Browser, CookieParam, GoToOptions, HTTPRequest, HTTPResponse, Page, Viewport } from 'puppeteer';\nimport type { Cookie } from 'set-cookie-parser';\nimport puppeteer, { TimeoutError } from 'puppeteer';\n\nimport { Defer, Deferred } from 'civkit/defer';\nimport { AssertionFailureError, ParamValidationError } from 'civkit/civ-rpc';\nimport { AsyncService } from 'civkit/async-service';\nimport { FancyFile } from 'civkit/fancy-file';\nimport { delay } from 'civkit/timeout';\n\nimport { SecurityCompromiseError, ServiceCrashedError, ServiceNodeResourceDrainError } from '../shared/lib/errors';\nimport { CurlControl } from './curl';\nimport { BlackHoleDetector } from './blackhole-detector';\nimport { AsyncLocalContext } from './async-context';\nimport { GlobalLogger } from './logger';\nimport { minimalStealth } from './minimal-stealth';\nconst tldExtract = require('tld-extract');\n\nconst READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8');\n\n\nexport interface ImgBrief {\n src: string;\n loaded?: boolean;\n width?: number;\n height?: number;\n naturalWidth?: number;\n naturalHeight?: number;\n alt?: string;\n}\n\nexport interface ReadabilityParsed {\n title: string;\n content: string;\n textContent: string;\n length: number;\n excerpt: string;\n byline: string;\n dir: string;\n siteName: string;\n lang: string;\n publishedTime: string;\n}\n\nexport interface PageSnapshot {\n title: string;\n description?: string;\n href: string;\n rebase?: string;\n html: string;\n htmlSignificantlyModifiedByJs?: boolean;\n shadowExpanded?: string;\n text: string;\n status?: number;\n statusText?: string;\n parsed?: Partial | null;\n screenshot?: Buffer;\n pageshot?: Buffer;\n imgs?: ImgBrief[];\n pdfs?: string[];\n maxElemDepth?: number;\n elemCount?: number;\n childFrames?: PageSnapshot[];\n isIntermediate?: boolean;\n isFromCache?: boolean;\n lastMutationIdle?: number;\n lastContentResourceLoaded?: number;\n lastMediaResourceLoaded?: number;\n}\n\nexport interface ExtendedSnapshot extends PageSnapshot {\n links: [string, string][];\n imgs: ImgBrief[];\n}\n\nexport interface ScrappingOptions {\n proxyUrl?: string;\n cookies?: Cookie[];\n favorScreenshot?: boolean;\n waitForSelector?: string | string[];\n minIntervalMs?: number;\n overrideUserAgent?: string;\n timeoutMs?: number;\n locale?: string;\n referer?: string;\n extraHeaders?: Record;\n injectFrameScripts?: string[];\n injectPageScripts?: string[];\n viewport?: Viewport;\n proxyResources?: boolean;\n\n sideLoad?: {\n impersonate: {\n [url: string]: {\n status: number;\n headers: { [k: string]: string | string[]; };\n contentType?: string;\n body?: FancyFile;\n };\n };\n proxyOrigin: { [origin: string]: string; };\n };\n\n}\n\nconst SIMULATE_SCROLL = `\n(function () {\n function createIntersectionObserverEntry(target, isIntersecting, timestamp) {\n const targetRect = target.getBoundingClientRect();\n const record = {\n target,\n isIntersecting,\n time: timestamp,\n // If intersecting, intersectionRect matches boundingClientRect\n // If not intersecting, intersectionRect is empty (0x0)\n intersectionRect: isIntersecting\n ? targetRect\n : new DOMRectReadOnly(0, 0, 0, 0),\n // Current bounding client rect of the target\n boundingClientRect: targetRect,\n // Intersection ratio is either 0 (not intersecting) or 1 (fully intersecting)\n intersectionRatio: isIntersecting ? 1 : 0,\n // Root bounds (viewport in our case)\n rootBounds: new DOMRectReadOnly(\n 0,\n 0,\n window.innerWidth,\n window.innerHeight\n )\n };\n Object.setPrototypeOf(record, window.IntersectionObserverEntry.prototype);\n return record;\n }\n function cloneIntersectionObserverEntry(entry) {\n const record = {\n target: entry.target,\n isIntersecting: entry.isIntersecting,\n time: entry.time,\n intersectionRect: entry.intersectionRect,\n boundingClientRect: entry.boundingClientRect,\n intersectionRatio: entry.intersectionRatio,\n rootBounds: entry.rootBounds\n };\n Object.setPrototypeOf(record, window.IntersectionObserverEntry.prototype);\n return record;\n }\n const orig = window.IntersectionObserver;\n const kCallback = Symbol('callback');\n const kLastEntryMap = Symbol('lastEntryMap');\n const liveObservers = new Map();\n class MangledIntersectionObserver extends orig {\n constructor(callback, options) {\n super((entries, observer) => {\n const lastEntryMap = observer[kLastEntryMap];\n const lastEntry = entries[entries.length - 1];\n lastEntryMap.set(lastEntry.target, lastEntry);\n return callback(entries, observer);\n }, options);\n this[kCallback] = callback;\n this[kLastEntryMap] = new WeakMap();\n liveObservers.set(this, new Set());\n }\n disconnect() {\n liveObservers.get(this)?.clear();\n liveObservers.delete(this);\n return super.disconnect();\n }\n observe(target) {\n const observer = liveObservers.get(this);\n observer?.add(target);\n return super.observe(target);\n }\n unobserve(target) {\n const observer = liveObservers.get(this);\n observer?.delete(target);\n return super.unobserve(target);\n }\n }\n Object.defineProperty(MangledIntersectionObserver, 'name', { value: 'IntersectionObserver', writable: false });\n window.IntersectionObserver = MangledIntersectionObserver;\n function simulateScroll() {\n for (const [observer, targets] of liveObservers.entries()) {\n const t0 = performance.now();\n for (const target of targets) {\n const entry = createIntersectionObserverEntry(target, true, t0);\n observer[kCallback]([entry], observer);\n setTimeout(() => {\n const t1 = performance.now();\n const lastEntry = observer[kLastEntryMap].get(target);\n if (!lastEntry) {\n return;\n }\n const entry2 = { ...cloneIntersectionObserverEntry(lastEntry), time: t1 };\n observer[kCallback]([entry2], observer);\n });\n }\n }\n }\n window.simulateScroll = simulateScroll;\n})();\n`;\n\nconst MUTATION_IDLE_WATCH = `\n(function () {\n let timeout;\n const sendMsg = ()=> {\n document.dispatchEvent(new CustomEvent('mutationIdle'));\n };\n\n const cb = () => {\n if (timeout) {\n clearTimeout(timeout);\n timeout = setTimeout(sendMsg, 200);\n }\n };\n const mutationObserver = new MutationObserver(cb);\n\n document.addEventListener('DOMContentLoaded', () => {\n mutationObserver.observe(document.documentElement, {\n childList: true,\n subtree: true,\n });\n timeout = setTimeout(sendMsg, 200);\n }, { once: true })\n})();\n`;\n\nconst SCRIPT_TO_INJECT_INTO_FRAME = `\n${READABILITY_JS}\n${SIMULATE_SCROLL}\n${MUTATION_IDLE_WATCH}\n(${minimalStealth.toString()})();\n\n(function(){\nfunction briefImgs(elem) {\n const imageTags = Array.from((elem || document).querySelectorAll('img[src],img[data-src]'));\n\n return imageTags.map((x)=> {\n let linkPreferredSrc = x.src;\n if (linkPreferredSrc.startsWith('data:')) {\n if (typeof x.dataset?.src === 'string' && !x.dataset.src.startsWith('data:')) {\n linkPreferredSrc = x.dataset.src;\n }\n }\n\n return {\n src: new URL(linkPreferredSrc, document.baseURI).toString(),\n loaded: x.complete,\n width: x.width,\n height: x.height,\n naturalWidth: x.naturalWidth,\n naturalHeight: x.naturalHeight,\n alt: x.alt || x.title,\n };\n });\n}\nfunction getMaxDepthAndElemCountUsingTreeWalker(root=document.documentElement) {\n let maxDepth = 0;\n let currentDepth = 0;\n let elementCount = 0;\n\n const treeWalker = document.createTreeWalker(\n root,\n NodeFilter.SHOW_ELEMENT,\n (node) => {\n const nodeName = node.nodeName?.toLowerCase();\n return (nodeName === 'svg') ? NodeFilter.FILTER_REJECT : NodeFilter.FILTER_ACCEPT;\n },\n false\n );\n\n while (true) {\n maxDepth = Math.max(maxDepth, currentDepth);\n elementCount++; // Increment the count for the current node\n\n if (treeWalker.firstChild()) {\n currentDepth++;\n } else {\n while (!treeWalker.nextSibling() && currentDepth > 0) {\n treeWalker.parentNode();\n currentDepth--;\n }\n\n if (currentDepth <= 0) {\n break;\n }\n }\n }\n\n return {\n maxDepth: maxDepth + 1,\n elementCount: elementCount\n };\n}\n\nfunction cloneAndExpandShadowRoots(rootElement = document.documentElement) {\n // Create a shallow clone of the root element\n const clone = rootElement.cloneNode(false);\n // Function to process an element and its shadow root\n function processShadowRoot(original, cloned) {\n if (original.shadowRoot && original.shadowRoot.mode === 'open') {\n shadowDomPresents = true;\n const shadowContent = document.createDocumentFragment();\n\n // Clone shadow root content normally\n original.shadowRoot.childNodes.forEach(childNode => {\n const clonedNode = childNode.cloneNode(true);\n shadowContent.appendChild(clonedNode);\n });\n\n // Handle slots\n const slots = shadowContent.querySelectorAll('slot');\n slots.forEach(slot => {\n const slotName = slot.getAttribute('name') || '';\n const assignedElements = original.querySelectorAll(\n slotName ? \\`[slot=\"\\${slotName}\"]\\` : ':not([slot])'\n );\n\n if (assignedElements.length > 0) {\n const slotContent = document.createDocumentFragment();\n assignedElements.forEach(el => {\n const clonedEl = el.cloneNode(true);\n slotContent.appendChild(clonedEl);\n });\n slot.parentNode.replaceChild(slotContent, slot);\n } else if (!slotName) {\n // Keep default slot content\n // No need to do anything as it's already cloned\n }\n });\n\n cloned.appendChild(shadowContent);\n }\n }\n\n // Use a TreeWalker on the original root to clone the entire structure\n const treeWalker = document.createTreeWalker(\n rootElement,\n NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT\n );\n\n const elementMap = new Map([[rootElement, clone]]);\n\n let currentNode;\n while (currentNode = treeWalker.nextNode()) {\n const parentClone = elementMap.get(currentNode.parentNode);\n const clonedNode = currentNode.cloneNode(false);\n parentClone.appendChild(clonedNode);\n\n if (currentNode.nodeType === Node.ELEMENT_NODE) {\n elementMap.set(currentNode, clonedNode);\n processShadowRoot(currentNode, clonedNode);\n }\n }\n\n return clone;\n}\n\nfunction shadowDomPresent(rootElement = document.documentElement) {\n const elems = rootElement.querySelectorAll('*');\n for (const x of elems) {\n if (x.shadowRoot && x.shadowRoot.mode === 'open') {\n return true;\n }\n }\n return false;\n}\n\nlet lastMutationIdle = 0;\nlet initialAnalytics;\ndocument.addEventListener('mutationIdle', ()=> lastMutationIdle = Date.now());\n\nfunction giveSnapshot(stopActiveSnapshot, overrideDomAnalysis) {\n if (stopActiveSnapshot) {\n window.haltSnapshot = true;\n }\n let parsed;\n try {\n parsed = new Readability(document.cloneNode(true)).parse();\n } catch (err) {\n void 0;\n }\n const domAnalysis = overrideDomAnalysis || getMaxDepthAndElemCountUsingTreeWalker(document.documentElement);\n initialAnalytics ??= domAnalysis;\n\n const thisElemCount = domAnalysis.elementCount;\n const initialElemCount = initialAnalytics.elementCount;\n Math.abs(thisElemCount - initialElemCount) / (initialElemCount + Number.EPSILON)\n const r = {\n title: document.title,\n description: document.head?.querySelector('meta[name=\"description\"]')?.getAttribute('content') ?? '',\n href: document.location.href,\n html: document.documentElement?.outerHTML,\n htmlSignificantlyModifiedByJs: Boolean(Math.abs(thisElemCount - initialElemCount) / (initialElemCount + Number.EPSILON) > 0.05),\n text: document.body?.innerText,\n shadowExpanded: shadowDomPresent() ? cloneAndExpandShadowRoots()?.outerHTML : undefined,\n parsed: parsed,\n imgs: [],\n maxElemDepth: domAnalysis.maxDepth,\n elemCount: domAnalysis.elementCount,\n lastMutationIdle,\n };\n if (document.baseURI !== r.href) {\n r.rebase = document.baseURI;\n }\n r.imgs = briefImgs();\n\n return r;\n}\nfunction waitForSelector(selectorText) {\n return new Promise((resolve) => {\n const existing = document.querySelector(selectorText);\n if (existing) {\n resolve(existing);\n return;\n }\n const observer = new MutationObserver(() => {\n const elem = document.querySelector(selectorText);\n if (elem) {\n resolve(document.querySelector(selectorText));\n observer.disconnect();\n }\n });\n observer.observe(document.documentElement, {\n childList: true,\n subtree: true\n });\n });\n}\nwindow.getMaxDepthAndElemCountUsingTreeWalker = getMaxDepthAndElemCountUsingTreeWalker;\nwindow.waitForSelector = waitForSelector;\nwindow.giveSnapshot = giveSnapshot;\nwindow.briefImgs = briefImgs;\n})();\n`;\n\nconst documentResourceTypes = new Set([\n 'document', 'script', 'xhr', 'fetch', 'prefetch', 'eventsource', 'websocket', 'preflight'\n]);\nconst mediaResourceTypes = new Set([\n 'stylesheet', 'image', 'font', 'media'\n]);\n\n\nclass PageReqCtrlKit {\n reqSet: Set = new Set();\n blockers: Deferred[] = [];\n lastResourceLoadedAt: number = 0;\n lastContentResourceLoadedAt: number = 0;\n lastMediaResourceLoadedAt: number = 0;\n\n constructor(\n public concurrency: number,\n ) {\n if (isNaN(concurrency) || concurrency < 1) {\n throw new AssertionFailureError(`Invalid concurrency: ${concurrency}`);\n }\n }\n\n onNewRequest(req: HTTPRequest) {\n this.reqSet.add(req);\n if (this.reqSet.size <= this.concurrency) {\n return Promise.resolve();\n }\n const deferred = Defer();\n this.blockers.push(deferred);\n\n return deferred.promise;\n }\n\n onFinishRequest(req: HTTPRequest) {\n this.reqSet.delete(req);\n const deferred = this.blockers.shift();\n deferred?.resolve();\n const now = Date.now();\n this.lastResourceLoadedAt = now;\n // Beware req being undefined\n // https://pptr.dev/api/puppeteer.pageevent#:~:text=For%20certain%20requests%2C%20might%20contain%20undefined.\n const typ = req?.resourceType();\n if (!typ) {\n return;\n }\n if (documentResourceTypes.has(typ)) {\n this.lastContentResourceLoadedAt = now;\n }\n if (mediaResourceTypes.has(typ)) {\n this.lastMediaResourceLoadedAt = now;\n }\n }\n}\n\n@singleton()\nexport class PuppeteerControl extends AsyncService {\n\n _sn = 0;\n browser!: Browser;\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n __loadedPage: Page[] = [];\n\n finalizerMap = new WeakMap>();\n snMap = new WeakMap();\n livePages = new Set();\n pagePhase = new WeakMap();\n lastPageCratedAt: number = 0;\n ua: string = '';\n effectiveUA: string = '';\n\n concurrentRequestsPerPage: number = 32;\n pageReqCtrl = new WeakMap();\n\n lastReqSentAt: number = 0;\n\n circuitBreakerHosts: Set = new Set();\n\n lifeCycleTrack = new WeakMap();\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected asyncLocalContext: AsyncLocalContext,\n protected curlControl: CurlControl,\n protected blackHoleDetector: BlackHoleDetector,\n ) {\n super(...arguments);\n this.setMaxListeners(Infinity);\n\n let crippledTimes = 0;\n this.on('crippled', () => {\n crippledTimes += 1;\n this.__loadedPage.length = 0;\n this.livePages.clear();\n if (crippledTimes > 5) {\n process.nextTick(() => {\n this.emit('error', new Error('Browser crashed too many times, quitting...'));\n // process.exit(1);\n });\n }\n });\n }\n\n override async init() {\n await this.dependencyReady();\n if (process.env.NODE_ENV?.includes('dry-run')) {\n this.emit('ready');\n return;\n }\n\n if (this.browser) {\n if (this.browser.connected) {\n await this.browser.close();\n } else {\n this.browser.process()?.kill('SIGKILL');\n }\n }\n this.browser = await puppeteer.launch({\n timeout: 10_000,\n headless: !Boolean(process.env.DEBUG_BROWSER),\n executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH,\n args: [\n '--disable-dev-shm-usage',\n '--disable-blink-features=AutomationControlled'\n ]\n }).catch((err: any) => {\n this.logger.error(`Unknown firebase issue, just die fast.`, { err });\n process.nextTick(() => {\n this.emit('error', err);\n // process.exit(1);\n });\n return Promise.reject(err);\n });\n this.browser.once('disconnected', () => {\n this.logger.warn(`Browser disconnected`);\n if (this.browser) {\n this.emit('crippled');\n }\n process.nextTick(() => this.serviceReady());\n });\n this.ua = await this.browser.userAgent();\n this.logger.info(`Browser launched: ${this.browser.process()?.pid}, ${this.ua}`);\n this.effectiveUA = this.ua.replace(/Headless/i, '').replace('Mozilla/5.0 (X11; Linux x86_64)', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)');\n this.curlControl.impersonateChrome(this.effectiveUA);\n\n await this.newPage('beware_deadlock').then((r) => this.__loadedPage.push(r));\n\n this.emit('ready');\n }\n\n protected getRpsControlKit(page: Page) {\n let kit = this.pageReqCtrl.get(page);\n if (!kit) {\n kit = new PageReqCtrlKit(this.concurrentRequestsPerPage);\n this.pageReqCtrl.set(page, kit);\n }\n\n return kit;\n }\n\n async newPage(bewareDeadLock: any = false) {\n if (!bewareDeadLock) {\n await this.serviceReady();\n }\n const sn = this._sn++;\n let page;\n try {\n const dedicatedContext = await this.browser.createBrowserContext();\n page = await dedicatedContext.newPage();\n } catch (err: any) {\n this.logger.warn(`Failed to create page ${sn}`, { err });\n this.browser.process()?.kill('SIGKILL');\n throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`);\n }\n const preparations = [];\n\n preparations.push(page.setUserAgent(this.effectiveUA));\n // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));\n // preparations.push(page.setUserAgent(`Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`));\n preparations.push(page.setBypassCSP(true));\n preparations.push(page.setViewport({ width: 1024, height: 1024 }));\n preparations.push(page.exposeFunction('reportSnapshot', (snapshot: PageSnapshot) => {\n if (snapshot.href === 'about:blank') {\n return;\n }\n page.emit('snapshot', snapshot);\n }));\n preparations.push(page.exposeFunction('setViewport', (viewport: Viewport | null) => {\n page.setViewport(viewport).catch(() => undefined);\n }));\n preparations.push(page.evaluateOnNewDocument(SCRIPT_TO_INJECT_INTO_FRAME));\n preparations.push(page.setRequestInterception(true));\n\n await Promise.all(preparations);\n\n await page.goto('about:blank', { waitUntil: 'domcontentloaded' });\n\n const domainSet = new Set();\n let reqCounter = 0;\n let t0: number | undefined;\n let halt = false;\n\n page.on('request', async (req) => {\n reqCounter++;\n if (halt) {\n return req.abort('blockedbyclient', 1000);\n }\n const requestUrl = req.url();\n if (!requestUrl.startsWith('http:') && !requestUrl.startsWith('https:') && !requestUrl.startsWith('chrome-extension:') && requestUrl !== 'about:blank') {\n return req.abort('blockedbyclient', 1000);\n }\n t0 ??= Date.now();\n\n const parsedUrl = new URL(requestUrl);\n if (isIP(parsedUrl.hostname)) {\n domainSet.add(parsedUrl.hostname);\n } else {\n try {\n const tldParsed = tldExtract(requestUrl);\n domainSet.add(tldParsed.domain);\n } catch (_err) {\n domainSet.add(parsedUrl.hostname);\n }\n }\n\n if (this.circuitBreakerHosts.has(parsedUrl.hostname.toLowerCase())) {\n page.emit('abuse', { url: requestUrl, page, sn, reason: `Abusive request: ${requestUrl}` });\n return req.abort('blockedbyclient', 1000);\n }\n\n if (\n parsedUrl.hostname === 'localhost' ||\n parsedUrl.hostname.startsWith('127.')\n ) {\n page.emit('abuse', { url: requestUrl, page, sn, reason: `Suspicious action: Request to localhost: ${requestUrl}` });\n\n return req.abort('blockedbyclient', 1000);\n }\n\n const dt = Math.ceil((Date.now() - t0) / 1000);\n const rps = reqCounter / dt;\n // console.log(`rps: ${rps}`);\n const pagePhase = this.pagePhase.get(page);\n if (pagePhase === 'background') {\n if (rps > 10 || reqCounter > 1000) {\n halt = true;\n\n return req.abort('blockedbyclient', 1000);\n }\n }\n if (reqCounter > 1000) {\n if (rps > 60 || reqCounter > 2000) {\n page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many requests` });\n halt = true;\n\n return req.abort('blockedbyclient', 1000);\n }\n }\n\n if (domainSet.size > 200) {\n page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many domains` });\n halt = true;\n\n return req.abort('blockedbyclient', 1000);\n }\n\n if (requestUrl.startsWith('http')) {\n const kit = this.getRpsControlKit(page);\n await kit.onNewRequest(req);\n }\n\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n\n const continueArgs = req.continueRequestOverrides\n ? [req.continueRequestOverrides(), 0] as const\n : [];\n\n return req.continue(continueArgs[0], continueArgs[1]);\n });\n const reqFinishHandler = (req: HTTPRequest) => {\n const kit = this.getRpsControlKit(page);\n kit.onFinishRequest(req);\n };\n page.on('requestfinished', reqFinishHandler);\n page.on('requestfailed', reqFinishHandler);\n page.on('requestservedfromcache', reqFinishHandler);\n\n await page.evaluateOnNewDocument(`\n(function () {\n if (window.self === window.top) {\n let lastAnalytics;\n let lastReportedAt = 0;\n const handlePageLoad = () => {\n const now = Date.now();\n const dt = now - lastReportedAt;\n const previousAnalytics = lastAnalytics;\n const thisAnalytics = getMaxDepthAndElemCountUsingTreeWalker();\n let dElem = 0;\n\n if (window.haltSnapshot) {\n return;\n }\n\n const thisElemCount = thisAnalytics.elementCount;\n if (previousAnalytics) {\n const previousElemCount = previousAnalytics.elementCount;\n\n const delta = Math.abs(thisElemCount - previousElemCount);\n dElem = delta /(previousElemCount + Number.EPSILON);\n }\n\n if (dt < 1200 && dElem < 0.05) {\n return;\n }\n\n lastAnalytics = thisAnalytics;\n lastReportedAt = now;\n\n const r = giveSnapshot(false, lastAnalytics);\n window.reportSnapshot(r);\n };\n document.addEventListener('readystatechange', ()=> {\n if (document.readyState === 'interactive') {\n handlePageLoad();\n }\n });\n document.addEventListener('load', handlePageLoad);\n window.addEventListener('load', handlePageLoad);\n document.addEventListener('DOMContentLoaded', handlePageLoad);\n document.addEventListener('mutationIdle', handlePageLoad);\n }\n document.addEventListener('DOMContentLoaded', ()=> window.simulateScroll(), { once: true });\n})();\n`);\n\n this.snMap.set(page, sn);\n this.logger.debug(`Page ${sn} created.`);\n this.lastPageCratedAt = Date.now();\n this.livePages.add(page);\n this.pagePhase.set(page, 'idle');\n\n return page;\n }\n\n async getNextPage() {\n let thePage: Page | undefined;\n if (this.__loadedPage.length) {\n thePage = this.__loadedPage.shift();\n if (this.__loadedPage.length <= 1) {\n process.nextTick(() => {\n this.newPage()\n .then((r) => this.__loadedPage.push(r))\n .catch((err) => {\n this.logger.warn(`Failed to load new page ahead of time`, { err });\n });\n });\n }\n }\n\n if (!thePage) {\n thePage = await this.newPage();\n }\n\n const timer = setTimeout(() => {\n this.logger.warn(`Page is not allowed to live past 5 minutes, ditching page ${this.snMap.get(thePage!)}...`);\n this.ditchPage(thePage!);\n }, 300 * 1000);\n\n this.finalizerMap.set(thePage, timer);\n\n return thePage;\n }\n\n async ditchPage(page: Page) {\n if (this.finalizerMap.has(page)) {\n clearTimeout(this.finalizerMap.get(page)!);\n this.finalizerMap.delete(page);\n }\n if (page.isClosed()) {\n return;\n }\n const sn = this.snMap.get(page);\n this.logger.debug(`Closing page ${sn}`);\n await Promise.race([\n (async () => {\n const ctx = page.browserContext();\n try {\n await page.close();\n } finally {\n await ctx.close();\n }\n })(),\n delay(5000)\n ]).catch((err) => {\n this.logger.error(`Failed to destroy page ${sn}`, { err });\n });\n this.livePages.delete(page);\n this.pagePhase.delete(page);\n }\n\n async *scrap(parsedUrl: URL, options: ScrappingOptions = {}): AsyncGenerator {\n // parsedUrl.search = '';\n const url = parsedUrl.toString();\n let snapshot: PageSnapshot | undefined;\n let screenshot: Buffer | undefined;\n let pageshot: Buffer | undefined;\n const pdfUrls: string[] = [];\n let navigationResponse: HTTPResponse | undefined;\n const page = await this.getNextPage();\n this.lifeCycleTrack.set(page, this.asyncLocalContext.ctx);\n this.pagePhase.set(page, 'active');\n page.on('response', (resp) => {\n this.blackHoleDetector.itWorked();\n const req = resp.request();\n if (req.frame() === page.mainFrame() && req.isNavigationRequest()) {\n navigationResponse = resp;\n }\n if (!resp.ok()) {\n return;\n }\n const headers = resp.headers();\n const url = resp.url();\n const contentType = headers['content-type'];\n if (contentType?.toLowerCase().includes('application/pdf')) {\n pdfUrls.push(url);\n }\n });\n page.on('request', async (req) => {\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n const reqUrlParsed = new URL(req.url());\n if (!reqUrlParsed.protocol.startsWith('http')) {\n const overrides = req.continueRequestOverrides();\n\n return req.continue(overrides, 0);\n }\n const typ = req.resourceType();\n if (typ === 'media') {\n // Non-cooperative answer to block all media requests.\n return req.abort('blockedbyclient');\n }\n if (!options.proxyResources) {\n const isDocRequest = ['document', 'xhr', 'fetch', 'websocket', 'prefetch', 'eventsource', 'ping'].includes(typ);\n if (!isDocRequest) {\n if (options.extraHeaders) {\n const overrides = req.continueRequestOverrides();\n const continueArgs = [{\n ...overrides,\n headers: {\n ...req.headers(),\n ...overrides?.headers,\n ...options.extraHeaders,\n }\n }, 1] as const;\n\n return req.continue(continueArgs[0], continueArgs[1]);\n }\n const overrides = req.continueRequestOverrides();\n\n return req.continue(overrides, 0);\n }\n }\n const sideload = options.sideLoad;\n\n const impersonate = sideload?.impersonate[reqUrlParsed.href];\n if (impersonate) {\n let body;\n if (impersonate.body) {\n body = await readFile(await impersonate.body.filePath);\n if (req.isInterceptResolutionHandled()) {\n return;\n }\n }\n return req.respond({\n status: impersonate.status,\n headers: impersonate.headers,\n contentType: impersonate.contentType,\n body: body ? Uint8Array.from(body) : undefined,\n }, 999);\n }\n\n const proxy = options.proxyUrl || sideload?.proxyOrigin?.[reqUrlParsed.origin];\n const ctx = this.lifeCycleTrack.get(page);\n if (proxy && ctx) {\n return await this.asyncLocalContext.bridge(ctx, async () => {\n try {\n const curled = await this.curlControl.sideLoad(reqUrlParsed, {\n ...options,\n method: req.method(),\n body: req.postData(),\n extraHeaders: {\n ...req.headers(),\n ...options.extraHeaders,\n },\n proxyUrl: proxy\n });\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n\n if (curled.chain.length === 1) {\n if (!curled.file) {\n return req.respond({\n status: curled.status,\n headers: _.omit(curled.headers, 'result'),\n contentType: curled.contentType,\n }, 3);\n }\n const body = await readFile(await curled.file.filePath);\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n return req.respond({\n status: curled.status,\n headers: _.omit(curled.headers, 'result'),\n contentType: curled.contentType,\n body: Uint8Array.from(body),\n }, 3);\n }\n options.sideLoad ??= curled.sideLoadOpts;\n _.merge(options.sideLoad, curled.sideLoadOpts);\n const firstReq = curled.chain[0];\n\n return req.respond({\n status: firstReq.result!.code,\n headers: _.omit(firstReq, 'result'),\n }, 3);\n } catch (err: any) {\n this.logger.warn(`Failed to sideload browser request ${reqUrlParsed.origin}`, { href: reqUrlParsed.href, err, proxy });\n }\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n const overrides = req.continueRequestOverrides();\n const continueArgs = [{\n ...overrides,\n headers: {\n ...req.headers(),\n ...overrides?.headers,\n ...options.extraHeaders,\n }\n }, 1] as const;\n\n return req.continue(continueArgs[0], continueArgs[1]);\n });\n }\n\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n const overrides = req.continueRequestOverrides();\n const continueArgs = [{\n ...overrides,\n headers: {\n ...req.headers(),\n ...overrides?.headers,\n ...options.extraHeaders,\n }\n }, 1] as const;\n\n return req.continue(continueArgs[0], continueArgs[1]);\n });\n let pageScriptEvaluations: Promise[] = [];\n let frameScriptEvaluations: Promise[] = [];\n if (options.injectPageScripts?.length) {\n page.on('framenavigated', (frame) => {\n if (frame !== page.mainFrame()) {\n return;\n }\n\n pageScriptEvaluations.push(\n Promise.allSettled(options.injectPageScripts!.map((x) => frame.evaluate(x).catch((err) => {\n this.logger.warn(`Error in evaluation of page scripts`, { err });\n })))\n );\n });\n }\n if (options.injectFrameScripts?.length) {\n page.on('framenavigated', (frame) => {\n frameScriptEvaluations.push(\n Promise.allSettled(options.injectFrameScripts!.map((x) => frame.evaluate(x).catch((err) => {\n this.logger.warn(`Error in evaluation of frame scripts`, { err });\n })))\n );\n });\n }\n const sn = this.snMap.get(page);\n this.logger.info(`Page ${sn}: Scraping ${url}`, { url });\n if (options.locale) {\n // Add headers via request interception to walk around this bug\n // https://github.com/puppeteer/puppeteer/issues/10235\n // await page.setExtraHTTPHeaders({\n // 'Accept-Language': options.locale\n // });\n\n await page.evaluateOnNewDocument(() => {\n Object.defineProperty(navigator, \"language\", {\n get: function () {\n return options.locale;\n }\n });\n Object.defineProperty(navigator, \"languages\", {\n get: function () {\n return [options.locale];\n }\n });\n });\n }\n\n if (options.cookies) {\n const mapped = options.cookies.map((x) => {\n const draft: CookieParam = {\n name: x.name,\n value: encodeURIComponent(x.value),\n secure: x.secure,\n domain: x.domain,\n path: x.path,\n expires: x.expires ? Math.floor(x.expires.valueOf() / 1000) : undefined,\n sameSite: x.sameSite as any,\n };\n if (!draft.expires && x.maxAge) {\n draft.expires = Math.floor(Date.now() / 1000) + x.maxAge;\n }\n if (!draft.domain) {\n draft.url = parsedUrl.toString();\n }\n\n return draft;\n });\n try {\n await page.setCookie(...mapped);\n } catch (err: any) {\n this.logger.warn(`Page ${sn}: Failed to set cookies`, { err });\n throw new ParamValidationError({\n path: 'cookies',\n message: `Failed to set cookies: ${err?.message}`\n });\n }\n }\n if (options.overrideUserAgent) {\n await page.setUserAgent(options.overrideUserAgent);\n }\n if (options.viewport) {\n await page.setViewport(options.viewport);\n }\n\n let nextSnapshotDeferred = Defer();\n const crippleListener = () => nextSnapshotDeferred.reject(new ServiceCrashedError({ message: `Browser crashed, try again` }));\n this.once('crippled', crippleListener);\n nextSnapshotDeferred.promise.finally(() => {\n this.off('crippled', crippleListener);\n });\n let successfullyDone;\n const hdl = (s: any) => {\n if (snapshot === s) {\n return;\n }\n snapshot = s;\n if (snapshot) {\n const kit = this.pageReqCtrl.get(page);\n snapshot.lastContentResourceLoaded = kit?.lastContentResourceLoadedAt;\n snapshot.lastMediaResourceLoaded = kit?.lastMediaResourceLoadedAt;\n }\n if (s?.maxElemDepth && s.maxElemDepth > 256) {\n return;\n }\n if (s?.elemCount && s.elemCount > 10_000) {\n return;\n }\n nextSnapshotDeferred.resolve(s);\n nextSnapshotDeferred = Defer();\n this.once('crippled', crippleListener);\n nextSnapshotDeferred.promise.finally(() => {\n this.off('crippled', crippleListener);\n });\n };\n page.on('snapshot', hdl);\n page.once('abuse', (event: any) => {\n this.emit('abuse', { ...event, url: parsedUrl });\n if (snapshot?.href && parsedUrl.href !== snapshot.href) {\n this.emit('abuse', { ...event, url: snapshot.href });\n }\n\n nextSnapshotDeferred.reject(\n new SecurityCompromiseError(`Abuse detected: ${event.reason}`)\n );\n });\n\n const timeout = options.timeoutMs || 30_000;\n const goToOptions: GoToOptions = {\n waitUntil: ['load', 'domcontentloaded', 'networkidle0'],\n timeout,\n };\n\n if (options.referer) {\n goToOptions.referer = options.referer;\n }\n\n let waitForPromise: Promise | undefined;\n let finalizationPromise: Promise | undefined;\n const doFinalization = async () => {\n if (waitForPromise) {\n // SuccessfullyDone is meant for the finish of the page.\n // It doesn't matter if you are expecting something and it didn't show up.\n await waitForPromise.catch(() => void 0);\n }\n successfullyDone ??= true;\n try {\n const pSubFrameSnapshots = this.snapshotChildFrames(page);\n snapshot = await page.evaluate('giveSnapshot(true)') as PageSnapshot;\n screenshot = (await this.takeScreenShot(page)) || screenshot;\n pageshot = (await this.takeScreenShot(page, { fullPage: true })) || pageshot;\n if (snapshot) {\n snapshot.childFrames = await pSubFrameSnapshots;\n }\n } catch (err: any) {\n this.logger.warn(`Page ${sn}: Failed to finalize ${url}`, { err });\n }\n if (!snapshot?.html) {\n return;\n }\n\n this.logger.info(`Page ${sn}: Snapshot of ${url} done`, { url, title: snapshot?.title, href: snapshot?.href });\n this.emit(\n 'crawled',\n {\n ...snapshot,\n status: navigationResponse?.status(),\n statusText: navigationResponse?.statusText(),\n pdfs: _.uniq(pdfUrls), screenshot, pageshot,\n },\n { ...options, url: parsedUrl }\n );\n };\n const delayPromise = delay(timeout);\n const gotoPromise = page.goto(url, goToOptions)\n .catch((err) => {\n if (err instanceof TimeoutError) {\n this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err });\n return new AssertionFailureError({\n message: `Failed to goto ${url}: ${err}`,\n cause: err,\n });\n }\n if (err?.message?.startsWith('net::ERR_ABORTED')) {\n if (pdfUrls.length) {\n // Not throw for pdf mode.\n return;\n }\n }\n\n this.logger.warn(`Page ${sn}: Browsing of ${url} failed`, { err });\n return new AssertionFailureError({\n message: `Failed to goto ${url}: ${err}`,\n cause: err,\n });\n }).then(async (stuff) => {\n // This check is necessary because without snapshot, the condition of the page is unclear\n // Calling evaluate directly may stall the process.\n if (!snapshot) {\n if (stuff instanceof Error) {\n throw stuff;\n }\n }\n await Promise.race([Promise.allSettled([...pageScriptEvaluations, ...frameScriptEvaluations]), delayPromise])\n .catch(() => void 0);\n return stuff;\n });\n if (options.waitForSelector) {\n const t0 = Date.now();\n waitForPromise = nextSnapshotDeferred.promise.then(() => {\n const t1 = Date.now();\n const elapsed = t1 - t0;\n const remaining = timeout - elapsed;\n const thisTimeout = remaining > 100 ? remaining : 100;\n const p = (Array.isArray(options.waitForSelector) ?\n Promise.all(options.waitForSelector.map((x) => page.waitForSelector(x, { timeout: thisTimeout }))) :\n page.waitForSelector(options.waitForSelector!, { timeout: thisTimeout }))\n .then(() => {\n successfullyDone = true;\n })\n .catch((err) => {\n waitForPromise = undefined;\n this.logger.warn(`Page ${sn}: Failed to wait for selector ${options.waitForSelector}`, { err });\n });\n return p as any;\n });\n finalizationPromise = Promise.allSettled([waitForPromise, gotoPromise]).then(doFinalization);\n } else {\n finalizationPromise = gotoPromise.then(doFinalization);\n }\n\n try {\n let lastHTML = snapshot?.html;\n while (true) {\n const ckpt = [nextSnapshotDeferred.promise, waitForPromise ?? gotoPromise];\n\n if (options.minIntervalMs) {\n ckpt.push(delay(options.minIntervalMs));\n }\n let error;\n await Promise.race(ckpt).catch((err) => error = err);\n if (successfullyDone && !error) {\n if (!snapshot && !screenshot) {\n throw new AssertionFailureError(`Could not extract any meaningful content from the page`);\n }\n yield {\n ...snapshot,\n status: navigationResponse?.status(),\n statusText: navigationResponse?.statusText(),\n pdfs: _.uniq(pdfUrls), screenshot, pageshot\n } as PageSnapshot;\n break;\n }\n if (options.favorScreenshot && snapshot?.title && snapshot?.html !== lastHTML) {\n screenshot = (await this.takeScreenShot(page)) || screenshot;\n pageshot = (await this.takeScreenShot(page, { fullPage: true })) || pageshot;\n lastHTML = snapshot.html;\n }\n if (snapshot || screenshot) {\n yield {\n ...snapshot,\n status: navigationResponse?.status(),\n statusText: navigationResponse?.statusText(),\n pdfs: _.uniq(pdfUrls), screenshot, pageshot,\n isIntermediate: true,\n } as PageSnapshot;\n }\n if (error) {\n throw error;\n }\n if (successfullyDone) {\n break;\n }\n }\n await finalizationPromise;\n yield {\n ...snapshot,\n status: navigationResponse?.status(),\n statusText: navigationResponse?.statusText(),\n pdfs: _.uniq(pdfUrls), screenshot, pageshot\n } as PageSnapshot;\n } finally {\n this.pagePhase.set(page, 'background');\n Promise.allSettled([gotoPromise, waitForPromise, finalizationPromise]).finally(() => {\n page.off('snapshot', hdl);\n this.ditchPage(page);\n });\n nextSnapshotDeferred.resolve();\n }\n }\n\n protected async takeScreenShot(page: Page, opts?: Parameters[0]): Promise {\n const r = await page.screenshot(opts).catch((err) => {\n this.logger.warn(`Failed to take screenshot`, { err });\n });\n\n if (r) {\n return Buffer.from(r);\n }\n\n return undefined;\n }\n\n async snapshotChildFrames(page: Page): Promise {\n const childFrames = page.mainFrame().childFrames();\n const r = await Promise.all(childFrames.map(async (x) => {\n const thisUrl = x.url();\n if (!thisUrl || thisUrl === 'about:blank') {\n return undefined;\n }\n try {\n await x.evaluate(SCRIPT_TO_INJECT_INTO_FRAME);\n\n return await x.evaluate(`giveSnapshot()`);\n } catch (err) {\n this.logger.warn(`Failed to snapshot child frame ${thisUrl}`, { err });\n return undefined;\n }\n })) as PageSnapshot[];\n\n return r.filter(Boolean);\n }\n\n}\n\nconst puppeteerControl = container.resolve(PuppeteerControl);\n\nexport default puppeteerControl;\n" }, { "path": "src/services/registry.ts", "content": "import { propertyInjectorFactory } from 'civkit/property-injector';\nimport { KoaRPCRegistry } from 'civkit/civ-rpc/koa';\nimport { container, singleton } from 'tsyringe';\nimport { IntegrityEnvelope } from 'civkit/civ-rpc';\nimport bodyParser from '@koa/bodyparser';\n\nimport { GlobalLogger } from './logger';\nimport { TempFileManager } from './temp-file';\nimport { AsyncLocalContext } from './async-context';\nimport { BlackHoleDetector } from './blackhole-detector';\nexport { Context } from 'koa';\n\nexport const InjectProperty = propertyInjectorFactory(container);\n\n@singleton()\nexport class RPCRegistry extends KoaRPCRegistry {\n\n title = 'Jina Reader API';\n container = container;\n logger = this.globalLogger.child({ service: this.constructor.name });\n static override envelope = IntegrityEnvelope;\n override _BODY_PARSER_LIMIT = '102mb';\n override _RESPONSE_STREAM_MODE = 'koa' as const;\n\n override koaMiddlewares = [\n this.__CORSAllowAllMiddleware.bind(this),\n bodyParser({\n encoding: 'utf-8',\n enableTypes: ['json', 'form'],\n jsonLimit: this._BODY_PARSER_LIMIT,\n xmlLimit: this._BODY_PARSER_LIMIT,\n formLimit: this._BODY_PARSER_LIMIT,\n }),\n this.__multiParse.bind(this),\n this.__binaryParse.bind(this),\n ];\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected ctxMgr: AsyncLocalContext,\n protected tempFileManager: TempFileManager,\n protected blackHoleDetector: BlackHoleDetector,\n ) {\n super(...arguments);\n\n this.on('run', () => this.blackHoleDetector.incomingRequest());\n this.on('ran', () => this.blackHoleDetector.doneWithRequest());\n this.on('fail', () => this.blackHoleDetector.doneWithRequest());\n }\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n }\n\n}\n\nconst instance = container.resolve(RPCRegistry);\nexport default instance;\nexport const { Method, RPCMethod, RPCReflect, Param, Ctx, } = instance.decorators();\n" }, { "path": "src/services/robots-text.ts", "content": "import { singleton } from 'tsyringe';\nimport { URL } from 'url';\nimport { AssertionFailureError, DownstreamServiceFailureError, ResourcePolicyDenyError } from 'civkit/civ-rpc';\nimport { AsyncService } from 'civkit/async-service';\nimport { HashManager } from 'civkit/hash';\nimport { marshalErrorLike } from 'civkit/lang';\n\nimport { GlobalLogger } from './logger';\nimport { FirebaseStorageBucketControl } from '../shared/services/firebase-storage-bucket';\nimport { Threaded } from '../services/threaded';\n\n\nexport const md5Hasher = new HashManager('md5', 'hex');\n\n@singleton()\nexport class RobotsTxtService extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected firebaseStorageBucketControl: FirebaseStorageBucketControl,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n }\n\n async getCachedRobotTxt(origin: string) {\n const digest = md5Hasher.hash(origin.toLowerCase());\n const cacheLoc = `robots-txt/${digest}`;\n let buff;\n buff = await this.firebaseStorageBucketControl.downloadFile(cacheLoc).catch(() => undefined);\n if (buff) {\n return buff.toString();\n }\n\n const r = await fetch(new URL('robots.txt', origin).href, { signal: AbortSignal.timeout(5000) });\n if (!r.ok) {\n throw new DownstreamServiceFailureError(`Failed to fetch robots.txt from ${origin}: ${r.status} ${r.statusText}`);\n }\n buff = Buffer.from(await r.arrayBuffer());\n\n this.firebaseStorageBucketControl.saveFile(cacheLoc, buff, {\n contentType: 'text/plain'\n }).catch((err) => {\n this.logger.warn(`Failed to save robots.txt to cache: ${err}`, { err: marshalErrorLike(err) });\n });\n\n return buff.toString();\n }\n\n @Threaded()\n async assertAccessAllowed(url: URL, inputMyUa = '*') {\n let robotTxt: string = '';\n try {\n robotTxt = await this.getCachedRobotTxt(url.origin);\n } catch (err) {\n if (err instanceof DownstreamServiceFailureError) {\n // Remote server is reachable but cannot provide a robot.txt; this is treated as public access\n return true;\n }\n throw new AssertionFailureError(`Failed to load robots.txt from ${url.origin}: ${err}`);\n }\n const myUa = inputMyUa.toLowerCase();\n const lines = robotTxt.split(/\\r?\\n/g);\n\n let currentUa = myUa || '*';\n let uaLine = 'User-Agent: *';\n const pathNormalized = `${url.pathname}?`;\n\n for (const line of lines) {\n const trimmed = line.trim();\n if (trimmed.startsWith('#') || !trimmed) {\n continue;\n }\n const [k, ...rest] = trimmed.split(':');\n const key = k.trim().toLowerCase();\n const value = rest.join(':').trim();\n\n if (key === 'user-agent') {\n currentUa = value.toLowerCase();\n if (value === '*') {\n currentUa = myUa;\n }\n uaLine = line;\n continue;\n }\n\n if (currentUa !== myUa) {\n continue;\n }\n\n if (key === 'disallow') {\n if (!value) {\n return true;\n }\n if (value.includes('*')) {\n const [head, tail] = value.split('*');\n if (url.pathname.startsWith(head) && url.pathname.endsWith(tail)) {\n throw new ResourcePolicyDenyError(`Access to ${url.href} is disallowed by site robots.txt: For ${uaLine}, ${line}`);\n }\n } else if (pathNormalized.startsWith(value)) {\n throw new ResourcePolicyDenyError(`Access to ${url.href} is disallowed by site robots.txt: For ${uaLine}, ${line}`);\n }\n\n continue;\n }\n\n if (key === 'allow') {\n if (!value) {\n return true;\n }\n if (pathNormalized.startsWith(value)) {\n return true;\n }\n continue;\n }\n }\n\n return true;\n }\n\n}\n" }, { "path": "src/services/serp/compat.ts", "content": "export interface WebSearchEntry {\n link: string;\n title: string;\n source?: string;\n date?: string;\n snippet?: string;\n imageUrl?: string;\n siteLinks?: {\n link: string; title: string; snippet?: string;\n }[];\n variant?: 'web' | 'images' | 'news';\n}" }, { "path": "src/services/serp/google.ts", "content": "import { singleton } from 'tsyringe';\nimport { AsyncService } from 'civkit/async-service';\nimport { GlobalLogger } from '../logger';\nimport { JSDomControl } from '../jsdom';\nimport { isMainThread } from 'worker_threads';\nimport _ from 'lodash';\nimport { WebSearchEntry } from './compat';\nimport { ScrappingOptions, SERPSpecializedPuppeteerControl } from './puppeteer';\nimport { CurlControl } from '../curl';\nimport { readFile } from 'fs/promises';\nimport { ApplicationError } from 'civkit/civ-rpc';\nimport { ServiceBadApproachError, ServiceBadAttemptError } from '../errors';\nimport { parseJSONText } from 'civkit/vectorize';\nimport { retryWith } from 'civkit/decorators';\nimport { ProxyProviderService } from '../../shared/services/proxy-provider';\n\n@singleton()\nexport class GoogleSERP extends AsyncService {\n logger = this.globalLogger.child({ service: this.constructor.name });\n googleDomain = process.env.OVERRIDE_GOOGLE_DOMAIN || 'www.google.com';\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected puppeteerControl: SERPSpecializedPuppeteerControl,\n protected jsDomControl: JSDomControl,\n protected curlControl: CurlControl,\n protected proxyProvider: ProxyProviderService,\n ) {\n const filteredDeps = isMainThread ? arguments : _.without(arguments, puppeteerControl);\n super(...filteredDeps);\n }\n\n override async init() {\n await this.dependencyReady();\n\n this.emit('ready');\n }\n\n @retryWith((err) => {\n if (err instanceof ServiceBadApproachError) {\n return false;\n }\n if (err instanceof ServiceBadAttemptError) {\n // Keep trying\n return true;\n }\n if (err instanceof ApplicationError) {\n // Quit with this error\n return false;\n }\n return undefined;\n }, 3)\n async sideLoadWithAllocatedProxy(url: URL, opts?: ScrappingOptions) {\n if (opts?.allocProxy === 'none') {\n return this.curlControl.sideLoad(url, opts);\n }\n\n const proxy = await this.proxyProvider.alloc(\n process.env.PREFERRED_PROXY_COUNTRY || 'auto'\n );\n this.logger.debug(`Proxy allocated`, { proxy: proxy.href });\n const r = await this.curlControl.sideLoad(url, {\n ...opts,\n proxyUrl: proxy.href,\n });\n\n if (r.status === 429) {\n throw new ServiceBadAttemptError('Google returned a 429 error. This may happen due to various reasons, including rate limiting or other issues.');\n }\n\n if (opts && opts.allocProxy) {\n opts.proxyUrl ??= proxy.href;\n }\n\n return { ...r, proxy };\n }\n\n digestQuery(query: { [k: string]: any; }) {\n const url = new URL(`https://${this.googleDomain}/search`);\n const clone = { ...query };\n const num = clone.num || 10;\n if (clone.page) {\n const page = parseInt(clone.page);\n delete clone.page;\n clone.start = (page - 1) * num;\n if (clone.start === 0) {\n delete clone.start;\n }\n }\n if (clone.location) {\n delete clone.location;\n }\n\n for (const [k, v] of Object.entries(clone)) {\n if (v === undefined || v === null) {\n continue;\n }\n url.searchParams.set(k, `${v}`);\n }\n\n return url;\n }\n\n async webSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) {\n const url = this.digestQuery(query);\n\n const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts);\n if (opts && sideLoaded.sideLoadOpts) {\n opts.sideLoad = sideLoaded.sideLoadOpts;\n }\n\n const snapshot = await this.puppeteerControl.controlledScrap(url, getWebSearchResults, opts);\n\n return snapshot;\n }\n\n async newsSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) {\n const url = this.digestQuery(query);\n\n url.searchParams.set('tbm', 'nws');\n\n const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts);\n if (opts && sideLoaded.sideLoadOpts) {\n opts.sideLoad = sideLoaded.sideLoadOpts;\n }\n\n const snapshot = await this.puppeteerControl.controlledScrap(url, getNewsSearchResults, opts);\n\n return snapshot;\n }\n\n async imageSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) {\n const url = this.digestQuery(query);\n\n url.searchParams.set('tbm', 'isch');\n url.searchParams.set('asearch', 'isch');\n url.searchParams.set('async', `_fmt:json,p:1,ijn:${query.start ? Math.floor(query.start / (query.num || 10)) : 0}`);\n\n const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts);\n\n if (sideLoaded.status !== 200 || !sideLoaded.file) {\n throw new ServiceBadAttemptError('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.');\n }\n\n const jsonTxt = (await readFile((await sideLoaded.file.filePath))).toString();\n const rJSON = parseJSONText(jsonTxt.slice(jsonTxt.indexOf('{\"ischj\":')));\n\n return _.get(rJSON, 'ischj.metadata').map((x: any) => {\n\n return {\n link: _.get(x, 'result.referrer_url'),\n title: _.get(x, 'result.page_title'),\n snippet: _.get(x, 'text_in_grid.snippet'),\n source: _.get(x, 'result.site_title'),\n imageWidth: _.get(x, 'original_image.width'),\n imageHeight: _.get(x, 'original_image.height'),\n imageUrl: _.get(x, 'original_image.url'),\n variant: 'images',\n };\n }) as WebSearchEntry[];\n }\n}\n\nasync function getWebSearchResults() {\n if (location.pathname.startsWith('/sorry') || location.pathname.startsWith('/error')) {\n throw new Error('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.');\n }\n\n // @ts-ignore\n await Promise.race([window.waitForSelector('div[data-async-context^=\"query\"]'), window.waitForSelector('#botstuff .mnr-c')]);\n\n const wrapper1 = document.querySelector('div[data-async-context^=\"query\"]');\n\n if (!wrapper1) {\n return undefined;\n }\n\n const query = decodeURIComponent(wrapper1.getAttribute('data-async-context')?.split('query:')[1] || '');\n\n if (!query) {\n return undefined;\n }\n\n const candidates = Array.from(wrapper1.querySelectorAll('div[lang],div[data-surl]'));\n\n return candidates.map((x, pos) => {\n const primaryLink = x.querySelector('a:not([href=\"#\"])');\n if (!primaryLink) {\n return undefined;\n }\n const url = primaryLink.getAttribute('href');\n\n if (primaryLink.querySelector('div[role=\"heading\"]')) {\n // const spans = primaryLink.querySelectorAll('span');\n // const title = spans[0]?.textContent;\n // const source = spans[1]?.textContent;\n // const date = spans[spans.length - 1].textContent;\n\n // return {\n // link: url,\n // title,\n // source,\n // date,\n // variant: 'video'\n // };\n return undefined;\n }\n\n const title = primaryLink.querySelector('h3')?.textContent;\n const source = Array.from(primaryLink.querySelectorAll('span')).find((x) => x.textContent)?.textContent;\n const cite = primaryLink.querySelector('cite[role=text]')?.textContent;\n let date = cite?.split('·')[1]?.trim();\n const snippets = Array.from(x.querySelectorAll('div[data-sncf*=\"1\"] span'));\n let snippet = snippets[snippets.length - 1]?.textContent;\n if (!snippet) {\n snippet = x.querySelector('div.IsZvec')?.textContent?.trim() || null;\n }\n date ??= snippets[snippets.length - 2]?.textContent?.trim();\n const imageUrl = x.querySelector('div[data-sncf*=\"1\"] img[src]:not(img[src^=\"data\"])')?.getAttribute('src');\n let siteLinks = Array.from(x.querySelectorAll('div[data-sncf*=\"3\"] a[href]')).map((l) => {\n return {\n link: l.getAttribute('href'),\n title: l.textContent,\n };\n });\n const perhapsParent = x.parentElement?.closest('div[data-hveid]');\n if (!siteLinks?.length && perhapsParent) {\n const candidates = Array.from(perhapsParent.querySelectorAll('td h3'));\n if (candidates.length) {\n siteLinks = candidates.map((l) => {\n const link = l.querySelector('a');\n if (!link) {\n return undefined;\n }\n const snippet = l.nextElementSibling?.textContent;\n return {\n link: link.getAttribute('href'),\n title: link.textContent,\n snippet,\n };\n }).filter(Boolean) as any;\n }\n }\n\n return {\n link: url,\n title,\n source,\n date,\n snippet: snippet ?? undefined,\n imageUrl: imageUrl?.startsWith('data:') ? undefined : imageUrl,\n siteLinks: siteLinks.length ? siteLinks : undefined,\n variant: 'web',\n };\n }).filter(Boolean) as WebSearchEntry[];\n}\n\nasync function getNewsSearchResults() {\n if (location.pathname.startsWith('/sorry') || location.pathname.startsWith('/error')) {\n throw new Error('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.');\n }\n\n // @ts-ignore\n await Promise.race([window.waitForSelector('div[data-async-context^=\"query\"]'), window.waitForSelector('#botstuff .mnr-c')]);\n\n const wrapper1 = document.querySelector('div[data-async-context^=\"query\"]');\n\n if (!wrapper1) {\n return undefined;\n }\n\n const query = decodeURIComponent(wrapper1.getAttribute('data-async-context')?.split('query:')[1] || '');\n\n if (!query) {\n return undefined;\n }\n\n const candidates = Array.from(wrapper1.querySelectorAll('div[data-news-doc-id]'));\n\n return candidates.map((x) => {\n const primaryLink = x.querySelector('a:not([href=\"#\"])');\n if (!primaryLink) {\n return undefined;\n }\n const url = primaryLink.getAttribute('href');\n const titleElem = primaryLink.querySelector('div[role=\"heading\"]');\n\n if (!titleElem) {\n return undefined;\n }\n\n const title = titleElem.textContent?.trim();\n const source = titleElem.previousElementSibling?.textContent?.trim();\n const snippet = titleElem.nextElementSibling?.textContent?.trim();\n\n const innerSpans = Array.from(titleElem.parentElement?.querySelectorAll('span') || []);\n const date = innerSpans[innerSpans.length - 1]?.textContent?.trim();\n\n return {\n link: url,\n title,\n source,\n date,\n snippet,\n variant: 'news',\n };\n }).filter(Boolean) as WebSearchEntry[];\n}" }, { "path": "src/services/serp/internal.ts", "content": "\nimport { singleton } from 'tsyringe';\nimport { GlobalLogger } from '../logger';\nimport { SecretExposer } from '../../shared/services/secrets';\nimport { AsyncLocalContext } from '../async-context';\nimport { SerperSearchQueryParams } from '../../shared/3rd-party/serper-search';\nimport { BlackHoleDetector } from '../blackhole-detector';\nimport { AsyncService } from 'civkit/async-service';\nimport { JinaSerpApiHTTP } from '../../shared/3rd-party/internal-serp';\nimport { WebSearchEntry } from './compat';\n\n@singleton()\nexport class InternalJinaSerpService extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n client!: JinaSerpApiHTTP;\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected secretExposer: SecretExposer,\n protected threadLocal: AsyncLocalContext,\n protected blackHoleDetector: BlackHoleDetector,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n\n this.client = new JinaSerpApiHTTP(this.secretExposer.JINA_SERP_API_KEY);\n }\n\n\n async doSearch(variant: 'web' | 'images' | 'news', query: SerperSearchQueryParams) {\n this.logger.debug(`Doing external search`, query);\n let results;\n switch (variant) {\n // case 'images': {\n // const r = await this.client.imageSearch(query);\n\n // results = r.parsed.images;\n // break;\n // }\n // case 'news': {\n // const r = await this.client.newsSearch(query);\n\n // results = r.parsed.news;\n // break;\n // }\n case 'web':\n default: {\n const r = await this.client.webSearch(query);\n\n results = r.parsed.results?.map((x) => ({ ...x, link: x.url }));\n break;\n }\n }\n\n this.blackHoleDetector.itWorked();\n\n return results as WebSearchEntry[];\n }\n\n\n async webSearch(query: SerperSearchQueryParams) {\n return this.doSearch('web', query);\n }\n async imageSearch(query: SerperSearchQueryParams) {\n return this.doSearch('images', query);\n }\n async newsSearch(query: SerperSearchQueryParams) {\n return this.doSearch('news', query);\n }\n\n}\n" }, { "path": "src/services/serp/puppeteer.ts", "content": "import _ from 'lodash';\nimport { readFile } from 'fs/promises';\nimport { container, singleton } from 'tsyringe';\n\nimport type { Browser, CookieParam, GoToOptions, Page, Viewport } from 'puppeteer';\nimport type { Cookie } from 'set-cookie-parser';\nimport puppeteer, { TimeoutError } from 'puppeteer';\n\nimport { Defer } from 'civkit/defer';\nimport { AssertionFailureError, ParamValidationError } from 'civkit/civ-rpc';\nimport { AsyncService } from 'civkit/async-service';\nimport { FancyFile } from 'civkit/fancy-file';\nimport { delay } from 'civkit/timeout';\n\nimport { SecurityCompromiseError, ServiceCrashedError, ServiceNodeResourceDrainError } from '../../shared/lib/errors';\nimport { CurlControl } from '../curl';\nimport { AsyncLocalContext } from '../async-context';\nimport { GlobalLogger } from '../logger';\nimport { minimalStealth } from '../minimal-stealth';\nimport { BlackHoleDetector } from '../blackhole-detector';\n\n\nexport interface ScrappingOptions {\n proxyUrl?: string;\n cookies?: Cookie[];\n overrideUserAgent?: string;\n timeoutMs?: number;\n locale?: string;\n referer?: string;\n extraHeaders?: Record;\n viewport?: Viewport;\n proxyResources?: boolean;\n allocProxy?: string;\n\n sideLoad?: {\n impersonate: {\n [url: string]: {\n status: number;\n headers: { [k: string]: string | string[]; };\n contentType?: string;\n body?: FancyFile;\n };\n };\n proxyOrigin: { [origin: string]: string; };\n };\n\n}\n\nconst SIMULATE_SCROLL = `\n(function () {\n function createIntersectionObserverEntry(target, isIntersecting, timestamp) {\n const targetRect = target.getBoundingClientRect();\n const record = {\n target,\n isIntersecting,\n time: timestamp,\n // If intersecting, intersectionRect matches boundingClientRect\n // If not intersecting, intersectionRect is empty (0x0)\n intersectionRect: isIntersecting\n ? targetRect\n : new DOMRectReadOnly(0, 0, 0, 0),\n // Current bounding client rect of the target\n boundingClientRect: targetRect,\n // Intersection ratio is either 0 (not intersecting) or 1 (fully intersecting)\n intersectionRatio: isIntersecting ? 1 : 0,\n // Root bounds (viewport in our case)\n rootBounds: new DOMRectReadOnly(\n 0,\n 0,\n window.innerWidth,\n window.innerHeight\n )\n };\n Object.setPrototypeOf(record, window.IntersectionObserverEntry.prototype);\n return record;\n }\n function cloneIntersectionObserverEntry(entry) {\n const record = {\n target: entry.target,\n isIntersecting: entry.isIntersecting,\n time: entry.time,\n intersectionRect: entry.intersectionRect,\n boundingClientRect: entry.boundingClientRect,\n intersectionRatio: entry.intersectionRatio,\n rootBounds: entry.rootBounds\n };\n Object.setPrototypeOf(record, window.IntersectionObserverEntry.prototype);\n return record;\n }\n const orig = window.IntersectionObserver;\n const kCallback = Symbol('callback');\n const kLastEntryMap = Symbol('lastEntryMap');\n const liveObservers = new Map();\n class MangledIntersectionObserver extends orig {\n constructor(callback, options) {\n super((entries, observer) => {\n const lastEntryMap = observer[kLastEntryMap];\n const lastEntry = entries[entries.length - 1];\n lastEntryMap.set(lastEntry.target, lastEntry);\n return callback(entries, observer);\n }, options);\n this[kCallback] = callback;\n this[kLastEntryMap] = new WeakMap();\n liveObservers.set(this, new Set());\n }\n disconnect() {\n liveObservers.get(this)?.clear();\n liveObservers.delete(this);\n return super.disconnect();\n }\n observe(target) {\n const observer = liveObservers.get(this);\n observer?.add(target);\n return super.observe(target);\n }\n unobserve(target) {\n const observer = liveObservers.get(this);\n observer?.delete(target);\n return super.unobserve(target);\n }\n }\n Object.defineProperty(MangledIntersectionObserver, 'name', { value: 'IntersectionObserver', writable: false });\n window.IntersectionObserver = MangledIntersectionObserver;\n function simulateScroll() {\n for (const [observer, targets] of liveObservers.entries()) {\n const t0 = performance.now();\n for (const target of targets) {\n const entry = createIntersectionObserverEntry(target, true, t0);\n observer[kCallback]([entry], observer);\n setTimeout(() => {\n const t1 = performance.now();\n const lastEntry = observer[kLastEntryMap].get(target);\n if (!lastEntry) {\n return;\n }\n const entry2 = { ...cloneIntersectionObserverEntry(lastEntry), time: t1 };\n observer[kCallback]([entry2], observer);\n });\n }\n }\n }\n window.simulateScroll = simulateScroll;\n})();\n`;\n\nconst MUTATION_IDLE_WATCH = `\n(function () {\n let timeout;\n const sendMsg = ()=> {\n document.dispatchEvent(new CustomEvent('mutationIdle'));\n };\n\n const cb = () => {\n if (timeout) {\n clearTimeout(timeout);\n timeout = setTimeout(sendMsg, 200);\n }\n };\n const mutationObserver = new MutationObserver(cb);\n\n document.addEventListener('DOMContentLoaded', () => {\n mutationObserver.observe(document.documentElement, {\n childList: true,\n subtree: true,\n });\n timeout = setTimeout(sendMsg, 200);\n }, { once: true })\n})();\n`;\n\nconst SCRIPT_TO_INJECT_INTO_FRAME = `\n${SIMULATE_SCROLL}\n${MUTATION_IDLE_WATCH}\n(${minimalStealth.toString()})();\n\n(function(){\n\nlet lastMutationIdle = 0;\nlet initialAnalytics;\ndocument.addEventListener('mutationIdle', ()=> lastMutationIdle = Date.now());\n\nfunction waitForSelector(selectorText) {\n return new Promise((resolve) => {\n const existing = document.querySelector(selectorText);\n if (existing) {\n resolve(existing);\n return;\n }\n if (document.readyState === 'loading') {\n document.addEventListener('DOMContentLoaded', () => {\n const observer = new MutationObserver(() => {\n const elem = document.querySelector(selectorText);\n if (elem) {\n resolve(document.querySelector(selectorText));\n observer.disconnect();\n }\n });\n observer.observe(document.documentElement, {\n childList: true,\n subtree: true\n });\n });\n return;\n }\n const observer = new MutationObserver(() => {\n const elem = document.querySelector(selectorText);\n if (elem) {\n resolve(document.querySelector(selectorText));\n observer.disconnect();\n }\n });\n observer.observe(document.documentElement, {\n childList: true,\n subtree: true\n });\n });\n}\nwindow.waitForSelector = waitForSelector;\n})();\n`;\n\n@singleton()\nexport class SERPSpecializedPuppeteerControl extends AsyncService {\n\n _sn = 0;\n browser!: Browser;\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n __loadedPage: Page[] = [];\n\n finalizerMap = new WeakMap>();\n snMap = new WeakMap();\n livePages = new Set();\n lastPageCratedAt: number = 0;\n ua: string = '';\n effectiveUA: string = '';\n\n protected _REPORT_FUNCTION_NAME = 'bingo';\n\n lifeCycleTrack = new WeakMap();\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected asyncLocalContext: AsyncLocalContext,\n protected curlControl: CurlControl,\n protected blackHoleDetector: BlackHoleDetector,\n ) {\n super(...arguments);\n this.setMaxListeners(Infinity);\n\n let crippledTimes = 0;\n this.on('crippled', () => {\n crippledTimes += 1;\n this.__loadedPage.length = 0;\n this.livePages.clear();\n if (crippledTimes > 5) {\n process.nextTick(() => {\n this.emit('error', new Error('Browser crashed too many times, quitting...'));\n // process.exit(1);\n });\n }\n });\n }\n\n override async init() {\n await this.dependencyReady();\n if (process.env.NODE_ENV?.includes('dry-run')) {\n this.emit('ready');\n return;\n }\n\n if (this.browser) {\n if (this.browser.connected) {\n await this.browser.close();\n } else {\n this.browser.process()?.kill('SIGKILL');\n }\n }\n this.browser = await puppeteer.launch({\n timeout: 10_000,\n headless: !Boolean(process.env.DEBUG_BROWSER),\n executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH,\n args: [\n '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'\n ]\n }).catch((err: any) => {\n this.logger.error(`Unknown firebase issue, just die fast.`, { err });\n process.nextTick(() => {\n this.emit('error', err);\n // process.exit(1);\n });\n return Promise.reject(err);\n });\n this.browser.once('disconnected', () => {\n this.logger.warn(`Browser disconnected`);\n if (this.browser) {\n this.emit('crippled');\n }\n process.nextTick(() => this.serviceReady());\n });\n this.ua = await this.browser.userAgent();\n this.logger.info(`Browser launched: ${this.browser.process()?.pid}, ${this.ua}`);\n this.effectiveUA = this.ua.replace(/Headless/i, '').replace('Mozilla/5.0 (X11; Linux x86_64)', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)');\n this.curlControl.impersonateChrome(this.effectiveUA);\n\n await this.newPage('beware_deadlock').then((r) => this.__loadedPage.push(r));\n\n this.emit('ready');\n }\n\n async newPage(bewareDeadLock: any = false) {\n if (!bewareDeadLock) {\n await this.serviceReady();\n }\n const sn = this._sn++;\n let page;\n try {\n const dedicatedContext = await this.browser.createBrowserContext();\n page = await dedicatedContext.newPage();\n } catch (err: any) {\n this.logger.warn(`Failed to create page ${sn}`, { err });\n this.browser.process()?.kill('SIGKILL');\n throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`);\n }\n const preparations = [];\n\n preparations.push(page.setUserAgent(this.effectiveUA));\n // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));\n // preparations.push(page.setUserAgent(`Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`));\n preparations.push(page.setBypassCSP(true));\n preparations.push(page.setViewport({ width: 1024, height: 1024 }));\n preparations.push(page.exposeFunction(this._REPORT_FUNCTION_NAME, (thing: T) => {\n page.emit(this._REPORT_FUNCTION_NAME, thing);\n }));\n preparations.push(page.exposeFunction('setViewport', (viewport: Viewport | null) => {\n page.setViewport(viewport).catch(() => undefined);\n }));\n preparations.push(page.evaluateOnNewDocument(SCRIPT_TO_INJECT_INTO_FRAME));\n\n await Promise.all(preparations);\n\n this.snMap.set(page, sn);\n this.logger.debug(`Page ${sn} created.`);\n this.lastPageCratedAt = Date.now();\n this.livePages.add(page);\n\n return page;\n }\n\n async getNextPage() {\n let thePage: Page | undefined;\n if (this.__loadedPage.length) {\n thePage = this.__loadedPage.shift();\n if (this.__loadedPage.length <= 1) {\n process.nextTick(() => {\n this.newPage()\n .then((r) => this.__loadedPage.push(r))\n .catch((err) => {\n this.logger.warn(`Failed to load new page ahead of time`, { err });\n });\n });\n }\n }\n\n if (!thePage) {\n thePage = await this.newPage();\n }\n\n const timer = setTimeout(() => {\n this.logger.warn(`Page is not allowed to live past 5 minutes, ditching page ${this.snMap.get(thePage!)}...`);\n this.ditchPage(thePage!);\n }, 300 * 1000);\n\n this.finalizerMap.set(thePage, timer);\n\n return thePage;\n }\n\n async ditchPage(page: Page) {\n if (this.finalizerMap.has(page)) {\n clearTimeout(this.finalizerMap.get(page)!);\n this.finalizerMap.delete(page);\n }\n if (page.isClosed()) {\n return;\n }\n const sn = this.snMap.get(page);\n this.logger.debug(`Closing page ${sn}`);\n await Promise.race([\n (async () => {\n const ctx = page.browserContext();\n try {\n await page.close();\n } finally {\n await ctx.close();\n }\n })(),\n delay(5000)\n ]).catch((err) => {\n this.logger.error(`Failed to destroy page ${sn}`, { err });\n });\n this.livePages.delete(page);\n }\n\n async controlledScrap(parsedUrl: URL, func: (this: void) => Promise, options: ScrappingOptions = {}): Promise {\n // parsedUrl.search = '';\n const url = parsedUrl.toString();\n const page = await this.getNextPage();\n this.lifeCycleTrack.set(page, this.asyncLocalContext.ctx);\n page.on('response', (_resp) => {\n this.blackHoleDetector.itWorked();\n });\n page.on('request', async (req) => {\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n const reqUrlParsed = new URL(req.url());\n if (!reqUrlParsed.protocol.startsWith('http')) {\n const overrides = req.continueRequestOverrides();\n\n return req.continue(overrides, 0);\n }\n const typ = req.resourceType();\n if (typ === 'media' || typ === 'font' || typ === 'image' || typ === 'stylesheet') {\n // Non-cooperative answer to block all media requests.\n return req.abort('blockedbyclient');\n }\n if (!options.proxyResources) {\n const isDocRequest = ['document', 'xhr', 'fetch', 'websocket', 'prefetch', 'eventsource', 'ping'].includes(typ);\n if (!isDocRequest) {\n if (options.extraHeaders) {\n const overrides = req.continueRequestOverrides();\n const continueArgs = [{\n ...overrides,\n headers: {\n ...req.headers(),\n ...overrides?.headers,\n ...options.extraHeaders,\n }\n }, 1] as const;\n\n return req.continue(continueArgs[0], continueArgs[1]);\n }\n const overrides = req.continueRequestOverrides();\n\n return req.continue(overrides, 0);\n }\n }\n const sideload = options.sideLoad;\n\n const impersonate = sideload?.impersonate[reqUrlParsed.href];\n if (impersonate) {\n let body;\n if (impersonate.body) {\n body = await readFile(await impersonate.body.filePath);\n if (req.isInterceptResolutionHandled()) {\n return;\n }\n }\n return req.respond({\n status: impersonate.status,\n headers: impersonate.headers,\n contentType: impersonate.contentType,\n body: body ? Uint8Array.from(body) : undefined,\n }, 999);\n }\n\n const proxy = options.proxyUrl || sideload?.proxyOrigin?.[reqUrlParsed.origin];\n const ctx = this.lifeCycleTrack.get(page);\n if (proxy && ctx) {\n return await this.asyncLocalContext.bridge(ctx, async () => {\n try {\n const curled = await this.curlControl.sideLoad(reqUrlParsed, {\n ...options,\n method: req.method(),\n body: req.postData(),\n extraHeaders: {\n ...req.headers(),\n ...options.extraHeaders,\n },\n proxyUrl: proxy\n });\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n\n if (curled.chain.length === 1) {\n if (!curled.file) {\n return req.respond({\n status: curled.status,\n headers: _.omit(curled.headers, 'result'),\n contentType: curled.contentType,\n }, 3);\n }\n const body = await readFile(await curled.file.filePath);\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n return req.respond({\n status: curled.status,\n headers: _.omit(curled.headers, 'result'),\n contentType: curled.contentType,\n body: Uint8Array.from(body),\n }, 3);\n }\n options.sideLoad ??= curled.sideLoadOpts;\n _.merge(options.sideLoad, curled.sideLoadOpts);\n const firstReq = curled.chain[0];\n\n return req.respond({\n status: firstReq.result!.code,\n headers: _.omit(firstReq, 'result'),\n }, 3);\n } catch (err: any) {\n this.logger.warn(`Failed to sideload browser request ${reqUrlParsed.origin}`, { href: reqUrlParsed.href, err, proxy });\n }\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n const overrides = req.continueRequestOverrides();\n const continueArgs = [{\n ...overrides,\n headers: {\n ...req.headers(),\n ...overrides?.headers,\n ...options.extraHeaders,\n }\n }, 1] as const;\n\n return req.continue(continueArgs[0], continueArgs[1]);\n });\n }\n\n if (req.isInterceptResolutionHandled()) {\n return;\n };\n const overrides = req.continueRequestOverrides();\n const continueArgs = [{\n ...overrides,\n headers: {\n ...req.headers(),\n ...overrides?.headers,\n ...options.extraHeaders,\n }\n }, 1] as const;\n\n return req.continue(continueArgs[0], continueArgs[1]);\n });\n await page.setRequestInterception(true);\n\n const sn = this.snMap.get(page);\n this.logger.info(`Page ${sn}: Scraping ${url}`, { url });\n\n await page.evaluateOnNewDocument(`(function () {\nif (window.top !== window.self) {\n return;\n}\nconst func = ${func.toString()};\n\nfunc().then((result) => {\n window.${this._REPORT_FUNCTION_NAME}({data: result});\n}).catch((err) => {\n window.${this._REPORT_FUNCTION_NAME}({err: err});\n});\n\n})();`);\n\n if (options.locale) {\n // Add headers via request interception to walk around this bug\n // https://github.com/puppeteer/puppeteer/issues/10235\n // await page.setExtraHTTPHeaders({\n // 'Accept-Language': options.locale\n // });\n\n await page.evaluateOnNewDocument(() => {\n Object.defineProperty(navigator, \"language\", {\n get: function () {\n return options.locale;\n }\n });\n Object.defineProperty(navigator, \"languages\", {\n get: function () {\n return [options.locale];\n }\n });\n });\n }\n\n if (options.cookies) {\n const mapped = options.cookies.map((x) => {\n const draft: CookieParam = {\n name: x.name,\n value: encodeURIComponent(x.value),\n secure: x.secure,\n domain: x.domain,\n path: x.path,\n expires: x.expires ? Math.floor(x.expires.valueOf() / 1000) : undefined,\n sameSite: x.sameSite as any,\n };\n if (!draft.expires && x.maxAge) {\n draft.expires = Math.floor(Date.now() / 1000) + x.maxAge;\n }\n if (!draft.domain) {\n draft.url = parsedUrl.toString();\n }\n\n return draft;\n });\n try {\n await page.setCookie(...mapped);\n } catch (err: any) {\n this.logger.warn(`Page ${sn}: Failed to set cookies`, { err });\n throw new ParamValidationError({\n path: 'cookies',\n message: `Failed to set cookies: ${err?.message}`\n });\n }\n }\n if (options.overrideUserAgent) {\n await page.setUserAgent(options.overrideUserAgent);\n }\n if (options.viewport) {\n await page.setViewport(options.viewport);\n }\n\n const resultDeferred = Defer();\n const crippleListener = () => resultDeferred.reject(new ServiceCrashedError({ message: `Browser crashed, try again` }));\n this.once('crippled', crippleListener);\n resultDeferred.promise.finally(() => {\n this.off('crippled', crippleListener);\n });\n const hdl = (s: {\n err?: any;\n data?: T;\n }) => {\n if (s.err) {\n resultDeferred.reject(s.err);\n }\n resultDeferred.resolve(s.data);\n };\n page.on(this._REPORT_FUNCTION_NAME, hdl as any);\n page.once('abuse', (event: any) => {\n this.emit('abuse', { ...event, url: parsedUrl });\n\n resultDeferred.reject(\n new SecurityCompromiseError(`Abuse detected: ${event.reason}`)\n );\n });\n\n const timeout = options.timeoutMs || 30_000;\n const goToOptions: GoToOptions = {\n waitUntil: ['load', 'domcontentloaded', 'networkidle0'],\n timeout,\n };\n\n if (options.referer) {\n goToOptions.referer = options.referer;\n }\n\n\n const gotoPromise = page.goto(url, goToOptions)\n .catch((err) => {\n if (err instanceof TimeoutError) {\n this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err });\n return new AssertionFailureError({\n message: `Failed to goto ${url}: ${err}`,\n cause: err,\n });\n }\n\n this.logger.warn(`Page ${sn}: Browsing of ${url} aborted`, { err });\n return undefined;\n }).then(async (r) => {\n await delay(5000);\n resultDeferred.reject(new TimeoutError(`Control function did not respond in time`));\n return r;\n });\n\n try {\n await Promise.race([resultDeferred.promise, gotoPromise]);\n\n return resultDeferred.promise;\n } finally {\n page.off(this._REPORT_FUNCTION_NAME, hdl as any);\n this.ditchPage(page);\n resultDeferred.resolve();\n }\n }\n\n}\n\nconst puppeteerControl = container.resolve(SERPSpecializedPuppeteerControl);\n\nexport default puppeteerControl;\n" }, { "path": "src/services/serp/serper.ts", "content": "\nimport { singleton } from 'tsyringe';\nimport { GlobalLogger } from '../logger';\nimport { SecretExposer } from '../../shared/services/secrets';\nimport { AsyncLocalContext } from '../async-context';\nimport { SerperBingHTTP, SerperGoogleHTTP, SerperImageSearchResponse, SerperNewsSearchResponse, SerperSearchQueryParams, SerperWebSearchResponse } from '../../shared/3rd-party/serper-search';\nimport { BlackHoleDetector } from '../blackhole-detector';\nimport { Context } from '../registry';\nimport { AsyncService } from 'civkit/async-service';\nimport { AutoCastable, Prop, RPC_CALL_ENVIRONMENT } from 'civkit/civ-rpc';\n\n@singleton()\nexport class SerperGoogleSearchService extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n client!: SerperGoogleHTTP;\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected secretExposer: SecretExposer,\n protected threadLocal: AsyncLocalContext,\n protected blackHoleDetector: BlackHoleDetector,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n\n this.client = new SerperGoogleHTTP(this.secretExposer.SERPER_SEARCH_API_KEY);\n }\n\n\n doSearch(variant: 'web', query: SerperSearchQueryParams): Promise;\n doSearch(variant: 'images', query: SerperSearchQueryParams): Promise;\n doSearch(variant: 'news', query: SerperSearchQueryParams): Promise;\n async doSearch(variant: 'web' | 'images' | 'news', query: SerperSearchQueryParams) {\n this.logger.debug(`Doing external search`, query);\n let results;\n switch (variant) {\n case 'images': {\n const r = await this.client.imageSearch(query);\n\n results = r.parsed.images;\n break;\n }\n case 'news': {\n const r = await this.client.newsSearch(query);\n\n results = r.parsed.news;\n break;\n }\n case 'web':\n default: {\n const r = await this.client.webSearch(query);\n\n results = r.parsed.organic;\n break;\n }\n }\n\n this.blackHoleDetector.itWorked();\n\n return results;\n }\n\n\n async webSearch(query: SerperSearchQueryParams) {\n return this.doSearch('web', query);\n }\n async imageSearch(query: SerperSearchQueryParams) {\n return this.doSearch('images', query);\n }\n async newsSearch(query: SerperSearchQueryParams) {\n return this.doSearch('news', query);\n }\n\n}\n\n@singleton()\nexport class SerperBingSearchService extends SerperGoogleSearchService {\n override client!: SerperBingHTTP;\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n\n this.client = new SerperBingHTTP(this.secretExposer.SERPER_SEARCH_API_KEY);\n }\n}\n\nexport class GoogleSearchExplicitOperatorsDto extends AutoCastable {\n @Prop({\n arrayOf: String,\n desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.`\n })\n ext?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.`\n })\n filetype?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.`\n })\n intitle?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`\n })\n loc?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.`\n })\n site?: string | string[];\n\n addTo(searchTerm: string) {\n const chunks = [];\n for (const [key, value] of Object.entries(this)) {\n if (value) {\n const values = Array.isArray(value) ? value : [value];\n const textValue = values.map((v) => `${key}:${v}`).join(' OR ');\n if (textValue) {\n chunks.push(textValue);\n }\n }\n }\n const opPart = chunks.length > 1 ? chunks.map((x) => `(${x})`).join(' AND ') : chunks;\n\n if (opPart.length) {\n return [searchTerm, opPart].join(' ');\n }\n\n return searchTerm;\n }\n\n static override from(input: any) {\n const instance = super.from(input) as GoogleSearchExplicitOperatorsDto;\n const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined;\n\n const params = ['ext', 'filetype', 'intitle', 'loc', 'site'];\n\n for (const p of params) {\n const customValue = ctx?.get(`x-${p}`) || ctx?.get(`${p}`);\n if (!customValue) {\n continue;\n }\n\n const filtered = customValue.split(', ').filter(Boolean);\n if (filtered.length) {\n Reflect.set(instance, p, filtered);\n }\n }\n\n return instance;\n }\n}\n" }, { "path": "src/services/serper-search.ts", "content": "import { AsyncService, AutoCastable, DownstreamServiceFailureError, Prop, RPC_CALL_ENVIRONMENT, delay, marshalErrorLike } from 'civkit';\nimport { singleton } from 'tsyringe';\nimport { GlobalLogger } from './logger';\nimport { SecretExposer } from '../shared/services/secrets';\nimport { AsyncLocalContext } from './async-context';\nimport { SerperBingHTTP, SerperGoogleHTTP, SerperImageSearchResponse, SerperNewsSearchResponse, SerperSearchQueryParams, SerperWebSearchResponse } from '../shared/3rd-party/serper-search';\nimport { BlackHoleDetector } from './blackhole-detector';\nimport { Context } from './registry';\nimport { ServiceBadAttemptError } from '../shared';\n\n@singleton()\nexport class SerperSearchService extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n serperGoogleSearchHTTP!: SerperGoogleHTTP;\n serperBingSearchHTTP!: SerperBingHTTP;\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected secretExposer: SecretExposer,\n protected threadLocal: AsyncLocalContext,\n protected blackHoleDetector: BlackHoleDetector,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n\n this.serperGoogleSearchHTTP = new SerperGoogleHTTP(this.secretExposer.SERPER_SEARCH_API_KEY);\n this.serperBingSearchHTTP = new SerperBingHTTP(this.secretExposer.SERPER_SEARCH_API_KEY);\n }\n\n *iterClient() {\n const preferBingSearch = this.threadLocal.get('bing-preferred');\n if (preferBingSearch) {\n yield this.serperBingSearchHTTP;\n }\n while (true) {\n yield this.serperGoogleSearchHTTP;\n }\n }\n\n doSearch(variant: 'web', query: SerperSearchQueryParams): Promise;\n doSearch(variant: 'images', query: SerperSearchQueryParams): Promise;\n doSearch(variant: 'news', query: SerperSearchQueryParams): Promise;\n async doSearch(variant: 'web' | 'images' | 'news', query: SerperSearchQueryParams) {\n const clientIt = this.iterClient();\n let client = clientIt.next().value;\n if (!client) {\n throw new Error(`Error iterating serper client`);\n }\n\n let maxTries = 3;\n\n while (maxTries--) {\n const t0 = Date.now();\n try {\n this.logger.debug(`Doing external search`, query);\n let r;\n switch (variant) {\n case 'images': {\n r = await client.imageSearch(query);\n const nextClient = clientIt.next().value;\n if (nextClient && nextClient !== client) {\n const results = r.parsed.images;\n if (!results.length) {\n client = nextClient;\n throw new ServiceBadAttemptError('No results found');\n }\n }\n\n break;\n }\n case 'news': {\n r = await client.newsSearch(query);\n const nextClient = clientIt.next().value;\n if (nextClient && nextClient !== client) {\n const results = r.parsed.news;\n if (!results.length) {\n client = nextClient;\n throw new ServiceBadAttemptError('No results found');\n }\n }\n\n break;\n }\n case 'web':\n default: {\n r = await client.webSearch(query);\n const nextClient = clientIt.next().value;\n if (nextClient && nextClient !== client) {\n const results = r.parsed.organic;\n if (!results.length) {\n client = nextClient;\n throw new ServiceBadAttemptError('No results found');\n }\n }\n\n break;\n }\n }\n const dt = Date.now() - t0;\n this.blackHoleDetector.itWorked();\n this.logger.debug(`External search took ${dt}ms`, { searchDt: dt, variant });\n\n return r.parsed;\n } catch (err: any) {\n const dt = Date.now() - t0;\n this.logger.error(`${variant} search failed: ${err?.message}`, { searchDt: dt, err: marshalErrorLike(err) });\n if (err?.status === 429) {\n await delay(500 + 1000 * Math.random());\n continue;\n }\n if (err instanceof ServiceBadAttemptError) {\n continue;\n }\n\n throw new DownstreamServiceFailureError({ message: `Search(${variant}) failed` });\n }\n }\n\n throw new DownstreamServiceFailureError({ message: `Search(${variant}) failed` });\n }\n\n\n async webSearch(query: SerperSearchQueryParams) {\n return this.doSearch('web', query);\n }\n async imageSearch(query: SerperSearchQueryParams) {\n return this.doSearch('images', query);\n }\n async newsSearch(query: SerperSearchQueryParams) {\n return this.doSearch('news', query);\n }\n\n}\n\nexport class GoogleSearchExplicitOperatorsDto extends AutoCastable {\n @Prop({\n arrayOf: String,\n desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.`\n })\n ext?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.`\n })\n filetype?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.`\n })\n intitle?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`\n })\n loc?: string | string[];\n\n @Prop({\n arrayOf: String,\n desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.`\n })\n site?: string | string[];\n\n addTo(searchTerm: string) {\n const chunks = [];\n for (const [key, value] of Object.entries(this)) {\n if (value) {\n const values = Array.isArray(value) ? value : [value];\n const textValue = values.map((v) => `${key}:${v}`).join(' OR ');\n if (textValue) {\n chunks.push(textValue);\n }\n }\n }\n const opPart = chunks.length > 1 ? chunks.map((x) => `(${x})`).join(' AND ') : chunks;\n\n if (opPart.length) {\n return [searchTerm, opPart].join(' ');\n }\n\n return searchTerm;\n }\n\n static override from(input: any) {\n const instance = super.from(input) as GoogleSearchExplicitOperatorsDto;\n const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined;\n\n const params = ['ext', 'filetype', 'intitle', 'loc', 'site'];\n\n for (const p of params) {\n const customValue = ctx?.get(`x-${p}`) || ctx?.get(`${p}`);\n if (!customValue) {\n continue;\n }\n\n const filtered = customValue.split(', ').filter(Boolean);\n if (filtered.length) {\n Reflect.set(instance, p, filtered);\n }\n }\n\n return instance;\n }\n}\n" }, { "path": "src/services/snapshot-formatter.ts", "content": "import { randomUUID } from 'crypto';\nimport { container, singleton } from 'tsyringe';\nimport { AssertionFailureError, AsyncService, DataStreamBrokenError, FancyFile, HashManager, marshalErrorLike } from 'civkit';\nimport TurndownService, { Filter, Rule } from 'turndown';\nimport { GlobalLogger } from './logger';\nimport { PageSnapshot } from './puppeteer';\nimport { FirebaseStorageBucketControl } from '../shared/services/firebase-storage-bucket';\nimport { AsyncContext } from '../shared/services/async-context';\nimport { Threaded } from '../services/threaded';\nimport { JSDomControl } from './jsdom';\nimport { AltTextService } from './alt-text';\nimport { PDFExtractor } from './pdf-extract';\nimport { cleanAttribute } from '../utils/misc';\nimport _ from 'lodash';\nimport { STATUS_CODES } from 'http';\nimport type { CrawlerOptions } from '../dto/crawler-options';\nimport { readFile } from '../utils/encoding';\nimport { pathToFileURL } from 'url';\nimport { countGPTToken } from '../shared/utils/openai';\n\n\nexport interface FormattedPage {\n title?: string;\n description?: string;\n url?: string;\n content?: string;\n publishedTime?: string;\n html?: string;\n text?: string;\n screenshotUrl?: string;\n screenshot?: Buffer;\n pageshotUrl?: string;\n pageshot?: Buffer;\n links?: { [k: string]: string; } | [string, string][];\n images?: { [k: string]: string; } | [string, string][];\n warning?: string;\n usage?: {\n total_tokens?: number;\n totalTokens?: number;\n tokens?: number;\n };\n\n textRepresentation?: string;\n\n [Symbol.dispose]?: () => void;\n}\n\nexport const md5Hasher = new HashManager('md5', 'hex');\n\nconst gfmPlugin = require('turndown-plugin-gfm');\nconst highlightRegExp = /highlight-(?:text|source)-([a-z0-9]+)/;\n\nexport function highlightedCodeBlock(turndownService: TurndownService) {\n turndownService.addRule('highlightedCodeBlock', {\n filter: (node) => {\n return (\n node.nodeName === 'DIV' &&\n node.firstChild?.nodeName === 'PRE' &&\n highlightRegExp.test(node.className)\n );\n },\n replacement: (_content, node, options) => {\n const className = (node as any).className || '';\n const language = (className.match(highlightRegExp) || [null, ''])[1];\n\n return (\n '\\n\\n' + options.fence + language + '\\n' +\n node.firstChild!.textContent +\n '\\n' + options.fence + '\\n\\n'\n );\n }\n });\n}\n\n@singleton()\nexport class SnapshotFormatter extends AsyncService {\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n gfmPlugin = [gfmPlugin.tables, highlightedCodeBlock, gfmPlugin.strikethrough, gfmPlugin.taskListItems];\n gfmNoTable = [highlightedCodeBlock, gfmPlugin.strikethrough, gfmPlugin.taskListItems];\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected jsdomControl: JSDomControl,\n protected altTextService: AltTextService,\n protected pdfExtractor: PDFExtractor,\n protected threadLocal: AsyncContext,\n protected firebaseObjectStorage: FirebaseStorageBucketControl,\n ) {\n super(...arguments);\n }\n\n override async init() {\n await this.dependencyReady();\n this.emit('ready');\n }\n\n\n @Threaded()\n async formatSnapshot(mode: string | 'markdown' | 'html' | 'text' | 'screenshot' | 'pageshot', snapshot: PageSnapshot & {\n screenshotUrl?: string;\n pageshotUrl?: string;\n }, nominalUrl?: URL, urlValidMs = 3600 * 1000 * 4) {\n const t0 = Date.now();\n const f = {\n ...(await this.getGeneralSnapshotMixins(snapshot)),\n };\n let modeOK = false;\n\n if (mode.includes('screenshot')) {\n modeOK = true;\n if (snapshot.screenshot && !snapshot.screenshotUrl) {\n const fid = `instant-screenshots/${randomUUID()}`;\n await this.firebaseObjectStorage.saveFile(fid, snapshot.screenshot, {\n metadata: {\n contentType: 'image/png',\n }\n });\n snapshot.screenshotUrl = await this.firebaseObjectStorage.signDownloadUrl(fid, Date.now() + urlValidMs);\n }\n Object.assign(f, {\n screenshotUrl: snapshot.screenshotUrl,\n });\n\n Object.defineProperty(f, 'textRepresentation', { value: `${f.screenshotUrl}\\n`, enumerable: false, configurable: true });\n }\n if (mode.includes('pageshot')) {\n modeOK = true;\n if (snapshot.pageshot && !snapshot.pageshotUrl) {\n const fid = `instant-screenshots/${randomUUID()}`;\n await this.firebaseObjectStorage.saveFile(fid, snapshot.pageshot, {\n metadata: {\n contentType: 'image/png',\n }\n });\n snapshot.pageshotUrl = await this.firebaseObjectStorage.signDownloadUrl(fid, Date.now() + urlValidMs);\n }\n Object.assign(f, {\n html: snapshot.html,\n pageshotUrl: snapshot.pageshotUrl,\n });\n Object.defineProperty(f, 'textRepresentation', { value: `${f.pageshotUrl}\\n`, enumerable: false, configurable: true });\n }\n if (mode.includes('html')) {\n modeOK = true;\n Object.assign(f, {\n html: snapshot.html,\n });\n\n Object.defineProperty(f, 'textRepresentation', { value: snapshot.html, enumerable: false, configurable: true });\n }\n\n let pdfMode = false;\n // in case of Google Web Cache content\n if (snapshot.pdfs?.length && (!snapshot.title || snapshot.title.startsWith('cache:'))) {\n const pdf = await this.pdfExtractor.cachedExtract(snapshot.pdfs[0],\n this.threadLocal.get('cacheTolerance'),\n snapshot.pdfs[0].startsWith('http') ? undefined : snapshot.href,\n );\n if (pdf) {\n pdfMode = true;\n snapshot.title = pdf.meta?.Title;\n snapshot.text = pdf.text || snapshot.text;\n snapshot.parsed = {\n content: pdf.content,\n textContent: pdf.content,\n length: pdf.content?.length,\n byline: pdf.meta?.Author,\n lang: pdf.meta?.Language || undefined,\n title: pdf.meta?.Title,\n publishedTime: this.pdfExtractor.parsePdfDate(pdf.meta?.ModDate || pdf.meta?.CreationDate)?.toISOString(),\n };\n }\n }\n\n if (mode.includes('text')) {\n modeOK = true;\n Object.assign(f, {\n text: snapshot.text,\n });\n Object.defineProperty(f, 'textRepresentation', { value: snapshot.text, enumerable: false, configurable: true });\n }\n\n if (mode.includes('lm')) {\n modeOK = true;\n f.content = snapshot.parsed?.textContent;\n }\n\n if (modeOK && (mode.includes('lm') ||\n (!mode.includes('markdown') && !mode.includes('content')))\n ) {\n const dt = Date.now() - t0;\n this.logger.debug(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });\n\n const formatted: FormattedPage = {\n title: (snapshot.parsed?.title || snapshot.title || '').trim(),\n description: (snapshot.description || '').trim(),\n url: nominalUrl?.toString() || snapshot.href?.trim(),\n publishedTime: snapshot.parsed?.publishedTime || undefined,\n };\n\n Object.assign(f, formatted);\n\n return f;\n }\n\n const imgDataUrlToObjectUrl = !Boolean(this.threadLocal.get('keepImgDataUrl'));\n\n let contentText = '';\n const imageSummary = {} as { [k: string]: string; };\n const imageIdxTrack = new Map();\n const uid = this.threadLocal.get('uid');\n\n do {\n if (pdfMode) {\n contentText = (snapshot.parsed?.content || snapshot.text || '').trim();\n break;\n }\n\n if (\n snapshot.maxElemDepth! > 256 ||\n (!uid && snapshot.elemCount! > 10_000) ||\n snapshot.elemCount! > 80_000\n ) {\n this.logger.warn('Degrading to text to protect the server', { url: snapshot.href, elemDepth: snapshot.maxElemDepth, elemCount: snapshot.elemCount });\n contentText = (snapshot.text || '').trimEnd();\n break;\n }\n\n const noGFMOpts = this.threadLocal.get('noGfm');\n const imageRetention = this.threadLocal.get('retainImages') as CrawlerOptions['retainImages'];\n let imgIdx = 0;\n const urlToAltMap: { [k: string]: string | undefined; } = {};\n const customRules: { [k: string]: Rule; } = {\n 'img-retention': {\n filter: 'img',\n replacement: (_content: string, node: HTMLElement) => {\n if (imageRetention === 'none') {\n return '';\n }\n const alt = cleanAttribute(node.getAttribute('alt'));\n\n if (imageRetention === 'alt') {\n return alt ? `(Image ${++imgIdx}: ${alt})` : '';\n }\n const originalSrc = (node.getAttribute('src') || '').trim();\n let linkPreferredSrc = originalSrc;\n const maybeSrcSet: string = (node.getAttribute('srcset') || '').trim();\n if (!linkPreferredSrc && maybeSrcSet) {\n linkPreferredSrc = maybeSrcSet.split(',').map((x) => x.trim()).filter(Boolean)[0];\n }\n if (!linkPreferredSrc || linkPreferredSrc.startsWith('data:')) {\n const dataSrc = (node.getAttribute('data-src') || '').trim();\n if (dataSrc && !dataSrc.startsWith('data:')) {\n linkPreferredSrc = dataSrc;\n }\n }\n\n let src;\n try {\n src = new URL(linkPreferredSrc, snapshot.rebase || nominalUrl).toString();\n } catch (_err) {\n void 0;\n }\n if (!src) {\n return '';\n }\n\n const keySrc = (originalSrc.startsWith('data:') ? this.dataUrlToBlobUrl(originalSrc, snapshot.rebase) : src).trim();\n const mapped = urlToAltMap[keySrc];\n const imgSerial = ++imgIdx;\n const idxArr = imageIdxTrack.has(keySrc) ? imageIdxTrack.get(keySrc)! : [];\n idxArr.push(imgSerial);\n imageIdxTrack.set(keySrc, idxArr);\n\n if (mapped) {\n imageSummary[keySrc] = mapped || alt;\n\n if (imageRetention === 'alt_p') {\n return `(Image ${imgSerial}: ${mapped || alt})`;\n }\n\n if (imgDataUrlToObjectUrl) {\n return `![Image ${imgSerial}: ${mapped || alt}](${keySrc})`;\n }\n\n return `![Image ${imgSerial}: ${mapped || alt}](${src})`;\n } else if (imageRetention === 'alt_p') {\n return alt ? `(Image ${imgSerial}: ${alt})` : '';\n }\n\n imageSummary[keySrc] = alt || '';\n\n if (imgDataUrlToObjectUrl) {\n return alt ? `![Image ${imgSerial}: ${alt}](${keySrc})` : `![Image ${imgSerial}](${keySrc})`;\n }\n\n return alt ? `![Image ${imgSerial}: ${alt}](${src})` : `![Image ${imgSerial}](${src})`;\n }\n } as Rule\n };\n const optsMixin = {\n url: snapshot.rebase || nominalUrl,\n customRules,\n customKeep: noGFMOpts === 'table' ? 'table' : undefined,\n imgDataUrlToObjectUrl,\n } as const;\n\n const jsDomElementOfHTML = this.jsdomControl.snippetToElement(snapshot.html, snapshot.href);\n let toBeTurnedToMd = jsDomElementOfHTML;\n let turnDownService = this.getTurndown({ ...optsMixin });\n if (!mode.includes('markdown') && snapshot.parsed?.content) {\n const jsDomElementOfParsed = this.jsdomControl.snippetToElement(snapshot.parsed.content, snapshot.href);\n const par1 = this.jsdomControl.runTurndown(turnDownService, jsDomElementOfHTML);\n imgIdx = 0;\n const par2 = snapshot.parsed.content ? this.jsdomControl.runTurndown(turnDownService, jsDomElementOfParsed) : '';\n\n // If Readability did its job\n if (par2.length >= 0.3 * par1.length) {\n turnDownService = this.getTurndown({ noRules: true, ...optsMixin });\n imgIdx = 0;\n if (snapshot.parsed.content) {\n toBeTurnedToMd = jsDomElementOfParsed;\n }\n }\n }\n\n if (!noGFMOpts) {\n turnDownService = turnDownService.use(noGFMOpts === 'table' ? this.gfmNoTable : this.gfmPlugin);\n }\n\n // _p is the special suffix for withGeneratedAlt\n if (snapshot.imgs?.length && imageRetention?.endsWith('_p')) {\n const tasks = _.uniqBy((snapshot.imgs || []), 'src').map(async (x) => {\n const r = await this.altTextService.getAltText(x).catch((err: any) => {\n this.logger.warn(`Failed to get alt text for ${x.src}`, { err: marshalErrorLike(err) });\n return undefined;\n });\n if (r && x.src) {\n // note x.src here is already rebased to absolute url by browser/upstream.\n const keySrc = (x.src.startsWith('data:') ? this.dataUrlToBlobUrl(x.src, snapshot.rebase) : x.src).trim();\n urlToAltMap[keySrc] = r;\n }\n });\n\n await Promise.all(tasks);\n }\n\n if (toBeTurnedToMd) {\n try {\n contentText = this.jsdomControl.runTurndown(turnDownService, toBeTurnedToMd).trim();\n imgIdx = 0;\n } catch (err) {\n this.logger.warn(`Turndown failed to run, retrying without plugins`, { err });\n const vanillaTurnDownService = this.getTurndown({ ...optsMixin });\n try {\n contentText = this.jsdomControl.runTurndown(vanillaTurnDownService, toBeTurnedToMd).trim();\n imgIdx = 0;\n } catch (err2) {\n this.logger.warn(`Turndown failed to run, giving up`, { err: err2 });\n }\n }\n }\n\n if (\n this.isPoorlyTransformed(contentText, toBeTurnedToMd)\n && toBeTurnedToMd !== jsDomElementOfHTML\n ) {\n toBeTurnedToMd = jsDomElementOfHTML;\n try {\n contentText = this.jsdomControl.runTurndown(turnDownService, jsDomElementOfHTML).trim();\n imgIdx = 0;\n } catch (err) {\n this.logger.warn(`Turndown failed to run, retrying without plugins`, { err });\n const vanillaTurnDownService = this.getTurndown({ ...optsMixin });\n try {\n contentText = this.jsdomControl.runTurndown(vanillaTurnDownService, jsDomElementOfHTML).trim();\n imgIdx = 0;\n } catch (err2) {\n this.logger.warn(`Turndown failed to run, giving up`, { err: err2 });\n }\n }\n }\n if (mode === 'content' && this.isPoorlyTransformed(contentText, toBeTurnedToMd)) {\n contentText = (snapshot.text || '').trimEnd();\n }\n } while (false);\n\n const formatted: FormattedPage = {\n title: (snapshot.parsed?.title || snapshot.title || '').trim(),\n description: (snapshot.description || '').trim(),\n url: nominalUrl?.toString() || snapshot.href?.trim(),\n content: contentText,\n publishedTime: snapshot.parsed?.publishedTime || undefined,\n };\n\n if (snapshot.status) {\n const code = snapshot.status;\n const n = code - 200;\n if (n < 0 || n >= 200) {\n const text = snapshot.statusText || STATUS_CODES[code];\n formatted.warning ??= '';\n const msg = `Target URL returned error ${code}${text ? `: ${text}` : ''}`;\n formatted.warning = `${formatted.warning}${formatted.warning ? '\\n' : ''}${msg}`;\n }\n }\n\n if (this.threadLocal.get('withImagesSummary')) {\n formatted.images =\n _(imageSummary)\n .toPairs()\n .map(\n ([url, alt], i) => {\n const idxTrack = imageIdxTrack.get(url);\n const tag = idxTrack?.length ? `Image ${_.uniq(idxTrack).join(',')}` : `Hidden Image ${i + 1}`;\n\n return [`${tag}${alt ? `: ${alt}` : ''}`, url];\n }\n ).fromPairs()\n .value();\n }\n if (this.threadLocal.get('withLinksSummary')) {\n const links = (await this.jsdomControl.inferSnapshot(snapshot)).links;\n\n if (this.threadLocal.get('withLinksSummary') === 'all') {\n formatted.links = links;\n } else {\n formatted.links = _(links).filter(([_label, href]) => !href.startsWith('file:') && !href.startsWith('javascript:')).uniqBy(1).fromPairs().value();\n }\n }\n\n if (countGPTToken(formatted.content) < 200) {\n formatted.warning ??= '';\n if (snapshot.isIntermediate) {\n const msg = 'This page maybe not yet fully loaded, consider explicitly specify a timeout.';\n formatted.warning = `${formatted.warning}${formatted.warning ? '\\n' : ''}${msg}`;\n }\n if (snapshot.childFrames?.length && !this.threadLocal.get('withIframe')) {\n const msg = 'This page contains iframe that are currently hidden, consider enabling iframe processing.';\n formatted.warning = `${formatted.warning}${formatted.warning ? '\\n' : ''}${msg}`;\n }\n if (snapshot.shadowExpanded && !this.threadLocal.get('withShadowDom')) {\n const msg = 'This page contains shadow DOM that are currently hidden, consider enabling shadow DOM processing.';\n formatted.warning = `${formatted.warning}${formatted.warning ? '\\n' : ''}${msg}`;\n }\n if (snapshot.html.includes('captcha') || snapshot.html.includes('cf-turnstile-response')) {\n const msg = 'This page maybe requiring CAPTCHA, please make sure you are authorized to access this page.';\n formatted.warning = `${formatted.warning}${formatted.warning ? '\\n' : ''}${msg}`;\n }\n if (snapshot.isFromCache) {\n const msg = 'This is a cached snapshot of the original page, consider retry with caching opt-out.';\n formatted.warning = `${formatted.warning}${formatted.warning ? '\\n' : ''}${msg}`;\n }\n }\n\n Object.assign(f, formatted);\n\n const textRepresentation = (function (this: typeof formatted) {\n const mixins = [];\n if (this.publishedTime) {\n mixins.push(`Published Time: ${this.publishedTime}`);\n }\n const suffixMixins = [];\n if (this.images) {\n const imageSummaryChunks = ['Images:'];\n for (const [k, v] of Object.entries(this.images)) {\n imageSummaryChunks.push(`- ![${k}](${v})`);\n }\n if (imageSummaryChunks.length === 1) {\n imageSummaryChunks.push('This page does not seem to contain any images.');\n }\n suffixMixins.push(imageSummaryChunks.join('\\n'));\n }\n if (this.links) {\n const linkSummaryChunks = ['Links/Buttons:'];\n if (Array.isArray(this.links)) {\n for (const [k, v] of this.links) {\n linkSummaryChunks.push(`- [${k}](${v})`);\n }\n } else {\n for (const [k, v] of Object.entries(this.links)) {\n linkSummaryChunks.push(`- [${k}](${v})`);\n }\n }\n if (linkSummaryChunks.length === 1) {\n linkSummaryChunks.push('This page does not seem to contain any buttons/links.');\n }\n suffixMixins.push(linkSummaryChunks.join('\\n'));\n }\n\n if (this.warning) {\n mixins.push(this.warning.split('\\n').map((v) => `Warning: ${v}`).join('\\n'));\n }\n\n if (mode.includes('markdown')) {\n return `${mixins.length ? `${mixins.join('\\n\\n')}\\n\\n` : ''}${this.content}\n${suffixMixins.length ? `\\n${suffixMixins.join('\\n\\n')}\\n` : ''}`;\n }\n\n return `Title: ${this.title}\n\nURL Source: ${this.url}\n${mixins.length ? `\\n${mixins.join('\\n\\n')}\\n` : ''}\nMarkdown Content:\n${this.content}\n${suffixMixins.length ? `\\n${suffixMixins.join('\\n\\n')}\\n` : ''}`;\n }).call(formatted);\n\n Object.defineProperty(f, 'textRepresentation', { value: textRepresentation, enumerable: false });\n\n const dt = Date.now() - t0;\n this.logger.debug(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });\n\n return f as FormattedPage;\n }\n\n dataUrlToBlobUrl(dataUrl: string, baseUrl: string = 'http://localhost/') {\n const refUrl = new URL(baseUrl);\n const mappedUrl = new URL(`blob:${refUrl.origin || 'localhost'}/${md5Hasher.hash(dataUrl)}`);\n\n return mappedUrl.href;\n }\n\n async getGeneralSnapshotMixins(snapshot: PageSnapshot) {\n let inferred;\n const mixin: any = {};\n if (this.threadLocal.get('withImagesSummary')) {\n inferred ??= await this.jsdomControl.inferSnapshot(snapshot);\n const imageSummary = {} as { [k: string]: string; };\n const imageIdxTrack = new Map();\n\n let imgIdx = 0;\n\n for (const img of inferred.imgs) {\n const imgSerial = ++imgIdx;\n const keySrc = (img.src.startsWith('data:') ? this.dataUrlToBlobUrl(img.src, snapshot.rebase) : img.src).trim();\n const idxArr = imageIdxTrack.has(keySrc) ? imageIdxTrack.get(keySrc)! : [];\n idxArr.push(imgSerial);\n imageIdxTrack.set(keySrc, idxArr);\n imageSummary[keySrc] = img.alt || '';\n }\n\n mixin.images =\n _(imageSummary)\n .toPairs()\n .map(\n ([url, alt], i) => {\n const idxTrack = imageIdxTrack.get(url);\n const tag = idxTrack?.length ? `Image ${_.uniq(idxTrack).join(',')}` : `Hidden Image ${i + 1}`;\n\n return [`${tag}${alt ? `: ${alt}` : ''}`, url];\n }\n ).fromPairs()\n .value();\n }\n if (this.threadLocal.get('withLinksSummary')) {\n inferred ??= await this.jsdomControl.inferSnapshot(snapshot);\n if (this.threadLocal.get('withLinksSummary') === 'all') {\n mixin.links = inferred.links;\n } else {\n mixin.links = _(inferred.links).filter(([_label, href]) => !href.startsWith('file:') && !href.startsWith('javascript:')).uniqBy(1).fromPairs().value();\n }\n }\n if (snapshot.status) {\n const code = snapshot.status;\n const n = code - 200;\n if (n < 0 || n >= 200) {\n const text = snapshot.statusText || STATUS_CODES[code];\n mixin.warning ??= '';\n const msg = `Target URL returned error ${code}${text ? `: ${text}` : ''}`;\n mixin.warning = `${mixin.warning}${mixin.warning ? '\\n' : ''}${msg}`;\n }\n }\n\n return mixin;\n }\n\n getTurndown(options?: {\n noRules?: boolean | string,\n url?: string | URL;\n imgDataUrlToObjectUrl?: boolean;\n removeImages?: boolean | 'src';\n customRules?: { [k: string]: Rule; };\n customKeep?: Filter;\n }) {\n const turndownOpts = this.threadLocal.get('turndownOpts');\n const turnDownService = new TurndownService({\n ...turndownOpts,\n codeBlockStyle: 'fenced',\n preformattedCode: true,\n } as any);\n if (options?.customKeep) {\n turnDownService.keep(options.customKeep);\n }\n if (!options?.noRules) {\n turnDownService.addRule('remove-irrelevant', {\n filter: ['meta', 'style', 'script', 'noscript', 'link', 'textarea', 'select'],\n replacement: () => ''\n });\n turnDownService.addRule('truncate-svg', {\n filter: 'svg' as any,\n replacement: () => ''\n });\n turnDownService.addRule('title-as-h1', {\n filter: ['title'],\n replacement: (innerText) => `${innerText}\\n===============\\n`\n });\n }\n\n if (options?.imgDataUrlToObjectUrl) {\n turnDownService.addRule('data-url-to-pseudo-object-url', {\n filter: (node) => Boolean(node.tagName === 'IMG' && node.getAttribute('src')?.startsWith('data:')),\n replacement: (_content, node: any) => {\n const src = (node.getAttribute('src') || '').trim();\n const alt = cleanAttribute(node.getAttribute('alt')) || '';\n\n const blobUrl = this.dataUrlToBlobUrl(src, options.url?.toString());\n\n return `![${alt}](${blobUrl})`;\n }\n });\n }\n\n if (options?.customRules) {\n for (const [k, v] of Object.entries(options.customRules)) {\n turnDownService.addRule(k, v);\n }\n }\n\n turnDownService.addRule('improved-heading', {\n filter: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'],\n replacement: (content, node, options) => {\n const hLevel = Number(node.nodeName.charAt(1));\n if (options.headingStyle === 'setext' && hLevel < 3) {\n const underline = _.repeat((hLevel === 1 ? '=' : '-'), Math.min(128, content.length));\n return (\n '\\n\\n' + content + '\\n' + underline + '\\n\\n'\n );\n } else {\n return '\\n\\n' + _.repeat('#', hLevel) + ' ' + content + '\\n\\n';\n }\n }\n });\n\n turnDownService.addRule('improved-paragraph', {\n filter: 'p',\n replacement: (innerText) => {\n const trimmed = innerText.trim();\n if (!trimmed) {\n return '';\n }\n\n return `${trimmed.replace(/\\n{3,}/g, '\\n\\n')}\\n\\n`;\n }\n });\n\n let realLinkStyle: 'inlined' | 'collapsed' | 'shortcut' | 'referenced' | 'discarded' = 'inlined';\n if (turndownOpts?.linkStyle === 'referenced' || turndownOpts?.linkReferenceStyle) {\n realLinkStyle = 'referenced';\n if (turndownOpts?.linkReferenceStyle === 'collapsed') {\n realLinkStyle = 'collapsed';\n } else if (turndownOpts?.linkReferenceStyle === 'shortcut') {\n realLinkStyle = 'shortcut';\n } else if (turndownOpts?.linkReferenceStyle === 'discarded') {\n realLinkStyle = 'discarded';\n }\n } else if (turndownOpts?.linkStyle === 'discarded') {\n realLinkStyle = 'discarded';\n }\n\n turnDownService.addRule('improved-link', {\n filter: function (node, _options) {\n return Boolean(\n node.nodeName === 'A' &&\n node.getAttribute('href')\n );\n },\n\n replacement: function (this: { references: string[]; }, content, node: any) {\n var href = node.getAttribute('href');\n let title = cleanAttribute(node.getAttribute('title'));\n if (title) title = ` \"${title.replace(/\"/g, '\\\\\"')}\"`;\n let replacement;\n let reference;\n const fixedContent = content.replace(/\\s+/g, ' ').trim();\n let fixedHref = href;\n if (options?.url) {\n try {\n fixedHref = new URL(fixedHref, options.url).toString();\n } catch (_err) {\n void 0;\n }\n }\n\n switch (realLinkStyle) {\n case 'inlined':\n replacement = `[${fixedContent}](${fixedHref}${title || ''})`;\n reference = undefined;\n break;\n case 'collapsed':\n replacement = `[${fixedContent}][]`;\n reference = `[${fixedContent}]: ${fixedHref}${title}`;\n break;\n case 'shortcut':\n replacement = `[${fixedContent}]`;\n reference = `[${fixedContent}]: ${fixedHref}${title}`;\n break;\n case 'discarded':\n replacement = content;\n reference = undefined;\n break;\n default:\n const id = this.references.length + 1;\n replacement = `[${fixedContent}][${id}]`;\n reference = `[${id}]${fixedHref}${title}`;\n }\n\n if (reference) {\n this.references.push(reference);\n }\n\n return replacement;\n },\n\n // @ts-ignore\n references: [],\n\n append: function (this: { references: string[]; }) {\n let references = '';\n if (this.references.length) {\n references = `\\n\\n${this.references.join('\\n')}\\n\\n`;\n this.references = []; // Reset references\n }\n return references;\n }\n });\n turnDownService.addRule('improved-code', {\n filter: function (node: any) {\n let hasSiblings = node.previousSibling || node.nextSibling;\n let isCodeBlock = node.parentNode.nodeName === 'PRE' && !hasSiblings;\n\n return node.nodeName === 'CODE' && !isCodeBlock;\n },\n\n replacement: function (inputContent: any) {\n if (!inputContent) return '';\n let content = inputContent;\n\n let delimiter = '`';\n let matches = content.match(/`+/gm) || [];\n while (matches.indexOf(delimiter) !== -1) delimiter = delimiter + '`';\n if (content.includes('\\n')) {\n delimiter = '```';\n }\n\n let extraSpace = delimiter === '```' ? '\\n' : /^`|^ .*?[^ ].* $|`$/.test(content) ? ' ' : '';\n\n return delimiter + extraSpace + content + (delimiter === '```' && !content.endsWith(extraSpace) ? extraSpace : '') + delimiter;\n }\n });\n\n return turnDownService;\n }\n\n\n isPoorlyTransformed(content?: string, node?: Element) {\n if (!content) {\n return true;\n }\n\n if (content.startsWith('<') && content.endsWith('>')) {\n return true;\n }\n\n if (!this.threadLocal.get('noGfm') && content.includes('')) {\n if (node?.textContent && content.length > node.textContent.length * 0.8) {\n return true;\n }\n\n const tableElms = node?.querySelectorAll('table') || [];\n const deepTableElms = node?.querySelectorAll('table table');\n if (node && tableElms.length) {\n const wrappingTables = _.without(tableElms, ...Array.from(deepTableElms || []));\n const tableTextsLength = _.sum(wrappingTables.map((x) => (x.innerHTML?.length || 0)));\n\n if (tableTextsLength / (content.length) > 0.6) {\n return true;\n }\n }\n\n const tbodyElms = node?.querySelectorAll('tbody') || [];\n const deepTbodyElms = node?.querySelectorAll('tbody tbody');\n if ((deepTbodyElms?.length || 0) / tbodyElms.length > 0.6) {\n return true;\n }\n }\n\n return false;\n }\n\n async createSnapshotFromFile(url: URL, file: FancyFile, overrideContentType?: string, overrideFileName?: string) {\n if (overrideContentType === 'application/octet-stream') {\n overrideContentType = undefined;\n }\n\n const contentType: string = (overrideContentType || await file.mimeType).toLowerCase();\n const fileName = overrideFileName || `${url.origin}${url.pathname}`;\n const snapshot: PageSnapshot = {\n title: '',\n href: url.href,\n html: '',\n text: ''\n };\n\n if (contentType.startsWith('image/')) {\n snapshot.html = `${fileName}`;\n snapshot.title = fileName;\n snapshot.imgs = [{ src: url.href }];\n\n return snapshot;\n }\n try {\n const encoding: string | undefined = contentType.includes('charset=') ? contentType.split('charset=')[1]?.trim().toLowerCase() : 'utf-8';\n if (contentType.startsWith('text/html')) {\n if ((await file.size) > 1024 * 1024 * 32) {\n throw new AssertionFailureError(`Failed to access ${url}: file too large`);\n }\n snapshot.html = await readFile(await file.filePath, encoding);\n let innerCharset;\n const peek = snapshot.html.slice(0, 1024);\n innerCharset ??= peek.match(/]+text\\/html;\\s*?charset=([^>\"]+)/i)?.[1]?.toLowerCase();\n innerCharset ??= peek.match(/]+charset=\"([^>\"]+)\\\"/i)?.[1]?.toLowerCase();\n if (innerCharset && innerCharset !== encoding) {\n snapshot.html = await readFile(await file.filePath, innerCharset);\n }\n\n return snapshot;\n }\n if (contentType.startsWith('text/') || contentType.startsWith('application/json')) {\n if ((await file.size) > 1024 * 1024 * 32) {\n throw new AssertionFailureError(`Failed to access ${url}: file too large`);\n }\n snapshot.text = await readFile(await file.filePath, encoding);\n snapshot.html = `
${snapshot.text}
`;\n\n return snapshot;\n }\n if (contentType.startsWith('application/pdf')) {\n snapshot.pdfs = [pathToFileURL(await file.filePath).href];\n\n return snapshot;\n }\n } catch (err: any) {\n this.logger.warn(`Failed to read from file: ${url}`, { err, url });\n throw new DataStreamBrokenError(`Failed to access ${url}: ${err?.message}`);\n }\n\n throw new AssertionFailureError(`Failed to access ${url}: unexpected type ${contentType}`);\n }\n}\n\nconst snapshotFormatter = container.resolve(SnapshotFormatter);\n\nexport default snapshotFormatter;\n" }, { "path": "src/services/temp-file.ts", "content": "import { AbstractTempFileManger } from 'civkit/temp';\nimport { rm } from 'fs/promises';\nimport { singleton } from 'tsyringe';\nimport { Finalizer } from './finalizer';\n\n@singleton()\nexport class TempFileManager extends AbstractTempFileManger {\n\n rootDir = '';\n\n override async init() {\n await this.dependencyReady();\n await super.init();\n this.emit('ready');\n }\n\n @Finalizer()\n override async standDown() {\n await super.standDown();\n\n await rm(this.rootDir, { recursive: true, force: true });\n }\n}\n" }, { "path": "src/services/threaded.ts", "content": "import 'reflect-metadata';\n\nimport { singleton, container } from 'tsyringe';\nimport { AbstractThreadedServiceRegistry } from 'civkit/threaded';\nimport _ from 'lodash';\n\nimport { GlobalLogger } from './logger';\nimport { AsyncLocalContext } from './async-context';\nimport { PseudoTransfer } from './pseudo-transfer';\nimport { cpus } from 'os';\nimport { isMainThread } from 'worker_threads';\n\n@singleton()\nexport class ThreadedServiceRegistry extends AbstractThreadedServiceRegistry {\n container = container;\n\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n constructor(\n protected globalLogger: GlobalLogger,\n public asyncContext: AsyncLocalContext,\n public pseudoTransfer: PseudoTransfer,\n ) {\n super(...arguments);\n }\n\n setMaxWorkersByCpu() {\n const cpuStat = cpus();\n\n const evenCpuCycles = cpuStat.filter((_cpu, i) => i % 2 === 0).reduce((acc, cpu) => acc + cpu.times.user + cpu.times.sys, 0);\n const oddCpuCycles = cpuStat.filter((_cpu, i) => i % 2 === 1).reduce((acc, cpu) => acc + cpu.times.user + cpu.times.sys, 0);\n\n const isLikelyHyperThreaded = (oddCpuCycles / evenCpuCycles) < 0.5;\n\n this.maxWorkers = isLikelyHyperThreaded ? cpuStat.length / 2 : cpuStat.length;\n }\n\n override async init() {\n await this.dependencyReady();\n await super.init();\n\n if (isMainThread) {\n this.setMaxWorkersByCpu();\n await Promise.all(\n _.range(0, 2).map(\n (_n) =>\n new Promise(\n (resolve, reject) => {\n this.createWorker()\n .once('message', resolve)\n .once('error', reject);\n }\n )\n )\n );\n }\n\n this.emit('ready');\n }\n\n}\n\n\nconst instance = container.resolve(ThreadedServiceRegistry);\nexport default instance;\nexport const { Method, Param, Ctx, RPCReflect, Threaded } = instance.decorators();\n" }, { "path": "src/stand-alone/crawl.ts", "content": "import 'reflect-metadata';\nimport { container, singleton } from 'tsyringe';\n\nimport { KoaServer } from 'civkit/civ-rpc/koa';\nimport http2 from 'http2';\nimport http from 'http';\nimport { CrawlerHost } from '../api/crawler';\nimport { FsWalk, WalkOutEntity } from 'civkit/fswalk';\nimport path from 'path';\nimport fs from 'fs';\nimport { mimeOfExt } from 'civkit/mime';\nimport { Context, Next } from 'koa';\nimport { RPCRegistry } from '../services/registry';\nimport { AsyncResource } from 'async_hooks';\nimport { runOnce } from 'civkit/decorators';\nimport { randomUUID } from 'crypto';\nimport { ThreadedServiceRegistry } from '../services/threaded';\nimport { GlobalLogger } from '../services/logger';\nimport { AsyncLocalContext } from '../services/async-context';\nimport finalizer, { Finalizer } from '../services/finalizer';\nimport koaCompress from 'koa-compress';\n\n@singleton()\nexport class CrawlStandAloneServer extends KoaServer {\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n httpAlternativeServer?: typeof this['httpServer'];\n assets = new Map();\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected registry: RPCRegistry,\n protected crawlerHost: CrawlerHost,\n protected threadLocal: AsyncLocalContext,\n protected threads: ThreadedServiceRegistry,\n ) {\n super(...arguments);\n }\n\n h2c() {\n this.httpAlternativeServer = this.httpServer;\n const fn = this.koaApp.callback();\n this.httpServer = http2.createServer((req, res) => {\n const ar = new AsyncResource('HTTP2ServerRequest');\n ar.runInAsyncScope(fn, this.koaApp, req, res);\n });\n // useResourceBasedDefaultTracker();\n\n return this;\n }\n\n override async init() {\n await this.walkForAssets();\n await super.init();\n }\n\n async walkForAssets() {\n const files = await FsWalk.walkOut(path.resolve(__dirname, '..', '..', 'public'));\n\n for (const file of files) {\n if (file.type !== 'file') {\n continue;\n }\n this.assets.set(file.relativePath.toString(), file);\n }\n }\n\n override listen(port: number) {\n const r = super.listen(port);\n if (this.httpAlternativeServer) {\n const altPort = port + 1;\n this.httpAlternativeServer.listen(altPort, () => {\n this.logger.info(`Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`);\n });\n }\n\n return r;\n }\n\n makeAssetsServingController() {\n return (ctx: Context, next: Next) => {\n const requestPath = ctx.path;\n const file = requestPath.slice(1);\n if (!file) {\n return next();\n }\n\n const asset = this.assets.get(file);\n if (asset?.type !== 'file') {\n return next();\n }\n\n ctx.body = fs.createReadStream(asset.path);\n ctx.type = mimeOfExt(path.extname(asset.path.toString())) || 'application/octet-stream';\n ctx.set('Content-Length', asset.stats.size.toString());\n\n return;\n };\n }\n\n registerRoutes(): void {\n this.koaApp.use(koaCompress({\n filter(type) {\n if (type.startsWith('text/')) {\n return true;\n }\n\n if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {\n return true;\n }\n\n if (type.includes('application/x-ndjson')) {\n return true;\n }\n\n return false;\n }\n }));\n this.koaApp.use(this.makeAssetsServingController());\n this.koaApp.use(this.registry.makeShimController());\n }\n\n // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context\n // TraceId is expected to be request-bound and unique. So these two has to be distinguished.\n @runOnce()\n override insertAsyncHookMiddleware() {\n const asyncHookMiddleware = async (ctx: Context, next: () => Promise) => {\n const googleTraceId = ctx.get('x-cloud-trace-context').split('/')?.[0];\n this.threadLocal.setup({\n traceId: randomUUID(),\n traceT0: new Date(),\n googleTraceId,\n });\n\n return next();\n };\n\n this.koaApp.use(asyncHookMiddleware);\n }\n\n @Finalizer()\n override async standDown() {\n const tasks: Promise[] = [];\n if (this.httpAlternativeServer?.listening) {\n (this.httpAlternativeServer as http.Server).closeIdleConnections?.();\n this.httpAlternativeServer.close();\n tasks.push(new Promise((resolve, reject) => {\n this.httpAlternativeServer!.close((err) => {\n if (err) {\n return reject(err);\n }\n resolve();\n });\n }));\n }\n tasks.push(super.standDown());\n await Promise.all(tasks);\n }\n\n}\nconst instance = container.resolve(CrawlStandAloneServer);\n\nexport default instance;\n\nif (process.env.NODE_ENV?.includes('dry-run')) {\n instance.serviceReady().then(() => finalizer.terminate());\n} else {\n instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000));\n}" }, { "path": "src/stand-alone/search.ts", "content": "import 'reflect-metadata';\nimport { container, singleton } from 'tsyringe';\n\nimport { KoaServer } from 'civkit/civ-rpc/koa';\nimport http2 from 'http2';\nimport http from 'http';\nimport { SearcherHost } from '../api/searcher';\nimport { FsWalk, WalkOutEntity } from 'civkit/fswalk';\nimport path from 'path';\nimport fs from 'fs';\nimport { mimeOfExt } from 'civkit/mime';\nimport { Context, Next } from 'koa';\nimport { RPCRegistry } from '../services/registry';\nimport { AsyncResource } from 'async_hooks';\nimport { runOnce } from 'civkit/decorators';\nimport { randomUUID } from 'crypto';\nimport { ThreadedServiceRegistry } from '../services/threaded';\nimport { GlobalLogger } from '../services/logger';\nimport { AsyncLocalContext } from '../services/async-context';\nimport finalizer, { Finalizer } from '../services/finalizer';\nimport koaCompress from 'koa-compress';\n\n@singleton()\nexport class SearchStandAloneServer extends KoaServer {\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n httpAlternativeServer?: typeof this['httpServer'];\n assets = new Map();\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected registry: RPCRegistry,\n protected searcherHost: SearcherHost,\n protected threadLocal: AsyncLocalContext,\n protected threads: ThreadedServiceRegistry,\n ) {\n super(...arguments);\n }\n\n h2c() {\n this.httpAlternativeServer = this.httpServer;\n const fn = this.koaApp.callback();\n this.httpServer = http2.createServer((req, res) => {\n const ar = new AsyncResource('HTTP2ServerRequest');\n ar.runInAsyncScope(fn, this.koaApp, req, res);\n });\n // useResourceBasedDefaultTracker();\n\n return this;\n }\n\n override async init() {\n await this.walkForAssets();\n await this.dependencyReady();\n\n for (const [k, v] of this.registry.conf.entries()) {\n if (v.tags?.includes('crawl')) {\n this.registry.conf.delete(k);\n }\n }\n\n await super.init();\n }\n\n async walkForAssets() {\n const files = await FsWalk.walkOut(path.resolve(__dirname, '..', '..', 'public'));\n\n for (const file of files) {\n if (file.type !== 'file') {\n continue;\n }\n this.assets.set(file.relativePath.toString(), file);\n }\n }\n\n override listen(port: number) {\n const r = super.listen(port);\n if (this.httpAlternativeServer) {\n const altPort = port + 1;\n this.httpAlternativeServer.listen(altPort, () => {\n this.logger.info(`Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`);\n });\n }\n\n return r;\n }\n\n makeAssetsServingController() {\n return (ctx: Context, next: Next) => {\n const requestPath = ctx.path;\n const file = requestPath.slice(1);\n if (!file) {\n return next();\n }\n\n const asset = this.assets.get(file);\n if (asset?.type !== 'file') {\n return next();\n }\n\n ctx.body = fs.createReadStream(asset.path);\n ctx.type = mimeOfExt(path.extname(asset.path.toString())) || 'application/octet-stream';\n ctx.set('Content-Length', asset.stats.size.toString());\n\n return;\n };\n }\n\n registerRoutes(): void {\n this.koaApp.use(koaCompress({\n filter(type) {\n if (type.startsWith('text/')) {\n return true;\n }\n\n if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {\n return true;\n }\n\n if (type.includes('application/x-ndjson')) {\n return true;\n }\n\n return false;\n }\n }));\n this.koaApp.use(this.makeAssetsServingController());\n this.koaApp.use(this.registry.makeShimController());\n }\n\n\n // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context\n // TraceId is expected to be request-bound and unique. So these two has to be distinguished.\n @runOnce()\n override insertAsyncHookMiddleware() {\n const asyncHookMiddleware = async (ctx: Context, next: () => Promise) => {\n const googleTraceId = ctx.get('x-cloud-trace-context').split('/')?.[0];\n this.threadLocal.setup({\n traceId: randomUUID(),\n traceT0: new Date(),\n googleTraceId,\n });\n\n return next();\n };\n\n this.koaApp.use(asyncHookMiddleware);\n }\n\n @Finalizer()\n override async standDown() {\n const tasks: Promise[] = [];\n if (this.httpAlternativeServer?.listening) {\n (this.httpAlternativeServer as http.Server).closeIdleConnections?.();\n this.httpAlternativeServer.close();\n tasks.push(new Promise((resolve, reject) => {\n this.httpAlternativeServer!.close((err) => {\n if (err) {\n return reject(err);\n }\n resolve();\n });\n }));\n }\n tasks.push(super.standDown());\n await Promise.all(tasks);\n }\n\n}\nconst instance = container.resolve(SearchStandAloneServer);\n\nexport default instance;\n\nif (process.env.NODE_ENV?.includes('dry-run')) {\n instance.serviceReady().then(() => finalizer.terminate());\n} else {\n instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000));\n}\n" }, { "path": "src/stand-alone/serp.ts", "content": "import 'reflect-metadata';\nimport { container, singleton } from 'tsyringe';\n\nimport { KoaServer } from 'civkit/civ-rpc/koa';\nimport http2 from 'http2';\nimport http from 'http';\nimport { FsWalk, WalkOutEntity } from 'civkit/fswalk';\nimport path from 'path';\nimport fs from 'fs';\nimport { mimeOfExt } from 'civkit/mime';\nimport { Context, Next } from 'koa';\nimport { RPCRegistry } from '../services/registry';\nimport { AsyncResource } from 'async_hooks';\nimport { runOnce } from 'civkit/decorators';\nimport { randomUUID } from 'crypto';\nimport { ThreadedServiceRegistry } from '../services/threaded';\nimport { GlobalLogger } from '../services/logger';\nimport { AsyncLocalContext } from '../services/async-context';\nimport finalizer, { Finalizer } from '../services/finalizer';\nimport { SerpHost } from '../api/serp';\nimport koaCompress from 'koa-compress';\nimport { getAuditionMiddleware } from '../shared/utils/audition';\n\n@singleton()\nexport class SERPStandAloneServer extends KoaServer {\n logger = this.globalLogger.child({ service: this.constructor.name });\n\n httpAlternativeServer?: typeof this['httpServer'];\n assets = new Map();\n\n constructor(\n protected globalLogger: GlobalLogger,\n protected registry: RPCRegistry,\n protected serpHost: SerpHost,\n protected threadLocal: AsyncLocalContext,\n protected threads: ThreadedServiceRegistry,\n ) {\n super(...arguments);\n }\n\n h2c() {\n this.httpAlternativeServer = this.httpServer;\n const fn = this.koaApp.callback();\n this.httpServer = http2.createServer((req, res) => {\n const ar = new AsyncResource('HTTP2ServerRequest');\n ar.runInAsyncScope(fn, this.koaApp, req, res);\n });\n // useResourceBasedDefaultTracker();\n\n return this;\n }\n\n override async init() {\n await this.walkForAssets();\n await this.dependencyReady();\n\n for (const [k, v] of this.registry.conf.entries()) {\n if (v.tags?.includes('crawl')) {\n this.registry.conf.delete(k);\n }\n }\n\n await super.init();\n }\n\n async walkForAssets() {\n const files = await FsWalk.walkOut(path.resolve(__dirname, '..', '..', 'public'));\n\n for (const file of files) {\n if (file.type !== 'file') {\n continue;\n }\n this.assets.set(file.relativePath.toString(), file);\n }\n }\n\n override listen(port: number) {\n const r = super.listen(port);\n if (this.httpAlternativeServer) {\n const altPort = port + 1;\n this.httpAlternativeServer.listen(altPort, () => {\n this.logger.info(`Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`);\n });\n }\n\n return r;\n }\n\n makeAssetsServingController() {\n return (ctx: Context, next: Next) => {\n const requestPath = ctx.path;\n const file = requestPath.slice(1);\n if (!file) {\n return next();\n }\n\n const asset = this.assets.get(file);\n if (asset?.type !== 'file') {\n return next();\n }\n\n ctx.body = fs.createReadStream(asset.path);\n ctx.type = mimeOfExt(path.extname(asset.path.toString())) || 'application/octet-stream';\n ctx.set('Content-Length', asset.stats.size.toString());\n\n return;\n };\n }\n\n registerRoutes(): void {\n this.koaApp.use(getAuditionMiddleware());\n this.koaApp.use(koaCompress({\n filter(type) {\n if (type.startsWith('text/')) {\n return true;\n }\n\n if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {\n return true;\n }\n\n if (type.includes('application/x-ndjson')) {\n return true;\n }\n\n return false;\n }\n }));\n this.koaApp.use(this.makeAssetsServingController());\n this.koaApp.use(this.registry.makeShimController());\n }\n\n\n // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context\n // TraceId is expected to be request-bound and unique. So these two has to be distinguished.\n @runOnce()\n override insertAsyncHookMiddleware() {\n const asyncHookMiddleware = async (ctx: Context, next: () => Promise) => {\n const googleTraceId = ctx.get('x-cloud-trace-context').split('/')?.[0];\n this.threadLocal.setup({\n traceId: randomUUID(),\n traceT0: new Date(),\n googleTraceId,\n });\n\n return next();\n };\n\n this.koaApp.use(asyncHookMiddleware);\n }\n\n @Finalizer()\n override async standDown() {\n const tasks: Promise[] = [];\n if (this.httpAlternativeServer?.listening) {\n (this.httpAlternativeServer as http.Server).closeIdleConnections?.();\n this.httpAlternativeServer.close();\n tasks.push(new Promise((resolve, reject) => {\n this.httpAlternativeServer!.close((err) => {\n if (err) {\n return reject(err);\n }\n resolve();\n });\n }));\n }\n tasks.push(super.standDown());\n await Promise.all(tasks);\n }\n\n}\nconst instance = container.resolve(SERPStandAloneServer);\n\nexport default instance;\n\nif (process.env.NODE_ENV?.includes('dry-run')) {\n instance.serviceReady().then(() => finalizer.terminate());\n} else {\n instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000));\n}\n" }, { "path": "src/types.d.ts", "content": "declare module 'langdetect' {\n interface DetectionResult {\n lang: string;\n prob: number;\n }\n\n export function detect(text: string): DetectionResult[];\n export function detectOne(text: string): string | null;\n}\n\ndeclare module 'jsdom' {\n import EventEmitter from 'events';\n export class JSDOM {\n constructor(html: string, options?: any);\n window: typeof window;\n }\n export class VirtualConsole extends EventEmitter {\n constructor();\n sendTo(console: any, options?: any);\n }\n}\n\ndeclare module 'simple-zstd' {\n import { Duplex } from 'stream';\n export function ZSTDCompress(lvl: Number): Duplex;\n export function ZSTDDecompress(): Duplex;\n export function ZSTDDecompressMaybe(): Duplex;\n}\n" }, { "path": "src/utils/encoding.ts", "content": "import { createReadStream } from 'fs';\nimport { Readable } from 'stream';\nimport { TextDecoderStream } from 'stream/web';\n\nexport async function decodeFileStream(\n fileStream: Readable,\n encoding: string = 'utf-8',\n): Promise {\n const decodeStream = new TextDecoderStream(encoding, { fatal: false, ignoreBOM: false });\n Readable.toWeb(fileStream).pipeThrough(decodeStream);\n const chunks = [];\n\n for await (const chunk of decodeStream.readable) {\n chunks.push(chunk);\n }\n\n return chunks.join('');\n}\n\n\nexport async function readFile(\n filePath: string,\n encoding: string = 'utf-8',\n): Promise {\n const decodeStream = new TextDecoderStream(encoding, { fatal: false, ignoreBOM: false });\n Readable.toWeb(createReadStream(filePath)).pipeThrough(decodeStream);\n const chunks = [];\n\n for await (const chunk of decodeStream.readable) {\n chunks.push(chunk);\n }\n\n return chunks.join('');\n}" }, { "path": "src/utils/get-function-url.ts", "content": "import { GoogleAuth } from 'google-auth-library';\n\n/**\n * Get the URL of a given v2 cloud function.\n *\n * @param {string} name the function's name\n * @param {string} location the function's location\n * @return {Promise} The URL of the function\n */\nexport async function getFunctionUrl(name: string, location = \"us-central1\") {\n const projectId = `reader-6b7dc`;\n const url = \"https://cloudfunctions.googleapis.com/v2beta/\" +\n `projects/${projectId}/locations/${location}/functions/${name}`;\n const auth = new GoogleAuth({\n scopes: 'https://www.googleapis.com/auth/cloud-platform',\n });\n const client = await auth.getClient();\n const res = await client.request({ url });\n const uri = res.data?.serviceConfig?.uri;\n if (!uri) {\n throw new Error(`Unable to retreive uri for function at ${url}`);\n }\n return uri;\n}\n" }, { "path": "src/utils/ip.ts", "content": "import { isIPv4, isIPv6 } from 'net';\n\nexport function parseIp(ip: string): Buffer {\n if (isIPv4(ip)) {\n const [a, b, c, d] = ip.split('.').map(Number);\n\n const buf = Buffer.alloc(4);\n buf.writeUInt8(a, 0);\n buf.writeUInt8(b, 1);\n buf.writeUInt8(c, 2);\n buf.writeUInt8(d, 3);\n\n return buf;\n }\n\n if (isIPv6(ip)) {\n if (ip.includes('.')) {\n const parts = ip.split(':');\n const ipv4Part = parts.pop();\n if (!ipv4Part) throw new Error('Invalid IPv6 address');\n const ipv4Bytes = parseIp(ipv4Part);\n parts.push('0');\n const ipv6Bytes = parseIp(parts.join(':'));\n ipv6Bytes.writeUInt32BE(ipv4Bytes.readUInt32BE(0), 12);\n\n return ipv6Bytes;\n }\n\n const buf = Buffer.alloc(16);\n\n // Expand :: notation\n let expanded = ip;\n if (ip.includes('::')) {\n const sides = ip.split('::');\n const left = sides[0] ? sides[0].split(':') : [];\n const right = sides[1] ? sides[1].split(':') : [];\n const middle = Array(8 - left.length - right.length).fill('0');\n expanded = [...left, ...middle, ...right].join(':');\n }\n\n // Convert to buffer\n const parts = expanded.split(':');\n let offset = 0;\n for (const part of parts) {\n buf.writeUInt16BE(parseInt(part, 16), offset);\n offset += 2;\n }\n\n return buf;\n }\n\n throw new Error('Invalid IP address');\n}\n\n\nexport function parseCIDR(cidr: string): [Buffer, Buffer] {\n const [ip, prefixTxt] = cidr.split('/');\n const buf = parseIp(ip);\n const maskBuf = Buffer.alloc(buf.byteLength, 0xff);\n const prefixBits = parseInt(prefixTxt);\n\n let offsetBits = 0;\n while (offsetBits < (buf.byteLength * 8)) {\n if (offsetBits <= (prefixBits - 8)) {\n offsetBits += 8;\n continue;\n }\n const bitsRemain = prefixBits - offsetBits;\n const byteOffset = Math.floor(offsetBits / 8);\n\n if (bitsRemain > 0) {\n const theByte = buf[byteOffset];\n const mask = 0xff << (8 - bitsRemain);\n maskBuf[byteOffset] = mask;\n buf[byteOffset] = theByte & mask;\n\n offsetBits += 8;\n continue;\n };\n buf[byteOffset] = 0;\n maskBuf[byteOffset] = 0;\n\n offsetBits += 8;\n }\n\n return [buf, maskBuf];\n}\n\nexport class CIDR {\n buff: Buffer;\n mask: Buffer;\n text: string;\n constructor(cidr: string) {\n this.text = cidr;\n [this.buff, this.mask] = parseCIDR(cidr);\n }\n\n toString() {\n return this.text;\n }\n\n get family() {\n return this.buff.byteLength === 4 ? 4 : 6;\n }\n\n test(ip: string | Buffer): boolean {\n const parsedIp = typeof ip === 'string' ? parseIp(ip) : ip;\n\n if (parsedIp.byteLength !== this.buff.byteLength) {\n return false;\n }\n\n for (const i of Array(this.buff.byteLength).keys()) {\n const t = parsedIp[i];\n const m = this.mask[i];\n\n if (m === 0) {\n return true;\n }\n\n const r = this.buff[i];\n if ((t & m) !== r) {\n return false;\n }\n }\n\n return true;\n }\n}\n\nconst nonPublicNetworks4 = [\n '10.0.0.0/8',\n '172.16.0.0/12',\n '192.168.0.0/16',\n\n '127.0.0.0/8',\n '255.255.255.255/32',\n '169.254.0.0/16',\n '224.0.0.0/4',\n\n '100.64.0.0/10',\n '0.0.0.0/32',\n];\n\n\nconst nonPublicNetworks6 = [\n 'fc00::/7',\n 'fe80::/10',\n 'ff00::/8',\n\n '::127.0.0.0/104',\n '::/128',\n];\n\nconst nonPublicCIDRs = [...nonPublicNetworks4, ...nonPublicNetworks6].map(cidr => new CIDR(cidr));\n\nexport function isIPInNonPublicRange(ip: string) {\n const parsed = parseIp(ip);\n\n for (const cidr of nonPublicCIDRs) {\n if (cidr.test(parsed)) {\n return true;\n }\n }\n\n return false;\n}\n" }, { "path": "src/utils/markdown.ts", "content": "\nexport function tidyMarkdown(markdown: string): string {\n\n // Step 1: Handle complex broken links with text and optional images spread across multiple lines\n let normalizedMarkdown = markdown.replace(/\\[\\s*([^\\]\\n]+?)\\s*\\]\\s*\\(\\s*([^)]+)\\s*\\)/g, (match, text, url) => {\n // Remove internal new lines and excessive spaces within the text\n text = text.replace(/\\s+/g, ' ').trim();\n url = url.replace(/\\s+/g, '').trim();\n return `[${text}](${url})`;\n });\n\n normalizedMarkdown = normalizedMarkdown.replace(/\\[\\s*([^\\]\\n!]*?)\\s*\\n*(?:!\\[([^\\]]*)\\]\\((.*?)\\))?\\s*\\n*\\]\\s*\\(\\s*([^)]+)\\s*\\)/g, (match, text, alt, imgUrl, linkUrl) => {\n // Normalize by removing excessive spaces and new lines\n text = text.replace(/\\s+/g, ' ').trim();\n alt = alt ? alt.replace(/\\s+/g, ' ').trim() : '';\n imgUrl = imgUrl ? imgUrl.replace(/\\s+/g, '').trim() : '';\n linkUrl = linkUrl.replace(/\\s+/g, '').trim();\n if (imgUrl) {\n return `[${text} ![${alt}](${imgUrl})](${linkUrl})`;\n } else {\n return `[${text}](${linkUrl})`;\n }\n });\n\n // Step 2: Normalize regular links that may be broken across lines\n normalizedMarkdown = normalizedMarkdown.replace(/\\[\\s*([^\\]]+)\\]\\s*\\(\\s*([^)]+)\\)/g, (match, text, url) => {\n text = text.replace(/\\s+/g, ' ').trim();\n url = url.replace(/\\s+/g, '').trim();\n return `[${text}](${url})`;\n });\n\n // Step 3: Replace more than two consecutive empty lines with exactly two empty lines\n normalizedMarkdown = normalizedMarkdown.replace(/\\n{3,}/g, '\\n\\n');\n\n // Step 4: Remove leading spaces from each line\n normalizedMarkdown = normalizedMarkdown.replace(/^[ \\t]+/gm, '');\n\n return normalizedMarkdown.trim();\n}\n" }, { "path": "src/utils/misc.ts", "content": "import { ParamValidationError } from 'civkit';\n\nexport function cleanAttribute(attribute: string | null) {\n return attribute ? attribute.replace(/(\\n+\\s*)+/g, '\\n') : '';\n}\n\n\nexport function tryDecodeURIComponent(input: string) {\n try {\n return decodeURIComponent(input);\n } catch (err) {\n if (URL.canParse(input, 'http://localhost:3000')) {\n return input;\n }\n\n throw new ParamValidationError(`Invalid URIComponent: ${input}`);\n }\n}\n\n\nexport async function* toAsyncGenerator(val: T) {\n yield val;\n}\n\nexport async function* toGenerator(val: T) {\n yield val;\n}" }, { "path": "src/utils/tailwind-classes.ts", "content": "export const tailwindClasses = new Set([\n \"aspect-auto\",\n \"aspect-square\",\n \"aspect-video\",\n \"container\",\n \"columns-1\",\n \"columns-2\",\n \"columns-3\",\n \"columns-4\",\n \"columns-5\",\n \"columns-6\",\n \"columns-7\",\n \"columns-8\",\n \"columns-9\",\n \"columns-10\",\n \"columns-11\",\n \"columns-12\",\n \"columns-auto\",\n \"columns-3xs\",\n \"columns-2xs\",\n \"columns-xs\",\n \"columns-sm\",\n \"columns-md\",\n \"columns-lg\",\n \"columns-xl\",\n \"columns-2xl\",\n \"columns-3xl\",\n \"columns-4xl\",\n \"columns-5xl\",\n \"columns-6xl\",\n \"columns-7xl\",\n \"break-after-auto\",\n \"break-after-avoid\",\n \"break-after-all\",\n \"break-after-avoid-page\",\n \"break-after-page\",\n \"break-after-left\",\n \"break-after-right\",\n \"break-after-column\",\n \"break-before-auto\",\n \"break-before-avoid\",\n \"break-before-all\",\n \"break-before-avoid-page\",\n \"break-before-page\",\n \"break-before-left\",\n \"break-before-right\",\n \"break-before-column\",\n \"break-inside-auto\",\n \"break-inside-avoid\",\n \"break-inside-avoid-page\",\n \"break-inside-avoid-column\",\n \"box-decoration-clone\",\n \"box-decoration-slice\",\n \"box-border\",\n \"box-content\",\n \"block\",\n \"inline-block\",\n \"inline\",\n \"flex\",\n \"inline-flex\",\n \"table\",\n \"inline-table\",\n \"table-caption\",\n \"table-cell\",\n \"table-column\",\n \"table-column-group\",\n \"table-footer-group\",\n \"table-header-group\",\n \"table-row-group\",\n \"table-row\",\n \"flow-root\",\n \"grid\",\n \"inline-grid\",\n \"contents\",\n \"list-item\",\n // \"hidden\",\n \"float-start\",\n \"float-end\",\n \"float-right\",\n \"float-left\",\n \"float-none\",\n \"clear-start\",\n \"clear-end\",\n \"clear-left\",\n \"clear-right\",\n \"clear-both\",\n \"clear-none\",\n \"isolate\",\n \"isolation-auto\",\n \"object-contain\",\n \"object-cover\",\n \"object-fill\",\n \"object-none\",\n \"object-scale-down\",\n \"object-bottom\",\n \"object-center\",\n \"object-left\",\n \"object-left-bottom\",\n \"object-left-top\",\n \"object-right\",\n \"object-right-bottom\",\n \"object-right-top\",\n \"object-top\",\n \"overflow-auto\",\n \"overflow-hidden\",\n \"overflow-clip\",\n \"overflow-visible\",\n \"overflow-scroll\",\n \"overflow-x-auto\",\n \"overflow-y-auto\",\n \"overflow-x-hidden\",\n \"overflow-y-hidden\",\n \"overflow-x-clip\",\n \"overflow-y-clip\",\n \"overflow-x-visible\",\n \"overflow-y-visible\",\n \"overflow-x-scroll\",\n \"overflow-y-scroll\",\n \"overscroll-auto\",\n \"overscroll-contain\",\n \"overscroll-none\",\n \"overscroll-y-auto\",\n \"overscroll-y-contain\",\n \"overscroll-y-none\",\n \"overscroll-x-auto\",\n \"overscroll-x-contain\",\n \"overscroll-x-none\",\n \"static\",\n \"fixed\",\n \"absolute\",\n \"relative\",\n \"sticky\",\n \"inset-0\",\n \"inset-x-0\",\n \"inset-y-0\",\n \"start-0\",\n \"end-0\",\n \"top-0\",\n \"right-0\",\n \"bottom-0\",\n \"left-0\",\n \"inset-px\",\n \"inset-x-px\",\n \"inset-y-px\",\n \"start-px\",\n \"end-px\",\n \"top-px\",\n \"right-px\",\n \"bottom-px\",\n \"left-px\",\n \"inset-0.5\",\n \"inset-x-0.5\",\n \"inset-y-0.5\",\n \"start-0.5\",\n \"end-0.5\",\n \"top-0.5\",\n \"right-0.5\",\n \"bottom-0.5\",\n \"left-0.5\",\n \"inset-1\",\n \"inset-x-1\",\n \"inset-y-1\",\n \"start-1\",\n \"end-1\",\n \"top-1\",\n \"right-1\",\n \"bottom-1\",\n \"left-1\",\n \"inset-1.5\",\n \"inset-x-1.5\",\n \"inset-y-1.5\",\n \"start-1.5\",\n \"end-1.5\",\n \"top-1.5\",\n \"right-1.5\",\n \"bottom-1.5\",\n \"left-1.5\",\n \"inset-2\",\n \"inset-x-2\",\n \"inset-y-2\",\n \"start-2\",\n \"end-2\",\n \"top-2\",\n \"right-2\",\n \"bottom-2\",\n \"left-2\",\n \"inset-2.5\",\n \"inset-x-2.5\",\n \"inset-y-2.5\",\n \"start-2.5\",\n \"end-2.5\",\n \"top-2.5\",\n \"right-2.5\",\n \"bottom-2.5\",\n \"left-2.5\",\n \"inset-3\",\n \"inset-x-3\",\n \"inset-y-3\",\n \"start-3\",\n \"end-3\",\n \"top-3\",\n \"right-3\",\n \"bottom-3\",\n \"left-3\",\n \"inset-3.5\",\n \"inset-x-3.5\",\n \"inset-y-3.5\",\n \"start-3.5\",\n \"end-3.5\",\n \"top-3.5\",\n \"right-3.5\",\n \"bottom-3.5\",\n \"left-3.5\",\n \"inset-4\",\n \"inset-x-4\",\n \"inset-y-4\",\n \"start-4\",\n \"end-4\",\n \"top-4\",\n \"right-4\",\n \"bottom-4\",\n \"left-4\",\n \"inset-5\",\n \"inset-x-5\",\n \"inset-y-5\",\n \"start-5\",\n \"end-5\",\n \"top-5\",\n \"right-5\",\n \"bottom-5\",\n \"left-5\",\n \"inset-6\",\n \"inset-x-6\",\n \"inset-y-6\",\n \"start-6\",\n \"end-6\",\n \"top-6\",\n \"right-6\",\n \"bottom-6\",\n \"left-6\",\n \"inset-7\",\n \"inset-x-7\",\n \"inset-y-7\",\n \"start-7\",\n \"end-7\",\n \"top-7\",\n \"right-7\",\n \"bottom-7\",\n \"left-7\",\n \"inset-8\",\n \"inset-x-8\",\n \"inset-y-8\",\n \"start-8\",\n \"end-8\",\n \"top-8\",\n \"right-8\",\n \"bottom-8\",\n \"left-8\",\n \"inset-9\",\n \"inset-x-9\",\n \"inset-y-9\",\n \"start-9\",\n \"end-9\",\n \"top-9\",\n \"right-9\",\n \"bottom-9\",\n \"left-9\",\n \"inset-10\",\n \"inset-x-10\",\n \"inset-y-10\",\n \"start-10\",\n \"end-10\",\n \"top-10\",\n \"right-10\",\n \"bottom-10\",\n \"left-10\",\n \"inset-11\",\n \"inset-x-11\",\n \"inset-y-11\",\n \"start-11\",\n \"end-11\",\n \"top-11\",\n \"right-11\",\n \"bottom-11\",\n \"left-11\",\n \"inset-12\",\n \"inset-x-12\",\n \"inset-y-12\",\n \"start-12\",\n \"end-12\",\n \"top-12\",\n \"right-12\",\n \"bottom-12\",\n \"left-12\",\n \"inset-14\",\n \"inset-x-14\",\n \"inset-y-14\",\n \"start-14\",\n \"end-14\",\n \"top-14\",\n \"right-14\",\n \"bottom-14\",\n \"left-14\",\n \"inset-16\",\n \"inset-x-16\",\n \"inset-y-16\",\n \"start-16\",\n \"end-16\",\n \"top-16\",\n \"right-16\",\n \"bottom-16\",\n \"left-16\",\n \"inset-20\",\n \"inset-x-20\",\n \"inset-y-20\",\n \"start-20\",\n \"end-20\",\n \"top-20\",\n \"right-20\",\n \"bottom-20\",\n \"left-20\",\n \"inset-24\",\n \"inset-x-24\",\n \"inset-y-24\",\n \"start-24\",\n \"end-24\",\n \"top-24\",\n \"right-24\",\n \"bottom-24\",\n \"left-24\",\n \"inset-28\",\n \"inset-x-28\",\n \"inset-y-28\",\n \"start-28\",\n \"end-28\",\n \"top-28\",\n \"right-28\",\n \"bottom-28\",\n \"left-28\",\n \"inset-32\",\n \"inset-x-32\",\n \"inset-y-32\",\n \"start-32\",\n \"end-32\",\n \"top-32\",\n \"right-32\",\n \"bottom-32\",\n \"left-32\",\n \"inset-36\",\n \"inset-x-36\",\n \"inset-y-36\",\n \"start-36\",\n \"end-36\",\n \"top-36\",\n \"right-36\",\n \"bottom-36\",\n \"left-36\",\n \"inset-40\",\n \"inset-x-40\",\n \"inset-y-40\",\n \"start-40\",\n \"end-40\",\n \"top-40\",\n \"right-40\",\n \"bottom-40\",\n \"left-40\",\n \"inset-44\",\n \"inset-x-44\",\n \"inset-y-44\",\n \"start-44\",\n \"end-44\",\n \"top-44\",\n \"right-44\",\n \"bottom-44\",\n \"left-44\",\n \"inset-48\",\n \"inset-x-48\",\n \"inset-y-48\",\n \"start-48\",\n \"end-48\",\n \"top-48\",\n \"right-48\",\n \"bottom-48\",\n \"left-48\",\n \"inset-52\",\n \"inset-x-52\",\n \"inset-y-52\",\n \"start-52\",\n \"end-52\",\n \"top-52\",\n \"right-52\",\n \"bottom-52\",\n \"left-52\",\n \"inset-56\",\n \"inset-x-56\",\n \"inset-y-56\",\n \"start-56\",\n \"end-56\",\n \"top-56\",\n \"right-56\",\n \"bottom-56\",\n \"left-56\",\n \"inset-60\",\n \"inset-x-60\",\n \"inset-y-60\",\n \"start-60\",\n \"end-60\",\n \"top-60\",\n \"right-60\",\n \"bottom-60\",\n \"left-60\",\n \"inset-64\",\n \"inset-x-64\",\n \"inset-y-64\",\n \"start-64\",\n \"end-64\",\n \"top-64\",\n \"right-64\",\n \"bottom-64\",\n \"left-64\",\n \"inset-72\",\n \"inset-x-72\",\n \"inset-y-72\",\n \"start-72\",\n \"end-72\",\n \"top-72\",\n \"right-72\",\n \"bottom-72\",\n \"left-72\",\n \"inset-80\",\n \"inset-x-80\",\n \"inset-y-80\",\n \"start-80\",\n \"end-80\",\n \"top-80\",\n \"right-80\",\n \"bottom-80\",\n \"left-80\",\n \"inset-96\",\n \"inset-x-96\",\n \"inset-y-96\",\n \"start-96\",\n \"end-96\",\n \"top-96\",\n \"right-96\",\n \"bottom-96\",\n \"left-96\",\n \"inset-auto\",\n \"inset-1/2\",\n \"inset-1/3\",\n \"inset-2/3\",\n \"inset-1/4\",\n \"inset-2/4\",\n \"inset-3/4\",\n \"inset-full\",\n \"inset-x-auto\",\n \"inset-x-1/2\",\n \"inset-x-1/3\",\n \"inset-x-2/3\",\n \"inset-x-1/4\",\n \"inset-x-2/4\",\n \"inset-x-3/4\",\n \"inset-x-full\",\n \"inset-y-auto\",\n \"inset-y-1/2\",\n \"inset-y-1/3\",\n \"inset-y-2/3\",\n \"inset-y-1/4\",\n \"inset-y-2/4\",\n \"inset-y-3/4\",\n \"inset-y-full\",\n \"start-auto\",\n \"start-1/2\",\n \"start-1/3\",\n \"start-2/3\",\n \"start-1/4\",\n \"start-2/4\",\n \"start-3/4\",\n \"start-full\",\n \"end-auto\",\n \"end-1/2\",\n \"end-1/3\",\n \"end-2/3\",\n \"end-1/4\",\n \"end-2/4\",\n \"end-3/4\",\n \"end-full\",\n \"top-auto\",\n \"top-1/2\",\n \"top-1/3\",\n \"top-2/3\",\n \"top-1/4\",\n \"top-2/4\",\n \"top-3/4\",\n \"top-full\",\n \"right-auto\",\n \"right-1/2\",\n \"right-1/3\",\n \"right-2/3\",\n \"right-1/4\",\n \"right-2/4\",\n \"right-3/4\",\n \"right-full\",\n \"bottom-auto\",\n \"bottom-1/2\",\n \"bottom-1/3\",\n \"bottom-2/3\",\n \"bottom-1/4\",\n \"bottom-2/4\",\n \"bottom-3/4\",\n \"bottom-full\",\n \"left-auto\",\n \"left-1/2\",\n \"left-1/3\",\n \"left-2/3\",\n \"left-1/4\",\n \"left-2/4\",\n \"left-3/4\",\n \"left-full\",\n \"visible\",\n \"invisible\",\n \"collapse\",\n \"z-0\",\n \"z-10\",\n \"z-20\",\n \"z-30\",\n \"z-40\",\n \"z-50\",\n \"z-auto\",\n \"basis-0\",\n \"basis-1\",\n \"basis-2\",\n \"basis-3\",\n \"basis-4\",\n \"basis-5\",\n \"basis-6\",\n \"basis-7\",\n \"basis-8\",\n \"basis-9\",\n \"basis-10\",\n \"basis-11\",\n \"basis-12\",\n \"basis-14\",\n \"basis-16\",\n \"basis-20\",\n \"basis-24\",\n \"basis-28\",\n \"basis-32\",\n \"basis-36\",\n \"basis-40\",\n \"basis-44\",\n \"basis-48\",\n \"basis-52\",\n \"basis-56\",\n \"basis-60\",\n \"basis-64\",\n \"basis-72\",\n \"basis-80\",\n \"basis-96\",\n \"basis-auto\",\n \"basis-px\",\n \"basis-0.5\",\n \"basis-1.5\",\n \"basis-2.5\",\n \"basis-3.5\",\n \"basis-1/2\",\n \"basis-1/3\",\n \"basis-2/3\",\n \"basis-1/4\",\n \"basis-2/4\",\n \"basis-3/4\",\n \"basis-1/5\",\n \"basis-2/5\",\n \"basis-3/5\",\n \"basis-4/5\",\n \"basis-1/6\",\n \"basis-2/6\",\n \"basis-3/6\",\n \"basis-4/6\",\n \"basis-5/6\",\n \"basis-1/12\",\n \"basis-2/12\",\n \"basis-3/12\",\n \"basis-4/12\",\n \"basis-5/12\",\n \"basis-6/12\",\n \"basis-7/12\",\n \"basis-8/12\",\n \"basis-9/12\",\n \"basis-10/12\",\n \"basis-11/12\",\n \"basis-full\",\n \"flex-row\",\n \"flex-row-reverse\",\n \"flex-col\",\n \"flex-col-reverse\",\n \"flex-wrap\",\n \"flex-wrap-reverse\",\n \"flex-nowrap\",\n \"flex-1\",\n \"flex-auto\",\n \"flex-initial\",\n \"flex-none\",\n \"grow\",\n \"grow-0\",\n \"shrink\",\n \"shrink-0\",\n \"order-1\",\n \"order-2\",\n \"order-3\",\n \"order-4\",\n \"order-5\",\n \"order-6\",\n \"order-7\",\n \"order-8\",\n \"order-9\",\n \"order-10\",\n \"order-11\",\n \"order-12\",\n \"order-first\",\n \"order-last\",\n \"order-none\",\n \"grid-cols-1\",\n \"grid-cols-2\",\n \"grid-cols-3\",\n \"grid-cols-4\",\n \"grid-cols-5\",\n \"grid-cols-6\",\n \"grid-cols-7\",\n \"grid-cols-8\",\n \"grid-cols-9\",\n \"grid-cols-10\",\n \"grid-cols-11\",\n \"grid-cols-12\",\n \"grid-cols-none\",\n \"grid-cols-subgrid\",\n \"col-auto\",\n \"col-span-1\",\n \"col-span-2\",\n \"col-span-3\",\n \"col-span-4\",\n \"col-span-5\",\n \"col-span-6\",\n \"col-span-7\",\n \"col-span-8\",\n \"col-span-9\",\n \"col-span-10\",\n \"col-span-11\",\n \"col-span-12\",\n \"col-span-full\",\n \"col-start-1\",\n \"col-start-2\",\n \"col-start-3\",\n \"col-start-4\",\n \"col-start-5\",\n \"col-start-6\",\n \"col-start-7\",\n \"col-start-8\",\n \"col-start-9\",\n \"col-start-10\",\n \"col-start-11\",\n \"col-start-12\",\n \"col-start-13\",\n \"col-start-auto\",\n \"col-end-1\",\n \"col-end-2\",\n \"col-end-3\",\n \"col-end-4\",\n \"col-end-5\",\n \"col-end-6\",\n \"col-end-7\",\n \"col-end-8\",\n \"col-end-9\",\n \"col-end-10\",\n \"col-end-11\",\n \"col-end-12\",\n \"col-end-13\",\n \"col-end-auto\",\n \"grid-rows-1\",\n \"grid-rows-2\",\n \"grid-rows-3\",\n \"grid-rows-4\",\n \"grid-rows-5\",\n \"grid-rows-6\",\n \"grid-rows-7\",\n \"grid-rows-8\",\n \"grid-rows-9\",\n \"grid-rows-10\",\n \"grid-rows-11\",\n \"grid-rows-12\",\n \"grid-rows-none\",\n \"grid-rows-subgrid\",\n \"row-auto\",\n \"row-span-1\",\n \"row-span-2\",\n \"row-span-3\",\n \"row-span-4\",\n \"row-span-5\",\n \"row-span-6\",\n \"row-span-7\",\n \"row-span-8\",\n \"row-span-9\",\n \"row-span-10\",\n \"row-span-11\",\n \"row-span-12\",\n \"row-span-full\",\n \"row-start-1\",\n \"row-start-2\",\n \"row-start-3\",\n \"row-start-4\",\n \"row-start-5\",\n \"row-start-6\",\n \"row-start-7\",\n \"row-start-8\",\n \"row-start-9\",\n \"row-start-10\",\n \"row-start-11\",\n \"row-start-12\",\n \"row-start-13\",\n \"row-start-auto\",\n \"row-end-1\",\n \"row-end-2\",\n \"row-end-3\",\n \"row-end-4\",\n \"row-end-5\",\n \"row-end-6\",\n \"row-end-7\",\n \"row-end-8\",\n \"row-end-9\",\n \"row-end-10\",\n \"row-end-11\",\n \"row-end-12\",\n \"row-end-13\",\n \"row-end-auto\",\n \"grid-flow-row\",\n \"grid-flow-col\",\n \"grid-flow-dense\",\n \"grid-flow-row-dense\",\n \"grid-flow-col-dense\",\n \"auto-cols-auto\",\n \"auto-cols-min\",\n \"auto-cols-max\",\n \"auto-cols-fr\",\n \"auto-rows-auto\",\n \"auto-rows-min\",\n \"auto-rows-max\",\n \"auto-rows-fr\",\n \"gap-0\",\n \"gap-x-0\",\n \"gap-y-0\",\n \"gap-px\",\n \"gap-x-px\",\n \"gap-y-px\",\n \"gap-0.5\",\n \"gap-x-0.5\",\n \"gap-y-0.5\",\n \"gap-1\",\n \"gap-x-1\",\n \"gap-y-1\",\n \"gap-1.5\",\n \"gap-x-1.5\",\n \"gap-y-1.5\",\n \"gap-2\",\n \"gap-x-2\",\n \"gap-y-2\",\n \"gap-2.5\",\n \"gap-x-2.5\",\n \"gap-y-2.5\",\n \"gap-3\",\n \"gap-x-3\",\n \"gap-y-3\",\n \"gap-3.5\",\n \"gap-x-3.5\",\n \"gap-y-3.5\",\n \"gap-4\",\n \"gap-x-4\",\n \"gap-y-4\",\n \"gap-5\",\n \"gap-x-5\",\n \"gap-y-5\",\n \"gap-6\",\n \"gap-x-6\",\n \"gap-y-6\",\n \"gap-7\",\n \"gap-x-7\",\n \"gap-y-7\",\n \"gap-8\",\n \"gap-x-8\",\n \"gap-y-8\",\n \"gap-9\",\n \"gap-x-9\",\n \"gap-y-9\",\n \"gap-10\",\n \"gap-x-10\",\n \"gap-y-10\",\n \"gap-11\",\n \"gap-x-11\",\n \"gap-y-11\",\n \"gap-12\",\n \"gap-x-12\",\n \"gap-y-12\",\n \"gap-14\",\n \"gap-x-14\",\n \"gap-y-14\",\n \"gap-16\",\n \"gap-x-16\",\n \"gap-y-16\",\n \"gap-20\",\n \"gap-x-20\",\n \"gap-y-20\",\n \"gap-24\",\n \"gap-x-24\",\n \"gap-y-24\",\n \"gap-28\",\n \"gap-x-28\",\n \"gap-y-28\",\n \"gap-32\",\n \"gap-x-32\",\n \"gap-y-32\",\n \"gap-36\",\n \"gap-x-36\",\n \"gap-y-36\",\n \"gap-40\",\n \"gap-x-40\",\n \"gap-y-40\",\n \"gap-44\",\n \"gap-x-44\",\n \"gap-y-44\",\n \"gap-48\",\n \"gap-x-48\",\n \"gap-y-48\",\n \"gap-52\",\n \"gap-x-52\",\n \"gap-y-52\",\n \"gap-56\",\n \"gap-x-56\",\n \"gap-y-56\",\n \"gap-60\",\n \"gap-x-60\",\n \"gap-y-60\",\n \"gap-64\",\n \"gap-x-64\",\n \"gap-y-64\",\n \"gap-72\",\n \"gap-x-72\",\n \"gap-y-72\",\n \"gap-80\",\n \"gap-x-80\",\n \"gap-y-80\",\n \"gap-96\",\n \"gap-x-96\",\n \"gap-y-96\",\n \"justify-normal\",\n \"justify-start\",\n \"justify-end\",\n \"justify-center\",\n \"justify-between\",\n \"justify-around\",\n \"justify-evenly\",\n \"justify-stretch\",\n \"justify-items-start\",\n \"justify-items-end\",\n \"justify-items-center\",\n \"justify-items-stretch\",\n \"justify-self-auto\",\n \"justify-self-start\",\n \"justify-self-end\",\n \"justify-self-center\",\n \"justify-self-stretch\",\n \"content-normal\",\n \"content-center\",\n \"content-start\",\n \"content-end\",\n \"content-between\",\n \"content-around\",\n \"content-evenly\",\n \"content-baseline\",\n \"content-stretch\",\n \"items-start\",\n \"items-end\",\n \"items-center\",\n \"items-baseline\",\n \"items-stretch\",\n \"self-auto\",\n \"self-start\",\n \"self-end\",\n \"self-center\",\n \"self-stretch\",\n \"self-baseline\",\n \"place-content-center\",\n \"place-content-start\",\n \"place-content-end\",\n \"place-content-between\",\n \"place-content-around\",\n \"place-content-evenly\",\n \"place-content-baseline\",\n \"place-content-stretch\",\n \"place-items-start\",\n \"place-items-end\",\n \"place-items-center\",\n \"place-items-baseline\",\n \"place-items-stretch\",\n \"place-self-auto\",\n \"place-self-start\",\n \"place-self-end\",\n \"place-self-center\",\n \"place-self-stretch\",\n \"p-0\",\n \"px-0\",\n \"py-0\",\n \"ps-0\",\n \"pe-0\",\n \"pt-0\",\n \"pr-0\",\n \"pb-0\",\n \"pl-0\",\n \"p-px\",\n \"px-px\",\n \"py-px\",\n \"ps-px\",\n \"pe-px\",\n \"pt-px\",\n \"pr-px\",\n \"pb-px\",\n \"pl-px\",\n \"p-0.5\",\n \"px-0.5\",\n \"py-0.5\",\n \"ps-0.5\",\n \"pe-0.5\",\n \"pt-0.5\",\n \"pr-0.5\",\n \"pb-0.5\",\n \"pl-0.5\",\n \"p-1\",\n \"px-1\",\n \"py-1\",\n \"ps-1\",\n \"pe-1\",\n \"pt-1\",\n \"pr-1\",\n \"pb-1\",\n \"pl-1\",\n \"p-1.5\",\n \"px-1.5\",\n \"py-1.5\",\n \"ps-1.5\",\n \"pe-1.5\",\n \"pt-1.5\",\n \"pr-1.5\",\n \"pb-1.5\",\n \"pl-1.5\",\n \"p-2\",\n \"px-2\",\n \"py-2\",\n \"ps-2\",\n \"pe-2\",\n \"pt-2\",\n \"pr-2\",\n \"pb-2\",\n \"pl-2\",\n \"p-2.5\",\n \"px-2.5\",\n \"py-2.5\",\n \"ps-2.5\",\n \"pe-2.5\",\n \"pt-2.5\",\n \"pr-2.5\",\n \"pb-2.5\",\n \"pl-2.5\",\n \"p-3\",\n \"px-3\",\n \"py-3\",\n \"ps-3\",\n \"pe-3\",\n \"pt-3\",\n \"pr-3\",\n \"pb-3\",\n \"pl-3\",\n \"p-3.5\",\n \"px-3.5\",\n \"py-3.5\",\n \"ps-3.5\",\n \"pe-3.5\",\n \"pt-3.5\",\n \"pr-3.5\",\n \"pb-3.5\",\n \"pl-3.5\",\n \"p-4\",\n \"px-4\",\n \"py-4\",\n \"ps-4\",\n \"pe-4\",\n \"pt-4\",\n \"pr-4\",\n \"pb-4\",\n \"pl-4\",\n \"p-5\",\n \"px-5\",\n \"py-5\",\n \"ps-5\",\n \"pe-5\",\n \"pt-5\",\n \"pr-5\",\n \"pb-5\",\n \"pl-5\",\n \"p-6\",\n \"px-6\",\n \"py-6\",\n \"ps-6\",\n \"pe-6\",\n \"pt-6\",\n \"pr-6\",\n \"pb-6\",\n \"pl-6\",\n \"p-7\",\n \"px-7\",\n \"py-7\",\n \"ps-7\",\n \"pe-7\",\n \"pt-7\",\n \"pr-7\",\n \"pb-7\",\n \"pl-7\",\n \"p-8\",\n \"px-8\",\n \"py-8\",\n \"ps-8\",\n \"pe-8\",\n \"pt-8\",\n \"pr-8\",\n \"pb-8\",\n \"pl-8\",\n \"p-9\",\n \"px-9\",\n \"py-9\",\n \"ps-9\",\n \"pe-9\",\n \"pt-9\",\n \"pr-9\",\n \"pb-9\",\n \"pl-9\",\n \"p-10\",\n \"px-10\",\n \"py-10\",\n \"ps-10\",\n \"pe-10\",\n \"pt-10\",\n \"pr-10\",\n \"pb-10\",\n \"pl-10\",\n \"p-11\",\n \"px-11\",\n \"py-11\",\n \"ps-11\",\n \"pe-11\",\n \"pt-11\",\n \"pr-11\",\n \"pb-11\",\n \"pl-11\",\n \"p-12\",\n \"px-12\",\n \"py-12\",\n \"ps-12\",\n \"pe-12\",\n \"pt-12\",\n \"pr-12\",\n \"pb-12\",\n \"pl-12\",\n \"p-14\",\n \"px-14\",\n \"py-14\",\n \"ps-14\",\n \"pe-14\",\n \"pt-14\",\n \"pr-14\",\n \"pb-14\",\n \"pl-14\",\n \"p-16\",\n \"px-16\",\n \"py-16\",\n \"ps-16\",\n \"pe-16\",\n \"pt-16\",\n \"pr-16\",\n \"pb-16\",\n \"pl-16\",\n \"p-20\",\n \"px-20\",\n \"py-20\",\n \"ps-20\",\n \"pe-20\",\n \"pt-20\",\n \"pr-20\",\n \"pb-20\",\n \"pl-20\",\n \"p-24\",\n \"px-24\",\n \"py-24\",\n \"ps-24\",\n \"pe-24\",\n \"pt-24\",\n \"pr-24\",\n \"pb-24\",\n \"pl-24\",\n \"p-28\",\n \"px-28\",\n \"py-28\",\n \"ps-28\",\n \"pe-28\",\n \"pt-28\",\n \"pr-28\",\n \"pb-28\",\n \"pl-28\",\n \"p-32\",\n \"px-32\",\n \"py-32\",\n \"ps-32\",\n \"pe-32\",\n \"pt-32\",\n \"pr-32\",\n \"pb-32\",\n \"pl-32\",\n \"p-36\",\n \"px-36\",\n \"py-36\",\n \"ps-36\",\n \"pe-36\",\n \"pt-36\",\n \"pr-36\",\n \"pb-36\",\n \"pl-36\",\n \"p-40\",\n \"px-40\",\n \"py-40\",\n \"ps-40\",\n \"pe-40\",\n \"pt-40\",\n \"pr-40\",\n \"pb-40\",\n \"pl-40\",\n \"p-44\",\n \"px-44\",\n \"py-44\",\n \"ps-44\",\n \"pe-44\",\n \"pt-44\",\n \"pr-44\",\n \"pb-44\",\n \"pl-44\",\n \"p-48\",\n \"px-48\",\n \"py-48\",\n \"ps-48\",\n \"pe-48\",\n \"pt-48\",\n \"pr-48\",\n \"pb-48\",\n \"pl-48\",\n \"p-52\",\n \"px-52\",\n \"py-52\",\n \"ps-52\",\n \"pe-52\",\n \"pt-52\",\n \"pr-52\",\n \"pb-52\",\n \"pl-52\",\n \"p-56\",\n \"px-56\",\n \"py-56\",\n \"ps-56\",\n \"pe-56\",\n \"pt-56\",\n \"pr-56\",\n \"pb-56\",\n \"pl-56\",\n \"p-60\",\n \"px-60\",\n \"py-60\",\n \"ps-60\",\n \"pe-60\",\n \"pt-60\",\n \"pr-60\",\n \"pb-60\",\n \"pl-60\",\n \"p-64\",\n \"px-64\",\n \"py-64\",\n \"ps-64\",\n \"pe-64\",\n \"pt-64\",\n \"pr-64\",\n \"pb-64\",\n \"pl-64\",\n \"p-72\",\n \"px-72\",\n \"py-72\",\n \"ps-72\",\n \"pe-72\",\n \"pt-72\",\n \"pr-72\",\n \"pb-72\",\n \"pl-72\",\n \"p-80\",\n \"px-80\",\n \"py-80\",\n \"ps-80\",\n \"pe-80\",\n \"pt-80\",\n \"pr-80\",\n \"pb-80\",\n \"pl-80\",\n \"p-96\",\n \"px-96\",\n \"py-96\",\n \"ps-96\",\n \"pe-96\",\n \"pt-96\",\n \"pr-96\",\n \"pb-96\",\n \"pl-96\",\n \"m-0\",\n \"mx-0\",\n \"my-0\",\n \"ms-0\",\n \"me-0\",\n \"mt-0\",\n \"mr-0\",\n \"mb-0\",\n \"ml-0\",\n \"m-px\",\n \"mx-px\",\n \"my-px\",\n \"ms-px\",\n \"me-px\",\n \"mt-px\",\n \"mr-px\",\n \"mb-px\",\n \"ml-px\",\n \"m-0.5\",\n \"mx-0.5\",\n \"my-0.5\",\n \"ms-0.5\",\n \"me-0.5\",\n \"mt-0.5\",\n \"mr-0.5\",\n \"mb-0.5\",\n \"ml-0.5\",\n \"m-1\",\n \"mx-1\",\n \"my-1\",\n \"ms-1\",\n \"me-1\",\n \"mt-1\",\n \"mr-1\",\n \"mb-1\",\n \"ml-1\",\n \"m-1.5\",\n \"mx-1.5\",\n \"my-1.5\",\n \"ms-1.5\",\n \"me-1.5\",\n \"mt-1.5\",\n \"mr-1.5\",\n \"mb-1.5\",\n \"ml-1.5\",\n \"m-2\",\n \"mx-2\",\n \"my-2\",\n \"ms-2\",\n \"me-2\",\n \"mt-2\",\n \"mr-2\",\n \"mb-2\",\n \"ml-2\",\n \"m-2.5\",\n \"mx-2.5\",\n \"my-2.5\",\n \"ms-2.5\",\n \"me-2.5\",\n \"mt-2.5\",\n \"mr-2.5\",\n \"mb-2.5\",\n \"ml-2.5\",\n \"m-3\",\n \"mx-3\",\n \"my-3\",\n \"ms-3\",\n \"me-3\",\n \"mt-3\",\n \"mr-3\",\n \"mb-3\",\n \"ml-3\",\n \"m-3.5\",\n \"mx-3.5\",\n \"my-3.5\",\n \"ms-3.5\",\n \"me-3.5\",\n \"mt-3.5\",\n \"mr-3.5\",\n \"mb-3.5\",\n \"ml-3.5\",\n \"m-4\",\n \"mx-4\",\n \"my-4\",\n \"ms-4\",\n \"me-4\",\n \"mt-4\",\n \"mr-4\",\n \"mb-4\",\n \"ml-4\",\n \"m-5\",\n \"mx-5\",\n \"my-5\",\n \"ms-5\",\n \"me-5\",\n \"mt-5\",\n \"mr-5\",\n \"mb-5\",\n \"ml-5\",\n \"m-6\",\n \"mx-6\",\n \"my-6\",\n \"ms-6\",\n \"me-6\",\n \"mt-6\",\n \"mr-6\",\n \"mb-6\",\n \"ml-6\",\n \"m-7\",\n \"mx-7\",\n \"my-7\",\n \"ms-7\",\n \"me-7\",\n \"mt-7\",\n \"mr-7\",\n \"mb-7\",\n \"ml-7\",\n \"m-8\",\n \"mx-8\",\n \"my-8\",\n \"ms-8\",\n \"me-8\",\n \"mt-8\",\n \"mr-8\",\n \"mb-8\",\n \"ml-8\",\n \"m-9\",\n \"mx-9\",\n \"my-9\",\n \"ms-9\",\n \"me-9\",\n \"mt-9\",\n \"mr-9\",\n \"mb-9\",\n \"ml-9\",\n \"m-10\",\n \"mx-10\",\n \"my-10\",\n \"ms-10\",\n \"me-10\",\n \"mt-10\",\n \"mr-10\",\n \"mb-10\",\n \"ml-10\",\n \"m-11\",\n \"mx-11\",\n \"my-11\",\n \"ms-11\",\n \"me-11\",\n \"mt-11\",\n \"mr-11\",\n \"mb-11\",\n \"ml-11\",\n \"m-12\",\n \"mx-12\",\n \"my-12\",\n \"ms-12\",\n \"me-12\",\n \"mt-12\",\n \"mr-12\",\n \"mb-12\",\n \"ml-12\",\n \"m-14\",\n \"mx-14\",\n \"my-14\",\n \"ms-14\",\n \"me-14\",\n \"mt-14\",\n \"mr-14\",\n \"mb-14\",\n \"ml-14\",\n \"m-16\",\n \"mx-16\",\n \"my-16\",\n \"ms-16\",\n \"me-16\",\n \"mt-16\",\n \"mr-16\",\n \"mb-16\",\n \"ml-16\",\n \"m-20\",\n \"mx-20\",\n \"my-20\",\n \"ms-20\",\n \"me-20\",\n \"mt-20\",\n \"mr-20\",\n \"mb-20\",\n \"ml-20\",\n \"m-24\",\n \"mx-24\",\n \"my-24\",\n \"ms-24\",\n \"me-24\",\n \"mt-24\",\n \"mr-24\",\n \"mb-24\",\n \"ml-24\",\n \"m-28\",\n \"mx-28\",\n \"my-28\",\n \"ms-28\",\n \"me-28\",\n \"mt-28\",\n \"mr-28\",\n \"mb-28\",\n \"ml-28\",\n \"m-32\",\n \"mx-32\",\n \"my-32\",\n \"ms-32\",\n \"me-32\",\n \"mt-32\",\n \"mr-32\",\n \"mb-32\",\n \"ml-32\",\n \"m-36\",\n \"mx-36\",\n \"my-36\",\n \"ms-36\",\n \"me-36\",\n \"mt-36\",\n \"mr-36\",\n \"mb-36\",\n \"ml-36\",\n \"m-40\",\n \"mx-40\",\n \"my-40\",\n \"ms-40\",\n \"me-40\",\n \"mt-40\",\n \"mr-40\",\n \"mb-40\",\n \"ml-40\",\n \"m-44\",\n \"mx-44\",\n \"my-44\",\n \"ms-44\",\n \"me-44\",\n \"mt-44\",\n \"mr-44\",\n \"mb-44\",\n \"ml-44\",\n \"m-48\",\n \"mx-48\",\n \"my-48\",\n \"ms-48\",\n \"me-48\",\n \"mt-48\",\n \"mr-48\",\n \"mb-48\",\n \"ml-48\",\n \"m-52\",\n \"mx-52\",\n \"my-52\",\n \"ms-52\",\n \"me-52\",\n \"mt-52\",\n \"mr-52\",\n \"mb-52\",\n \"ml-52\",\n \"m-56\",\n \"mx-56\",\n \"my-56\",\n \"ms-56\",\n \"me-56\",\n \"mt-56\",\n \"mr-56\",\n \"mb-56\",\n \"ml-56\",\n \"m-60\",\n \"mx-60\",\n \"my-60\",\n \"ms-60\",\n \"me-60\",\n \"mt-60\",\n \"mr-60\",\n \"mb-60\",\n \"ml-60\",\n \"m-64\",\n \"mx-64\",\n \"my-64\",\n \"ms-64\",\n \"me-64\",\n \"mt-64\",\n \"mr-64\",\n \"mb-64\",\n \"ml-64\",\n \"m-72\",\n \"mx-72\",\n \"my-72\",\n \"ms-72\",\n \"me-72\",\n \"mt-72\",\n \"mr-72\",\n \"mb-72\",\n \"ml-72\",\n \"m-80\",\n \"mx-80\",\n \"my-80\",\n \"ms-80\",\n \"me-80\",\n \"mt-80\",\n \"mr-80\",\n \"mb-80\",\n \"ml-80\",\n \"m-96\",\n \"mx-96\",\n \"my-96\",\n \"ms-96\",\n \"me-96\",\n \"mt-96\",\n \"mr-96\",\n \"mb-96\",\n \"ml-96\",\n \"m-auto\",\n \"mx-auto\",\n \"my-auto\",\n \"ms-auto\",\n \"me-auto\",\n \"mt-auto\",\n \"mr-auto\",\n \"mb-auto\",\n \"ml-auto\",\n \"space-x-0\",\n \"space-y-0\",\n \"space-x-0.5\",\n \"space-y-0.5\",\n \"space-x-1\",\n \"space-y-1\",\n \"space-x-1.5\",\n \"space-y-1.5\",\n \"space-x-2\",\n \"space-y-2\",\n \"space-x-2.5\",\n \"space-y-2.5\",\n \"space-x-3\",\n \"space-y-3\",\n \"space-x-3.5\",\n \"space-y-3.5\",\n \"space-x-4\",\n \"space-y-4\",\n \"space-x-5\",\n \"space-y-5\",\n \"space-x-6\",\n \"space-y-6\",\n \"space-x-7\",\n \"space-y-7\",\n \"space-x-8\",\n \"space-y-8\",\n \"space-x-9\",\n \"space-y-9\",\n \"space-x-10\",\n \"space-y-10\",\n \"space-x-11\",\n \"space-y-11\",\n \"space-x-12\",\n \"space-y-12\",\n \"space-x-14\",\n \"space-y-14\",\n \"space-x-16\",\n \"space-y-16\",\n \"space-x-20\",\n \"space-y-20\",\n \"space-x-24\",\n \"space-y-24\",\n \"space-x-28\",\n \"space-y-28\",\n \"space-x-32\",\n \"space-y-32\",\n \"space-x-36\",\n \"space-y-36\",\n \"space-x-40\",\n \"space-y-40\",\n \"space-x-44\",\n \"space-y-44\",\n \"space-x-48\",\n \"space-y-48\",\n \"space-x-52\",\n \"space-y-52\",\n \"space-x-56\",\n \"space-y-56\",\n \"space-x-60\",\n \"space-y-60\",\n \"space-x-64\",\n \"space-y-64\",\n \"space-x-72\",\n \"space-y-72\",\n \"space-x-80\",\n \"space-y-80\",\n \"space-x-96\",\n \"space-y-96\",\n \"space-x-px\",\n \"space-y-px\",\n \"space-y-reverse\",\n \"space-x-reverse\",\n \"w-0\",\n \"w-px\",\n \"w-0.5\",\n \"w-1\",\n \"w-1.5\",\n \"w-2\",\n \"w-2.5\",\n \"w-3\",\n \"w-3.5\",\n \"w-4\",\n \"w-5\",\n \"w-6\",\n \"w-7\",\n \"w-8\",\n \"w-9\",\n \"w-10\",\n \"w-11\",\n \"w-12\",\n \"w-14\",\n \"w-16\",\n \"w-20\",\n \"w-24\",\n \"w-28\",\n \"w-32\",\n \"w-36\",\n \"w-40\",\n \"w-44\",\n \"w-48\",\n \"w-52\",\n \"w-56\",\n \"w-60\",\n \"w-64\",\n \"w-72\",\n \"w-80\",\n \"w-96\",\n \"w-auto\",\n \"w-1/2\",\n \"w-1/3\",\n \"w-2/3\",\n \"w-1/4\",\n \"w-2/4\",\n \"w-3/4\",\n \"w-1/5\",\n \"w-2/5\",\n \"w-3/5\",\n \"w-4/5\",\n \"w-1/6\",\n \"w-2/6\",\n \"w-3/6\",\n \"w-4/6\",\n \"w-5/6\",\n \"w-1/12\",\n \"w-2/12\",\n \"w-3/12\",\n \"w-4/12\",\n \"w-5/12\",\n \"w-6/12\",\n \"w-7/12\",\n \"w-8/12\",\n \"w-9/12\",\n \"w-10/12\",\n \"w-11/12\",\n \"w-full\",\n \"w-screen\",\n \"w-svw\",\n \"w-lvw\",\n \"w-dvw\",\n \"w-min\",\n \"w-max\",\n \"w-fit\",\n \"min-w-0\",\n \"min-w-1\",\n \"min-w-2\",\n \"min-w-3\",\n \"min-w-4\",\n \"min-w-5\",\n \"min-w-6\",\n \"min-w-7\",\n \"min-w-8\",\n \"min-w-9\",\n \"min-w-10\",\n \"min-w-11\",\n \"min-w-12\",\n \"min-w-14\",\n \"min-w-16\",\n \"min-w-20\",\n \"min-w-24\",\n \"min-w-28\",\n \"min-w-32\",\n \"min-w-36\",\n \"min-w-40\",\n \"min-w-44\",\n \"min-w-48\",\n \"min-w-52\",\n \"min-w-56\",\n \"min-w-60\",\n \"min-w-64\",\n \"min-w-72\",\n \"min-w-80\",\n \"min-w-96\",\n \"min-w-px\",\n \"min-w-0.5\",\n \"min-w-1.5\",\n \"min-w-2.5\",\n \"min-w-3.5\",\n \"min-w-full\",\n \"min-w-min\",\n \"min-w-max\",\n \"min-w-fit\",\n \"max-w-0\",\n \"max-w-px\",\n \"max-w-0.5\",\n \"max-w-1\",\n \"max-w-1.5\",\n \"max-w-2\",\n \"max-w-2.5\",\n \"max-w-3\",\n \"max-w-3.5\",\n \"max-w-4\",\n \"max-w-5\",\n \"max-w-6\",\n \"max-w-7\",\n \"max-w-8\",\n \"max-w-9\",\n \"max-w-10\",\n \"max-w-11\",\n \"max-w-12\",\n \"max-w-14\",\n \"max-w-16\",\n \"max-w-20\",\n \"max-w-24\",\n \"max-w-28\",\n \"max-w-32\",\n \"max-w-36\",\n \"max-w-40\",\n \"max-w-44\",\n \"max-w-48\",\n \"max-w-52\",\n \"max-w-56\",\n \"max-w-60\",\n \"max-w-64\",\n \"max-w-72\",\n \"max-w-80\",\n \"max-w-96\",\n \"max-w-none\",\n \"max-w-xs\",\n \"max-w-sm\",\n \"max-w-md\",\n \"max-w-lg\",\n \"max-w-xl\",\n \"max-w-2xl\",\n \"max-w-3xl\",\n \"max-w-4xl\",\n \"max-w-5xl\",\n \"max-w-6xl\",\n \"max-w-7xl\",\n \"max-w-full\",\n \"max-w-min\",\n \"max-w-max\",\n \"max-w-fit\",\n \"max-w-prose\",\n \"max-w-screen-sm\",\n \"max-w-screen-md\",\n \"max-w-screen-lg\",\n \"max-w-screen-xl\",\n \"max-w-screen-2xl\",\n \"h-0\",\n \"h-px\",\n \"h-0.5\",\n \"h-1\",\n \"h-1.5\",\n \"h-2\",\n \"h-2.5\",\n \"h-3\",\n \"h-3.5\",\n \"h-4\",\n \"h-5\",\n \"h-6\",\n \"h-7\",\n \"h-8\",\n \"h-9\",\n \"h-10\",\n \"h-11\",\n \"h-12\",\n \"h-14\",\n \"h-16\",\n \"h-20\",\n \"h-24\",\n \"h-28\",\n \"h-32\",\n \"h-36\",\n \"h-40\",\n \"h-44\",\n \"h-48\",\n \"h-52\",\n \"h-56\",\n \"h-60\",\n \"h-64\",\n \"h-72\",\n \"h-80\",\n \"h-96\",\n \"h-auto\",\n \"h-1/2\",\n \"h-1/3\",\n \"h-2/3\",\n \"h-1/4\",\n \"h-2/4\",\n \"h-3/4\",\n \"h-1/5\",\n \"h-2/5\",\n \"h-3/5\",\n \"h-4/5\",\n \"h-1/6\",\n \"h-2/6\",\n \"h-3/6\",\n \"h-4/6\",\n \"h-5/6\",\n \"h-full\",\n \"h-screen\",\n \"h-svh\",\n \"h-lvh\",\n \"h-dvh\",\n \"h-min\",\n \"h-max\",\n \"h-fit\",\n \"min-h-0\",\n \"min-h-1\",\n \"min-h-2\",\n \"min-h-3\",\n \"min-h-4\",\n \"min-h-5\",\n \"min-h-6\",\n \"min-h-7\",\n \"min-h-8\",\n \"min-h-9\",\n \"min-h-10\",\n \"min-h-11\",\n \"min-h-12\",\n \"min-h-14\",\n \"min-h-16\",\n \"min-h-20\",\n \"min-h-24\",\n \"min-h-28\",\n \"min-h-32\",\n \"min-h-36\",\n \"min-h-40\",\n \"min-h-44\",\n \"min-h-48\",\n \"min-h-52\",\n \"min-h-56\",\n \"min-h-60\",\n \"min-h-64\",\n \"min-h-72\",\n \"min-h-80\",\n \"min-h-96\",\n \"min-h-px\",\n \"min-h-0.5\",\n \"min-h-1.5\",\n \"min-h-2.5\",\n \"min-h-3.5\",\n \"min-h-full\",\n \"min-h-screen\",\n \"min-h-svh\",\n \"min-h-lvh\",\n \"min-h-dvh\",\n \"min-h-min\",\n \"min-h-max\",\n \"min-h-fit\",\n \"max-h-0\",\n \"max-h-px\",\n \"max-h-0.5\",\n \"max-h-1\",\n \"max-h-1.5\",\n \"max-h-2\",\n \"max-h-2.5\",\n \"max-h-3\",\n \"max-h-3.5\",\n \"max-h-4\",\n \"max-h-5\",\n \"max-h-6\",\n \"max-h-7\",\n \"max-h-8\",\n \"max-h-9\",\n \"max-h-10\",\n \"max-h-11\",\n \"max-h-12\",\n \"max-h-14\",\n \"max-h-16\",\n \"max-h-20\",\n \"max-h-24\",\n \"max-h-28\",\n \"max-h-32\",\n \"max-h-36\",\n \"max-h-40\",\n \"max-h-44\",\n \"max-h-48\",\n \"max-h-52\",\n \"max-h-56\",\n \"max-h-60\",\n \"max-h-64\",\n \"max-h-72\",\n \"max-h-80\",\n \"max-h-96\",\n \"max-h-none\",\n \"max-h-full\",\n \"max-h-screen\",\n \"max-h-svh\",\n \"max-h-lvh\",\n \"max-h-dvh\",\n \"max-h-min\",\n \"max-h-max\",\n \"max-h-fit\",\n \"size-0\",\n \"size-px\",\n \"size-0.5\",\n \"size-1\",\n \"size-1.5\",\n \"size-2\",\n \"size-2.5\",\n \"size-3\",\n \"size-3.5\",\n \"size-4\",\n \"size-5\",\n \"size-6\",\n \"size-7\",\n \"size-8\",\n \"size-9\",\n \"size-10\",\n \"size-11\",\n \"size-12\",\n \"size-14\",\n \"size-16\",\n \"size-20\",\n \"size-24\",\n \"size-28\",\n \"size-32\",\n \"size-36\",\n \"size-40\",\n \"size-44\",\n \"size-48\",\n \"size-52\",\n \"size-56\",\n \"size-60\",\n \"size-64\",\n \"size-72\",\n \"size-80\",\n \"size-96\",\n \"size-auto\",\n \"size-1/2\",\n \"size-1/3\",\n \"size-2/3\",\n \"size-1/4\",\n \"size-2/4\",\n \"size-3/4\",\n \"size-1/5\",\n \"size-2/5\",\n \"size-3/5\",\n \"size-4/5\",\n \"size-1/6\",\n \"size-2/6\",\n \"size-3/6\",\n \"size-4/6\",\n \"size-5/6\",\n \"size-1/12\",\n \"size-2/12\",\n \"size-3/12\",\n \"size-4/12\",\n \"size-5/12\",\n \"size-6/12\",\n \"size-7/12\",\n \"size-8/12\",\n \"size-9/12\",\n \"size-10/12\",\n \"size-11/12\",\n \"size-full\",\n \"size-min\",\n \"size-max\",\n \"size-fit\",\n \"font-sans\",\n \"font-serif\",\n \"font-mono\",\n \"text-xs\",\n \"text-sm\",\n \"text-base\",\n \"text-lg\",\n \"text-xl\",\n \"text-2xl\",\n \"text-3xl\",\n \"text-4xl\",\n \"text-5xl\",\n \"text-6xl\",\n \"text-7xl\",\n \"text-8xl\",\n \"text-9xl\",\n \"antialiased\",\n \"subpixel-antialiased\",\n \"italic\",\n \"not-italic\",\n \"font-thin\",\n \"font-extralight\",\n \"font-light\",\n \"font-normal\",\n \"font-medium\",\n \"font-semibold\",\n \"font-bold\",\n \"font-extrabold\",\n \"font-black\",\n \"normal-nums\",\n \"ordinal\",\n \"slashed-zero\",\n \"lining-nums\",\n \"oldstyle-nums\",\n \"proportional-nums\",\n \"tabular-nums\",\n \"diagonal-fractions\",\n \"stacked-fractions\",\n \"tracking-tighter\",\n \"tracking-tight\",\n \"tracking-normal\",\n \"tracking-wide\",\n \"tracking-wider\",\n \"tracking-widest\",\n \"line-clamp-1\",\n \"line-clamp-2\",\n \"line-clamp-3\",\n \"line-clamp-4\",\n \"line-clamp-5\",\n \"line-clamp-6\",\n \"line-clamp-none\",\n \"leading-3\",\n \"leading-4\",\n \"leading-5\",\n \"leading-6\",\n \"leading-7\",\n \"leading-8\",\n \"leading-9\",\n \"leading-10\",\n \"leading-none\",\n \"leading-tight\",\n \"leading-snug\",\n \"leading-normal\",\n \"leading-relaxed\",\n \"leading-loose\",\n \"list-image-none\",\n \"list-inside\",\n \"list-outside\",\n \"list-none\",\n \"list-disc\",\n \"list-decimal\",\n \"text-left\",\n \"text-center\",\n \"text-right\",\n \"text-justify\",\n \"text-start\",\n \"text-end\",\n \"text-inherit\",\n \"text-current\",\n \"text-transparent\",\n \"text-black\",\n \"text-white\",\n \"text-slate-50\",\n \"text-slate-100\",\n \"text-slate-200\",\n \"text-slate-300\",\n \"text-slate-400\",\n \"text-slate-500\",\n \"text-slate-600\",\n \"text-slate-700\",\n \"text-slate-800\",\n \"text-slate-900\",\n \"text-slate-950\",\n \"text-gray-50\",\n \"text-gray-100\",\n \"text-gray-200\",\n \"text-gray-300\",\n \"text-gray-400\",\n \"text-gray-500\",\n \"text-gray-600\",\n \"text-gray-700\",\n \"text-gray-800\",\n \"text-gray-900\",\n \"text-gray-950\",\n \"text-zinc-50\",\n \"text-zinc-100\",\n \"text-zinc-200\",\n \"text-zinc-300\",\n \"text-zinc-400\",\n \"text-zinc-500\",\n \"text-zinc-600\",\n \"text-zinc-700\",\n \"text-zinc-800\",\n \"text-zinc-900\",\n \"text-zinc-950\",\n \"text-neutral-50\",\n \"text-neutral-100\",\n \"text-neutral-200\",\n \"text-neutral-300\",\n \"text-neutral-400\",\n \"text-neutral-500\",\n \"text-neutral-600\",\n \"text-neutral-700\",\n \"text-neutral-800\",\n \"text-neutral-900\",\n \"text-neutral-950\",\n \"text-stone-50\",\n \"text-stone-100\",\n \"text-stone-200\",\n \"text-stone-300\",\n \"text-stone-400\",\n \"text-stone-500\",\n \"text-stone-600\",\n \"text-stone-700\",\n \"text-stone-800\",\n \"text-stone-900\",\n \"text-stone-950\",\n \"text-red-50\",\n \"text-red-100\",\n \"text-red-200\",\n \"text-red-300\",\n \"text-red-400\",\n \"text-red-500\",\n \"text-red-600\",\n \"text-red-700\",\n \"text-red-800\",\n \"text-red-900\",\n \"text-red-950\",\n \"text-orange-50\",\n \"text-orange-100\",\n \"text-orange-200\",\n \"text-orange-300\",\n \"text-orange-400\",\n \"text-orange-500\",\n \"text-orange-600\",\n \"text-orange-700\",\n \"text-orange-800\",\n \"text-orange-900\",\n \"text-orange-950\",\n \"text-amber-50\",\n \"text-amber-100\",\n \"text-amber-200\",\n \"text-amber-300\",\n \"text-amber-400\",\n \"text-amber-500\",\n \"text-amber-600\",\n \"text-amber-700\",\n \"text-amber-800\",\n \"text-amber-900\",\n \"text-amber-950\",\n \"text-yellow-50\",\n \"text-yellow-100\",\n \"text-yellow-200\",\n \"text-yellow-300\",\n \"text-yellow-400\",\n \"text-yellow-500\",\n \"text-yellow-600\",\n \"text-yellow-700\",\n \"text-yellow-800\",\n \"text-yellow-900\",\n \"text-yellow-950\",\n \"text-lime-50\",\n \"text-lime-100\",\n \"text-lime-200\",\n \"text-lime-300\",\n \"text-lime-400\",\n \"text-lime-500\",\n \"text-lime-600\",\n \"text-lime-700\",\n \"text-lime-800\",\n \"text-lime-900\",\n \"text-lime-950\",\n \"text-green-50\",\n \"text-green-100\",\n \"text-green-200\",\n \"text-green-300\",\n \"text-green-400\",\n \"text-green-500\",\n \"text-green-600\",\n \"text-green-700\",\n \"text-green-800\",\n \"text-green-900\",\n \"text-green-950\",\n \"text-emerald-50\",\n \"text-emerald-100\",\n \"text-emerald-200\",\n \"text-emerald-300\",\n \"text-emerald-400\",\n \"text-emerald-500\",\n \"text-emerald-600\",\n \"text-emerald-700\",\n \"text-emerald-800\",\n \"text-emerald-900\",\n \"text-emerald-950\",\n \"text-teal-50\",\n \"text-teal-100\",\n \"text-teal-200\",\n \"text-teal-300\",\n \"text-teal-400\",\n \"text-teal-500\",\n \"text-teal-600\",\n \"text-teal-700\",\n \"text-teal-800\",\n \"text-teal-900\",\n \"text-teal-950\",\n \"text-cyan-50\",\n \"text-cyan-100\",\n \"text-cyan-200\",\n \"text-cyan-300\",\n \"text-cyan-400\",\n \"text-cyan-500\",\n \"text-cyan-600\",\n \"text-cyan-700\",\n \"text-cyan-800\",\n \"text-cyan-900\",\n \"text-cyan-950\",\n \"text-sky-50\",\n \"text-sky-100\",\n \"text-sky-200\",\n \"text-sky-300\",\n \"text-sky-400\",\n \"text-sky-500\",\n \"text-sky-600\",\n \"text-sky-700\",\n \"text-sky-800\",\n \"text-sky-900\",\n \"text-sky-950\",\n \"text-blue-50\",\n \"text-blue-100\",\n \"text-blue-200\",\n \"text-blue-300\",\n \"text-blue-400\",\n \"text-blue-500\",\n \"text-blue-600\",\n \"text-blue-700\",\n \"text-blue-800\",\n \"text-blue-900\",\n \"text-blue-950\",\n \"text-indigo-50\",\n \"text-indigo-100\",\n \"text-indigo-200\",\n \"text-indigo-300\",\n \"text-indigo-400\",\n \"text-indigo-500\",\n \"text-indigo-600\",\n \"text-indigo-700\",\n \"text-indigo-800\",\n \"text-indigo-900\",\n \"text-indigo-950\",\n \"text-violet-50\",\n \"text-violet-100\",\n \"text-violet-200\",\n \"text-violet-300\",\n \"text-violet-400\",\n \"text-violet-500\",\n \"text-violet-600\",\n \"text-violet-700\",\n \"text-violet-800\",\n \"text-violet-900\",\n \"text-violet-950\",\n \"text-purple-50\",\n \"text-purple-100\",\n \"text-purple-200\",\n \"text-purple-300\",\n \"text-purple-400\",\n \"text-purple-500\",\n \"text-purple-600\",\n \"text-purple-700\",\n \"text-purple-800\",\n \"text-purple-900\",\n \"text-purple-950\",\n \"text-fuchsia-50\",\n \"text-fuchsia-100\",\n \"text-fuchsia-200\",\n \"text-fuchsia-300\",\n \"text-fuchsia-400\",\n \"text-fuchsia-500\",\n \"text-fuchsia-600\",\n \"text-fuchsia-700\",\n \"text-fuchsia-800\",\n \"text-fuchsia-900\",\n \"text-fuchsia-950\",\n \"text-pink-50\",\n \"text-pink-100\",\n \"text-pink-200\",\n \"text-pink-300\",\n \"text-pink-400\",\n \"text-pink-500\",\n \"text-pink-600\",\n \"text-pink-700\",\n \"text-pink-800\",\n \"text-pink-900\",\n \"text-pink-950\",\n \"text-rose-50\",\n \"text-rose-100\",\n \"text-rose-200\",\n \"text-rose-300\",\n \"text-rose-400\",\n \"text-rose-500\",\n \"text-rose-600\",\n \"text-rose-700\",\n \"text-rose-800\",\n \"text-rose-900\",\n \"text-rose-950\",\n \"underline\",\n \"overline\",\n \"line-through\",\n \"no-underline\",\n \"decoration-inherit\",\n \"decoration-current\",\n \"decoration-transparent\",\n \"decoration-black\",\n \"decoration-white\",\n \"decoration-slate-50\",\n \"decoration-slate-100\",\n \"decoration-slate-200\",\n \"decoration-slate-300\",\n \"decoration-slate-400\",\n \"decoration-slate-500\",\n \"decoration-slate-600\",\n \"decoration-slate-700\",\n \"decoration-slate-800\",\n \"decoration-slate-900\",\n \"decoration-slate-950\",\n \"decoration-gray-50\",\n \"decoration-gray-100\",\n \"decoration-gray-200\",\n \"decoration-gray-300\",\n \"decoration-gray-400\",\n \"decoration-gray-500\",\n \"decoration-gray-600\",\n \"decoration-gray-700\",\n \"decoration-gray-800\",\n \"decoration-gray-900\",\n \"decoration-gray-950\",\n \"decoration-zinc-50\",\n \"decoration-zinc-100\",\n \"decoration-zinc-200\",\n \"decoration-zinc-300\",\n \"decoration-zinc-400\",\n \"decoration-zinc-500\",\n \"decoration-zinc-600\",\n \"decoration-zinc-700\",\n \"decoration-zinc-800\",\n \"decoration-zinc-900\",\n \"decoration-zinc-950\",\n \"decoration-neutral-50\",\n \"decoration-neutral-100\",\n \"decoration-neutral-200\",\n \"decoration-neutral-300\",\n \"decoration-neutral-400\",\n \"decoration-neutral-500\",\n \"decoration-neutral-600\",\n \"decoration-neutral-700\",\n \"decoration-neutral-800\",\n \"decoration-neutral-900\",\n \"decoration-neutral-950\",\n \"decoration-stone-50\",\n \"decoration-stone-100\",\n \"decoration-stone-200\",\n \"decoration-stone-300\",\n \"decoration-stone-400\",\n \"decoration-stone-500\",\n \"decoration-stone-600\",\n \"decoration-stone-700\",\n \"decoration-stone-800\",\n \"decoration-stone-900\",\n \"decoration-stone-950\",\n \"decoration-red-50\",\n \"decoration-red-100\",\n \"decoration-red-200\",\n \"decoration-red-300\",\n \"decoration-red-400\",\n \"decoration-red-500\",\n \"decoration-red-600\",\n \"decoration-red-700\",\n \"decoration-red-800\",\n \"decoration-red-900\",\n \"decoration-red-950\",\n \"decoration-orange-50\",\n \"decoration-orange-100\",\n \"decoration-orange-200\",\n \"decoration-orange-300\",\n \"decoration-orange-400\",\n \"decoration-orange-500\",\n \"decoration-orange-600\",\n \"decoration-orange-700\",\n \"decoration-orange-800\",\n \"decoration-orange-900\",\n \"decoration-orange-950\",\n \"decoration-amber-50\",\n \"decoration-amber-100\",\n \"decoration-amber-200\",\n \"decoration-amber-300\",\n \"decoration-amber-400\",\n \"decoration-amber-500\",\n \"decoration-amber-600\",\n \"decoration-amber-700\",\n \"decoration-amber-800\",\n \"decoration-amber-900\",\n \"decoration-amber-950\",\n \"decoration-yellow-50\",\n \"decoration-yellow-100\",\n \"decoration-yellow-200\",\n \"decoration-yellow-300\",\n \"decoration-yellow-400\",\n \"decoration-yellow-500\",\n \"decoration-yellow-600\",\n \"decoration-yellow-700\",\n \"decoration-yellow-800\",\n \"decoration-yellow-900\",\n \"decoration-yellow-950\",\n \"decoration-lime-50\",\n \"decoration-lime-100\",\n \"decoration-lime-200\",\n \"decoration-lime-300\",\n \"decoration-lime-400\",\n \"decoration-lime-500\",\n \"decoration-lime-600\",\n \"decoration-lime-700\",\n \"decoration-lime-800\",\n \"decoration-lime-900\",\n \"decoration-lime-950\",\n \"decoration-green-50\",\n \"decoration-green-100\",\n \"decoration-green-200\",\n \"decoration-green-300\",\n \"decoration-green-400\",\n \"decoration-green-500\",\n \"decoration-green-600\",\n \"decoration-green-700\",\n \"decoration-green-800\",\n \"decoration-green-900\",\n \"decoration-green-950\",\n \"decoration-emerald-50\",\n \"decoration-emerald-100\",\n \"decoration-emerald-200\",\n \"decoration-emerald-300\",\n \"decoration-emerald-400\",\n \"decoration-emerald-500\",\n \"decoration-emerald-600\",\n \"decoration-emerald-700\",\n \"decoration-emerald-800\",\n \"decoration-emerald-900\",\n \"decoration-emerald-950\",\n \"decoration-teal-50\",\n \"decoration-teal-100\",\n \"decoration-teal-200\",\n \"decoration-teal-300\",\n \"decoration-teal-400\",\n \"decoration-teal-500\",\n \"decoration-teal-600\",\n \"decoration-teal-700\",\n \"decoration-teal-800\",\n \"decoration-teal-900\",\n \"decoration-teal-950\",\n \"decoration-cyan-50\",\n \"decoration-cyan-100\",\n \"decoration-cyan-200\",\n \"decoration-cyan-300\",\n \"decoration-cyan-400\",\n \"decoration-cyan-500\",\n \"decoration-cyan-600\",\n \"decoration-cyan-700\",\n \"decoration-cyan-800\",\n \"decoration-cyan-900\",\n \"decoration-cyan-950\",\n \"decoration-sky-50\",\n \"decoration-sky-100\",\n \"decoration-sky-200\",\n \"decoration-sky-300\",\n \"decoration-sky-400\",\n \"decoration-sky-500\",\n \"decoration-sky-600\",\n \"decoration-sky-700\",\n \"decoration-sky-800\",\n \"decoration-sky-900\",\n \"decoration-sky-950\",\n \"decoration-blue-50\",\n \"decoration-blue-100\",\n \"decoration-blue-200\",\n \"decoration-blue-300\",\n \"decoration-blue-400\",\n \"decoration-blue-500\",\n \"decoration-blue-600\",\n \"decoration-blue-700\",\n \"decoration-blue-800\",\n \"decoration-blue-900\",\n \"decoration-blue-950\",\n \"decoration-indigo-50\",\n \"decoration-indigo-100\",\n \"decoration-indigo-200\",\n \"decoration-indigo-300\",\n \"decoration-indigo-400\",\n \"decoration-indigo-500\",\n \"decoration-indigo-600\",\n \"decoration-indigo-700\",\n \"decoration-indigo-800\",\n \"decoration-indigo-900\",\n \"decoration-indigo-950\",\n \"decoration-violet-50\",\n \"decoration-violet-100\",\n \"decoration-violet-200\",\n \"decoration-violet-300\",\n \"decoration-violet-400\",\n \"decoration-violet-500\",\n \"decoration-violet-600\",\n \"decoration-violet-700\",\n \"decoration-violet-800\",\n \"decoration-violet-900\",\n \"decoration-violet-950\",\n \"decoration-purple-50\",\n \"decoration-purple-100\",\n \"decoration-purple-200\",\n \"decoration-purple-300\",\n \"decoration-purple-400\",\n \"decoration-purple-500\",\n \"decoration-purple-600\",\n \"decoration-purple-700\",\n \"decoration-purple-800\",\n \"decoration-purple-900\",\n \"decoration-purple-950\",\n \"decoration-fuchsia-50\",\n \"decoration-fuchsia-100\",\n \"decoration-fuchsia-200\",\n \"decoration-fuchsia-300\",\n \"decoration-fuchsia-400\",\n \"decoration-fuchsia-500\",\n \"decoration-fuchsia-600\",\n \"decoration-fuchsia-700\",\n \"decoration-fuchsia-800\",\n \"decoration-fuchsia-900\",\n \"decoration-fuchsia-950\",\n \"decoration-pink-50\",\n \"decoration-pink-100\",\n \"decoration-pink-200\",\n \"decoration-pink-300\",\n \"decoration-pink-400\",\n \"decoration-pink-500\",\n \"decoration-pink-600\",\n \"decoration-pink-700\",\n \"decoration-pink-800\",\n \"decoration-pink-900\",\n \"decoration-pink-950\",\n \"decoration-rose-50\",\n \"decoration-rose-100\",\n \"decoration-rose-200\",\n \"decoration-rose-300\",\n \"decoration-rose-400\",\n \"decoration-rose-500\",\n \"decoration-rose-600\",\n \"decoration-rose-700\",\n \"decoration-rose-800\",\n \"decoration-rose-900\",\n \"decoration-rose-950\",\n \"decoration-solid\",\n \"decoration-double\",\n \"decoration-dotted\",\n \"decoration-dashed\",\n \"decoration-wavy\",\n \"decoration-auto\",\n \"decoration-from-font\",\n \"decoration-0\",\n \"decoration-1\",\n \"decoration-2\",\n \"decoration-4\",\n \"decoration-8\",\n \"underline-offset-auto\",\n \"underline-offset-0\",\n \"underline-offset-1\",\n \"underline-offset-2\",\n \"underline-offset-4\",\n \"underline-offset-8\",\n \"uppercase\",\n \"lowercase\",\n \"capitalize\",\n \"normal-case\",\n \"truncate\",\n \"text-ellipsis\",\n \"text-clip\",\n \"text-wrap\",\n \"text-nowrap\",\n \"text-balance\",\n \"text-pretty\",\n \"indent-0\",\n \"indent-px\",\n \"indent-0.5\",\n \"indent-1\",\n \"indent-1.5\",\n \"indent-2\",\n \"indent-2.5\",\n \"indent-3\",\n \"indent-3.5\",\n \"indent-4\",\n \"indent-5\",\n \"indent-6\",\n \"indent-7\",\n \"indent-8\",\n \"indent-9\",\n \"indent-10\",\n \"indent-11\",\n \"indent-12\",\n \"indent-14\",\n \"indent-16\",\n \"indent-20\",\n \"indent-24\",\n \"indent-28\",\n \"indent-32\",\n \"indent-36\",\n \"indent-40\",\n \"indent-44\",\n \"indent-48\",\n \"indent-52\",\n \"indent-56\",\n \"indent-60\",\n \"indent-64\",\n \"indent-72\",\n \"indent-80\",\n \"indent-96\",\n \"align-baseline\",\n \"align-top\",\n \"align-middle\",\n \"align-bottom\",\n \"align-text-top\",\n \"align-text-bottom\",\n \"align-sub\",\n \"align-super\",\n \"whitespace-normal\",\n \"whitespace-nowrap\",\n \"whitespace-pre\",\n \"whitespace-pre-line\",\n \"whitespace-pre-wrap\",\n \"whitespace-break-spaces\",\n \"break-normal\",\n \"break-words\",\n \"break-all\",\n \"break-keep\",\n \"hyphens-none\",\n \"hyphens-manual\",\n \"hyphens-auto\",\n \"content-none\",\n \"bg-fixed\",\n \"bg-local\",\n \"bg-scroll\",\n \"bg-clip-border\",\n \"bg-clip-padding\",\n \"bg-clip-content\",\n \"bg-clip-text\",\n \"bg-inherit\",\n \"bg-current\",\n \"bg-transparent\",\n \"bg-black\",\n \"bg-white\",\n \"bg-slate-50\",\n \"bg-slate-100\",\n \"bg-slate-200\",\n \"bg-slate-300\",\n \"bg-slate-400\",\n \"bg-slate-500\",\n \"bg-slate-600\",\n \"bg-slate-700\",\n \"bg-slate-800\",\n \"bg-slate-900\",\n \"bg-slate-950\",\n \"bg-gray-50\",\n \"bg-gray-100\",\n \"bg-gray-200\",\n \"bg-gray-300\",\n \"bg-gray-400\",\n \"bg-gray-500\",\n \"bg-gray-600\",\n \"bg-gray-700\",\n \"bg-gray-800\",\n \"bg-gray-900\",\n \"bg-gray-950\",\n \"bg-zinc-50\",\n \"bg-zinc-100\",\n \"bg-zinc-200\",\n \"bg-zinc-300\",\n \"bg-zinc-400\",\n \"bg-zinc-500\",\n \"bg-zinc-600\",\n \"bg-zinc-700\",\n \"bg-zinc-800\",\n \"bg-zinc-900\",\n \"bg-zinc-950\",\n \"bg-neutral-50\",\n \"bg-neutral-100\",\n \"bg-neutral-200\",\n \"bg-neutral-300\",\n \"bg-neutral-400\",\n \"bg-neutral-500\",\n \"bg-neutral-600\",\n \"bg-neutral-700\",\n \"bg-neutral-800\",\n \"bg-neutral-900\",\n \"bg-neutral-950\",\n \"bg-stone-50\",\n \"bg-stone-100\",\n \"bg-stone-200\",\n \"bg-stone-300\",\n \"bg-stone-400\",\n \"bg-stone-500\",\n \"bg-stone-600\",\n \"bg-stone-700\",\n \"bg-stone-800\",\n \"bg-stone-900\",\n \"bg-stone-950\",\n \"bg-red-50\",\n \"bg-red-100\",\n \"bg-red-200\",\n \"bg-red-300\",\n \"bg-red-400\",\n \"bg-red-500\",\n \"bg-red-600\",\n \"bg-red-700\",\n \"bg-red-800\",\n \"bg-red-900\",\n \"bg-red-950\",\n \"bg-orange-50\",\n \"bg-orange-100\",\n \"bg-orange-200\",\n \"bg-orange-300\",\n \"bg-orange-400\",\n \"bg-orange-500\",\n \"bg-orange-600\",\n \"bg-orange-700\",\n \"bg-orange-800\",\n \"bg-orange-900\",\n \"bg-orange-950\",\n \"bg-amber-50\",\n \"bg-amber-100\",\n \"bg-amber-200\",\n \"bg-amber-300\",\n \"bg-amber-400\",\n \"bg-amber-500\",\n \"bg-amber-600\",\n \"bg-amber-700\",\n \"bg-amber-800\",\n \"bg-amber-900\",\n \"bg-amber-950\",\n \"bg-yellow-50\",\n \"bg-yellow-100\",\n \"bg-yellow-200\",\n \"bg-yellow-300\",\n \"bg-yellow-400\",\n \"bg-yellow-500\",\n \"bg-yellow-600\",\n \"bg-yellow-700\",\n \"bg-yellow-800\",\n \"bg-yellow-900\",\n \"bg-yellow-950\",\n \"bg-lime-50\",\n \"bg-lime-100\",\n \"bg-lime-200\",\n \"bg-lime-300\",\n \"bg-lime-400\",\n \"bg-lime-500\",\n \"bg-lime-600\",\n \"bg-lime-700\",\n \"bg-lime-800\",\n \"bg-lime-900\",\n \"bg-lime-950\",\n \"bg-green-50\",\n \"bg-green-100\",\n \"bg-green-200\",\n \"bg-green-300\",\n \"bg-green-400\",\n \"bg-green-500\",\n \"bg-green-600\",\n \"bg-green-700\",\n \"bg-green-800\",\n \"bg-green-900\",\n \"bg-green-950\",\n \"bg-emerald-50\",\n \"bg-emerald-100\",\n \"bg-emerald-200\",\n \"bg-emerald-300\",\n \"bg-emerald-400\",\n \"bg-emerald-500\",\n \"bg-emerald-600\",\n \"bg-emerald-700\",\n \"bg-emerald-800\",\n \"bg-emerald-900\",\n \"bg-emerald-950\",\n \"bg-teal-50\",\n \"bg-teal-100\",\n \"bg-teal-200\",\n \"bg-teal-300\",\n \"bg-teal-400\",\n \"bg-teal-500\",\n \"bg-teal-600\",\n \"bg-teal-700\",\n \"bg-teal-800\",\n \"bg-teal-900\",\n \"bg-teal-950\",\n \"bg-cyan-50\",\n \"bg-cyan-100\",\n \"bg-cyan-200\",\n \"bg-cyan-300\",\n \"bg-cyan-400\",\n \"bg-cyan-500\",\n \"bg-cyan-600\",\n \"bg-cyan-700\",\n \"bg-cyan-800\",\n \"bg-cyan-900\",\n \"bg-cyan-950\",\n \"bg-sky-50\",\n \"bg-sky-100\",\n \"bg-sky-200\",\n \"bg-sky-300\",\n \"bg-sky-400\",\n \"bg-sky-500\",\n \"bg-sky-600\",\n \"bg-sky-700\",\n \"bg-sky-800\",\n \"bg-sky-900\",\n \"bg-sky-950\",\n \"bg-blue-50\",\n \"bg-blue-100\",\n \"bg-blue-200\",\n \"bg-blue-300\",\n \"bg-blue-400\",\n \"bg-blue-500\",\n \"bg-blue-600\",\n \"bg-blue-700\",\n \"bg-blue-800\",\n \"bg-blue-900\",\n \"bg-blue-950\",\n \"bg-indigo-50\",\n \"bg-indigo-100\",\n \"bg-indigo-200\",\n \"bg-indigo-300\",\n \"bg-indigo-400\",\n \"bg-indigo-500\",\n \"bg-indigo-600\",\n \"bg-indigo-700\",\n \"bg-indigo-800\",\n \"bg-indigo-900\",\n \"bg-indigo-950\",\n \"bg-violet-50\",\n \"bg-violet-100\",\n \"bg-violet-200\",\n \"bg-violet-300\",\n \"bg-violet-400\",\n \"bg-violet-500\",\n \"bg-violet-600\",\n \"bg-violet-700\",\n \"bg-violet-800\",\n \"bg-violet-900\",\n \"bg-violet-950\",\n \"bg-purple-50\",\n \"bg-purple-100\",\n \"bg-purple-200\",\n \"bg-purple-300\",\n \"bg-purple-400\",\n \"bg-purple-500\",\n \"bg-purple-600\",\n \"bg-purple-700\",\n \"bg-purple-800\",\n \"bg-purple-900\",\n \"bg-purple-950\",\n \"bg-fuchsia-50\",\n \"bg-fuchsia-100\",\n \"bg-fuchsia-200\",\n \"bg-fuchsia-300\",\n \"bg-fuchsia-400\",\n \"bg-fuchsia-500\",\n \"bg-fuchsia-600\",\n \"bg-fuchsia-700\",\n \"bg-fuchsia-800\",\n \"bg-fuchsia-900\",\n \"bg-fuchsia-950\",\n \"bg-pink-50\",\n \"bg-pink-100\",\n \"bg-pink-200\",\n \"bg-pink-300\",\n \"bg-pink-400\",\n \"bg-pink-500\",\n \"bg-pink-600\",\n \"bg-pink-700\",\n \"bg-pink-800\",\n \"bg-pink-900\",\n \"bg-pink-950\",\n \"bg-rose-50\",\n \"bg-rose-100\",\n \"bg-rose-200\",\n \"bg-rose-300\",\n \"bg-rose-400\",\n \"bg-rose-500\",\n \"bg-rose-600\",\n \"bg-rose-700\",\n \"bg-rose-800\",\n \"bg-rose-900\",\n \"bg-rose-950\",\n \"bg-origin-border\",\n \"bg-origin-padding\",\n \"bg-origin-content\",\n \"bg-bottom\",\n \"bg-center\",\n \"bg-left\",\n \"bg-left-bottom\",\n \"bg-left-top\",\n \"bg-right\",\n \"bg-right-bottom\",\n \"bg-right-top\",\n \"bg-top\",\n \"bg-repeat\",\n \"bg-no-repeat\",\n \"bg-repeat-x\",\n \"bg-repeat-y\",\n \"bg-repeat-round\",\n \"bg-repeat-space\",\n \"bg-auto\",\n \"bg-cover\",\n \"bg-contain\",\n \"bg-none\",\n \"bg-gradient-to-t\",\n \"bg-gradient-to-tr\",\n \"bg-gradient-to-r\",\n \"bg-gradient-to-br\",\n \"bg-gradient-to-b\",\n \"bg-gradient-to-bl\",\n \"bg-gradient-to-l\",\n \"bg-gradient-to-tl\",\n \"from-inherit\",\n \"from-current\",\n \"from-transparent\",\n \"from-black\",\n \"from-white\",\n \"from-slate-50\",\n \"from-slate-100\",\n \"from-slate-200\",\n \"from-slate-300\",\n \"from-slate-400\",\n \"from-slate-500\",\n \"from-slate-600\",\n \"from-slate-700\",\n \"from-slate-800\",\n \"from-slate-900\",\n \"from-slate-950\",\n \"from-gray-50\",\n \"from-gray-100\",\n \"from-gray-200\",\n \"from-gray-300\",\n \"from-gray-400\",\n \"from-gray-500\",\n \"from-gray-600\",\n \"from-gray-700\",\n \"from-gray-800\",\n \"from-gray-900\",\n \"from-gray-950\",\n \"from-zinc-50\",\n \"from-zinc-100\",\n \"from-zinc-200\",\n \"from-zinc-300\",\n \"from-zinc-400\",\n \"from-zinc-500\",\n \"from-zinc-600\",\n \"from-zinc-700\",\n \"from-zinc-800\",\n \"from-zinc-900\",\n \"from-zinc-950\",\n \"from-neutral-50\",\n \"from-neutral-100\",\n \"from-neutral-200\",\n \"from-neutral-300\",\n \"from-neutral-400\",\n \"from-neutral-500\",\n \"from-neutral-600\",\n \"from-neutral-700\",\n \"from-neutral-800\",\n \"from-neutral-900\",\n \"from-neutral-950\",\n \"from-stone-50\",\n \"from-stone-100\",\n \"from-stone-200\",\n \"from-stone-300\",\n \"from-stone-400\",\n \"from-stone-500\",\n \"from-stone-600\",\n \"from-stone-700\",\n \"from-stone-800\",\n \"from-stone-900\",\n \"from-stone-950\",\n \"from-red-50\",\n \"from-red-100\",\n \"from-red-200\",\n \"from-red-300\",\n \"from-red-400\",\n \"from-red-500\",\n \"from-red-600\",\n \"from-red-700\",\n \"from-red-800\",\n \"from-red-900\",\n \"from-red-950\",\n \"from-orange-50\",\n \"from-orange-100\",\n \"from-orange-200\",\n \"from-orange-300\",\n \"from-orange-400\",\n \"from-orange-500\",\n \"from-orange-600\",\n \"from-orange-700\",\n \"from-orange-800\",\n \"from-orange-900\",\n \"from-orange-950\",\n \"from-amber-50\",\n \"from-amber-100\",\n \"from-amber-200\",\n \"from-amber-300\",\n \"from-amber-400\",\n \"from-amber-500\",\n \"from-amber-600\",\n \"from-amber-700\",\n \"from-amber-800\",\n \"from-amber-900\",\n \"from-amber-950\",\n \"from-yellow-50\",\n \"from-yellow-100\",\n \"from-yellow-200\",\n \"from-yellow-300\",\n \"from-yellow-400\",\n \"from-yellow-500\",\n \"from-yellow-600\",\n \"from-yellow-700\",\n \"from-yellow-800\",\n \"from-yellow-900\",\n \"from-yellow-950\",\n \"from-lime-50\",\n \"from-lime-100\",\n \"from-lime-200\",\n \"from-lime-300\",\n \"from-lime-400\",\n \"from-lime-500\",\n \"from-lime-600\",\n \"from-lime-700\",\n \"from-lime-800\",\n \"from-lime-900\",\n \"from-lime-950\",\n \"from-green-50\",\n \"from-green-100\",\n \"from-green-200\",\n \"from-green-300\",\n \"from-green-400\",\n \"from-green-500\",\n \"from-green-600\",\n \"from-green-700\",\n \"from-green-800\",\n \"from-green-900\",\n \"from-green-950\",\n \"from-emerald-50\",\n \"from-emerald-100\",\n \"from-emerald-200\",\n \"from-emerald-300\",\n \"from-emerald-400\",\n \"from-emerald-500\",\n \"from-emerald-600\",\n \"from-emerald-700\",\n \"from-emerald-800\",\n \"from-emerald-900\",\n \"from-emerald-950\",\n \"from-teal-50\",\n \"from-teal-100\",\n \"from-teal-200\",\n \"from-teal-300\",\n \"from-teal-400\",\n \"from-teal-500\",\n \"from-teal-600\",\n \"from-teal-700\",\n \"from-teal-800\",\n \"from-teal-900\",\n \"from-teal-950\",\n \"from-cyan-50\",\n \"from-cyan-100\",\n \"from-cyan-200\",\n \"from-cyan-300\",\n \"from-cyan-400\",\n \"from-cyan-500\",\n \"from-cyan-600\",\n \"from-cyan-700\",\n \"from-cyan-800\",\n \"from-cyan-900\",\n \"from-cyan-950\",\n \"from-sky-50\",\n \"from-sky-100\",\n \"from-sky-200\",\n \"from-sky-300\",\n \"from-sky-400\",\n \"from-sky-500\",\n \"from-sky-600\",\n \"from-sky-700\",\n \"from-sky-800\",\n \"from-sky-900\",\n \"from-sky-950\",\n \"from-blue-50\",\n \"from-blue-100\",\n \"from-blue-200\",\n \"from-blue-300\",\n \"from-blue-400\",\n \"from-blue-500\",\n \"from-blue-600\",\n \"from-blue-700\",\n \"from-blue-800\",\n \"from-blue-900\",\n \"from-blue-950\",\n \"from-indigo-50\",\n \"from-indigo-100\",\n \"from-indigo-200\",\n \"from-indigo-300\",\n \"from-indigo-400\",\n \"from-indigo-500\",\n \"from-indigo-600\",\n \"from-indigo-700\",\n \"from-indigo-800\",\n \"from-indigo-900\",\n \"from-indigo-950\",\n \"from-violet-50\",\n \"from-violet-100\",\n \"from-violet-200\",\n \"from-violet-300\",\n \"from-violet-400\",\n \"from-violet-500\",\n \"from-violet-600\",\n \"from-violet-700\",\n \"from-violet-800\",\n \"from-violet-900\",\n \"from-violet-950\",\n \"from-purple-50\",\n \"from-purple-100\",\n \"from-purple-200\",\n \"from-purple-300\",\n \"from-purple-400\",\n \"from-purple-500\",\n \"from-purple-600\",\n \"from-purple-700\",\n \"from-purple-800\",\n \"from-purple-900\",\n \"from-purple-950\",\n \"from-fuchsia-50\",\n \"from-fuchsia-100\",\n \"from-fuchsia-200\",\n \"from-fuchsia-300\",\n \"from-fuchsia-400\",\n \"from-fuchsia-500\",\n \"from-fuchsia-600\",\n \"from-fuchsia-700\",\n \"from-fuchsia-800\",\n \"from-fuchsia-900\",\n \"from-fuchsia-950\",\n \"from-pink-50\",\n \"from-pink-100\",\n \"from-pink-200\",\n \"from-pink-300\",\n \"from-pink-400\",\n \"from-pink-500\",\n \"from-pink-600\",\n \"from-pink-700\",\n \"from-pink-800\",\n \"from-pink-900\",\n \"from-pink-950\",\n \"from-rose-50\",\n \"from-rose-100\",\n \"from-rose-200\",\n \"from-rose-300\",\n \"from-rose-400\",\n \"from-rose-500\",\n \"from-rose-600\",\n \"from-rose-700\",\n \"from-rose-800\",\n \"from-rose-900\",\n \"from-rose-950\",\n \"from-0%\",\n \"from-5%\",\n \"from-10%\",\n \"from-15%\",\n \"from-20%\",\n \"from-25%\",\n \"from-30%\",\n \"from-35%\",\n \"from-40%\",\n \"from-45%\",\n \"from-50%\",\n \"from-55%\",\n \"from-60%\",\n \"from-65%\",\n \"from-70%\",\n \"from-75%\",\n \"from-80%\",\n \"from-85%\",\n \"from-90%\",\n \"from-95%\",\n \"from-100%\",\n \"via-inherit\",\n \"via-current\",\n \"via-transparent\",\n \"via-black\",\n \"via-white\",\n \"via-slate-50\",\n \"via-slate-100\",\n \"via-slate-200\",\n \"via-slate-300\",\n \"via-slate-400\",\n \"via-slate-500\",\n \"via-slate-600\",\n \"via-slate-700\",\n \"via-slate-800\",\n \"via-slate-900\",\n \"via-slate-950\",\n \"via-gray-50\",\n \"via-gray-100\",\n \"via-gray-200\",\n \"via-gray-300\",\n \"via-gray-400\",\n \"via-gray-500\",\n \"via-gray-600\",\n \"via-gray-700\",\n \"via-gray-800\",\n \"via-gray-900\",\n \"via-gray-950\",\n \"via-zinc-50\",\n \"via-zinc-100\",\n \"via-zinc-200\",\n \"via-zinc-300\",\n \"via-zinc-400\",\n \"via-zinc-500\",\n \"via-zinc-600\",\n \"via-zinc-700\",\n \"via-zinc-800\",\n \"via-zinc-900\",\n \"via-zinc-950\",\n \"via-neutral-50\",\n \"via-neutral-100\",\n \"via-neutral-200\",\n \"via-neutral-300\",\n \"via-neutral-400\",\n \"via-neutral-500\",\n \"via-neutral-600\",\n \"via-neutral-700\",\n \"via-neutral-800\",\n \"via-neutral-900\",\n \"via-neutral-950\",\n \"via-stone-50\",\n \"via-stone-100\",\n \"via-stone-200\",\n \"via-stone-300\",\n \"via-stone-400\",\n \"via-stone-500\",\n \"via-stone-600\",\n \"via-stone-700\",\n \"via-stone-800\",\n \"via-stone-900\",\n \"via-stone-950\",\n \"via-red-50\",\n \"via-red-100\",\n \"via-red-200\",\n \"via-red-300\",\n \"via-red-400\",\n \"via-red-500\",\n \"via-red-600\",\n \"via-red-700\",\n \"via-red-800\",\n \"via-red-900\",\n \"via-red-950\",\n \"via-orange-50\",\n \"via-orange-100\",\n \"via-orange-200\",\n \"via-orange-300\",\n \"via-orange-400\",\n \"via-orange-500\",\n \"via-orange-600\",\n \"via-orange-700\",\n \"via-orange-800\",\n \"via-orange-900\",\n \"via-orange-950\",\n \"via-amber-50\",\n \"via-amber-100\",\n \"via-amber-200\",\n \"via-amber-300\",\n \"via-amber-400\",\n \"via-amber-500\",\n \"via-amber-600\",\n \"via-amber-700\",\n \"via-amber-800\",\n \"via-amber-900\",\n \"via-amber-950\",\n \"via-yellow-50\",\n \"via-yellow-100\",\n \"via-yellow-200\",\n \"via-yellow-300\",\n \"via-yellow-400\",\n \"via-yellow-500\",\n \"via-yellow-600\",\n \"via-yellow-700\",\n \"via-yellow-800\",\n \"via-yellow-900\",\n \"via-yellow-950\",\n \"via-lime-50\",\n \"via-lime-100\",\n \"via-lime-200\",\n \"via-lime-300\",\n \"via-lime-400\",\n \"via-lime-500\",\n \"via-lime-600\",\n \"via-lime-700\",\n \"via-lime-800\",\n \"via-lime-900\",\n \"via-lime-950\",\n \"via-green-50\",\n \"via-green-100\",\n \"via-green-200\",\n \"via-green-300\",\n \"via-green-400\",\n \"via-green-500\",\n \"via-green-600\",\n \"via-green-700\",\n \"via-green-800\",\n \"via-green-900\",\n \"via-green-950\",\n \"via-emerald-50\",\n \"via-emerald-100\",\n \"via-emerald-200\",\n \"via-emerald-300\",\n \"via-emerald-400\",\n \"via-emerald-500\",\n \"via-emerald-600\",\n \"via-emerald-700\",\n \"via-emerald-800\",\n \"via-emerald-900\",\n \"via-emerald-950\",\n \"via-teal-50\",\n \"via-teal-100\",\n \"via-teal-200\",\n \"via-teal-300\",\n \"via-teal-400\",\n \"via-teal-500\",\n \"via-teal-600\",\n \"via-teal-700\",\n \"via-teal-800\",\n \"via-teal-900\",\n \"via-teal-950\",\n \"via-cyan-50\",\n \"via-cyan-100\",\n \"via-cyan-200\",\n \"via-cyan-300\",\n \"via-cyan-400\",\n \"via-cyan-500\",\n \"via-cyan-600\",\n \"via-cyan-700\",\n \"via-cyan-800\",\n \"via-cyan-900\",\n \"via-cyan-950\",\n \"via-sky-50\",\n \"via-sky-100\",\n \"via-sky-200\",\n \"via-sky-300\",\n \"via-sky-400\",\n \"via-sky-500\",\n \"via-sky-600\",\n \"via-sky-700\",\n \"via-sky-800\",\n \"via-sky-900\",\n \"via-sky-950\",\n \"via-blue-50\",\n \"via-blue-100\",\n \"via-blue-200\",\n \"via-blue-300\",\n \"via-blue-400\",\n \"via-blue-500\",\n \"via-blue-600\",\n \"via-blue-700\",\n \"via-blue-800\",\n \"via-blue-900\",\n \"via-blue-950\",\n \"via-indigo-50\",\n \"via-indigo-100\",\n \"via-indigo-200\",\n \"via-indigo-300\",\n \"via-indigo-400\",\n \"via-indigo-500\",\n \"via-indigo-600\",\n \"via-indigo-700\",\n \"via-indigo-800\",\n \"via-indigo-900\",\n \"via-indigo-950\",\n \"via-violet-50\",\n \"via-violet-100\",\n \"via-violet-200\",\n \"via-violet-300\",\n \"via-violet-400\",\n \"via-violet-500\",\n \"via-violet-600\",\n \"via-violet-700\",\n \"via-violet-800\",\n \"via-violet-900\",\n \"via-violet-950\",\n \"via-purple-50\",\n \"via-purple-100\",\n \"via-purple-200\",\n \"via-purple-300\",\n \"via-purple-400\",\n \"via-purple-500\",\n \"via-purple-600\",\n \"via-purple-700\",\n \"via-purple-800\",\n \"via-purple-900\",\n \"via-purple-950\",\n \"via-fuchsia-50\",\n \"via-fuchsia-100\",\n \"via-fuchsia-200\",\n \"via-fuchsia-300\",\n \"via-fuchsia-400\",\n \"via-fuchsia-500\",\n \"via-fuchsia-600\",\n \"via-fuchsia-700\",\n \"via-fuchsia-800\",\n \"via-fuchsia-900\",\n \"via-fuchsia-950\",\n \"via-pink-50\",\n \"via-pink-100\",\n \"via-pink-200\",\n \"via-pink-300\",\n \"via-pink-400\",\n \"via-pink-500\",\n \"via-pink-600\",\n \"via-pink-700\",\n \"via-pink-800\",\n \"via-pink-900\",\n \"via-pink-950\",\n \"via-rose-50\",\n \"via-rose-100\",\n \"via-rose-200\",\n \"via-rose-300\",\n \"via-rose-400\",\n \"via-rose-500\",\n \"via-rose-600\",\n \"via-rose-700\",\n \"via-rose-800\",\n \"via-rose-900\",\n \"via-rose-950\",\n \"via-0%\",\n \"via-5%\",\n \"via-10%\",\n \"via-15%\",\n \"via-20%\",\n \"via-25%\",\n \"via-30%\",\n \"via-35%\",\n \"via-40%\",\n \"via-45%\",\n \"via-50%\",\n \"via-55%\",\n \"via-60%\",\n \"via-65%\",\n \"via-70%\",\n \"via-75%\",\n \"via-80%\",\n \"via-85%\",\n \"via-90%\",\n \"via-95%\",\n \"via-100%\",\n \"to-inherit\",\n \"to-current\",\n \"to-transparent\",\n \"to-black\",\n \"to-white\",\n \"to-slate-50\",\n \"to-slate-100\",\n \"to-slate-200\",\n \"to-slate-300\",\n \"to-slate-400\",\n \"to-slate-500\",\n \"to-slate-600\",\n \"to-slate-700\",\n \"to-slate-800\",\n \"to-slate-900\",\n \"to-slate-950\",\n \"to-gray-50\",\n \"to-gray-100\",\n \"to-gray-200\",\n \"to-gray-300\",\n \"to-gray-400\",\n \"to-gray-500\",\n \"to-gray-600\",\n \"to-gray-700\",\n \"to-gray-800\",\n \"to-gray-900\",\n \"to-gray-950\",\n \"to-zinc-50\",\n \"to-zinc-100\",\n \"to-zinc-200\",\n \"to-zinc-300\",\n \"to-zinc-400\",\n \"to-zinc-500\",\n \"to-zinc-600\",\n \"to-zinc-700\",\n \"to-zinc-800\",\n \"to-zinc-900\",\n \"to-zinc-950\",\n \"to-neutral-50\",\n \"to-neutral-100\",\n \"to-neutral-200\",\n \"to-neutral-300\",\n \"to-neutral-400\",\n \"to-neutral-500\",\n \"to-neutral-600\",\n \"to-neutral-700\",\n \"to-neutral-800\",\n \"to-neutral-900\",\n \"to-neutral-950\",\n \"to-stone-50\",\n \"to-stone-100\",\n \"to-stone-200\",\n \"to-stone-300\",\n \"to-stone-400\",\n \"to-stone-500\",\n \"to-stone-600\",\n \"to-stone-700\",\n \"to-stone-800\",\n \"to-stone-900\",\n \"to-stone-950\",\n \"to-red-50\",\n \"to-red-100\",\n \"to-red-200\",\n \"to-red-300\",\n \"to-red-400\",\n \"to-red-500\",\n \"to-red-600\",\n \"to-red-700\",\n \"to-red-800\",\n \"to-red-900\",\n \"to-red-950\",\n \"to-orange-50\",\n \"to-orange-100\",\n \"to-orange-200\",\n \"to-orange-300\",\n \"to-orange-400\",\n \"to-orange-500\",\n \"to-orange-600\",\n \"to-orange-700\",\n \"to-orange-800\",\n \"to-orange-900\",\n \"to-orange-950\",\n \"to-amber-50\",\n \"to-amber-100\",\n \"to-amber-200\",\n \"to-amber-300\",\n \"to-amber-400\",\n \"to-amber-500\",\n \"to-amber-600\",\n \"to-amber-700\",\n \"to-amber-800\",\n \"to-amber-900\",\n \"to-amber-950\",\n \"to-yellow-50\",\n \"to-yellow-100\",\n \"to-yellow-200\",\n \"to-yellow-300\",\n \"to-yellow-400\",\n \"to-yellow-500\",\n \"to-yellow-600\",\n \"to-yellow-700\",\n \"to-yellow-800\",\n \"to-yellow-900\",\n \"to-yellow-950\",\n \"to-lime-50\",\n \"to-lime-100\",\n \"to-lime-200\",\n \"to-lime-300\",\n \"to-lime-400\",\n \"to-lime-500\",\n \"to-lime-600\",\n \"to-lime-700\",\n \"to-lime-800\",\n \"to-lime-900\",\n \"to-lime-950\",\n \"to-green-50\",\n \"to-green-100\",\n \"to-green-200\",\n \"to-green-300\",\n \"to-green-400\",\n \"to-green-500\",\n \"to-green-600\",\n \"to-green-700\",\n \"to-green-800\",\n \"to-green-900\",\n \"to-green-950\",\n \"to-emerald-50\",\n \"to-emerald-100\",\n \"to-emerald-200\",\n \"to-emerald-300\",\n \"to-emerald-400\",\n \"to-emerald-500\",\n \"to-emerald-600\",\n \"to-emerald-700\",\n \"to-emerald-800\",\n \"to-emerald-900\",\n \"to-emerald-950\",\n \"to-teal-50\",\n \"to-teal-100\",\n \"to-teal-200\",\n \"to-teal-300\",\n \"to-teal-400\",\n \"to-teal-500\",\n \"to-teal-600\",\n \"to-teal-700\",\n \"to-teal-800\",\n \"to-teal-900\",\n \"to-teal-950\",\n \"to-cyan-50\",\n \"to-cyan-100\",\n \"to-cyan-200\",\n \"to-cyan-300\",\n \"to-cyan-400\",\n \"to-cyan-500\",\n \"to-cyan-600\",\n \"to-cyan-700\",\n \"to-cyan-800\",\n \"to-cyan-900\",\n \"to-cyan-950\",\n \"to-sky-50\",\n \"to-sky-100\",\n \"to-sky-200\",\n \"to-sky-300\",\n \"to-sky-400\",\n \"to-sky-500\",\n \"to-sky-600\",\n \"to-sky-700\",\n \"to-sky-800\",\n \"to-sky-900\",\n \"to-sky-950\",\n \"to-blue-50\",\n \"to-blue-100\",\n \"to-blue-200\",\n \"to-blue-300\",\n \"to-blue-400\",\n \"to-blue-500\",\n \"to-blue-600\",\n \"to-blue-700\",\n \"to-blue-800\",\n \"to-blue-900\",\n \"to-blue-950\",\n \"to-indigo-50\",\n \"to-indigo-100\",\n \"to-indigo-200\",\n \"to-indigo-300\",\n \"to-indigo-400\",\n \"to-indigo-500\",\n \"to-indigo-600\",\n \"to-indigo-700\",\n \"to-indigo-800\",\n \"to-indigo-900\",\n \"to-indigo-950\",\n \"to-violet-50\",\n \"to-violet-100\",\n \"to-violet-200\",\n \"to-violet-300\",\n \"to-violet-400\",\n \"to-violet-500\",\n \"to-violet-600\",\n \"to-violet-700\",\n \"to-violet-800\",\n \"to-violet-900\",\n \"to-violet-950\",\n \"to-purple-50\",\n \"to-purple-100\",\n \"to-purple-200\",\n \"to-purple-300\",\n \"to-purple-400\",\n \"to-purple-500\",\n \"to-purple-600\",\n \"to-purple-700\",\n \"to-purple-800\",\n \"to-purple-900\",\n \"to-purple-950\",\n \"to-fuchsia-50\",\n \"to-fuchsia-100\",\n \"to-fuchsia-200\",\n \"to-fuchsia-300\",\n \"to-fuchsia-400\",\n \"to-fuchsia-500\",\n \"to-fuchsia-600\",\n \"to-fuchsia-700\",\n \"to-fuchsia-800\",\n \"to-fuchsia-900\",\n \"to-fuchsia-950\",\n \"to-pink-50\",\n \"to-pink-100\",\n \"to-pink-200\",\n \"to-pink-300\",\n \"to-pink-400\",\n \"to-pink-500\",\n \"to-pink-600\",\n \"to-pink-700\",\n \"to-pink-800\",\n \"to-pink-900\",\n \"to-pink-950\",\n \"to-rose-50\",\n \"to-rose-100\",\n \"to-rose-200\",\n \"to-rose-300\",\n \"to-rose-400\",\n \"to-rose-500\",\n \"to-rose-600\",\n \"to-rose-700\",\n \"to-rose-800\",\n \"to-rose-900\",\n \"to-rose-950\",\n \"to-0%\",\n \"to-5%\",\n \"to-10%\",\n \"to-15%\",\n \"to-20%\",\n \"to-25%\",\n \"to-30%\",\n \"to-35%\",\n \"to-40%\",\n \"to-45%\",\n \"to-50%\",\n \"to-55%\",\n \"to-60%\",\n \"to-65%\",\n \"to-70%\",\n \"to-75%\",\n \"to-80%\",\n \"to-85%\",\n \"to-90%\",\n \"to-95%\",\n \"to-100%\",\n \"rounded-none\",\n \"rounded-sm\",\n \"rounded\",\n \"rounded-md\",\n \"rounded-lg\",\n \"rounded-xl\",\n \"rounded-2xl\",\n \"rounded-3xl\",\n \"rounded-full\",\n \"rounded-s-none\",\n \"rounded-s-sm\",\n \"rounded-s\",\n \"rounded-s-md\",\n \"rounded-s-lg\",\n \"rounded-s-xl\",\n \"rounded-s-2xl\",\n \"rounded-s-3xl\",\n \"rounded-s-full\",\n \"rounded-e-none\",\n \"rounded-e-sm\",\n \"rounded-e\",\n \"rounded-e-md\",\n \"rounded-e-lg\",\n \"rounded-e-xl\",\n \"rounded-e-2xl\",\n \"rounded-e-3xl\",\n \"rounded-e-full\",\n \"rounded-t-none\",\n \"rounded-t-sm\",\n \"rounded-t\",\n \"rounded-t-md\",\n \"rounded-t-lg\",\n \"rounded-t-xl\",\n \"rounded-t-2xl\",\n \"rounded-t-3xl\",\n \"rounded-t-full\",\n \"rounded-r-none\",\n \"rounded-r-sm\",\n \"rounded-r\",\n \"rounded-r-md\",\n \"rounded-r-lg\",\n \"rounded-r-xl\",\n \"rounded-r-2xl\",\n \"rounded-r-3xl\",\n \"rounded-r-full\",\n \"rounded-b-none\",\n \"rounded-b-sm\",\n \"rounded-b\",\n \"rounded-b-md\",\n \"rounded-b-lg\",\n \"rounded-b-xl\",\n \"rounded-b-2xl\",\n \"rounded-b-3xl\",\n \"rounded-b-full\",\n \"rounded-l-none\",\n \"rounded-l-sm\",\n \"rounded-l\",\n \"rounded-l-md\",\n \"rounded-l-lg\",\n \"rounded-l-xl\",\n \"rounded-l-2xl\",\n \"rounded-l-3xl\",\n \"rounded-l-full\",\n \"rounded-ss-none\",\n \"rounded-ss-sm\",\n \"rounded-ss\",\n \"rounded-ss-md\",\n \"rounded-ss-lg\",\n \"rounded-ss-xl\",\n \"rounded-ss-2xl\",\n \"rounded-ss-3xl\",\n \"rounded-ss-full\",\n \"rounded-se-none\",\n \"rounded-se-sm\",\n \"rounded-se\",\n \"rounded-se-md\",\n \"rounded-se-lg\",\n \"rounded-se-xl\",\n \"rounded-se-2xl\",\n \"rounded-se-3xl\",\n \"rounded-se-full\",\n \"rounded-ee-none\",\n \"rounded-ee-sm\",\n \"rounded-ee\",\n \"rounded-ee-md\",\n \"rounded-ee-lg\",\n \"rounded-ee-xl\",\n \"rounded-ee-2xl\",\n \"rounded-ee-3xl\",\n \"rounded-ee-full\",\n \"rounded-es-none\",\n \"rounded-es-sm\",\n \"rounded-es\",\n \"rounded-es-md\",\n \"rounded-es-lg\",\n \"rounded-es-xl\",\n \"rounded-es-2xl\",\n \"rounded-es-3xl\",\n \"rounded-es-full\",\n \"rounded-tl-none\",\n \"rounded-tl-sm\",\n \"rounded-tl\",\n \"rounded-tl-md\",\n \"rounded-tl-lg\",\n \"rounded-tl-xl\",\n \"rounded-tl-2xl\",\n \"rounded-tl-3xl\",\n \"rounded-tl-full\",\n \"rounded-tr-none\",\n \"rounded-tr-sm\",\n \"rounded-tr\",\n \"rounded-tr-md\",\n \"rounded-tr-lg\",\n \"rounded-tr-xl\",\n \"rounded-tr-2xl\",\n \"rounded-tr-3xl\",\n \"rounded-tr-full\",\n \"rounded-br-none\",\n \"rounded-br-sm\",\n \"rounded-br\",\n \"rounded-br-md\",\n \"rounded-br-lg\",\n \"rounded-br-xl\",\n \"rounded-br-2xl\",\n \"rounded-br-3xl\",\n \"rounded-br-full\",\n \"rounded-bl-none\",\n \"rounded-bl-sm\",\n \"rounded-bl\",\n \"rounded-bl-md\",\n \"rounded-bl-lg\",\n \"rounded-bl-xl\",\n \"rounded-bl-2xl\",\n \"rounded-bl-3xl\",\n \"rounded-bl-full\",\n \"rounded-s-*\",\n \"rounded-e-*\",\n \"rounded-ss-*\",\n \"rounded-se-*\",\n \"rounded-es-*\",\n \"rounded-ee-*\",\n \"border-0\",\n \"border-2\",\n \"border-4\",\n \"border-8\",\n \"border\",\n \"border-x-0\",\n \"border-x-2\",\n \"border-x-4\",\n \"border-x-8\",\n \"border-x\",\n \"border-y-0\",\n \"border-y-2\",\n \"border-y-4\",\n \"border-y-8\",\n \"border-y\",\n \"border-s-0\",\n \"border-s-2\",\n \"border-s-4\",\n \"border-s-8\",\n \"border-s\",\n \"border-e-0\",\n \"border-e-2\",\n \"border-e-4\",\n \"border-e-8\",\n \"border-e\",\n \"border-t-0\",\n \"border-t-2\",\n \"border-t-4\",\n \"border-t-8\",\n \"border-t\",\n \"border-r-0\",\n \"border-r-2\",\n \"border-r-4\",\n \"border-r-8\",\n \"border-r\",\n \"border-b-0\",\n \"border-b-2\",\n \"border-b-4\",\n \"border-b-8\",\n \"border-b\",\n \"border-l-0\",\n \"border-l-2\",\n \"border-l-4\",\n \"border-l-8\",\n \"border-l\",\n \"border-inherit\",\n \"border-current\",\n \"border-transparent\",\n \"border-black\",\n \"border-white\",\n \"border-slate-50\",\n \"border-slate-100\",\n \"border-slate-200\",\n \"border-slate-300\",\n \"border-slate-400\",\n \"border-slate-500\",\n \"border-slate-600\",\n \"border-slate-700\",\n \"border-slate-800\",\n \"border-slate-900\",\n \"border-slate-950\",\n \"border-gray-50\",\n \"border-gray-100\",\n \"border-gray-200\",\n \"border-gray-300\",\n \"border-gray-400\",\n \"border-gray-500\",\n \"border-gray-600\",\n \"border-gray-700\",\n \"border-gray-800\",\n \"border-gray-900\",\n \"border-gray-950\",\n \"border-zinc-50\",\n \"border-zinc-100\",\n \"border-zinc-200\",\n \"border-zinc-300\",\n \"border-zinc-400\",\n \"border-zinc-500\",\n \"border-zinc-600\",\n \"border-zinc-700\",\n \"border-zinc-800\",\n \"border-zinc-900\",\n \"border-zinc-950\",\n \"border-neutral-50\",\n \"border-neutral-100\",\n \"border-neutral-200\",\n \"border-neutral-300\",\n \"border-neutral-400\",\n \"border-neutral-500\",\n \"border-neutral-600\",\n \"border-neutral-700\",\n \"border-neutral-800\",\n \"border-neutral-900\",\n \"border-neutral-950\",\n \"border-stone-50\",\n \"border-stone-100\",\n \"border-stone-200\",\n \"border-stone-300\",\n \"border-stone-400\",\n \"border-stone-500\",\n \"border-stone-600\",\n \"border-stone-700\",\n \"border-stone-800\",\n \"border-stone-900\",\n \"border-stone-950\",\n \"border-red-50\",\n \"border-red-100\",\n \"border-red-200\",\n \"border-red-300\",\n \"border-red-400\",\n \"border-red-500\",\n \"border-red-600\",\n \"border-red-700\",\n \"border-red-800\",\n \"border-red-900\",\n \"border-red-950\",\n \"border-orange-50\",\n \"border-orange-100\",\n \"border-orange-200\",\n \"border-orange-300\",\n \"border-orange-400\",\n \"border-orange-500\",\n \"border-orange-600\",\n \"border-orange-700\",\n \"border-orange-800\",\n \"border-orange-900\",\n \"border-orange-950\",\n \"border-amber-50\",\n \"border-amber-100\",\n \"border-amber-200\",\n \"border-amber-300\",\n \"border-amber-400\",\n \"border-amber-500\",\n \"border-amber-600\",\n \"border-amber-700\",\n \"border-amber-800\",\n \"border-amber-900\",\n \"border-amber-950\",\n \"border-yellow-50\",\n \"border-yellow-100\",\n \"border-yellow-200\",\n \"border-yellow-300\",\n \"border-yellow-400\",\n \"border-yellow-500\",\n \"border-yellow-600\",\n \"border-yellow-700\",\n \"border-yellow-800\",\n \"border-yellow-900\",\n \"border-yellow-950\",\n \"border-lime-50\",\n \"border-lime-100\",\n \"border-lime-200\",\n \"border-lime-300\",\n \"border-lime-400\",\n \"border-lime-500\",\n \"border-lime-600\",\n \"border-lime-700\",\n \"border-lime-800\",\n \"border-lime-900\",\n \"border-lime-950\",\n \"border-green-50\",\n \"border-green-100\",\n \"border-green-200\",\n \"border-green-300\",\n \"border-green-400\",\n \"border-green-500\",\n \"border-green-600\",\n \"border-green-700\",\n \"border-green-800\",\n \"border-green-900\",\n \"border-green-950\",\n \"border-emerald-50\",\n \"border-emerald-100\",\n \"border-emerald-200\",\n \"border-emerald-300\",\n \"border-emerald-400\",\n \"border-emerald-500\",\n \"border-emerald-600\",\n \"border-emerald-700\",\n \"border-emerald-800\",\n \"border-emerald-900\",\n \"border-emerald-950\",\n \"border-teal-50\",\n \"border-teal-100\",\n \"border-teal-200\",\n \"border-teal-300\",\n \"border-teal-400\",\n \"border-teal-500\",\n \"border-teal-600\",\n \"border-teal-700\",\n \"border-teal-800\",\n \"border-teal-900\",\n \"border-teal-950\",\n \"border-cyan-50\",\n \"border-cyan-100\",\n \"border-cyan-200\",\n \"border-cyan-300\",\n \"border-cyan-400\",\n \"border-cyan-500\",\n \"border-cyan-600\",\n \"border-cyan-700\",\n \"border-cyan-800\",\n \"border-cyan-900\",\n \"border-cyan-950\",\n \"border-sky-50\",\n \"border-sky-100\",\n \"border-sky-200\",\n \"border-sky-300\",\n \"border-sky-400\",\n \"border-sky-500\",\n \"border-sky-600\",\n \"border-sky-700\",\n \"border-sky-800\",\n \"border-sky-900\",\n \"border-sky-950\",\n \"border-blue-50\",\n \"border-blue-100\",\n \"border-blue-200\",\n \"border-blue-300\",\n \"border-blue-400\",\n \"border-blue-500\",\n \"border-blue-600\",\n \"border-blue-700\",\n \"border-blue-800\",\n \"border-blue-900\",\n \"border-blue-950\",\n \"border-indigo-50\",\n \"border-indigo-100\",\n \"border-indigo-200\",\n \"border-indigo-300\",\n \"border-indigo-400\",\n \"border-indigo-500\",\n \"border-indigo-600\",\n \"border-indigo-700\",\n \"border-indigo-800\",\n \"border-indigo-900\",\n \"border-indigo-950\",\n \"border-violet-50\",\n \"border-violet-100\",\n \"border-violet-200\",\n \"border-violet-300\",\n \"border-violet-400\",\n \"border-violet-500\",\n \"border-violet-600\",\n \"border-violet-700\",\n \"border-violet-800\",\n \"border-violet-900\",\n \"border-violet-950\",\n \"border-purple-50\",\n \"border-purple-100\",\n \"border-purple-200\",\n \"border-purple-300\",\n \"border-purple-400\",\n \"border-purple-500\",\n \"border-purple-600\",\n \"border-purple-700\",\n \"border-purple-800\",\n \"border-purple-900\",\n \"border-purple-950\",\n \"border-fuchsia-50\",\n \"border-fuchsia-100\",\n \"border-fuchsia-200\",\n \"border-fuchsia-300\",\n \"border-fuchsia-400\",\n \"border-fuchsia-500\",\n \"border-fuchsia-600\",\n \"border-fuchsia-700\",\n \"border-fuchsia-800\",\n \"border-fuchsia-900\",\n \"border-fuchsia-950\",\n \"border-pink-50\",\n \"border-pink-100\",\n \"border-pink-200\",\n \"border-pink-300\",\n \"border-pink-400\",\n \"border-pink-500\",\n \"border-pink-600\",\n \"border-pink-700\",\n \"border-pink-800\",\n \"border-pink-900\",\n \"border-pink-950\",\n \"border-rose-50\",\n \"border-rose-100\",\n \"border-rose-200\",\n \"border-rose-300\",\n \"border-rose-400\",\n \"border-rose-500\",\n \"border-rose-600\",\n \"border-rose-700\",\n \"border-rose-800\",\n \"border-rose-900\",\n \"border-rose-950\",\n \"border-x-inherit\",\n \"border-x-current\",\n \"border-x-transparent\",\n \"border-x-black\",\n \"border-x-white\",\n \"border-x-slate-50\",\n \"border-x-slate-100\",\n \"border-x-slate-200\",\n \"border-x-slate-300\",\n \"border-x-slate-400\",\n \"border-x-slate-500\",\n \"border-x-slate-600\",\n \"border-x-slate-700\",\n \"border-x-slate-800\",\n \"border-x-slate-900\",\n \"border-x-slate-950\",\n \"border-x-gray-50\",\n \"border-x-gray-100\",\n \"border-x-gray-200\",\n \"border-x-gray-300\",\n \"border-x-gray-400\",\n \"border-x-gray-500\",\n \"border-x-gray-600\",\n \"border-x-gray-700\",\n \"border-x-gray-800\",\n \"border-x-gray-900\",\n \"border-x-gray-950\",\n \"border-x-zinc-50\",\n \"border-x-zinc-100\",\n \"border-x-zinc-200\",\n \"border-x-zinc-300\",\n \"border-x-zinc-400\",\n \"border-x-zinc-500\",\n \"border-x-zinc-600\",\n \"border-x-zinc-700\",\n \"border-x-zinc-800\",\n \"border-x-zinc-900\",\n \"border-x-zinc-950\",\n \"border-x-neutral-50\",\n \"border-x-neutral-100\",\n \"border-x-neutral-200\",\n \"border-x-neutral-300\",\n \"border-x-neutral-400\",\n \"border-x-neutral-500\",\n \"border-x-neutral-600\",\n \"border-x-neutral-700\",\n \"border-x-neutral-800\",\n \"border-x-neutral-900\",\n \"border-x-neutral-950\",\n \"border-x-stone-50\",\n \"border-x-stone-100\",\n \"border-x-stone-200\",\n \"border-x-stone-300\",\n \"border-x-stone-400\",\n \"border-x-stone-500\",\n \"border-x-stone-600\",\n \"border-x-stone-700\",\n \"border-x-stone-800\",\n \"border-x-stone-900\",\n \"border-x-stone-950\",\n \"border-x-red-50\",\n \"border-x-red-100\",\n \"border-x-red-200\",\n \"border-x-red-300\",\n \"border-x-red-400\",\n \"border-x-red-500\",\n \"border-x-red-600\",\n \"border-x-red-700\",\n \"border-x-red-800\",\n \"border-x-red-900\",\n \"border-x-red-950\",\n \"border-x-orange-50\",\n \"border-x-orange-100\",\n \"border-x-orange-200\",\n \"border-x-orange-300\",\n \"border-x-orange-400\",\n \"border-x-orange-500\",\n \"border-x-orange-600\",\n \"border-x-orange-700\",\n \"border-x-orange-800\",\n \"border-x-orange-900\",\n \"border-x-orange-950\",\n \"border-x-amber-50\",\n \"border-x-amber-100\",\n \"border-x-amber-200\",\n \"border-x-amber-300\",\n \"border-x-amber-400\",\n \"border-x-amber-500\",\n \"border-x-amber-600\",\n \"border-x-amber-700\",\n \"border-x-amber-800\",\n \"border-x-amber-900\",\n \"border-x-amber-950\",\n \"border-x-yellow-50\",\n \"border-x-yellow-100\",\n \"border-x-yellow-200\",\n \"border-x-yellow-300\",\n \"border-x-yellow-400\",\n \"border-x-yellow-500\",\n \"border-x-yellow-600\",\n \"border-x-yellow-700\",\n \"border-x-yellow-800\",\n \"border-x-yellow-900\",\n \"border-x-yellow-950\",\n \"border-x-lime-50\",\n \"border-x-lime-100\",\n \"border-x-lime-200\",\n \"border-x-lime-300\",\n \"border-x-lime-400\",\n \"border-x-lime-500\",\n \"border-x-lime-600\",\n \"border-x-lime-700\",\n \"border-x-lime-800\",\n \"border-x-lime-900\",\n \"border-x-lime-950\",\n \"border-x-green-50\",\n \"border-x-green-100\",\n \"border-x-green-200\",\n \"border-x-green-300\",\n \"border-x-green-400\",\n \"border-x-green-500\",\n \"border-x-green-600\",\n \"border-x-green-700\",\n \"border-x-green-800\",\n \"border-x-green-900\",\n \"border-x-green-950\",\n \"border-x-emerald-50\",\n \"border-x-emerald-100\",\n \"border-x-emerald-200\",\n \"border-x-emerald-300\",\n \"border-x-emerald-400\",\n \"border-x-emerald-500\",\n \"border-x-emerald-600\",\n \"border-x-emerald-700\",\n \"border-x-emerald-800\",\n \"border-x-emerald-900\",\n \"border-x-emerald-950\",\n \"border-x-teal-50\",\n \"border-x-teal-100\",\n \"border-x-teal-200\",\n \"border-x-teal-300\",\n \"border-x-teal-400\",\n \"border-x-teal-500\",\n \"border-x-teal-600\",\n \"border-x-teal-700\",\n \"border-x-teal-800\",\n \"border-x-teal-900\",\n \"border-x-teal-950\",\n \"border-x-cyan-50\",\n \"border-x-cyan-100\",\n \"border-x-cyan-200\",\n \"border-x-cyan-300\",\n \"border-x-cyan-400\",\n \"border-x-cyan-500\",\n \"border-x-cyan-600\",\n \"border-x-cyan-700\",\n \"border-x-cyan-800\",\n \"border-x-cyan-900\",\n \"border-x-cyan-950\",\n \"border-x-sky-50\",\n \"border-x-sky-100\",\n \"border-x-sky-200\",\n \"border-x-sky-300\",\n \"border-x-sky-400\",\n \"border-x-sky-500\",\n \"border-x-sky-600\",\n \"border-x-sky-700\",\n \"border-x-sky-800\",\n \"border-x-sky-900\",\n \"border-x-sky-950\",\n \"border-x-blue-50\",\n \"border-x-blue-100\",\n \"border-x-blue-200\",\n \"border-x-blue-300\",\n \"border-x-blue-400\",\n \"border-x-blue-500\",\n \"border-x-blue-600\",\n \"border-x-blue-700\",\n \"border-x-blue-800\",\n \"border-x-blue-900\",\n \"border-x-blue-950\",\n \"border-x-indigo-50\",\n \"border-x-indigo-100\",\n \"border-x-indigo-200\",\n \"border-x-indigo-300\",\n \"border-x-indigo-400\",\n \"border-x-indigo-500\",\n \"border-x-indigo-600\",\n \"border-x-indigo-700\",\n \"border-x-indigo-800\",\n \"border-x-indigo-900\",\n \"border-x-indigo-950\",\n \"border-x-violet-50\",\n \"border-x-violet-100\",\n \"border-x-violet-200\",\n \"border-x-violet-300\",\n \"border-x-violet-400\",\n \"border-x-violet-500\",\n \"border-x-violet-600\",\n \"border-x-violet-700\",\n \"border-x-violet-800\",\n \"border-x-violet-900\",\n \"border-x-violet-950\",\n \"border-x-purple-50\",\n \"border-x-purple-100\",\n \"border-x-purple-200\",\n \"border-x-purple-300\",\n \"border-x-purple-400\",\n \"border-x-purple-500\",\n \"border-x-purple-600\",\n \"border-x-purple-700\",\n \"border-x-purple-800\",\n \"border-x-purple-900\",\n \"border-x-purple-950\",\n \"border-x-fuchsia-50\",\n \"border-x-fuchsia-100\",\n \"border-x-fuchsia-200\",\n \"border-x-fuchsia-300\",\n \"border-x-fuchsia-400\",\n \"border-x-fuchsia-500\",\n \"border-x-fuchsia-600\",\n \"border-x-fuchsia-700\",\n \"border-x-fuchsia-800\",\n \"border-x-fuchsia-900\",\n \"border-x-fuchsia-950\",\n \"border-x-pink-50\",\n \"border-x-pink-100\",\n \"border-x-pink-200\",\n \"border-x-pink-300\",\n \"border-x-pink-400\",\n \"border-x-pink-500\",\n \"border-x-pink-600\",\n \"border-x-pink-700\",\n \"border-x-pink-800\",\n \"border-x-pink-900\",\n \"border-x-pink-950\",\n \"border-x-rose-50\",\n \"border-x-rose-100\",\n \"border-x-rose-200\",\n \"border-x-rose-300\",\n \"border-x-rose-400\",\n \"border-x-rose-500\",\n \"border-x-rose-600\",\n \"border-x-rose-700\",\n \"border-x-rose-800\",\n \"border-x-rose-900\",\n \"border-x-rose-950\",\n \"border-y-inherit\",\n \"border-y-current\",\n \"border-y-transparent\",\n \"border-y-black\",\n \"border-y-white\",\n \"border-y-slate-50\",\n \"border-y-slate-100\",\n \"border-y-slate-200\",\n \"border-y-slate-300\",\n \"border-y-slate-400\",\n \"border-y-slate-500\",\n \"border-y-slate-600\",\n \"border-y-slate-700\",\n \"border-y-slate-800\",\n \"border-y-slate-900\",\n \"border-y-slate-950\",\n \"border-y-gray-50\",\n \"border-y-gray-100\",\n \"border-y-gray-200\",\n \"border-y-gray-300\",\n \"border-y-gray-400\",\n \"border-y-gray-500\",\n \"border-y-gray-600\",\n \"border-y-gray-700\",\n \"border-y-gray-800\",\n \"border-y-gray-900\",\n \"border-y-gray-950\",\n \"border-y-zinc-50\",\n \"border-y-zinc-100\",\n \"border-y-zinc-200\",\n \"border-y-zinc-300\",\n \"border-y-zinc-400\",\n \"border-y-zinc-500\",\n \"border-y-zinc-600\",\n \"border-y-zinc-700\",\n \"border-y-zinc-800\",\n \"border-y-zinc-900\",\n \"border-y-zinc-950\",\n \"border-y-neutral-50\",\n \"border-y-neutral-100\",\n \"border-y-neutral-200\",\n \"border-y-neutral-300\",\n \"border-y-neutral-400\",\n \"border-y-neutral-500\",\n \"border-y-neutral-600\",\n \"border-y-neutral-700\",\n \"border-y-neutral-800\",\n \"border-y-neutral-900\",\n \"border-y-neutral-950\",\n \"border-y-stone-50\",\n \"border-y-stone-100\",\n \"border-y-stone-200\",\n \"border-y-stone-300\",\n \"border-y-stone-400\",\n \"border-y-stone-500\",\n \"border-y-stone-600\",\n \"border-y-stone-700\",\n \"border-y-stone-800\",\n \"border-y-stone-900\",\n \"border-y-stone-950\",\n \"border-y-red-50\",\n \"border-y-red-100\",\n \"border-y-red-200\",\n \"border-y-red-300\",\n \"border-y-red-400\",\n \"border-y-red-500\",\n \"border-y-red-600\",\n \"border-y-red-700\",\n \"border-y-red-800\",\n \"border-y-red-900\",\n \"border-y-red-950\",\n \"border-y-orange-50\",\n \"border-y-orange-100\",\n \"border-y-orange-200\",\n \"border-y-orange-300\",\n \"border-y-orange-400\",\n \"border-y-orange-500\",\n \"border-y-orange-600\",\n \"border-y-orange-700\",\n \"border-y-orange-800\",\n \"border-y-orange-900\",\n \"border-y-orange-950\",\n \"border-y-amber-50\",\n \"border-y-amber-100\",\n \"border-y-amber-200\",\n \"border-y-amber-300\",\n \"border-y-amber-400\",\n \"border-y-amber-500\",\n \"border-y-amber-600\",\n \"border-y-amber-700\",\n \"border-y-amber-800\",\n \"border-y-amber-900\",\n \"border-y-amber-950\",\n \"border-y-yellow-50\",\n \"border-y-yellow-100\",\n \"border-y-yellow-200\",\n \"border-y-yellow-300\",\n \"border-y-yellow-400\",\n \"border-y-yellow-500\",\n \"border-y-yellow-600\",\n \"border-y-yellow-700\",\n \"border-y-yellow-800\",\n \"border-y-yellow-900\",\n \"border-y-yellow-950\",\n \"border-y-lime-50\",\n \"border-y-lime-100\",\n \"border-y-lime-200\",\n \"border-y-lime-300\",\n \"border-y-lime-400\",\n \"border-y-lime-500\",\n \"border-y-lime-600\",\n \"border-y-lime-700\",\n \"border-y-lime-800\",\n \"border-y-lime-900\",\n \"border-y-lime-950\",\n \"border-y-green-50\",\n \"border-y-green-100\",\n \"border-y-green-200\",\n \"border-y-green-300\",\n \"border-y-green-400\",\n \"border-y-green-500\",\n \"border-y-green-600\",\n \"border-y-green-700\",\n \"border-y-green-800\",\n \"border-y-green-900\",\n \"border-y-green-950\",\n \"border-y-emerald-50\",\n \"border-y-emerald-100\",\n \"border-y-emerald-200\",\n \"border-y-emerald-300\",\n \"border-y-emerald-400\",\n \"border-y-emerald-500\",\n \"border-y-emerald-600\",\n \"border-y-emerald-700\",\n \"border-y-emerald-800\",\n \"border-y-emerald-900\",\n \"border-y-emerald-950\",\n \"border-y-teal-50\",\n \"border-y-teal-100\",\n \"border-y-teal-200\",\n \"border-y-teal-300\",\n \"border-y-teal-400\",\n \"border-y-teal-500\",\n \"border-y-teal-600\",\n \"border-y-teal-700\",\n \"border-y-teal-800\",\n \"border-y-teal-900\",\n \"border-y-teal-950\",\n \"border-y-cyan-50\",\n \"border-y-cyan-100\",\n \"border-y-cyan-200\",\n \"border-y-cyan-300\",\n \"border-y-cyan-400\",\n \"border-y-cyan-500\",\n \"border-y-cyan-600\",\n \"border-y-cyan-700\",\n \"border-y-cyan-800\",\n \"border-y-cyan-900\",\n \"border-y-cyan-950\",\n \"border-y-sky-50\",\n \"border-y-sky-100\",\n \"border-y-sky-200\",\n \"border-y-sky-300\",\n \"border-y-sky-400\",\n \"border-y-sky-500\",\n \"border-y-sky-600\",\n \"border-y-sky-700\",\n \"border-y-sky-800\",\n \"border-y-sky-900\",\n \"border-y-sky-950\",\n \"border-y-blue-50\",\n \"border-y-blue-100\",\n \"border-y-blue-200\",\n \"border-y-blue-300\",\n \"border-y-blue-400\",\n \"border-y-blue-500\",\n \"border-y-blue-600\",\n \"border-y-blue-700\",\n \"border-y-blue-800\",\n \"border-y-blue-900\",\n \"border-y-blue-950\",\n \"border-y-indigo-50\",\n \"border-y-indigo-100\",\n \"border-y-indigo-200\",\n \"border-y-indigo-300\",\n \"border-y-indigo-400\",\n \"border-y-indigo-500\",\n \"border-y-indigo-600\",\n \"border-y-indigo-700\",\n \"border-y-indigo-800\",\n \"border-y-indigo-900\",\n \"border-y-indigo-950\",\n \"border-y-violet-50\",\n \"border-y-violet-100\",\n \"border-y-violet-200\",\n \"border-y-violet-300\",\n \"border-y-violet-400\",\n \"border-y-violet-500\",\n \"border-y-violet-600\",\n \"border-y-violet-700\",\n \"border-y-violet-800\",\n \"border-y-violet-900\",\n \"border-y-violet-950\",\n \"border-y-purple-50\",\n \"border-y-purple-100\",\n \"border-y-purple-200\",\n \"border-y-purple-300\",\n \"border-y-purple-400\",\n \"border-y-purple-500\",\n \"border-y-purple-600\",\n \"border-y-purple-700\",\n \"border-y-purple-800\",\n \"border-y-purple-900\",\n \"border-y-purple-950\",\n \"border-y-fuchsia-50\",\n \"border-y-fuchsia-100\",\n \"border-y-fuchsia-200\",\n \"border-y-fuchsia-300\",\n \"border-y-fuchsia-400\",\n \"border-y-fuchsia-500\",\n \"border-y-fuchsia-600\",\n \"border-y-fuchsia-700\",\n \"border-y-fuchsia-800\",\n \"border-y-fuchsia-900\",\n \"border-y-fuchsia-950\",\n \"border-y-pink-50\",\n \"border-y-pink-100\",\n \"border-y-pink-200\",\n \"border-y-pink-300\",\n \"border-y-pink-400\",\n \"border-y-pink-500\",\n \"border-y-pink-600\",\n \"border-y-pink-700\",\n \"border-y-pink-800\",\n \"border-y-pink-900\",\n \"border-y-pink-950\",\n \"border-y-rose-50\",\n \"border-y-rose-100\",\n \"border-y-rose-200\",\n \"border-y-rose-300\",\n \"border-y-rose-400\",\n \"border-y-rose-500\",\n \"border-y-rose-600\",\n \"border-y-rose-700\",\n \"border-y-rose-800\",\n \"border-y-rose-900\",\n \"border-y-rose-950\",\n \"border-s-inherit\",\n \"border-s-current\",\n \"border-s-transparent\",\n \"border-s-black\",\n \"border-s-white\",\n \"border-s-slate-50\",\n \"border-s-slate-100\",\n \"border-s-slate-200\",\n \"border-s-slate-300\",\n \"border-s-slate-400\",\n \"border-s-slate-500\",\n \"border-s-slate-600\",\n \"border-s-slate-700\",\n \"border-s-slate-800\",\n \"border-s-slate-900\",\n \"border-s-slate-950\",\n \"border-s-gray-50\",\n \"border-s-gray-100\",\n \"border-s-gray-200\",\n \"border-s-gray-300\",\n \"border-s-gray-400\",\n \"border-s-gray-500\",\n \"border-s-gray-600\",\n \"border-s-gray-700\",\n \"border-s-gray-800\",\n \"border-s-gray-900\",\n \"border-s-gray-950\",\n \"border-s-zinc-50\",\n \"border-s-zinc-100\",\n \"border-s-zinc-200\",\n \"border-s-zinc-300\",\n \"border-s-zinc-400\",\n \"border-s-zinc-500\",\n \"border-s-zinc-600\",\n \"border-s-zinc-700\",\n \"border-s-zinc-800\",\n \"border-s-zinc-900\",\n \"border-s-zinc-950\",\n \"border-s-neutral-50\",\n \"border-s-neutral-100\",\n \"border-s-neutral-200\",\n \"border-s-neutral-300\",\n \"border-s-neutral-400\",\n \"border-s-neutral-500\",\n \"border-s-neutral-600\",\n \"border-s-neutral-700\",\n \"border-s-neutral-800\",\n \"border-s-neutral-900\",\n \"border-s-neutral-950\",\n \"border-s-stone-50\",\n \"border-s-stone-100\",\n \"border-s-stone-200\",\n \"border-s-stone-300\",\n \"border-s-stone-400\",\n \"border-s-stone-500\",\n \"border-s-stone-600\",\n \"border-s-stone-700\",\n \"border-s-stone-800\",\n \"border-s-stone-900\",\n \"border-s-stone-950\",\n \"border-s-red-50\",\n \"border-s-red-100\",\n \"border-s-red-200\",\n \"border-s-red-300\",\n \"border-s-red-400\",\n \"border-s-red-500\",\n \"border-s-red-600\",\n \"border-s-red-700\",\n \"border-s-red-800\",\n \"border-s-red-900\",\n \"border-s-red-950\",\n \"border-s-orange-50\",\n \"border-s-orange-100\",\n \"border-s-orange-200\",\n \"border-s-orange-300\",\n \"border-s-orange-400\",\n \"border-s-orange-500\",\n \"border-s-orange-600\",\n \"border-s-orange-700\",\n \"border-s-orange-800\",\n \"border-s-orange-900\",\n \"border-s-orange-950\",\n \"border-s-amber-50\",\n \"border-s-amber-100\",\n \"border-s-amber-200\",\n \"border-s-amber-300\",\n \"border-s-amber-400\",\n \"border-s-amber-500\",\n \"border-s-amber-600\",\n \"border-s-amber-700\",\n \"border-s-amber-800\",\n \"border-s-amber-900\",\n \"border-s-amber-950\",\n \"border-s-yellow-50\",\n \"border-s-yellow-100\",\n \"border-s-yellow-200\",\n \"border-s-yellow-300\",\n \"border-s-yellow-400\",\n \"border-s-yellow-500\",\n \"border-s-yellow-600\",\n \"border-s-yellow-700\",\n \"border-s-yellow-800\",\n \"border-s-yellow-900\",\n \"border-s-yellow-950\",\n \"border-s-lime-50\",\n \"border-s-lime-100\",\n \"border-s-lime-200\",\n \"border-s-lime-300\",\n \"border-s-lime-400\",\n \"border-s-lime-500\",\n \"border-s-lime-600\",\n \"border-s-lime-700\",\n \"border-s-lime-800\",\n \"border-s-lime-900\",\n \"border-s-lime-950\",\n \"border-s-green-50\",\n \"border-s-green-100\",\n \"border-s-green-200\",\n \"border-s-green-300\",\n \"border-s-green-400\",\n \"border-s-green-500\",\n \"border-s-green-600\",\n \"border-s-green-700\",\n \"border-s-green-800\",\n \"border-s-green-900\",\n \"border-s-green-950\",\n \"border-s-emerald-50\",\n \"border-s-emerald-100\",\n \"border-s-emerald-200\",\n \"border-s-emerald-300\",\n \"border-s-emerald-400\",\n \"border-s-emerald-500\",\n \"border-s-emerald-600\",\n \"border-s-emerald-700\",\n \"border-s-emerald-800\",\n \"border-s-emerald-900\",\n \"border-s-emerald-950\",\n \"border-s-teal-50\",\n \"border-s-teal-100\",\n \"border-s-teal-200\",\n \"border-s-teal-300\",\n \"border-s-teal-400\",\n \"border-s-teal-500\",\n \"border-s-teal-600\",\n \"border-s-teal-700\",\n \"border-s-teal-800\",\n \"border-s-teal-900\",\n \"border-s-teal-950\",\n \"border-s-cyan-50\",\n \"border-s-cyan-100\",\n \"border-s-cyan-200\",\n \"border-s-cyan-300\",\n \"border-s-cyan-400\",\n \"border-s-cyan-500\",\n \"border-s-cyan-600\",\n \"border-s-cyan-700\",\n \"border-s-cyan-800\",\n \"border-s-cyan-900\",\n \"border-s-cyan-950\",\n \"border-s-sky-50\",\n \"border-s-sky-100\",\n \"border-s-sky-200\",\n \"border-s-sky-300\",\n \"border-s-sky-400\",\n \"border-s-sky-500\",\n \"border-s-sky-600\",\n \"border-s-sky-700\",\n \"border-s-sky-800\",\n \"border-s-sky-900\",\n \"border-s-sky-950\",\n \"border-s-blue-50\",\n \"border-s-blue-100\",\n \"border-s-blue-200\",\n \"border-s-blue-300\",\n \"border-s-blue-400\",\n \"border-s-blue-500\",\n \"border-s-blue-600\",\n \"border-s-blue-700\",\n \"border-s-blue-800\",\n \"border-s-blue-900\",\n \"border-s-blue-950\",\n \"border-s-indigo-50\",\n \"border-s-indigo-100\",\n \"border-s-indigo-200\",\n \"border-s-indigo-300\",\n \"border-s-indigo-400\",\n \"border-s-indigo-500\",\n \"border-s-indigo-600\",\n \"border-s-indigo-700\",\n \"border-s-indigo-800\",\n \"border-s-indigo-900\",\n \"border-s-indigo-950\",\n \"border-s-violet-50\",\n \"border-s-violet-100\",\n \"border-s-violet-200\",\n \"border-s-violet-300\",\n \"border-s-violet-400\",\n \"border-s-violet-500\",\n \"border-s-violet-600\",\n \"border-s-violet-700\",\n \"border-s-violet-800\",\n \"border-s-violet-900\",\n \"border-s-violet-950\",\n \"border-s-purple-50\",\n \"border-s-purple-100\",\n \"border-s-purple-200\",\n \"border-s-purple-300\",\n \"border-s-purple-400\",\n \"border-s-purple-500\",\n \"border-s-purple-600\",\n \"border-s-purple-700\",\n \"border-s-purple-800\",\n \"border-s-purple-900\",\n \"border-s-purple-950\",\n \"border-s-fuchsia-50\",\n \"border-s-fuchsia-100\",\n \"border-s-fuchsia-200\",\n \"border-s-fuchsia-300\",\n \"border-s-fuchsia-400\",\n \"border-s-fuchsia-500\",\n \"border-s-fuchsia-600\",\n \"border-s-fuchsia-700\",\n \"border-s-fuchsia-800\",\n \"border-s-fuchsia-900\",\n \"border-s-fuchsia-950\",\n \"border-s-pink-50\",\n \"border-s-pink-100\",\n \"border-s-pink-200\",\n \"border-s-pink-300\",\n \"border-s-pink-400\",\n \"border-s-pink-500\",\n \"border-s-pink-600\",\n \"border-s-pink-700\",\n \"border-s-pink-800\",\n \"border-s-pink-900\",\n \"border-s-pink-950\",\n \"border-s-rose-50\",\n \"border-s-rose-100\",\n \"border-s-rose-200\",\n \"border-s-rose-300\",\n \"border-s-rose-400\",\n \"border-s-rose-500\",\n \"border-s-rose-600\",\n \"border-s-rose-700\",\n \"border-s-rose-800\",\n \"border-s-rose-900\",\n \"border-s-rose-950\",\n \"border-e-inherit\",\n \"border-e-current\",\n \"border-e-transparent\",\n \"border-e-black\",\n \"border-e-white\",\n \"border-e-slate-50\",\n \"border-e-slate-100\",\n \"border-e-slate-200\",\n \"border-e-slate-300\",\n \"border-e-slate-400\",\n \"border-e-slate-500\",\n \"border-e-slate-600\",\n \"border-e-slate-700\",\n \"border-e-slate-800\",\n \"border-e-slate-900\",\n \"border-e-slate-950\",\n \"border-e-gray-50\",\n \"border-e-gray-100\",\n \"border-e-gray-200\",\n \"border-e-gray-300\",\n \"border-e-gray-400\",\n \"border-e-gray-500\",\n \"border-e-gray-600\",\n \"border-e-gray-700\",\n \"border-e-gray-800\",\n \"border-e-gray-900\",\n \"border-e-gray-950\",\n \"border-e-zinc-50\",\n \"border-e-zinc-100\",\n \"border-e-zinc-200\",\n \"border-e-zinc-300\",\n \"border-e-zinc-400\",\n \"border-e-zinc-500\",\n \"border-e-zinc-600\",\n \"border-e-zinc-700\",\n \"border-e-zinc-800\",\n \"border-e-zinc-900\",\n \"border-e-zinc-950\",\n \"border-e-neutral-50\",\n \"border-e-neutral-100\",\n \"border-e-neutral-200\",\n \"border-e-neutral-300\",\n \"border-e-neutral-400\",\n \"border-e-neutral-500\",\n \"border-e-neutral-600\",\n \"border-e-neutral-700\",\n \"border-e-neutral-800\",\n \"border-e-neutral-900\",\n \"border-e-neutral-950\",\n \"border-e-stone-50\",\n \"border-e-stone-100\",\n \"border-e-stone-200\",\n \"border-e-stone-300\",\n \"border-e-stone-400\",\n \"border-e-stone-500\",\n \"border-e-stone-600\",\n \"border-e-stone-700\",\n \"border-e-stone-800\",\n \"border-e-stone-900\",\n \"border-e-stone-950\",\n \"border-e-red-50\",\n \"border-e-red-100\",\n \"border-e-red-200\",\n \"border-e-red-300\",\n \"border-e-red-400\",\n \"border-e-red-500\",\n \"border-e-red-600\",\n \"border-e-red-700\",\n \"border-e-red-800\",\n \"border-e-red-900\",\n \"border-e-red-950\",\n \"border-e-orange-50\",\n \"border-e-orange-100\",\n \"border-e-orange-200\",\n \"border-e-orange-300\",\n \"border-e-orange-400\",\n \"border-e-orange-500\",\n \"border-e-orange-600\",\n \"border-e-orange-700\",\n \"border-e-orange-800\",\n \"border-e-orange-900\",\n \"border-e-orange-950\",\n \"border-e-amber-50\",\n \"border-e-amber-100\",\n \"border-e-amber-200\",\n \"border-e-amber-300\",\n \"border-e-amber-400\",\n \"border-e-amber-500\",\n \"border-e-amber-600\",\n \"border-e-amber-700\",\n \"border-e-amber-800\",\n \"border-e-amber-900\",\n \"border-e-amber-950\",\n \"border-e-yellow-50\",\n \"border-e-yellow-100\",\n \"border-e-yellow-200\",\n \"border-e-yellow-300\",\n \"border-e-yellow-400\",\n \"border-e-yellow-500\",\n \"border-e-yellow-600\",\n \"border-e-yellow-700\",\n \"border-e-yellow-800\",\n \"border-e-yellow-900\",\n \"border-e-yellow-950\",\n \"border-e-lime-50\",\n \"border-e-lime-100\",\n \"border-e-lime-200\",\n \"border-e-lime-300\",\n \"border-e-lime-400\",\n \"border-e-lime-500\",\n \"border-e-lime-600\",\n \"border-e-lime-700\",\n \"border-e-lime-800\",\n \"border-e-lime-900\",\n \"border-e-lime-950\",\n \"border-e-green-50\",\n \"border-e-green-100\",\n \"border-e-green-200\",\n \"border-e-green-300\",\n \"border-e-green-400\",\n \"border-e-green-500\",\n \"border-e-green-600\",\n \"border-e-green-700\",\n \"border-e-green-800\",\n \"border-e-green-900\",\n \"border-e-green-950\",\n \"border-e-emerald-50\",\n \"border-e-emerald-100\",\n \"border-e-emerald-200\",\n \"border-e-emerald-300\",\n \"border-e-emerald-400\",\n \"border-e-emerald-500\",\n \"border-e-emerald-600\",\n \"border-e-emerald-700\",\n \"border-e-emerald-800\",\n \"border-e-emerald-900\",\n \"border-e-emerald-950\",\n \"border-e-teal-50\",\n \"border-e-teal-100\",\n \"border-e-teal-200\",\n \"border-e-teal-300\",\n \"border-e-teal-400\",\n \"border-e-teal-500\",\n \"border-e-teal-600\",\n \"border-e-teal-700\",\n \"border-e-teal-800\",\n \"border-e-teal-900\",\n \"border-e-teal-950\",\n \"border-e-cyan-50\",\n \"border-e-cyan-100\",\n \"border-e-cyan-200\",\n \"border-e-cyan-300\",\n \"border-e-cyan-400\",\n \"border-e-cyan-500\",\n \"border-e-cyan-600\",\n \"border-e-cyan-700\",\n \"border-e-cyan-800\",\n \"border-e-cyan-900\",\n \"border-e-cyan-950\",\n \"border-e-sky-50\",\n \"border-e-sky-100\",\n \"border-e-sky-200\",\n \"border-e-sky-300\",\n \"border-e-sky-400\",\n \"border-e-sky-500\",\n \"border-e-sky-600\",\n \"border-e-sky-700\",\n \"border-e-sky-800\",\n \"border-e-sky-900\",\n \"border-e-sky-950\",\n \"border-e-blue-50\",\n \"border-e-blue-100\",\n \"border-e-blue-200\",\n \"border-e-blue-300\",\n \"border-e-blue-400\",\n \"border-e-blue-500\",\n \"border-e-blue-600\",\n \"border-e-blue-700\",\n \"border-e-blue-800\",\n \"border-e-blue-900\",\n \"border-e-blue-950\",\n \"border-e-indigo-50\",\n \"border-e-indigo-100\",\n \"border-e-indigo-200\",\n \"border-e-indigo-300\",\n \"border-e-indigo-400\",\n \"border-e-indigo-500\",\n \"border-e-indigo-600\",\n \"border-e-indigo-700\",\n \"border-e-indigo-800\",\n \"border-e-indigo-900\",\n \"border-e-indigo-950\",\n \"border-e-violet-50\",\n \"border-e-violet-100\",\n \"border-e-violet-200\",\n \"border-e-violet-300\",\n \"border-e-violet-400\",\n \"border-e-violet-500\",\n \"border-e-violet-600\",\n \"border-e-violet-700\",\n \"border-e-violet-800\",\n \"border-e-violet-900\",\n \"border-e-violet-950\",\n \"border-e-purple-50\",\n \"border-e-purple-100\",\n \"border-e-purple-200\",\n \"border-e-purple-300\",\n \"border-e-purple-400\",\n \"border-e-purple-500\",\n \"border-e-purple-600\",\n \"border-e-purple-700\",\n \"border-e-purple-800\",\n \"border-e-purple-900\",\n \"border-e-purple-950\",\n \"border-e-fuchsia-50\",\n \"border-e-fuchsia-100\",\n \"border-e-fuchsia-200\",\n \"border-e-fuchsia-300\",\n \"border-e-fuchsia-400\",\n \"border-e-fuchsia-500\",\n \"border-e-fuchsia-600\",\n \"border-e-fuchsia-700\",\n \"border-e-fuchsia-800\",\n \"border-e-fuchsia-900\",\n \"border-e-fuchsia-950\",\n \"border-e-pink-50\",\n \"border-e-pink-100\",\n \"border-e-pink-200\",\n \"border-e-pink-300\",\n \"border-e-pink-400\",\n \"border-e-pink-500\",\n \"border-e-pink-600\",\n \"border-e-pink-700\",\n \"border-e-pink-800\",\n \"border-e-pink-900\",\n \"border-e-pink-950\",\n \"border-e-rose-50\",\n \"border-e-rose-100\",\n \"border-e-rose-200\",\n \"border-e-rose-300\",\n \"border-e-rose-400\",\n \"border-e-rose-500\",\n \"border-e-rose-600\",\n \"border-e-rose-700\",\n \"border-e-rose-800\",\n \"border-e-rose-900\",\n \"border-e-rose-950\",\n \"border-t-inherit\",\n \"border-t-current\",\n \"border-t-transparent\",\n \"border-t-black\",\n \"border-t-white\",\n \"border-t-slate-50\",\n \"border-t-slate-100\",\n \"border-t-slate-200\",\n \"border-t-slate-300\",\n \"border-t-slate-400\",\n \"border-t-slate-500\",\n \"border-t-slate-600\",\n \"border-t-slate-700\",\n \"border-t-slate-800\",\n \"border-t-slate-900\",\n \"border-t-slate-950\",\n \"border-t-gray-50\",\n \"border-t-gray-100\",\n \"border-t-gray-200\",\n \"border-t-gray-300\",\n \"border-t-gray-400\",\n \"border-t-gray-500\",\n \"border-t-gray-600\",\n \"border-t-gray-700\",\n \"border-t-gray-800\",\n \"border-t-gray-900\",\n \"border-t-gray-950\",\n \"border-t-zinc-50\",\n \"border-t-zinc-100\",\n \"border-t-zinc-200\",\n \"border-t-zinc-300\",\n \"border-t-zinc-400\",\n \"border-t-zinc-500\",\n \"border-t-zinc-600\",\n \"border-t-zinc-700\",\n \"border-t-zinc-800\",\n \"border-t-zinc-900\",\n \"border-t-zinc-950\",\n \"border-t-neutral-50\",\n \"border-t-neutral-100\",\n \"border-t-neutral-200\",\n \"border-t-neutral-300\",\n \"border-t-neutral-400\",\n \"border-t-neutral-500\",\n \"border-t-neutral-600\",\n \"border-t-neutral-700\",\n \"border-t-neutral-800\",\n \"border-t-neutral-900\",\n \"border-t-neutral-950\",\n \"border-t-stone-50\",\n \"border-t-stone-100\",\n \"border-t-stone-200\",\n \"border-t-stone-300\",\n \"border-t-stone-400\",\n \"border-t-stone-500\",\n \"border-t-stone-600\",\n \"border-t-stone-700\",\n \"border-t-stone-800\",\n \"border-t-stone-900\",\n \"border-t-stone-950\",\n \"border-t-red-50\",\n \"border-t-red-100\",\n \"border-t-red-200\",\n \"border-t-red-300\",\n \"border-t-red-400\",\n \"border-t-red-500\",\n \"border-t-red-600\",\n \"border-t-red-700\",\n \"border-t-red-800\",\n \"border-t-red-900\",\n \"border-t-red-950\",\n \"border-t-orange-50\",\n \"border-t-orange-100\",\n \"border-t-orange-200\",\n \"border-t-orange-300\",\n \"border-t-orange-400\",\n \"border-t-orange-500\",\n \"border-t-orange-600\",\n \"border-t-orange-700\",\n \"border-t-orange-800\",\n \"border-t-orange-900\",\n \"border-t-orange-950\",\n \"border-t-amber-50\",\n \"border-t-amber-100\",\n \"border-t-amber-200\",\n \"border-t-amber-300\",\n \"border-t-amber-400\",\n \"border-t-amber-500\",\n \"border-t-amber-600\",\n \"border-t-amber-700\",\n \"border-t-amber-800\",\n \"border-t-amber-900\",\n \"border-t-amber-950\",\n \"border-t-yellow-50\",\n \"border-t-yellow-100\",\n \"border-t-yellow-200\",\n \"border-t-yellow-300\",\n \"border-t-yellow-400\",\n \"border-t-yellow-500\",\n \"border-t-yellow-600\",\n \"border-t-yellow-700\",\n \"border-t-yellow-800\",\n \"border-t-yellow-900\",\n \"border-t-yellow-950\",\n \"border-t-lime-50\",\n \"border-t-lime-100\",\n \"border-t-lime-200\",\n \"border-t-lime-300\",\n \"border-t-lime-400\",\n \"border-t-lime-500\",\n \"border-t-lime-600\",\n \"border-t-lime-700\",\n \"border-t-lime-800\",\n \"border-t-lime-900\",\n \"border-t-lime-950\",\n \"border-t-green-50\",\n \"border-t-green-100\",\n \"border-t-green-200\",\n \"border-t-green-300\",\n \"border-t-green-400\",\n \"border-t-green-500\",\n \"border-t-green-600\",\n \"border-t-green-700\",\n \"border-t-green-800\",\n \"border-t-green-900\",\n \"border-t-green-950\",\n \"border-t-emerald-50\",\n \"border-t-emerald-100\",\n \"border-t-emerald-200\",\n \"border-t-emerald-300\",\n \"border-t-emerald-400\",\n \"border-t-emerald-500\",\n \"border-t-emerald-600\",\n \"border-t-emerald-700\",\n \"border-t-emerald-800\",\n \"border-t-emerald-900\",\n \"border-t-emerald-950\",\n \"border-t-teal-50\",\n \"border-t-teal-100\",\n \"border-t-teal-200\",\n \"border-t-teal-300\",\n \"border-t-teal-400\",\n \"border-t-teal-500\",\n \"border-t-teal-600\",\n \"border-t-teal-700\",\n \"border-t-teal-800\",\n \"border-t-teal-900\",\n \"border-t-teal-950\",\n \"border-t-cyan-50\",\n \"border-t-cyan-100\",\n \"border-t-cyan-200\",\n \"border-t-cyan-300\",\n \"border-t-cyan-400\",\n \"border-t-cyan-500\",\n \"border-t-cyan-600\",\n \"border-t-cyan-700\",\n \"border-t-cyan-800\",\n \"border-t-cyan-900\",\n \"border-t-cyan-950\",\n \"border-t-sky-50\",\n \"border-t-sky-100\",\n \"border-t-sky-200\",\n \"border-t-sky-300\",\n \"border-t-sky-400\",\n \"border-t-sky-500\",\n \"border-t-sky-600\",\n \"border-t-sky-700\",\n \"border-t-sky-800\",\n \"border-t-sky-900\",\n \"border-t-sky-950\",\n \"border-t-blue-50\",\n \"border-t-blue-100\",\n \"border-t-blue-200\",\n \"border-t-blue-300\",\n \"border-t-blue-400\",\n \"border-t-blue-500\",\n \"border-t-blue-600\",\n \"border-t-blue-700\",\n \"border-t-blue-800\",\n \"border-t-blue-900\",\n \"border-t-blue-950\",\n \"border-t-indigo-50\",\n \"border-t-indigo-100\",\n \"border-t-indigo-200\",\n \"border-t-indigo-300\",\n \"border-t-indigo-400\",\n \"border-t-indigo-500\",\n \"border-t-indigo-600\",\n \"border-t-indigo-700\",\n \"border-t-indigo-800\",\n \"border-t-indigo-900\",\n \"border-t-indigo-950\",\n \"border-t-violet-50\",\n \"border-t-violet-100\",\n \"border-t-violet-200\",\n \"border-t-violet-300\",\n \"border-t-violet-400\",\n \"border-t-violet-500\",\n \"border-t-violet-600\",\n \"border-t-violet-700\",\n \"border-t-violet-800\",\n \"border-t-violet-900\",\n \"border-t-violet-950\",\n \"border-t-purple-50\",\n \"border-t-purple-100\",\n \"border-t-purple-200\",\n \"border-t-purple-300\",\n \"border-t-purple-400\",\n \"border-t-purple-500\",\n \"border-t-purple-600\",\n \"border-t-purple-700\",\n \"border-t-purple-800\",\n \"border-t-purple-900\",\n \"border-t-purple-950\",\n \"border-t-fuchsia-50\",\n \"border-t-fuchsia-100\",\n \"border-t-fuchsia-200\",\n \"border-t-fuchsia-300\",\n \"border-t-fuchsia-400\",\n \"border-t-fuchsia-500\",\n \"border-t-fuchsia-600\",\n \"border-t-fuchsia-700\",\n \"border-t-fuchsia-800\",\n \"border-t-fuchsia-900\",\n \"border-t-fuchsia-950\",\n \"border-t-pink-50\",\n \"border-t-pink-100\",\n \"border-t-pink-200\",\n \"border-t-pink-300\",\n \"border-t-pink-400\",\n \"border-t-pink-500\",\n \"border-t-pink-600\",\n \"border-t-pink-700\",\n \"border-t-pink-800\",\n \"border-t-pink-900\",\n \"border-t-pink-950\",\n \"border-t-rose-50\",\n \"border-t-rose-100\",\n \"border-t-rose-200\",\n \"border-t-rose-300\",\n \"border-t-rose-400\",\n \"border-t-rose-500\",\n \"border-t-rose-600\",\n \"border-t-rose-700\",\n \"border-t-rose-800\",\n \"border-t-rose-900\",\n \"border-t-rose-950\",\n \"border-r-inherit\",\n \"border-r-current\",\n \"border-r-transparent\",\n \"border-r-black\",\n \"border-r-white\",\n \"border-r-slate-50\",\n \"border-r-slate-100\",\n \"border-r-slate-200\",\n \"border-r-slate-300\",\n \"border-r-slate-400\",\n \"border-r-slate-500\",\n \"border-r-slate-600\",\n \"border-r-slate-700\",\n \"border-r-slate-800\",\n \"border-r-slate-900\",\n \"border-r-slate-950\",\n \"border-r-gray-50\",\n \"border-r-gray-100\",\n \"border-r-gray-200\",\n \"border-r-gray-300\",\n \"border-r-gray-400\",\n \"border-r-gray-500\",\n \"border-r-gray-600\",\n \"border-r-gray-700\",\n \"border-r-gray-800\",\n \"border-r-gray-900\",\n \"border-r-gray-950\",\n \"border-r-zinc-50\",\n \"border-r-zinc-100\",\n \"border-r-zinc-200\",\n \"border-r-zinc-300\",\n \"border-r-zinc-400\",\n \"border-r-zinc-500\",\n \"border-r-zinc-600\",\n \"border-r-zinc-700\",\n \"border-r-zinc-800\",\n \"border-r-zinc-900\",\n \"border-r-zinc-950\",\n \"border-r-neutral-50\",\n \"border-r-neutral-100\",\n \"border-r-neutral-200\",\n \"border-r-neutral-300\",\n \"border-r-neutral-400\",\n \"border-r-neutral-500\",\n \"border-r-neutral-600\",\n \"border-r-neutral-700\",\n \"border-r-neutral-800\",\n \"border-r-neutral-900\",\n \"border-r-neutral-950\",\n \"border-r-stone-50\",\n \"border-r-stone-100\",\n \"border-r-stone-200\",\n \"border-r-stone-300\",\n \"border-r-stone-400\",\n \"border-r-stone-500\",\n \"border-r-stone-600\",\n \"border-r-stone-700\",\n \"border-r-stone-800\",\n \"border-r-stone-900\",\n \"border-r-stone-950\",\n \"border-r-red-50\",\n \"border-r-red-100\",\n \"border-r-red-200\",\n \"border-r-red-300\",\n \"border-r-red-400\",\n \"border-r-red-500\",\n \"border-r-red-600\",\n \"border-r-red-700\",\n \"border-r-red-800\",\n \"border-r-red-900\",\n \"border-r-red-950\",\n \"border-r-orange-50\",\n \"border-r-orange-100\",\n \"border-r-orange-200\",\n \"border-r-orange-300\",\n \"border-r-orange-400\",\n \"border-r-orange-500\",\n \"border-r-orange-600\",\n \"border-r-orange-700\",\n \"border-r-orange-800\",\n \"border-r-orange-900\",\n \"border-r-orange-950\",\n \"border-r-amber-50\",\n \"border-r-amber-100\",\n \"border-r-amber-200\",\n \"border-r-amber-300\",\n \"border-r-amber-400\",\n \"border-r-amber-500\",\n \"border-r-amber-600\",\n \"border-r-amber-700\",\n \"border-r-amber-800\",\n \"border-r-amber-900\",\n \"border-r-amber-950\",\n \"border-r-yellow-50\",\n \"border-r-yellow-100\",\n \"border-r-yellow-200\",\n \"border-r-yellow-300\",\n \"border-r-yellow-400\",\n \"border-r-yellow-500\",\n \"border-r-yellow-600\",\n \"border-r-yellow-700\",\n \"border-r-yellow-800\",\n \"border-r-yellow-900\",\n \"border-r-yellow-950\",\n \"border-r-lime-50\",\n \"border-r-lime-100\",\n \"border-r-lime-200\",\n \"border-r-lime-300\",\n \"border-r-lime-400\",\n \"border-r-lime-500\",\n \"border-r-lime-600\",\n \"border-r-lime-700\",\n \"border-r-lime-800\",\n \"border-r-lime-900\",\n \"border-r-lime-950\",\n \"border-r-green-50\",\n \"border-r-green-100\",\n \"border-r-green-200\",\n \"border-r-green-300\",\n \"border-r-green-400\",\n \"border-r-green-500\",\n \"border-r-green-600\",\n \"border-r-green-700\",\n \"border-r-green-800\",\n \"border-r-green-900\",\n \"border-r-green-950\",\n \"border-r-emerald-50\",\n \"border-r-emerald-100\",\n \"border-r-emerald-200\",\n \"border-r-emerald-300\",\n \"border-r-emerald-400\",\n \"border-r-emerald-500\",\n \"border-r-emerald-600\",\n \"border-r-emerald-700\",\n \"border-r-emerald-800\",\n \"border-r-emerald-900\",\n \"border-r-emerald-950\",\n \"border-r-teal-50\",\n \"border-r-teal-100\",\n \"border-r-teal-200\",\n \"border-r-teal-300\",\n \"border-r-teal-400\",\n \"border-r-teal-500\",\n \"border-r-teal-600\",\n \"border-r-teal-700\",\n \"border-r-teal-800\",\n \"border-r-teal-900\",\n \"border-r-teal-950\",\n \"border-r-cyan-50\",\n \"border-r-cyan-100\",\n \"border-r-cyan-200\",\n \"border-r-cyan-300\",\n \"border-r-cyan-400\",\n \"border-r-cyan-500\",\n \"border-r-cyan-600\",\n \"border-r-cyan-700\",\n \"border-r-cyan-800\",\n \"border-r-cyan-900\",\n \"border-r-cyan-950\",\n \"border-r-sky-50\",\n \"border-r-sky-100\",\n \"border-r-sky-200\",\n \"border-r-sky-300\",\n \"border-r-sky-400\",\n \"border-r-sky-500\",\n \"border-r-sky-600\",\n \"border-r-sky-700\",\n \"border-r-sky-800\",\n \"border-r-sky-900\",\n \"border-r-sky-950\",\n \"border-r-blue-50\",\n \"border-r-blue-100\",\n \"border-r-blue-200\",\n \"border-r-blue-300\",\n \"border-r-blue-400\",\n \"border-r-blue-500\",\n \"border-r-blue-600\",\n \"border-r-blue-700\",\n \"border-r-blue-800\",\n \"border-r-blue-900\",\n \"border-r-blue-950\",\n \"border-r-indigo-50\",\n \"border-r-indigo-100\",\n \"border-r-indigo-200\",\n \"border-r-indigo-300\",\n \"border-r-indigo-400\",\n \"border-r-indigo-500\",\n \"border-r-indigo-600\",\n \"border-r-indigo-700\",\n \"border-r-indigo-800\",\n \"border-r-indigo-900\",\n \"border-r-indigo-950\",\n \"border-r-violet-50\",\n \"border-r-violet-100\",\n \"border-r-violet-200\",\n \"border-r-violet-300\",\n \"border-r-violet-400\",\n \"border-r-violet-500\",\n \"border-r-violet-600\",\n \"border-r-violet-700\",\n \"border-r-violet-800\",\n \"border-r-violet-900\",\n \"border-r-violet-950\",\n \"border-r-purple-50\",\n \"border-r-purple-100\",\n \"border-r-purple-200\",\n \"border-r-purple-300\",\n \"border-r-purple-400\",\n \"border-r-purple-500\",\n \"border-r-purple-600\",\n \"border-r-purple-700\",\n \"border-r-purple-800\",\n \"border-r-purple-900\",\n \"border-r-purple-950\",\n \"border-r-fuchsia-50\",\n \"border-r-fuchsia-100\",\n \"border-r-fuchsia-200\",\n \"border-r-fuchsia-300\",\n \"border-r-fuchsia-400\",\n \"border-r-fuchsia-500\",\n \"border-r-fuchsia-600\",\n \"border-r-fuchsia-700\",\n \"border-r-fuchsia-800\",\n \"border-r-fuchsia-900\",\n \"border-r-fuchsia-950\",\n \"border-r-pink-50\",\n \"border-r-pink-100\",\n \"border-r-pink-200\",\n \"border-r-pink-300\",\n \"border-r-pink-400\",\n \"border-r-pink-500\",\n \"border-r-pink-600\",\n \"border-r-pink-700\",\n \"border-r-pink-800\",\n \"border-r-pink-900\",\n \"border-r-pink-950\",\n \"border-r-rose-50\",\n \"border-r-rose-100\",\n \"border-r-rose-200\",\n \"border-r-rose-300\",\n \"border-r-rose-400\",\n \"border-r-rose-500\",\n \"border-r-rose-600\",\n \"border-r-rose-700\",\n \"border-r-rose-800\",\n \"border-r-rose-900\",\n \"border-r-rose-950\",\n \"border-b-inherit\",\n \"border-b-current\",\n \"border-b-transparent\",\n \"border-b-black\",\n \"border-b-white\",\n \"border-b-slate-50\",\n \"border-b-slate-100\",\n \"border-b-slate-200\",\n \"border-b-slate-300\",\n \"border-b-slate-400\",\n \"border-b-slate-500\",\n \"border-b-slate-600\",\n \"border-b-slate-700\",\n \"border-b-slate-800\",\n \"border-b-slate-900\",\n \"border-b-slate-950\",\n \"border-b-gray-50\",\n \"border-b-gray-100\",\n \"border-b-gray-200\",\n \"border-b-gray-300\",\n \"border-b-gray-400\",\n \"border-b-gray-500\",\n \"border-b-gray-600\",\n \"border-b-gray-700\",\n \"border-b-gray-800\",\n \"border-b-gray-900\",\n \"border-b-gray-950\",\n \"border-b-zinc-50\",\n \"border-b-zinc-100\",\n \"border-b-zinc-200\",\n \"border-b-zinc-300\",\n \"border-b-zinc-400\",\n \"border-b-zinc-500\",\n \"border-b-zinc-600\",\n \"border-b-zinc-700\",\n \"border-b-zinc-800\",\n \"border-b-zinc-900\",\n \"border-b-zinc-950\",\n \"border-b-neutral-50\",\n \"border-b-neutral-100\",\n \"border-b-neutral-200\",\n \"border-b-neutral-300\",\n \"border-b-neutral-400\",\n \"border-b-neutral-500\",\n \"border-b-neutral-600\",\n \"border-b-neutral-700\",\n \"border-b-neutral-800\",\n \"border-b-neutral-900\",\n \"border-b-neutral-950\",\n \"border-b-stone-50\",\n \"border-b-stone-100\",\n \"border-b-stone-200\",\n \"border-b-stone-300\",\n \"border-b-stone-400\",\n \"border-b-stone-500\",\n \"border-b-stone-600\",\n \"border-b-stone-700\",\n \"border-b-stone-800\",\n \"border-b-stone-900\",\n \"border-b-stone-950\",\n \"border-b-red-50\",\n \"border-b-red-100\",\n \"border-b-red-200\",\n \"border-b-red-300\",\n \"border-b-red-400\",\n \"border-b-red-500\",\n \"border-b-red-600\",\n \"border-b-red-700\",\n \"border-b-red-800\",\n \"border-b-red-900\",\n \"border-b-red-950\",\n \"border-b-orange-50\",\n \"border-b-orange-100\",\n \"border-b-orange-200\",\n \"border-b-orange-300\",\n \"border-b-orange-400\",\n \"border-b-orange-500\",\n \"border-b-orange-600\",\n \"border-b-orange-700\",\n \"border-b-orange-800\",\n \"border-b-orange-900\",\n \"border-b-orange-950\",\n \"border-b-amber-50\",\n \"border-b-amber-100\",\n \"border-b-amber-200\",\n \"border-b-amber-300\",\n \"border-b-amber-400\",\n \"border-b-amber-500\",\n \"border-b-amber-600\",\n \"border-b-amber-700\",\n \"border-b-amber-800\",\n \"border-b-amber-900\",\n \"border-b-amber-950\",\n \"border-b-yellow-50\",\n \"border-b-yellow-100\",\n \"border-b-yellow-200\",\n \"border-b-yellow-300\",\n \"border-b-yellow-400\",\n \"border-b-yellow-500\",\n \"border-b-yellow-600\",\n \"border-b-yellow-700\",\n \"border-b-yellow-800\",\n \"border-b-yellow-900\",\n \"border-b-yellow-950\",\n \"border-b-lime-50\",\n \"border-b-lime-100\",\n \"border-b-lime-200\",\n \"border-b-lime-300\",\n \"border-b-lime-400\",\n \"border-b-lime-500\",\n \"border-b-lime-600\",\n \"border-b-lime-700\",\n \"border-b-lime-800\",\n \"border-b-lime-900\",\n \"border-b-lime-950\",\n \"border-b-green-50\",\n \"border-b-green-100\",\n \"border-b-green-200\",\n \"border-b-green-300\",\n \"border-b-green-400\",\n \"border-b-green-500\",\n \"border-b-green-600\",\n \"border-b-green-700\",\n \"border-b-green-800\",\n \"border-b-green-900\",\n \"border-b-green-950\",\n \"border-b-emerald-50\",\n \"border-b-emerald-100\",\n \"border-b-emerald-200\",\n \"border-b-emerald-300\",\n \"border-b-emerald-400\",\n \"border-b-emerald-500\",\n \"border-b-emerald-600\",\n \"border-b-emerald-700\",\n \"border-b-emerald-800\",\n \"border-b-emerald-900\",\n \"border-b-emerald-950\",\n \"border-b-teal-50\",\n \"border-b-teal-100\",\n \"border-b-teal-200\",\n \"border-b-teal-300\",\n \"border-b-teal-400\",\n \"border-b-teal-500\",\n \"border-b-teal-600\",\n \"border-b-teal-700\",\n \"border-b-teal-800\",\n \"border-b-teal-900\",\n \"border-b-teal-950\",\n \"border-b-cyan-50\",\n \"border-b-cyan-100\",\n \"border-b-cyan-200\",\n \"border-b-cyan-300\",\n \"border-b-cyan-400\",\n \"border-b-cyan-500\",\n \"border-b-cyan-600\",\n \"border-b-cyan-700\",\n \"border-b-cyan-800\",\n \"border-b-cyan-900\",\n \"border-b-cyan-950\",\n \"border-b-sky-50\",\n \"border-b-sky-100\",\n \"border-b-sky-200\",\n \"border-b-sky-300\",\n \"border-b-sky-400\",\n \"border-b-sky-500\",\n \"border-b-sky-600\",\n \"border-b-sky-700\",\n \"border-b-sky-800\",\n \"border-b-sky-900\",\n \"border-b-sky-950\",\n \"border-b-blue-50\",\n \"border-b-blue-100\",\n \"border-b-blue-200\",\n \"border-b-blue-300\",\n \"border-b-blue-400\",\n \"border-b-blue-500\",\n \"border-b-blue-600\",\n \"border-b-blue-700\",\n \"border-b-blue-800\",\n \"border-b-blue-900\",\n \"border-b-blue-950\",\n \"border-b-indigo-50\",\n \"border-b-indigo-100\",\n \"border-b-indigo-200\",\n \"border-b-indigo-300\",\n \"border-b-indigo-400\",\n \"border-b-indigo-500\",\n \"border-b-indigo-600\",\n \"border-b-indigo-700\",\n \"border-b-indigo-800\",\n \"border-b-indigo-900\",\n \"border-b-indigo-950\",\n \"border-b-violet-50\",\n \"border-b-violet-100\",\n \"border-b-violet-200\",\n \"border-b-violet-300\",\n \"border-b-violet-400\",\n \"border-b-violet-500\",\n \"border-b-violet-600\",\n \"border-b-violet-700\",\n \"border-b-violet-800\",\n \"border-b-violet-900\",\n \"border-b-violet-950\",\n \"border-b-purple-50\",\n \"border-b-purple-100\",\n \"border-b-purple-200\",\n \"border-b-purple-300\",\n \"border-b-purple-400\",\n \"border-b-purple-500\",\n \"border-b-purple-600\",\n \"border-b-purple-700\",\n \"border-b-purple-800\",\n \"border-b-purple-900\",\n \"border-b-purple-950\",\n \"border-b-fuchsia-50\",\n \"border-b-fuchsia-100\",\n \"border-b-fuchsia-200\",\n \"border-b-fuchsia-300\",\n \"border-b-fuchsia-400\",\n \"border-b-fuchsia-500\",\n \"border-b-fuchsia-600\",\n \"border-b-fuchsia-700\",\n \"border-b-fuchsia-800\",\n \"border-b-fuchsia-900\",\n \"border-b-fuchsia-950\",\n \"border-b-pink-50\",\n \"border-b-pink-100\",\n \"border-b-pink-200\",\n \"border-b-pink-300\",\n \"border-b-pink-400\",\n \"border-b-pink-500\",\n \"border-b-pink-600\",\n \"border-b-pink-700\",\n \"border-b-pink-800\",\n \"border-b-pink-900\",\n \"border-b-pink-950\",\n \"border-b-rose-50\",\n \"border-b-rose-100\",\n \"border-b-rose-200\",\n \"border-b-rose-300\",\n \"border-b-rose-400\",\n \"border-b-rose-500\",\n \"border-b-rose-600\",\n \"border-b-rose-700\",\n \"border-b-rose-800\",\n \"border-b-rose-900\",\n \"border-b-rose-950\",\n \"border-l-inherit\",\n \"border-l-current\",\n \"border-l-transparent\",\n \"border-l-black\",\n \"border-l-white\",\n \"border-l-slate-50\",\n \"border-l-slate-100\",\n \"border-l-slate-200\",\n \"border-l-slate-300\",\n \"border-l-slate-400\",\n \"border-l-slate-500\",\n \"border-l-slate-600\",\n \"border-l-slate-700\",\n \"border-l-slate-800\",\n \"border-l-slate-900\",\n \"border-l-slate-950\",\n \"border-l-gray-50\",\n \"border-l-gray-100\",\n \"border-l-gray-200\",\n \"border-l-gray-300\",\n \"border-l-gray-400\",\n \"border-l-gray-500\",\n \"border-l-gray-600\",\n \"border-l-gray-700\",\n \"border-l-gray-800\",\n \"border-l-gray-900\",\n \"border-l-gray-950\",\n \"border-l-zinc-50\",\n \"border-l-zinc-100\",\n \"border-l-zinc-200\",\n \"border-l-zinc-300\",\n \"border-l-zinc-400\",\n \"border-l-zinc-500\",\n \"border-l-zinc-600\",\n \"border-l-zinc-700\",\n \"border-l-zinc-800\",\n \"border-l-zinc-900\",\n \"border-l-zinc-950\",\n \"border-l-neutral-50\",\n \"border-l-neutral-100\",\n \"border-l-neutral-200\",\n \"border-l-neutral-300\",\n \"border-l-neutral-400\",\n \"border-l-neutral-500\",\n \"border-l-neutral-600\",\n \"border-l-neutral-700\",\n \"border-l-neutral-800\",\n \"border-l-neutral-900\",\n \"border-l-neutral-950\",\n \"border-l-stone-50\",\n \"border-l-stone-100\",\n \"border-l-stone-200\",\n \"border-l-stone-300\",\n \"border-l-stone-400\",\n \"border-l-stone-500\",\n \"border-l-stone-600\",\n \"border-l-stone-700\",\n \"border-l-stone-800\",\n \"border-l-stone-900\",\n \"border-l-stone-950\",\n \"border-l-red-50\",\n \"border-l-red-100\",\n \"border-l-red-200\",\n \"border-l-red-300\",\n \"border-l-red-400\",\n \"border-l-red-500\",\n \"border-l-red-600\",\n \"border-l-red-700\",\n \"border-l-red-800\",\n \"border-l-red-900\",\n \"border-l-red-950\",\n \"border-l-orange-50\",\n \"border-l-orange-100\",\n \"border-l-orange-200\",\n \"border-l-orange-300\",\n \"border-l-orange-400\",\n \"border-l-orange-500\",\n \"border-l-orange-600\",\n \"border-l-orange-700\",\n \"border-l-orange-800\",\n \"border-l-orange-900\",\n \"border-l-orange-950\",\n \"border-l-amber-50\",\n \"border-l-amber-100\",\n \"border-l-amber-200\",\n \"border-l-amber-300\",\n \"border-l-amber-400\",\n \"border-l-amber-500\",\n \"border-l-amber-600\",\n \"border-l-amber-700\",\n \"border-l-amber-800\",\n \"border-l-amber-900\",\n \"border-l-amber-950\",\n \"border-l-yellow-50\",\n \"border-l-yellow-100\",\n \"border-l-yellow-200\",\n \"border-l-yellow-300\",\n \"border-l-yellow-400\",\n \"border-l-yellow-500\",\n \"border-l-yellow-600\",\n \"border-l-yellow-700\",\n \"border-l-yellow-800\",\n \"border-l-yellow-900\",\n \"border-l-yellow-950\",\n \"border-l-lime-50\",\n \"border-l-lime-100\",\n \"border-l-lime-200\",\n \"border-l-lime-300\",\n \"border-l-lime-400\",\n \"border-l-lime-500\",\n \"border-l-lime-600\",\n \"border-l-lime-700\",\n \"border-l-lime-800\",\n \"border-l-lime-900\",\n \"border-l-lime-950\",\n \"border-l-green-50\",\n \"border-l-green-100\",\n \"border-l-green-200\",\n \"border-l-green-300\",\n \"border-l-green-400\",\n \"border-l-green-500\",\n \"border-l-green-600\",\n \"border-l-green-700\",\n \"border-l-green-800\",\n \"border-l-green-900\",\n \"border-l-green-950\",\n \"border-l-emerald-50\",\n \"border-l-emerald-100\",\n \"border-l-emerald-200\",\n \"border-l-emerald-300\",\n \"border-l-emerald-400\",\n \"border-l-emerald-500\",\n \"border-l-emerald-600\",\n \"border-l-emerald-700\",\n \"border-l-emerald-800\",\n \"border-l-emerald-900\",\n \"border-l-emerald-950\",\n \"border-l-teal-50\",\n \"border-l-teal-100\",\n \"border-l-teal-200\",\n \"border-l-teal-300\",\n \"border-l-teal-400\",\n \"border-l-teal-500\",\n \"border-l-teal-600\",\n \"border-l-teal-700\",\n \"border-l-teal-800\",\n \"border-l-teal-900\",\n \"border-l-teal-950\",\n \"border-l-cyan-50\",\n \"border-l-cyan-100\",\n \"border-l-cyan-200\",\n \"border-l-cyan-300\",\n \"border-l-cyan-400\",\n \"border-l-cyan-500\",\n \"border-l-cyan-600\",\n \"border-l-cyan-700\",\n \"border-l-cyan-800\",\n \"border-l-cyan-900\",\n \"border-l-cyan-950\",\n \"border-l-sky-50\",\n \"border-l-sky-100\",\n \"border-l-sky-200\",\n \"border-l-sky-300\",\n \"border-l-sky-400\",\n \"border-l-sky-500\",\n \"border-l-sky-600\",\n \"border-l-sky-700\",\n \"border-l-sky-800\",\n \"border-l-sky-900\",\n \"border-l-sky-950\",\n \"border-l-blue-50\",\n \"border-l-blue-100\",\n \"border-l-blue-200\",\n \"border-l-blue-300\",\n \"border-l-blue-400\",\n \"border-l-blue-500\",\n \"border-l-blue-600\",\n \"border-l-blue-700\",\n \"border-l-blue-800\",\n \"border-l-blue-900\",\n \"border-l-blue-950\",\n \"border-l-indigo-50\",\n \"border-l-indigo-100\",\n \"border-l-indigo-200\",\n \"border-l-indigo-300\",\n \"border-l-indigo-400\",\n \"border-l-indigo-500\",\n \"border-l-indigo-600\",\n \"border-l-indigo-700\",\n \"border-l-indigo-800\",\n \"border-l-indigo-900\",\n \"border-l-indigo-950\",\n \"border-l-violet-50\",\n \"border-l-violet-100\",\n \"border-l-violet-200\",\n \"border-l-violet-300\",\n \"border-l-violet-400\",\n \"border-l-violet-500\",\n \"border-l-violet-600\",\n \"border-l-violet-700\",\n \"border-l-violet-800\",\n \"border-l-violet-900\",\n \"border-l-violet-950\",\n \"border-l-purple-50\",\n \"border-l-purple-100\",\n \"border-l-purple-200\",\n \"border-l-purple-300\",\n \"border-l-purple-400\",\n \"border-l-purple-500\",\n \"border-l-purple-600\",\n \"border-l-purple-700\",\n \"border-l-purple-800\",\n \"border-l-purple-900\",\n \"border-l-purple-950\",\n \"border-l-fuchsia-50\",\n \"border-l-fuchsia-100\",\n \"border-l-fuchsia-200\",\n \"border-l-fuchsia-300\",\n \"border-l-fuchsia-400\",\n \"border-l-fuchsia-500\",\n \"border-l-fuchsia-600\",\n \"border-l-fuchsia-700\",\n \"border-l-fuchsia-800\",\n \"border-l-fuchsia-900\",\n \"border-l-fuchsia-950\",\n \"border-l-pink-50\",\n \"border-l-pink-100\",\n \"border-l-pink-200\",\n \"border-l-pink-300\",\n \"border-l-pink-400\",\n \"border-l-pink-500\",\n \"border-l-pink-600\",\n \"border-l-pink-700\",\n \"border-l-pink-800\",\n \"border-l-pink-900\",\n \"border-l-pink-950\",\n \"border-l-rose-50\",\n \"border-l-rose-100\",\n \"border-l-rose-200\",\n \"border-l-rose-300\",\n \"border-l-rose-400\",\n \"border-l-rose-500\",\n \"border-l-rose-600\",\n \"border-l-rose-700\",\n \"border-l-rose-800\",\n \"border-l-rose-900\",\n \"border-l-rose-950\",\n \"border-solid\",\n \"border-dashed\",\n \"border-dotted\",\n \"border-double\",\n \"border-hidden\",\n \"border-none\",\n \"divide-x-0\",\n \"divide-x-2\",\n \"divide-x-4\",\n \"divide-x-8\",\n \"divide-x\",\n \"divide-y-0\",\n \"divide-y-2\",\n \"divide-y-4\",\n \"divide-y-8\",\n \"divide-y\",\n \"divide-y-reverse\",\n \"divide-x-reverse\",\n \"divide-inherit\",\n \"divide-current\",\n \"divide-transparent\",\n \"divide-black\",\n \"divide-white\",\n \"divide-slate-50\",\n \"divide-slate-100\",\n \"divide-slate-200\",\n \"divide-slate-300\",\n \"divide-slate-400\",\n \"divide-slate-500\",\n \"divide-slate-600\",\n \"divide-slate-700\",\n \"divide-slate-800\",\n \"divide-slate-900\",\n \"divide-slate-950\",\n \"divide-gray-50\",\n \"divide-gray-100\",\n \"divide-gray-200\",\n \"divide-gray-300\",\n \"divide-gray-400\",\n \"divide-gray-500\",\n \"divide-gray-600\",\n \"divide-gray-700\",\n \"divide-gray-800\",\n \"divide-gray-900\",\n \"divide-gray-950\",\n \"divide-zinc-50\",\n \"divide-zinc-100\",\n \"divide-zinc-200\",\n \"divide-zinc-300\",\n \"divide-zinc-400\",\n \"divide-zinc-500\",\n \"divide-zinc-600\",\n \"divide-zinc-700\",\n \"divide-zinc-800\",\n \"divide-zinc-900\",\n \"divide-zinc-950\",\n \"divide-neutral-50\",\n \"divide-neutral-100\",\n \"divide-neutral-200\",\n \"divide-neutral-300\",\n \"divide-neutral-400\",\n \"divide-neutral-500\",\n \"divide-neutral-600\",\n \"divide-neutral-700\",\n \"divide-neutral-800\",\n \"divide-neutral-900\",\n \"divide-neutral-950\",\n \"divide-stone-50\",\n \"divide-stone-100\",\n \"divide-stone-200\",\n \"divide-stone-300\",\n \"divide-stone-400\",\n \"divide-stone-500\",\n \"divide-stone-600\",\n \"divide-stone-700\",\n \"divide-stone-800\",\n \"divide-stone-900\",\n \"divide-stone-950\",\n \"divide-red-50\",\n \"divide-red-100\",\n \"divide-red-200\",\n \"divide-red-300\",\n \"divide-red-400\",\n \"divide-red-500\",\n \"divide-red-600\",\n \"divide-red-700\",\n \"divide-red-800\",\n \"divide-red-900\",\n \"divide-red-950\",\n \"divide-orange-50\",\n \"divide-orange-100\",\n \"divide-orange-200\",\n \"divide-orange-300\",\n \"divide-orange-400\",\n \"divide-orange-500\",\n \"divide-orange-600\",\n \"divide-orange-700\",\n \"divide-orange-800\",\n \"divide-orange-900\",\n \"divide-orange-950\",\n \"divide-amber-50\",\n \"divide-amber-100\",\n \"divide-amber-200\",\n \"divide-amber-300\",\n \"divide-amber-400\",\n \"divide-amber-500\",\n \"divide-amber-600\",\n \"divide-amber-700\",\n \"divide-amber-800\",\n \"divide-amber-900\",\n \"divide-amber-950\",\n \"divide-yellow-50\",\n \"divide-yellow-100\",\n \"divide-yellow-200\",\n \"divide-yellow-300\",\n \"divide-yellow-400\",\n \"divide-yellow-500\",\n \"divide-yellow-600\",\n \"divide-yellow-700\",\n \"divide-yellow-800\",\n \"divide-yellow-900\",\n \"divide-yellow-950\",\n \"divide-lime-50\",\n \"divide-lime-100\",\n \"divide-lime-200\",\n \"divide-lime-300\",\n \"divide-lime-400\",\n \"divide-lime-500\",\n \"divide-lime-600\",\n \"divide-lime-700\",\n \"divide-lime-800\",\n \"divide-lime-900\",\n \"divide-lime-950\",\n \"divide-green-50\",\n \"divide-green-100\",\n \"divide-green-200\",\n \"divide-green-300\",\n \"divide-green-400\",\n \"divide-green-500\",\n \"divide-green-600\",\n \"divide-green-700\",\n \"divide-green-800\",\n \"divide-green-900\",\n \"divide-green-950\",\n \"divide-emerald-50\",\n \"divide-emerald-100\",\n \"divide-emerald-200\",\n \"divide-emerald-300\",\n \"divide-emerald-400\",\n \"divide-emerald-500\",\n \"divide-emerald-600\",\n \"divide-emerald-700\",\n \"divide-emerald-800\",\n \"divide-emerald-900\",\n \"divide-emerald-950\",\n \"divide-teal-50\",\n \"divide-teal-100\",\n \"divide-teal-200\",\n \"divide-teal-300\",\n \"divide-teal-400\",\n \"divide-teal-500\",\n \"divide-teal-600\",\n \"divide-teal-700\",\n \"divide-teal-800\",\n \"divide-teal-900\",\n \"divide-teal-950\",\n \"divide-cyan-50\",\n \"divide-cyan-100\",\n \"divide-cyan-200\",\n \"divide-cyan-300\",\n \"divide-cyan-400\",\n \"divide-cyan-500\",\n \"divide-cyan-600\",\n \"divide-cyan-700\",\n \"divide-cyan-800\",\n \"divide-cyan-900\",\n \"divide-cyan-950\",\n \"divide-sky-50\",\n \"divide-sky-100\",\n \"divide-sky-200\",\n \"divide-sky-300\",\n \"divide-sky-400\",\n \"divide-sky-500\",\n \"divide-sky-600\",\n \"divide-sky-700\",\n \"divide-sky-800\",\n \"divide-sky-900\",\n \"divide-sky-950\",\n \"divide-blue-50\",\n \"divide-blue-100\",\n \"divide-blue-200\",\n \"divide-blue-300\",\n \"divide-blue-400\",\n \"divide-blue-500\",\n \"divide-blue-600\",\n \"divide-blue-700\",\n \"divide-blue-800\",\n \"divide-blue-900\",\n \"divide-blue-950\",\n \"divide-indigo-50\",\n \"divide-indigo-100\",\n \"divide-indigo-200\",\n \"divide-indigo-300\",\n \"divide-indigo-400\",\n \"divide-indigo-500\",\n \"divide-indigo-600\",\n \"divide-indigo-700\",\n \"divide-indigo-800\",\n \"divide-indigo-900\",\n \"divide-indigo-950\",\n \"divide-violet-50\",\n \"divide-violet-100\",\n \"divide-violet-200\",\n \"divide-violet-300\",\n \"divide-violet-400\",\n \"divide-violet-500\",\n \"divide-violet-600\",\n \"divide-violet-700\",\n \"divide-violet-800\",\n \"divide-violet-900\",\n \"divide-violet-950\",\n \"divide-purple-50\",\n \"divide-purple-100\",\n \"divide-purple-200\",\n \"divide-purple-300\",\n \"divide-purple-400\",\n \"divide-purple-500\",\n \"divide-purple-600\",\n \"divide-purple-700\",\n \"divide-purple-800\",\n \"divide-purple-900\",\n \"divide-purple-950\",\n \"divide-fuchsia-50\",\n \"divide-fuchsia-100\",\n \"divide-fuchsia-200\",\n \"divide-fuchsia-300\",\n \"divide-fuchsia-400\",\n \"divide-fuchsia-500\",\n \"divide-fuchsia-600\",\n \"divide-fuchsia-700\",\n \"divide-fuchsia-800\",\n \"divide-fuchsia-900\",\n \"divide-fuchsia-950\",\n \"divide-pink-50\",\n \"divide-pink-100\",\n \"divide-pink-200\",\n \"divide-pink-300\",\n \"divide-pink-400\",\n \"divide-pink-500\",\n \"divide-pink-600\",\n \"divide-pink-700\",\n \"divide-pink-800\",\n \"divide-pink-900\",\n \"divide-pink-950\",\n \"divide-rose-50\",\n \"divide-rose-100\",\n \"divide-rose-200\",\n \"divide-rose-300\",\n \"divide-rose-400\",\n \"divide-rose-500\",\n \"divide-rose-600\",\n \"divide-rose-700\",\n \"divide-rose-800\",\n \"divide-rose-900\",\n \"divide-rose-950\",\n \"divide-solid\",\n \"divide-dashed\",\n \"divide-dotted\",\n \"divide-double\",\n \"divide-none\",\n \"outline-0\",\n \"outline-1\",\n \"outline-2\",\n \"outline-4\",\n \"outline-8\",\n \"outline-inherit\",\n \"outline-current\",\n \"outline-transparent\",\n \"outline-black\",\n \"outline-white\",\n \"outline-slate-50\",\n \"outline-slate-100\",\n \"outline-slate-200\",\n \"outline-slate-300\",\n \"outline-slate-400\",\n \"outline-slate-500\",\n \"outline-slate-600\",\n \"outline-slate-700\",\n \"outline-slate-800\",\n \"outline-slate-900\",\n \"outline-slate-950\",\n \"outline-gray-50\",\n \"outline-gray-100\",\n \"outline-gray-200\",\n \"outline-gray-300\",\n \"outline-gray-400\",\n \"outline-gray-500\",\n \"outline-gray-600\",\n \"outline-gray-700\",\n \"outline-gray-800\",\n \"outline-gray-900\",\n \"outline-gray-950\",\n \"outline-zinc-50\",\n \"outline-zinc-100\",\n \"outline-zinc-200\",\n \"outline-zinc-300\",\n \"outline-zinc-400\",\n \"outline-zinc-500\",\n \"outline-zinc-600\",\n \"outline-zinc-700\",\n \"outline-zinc-800\",\n \"outline-zinc-900\",\n \"outline-zinc-950\",\n \"outline-neutral-50\",\n \"outline-neutral-100\",\n \"outline-neutral-200\",\n \"outline-neutral-300\",\n \"outline-neutral-400\",\n \"outline-neutral-500\",\n \"outline-neutral-600\",\n \"outline-neutral-700\",\n \"outline-neutral-800\",\n \"outline-neutral-900\",\n \"outline-neutral-950\",\n \"outline-stone-50\",\n \"outline-stone-100\",\n \"outline-stone-200\",\n \"outline-stone-300\",\n \"outline-stone-400\",\n \"outline-stone-500\",\n \"outline-stone-600\",\n \"outline-stone-700\",\n \"outline-stone-800\",\n \"outline-stone-900\",\n \"outline-stone-950\",\n \"outline-red-50\",\n \"outline-red-100\",\n \"outline-red-200\",\n \"outline-red-300\",\n \"outline-red-400\",\n \"outline-red-500\",\n \"outline-red-600\",\n \"outline-red-700\",\n \"outline-red-800\",\n \"outline-red-900\",\n \"outline-red-950\",\n \"outline-orange-50\",\n \"outline-orange-100\",\n \"outline-orange-200\",\n \"outline-orange-300\",\n \"outline-orange-400\",\n \"outline-orange-500\",\n \"outline-orange-600\",\n \"outline-orange-700\",\n \"outline-orange-800\",\n \"outline-orange-900\",\n \"outline-orange-950\",\n \"outline-amber-50\",\n \"outline-amber-100\",\n \"outline-amber-200\",\n \"outline-amber-300\",\n \"outline-amber-400\",\n \"outline-amber-500\",\n \"outline-amber-600\",\n \"outline-amber-700\",\n \"outline-amber-800\",\n \"outline-amber-900\",\n \"outline-amber-950\",\n \"outline-yellow-50\",\n \"outline-yellow-100\",\n \"outline-yellow-200\",\n \"outline-yellow-300\",\n \"outline-yellow-400\",\n \"outline-yellow-500\",\n \"outline-yellow-600\",\n \"outline-yellow-700\",\n \"outline-yellow-800\",\n \"outline-yellow-900\",\n \"outline-yellow-950\",\n \"outline-lime-50\",\n \"outline-lime-100\",\n \"outline-lime-200\",\n \"outline-lime-300\",\n \"outline-lime-400\",\n \"outline-lime-500\",\n \"outline-lime-600\",\n \"outline-lime-700\",\n \"outline-lime-800\",\n \"outline-lime-900\",\n \"outline-lime-950\",\n \"outline-green-50\",\n \"outline-green-100\",\n \"outline-green-200\",\n \"outline-green-300\",\n \"outline-green-400\",\n \"outline-green-500\",\n \"outline-green-600\",\n \"outline-green-700\",\n \"outline-green-800\",\n \"outline-green-900\",\n \"outline-green-950\",\n \"outline-emerald-50\",\n \"outline-emerald-100\",\n \"outline-emerald-200\",\n \"outline-emerald-300\",\n \"outline-emerald-400\",\n \"outline-emerald-500\",\n \"outline-emerald-600\",\n \"outline-emerald-700\",\n \"outline-emerald-800\",\n \"outline-emerald-900\",\n \"outline-emerald-950\",\n \"outline-teal-50\",\n \"outline-teal-100\",\n \"outline-teal-200\",\n \"outline-teal-300\",\n \"outline-teal-400\",\n \"outline-teal-500\",\n \"outline-teal-600\",\n \"outline-teal-700\",\n \"outline-teal-800\",\n \"outline-teal-900\",\n \"outline-teal-950\",\n \"outline-cyan-50\",\n \"outline-cyan-100\",\n \"outline-cyan-200\",\n \"outline-cyan-300\",\n \"outline-cyan-400\",\n \"outline-cyan-500\",\n \"outline-cyan-600\",\n \"outline-cyan-700\",\n \"outline-cyan-800\",\n \"outline-cyan-900\",\n \"outline-cyan-950\",\n \"outline-sky-50\",\n \"outline-sky-100\",\n \"outline-sky-200\",\n \"outline-sky-300\",\n \"outline-sky-400\",\n \"outline-sky-500\",\n \"outline-sky-600\",\n \"outline-sky-700\",\n \"outline-sky-800\",\n \"outline-sky-900\",\n \"outline-sky-950\",\n \"outline-blue-50\",\n \"outline-blue-100\",\n \"outline-blue-200\",\n \"outline-blue-300\",\n \"outline-blue-400\",\n \"outline-blue-500\",\n \"outline-blue-600\",\n \"outline-blue-700\",\n \"outline-blue-800\",\n \"outline-blue-900\",\n \"outline-blue-950\",\n \"outline-indigo-50\",\n \"outline-indigo-100\",\n \"outline-indigo-200\",\n \"outline-indigo-300\",\n \"outline-indigo-400\",\n \"outline-indigo-500\",\n \"outline-indigo-600\",\n \"outline-indigo-700\",\n \"outline-indigo-800\",\n \"outline-indigo-900\",\n \"outline-indigo-950\",\n \"outline-violet-50\",\n \"outline-violet-100\",\n \"outline-violet-200\",\n \"outline-violet-300\",\n \"outline-violet-400\",\n \"outline-violet-500\",\n \"outline-violet-600\",\n \"outline-violet-700\",\n \"outline-violet-800\",\n \"outline-violet-900\",\n \"outline-violet-950\",\n \"outline-purple-50\",\n \"outline-purple-100\",\n \"outline-purple-200\",\n \"outline-purple-300\",\n \"outline-purple-400\",\n \"outline-purple-500\",\n \"outline-purple-600\",\n \"outline-purple-700\",\n \"outline-purple-800\",\n \"outline-purple-900\",\n \"outline-purple-950\",\n \"outline-fuchsia-50\",\n \"outline-fuchsia-100\",\n \"outline-fuchsia-200\",\n \"outline-fuchsia-300\",\n \"outline-fuchsia-400\",\n \"outline-fuchsia-500\",\n \"outline-fuchsia-600\",\n \"outline-fuchsia-700\",\n \"outline-fuchsia-800\",\n \"outline-fuchsia-900\",\n \"outline-fuchsia-950\",\n \"outline-pink-50\",\n \"outline-pink-100\",\n \"outline-pink-200\",\n \"outline-pink-300\",\n \"outline-pink-400\",\n \"outline-pink-500\",\n \"outline-pink-600\",\n \"outline-pink-700\",\n \"outline-pink-800\",\n \"outline-pink-900\",\n \"outline-pink-950\",\n \"outline-rose-50\",\n \"outline-rose-100\",\n \"outline-rose-200\",\n \"outline-rose-300\",\n \"outline-rose-400\",\n \"outline-rose-500\",\n \"outline-rose-600\",\n \"outline-rose-700\",\n \"outline-rose-800\",\n \"outline-rose-900\",\n \"outline-rose-950\",\n \"outline-none\",\n \"outline\",\n \"outline-dashed\",\n \"outline-dotted\",\n \"outline-double\",\n \"outline-offset-0\",\n \"outline-offset-1\",\n \"outline-offset-2\",\n \"outline-offset-4\",\n \"outline-offset-8\",\n \"ring-0\",\n \"ring-1\",\n \"ring-2\",\n \"ring\",\n \"ring-4\",\n \"ring-8\",\n \"ring-inset\",\n \"ring-inherit\",\n \"ring-current\",\n \"ring-transparent\",\n \"ring-black\",\n \"ring-white\",\n \"ring-slate-50\",\n \"ring-slate-100\",\n \"ring-slate-200\",\n \"ring-slate-300\",\n \"ring-slate-400\",\n \"ring-slate-500\",\n \"ring-slate-600\",\n \"ring-slate-700\",\n \"ring-slate-800\",\n \"ring-slate-900\",\n \"ring-slate-950\",\n \"ring-gray-50\",\n \"ring-gray-100\",\n \"ring-gray-200\",\n \"ring-gray-300\",\n \"ring-gray-400\",\n \"ring-gray-500\",\n \"ring-gray-600\",\n \"ring-gray-700\",\n \"ring-gray-800\",\n \"ring-gray-900\",\n \"ring-gray-950\",\n \"ring-zinc-50\",\n \"ring-zinc-100\",\n \"ring-zinc-200\",\n \"ring-zinc-300\",\n \"ring-zinc-400\",\n \"ring-zinc-500\",\n \"ring-zinc-600\",\n \"ring-zinc-700\",\n \"ring-zinc-800\",\n \"ring-zinc-900\",\n \"ring-zinc-950\",\n \"ring-neutral-50\",\n \"ring-neutral-100\",\n \"ring-neutral-200\",\n \"ring-neutral-300\",\n \"ring-neutral-400\",\n \"ring-neutral-500\",\n \"ring-neutral-600\",\n \"ring-neutral-700\",\n \"ring-neutral-800\",\n \"ring-neutral-900\",\n \"ring-neutral-950\",\n \"ring-stone-50\",\n \"ring-stone-100\",\n \"ring-stone-200\",\n \"ring-stone-300\",\n \"ring-stone-400\",\n \"ring-stone-500\",\n \"ring-stone-600\",\n \"ring-stone-700\",\n \"ring-stone-800\",\n \"ring-stone-900\",\n \"ring-stone-950\",\n \"ring-red-50\",\n \"ring-red-100\",\n \"ring-red-200\",\n \"ring-red-300\",\n \"ring-red-400\",\n \"ring-red-500\",\n \"ring-red-600\",\n \"ring-red-700\",\n \"ring-red-800\",\n \"ring-red-900\",\n \"ring-red-950\",\n \"ring-orange-50\",\n \"ring-orange-100\",\n \"ring-orange-200\",\n \"ring-orange-300\",\n \"ring-orange-400\",\n \"ring-orange-500\",\n \"ring-orange-600\",\n \"ring-orange-700\",\n \"ring-orange-800\",\n \"ring-orange-900\",\n \"ring-orange-950\",\n \"ring-amber-50\",\n \"ring-amber-100\",\n \"ring-amber-200\",\n \"ring-amber-300\",\n \"ring-amber-400\",\n \"ring-amber-500\",\n \"ring-amber-600\",\n \"ring-amber-700\",\n \"ring-amber-800\",\n \"ring-amber-900\",\n \"ring-amber-950\",\n \"ring-yellow-50\",\n \"ring-yellow-100\",\n \"ring-yellow-200\",\n \"ring-yellow-300\",\n \"ring-yellow-400\",\n \"ring-yellow-500\",\n \"ring-yellow-600\",\n \"ring-yellow-700\",\n \"ring-yellow-800\",\n \"ring-yellow-900\",\n \"ring-yellow-950\",\n \"ring-lime-50\",\n \"ring-lime-100\",\n \"ring-lime-200\",\n \"ring-lime-300\",\n \"ring-lime-400\",\n \"ring-lime-500\",\n \"ring-lime-600\",\n \"ring-lime-700\",\n \"ring-lime-800\",\n \"ring-lime-900\",\n \"ring-lime-950\",\n \"ring-green-50\",\n \"ring-green-100\",\n \"ring-green-200\",\n \"ring-green-300\",\n \"ring-green-400\",\n \"ring-green-500\",\n \"ring-green-600\",\n \"ring-green-700\",\n \"ring-green-800\",\n \"ring-green-900\",\n \"ring-green-950\",\n \"ring-emerald-50\",\n \"ring-emerald-100\",\n \"ring-emerald-200\",\n \"ring-emerald-300\",\n \"ring-emerald-400\",\n \"ring-emerald-500\",\n \"ring-emerald-600\",\n \"ring-emerald-700\",\n \"ring-emerald-800\",\n \"ring-emerald-900\",\n \"ring-emerald-950\",\n \"ring-teal-50\",\n \"ring-teal-100\",\n \"ring-teal-200\",\n \"ring-teal-300\",\n \"ring-teal-400\",\n \"ring-teal-500\",\n \"ring-teal-600\",\n \"ring-teal-700\",\n \"ring-teal-800\",\n \"ring-teal-900\",\n \"ring-teal-950\",\n \"ring-cyan-50\",\n \"ring-cyan-100\",\n \"ring-cyan-200\",\n \"ring-cyan-300\",\n \"ring-cyan-400\",\n \"ring-cyan-500\",\n \"ring-cyan-600\",\n \"ring-cyan-700\",\n \"ring-cyan-800\",\n \"ring-cyan-900\",\n \"ring-cyan-950\",\n \"ring-sky-50\",\n \"ring-sky-100\",\n \"ring-sky-200\",\n \"ring-sky-300\",\n \"ring-sky-400\",\n \"ring-sky-500\",\n \"ring-sky-600\",\n \"ring-sky-700\",\n \"ring-sky-800\",\n \"ring-sky-900\",\n \"ring-sky-950\",\n \"ring-blue-50\",\n \"ring-blue-100\",\n \"ring-blue-200\",\n \"ring-blue-300\",\n \"ring-blue-400\",\n \"ring-blue-500\",\n \"ring-blue-600\",\n \"ring-blue-700\",\n \"ring-blue-800\",\n \"ring-blue-900\",\n \"ring-blue-950\",\n \"ring-indigo-50\",\n \"ring-indigo-100\",\n \"ring-indigo-200\",\n \"ring-indigo-300\",\n \"ring-indigo-400\",\n \"ring-indigo-500\",\n \"ring-indigo-600\",\n \"ring-indigo-700\",\n \"ring-indigo-800\",\n \"ring-indigo-900\",\n \"ring-indigo-950\",\n \"ring-violet-50\",\n \"ring-violet-100\",\n \"ring-violet-200\",\n \"ring-violet-300\",\n \"ring-violet-400\",\n \"ring-violet-500\",\n \"ring-violet-600\",\n \"ring-violet-700\",\n \"ring-violet-800\",\n \"ring-violet-900\",\n \"ring-violet-950\",\n \"ring-purple-50\",\n \"ring-purple-100\",\n \"ring-purple-200\",\n \"ring-purple-300\",\n \"ring-purple-400\",\n \"ring-purple-500\",\n \"ring-purple-600\",\n \"ring-purple-700\",\n \"ring-purple-800\",\n \"ring-purple-900\",\n \"ring-purple-950\",\n \"ring-fuchsia-50\",\n \"ring-fuchsia-100\",\n \"ring-fuchsia-200\",\n \"ring-fuchsia-300\",\n \"ring-fuchsia-400\",\n \"ring-fuchsia-500\",\n \"ring-fuchsia-600\",\n \"ring-fuchsia-700\",\n \"ring-fuchsia-800\",\n \"ring-fuchsia-900\",\n \"ring-fuchsia-950\",\n \"ring-pink-50\",\n \"ring-pink-100\",\n \"ring-pink-200\",\n \"ring-pink-300\",\n \"ring-pink-400\",\n \"ring-pink-500\",\n \"ring-pink-600\",\n \"ring-pink-700\",\n \"ring-pink-800\",\n \"ring-pink-900\",\n \"ring-pink-950\",\n \"ring-rose-50\",\n \"ring-rose-100\",\n \"ring-rose-200\",\n \"ring-rose-300\",\n \"ring-rose-400\",\n \"ring-rose-500\",\n \"ring-rose-600\",\n \"ring-rose-700\",\n \"ring-rose-800\",\n \"ring-rose-900\",\n \"ring-rose-950\",\n \"ring-offset-0\",\n \"ring-offset-1\",\n \"ring-offset-2\",\n \"ring-offset-4\",\n \"ring-offset-8\",\n \"ring-offset-inherit\",\n \"ring-offset-current\",\n \"ring-offset-transparent\",\n \"ring-offset-black\",\n \"ring-offset-white\",\n \"ring-offset-slate-50\",\n \"ring-offset-slate-100\",\n \"ring-offset-slate-200\",\n \"ring-offset-slate-300\",\n \"ring-offset-slate-400\",\n \"ring-offset-slate-500\",\n \"ring-offset-slate-600\",\n \"ring-offset-slate-700\",\n \"ring-offset-slate-800\",\n \"ring-offset-slate-900\",\n \"ring-offset-slate-950\",\n \"ring-offset-gray-50\",\n \"ring-offset-gray-100\",\n \"ring-offset-gray-200\",\n \"ring-offset-gray-300\",\n \"ring-offset-gray-400\",\n \"ring-offset-gray-500\",\n \"ring-offset-gray-600\",\n \"ring-offset-gray-700\",\n \"ring-offset-gray-800\",\n \"ring-offset-gray-900\",\n \"ring-offset-gray-950\",\n \"ring-offset-zinc-50\",\n \"ring-offset-zinc-100\",\n \"ring-offset-zinc-200\",\n \"ring-offset-zinc-300\",\n \"ring-offset-zinc-400\",\n \"ring-offset-zinc-500\",\n \"ring-offset-zinc-600\",\n \"ring-offset-zinc-700\",\n \"ring-offset-zinc-800\",\n \"ring-offset-zinc-900\",\n \"ring-offset-zinc-950\",\n \"ring-offset-neutral-50\",\n \"ring-offset-neutral-100\",\n \"ring-offset-neutral-200\",\n \"ring-offset-neutral-300\",\n \"ring-offset-neutral-400\",\n \"ring-offset-neutral-500\",\n \"ring-offset-neutral-600\",\n \"ring-offset-neutral-700\",\n \"ring-offset-neutral-800\",\n \"ring-offset-neutral-900\",\n \"ring-offset-neutral-950\",\n \"ring-offset-stone-50\",\n \"ring-offset-stone-100\",\n \"ring-offset-stone-200\",\n \"ring-offset-stone-300\",\n \"ring-offset-stone-400\",\n \"ring-offset-stone-500\",\n \"ring-offset-stone-600\",\n \"ring-offset-stone-700\",\n \"ring-offset-stone-800\",\n \"ring-offset-stone-900\",\n \"ring-offset-stone-950\",\n \"ring-offset-red-50\",\n \"ring-offset-red-100\",\n \"ring-offset-red-200\",\n \"ring-offset-red-300\",\n \"ring-offset-red-400\",\n \"ring-offset-red-500\",\n \"ring-offset-red-600\",\n \"ring-offset-red-700\",\n \"ring-offset-red-800\",\n \"ring-offset-red-900\",\n \"ring-offset-red-950\",\n \"ring-offset-orange-50\",\n \"ring-offset-orange-100\",\n \"ring-offset-orange-200\",\n \"ring-offset-orange-300\",\n \"ring-offset-orange-400\",\n \"ring-offset-orange-500\",\n \"ring-offset-orange-600\",\n \"ring-offset-orange-700\",\n \"ring-offset-orange-800\",\n \"ring-offset-orange-900\",\n \"ring-offset-orange-950\",\n \"ring-offset-amber-50\",\n \"ring-offset-amber-100\",\n \"ring-offset-amber-200\",\n \"ring-offset-amber-300\",\n \"ring-offset-amber-400\",\n \"ring-offset-amber-500\",\n \"ring-offset-amber-600\",\n \"ring-offset-amber-700\",\n \"ring-offset-amber-800\",\n \"ring-offset-amber-900\",\n \"ring-offset-amber-950\",\n \"ring-offset-yellow-50\",\n \"ring-offset-yellow-100\",\n \"ring-offset-yellow-200\",\n \"ring-offset-yellow-300\",\n \"ring-offset-yellow-400\",\n \"ring-offset-yellow-500\",\n \"ring-offset-yellow-600\",\n \"ring-offset-yellow-700\",\n \"ring-offset-yellow-800\",\n \"ring-offset-yellow-900\",\n \"ring-offset-yellow-950\",\n \"ring-offset-lime-50\",\n \"ring-offset-lime-100\",\n \"ring-offset-lime-200\",\n \"ring-offset-lime-300\",\n \"ring-offset-lime-400\",\n \"ring-offset-lime-500\",\n \"ring-offset-lime-600\",\n \"ring-offset-lime-700\",\n \"ring-offset-lime-800\",\n \"ring-offset-lime-900\",\n \"ring-offset-lime-950\",\n \"ring-offset-green-50\",\n \"ring-offset-green-100\",\n \"ring-offset-green-200\",\n \"ring-offset-green-300\",\n \"ring-offset-green-400\",\n \"ring-offset-green-500\",\n \"ring-offset-green-600\",\n \"ring-offset-green-700\",\n \"ring-offset-green-800\",\n \"ring-offset-green-900\",\n \"ring-offset-green-950\",\n \"ring-offset-emerald-50\",\n \"ring-offset-emerald-100\",\n \"ring-offset-emerald-200\",\n \"ring-offset-emerald-300\",\n \"ring-offset-emerald-400\",\n \"ring-offset-emerald-500\",\n \"ring-offset-emerald-600\",\n \"ring-offset-emerald-700\",\n \"ring-offset-emerald-800\",\n \"ring-offset-emerald-900\",\n \"ring-offset-emerald-950\",\n \"ring-offset-teal-50\",\n \"ring-offset-teal-100\",\n \"ring-offset-teal-200\",\n \"ring-offset-teal-300\",\n \"ring-offset-teal-400\",\n \"ring-offset-teal-500\",\n \"ring-offset-teal-600\",\n \"ring-offset-teal-700\",\n \"ring-offset-teal-800\",\n \"ring-offset-teal-900\",\n \"ring-offset-teal-950\",\n \"ring-offset-cyan-50\",\n \"ring-offset-cyan-100\",\n \"ring-offset-cyan-200\",\n \"ring-offset-cyan-300\",\n \"ring-offset-cyan-400\",\n \"ring-offset-cyan-500\",\n \"ring-offset-cyan-600\",\n \"ring-offset-cyan-700\",\n \"ring-offset-cyan-800\",\n \"ring-offset-cyan-900\",\n \"ring-offset-cyan-950\",\n \"ring-offset-sky-50\",\n \"ring-offset-sky-100\",\n \"ring-offset-sky-200\",\n \"ring-offset-sky-300\",\n \"ring-offset-sky-400\",\n \"ring-offset-sky-500\",\n \"ring-offset-sky-600\",\n \"ring-offset-sky-700\",\n \"ring-offset-sky-800\",\n \"ring-offset-sky-900\",\n \"ring-offset-sky-950\",\n \"ring-offset-blue-50\",\n \"ring-offset-blue-100\",\n \"ring-offset-blue-200\",\n \"ring-offset-blue-300\",\n \"ring-offset-blue-400\",\n \"ring-offset-blue-500\",\n \"ring-offset-blue-600\",\n \"ring-offset-blue-700\",\n \"ring-offset-blue-800\",\n \"ring-offset-blue-900\",\n \"ring-offset-blue-950\",\n \"ring-offset-indigo-50\",\n \"ring-offset-indigo-100\",\n \"ring-offset-indigo-200\",\n \"ring-offset-indigo-300\",\n \"ring-offset-indigo-400\",\n \"ring-offset-indigo-500\",\n \"ring-offset-indigo-600\",\n \"ring-offset-indigo-700\",\n \"ring-offset-indigo-800\",\n \"ring-offset-indigo-900\",\n \"ring-offset-indigo-950\",\n \"ring-offset-violet-50\",\n \"ring-offset-violet-100\",\n \"ring-offset-violet-200\",\n \"ring-offset-violet-300\",\n \"ring-offset-violet-400\",\n \"ring-offset-violet-500\",\n \"ring-offset-violet-600\",\n \"ring-offset-violet-700\",\n \"ring-offset-violet-800\",\n \"ring-offset-violet-900\",\n \"ring-offset-violet-950\",\n \"ring-offset-purple-50\",\n \"ring-offset-purple-100\",\n \"ring-offset-purple-200\",\n \"ring-offset-purple-300\",\n \"ring-offset-purple-400\",\n \"ring-offset-purple-500\",\n \"ring-offset-purple-600\",\n \"ring-offset-purple-700\",\n \"ring-offset-purple-800\",\n \"ring-offset-purple-900\",\n \"ring-offset-purple-950\",\n \"ring-offset-fuchsia-50\",\n \"ring-offset-fuchsia-100\",\n \"ring-offset-fuchsia-200\",\n \"ring-offset-fuchsia-300\",\n \"ring-offset-fuchsia-400\",\n \"ring-offset-fuchsia-500\",\n \"ring-offset-fuchsia-600\",\n \"ring-offset-fuchsia-700\",\n \"ring-offset-fuchsia-800\",\n \"ring-offset-fuchsia-900\",\n \"ring-offset-fuchsia-950\",\n \"ring-offset-pink-50\",\n \"ring-offset-pink-100\",\n \"ring-offset-pink-200\",\n \"ring-offset-pink-300\",\n \"ring-offset-pink-400\",\n \"ring-offset-pink-500\",\n \"ring-offset-pink-600\",\n \"ring-offset-pink-700\",\n \"ring-offset-pink-800\",\n \"ring-offset-pink-900\",\n \"ring-offset-pink-950\",\n \"ring-offset-rose-50\",\n \"ring-offset-rose-100\",\n \"ring-offset-rose-200\",\n \"ring-offset-rose-300\",\n \"ring-offset-rose-400\",\n \"ring-offset-rose-500\",\n \"ring-offset-rose-600\",\n \"ring-offset-rose-700\",\n \"ring-offset-rose-800\",\n \"ring-offset-rose-900\",\n \"ring-offset-rose-950\",\n \"shadow-sm\",\n \"shadow\",\n \"shadow-md\",\n \"shadow-lg\",\n \"shadow-xl\",\n \"shadow-2xl\",\n \"shadow-inner\",\n \"shadow-none\",\n \"shadow-inherit\",\n \"shadow-current\",\n \"shadow-transparent\",\n \"shadow-black\",\n \"shadow-white\",\n \"shadow-slate-50\",\n \"shadow-slate-100\",\n \"shadow-slate-200\",\n \"shadow-slate-300\",\n \"shadow-slate-400\",\n \"shadow-slate-500\",\n \"shadow-slate-600\",\n \"shadow-slate-700\",\n \"shadow-slate-800\",\n \"shadow-slate-900\",\n \"shadow-slate-950\",\n \"shadow-gray-50\",\n \"shadow-gray-100\",\n \"shadow-gray-200\",\n \"shadow-gray-300\",\n \"shadow-gray-400\",\n \"shadow-gray-500\",\n \"shadow-gray-600\",\n \"shadow-gray-700\",\n \"shadow-gray-800\",\n \"shadow-gray-900\",\n \"shadow-gray-950\",\n \"shadow-zinc-50\",\n \"shadow-zinc-100\",\n \"shadow-zinc-200\",\n \"shadow-zinc-300\",\n \"shadow-zinc-400\",\n \"shadow-zinc-500\",\n \"shadow-zinc-600\",\n \"shadow-zinc-700\",\n \"shadow-zinc-800\",\n \"shadow-zinc-900\",\n \"shadow-zinc-950\",\n \"shadow-neutral-50\",\n \"shadow-neutral-100\",\n \"shadow-neutral-200\",\n \"shadow-neutral-300\",\n \"shadow-neutral-400\",\n \"shadow-neutral-500\",\n \"shadow-neutral-600\",\n \"shadow-neutral-700\",\n \"shadow-neutral-800\",\n \"shadow-neutral-900\",\n \"shadow-neutral-950\",\n \"shadow-stone-50\",\n \"shadow-stone-100\",\n \"shadow-stone-200\",\n \"shadow-stone-300\",\n \"shadow-stone-400\",\n \"shadow-stone-500\",\n \"shadow-stone-600\",\n \"shadow-stone-700\",\n \"shadow-stone-800\",\n \"shadow-stone-900\",\n \"shadow-stone-950\",\n \"shadow-red-50\",\n \"shadow-red-100\",\n \"shadow-red-200\",\n \"shadow-red-300\",\n \"shadow-red-400\",\n \"shadow-red-500\",\n \"shadow-red-600\",\n \"shadow-red-700\",\n \"shadow-red-800\",\n \"shadow-red-900\",\n \"shadow-red-950\",\n \"shadow-orange-50\",\n \"shadow-orange-100\",\n \"shadow-orange-200\",\n \"shadow-orange-300\",\n \"shadow-orange-400\",\n \"shadow-orange-500\",\n \"shadow-orange-600\",\n \"shadow-orange-700\",\n \"shadow-orange-800\",\n \"shadow-orange-900\",\n \"shadow-orange-950\",\n \"shadow-amber-50\",\n \"shadow-amber-100\",\n \"shadow-amber-200\",\n \"shadow-amber-300\",\n \"shadow-amber-400\",\n \"shadow-amber-500\",\n \"shadow-amber-600\",\n \"shadow-amber-700\",\n \"shadow-amber-800\",\n \"shadow-amber-900\",\n \"shadow-amber-950\",\n \"shadow-yellow-50\",\n \"shadow-yellow-100\",\n \"shadow-yellow-200\",\n \"shadow-yellow-300\",\n \"shadow-yellow-400\",\n \"shadow-yellow-500\",\n \"shadow-yellow-600\",\n \"shadow-yellow-700\",\n \"shadow-yellow-800\",\n \"shadow-yellow-900\",\n \"shadow-yellow-950\",\n \"shadow-lime-50\",\n \"shadow-lime-100\",\n \"shadow-lime-200\",\n \"shadow-lime-300\",\n \"shadow-lime-400\",\n \"shadow-lime-500\",\n \"shadow-lime-600\",\n \"shadow-lime-700\",\n \"shadow-lime-800\",\n \"shadow-lime-900\",\n \"shadow-lime-950\",\n \"shadow-green-50\",\n \"shadow-green-100\",\n \"shadow-green-200\",\n \"shadow-green-300\",\n \"shadow-green-400\",\n \"shadow-green-500\",\n \"shadow-green-600\",\n \"shadow-green-700\",\n \"shadow-green-800\",\n \"shadow-green-900\",\n \"shadow-green-950\",\n \"shadow-emerald-50\",\n \"shadow-emerald-100\",\n \"shadow-emerald-200\",\n \"shadow-emerald-300\",\n \"shadow-emerald-400\",\n \"shadow-emerald-500\",\n \"shadow-emerald-600\",\n \"shadow-emerald-700\",\n \"shadow-emerald-800\",\n \"shadow-emerald-900\",\n \"shadow-emerald-950\",\n \"shadow-teal-50\",\n \"shadow-teal-100\",\n \"shadow-teal-200\",\n \"shadow-teal-300\",\n \"shadow-teal-400\",\n \"shadow-teal-500\",\n \"shadow-teal-600\",\n \"shadow-teal-700\",\n \"shadow-teal-800\",\n \"shadow-teal-900\",\n \"shadow-teal-950\",\n \"shadow-cyan-50\",\n \"shadow-cyan-100\",\n \"shadow-cyan-200\",\n \"shadow-cyan-300\",\n \"shadow-cyan-400\",\n \"shadow-cyan-500\",\n \"shadow-cyan-600\",\n \"shadow-cyan-700\",\n \"shadow-cyan-800\",\n \"shadow-cyan-900\",\n \"shadow-cyan-950\",\n \"shadow-sky-50\",\n \"shadow-sky-100\",\n \"shadow-sky-200\",\n \"shadow-sky-300\",\n \"shadow-sky-400\",\n \"shadow-sky-500\",\n \"shadow-sky-600\",\n \"shadow-sky-700\",\n \"shadow-sky-800\",\n \"shadow-sky-900\",\n \"shadow-sky-950\",\n \"shadow-blue-50\",\n \"shadow-blue-100\",\n \"shadow-blue-200\",\n \"shadow-blue-300\",\n \"shadow-blue-400\",\n \"shadow-blue-500\",\n \"shadow-blue-600\",\n \"shadow-blue-700\",\n \"shadow-blue-800\",\n \"shadow-blue-900\",\n \"shadow-blue-950\",\n \"shadow-indigo-50\",\n \"shadow-indigo-100\",\n \"shadow-indigo-200\",\n \"shadow-indigo-300\",\n \"shadow-indigo-400\",\n \"shadow-indigo-500\",\n \"shadow-indigo-600\",\n \"shadow-indigo-700\",\n \"shadow-indigo-800\",\n \"shadow-indigo-900\",\n \"shadow-indigo-950\",\n \"shadow-violet-50\",\n \"shadow-violet-100\",\n \"shadow-violet-200\",\n \"shadow-violet-300\",\n \"shadow-violet-400\",\n \"shadow-violet-500\",\n \"shadow-violet-600\",\n \"shadow-violet-700\",\n \"shadow-violet-800\",\n \"shadow-violet-900\",\n \"shadow-violet-950\",\n \"shadow-purple-50\",\n \"shadow-purple-100\",\n \"shadow-purple-200\",\n \"shadow-purple-300\",\n \"shadow-purple-400\",\n \"shadow-purple-500\",\n \"shadow-purple-600\",\n \"shadow-purple-700\",\n \"shadow-purple-800\",\n \"shadow-purple-900\",\n \"shadow-purple-950\",\n \"shadow-fuchsia-50\",\n \"shadow-fuchsia-100\",\n \"shadow-fuchsia-200\",\n \"shadow-fuchsia-300\",\n \"shadow-fuchsia-400\",\n \"shadow-fuchsia-500\",\n \"shadow-fuchsia-600\",\n \"shadow-fuchsia-700\",\n \"shadow-fuchsia-800\",\n \"shadow-fuchsia-900\",\n \"shadow-fuchsia-950\",\n \"shadow-pink-50\",\n \"shadow-pink-100\",\n \"shadow-pink-200\",\n \"shadow-pink-300\",\n \"shadow-pink-400\",\n \"shadow-pink-500\",\n \"shadow-pink-600\",\n \"shadow-pink-700\",\n \"shadow-pink-800\",\n \"shadow-pink-900\",\n \"shadow-pink-950\",\n \"shadow-rose-50\",\n \"shadow-rose-100\",\n \"shadow-rose-200\",\n \"shadow-rose-300\",\n \"shadow-rose-400\",\n \"shadow-rose-500\",\n \"shadow-rose-600\",\n \"shadow-rose-700\",\n \"shadow-rose-800\",\n \"shadow-rose-900\",\n \"shadow-rose-950\",\n \"opacity-0\",\n \"opacity-5\",\n \"opacity-10\",\n \"opacity-15\",\n \"opacity-20\",\n \"opacity-25\",\n \"opacity-30\",\n \"opacity-35\",\n \"opacity-40\",\n \"opacity-45\",\n \"opacity-50\",\n \"opacity-55\",\n \"opacity-60\",\n \"opacity-65\",\n \"opacity-70\",\n \"opacity-75\",\n \"opacity-80\",\n \"opacity-85\",\n \"opacity-90\",\n \"opacity-95\",\n \"opacity-100\",\n \"mix-blend-normal\",\n \"mix-blend-multiply\",\n \"mix-blend-screen\",\n \"mix-blend-overlay\",\n \"mix-blend-darken\",\n \"mix-blend-lighten\",\n \"mix-blend-color-dodge\",\n \"mix-blend-color-burn\",\n \"mix-blend-hard-light\",\n \"mix-blend-soft-light\",\n \"mix-blend-difference\",\n \"mix-blend-exclusion\",\n \"mix-blend-hue\",\n \"mix-blend-saturation\",\n \"mix-blend-color\",\n \"mix-blend-luminosity\",\n \"mix-blend-plus-darker\",\n \"mix-blend-plus-lighter\",\n \"bg-blend-normal\",\n \"bg-blend-multiply\",\n \"bg-blend-screen\",\n \"bg-blend-overlay\",\n \"bg-blend-darken\",\n \"bg-blend-lighten\",\n \"bg-blend-color-dodge\",\n \"bg-blend-color-burn\",\n \"bg-blend-hard-light\",\n \"bg-blend-soft-light\",\n \"bg-blend-difference\",\n \"bg-blend-exclusion\",\n \"bg-blend-hue\",\n \"bg-blend-saturation\",\n \"bg-blend-color\",\n \"bg-blend-luminosity\",\n \"blur-none\",\n \"blur-sm\",\n \"blur\",\n \"blur-md\",\n \"blur-lg\",\n \"blur-xl\",\n \"blur-2xl\",\n \"blur-3xl\",\n \"brightness-0\",\n \"brightness-50\",\n \"brightness-75\",\n \"brightness-90\",\n \"brightness-95\",\n \"brightness-100\",\n \"brightness-105\",\n \"brightness-110\",\n \"brightness-125\",\n \"brightness-150\",\n \"brightness-200\",\n \"contrast-0\",\n \"contrast-50\",\n \"contrast-75\",\n \"contrast-100\",\n \"contrast-125\",\n \"contrast-150\",\n \"contrast-200\",\n \"drop-shadow-sm\",\n \"drop-shadow\",\n \"drop-shadow-md\",\n \"drop-shadow-lg\",\n \"drop-shadow-xl\",\n \"drop-shadow-2xl\",\n \"drop-shadow-none\",\n \"grayscale-0\",\n \"grayscale\",\n \"hue-rotate-0\",\n \"hue-rotate-15\",\n \"hue-rotate-30\",\n \"hue-rotate-60\",\n \"hue-rotate-90\",\n \"hue-rotate-180\",\n \"invert-0\",\n \"invert\",\n \"saturate-0\",\n \"saturate-50\",\n \"saturate-100\",\n \"saturate-150\",\n \"saturate-200\",\n \"sepia-0\",\n \"sepia\",\n \"backdrop-blur-none\",\n \"backdrop-blur-sm\",\n \"backdrop-blur\",\n \"backdrop-blur-md\",\n \"backdrop-blur-lg\",\n \"backdrop-blur-xl\",\n \"backdrop-blur-2xl\",\n \"backdrop-blur-3xl\",\n \"backdrop-brightness-0\",\n \"backdrop-brightness-50\",\n \"backdrop-brightness-75\",\n \"backdrop-brightness-90\",\n \"backdrop-brightness-95\",\n \"backdrop-brightness-100\",\n \"backdrop-brightness-105\",\n \"backdrop-brightness-110\",\n \"backdrop-brightness-125\",\n \"backdrop-brightness-150\",\n \"backdrop-brightness-200\",\n \"backdrop-contrast-0\",\n \"backdrop-contrast-50\",\n \"backdrop-contrast-75\",\n \"backdrop-contrast-100\",\n \"backdrop-contrast-125\",\n \"backdrop-contrast-150\",\n \"backdrop-contrast-200\",\n \"backdrop-grayscale-0\",\n \"backdrop-grayscale\",\n \"backdrop-hue-rotate-0\",\n \"backdrop-hue-rotate-15\",\n \"backdrop-hue-rotate-30\",\n \"backdrop-hue-rotate-60\",\n \"backdrop-hue-rotate-90\",\n \"backdrop-hue-rotate-180\",\n \"backdrop-invert-0\",\n \"backdrop-invert\",\n \"backdrop-opacity-0\",\n \"backdrop-opacity-5\",\n \"backdrop-opacity-10\",\n \"backdrop-opacity-15\",\n \"backdrop-opacity-20\",\n \"backdrop-opacity-25\",\n \"backdrop-opacity-30\",\n \"backdrop-opacity-35\",\n \"backdrop-opacity-40\",\n \"backdrop-opacity-45\",\n \"backdrop-opacity-50\",\n \"backdrop-opacity-55\",\n \"backdrop-opacity-60\",\n \"backdrop-opacity-65\",\n \"backdrop-opacity-70\",\n \"backdrop-opacity-75\",\n \"backdrop-opacity-80\",\n \"backdrop-opacity-85\",\n \"backdrop-opacity-90\",\n \"backdrop-opacity-95\",\n \"backdrop-opacity-100\",\n \"backdrop-saturate-0\",\n \"backdrop-saturate-50\",\n \"backdrop-saturate-100\",\n \"backdrop-saturate-150\",\n \"backdrop-saturate-200\",\n \"backdrop-sepia-0\",\n \"backdrop-sepia\",\n \"border-collapse\",\n \"border-separate\",\n \"Indiana\",\n \"Ohio\",\n \"Michigan\",\n \"Indiana\",\n \"Ohio\",\n \"Michigan\",\n \"border-spacing-0\",\n \"border-spacing-x-0\",\n \"border-spacing-y-0\",\n \"border-spacing-px\",\n \"border-spacing-x-px\",\n \"border-spacing-y-px\",\n \"border-spacing-0.5\",\n \"border-spacing-x-0.5\",\n \"border-spacing-y-0.5\",\n \"border-spacing-1\",\n \"border-spacing-x-1\",\n \"border-spacing-y-1\",\n \"border-spacing-1.5\",\n \"border-spacing-x-1.5\",\n \"border-spacing-y-1.5\",\n \"border-spacing-2\",\n \"border-spacing-x-2\",\n \"border-spacing-y-2\",\n \"border-spacing-2.5\",\n \"border-spacing-x-2.5\",\n \"border-spacing-y-2.5\",\n \"border-spacing-3\",\n \"border-spacing-x-3\",\n \"border-spacing-y-3\",\n \"border-spacing-3.5\",\n \"border-spacing-x-3.5\",\n \"border-spacing-y-3.5\",\n \"border-spacing-4\",\n \"border-spacing-x-4\",\n \"border-spacing-y-4\",\n \"border-spacing-5\",\n \"border-spacing-x-5\",\n \"border-spacing-y-5\",\n \"border-spacing-6\",\n \"border-spacing-x-6\",\n \"border-spacing-y-6\",\n \"border-spacing-7\",\n \"border-spacing-x-7\",\n \"border-spacing-y-7\",\n \"border-spacing-8\",\n \"border-spacing-x-8\",\n \"border-spacing-y-8\",\n \"border-spacing-9\",\n \"border-spacing-x-9\",\n \"border-spacing-y-9\",\n \"border-spacing-10\",\n \"border-spacing-x-10\",\n \"border-spacing-y-10\",\n \"border-spacing-11\",\n \"border-spacing-x-11\",\n \"border-spacing-y-11\",\n \"border-spacing-12\",\n \"border-spacing-x-12\",\n \"border-spacing-y-12\",\n \"border-spacing-14\",\n \"border-spacing-x-14\",\n \"border-spacing-y-14\",\n \"border-spacing-16\",\n \"border-spacing-x-16\",\n \"border-spacing-y-16\",\n \"border-spacing-20\",\n \"border-spacing-x-20\",\n \"border-spacing-y-20\",\n \"border-spacing-24\",\n \"border-spacing-x-24\",\n \"border-spacing-y-24\",\n \"border-spacing-28\",\n \"border-spacing-x-28\",\n \"border-spacing-y-28\",\n \"border-spacing-32\",\n \"border-spacing-x-32\",\n \"border-spacing-y-32\",\n \"border-spacing-36\",\n \"border-spacing-x-36\",\n \"border-spacing-y-36\",\n \"border-spacing-40\",\n \"border-spacing-x-40\",\n \"border-spacing-y-40\",\n \"border-spacing-44\",\n \"border-spacing-x-44\",\n \"border-spacing-y-44\",\n \"border-spacing-48\",\n \"border-spacing-x-48\",\n \"border-spacing-y-48\",\n \"border-spacing-52\",\n \"border-spacing-x-52\",\n \"border-spacing-y-52\",\n \"border-spacing-56\",\n \"border-spacing-x-56\",\n \"border-spacing-y-56\",\n \"border-spacing-60\",\n \"border-spacing-x-60\",\n \"border-spacing-y-60\",\n \"border-spacing-64\",\n \"border-spacing-x-64\",\n \"border-spacing-y-64\",\n \"border-spacing-72\",\n \"border-spacing-x-72\",\n \"border-spacing-y-72\",\n \"border-spacing-80\",\n \"border-spacing-x-80\",\n \"border-spacing-y-80\",\n \"border-spacing-96\",\n \"border-spacing-x-96\",\n \"border-spacing-y-96\",\n \"Indiana\",\n \"Ohio\",\n \"Michigan\",\n \"table-auto\",\n \"table-fixed\",\n \"caption-top\",\n \"caption-bottom\",\n \"transition-none\",\n \"transition-all\",\n \"transition\",\n \"transition-colors\",\n \"transition-opacity\",\n \"transition-shadow\",\n \"transition-transform\",\n \"duration-0\",\n \"duration-75\",\n \"duration-100\",\n \"duration-150\",\n \"duration-200\",\n \"duration-300\",\n \"duration-500\",\n \"duration-700\",\n \"duration-1000\",\n \"ease-linear\",\n \"ease-in\",\n \"ease-out\",\n \"ease-in-out\",\n \"delay-0\",\n \"delay-75\",\n \"delay-100\",\n \"delay-150\",\n \"delay-200\",\n \"delay-300\",\n \"delay-500\",\n \"delay-700\",\n \"delay-1000\",\n \"animate-none\",\n \"animate-spin\",\n \"animate-ping\",\n \"animate-pulse\",\n \"animate-bounce\",\n \"scale-0\",\n \"scale-x-0\",\n \"scale-y-0\",\n \"scale-50\",\n \"scale-x-50\",\n \"scale-y-50\",\n \"scale-75\",\n \"scale-x-75\",\n \"scale-y-75\",\n \"scale-90\",\n \"scale-x-90\",\n \"scale-y-90\",\n \"scale-95\",\n \"scale-x-95\",\n \"scale-y-95\",\n \"scale-100\",\n \"scale-x-100\",\n \"scale-y-100\",\n \"scale-105\",\n \"scale-x-105\",\n \"scale-y-105\",\n \"scale-110\",\n \"scale-x-110\",\n \"scale-y-110\",\n \"scale-125\",\n \"scale-x-125\",\n \"scale-y-125\",\n \"scale-150\",\n \"scale-x-150\",\n \"scale-y-150\",\n \"rotate-0\",\n \"rotate-1\",\n \"rotate-2\",\n \"rotate-3\",\n \"rotate-6\",\n \"rotate-12\",\n \"rotate-45\",\n \"rotate-90\",\n \"rotate-180\",\n \"translate-x-0\",\n \"translate-y-0\",\n \"translate-x-px\",\n \"translate-y-px\",\n \"translate-x-0.5\",\n \"translate-y-0.5\",\n \"translate-x-1\",\n \"translate-y-1\",\n \"translate-x-1.5\",\n \"translate-y-1.5\",\n \"translate-x-2\",\n \"translate-y-2\",\n \"translate-x-2.5\",\n \"translate-y-2.5\",\n \"translate-x-3\",\n \"translate-y-3\",\n \"translate-x-3.5\",\n \"translate-y-3.5\",\n \"translate-x-4\",\n \"translate-y-4\",\n \"translate-x-5\",\n \"translate-y-5\",\n \"translate-x-6\",\n \"translate-y-6\",\n \"translate-x-7\",\n \"translate-y-7\",\n \"translate-x-8\",\n \"translate-y-8\",\n \"translate-x-9\",\n \"translate-y-9\",\n \"translate-x-10\",\n \"translate-y-10\",\n \"translate-x-11\",\n \"translate-y-11\",\n \"translate-x-12\",\n \"translate-y-12\",\n \"translate-x-14\",\n \"translate-y-14\",\n \"translate-x-16\",\n \"translate-y-16\",\n \"translate-x-20\",\n \"translate-y-20\",\n \"translate-x-24\",\n \"translate-y-24\",\n \"translate-x-28\",\n \"translate-y-28\",\n \"translate-x-32\",\n \"translate-y-32\",\n \"translate-x-36\",\n \"translate-y-36\",\n \"translate-x-40\",\n \"translate-y-40\",\n \"translate-x-44\",\n \"translate-y-44\",\n \"translate-x-48\",\n \"translate-y-48\",\n \"translate-x-52\",\n \"translate-y-52\",\n \"translate-x-56\",\n \"translate-y-56\",\n \"translate-x-60\",\n \"translate-y-60\",\n \"translate-x-64\",\n \"translate-y-64\",\n \"translate-x-72\",\n \"translate-y-72\",\n \"translate-x-80\",\n \"translate-y-80\",\n \"translate-x-96\",\n \"translate-y-96\",\n \"translate-x-1/2\",\n \"translate-x-1/3\",\n \"translate-x-2/3\",\n \"translate-x-1/4\",\n \"translate-x-2/4\",\n \"translate-x-3/4\",\n \"translate-x-full\",\n \"translate-y-1/2\",\n \"translate-y-1/3\",\n \"translate-y-2/3\",\n \"translate-y-1/4\",\n \"translate-y-2/4\",\n \"translate-y-3/4\",\n \"translate-y-full\",\n \"skew-x-0\",\n \"skew-y-0\",\n \"skew-x-1\",\n \"skew-y-1\",\n \"skew-x-2\",\n \"skew-y-2\",\n \"skew-x-3\",\n \"skew-y-3\",\n \"skew-x-6\",\n \"skew-y-6\",\n \"skew-x-12\",\n \"skew-y-12\",\n \"origin-center\",\n \"origin-top\",\n \"origin-top-right\",\n \"origin-right\",\n \"origin-bottom-right\",\n \"origin-bottom\",\n \"origin-bottom-left\",\n \"origin-left\",\n \"origin-top-left\",\n \"accent-inherit\",\n \"accent-current\",\n \"accent-transparent\",\n \"accent-black\",\n \"accent-white\",\n \"accent-slate-50\",\n \"accent-slate-100\",\n \"accent-slate-200\",\n \"accent-slate-300\",\n \"accent-slate-400\",\n \"accent-slate-500\",\n \"accent-slate-600\",\n \"accent-slate-700\",\n \"accent-slate-800\",\n \"accent-slate-900\",\n \"accent-slate-950\",\n \"accent-gray-50\",\n \"accent-gray-100\",\n \"accent-gray-200\",\n \"accent-gray-300\",\n \"accent-gray-400\",\n \"accent-gray-500\",\n \"accent-gray-600\",\n \"accent-gray-700\",\n \"accent-gray-800\",\n \"accent-gray-900\",\n \"accent-gray-950\",\n \"accent-zinc-50\",\n \"accent-zinc-100\",\n \"accent-zinc-200\",\n \"accent-zinc-300\",\n \"accent-zinc-400\",\n \"accent-zinc-500\",\n \"accent-zinc-600\",\n \"accent-zinc-700\",\n \"accent-zinc-800\",\n \"accent-zinc-900\",\n \"accent-zinc-950\",\n \"accent-neutral-50\",\n \"accent-neutral-100\",\n \"accent-neutral-200\",\n \"accent-neutral-300\",\n \"accent-neutral-400\",\n \"accent-neutral-500\",\n \"accent-neutral-600\",\n \"accent-neutral-700\",\n \"accent-neutral-800\",\n \"accent-neutral-900\",\n \"accent-neutral-950\",\n \"accent-stone-50\",\n \"accent-stone-100\",\n \"accent-stone-200\",\n \"accent-stone-300\",\n \"accent-stone-400\",\n \"accent-stone-500\",\n \"accent-stone-600\",\n \"accent-stone-700\",\n \"accent-stone-800\",\n \"accent-stone-900\",\n \"accent-stone-950\",\n \"accent-red-50\",\n \"accent-red-100\",\n \"accent-red-200\",\n \"accent-red-300\",\n \"accent-red-400\",\n \"accent-red-500\",\n \"accent-red-600\",\n \"accent-red-700\",\n \"accent-red-800\",\n \"accent-red-900\",\n \"accent-red-950\",\n \"accent-orange-50\",\n \"accent-orange-100\",\n \"accent-orange-200\",\n \"accent-orange-300\",\n \"accent-orange-400\",\n \"accent-orange-500\",\n \"accent-orange-600\",\n \"accent-orange-700\",\n \"accent-orange-800\",\n \"accent-orange-900\",\n \"accent-orange-950\",\n \"accent-amber-50\",\n \"accent-amber-100\",\n \"accent-amber-200\",\n \"accent-amber-300\",\n \"accent-amber-400\",\n \"accent-amber-500\",\n \"accent-amber-600\",\n \"accent-amber-700\",\n \"accent-amber-800\",\n \"accent-amber-900\",\n \"accent-amber-950\",\n \"accent-yellow-50\",\n \"accent-yellow-100\",\n \"accent-yellow-200\",\n \"accent-yellow-300\",\n \"accent-yellow-400\",\n \"accent-yellow-500\",\n \"accent-yellow-600\",\n \"accent-yellow-700\",\n \"accent-yellow-800\",\n \"accent-yellow-900\",\n \"accent-yellow-950\",\n \"accent-lime-50\",\n \"accent-lime-100\",\n \"accent-lime-200\",\n \"accent-lime-300\",\n \"accent-lime-400\",\n \"accent-lime-500\",\n \"accent-lime-600\",\n \"accent-lime-700\",\n \"accent-lime-800\",\n \"accent-lime-900\",\n \"accent-lime-950\",\n \"accent-green-50\",\n \"accent-green-100\",\n \"accent-green-200\",\n \"accent-green-300\",\n \"accent-green-400\",\n \"accent-green-500\",\n \"accent-green-600\",\n \"accent-green-700\",\n \"accent-green-800\",\n \"accent-green-900\",\n \"accent-green-950\",\n \"accent-emerald-50\",\n \"accent-emerald-100\",\n \"accent-emerald-200\",\n \"accent-emerald-300\",\n \"accent-emerald-400\",\n \"accent-emerald-500\",\n \"accent-emerald-600\",\n \"accent-emerald-700\",\n \"accent-emerald-800\",\n \"accent-emerald-900\",\n \"accent-emerald-950\",\n \"accent-teal-50\",\n \"accent-teal-100\",\n \"accent-teal-200\",\n \"accent-teal-300\",\n \"accent-teal-400\",\n \"accent-teal-500\",\n \"accent-teal-600\",\n \"accent-teal-700\",\n \"accent-teal-800\",\n \"accent-teal-900\",\n \"accent-teal-950\",\n \"accent-cyan-50\",\n \"accent-cyan-100\",\n \"accent-cyan-200\",\n \"accent-cyan-300\",\n \"accent-cyan-400\",\n \"accent-cyan-500\",\n \"accent-cyan-600\",\n \"accent-cyan-700\",\n \"accent-cyan-800\",\n \"accent-cyan-900\",\n \"accent-cyan-950\",\n \"accent-sky-50\",\n \"accent-sky-100\",\n \"accent-sky-200\",\n \"accent-sky-300\",\n \"accent-sky-400\",\n \"accent-sky-500\",\n \"accent-sky-600\",\n \"accent-sky-700\",\n \"accent-sky-800\",\n \"accent-sky-900\",\n \"accent-sky-950\",\n \"accent-blue-50\",\n \"accent-blue-100\",\n \"accent-blue-200\",\n \"accent-blue-300\",\n \"accent-blue-400\",\n \"accent-blue-500\",\n \"accent-blue-600\",\n \"accent-blue-700\",\n \"accent-blue-800\",\n \"accent-blue-900\",\n \"accent-blue-950\",\n \"accent-indigo-50\",\n \"accent-indigo-100\",\n \"accent-indigo-200\",\n \"accent-indigo-300\",\n \"accent-indigo-400\",\n \"accent-indigo-500\",\n \"accent-indigo-600\",\n \"accent-indigo-700\",\n \"accent-indigo-800\",\n \"accent-indigo-900\",\n \"accent-indigo-950\",\n \"accent-violet-50\",\n \"accent-violet-100\",\n \"accent-violet-200\",\n \"accent-violet-300\",\n \"accent-violet-400\",\n \"accent-violet-500\",\n \"accent-violet-600\",\n \"accent-violet-700\",\n \"accent-violet-800\",\n \"accent-violet-900\",\n \"accent-violet-950\",\n \"accent-purple-50\",\n \"accent-purple-100\",\n \"accent-purple-200\",\n \"accent-purple-300\",\n \"accent-purple-400\",\n \"accent-purple-500\",\n \"accent-purple-600\",\n \"accent-purple-700\",\n \"accent-purple-800\",\n \"accent-purple-900\",\n \"accent-purple-950\",\n \"accent-fuchsia-50\",\n \"accent-fuchsia-100\",\n \"accent-fuchsia-200\",\n \"accent-fuchsia-300\",\n \"accent-fuchsia-400\",\n \"accent-fuchsia-500\",\n \"accent-fuchsia-600\",\n \"accent-fuchsia-700\",\n \"accent-fuchsia-800\",\n \"accent-fuchsia-900\",\n \"accent-fuchsia-950\",\n \"accent-pink-50\",\n \"accent-pink-100\",\n \"accent-pink-200\",\n \"accent-pink-300\",\n \"accent-pink-400\",\n \"accent-pink-500\",\n \"accent-pink-600\",\n \"accent-pink-700\",\n \"accent-pink-800\",\n \"accent-pink-900\",\n \"accent-pink-950\",\n \"accent-rose-50\",\n \"accent-rose-100\",\n \"accent-rose-200\",\n \"accent-rose-300\",\n \"accent-rose-400\",\n \"accent-rose-500\",\n \"accent-rose-600\",\n \"accent-rose-700\",\n \"accent-rose-800\",\n \"accent-rose-900\",\n \"accent-rose-950\",\n \"accent-auto\",\n \"appearance-none\",\n \"appearance-auto\",\n \"cursor-auto\",\n \"cursor-default\",\n \"cursor-pointer\",\n \"cursor-wait\",\n \"cursor-text\",\n \"cursor-move\",\n \"cursor-help\",\n \"cursor-not-allowed\",\n \"cursor-none\",\n \"cursor-context-menu\",\n \"cursor-progress\",\n \"cursor-cell\",\n \"cursor-crosshair\",\n \"cursor-vertical-text\",\n \"cursor-alias\",\n \"cursor-copy\",\n \"cursor-no-drop\",\n \"cursor-grab\",\n \"cursor-grabbing\",\n \"cursor-all-scroll\",\n \"cursor-col-resize\",\n \"cursor-row-resize\",\n \"cursor-n-resize\",\n \"cursor-e-resize\",\n \"cursor-s-resize\",\n \"cursor-w-resize\",\n \"cursor-ne-resize\",\n \"cursor-nw-resize\",\n \"cursor-se-resize\",\n \"cursor-sw-resize\",\n \"cursor-ew-resize\",\n \"cursor-ns-resize\",\n \"cursor-nesw-resize\",\n \"cursor-nwse-resize\",\n \"cursor-zoom-in\",\n \"cursor-zoom-out\",\n \"caret-inherit\",\n \"caret-current\",\n \"caret-transparent\",\n \"caret-black\",\n \"caret-white\",\n \"caret-slate-50\",\n \"caret-slate-100\",\n \"caret-slate-200\",\n \"caret-slate-300\",\n \"caret-slate-400\",\n \"caret-slate-500\",\n \"caret-slate-600\",\n \"caret-slate-700\",\n \"caret-slate-800\",\n \"caret-slate-900\",\n \"caret-slate-950\",\n \"caret-gray-50\",\n \"caret-gray-100\",\n \"caret-gray-200\",\n \"caret-gray-300\",\n \"caret-gray-400\",\n \"caret-gray-500\",\n \"caret-gray-600\",\n \"caret-gray-700\",\n \"caret-gray-800\",\n \"caret-gray-900\",\n \"caret-gray-950\",\n \"caret-zinc-50\",\n \"caret-zinc-100\",\n \"caret-zinc-200\",\n \"caret-zinc-300\",\n \"caret-zinc-400\",\n \"caret-zinc-500\",\n \"caret-zinc-600\",\n \"caret-zinc-700\",\n \"caret-zinc-800\",\n \"caret-zinc-900\",\n \"caret-zinc-950\",\n \"caret-neutral-50\",\n \"caret-neutral-100\",\n \"caret-neutral-200\",\n \"caret-neutral-300\",\n \"caret-neutral-400\",\n \"caret-neutral-500\",\n \"caret-neutral-600\",\n \"caret-neutral-700\",\n \"caret-neutral-800\",\n \"caret-neutral-900\",\n \"caret-neutral-950\",\n \"caret-stone-50\",\n \"caret-stone-100\",\n \"caret-stone-200\",\n \"caret-stone-300\",\n \"caret-stone-400\",\n \"caret-stone-500\",\n \"caret-stone-600\",\n \"caret-stone-700\",\n \"caret-stone-800\",\n \"caret-stone-900\",\n \"caret-stone-950\",\n \"caret-red-50\",\n \"caret-red-100\",\n \"caret-red-200\",\n \"caret-red-300\",\n \"caret-red-400\",\n \"caret-red-500\",\n \"caret-red-600\",\n \"caret-red-700\",\n \"caret-red-800\",\n \"caret-red-900\",\n \"caret-red-950\",\n \"caret-orange-50\",\n \"caret-orange-100\",\n \"caret-orange-200\",\n \"caret-orange-300\",\n \"caret-orange-400\",\n \"caret-orange-500\",\n \"caret-orange-600\",\n \"caret-orange-700\",\n \"caret-orange-800\",\n \"caret-orange-900\",\n \"caret-orange-950\",\n \"caret-amber-50\",\n \"caret-amber-100\",\n \"caret-amber-200\",\n \"caret-amber-300\",\n \"caret-amber-400\",\n \"caret-amber-500\",\n \"caret-amber-600\",\n \"caret-amber-700\",\n \"caret-amber-800\",\n \"caret-amber-900\",\n \"caret-amber-950\",\n \"caret-yellow-50\",\n \"caret-yellow-100\",\n \"caret-yellow-200\",\n \"caret-yellow-300\",\n \"caret-yellow-400\",\n \"caret-yellow-500\",\n \"caret-yellow-600\",\n \"caret-yellow-700\",\n \"caret-yellow-800\",\n \"caret-yellow-900\",\n \"caret-yellow-950\",\n \"caret-lime-50\",\n \"caret-lime-100\",\n \"caret-lime-200\",\n \"caret-lime-300\",\n \"caret-lime-400\",\n \"caret-lime-500\",\n \"caret-lime-600\",\n \"caret-lime-700\",\n \"caret-lime-800\",\n \"caret-lime-900\",\n \"caret-lime-950\",\n \"caret-green-50\",\n \"caret-green-100\",\n \"caret-green-200\",\n \"caret-green-300\",\n \"caret-green-400\",\n \"caret-green-500\",\n \"caret-green-600\",\n \"caret-green-700\",\n \"caret-green-800\",\n \"caret-green-900\",\n \"caret-green-950\",\n \"caret-emerald-50\",\n \"caret-emerald-100\",\n \"caret-emerald-200\",\n \"caret-emerald-300\",\n \"caret-emerald-400\",\n \"caret-emerald-500\",\n \"caret-emerald-600\",\n \"caret-emerald-700\",\n \"caret-emerald-800\",\n \"caret-emerald-900\",\n \"caret-emerald-950\",\n \"caret-teal-50\",\n \"caret-teal-100\",\n \"caret-teal-200\",\n \"caret-teal-300\",\n \"caret-teal-400\",\n \"caret-teal-500\",\n \"caret-teal-600\",\n \"caret-teal-700\",\n \"caret-teal-800\",\n \"caret-teal-900\",\n \"caret-teal-950\",\n \"caret-cyan-50\",\n \"caret-cyan-100\",\n \"caret-cyan-200\",\n \"caret-cyan-300\",\n \"caret-cyan-400\",\n \"caret-cyan-500\",\n \"caret-cyan-600\",\n \"caret-cyan-700\",\n \"caret-cyan-800\",\n \"caret-cyan-900\",\n \"caret-cyan-950\",\n \"caret-sky-50\",\n \"caret-sky-100\",\n \"caret-sky-200\",\n \"caret-sky-300\",\n \"caret-sky-400\",\n \"caret-sky-500\",\n \"caret-sky-600\",\n \"caret-sky-700\",\n \"caret-sky-800\",\n \"caret-sky-900\",\n \"caret-sky-950\",\n \"caret-blue-50\",\n \"caret-blue-100\",\n \"caret-blue-200\",\n \"caret-blue-300\",\n \"caret-blue-400\",\n \"caret-blue-500\",\n \"caret-blue-600\",\n \"caret-blue-700\",\n \"caret-blue-800\",\n \"caret-blue-900\",\n \"caret-blue-950\",\n \"caret-indigo-50\",\n \"caret-indigo-100\",\n \"caret-indigo-200\",\n \"caret-indigo-300\",\n \"caret-indigo-400\",\n \"caret-indigo-500\",\n \"caret-indigo-600\",\n \"caret-indigo-700\",\n \"caret-indigo-800\",\n \"caret-indigo-900\",\n \"caret-indigo-950\",\n \"caret-violet-50\",\n \"caret-violet-100\",\n \"caret-violet-200\",\n \"caret-violet-300\",\n \"caret-violet-400\",\n \"caret-violet-500\",\n \"caret-violet-600\",\n \"caret-violet-700\",\n \"caret-violet-800\",\n \"caret-violet-900\",\n \"caret-violet-950\",\n \"caret-purple-50\",\n \"caret-purple-100\",\n \"caret-purple-200\",\n \"caret-purple-300\",\n \"caret-purple-400\",\n \"caret-purple-500\",\n \"caret-purple-600\",\n \"caret-purple-700\",\n \"caret-purple-800\",\n \"caret-purple-900\",\n \"caret-purple-950\",\n \"caret-fuchsia-50\",\n \"caret-fuchsia-100\",\n \"caret-fuchsia-200\",\n \"caret-fuchsia-300\",\n \"caret-fuchsia-400\",\n \"caret-fuchsia-500\",\n \"caret-fuchsia-600\",\n \"caret-fuchsia-700\",\n \"caret-fuchsia-800\",\n \"caret-fuchsia-900\",\n \"caret-fuchsia-950\",\n \"caret-pink-50\",\n \"caret-pink-100\",\n \"caret-pink-200\",\n \"caret-pink-300\",\n \"caret-pink-400\",\n \"caret-pink-500\",\n \"caret-pink-600\",\n \"caret-pink-700\",\n \"caret-pink-800\",\n \"caret-pink-900\",\n \"caret-pink-950\",\n \"caret-rose-50\",\n \"caret-rose-100\",\n \"caret-rose-200\",\n \"caret-rose-300\",\n \"caret-rose-400\",\n \"caret-rose-500\",\n \"caret-rose-600\",\n \"caret-rose-700\",\n \"caret-rose-800\",\n \"caret-rose-900\",\n \"caret-rose-950\",\n \"pointer-events-none\",\n \"pointer-events-auto\",\n \"resize-none\",\n \"resize-y\",\n \"resize-x\",\n \"resize\",\n \"scroll-auto\",\n \"scroll-smooth\",\n \"scroll-m-0\",\n \"scroll-mx-0\",\n \"scroll-my-0\",\n \"scroll-ms-0\",\n \"scroll-me-0\",\n \"scroll-mt-0\",\n \"scroll-mr-0\",\n \"scroll-mb-0\",\n \"scroll-ml-0\",\n \"scroll-m-px\",\n \"scroll-mx-px\",\n \"scroll-my-px\",\n \"scroll-ms-px\",\n \"scroll-me-px\",\n \"scroll-mt-px\",\n \"scroll-mr-px\",\n \"scroll-mb-px\",\n \"scroll-ml-px\",\n \"scroll-m-0.5\",\n \"scroll-mx-0.5\",\n \"scroll-my-0.5\",\n \"scroll-ms-0.5\",\n \"scroll-me-0.5\",\n \"scroll-mt-0.5\",\n \"scroll-mr-0.5\",\n \"scroll-mb-0.5\",\n \"scroll-ml-0.5\",\n \"scroll-m-1\",\n \"scroll-mx-1\",\n \"scroll-my-1\",\n \"scroll-ms-1\",\n \"scroll-me-1\",\n \"scroll-mt-1\",\n \"scroll-mr-1\",\n \"scroll-mb-1\",\n \"scroll-ml-1\",\n \"scroll-m-1.5\",\n \"scroll-mx-1.5\",\n \"scroll-my-1.5\",\n \"scroll-ms-1.5\",\n \"scroll-me-1.5\",\n \"scroll-mt-1.5\",\n \"scroll-mr-1.5\",\n \"scroll-mb-1.5\",\n \"scroll-ml-1.5\",\n \"scroll-m-2\",\n \"scroll-mx-2\",\n \"scroll-my-2\",\n \"scroll-ms-2\",\n \"scroll-me-2\",\n \"scroll-mt-2\",\n \"scroll-mr-2\",\n \"scroll-mb-2\",\n \"scroll-ml-2\",\n \"scroll-m-2.5\",\n \"scroll-mx-2.5\",\n \"scroll-my-2.5\",\n \"scroll-ms-2.5\",\n \"scroll-me-2.5\",\n \"scroll-mt-2.5\",\n \"scroll-mr-2.5\",\n \"scroll-mb-2.5\",\n \"scroll-ml-2.5\",\n \"scroll-m-3\",\n \"scroll-mx-3\",\n \"scroll-my-3\",\n \"scroll-ms-3\",\n \"scroll-me-3\",\n \"scroll-mt-3\",\n \"scroll-mr-3\",\n \"scroll-mb-3\",\n \"scroll-ml-3\",\n \"scroll-m-3.5\",\n \"scroll-mx-3.5\",\n \"scroll-my-3.5\",\n \"scroll-ms-3.5\",\n \"scroll-me-3.5\",\n \"scroll-mt-3.5\",\n \"scroll-mr-3.5\",\n \"scroll-mb-3.5\",\n \"scroll-ml-3.5\",\n \"scroll-m-4\",\n \"scroll-mx-4\",\n \"scroll-my-4\",\n \"scroll-ms-4\",\n \"scroll-me-4\",\n \"scroll-mt-4\",\n \"scroll-mr-4\",\n \"scroll-mb-4\",\n \"scroll-ml-4\",\n \"scroll-m-5\",\n \"scroll-mx-5\",\n \"scroll-my-5\",\n \"scroll-ms-5\",\n \"scroll-me-5\",\n \"scroll-mt-5\",\n \"scroll-mr-5\",\n \"scroll-mb-5\",\n \"scroll-ml-5\",\n \"scroll-m-6\",\n \"scroll-mx-6\",\n \"scroll-my-6\",\n \"scroll-ms-6\",\n \"scroll-me-6\",\n \"scroll-mt-6\",\n \"scroll-mr-6\",\n \"scroll-mb-6\",\n \"scroll-ml-6\",\n \"scroll-m-7\",\n \"scroll-mx-7\",\n \"scroll-my-7\",\n \"scroll-ms-7\",\n \"scroll-me-7\",\n \"scroll-mt-7\",\n \"scroll-mr-7\",\n \"scroll-mb-7\",\n \"scroll-ml-7\",\n \"scroll-m-8\",\n \"scroll-mx-8\",\n \"scroll-my-8\",\n \"scroll-ms-8\",\n \"scroll-me-8\",\n \"scroll-mt-8\",\n \"scroll-mr-8\",\n \"scroll-mb-8\",\n \"scroll-ml-8\",\n \"scroll-m-9\",\n \"scroll-mx-9\",\n \"scroll-my-9\",\n \"scroll-ms-9\",\n \"scroll-me-9\",\n \"scroll-mt-9\",\n \"scroll-mr-9\",\n \"scroll-mb-9\",\n \"scroll-ml-9\",\n \"scroll-m-10\",\n \"scroll-mx-10\",\n \"scroll-my-10\",\n \"scroll-ms-10\",\n \"scroll-me-10\",\n \"scroll-mt-10\",\n \"scroll-mr-10\",\n \"scroll-mb-10\",\n \"scroll-ml-10\",\n \"scroll-m-11\",\n \"scroll-mx-11\",\n \"scroll-my-11\",\n \"scroll-ms-11\",\n \"scroll-me-11\",\n \"scroll-mt-11\",\n \"scroll-mr-11\",\n \"scroll-mb-11\",\n \"scroll-ml-11\",\n \"scroll-m-12\",\n \"scroll-mx-12\",\n \"scroll-my-12\",\n \"scroll-ms-12\",\n \"scroll-me-12\",\n \"scroll-mt-12\",\n \"scroll-mr-12\",\n \"scroll-mb-12\",\n \"scroll-ml-12\",\n \"scroll-m-14\",\n \"scroll-mx-14\",\n \"scroll-my-14\",\n \"scroll-ms-14\",\n \"scroll-me-14\",\n \"scroll-mt-14\",\n \"scroll-mr-14\",\n \"scroll-mb-14\",\n \"scroll-ml-14\",\n \"scroll-m-16\",\n \"scroll-mx-16\",\n \"scroll-my-16\",\n \"scroll-ms-16\",\n \"scroll-me-16\",\n \"scroll-mt-16\",\n \"scroll-mr-16\",\n \"scroll-mb-16\",\n \"scroll-ml-16\",\n \"scroll-m-20\",\n \"scroll-mx-20\",\n \"scroll-my-20\",\n \"scroll-ms-20\",\n \"scroll-me-20\",\n \"scroll-mt-20\",\n \"scroll-mr-20\",\n \"scroll-mb-20\",\n \"scroll-ml-20\",\n \"scroll-m-24\",\n \"scroll-mx-24\",\n \"scroll-my-24\",\n \"scroll-ms-24\",\n \"scroll-me-24\",\n \"scroll-mt-24\",\n \"scroll-mr-24\",\n \"scroll-mb-24\",\n \"scroll-ml-24\",\n \"scroll-m-28\",\n \"scroll-mx-28\",\n \"scroll-my-28\",\n \"scroll-ms-28\",\n \"scroll-me-28\",\n \"scroll-mt-28\",\n \"scroll-mr-28\",\n \"scroll-mb-28\",\n \"scroll-ml-28\",\n \"scroll-m-32\",\n \"scroll-mx-32\",\n \"scroll-my-32\",\n \"scroll-ms-32\",\n \"scroll-me-32\",\n \"scroll-mt-32\",\n \"scroll-mr-32\",\n \"scroll-mb-32\",\n \"scroll-ml-32\",\n \"scroll-m-36\",\n \"scroll-mx-36\",\n \"scroll-my-36\",\n \"scroll-ms-36\",\n \"scroll-me-36\",\n \"scroll-mt-36\",\n \"scroll-mr-36\",\n \"scroll-mb-36\",\n \"scroll-ml-36\",\n \"scroll-m-40\",\n \"scroll-mx-40\",\n \"scroll-my-40\",\n \"scroll-ms-40\",\n \"scroll-me-40\",\n \"scroll-mt-40\",\n \"scroll-mr-40\",\n \"scroll-mb-40\",\n \"scroll-ml-40\",\n \"scroll-m-44\",\n \"scroll-mx-44\",\n \"scroll-my-44\",\n \"scroll-ms-44\",\n \"scroll-me-44\",\n \"scroll-mt-44\",\n \"scroll-mr-44\",\n \"scroll-mb-44\",\n \"scroll-ml-44\",\n \"scroll-m-48\",\n \"scroll-mx-48\",\n \"scroll-my-48\",\n \"scroll-ms-48\",\n \"scroll-me-48\",\n \"scroll-mt-48\",\n \"scroll-mr-48\",\n \"scroll-mb-48\",\n \"scroll-ml-48\",\n \"scroll-m-52\",\n \"scroll-mx-52\",\n \"scroll-my-52\",\n \"scroll-ms-52\",\n \"scroll-me-52\",\n \"scroll-mt-52\",\n \"scroll-mr-52\",\n \"scroll-mb-52\",\n \"scroll-ml-52\",\n \"scroll-m-56\",\n \"scroll-mx-56\",\n \"scroll-my-56\",\n \"scroll-ms-56\",\n \"scroll-me-56\",\n \"scroll-mt-56\",\n \"scroll-mr-56\",\n \"scroll-mb-56\",\n \"scroll-ml-56\",\n \"scroll-m-60\",\n \"scroll-mx-60\",\n \"scroll-my-60\",\n \"scroll-ms-60\",\n \"scroll-me-60\",\n \"scroll-mt-60\",\n \"scroll-mr-60\",\n \"scroll-mb-60\",\n \"scroll-ml-60\",\n \"scroll-m-64\",\n \"scroll-mx-64\",\n \"scroll-my-64\",\n \"scroll-ms-64\",\n \"scroll-me-64\",\n \"scroll-mt-64\",\n \"scroll-mr-64\",\n \"scroll-mb-64\",\n \"scroll-ml-64\",\n \"scroll-m-72\",\n \"scroll-mx-72\",\n \"scroll-my-72\",\n \"scroll-ms-72\",\n \"scroll-me-72\",\n \"scroll-mt-72\",\n \"scroll-mr-72\",\n \"scroll-mb-72\",\n \"scroll-ml-72\",\n \"scroll-m-80\",\n \"scroll-mx-80\",\n \"scroll-my-80\",\n \"scroll-ms-80\",\n \"scroll-me-80\",\n \"scroll-mt-80\",\n \"scroll-mr-80\",\n \"scroll-mb-80\",\n \"scroll-ml-80\",\n \"scroll-m-96\",\n \"scroll-mx-96\",\n \"scroll-my-96\",\n \"scroll-ms-96\",\n \"scroll-me-96\",\n \"scroll-mt-96\",\n \"scroll-mr-96\",\n \"scroll-mb-96\",\n \"scroll-ml-96\",\n \"scroll-p-0\",\n \"scroll-px-0\",\n \"scroll-py-0\",\n \"scroll-ps-0\",\n \"scroll-pe-0\",\n \"scroll-pt-0\",\n \"scroll-pr-0\",\n \"scroll-pb-0\",\n \"scroll-pl-0\",\n \"scroll-p-px\",\n \"scroll-px-px\",\n \"scroll-py-px\",\n \"scroll-ps-px\",\n \"scroll-pe-px\",\n \"scroll-pt-px\",\n \"scroll-pr-px\",\n \"scroll-pb-px\",\n \"scroll-pl-px\",\n \"scroll-p-0.5\",\n \"scroll-px-0.5\",\n \"scroll-py-0.5\",\n \"scroll-ps-0.5\",\n \"scroll-pe-0.5\",\n \"scroll-pt-0.5\",\n \"scroll-pr-0.5\",\n \"scroll-pb-0.5\",\n \"scroll-pl-0.5\",\n \"scroll-p-1\",\n \"scroll-px-1\",\n \"scroll-py-1\",\n \"scroll-ps-1\",\n \"scroll-pe-1\",\n \"scroll-pt-1\",\n \"scroll-pr-1\",\n \"scroll-pb-1\",\n \"scroll-pl-1\",\n \"scroll-p-1.5\",\n \"scroll-px-1.5\",\n \"scroll-py-1.5\",\n \"scroll-ps-1.5\",\n \"scroll-pe-1.5\",\n \"scroll-pt-1.5\",\n \"scroll-pr-1.5\",\n \"scroll-pb-1.5\",\n \"scroll-pl-1.5\",\n \"scroll-p-2\",\n \"scroll-px-2\",\n \"scroll-py-2\",\n \"scroll-ps-2\",\n \"scroll-pe-2\",\n \"scroll-pt-2\",\n \"scroll-pr-2\",\n \"scroll-pb-2\",\n \"scroll-pl-2\",\n \"scroll-p-2.5\",\n \"scroll-px-2.5\",\n \"scroll-py-2.5\",\n \"scroll-ps-2.5\",\n \"scroll-pe-2.5\",\n \"scroll-pt-2.5\",\n \"scroll-pr-2.5\",\n \"scroll-pb-2.5\",\n \"scroll-pl-2.5\",\n \"scroll-p-3\",\n \"scroll-px-3\",\n \"scroll-py-3\",\n \"scroll-ps-3\",\n \"scroll-pe-3\",\n \"scroll-pt-3\",\n \"scroll-pr-3\",\n \"scroll-pb-3\",\n \"scroll-pl-3\",\n \"scroll-p-3.5\",\n \"scroll-px-3.5\",\n \"scroll-py-3.5\",\n \"scroll-ps-3.5\",\n \"scroll-pe-3.5\",\n \"scroll-pt-3.5\",\n \"scroll-pr-3.5\",\n \"scroll-pb-3.5\",\n \"scroll-pl-3.5\",\n \"scroll-p-4\",\n \"scroll-px-4\",\n \"scroll-py-4\",\n \"scroll-ps-4\",\n \"scroll-pe-4\",\n \"scroll-pt-4\",\n \"scroll-pr-4\",\n \"scroll-pb-4\",\n \"scroll-pl-4\",\n \"scroll-p-5\",\n \"scroll-px-5\",\n \"scroll-py-5\",\n \"scroll-ps-5\",\n \"scroll-pe-5\",\n \"scroll-pt-5\",\n \"scroll-pr-5\",\n \"scroll-pb-5\",\n \"scroll-pl-5\",\n \"scroll-p-6\",\n \"scroll-px-6\",\n \"scroll-py-6\",\n \"scroll-ps-6\",\n \"scroll-pe-6\",\n \"scroll-pt-6\",\n \"scroll-pr-6\",\n \"scroll-pb-6\",\n \"scroll-pl-6\",\n \"scroll-p-7\",\n \"scroll-px-7\",\n \"scroll-py-7\",\n \"scroll-ps-7\",\n \"scroll-pe-7\",\n \"scroll-pt-7\",\n \"scroll-pr-7\",\n \"scroll-pb-7\",\n \"scroll-pl-7\",\n \"scroll-p-8\",\n \"scroll-px-8\",\n \"scroll-py-8\",\n \"scroll-ps-8\",\n \"scroll-pe-8\",\n \"scroll-pt-8\",\n \"scroll-pr-8\",\n \"scroll-pb-8\",\n \"scroll-pl-8\",\n \"scroll-p-9\",\n \"scroll-px-9\",\n \"scroll-py-9\",\n \"scroll-ps-9\",\n \"scroll-pe-9\",\n \"scroll-pt-9\",\n \"scroll-pr-9\",\n \"scroll-pb-9\",\n \"scroll-pl-9\",\n \"scroll-p-10\",\n \"scroll-px-10\",\n \"scroll-py-10\",\n \"scroll-ps-10\",\n \"scroll-pe-10\",\n \"scroll-pt-10\",\n \"scroll-pr-10\",\n \"scroll-pb-10\",\n \"scroll-pl-10\",\n \"scroll-p-11\",\n \"scroll-px-11\",\n \"scroll-py-11\",\n \"scroll-ps-11\",\n \"scroll-pe-11\",\n \"scroll-pt-11\",\n \"scroll-pr-11\",\n \"scroll-pb-11\",\n \"scroll-pl-11\",\n \"scroll-p-12\",\n \"scroll-px-12\",\n \"scroll-py-12\",\n \"scroll-ps-12\",\n \"scroll-pe-12\",\n \"scroll-pt-12\",\n \"scroll-pr-12\",\n \"scroll-pb-12\",\n \"scroll-pl-12\",\n \"scroll-p-14\",\n \"scroll-px-14\",\n \"scroll-py-14\",\n \"scroll-ps-14\",\n \"scroll-pe-14\",\n \"scroll-pt-14\",\n \"scroll-pr-14\",\n \"scroll-pb-14\",\n \"scroll-pl-14\",\n \"scroll-p-16\",\n \"scroll-px-16\",\n \"scroll-py-16\",\n \"scroll-ps-16\",\n \"scroll-pe-16\",\n \"scroll-pt-16\",\n \"scroll-pr-16\",\n \"scroll-pb-16\",\n \"scroll-pl-16\",\n \"scroll-p-20\",\n \"scroll-px-20\",\n \"scroll-py-20\",\n \"scroll-ps-20\",\n \"scroll-pe-20\",\n \"scroll-pt-20\",\n \"scroll-pr-20\",\n \"scroll-pb-20\",\n \"scroll-pl-20\",\n \"scroll-p-24\",\n \"scroll-px-24\",\n \"scroll-py-24\",\n \"scroll-ps-24\",\n \"scroll-pe-24\",\n \"scroll-pt-24\",\n \"scroll-pr-24\",\n \"scroll-pb-24\",\n \"scroll-pl-24\",\n \"scroll-p-28\",\n \"scroll-px-28\",\n \"scroll-py-28\",\n \"scroll-ps-28\",\n \"scroll-pe-28\",\n \"scroll-pt-28\",\n \"scroll-pr-28\",\n \"scroll-pb-28\",\n \"scroll-pl-28\",\n \"scroll-p-32\",\n \"scroll-px-32\",\n \"scroll-py-32\",\n \"scroll-ps-32\",\n \"scroll-pe-32\",\n \"scroll-pt-32\",\n \"scroll-pr-32\",\n \"scroll-pb-32\",\n \"scroll-pl-32\",\n \"scroll-p-36\",\n \"scroll-px-36\",\n \"scroll-py-36\",\n \"scroll-ps-36\",\n \"scroll-pe-36\",\n \"scroll-pt-36\",\n \"scroll-pr-36\",\n \"scroll-pb-36\",\n \"scroll-pl-36\",\n \"scroll-p-40\",\n \"scroll-px-40\",\n \"scroll-py-40\",\n \"scroll-ps-40\",\n \"scroll-pe-40\",\n \"scroll-pt-40\",\n \"scroll-pr-40\",\n \"scroll-pb-40\",\n \"scroll-pl-40\",\n \"scroll-p-44\",\n \"scroll-px-44\",\n \"scroll-py-44\",\n \"scroll-ps-44\",\n \"scroll-pe-44\",\n \"scroll-pt-44\",\n \"scroll-pr-44\",\n \"scroll-pb-44\",\n \"scroll-pl-44\",\n \"scroll-p-48\",\n \"scroll-px-48\",\n \"scroll-py-48\",\n \"scroll-ps-48\",\n \"scroll-pe-48\",\n \"scroll-pt-48\",\n \"scroll-pr-48\",\n \"scroll-pb-48\",\n \"scroll-pl-48\",\n \"scroll-p-52\",\n \"scroll-px-52\",\n \"scroll-py-52\",\n \"scroll-ps-52\",\n \"scroll-pe-52\",\n \"scroll-pt-52\",\n \"scroll-pr-52\",\n \"scroll-pb-52\",\n \"scroll-pl-52\",\n \"scroll-p-56\",\n \"scroll-px-56\",\n \"scroll-py-56\",\n \"scroll-ps-56\",\n \"scroll-pe-56\",\n \"scroll-pt-56\",\n \"scroll-pr-56\",\n \"scroll-pb-56\",\n \"scroll-pl-56\",\n \"scroll-p-60\",\n \"scroll-px-60\",\n \"scroll-py-60\",\n \"scroll-ps-60\",\n \"scroll-pe-60\",\n \"scroll-pt-60\",\n \"scroll-pr-60\",\n \"scroll-pb-60\",\n \"scroll-pl-60\",\n \"scroll-p-64\",\n \"scroll-px-64\",\n \"scroll-py-64\",\n \"scroll-ps-64\",\n \"scroll-pe-64\",\n \"scroll-pt-64\",\n \"scroll-pr-64\",\n \"scroll-pb-64\",\n \"scroll-pl-64\",\n \"scroll-p-72\",\n \"scroll-px-72\",\n \"scroll-py-72\",\n \"scroll-ps-72\",\n \"scroll-pe-72\",\n \"scroll-pt-72\",\n \"scroll-pr-72\",\n \"scroll-pb-72\",\n \"scroll-pl-72\",\n \"scroll-p-80\",\n \"scroll-px-80\",\n \"scroll-py-80\",\n \"scroll-ps-80\",\n \"scroll-pe-80\",\n \"scroll-pt-80\",\n \"scroll-pr-80\",\n \"scroll-pb-80\",\n \"scroll-pl-80\",\n \"scroll-p-96\",\n \"scroll-px-96\",\n \"scroll-py-96\",\n \"scroll-ps-96\",\n \"scroll-pe-96\",\n \"scroll-pt-96\",\n \"scroll-pr-96\",\n \"scroll-pb-96\",\n \"scroll-pl-96\",\n \"snap-start\",\n \"snap-end\",\n \"snap-center\",\n \"snap-align-none\",\n \"snap-normal\",\n \"snap-always\",\n \"snap-none\",\n \"snap-x\",\n \"snap-y\",\n \"snap-both\",\n \"snap-mandatory\",\n \"snap-proximity\",\n \"touch-auto\",\n \"touch-none\",\n \"touch-pan-x\",\n \"touch-pan-left\",\n \"touch-pan-right\",\n \"touch-pan-y\",\n \"touch-pan-up\",\n \"touch-pan-down\",\n \"touch-pinch-zoom\",\n \"touch-manipulation\",\n \"select-none\",\n \"select-text\",\n \"select-all\",\n \"select-auto\",\n \"will-change-auto\",\n \"will-change-scroll\",\n \"will-change-contents\",\n \"will-change-transform\",\n \"fill-none\",\n \"fill-inherit\",\n \"fill-current\",\n \"fill-transparent\",\n \"fill-black\",\n \"fill-white\",\n \"fill-slate-50\",\n \"fill-slate-100\",\n \"fill-slate-200\",\n \"fill-slate-300\",\n \"fill-slate-400\",\n \"fill-slate-500\",\n \"fill-slate-600\",\n \"fill-slate-700\",\n \"fill-slate-800\",\n \"fill-slate-900\",\n \"fill-slate-950\",\n \"fill-gray-50\",\n \"fill-gray-100\",\n \"fill-gray-200\",\n \"fill-gray-300\",\n \"fill-gray-400\",\n \"fill-gray-500\",\n \"fill-gray-600\",\n \"fill-gray-700\",\n \"fill-gray-800\",\n \"fill-gray-900\",\n \"fill-gray-950\",\n \"fill-zinc-50\",\n \"fill-zinc-100\",\n \"fill-zinc-200\",\n \"fill-zinc-300\",\n \"fill-zinc-400\",\n \"fill-zinc-500\",\n \"fill-zinc-600\",\n \"fill-zinc-700\",\n \"fill-zinc-800\",\n \"fill-zinc-900\",\n \"fill-zinc-950\",\n \"fill-neutral-50\",\n \"fill-neutral-100\",\n \"fill-neutral-200\",\n \"fill-neutral-300\",\n \"fill-neutral-400\",\n \"fill-neutral-500\",\n \"fill-neutral-600\",\n \"fill-neutral-700\",\n \"fill-neutral-800\",\n \"fill-neutral-900\",\n \"fill-neutral-950\",\n \"fill-stone-50\",\n \"fill-stone-100\",\n \"fill-stone-200\",\n \"fill-stone-300\",\n \"fill-stone-400\",\n \"fill-stone-500\",\n \"fill-stone-600\",\n \"fill-stone-700\",\n \"fill-stone-800\",\n \"fill-stone-900\",\n \"fill-stone-950\",\n \"fill-red-50\",\n \"fill-red-100\",\n \"fill-red-200\",\n \"fill-red-300\",\n \"fill-red-400\",\n \"fill-red-500\",\n \"fill-red-600\",\n \"fill-red-700\",\n \"fill-red-800\",\n \"fill-red-900\",\n \"fill-red-950\",\n \"fill-orange-50\",\n \"fill-orange-100\",\n \"fill-orange-200\",\n \"fill-orange-300\",\n \"fill-orange-400\",\n \"fill-orange-500\",\n \"fill-orange-600\",\n \"fill-orange-700\",\n \"fill-orange-800\",\n \"fill-orange-900\",\n \"fill-orange-950\",\n \"fill-amber-50\",\n \"fill-amber-100\",\n \"fill-amber-200\",\n \"fill-amber-300\",\n \"fill-amber-400\",\n \"fill-amber-500\",\n \"fill-amber-600\",\n \"fill-amber-700\",\n \"fill-amber-800\",\n \"fill-amber-900\",\n \"fill-amber-950\",\n \"fill-yellow-50\",\n \"fill-yellow-100\",\n \"fill-yellow-200\",\n \"fill-yellow-300\",\n \"fill-yellow-400\",\n \"fill-yellow-500\",\n \"fill-yellow-600\",\n \"fill-yellow-700\",\n \"fill-yellow-800\",\n \"fill-yellow-900\",\n \"fill-yellow-950\",\n \"fill-lime-50\",\n \"fill-lime-100\",\n \"fill-lime-200\",\n \"fill-lime-300\",\n \"fill-lime-400\",\n \"fill-lime-500\",\n \"fill-lime-600\",\n \"fill-lime-700\",\n \"fill-lime-800\",\n \"fill-lime-900\",\n \"fill-lime-950\",\n \"fill-green-50\",\n \"fill-green-100\",\n \"fill-green-200\",\n \"fill-green-300\",\n \"fill-green-400\",\n \"fill-green-500\",\n \"fill-green-600\",\n \"fill-green-700\",\n \"fill-green-800\",\n \"fill-green-900\",\n \"fill-green-950\",\n \"fill-emerald-50\",\n \"fill-emerald-100\",\n \"fill-emerald-200\",\n \"fill-emerald-300\",\n \"fill-emerald-400\",\n \"fill-emerald-500\",\n \"fill-emerald-600\",\n \"fill-emerald-700\",\n \"fill-emerald-800\",\n \"fill-emerald-900\",\n \"fill-emerald-950\",\n \"fill-teal-50\",\n \"fill-teal-100\",\n \"fill-teal-200\",\n \"fill-teal-300\",\n \"fill-teal-400\",\n \"fill-teal-500\",\n \"fill-teal-600\",\n \"fill-teal-700\",\n \"fill-teal-800\",\n \"fill-teal-900\",\n \"fill-teal-950\",\n \"fill-cyan-50\",\n \"fill-cyan-100\",\n \"fill-cyan-200\",\n \"fill-cyan-300\",\n \"fill-cyan-400\",\n \"fill-cyan-500\",\n \"fill-cyan-600\",\n \"fill-cyan-700\",\n \"fill-cyan-800\",\n \"fill-cyan-900\",\n \"fill-cyan-950\",\n \"fill-sky-50\",\n \"fill-sky-100\",\n \"fill-sky-200\",\n \"fill-sky-300\",\n \"fill-sky-400\",\n \"fill-sky-500\",\n \"fill-sky-600\",\n \"fill-sky-700\",\n \"fill-sky-800\",\n \"fill-sky-900\",\n \"fill-sky-950\",\n \"fill-blue-50\",\n \"fill-blue-100\",\n \"fill-blue-200\",\n \"fill-blue-300\",\n \"fill-blue-400\",\n \"fill-blue-500\",\n \"fill-blue-600\",\n \"fill-blue-700\",\n \"fill-blue-800\",\n \"fill-blue-900\",\n \"fill-blue-950\",\n \"fill-indigo-50\",\n \"fill-indigo-100\",\n \"fill-indigo-200\",\n \"fill-indigo-300\",\n \"fill-indigo-400\",\n \"fill-indigo-500\",\n \"fill-indigo-600\",\n \"fill-indigo-700\",\n \"fill-indigo-800\",\n \"fill-indigo-900\",\n \"fill-indigo-950\",\n \"fill-violet-50\",\n \"fill-violet-100\",\n \"fill-violet-200\",\n \"fill-violet-300\",\n \"fill-violet-400\",\n \"fill-violet-500\",\n \"fill-violet-600\",\n \"fill-violet-700\",\n \"fill-violet-800\",\n \"fill-violet-900\",\n \"fill-violet-950\",\n \"fill-purple-50\",\n \"fill-purple-100\",\n \"fill-purple-200\",\n \"fill-purple-300\",\n \"fill-purple-400\",\n \"fill-purple-500\",\n \"fill-purple-600\",\n \"fill-purple-700\",\n \"fill-purple-800\",\n \"fill-purple-900\",\n \"fill-purple-950\",\n \"fill-fuchsia-50\",\n \"fill-fuchsia-100\",\n \"fill-fuchsia-200\",\n \"fill-fuchsia-300\",\n \"fill-fuchsia-400\",\n \"fill-fuchsia-500\",\n \"fill-fuchsia-600\",\n \"fill-fuchsia-700\",\n \"fill-fuchsia-800\",\n \"fill-fuchsia-900\",\n \"fill-fuchsia-950\",\n \"fill-pink-50\",\n \"fill-pink-100\",\n \"fill-pink-200\",\n \"fill-pink-300\",\n \"fill-pink-400\",\n \"fill-pink-500\",\n \"fill-pink-600\",\n \"fill-pink-700\",\n \"fill-pink-800\",\n \"fill-pink-900\",\n \"fill-pink-950\",\n \"fill-rose-50\",\n \"fill-rose-100\",\n \"fill-rose-200\",\n \"fill-rose-300\",\n \"fill-rose-400\",\n \"fill-rose-500\",\n \"fill-rose-600\",\n \"fill-rose-700\",\n \"fill-rose-800\",\n \"fill-rose-900\",\n \"fill-rose-950\",\n \"stroke-none\",\n \"stroke-inherit\",\n \"stroke-current\",\n \"stroke-transparent\",\n \"stroke-black\",\n \"stroke-white\",\n \"stroke-slate-50\",\n \"stroke-slate-100\",\n \"stroke-slate-200\",\n \"stroke-slate-300\",\n \"stroke-slate-400\",\n \"stroke-slate-500\",\n \"stroke-slate-600\",\n \"stroke-slate-700\",\n \"stroke-slate-800\",\n \"stroke-slate-900\",\n \"stroke-slate-950\",\n \"stroke-gray-50\",\n \"stroke-gray-100\",\n \"stroke-gray-200\",\n \"stroke-gray-300\",\n \"stroke-gray-400\",\n \"stroke-gray-500\",\n \"stroke-gray-600\",\n \"stroke-gray-700\",\n \"stroke-gray-800\",\n \"stroke-gray-900\",\n \"stroke-gray-950\",\n \"stroke-zinc-50\",\n \"stroke-zinc-100\",\n \"stroke-zinc-200\",\n \"stroke-zinc-300\",\n \"stroke-zinc-400\",\n \"stroke-zinc-500\",\n \"stroke-zinc-600\",\n \"stroke-zinc-700\",\n \"stroke-zinc-800\",\n \"stroke-zinc-900\",\n \"stroke-zinc-950\",\n \"stroke-neutral-50\",\n \"stroke-neutral-100\",\n \"stroke-neutral-200\",\n \"stroke-neutral-300\",\n \"stroke-neutral-400\",\n \"stroke-neutral-500\",\n \"stroke-neutral-600\",\n \"stroke-neutral-700\",\n \"stroke-neutral-800\",\n \"stroke-neutral-900\",\n \"stroke-neutral-950\",\n \"stroke-stone-50\",\n \"stroke-stone-100\",\n \"stroke-stone-200\",\n \"stroke-stone-300\",\n \"stroke-stone-400\",\n \"stroke-stone-500\",\n \"stroke-stone-600\",\n \"stroke-stone-700\",\n \"stroke-stone-800\",\n \"stroke-stone-900\",\n \"stroke-stone-950\",\n \"stroke-red-50\",\n \"stroke-red-100\",\n \"stroke-red-200\",\n \"stroke-red-300\",\n \"stroke-red-400\",\n \"stroke-red-500\",\n \"stroke-red-600\",\n \"stroke-red-700\",\n \"stroke-red-800\",\n \"stroke-red-900\",\n \"stroke-red-950\",\n \"stroke-orange-50\",\n \"stroke-orange-100\",\n \"stroke-orange-200\",\n \"stroke-orange-300\",\n \"stroke-orange-400\",\n \"stroke-orange-500\",\n \"stroke-orange-600\",\n \"stroke-orange-700\",\n \"stroke-orange-800\",\n \"stroke-orange-900\",\n \"stroke-orange-950\",\n \"stroke-amber-50\",\n \"stroke-amber-100\",\n \"stroke-amber-200\",\n \"stroke-amber-300\",\n \"stroke-amber-400\",\n \"stroke-amber-500\",\n \"stroke-amber-600\",\n \"stroke-amber-700\",\n \"stroke-amber-800\",\n \"stroke-amber-900\",\n \"stroke-amber-950\",\n \"stroke-yellow-50\",\n \"stroke-yellow-100\",\n \"stroke-yellow-200\",\n \"stroke-yellow-300\",\n \"stroke-yellow-400\",\n \"stroke-yellow-500\",\n \"stroke-yellow-600\",\n \"stroke-yellow-700\",\n \"stroke-yellow-800\",\n \"stroke-yellow-900\",\n \"stroke-yellow-950\",\n \"stroke-lime-50\",\n \"stroke-lime-100\",\n \"stroke-lime-200\",\n \"stroke-lime-300\",\n \"stroke-lime-400\",\n \"stroke-lime-500\",\n \"stroke-lime-600\",\n \"stroke-lime-700\",\n \"stroke-lime-800\",\n \"stroke-lime-900\",\n \"stroke-lime-950\",\n \"stroke-green-50\",\n \"stroke-green-100\",\n \"stroke-green-200\",\n \"stroke-green-300\",\n \"stroke-green-400\",\n \"stroke-green-500\",\n \"stroke-green-600\",\n \"stroke-green-700\",\n \"stroke-green-800\",\n \"stroke-green-900\",\n \"stroke-green-950\",\n \"stroke-emerald-50\",\n \"stroke-emerald-100\",\n \"stroke-emerald-200\",\n \"stroke-emerald-300\",\n \"stroke-emerald-400\",\n \"stroke-emerald-500\",\n \"stroke-emerald-600\",\n \"stroke-emerald-700\",\n \"stroke-emerald-800\",\n \"stroke-emerald-900\",\n \"stroke-emerald-950\",\n \"stroke-teal-50\",\n \"stroke-teal-100\",\n \"stroke-teal-200\",\n \"stroke-teal-300\",\n \"stroke-teal-400\",\n \"stroke-teal-500\",\n \"stroke-teal-600\",\n \"stroke-teal-700\",\n \"stroke-teal-800\",\n \"stroke-teal-900\",\n \"stroke-teal-950\",\n \"stroke-cyan-50\",\n \"stroke-cyan-100\",\n \"stroke-cyan-200\",\n \"stroke-cyan-300\",\n \"stroke-cyan-400\",\n \"stroke-cyan-500\",\n \"stroke-cyan-600\",\n \"stroke-cyan-700\",\n \"stroke-cyan-800\",\n \"stroke-cyan-900\",\n \"stroke-cyan-950\",\n \"stroke-sky-50\",\n \"stroke-sky-100\",\n \"stroke-sky-200\",\n \"stroke-sky-300\",\n \"stroke-sky-400\",\n \"stroke-sky-500\",\n \"stroke-sky-600\",\n \"stroke-sky-700\",\n \"stroke-sky-800\",\n \"stroke-sky-900\",\n \"stroke-sky-950\",\n \"stroke-blue-50\",\n \"stroke-blue-100\",\n \"stroke-blue-200\",\n \"stroke-blue-300\",\n \"stroke-blue-400\",\n \"stroke-blue-500\",\n \"stroke-blue-600\",\n \"stroke-blue-700\",\n \"stroke-blue-800\",\n \"stroke-blue-900\",\n \"stroke-blue-950\",\n \"stroke-indigo-50\",\n \"stroke-indigo-100\",\n \"stroke-indigo-200\",\n \"stroke-indigo-300\",\n \"stroke-indigo-400\",\n \"stroke-indigo-500\",\n \"stroke-indigo-600\",\n \"stroke-indigo-700\",\n \"stroke-indigo-800\",\n \"stroke-indigo-900\",\n \"stroke-indigo-950\",\n \"stroke-violet-50\",\n \"stroke-violet-100\",\n \"stroke-violet-200\",\n \"stroke-violet-300\",\n \"stroke-violet-400\",\n \"stroke-violet-500\",\n \"stroke-violet-600\",\n \"stroke-violet-700\",\n \"stroke-violet-800\",\n \"stroke-violet-900\",\n \"stroke-violet-950\",\n \"stroke-purple-50\",\n \"stroke-purple-100\",\n \"stroke-purple-200\",\n \"stroke-purple-300\",\n \"stroke-purple-400\",\n \"stroke-purple-500\",\n \"stroke-purple-600\",\n \"stroke-purple-700\",\n \"stroke-purple-800\",\n \"stroke-purple-900\",\n \"stroke-purple-950\",\n \"stroke-fuchsia-50\",\n \"stroke-fuchsia-100\",\n \"stroke-fuchsia-200\",\n \"stroke-fuchsia-300\",\n \"stroke-fuchsia-400\",\n \"stroke-fuchsia-500\",\n \"stroke-fuchsia-600\",\n \"stroke-fuchsia-700\",\n \"stroke-fuchsia-800\",\n \"stroke-fuchsia-900\",\n \"stroke-fuchsia-950\",\n \"stroke-pink-50\",\n \"stroke-pink-100\",\n \"stroke-pink-200\",\n \"stroke-pink-300\",\n \"stroke-pink-400\",\n \"stroke-pink-500\",\n \"stroke-pink-600\",\n \"stroke-pink-700\",\n \"stroke-pink-800\",\n \"stroke-pink-900\",\n \"stroke-pink-950\",\n \"stroke-rose-50\",\n \"stroke-rose-100\",\n \"stroke-rose-200\",\n \"stroke-rose-300\",\n \"stroke-rose-400\",\n \"stroke-rose-500\",\n \"stroke-rose-600\",\n \"stroke-rose-700\",\n \"stroke-rose-800\",\n \"stroke-rose-900\",\n \"stroke-rose-950\",\n \"stroke-0\",\n \"stroke-1\",\n \"stroke-2\",\n \"sr-only\",\n \"not-sr-only\",\n \"forced-color-adjust-auto\",\n \"forced-color-adjust-none\",\n \"transform\",\n \"transform-none\",\n \"transform-gpu\",\n \"transform-cpu\",\n \"group\",\n \"peer\"\n]);\n" }, { "path": "tsconfig.json", "content": "{\n \"compilerOptions\": {\n \"module\": \"node16\",\n\n \"noImplicitReturns\": true,\n \"noUnusedLocals\": true,\n \"outDir\": \"build\",\n \"sourceMap\": true,\n \"strict\": true,\n \"allowJs\": true,\n \"target\": \"es2022\",\n \"lib\": [\"es2022\"],\n \"skipLibCheck\": true,\n \"useDefineForClassFields\": false,\n \"experimentalDecorators\": true,\n \"emitDecoratorMetadata\": true,\n \"esModuleInterop\": true,\n \"noImplicitOverride\": true,\n },\n \"compileOnSave\": true,\n \"include\": [\"src\"]\n}\n" } ]