[ { "path": ".actor/Dockerfile", "content": "FROM node:alpine\n\nRUN apk --no-cache add curl bash git monolith jq\nRUN npm -g install apify-cli\nCOPY .actor .actor\nCMD ./.actor/bin/actor.sh\n" }, { "path": ".actor/README.md", "content": "# Monolith Actor on Apify\n\n[![Monolith Actor](https://apify.com/actor-badge?actor=snshn/monolith)](https://apify.com/snshn/monolith?fpr=snshn)\n\nThis Actor wraps [Monolith](https://crates.io/crates/monolith) to crawl a web page URL and bundle the entire content in a single HTML file, without installing and running the tool locally.\n\n## What are Actors?\n[Actors](https://docs.apify.com/platform/actors?fpr=snshn) are serverless microservices running on the [Apify Platform](https://apify.com/?fpr=snshn). They are based on the [Actor SDK](https://docs.apify.com/sdk/js?fpr=snshn) and can be found in the [Apify Store](https://apify.com/store?fpr=snshn). Learn more about Actors in the [Apify Whitepaper](https://whitepaper.actor?fpr=snshn).\n\n## Usage\n\n### Apify Console\n\n1. Go to the Apify Actor page\n2. Click \"Run\"\n3. In the input form, fill in **URL(s)** to crawl and bundle\n4. The Actor will run and :\n - save the bundled HTML files in the run's default key-value store\n - save the links to the KVS with original URL and monolith process exit status to the dataset\n\n\n### Apify CLI\n\n```bash\napify call snshn/monolith --input='{\n \"urls\": [\"https://news.ycombinator.com/\"]\n}'\n```\n\n### Using Apify API\n\n```bash\ncurl --request POST \\\n --url \"https://api.apify.com/v2/acts/snshn~monolith/run\" \\\n --header 'Content-Type: application/json' \\\n --header 'Authorization: Bearer YOUR_API_TOKEN' \\\n --data '{\n \"urls\": [\"https://news.ycombinator.com/\"],\n }\n}'\n```\n\n## Input Parameters\n\nThe Actor accepts a JSON schema with the following structure:\n\n| Field | Type | Required | Default | Description |\n|-------|------|----------|---------|-------------|\n| `urls` | array | Yes | - | List of URLs to monolith |\n| `urls[]` | string | Yes | - | URL to monolith |\n\n\n### Example Input\n\n```json\n{\n \"urls\": [\"https://news.ycombinator.com/\"],\n}\n```\n\n## Output\n\nThe Actor provides three types of outputs:\n\n### Dataset Record\n\n| Field | Type | Required | Description |\n|-------|------|----------|-------------|\n| `url` | string | Yes | A link to the Apify key-value store object where the monolithic html is available for download |\n| `kvsUrl` | array | Yes | Exit status of the monolith process |\n| `status`| number | No | The original start URL for the monolith process |\n\n### Example Dataset Item (JSON)\n\n```json\n{\n \"url\": \"https://news.ycombinator.com/\",\n \"kvsUrl\": \"https://api.apify.com/v2/key-value-stores/JRFLHRy9DOtdKGpdm/records/https___news.ycombinator.com_\",\n \"status\": \"0\"\n}\n```\n\n## Performance & Resources\n\n- **Memory Requirements**:\n - Minimum: 4168 MB RAM\n- **Processing Time**:\n - 30s per complex page like [bbc.co.uk](https://bbc.co.uk)\n\n\nFor more help, check the [Monolith Project documentation](https://github.com/Y2Z/monolith) or raise an issue in the [Actor page detail](https://apify.com/snshn/monolith?fpr=snshn) on Apify.\n\n\n" }, { "path": ".actor/actor.json", "content": "{\n\t\"actorSpecification\": 1,\n\t\"name\": \"monolith\",\n\t\"version\": \"0.0\",\n\t\"buildTag\": \"latest\",\n\t\"environmentVariables\": {},\n \"dockerFile\": \"./Dockerfile\", \n \"dockerContext\": \"../\",\n \"input\": \"./input_schema.json\",\n \"storages\": {\n \"dataset\": \"./dataset_schema.json\"\n }\n}\n" }, { "path": ".actor/bin/actor.sh", "content": "#!/bin/bash\n#pwd\n#find ./storage\napify actor:get-input > /dev/null\nINPUT=`apify actor:get-input | jq -r .urls[] | xargs echo`\necho \"INPUT: $INPUT\"\n\nfor url in $INPUT; do\n # support for local usage\n # sanitize url to a safe *nix filename - replace nonalfanumerical characters\n # https://stackoverflow.com/questions/9847288/is-it-possible-to-use-in-a-filename\n # https://serverfault.com/questions/348482/how-to-remove-invalid-characters-from-filenames\n safe_filename=`echo $url | sed -e 's/[^A-Za-z0-9._-]/_/g'`\n echo \"Monolith-ing $url to key $safe_filename\"\n monolith $url | apify actor:set-value \"$safe_filename\" --contentType=text/html\n kvs_url=\"https://api.apify.com/v2/key-value-stores/${APIFY_DEFAULT_KEY_VALUE_STORE_ID}/records/${safe_filename}\"\n result=$?\n echo \"Pushing result item to the datastore\"\n echo \"{\\\"url\\\":\\\"${url}\\\",\\\"status\\\":\\\"${result}\\\", \\\"kvsUrl\\\":\\\"${kvs_url}\\\"}\" | apify actor:push-data\ndone\n\nexit 0\n" }, { "path": ".actor/dataset_schema.json", "content": "{\n \"actorSpecification\": 1,\n \"fields\":{\n \"title\": \"Sherlock actor input\",\n \"description\": \"This is actor input schema\",\n \"type\": \"object\",\n \"schemaVersion\": 1,\n \"properties\": {\n \"kvsUrl\": {\n \"title\": \"Object URL\",\n \"type\": \"string\",\n \"description\": \"A link to the Apify key-value store object where the monolithic html is available\"\n },\n \"status\": {\n \"title\": \"Exist status\",\n \"type\": \"string\",\n \"description\": \"Exit status of the monolith process\"\n },\n \"url\": {\n \"title\": \"URL\",\n \"type\": \"string\",\n \"description\": \"The original start URL for the monolith process \"\n }\n \n },\n \"required\": [\n \"kvsUrl\", \n \"status\",\n \"url\"\n ]\n },\n \"views\": {\n \"overview\": {\n \"title\": \"Overview\",\n \"transformation\": {\n \"fields\": [\n \"url\",\n \"kvsUrl\",\n \"status\"\n ],\n },\n \"display\": {\n \"component\": \"table\",\n \"url\": {\n \"label\": \"Page URL\"\n },\n \"kvsUrl\": {\n \"label\": \"KVS URL\" \n },\n \"status\": {\n \"label\": \"Status\"\n }\n }\n }\n }\n}\n" }, { "path": ".actor/input_schema.json", "content": "{\n \"title\": \"Sherlock actor input\",\n \"description\": \"This is actor input schema\",\n \"type\": \"object\",\n \"schemaVersion\": 1,\n \"properties\": {\n \"urls\": {\n \"title\": \"Urls\",\n \"type\": \"array\",\n \"description\": \"A list of urls of pages to bundle into single HTML document\",\n \"editor\": \"stringList\",\n \"prefill\": [\"http://www.google.com\"]\n }\n },\n \"required\": [\n \"urls\"\n ]\n}\n" }, { "path": ".dockerignore", "content": "/target/\n" }, { "path": ".github/FUNDING.yml", "content": "# These are supported funding model platforms\n\ngithub: snshn\n" }, { "path": ".github/workflows/build_gnu_linux.yml", "content": "name: GNU/Linux\n\non:\n push:\n branches: [ master ]\n paths-ignore:\n - 'assets/'\n - 'dist/'\n - 'snap/'\n - 'Dockerfile'\n - 'LICENSE'\n - 'Makefile'\n - 'monolith.nuspec'\n - 'README.md'\n\njobs:\n build:\n\n strategy:\n matrix:\n os:\n - ubuntu-latest\n rust:\n - stable\n runs-on: ${{ matrix.os }}\n\n steps:\n - run: git config --global core.autocrlf false\n\n - uses: actions/checkout@v2\n\n - name: Build\n run: cargo build --all --locked --verbose\n" }, { "path": ".github/workflows/build_macos.yml", "content": "name: macOS\n\non:\n push:\n branches: [ master ]\n paths-ignore:\n - 'assets/'\n - 'dist/'\n - 'snap/'\n - 'Dockerfile'\n - 'LICENSE'\n - 'Makefile'\n - 'monolith.nuspec'\n - 'README.md'\n\njobs:\n build:\n\n strategy:\n matrix:\n os:\n - macos-latest\n rust:\n - stable\n runs-on: ${{ matrix.os }}\n\n steps:\n - run: git config --global core.autocrlf false\n\n - uses: actions/checkout@v2\n\n - name: Build\n run: cargo build --all --locked --verbose\n" }, { "path": ".github/workflows/build_windows.yml", "content": "name: Windows\n\non:\n push:\n branches: [ master ]\n paths-ignore:\n - 'assets/'\n - 'dist/'\n - 'snap/'\n - 'Dockerfile'\n - 'LICENSE'\n - 'Makefile'\n - 'monolith.nuspec'\n - 'README.md'\n\njobs:\n build:\n\n strategy:\n matrix:\n os:\n - windows-latest\n rust:\n - stable\n runs-on: ${{ matrix.os }}\n\n steps:\n - run: git config --global core.autocrlf false\n\n - uses: actions/checkout@v2\n\n - name: Build\n run: cargo build --all --locked --verbose\n" }, { "path": ".github/workflows/cd.yml", "content": "# CD GitHub Actions workflow for monolith\n\nname: CD\n\non:\n release:\n types:\n - created\n\njobs:\n\n gnu_linux_aarch64:\n runs-on: ubuntu-20.04\n steps:\n - name: Checkout the repository\n uses: actions/checkout@v4\n\n - name: Prepare cross-platform environment\n run: |\n sudo mkdir /cross-build\n sudo touch /etc/apt/sources.list.d/arm64.list\n echo \"deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ focal main\" | sudo tee -a /etc/apt/sources.list.d/arm64.list\n sudo apt-get update\n sudo apt-get install -y gcc-aarch64-linux-gnu libc6-arm64-cross libc6-dev-arm64-cross\n sudo apt-get download libssl1.1:arm64 libssl-dev:arm64\n sudo dpkg -x libssl1.1*.deb /cross-build\n sudo dpkg -x libssl-dev*.deb /cross-build\n rustup target add aarch64-unknown-linux-gnu\n echo \"C_INCLUDE_PATH=/cross-build/usr/include\" >> $GITHUB_ENV\n echo \"OPENSSL_INCLUDE_DIR=/cross-build/usr/include/aarch64-linux-gnu\" >> $GITHUB_ENV\n echo \"OPENSSL_LIB_DIR=/cross-build/usr/lib/aarch64-linux-gnu\" >> $GITHUB_ENV\n echo \"PKG_CONFIG_ALLOW_CROSS=1\" >> $GITHUB_ENV\n echo \"RUSTFLAGS=-C linker=aarch64-linux-gnu-gcc -L/usr/aarch64-linux-gnu/lib -L/cross-build/usr/lib/aarch64-linux-gnu\" >> $GITHUB_ENV\n\n - name: Build the executable\n run: cargo build --release --target=aarch64-unknown-linux-gnu --no-default-features --features cli\n\n - name: Attach artifact to the release\n uses: Shopify/upload-to-release@v2.0.0\n with:\n name: monolith-gnu-linux-aarch64\n path: target/aarch64-unknown-linux-gnu/release/monolith\n repo-token: ${{ secrets.GITHUB_TOKEN }}\n\n gnu_linux_armhf:\n runs-on: ubuntu-20.04\n steps:\n - name: Checkout the repository\n uses: actions/checkout@v4\n\n - name: Prepare cross-platform environment\n run: |\n sudo mkdir /cross-build\n sudo touch /etc/apt/sources.list.d/armhf.list\n echo \"deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ focal main\" | sudo tee -a /etc/apt/sources.list.d/armhf.list\n sudo apt-get update\n sudo apt-get install -y gcc-arm-linux-gnueabihf libc6-armhf-cross libc6-dev-armhf-cross\n sudo apt-get download libssl1.1:armhf libssl-dev:armhf\n sudo dpkg -x libssl1.1*.deb /cross-build\n sudo dpkg -x libssl-dev*.deb /cross-build\n rustup target add arm-unknown-linux-gnueabihf\n echo \"C_INCLUDE_PATH=/cross-build/usr/include\" >> $GITHUB_ENV\n echo \"OPENSSL_INCLUDE_DIR=/cross-build/usr/include/arm-linux-gnueabihf\" >> $GITHUB_ENV\n echo \"OPENSSL_LIB_DIR=/cross-build/usr/lib/arm-linux-gnueabihf\" >> $GITHUB_ENV\n echo \"PKG_CONFIG_ALLOW_CROSS=1\" >> $GITHUB_ENV\n echo \"RUSTFLAGS=-C linker=arm-linux-gnueabihf-gcc -L/usr/arm-linux-gnueabihf/lib -L/cross-build/usr/lib/arm-linux-gnueabihf -L/cross-build/lib/arm-linux-gnueabihf\" >> $GITHUB_ENV\n\n - name: Build the executable\n run: cargo build --release --target=arm-unknown-linux-gnueabihf --no-default-features --features cli\n\n - name: Attach artifact to the release\n uses: Shopify/upload-to-release@v2.0.0\n with:\n name: monolith-gnu-linux-armhf\n path: target/arm-unknown-linux-gnueabihf/release/monolith\n repo-token: ${{ secrets.GITHUB_TOKEN }}\n\n gnu_linux_x86_64:\n runs-on: ubuntu-20.04\n steps:\n - name: Checkout the repository\n uses: actions/checkout@v4\n\n - name: Build the executable\n run: cargo build --release\n\n - uses: Shopify/upload-to-release@v2.0.0\n with:\n name: monolith-gnu-linux-x86_64\n path: target/release/monolith\n repo-token: ${{ secrets.GITHUB_TOKEN }}\n\n windows:\n runs-on: windows-2019\n steps:\n - run: git config --global core.autocrlf false\n\n - name: Checkout the repository\n uses: actions/checkout@v4\n\n - name: Build the executable\n run: cargo build --release\n\n - uses: Shopify/upload-to-release@v2.0.0\n with:\n name: monolith.exe\n path: target\\release\\monolith.exe\n repo-token: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".github/workflows/ci-netbsd.yml", "content": "# CI NetBSD GitHub Actions workflow for monolith\n\nname: CI (NetBSD)\n\non:\n pull_request:\n branches: [ master ]\n paths-ignore:\n - 'assets/'\n - 'dist/'\n - 'snap/'\n - 'Dockerfile'\n - 'LICENSE'\n - 'Makefile'\n - 'monolith.nuspec'\n - 'README.md'\n\njobs:\n build_and_test:\n runs-on: ubuntu-latest\n name: Build and test (netbsd)\n steps:\n - name: \"Checkout repository\"\n uses: actions/checkout@v4\n\n - name: Test in NetBSD\n uses: vmactions/netbsd-vm@v1\n with:\n usesh: true\n prepare: |\n /usr/sbin/pkg_add cwrappers gmake mktools pkgconf rust\n run: |\n cargo build --all --locked --verbose --no-default-features --features cli\n cargo test --all --locked --verbose --no-default-features --features cli\n" }, { "path": ".github/workflows/ci.yml", "content": "# CI GitHub Actions workflow for monolith\n\nname: CI\n\non:\n pull_request:\n branches: [ master ]\n paths-ignore:\n - 'assets/'\n - 'dist/'\n - 'snap/'\n - 'Dockerfile'\n - 'LICENSE'\n - 'Makefile'\n - 'monolith.nuspec'\n - 'README.md'\n\njobs:\n build_and_test:\n name: Build and test\n strategy:\n matrix:\n os:\n - ubuntu-latest\n - macos-latest\n - windows-latest\n runs-on: ${{ matrix.os }}\n steps:\n - run: git config --global core.autocrlf false\n\n - name: \"Checkout repository\"\n uses: actions/checkout@v4\n\n - name: Build\n run: cargo build --all --locked --verbose\n\n - name: Run tests\n run: cargo test --all --locked --verbose\n\n - name: Check code formatting\n run: |\n rustup component add rustfmt\n cargo fmt --all -- --check\n" }, { "path": ".gitignore", "content": "# Generated by Cargo\n# will have compiled files and executables\n/target/\n\n# These are backup files generated by rustfmt\n**/*.rs.bk\n\n# Added by Apify CLI\nstorage\nnode_modules\n.venv\n" }, { "path": "Cargo.toml", "content": "[package]\nname = \"monolith\"\nversion = \"2.11.0\"\nauthors = [\n \"Sunshine \",\n \"Mahdi Robatipoor \",\n \"Emmanuel Delaborde \",\n \"Emi Simpson \",\n \"rhysd \",\n \"Andriy Rakhnin \",\n]\nedition = \"2021\"\ndescription = \"CLI tool and library for saving web pages as a single HTML file\"\nhomepage = \"https://github.com/Y2Z/monolith\"\nrepository = \"https://github.com/Y2Z/monolith\"\nreadme = \"README.md\"\nkeywords = [\"web\", \"http\", \"html\", \"download\", \"command-line\"]\ncategories = [\"command-line-utilities\", \"web-programming\"]\ninclude = [\"src/*.rs\", \"Cargo.toml\"]\nlicense = \"CC0-1.0\"\n\n[dependencies]\natty = \"=0.2.14\" # Used for highlighting network errors\nbase64 = \"=0.22.1\" # Used for integrity attributes\nchrono = \"=0.4.41\" # Used for formatting timestamps\nclap = { version = \"=4.5.37\", features = [\n \"derive\",\n], optional = true } # Used for processing CLI arguments\ncssparser = \"=0.35.0\" # Used for dealing with CSS\ndirectories = { version = \"=6.0.0\", optional = true } # Used for GUI\ndruid = { version = \"=0.8.3\", optional = true } # Used for GUI\nencoding_rs = \"=0.8.35\" # Used for parsing and converting document charsets\nhtml5ever = \"=0.29.1\" # Used for all things DOM\nmarkup5ever_rcdom = \"=0.5.0-unofficial\" # Used for manipulating DOM\npercent-encoding = \"=2.3.1\" # Used for encoding URLs\nsha2 = \"=0.10.9\" # Used for calculating checksums during integrity checks\nredb = \"=2.4.0\" # Used for on-disk caching of remote assets\ntempfile = { version = \"=3.19.1\", optional = true } # Used for on-disk caching of remote assets\nurl = \"=2.5.4\" # Used for parsing URLs\nopenssl = \"=0.10.72\" # Used for static linking of the OpenSSL library\n\n# Used for unwrapping NOSCRIPT\n[dependencies.regex]\nversion = \"=1.11.1\"\ndefault-features = false\nfeatures = [\"std\", \"perf-dfa\", \"unicode-perl\"]\n\n# Used for making network requests\n[dependencies.reqwest]\nversion = \"=0.12.15\"\ndefault-features = false\nfeatures = [\"default-tls\", \"blocking\", \"gzip\", \"brotli\", \"deflate\"]\n\n[dev-dependencies]\nassert_cmd = \"=2.0.17\"\n\n[features]\ndefault = [\"cli\", \"vendored-openssl\"]\ncli = [\"clap\", \"tempfile\"] # Build a CLI tool that includes main() function\ngui = [\n \"directories\",\n \"druid\",\n \"tempfile\",\n] # Build a GUI executable that includes main() function\nvendored-openssl = [\n \"openssl/vendored\",\n] # Compile and statically link a copy of OpenSSL\n\n[lib]\nname = \"monolith\"\npath = \"src/lib.rs\"\n\n[[bin]]\nname = \"monolith\"\npath = \"src/main.rs\"\nrequired-features = [\"cli\"]\n\n[[bin]]\nname = \"monolith-gui\"\npath = \"src/gui.rs\"\nrequired-features = [\"gui\"]\n" }, { "path": "Dockerfile", "content": "FROM clux/muslrust:stable as builder\n\nRUN curl -L -o monolith.tar.gz $(curl -s https://api.github.com/repos/y2z/monolith/releases/latest \\\n | grep \"tarball_url.*\\\",\" \\\n | cut -d '\"' -f 4)\nRUN tar xfz monolith.tar.gz \\\n && mv Y2Z-monolith-* monolith \\\n && rm monolith.tar.gz\n\nWORKDIR monolith/\nRUN make install\n\n\nFROM alpine\n\nRUN apk update && \\\n apk add --no-cache openssl && \\\n rm -rf \"/var/cache/apk/*\"\n\nCOPY --from=builder /root/.cargo/bin/monolith /usr/bin/monolith\nWORKDIR /tmp\nENTRYPOINT [\"/usr/bin/monolith\"]\n" }, { "path": "LICENSE", "content": "Creative Commons Legal Code\n\nCC0 1.0 Universal\n\n CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE\n LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN\n ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS\n INFORMATION ON AN \"AS-IS\" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES\n REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS\n PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM\n THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED\n HEREUNDER.\n\nStatement of Purpose\n\nThe laws of most jurisdictions throughout the world automatically confer\nexclusive Copyright and Related Rights (defined below) upon the creator\nand subsequent owner(s) (each and all, an \"owner\") of an original work of\nauthorship and/or a database (each, a \"Work\").\n\nCertain owners wish to permanently relinquish those rights to a Work for\nthe purpose of contributing to a commons of creative, cultural and\nscientific works (\"Commons\") that the public can reliably and without fear\nof later claims of infringement build upon, modify, incorporate in other\nworks, reuse and redistribute as freely as possible in any form whatsoever\nand for any purposes, including without limitation commercial purposes.\nThese owners may contribute to the Commons to promote the ideal of a free\nculture and the further production of creative, cultural and scientific\nworks, or to gain reputation or greater distribution for their Work in\npart through the use and efforts of others.\n\nFor these and/or other purposes and motivations, and without any\nexpectation of additional consideration or compensation, the person\nassociating CC0 with a Work (the \"Affirmer\"), to the extent that he or she\nis an owner of Copyright and Related Rights in the Work, voluntarily\nelects to apply CC0 to the Work and publicly distribute the Work under its\nterms, with knowledge of his or her Copyright and Related Rights in the\nWork and the meaning and intended legal effect of CC0 on those rights.\n\n1. Copyright and Related Rights. A Work made available under CC0 may be\nprotected by copyright and related or neighboring rights (\"Copyright and\nRelated Rights\"). Copyright and Related Rights include, but are not\nlimited to, the following:\n\n i. the right to reproduce, adapt, distribute, perform, display,\n communicate, and translate a Work;\n ii. moral rights retained by the original author(s) and/or performer(s);\niii. publicity and privacy rights pertaining to a person's image or\n likeness depicted in a Work;\n iv. rights protecting against unfair competition in regards to a Work,\n subject to the limitations in paragraph 4(a), below;\n v. rights protecting the extraction, dissemination, use and reuse of data\n in a Work;\n vi. database rights (such as those arising under Directive 96/9/EC of the\n European Parliament and of the Council of 11 March 1996 on the legal\n protection of databases, and under any national implementation\n thereof, including any amended or successor version of such\n directive); and\nvii. other similar, equivalent or corresponding rights throughout the\n world based on applicable law or treaty, and any national\n implementations thereof.\n\n2. Waiver. To the greatest extent permitted by, but not in contravention\nof, applicable law, Affirmer hereby overtly, fully, permanently,\nirrevocably and unconditionally waives, abandons, and surrenders all of\nAffirmer's Copyright and Related Rights and associated claims and causes\nof action, whether now known or unknown (including existing as well as\nfuture claims and causes of action), in the Work (i) in all territories\nworldwide, (ii) for the maximum duration provided by applicable law or\ntreaty (including future time extensions), (iii) in any current or future\nmedium and for any number of copies, and (iv) for any purpose whatsoever,\nincluding without limitation commercial, advertising or promotional\npurposes (the \"Waiver\"). Affirmer makes the Waiver for the benefit of each\nmember of the public at large and to the detriment of Affirmer's heirs and\nsuccessors, fully intending that such Waiver shall not be subject to\nrevocation, rescission, cancellation, termination, or any other legal or\nequitable action to disrupt the quiet enjoyment of the Work by the public\nas contemplated by Affirmer's express Statement of Purpose.\n\n3. Public License Fallback. Should any part of the Waiver for any reason\nbe judged legally invalid or ineffective under applicable law, then the\nWaiver shall be preserved to the maximum extent permitted taking into\naccount Affirmer's express Statement of Purpose. In addition, to the\nextent the Waiver is so judged Affirmer hereby grants to each affected\nperson a royalty-free, non transferable, non sublicensable, non exclusive,\nirrevocable and unconditional license to exercise Affirmer's Copyright and\nRelated Rights in the Work (i) in all territories worldwide, (ii) for the\nmaximum duration provided by applicable law or treaty (including future\ntime extensions), (iii) in any current or future medium and for any number\nof copies, and (iv) for any purpose whatsoever, including without\nlimitation commercial, advertising or promotional purposes (the\n\"License\"). The License shall be deemed effective as of the date CC0 was\napplied by Affirmer to the Work. Should any part of the License for any\nreason be judged legally invalid or ineffective under applicable law, such\npartial invalidity or ineffectiveness shall not invalidate the remainder\nof the License, and in such case Affirmer hereby affirms that he or she\nwill not (i) exercise any of his or her remaining Copyright and Related\nRights in the Work or (ii) assert any associated claims and causes of\naction with respect to the Work, in either case contrary to Affirmer's\nexpress Statement of Purpose.\n\n4. Limitations and Disclaimers.\n\n a. No trademark or patent rights held by Affirmer are waived, abandoned,\n surrendered, licensed or otherwise affected by this document.\n b. Affirmer offers the Work as-is and makes no representations or\n warranties of any kind concerning the Work, express, implied,\n statutory or otherwise, including without limitation warranties of\n title, merchantability, fitness for a particular purpose, non\n infringement, or the absence of latent or other defects, accuracy, or\n the present or absence of errors, whether or not discoverable, all to\n the greatest extent permissible under applicable law.\n c. Affirmer disclaims responsibility for clearing rights of other persons\n that may apply to the Work or any use thereof, including without\n limitation any person's Copyright and Related Rights in the Work.\n Further, Affirmer disclaims responsibility for obtaining any necessary\n consents, permissions or other rights required for any use of the\n Work.\n d. Affirmer understands and acknowledges that Creative Commons is not a\n party to this document and has no duty or obligation with respect to\n this CC0 or use of the Work.\n" }, { "path": "Makefile", "content": "# Makefile for monolith\n\nall: build build-gui\n.PHONY: all\n\nbuild:\n\t@cargo build --locked\n.PHONY: build\n\nbuild-gui:\n\t@cargo build --locked --bin monolith-gui --features=\"gui\"\n.PHONY: build_gui\n\nclean:\n\t@cargo clean\n.PHONY: clean\n\nformat:\n\t@cargo fmt --all --\n.PHONY: format\n\nformat-check:\n\t@cargo fmt --all -- --check\n.PHONY: format\n\ninstall:\n\t@cargo install --force --locked --path .\n.PHONY: install\n\nlint:\n\t@cargo clippy --fix --allow-dirty --allow-staged\n# \t@cargo fix --allow-dirty --allow-staged\n.PHONY: lint\n\nlint-check:\n\t@cargo clippy --\n.PHONY: lint_check\n\ntest: build\n\t@cargo test --locked\n.PHONY: test\n\nuninstall:\n\t@cargo uninstall\n.PHONY: uninstall\n\nupdate-lock-file:\n\t@cargo update\n.PHONY: clean\n" }, { "path": "README.md", "content": "[![monolith build status on GNU/Linux](https://github.com/Y2Z/monolith/workflows/GNU%2FLinux/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AGNU%2FLinux)\n[![monolith build status on macOS](https://github.com/Y2Z/monolith/workflows/macOS/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AmacOS)\n[![monolith build status on Windows](https://github.com/Y2Z/monolith/workflows/Windows/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AWindows)\n[![Monolith Actor on Apify](https://apify.com/actor-badge?actor=snshn/monolith)](https://apify.com/snshn/monolith?fpr=snshn)\n\n\n```\n _____ _____________ __________ ___________________ ___\n| \\ / \\ | | | | | |\n| \\/ __ \\| __ | | ___ ___ |__| |\n| | | | | | | | | | | |\n| |\\ /| |__| |__| |___| | | | | __ |\n| | \\__/ | |\\ | | | | | | |\n|___| |__________| \\___________________| |___| |___| |___|\n```\n\nA data hoarder’s dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.\n\nUnlike the conventional “Save page as”, `monolith` not only saves the target document, it embeds CSS, image, and JavaScript assets **all at once**, producing a single HTML5 document that is a joy to store and share.\n\nIf compared to saving websites with `wget -mpk`, this tool embeds all assets as data URLs and therefore lets browsers render the saved page exactly the way it was on the Internet, even when no network connection is available.\n\n\n---------------------------------------------------\n\n\n## Installation\n\n#### Using [Cargo](https://crates.io/crates/monolith) (cross-platform)\n\n```console\ncargo install monolith\n```\n\n#### Via [Homebrew](https://formulae.brew.sh/formula/monolith) (macOS and GNU/Linux)\n\n```console\nbrew install monolith\n```\n\n#### Via [Chocolatey](https://community.chocolatey.org/packages/monolith) (Windows)\n\n```console\nchoco install monolith\n```\n\n#### Via [Scoop](https://scoop.sh/#/apps?q=monolith) (Windows)\n\n```console\nscoop install main/monolith\n```\n\n#### Via [Winget](https://winstall.app/apps/Y2Z.Monolith) (Windows)\n\n```console\nwinget install --id=Y2Z.Monolith -e\n```\n\n#### Via [MacPorts](https://ports.macports.org/port/monolith/summary) (macOS)\n\n```console\nsudo port install monolith\n```\n\n#### Using [Snapcraft](https://snapcraft.io/monolith) (GNU/Linux)\n\n```console\nsnap install monolith\n```\n\n#### Using [Guix](https://packages.guix.gnu.org/packages/monolith) (GNU/Linux)\n\n```console\nguix install monolith\n```\n\n#### Using [NixPkgs](https://search.nixos.org/packages?channel=unstable&show=monolith&query=monolith)\n\n```console\nnix-env -iA nixpkgs.monolith\n```\n\n#### Using [Flox](https://flox.dev)\n\n```console\nflox install monolith\n```\n\n#### Using [Pacman](https://archlinux.org/packages/extra/x86_64/monolith) (Arch Linux)\n\n```console\npacman -S monolith\n```\n\n#### Using [aports](https://pkgs.alpinelinux.org/packages?name=monolith) (Alpine Linux)\n\n```console\napk add monolith\n```\n\n#### Using [XBPS Package Manager](https://voidlinux.org/packages/?q=monolith) (Void Linux)\n\n```console\nxbps-install -S monolith\n```\n\n#### Using [FreeBSD packages](https://svnweb.freebsd.org/ports/head/www/monolith/) (FreeBSD)\n\n```console\npkg install monolith\n```\n\n#### Using [FreeBSD ports](https://www.freshports.org/www/monolith/) (FreeBSD)\n\n```console\ncd /usr/ports/www/monolith/\nmake install clean\n```\n\n#### Using [pkgsrc](https://pkgsrc.se/www/monolith) (NetBSD, OpenBSD, Haiku, etc)\n\n```console\ncd /usr/pkgsrc/www/monolith\nmake install clean\n```\n\n#### Using [containers](https://www.docker.com/)\n\n```console\ndocker build -t y2z/monolith .\nsudo install -b dist/run-in-container.sh /usr/local/bin/monolith\n```\n\n#### From [source](https://github.com/Y2Z/monolith)\n\nDependencies: `libssl`, `cargo`\n\n
\n Install cargo (GNU/Linux)\n Check if cargo is installed\n\n ```console\n cargo -v\n ```\n\n If cargo is not already installed, install and add it to your existing ```$PATH``` (paraphrasing the [official installation instructions](https://doc.rust-lang.org/cargo/getting-started/installation.html)):\n\n ```console\n curl https://sh.rustup.rs -sSf | sh\n . \"$HOME/.cargo/env\"\n ```\n\nProceed with installing from source:\n
\n\n```console\ngit clone https://github.com/Y2Z/monolith.git\ncd monolith\nmake install\n```\n\n#### Using [pre-built binaries](https://github.com/Y2Z/monolith/releases) (Windows, ARM-based devices, etc)\n\nEvery release contains pre-built binaries for Windows, GNU/Linux, as well as platforms with non-standard CPU architecture.\n\n\n---------------------------------------------------\n\n\n## Usage\n\n```console\nmonolith https://lyrics.github.io/db/P/Portishead/Dummy/Roads/ -o %title%.%timestamp%.html\n```\n\n```console\ncat some-site-page.html | monolith -aIiFfcMv -b https://some.site/ - > some-site-page-with-assets.html\n```\n\n\n---------------------------------------------------\n\n\n## Options\n\n - `-a`: Exclude audio sources\n - `-b`: Use `custom base URL`\n - `-B`: Forbid retrieving assets from specified domain(s)\n - `-c`: Exclude CSS\n - `-C`: Read cookies from `file`\n - `-d`: Allow retrieving assets only from specified `domain(s)`\n - `-e`: Ignore network errors\n - `-E`: Save document using `custom encoding`\n - `-f`: Omit frames\n - `-F`: Exclude web fonts\n - `-h`: Print help information\n - `-i`: Remove images\n - `-I`: Isolate the document\n - `-j`: Exclude JavaScript\n - `-k`: Accept invalid X.509 (TLS) certificates\n - `-m`: Output in MHTML format instead of HTML\n - `-M`: Don't add timestamp and URL information\n - `-n`: Extract contents of NOSCRIPT elements\n - `-o`: Write output to `file` (use “-” for STDOUT)\n - `-q`: Be quiet\n - `-t`: Adjust `network request timeout`\n - `-u`: Provide `custom User-Agent`\n - `-v`: Exclude videos\n - `-V`: Print version number\n\n\n---------------------------------------------------\n\n\n## Whitelisting and blacklisting domains\n\nOptions `-d` and `-B` provide control over what domains can be used to retrieve assets from, e.g.:\n\n```console\nmonolith -I -d example.com -d www.example.com https://example.com -o example-only.html\n```\n\n```console\nmonolith -I -B -d .googleusercontent.com -d googleanalytics.com -d .google.com https://example.com -o example-no-ads.html\n```\n\n\n---------------------------------------------------\n\n\n## Dynamic content\n\nMonolith doesn't feature a JavaScript engine, hence websites that retrieve and display data after initial load may require usage of additional tools.\n\nFor example, Chromium (Chrome) can be used to act as a pre-processor for such pages:\n\n```console\nchromium --headless --window-size=1920,1080 --run-all-compositor-stages-before-draw --virtual-time-budget=9000 --incognito --dump-dom https://github.com | monolith - -I -b https://github.com -o github.html\n```\n\n\n---------------------------------------------------\n\n\n## Authentication\n\n```console\nmonolith https://username:password@example.com -o example-basic-auth.html\n```\n\n\n---------------------------------------------------\n\n\n## Proxies\n\nPlease set `https_proxy`, `http_proxy`, and `no_proxy` environment variables.\n\n\n---------------------------------------------------\n\n### Apify Actor Usage\n\n\"Run\n\nYou can run Monolith in the cloud without installation using the [Monolith Actor](https://apify.com/snshn/monolith?fpr=snshn) on [Apify](https://apify.com?fpr=snshn) free of charge.\n\n``` bash\necho '{\"urls\": [\"https://news.ycombinator.com/\"]}' | apify call -so snshn/monolith\n[{\n \"url\": \"https://news.ycombinator.com/\",\n \"status\": \"0\",\n \"kvsUrl\": \"https://api.apify.com/v2/key-value-stores/of9xNgvpon4elPLbc/records/https___news.ycombinator.com_\"\n}]\n```\n\nRead more about the [Monolith Actor](.actor/README.md), including how to use it via the Apify UI, API and CLI without installation.\n\n---------------------------------------------------\n\n\n## Contributing\n\nPlease open an issue if something is wrong, that helps make this project better.\n\n\n---------------------------------------------------\n\n\n## License\n\nTo the extent possible under law, the author(s) have dedicated all copyright related and neighboring rights to this software to the public domain worldwide.\nThis software is distributed without any warranty.\n" }, { "path": "dist/run-in-container.sh", "content": "#!/bin/sh\n\nDOCKER=docker\nif which podman 2>&1 > /dev/null; then\n DOCKER=podman\nfi\nORG_NAME=y2z\nPROG_NAME=monolith\n\n$DOCKER run --rm $ORG_NAME/$PROG_NAME \"$@\"\n" }, { "path": "monolith.nuspec", "content": "\n\n \n monolith\n 2.8.1\n Monolith\n Sunshine, Mahdi Robatipoor, Emmanuel Delaborde, Emi Simpson, rhysd\n https://github.com/Y2Z/monolith\n https://raw.githubusercontent.com/Y2Z/monolith/master/assets/icon/icon.png\n https://raw.githubusercontent.com/Y2Z/monolith/master/LICENSE\n false\n CLI tool for saving complete web pages as a single HTML file\n\nA data hoarder’s dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.\n\nUnlike the conventional “Save page as”, monolith not only saves the target document, it embeds CSS, image, and JavaScript assets all at once, producing a single HTML5 document that is a joy to store and share.\n\nIf compared to saving websites using wget, this tool embeds all assets as data URLs and therefore lets browsers render the saved page exactly the way it was on the Internet, even when no network connection is available.\n \n Public Domain\n en-US\n scraping archiving\n https://github.com/Y2Z/monolith/blob/master/README.md\n \n\n" }, { "path": "snap/snapcraft.yaml", "content": "name: monolith\nbase: core18 \n# Version data defined inside the monolith part below\nadopt-info: monolith\nsummary: Monolith - Save HTML pages with ease \ndescription: |\n A data hoarder's dream come true: bundle any web page into a single\n HTML file. You can finally replace that gazillion of open tabs with\n a gazillion of .html files stored somewhere on your precious little\n drive.\n Unlike conventional \"Save page as…\", monolith not only saves the\n target document, it embeds CSS, image, and JavaScript assets all\n at once, producing a single HTML5 document that is a joy to store\n and share.\n If compared to saving websites with wget -mpk, monolith embeds\n all assets as data URLs and therefore displays the saved page\n exactly the same, being completely separated from the Internet.\n\nconfinement: strict\n\narchitectures:\n - build-on: amd64\n - build-on: arm64\n - build-on: armhf\n - build-on: i386\n - build-on: ppc64el\n - build-on: s390x\n\nparts:\n monolith:\n plugin: rust\n source: .\n build-packages:\n - libssl-dev\n - pkg-config\n override-pull: |\n snapcraftctl pull\n # Determine the current tag\n last_committed_tag=\"$(git describe --tags --abbrev=0)\"\n last_committed_tag_ver=\"$(echo ${last_committed_tag} | sed 's/v//')\"\n # Determine the most recent version in the beta channel in the Snap Store\n last_released_tag=\"$(snap info $SNAPCRAFT_PROJECT_NAME | awk '$1 == \"beta:\" { print $2 }')\"\n # If the latest tag from the upstream project has not been released to\n # beta, build that tag instead of master.\n if [ \"${last_committed_tag_ver}\" != \"${last_released_tag}\" ]; then\n git fetch\n git checkout \"${last_committed_tag}\"\n fi\n # set version number of the snap based on what we did above\n snapcraftctl set-version $(git describe --tags --abbrev=0)\n\napps:\n monolith:\n command: monolith\n plugs:\n - home\n - network\n - removable-media\n" }, { "path": "src/cache.rs", "content": "use std::collections::HashMap;\nuse std::fs::File;\nuse std::io::{BufWriter, Write};\nuse std::path::Path;\n\nuse redb::{Database, Error, TableDefinition};\n\npub struct CacheMetadataItem {\n data: Option>, // Asset's blob; used for caching small files or if on-disk database isn't utilized\n media_type: Option, // MIME-type, things like \"text/plain\", \"image/png\"...\n charset: Option, // \"UTF-8\", \"UTF-16\"...\n}\n\n// #[derive(Debug)]\npub struct Cache {\n min_file_size: usize, // Only use database for assets larger than this size (in bytes), otherwise keep them in RAM\n metadata: HashMap, // Dictionary of metadata (and occasionally data [mostly for very small files])\n db: Option, // Pointer to database instance; None if not yet initialized or if failed to initialize\n db_ok: Option, // None by default, Some(true) if was able to initialize database, Some (false) if an error occurred\n db_file_path: Option, // Filesystem path to file used for storing database\n}\n\nconst FILE_WRITE_BUF_LEN: usize = 1024 * 100; // On-disk cache file write buffer size (in bytes)\nconst TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new(\"_\");\n\nimpl Cache {\n pub fn new(min_file_size: usize, db_file_path: Option) -> Cache {\n let mut cache = Cache {\n min_file_size,\n metadata: HashMap::new(),\n db: None,\n db_ok: None,\n db_file_path: db_file_path.clone(),\n };\n\n if db_file_path.is_some() {\n // Attempt to initialize on-disk database\n match Database::create(Path::new(&db_file_path.unwrap())) {\n Ok(db) => {\n cache.db = Some(db);\n cache.db_ok = Some(true);\n cache\n }\n Err(..) => {\n cache.db_ok = Some(false);\n cache\n }\n }\n } else {\n cache.db_ok = Some(false);\n cache\n }\n }\n\n pub fn set(&mut self, key: &str, data: &Vec, media_type: String, charset: String) {\n let mut cache_metadata_item: CacheMetadataItem = CacheMetadataItem {\n data: if self.db_ok.is_some() && self.db_ok.unwrap() {\n None\n } else {\n Some(data.to_owned().to_vec())\n },\n media_type: Some(media_type.to_owned()),\n charset: Some(charset),\n };\n\n if (self.db_ok.is_none() || !self.db_ok.unwrap()) || data.len() <= self.min_file_size {\n cache_metadata_item.data = Some(data.to_owned().to_vec());\n } else {\n match self.db.as_ref().unwrap().begin_write() {\n Ok(write_txn) => {\n {\n let mut table = write_txn.open_table(TABLE).unwrap();\n table.insert(key, &*data.to_owned()).unwrap();\n }\n write_txn.commit().unwrap();\n }\n Err(..) => {\n // Fall back to caching everything in memory\n cache_metadata_item.data = Some(data.to_owned().to_vec());\n }\n }\n }\n\n self.metadata\n .insert((*key).to_string(), cache_metadata_item);\n }\n\n pub fn get(&self, key: &str) -> Result<(Vec, String, String), Error> {\n if self.metadata.contains_key(key) {\n let metadata_item = self.metadata.get(key).unwrap();\n\n if metadata_item.data.is_some() {\n return Ok((\n metadata_item.data.as_ref().unwrap().to_vec(),\n metadata_item.media_type.as_ref().expect(\"\").to_string(),\n metadata_item.charset.as_ref().expect(\"\").to_string(),\n ));\n } else if self.db_ok.is_some() && self.db_ok.unwrap() {\n let read_txn = self.db.as_ref().unwrap().begin_read()?;\n let table = read_txn.open_table(TABLE)?;\n let data = table.get(key)?;\n let bytes = data.unwrap();\n\n return Ok((\n bytes.value().to_vec(),\n metadata_item.media_type.as_ref().expect(\"\").to_string(),\n metadata_item.charset.as_ref().expect(\"\").to_string(),\n ));\n }\n }\n\n Err(Error::TransactionInProgress) // XXX\n }\n\n pub fn contains_key(&self, key: &str) -> bool {\n self.metadata.contains_key(key)\n }\n\n pub fn destroy_database_file(&mut self) {\n if self.db_ok.is_none() || !self.db_ok.unwrap() {\n return;\n }\n\n // Destroy database instance (prevents writes into file)\n self.db = None;\n self.db_ok = Some(false);\n\n // Wipe database file\n if let Some(db_file_path) = self.db_file_path.to_owned() {\n // Overwrite file with zeroes\n if let Ok(temp_file) = File::options()\n .read(true)\n .write(true)\n .open(db_file_path.clone())\n {\n let mut buffer = [0; FILE_WRITE_BUF_LEN];\n let mut remaining_size: usize = temp_file.metadata().unwrap().len() as usize;\n let mut writer = BufWriter::new(temp_file);\n\n while remaining_size > 0 {\n let bytes_to_write: usize = if remaining_size < FILE_WRITE_BUF_LEN {\n remaining_size\n } else {\n FILE_WRITE_BUF_LEN\n };\n let buffer = &mut buffer[..bytes_to_write];\n writer.write(buffer).unwrap();\n\n remaining_size -= bytes_to_write;\n }\n }\n }\n }\n}\n" }, { "path": "src/cookies.rs", "content": "use std::time::{SystemTime, UNIX_EPOCH};\n\nuse crate::url::Url;\n\npub struct Cookie {\n pub domain: String,\n pub include_subdomains: bool,\n pub path: String,\n pub https_only: bool,\n pub expires: u64,\n pub name: String,\n pub value: String,\n}\n\n#[derive(Debug)]\npub enum CookieFileContentsParseError {\n InvalidHeader,\n}\n\nimpl Cookie {\n pub fn is_expired(&self) -> bool {\n if self.expires == 0 {\n return false; // Session, never expires\n }\n\n let start = SystemTime::now();\n let since_the_epoch = start\n .duration_since(UNIX_EPOCH)\n .expect(\"Time went backwards\");\n\n self.expires < since_the_epoch.as_secs()\n }\n\n pub fn matches_url(&self, url: &str) -> bool {\n match Url::parse(url) {\n Ok(url) => {\n // Check protocol scheme\n match url.scheme() {\n \"http\" => {\n if self.https_only {\n return false;\n }\n }\n \"https\" => {}\n _ => {\n // Should never match URLs of protocols other than HTTP(S)\n return false;\n }\n }\n\n // Check host\n if let Some(url_host) = url.host_str() {\n if self.domain.starts_with(\".\") && self.include_subdomains {\n if !url_host.to_lowercase().ends_with(&self.domain)\n && !url_host\n .eq_ignore_ascii_case(&self.domain[1..self.domain.len() - 1])\n {\n return false;\n }\n } else if !url_host.eq_ignore_ascii_case(&self.domain) {\n return false;\n }\n } else {\n return false;\n }\n\n // Check path\n if !url.path().eq_ignore_ascii_case(&self.path)\n && !url.path().starts_with(&self.path)\n {\n return false;\n }\n }\n Err(_) => {\n return false;\n }\n }\n\n true\n }\n}\n\npub fn parse_cookie_file_contents(\n cookie_file_contents: &str,\n) -> Result, CookieFileContentsParseError> {\n let mut cookies: Vec = Vec::new();\n\n for (i, line) in cookie_file_contents.lines().enumerate() {\n if i == 0 {\n // Parsing first line\n if !line.eq(\"# HTTP Cookie File\") && !line.eq(\"# Netscape HTTP Cookie File\") {\n return Err(CookieFileContentsParseError::InvalidHeader);\n }\n } else {\n // Ignore comment lines\n if line.starts_with(\"#\") {\n continue;\n }\n\n // Attempt to parse values\n let mut fields = line.split(\"\\t\");\n if fields.clone().count() != 7 {\n continue;\n }\n cookies.push(Cookie {\n domain: fields.next().unwrap().to_string().to_lowercase(),\n include_subdomains: fields.next().unwrap() == \"TRUE\",\n path: fields.next().unwrap().to_string(),\n https_only: fields.next().unwrap() == \"TRUE\",\n expires: fields.next().unwrap().parse::().unwrap(),\n name: fields.next().unwrap().to_string(),\n value: fields.next().unwrap().to_string(),\n });\n }\n }\n\n Ok(cookies)\n}\n" }, { "path": "src/core.rs", "content": "use std::env;\nuse std::error::Error;\nuse std::fmt;\nuse std::fs;\nuse std::io::{self, Write};\nuse std::path::Path;\n\nuse chrono::{SecondsFormat, Utc};\nuse encoding_rs::Encoding;\nuse markup5ever_rcdom::RcDom;\nuse url::Url;\n\nuse crate::html::{\n add_favicon, create_metadata_tag, get_base_url, get_charset, get_robots, get_title,\n has_favicon, html_to_dom, serialize_document, set_base_url, set_charset, set_robots, walk,\n};\nuse crate::session::Session;\nuse crate::url::{create_data_url, resolve_url};\n\n#[derive(Debug)]\npub struct MonolithError {\n details: String,\n}\n\nimpl MonolithError {\n fn new(msg: &str) -> MonolithError {\n MonolithError {\n details: msg.to_string(),\n }\n }\n}\n\nimpl fmt::Display for MonolithError {\n fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n write!(f, \"{}\", self.details)\n }\n}\n\nimpl Error for MonolithError {\n fn description(&self) -> &str {\n &self.details\n }\n}\n\n#[derive(Clone, Debug, PartialEq, Eq, Default)]\npub enum MonolithOutputFormat {\n #[default]\n HTML,\n MHTML,\n // WARC,\n // ZIM,\n // HAR,\n}\n\n#[derive(Default)]\npub struct MonolithOptions {\n pub base_url: Option,\n pub blacklist_domains: bool,\n pub domains: Option>,\n pub encoding: Option,\n pub ignore_errors: bool,\n pub insecure: bool,\n pub isolate: bool,\n pub no_audio: bool,\n pub no_css: bool,\n pub no_fonts: bool,\n pub no_frames: bool,\n pub no_images: bool,\n pub no_js: bool,\n pub no_metadata: bool,\n pub no_video: bool,\n pub output_format: MonolithOutputFormat,\n pub silent: bool,\n pub timeout: u64,\n pub unwrap_noscript: bool,\n pub user_agent: Option,\n}\n\nconst ANSI_COLOR_RED: &str = \"\\x1b[31m\";\nconst ANSI_COLOR_RESET: &str = \"\\x1b[0m\";\nconst FILE_SIGNATURES: [[&[u8]; 2]; 18] = [\n // Image\n [b\"GIF87a\", b\"image/gif\"],\n [b\"GIF89a\", b\"image/gif\"],\n [b\"\\xFF\\xD8\\xFF\", b\"image/jpeg\"],\n [b\"\\x89PNG\\x0D\\x0A\\x1A\\x0A\", b\"image/png\"],\n [b\",\n input_encoding: Option,\n input_target: Option,\n) -> Result<(Vec, Option), MonolithError> {\n // Validate options\n {\n // Check if custom encoding value is acceptable\n if let Some(custom_output_encoding) = session.options.encoding.clone() {\n if Encoding::for_label_no_replacement(custom_output_encoding.as_bytes()).is_none() {\n return Err(MonolithError::new(&format!(\n \"unknown encoding \\\"{}\\\"\",\n &custom_output_encoding\n )));\n }\n }\n }\n\n let mut base_url: Url = if input_target.is_some() {\n Url::parse(&input_target.clone().unwrap()).unwrap()\n } else {\n Url::parse(\"data:text/html,\").unwrap()\n };\n let mut document_encoding: String = input_encoding.clone().unwrap_or(\"utf-8\".to_string());\n let mut dom: RcDom;\n\n // Initial parse\n dom = html_to_dom(&input_data, document_encoding.clone());\n\n // Attempt to determine document's encoding\n if let Some(html_charset) = get_charset(&dom.document) {\n if !html_charset.is_empty() {\n // Check if the charset specified inside HTML is valid\n if let Some(document_charset) =\n Encoding::for_label_no_replacement(html_charset.as_bytes())\n {\n document_encoding = html_charset;\n dom = html_to_dom(&input_data, document_charset.name().to_string());\n }\n }\n }\n\n // Use custom base URL if specified; read and use what's in the DOM otherwise\n let custom_base_url: String = session.options.base_url.clone().unwrap_or_default();\n if custom_base_url.is_empty() {\n // No custom base URL is specified; try to see if document has BASE element\n if let Some(existing_base_url) = get_base_url(&dom.document) {\n base_url = resolve_url(&base_url, &existing_base_url);\n }\n } else {\n // Custom base URL provided\n match Url::parse(&custom_base_url) {\n Ok(parsed_url) => {\n if parsed_url.scheme() == \"file\" {\n // File base URLs can only work with documents saved from filesystem\n if base_url.scheme() == \"file\" {\n base_url = parsed_url;\n }\n } else {\n base_url = parsed_url;\n }\n }\n Err(_) => {\n // Failed to parse given base URL, perhaps it's a filesystem path?\n if base_url.scheme() == \"file\" {\n // Relative paths could work for documents saved from filesystem\n let path: &Path = Path::new(&custom_base_url);\n if path.exists() {\n match Url::from_file_path(fs::canonicalize(path).unwrap()) {\n Ok(file_url) => {\n base_url = file_url;\n }\n Err(_) => {\n return Err(MonolithError::new(&format!(\n \"could not map given path to base URL \\\"{}\\\"\",\n custom_base_url\n )));\n }\n }\n }\n }\n }\n }\n }\n\n // Traverse through the document and embed remote assets\n walk(&mut session, &base_url, &dom.document);\n\n // Update or add new BASE element to reroute network requests and hash-links\n if let Some(new_base_url) = session.options.base_url.clone() {\n dom = set_base_url(&dom.document, new_base_url);\n }\n\n // Request and embed /favicon.ico (unless it's already linked in the document)\n if !session.options.no_images\n && (base_url.scheme() == \"http\" || base_url.scheme() == \"https\")\n && (input_target.is_some()\n && (input_target.as_ref().unwrap().starts_with(\"http:\")\n || input_target.as_ref().unwrap().starts_with(\"https:\")))\n && !has_favicon(&dom.document)\n {\n let favicon_ico_url: Url = resolve_url(&base_url, \"/favicon.ico\");\n\n match session.retrieve_asset(/*&target_url, */ &base_url, &favicon_ico_url) {\n Ok((data, final_url, media_type, charset)) => {\n let favicon_data_url: Url =\n create_data_url(&media_type, &charset, &data, &final_url);\n dom = add_favicon(&dom.document, favicon_data_url.to_string());\n }\n Err(_) => {\n // Failed to retrieve /favicon.ico\n }\n }\n }\n\n // Append noindex META-tag\n let meta_robots_content_value = get_robots(&dom.document).unwrap_or_default();\n if meta_robots_content_value.trim().is_empty() || meta_robots_content_value != \"none\" {\n dom = set_robots(dom, \"none\");\n }\n\n // Save using specified charset, if given\n if let Some(custom_encoding) = session.options.encoding.clone() {\n document_encoding = custom_encoding;\n dom = set_charset(dom, document_encoding.clone());\n }\n\n let document_title: Option = get_title(&dom.document);\n\n if session.options.output_format == MonolithOutputFormat::HTML {\n // Serialize DOM tree\n let mut result: Vec = serialize_document(dom, document_encoding, &session.options);\n\n // Prepend metadata comment tag\n if !session.options.no_metadata && !input_target.clone().unwrap_or_default().is_empty() {\n let mut metadata_comment: String =\n create_metadata_tag(&Url::parse(&input_target.unwrap_or_default()).unwrap());\n // let mut metadata_comment: String = create_metadata_tag(target);\n metadata_comment += \"\\n\";\n result.splice(0..0, metadata_comment.as_bytes().to_vec());\n }\n\n // Ensure newline at end of result\n if result.last() != Some(&b\"\\n\"[0]) {\n result.extend_from_slice(b\"\\n\");\n }\n\n Ok((result, document_title))\n } else if session.options.output_format == MonolithOutputFormat::MHTML {\n // Serialize DOM tree\n let mut result: Vec = serialize_document(dom, document_encoding, &session.options);\n\n // Prepend metadata comment tag\n if !session.options.no_metadata && !input_target.clone().unwrap_or_default().is_empty() {\n let mut metadata_comment: String =\n create_metadata_tag(&Url::parse(&input_target.unwrap_or_default()).unwrap());\n // let mut metadata_comment: String = create_metadata_tag(target);\n metadata_comment += \"\\n\";\n result.splice(0..0, metadata_comment.as_bytes().to_vec());\n }\n\n // Extremely hacky way to convert output to MIME\n let mime = \"MIME-Version: 1.0\\r\\n\\\nContent-Type: multipart/related; boundary=\\\"----=_NextPart_000_0000\\\"\\r\\n\\\n\\r\\n\\\n------=_NextPart_000_0000\\r\\n\\\nContent-Type: text/html; charset=\\\"utf-8\\\"\\r\\n\\\nContent-Location: http://example.com/\\r\\n\\\n\\r\\n\";\n\n result.splice(0..0, mime.as_bytes().to_vec());\n\n let mime = \"\\r\\n------=_NextPart_000_0000--\\r\\n\";\n\n result.extend_from_slice(mime.as_bytes());\n\n Ok((result, document_title))\n } else {\n Ok((vec![], document_title))\n }\n}\n\npub fn create_monolithic_document(\n mut session: Session,\n target: String,\n) -> Result<(Vec, Option), MonolithError> {\n // Check if target was provided\n if target.is_empty() {\n return Err(MonolithError::new(\"no target specified\"));\n }\n\n // Validate options\n {\n // Check if custom encoding value is acceptable\n if let Some(custom_encoding) = session.options.encoding.clone() {\n if Encoding::for_label_no_replacement(custom_encoding.as_bytes()).is_none() {\n return Err(MonolithError::new(&format!(\n \"unknown encoding \\\"{}\\\"\",\n &custom_encoding\n )));\n }\n }\n }\n\n let mut target_url = match target.as_str() {\n target_str => match Url::parse(target_str) {\n Ok(target_url) => match target_url.scheme() {\n \"data\" | \"file\" | \"http\" | \"https\" => target_url,\n unsupported_scheme => {\n return Err(MonolithError::new(&format!(\n \"unsupported target URL scheme \\\"{}\\\"\",\n unsupported_scheme\n )));\n }\n },\n Err(_) => {\n // Failed to parse given base URL (perhaps it's a filesystem path?)\n let path: &Path = Path::new(&target_str);\n\n match path.exists() {\n true => match path.is_file() {\n true => {\n let canonical_path = fs::canonicalize(path).unwrap();\n\n match Url::from_file_path(canonical_path) {\n Ok(url) => url,\n Err(_) => {\n return Err(MonolithError::new(&format!(\n \"could not generate file URL out of given path \\\"{}\\\"\",\n &target_str\n )));\n }\n }\n }\n false => {\n return Err(MonolithError::new(&format!(\n \"local target \\\"{}\\\" is not a file\",\n &target_str\n )));\n }\n },\n false => {\n // It is not a FS path, now we do what browsers do:\n // prepend \"http://\" and hope it points to a website\n Url::parse(&format!(\"http://{}\", &target_str)).unwrap()\n }\n }\n }\n },\n };\n\n let data: Vec;\n let document_encoding: Option;\n\n // Retrieve target document\n if target_url.scheme() == \"file\"\n || target_url.scheme() == \"http\"\n || target_url.scheme() == \"https\"\n || target_url.scheme() == \"data\"\n {\n match session.retrieve_asset(&target_url, &target_url) {\n Ok((retrieved_data, final_url, media_type, charset)) => {\n if !media_type.eq_ignore_ascii_case(\"text/html\")\n && !media_type.eq_ignore_ascii_case(\"application/xhtml+xml\")\n {\n // Provide output as text (without processing it, the way browsers do)\n return Ok((retrieved_data, None));\n }\n\n // If got redirected, set target_url to that\n if final_url != target_url {\n target_url = final_url.clone();\n }\n\n data = retrieved_data;\n document_encoding = Some(charset);\n }\n Err(_) => {\n return Err(MonolithError::new(\"could not retrieve target document\"));\n }\n }\n } else {\n return Err(MonolithError::new(\"unsupported target\"));\n }\n\n create_monolithic_document_from_data(\n session,\n data,\n document_encoding,\n Some(target_url.to_string()),\n )\n}\n\npub fn detect_media_type(data: &[u8], url: &Url) -> String {\n // At first attempt to read file's header\n for file_signature in FILE_SIGNATURES.iter() {\n if data.starts_with(file_signature[0]) {\n return String::from_utf8(file_signature[1].to_vec()).unwrap();\n }\n }\n\n // If header didn't match any known magic signatures,\n // try to guess media type from file name\n let parts: Vec<&str> = url.path().split('/').collect();\n detect_media_type_by_file_name(parts.last().unwrap())\n}\n\npub fn detect_media_type_by_file_name(filename: &str) -> String {\n let filename_lowercased: &str = &filename.to_lowercase();\n let parts: Vec<&str> = filename_lowercased.split('.').collect();\n\n let mime: &str = match parts.last() {\n Some(v) => match *v {\n \"avi\" => \"video/avi\",\n \"bmp\" => \"image/bmp\",\n \"css\" => \"text/css\",\n \"flac\" => \"audio/flac\",\n \"gif\" => \"image/gif\",\n \"htm\" | \"html\" => \"text/html\",\n \"ico\" => \"image/x-icon\",\n \"jpeg\" | \"jpg\" => \"image/jpeg\",\n \"js\" => \"text/javascript\",\n \"json\" => \"application/json\",\n \"jsonld\" => \"application/ld+json\",\n \"mp3\" => \"audio/mpeg\",\n \"mp4\" | \"m4v\" => \"video/mp4\",\n \"ogg\" => \"audio/ogg\",\n \"ogv\" => \"video/ogg\",\n \"pdf\" => \"application/pdf\",\n \"png\" => \"image/png\",\n \"svg\" => \"image/svg+xml\",\n \"swf\" => \"application/x-shockwave-flash\",\n \"tif\" | \"tiff\" => \"image/tiff\",\n \"txt\" => \"text/plain\",\n \"wav\" => \"audio/wav\",\n \"webp\" => \"image/webp\",\n \"woff\" => \"font/woff\",\n \"woff2\" => \"font/woff2\",\n \"xhtml\" => \"application/xhtml+xml\",\n \"xml\" => \"text/xml\",\n &_ => \"\",\n },\n None => \"\",\n };\n mime.to_string()\n}\n\npub fn format_output_path(\n path: &str,\n document_title: &str,\n output_format: MonolithOutputFormat,\n) -> String {\n let datetime: &str = &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);\n\n path.replace(\"%timestamp%\", &datetime.replace(':', \"_\"))\n .replace(\n \"%title%\",\n document_title\n .to_string()\n .replace(['/', '\\\\'], \"_\")\n .replace('<', \"[\")\n .replace('>', \"]\")\n .replace(':', \" - \")\n .replace('\\\"', \"\")\n .replace('|', \"-\")\n .replace('?', \"\")\n .trim_start_matches('.'),\n )\n .replace(\n \"%ext%\",\n if output_format == MonolithOutputFormat::HTML {\n \"htm\"\n } else if output_format == MonolithOutputFormat::MHTML {\n \"mht\"\n } else {\n \"\"\n },\n )\n .replace(\n \"%extension%\",\n if output_format == MonolithOutputFormat::HTML {\n \"html\"\n } else if output_format == MonolithOutputFormat::MHTML {\n \"mhtml\"\n } else {\n \"\"\n },\n )\n .to_string()\n}\n\npub fn is_plaintext_media_type(media_type: &str) -> bool {\n media_type.to_lowercase().as_str().starts_with(\"text/\")\n || PLAINTEXT_MEDIA_TYPES.contains(&media_type.to_lowercase().as_str())\n}\n\npub fn parse_content_type(content_type: &str) -> (String, String, bool) {\n let mut media_type: String = \"text/plain\".to_string();\n let mut charset: String = \"US-ASCII\".to_string();\n let mut is_base64: bool = false;\n\n // Parse meta data\n let content_type_items: Vec<&str> = content_type.split(';').collect();\n let mut i: i8 = 0;\n for item in &content_type_items {\n if i == 0 {\n if !item.trim().is_empty() {\n media_type = item.trim().to_string();\n }\n } else if item.trim().eq_ignore_ascii_case(\"base64\") {\n is_base64 = true;\n } else if item.trim().starts_with(\"charset=\") {\n charset = item.trim().chars().skip(8).collect();\n }\n\n i += 1;\n }\n\n (media_type, charset, is_base64)\n}\n\npub fn print_error_message(text: &str) {\n let stderr = io::stderr();\n let mut handle = stderr.lock();\n\n const ENV_VAR_NO_COLOR: &str = \"NO_COLOR\";\n const ENV_VAR_TERM: &str = \"TERM\";\n\n let mut no_color = env::var_os(ENV_VAR_NO_COLOR).is_some() || atty::isnt(atty::Stream::Stderr);\n if let Some(term) = env::var_os(ENV_VAR_TERM) {\n if term == \"dumb\" {\n no_color = true;\n }\n }\n\n if handle\n .write_all(\n format!(\n \"{}{}{}\\n\",\n if no_color { \"\" } else { ANSI_COLOR_RED },\n &text,\n if no_color { \"\" } else { ANSI_COLOR_RESET },\n )\n .as_bytes(),\n )\n .is_ok()\n {}\n}\n\npub fn print_info_message(text: &str) {\n let stderr = io::stderr();\n let mut handle = stderr.lock();\n\n if handle.write_all(format!(\"{}\\n\", &text).as_bytes()).is_ok() {}\n}\n" }, { "path": "src/css.rs", "content": "use cssparser::{\n serialize_identifier, serialize_string, ParseError, Parser, ParserInput, SourcePosition, Token,\n};\n\nuse crate::session::Session;\nuse crate::url::{create_data_url, resolve_url, Url, EMPTY_IMAGE_DATA_URL};\n\nconst CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[\n // Universal\n \"background\",\n \"background-image\",\n \"border-image\",\n \"border-image-source\",\n \"content\",\n \"cursor\",\n \"list-style\",\n \"list-style-image\",\n \"mask\",\n \"mask-image\",\n // Specific to @counter-style\n \"additive-symbols\",\n \"negative\",\n \"pad\",\n \"prefix\",\n \"suffix\",\n \"symbols\",\n];\n\npub fn embed_css(session: &mut Session, document_url: &Url, css: &str) -> String {\n let mut input = ParserInput::new(css);\n let mut parser = Parser::new(&mut input);\n\n process_css(session, document_url, &mut parser, \"\", \"\", \"\").unwrap()\n}\n\npub fn format_ident(ident: &str) -> String {\n let mut res: String = \"\".to_string();\n let _ = serialize_identifier(ident, &mut res);\n res = res.trim_end().to_string();\n res\n}\n\npub fn format_quoted_string(string: &str) -> String {\n let mut res: String = \"\".to_string();\n let _ = serialize_string(string, &mut res);\n res\n}\n\npub fn is_image_url_prop(prop_name: &str) -> bool {\n CSS_PROPS_WITH_IMAGE_URLS\n .iter()\n .any(|p| prop_name.eq_ignore_ascii_case(p))\n}\n\npub fn process_css<'a>(\n session: &mut Session,\n document_url: &Url,\n parser: &mut Parser,\n rule_name: &str,\n prop_name: &str,\n func_name: &str,\n) -> Result> {\n let mut result: String = \"\".to_string();\n\n let mut curr_rule: String = rule_name.to_string();\n let mut curr_prop: String = prop_name.to_string();\n let mut token: &Token;\n let mut token_offset: SourcePosition;\n\n loop {\n token_offset = parser.position();\n token = match parser.next_including_whitespace_and_comments() {\n Ok(token) => token,\n Err(_) => {\n break;\n }\n };\n\n match *token {\n Token::Comment(_) => {\n let token_slice = parser.slice_from(token_offset);\n result.push_str(token_slice);\n }\n Token::Semicolon => result.push(';'),\n Token::Colon => result.push(':'),\n Token::Comma => result.push(','),\n Token::ParenthesisBlock | Token::SquareBracketBlock | Token::CurlyBracketBlock => {\n if session.options.no_fonts && curr_rule == \"font-face\" {\n continue;\n }\n\n let closure: &str;\n if token == &Token::ParenthesisBlock {\n result.push('(');\n closure = \")\";\n } else if token == &Token::SquareBracketBlock {\n result.push('[');\n closure = \"]\";\n } else {\n result.push('{');\n closure = \"}\";\n }\n\n let block_css: String = parser\n .parse_nested_block(|parser| {\n process_css(\n session,\n document_url,\n parser,\n rule_name,\n curr_prop.as_str(),\n func_name,\n )\n })\n .unwrap();\n result.push_str(block_css.as_str());\n\n result.push_str(closure);\n }\n Token::CloseParenthesis => result.push(')'),\n Token::CloseSquareBracket => result.push(']'),\n Token::CloseCurlyBracket => result.push('}'),\n Token::IncludeMatch => result.push_str(\"~=\"),\n Token::DashMatch => result.push_str(\"|=\"),\n Token::PrefixMatch => result.push_str(\"^=\"),\n Token::SuffixMatch => result.push_str(\"$=\"),\n Token::SubstringMatch => result.push_str(\"*=\"),\n Token::CDO => result.push_str(\"\"),\n Token::WhiteSpace(value) => {\n result.push_str(value);\n }\n // div...\n Token::Ident(ref value) => {\n curr_rule = \"\".to_string();\n curr_prop = value.to_string();\n result.push_str(&format_ident(value));\n }\n // @import, @font-face, @charset, @media...\n Token::AtKeyword(ref value) => {\n curr_rule = value.to_string();\n if session.options.no_fonts && curr_rule == \"font-face\" {\n continue;\n }\n result.push('@');\n result.push_str(value);\n }\n Token::Hash(ref value) => {\n result.push('#');\n result.push_str(value);\n }\n Token::QuotedString(ref value) => {\n if curr_rule == \"import\" {\n // Reset current at-rule value\n curr_rule = \"\".to_string();\n\n // Skip empty import values\n if value.len() == 0 {\n result.push_str(\"''\");\n continue;\n }\n\n let import_full_url: Url = resolve_url(document_url, value);\n match session.retrieve_asset(document_url, &import_full_url) {\n Ok((\n import_contents,\n import_final_url,\n import_media_type,\n import_charset,\n )) => {\n let mut import_data_url = create_data_url(\n &import_media_type,\n &import_charset,\n embed_css(\n session,\n &import_final_url,\n &String::from_utf8_lossy(&import_contents),\n )\n .as_bytes(),\n &import_final_url,\n );\n import_data_url.set_fragment(import_full_url.fragment());\n result\n .push_str(format_quoted_string(import_data_url.as_ref()).as_str());\n }\n Err(_) => {\n // Keep remote reference if unable to retrieve the asset\n if import_full_url.scheme() == \"http\"\n || import_full_url.scheme() == \"https\"\n {\n result.push_str(\n format_quoted_string(import_full_url.as_ref()).as_str(),\n );\n }\n }\n }\n } else if func_name == \"url\" {\n // Skip empty url()'s\n if value.len() == 0 {\n continue;\n }\n\n if session.options.no_images && is_image_url_prop(curr_prop.as_str()) {\n result.push_str(format_quoted_string(EMPTY_IMAGE_DATA_URL).as_str());\n } else {\n let resolved_url: Url = resolve_url(document_url, value);\n\n match session.retrieve_asset(document_url, &resolved_url) {\n Ok((data, final_url, media_type, charset)) => {\n // TODO: if it's @font-face, exclude definitions of non-woff/woff-2 fonts (if woff/woff-2 are present)\n let mut data_url =\n create_data_url(&media_type, &charset, &data, &final_url);\n data_url.set_fragment(resolved_url.fragment());\n result.push_str(format_quoted_string(data_url.as_ref()).as_str());\n }\n Err(_) => {\n // Keep remote reference if unable to retrieve the asset\n if resolved_url.scheme() == \"http\"\n || resolved_url.scheme() == \"https\"\n {\n result.push_str(\n format_quoted_string(resolved_url.as_ref()).as_str(),\n );\n }\n }\n }\n }\n } else {\n result.push_str(format_quoted_string(value).as_str());\n }\n }\n Token::Number {\n ref has_sign,\n ref value,\n ..\n } => {\n if *has_sign && *value >= 0. {\n result.push('+');\n }\n result.push_str(&value.to_string())\n }\n Token::Percentage {\n ref has_sign,\n ref unit_value,\n ..\n } => {\n if *has_sign && *unit_value >= 0. {\n result.push('+');\n }\n result.push_str(&(unit_value * 100.0).to_string());\n result.push('%');\n }\n Token::Dimension {\n ref has_sign,\n ref value,\n ref unit,\n ..\n } => {\n if *has_sign && *value >= 0. {\n result.push('+');\n }\n result.push_str(&value.to_string());\n result.push_str(unit.as_ref());\n }\n // #selector, #id...\n Token::IDHash(ref value) => {\n curr_rule = \"\".to_string();\n result.push('#');\n result.push_str(&format_ident(value));\n }\n // url()\n Token::UnquotedUrl(ref value) => {\n let is_import: bool = curr_rule == \"import\";\n\n if is_import {\n // Reset current at-rule value\n curr_rule = \"\".to_string();\n }\n\n // Skip empty url()'s\n if value.len() < 1 {\n result.push_str(\"url()\");\n continue;\n } else if value.starts_with(\"#\") {\n result.push_str(\"url(\");\n result.push_str(value);\n result.push(')');\n continue;\n }\n\n result.push_str(\"url(\");\n if is_import {\n let full_url: Url = resolve_url(document_url, value);\n match session.retrieve_asset(document_url, &full_url) {\n Ok((css, final_url, media_type, charset)) => {\n let mut data_url = create_data_url(\n &media_type,\n &charset,\n embed_css(session, &final_url, &String::from_utf8_lossy(&css))\n .as_bytes(),\n &final_url,\n );\n data_url.set_fragment(full_url.fragment());\n result.push_str(format_quoted_string(data_url.as_ref()).as_str());\n }\n Err(_) => {\n // Keep remote reference if unable to retrieve the asset\n if full_url.scheme() == \"http\" || full_url.scheme() == \"https\" {\n result.push_str(format_quoted_string(full_url.as_ref()).as_str());\n }\n }\n }\n } else if is_image_url_prop(curr_prop.as_str()) && session.options.no_images {\n result.push_str(format_quoted_string(EMPTY_IMAGE_DATA_URL).as_str());\n } else {\n let full_url: Url = resolve_url(document_url, value);\n match session.retrieve_asset(document_url, &full_url) {\n Ok((data, final_url, media_type, charset)) => {\n let mut data_url =\n create_data_url(&media_type, &charset, &data, &final_url);\n data_url.set_fragment(full_url.fragment());\n result.push_str(format_quoted_string(data_url.as_ref()).as_str());\n }\n Err(_) => {\n // Keep remote reference if unable to retrieve the asset\n if full_url.scheme() == \"http\" || full_url.scheme() == \"https\" {\n result.push_str(format_quoted_string(full_url.as_ref()).as_str());\n }\n }\n }\n }\n result.push(')');\n }\n // =\n Token::Delim(ref value) => result.push(*value),\n Token::Function(ref name) => {\n let function_name: &str = &name.clone();\n result.push_str(function_name);\n result.push('(');\n\n let block_css: String = parser\n .parse_nested_block(|parser| {\n process_css(\n session,\n document_url,\n parser,\n curr_rule.as_str(),\n curr_prop.as_str(),\n function_name,\n )\n })\n .unwrap();\n result.push_str(block_css.as_str());\n\n result.push(')');\n }\n Token::BadUrl(_) | Token::BadString(_) => {}\n }\n }\n\n // Ensure empty CSS is really empty\n if !result.is_empty() && result.trim().is_empty() {\n result = result.trim().to_string()\n }\n\n Ok(result)\n}\n" }, { "path": "src/gui.rs", "content": "use std::fs;\nuse std::io::Write;\nuse std::path;\nuse std::thread;\n\nuse directories::UserDirs;\nuse druid::widget::{Button, Checkbox, Either, Flex, Label, Spinner, TextBox};\nuse druid::{\n commands, AppDelegate, AppLauncher, Command, Data, DelegateCtx, Env, FileDialogOptions,\n FileSpec, Handled, Lens, LocalizedString, PlatformError, Target, Widget, WidgetExt, WindowDesc,\n};\nuse tempfile::{Builder, NamedTempFile};\n\nuse monolith::cache::Cache;\nuse monolith::core::{\n create_monolithic_document, format_output_path, MonolithError, MonolithOptions,\n MonolithOutputFormat,\n};\nuse monolith::session::Session;\n\nconst CACHE_ASSET_FILE_SIZE_THRESHOLD: usize = 1024 * 20; // Minimum file size for on-disk caching (in bytes)\nconst FILESPEC_HTML: FileSpec = FileSpec::new(\"HTML files\", &[\"html\"]);\nconst MONOLITH_GUI_WRITE_OUTPUT: druid::Selector<(Vec, Option)> =\n druid::Selector::new(\"monolith-gui.write-output\");\nconst MONOLITH_GUI_ERROR: druid::Selector =\n druid::Selector::new(\"monolith-gui.error\");\nconst TEXT_BOX_WIDTH: f64 = 512_f64;\n\nstruct Delegate;\n\n#[derive(Clone, Data, Lens)]\nstruct AppState {\n target: String,\n keep_fonts: bool,\n keep_frames: bool,\n keep_images: bool,\n keep_scripts: bool,\n keep_styles: bool,\n output_path: String,\n isolate: bool,\n unwrap_noscript: bool,\n busy: bool,\n}\n\nfn main() -> Result<(), PlatformError> {\n let mut program_name: String = env!(\"CARGO_PKG_NAME\").to_string();\n if let Some(l) = program_name.get_mut(0..1) {\n l.make_ascii_uppercase();\n }\n let main_window = WindowDesc::new(ui_builder())\n .title(program_name)\n .with_min_size((720_f64, 360_f64));\n let state = AppState {\n target: \"\".to_string(),\n keep_fonts: false,\n keep_frames: true,\n keep_images: true,\n keep_scripts: true,\n keep_styles: true,\n output_path: if let Some(base_dirs) = UserDirs::new() {\n base_dirs.download_dir().unwrap().display().to_string()\n + &path::MAIN_SEPARATOR.to_string()\n + \"%title%.%ext%\"\n } else {\n \"%title%.%ext%\".to_string()\n },\n isolate: true,\n unwrap_noscript: false,\n busy: false,\n };\n\n AppLauncher::with_window(main_window)\n .delegate(Delegate)\n .launch(state)\n}\n\nfn ui_builder() -> impl Widget {\n let target_label: Label = Label::new(\"Target:\");\n let target_input = TextBox::new()\n .with_placeholder(\"URL or filesystem path\")\n .fix_width(TEXT_BOX_WIDTH)\n .lens(AppState::target)\n .disabled_if(|state: &AppState, _env| state.busy);\n let target_button = Button::new(LocalizedString::new(\"Open file\"))\n .on_click(|ctx, _, _| {\n ctx.submit_command(\n commands::SHOW_OPEN_PANEL.with(\n FileDialogOptions::new()\n .allowed_types(vec![FILESPEC_HTML])\n .default_type(FILESPEC_HTML),\n ),\n )\n })\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let output_path_label: Label = Label::new(\"Output path:\");\n let output_path_input = TextBox::new()\n .with_placeholder(\"Filesystem path\")\n .fix_width(TEXT_BOX_WIDTH)\n .lens(AppState::output_path)\n .disabled_if(|state: &AppState, _env| state.busy);\n let output_path_button = Button::new(LocalizedString::new(\"Browse\"))\n .on_click(|ctx, state: &mut AppState, _env| {\n ctx.submit_command(\n commands::SHOW_SAVE_PANEL.with(\n FileDialogOptions::new()\n // .force_starting_directory(\n // state\n // .output_path.clone()\n // .split(path::MAIN_SEPARATOR).collect::>()[..2]\n // .join(&path::MAIN_SEPARATOR.to_string())\n // )\n .default_name(\n state\n .output_path\n .clone()\n .split(path::MAIN_SEPARATOR)\n .last()\n .unwrap_or_default(),\n ),\n ),\n )\n })\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let fonts_checkbox = Checkbox::new(\"Include fonts\")\n .lens(AppState::keep_fonts)\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let frames_checkbox = Checkbox::new(\"Include frames\")\n .lens(AppState::keep_frames)\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let images_checkbox = Checkbox::new(\"Include images\")\n .lens(AppState::keep_images)\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let styles_checkbox = Checkbox::new(\"Include styles\")\n .lens(AppState::keep_styles)\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let scripts_checkbox = Checkbox::new(\"Include scripts\")\n .lens(AppState::keep_scripts)\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let isolate_checkbox = Checkbox::new(\"Isolate document\")\n .lens(AppState::isolate)\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let unwrap_noscript_checkbox = Checkbox::new(\"Unwrap NOSCRIPT\")\n .lens(AppState::unwrap_noscript)\n .disabled_if(|state: &AppState, _env| state.busy)\n .padding(5.0);\n let start_stop_button = Button::new(LocalizedString::new(\"Start\"))\n .on_click(|ctx, state: &mut AppState, _env| {\n if state.busy {\n return;\n }\n\n let mut options: MonolithOptions = MonolithOptions::default();\n options.ignore_errors = true;\n options.insecure = true;\n options.silent = true;\n options.no_frames = !state.keep_frames;\n options.no_fonts = !state.keep_fonts;\n options.no_images = !state.keep_images;\n options.no_css = !state.keep_styles;\n options.no_js = !state.keep_scripts;\n options.isolate = state.isolate;\n options.unwrap_noscript = state.unwrap_noscript;\n\n let handle = ctx.get_external_handle();\n let thread_state = state.clone();\n\n state.busy = true;\n\n // Set up cache (attempt to create temporary file)\n let temp_cache_file: Option =\n match Builder::new().prefix(\"monolith-\").tempfile() {\n Ok(tempfile) => Some(tempfile),\n Err(_) => None,\n };\n let cache = Some(Cache::new(\n CACHE_ASSET_FILE_SIZE_THRESHOLD,\n if temp_cache_file.is_some() {\n Some(\n temp_cache_file\n .as_ref()\n .unwrap()\n .path()\n .display()\n .to_string(),\n )\n } else {\n None\n },\n ));\n\n let session: Session = Session::new(cache, None, options);\n\n thread::spawn(\n move || match create_monolithic_document(session, thread_state.target) {\n Ok(result) => {\n handle\n .submit_command(MONOLITH_GUI_WRITE_OUTPUT, result, Target::Auto)\n .unwrap();\n\n // TODO: make it work again\n //cache.unwrap().destroy_database_file();\n }\n Err(error) => {\n handle\n .submit_command(MONOLITH_GUI_ERROR, error, Target::Auto)\n .unwrap();\n\n // TODO: make it work again\n //cache.unwrap().destroy_database_file();\n }\n },\n );\n })\n .disabled_if(|state: &AppState, _env| {\n state.busy || state.target.is_empty() || state.output_path.is_empty()\n })\n .padding(5.0);\n let spinner = Either::new(\n |sate: &AppState, _env| sate.busy,\n Spinner::new(),\n Label::new(\"\"),\n )\n .padding(5.0);\n\n Flex::column()\n .with_spacer(5_f64)\n .with_child(\n Flex::row()\n .with_child(target_label)\n .with_spacer(5_f64)\n .with_child(target_input)\n .with_child(target_button),\n )\n .with_child(fonts_checkbox)\n .with_child(frames_checkbox)\n .with_child(images_checkbox)\n .with_child(scripts_checkbox)\n .with_child(styles_checkbox)\n .with_child(\n Flex::row()\n .with_child(output_path_label)\n .with_spacer(5_f64)\n .with_child(output_path_input)\n .with_child(output_path_button),\n )\n .with_child(\n Flex::row()\n .with_child(isolate_checkbox)\n .with_child(unwrap_noscript_checkbox),\n )\n .with_child(start_stop_button)\n .with_child(spinner)\n .with_spacer(5_f64)\n}\n\nimpl AppDelegate for Delegate {\n fn command(\n &mut self,\n _ctx: &mut DelegateCtx,\n _target: Target,\n cmd: &Command,\n state: &mut AppState,\n _env: &Env,\n ) -> Handled {\n // Handle \"Open file\" button next to target input\n if let Some(file_info) = cmd.get(commands::OPEN_FILE) {\n state.target = file_info.path().display().to_string();\n\n return Handled::Yes;\n }\n // Handle \"Browse\" button next to output path input\n else if let Some(file_info) = cmd.get(commands::SAVE_FILE_AS) {\n state.output_path = file_info.path().display().to_string();\n\n return Handled::Yes;\n }\n // Write output\n else if let Some(result) = cmd.get(MONOLITH_GUI_WRITE_OUTPUT) {\n let (html, title) = result;\n\n if !state.output_path.is_empty() {\n match fs::File::create(format_output_path(\n &state.output_path,\n &title.clone().unwrap_or_default(),\n MonolithOutputFormat::HTML,\n )) {\n Ok(mut file) => {\n let _ = file.write(&html);\n }\n Err(_) => {\n eprintln!(\"Error: could not write output\");\n }\n }\n } else {\n eprintln!(\"Error: no output specified\");\n }\n\n state.busy = false;\n return Handled::Yes;\n }\n // Handle errors\n else if let Some(_error) = cmd.get(MONOLITH_GUI_ERROR) {\n state.busy = false;\n return Handled::Yes;\n }\n\n Handled::No\n }\n}\n" }, { "path": "src/html.rs", "content": "use base64::{prelude::BASE64_STANDARD, Engine};\nuse chrono::{SecondsFormat, Utc};\nuse encoding_rs::Encoding;\nuse html5ever::interface::{Attribute, QualName};\nuse html5ever::parse_document;\nuse html5ever::serialize::{serialize, SerializeOpts};\nuse html5ever::tendril::{format_tendril, TendrilSink};\nuse html5ever::tree_builder::{create_element, TreeSink};\nuse html5ever::{namespace_url, ns, LocalName};\nuse markup5ever_rcdom::{Handle, NodeData, RcDom, SerializableHandle};\nuse regex::Regex;\nuse sha2::{Digest, Sha256, Sha384, Sha512};\nuse std::default::Default;\n\nuse crate::core::{parse_content_type, MonolithOptions};\nuse crate::css::embed_css;\nuse crate::js::attr_is_event_handler;\nuse crate::session::Session;\nuse crate::url::{\n clean_url, create_data_url, is_url_and_has_protocol, resolve_url, Url, EMPTY_IMAGE_DATA_URL,\n};\n\nconst FAVICON_VALUES: &[&str] = &[\"icon\", \"shortcut icon\"];\nconst WHITESPACES: &[char] = &[' ', '\\t', '\\n', '\\x0c', '\\r']; // ASCII whitespaces\n\n#[derive(PartialEq, Eq)]\npub enum LinkType {\n Alternate,\n AppleTouchIcon,\n DnsPrefetch,\n Favicon,\n Preload,\n Stylesheet,\n}\n\npub struct SrcSetItem<'a> {\n pub path: &'a str,\n pub descriptor: &'a str, // Width or pixel density descriptor\n}\n\npub fn add_favicon(document: &Handle, favicon_data_url: String) -> RcDom {\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(document.clone()),\n SerializeOpts::default(),\n )\n .expect(\"unable to serialize DOM into buffer\");\n\n let dom = html_to_dom(&buf, \"utf-8\".to_string());\n for head in find_nodes(&dom.document, vec![\"html\", \"head\"]).iter() {\n let favicon_node = create_element(\n &dom,\n QualName::new(None, ns!(), LocalName::from(\"link\")),\n vec![\n Attribute {\n name: QualName::new(None, ns!(), LocalName::from(\"rel\")),\n value: format_tendril!(\"icon\"),\n },\n Attribute {\n name: QualName::new(None, ns!(), LocalName::from(\"href\")),\n value: format_tendril!(\"{}\", favicon_data_url),\n },\n ],\n );\n\n // Insert favicon LINK tag into HEAD\n head.children.borrow_mut().push(favicon_node.clone());\n }\n\n dom\n}\n\npub fn check_integrity(data: &[u8], integrity: &str) -> bool {\n if integrity.starts_with(\"sha256-\") {\n let mut hasher = Sha256::new();\n hasher.update(data);\n BASE64_STANDARD.encode(hasher.finalize()) == integrity[7..]\n } else if integrity.starts_with(\"sha384-\") {\n let mut hasher = Sha384::new();\n hasher.update(data);\n BASE64_STANDARD.encode(hasher.finalize()) == integrity[7..]\n } else if integrity.starts_with(\"sha512-\") {\n let mut hasher = Sha512::new();\n hasher.update(data);\n BASE64_STANDARD.encode(hasher.finalize()) == integrity[7..]\n } else {\n false\n }\n}\n\npub fn compose_csp(options: &MonolithOptions) -> String {\n let mut string_list = vec![];\n\n if options.isolate {\n string_list.push(\"default-src 'unsafe-eval' 'unsafe-inline' data:;\");\n }\n\n if options.no_css {\n string_list.push(\"style-src 'none';\");\n }\n\n if options.no_fonts {\n string_list.push(\"font-src 'none';\");\n }\n\n if options.no_frames {\n string_list.push(\"frame-src 'none';\");\n string_list.push(\"child-src 'none';\");\n }\n\n if options.no_js {\n string_list.push(\"script-src 'none';\");\n }\n\n if options.no_images {\n // Note: \"data:\" is required for transparent pixel images to work\n string_list.push(\"img-src data:;\");\n }\n\n string_list.join(\" \")\n}\n\npub fn create_metadata_tag(url: &Url) -> String {\n let datetime: &str = &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);\n let mut clean_url: Url = clean_url(url.clone());\n\n // Prevent credentials from getting into metadata\n if clean_url.scheme() == \"http\" || clean_url.scheme() == \"https\" {\n // Only HTTP(S) URLs can contain credentials\n clean_url.set_username(\"\").unwrap();\n clean_url.set_password(None).unwrap();\n }\n\n format!(\n \"\",\n if clean_url.scheme() == \"http\" || clean_url.scheme() == \"https\" {\n clean_url.as_str()\n } else {\n \"local source\"\n },\n datetime,\n env!(\"CARGO_PKG_NAME\"),\n env!(\"CARGO_PKG_VERSION\"),\n )\n}\n\npub fn embed_srcset(session: &mut Session, document_url: &Url, srcset: &str) -> String {\n let srcset_items: Vec = parse_srcset(srcset);\n\n // Embed assets\n let mut result: String = \"\".to_string();\n let mut i: usize = srcset_items.len();\n for srcset_item in srcset_items {\n if session.options.no_images {\n result.push_str(EMPTY_IMAGE_DATA_URL);\n } else {\n let image_full_url: Url = resolve_url(document_url, srcset_item.path);\n match session.retrieve_asset(document_url, &image_full_url) {\n Ok((image_data, image_final_url, image_media_type, image_charset)) => {\n let mut image_data_url = create_data_url(\n &image_media_type,\n &image_charset,\n &image_data,\n &image_final_url,\n );\n // Append retrieved asset as a data URL\n image_data_url.set_fragment(image_full_url.fragment());\n result.push_str(image_data_url.as_ref());\n }\n Err(_) => {\n // Keep remote reference if unable to retrieve the asset\n if image_full_url.scheme() == \"http\" || image_full_url.scheme() == \"https\" {\n result.push_str(image_full_url.as_ref());\n } else {\n // Avoid breaking the structure in case if not an HTTP(S) URL\n result.push_str(EMPTY_IMAGE_DATA_URL);\n }\n }\n }\n }\n\n if !srcset_item.descriptor.is_empty() {\n result.push(' ');\n result.push_str(srcset_item.descriptor);\n }\n\n if i > 1 {\n result.push_str(\", \");\n }\n\n i -= 1;\n }\n\n result\n}\n\npub fn find_nodes(node: &Handle, mut path: Vec<&str>) -> Vec {\n let mut result = vec![];\n\n while !path.is_empty() {\n match node.data {\n NodeData::Document | NodeData::Element { .. } => {\n // Dig deeper\n for child in node.children.borrow().iter() {\n if get_node_name(child)\n .unwrap_or_default()\n .eq_ignore_ascii_case(path[0])\n {\n if path.len() == 1 {\n result.push(child.clone());\n } else {\n result.append(&mut find_nodes(child, path[1..].to_vec()));\n }\n }\n }\n }\n _ => {}\n }\n\n path.remove(0);\n }\n\n result\n}\n\npub fn get_base_url(handle: &Handle) -> Option {\n for base_node in find_nodes(handle, vec![\"html\", \"head\", \"base\"]).iter() {\n // Only the first base tag matters (we ignore the rest, if there's any)\n return get_node_attr(base_node, \"href\");\n }\n\n None\n}\n\npub fn get_charset(node: &Handle) -> Option {\n for meta_node in find_nodes(node, vec![\"html\", \"head\", \"meta\"]).iter() {\n if let Some(meta_charset_node_attr_value) = get_node_attr(meta_node, \"charset\") {\n // Processing \n return Some(meta_charset_node_attr_value);\n }\n\n if get_node_attr(meta_node, \"http-equiv\")\n .unwrap_or_default()\n .eq_ignore_ascii_case(\"content-type\")\n {\n if let Some(meta_content_type_node_attr_value) = get_node_attr(meta_node, \"content\") {\n // Processing \n let (_media_type, charset, _is_base64) =\n parse_content_type(&meta_content_type_node_attr_value);\n return Some(charset);\n }\n }\n }\n\n None\n}\n\n// TODO: get rid of this function (replace with find_nodes)\npub fn get_child_node_by_name(parent: &Handle, node_name: &str) -> Option {\n let children = parent.children.borrow();\n let matching_children = children.iter().find(|child| match child.data {\n NodeData::Element { ref name, .. } => &*name.local == node_name,\n _ => false,\n });\n matching_children.cloned()\n}\n\npub fn get_node_attr(node: &Handle, attr_name: &str) -> Option {\n match &node.data {\n NodeData::Element { attrs, .. } => {\n for attr in attrs.borrow().iter() {\n if &*attr.name.local == attr_name {\n return Some(attr.value.to_string());\n }\n }\n None\n }\n _ => None,\n }\n}\n\npub fn get_node_name(node: &Handle) -> Option<&'_ str> {\n match &node.data {\n NodeData::Element { name, .. } => Some(name.local.as_ref()),\n _ => None,\n }\n}\n\npub fn get_parent_node(child: &Handle) -> Handle {\n let parent = child.parent.take().clone();\n parent.and_then(|node| node.upgrade()).unwrap()\n}\n\npub fn get_robots(handle: &Handle) -> Option {\n for meta_node in find_nodes(handle, vec![\"html\", \"head\", \"meta\"]).iter() {\n // Only the first base tag matters (we ignore the rest, if there's any)\n if get_node_attr(meta_node, \"name\")\n .unwrap_or_default()\n .eq_ignore_ascii_case(\"robots\")\n {\n return get_node_attr(meta_node, \"content\");\n }\n }\n\n None\n}\n\npub fn get_title(node: &Handle) -> Option {\n for title_node in find_nodes(node, vec![\"html\", \"head\", \"title\"]).iter() {\n for child_node in title_node.children.borrow().iter() {\n if let NodeData::Text { ref contents } = child_node.data {\n return Some(contents.borrow().to_string());\n }\n }\n }\n\n None\n}\n\npub fn has_favicon(handle: &Handle) -> bool {\n let mut found_favicon: bool = false;\n\n for link_node in find_nodes(handle, vec![\"html\", \"head\", \"link\"]).iter() {\n if let Some(attr_value) = get_node_attr(link_node, \"rel\") {\n if is_favicon(attr_value.trim()) {\n found_favicon = true;\n break;\n }\n }\n }\n\n found_favicon\n}\n\npub fn html_to_dom(data: &Vec, document_encoding: String) -> RcDom {\n let s: String;\n\n if let Some(encoding) = Encoding::for_label(document_encoding.as_bytes()) {\n let (string, _, _) = encoding.decode(data);\n s = string.to_string();\n } else {\n s = String::from_utf8_lossy(data).to_string();\n }\n\n parse_document(RcDom::default(), Default::default())\n .from_utf8()\n .read_from(&mut s.as_bytes())\n .unwrap()\n}\n\npub fn is_favicon(attr_value: &str) -> bool {\n FAVICON_VALUES.contains(&attr_value.to_lowercase().as_str())\n}\n\npub fn parse_link_type(link_attr_rel_value: &str) -> Vec {\n let mut types: Vec = vec![];\n\n for link_attr_rel_type in link_attr_rel_value.split_whitespace() {\n if link_attr_rel_type.eq_ignore_ascii_case(\"alternate\") {\n types.push(LinkType::Alternate);\n } else if link_attr_rel_type.eq_ignore_ascii_case(\"dns-prefetch\") {\n types.push(LinkType::DnsPrefetch);\n } else if link_attr_rel_type.eq_ignore_ascii_case(\"preload\") {\n types.push(LinkType::Preload);\n } else if link_attr_rel_type.eq_ignore_ascii_case(\"stylesheet\") {\n types.push(LinkType::Stylesheet);\n } else if is_favicon(link_attr_rel_type) {\n types.push(LinkType::Favicon);\n } else if link_attr_rel_type.eq_ignore_ascii_case(\"apple-touch-icon\") {\n types.push(LinkType::AppleTouchIcon);\n }\n }\n\n types\n}\n\npub fn parse_srcset(srcset: &str) -> Vec {\n let mut srcset_items: Vec = vec![];\n\n // Parse srcset\n let mut partials: Vec<&str> = srcset.split(WHITESPACES).collect();\n let mut path: Option<&str> = None;\n let mut descriptor: Option<&str> = None;\n let mut i = 0;\n while i < partials.len() {\n let partial = partials[i];\n\n i += 1;\n\n // Skip empty strings\n if partial.is_empty() {\n continue;\n }\n\n if partial.ends_with(',') {\n if path.is_none() {\n path = Some(partial.strip_suffix(',').unwrap());\n descriptor = Some(\"\")\n } else {\n descriptor = Some(partial.strip_suffix(',').unwrap());\n }\n } else if path.is_none() {\n path = Some(partial);\n } else {\n let mut chunks: Vec<&str> = partial.split(',').collect();\n\n if !chunks.is_empty() && chunks.first().unwrap().ends_with(['x', 'w']) {\n descriptor = Some(chunks.first().unwrap());\n\n chunks.remove(0);\n }\n\n if !chunks.is_empty() {\n if descriptor.is_some() {\n partials.insert(0, &partial[descriptor.unwrap().len()..]);\n } else {\n partials.insert(0, partial);\n }\n }\n }\n\n if path.is_some() && descriptor.is_some() {\n srcset_items.push(SrcSetItem {\n path: path.unwrap(),\n descriptor: descriptor.unwrap(),\n });\n\n path = None;\n descriptor = None;\n }\n }\n\n // Final attempt to process what was found\n if path.is_some() {\n srcset_items.push(SrcSetItem {\n path: path.unwrap(),\n descriptor: descriptor.unwrap_or_default(),\n });\n }\n\n srcset_items\n}\n\npub fn set_base_url(document: &Handle, base_href_value: String) -> RcDom {\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(document.clone()),\n SerializeOpts::default(),\n )\n .expect(\"unable to serialize DOM into buffer\");\n let dom = html_to_dom(&buf, \"utf-8\".to_string());\n\n if let Some(html_node) = get_child_node_by_name(&dom.document, \"html\") {\n if let Some(head_node) = get_child_node_by_name(&html_node, \"head\") {\n // Check if BASE node already exists in the DOM tree\n if let Some(base_node) = get_child_node_by_name(&head_node, \"base\") {\n set_node_attr(&base_node, \"href\", Some(base_href_value));\n } else {\n let base_node = create_element(\n &dom,\n QualName::new(None, ns!(), LocalName::from(\"base\")),\n vec![Attribute {\n name: QualName::new(None, ns!(), LocalName::from(\"href\")),\n value: format_tendril!(\"{}\", base_href_value),\n }],\n );\n\n // Insert newly created BASE node into HEAD\n head_node.children.borrow_mut().push(base_node.clone());\n }\n }\n }\n\n dom\n}\n\npub fn set_charset(dom: RcDom, charset: String) -> RcDom {\n for meta_node in find_nodes(&dom.document, vec![\"html\", \"head\", \"meta\"]).iter() {\n if get_node_attr(meta_node, \"charset\").is_some() {\n set_node_attr(meta_node, \"charset\", Some(charset));\n return dom;\n }\n\n if get_node_attr(meta_node, \"http-equiv\")\n .unwrap_or_default()\n .eq_ignore_ascii_case(\"content-type\")\n && get_node_attr(meta_node, \"content\").is_some()\n {\n set_node_attr(\n meta_node,\n \"content\",\n Some(format!(\"text/html;charset={}\", charset)),\n );\n return dom;\n }\n }\n\n // Manually append charset META node to HEAD\n {\n let meta_charset_node: Handle = create_element(\n &dom,\n QualName::new(None, ns!(), LocalName::from(\"meta\")),\n vec![Attribute {\n name: QualName::new(None, ns!(), LocalName::from(\"charset\")),\n value: format_tendril!(\"{}\", charset),\n }],\n );\n\n // Insert newly created META charset node into HEAD\n for head_node in find_nodes(&dom.document, vec![\"html\", \"head\"]).iter() {\n head_node\n .children\n .borrow_mut()\n .push(meta_charset_node.clone());\n break;\n }\n }\n\n dom\n}\n\npub fn set_node_attr(node: &Handle, attr_name: &str, attr_value: Option) {\n if let NodeData::Element { attrs, .. } = &node.data {\n let attrs_mut = &mut attrs.borrow_mut();\n let mut i = 0;\n let mut found_existing_attr: bool = false;\n\n while i < attrs_mut.len() {\n if &attrs_mut[i].name.local == attr_name {\n found_existing_attr = true;\n\n if let Some(attr_value) = attr_value.clone() {\n let _ = &attrs_mut[i].value.clear();\n let _ = &attrs_mut[i].value.push_slice(attr_value.as_str());\n } else {\n // Remove attr completely if attr_value is not defined\n attrs_mut.remove(i);\n continue;\n }\n }\n\n i += 1;\n }\n\n if !found_existing_attr {\n // Add new attribute (since originally the target node didn't have it)\n if let Some(attr_value) = attr_value.clone() {\n let name = LocalName::from(attr_name);\n\n attrs_mut.push(Attribute {\n name: QualName::new(None, ns!(), name),\n value: format_tendril!(\"{}\", attr_value),\n });\n }\n }\n };\n}\n\npub fn set_robots(dom: RcDom, content_value: &str) -> RcDom {\n for meta_node in find_nodes(&dom.document, vec![\"html\", \"head\", \"meta\"]).iter() {\n if get_node_attr(meta_node, \"name\")\n .unwrap_or_default()\n .eq_ignore_ascii_case(\"robots\")\n {\n set_node_attr(meta_node, \"content\", Some(content_value.to_string()));\n return dom;\n }\n }\n\n // Manually append robots META node to HEAD\n {\n let meta_charset_node: Handle = create_element(\n &dom,\n QualName::new(None, ns!(), LocalName::from(\"meta\")),\n vec![\n Attribute {\n name: QualName::new(None, ns!(), LocalName::from(\"name\")),\n value: format_tendril!(\"robots\"),\n },\n Attribute {\n name: QualName::new(None, ns!(), LocalName::from(\"content\")),\n value: format_tendril!(\"{}\", content_value),\n },\n ],\n );\n\n // Insert newly created META charset node into HEAD\n for head_node in find_nodes(&dom.document, vec![\"html\", \"head\"]).iter() {\n head_node\n .children\n .borrow_mut()\n .push(meta_charset_node.clone());\n break;\n }\n }\n\n dom\n}\n\npub fn serialize_document(\n dom: RcDom,\n document_encoding: String,\n options: &MonolithOptions,\n) -> Vec {\n let mut buf: Vec = Vec::new();\n\n if options.isolate\n || options.no_css\n || options.no_fonts\n || options.no_frames\n || options.no_js\n || options.no_images\n {\n // Take care of CSP\n if let Some(html) = get_child_node_by_name(&dom.document, \"html\") {\n if let Some(head) = get_child_node_by_name(&html, \"head\") {\n let meta = create_element(\n &dom,\n QualName::new(None, ns!(), LocalName::from(\"meta\")),\n vec![\n Attribute {\n name: QualName::new(None, ns!(), LocalName::from(\"http-equiv\")),\n value: format_tendril!(\"Content-Security-Policy\"),\n },\n Attribute {\n name: QualName::new(None, ns!(), LocalName::from(\"content\")),\n value: format_tendril!(\"{}\", compose_csp(options)),\n },\n ],\n );\n // The CSP meta-tag has to be prepended, never appended,\n // since there already may be one defined in the original document,\n // and browsers don't allow re-defining them (for obvious reasons)\n head.children.borrow_mut().reverse();\n head.children.borrow_mut().push(meta.clone());\n head.children.borrow_mut().reverse();\n }\n }\n }\n\n let serializable: SerializableHandle = dom.document.into();\n serialize(&mut buf, &serializable, SerializeOpts::default())\n .expect(\"Unable to serialize DOM into buffer\");\n\n // Unwrap NOSCRIPT elements\n if options.unwrap_noscript {\n let s: &str = &String::from_utf8_lossy(&buf);\n let noscript_re = Regex::new(r\"<(?P/?noscript[^>]*)>\").unwrap();\n buf = noscript_re.replace_all(s, \"\").as_bytes().to_vec();\n }\n\n if !document_encoding.is_empty() {\n if let Some(encoding) = Encoding::for_label(document_encoding.as_bytes()) {\n let s: &str = &String::from_utf8_lossy(&buf);\n let (data, _, _) = encoding.encode(s);\n buf = data.to_vec();\n }\n }\n\n buf\n}\n\npub fn retrieve_and_embed_asset(\n session: &mut Session,\n document_url: &Url,\n node: &Handle,\n attr_name: &str,\n attr_value: &str,\n) {\n let resolved_url: Url = resolve_url(document_url, attr_value);\n\n match session.retrieve_asset(&document_url.clone(), &resolved_url) {\n Ok((data, final_url, media_type, charset)) => {\n let node_name: &str = get_node_name(node).unwrap();\n\n // Check integrity if it's a LINK or SCRIPT element\n let mut ok_to_include: bool = true;\n if node_name == \"link\" || node_name == \"script\" {\n // Check integrity\n if let Some(node_integrity_attr_value) = get_node_attr(node, \"integrity\") {\n if !node_integrity_attr_value.is_empty() {\n ok_to_include = check_integrity(&data, &node_integrity_attr_value);\n }\n\n // Wipe the integrity attribute\n set_node_attr(node, \"integrity\", None);\n }\n }\n\n if ok_to_include {\n if node_name == \"link\"\n && parse_link_type(&get_node_attr(node, \"rel\").unwrap_or(String::from(\"\")))\n .contains(&LinkType::Stylesheet)\n {\n let stylesheet: String;\n if let Some(encoding) = Encoding::for_label(charset.as_bytes()) {\n let (string, _, _) = encoding.decode(&data);\n stylesheet = string.to_string();\n } else {\n stylesheet = String::from_utf8_lossy(&data).to_string();\n }\n\n // Stylesheet LINK elements require special treatment\n let css: String = embed_css(session, &final_url, &stylesheet);\n\n // Create and embed data URL\n let css_data_url =\n create_data_url(&media_type, &charset, css.as_bytes(), &final_url);\n set_node_attr(node, attr_name, Some(css_data_url.to_string()));\n } else if node_name == \"frame\" || node_name == \"iframe\" {\n // (I)FRAMEs are also quite different from conventional resources\n let frame_dom = html_to_dom(&data, charset.clone());\n walk(session, &final_url, &frame_dom.document);\n\n let mut frame_data: Vec = Vec::new();\n let serializable: SerializableHandle = frame_dom.document.into();\n serialize(&mut frame_data, &serializable, SerializeOpts::default()).unwrap();\n\n // Create and embed data URL\n let mut frame_data_url =\n create_data_url(&media_type, &charset, &frame_data, &final_url);\n frame_data_url.set_fragment(resolved_url.fragment());\n set_node_attr(node, attr_name, Some(frame_data_url.to_string()));\n } else {\n // Every other type of element gets processed here\n\n // Parse media type for SCRIPT elements\n if node_name == \"script\" {\n let script_media_type =\n get_node_attr(node, \"type\").unwrap_or(String::from(\"text/javascript\"));\n\n if script_media_type == \"text/javascript\"\n || script_media_type == \"application/javascript\"\n {\n // Embed javascript code instead of using data URLs\n let script_dom: RcDom =\n parse_document(RcDom::default(), Default::default())\n .one(\"\");\n for script_node in\n find_nodes(&script_dom.document, vec![\"html\", \"head\", \"script\"])\n .iter()\n {\n let text_node = &script_node.children.borrow()[0];\n\n if let NodeData::Text { ref contents } = text_node.data {\n let mut tendril = contents.borrow_mut();\n tendril.clear();\n tendril.push_slice(\n &String::from_utf8_lossy(&data)\n .replace(\"\", \"<\\\\/script>\"),\n );\n }\n\n node.children.borrow_mut().push(text_node.clone());\n set_node_attr(node, attr_name, None);\n }\n } else {\n // Create and embed data URL\n let mut data_url =\n create_data_url(&script_media_type, &charset, &data, &final_url);\n data_url.set_fragment(resolved_url.fragment());\n set_node_attr(node, attr_name, Some(data_url.to_string()));\n }\n } else {\n // Create and embed data URL\n let mut data_url =\n create_data_url(&media_type, &charset, &data, &final_url);\n data_url.set_fragment(resolved_url.fragment());\n set_node_attr(node, attr_name, Some(data_url.to_string()));\n }\n }\n }\n }\n Err(_) => {\n if resolved_url.scheme() == \"http\" || resolved_url.scheme() == \"https\" {\n // Keep remote references if unable to retrieve the asset\n set_node_attr(node, attr_name, Some(resolved_url.to_string()));\n } else {\n // Remove local references if they can't be successfully embedded as data URLs\n set_node_attr(node, attr_name, None);\n }\n }\n }\n}\n\npub fn walk(session: &mut Session, document_url: &Url, node: &Handle) {\n match node.data {\n NodeData::Document => {\n // Dig deeper\n for child_node in node.children.borrow().iter() {\n walk(session, document_url, child_node);\n }\n }\n NodeData::Element {\n ref name,\n ref attrs,\n ..\n } => {\n match name.local.as_ref() {\n \"meta\" => {\n if let Some(meta_attr_http_equiv_value) = get_node_attr(node, \"http-equiv\") {\n let meta_attr_http_equiv_value: &str = &meta_attr_http_equiv_value;\n if meta_attr_http_equiv_value.eq_ignore_ascii_case(\"refresh\")\n || meta_attr_http_equiv_value.eq_ignore_ascii_case(\"location\")\n {\n // Remove http-equiv attributes from META nodes if they're able to control the page\n set_node_attr(node, \"http-equiv\", None);\n }\n }\n }\n \"link\" => {\n let link_node_types: Vec =\n parse_link_type(&get_node_attr(node, \"rel\").unwrap_or(String::from(\"\")));\n\n if link_node_types.contains(&LinkType::Favicon)\n || link_node_types.contains(&LinkType::AppleTouchIcon)\n {\n // Find and resolve LINK's href attribute\n if let Some(link_attr_href_value) = get_node_attr(node, \"href\") {\n if !session.options.no_images && !link_attr_href_value.is_empty() {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"href\",\n &link_attr_href_value,\n );\n } else {\n set_node_attr(node, \"href\", None);\n }\n }\n } else if link_node_types.contains(&LinkType::Stylesheet) {\n // Resolve LINK's href attribute\n if let Some(link_attr_href_value) = get_node_attr(node, \"href\") {\n if session.options.no_css {\n set_node_attr(node, \"href\", None);\n // Wipe integrity attribute\n set_node_attr(node, \"integrity\", None);\n } else if !link_attr_href_value.is_empty() {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"href\",\n &link_attr_href_value,\n );\n }\n }\n } else if link_node_types.contains(&LinkType::Preload)\n || link_node_types.contains(&LinkType::DnsPrefetch)\n {\n // Since all resources are embedded as data URLs, preloading and prefetching are not necessary\n set_node_attr(node, \"rel\", None);\n } else {\n // Make sure that all other LINKs' href attributes are full URLs\n if let Some(link_attr_href_value) = get_node_attr(node, \"href\") {\n let href_full_url: Url =\n resolve_url(document_url, &link_attr_href_value);\n set_node_attr(node, \"href\", Some(href_full_url.to_string()));\n }\n }\n }\n \"base\" => {\n if document_url.scheme() == \"http\" || document_url.scheme() == \"https\" {\n // Ensure the BASE node doesn't have a relative URL\n if let Some(base_attr_href_value) = get_node_attr(node, \"href\") {\n let href_full_url: Url =\n resolve_url(document_url, &base_attr_href_value);\n set_node_attr(node, \"href\", Some(href_full_url.to_string()));\n }\n }\n }\n \"body\" => {\n // Read and remember background attribute value of this BODY node\n if let Some(body_attr_background_value) = get_node_attr(node, \"background\") {\n // Remove background BODY node attribute by default\n set_node_attr(node, \"background\", None);\n\n if !session.options.no_images && !body_attr_background_value.is_empty() {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"background\",\n &body_attr_background_value,\n );\n }\n }\n }\n \"img\" => {\n // Find src and data-src attribute(s)\n let img_attr_src_value: Option = get_node_attr(node, \"src\");\n let img_attr_data_src_value: Option = get_node_attr(node, \"data-src\");\n\n if session.options.no_images {\n // Put empty images into src and data-src attributes\n if img_attr_src_value.is_some() {\n set_node_attr(node, \"src\", Some(EMPTY_IMAGE_DATA_URL.to_string()));\n }\n if img_attr_data_src_value.is_some() {\n set_node_attr(node, \"data-src\", Some(EMPTY_IMAGE_DATA_URL.to_string()));\n }\n } else if img_attr_src_value.clone().unwrap_or_default().is_empty()\n && img_attr_data_src_value\n .clone()\n .unwrap_or_default()\n .is_empty()\n {\n // Add empty src attribute\n set_node_attr(node, \"src\", Some(\"\".to_string()));\n } else {\n // Add data URL src attribute\n let img_full_url: String = if !img_attr_data_src_value\n .clone()\n .unwrap_or_default()\n .is_empty()\n {\n img_attr_data_src_value.unwrap_or_default()\n } else {\n img_attr_src_value.unwrap_or_default()\n };\n retrieve_and_embed_asset(session, document_url, node, \"src\", &img_full_url);\n }\n\n // Resolve srcset attribute\n if let Some(img_srcset) = get_node_attr(node, \"srcset\") {\n if !img_srcset.is_empty() {\n let resolved_srcset: String =\n embed_srcset(session, document_url, &img_srcset);\n set_node_attr(node, \"srcset\", Some(resolved_srcset));\n }\n }\n }\n \"input\" => {\n if let Some(input_attr_type_value) = get_node_attr(node, \"type\") {\n if input_attr_type_value.eq_ignore_ascii_case(\"image\") {\n if let Some(input_attr_src_value) = get_node_attr(node, \"src\") {\n if session.options.no_images || input_attr_src_value.is_empty() {\n let value = if input_attr_src_value.is_empty() {\n \"\"\n } else {\n EMPTY_IMAGE_DATA_URL\n };\n set_node_attr(node, \"src\", Some(value.to_string()));\n } else {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"src\",\n &input_attr_src_value,\n );\n }\n }\n }\n }\n }\n \"svg\" => {\n if session.options.no_images {\n // Remove all children\n node.children.borrow_mut().clear();\n }\n }\n \"image\" => {\n let attr_names: [&str; 2] = [\"href\", \"xlink:href\"];\n\n for attr_name in attr_names.into_iter() {\n if let Some(image_attr_href_value) = get_node_attr(node, attr_name) {\n if session.options.no_images {\n set_node_attr(node, attr_name, None);\n } else {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n attr_name,\n &image_attr_href_value,\n );\n }\n }\n }\n }\n \"use\" => {\n let attr_names: [&str; 2] = [\"href\", \"xlink:href\"];\n\n for attr_name in attr_names.into_iter() {\n if let Some(use_attr_href_value) = get_node_attr(node, attr_name) {\n if session.options.no_images {\n set_node_attr(node, attr_name, None);\n } else {\n let image_asset_url: Url =\n resolve_url(document_url, &use_attr_href_value);\n\n match session.retrieve_asset(document_url, &image_asset_url) {\n Ok((data, final_url, media_type, charset)) => {\n if media_type == \"image/svg+xml\" {\n // Parse SVG\n let svg_dom: RcDom = parse_document(\n RcDom::default(),\n Default::default(),\n )\n .from_utf8()\n .read_from(&mut data.as_slice())\n .unwrap();\n\n if image_asset_url.fragment().is_some() {\n // Take only that one #fragment symbol from SVG and replace this image|use with that node\n let single_symbol_node = create_element(\n &svg_dom,\n QualName::new(\n None,\n ns!(),\n LocalName::from(\"symbol\"),\n ),\n vec![],\n );\n for symbol_node in find_nodes(\n &svg_dom.document,\n vec![\"html\", \"body\", \"svg\", \"defs\", \"symbol\"],\n )\n .iter()\n {\n if get_node_attr(symbol_node, \"id\")\n .unwrap_or_default()\n == image_asset_url.fragment().unwrap()\n {\n svg_dom.reparent_children(\n symbol_node,\n &single_symbol_node,\n );\n set_node_attr(\n &single_symbol_node,\n \"id\",\n Some(\n image_asset_url\n .fragment()\n .unwrap()\n .to_string(),\n ),\n );\n\n set_node_attr(\n node,\n attr_name,\n Some(format!(\n \"#{}\",\n image_asset_url.fragment().unwrap()\n )),\n );\n\n break;\n }\n }\n\n node.children\n .borrow_mut()\n .push(single_symbol_node.clone());\n } else {\n // Replace this image|use with whole DOM of that SVG file\n for svg_node in find_nodes(\n &svg_dom.document,\n vec![\"html\", \"body\", \"svg\"],\n )\n .iter()\n {\n svg_dom.reparent_children(svg_node, node);\n break;\n }\n // TODO: decide if we resort to using data URL here or stick with embedding the DOM\n }\n } else {\n // It's likely a raster image; embed it as data URL\n let image_asset_data: Url = create_data_url(\n &media_type,\n &charset,\n &data,\n &final_url,\n );\n set_node_attr(\n node,\n attr_name,\n Some(image_asset_data.to_string()),\n );\n }\n }\n Err(_) => {\n set_node_attr(\n node,\n attr_name,\n Some(image_asset_url.to_string()),\n );\n }\n }\n }\n }\n }\n }\n \"source\" => {\n let parent_node = get_parent_node(node);\n let parent_node_name: &str = get_node_name(&parent_node).unwrap_or_default();\n\n if let Some(source_attr_src_value) = get_node_attr(node, \"src\") {\n if parent_node_name == \"audio\" {\n if session.options.no_audio {\n set_node_attr(node, \"src\", None);\n } else {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"src\",\n &source_attr_src_value,\n );\n }\n } else if parent_node_name == \"video\" {\n if session.options.no_video {\n set_node_attr(node, \"src\", None);\n } else {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"src\",\n &source_attr_src_value,\n );\n }\n }\n }\n\n if let Some(source_attr_srcset_value) = get_node_attr(node, \"srcset\") {\n if parent_node_name == \"picture\" && !source_attr_srcset_value.is_empty() {\n if session.options.no_images {\n set_node_attr(\n node,\n \"srcset\",\n Some(EMPTY_IMAGE_DATA_URL.to_string()),\n );\n } else {\n let resolved_srcset: String =\n embed_srcset(session, document_url, &source_attr_srcset_value);\n set_node_attr(node, \"srcset\", Some(resolved_srcset));\n }\n }\n }\n }\n \"a\" | \"area\" => {\n if let Some(anchor_attr_href_value) = get_node_attr(node, \"href\") {\n if anchor_attr_href_value\n .clone()\n .trim()\n .starts_with(\"javascript:\")\n {\n if session.options.no_js {\n // Replace with empty JS call to preserve original behavior\n set_node_attr(node, \"href\", Some(\"javascript:;\".to_string()));\n }\n } else {\n // Don't touch mailto: links or hrefs which begin with a hash sign\n if !anchor_attr_href_value.clone().starts_with('#')\n && !is_url_and_has_protocol(&anchor_attr_href_value.clone())\n {\n let href_full_url: Url =\n resolve_url(document_url, &anchor_attr_href_value);\n set_node_attr(node, \"href\", Some(href_full_url.to_string()));\n }\n }\n }\n }\n \"script\" => {\n // Read values of integrity and src attributes\n let script_attr_src: &str = &get_node_attr(node, \"src\").unwrap_or_default();\n\n if session.options.no_js {\n // Empty inner content\n node.children.borrow_mut().clear();\n // Remove src attribute\n if !script_attr_src.is_empty() {\n set_node_attr(node, \"src\", None);\n // Wipe integrity attribute\n set_node_attr(node, \"integrity\", None);\n }\n } else if !script_attr_src.is_empty() {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"src\",\n script_attr_src,\n );\n }\n }\n \"style\" => {\n if session.options.no_css {\n // Empty inner content of STYLE tags\n node.children.borrow_mut().clear();\n } else {\n for child_node in node.children.borrow_mut().iter_mut() {\n if let NodeData::Text { ref contents } = child_node.data {\n let mut tendril = contents.borrow_mut();\n let replacement =\n embed_css(session, document_url, tendril.as_ref());\n tendril.clear();\n tendril.push_slice(&replacement);\n }\n }\n }\n }\n \"form\" => {\n if let Some(form_attr_action_value) = get_node_attr(node, \"action\") {\n // Modify action property to ensure it's a full URL\n let form_action_full_url: Url =\n resolve_url(document_url, &form_attr_action_value);\n set_node_attr(node, \"action\", Some(form_action_full_url.to_string()));\n }\n }\n \"frame\" | \"iframe\" => {\n if let Some(frame_attr_src_value) = get_node_attr(node, \"src\") {\n if session.options.no_frames {\n // Empty the src attribute\n set_node_attr(node, \"src\", Some(\"\".to_string()));\n } else {\n // Ignore (i)frames with empty source (they cause infinite loops)\n if !frame_attr_src_value.trim().is_empty() {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"src\",\n &frame_attr_src_value,\n );\n }\n }\n }\n }\n \"audio\" => {\n // Embed audio source\n if let Some(audio_attr_src_value) = get_node_attr(node, \"src\") {\n if session.options.no_audio {\n set_node_attr(node, \"src\", None);\n } else {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"src\",\n &audio_attr_src_value,\n );\n }\n }\n }\n \"video\" => {\n // Embed video source\n if let Some(video_attr_src_value) = get_node_attr(node, \"src\") {\n if session.options.no_video {\n set_node_attr(node, \"src\", None);\n } else {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"src\",\n &video_attr_src_value,\n );\n }\n }\n\n // Embed poster images\n if let Some(video_attr_poster_value) = get_node_attr(node, \"poster\") {\n // Skip posters with empty source\n if !video_attr_poster_value.is_empty() {\n if session.options.no_images {\n set_node_attr(\n node,\n \"poster\",\n Some(EMPTY_IMAGE_DATA_URL.to_string()),\n );\n } else {\n retrieve_and_embed_asset(\n session,\n document_url,\n node,\n \"poster\",\n &video_attr_poster_value,\n );\n }\n }\n }\n }\n \"noscript\" => {\n for child_node in node.children.borrow_mut().iter_mut() {\n if let NodeData::Text { ref contents } = child_node.data {\n // Get contents of NOSCRIPT node\n let mut noscript_contents = contents.borrow_mut();\n // Parse contents of NOSCRIPT node as DOM\n let noscript_contents_dom: RcDom =\n html_to_dom(&noscript_contents.as_bytes().to_vec(), \"\".to_string());\n // Embed assets of NOSCRIPT node contents\n walk(session, document_url, &noscript_contents_dom.document);\n // Get rid of original contents\n noscript_contents.clear();\n // Insert HTML containing embedded assets into NOSCRIPT node\n if let Some(html) =\n get_child_node_by_name(&noscript_contents_dom.document, \"html\")\n {\n if let Some(body) = get_child_node_by_name(&html, \"body\") {\n let mut buf: Vec = Vec::new();\n let serializable: SerializableHandle = body.into();\n serialize(&mut buf, &serializable, SerializeOpts::default())\n .expect(\"Unable to serialize DOM into buffer\");\n let result = String::from_utf8_lossy(&buf);\n noscript_contents.push_slice(&result);\n }\n }\n }\n }\n }\n _ => {}\n }\n\n // Process style attributes\n if session.options.no_css {\n // Get rid of style attributes\n set_node_attr(node, \"style\", None);\n } else {\n // Embed URLs found within the style attribute of this node\n if let Some(node_attr_style_value) = get_node_attr(node, \"style\") {\n let embedded_style = embed_css(session, document_url, &node_attr_style_value);\n set_node_attr(node, \"style\", Some(embedded_style));\n }\n }\n\n // Strip all JS from document\n if session.options.no_js {\n let attrs_mut = &mut attrs.borrow_mut();\n // Get rid of JS event attributes\n let mut js_attr_indexes = Vec::new();\n for (i, attr) in attrs_mut.iter().enumerate() {\n if attr_is_event_handler(&attr.name.local) {\n js_attr_indexes.push(i);\n }\n }\n js_attr_indexes.reverse();\n for attr_index in js_attr_indexes {\n attrs_mut.remove(attr_index);\n }\n }\n\n // Dig deeper\n for child_node in node.children.borrow().iter() {\n walk(session, document_url, child_node);\n }\n }\n _ => {\n // Note: in case of options.no_js being set to true, there's no need to worry about\n // getting rid of comments that may contain scripts, e.g. \\n\\n\"\n );\n\n // Exit code should be 0\n out.assert().code(0);\n }\n\n #[test]\n fn unwrap_noscript_contents_nested() {\n let mut cmd = Command::cargo_bin(env!(\"CARGO_PKG_NAME\")).unwrap();\n let path_html: &Path = Path::new(\"tests/_data_/noscript/nested.html\");\n let path_svg: &Path = Path::new(\"tests/_data_/noscript/image.svg\");\n\n let out = cmd.arg(\"-Mn\").arg(path_html.as_os_str()).output().unwrap();\n\n // STDERR should contain target HTML and embedded SVG files\n assert_eq!(\n String::from_utf8_lossy(&out.stderr),\n format!(\n \"\\\n {file_url_html}\\n\\\n {file_url_svg}\\n\\\n \",\n file_url_html = Url::from_file_path(fs::canonicalize(path_html).unwrap()).unwrap(),\n file_url_svg = Url::from_file_path(fs::canonicalize(path_svg).unwrap()).unwrap(),\n )\n );\n\n // STDOUT should contain HTML with no CSS\n assert_eq!(\n String::from_utf8_lossy(&out.stdout),\n \"

JS is not active

\\n\\n\"\n );\n\n // Exit code should be 0\n out.assert().code(0);\n }\n\n #[test]\n fn unwrap_noscript_contents_with_script() {\n let mut cmd = Command::cargo_bin(env!(\"CARGO_PKG_NAME\")).unwrap();\n let path_html: &Path = Path::new(\"tests/_data_/noscript/script.html\");\n let path_svg: &Path = Path::new(\"tests/_data_/noscript/image.svg\");\n\n let out = cmd.arg(\"-Mn\").arg(path_html.as_os_str()).output().unwrap();\n\n // STDERR should contain target HTML and embedded SVG files\n assert_eq!(\n String::from_utf8_lossy(&out.stderr),\n format!(\n \"\\\n {file_url_html}\\n\\\n {file_url_svg}\\n\\\n \",\n file_url_html = Url::from_file_path(fs::canonicalize(path_html).unwrap()).unwrap(),\n file_url_svg = Url::from_file_path(fs::canonicalize(path_svg).unwrap()).unwrap(),\n )\n );\n\n // STDOUT should contain HTML with no CSS\n assert_eq!(\n String::from_utf8_lossy(&out.stdout),\n r#\"\n\n\"#\n );\n\n // Exit code should be 0\n out.assert().code(0);\n }\n\n #[test]\n fn unwrap_noscript_contents_attr_data_url() {\n let mut cmd = Command::cargo_bin(env!(\"CARGO_PKG_NAME\")).unwrap();\n let out = cmd\n .arg(\"-M\")\n .arg(\"-n\")\n .arg(\"data:text/html,\")\n .output()\n .unwrap();\n\n // STDERR should be empty\n assert_eq!(String::from_utf8_lossy(&out.stderr), \"\");\n\n // STDOUT should contain unwrapped contents of NOSCRIPT element\n assert_eq!(\n String::from_utf8_lossy(&out.stdout),\n r#\"test\n\"#\n );\n\n // Exit code should be 0\n out.assert().code(0);\n }\n}\n" }, { "path": "tests/cli/unusual_encodings.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use assert_cmd::prelude::*;\n use encoding_rs::Encoding;\n use std::env;\n use std::path::MAIN_SEPARATOR;\n use std::process::{Command, Stdio};\n\n #[test]\n fn properly_save_document_with_gb2312() {\n let cwd = env::current_dir().unwrap();\n let cwd_normalized: String = cwd.to_str().unwrap().replace(\"\\\\\", \"/\");\n let mut cmd = Command::cargo_bin(env!(\"CARGO_PKG_NAME\")).unwrap();\n let out = cmd\n .arg(\"-M\")\n .arg(format!(\n \"tests{s}_data_{s}unusual_encodings{s}gb2312.html\",\n s = MAIN_SEPARATOR\n ))\n .output()\n .unwrap();\n let file_url_protocol: &str = if cfg!(windows) { \"file:///\" } else { \"file://\" };\n\n // STDERR should contain only the target file\n assert_eq!(\n String::from_utf8_lossy(&out.stderr),\n format!(\n \"{file}{cwd}/tests/_data_/unusual_encodings/gb2312.html\\n\",\n file = file_url_protocol,\n cwd = cwd_normalized,\n )\n );\n\n // STDOUT should contain original document without any modifications\n let s: String;\n if let Some(encoding) = Encoding::for_label(b\"gb2312\") {\n let (string, _, _) = encoding.decode(&out.stdout);\n s = string.to_string();\n } else {\n s = String::from_utf8_lossy(&out.stdout).to_string();\n }\n assert_eq!(\n s,\n r##\"\n \n 近七成人减少线下需求　银行数字化转型提速--经济·科技--人民网 \n\n\n

近七成人减少线下需求　银行数字化转型提速

\n\n\n\n\"##\n );\n\n // Exit code should be 0\n out.assert().code(0);\n }\n\n #[test]\n fn properly_save_document_with_gb2312_from_stdin() {\n let mut echo = Command::new(\"cat\")\n .arg(format!(\n \"tests{s}_data_{s}unusual_encodings{s}gb2312.html\",\n s = MAIN_SEPARATOR\n ))\n .stdout(Stdio::piped())\n .spawn()\n .unwrap();\n let echo_out = echo.stdout.take().unwrap();\n echo.wait().unwrap();\n\n let mut cmd = Command::cargo_bin(env!(\"CARGO_PKG_NAME\")).unwrap();\n cmd.stdin(echo_out);\n let out = cmd.arg(\"-M\").arg(\"-\").output().unwrap();\n\n // STDERR should be empty\n assert_eq!(String::from_utf8_lossy(&out.stderr), \"\");\n\n // STDOUT should contain HTML created out of STDIN\n let s: String;\n if let Some(encoding) = Encoding::for_label(b\"gb2312\") {\n let (string, _, _) = encoding.decode(&out.stdout);\n s = string.to_string();\n } else {\n s = String::from_utf8_lossy(&out.stdout).to_string();\n }\n assert_eq!(\n s,\n r##\"\n \n 近七成人减少线下需求　银行数字化转型提速--经济·科技--人民网 \n\n\n

近七成人减少线下需求　银行数字化转型提速

\n\n\n\n\"##\n );\n\n // Exit code should be 0\n out.assert().code(0);\n }\n\n #[test]\n fn properly_save_document_with_gb2312_custom_charset() {\n let cwd = env::current_dir().unwrap();\n let cwd_normalized: String = cwd.to_str().unwrap().replace(\"\\\\\", \"/\");\n let mut cmd = Command::cargo_bin(env!(\"CARGO_PKG_NAME\")).unwrap();\n let out = cmd\n .arg(\"-M\")\n .arg(\"-E\")\n .arg(\"utf8\")\n .arg(format!(\n \"tests{s}_data_{s}unusual_encodings{s}gb2312.html\",\n s = MAIN_SEPARATOR\n ))\n .output()\n .unwrap();\n let file_url_protocol: &str = if cfg!(windows) { \"file:///\" } else { \"file://\" };\n\n // STDERR should contain only the target file\n assert_eq!(\n String::from_utf8_lossy(&out.stderr),\n format!(\n \"{file}{cwd}/tests/_data_/unusual_encodings/gb2312.html\\n\",\n file = file_url_protocol,\n cwd = cwd_normalized,\n )\n );\n\n // STDOUT should contain original document without any modifications\n assert_eq!(\n String::from_utf8_lossy(&out.stdout).to_string(),\n r#\"\n \n 近七成人减少线下需求　银行数字化转型提速--经济·科技--人民网 \n\n\n

近七成人减少线下需求　银行数字化转型提速

\n\n\n\n\"#\n );\n\n // Exit code should be 0\n out.assert().code(0);\n }\n\n #[test]\n fn properly_save_document_with_gb2312_custom_charset_bad() {\n let mut cmd = Command::cargo_bin(env!(\"CARGO_PKG_NAME\")).unwrap();\n let out = cmd\n .arg(\"-M\")\n .arg(\"-E\")\n .arg(\"utf0\")\n .arg(format!(\n \"tests{s}_data_{s}unusual_encodings{s}gb2312.html\",\n s = MAIN_SEPARATOR\n ))\n .output()\n .unwrap();\n\n // STDERR should contain error message\n assert_eq!(\n String::from_utf8_lossy(&out.stderr),\n \"Error: unknown encoding \\\"utf0\\\"\\n\"\n );\n\n // STDOUT should be empty\n assert_eq!(String::from_utf8_lossy(&out.stdout).to_string(), \"\");\n\n // Exit code should be 1\n out.assert().code(1);\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use assert_cmd::prelude::*;\n use std::env;\n use std::path::MAIN_SEPARATOR;\n use std::process::Command;\n\n #[test]\n fn change_iso88591_to_utf8_to_properly_display_html_entities() {\n let cwd = env::current_dir().unwrap();\n let cwd_normalized: String = cwd.to_str().unwrap().replace(\"\\\\\", \"/\");\n let mut cmd = Command::cargo_bin(env!(\"CARGO_PKG_NAME\")).unwrap();\n let out = cmd\n .arg(\"-M\")\n .arg(format!(\n \"tests{s}_data_{s}unusual_encodings{s}iso-8859-1.html\",\n s = MAIN_SEPARATOR\n ))\n .output()\n .unwrap();\n let file_url_protocol: &str = if cfg!(windows) { \"file:///\" } else { \"file://\" };\n\n // STDERR should contain only the target file\n assert_eq!(\n String::from_utf8_lossy(&out.stderr),\n format!(\n \"{file}{cwd}/tests/_data_/unusual_encodings/iso-8859-1.html\\n\",\n file = file_url_protocol,\n cwd = cwd_normalized,\n )\n );\n\n // STDOUT should contain original document but with UTF-8 charset\n assert_eq!(\n String::from_utf8_lossy(&out.stdout),\n r##\"\n \n \n \n � Some Company\n \n\n\n\"##\n );\n\n // Exit code should be 0\n out.assert().code(0);\n }\n}\n" }, { "path": "tests/cookies/cookie/is_expired.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::cookies;\n\n #[test]\n fn never_expires() {\n let cookie = cookies::Cookie {\n domain: String::from(\"127.0.0.1\"),\n include_subdomains: true,\n path: String::from(\"/\"),\n https_only: false,\n expires: 0,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n\n assert!(!cookie.is_expired());\n }\n\n #[test]\n fn expires_long_from_now() {\n let cookie = cookies::Cookie {\n domain: String::from(\"127.0.0.1\"),\n include_subdomains: true,\n path: String::from(\"/\"),\n https_only: false,\n expires: 9999999999,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n\n assert!(!cookie.is_expired());\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::cookies;\n\n #[test]\n fn expired() {\n let cookie = cookies::Cookie {\n domain: String::from(\"127.0.0.1\"),\n include_subdomains: true,\n path: String::from(\"/\"),\n https_only: false,\n expires: 1,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n\n assert!(cookie.is_expired());\n }\n}\n" }, { "path": "tests/cookies/cookie/matches_url.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::cookies;\n\n #[test]\n fn secure_url() {\n let cookie = cookies::Cookie {\n domain: String::from(\"127.0.0.1\"),\n include_subdomains: true,\n path: String::from(\"/\"),\n https_only: true,\n expires: 0,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n assert!(cookie.matches_url(\"https://127.0.0.1/something\"));\n }\n\n #[test]\n fn non_secure_url() {\n let cookie = cookies::Cookie {\n domain: String::from(\"127.0.0.1\"),\n include_subdomains: true,\n path: String::from(\"/\"),\n https_only: false,\n expires: 0,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n assert!(cookie.matches_url(\"http://127.0.0.1/something\"));\n }\n\n #[test]\n fn subdomain() {\n let cookie = cookies::Cookie {\n domain: String::from(\".somethingsomething.com\"),\n include_subdomains: true,\n path: String::from(\"/\"),\n https_only: true,\n expires: 0,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n assert!(cookie.matches_url(\"https://cdn.somethingsomething.com/something\"));\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::cookies;\n\n #[test]\n fn empty_url() {\n let cookie = cookies::Cookie {\n domain: String::from(\"127.0.0.1\"),\n include_subdomains: true,\n path: String::from(\"/\"),\n https_only: false,\n expires: 0,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n assert!(!cookie.matches_url(\"\"));\n }\n\n #[test]\n fn wrong_hostname() {\n let cookie = cookies::Cookie {\n domain: String::from(\"127.0.0.1\"),\n include_subdomains: true,\n path: String::from(\"/\"),\n https_only: false,\n expires: 0,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n assert!(!cookie.matches_url(\"http://0.0.0.0/\"));\n }\n\n #[test]\n fn wrong_path() {\n let cookie = cookies::Cookie {\n domain: String::from(\"127.0.0.1\"),\n include_subdomains: false,\n path: String::from(\"/\"),\n https_only: false,\n expires: 0,\n name: String::from(\"\"),\n value: String::from(\"\"),\n };\n assert!(!cookie.matches_url(\"http://0.0.0.0/path\"));\n }\n}\n" }, { "path": "tests/cookies/cookie/mod.rs", "content": "mod is_expired;\nmod matches_url;\n" }, { "path": "tests/cookies/mod.rs", "content": "mod cookie;\nmod parse_cookie_file_contents;\n" }, { "path": "tests/cookies/parse_cookie_file_contents.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::cookies;\n\n #[test]\n fn parse_file() {\n let file_contents = r#\"# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t0\tUSER_TOKEN\tin\"#;\n let result = cookies::parse_cookie_file_contents(file_contents).unwrap();\n assert_eq!(result.len(), 1);\n assert_eq!(result[0].domain, \"127.0.0.1\");\n assert!(!result[0].include_subdomains);\n assert_eq!(result[0].path, \"/\");\n assert!(!result[0].https_only);\n assert_eq!(result[0].expires, 0);\n assert_eq!(result[0].name, \"USER_TOKEN\");\n assert_eq!(result[0].value, \"in\");\n }\n\n #[test]\n fn parse_multiline_file() {\n let file_contents = r#\"# HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t0\tUSER_TOKEN\tin\n127.0.0.1\tTRUE\t/\tTRUE\t9\tUSER_TOKEN\tout\n\n\"#;\n let result = cookies::parse_cookie_file_contents(file_contents).unwrap();\n assert_eq!(result.len(), 2);\n assert_eq!(result[0].domain, \"127.0.0.1\");\n assert!(!result[0].include_subdomains);\n assert_eq!(result[0].path, \"/\");\n assert!(!result[0].https_only);\n assert_eq!(result[0].expires, 0);\n assert_eq!(result[0].name, \"USER_TOKEN\");\n assert_eq!(result[0].value, \"in\");\n assert_eq!(result[1].domain, \"127.0.0.1\");\n assert!(result[1].include_subdomains);\n assert_eq!(result[1].path, \"/\");\n assert!(result[1].https_only);\n assert_eq!(result[1].expires, 9);\n assert_eq!(result[1].name, \"USER_TOKEN\");\n assert_eq!(result[1].value, \"out\");\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::cookies;\n\n #[test]\n fn empty() {\n let file_contents = \"\";\n let result = cookies::parse_cookie_file_contents(file_contents).unwrap();\n assert_eq!(result.len(), 0);\n }\n\n #[test]\n fn no_header() {\n let file_contents = \"127.0.0.1\tFALSE\t/\tFALSE\t0\tUSER_TOKEN\tin\";\n match cookies::parse_cookie_file_contents(file_contents) {\n Ok(_result) => {\n assert!(false);\n }\n Err(_e) => {\n assert!(true);\n }\n }\n }\n\n #[test]\n fn spaces_instead_of_tabs() {\n let file_contents =\n \"# HTTP Cookie File\\n127.0.0.1 FALSE / FALSE 0 USER_TOKEN in\";\n let result = cookies::parse_cookie_file_contents(file_contents).unwrap();\n assert_eq!(result.len(), 0);\n }\n}\n" }, { "path": "tests/core/detect_media_type.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n\n use monolith::core::detect_media_type;\n\n #[test]\n fn image_gif87() {\n let dummy_url: Url = Url::parse(\"data:,\").unwrap();\n assert_eq!(detect_media_type(b\"GIF87a\", &dummy_url), \"image/gif\");\n }\n\n #[test]\n fn image_gif89() {\n let dummy_url: Url = Url::parse(\"data:,\").unwrap();\n assert_eq!(detect_media_type(b\"GIF89a\", &dummy_url), \"image/gif\");\n }\n\n #[test]\n fn image_jpeg() {\n let dummy_url: Url = Url::parse(\"data:,\").unwrap();\n assert_eq!(detect_media_type(b\"\\xFF\\xD8\\xFF\", &dummy_url), \"image/jpeg\");\n }\n\n #[test]\n fn image_png() {\n let dummy_url: Url = Url::parse(\"data:,\").unwrap();\n assert_eq!(\n detect_media_type(b\"\\x89PNG\\x0D\\x0A\\x1A\\x0A\", &dummy_url),\n \"image/png\"\n );\n }\n\n #[test]\n fn image_svg() {\n let dummy_url: Url = Url::parse(\"data:,\").unwrap();\n assert_eq!(detect_media_type(b\":\"|?/%title%.html\"#,\n r#\"/\\<>:\"|?\"#,\n MonolithOutputFormat::HTML,\n );\n\n assert_eq!(\n final_destination,\n r#\"/home/username/Downloads/<>:\"|?/__[] - -.html\"#\n );\n }\n\n #[test]\n fn level_up() {\n let final_destination =\n format_output_path(\"../%title%.html\", \".Title\", MonolithOutputFormat::HTML);\n\n assert_eq!(final_destination, r#\"../Title.html\"#);\n }\n\n #[test]\n fn file_name_extension() {\n let final_destination =\n format_output_path(\"%title%.%extension%\", \"Title\", MonolithOutputFormat::HTML);\n\n assert_eq!(final_destination, r#\"Title.html\"#);\n }\n\n #[test]\n fn file_name_extension_mhtml() {\n let final_destination =\n format_output_path(\"%title%.%extension%\", \"Title\", MonolithOutputFormat::MHTML);\n\n assert_eq!(final_destination, r#\"Title.mhtml\"#);\n }\n\n #[test]\n fn file_name_extension_short() {\n let final_destination =\n format_output_path(\"%title%.%ext%\", \"Title\", MonolithOutputFormat::HTML);\n\n assert_eq!(final_destination, r#\"Title.htm\"#);\n }\n\n #[test]\n fn file_name_extension_short_mhtml() {\n let final_destination =\n format_output_path(\"%title%.%ext%\", \"Title\", MonolithOutputFormat::MHTML);\n\n assert_eq!(final_destination, r#\"Title.mht\"#);\n }\n}\n" }, { "path": "tests/core/mod.rs", "content": "mod detect_media_type;\nmod format_output_path;\nmod options;\nmod parse_content_type;\n" }, { "path": "tests/core/options.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::core::{MonolithOptions, MonolithOutputFormat};\n\n #[test]\n fn defaults() {\n let options: MonolithOptions = MonolithOptions::default();\n\n assert!(!options.no_audio);\n assert_eq!(options.base_url, None);\n assert!(!options.no_css);\n assert_eq!(options.encoding, None);\n assert!(!options.no_frames);\n assert!(!options.no_fonts);\n assert!(!options.no_images);\n assert!(!options.isolate);\n assert!(!options.no_js);\n assert!(!options.insecure);\n assert!(!options.no_metadata);\n assert_eq!(options.output_format, MonolithOutputFormat::HTML);\n assert!(!options.silent);\n assert_eq!(options.timeout, 0);\n assert_eq!(options.user_agent, None);\n assert!(!options.no_video);\n }\n}\n" }, { "path": "tests/core/parse_content_type.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::core::parse_content_type;\n\n #[test]\n fn text_plain_utf8() {\n let (media_type, charset, is_base64) = parse_content_type(\"text/plain;charset=utf8\");\n assert_eq!(media_type, \"text/plain\");\n assert_eq!(charset, \"utf8\");\n assert!(!is_base64);\n }\n\n #[test]\n fn text_plain_utf8_spaces() {\n let (media_type, charset, is_base64) = parse_content_type(\" text/plain ; charset=utf8 \");\n assert_eq!(media_type, \"text/plain\");\n assert_eq!(charset, \"utf8\");\n assert!(!is_base64);\n }\n\n #[test]\n fn empty() {\n let (media_type, charset, is_base64) = parse_content_type(\"\");\n assert_eq!(media_type, \"text/plain\");\n assert_eq!(charset, \"US-ASCII\");\n assert!(!is_base64);\n }\n\n #[test]\n fn base64() {\n let (media_type, charset, is_base64) = parse_content_type(\";base64\");\n assert_eq!(media_type, \"text/plain\");\n assert_eq!(charset, \"US-ASCII\");\n assert!(is_base64);\n }\n\n #[test]\n fn text_html_base64() {\n let (media_type, charset, is_base64) = parse_content_type(\"text/html;base64\");\n assert_eq!(media_type, \"text/html\");\n assert_eq!(charset, \"US-ASCII\");\n assert!(is_base64);\n }\n\n #[test]\n fn only_media_type() {\n let (media_type, charset, is_base64) = parse_content_type(\"text/html\");\n assert_eq!(media_type, \"text/html\");\n assert_eq!(charset, \"US-ASCII\");\n assert!(!is_base64);\n }\n\n #[test]\n fn only_media_type_colon() {\n let (media_type, charset, is_base64) = parse_content_type(\"text/html;\");\n assert_eq!(media_type, \"text/html\");\n assert_eq!(charset, \"US-ASCII\");\n assert!(!is_base64);\n }\n\n #[test]\n fn media_type_gb2312_filename() {\n let (media_type, charset, is_base64) =\n parse_content_type(\"text/html;charset=GB2312;filename=index.html\");\n assert_eq!(media_type, \"text/html\");\n assert_eq!(charset, \"GB2312\");\n assert!(!is_base64);\n }\n\n #[test]\n fn media_type_filename_gb2312() {\n let (media_type, charset, is_base64) =\n parse_content_type(\"text/html;filename=index.html;charset=GB2312\");\n assert_eq!(media_type, \"text/html\");\n assert_eq!(charset, \"GB2312\");\n assert!(!is_base64);\n }\n}\n" }, { "path": "tests/css/embed_css.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n\n use monolith::core::MonolithOptions;\n use monolith::css;\n use monolith::session::Session;\n use monolith::url::EMPTY_IMAGE_DATA_URL;\n\n #[test]\n fn empty_input() {\n let document_url: Url = Url::parse(\"data:,\").unwrap();\n let options = MonolithOptions::default();\n let mut session: Session = Session::new(None, None, options);\n\n assert_eq!(css::embed_css(&mut session, &document_url, \"\"), \"\");\n }\n\n #[test]\n fn trim_if_empty() {\n let document_url: Url = Url::parse(\"https://doesntmatter.local/\").unwrap();\n let options = MonolithOptions::default();\n let mut session: Session = Session::new(None, None, options);\n\n assert_eq!(\n css::embed_css(&mut session, &document_url, \"\\t \\t \"),\n \"\"\n );\n }\n\n #[test]\n fn style_exclude_unquoted_images() {\n let document_url: Url = Url::parse(\"https://doesntmatter.local/\").unwrap();\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const STYLE: &str = \"/* border: none;*/\\\n background-image: url(https://somewhere.com/bg.png); \\\n list-style: url(/assets/images/bullet.svg);\\\n width:99.998%; \\\n margin-top: -20px; \\\n line-height: -1; \\\n height: calc(100vh - 10pt)\";\n\n assert_eq!(\n css::embed_css(&mut session, &document_url, STYLE),\n format!(\n \"/* border: none;*/\\\n background-image: url(\\\"{empty_image}\\\"); \\\n list-style: url(\\\"{empty_image}\\\");\\\n width:99.998%; \\\n margin-top: -20px; \\\n line-height: -1; \\\n height: calc(100vh - 10pt)\",\n empty_image = EMPTY_IMAGE_DATA_URL\n )\n );\n }\n\n #[test]\n fn style_exclude_single_quoted_images() {\n let document_url: Url = Url::parse(\"data:,\").unwrap();\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const STYLE: &str = \"/* border: none;*/\\\n background-image: url('https://somewhere.com/bg.png'); \\\n list-style: url('/assets/images/bullet.svg');\\\n width:99.998%; \\\n margin-top: -20px; \\\n line-height: -1; \\\n height: calc(100vh - 10pt)\";\n\n assert_eq!(\n css::embed_css(&mut session, &document_url, STYLE),\n format!(\n \"/* border: none;*/\\\n background-image: url(\\\"{empty_image}\\\"); \\\n list-style: url(\\\"{empty_image}\\\");\\\n width:99.998%; \\\n margin-top: -20px; \\\n line-height: -1; \\\n height: calc(100vh - 10pt)\",\n empty_image = EMPTY_IMAGE_DATA_URL\n )\n );\n }\n\n #[test]\n fn style_block() {\n let document_url: Url = Url::parse(\"file:///\").unwrap();\n let mut options = MonolithOptions::default();\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n #id.class-name:not(:nth-child(3n+0)) {\\n \\\n // border: none;\\n \\\n background-image: url(\\\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\\\");\\n\\\n }\\n\\\n \\n\\\n html > body {}\";\n\n assert_eq!(css::embed_css(&mut session, &document_url, CSS), CSS);\n }\n\n #[test]\n fn attribute_selectors() {\n let document_url: Url = Url::parse(\"https://doesntmatter.local/\").unwrap();\n let mut options = MonolithOptions::default();\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n [data-value] {\n /* Attribute exists */\n }\n\n [data-value=\\\"foo\\\"] {\n /* Attribute has this exact value */\n }\n\n [data-value*=\\\"foo\\\"] {\n /* Attribute value contains this value somewhere in it */\n }\n\n [data-value~=\\\"foo\\\"] {\n /* Attribute has this value in a space-separated list somewhere */\n }\n\n [data-value^=\\\"foo\\\"] {\n /* Attribute value starts with this */\n }\n\n [data-value|=\\\"foo\\\"] {\n /* Attribute value starts with this in a dash-separated list */\n }\n\n [data-value$=\\\"foo\\\"] {\n /* Attribute value ends with this */\n }\n \";\n\n assert_eq!(css::embed_css(&mut session, &document_url, CSS), CSS);\n }\n\n #[test]\n fn import_string() {\n let document_url: Url = Url::parse(\"https://doesntmatter.local/\").unwrap();\n let mut options = MonolithOptions::default();\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n @charset 'UTF-8';\\n\\\n \\n\\\n @import 'data:text/css,html{background-color:%23000}';\\n\\\n \\n\\\n @import url('data:text/css,html{color:%23fff}')\\n\\\n \";\n\n assert_eq!(\n css::embed_css(&mut session, &document_url, CSS),\n \"\\\n @charset \\\"UTF-8\\\";\\n\\\n \\n\\\n @import \\\"data:text/css;base64,aHRtbHtiYWNrZ3JvdW5kLWNvbG9yOiMwMDB9\\\";\\n\\\n \\n\\\n @import url(\\\"data:text/css;base64,aHRtbHtjb2xvcjojZmZmfQ==\\\")\\n\\\n \"\n );\n }\n\n #[test]\n fn hash_urls() {\n let document_url: Url = Url::parse(\"https://doesntmatter.local/\").unwrap();\n let mut options = MonolithOptions::default();\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n body {\\n \\\n behavior: url(#default#something);\\n\\\n }\\n\\\n \\n\\\n .scissorHalf {\\n \\\n offset-path: url(#somePath);\\n\\\n }\\n\\\n \";\n\n assert_eq!(css::embed_css(&mut session, &document_url, CSS), CSS);\n }\n\n #[test]\n fn transform_percentages_and_degrees() {\n let document_url: Url = Url::parse(\"https://doesntmatter.local/\").unwrap();\n let mut options = MonolithOptions::default();\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n div {\\n \\\n transform: translate(-50%, -50%) rotate(-45deg);\\n\\\n transform: translate(50%, 50%) rotate(45deg);\\n\\\n transform: translate(+50%, +50%) rotate(+45deg);\\n\\\n }\\n\\\n \";\n\n assert_eq!(css::embed_css(&mut session, &document_url, CSS), CSS);\n }\n\n #[test]\n fn unusual_indents() {\n let document_url: Url = Url::parse(\"https://doesntmatter.local/\").unwrap();\n let mut options = MonolithOptions::default();\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n .is\\\\:good:hover {\\n \\\n color: green\\n\\\n }\\n\\\n \\n\\\n #\\\\~\\\\!\\\\@\\\\$\\\\%\\\\^\\\\&\\\\*\\\\(\\\\)\\\\+\\\\=\\\\,\\\\.\\\\/\\\\\\\\\\\\'\\\\\\\"\\\\;\\\\:\\\\?\\\\>\\\\<\\\\[\\\\]\\\\{\\\\}\\\\|\\\\`\\\\# {\\n \\\n color: black\\n\\\n }\\n\\\n \";\n\n assert_eq!(css::embed_css(&mut session, &document_url, CSS), CSS);\n }\n\n #[test]\n fn exclude_fonts() {\n let document_url: Url = Url::parse(\"https://doesntmatter.local/\").unwrap();\n let mut options = MonolithOptions::default();\n options.no_fonts = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n @font-face {\\n \\\n font-family: 'My Font';\\n \\\n src: url(my_font.woff);\\n\\\n }\\n\\\n \\n\\\n #identifier {\\n \\\n font-family: 'My Font' Arial\\n\\\n }\\n\\\n \\n\\\n @font-face {\\n \\\n font-family: 'My Font';\\n \\\n src: url(my_font.woff);\\n\\\n }\\n\\\n \\n\\\n div {\\n \\\n font-family: 'My Font' Verdana\\n\\\n }\\n\\\n \";\n const CSS_OUT: &str = \" \\\n \\n\\\n \\n\\\n #identifier {\\n \\\n font-family: \\\"My Font\\\" Arial\\n\\\n }\\n\\\n \\n \\\n \\n\\\n \\n\\\n div {\\n \\\n font-family: \\\"My Font\\\" Verdana\\n\\\n }\\n\\\n \";\n\n assert_eq!(css::embed_css(&mut session, &document_url, CSS), CSS_OUT);\n }\n\n #[test]\n fn content() {\n let document_url: Url = Url::parse(\"data:,\").unwrap();\n let mut options = MonolithOptions::default();\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n #language a[href=\\\"#translations\\\"]:before {\\n\\\n content: url(data:,) \\\"\\\\A\\\";\\n\\\n white-space: pre }\\n\\\n \";\n const CSS_OUT: &str = \"\\\n #language a[href=\\\"#translations\\\"]:before {\\n\\\n content: url(\\\"data:text/plain;base64,\\\") \\\"\\\\a \\\";\\n\\\n white-space: pre }\\n\\\n \";\n\n assert_eq!(css::embed_css(&mut session, &document_url, CSS), CSS_OUT);\n }\n\n #[test]\n fn ie_css_hack() {\n let document_url: Url = Url::parse(\"data:,\").unwrap();\n let mut options = MonolithOptions::default();\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n\n const CSS: &str = \"\\\n div#p>svg>foreignObject>section:not(\\\\9) {\\n\\\n width: 300px;\\n\\\n width: 500px\\\\9;\\n\\\n }\\n\\\n \";\n const CSS_OUT: &str = \"\\\n div#p>svg>foreignObject>section:not(\\\\9) {\\n\\\n width: 300px;\\n\\\n width: 500px\\t;\\n\\\n }\\n\\\n \";\n\n assert_eq!(css::embed_css(&mut session, &document_url, CSS), CSS_OUT);\n }\n}\n" }, { "path": "tests/css/is_image_url_prop.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::css;\n\n #[test]\n fn background() {\n assert!(css::is_image_url_prop(\"background\"));\n }\n\n #[test]\n fn background_image() {\n assert!(css::is_image_url_prop(\"background-image\"));\n }\n\n #[test]\n fn background_image_uppercase() {\n assert!(css::is_image_url_prop(\"BACKGROUND-IMAGE\"));\n }\n\n #[test]\n fn border_image() {\n assert!(css::is_image_url_prop(\"border-image\"));\n }\n\n #[test]\n fn content() {\n assert!(css::is_image_url_prop(\"content\"));\n }\n\n #[test]\n fn cursor() {\n assert!(css::is_image_url_prop(\"cursor\"));\n }\n\n #[test]\n fn list_style() {\n assert!(css::is_image_url_prop(\"list-style\"));\n }\n\n #[test]\n fn list_style_image() {\n assert!(css::is_image_url_prop(\"list-style-image\"));\n }\n\n #[test]\n fn mask_image() {\n assert!(css::is_image_url_prop(\"mask-image\"));\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::css;\n\n #[test]\n fn empty() {\n assert!(!css::is_image_url_prop(\"\"));\n }\n\n #[test]\n fn width() {\n assert!(!css::is_image_url_prop(\"width\"));\n }\n\n #[test]\n fn color() {\n assert!(!css::is_image_url_prop(\"color\"));\n }\n\n #[test]\n fn z_index() {\n assert!(!css::is_image_url_prop(\"z-index\"));\n }\n}\n" }, { "path": "tests/css/mod.rs", "content": "mod embed_css;\nmod is_image_url_prop;\n" }, { "path": "tests/html/add_favicon.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use html5ever::serialize::{serialize, SerializeOpts};\n use markup5ever_rcdom::SerializableHandle;\n\n use monolith::html;\n\n #[test]\n fn basic() {\n let html = \"
text
\";\n let mut dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n dom = html::add_favicon(&dom.document, \"I_AM_A_FAVICON_DATA_URL\".to_string());\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"
text
\"\n );\n }\n}\n" }, { "path": "tests/html/check_integrity.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::html;\n\n #[test]\n fn empty_input_sha256() {\n assert!(html::check_integrity(\n \"\".as_bytes(),\n \"sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\"\n ));\n }\n\n #[test]\n fn sha256() {\n assert!(html::check_integrity(\n \"abcdef0123456789\".as_bytes(),\n \"sha256-9EWAHgy4mSYsm54hmDaIDXPKLRsLnBX7lZyQ6xISNOM=\"\n ));\n }\n\n #[test]\n fn sha384() {\n assert!(html::check_integrity(\n \"abcdef0123456789\".as_bytes(),\n \"sha384-gc9l7omltke8C33bedgh15E12M7RrAQa5t63Yb8APlpe7ZhiqV23+oqiulSJl3Kw\"\n ));\n }\n\n #[test]\n fn sha512() {\n assert!(html::check_integrity(\n \"abcdef0123456789\".as_bytes(),\n \"sha512-zG5B88cYMqcdiMi9gz0XkOFYw2BpjeYdn5V6+oFrMgSNjRpqL7EF8JEwl17ztZbK3N7I/tTwp3kxQbN1RgFBww==\"\n ));\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::html;\n\n #[test]\n fn empty_hash() {\n assert!(!html::check_integrity(\"abcdef0123456789\".as_bytes(), \"\"));\n }\n\n #[test]\n fn empty_input_empty_hash() {\n assert!(!html::check_integrity(\"\".as_bytes(), \"\"));\n }\n\n #[test]\n fn sha256() {\n assert!(!html::check_integrity(\n \"abcdef0123456789\".as_bytes(),\n \"sha256-badhash\"\n ));\n }\n\n #[test]\n fn sha384() {\n assert!(!html::check_integrity(\n \"abcdef0123456789\".as_bytes(),\n \"sha384-badhash\"\n ));\n }\n\n #[test]\n fn sha512() {\n assert!(!html::check_integrity(\n \"abcdef0123456789\".as_bytes(),\n \"sha512-badhash\"\n ));\n }\n}\n" }, { "path": "tests/html/compose_csp.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::core::MonolithOptions;\n use monolith::html;\n\n #[test]\n fn isolated() {\n let mut options = MonolithOptions::default();\n options.isolate = true;\n let csp_content = html::compose_csp(&options);\n\n assert_eq!(\n csp_content,\n \"default-src 'unsafe-eval' 'unsafe-inline' data:;\"\n );\n }\n\n #[test]\n fn no_css() {\n let mut options = MonolithOptions::default();\n options.no_css = true;\n let csp_content = html::compose_csp(&options);\n\n assert_eq!(csp_content, \"style-src 'none';\");\n }\n\n #[test]\n fn no_fonts() {\n let mut options = MonolithOptions::default();\n options.no_fonts = true;\n let csp_content = html::compose_csp(&options);\n\n assert_eq!(csp_content, \"font-src 'none';\");\n }\n\n #[test]\n fn no_frames() {\n let mut options = MonolithOptions::default();\n options.no_frames = true;\n let csp_content = html::compose_csp(&options);\n\n assert_eq!(csp_content, \"frame-src 'none'; child-src 'none';\");\n }\n\n #[test]\n fn no_js() {\n let mut options = MonolithOptions::default();\n options.no_js = true;\n let csp_content = html::compose_csp(&options);\n\n assert_eq!(csp_content, \"script-src 'none';\");\n }\n\n #[test]\n fn no_images() {\n let mut options = MonolithOptions::default();\n options.no_images = true;\n let csp_content = html::compose_csp(&options);\n\n assert_eq!(csp_content, \"img-src data:;\");\n }\n\n #[test]\n fn all() {\n let mut options = MonolithOptions::default();\n options.isolate = true;\n options.no_css = true;\n options.no_fonts = true;\n options.no_frames = true;\n options.no_js = true;\n options.no_images = true;\n let csp_content = html::compose_csp(&options);\n\n assert_eq!(\n csp_content,\n \"default-src 'unsafe-eval' 'unsafe-inline' data:; style-src 'none'; font-src 'none'; frame-src 'none'; child-src 'none'; script-src 'none'; img-src data:;\"\n );\n }\n}\n" }, { "path": "tests/html/create_metadata_tag.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use chrono::prelude::*;\n use reqwest::Url;\n\n use monolith::html;\n\n #[test]\n fn http_url() {\n let url: Url = Url::parse(\"http://192.168.1.1/\").unwrap();\n let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);\n let metadata_comment: String = html::create_metadata_tag(&url);\n\n assert_eq!(\n metadata_comment,\n format!(\n \"\",\n &url,\n timestamp,\n env!(\"CARGO_PKG_NAME\"),\n env!(\"CARGO_PKG_VERSION\"),\n )\n );\n }\n\n #[test]\n fn file_url() {\n let url: Url = Url::parse(\"file:///home/monolith/index.html\").unwrap();\n let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);\n let metadata_comment: String = html::create_metadata_tag(&url);\n\n assert_eq!(\n metadata_comment,\n format!(\n \"\",\n timestamp,\n env!(\"CARGO_PKG_NAME\"),\n env!(\"CARGO_PKG_VERSION\"),\n )\n );\n }\n\n #[test]\n fn data_url() {\n let url: Url = Url::parse(\"data:text/html,Hello%2C%20World!\").unwrap();\n let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);\n let metadata_comment: String = html::create_metadata_tag(&url);\n\n assert_eq!(\n metadata_comment,\n format!(\n \"\",\n timestamp,\n env!(\"CARGO_PKG_NAME\"),\n env!(\"CARGO_PKG_VERSION\"),\n )\n );\n }\n}\n" }, { "path": "tests/html/embed_srcset.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n\n use monolith::core::MonolithOptions;\n use monolith::html;\n use monolith::session::Session;\n use monolith::url::EMPTY_IMAGE_DATA_URL;\n\n #[test]\n fn small_medium_large() {\n let srcset_value = \"small.png 1x, medium.png 1.5x, large.png 2x\";\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n let embedded_css =\n html::embed_srcset(&mut session, &Url::parse(\"data:,\").unwrap(), srcset_value);\n\n assert_eq!(\n embedded_css,\n format!(\n \"{dataurl} 1x, {dataurl} 1.5x, {dataurl} 2x\",\n dataurl = EMPTY_IMAGE_DATA_URL,\n ),\n );\n }\n\n #[test]\n fn small_medium_only_medium_has_scale() {\n let srcset_value = \"small.png, medium.png 1.5x\";\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n let embedded_css =\n html::embed_srcset(&mut session, &Url::parse(\"data:,\").unwrap(), srcset_value);\n\n assert_eq!(\n embedded_css,\n format!(\"{dataurl}, {dataurl} 1.5x\", dataurl = EMPTY_IMAGE_DATA_URL),\n );\n }\n\n #[test]\n fn commas_within_file_names() {\n let srcset_value = \"small,s.png 1x, large,l.png 2x\";\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n let embedded_css =\n html::embed_srcset(&mut session, &Url::parse(\"data:,\").unwrap(), srcset_value);\n\n assert_eq!(\n embedded_css,\n format!(\"{dataurl} 1x, {dataurl} 2x\", dataurl = EMPTY_IMAGE_DATA_URL),\n );\n }\n\n #[test]\n fn narrow_whitespaces_within_file_names() {\n let srcset_value = \"small\\u{202f}s.png 1x, large\\u{202f}l.png 2x\";\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n let embedded_css =\n html::embed_srcset(&mut session, &Url::parse(\"data:,\").unwrap(), srcset_value);\n\n assert_eq!(\n embedded_css,\n format!(\"{dataurl} 1x, {dataurl} 2x\", dataurl = EMPTY_IMAGE_DATA_URL),\n );\n }\n\n #[test]\n fn tabs_and_newlines_after_commas() {\n let srcset_value = \"small-s.png 1x,\\tmedium,m.png 2x,\\nlarge-l.png 3x\";\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n let embedded_css =\n html::embed_srcset(&mut session, &Url::parse(\"data:,\").unwrap(), srcset_value);\n\n assert_eq!(\n embedded_css,\n format!(\n \"{dataurl} 1x, {dataurl} 2x, {dataurl} 3x\",\n dataurl = EMPTY_IMAGE_DATA_URL\n ),\n );\n }\n\n #[test]\n fn no_whitespace_after_commas() {\n let srcset_value = \"small-s.png 1x,medium-m.png 2x,large-l.png 3x\";\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n let embedded_css =\n html::embed_srcset(&mut session, &Url::parse(\"data:,\").unwrap(), srcset_value);\n\n assert_eq!(\n embedded_css,\n format!(\n \"{dataurl} 1x, {dataurl} 2x, {dataurl} 3x\",\n dataurl = EMPTY_IMAGE_DATA_URL\n ),\n );\n }\n\n #[test]\n fn last_without_descriptor() {\n let srcset_value = \"small-s.png 400w, medium-m.png 800w, large-l.png\";\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n let embedded_css =\n html::embed_srcset(&mut session, &Url::parse(\"data:,\").unwrap(), srcset_value);\n\n assert_eq!(\n embedded_css,\n format!(\n \"{dataurl} 400w, {dataurl} 800w, {dataurl}\",\n dataurl = EMPTY_IMAGE_DATA_URL\n ),\n );\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use reqwest::Url;\n\n use monolith::core::MonolithOptions;\n use monolith::html;\n use monolith::session::Session;\n use monolith::url::EMPTY_IMAGE_DATA_URL;\n\n #[test]\n fn trailing_comma() {\n let srcset_value = \"small.png 1x, large.png 2x,\";\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n let mut session: Session = Session::new(None, None, options);\n let embedded_css =\n html::embed_srcset(&mut session, &Url::parse(\"data:,\").unwrap(), srcset_value);\n\n assert_eq!(\n embedded_css,\n format!(\"{dataurl} 1x, {dataurl} 2x\", dataurl = EMPTY_IMAGE_DATA_URL),\n );\n }\n}\n" }, { "path": "tests/html/get_base_url.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::html;\n\n #[test]\n fn present() {\n let html = \"\n\n \n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(\n html::get_base_url(&dom.document),\n Some(\"https://musicbrainz.org\".to_string())\n );\n }\n\n #[test]\n fn multiple_tags() {\n let html = \"\n\n \n \n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(\n html::get_base_url(&dom.document),\n Some(\"https://www.discogs.com/\".to_string())\n );\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::html;\n\n #[test]\n fn absent() {\n let html = \"\n\n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(html::get_base_url(&dom.document), None);\n }\n\n #[test]\n fn no_href() {\n let html = \"\n\n \n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(html::get_base_url(&dom.document), None);\n }\n\n #[test]\n fn empty_href() {\n let html = \"\n\n \n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(html::get_base_url(&dom.document), Some(\"\".to_string()));\n }\n}\n" }, { "path": "tests/html/get_charset.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::html;\n\n #[test]\n fn meta_content_type() {\n let html = \"\n\n \n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(html::get_charset(&dom.document), Some(\"GB2312\".to_string()));\n }\n\n #[test]\n fn meta_charset() {\n let html = \"\n\n \n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(html::get_charset(&dom.document), Some(\"GB2312\".to_string()));\n }\n\n #[test]\n fn multiple_conflicting_meta_charset_first() {\n let html = \"\n\n \n \n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(html::get_charset(&dom.document), Some(\"utf-8\".to_string()));\n }\n #[test]\n fn multiple_conflicting_meta_content_type_first() {\n let html = \"\n\n \n \n \n \n \n \n\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n\n assert_eq!(html::get_charset(&dom.document), Some(\"GB2312\".to_string()));\n }\n}\n" }, { "path": "tests/html/get_node_attr.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use markup5ever_rcdom::{Handle, NodeData};\n\n use monolith::html;\n\n #[test]\n fn div_two_style_attributes() {\n let html = \"
\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let mut count = 0;\n\n fn test_walk(node: &Handle, i: &mut i8) {\n *i += 1;\n\n match &node.data {\n NodeData::Document => {\n // Dig deeper\n for child in node.children.borrow().iter() {\n test_walk(child, &mut *i);\n }\n }\n NodeData::Element { name, .. } => {\n let node_name = name.local.as_ref().to_string();\n\n if node_name == \"body\" {\n assert_eq!(html::get_node_attr(node, \"class\"), None);\n } else if node_name == \"div\" {\n assert_eq!(\n html::get_node_attr(node, \"style\"),\n Some(\"color: blue;\".to_string())\n );\n }\n\n for child in node.children.borrow().iter() {\n test_walk(child, &mut *i);\n }\n }\n _ => (),\n };\n }\n\n test_walk(&dom.document, &mut count);\n\n assert_eq!(count, 6);\n }\n}\n" }, { "path": "tests/html/get_node_name.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use markup5ever_rcdom::{Handle, NodeData};\n\n use monolith::html;\n\n #[test]\n fn parent_node_names() {\n let html = \"

\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let mut count = 0;\n\n fn test_walk(node: &Handle, i: &mut i8) {\n *i += 1;\n\n match &node.data {\n NodeData::Document => {\n for child in node.children.borrow().iter() {\n test_walk(child, &mut *i);\n }\n }\n NodeData::Element { name, .. } => {\n let node_name = name.local.as_ref().to_string();\n let parent = html::get_parent_node(node);\n let parent_node_name = html::get_node_name(&parent);\n if node_name == \"head\" || node_name == \"body\" {\n assert_eq!(parent_node_name, Some(\"html\"));\n } else if node_name == \"div\" {\n assert_eq!(parent_node_name, Some(\"body\"));\n } else if node_name == \"p\" {\n assert_eq!(parent_node_name, Some(\"div\"));\n }\n\n for child in node.children.borrow().iter() {\n test_walk(child, &mut *i);\n }\n }\n _ => (),\n };\n }\n\n test_walk(&dom.document, &mut count);\n\n assert_eq!(count, 7);\n }\n}\n" }, { "path": "tests/html/has_favicon.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::html;\n\n #[test]\n fn icon() {\n let html = r#\"
text
\"#;\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let res: bool = html::has_favicon(&dom.document);\n\n assert!(res);\n }\n\n #[test]\n fn shortcut_icon() {\n let html = r#\"
text
\"#;\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let res: bool = html::has_favicon(&dom.document);\n\n assert!(res);\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::html;\n\n #[test]\n fn absent() {\n let html = \"
text
\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let res: bool = html::has_favicon(&dom.document);\n\n assert!(!res);\n }\n}\n" }, { "path": "tests/html/is_favicon.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::html::is_favicon;\n\n #[test]\n fn icon() {\n assert!(is_favicon(\"icon\"));\n }\n\n #[test]\n fn shortcut_icon_capitalized() {\n assert!(is_favicon(\"Shortcut Icon\"));\n }\n\n #[test]\n fn icon_uppercase() {\n assert!(is_favicon(\"ICON\"));\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::html::is_favicon;\n\n #[test]\n fn apple_touch_icon() {\n assert!(!is_favicon(\"apple-touch-icon\"));\n }\n\n #[test]\n fn mask_icon() {\n assert!(!is_favicon(\"mask-icon\"));\n }\n\n #[test]\n fn fluid_icon() {\n assert!(!is_favicon(\"fluid-icon\"));\n }\n\n #[test]\n fn stylesheet() {\n assert!(!is_favicon(\"stylesheet\"));\n }\n\n #[test]\n fn empty_string() {\n assert!(!is_favicon(\"\"));\n }\n}\n" }, { "path": "tests/html/mod.rs", "content": "mod add_favicon;\nmod check_integrity;\nmod compose_csp;\nmod create_metadata_tag;\nmod embed_srcset;\nmod get_base_url;\nmod get_charset;\nmod get_node_attr;\nmod get_node_name;\nmod has_favicon;\nmod is_favicon;\nmod parse_link_type;\nmod parse_srcset;\nmod serialize_document;\nmod set_node_attr;\nmod walk;\n" }, { "path": "tests/html/parse_link_type.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::html;\n\n #[test]\n fn icon() {\n assert!(html::parse_link_type(\"icon\").contains(&html::LinkType::Favicon));\n }\n\n #[test]\n fn shortcut_icon_capitalized() {\n assert!(html::parse_link_type(\"Shortcut Icon\").contains(&html::LinkType::Favicon));\n }\n\n #[test]\n fn stylesheet() {\n assert!(html::parse_link_type(\"stylesheet\").contains(&html::LinkType::Stylesheet));\n }\n\n #[test]\n fn preload_stylesheet() {\n assert!(html::parse_link_type(\"preload stylesheet\").contains(&html::LinkType::Stylesheet));\n }\n\n #[test]\n fn apple_touch_icon() {\n assert!(html::parse_link_type(\"apple-touch-icon\").contains(&html::LinkType::AppleTouchIcon));\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::html;\n\n #[test]\n fn mask_icon() {\n assert!(html::parse_link_type(\"mask-icon\").is_empty());\n }\n\n #[test]\n fn fluid_icon() {\n assert!(html::parse_link_type(\"fluid-icon\").is_empty());\n }\n\n #[test]\n fn empty_string() {\n assert!(html::parse_link_type(\"\").is_empty());\n }\n}\n" }, { "path": "tests/html/parse_srcset.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::html::{parse_srcset, SrcSetItem};\n\n #[test]\n fn three_items_with_width_descriptors_and_newlines() {\n let srcset = r#\"https://some-site.com/width/600/https://media2.some-site.com/2021/07/some-image-073362.jpg 600w,\n https://some-site.com/width/960/https://media2.some-site.com/2021/07/some-image-073362.jpg 960w,\n https://some-site.com/width/1200/https://media2.some-site.com/2021/07/some-image-073362.jpg 1200w\"#;\n let srcset_items: Vec = parse_srcset(srcset);\n\n assert_eq!(srcset_items.len(), 3);\n assert_eq!(srcset_items[0].path, \"https://some-site.com/width/600/https://media2.some-site.com/2021/07/some-image-073362.jpg\");\n assert_eq!(srcset_items[0].descriptor, \"600w\");\n assert_eq!(srcset_items[1].path, \"https://some-site.com/width/960/https://media2.some-site.com/2021/07/some-image-073362.jpg\");\n assert_eq!(srcset_items[1].descriptor, \"960w\");\n assert_eq!(srcset_items[2].path, \"https://some-site.com/width/1200/https://media2.some-site.com/2021/07/some-image-073362.jpg\");\n assert_eq!(srcset_items[2].descriptor, \"1200w\");\n }\n}\n" }, { "path": "tests/html/serialize_document.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::core::MonolithOptions;\n use monolith::html;\n\n #[test]\n fn div_as_root_element() {\n let html = \"
\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let options = MonolithOptions::default();\n\n assert_eq!(\n String::from_utf8_lossy(&html::serialize_document(dom, \"\".to_string(), &options)),\n \"
\"\n );\n }\n\n #[test]\n fn full_page_with_no_html_head_or_body() {\n let html = \"Isolated document\\\n \\\n \\\n
\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let mut options = MonolithOptions::default();\n options.isolate = true;\n\n assert_eq!(\n String::from_utf8_lossy(&html::serialize_document(dom, \"\".to_string(), &options)),\n \"\\\n \\\n \\\n Isolated document\\\n \\\n \\\n \\\n \\\n
\\\n \\\n
\\\n \\\n \"\n );\n }\n\n #[test]\n fn doctype_and_the_rest_no_html_head_or_body() {\n let html = \"\\\n Unstyled document\\\n \\\n
\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let mut options = MonolithOptions::default();\n options.no_css = true;\n\n assert_eq!(\n String::from_utf8_lossy(&html::serialize_document(dom, \"\".to_string(), &options)),\n \"\\\n \\\n \\\n \\\n Unstyled document\\\n \\\n \\\n
\\\n \"\n );\n }\n\n #[test]\n fn doctype_and_the_rest_no_html_head_or_body_forbid_frames() {\n let html = \"\\\n Frameless document\\\n \\\n
\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let mut options = MonolithOptions::default();\n options.no_frames = true;\n\n assert_eq!(\n String::from_utf8_lossy(&html::serialize_document(dom, \"\".to_string(), &options)),\n \"\\\n \\\n \\\n \\\n Frameless document\\\n \\\n \\\n
\\\n \"\n );\n }\n\n #[test]\n fn doctype_and_the_rest_all_forbidden() {\n let html = \"\\\n no-frame no-css no-js no-image isolated document\\\n \\\n \\\n
\\\n \\\n \\\n \\\n
\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let mut options = MonolithOptions::default();\n options.isolate = true;\n options.no_css = true;\n options.no_fonts = true;\n options.no_frames = true;\n options.no_js = true;\n options.no_images = true;\n\n assert_eq!(\n String::from_utf8_lossy(&html::serialize_document(dom, \"\".to_string(), &options)),\n \"\\\n \\\n \\\n \\\n no-frame no-css no-js no-image isolated document\\\n \\\n \\\n \\\n \\\n
\\\n \\\n \\\n \\\n
\\\n \\\n \"\n );\n }\n}\n" }, { "path": "tests/html/set_node_attr.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use markup5ever_rcdom::{Handle, NodeData};\n\n use monolith::html;\n\n #[test]\n fn html_lang_and_body_style() {\n let html = \"\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let mut count = 0;\n\n fn test_walk(node: &Handle, i: &mut i8) {\n *i += 1;\n\n match &node.data {\n NodeData::Document => {\n // Dig deeper\n for child in node.children.borrow().iter() {\n test_walk(child, &mut *i);\n }\n }\n NodeData::Element { name, .. } => {\n let node_name = name.local.as_ref().to_string();\n\n if node_name == \"html\" {\n assert_eq!(html::get_node_attr(node, \"lang\"), Some(\"en\".to_string()));\n\n html::set_node_attr(node, \"lang\", Some(\"de\".to_string()));\n assert_eq!(html::get_node_attr(node, \"lang\"), Some(\"de\".to_string()));\n\n html::set_node_attr(node, \"lang\", None);\n assert_eq!(html::get_node_attr(node, \"lang\"), None);\n\n html::set_node_attr(node, \"lang\", Some(\"\".to_string()));\n assert_eq!(html::get_node_attr(node, \"lang\"), Some(\"\".to_string()));\n } else if node_name == \"body\" {\n assert_eq!(html::get_node_attr(node, \"style\"), None);\n\n html::set_node_attr(node, \"style\", Some(\"display: none;\".to_string()));\n assert_eq!(\n html::get_node_attr(node, \"style\"),\n Some(\"display: none;\".to_string())\n );\n }\n\n for child in node.children.borrow().iter() {\n test_walk(child, &mut *i);\n }\n }\n _ => (),\n };\n }\n\n test_walk(&dom.document, &mut count);\n\n assert_eq!(count, 5);\n }\n\n #[test]\n fn body_background() {\n let html = \"\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let mut count = 0;\n\n fn test_walk(node: &Handle, i: &mut i8) {\n *i += 1;\n\n match &node.data {\n NodeData::Document => {\n // Dig deeper\n for child in node.children.borrow().iter() {\n test_walk(child, &mut *i);\n }\n }\n NodeData::Element { name, .. } => {\n let node_name = name.local.as_ref().to_string();\n\n if node_name == \"body\" {\n assert_eq!(\n html::get_node_attr(node, \"background\"),\n Some(\"1\".to_string())\n );\n\n html::set_node_attr(node, \"background\", None);\n assert_eq!(html::get_node_attr(node, \"background\"), None);\n }\n\n for child in node.children.borrow().iter() {\n test_walk(child, &mut *i);\n }\n }\n _ => (),\n };\n }\n\n test_walk(&dom.document, &mut count);\n\n assert_eq!(count, 5);\n }\n}\n" }, { "path": "tests/html/walk.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use html5ever::serialize::{serialize, SerializeOpts};\n use markup5ever_rcdom::SerializableHandle;\n use url::Url;\n\n use monolith::core::MonolithOptions;\n use monolith::html;\n use monolith::session::Session;\n use monolith::url::EMPTY_IMAGE_DATA_URL;\n\n #[test]\n fn basic() {\n let html: &str = \"

\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"

\"\n );\n }\n\n #[test]\n fn ensure_no_recursive_iframe() {\n let html = \"

\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"

\"\n );\n }\n\n #[test]\n fn ensure_no_recursive_frame() {\n let html = \"\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\"\n );\n }\n\n #[test]\n fn no_css() {\n let html = \"\\\n \\\n \\\n \\\n
\\\n \";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_css = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n \\\n \\\n \\\n \\\n \\\n
\\\n \\\n \\\n \"\n );\n }\n\n #[test]\n fn no_images() {\n let html = \"\\\n
\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n format!(\n \"\\\n \\\n \\\n \\\n \\\n
\\\n \\\n
\\\n \\\n \",\n empty_image = EMPTY_IMAGE_DATA_URL\n )\n );\n }\n\n #[test]\n fn no_body_background_images() {\n let html =\n \"\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\"\n );\n }\n\n #[test]\n fn no_frames() {\n let html = \"\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_frames = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n \\\n \\\n \\\n \\\n \\\n \"\n );\n }\n\n #[test]\n fn no_iframes() {\n let html = \"\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_frames = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n \\\n \\\n \\\n \\\n \"\n );\n }\n\n #[test]\n fn no_js() {\n let html = \"\\\n
\\\n \\\n \\\n
\\\n \";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_js = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n \\\n
\\\n \\\n \\\n
\\\n \\\n \\\n \"\n );\n }\n\n #[test]\n fn keeps_integrity_for_unfamiliar_links() {\n let html = \"Has integrity\\\n \";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n Has integrity\\\n \\\n \\\n \\\n \\\n \"\n );\n }\n\n #[test]\n fn discards_integrity_for_known_links_nojs_nocss() {\n let html = \"\\\n No integrity\\\n \\\n \\\n \";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_css = true;\n options.no_js = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n No integrity\\\n \\\n \\\n \\\n \\\n \\\n \"\n );\n }\n\n #[test]\n fn discards_integrity_for_embedded_assets() {\n let html = \"\\\n No integrity\\\n \\\n \\\n \";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_css = true;\n options.no_js = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n No integrity\\\n \\\n \\\n \\\n \\\n \\\n \\\n \"\n );\n }\n\n #[test]\n fn removes_unwanted_meta_tags() {\n let html = \"\\\n \\\n \\\n \\\n \\\n \\\n \\\n \\\n \\\n \";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_css = true;\n options.no_frames = true;\n options.no_js = true;\n options.no_images = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n \\\n \\\n \\\n \\\n \\\n \"\n );\n }\n\n #[test]\n fn processes_noscript_tags() {\n let html = \"\\\n \\\n \\\n \\\n \\\n \";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.no_images = true;\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n format!(\n \"\\\n \\\n \\\n \\\n \\\n \\\n \\\n \",\n EMPTY_IMAGE_DATA_URL,\n )\n );\n }\n\n #[test]\n fn preserves_script_type_json() {\n let html = \"\";\n let dom = html::html_to_dom(&html.as_bytes().to_vec(), \"\".to_string());\n let url: Url = Url::parse(\"http://localhost\").unwrap();\n\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n html::walk(&mut session, &url, &dom.document);\n\n let mut buf: Vec = Vec::new();\n serialize(\n &mut buf,\n &SerializableHandle::from(dom.document.clone()),\n SerializeOpts::default(),\n )\n .unwrap();\n\n assert_eq!(\n buf.iter().map(|&c| c as char).collect::(),\n \"\\\n \\\n \\\n \\\n \\\n \\\n \\\n \"\n );\n }\n}\n" }, { "path": "tests/js/attr_is_event_handler.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::js;\n\n #[test]\n fn onblur_camelcase() {\n assert!(js::attr_is_event_handler(\"onBlur\"));\n }\n\n #[test]\n fn onclick_lowercase() {\n assert!(js::attr_is_event_handler(\"onclick\"));\n }\n\n #[test]\n fn onclick_camelcase() {\n assert!(js::attr_is_event_handler(\"onClick\"));\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::js;\n\n #[test]\n fn href() {\n assert!(!js::attr_is_event_handler(\"href\"));\n }\n\n #[test]\n fn empty_string() {\n assert!(!js::attr_is_event_handler(\"\"));\n }\n\n #[test]\n fn class() {\n assert!(!js::attr_is_event_handler(\"class\"));\n }\n}\n" }, { "path": "tests/js/mod.rs", "content": "mod attr_is_event_handler;\n" }, { "path": "tests/mod.rs", "content": "mod cli;\nmod cookies;\nmod core;\nmod css;\nmod html;\nmod js;\nmod session;\nmod url;\n" }, { "path": "tests/session/mod.rs", "content": "mod retrieve_asset;\n" }, { "path": "tests/session/retrieve_asset.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n use std::env;\n\n use monolith::core::MonolithOptions;\n use monolith::session::Session;\n use monolith::url;\n\n #[test]\n fn read_data_url() {\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n // If both source and target are data URLs,\n // ensure the result contains target data URL\n let (data, final_url, media_type, charset) = session\n .retrieve_asset(\n &Url::parse(\"data:text/html;base64,c291cmNl\").unwrap(),\n &Url::parse(\"data:text/html;base64,dGFyZ2V0\").unwrap(),\n )\n .unwrap();\n assert_eq!(&media_type, \"text/html\");\n assert_eq!(&charset, \"US-ASCII\");\n assert_eq!(\n url::create_data_url(&media_type, &charset, &data, &final_url),\n Url::parse(\"data:text/html;base64,dGFyZ2V0\").unwrap(),\n );\n assert_eq!(\n final_url,\n Url::parse(\"data:text/html;base64,dGFyZ2V0\").unwrap(),\n );\n }\n\n #[test]\n fn read_local_file_with_file_url_parent() {\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n let file_url_protocol: &str = if cfg!(windows) { \"file:///\" } else { \"file://\" };\n\n // Inclusion of local assets from local sources should be allowed\n let cwd = env::current_dir().unwrap();\n let (data, final_url, media_type, charset) = session\n .retrieve_asset(\n &Url::parse(&format!(\n \"{file}{cwd}/tests/_data_/basic/local-file.html\",\n file = file_url_protocol,\n cwd = cwd.to_str().unwrap()\n ))\n .unwrap(),\n &Url::parse(&format!(\n \"{file}{cwd}/tests/_data_/basic/local-script.js\",\n file = file_url_protocol,\n cwd = cwd.to_str().unwrap()\n ))\n .unwrap(),\n )\n .unwrap();\n assert_eq!(&media_type, \"text/javascript\");\n assert_eq!(&charset, \"\");\n let data_url = \"data:text/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==\";\n assert_eq!(\n url::create_data_url(&media_type, &charset, &data, &final_url),\n Url::parse(data_url).unwrap()\n );\n assert_eq!(\n final_url,\n Url::parse(&format!(\n \"{file}{cwd}/tests/_data_/basic/local-script.js\",\n file = file_url_protocol,\n cwd = cwd.to_str().unwrap()\n ))\n .unwrap()\n );\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use reqwest::Url;\n\n use monolith::core::MonolithOptions;\n use monolith::session::Session;\n\n #[test]\n fn read_local_file_with_data_url_parent() {\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n // Inclusion of local assets from data URL sources should not be allowed\n match session.retrieve_asset(\n &Url::parse(\"data:text/html;base64,SoUrCe\").unwrap(),\n &Url::parse(\"file:///etc/passwd\").unwrap(),\n ) {\n Ok((..)) => {\n assert!(false);\n }\n Err(_) => {\n assert!(true);\n }\n }\n }\n\n #[test]\n fn read_local_file_with_https_parent() {\n let mut options = MonolithOptions::default();\n options.silent = true;\n\n let mut session: Session = Session::new(None, None, options);\n\n // Inclusion of local assets from remote sources should not be allowed\n match session.retrieve_asset(\n &Url::parse(\"https://kernel.org/\").unwrap(),\n &Url::parse(\"file:///etc/passwd\").unwrap(),\n ) {\n Ok((..)) => {\n assert!(false);\n }\n Err(_) => {\n assert!(true);\n }\n }\n }\n}\n" }, { "path": "tests/url/clean_url.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n\n use monolith::url;\n\n #[test]\n fn preserve_original() {\n let u: Url = Url::parse(\"https://somewhere.com/font.eot#iefix\").unwrap();\n\n let clean_u: Url = url::clean_url(u.clone());\n\n assert_eq!(clean_u.as_str(), \"https://somewhere.com/font.eot\");\n assert_eq!(u.as_str(), \"https://somewhere.com/font.eot#iefix\");\n }\n\n #[test]\n fn removes_fragment() {\n assert_eq!(\n url::clean_url(Url::parse(\"https://somewhere.com/font.eot#iefix\").unwrap()).as_str(),\n \"https://somewhere.com/font.eot\"\n );\n }\n\n #[test]\n fn removes_empty_fragment() {\n assert_eq!(\n url::clean_url(Url::parse(\"https://somewhere.com/font.eot#\").unwrap()).as_str(),\n \"https://somewhere.com/font.eot\"\n );\n }\n\n #[test]\n fn removes_empty_fragment_and_keeps_empty_query() {\n assert_eq!(\n url::clean_url(Url::parse(\"https://somewhere.com/font.eot?#\").unwrap()).as_str(),\n \"https://somewhere.com/font.eot?\"\n );\n }\n\n #[test]\n fn removes_empty_fragment_and_keeps_query() {\n assert_eq!(\n url::clean_url(Url::parse(\"https://somewhere.com/font.eot?a=b&#\").unwrap()).as_str(),\n \"https://somewhere.com/font.eot?a=b&\"\n );\n }\n\n #[test]\n fn keeps_credentials() {\n assert_eq!(\n url::clean_url(Url::parse(\"https://cookie:monster@gibson.internet/\").unwrap()).as_str(),\n \"https://cookie:monster@gibson.internet/\"\n );\n }\n}\n" }, { "path": "tests/url/create_data_url.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n\n use monolith::url;\n\n #[test]\n fn encode_string_with_specific_media_type() {\n let media_type = \"application/javascript\";\n let data = \"var word = 'hello';\\nalert(word);\\n\";\n let data_url = url::create_data_url(\n media_type,\n \"\",\n data.as_bytes(),\n &Url::parse(\"data:,\").unwrap(),\n );\n\n assert_eq!(\n data_url.as_str(),\n \"data:application/javascript;base64,dmFyIHdvcmQgPSAnaGVsbG8nOwphbGVydCh3b3JkKTsK\"\n );\n }\n\n #[test]\n fn encode_append_fragment() {\n let data = \"\\n\";\n let data_url = url::create_data_url(\n \"image/svg+xml\",\n \"\",\n data.as_bytes(),\n &Url::parse(\"data:,\").unwrap(),\n );\n\n assert_eq!(\n data_url.as_str(),\n \"data:image/svg+xml;base64,PHN2Zz48L3N2Zz4K\"\n );\n }\n\n #[test]\n fn encode_string_with_specific_media_type_and_charset() {\n let media_type = \"application/javascript\";\n let charset = \"utf8\";\n let data = \"var word = 'hello';\\nalert(word);\\n\";\n let data_url = url::create_data_url(\n media_type,\n charset,\n data.as_bytes(),\n &Url::parse(\"data:,\").unwrap(),\n );\n\n assert_eq!(\n data_url.as_str(),\n \"data:application/javascript;charset=utf8;base64,dmFyIHdvcmQgPSAnaGVsbG8nOwphbGVydCh3b3JkKTsK\"\n );\n }\n\n #[test]\n fn create_data_url_with_us_ascii_charset() {\n let media_type = \"\";\n let charset = \"us-ascii\";\n let data = \"\";\n let data_url = url::create_data_url(\n media_type,\n charset,\n data.as_bytes(),\n &Url::parse(\"data:,\").unwrap(),\n );\n\n assert_eq!(data_url.as_str(), \"data:;base64,\");\n }\n\n #[test]\n fn create_data_url_with_utf8_charset() {\n let media_type = \"\";\n let charset = \"utf8\";\n let data = \"\";\n let data_url = url::create_data_url(\n media_type,\n charset,\n data.as_bytes(),\n &Url::parse(\"data:,\").unwrap(),\n );\n\n assert_eq!(data_url.as_str(), \"data:;charset=utf8;base64,\");\n }\n\n #[test]\n fn create_data_url_with_media_type_text_plain_and_utf8_charset() {\n let media_type = \"text/plain\";\n let charset = \"utf8\";\n let data = \"\";\n let data_url = url::create_data_url(\n media_type,\n charset,\n data.as_bytes(),\n &Url::parse(\"data:,\").unwrap(),\n );\n\n assert_eq!(data_url.as_str(), \"data:text/plain;charset=utf8;base64,\");\n }\n}\n" }, { "path": "tests/url/domain_is_within_domain.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::url::domain_is_within_domain;\n\n #[test]\n fn sub_domain_is_within_dotted_sub_domain() {\n assert!(domain_is_within_domain(\n \"news.ycombinator.com\",\n \".news.ycombinator.com\"\n ));\n }\n\n #[test]\n fn domain_is_within_dotted_domain() {\n assert!(domain_is_within_domain(\n \"ycombinator.com\",\n \".ycombinator.com\"\n ));\n }\n\n #[test]\n fn sub_domain_is_within_dotted_domain() {\n assert!(domain_is_within_domain(\n \"news.ycombinator.com\",\n \".ycombinator.com\"\n ));\n }\n\n #[test]\n fn sub_domain_is_within_dotted_top_level_domain() {\n assert!(domain_is_within_domain(\"news.ycombinator.com\", \".com\"));\n }\n\n #[test]\n fn domain_is_within_itself() {\n assert!(domain_is_within_domain(\n \"ycombinator.com\",\n \"ycombinator.com\"\n ));\n }\n\n #[test]\n fn domain_with_trailing_dot_is_within_itself() {\n assert!(domain_is_within_domain(\n \"ycombinator.com.\",\n \"ycombinator.com\"\n ));\n }\n\n #[test]\n fn domain_with_trailing_dot_is_within_single_dot() {\n assert!(domain_is_within_domain(\"ycombinator.com.\", \".\"));\n }\n\n #[test]\n fn domain_matches_single_dot() {\n assert!(domain_is_within_domain(\"ycombinator.com\", \".\"));\n }\n\n #[test]\n fn dotted_domain_must_be_within_dotted_domain() {\n assert!(domain_is_within_domain(\n \".ycombinator.com\",\n \".ycombinator.com\"\n ));\n }\n\n #[test]\n fn empty_is_within_dot() {\n assert!(domain_is_within_domain(\"\", \".\"));\n }\n\n #[test]\n fn both_dots() {\n assert!(domain_is_within_domain(\".\", \".\"));\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::url::domain_is_within_domain;\n\n #[test]\n fn sub_domain_must_not_be_within_domain() {\n assert!(!domain_is_within_domain(\n \"news.ycombinator.com\",\n \"ycombinator.com\"\n ));\n }\n\n #[test]\n fn domain_must_not_be_within_top_level_domain() {\n assert!(!domain_is_within_domain(\"ycombinator.com\", \"com\"));\n }\n\n #[test]\n fn different_domains_must_not_be_within_one_another() {\n assert!(!domain_is_within_domain(\n \"news.ycombinator.com\",\n \"kernel.org\"\n ));\n }\n\n #[test]\n fn sub_domain_is_not_within_wrong_top_level_domain() {\n assert!(!domain_is_within_domain(\"news.ycombinator.com\", \"org\"));\n }\n\n #[test]\n fn dotted_domain_is_not_within_domain() {\n assert!(!domain_is_within_domain(\n \".ycombinator.com\",\n \"ycombinator.com\"\n ));\n }\n\n #[test]\n fn different_domain_is_not_within_dotted_domain() {\n assert!(!domain_is_within_domain(\n \"www.doodleoptimize.com\",\n \".ycombinator.com\"\n ));\n }\n\n #[test]\n fn no_domain_can_be_within_empty_domain() {\n assert!(!domain_is_within_domain(\"ycombinator.com\", \"\"));\n }\n\n #[test]\n fn both_can_not_be_empty() {\n assert!(!domain_is_within_domain(\"\", \"\"));\n }\n}\n" }, { "path": "tests/url/get_referer_url.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n\n use monolith::url;\n\n #[test]\n fn preserve_original() {\n let original_url: Url = Url::parse(\"https://somewhere.com/font.eot#iefix\").unwrap();\n let referer_url: Url = url::get_referer_url(original_url.clone());\n assert_eq!(referer_url.as_str(), \"https://somewhere.com/font.eot\");\n assert_eq!(\n original_url.as_str(),\n \"https://somewhere.com/font.eot#iefix\"\n );\n }\n\n #[test]\n fn removes_fragment() {\n assert_eq!(\n url::get_referer_url(Url::parse(\"https://somewhere.com/font.eot#iefix\").unwrap())\n .as_str(),\n \"https://somewhere.com/font.eot\"\n );\n }\n\n #[test]\n fn removes_empty_fragment() {\n assert_eq!(\n url::get_referer_url(Url::parse(\"https://somewhere.com/font.eot#\").unwrap()).as_str(),\n \"https://somewhere.com/font.eot\"\n );\n }\n\n #[test]\n fn removes_empty_fragment_and_keeps_empty_query() {\n assert_eq!(\n url::get_referer_url(Url::parse(\"https://somewhere.com/font.eot?#\").unwrap()).as_str(),\n \"https://somewhere.com/font.eot?\"\n );\n }\n\n #[test]\n fn removes_empty_fragment_and_keeps_query() {\n assert_eq!(\n url::get_referer_url(Url::parse(\"https://somewhere.com/font.eot?a=b&#\").unwrap())\n .as_str(),\n \"https://somewhere.com/font.eot?a=b&\"\n );\n }\n\n #[test]\n fn removes_credentials() {\n assert_eq!(\n url::get_referer_url(Url::parse(\"https://cookie:monster@gibson.lan/path\").unwrap())\n .as_str(),\n \"https://gibson.lan/path\"\n );\n }\n\n #[test]\n fn removes_empty_credentials() {\n assert_eq!(\n url::get_referer_url(Url::parse(\"https://@gibson.lan/path\").unwrap()).as_str(),\n \"https://gibson.lan/path\"\n );\n }\n\n #[test]\n fn removes_empty_username_credentials() {\n assert_eq!(\n url::get_referer_url(Url::parse(\"https://:monster@gibson.lan/path\").unwrap()).as_str(),\n \"https://gibson.lan/path\"\n );\n }\n\n #[test]\n fn removes_empty_password_credentials() {\n assert_eq!(\n url::get_referer_url(Url::parse(\"https://cookie@gibson.lan/path\").unwrap()).as_str(),\n \"https://gibson.lan/path\"\n );\n }\n}\n" }, { "path": "tests/url/is_url_and_has_protocol.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use monolith::url;\n\n #[test]\n fn mailto() {\n assert!(url::is_url_and_has_protocol(\n \"mailto:somebody@somewhere.com?subject=hello\"\n ));\n }\n\n #[test]\n fn tel() {\n assert!(url::is_url_and_has_protocol(\"tel:5551234567\"));\n }\n\n #[test]\n fn ftp_no_slashes() {\n assert!(url::is_url_and_has_protocol(\"ftp:some-ftp-server.com\"));\n }\n\n #[test]\n fn ftp_with_credentials() {\n assert!(url::is_url_and_has_protocol(\n \"ftp://user:password@some-ftp-server.com\"\n ));\n }\n\n #[test]\n fn javascript() {\n assert!(url::is_url_and_has_protocol(\"javascript:void(0)\"));\n }\n\n #[test]\n fn http() {\n assert!(url::is_url_and_has_protocol(\"http://news.ycombinator.com\"));\n }\n\n #[test]\n fn https() {\n assert!(url::is_url_and_has_protocol(\"https://github.com\"));\n }\n\n #[test]\n fn file() {\n assert!(url::is_url_and_has_protocol(\"file:///tmp/image.png\"));\n }\n\n #[test]\n fn mailto_uppercase() {\n assert!(url::is_url_and_has_protocol(\n \"MAILTO:somebody@somewhere.com?subject=hello\"\n ));\n }\n\n #[test]\n fn empty_data_url() {\n assert!(url::is_url_and_has_protocol(\"data:text/html,\"));\n }\n\n #[test]\n fn empty_data_url_surrounded_by_spaces() {\n assert!(url::is_url_and_has_protocol(\" data:text/html, \"));\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use monolith::url;\n\n #[test]\n fn url_with_no_protocol() {\n assert!(!url::is_url_and_has_protocol(\n \"//some-hostname.com/some-file.html\"\n ));\n }\n\n #[test]\n fn relative_path() {\n assert!(!url::is_url_and_has_protocol(\n \"some-hostname.com/some-file.html\"\n ));\n }\n\n #[test]\n fn relative_to_root_path() {\n assert!(!url::is_url_and_has_protocol(\"/some-file.html\"));\n }\n\n #[test]\n fn empty_string() {\n assert!(!url::is_url_and_has_protocol(\"\"));\n }\n}\n" }, { "path": "tests/url/mod.rs", "content": "mod clean_url;\nmod create_data_url;\nmod domain_is_within_domain;\nmod get_referer_url;\nmod is_url_and_has_protocol;\nmod parse_data_url;\nmod resolve_url;\n" }, { "path": "tests/url/parse_data_url.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n\n use monolith::url;\n\n #[test]\n fn parse_text_html_base64() {\n let (media_type, charset, data) = url::parse_data_url(&Url::parse(\"data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==\").unwrap());\n\n assert_eq!(media_type, \"text/html\");\n assert_eq!(charset, \"US-ASCII\");\n assert_eq!(\n String::from_utf8_lossy(&data),\n \"Work expands so as to fill the time available for its completion\"\n );\n }\n\n #[test]\n fn parse_text_html_utf8() {\n let (media_type, charset, data) = url::parse_data_url(\n &Url::parse(\"data:text/html;charset=utf8,Work expands so as to fill the time available for its completion\").unwrap(),\n );\n\n assert_eq!(media_type, \"text/html\");\n assert_eq!(charset, \"utf8\");\n assert_eq!(\n String::from_utf8_lossy(&data),\n \"Work expands so as to fill the time available for its completion\"\n );\n }\n\n #[test]\n fn parse_text_html_plaintext() {\n let (media_type, charset, data) = url::parse_data_url(\n &Url::parse(\n \"data:text/html,Work expands so as to fill the time available for its completion\",\n )\n .unwrap(),\n );\n\n assert_eq!(media_type, \"text/html\");\n assert_eq!(charset, \"US-ASCII\");\n assert_eq!(\n String::from_utf8_lossy(&data),\n \"Work expands so as to fill the time available for its completion\"\n );\n }\n\n #[test]\n fn parse_text_css_url_encoded() {\n let (media_type, charset, data) =\n url::parse_data_url(&Url::parse(\"data:text/css,div{background-color:%23000}\").unwrap());\n\n assert_eq!(media_type, \"text/css\");\n assert_eq!(charset, \"US-ASCII\");\n assert_eq!(String::from_utf8_lossy(&data), \"div{background-color:#000}\");\n }\n\n #[test]\n fn parse_no_media_type_base64() {\n let (media_type, charset, data) =\n url::parse_data_url(&Url::parse(\"data:;base64,dGVzdA==\").unwrap());\n\n assert_eq!(media_type, \"text/plain\");\n assert_eq!(charset, \"US-ASCII\");\n assert_eq!(String::from_utf8_lossy(&data), \"test\");\n }\n\n #[test]\n fn parse_no_media_type_no_encoding() {\n let (media_type, charset, data) =\n url::parse_data_url(&Url::parse(\"data:;,test%20test\").unwrap());\n\n assert_eq!(media_type, \"text/plain\");\n assert_eq!(charset, \"US-ASCII\");\n assert_eq!(String::from_utf8_lossy(&data), \"test test\");\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use reqwest::Url;\n\n use monolith::url;\n\n #[test]\n fn empty_data_url() {\n let (media_type, charset, data) = url::parse_data_url(&Url::parse(\"data:,\").unwrap());\n\n assert_eq!(media_type, \"text/plain\");\n assert_eq!(charset, \"US-ASCII\");\n assert_eq!(String::from_utf8_lossy(&data), \"\");\n }\n}\n" }, { "path": "tests/url/resolve_url.rs", "content": "// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗\n// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝\n// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗\n// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod passing {\n use reqwest::Url;\n\n use monolith::url;\n\n #[test]\n fn basic_httsp_relative() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"https://www.kernel.org\").unwrap(),\n \"category/signatures.html\"\n )\n .as_str(),\n Url::parse(\"https://www.kernel.org/category/signatures.html\")\n .unwrap()\n .as_str()\n );\n }\n\n #[test]\n fn basic_httsp_absolute() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"https://www.kernel.org\").unwrap(),\n \"/category/signatures.html\"\n )\n .as_str(),\n Url::parse(\"https://www.kernel.org/category/signatures.html\")\n .unwrap()\n .as_str()\n );\n }\n\n #[test]\n fn from_https_to_level_up_relative() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"https://www.kernel.org\").unwrap(),\n \"../category/signatures.html\"\n )\n .as_str(),\n Url::parse(\"https://www.kernel.org/category/signatures.html\")\n .unwrap()\n .as_str()\n );\n }\n\n #[test]\n fn from_https_url_to_url_with_no_protocol() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"https://www.kernel.org\").unwrap(),\n \"//www.kernel.org/theme/images/logos/tux.png\",\n )\n .as_str(),\n \"https://www.kernel.org/theme/images/logos/tux.png\"\n );\n }\n\n #[test]\n fn from_https_url_to_url_with_no_protocol_and_on_different_hostname() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"https://www.kernel.org\").unwrap(),\n \"//another-host.org/theme/images/logos/tux.png\",\n )\n .as_str(),\n \"https://another-host.org/theme/images/logos/tux.png\"\n );\n }\n\n #[test]\n fn from_https_url_to_absolute_path() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"https://www.kernel.org/category/signatures.html\").unwrap(),\n \"/theme/images/logos/tux.png\",\n )\n .as_str(),\n \"https://www.kernel.org/theme/images/logos/tux.png\"\n );\n }\n\n #[test]\n fn from_https_to_just_filename() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"https://www.w3schools.com/html/html_iframe.asp\").unwrap(),\n \"default.asp\",\n )\n .as_str(),\n \"https://www.w3schools.com/html/default.asp\"\n );\n }\n\n #[test]\n fn from_data_url_to_https() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h\")\n .unwrap(),\n \"https://www.kernel.org/category/signatures.html\",\n )\n .as_str(),\n \"https://www.kernel.org/category/signatures.html\"\n );\n }\n\n #[test]\n fn from_data_url_to_data_url() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h\")\n .unwrap(),\n \"data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K\",\n )\n .as_str(),\n \"data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K\"\n );\n }\n\n #[test]\n fn from_file_url_to_relative_path() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"file:///home/user/Websites/my-website/index.html\").unwrap(),\n \"assets/images/logo.png\",\n )\n .as_str(),\n \"file:///home/user/Websites/my-website/assets/images/logo.png\"\n );\n }\n\n #[test]\n fn from_file_url_to_relative_path_with_backslashes() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"file:\\\\\\\\\\\\home\\\\user\\\\Websites\\\\my-website\\\\index.html\").unwrap(),\n \"assets\\\\images\\\\logo.png\",\n )\n .as_str(),\n \"file:///home/user/Websites/my-website/assets/images/logo.png\"\n );\n }\n\n #[test]\n fn from_data_url_to_file_url() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h\")\n .unwrap(),\n \"file:///etc/passwd\",\n )\n .as_str(),\n \"file:///etc/passwd\"\n );\n }\n\n #[test]\n fn preserve_fragment() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"http://doesnt-matter.local/\").unwrap(),\n \"css/fonts/fontmarvelous.svg#fontmarvelous\",\n )\n .as_str(),\n \"http://doesnt-matter.local/css/fonts/fontmarvelous.svg#fontmarvelous\"\n );\n }\n\n #[test]\n fn resolve_from_file_url_to_file_url() {\n if cfg!(windows) {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"file:///c:/index.html\").unwrap(),\n \"file:///c:/image.png\"\n )\n .as_str(),\n \"file:///c:/image.png\"\n );\n } else {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"file:///tmp/index.html\").unwrap(),\n \"file:///tmp/image.png\"\n )\n .as_str(),\n \"file:///tmp/image.png\"\n );\n }\n }\n}\n\n// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗\n// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝\n// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗\n// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║\n// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝\n// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝\n\n#[cfg(test)]\nmod failing {\n use reqwest::Url;\n\n use monolith::url;\n\n #[test]\n fn from_data_url_to_url_with_no_protocol() {\n assert_eq!(\n url::resolve_url(\n &Url::parse(\"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h\")\n .unwrap(),\n \"//www.w3schools.com/html/html_iframe.asp\",\n )\n .as_str(),\n \"data:,\"\n );\n }\n}\n" } ]