Repository: mkb2091/blockconvert Branch: master Commit: 4dcb5af3bc14 Files: 107 Total size: 84.6 MB Directory structure: gitextract_u7y2p1i3/ ├── .cargo/ │ └── config.toml ├── .gitattributes ├── .gitignore ├── .vscode/ │ ├── launch.json │ └── settings.json ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSE ├── README.md ├── _config.yml ├── build.rs ├── end2end/ │ ├── package.json │ ├── playwright.config.ts │ └── tests/ │ └── example.spec.ts ├── filterlists.csv ├── internal/ │ ├── adblock.txt │ ├── allow_regex.txt │ ├── allowlist.txt │ ├── block_ipnets.txt │ ├── block_ips.txt │ ├── block_regex.txt │ └── blocklist.txt ├── local.toml ├── local2.toml ├── migrations/ │ ├── 20240222234126_filterlists.sql │ ├── 20240223010915_lastmodified.sql │ ├── 20240223011106_lastmodified.sql │ ├── 20240223011400_etag.sql │ ├── 20240224194439_rules.sql │ ├── 20240224195223_filter_list_contents.sql │ ├── 20240224203224_list_rules.sql │ ├── 20240225214254_list_source.sql │ ├── 20240225230839_index.sql │ ├── 20240225231841_index.sql │ ├── 20240225232249_index.sql │ ├── 20240226004619_change_date.sql │ ├── 20240226013547_source.sql │ ├── 20240226152939_temp.sql │ ├── 20240226215547_remove_column.sql │ ├── 20240226215929_remove_id.sql │ ├── 20240226220711_remove_fkey.sql │ ├── 20240226223817_domain_rules.sql │ ├── 20240226230317_domain_block.sql │ ├── 20240226230425_rename.sql │ ├── 20240227191530_drop_column.sql │ ├── 20240227194830_drop_column.sql │ ├── 20240227200629_drop_table.sql │ ├── 20240227201410_primary_key.sql │ ├── 20240228001134_domain_rules.sql │ ├── 20240228004601_extend_rules.sql │ ├── 20240228005409_drop_rule.sql │ ├── 20240228015906_change_unique.sql │ ├── 20240228164553_ip.sql │ ├── 20240228170646_ip.sql │ ├── 20240228175807_remove_unknown.sql │ ├── 20240228180753_index.sql │ ├── 20240229210101_domains.sql │ ├── 20240229212527_subdomains.sql │ ├── 20240301000455_more_indexes.sql │ ├── 20240301000900_subdomain_idx.sql │ ├── 20240301030213_subdomain_inde.sql │ ├── 20240302171950_expanded_subdomains.sql │ ├── 20240302184040_processed_subdomains.sql │ ├── 20240302192658_index.sql │ ├── 20240302194037_domain_rule_id_idx.sql │ ├── 20240302194733_not_null_parent.sql │ ├── 20240302205633_not_null#.sql │ ├── 20240302222401_dns.sql │ ├── 20240304235746_filterlist.sql │ ├── 20240305000257_filterlist.sql │ ├── 20240305000612_filterlist.sql │ ├── 20240305003411_filterlist.sql │ ├── 20240305200551_rule_matches.sql │ ├── 20240306123603_rule_count.sql │ ├── 20240306214217_index.sql │ ├── 20240307005702_lists.sql │ ├── 20240307012450_index.sql │ └── 20240307031445_idx.sql ├── output/ │ ├── adblock.txt │ ├── allowed_ips.txt │ ├── domains.rpz │ ├── domains.txt │ ├── hosts.txt │ ├── ip_blocklist.txt │ ├── whitelist_adblock.txt │ └── whitelist_domains.txt ├── package.json ├── rust-toolchain.toml ├── rustfmt.toml ├── src/ │ ├── app.rs │ ├── domain.rs │ ├── error_template.rs │ ├── fileserv.rs │ ├── filterlist.rs │ ├── home_page.rs │ ├── ip_view.rs │ ├── lib.rs │ ├── main.rs │ ├── rule.rs │ ├── server.rs │ ├── stats_view.rs │ └── tasks.rs ├── style/ │ ├── main.scss │ └── tailwind.css ├── tailwind.config.js ├── tld_list.txt └── update.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .cargo/config.toml ================================================ [target.x86_64-unknown-linux-gnu] rustflags = ["-Clink-arg=-fuse-ld=mold", "-Ctarget-cpu=native"] ================================================ FILE: .gitattributes ================================================ # Auto detect text files and perform LF normalization * text eol=lf ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ dns_cache.txt data/ db/ extracted/ passive_dns_db/ dns_db/ #Added by cargo /target Cargo.lock # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* lerna-debug.log* .pnpm-debug.log* # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage *.lcov # nyc test coverage .nyc_output # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) .grunt # Bower dependency directory (https://bower.io/) bower_components # node-waf configuration .lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release # Dependency directories node_modules/ jspm_packages/ # Snowpack dependency directory (https://snowpack.dev/) web_modules/ # TypeScript cache *.tsbuildinfo # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Optional stylelint cache .stylelintcache # Microbundle cache .rpt2_cache/ .rts2_cache_cjs/ .rts2_cache_es/ .rts2_cache_umd/ # Optional REPL history .node_repl_history # Output of 'npm pack' *.tgz # Yarn Integrity file .yarn-integrity # dotenv environment variable files .env .env.development.local .env.test.local .env.production.local .env.local # parcel-bundler cache (https://parceljs.org/) .cache .parcel-cache # Next.js build output .next out # Nuxt.js build / generate output .nuxt dist # Gatsby files .cache/ # Comment in the public line in if your project uses Gatsby and not Next.js # https://nextjs.org/blog/next-9-1#public-directory-support # public # vuepress build output .vuepress/dist # vuepress v2.x temp and cache directory .temp .cache # Docusaurus cache and generated files .docusaurus # Serverless directories .serverless/ # FuseBox cache .fusebox/ # DynamoDB Local files .dynamodb/ # TernJS port file .tern-port # Stores VSCode versions used for testing VSCode extensions .vscode-test # yarn v2 .yarn/cache .yarn/unplugged .yarn/build-state.yml .yarn/install-state.gz .pnp.* task_cmd ================================================ FILE: .vscode/launch.json ================================================ { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { "type": "lldb", "request": "launch", "name": "Debug", "program": "${workspaceFolder}/", "args": [], "cwd": "${workspaceFolder}" } ] } ================================================ FILE: .vscode/settings.json ================================================ { "rust-analyzer.linkedProjects": [ "./Cargo.toml", ], "rust-analyzer.cargo.buildScripts.overrideCommand": null, "emmet.includeLanguages": { "rust": "html", "*.rs": "html" }, "tailwindCSS.includeLanguages": { "rust": "html", "*.rs": "html" }, "files.associations": { "*.rs": "rust" }, "editor.quickSuggestions": { "other": "on", "comments": "on", "strings": true }, "css.validate": false, "editor.fontWeight": "normal", } ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing If you are contibuting, then thank you, it is appreciated. Below is what requirements a change must have for it to be added, and what information would be useful to have. ## Adding to whitelist Requirement(s): - Adding domain to whitelist must fix broken functionality/blocked first-party websites, advert domains will not be whitelisted. Information needed: - What website is blocked/broken, for third-party domains, just the blocked domains that need to be whitelisted isn't enough. ## Adding a new filter list Requirement(s): - URL must be to original host(unless original no longer exists), not a mirror/processed version - License must be compatible with GPLv3 Information needed: - URL to filter list - Whether filter list is blacklist or whitelist - Whether it is a list of malicious paths(will result in base domains being added) ## Adding a domain to blacklist Requirement(s): - Must not break pages Information needed: - Reason for adding domain, eg for adverts, example website using it, for malicious domains, virustotal report or other proof that it is malicious ================================================ FILE: Cargo.toml ================================================ [package] name = "blockconvert" version = "0.1.0" edition = "2021" [lib] crate-type = ["cdylib", "lib"] [dependencies] serde = { version = "1.0", features = ["derive", "rc"] } url = { version = "2.5.0", features = ["serde"] } axum = { version = "0.7", optional = true } console_error_panic_hook = { version = "0.1", optional = true } leptos = { git = "https://github.com/leptos-rs/leptos", version = "0.6", features = [ "nonce", ] } leptos_axum = { git = "https://github.com/leptos-rs/leptos", version = "0.6", features = [ "nonce", ], optional = true } leptos_meta = { git = "https://github.com/leptos-rs/leptos", version = "0.6", features = [ ] } leptos_router = { git = "https://github.com/leptos-rs/leptos", version = "0.6", features = [ ] } tokio = { version = "1", features = [ "rt-multi-thread", "parking_lot", "process", "signal", ], optional = true } tokio-util = { version = "0.7", optional = true } tower = { version = "0.4", optional = true } tower-http = { version = "0.5", features = [ "fs", "compression-br", "decompression-br", ], optional = true } wasm-bindgen = { version = "=0.2.92", default-features = false, optional = true } thiserror = "1" http = "1" csv = { version = "1.3.0", optional = true } env_logger = { version = "0.11.3", optional = true } log = "0.4.21" console_log = { version = "1.0", features = ["color"], optional = true } sqlx = { version = "0.7", features = [ "runtime-tokio", "chrono", "postgres", "ipnetwork", ], optional = true } reqwest = { version = "0.11.26", features = [ "native-tls", "brotli", ], default-features = false, optional = true } chrono = { version = "0.4", features = ["serde"] } dotenvy = { version = "0.15.7", optional = true } mimalloc = { version = "0.1.39", default-features = false, optional = true } hickory-resolver = { version = "0.24.0", features = [ "tokio-runtime", ], optional = true } addr = "0.15.6" ipnetwork = "0.20.0" futures = { version = "0.3.30", optional = true } hickory-proto = { version = "0.24.0", default-features = false } humantime = "2.1.0" notify = { version = "6.1.1", optional = true } async-channel = { version = "2.2.0", optional = true } tokio-tungstenite = { version = "0.21.0", features = [ "native-tls", ], optional = true } serde_json = { version = "1.0.114", optional = true } metrics = { version = "0.22", optional = true } metrics-exporter-prometheus = { version = "0.13.1", default-features = false, features = [ "push-gateway", ], optional = true } clap = { version = "4.5.2", features = ["derive"], optional = true } rand = {version = "0.8", optional = true} toml = {version = "0.8", optional = true} [features] hydrate = [ "leptos/hydrate", "leptos_meta/hydrate", "leptos_router/hydrate", "dep:console_log", "dep:console_error_panic_hook", "dep:wasm-bindgen", ] ssr = [ "dep:axum", "dep:tokio", "dep:tokio-util", "dep:tower", "dep:tower-http", "dep:leptos_axum", "leptos/ssr", "leptos_meta/ssr", "leptos_router/ssr", "dep:csv", "dep:env_logger", "dep:sqlx", "dep:reqwest", "dep:dotenvy", "dep:mimalloc", "dns_resolver", "dep:notify", "dep:async-channel", "dep:tokio-tungstenite", "dep:serde_json", "dep:futures", "dep:metrics", "dep:metrics-exporter-prometheus", "dep:clap", "dep:rand", "dep:toml", ] dns_resolver = ["dep:hickory-resolver"] default = ["ssr"] # Defines a size-optimized profile for the WASM bundle in release mode [profile.wasm-release] inherits = "release" opt-level = 'z' lto = true codegen-units = 1 panic = "abort" [profile.wasm-dev] inherits = "dev" debug = 0 [profile.wasm-dev.package."*"] opt-level = 's' [profile.server-dev] inherits = "dev" opt-level = 1 debug = 0 lto = "off" [profile.dev.package."*"] opt-level = 3 [package.metadata.leptos] # The name used by wasm-bindgen/cargo-leptos for the JS/WASM bundle. Defaults to the crate name output-name = "blockconvert" # The site root folder is where cargo-leptos generate all output. WARNING: all content of this folder will be erased on a rebuild. Use it in your server setup. site-root = "target/blockconvert" # The site-root relative folder where all compiled output (JS, WASM and CSS) is written # Defaults to pkg site-pkg-dir = "pkg" # [Optional] The source CSS file. If it ends with .sass or .scss then it will be compiled by dart-sass into CSS. The CSS is optimized by Lightning CSS before being written to //app.css style-file = "style/main.scss" # Assets source dir. All files found here will be copied and synchronized to site-root. # The assets-dir cannot have a sub directory with the same name/path as site-pkg-dir. # # Optional. Env: LEPTOS_ASSETS_DIR. assets-dir = "public" # The IP and port (ex: 127.0.0.1:3000) where the server serves the content. Use it in your server setup. site-addr = "127.0.0.1:3000" # The port to use for automatic reload monitoring reload-port = 3001 # [Optional] Command to use when running end2end tests. It will run in the end2end dir. # [Windows] for non-WSL use "npx.cmd playwright test" # This binary name can be checked in Powershell with Get-Command npx end2end-cmd = "npx playwright test" end2end-dir = "end2end" # The browserlist query used for optimizing the CSS. browserquery = "defaults" # Set by cargo-leptos watch when building with that tool. Controls whether autoreload JS will be included in the head watch = false # The environment Leptos will run in, usually either "DEV" or "PROD" env = "DEV" # The features to use when compiling the bin target # # Optional. Can be over-ridden with the command line parameter --bin-features bin-features = ["ssr"] # If the --no-default-features flag should be used when compiling the bin target # # Optional. Defaults to false. bin-default-features = false bin-profile-dev = "server-dev" # The features to use when compiling the lib target # # Optional. Can be over-ridden with the command line parameter --lib-features lib-features = ["hydrate"] # If the --no-default-features flag should be used when compiling the lib target # # Optional. Defaults to false. lib-default-features = false # The profile to use for the lib target when compiling for release # # Optional. Defaults to "release". lib-profile-release = "wasm-release" lib-profile-dev = "wasm-dev" # The tailwind input file. # # Optional, Activates the tailwind build tailwind-input-file = "style/tailwind.css" # The tailwind config file. # # Optional, defaults to "tailwind.config.js" which if is not present # is generated for you tailwind-config-file = "tailwind.config.js" ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2022 henrik Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # BlockConvert Malware, advert and tracking blocklist which consolidates and improves upon many other [blocklists](https://github.com/mkb2091/blockconvert/blob/master/filterlists.csv). ## What this blocks: - Malware/Phishing: Many malware lists are used in building this list, including multiple malware IP lists, which are used to find many more malware domains. - Adverts: Adblock syntax is partially supported, so this list is able to extract some advert domains. This list is pretty good at blocking adverts, but an in-browser adblocker such as uBlock Origin is recommended as well as relying on hosts/DNS blocking. - Trackers: Many tracking domains are extracted from the lists used, including Privacy Badger data files which automatically identify trackers. - Coin mining: A few coin mining blocklists are used to block browser-based coin mining from using cpu. ## Advantages of using this list: - Conversion of list types. As well as supporting many common filter list formats, it also supports Privacy Badger data file, which uses algorithms to detect trackers allowing newly created trackers to be quickly detected and added to this blocklist without a human needing to spot the tracker. - Reverse DNS and passive DNS on malware IP addresses. This allows finding all the domains which a malware IP blacklist suggests could be dangerous to be found and blocked. This allows blocking of malware domains that haven't yet been added to other malware domain lists. - Use of a whitelist. Using a hosts file doesn't allow whitelisting, and many DNS-based blockers don't have great whitelist support. This list has it's own whitelist, as well as using a few others to try to reduce false positives. This list supports "*" in subdomain and TLD to aid in easily fixing many false positives at once. (If you do find a false positive(a domain that shouldn't be blocked), then please make an issue and I will remove it) - Use of DNS to check if domains still exist. Many lists contain domains that have expired and no longer exist. This makes those lists larger than needed which wastes bandwidth, space and can slow blocking. ## How to use: - Pi-hole: Go to the web interface. Log in. Settings -> Blocklists. Copy domain list URL(Pi-hole currently only supports domain lists) from below in the links section, and paste it in the textbox. Click Save. - Blokada: Open Blokada. Click shield with black middle which says "{number} in blacklist". Click the plus in the circle at the bottom of the screen. Copy and paste hosts file from link sections. Click save. WARNING: This list is large and might slow down your phone - uBlock Origin: Click the uBlock Origin logo/uBlock Origin extension. Click open dashboard(3 horizontal lines under the disable uBlock Origin button, on the right). Click Filter lists. Scroll to the bottom, and click Import(in custom section). Copy and paste the Adblock style blocklist from the link section below. ## Links [Adblock Plus format](https://mkb2091.github.io/blockconvert/output/adblock.txt) [Hosts file format](https://mkb2091.github.io/blockconvert/output/hosts.txt) WARNING: Too large for Windows: https://github.com/mkb2091/blockconvert/issues/87 [Domain list](https://mkb2091.github.io/blockconvert/output/domains.txt) [Blocked IP address list](https://mkb2091.github.io/blockconvert/output/ip_blocklist.txt) [DNS Response Policy Zone(RPZ) format](https://mkb2091.github.io/blockconvert/output/domains.rpz) As well as generating blocklists, this project also generates whitelists which are used in the process. If you maintain your own blocklist, you may find one of the following whitelists useful: [Whitelisted domains](https://mkb2091.github.io/blockconvert/output/whitelist_domains.txt) [Whitelisted ABP format](https://mkb2091.github.io/blockconvert/output/whitelist_adblock.txt) ## The Process 1. Download all expired filterlists 2. Combine and split all the filterlists based on their type. This splits the lines into seperate groups: Adblock rules, blocked domains, regexes of blocked domains, allowed domains, regex of allowed domains, ips which are blocked, ips which are allowed, subnets which are blocked, subnets which are allowed. 3. Apply a regex to all the filterlists to extract domains and combine with other domains found via other means. 4. For each of those domains, use DNS to check if the domain is still active. If the domain isn't in the allowed domains list, doesn't match any of the allowed regexes, isn't in allowed by an adblock exception rule and it is blocked, or one of its cnames/ips is blocked then add it to the output. Sources: [Sources](https://github.com/mkb2091/blockconvert/blob/master/filterlists.csv) ================================================ FILE: _config.yml ================================================ theme: jekyll-theme-cayman ================================================ FILE: build.rs ================================================ // generated by `sqlx migrate build-script` fn main() { // trigger recompilation when a new migration is added println!("cargo:rerun-if-changed=migrations"); } ================================================ FILE: end2end/package.json ================================================ { "name": "end2end", "version": "1.0.0", "description": "", "main": "index.js", "scripts": {}, "keywords": [], "author": "", "license": "ISC", "devDependencies": { "@playwright/test": "^1.28.0" } } ================================================ FILE: end2end/playwright.config.ts ================================================ import type { PlaywrightTestConfig } from "@playwright/test"; import { devices } from "@playwright/test"; /** * Read environment variables from file. * https://github.com/motdotla/dotenv */ // require('dotenv').config(); /** * See https://playwright.dev/docs/test-configuration. */ const config: PlaywrightTestConfig = { testDir: "./tests", /* Maximum time one test can run for. */ timeout: 30 * 1000, expect: { /** * Maximum time expect() should wait for the condition to be met. * For example in `await expect(locator).toHaveText();` */ timeout: 5000, }, /* Run tests in files in parallel */ fullyParallel: true, /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, /* Retry on CI only */ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: "html", /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ actionTimeout: 0, /* Base URL to use in actions like `await page.goto('/')`. */ // baseURL: 'http://localhost:3000', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: "on-first-retry", }, /* Configure projects for major browsers */ projects: [ { name: "chromium", use: { ...devices["Desktop Chrome"], }, }, { name: "firefox", use: { ...devices["Desktop Firefox"], }, }, { name: "webkit", use: { ...devices["Desktop Safari"], }, }, /* Test against mobile viewports. */ // { // name: 'Mobile Chrome', // use: { // ...devices['Pixel 5'], // }, // }, // { // name: 'Mobile Safari', // use: { // ...devices['iPhone 12'], // }, // }, /* Test against branded browsers. */ // { // name: 'Microsoft Edge', // use: { // channel: 'msedge', // }, // }, // { // name: 'Google Chrome', // use: { // channel: 'chrome', // }, // }, ], /* Folder for test artifacts such as screenshots, videos, traces, etc. */ // outputDir: 'test-results/', /* Run your local dev server before starting the tests */ // webServer: { // command: 'npm run start', // port: 3000, // }, }; export default config; ================================================ FILE: end2end/tests/example.spec.ts ================================================ import { test, expect } from "@playwright/test"; test("homepage has title and links to intro page", async ({ page }) => { await page.goto("http://localhost:3000/"); await expect(page).toHaveTitle("Welcome to Leptos"); await expect(page.locator("h1")).toHaveText("Welcome to Leptos!"); }); ================================================ FILE: filterlists.csv ================================================ name,url,author,license,expires,list_type 🎮 Game Console Adblock List,https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt,,Dandelicence,345600,Adblock uBlock filters – Resource abuse,https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resource-abuse.txt,,GPLv3,345600,Adblock uBlock filters – Privacy,https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt,,GPLv3,345600,Adblock uBlock filters – Badware risks,https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt,,GPLv3,345600,Adblock uBlock filters,https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt,,GPLv3,345600,Adblock mobiletrackers,https://raw.githubusercontent.com/craiu/mobiletrackers/master/list.txt,,GPLv3,604800,DomainBlocklist iOS Tracker Blocklist,https://raw.githubusercontent.com/jakejarvis/ios-trackers/master/blocklist.txt,jakejarvis,MIT,864000,DomainBlocklistWithoutSubdomains hostsVN,https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts,,MIT,86400,Hostfile abuse.ch URLhaus Response Policy Zones,https://urlhaus.abuse.ch/downloads/hostfile,,CC0,86400,Hostfile abuse.ch SSLBL Botnet C2 IP Blacklist,https://sslbl.abuse.ch/blacklist/sslipblacklist.txt,,CC0,86400,IPBlocklist abuse.ch Feodo Tracker Botnet C2 IP Blocklist,https://feodotracker.abuse.ch/downloads/ipblocklist.txt,,CC0,86400,IPBlocklist Xiaomi DNS Blocklist,https://raw.githubusercontent.com/unknownFalleN/xiaomi-dns-blocklist/master/xiaomi_dns_block.lst,,GPLv3,604800,DomainBlocklist WindowsSpyBlocker - Hosts spy rules,https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt,,MIT,86400,Hostfile Whitelist,https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt,ookangzheng,CC BY-NC-SA 4.0(I received permission to add to this project),86400,DomainAllowlist Whitelist,https://raw.githubusercontent.com/NoExitTV/whitelist/master/domains/whitelist.txt,,MIT,604800,DomainAllowlist Trackers,https://git.herrbischoff.com/trackers/plain/trackers.txt,,The Unlicense,864000,DomainBlocklistWithoutSubdomains The Ultimate Hosts Blacklist whitelist,https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/whitelist/master/domains.list,,MIT,86400,DomainAllowlist The Hosts File Project,https://hblock.molinero.dev/hosts,Héctor Molinero Fernández ,MIT,86400,Hostfile The Block List Project Tracking,https://raw.githubusercontent.com/blocklistproject/Lists/master/tracking.txt,,The Unlicense,86400,Hostfile The Block List Project Phishing,https://raw.githubusercontent.com/blocklistproject/Lists/master/phishing.txt,,The Unlicense,86400,Hostfile The Block List Project Malware,https://raw.githubusercontent.com/blocklistproject/Lists/master/malware.txt,,The Unlicense,86400,Hostfile The Block List Project Ads,https://raw.githubusercontent.com/blocklistproject/Lists/master/ads.txt,,The Unlicense,86400,Hostfile StevenBlack/hosts,https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts,,MIT,86400,Hostfile Slovenian List,https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt,,The Unlicense,259200,Adblock Scams and Phishing,https://raw.githubusercontent.com/infinitytec/blocklists/master/scams-and-phishing.txt,infinitytec,MIT,864000,Hostfile Scam Blocklist by DurableNapkin,https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/hosts.txt,DurableNapkin,MIT,86400,Hostfile Regex Filters for Pi-hole Whitelist,https://raw.githubusercontent.com/mmotti/pihole-regex/master/whitelist.list,,With Permission from https://github.com/mmotti/pihole-regex/issues/30,864000,DomainAllowlist Regex Filters for Pi-hole,https://raw.githubusercontent.com/mmotti/pihole-regex/master/regex.list,,With Permission from https://github.com/mmotti/pihole-regex/issues/30,864000,RegexBlocklist Privacy filters,https://raw.githubusercontent.com/metaphoricgiraffe/tracking-filters/master/trackingfilters.txt,,The Unlicense,86400,Adblock Possibilities,https://raw.githubusercontent.com/infinitytec/blocklists/master/possibilities.txt,infinitytec,MIT,864000,Hostfile PiHoleBlocklist - SmartTV,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt,,MIT,86400,DomainBlocklist PiHoleBlocklist - SessionReplay,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SessionReplay.txt,,MIT,86400,DomainBlocklist PiHoleBlocklist - AndroidTracking,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt,,MIT,86400,DomainBlocklist PiHoleBlocklist - AmazonFireTV,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt,,MIT,86400,DomainBlocklist Pi-Hole blocklist,https://codeberg.org/spootle/blocklist/raw/branch/master/blocklist.txt,,GPL-3.0,86400,DomainBlocklist Perflyst's SmartTV Blocklist for Pi-hole - RegEx extension,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/regex.list,,MIT,86400,RegexBlocklist Nopelist,https://raw.githubusercontent.com/genediazjr/nopelist/master/nopelist.txt,Gene Diaz,MIT,604800,DomainBlocklist NoTrack Tracker Blocklist,https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt,QuidsUp,GPLv3,86400,DomainBlocklist NoTrack Malware Blocklist,https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt,QuidsUp,GPLv3,86400,DomainBlocklist NoCoin Filter List,https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt,,MIT,86400,Hostfile MinerBlock Filters,https://raw.githubusercontent.com/xd4rker/MinerBlock/master/assets/filters.txt,,MIT,86400,Adblock Malicious URL Blocklist,https://gitlab.com/curben/urlhaus-filter/raw/master/urlhaus-filter-agh.txt,,CC0,86400,Adblock Magneto Malware Skanner - Burner Domains,https://raw.githubusercontent.com/gwillem/magento-malware-scanner/master/rules/burner-domains.txt,,GPLv3,864000,DomainBlocklist Lightswitch05's Ads & Tracking,https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt,Daniel White,Apache2,172800,Hostfile Latvian List,https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt,,CC-BY-SA-4.0,86400,Adblock KADhosts,https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt,,CC-BY-SA-4,86400,Hostfile International List,https://raw.githubusercontent.com/betterwebleon/international-list/master/filters.txt,,The Unlicense,259200,Adblock I Hate Tracker,https://raw.githubusercontent.com/pirat28/IHateTracker/master/iHateTracker.txt,Pirat28,MIT,864000,DomainBlocklist Herm's Ad Black List,https://raw.githubusercontent.com/hermanjustinm/Herms-Blacklist/master/HermsAdBlacklist.txt,,MPL-2.0,864000,Adblock HackerList,https://pfblockerlists.smallbusinesstech.net/hackerlist.txt,Soren Stoutner,GPLv3,86400,IPBlocklist Frellwit's Swedish Hosts File,https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt,,GPL-3.0,86400,Hostfile First-party trackers host list,https://hostfiles.frogeye.fr/multiparty-trackers.txt,,MIT,604800,DomainBlocklist Fanboy's Annoyance List,https://easylist.to/easylist/fanboy-annoyance.txt,,GPLv3,345600,Adblock EasyPrivacy,https://easylist.to/easylist/easyprivacy.txt,,GPLv3,345600,Adblock EasyList Italy,https://easylist-downloads.adblockplus.org/easylistitaly.txt,,GPLv3,86400,Adblock EasyList Hebrew,https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt,,GPLv3,86400,Adblock EasyList Germany,https://easylist.to/easylistgermany/easylistgermany.txt,,GPLv3,86400,Adblock EasyList Dutch,https://easylist-downloads.adblockplus.org/easylistdutch.txt,,GPLv3,345600,Adblock EasyList China,https://easylist-downloads.adblockplus.org/easylistchina.txt,,GPLv3,345600,Adblock EasyList,https://easylist.to/easylist/easylist.txt,,GPLv3,345600,Adblock Commonly white listed domains for Pi-Hole,https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt,,MIT,86400,DomainAllowlist CJX's Annoyance List,https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt,,LGPLv3,345600,Adblock CB-Malicious-Domains,https://raw.githubusercontent.com/cb-software/CB-Malicious-Domains/master/block_lists/domains_only.txt,,MIT,86400,DomainBlocklist BlockConvert Internal IP Blocklist,internal/block_ips.txt,mkb2091,MIT License,30,IPBlocklist BlockConvert Internal Blocklist,internal/blocklist.txt,mkb2091,MIT License,30,DomainBlocklist BlockConvert Internal Allowlist,internal/allowlist.txt,mkb2091,MIT License,30,DomainAllowlist Basic tracking list by Disconnect,https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt,,GPLv3,86400,DomainBlocklist BarbBlock,https://paulgb.github.io/BarbBlock/blacklists/hosts-file.txt,,MIT,86400,DomainBlocklist BAR-list,https://raw.githubusercontent.com/zznidar/BAR/master/BAR-list,,GPL-3.0,864000,DomainBlocklist Anudeep's Blacklist,https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt,Anudeep ,MIT,86400,Hostfile Anti-WebMiner,https://raw.githubusercontent.com/greatis/Anti-WebMiner/master/blacklist.txt,,Apache2,86400,DomainBlocklist Ads and Trackers,https://raw.githubusercontent.com/infinitytec/blocklists/master/ads-and-trackers.txt,infinitytec,MIT,864000,Hostfile AdguardMobileAds,https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardMobileAds.txt,,GPLv3,86400,Hostfile AdguardDNS,https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardDNS.txt,,GPLv3,86400,Hostfile AdguardApps,https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardApps.txt,,GPLv3,86400,Hostfile Adblock List for Finland,https://raw.githubusercontent.com/finnish-easylist-addition/finnish-easylist-addition/master/Finland_adb.txt,,The Unlicense,432000,Adblock AdGuard Simplified Domain Names filter,https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt,,GPLv3,86400,Adblock AdBlock Farsi,https://raw.githubusercontent.com/SlashArash/adblockfa/master/adblockfa.txt,,The Beer-Ware License,432000,Adblock AdAway default blocklist,https://adaway.org/hosts.txt,,CC-BY-3,86400,Hostfile Ad filter list by Disconnect,https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt,,GPLv3,86400,DomainBlocklist AZORult Tracker,https://azorult-tracker.net/api/list/domain?format=plain,,Open Database License,86400,DomainBlocklist ABPindo,https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt,,GPLv3,345600,Adblock ,https://zerodot1.gitlab.io/CoinBlockerLists/list_browser.txt,,GPLv3,86400,DomainBlocklist ,https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt,,GPLv3,86400,DomainBlocklist ,https://raw.githubusercontent.com/xxcriticxx/.pl-host-file/master/hosts.txt,,GPLv3,86400,Hostfile ,https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hacked-domains.list,Mitchell Krog,MIT,86400,DomainBlocklist ,https://raw.githubusercontent.com/matomo-org/referrer-spam-blacklist/master/spammers.txt,,Public Domain,86400,DomainBlocklist ,https://raw.githubusercontent.com/ligyxy/Blocklist/master/BLOCKLIST,,MIT,86400,DomainBlocklist ,https://raw.githubusercontent.com/EFForg/privacybadger/master/src/data/yellowlist.txt,,GPLv3+,86400,DomainAllowlist ,https://cdn.jsdelivr.net/gh/realodix/AdBlockID@master/dist/adblockid.adfl.txt,,,86400,Adblock ,https://blocklists.kitsapcreator.com/scam-spam.txt,,The Unlicense,86400,DomainBlocklist ,https://blocklists.kitsapcreator.com/malware-malicious.txt,,The Unlicense,86400,DomainBlocklist ,https://blocklists.kitsapcreator.com/general.txt,,The Unlicense,86400,DomainBlocklist ,https://blocklists.kitsapcreator.com/ads.txt,,The Unlicense,86400,DomainBlocklist ================================================ FILE: internal/adblock.txt ================================================ ||cedexis.net^$third-party,badfilter ||episerver.net^$third-party,badfilter ================================================ FILE: internal/allow_regex.txt ================================================ ^gsp[0-9][0-9]-ssl\.ls\.apple\.com$ ================================================ FILE: internal/allowlist.txt ================================================ *.a-msedge.net *.adafruit.com *.b.akamaiedge.net *.bitcoin.it *.burst-alliance.org *.c-msedge.net *.cloudflare-dns.com # Cloudflare dns server *.coingeek.com *.delivery.mp.microsoft.com *.dl.delivery.mp.microsoft.com *.dsx.mp.microsoft.com *.e-msedge.net *.g.akamai.net *.g.akamaiedge.net *.gab.com *.md.mp.microsoft.com.* *.mp.microsoft.com *.prod.do.dsp.mp.microsoft.com *.s-msedge.net *.safelinks.protection.outlook.com *.search.msn.com *.simply.com *.skype.com *.smartscreen.microsoft.com *.splunk.com *.storage.live.com *.telecommand.telemetry.microsoft.com.akadns.net *.tlu.dl.delivery.mp.microsoft.com *.ugc.bazaarvoice.com *.update.microsoft.com *.vk.com *.windowsupdate.com *.wns.windows.com 1.www.s81c.com 1drv.ms 2-01-2c3e-003d.cdx.cedexis.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-783295583 Breaks LinkedIn.com 24timezones.com 33dim-trikal.tri.sch.gr https://github.com/mkb2091/blockconvert/issues/117 3p.ampproject.net 79423.analytics.edgekey.net 91-cdn.com 91mobiles.com a-msedge.net a.espncdn.com # https://github.com/mkb2091/blockconvert/issues/58 Breaks ESPN a248.e.akamai.net aaa.com aax-us-east.amazon-adsystem.com accounts.eu1.gigya.com accounts.us1.gigya.com # Needed for iRobot app https://github.com/mkb2091/blockconvert/issues/3#issuecomment-683157232 activation-v2.sls.microsoft.com activation.sls.microsoft.com ada.support # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Cricket cell phone customer support adapools.org # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-809546196 Crypto staking adaway.org adblock.ee add0n.com addons.mozilla.org adguard.com adguard.com # Adblocking tool adguardteam.github.io adl.windows.com # https://github.com/mkb2091/blockconvert/issues/92 Breaks Windows Update aeriagames.com aetna.com # blocks www.aetna.com which is a health website aftership.com aka.ms akismet.com alexa.com aliexpress.com alive.github.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-867284255 allure.com amazon.com androidauthority.com answers.com answers.microsoft.com api.bing.com api.cdp.microsoft.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-739536391 Required for Microsoft Edge Updates api.facebook.com api.mobile.immobilienscout24.de # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-715523256 Breaks the immoscout24 app api.mymonero.com api.particle.io api.qrserver.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-780859152 Breaks setting up Google 2FA api.vanilla.futurecdn.net api.videos.oovvuu.com appleid.apple.com.akadns.net appspot-preview.l.google.com apt.newrelic.com arc.msn.com arc.msn.com.nsatc.net arizona.edu ars.smartscreen.microsoft.com assets.bwbx.io # Breaks bloomberg.com layout assets.micpn.com assets.penny-arcade.com assets.zendesk.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-867284255 att.ada.support # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Cricket cell phone customer support attfeedback.uservoice.com au.download.windowsupdate.com auth.gfx.ms awin1.com ay.gy az416426.vo.msecnd.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-744499191 Breaks Airplus Mastercard software azorult-tracker.net b-graph.facebook.com b.akamaiedge.net badges.twitch.tv # https://github.com/mkb2091/blockconvert/issues/94 Breaks Twitch TV badges baidu.com battle.net bbci.co.uk behance.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-703781874 Not malware berush.com bestbuy.ca betfair.com better.fyi bgp.he.net # https://github.com/mkb2091/blockconvert/issues/86 Part of Hurricane Electric's Internet Toolkit bilder.bild.de # https://github.com/mkb2091/blockconvert/issues/57 Breaks images on https://www.bild.de/ binarydefense.com bing.com bitbucket.org bitcoin.com bitcoin.it bitlord.com bitly.com blackhatworld.com blob.weather.microsoft.com blog.hubspot.com blog.tube8.com blogs.sch.gr # https://github.com/mkb2091/blockconvert/issues/117 bountysource.com brightcove.map.fastly.net # https://github.com/mkb2091/blockconvert/issues/78 Breaks Brightcove video browser.pipe.aria.microsoft.com c-msedge.net c.footprint.net # https://github.com/mkb2091/blockconvert/issues/88 Blocks https://www.24ur.com/ on certain systems c.paypal.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-939979052 Breaks PayPal on some websites candycrushsoda.king.com cdn.ampproject.org # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687829847 CDN for AMP cdn.content.prod.cms.msn.com # Needed for Windows 10 News Images https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687662575 cdn.onenote.net cdn.privacy-mgmt.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-701373707 Breaks closing of a popup cdn2.editmysite.com cdnjs.cloudflare.com cdnp0.stackassets.com # https://github.com/mkb2091/blockconvert/issues/64 Static assets for stacksocial.com cdnp1.stackassets.com # https://github.com/mkb2091/blockconvert/issues/64 Static assets for stacksocial.com cdnp2.stackassets.com # https://github.com/mkb2091/blockconvert/issues/64 Static assets for stacksocial.com cdnp3.stackassets.com # https://github.com/mkb2091/blockconvert/issues/64 Static assets for stacksocial.com ce1.uicdn.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-769915219 Breaks login for many www.ionos.de and www.1und1.de services cesanta.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-819106039 Not malware change.org checkappexec.microsoft.com chng.it christianpost.com ci4.googleusercontent.com ci5.googleusercontent.com clickondetroit.com # https://github.com/mkb2091/blockconvert/issues/82 News site client-office365-tas.msedge.net client-s.gateway.messenger.live.com cloudflare-dns.com cloudfront.net cmail19.com cmail20.com cn-geo1.uber.com # https://github.com/mkb2091/blockconvert/issues/71 Breaks Uber geolocation cnet.com code.jquery.com code.tidio.co code.visualstudio.com codeberg.org cognito-identity.us-east-1.amazonaws.com # https://github.com/mkb2091/blockconvert/issues/101 Breaks AWS app login coinpot.co cointraffic.io completion.amazon.com config.edge.skype.com confirmit.com content.invisioncic.com countryflags.io credit.finance.intuit.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-788266121 Breaks checking credit score on Mint Mobile App cricket.att.ada.support # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Cricket cell phone customer support crunchyroll.com ct.sendgrid.net ctldl.windowsupdate.com cv.rdtcdn.com cy2.displaycatalog.md.mp.microsoft.com.akadns.net cy2.licensing.md.mp.microsoft.com.akadns.net cy2.settings.data.microsoft.com.akadns.net cybercrime-tracker.net d2405b0jymm2dk.cloudfront.net d25xi40x97liuc.cloudfront.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-798791030 Breaks Amazon Video d2gatte9o95jao.cloudfront.net dailymail.co.uk # Blocked by PrivacyBadger datadoghq-browser-agent.com dege.freeweb.hu # https://github.com/mkb2091/blockconvert/issues/81 Site that hosts tools for running old games delivery.mp.microsoft.com desktop.google.com dev.maxmind.com # Used for MaxMind developer guide developer.spotify.com # Breaks Spotify digicert.com digitalriver.com dim-aigeir.ach.sch.gr # https://github.com/mkb2091/blockconvert/issues/117 discord.com discuss.newrelic.com displaycatalog.mp.microsoft.com disq.us distilnetworks.com djvbdz1obemzo.cloudfront.net dl.delivery.mp.microsoft.com dlsrc.getmonero.org dm3p.wns.notify.windows.com.akadns.net dmd.metaservices.microsoft.com dmd.metaservices.microsoft.com.akadns.net dmqdd6hw24ucf.cloudfront.net # https://github.com/mkb2091/blockconvert/issues/63 Breaks Amazon Prime Subtitles dns.google.com do.co donate.getmonero.org download.db-ip.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-779885164 For Geo IP database download download.electrum.org download.maxmind.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-749884660 Used to download MaxMind GeoIP files download.newrelic.com download.visualware.com # https://github.com/mkb2091/blockconvert/issues/123 download.windowsupdate.com download.windowsupdate.com.c.footprint.net downloads.getmonero.org dpreview.com drh.img.digitalriver.com drh1.img.digitalriver.com drudgereport.com dsx.mp.microsoft.com duckdns.org duckduckgo.com duken.nl # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-869051876 A forum, not malware e-msedge.net e785s8hz.micpn.com easylist-downloads.adblockplus.org easylist.to ecdinterface.philips.com eec.crunchyroll.com electrum.org elinkeu.clickdimensions.com eloqua.com embed.spotify.com # Breaks Spotify emdl.ws.microsoft.com encrypted-tbn1.gstatic.com encrypted-tbn2.gstatic.com encrypted-tbn3.gstatic.com entireweb.com eu-central-1.protection.sophos.com # https://github.com/mkb2091/blockconvert/issues/93 Its a cloud security platform ev.rdtcdn.com everest.castbox.fm evoke-windowsservices-tas.msedge.net f.chtah.com f1.media.brightcove.com # https://github.com/mkb2091/blockconvert/issues/78 Breaks Brightcove video faast.schibsted.io facebook.com fandom.wikia.com fast.fonts.net fast.wistia.net faucethub.io fe2.update.microsoft.com fe2.update.microsoft.com.akadns.net fe3.delivery.dsp.mp.microsoft.com.nsatc.net fe3.delivery.mp.microsoft.com feodotracker.abuse.ch fiddle.jshell.net fide.com files.catbox.moe files.freedownloadmanager.org files.slack.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-771638966 Breaks user uploaded content on Slack files2.freedownloadmanager.org filterlists.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-787428332 Website with list of filterlists filtri-dns.ga firebaseremoteconfig.googleapis.com # Breaks android applications internally updating fiverr.com flipboard.com forms.office.com forum.getmonero.org forum.overclockers.co.uk # PC forum forums.storagereview.com fossbytes.com freedownloadmanager.org fs.microsoft.com ftpcontent.worldnow.com funk.eu g.akamai.net g.akamaiedge.net g.live.com g.msn.com.nsatc.net gadgetsnow.com gameofbitcoins.com gamespot.com gannettdigital.com gardenstatehelicopters.com gateway.reddit.com gcs.sc-cdn.net geo-prod.do.dsp.mp.microsoft.com geo-prod.dodsp.mp.microsoft.com.nsatc.net getmonero.org github.com github.developerdan.com gitlab.com gleam.io # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 global.edge.bamgrid.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687908780 Blocks access to Disney Plus global.ssl.fastly.net go.microsoft.com go2cloud.org google.* google.co.* google.com.* googletagmanager.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-780859152 Breaks setting up Google 2FA gpticketshop.com # https://github.com/mkb2091/blockconvert/issues/111 gs-loc.apple.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-695044303 Needed for Apple geolocation gsp-ssl-geomap.ls-apple.com.akadns.net # Breaks Apple Maps gsp-ssl.ls-apple.com.akadns.net # Breaks Apple Maps gsp-ssl.ls.apple.com # Break Apple Maps guce.advertising.com guce.huffpost.com h-ebay.online-metrix.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks EBay external logins hblock.molinero.dev help.mint.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Mint help heureka.cz highcharts.com homedepot.tt.omtrdc.net hostsfile.mine.nu howtogeek.com hubpages.com hwcdn.net i-am3p-cor003.api.p001.1drv.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks OneDrive/OutLook file storage i.ibb.co ibm.com ign.com ignorelist.com image.prntscr.com images.gounlimited.to imagizer.imageshack.com img-prod-cms-rt-microsoft-com.akamaized.net img-s-msn-com.akamaized.net # https://github.com/mkb2091/blockconvert/issues/67 Breaks msn.com images in.appcenter.ms # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687908780 Blocks access to Outlook 365 mail inc.com inference.location.live.net informer.com infowars.com infowarsstore.com insight.rapid7.com insights.eu.newrelic.com instagram.com int.whiteboard.microsoft.com io9.com ip-adress.com ip-tracker.org ip107307705.ahcdn.com # https://github.com/mkb2091/blockconvert/issues/75 Breaks videos on streaming XXX site ip107316446.ahcdn.com # https://github.com/mkb2091/blockconvert/issues/75 Breaks videos on streaming XXX site ipv4.login.msa.akadns6.net it.toolbox.com itch.io jaleco.com # Breaks download link for GIMP jarv.is joblo.com jpost.com # News website js.t.sinajs.cn kaspersky.com kasperskycontenthub.com knowyourmeme.com ku6.com larati.net leagueoflegends.com leanplum.com licensing.mp.microsoft.com linkedin.com links.mint.com linktr.ee livejasmin.com livejournal.com location-inference-westus.cloudapp.net location.services.mozilla.com login.live.com login.msa.akadns6.net login.windows.net logincdn.msauth.net lowendbox.com m.me m.stripe.network mail.com mail.getmonero.org mail.google.com mailchi.mp malwaredomainlist.com manager-magazin-de.manager-magazin.de # https://github.com/mkb2091/blockconvert/issues/60 Breaks accepting cookie banner manifest.prod.boltdns.net # https://github.com/mkb2091/blockconvert/issues/66 Breaks BrightCove player maps.windows.com mbl.is md.mp.microsoft.com.* media.pearsoncmg.com mediafire.com mediaredirect.microsoft.com mine.nu miro.medium.com # https://github.com/mkb2091/blockconvert/issues/77 and https://github.com/mkb2091/blockconvert/issues/3#issuecomment-748449378, Breaks images on medium.com mirror.co.uk mirrorservice.org # Breaks VLC update download mobile.tube8.com mobile.twitter.com modern.watson.data.microsoft.com.akadns.net monero.org moonbit.co.in motherboard.vice.com movable-ink-397.com mp.microsoft.com mscrl.microsoft.com msedge.api.cdp.microsoft.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-729147310 Needed for Microsoft Edge to check for updates msedge.net msftconnecttest.com mvaali1.ws.byteoversea.net mw1.wsj.net # https://github.com/mkb2091/blockconvert/issues/50 my.matterport.com # https://github.com/mkb2091/blockconvert/issues/72 Breaks a 3D space capture platform mymonero.com namecheap.com narkive.com naver.com netfort.com newrelic.com news.bitcoin.com newyorker.com # News website next-services.apps.microsoft.com nexus.ensighten.com nexusrules.officeapps.live.com ning.com notabug.org notifications.google.com now.sh ocation-inference-westus.cloudapp.net ocos-office365-s2s.msedge.ne ocos-office365-s2s.msedge.net ocsp.digicert.com ocsp.verisign.com # https://github.com/mkb2091/blockconvert/issues/94 Breaks verifying validity of certificates od.lk oem.twimg.com officeclient.microsoft.com ogs.google.com oneclient.sfx.ms onesignal.com onion.to online.jimmyjohns.com open.spotify.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-788015015 Breaks Spotify openwall.com oracle.com # Breaks java download orbitz.com osce11-0-en.url.trendmicro.com outlook.office365.com overclockers.co.uk # PC forum pastebin.com pastebin.org pasteboard.co pbs.twimg.com pcmag.com # Blocked by PrivacyBadger pcmatic.com phishtank.com play.spotify.com # Breaks Spotify play.spotify.edgekey.net # Breaks Spotify polyfill.io pool.ntp.org pornhub.com post.spmailtechnol.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Used for links in email powr.io privateinternetaccess.com prod.do.dsp.mp.microsoft.com prod.telemetry.ros.rockstargames.com proofpoint.com pti.store.microsoft.com purchase.mp.microsoft.com push.prod.netflix.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-809546196 Breaks Netflix DRM pushbullet.com # https://github.com/mkb2091/blockconvert/issues/115 qq.com query.prod.cms.rt.microsoft.com quickenloans.com quora.com rakuten.com ranker.com ransomwaretracker.abuse.ch raw.githubusercontent.com readcomiconline.to recruitics.com reddit.app.link reddit.com redditstatic.s3.amazonaws.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687908780 Breaks something redtube.com refinery29.com # Blog website report-uri.com rest-prde.immedia-semi.com returnpath.com ris-prod-atm.trafficmanager.net ris.api.iris.microsoft.com ris.api.iris.microsoft.com.akadns.net router.utorrent.com rumble.com # https://github.com/mkb2091/blockconvert/issues/76 Video Streaming Platform s-media-cache-ak0.pinimg.com s-msedge.net s.imgur.com s.ndemiccreations.com s.tradingview.com # https://github.com/mkb2091/blockconvert/issues/49 s0.ipstatp.com # https://github.com/mkb2091/blockconvert/issues/83 Breaks Captcha s3-ap-southeast-1.amazonaws.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-793400847 Breaks AWS S3 in SEA s3-eu-west-1.amazonaws.com s3-us-west-1.amazonaws.com s3.amazonaws.com s3.tradingview.com # https://github.com/mkb2091/blockconvert/issues/49 s7.orientaltrading.com safebrowsing-cache.google.com safebrowsing.google.com sat1.us1-1.nanorep.co # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Mint help scrapinghub.com sdks.shopifycdn.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-707148021 Needed for webshops search.msn.com searchenginejournal.com secure.indeed.com # https://github.com/mkb2091/blockconvert/issues/84 Breaks login page for indeed.com secure.mbl.is semrush.com sendgrid.com # https://github.com/mkb2091/blockconvert/issues/70 Base domain is not used for tracking serial.alcohol-soft.com # https://github.com/mkb2091/blockconvert/issues/80 CD and DVD burning software service.weather.microsoft.com settings-win.data.microsoft.com settings.data.microsoft.com sfdataservice.microsoft.com # Blocks Microsoft store payments sh.st shop.app # https://github.com/mkb2091/blockconvert/issues/98#issue-940336845 URL shortener for shopify shop.gadgetsnow.com si6ling.incestflix.com # https://github.com/mkb2091/blockconvert/issues/65 Static Assets similarweb.com sina.com.cn sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo sj14.mktossl.com # CNAME for safe.riskiq.com skype.com skypeecs-prod-usw-0-b.cloudapp.net sls.update.microsoft.com sls.update.microsoft.com.akadns.net smallseotools.com smartscreen-sn3p.smartscreen.microsoft.com smartscreen.microsoft.com socialize.us1.gigya.com # Needed for iRobot app https://github.com/mkb2091/blockconvert/issues/3#issuecomment-683157232 software.informer.com sohu.com som.aeroplan.com soundcloud.com sourceforge.net spcweb.ch # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-808365475 Breaks swiss parcel tracking speedcurve.com spiegel-de.spiegel.de # https://github.com/mkb2091/blockconvert/issues/61 spotify.com # Breaks Spotify squidblacklist.org src.ebay-us.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks EBay external logins ssl.bblck.me ssl.marfeelcdn.com sslbl.abuse.ch stacksocial.com staging-discuss.newrelic.com star-mini.c10r.facebook.com static-dot-virustotalcloud.appspot.com static-exp1.licdn.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-783295583 Breaks LinkedIn.com static.bbci.co.uk static.getmonero.org static.matterport.com # https://github.com/mkb2091/blockconvert/issues/72 Breaks a 3D space capture platform static.tacdn.com # https://github.com/mkb2091/blockconvert/issues/52 static.tidiochat.com static.wikia.nocookie.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-744090132 Breaks images on fandom.com wikis static.zdassets.com statics.streamable.com stats.foldingathome.org # https://github.com/mkb2091/blockconvert/issues/74 Regex False Positive stats.govt.nz # # https://github.com/mkb2091/blockconvert/issues/99 Government of New Zealand Statistics site stats.pingdom.com stats.stackexchange.com statsfe2.update.microsoft.com.akadns.net status.newrelic.com status.roll20.net statuscake-email.com statuscake.com storage.live.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks OneDrive/OutLook file storage store-images.microsoft.com store-images.s-microsoft.com storecatalogrevocation.storequality.microsoft.com storeedgefd.dsx.mp.microsoft.com stripecdn.map.fastly.net substrate.ms-acdc.office.com # CNAME for outlook.office.com substrate.office.com # CNAME for outlook.office.com supervisor.ext-twitch.tv # https://github.com/mkb2091/blockconvert/issues/100 Breaks twitch extensions support.microsoft.com survey.euro.confirmit.com survey.pearsoncmg.com techradar.com tecmint.com teespring.com telecommand.telemetry.microsoft.com.akadns.net texashomebase.com thebot.net theepochtimes.com # https://github.com/mkb2091/blockconvert/issues/73 News website theoldnet.com tiktok.com # Causes adblock to block www.tiktok.com tile-service.weather.microsoft.com time.samsungcloudsolution.com # Issues #3, blocks services like Plex, YouTube and Amazon Video on Samsung TV time.windows.com tlu.dl.delivery.mp.microsoft.com tmx.td.com # https://github.com/mkb2091/blockconvert/issues/102 Breaks Canadian finance website to-do.microsoft.com toptenreviews.com tracker.tvnihon.com tracker2.itzmx.com # https://github.com/mkb2091/blockconvert/issues/124 tracking.epicgames.com transfer.sh travelocity.com trial.alcohol-soft.com # https://github.com/mkb2091/blockconvert/issues/80 CD and DVD burning software trk.klclick1.com tsfe.trafficshaping.dsp.mp.microsoft.com tube8.com uk.com umich.qualtrics.com unitedstates.smartscreen-prod.microsoft.com update.microsoft.com urldefense.proofpoint.com urlhaus.abuse.ch us-east-1-a.route.herokuapp.com us.configsvc1.live.com.akadns.net use.typekit.net users.telenet.be uservoice.com v.firebog.net v10.events.data.microsoft.com v10.vortex-win.data.microsoft.com v20.events.data.microsoft.com validation-v2.sls.microsoft.com vanilla.futurecdn.net vanityfair.com version.hybrid.api.here.com # Needed for "Here We Go" map download https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687661883 vgwort.de videos.oovvuu.com vignette.wikia.nocookie.net vip5.afdorigin-prod-am02.afdogw.com vip5.afdorigin-prod-ch02.afdogw.com virustotalcloud.appspot.com vivo.com.br vmn.net vortex.accuweather.com vxvault.net w.soundcloud.com # https://github.com/mkb2091/blockconvert/issues/51 wac.phicdn.net wallet.microsoft.com watchdog-basic.energized.pro # Used to check if user is using Energized Protection watchdog-blu.energized.pro # Used to check if user is using Energized Protection watchdog-blugo.energized.pro # Used to check if user is using Energized Protection watchdog-dns.energized.pro # Used to check if user is using Energized Protection watchdog-ext.energized.pro # Used to check if user is using Energized Protection watchdog-porn.energized.pro # Used to check if user is using Energized Protection watchdog-spark.energized.pro # Used to check if user is using Energized Protection watchdog-ultimate.energized.pro # Used to check if user is using Energized Protection watchdog-unified.energized.pro # Used to check if user is using Energized Protection watchdog.energized.pro # Used to check if user is using Energized Protection watson.telemetry.microsoft.com wbd.ms wd-prod-cp-us-west-1-fe.westus.cloudapp.azure.com wd-prod-ss.trafficmanager.net wdcp.microsoft.* wdcp.microsoft.com web.getmonero.org web.tresorit.com webmasters.stackexchange.com webmd.com whiteboard.microsoft.com whiteboard.ms widget-mediator.zopim.com widgetdata.tradingview.com # https://github.com/mkb2091/blockconvert/issues/49 wikidot.com wildcard.twimg.com windowsupdate.com wl.spotify.com # Breaks spotify account confirmation wns.windows.com worldstream.nl wrapper-api.sp-prod.net wscont.apps.microsoft.com wscont1.apps.microsoft.com wscont2.apps.microsoft.com wsj.com ww.youporn.com ww2.sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo www.pushbullet.com # https://github.com/mkb2091/blockconvert/issues/115 wwww.youporn.com wx2.sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo wx3.sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo wx4.sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo x1.com xnxx.com # https://github.com/mkb2091/blockconvert/issues/56 yahoo.co.jp yahoo.uservoice.com yandex.ru # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-786252939 Breaks yandex yoast.com youporn.com zdnet.com zerodot1.gitlab.io zerohedge.com zeustracker.abuse.ch zoho.com ================================================ FILE: internal/block_ipnets.txt ================================================ 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.0.0.0/24 192.0.2.0/24 192.88.99.0/24 192.168.0.0/16 198.18.0.0/15 198.51.100.0/24 203.0.113.0/24 224.0.0.0/4 240.0.0.0/4 255.255.255.255/32 ================================================ FILE: internal/block_ips.txt ================================================ ================================================ FILE: internal/block_regex.txt ================================================ \.(?:com|co\.uk|net)\.(?:login|account|payment|verif) [_.-]analytics?[_.-] ================================================ FILE: internal/blocklist.txt ================================================ *.ct.sendgrid.net # Tracking https://github.com/mkb2091/blockconvert/issues/70 *.getemails.com # De-anonymization af.congressosiv-isv2018.it ag.bud-info.com.pl all-prize-giveaway.life an-geburtstag.fun # Spam app0973.vnbue32.live awfulcruelty9.live bestprizesday4.life bs.newlifecampania.it # Spam cd.nestwood.it cf.spettinatidautore.it # Spam cl.folignobenecomune.it competition4138.redirect-servers52.live converseboom24.live cp.milano2000.it dxnp7918p5axx.cloudfront.net ee.caleidoscopio-singoli-pisa.it getemails.com # De-anonymization installflash-s3.com mobile-global-app-market1.life noireblur2.live nonamedvlp49.live reward7978.nonamedvlp49.live rododesk.pl ysmk.pclatorre.it zmusic-online.com ================================================ FILE: local.toml ================================================ listen_port = 3000 [[peers]] url = "http://localhost/" get_dns_results = false ================================================ FILE: local2.toml ================================================ listen_port = 3000 [[peers]] url = "http://localhost:3000/" get_dns_results = false [[peers]] url = "http://localhost:3000/" get_dns_results = false ================================================ FILE: migrations/20240222234126_filterlists.sql ================================================ CREATE TABLE IF NOT EXISTS filterLists ( url TEXT PRIMARY KEY NOT NULL UNIQUE, contents TEXT NOT NULL, lastUpdated INTEGER NOT NULL ); ================================================ FILE: migrations/20240223010915_lastmodified.sql ================================================ -- Add migration script here ALTER TABLE filterLists ADD COLUMN lastModified INTEGER NOT NULL DEFAULT 0; ================================================ FILE: migrations/20240223011106_lastmodified.sql ================================================ -- Add migration script here alter table filterLists drop column lastmodified; ================================================ FILE: migrations/20240223011400_etag.sql ================================================ -- Add migration script here -- Add migration script here ALTER TABLE filterLists ADD COLUMN etag TEXT; ================================================ FILE: migrations/20240224194439_rules.sql ================================================ -- Add migration script here CREATE TABLE IF NOT EXISTS Rules ( id SERIAL PRIMARY KEY, rule TEXT NOT NULL UNIQUE ); ================================================ FILE: migrations/20240224195223_filter_list_contents.sql ================================================ -- Add migration script here ALTER TABLE filterLists RENAME TO OldFilterListContents; CREATE TABLE IF NOT EXISTS filterLists ( id SERIAL PRIMARY KEY, url TEXT NOT NULL UNIQUE, format TEXT NOT NULL, contents TEXT NOT NULL, lastUpdated INTEGER NOT NULL, etag TEXT ); INSERT INTO filterLists (url, format, contents, lastUpdated, etag) SELECT url, '', contents, lastUpdated, etag FROM filterLists; DROP TABLE OldFilterListContents; ================================================ FILE: migrations/20240224203224_list_rules.sql ================================================ -- Add migration script here CREATE table list_rules ( id SERIAL PRIMARY KEY, list_id INTEGER NOT NULL, rule_id INTEGER NOT NULL, FOREIGN KEY (list_id) REFERENCES filterLists(id), FOREIGN KEY (rule_id) REFERENCES rules(id) ); ================================================ FILE: migrations/20240225214254_list_source.sql ================================================ -- Add migration script here ALTER TABLE list_rules ADD COLUMN source TEXT; ================================================ FILE: migrations/20240225230839_index.sql ================================================ -- Add migration script here create unique index rule_index on Rules(rule); ================================================ FILE: migrations/20240225231841_index.sql ================================================ -- Add migration script here create index list_rules_index on list_rules(list_id, rule_id); ================================================ FILE: migrations/20240225232249_index.sql ================================================ -- Add migration script here drop index list_rules_index; create index list_rules_index_list_id on list_rules(list_id); create index list_rules_index_rule_id on list_rules(rule_id); ================================================ FILE: migrations/20240226004619_change_date.sql ================================================ -- Add migration script here alter table filterLists alter column lastUpdated type timestamp with time zone using to_timestamp(lastUpdated); ================================================ FILE: migrations/20240226013547_source.sql ================================================ -- Add migration script here CREATE TABLE rule_source ( id SERIAL PRIMARY KEY, source TEXT NOT NULL UNIQUE ); CREATE INDEX idx_rule_source_source ON rule_source (source); ALTER TABLE list_rules DROP COLUMN source; ALTER TABLE list_rules ADD COLUMN source_id INTEGER REFERENCES rule_source(id); ================================================ FILE: migrations/20240226152939_temp.sql ================================================ -- Add migration script here CREATE TABLE temp_rule_source ( idx SERIAL PRIMARY KEY, rule TEXT NOT NULL, source TEXT NOT NULL ); ================================================ FILE: migrations/20240226215547_remove_column.sql ================================================ -- Add migration script here ALTER TABLE temp_rule_source DROP COLUMN idx; ================================================ FILE: migrations/20240226215929_remove_id.sql ================================================ -- Add migration script here ALTER TABLE list_rules DROP COLUMN id; ================================================ FILE: migrations/20240226220711_remove_fkey.sql ================================================ -- Add migration script here ALTER TABLE list_rules DROP CONSTRAINT list_rules_list_id_fkey; ALTER TABLE list_rules DROP CONSTRAINT list_rules_rule_id_fkey; ALTER TABLE list_rules DROP CONSTRAINT list_rules_source_id_fkey; ================================================ FILE: migrations/20240226223817_domain_rules.sql ================================================ -- Add migration script here CREATE TABLE domain_rules ( id SERIAL PRIMARY KEY, domain TEXT NOT NULL ); CREATE INDEX domain_rules_idx_domain ON domain_rules (domain); ================================================ FILE: migrations/20240226230317_domain_block.sql ================================================ -- Add migration script here ALTER TABLE domain_rules ADD COLUMN block BOOLEAN NOT NULL; ================================================ FILE: migrations/20240226230425_rename.sql ================================================ -- Add migration script here ALTER TABLE domain_rules RENAME COLUMN id TO rule_id; ================================================ FILE: migrations/20240227191530_drop_column.sql ================================================ -- Add migration script here CREATE TEMPORARY TABLE temp_rule_source(source TEXT UNIQUE, rule_id INTEGER) ON COMMIT DROP; INSERT INTO temp_rule_source (source, rule_id) SELECT rule_source.source, list_rules.rule_id FROM list_rules INNER JOIN rule_source ON list_rules.source_id = rule_source.id ON CONFLICT DO NOTHING; DELETE FROM rule_source; ALTER TABLE rule_source ADD COLUMN rule_id INTEGER NOT NULL; INSERT INTO rule_source (source, rule_id) SELECT source, rule_id FROM temp_rule_source; ================================================ FILE: migrations/20240227194830_drop_column.sql ================================================ -- Add migration script here ALTER TABLE list_rules DROP COLUMN rule_id; ================================================ FILE: migrations/20240227200629_drop_table.sql ================================================ -- Add migration script here DROP TABLE temp_rule_source; ================================================ FILE: migrations/20240227201410_primary_key.sql ================================================ -- Add migration script here DELETE FROM list_rules; ALTER TABLE list_rules ADD PRIMARY KEY (list_id, source_id); ================================================ FILE: migrations/20240228001134_domain_rules.sql ================================================ -- Add migration script here ALTER TABLE domain_rules ADD COLUMN subdomain BOOLEAN NOT NULL DEFAULT FALSE; ALTER TABLE domain_rules ALTER COLUMN subdomain DROP DEFAULT; CREATE TEMPORARY TABLE temp_domain_rules ( domain TEXT NOT NULL, allow BOOLEAN NOT NULL, subdomain BOOLEAN NOT NULL ); INSERT INTO temp_domain_rules (domain, allow, subdomain) SELECT domain, block, subdomain FROM domain_rules; DELETE FROM domain_rules; ALTER TABLE domain_rules ADD COLUMN id SERIAL NOT NULL UNIQUE; ALTER TABLE domain_rules RENAME COLUMN block TO allow; ALTER TABLE domain_rules DROP column rule_id; ALTER TABLE domain_rules ADD CONSTRAINT domain_rule_unique UNIQUE (domain, allow, subdomain); INSERT INTO domain_rules (domain, allow, subdomain) SELECT domain, NOT allow, subdomain FROM temp_domain_rules; ================================================ FILE: migrations/20240228004601_extend_rules.sql ================================================ -- Add migration script here ALTER TABLE Rules ADD COLUMN domain_rule_id INTEGER; CREATE TABLE unknown_rules ( id SERIAL PRIMARY KEY, rule TEXT NOT NULL UNIQUE ); DELETE FROM Rules; ALTER TABLE Rules ADD COLUMN unknown_rule_id INTEGER; ALTER TABLE Rules ADD CONSTRAINT unique_rules UNIQUE NULLS NOT DISTINCT (domain_rule_id, unknown_rule_id); ================================================ FILE: migrations/20240228005409_drop_rule.sql ================================================ -- Add migration script here ALTER TABLE Rules DROP COLUMN rule; ================================================ FILE: migrations/20240228015906_change_unique.sql ================================================ -- Add migration script here ALTER TABLE rule_source DROP CONSTRAINT rule_source_source_key; ALTER TABLE rule_source ADD CONSTRAINT rule_source_unique UNIQUE (source, rule_id); ================================================ FILE: migrations/20240228164553_ip.sql ================================================ -- Add migration script here CREATE TABLE ip_rules ( id SERIAL PRIMARY KEY, ip_address inet NOT NULL, allow boolean NOT NULL, CONSTRAINT ip_rules_unique UNIQUE (ip_address, allow) ); ALTER TABLE Rules ADD COLUMN ip_rule_id INTEGER; ALTER TABLE Rules DROP CONSTRAINT unique_rules; ALTER TABLE Rules ADD CONSTRAINT unique_rules UNIQUE NULLS NOT DISTINCT (domain_rule_id, ip_rule_id, unknown_rule_id); ================================================ FILE: migrations/20240228170646_ip.sql ================================================ -- Add migration script here ALTER TABLE ip_rules RENAME COLUMN ip_address TO ip_network; ================================================ FILE: migrations/20240228175807_remove_unknown.sql ================================================ -- Add migration script here DROP TABLE unknown_rules; ALTER TABLE Rules DROP COLUMN unknown_rule_id; ================================================ FILE: migrations/20240228180753_index.sql ================================================ -- Add migration script here DELETE FROM Rules; ALTER TABLE Rules ADD CONSTRAINT rules_unique UNIQUE NULLS NOT DISTINCT (domain_rule_id, ip_rule_id); ================================================ FILE: migrations/20240229210101_domains.sql ================================================ -- Add migration script here CREATE TABLE domains ( id SERIAL PRIMARY KEY, domain TEXT NOT NULL UNIQUE ); INSERT INTO domains (domain) SELECT domain FROM domain_rules ON CONFLICT DO NOTHING; ALTER TABLE domain_rules RENAME TO domain_rules_old; CREATE TABLE domain_rules ( id SERIAL PRIMARY KEY, domain_id INTEGER NOT NULL, allow BOOLEAN NOT NULL, subdomain BOOLEAN NOT NULL, CONSTRAINT domain_rules_unique UNIQUE (domain_id, allow, subdomain) ); INSERT INTO domain_rules (domain_id, allow, subdomain) SELECT domains.id, domain_rules_old.allow, domain_rules_old.subdomain FROM domain_rules_old INNER JOIN domains ON domain_rules_old.domain = domains.domain; DROP TABLE domain_rules_old; ================================================ FILE: migrations/20240229212527_subdomains.sql ================================================ -- Add migration script here ALTER TABLE domains ALTER COLUMN id TYPE bigint; ALTER TABLE domain_rules ALTER COLUMN domain_id TYPE bigint; CREATE TABLE subdomains ( domain_id bigint NOT NULL UNIQUE, parent_domain_id bigint ); ================================================ FILE: migrations/20240301000455_more_indexes.sql ================================================ -- Add migration script here CREATE INDEX rule_source_rule_idx ON rule_source (rule_id); CREATE INDEX domain_rules_domain_idx ON domain_rules (domain_id); ================================================ FILE: migrations/20240301000900_subdomain_idx.sql ================================================ -- Add migration script here CREATE INDEX subdomain_domain_idx ON subdomains (domain_id); CREATE INDEX subdomain_parent_idx ON subdomains (parent_domain_id); ================================================ FILE: migrations/20240301030213_subdomain_inde.sql ================================================ -- Add migration script here CREATE INDEX domain_rules_subdomain_idx ON domain_rules (subdomain); ================================================ FILE: migrations/20240302171950_expanded_subdomains.sql ================================================ -- Add migration script here DELETE FROM subdomains WHERE parent_domain_id IS NOT NULL; ALTER TABLE subdomains DROP CONSTRAINT subdomains_domain_id_key; ALTER TABLE subdomains ADD CONSTRAINT subdomains_domain_id_key UNIQUE NULLS NOT DISTINCT (domain_id, parent_domain_id); ================================================ FILE: migrations/20240302184040_processed_subdomains.sql ================================================ DELETE FROM domains; DELETE FROM subdomains; DELETE FROM domain_rules; DELETE FROM Rules; DELETE FROM rule_source; ALTER TABLE domains ADD COLUMN processed_subdomains boolean DEFAULT false; ================================================ FILE: migrations/20240302192658_index.sql ================================================ -- Add migration script here CREATE INDEX domains_processed_subdomains_idx ON domains(processed_subdomains); ================================================ FILE: migrations/20240302194037_domain_rule_id_idx.sql ================================================ -- Add migration script here CREATE INDEX rules_domain_rule_id_idx ON rules(domain_rule_id); ================================================ FILE: migrations/20240302194733_not_null_parent.sql ================================================ -- Add migration script here DELETE FROM subdomains WHERE parent_domain_id IS NULL; ALTER TABLE subdomains ALTER COLUMN parent_domain_id SET NOT NULL; ================================================ FILE: migrations/20240302205633_not_null#.sql ================================================ -- Add migration script here ALTER TABLE domains ALTER COLUMN processed_subdomains SET NOT NULL; ================================================ FILE: migrations/20240302222401_dns.sql ================================================ -- Add migration script here ALTER TABLE domains ADD COLUMN last_checked_dns TIMESTAMP WITH TIME ZONE; CREATE INDEX domains_last_checked_dns_idx ON domains(last_checked_dns); CREATE TABLE dns_ips ( domain_id BIGINT NOT NULL, ip_address INET NOT NULL ); CREATE INDEX dns_ips_domain_id_idx ON dns_ips(domain_id); CREATE INDEX dns_ips_ip_address_idx ON dns_ips(ip_address); CREATE TABLE dns_cnames ( domain_id BIGINT NOT NULL, cname_domain_id BIGINT NOT NULL ); CREATE INDEX dns_cnames_domain_id_idx ON dns_cnames(domain_id); CREATE INDEX dns_cnames_cname_domain_id_idx ON dns_cnames(cname_domain_id); ================================================ FILE: migrations/20240304235746_filterlist.sql ================================================ -- Add migration script here ALTER TABLE filterlists ADD COLUMN name TEXT; ALTER TABLE filterlists ADD COLUMN author TEXT; ALTER TABLE filterlists ADD COLUMN expires INTEGER; ALTER TABLE filterlists ADD COLUMN license TEXT; ================================================ FILE: migrations/20240305000257_filterlist.sql ================================================ -- Add migration script here ALTER TABLE filterlists ALTER COLUMN contents DROP NOT NULL; ================================================ FILE: migrations/20240305000612_filterlist.sql ================================================ -- Add migration script here ALTER TABLE filterlists ALTER COLUMN lastUpdated DROP NOT NULL; ================================================ FILE: migrations/20240305003411_filterlist.sql ================================================ -- Add migration script here ALTER TABLE filterlists ALTER COLUMN expires SET NOT NULL; ================================================ FILE: migrations/20240305200551_rule_matches.sql ================================================ -- Add migration script here CREATE TABLE rule_matches ( rule_id INTEGER NOT NULL, domain_id BIGINT NOT NULL ); ALTER TABLE rule_matches ADD CONSTRAINT rule_matches_pk UNIQUE (rule_id, domain_id); CREATE INDEX rule_matches_rule_id_idx ON rule_matches (rule_id); CREATE INDEX rule_matches_domain_id_idx ON rule_matches (domain_id); ALTER TABLE Rules ADD COLUMN last_checked_matches TIMESTAMP; ================================================ FILE: migrations/20240306123603_rule_count.sql ================================================ -- Add migration script here ALTER TABLE filterLists ADD COLUMN rule_count INT NOT NULL DEFAULT 0; ================================================ FILE: migrations/20240306214217_index.sql ================================================ -- Add migration script here CREATE INDEX rules_last_checked_matches_idx ON rules (last_checked_matches); ================================================ FILE: migrations/20240307005702_lists.sql ================================================ -- Add migration script here CREATE TABLE allow_domains ( domain_id BIGINT UNIQUE NOT NULL ); CREATE TABLE block_domains ( domain_id BIGINT UNIQUE NOT NULL ); ================================================ FILE: migrations/20240307012450_index.sql ================================================ -- Add migration script here CREATE INDEX domain_rules_allow_idx ON domain_rules (allow); CREATE INDEX ip_rules_allow_idx ON ip_rules (allow); ================================================ FILE: migrations/20240307031445_idx.sql ================================================ -- Add migration script here CREATE INDEX ip_rules_network_idx ON ip_rules (ip_network); ================================================ FILE: output/adblock.txt ================================================ [File too large to display: 44.7 MB] ================================================ FILE: output/allowed_ips.txt ================================================ ================================================ FILE: output/domains.rpz ================================================ ================================================ FILE: output/domains.txt ================================================ [File too large to display: 39.7 MB] ================================================ FILE: output/hosts.txt ================================================ ================================================ FILE: output/ip_blocklist.txt ================================================ ================================================ FILE: output/whitelist_adblock.txt ================================================ ================================================ FILE: output/whitelist_domains.txt ================================================ ================================================ FILE: package.json ================================================ { "devDependencies": { "@tailwindcss/typography": "^0.5.10", "daisyui": "^4.7.2", "tailwindcss": "^3.4.1" } } ================================================ FILE: rust-toolchain.toml ================================================ [toolchain] channel = "stable" ================================================ FILE: rustfmt.toml ================================================ edition = "2021" ================================================ FILE: src/app.rs ================================================ use crate::{ domain::DomainViewPage, error_template::{AppError, ErrorTemplate}, filterlist::FilterListPage, home_page::HomePage, ip_view::IpView, rule::RuleViewPage, stats_view::StatsView, }; use leptos::*; use leptos_meta::*; use leptos_router::*; #[component] pub fn App() -> impl IntoView { // Provides context that manages stylesheets, titles, meta tags, etc. provide_meta_context(); view! { // content for this welcome page <Router fallback=|| { let mut outside_errors = Errors::default(); outside_errors.insert_with_default_key(AppError::NotFound); view! { <ErrorTemplate outside_errors/> }.into_view() }> <header class="p-4 text-white bg-indigo-600"> <nav class="container flex items-center justify-between mx-auto"> <A href="/" class="text-lg font-bold"> Home </A> <div class="space-x-4"> <A href="/tasks" class="hover:text-indigo-300"> Tasks </A> <A href="/stats" class="hover:text-indigo-300"> Stats </A> <A href="/login" class="px-4 py-2 text-indigo-600 bg-white rounded hover:bg-indigo-200" > Login </A> </div> </nav> </header> <main> <Routes> <Route path="" view=HomePage ssr=SsrMode::Async/> <Route path="tasks" view=crate::tasks::TaskView ssr=SsrMode::Async/> <Route path="stats" view=StatsView ssr=SsrMode::InOrder/> <Route path="list" view=FilterListPage ssr=SsrMode::InOrder/> <Route path="rule/:id" view=RuleViewPage ssr=SsrMode::InOrder/> <Route path="domain/:domain" view=DomainViewPage ssr=SsrMode::InOrder/> <Route path="ip/:ip" view=IpView ssr=SsrMode::InOrder/> </Routes> </main> </Router> } } #[component] pub fn Loading() -> impl IntoView { view! { <span class="loading loading-spinner loading-sm"></span> } } ================================================ FILE: src/domain.rs ================================================ #[cfg(feature = "ssr")] use crate::rule::RuleData; use crate::{ app::Loading, filterlist::{FilterListLink, FilterListUrl}, rule::DisplayRule, rule::RuleId, source::SourceId, }; use leptos::*; use leptos_router::*; #[cfg(feature = "ssr")] pub use lookup_dns_task::DomainResolver; use serde::{Deserialize, Serialize}; use std::{collections::BTreeSet, net::IpAddr, str::FromStr, sync::Arc}; #[cfg_attr(feature = "ssr", derive(sqlx::Encode, sqlx::Decode))] #[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct DomainId(i64); #[derive(Debug, Clone, thiserror::Error)] pub enum DomainParseError { Addr, HickoryProto, Custom, } impl std::fmt::Display for DomainParseError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Invalid domain") } } impl<'a> From<addr::error::Error<'a>> for DomainParseError { fn from(_: addr::error::Error) -> Self { DomainParseError::Addr } } impl From<hickory_proto::error::ProtoError> for DomainParseError { fn from(_: hickory_proto::error::ProtoError) -> Self { DomainParseError::HickoryProto } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(transparent)] pub struct Domain(Arc<str>); #[cfg(feature = "ssr")] impl sqlx::Type<sqlx::Postgres> for Domain { fn type_info() -> sqlx::postgres::PgTypeInfo { <&str as sqlx::Type<sqlx::Postgres>>::type_info() } fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool { <&str as sqlx::Type<sqlx::Postgres>>::compatible(ty) } } #[cfg(feature = "ssr")] impl sqlx::postgres::PgHasArrayType for Domain { fn array_type_info() -> sqlx::postgres::PgTypeInfo { <&str as sqlx::postgres::PgHasArrayType>::array_type_info() } fn array_compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool { <&str as sqlx::postgres::PgHasArrayType>::array_compatible(ty) } } #[cfg(feature = "ssr")] impl sqlx::Encode<'_, sqlx::Postgres> for Domain { fn encode_by_ref(&self, buf: &mut sqlx::postgres::PgArgumentBuffer) -> sqlx::encode::IsNull { <&str as sqlx::Encode<sqlx::Postgres>>::encode(self.as_ref(), buf) } } impl AsRef<str> for Domain { fn as_ref(&self) -> &str { &self.0 } } impl FromStr for Domain { type Err = DomainParseError; fn from_str(domain: &str) -> Result<Domain, Self::Err> { if domain.len() > 253 { return Err(DomainParseError::Custom); } let mut domain: Arc<str> = domain.into(); Arc::get_mut(&mut domain).unwrap().make_ascii_lowercase(); if domain.starts_with('*') || domain.ends_with('.') { return Err(DomainParseError::Custom); } if !addr::parse_dns_name(&domain)?.has_known_suffix() { return Err(DomainParseError::Addr); } let name = hickory_proto::rr::Name::from_str_relaxed(&domain)?; if name.num_labels() < 2 { return Err(DomainParseError::Custom); } if domain.contains('/') { log::warn!("Invalid domain: {:?}", domain); return Err(DomainParseError::Custom); } Ok(Domain(domain)) } } #[cfg(test)] mod tests { use crate::domain::Domain; #[test] fn valid_domain() { for domain in [ "amazonaws.com", "s3-website.us-east-1.amazonaws.com", "origin-mobile_mob.conduit.com", ] { let domain: Result<Domain, _> = domain.parse(); assert!(domain.is_ok()); } } #[test] fn invalid_domain() { for domain_str in [ "com", "@.amazonaws.com", "1234", "example.com,google.com", "example.com.", ] { let domain: Result<Domain, _> = domain_str.parse(); assert!(domain.is_err(), "{}", domain_str); } } #[test] fn makes_lowercase() { let domain: Domain = "EXAMPLE.COM".parse().unwrap(); assert_eq!(domain.as_ref(), "example.com"); } } #[cfg(feature = "ssr")] mod lookup_dns_task { use std::{collections::HashSet, sync::Mutex, time::Duration}; use super::*; use hickory_resolver::error::ResolveError; use tokio_util::sync::CancellationToken; fn parse_lookup_result( result: Result<hickory_resolver::lookup_ip::LookupIp, ResolveError>, ) -> Result<(Vec<ipnetwork::IpNetwork>, Vec<Domain>), ResolveError> { match result { Ok(result) => { let mut ips: Vec<ipnetwork::IpNetwork> = Vec::new(); let mut cnames = Vec::new(); let lookup = result.as_lookup(); for record in lookup.iter() { if let Some(a) = record.as_a() { let ip: IpAddr = a.0.into(); ips.push(ip.into()); } else if let Some(aaaa) = record.as_aaaa() { let ip: IpAddr = aaaa.0.into(); ips.push(ip.into()); } else if let Some(cname) = record.as_cname() { let mut cname = cname.0.to_ascii(); if cname.ends_with('.') { cname.pop(); } if let Ok(cname) = cname.parse() { cnames.push(cname); } else { log::warn!("Invalid CNAME {}", cname); } } else { log::info!("Unknown record type {:?}", record.record_type()); } } Ok((ips, cnames)) } Err(err) => match err.kind() { hickory_resolver::error::ResolveErrorKind::NoRecordsFound { query: _, soa: _, negative_ttl: _, response_code: _, trusted: _, } => Ok((vec![], vec![])), hickory_resolver::error::ResolveErrorKind::Proto(_) => Ok((vec![], vec![])), hickory_resolver::error::ResolveErrorKind::Timeout => Err(err), _ => Err(err), }, } } #[cfg(feature = "ssr")] type Resolver = Arc< hickory_resolver::AsyncResolver< hickory_resolver::name_server::GenericConnector< hickory_resolver::name_server::TokioRuntimeProvider, >, >, >; #[cfg(feature = "ssr")] type Task = (DomainId, Domain); #[cfg(feature = "ssr")] #[derive(Clone)] pub struct DomainResolver { resolvers: Vec<(Arc<str>, Resolver)>, tx: async_channel::Sender<Task>, rx: async_channel::Receiver<Task>, read_limit: i64, failed_cache_size: usize, failed_domains: Arc<Mutex<Vec<i64>>>, written_domains: Arc<Mutex<Vec<i64>>>, bad_domains: Arc<Mutex<Vec<i64>>>, looked_up_domains: Arc<Mutex<Vec<i64>>>, dns_ips: Arc<Mutex<(Vec<i64>, Vec<ipnetwork::IpNetwork>)>>, dns_cnames: Arc<Mutex<(Vec<i64>, Vec<Domain>)>>, token: CancellationToken, } #[cfg(feature = "ssr")] impl DomainResolver { pub fn new(token: CancellationToken) -> Result<Self, ServerFnError> { let _ = dotenvy::dotenv()?; let servers_str = std::env::var("DNS_SERVERS")?; let read_limit = std::env::var("READ_LIMIT")?.parse::<u32>()? as i64; let mut resolvers = Vec::new(); for server in servers_str.split(',') { let server: Arc<str> = server.into(); let (addr, port) = server .split_once(':') .ok_or_else(|| ServerFnError::new("Bad DNS_SERVER env"))?; let server_conf = hickory_resolver::config::NameServerConfigGroup::from_ips_clear( &[addr.parse()?], port.parse()?, true, ); let config = hickory_resolver::config::ResolverConfig::from_parts(None, vec![], server_conf); let mut opts = hickory_resolver::config::ResolverOpts::default(); opts.ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6; opts.cache_size = 32; opts.attempts = 3; opts.timeout = std::time::Duration::from_secs_f32(5.0); let resolver = Arc::new(hickory_resolver::AsyncResolver::tokio(config, opts)); resolvers.push((server, resolver)); } if resolvers.is_empty() { return Err(ServerFnError::new("Empty DNS server list")); } let (tx, rx) = async_channel::bounded(read_limit as usize); let bad_domains = Arc::new(Mutex::new(Vec::new())); let failed_cache_size = std::env::var("FAILED_CACHE_SIZE")?.parse()?; Ok(Self { resolvers, bad_domains, tx, rx, read_limit, failed_cache_size, failed_domains: Arc::new(Mutex::new(Vec::new())), written_domains: Arc::new(Mutex::new(Vec::new())), looked_up_domains: Arc::new(Mutex::new(Vec::new())), dns_ips: Default::default(), dns_cnames: Default::default(), token, }) } pub async fn run(&self) -> Result<(), ServerFnError> { dotenvy::dotenv()?; let concurrent_lookups: usize = std::env::var("CONCURRENT_LOOKUPS")?.parse()?; let mut tasks = tokio::task::JoinSet::new(); for resolver in &self.resolvers { for _ in 0..concurrent_lookups { let resolver_str = resolver.0.clone(); let resolver = resolver.1.clone(); let resolver_self = self.clone(); let task = async move { let token = resolver_self.token.clone(); tokio::select! { _ = token.cancelled() => { log::info!("Shutting down DNS resolver"); Ok(())}, res = resolver_self.run_task(resolver_str, resolver) => res} }; tasks.spawn(task); } } let selector = self.clone(); tasks.spawn(async move { let token = selector.token.clone(); tokio::select! { _ = token.cancelled() => { log::info!("Shutting down DNS selector"); Ok(())} res = selector.domain_selector() => res } }); let writer = self.clone(); tasks.spawn(async move { writer.write_to_db().await }); while let Some(result) = tasks.join_next().await { let _ = result?; } Ok(()) } async fn domain_selector(&self) -> Result<(), ServerFnError> { let pool = crate::server::get_db().await?; let mut started_domains = HashSet::new(); let mut failed_domains = std::collections::VecDeque::<i64>::new(); loop { { failed_domains.extend(std::mem::take(&mut *self.failed_domains.lock()?)); } while failed_domains.len() > self.failed_cache_size { if let Some(failed) = failed_domains.pop_front() { started_domains.remove(&failed); } } let written_domains = std::mem::take(&mut *self.written_domains.lock()?); for domain_id in written_domains { started_domains.remove(&domain_id); } let limit = self.read_limit + started_domains.len() as i64; let records = sqlx::query!( "SELECT id, domain FROM Domains WHERE last_checked_dns IS NULL ORDER BY id DESC NULLS FIRST LIMIT $1", limit ) .fetch_all(&pool) .await?; let recheck_domains = if records.len() < limit as usize { sqlx::query!( "SELECT id, domain FROM Domains ORDER BY last_checked_dns ASC NULLS FIRST LIMIT $1", limit ) .fetch_all(&pool) .await? } else { vec![] }; let records = records.into_iter().map(|record| (record.id, record.domain)); let recheck_domains = recheck_domains .into_iter() .map(|record| (record.id, record.domain)); let mut has_domains = false; for (domain_id, domain_str) in records.chain(recheck_domains) { has_domains = true; if !started_domains.insert(domain_id) { continue; } if let Ok(domain) = domain_str.parse::<Domain>() { if domain_str == domain.as_ref() { self.tx.send((DomainId(domain_id), domain)).await?; continue; } } log::warn!("Invalid domain: {}", domain_str); self.bad_domains.lock()?.push(domain_id); } if !has_domains { log::info!("No domains to check, sleeping"); tokio::time::sleep(Duration::from_secs(30)).await; } } } async fn run_task( &self, resolver_str: Arc<str>, resolver: Resolver, ) -> Result<(), ServerFnError> { while let Ok(task) = self.rx.recv().await { let (domain_id, domain) = task; let mut domain_str = domain.as_ref().to_string(); domain_str.push('.'); let result = resolver.lookup_ip(&domain_str).await; let result = parse_lookup_result(result); match result { Ok((ips, cnames)) => { self.looked_up_domains.lock()?.push(domain_id.0); { let mut dns_ips = self.dns_ips.lock()?; for ip in ips { dns_ips.0.push(domain_id.0); dns_ips.1.push(ip); } } { let mut dns_cnames = self.dns_cnames.lock()?; for cname in cnames { dns_cnames.0.push(domain_id.0); dns_cnames.1.push(cname); } } } Err(err) => { log::warn!( "Server: {} Error looking up domain {}: {}", resolver_str, domain.as_ref(), err ); self.failed_domains.lock()?.push(domain_id.0); } } } Ok(()) } async fn write_to_db(&self) -> Result<(), ServerFnError> { let pool = crate::server::get_db().await?; let write_frequency: u64 = std::env::var("WRITE_FREQUENCY")?.parse()?; let mut interval = tokio::time::interval(Duration::from_secs(write_frequency)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); interval.tick().await; loop { tokio::select! { _ = interval.tick() => {}, _ = self.token.cancelled() => { log::info!("Shutting down DNS writer") } } let looked_up_domains = std::mem::take(&mut *self.looked_up_domains.lock()?); let looked_up_domains_deduped = looked_up_domains.iter().cloned().collect::<HashSet<_>>(); assert_eq!(looked_up_domains.len(), looked_up_domains_deduped.len()); let dns_ips = std::mem::take(&mut *self.dns_ips.lock()?); let dns_ips_domain_ids = &dns_ips.0; let dns_ips_ips = &dns_ips.1; let dns_cnames = std::mem::take(&mut *self.dns_cnames.lock()?); let dns_cnames_domain_ids = &dns_cnames.0; let dns_cnames_cname = &dns_cnames.1; let bad_domains = std::mem::take(&mut *self.bad_domains.lock()?); let total_cnames = dns_cnames_cname .iter() .collect::<HashSet<_>>() .into_iter() .cloned() .collect::<Vec<Domain>>(); let new_domains_from_cnames = sqlx::query!( "INSERT INTO domains(domain) SELECT domain FROM UNNEST($1::text[]) as t(domain) ON CONFLICT DO NOTHING", &total_cnames[..] as _ ) .execute(&pool) .await? .rows_affected(); let mut tx = pool.begin().await?; sqlx::query!( "DELETE FROM dns_ips WHERE domain_id = ANY($1::bigint[])", &looked_up_domains[..] ) .execute(&mut *tx) .await?; sqlx::query!( "DELETE FROM dns_cnames WHERE domain_id = ANY($1::bigint[])", &looked_up_domains[..] ) .execute(&mut *tx) .await?; sqlx::query!( "INSERT INTO dns_ips(domain_id, ip_address) SELECT domain_id, ip FROM UNNEST($1::bigint[], $2::inet[]) as t(domain_id, ip)", &dns_ips_domain_ids[..], &dns_ips_ips[..] ) .execute(&mut *tx) .await?; sqlx::query!( "INSERT INTO dns_cnames(domain_id, cname_domain_id) SELECT domain_id, cname_domains.id FROM UNNEST($1::bigint[], $2::text[]) as t(domain_id, cname) INNER JOIN domains AS cname_domains ON cname_domains.domain = t.cname ", &dns_cnames_domain_ids[..], &dns_cnames_cname[..] as _ ) .execute(&mut *tx) .await?; let updated_domains = sqlx::query!( "UPDATE domains SET last_checked_dns = now() WHERE id = ANY($1::bigint[])", &looked_up_domains[..] ) .execute(&mut *tx) .await? .rows_affected(); assert_eq!(updated_domains, looked_up_domains.len() as u64); tx.commit().await?; self.written_domains.lock()?.extend(&looked_up_domains); if !bad_domains.is_empty() { log::info!("Removing {} bad domains", bad_domains.len()); sqlx::query!( "DELETE FROM domains WHERE id = ANY($1::bigint[])", &bad_domains[..] ) .execute(&pool) .await?; } self.written_domains.lock()?.extend(&bad_domains); log::info!( "Looked up {} domains, got {} ips, {} cnames ({} new)", looked_up_domains.len(), dns_ips_domain_ids.len(), dns_cnames_domain_ids.len(), new_domains_from_cnames ); if self.token.is_cancelled() { return Ok(()); } } } } } #[server] async fn get_dns_result( domain: Domain, ) -> Result<(BTreeSet<IpAddr>, BTreeSet<(DomainId, String)>), ServerFnError> { let records = sqlx::query!( r#"SELECT dns_ips.ip_address as "ip_address: Option<ipnetwork::IpNetwork>", cname_domains.id as "cname_domain_id: Option<i64>", cname_domains.domain as "cname_domain: Option<String>" FROM domains LEFT JOIN dns_ips ON dns_ips.domain_id=domains.id LEFT JOIN dns_cnames on dns_cnames.domain_id=domains.id LEFT JOIN domains AS cname_domains ON cname_domains.id=dns_cnames.cname_domain_id WHERE domains.domain = $1 "#r, domain.as_ref().to_string() ) .fetch_all(&crate::server::get_db().await?) .await?; let mut ip_addresses = BTreeSet::new(); let mut cnames = BTreeSet::new(); for record in records { if let Some(ip) = record.ip_address { ip_addresses.insert(ip.ip()); } if let (Some(id), Some(cname)) = (record.cname_domain_id, record.cname_domain) { cnames.insert((DomainId(id), cname)); } } Ok((ip_addresses, cnames)) } #[component] fn DnsResultView(domain: Domain) -> impl IntoView { view! { <Await future=move || { let domain = domain.clone(); async move { get_dns_result(domain.clone()).await } } let:dns_results > { let dns_results = dns_results.clone(); move || match dns_results.clone() { Ok((ips, cnames)) => { view! { <div class="grid grid-cols-2 gap-4"> <div> <h2 class="mb-2 text-lg font-bold">IP Addresses</h2> <ul class="grid grid-cols-2"> <For each=move || { ips.clone() } key=|ip| *ip children=|ip| { let href = format!("/ip/{ip}"); view! { <li> <A href=href class="link link-neutral"> {ip.to_string()} </A> </li> } } /> </ul> </div> <div> <h2 class="mb-2 text-lg font-bold">CNAMEs</h2> <ul> <For each=move || { cnames.clone() } key=|(id, _cname)| *id children=|(_id, cname)| { let href = format!("/domain/{cname}"); view! { <li> <A href=href class="link link-neutral"> {cname} </A> </li> } } /> </ul> </div> </div> } .into_view() } _ => view! { <p>"Error"</p> }.into_view(), } } </Await> } } #[server] async fn get_blocked_by( domain: String, ) -> Result<Vec<(FilterListUrl, RuleId, SourceId, crate::filterlist::RulePair)>, ServerFnError> { let records = sqlx::query!( r#" SELECT Rules.id as "rule_id: RuleId", domain_rules_domain.domain as "domain: Option<String>", domain_rules.allow as "domain_allow: Option<bool>", subdomain as "subdomain: Option<bool>", ip_rules.ip_network as "ip_network: Option<ipnetwork::IpNetwork>", ip_rules.allow as "ip_allow: Option<bool>", source_id AS "source_id: SourceId", source, url FROM domains INNER JOIN rule_matches ON domains.id = rule_matches.domain_id INNER JOIN Rules on Rules.id = rule_matches.rule_id INNER JOIN rule_source ON rules.id = rule_source.rule_id INNER JOIN list_rules ON rule_source.id = list_rules.source_id INNER JOIN filterLists ON list_rules.list_id = filterLists.id LEFT JOIN domain_rules ON rules.domain_rule_id = domain_rules.id LEFT JOIN domains AS domain_rules_domain ON domain_rules_domain.id = domain_rules.domain_id LEFT JOIN ip_rules ON rules.ip_rule_id = ip_rules.id WHERE domains.domain = $1 ORDER BY url LIMIT 100 "#r, domain ) .fetch_all(&crate::server::get_db().await?) .await?; let rules = records .into_iter() .map(|record| { let rule_data = RuleData { rule_id: record.rule_id, domain: record.domain.clone(), domain_allow: record.domain_allow, domain_subdomain: record.subdomain, ip_network: record.ip_network, ip_allow: record.ip_allow, }; let rule = rule_data.try_into()?; let source = record.source.clone(); let pair = crate::filterlist::RulePair::new(source.into(), rule); let url = record.url.clone(); Ok((url.parse()?, record.rule_id, record.source_id, pair)) }) .collect::<Result<Vec<_>, ServerFnError>>()?; Ok(rules) } #[component] fn BlockedBy(get_domain: Box<dyn Fn() -> Result<String, ParamsError>>) -> impl IntoView { let blocked_by = create_resource(get_domain, |domain| async move { let rules = get_blocked_by(domain?).await?; Ok::<_, ServerFnError>(rules) }); view! { <Transition fallback=move || { view! { <p>"Loading" <Loading/></p> } }> {move || match blocked_by.get() { Some(Ok(rules)) => { view! { <table class="table table-zebra"> <For each=move || { rules.clone() } key=|(_url, rule_id, source_id, _pair)| (*rule_id, *source_id) children=|(url, rule_id, _source_id, pair)| { let source = pair.get_source().to_string(); let rule = pair.get_rule().clone(); view! { <tr> <td> <FilterListLink url=url/> </td> <td>{source}</td> <td> <A href=rule_id.get_href() class="link link-neutral"> <DisplayRule rule=rule/> </A> </td> </tr> } } /> </table> } .into_view() } _ => view! { <p>"Error"</p> }.into_view(), }} </Transition> } } #[server] async fn get_subdomains(domain: String) -> Result<Vec<String>, ServerFnError> { let records = sqlx::query!( "SELECT subdomain_text.domain FROM domains INNER JOIN subdomains ON domains.id = subdomains.parent_domain_id INNER JOIN domains AS subdomain_text ON subdomains.domain_id = subdomain_text.id WHERE domains.domain = $1 ", domain ) .fetch_all(&crate::server::get_db().await?) .await?; let subdomains = records.into_iter().map(|record| record.domain).collect(); Ok(subdomains) } #[component] fn DisplaySubdomains(get_domain: Box<dyn Fn() -> Result<String, ParamsError>>) -> impl IntoView { let subdomains = create_resource(get_domain, |domain| async move { let subdomains = get_subdomains(domain?).await?; Ok::<_, ServerFnError>(subdomains) }); view! { <Transition fallback=move || { view! { <p>"Loading" <Loading/></p> } }> {move || match subdomains.get() { Some(Ok(subdomains)) => { view! { <table class="table table-zebra"> <For each=move || { subdomains.clone() } key=std::clone::Clone::clone children=|subdomain| { let domain_href = format!("/domain/{subdomain}"); view! { <tr> <td> <A href=domain_href class="link link-neutral"> {subdomain} </A> </td> </tr> } } /> </table> } .into_view() } _ => view! { <p>"Error"</p> }.into_view(), }} </Transition> } } #[derive(Params, PartialEq)] struct DomainParam { domain: Option<String>, } #[component] pub fn DomainViewPage() -> impl IntoView { let params = use_params::<DomainParam>(); let get_domain = move || { params.with(|param| { param.as_ref().map_err(Clone::clone).and_then(|param| { param .domain .clone() .ok_or_else(|| ParamsError::MissingParam("No domain".into())) }) }) }; let get_domain_parsed = move || { params.with(|param| { Ok::<_, ServerFnError>( param .as_ref()? .domain .as_ref() .ok_or_else(|| ParamsError::MissingParam("No domain".into()))? .parse::<Domain>()?, ) }) }; view! { <div> {move || { let domain = get_domain_parsed(); match domain { Ok(domain) => { view! { <h1 class="text-3xl">"Domain: " {domain.as_ref().to_string()}</h1> <DnsResultView domain=domain.clone()/> <p>"Filtered by"</p> <BlockedBy get_domain=Box::new(get_domain)/> <p>"Subdomains"</p> <DisplaySubdomains get_domain=Box::new(get_domain)/> } .into_view() } Err(err) => view! { <p>"Error: " {format!("{err:?}")}</p> }.into_view(), } }} </div> } } ================================================ FILE: src/error_template.rs ================================================ use http::status::StatusCode; use leptos::*; use thiserror::Error; #[derive(Clone, Debug, Error)] pub enum AppError { #[error("Not Found")] NotFound, } impl AppError { pub fn status_code(&self) -> StatusCode { match self { AppError::NotFound => StatusCode::NOT_FOUND, } } } // A basic function to display errors served by the error boundaries. // Feel free to do more complicated things here than just displaying the error. #[component] pub fn ErrorTemplate( #[prop(optional)] outside_errors: Option<Errors>, #[prop(optional)] errors: Option<RwSignal<Errors>>, ) -> impl IntoView { let errors = match outside_errors { Some(e) => create_rw_signal(e), None => match errors { Some(e) => e, None => panic!("No Errors found and we expected errors!"), }, }; // Get Errors from Signal let errors = errors.get_untracked(); // Downcast lets us take a type that implements `std::error::Error` let errors: Vec<AppError> = errors .into_iter() .filter_map(|(_k, v)| v.downcast_ref::<AppError>().cloned()) .collect(); println!("Errors: {errors:#?}"); // Only the response code for the first error is actually sent from the server // this may be customized by the specific application #[cfg(feature = "ssr")] { use leptos_axum::ResponseOptions; let response = use_context::<ResponseOptions>(); if let Some(response) = response { response.set_status(errors[0].status_code()); } } view! { <h1>{if errors.len() > 1 { "Errors" } else { "Error" }}</h1> <For // a function that returns the items we're iterating over; a signal is fine each=move || { errors.clone().into_iter().enumerate() } // a unique key for each item as a reference key=|(index, _error)| *index // renders each item to a view children=move |error| { let error_string = error.1.to_string(); let error_code = error.1.status_code(); view! { <h2>{error_code.to_string()}</h2> <p>"Error: " {error_string}</p> } } /> } } ================================================ FILE: src/fileserv.rs ================================================ use crate::app::App; use axum::response::Response as AxumResponse; use axum::{ body::Body, extract::State, http::{Request, Response, StatusCode, Uri}, response::IntoResponse, }; use leptos::*; use tower::ServiceExt; use tower_http::services::ServeDir; pub async fn file_and_error_handler( uri: Uri, State(options): State<LeptosOptions>, req: Request<Body>, ) -> AxumResponse { let root = options.site_root.clone(); let res = get_static_file(uri.clone(), &root).await.unwrap(); if res.status() == StatusCode::OK { res.into_response() } else { let handler = leptos_axum::render_app_to_stream(options.clone(), App); handler(req).await.into_response() } } async fn get_static_file(uri: Uri, root: &str) -> Result<Response<Body>, (StatusCode, String)> { let req = Request::builder() .uri(uri.clone()) .body(Body::empty()) .unwrap(); // `ServeDir` implements `tower::Service` so we can call it with `tower::ServiceExt::oneshot` // This path is relative to the cargo root match ServeDir::new(root).oneshot(req).await { Ok(res) => Ok(res.into_response()), Err(err) => Err(( StatusCode::INTERNAL_SERVER_ERROR, format!("Something went wrong: {err}"), )), } } ================================================ FILE: src/filterlist.rs ================================================ pub use crate::domain::Domain; use crate::PAGE_SIZE; use crate::{rule::RuleId, source::SourceId}; use leptos::*; use leptos::{server, ServerFnError}; use crate::rule::DisplayRule; #[cfg(feature = "ssr")] use crate::rule::RuleData; use leptos_router::*; use serde::{Deserialize, Serialize}; use std::str::FromStr; use std::sync::Arc; #[cfg_attr(feature = "ssr", derive(sqlx::Encode, sqlx::Decode))] #[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct ListId(i32); #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FilterListRecord { pub name: Arc<str>, pub list_format: FilterListType, pub author: Arc<str>, pub license: Arc<str>, pub expires: std::time::Duration, pub last_updated: Option<chrono::DateTime<chrono::Utc>>, pub list_size: usize, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord, Hash)] #[serde(transparent)] pub struct FilterListUrl { url: Arc<str>, } impl FilterListUrl { pub fn as_str(&self) -> &str { self.as_ref() } pub fn to_internal_path(&self) -> Option<std::path::PathBuf> { if self.as_str().starts_with("internal/") { Some(std::path::PathBuf::from(self.as_str())) } else { None } } } impl std::ops::Deref for FilterListUrl { type Target = str; fn deref(&self) -> &Self::Target { self.url.as_ref() } } impl FromStr for FilterListUrl { type Err = url::ParseError; fn from_str(s: &str) -> Result<Self, Self::Err> { match s { "internal/blocklist.txt" | "internal/block_ips.txt" | "internal/allowlist.txt" => { Ok(Self { url: s.into() }) } s => Ok(Self { url: url::Url::parse(s)?.as_str().into(), }), } } } #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum FilterListType { Adblock, DomainBlocklist, DomainBlocklistWithoutSubdomains, DomainAllowlist, IPBlocklist, IPAllowlist, IPNetBlocklist, DenyHosts, RegexAllowlist, RegexBlocklist, Hostfile, } impl FilterListType { pub fn as_str(&self) -> &'static str { match self { Self::Adblock => "Adblock", Self::DomainBlocklist => "DomainBlocklist", Self::DomainBlocklistWithoutSubdomains => "DomainBlocklistWithoutSubdomains", Self::DomainAllowlist => "DomainAllowlist", Self::IPBlocklist => "IPBlocklist", Self::IPAllowlist => "IPAllowlist", Self::IPNetBlocklist => "IPNetBlocklist", Self::DenyHosts => "DenyHosts", Self::RegexAllowlist => "RegexAllowlist", Self::RegexBlocklist => "RegexBlocklist", Self::Hostfile => "Hostfile", } } } #[derive(Debug, thiserror::Error)] pub struct InvalidFilterListTypeError; impl std::fmt::Display for InvalidFilterListTypeError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Invalid FilterListType") } } impl std::str::FromStr for FilterListType { type Err = InvalidFilterListTypeError; fn from_str(s: &str) -> Result<Self, Self::Err> { match s { "Adblock" => Ok(Self::Adblock), "DomainBlocklist" => Ok(Self::DomainBlocklist), "DomainBlocklistWithoutSubdomains" => Ok(Self::DomainBlocklistWithoutSubdomains), "DomainAllowlist" => Ok(Self::DomainAllowlist), "IPBlocklist" => Ok(Self::IPBlocklist), "IPAllowlist" => Ok(Self::IPAllowlist), "IPNetBlocklist" => Ok(Self::IPNetBlocklist), "DenyHosts" => Ok(Self::DenyHosts), "RegexAllowlist" => Ok(Self::RegexAllowlist), "RegexBlocklist" => Ok(Self::RegexBlocklist), "Hostfile" => Ok(Self::Hostfile), _ => Err(InvalidFilterListTypeError), } } } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FilterListMap( pub Vec<(FilterListUrl, FilterListRecord)>, // Just so it is consistently ordered ); #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct DomainRule { pub domain: Domain, pub allow: bool, pub subdomain: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct IpRule { pub ip: ipnetwork::IpNetwork, pub allow: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Rule { #[serde(rename = "d")] Domain(DomainRule), IpRule(IpRule), #[serde(rename = "u")] Unknown, #[serde(rename = "i")] Invalid, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(into = "(Arc<str>, Rule)")] #[serde(from = "(Arc<str>, Rule)")] pub struct RulePair { source: Arc<str>, rule: Rule, } impl RulePair { pub fn new(source: Arc<str>, rule: Rule) -> RulePair { RulePair { source, rule } } pub fn get_rule(&self) -> &Rule { &self.rule } pub fn get_source(&self) -> &Arc<str> { &self.source } } impl From<RulePair> for (Arc<str>, Rule) { fn from(val: RulePair) -> Self { (val.source, val.rule) } } impl From<(Arc<str>, Rule)> for RulePair { fn from((source, rule): (Arc<str>, Rule)) -> Self { Self { source, rule } } } #[cfg(feature = "ssr")] fn parse_lines(contents: &str, parser: &dyn Fn(&str) -> Option<Rule>) -> Vec<RulePair> { let mut rules = vec![]; for line in contents.lines() { let source = line; if line.is_empty() { continue; } if let Some(rule) = parser(line) { rules.push(RulePair { source: source.into(), rule, }); } } rules } #[cfg(feature = "ssr")] fn parse_domain_list_line(line: &str, allow: bool, subdomain: bool) -> Option<Rule> { let line = line.split('#').next()?; if line.is_empty() { return None; } let line = line.trim(); let mut segments = line.split_whitespace(); match (segments.next(), segments.next(), segments.next()) { (Some(domain), None, None) | (Some("127.0.0.1" | "0.0.0.0"), Some(domain), None) => { let (subdomain, domain) = domain .strip_prefix("*.") .map_or((subdomain, domain), |domain| (true, domain)); if let Ok(domain) = domain.parse() { let domain_rule = DomainRule { domain, allow, subdomain, }; Some(Rule::Domain(domain_rule)) } else { Some(Rule::Invalid) } } _ => Some(Rule::Invalid), } } #[cfg(feature = "ssr")] fn parse_domain_list(contents: &str, allow: bool, subdomain: bool) -> Vec<RulePair> { parse_lines(contents, &|line| { parse_domain_list_line(line, allow, subdomain) }) } #[cfg(feature = "ssr")] fn parse_adblock_line(line: &str) -> Option<Rule> { let rule = line; if rule.starts_with('!') // Comment || rule.contains('#') // CSS selector || !rule.trim_matches('.').contains('.') // Not a domain || matches! {rule, "[Adblock Plus 2.0]" | "[Adblock Plus 1.1]"} { return None; } let mut match_end_domain = false; let rule = if let Some((start, tags)) = rule.split_once('$') { let mut block_site = false; let mut has_specific_filters = false; let mut has_unknown_tag = false; for tag in tags.split(',') { if tag.starts_with('~') // Can't partially block a site || tag.starts_with("rewrite=") // Can't rewrite a site { return None; } else if let Some(domain_tag) = tag.strip_prefix("domain=") { for domain in domain_tag.split('|') { if !start.contains(domain) { return None; } } } else { match tag { "3p" | "doc" | "document" | "all" => { match_end_domain = true; block_site = true; } "popup" | "ghide" | "generichide" | "genericblock" | "image" | "script" | "third-party" | "xmlhttprequest" | "stylesheet" | "subdocument" | "media" | "csp" => { has_specific_filters = true; } "important" => {} _ => { has_unknown_tag = true; } } } } if has_specific_filters && !block_site { return None; } if has_unknown_tag { return Some(Rule::Unknown); } start } else { rule }; if let Some(rule) = rule.strip_prefix('/') { if let Some(_rule) = rule.strip_suffix('/') { // REGEX } else { return None; // Path selector } } let (rule, exception) = if let Some(rule) = rule.strip_prefix("@@") { (rule, true) } else { (rule, false) }; let (rule, mut match_start_domain, match_exact_start) = if let Some(rule) = rule.strip_prefix("||") { (rule, true, false) } else { let (rule, match_exact_start) = rule .strip_prefix('|') .map_or((rule, false), |rule| (rule, true)); (rule, false, match_exact_start) }; let (rule, match_end_domain_exact) = rule .strip_suffix('|') .map_or((rule, false), |rule| (rule, true)); let (mut rule, match_end_domain) = rule .strip_suffix('^') .map_or((rule, match_end_domain), |rule| (rule, true)); if rule.contains('/') { return None; // Path selector } if rule.contains('*') { return Some(Rule::Unknown); } if !match_start_domain { (match_start_domain, rule) = rule .strip_prefix('.') .or_else(|| rule.strip_prefix("*.")) .map_or((false, rule), |rule| (true, rule)); } if match_start_domain && (match_end_domain | match_end_domain_exact) { if let Ok(domain) = rule.parse() { let domain_rule = DomainRule { domain, allow: exception, subdomain: !match_exact_start, }; return Some(Rule::Domain(domain_rule)); } else if let Ok(ip) = rule.parse::<ipnetwork::IpNetwork>() { return Some(Rule::IpRule(IpRule { ip, allow: exception, })); } } Some(Rule::Unknown) } #[cfg(feature = "ssr")] fn parse_adblock(contents: &str) -> Vec<RulePair> { parse_lines(contents, &parse_adblock_line) } #[cfg(feature = "ssr")] fn parse_regex_line(line: &str) -> Option<Rule> { if line.starts_with('#') { return None; } if let Some(rule) = line.strip_prefix(r"(^|\.)") { if let Some(rule) = rule.strip_suffix('$') { let mut rule = rule.to_string(); rule.retain(|c| c != '/'); if let Ok(domain) = rule.parse() { let domain_rule = DomainRule { domain, allow: false, subdomain: true, }; return Some(Rule::Domain(domain_rule)); } } } Some(Rule::Unknown) } #[cfg(feature = "ssr")] fn parse_ip_network_line(line: &str, allow: bool) -> Option<Rule> { let line = line.trim(); if line.is_empty() || line.starts_with('#') { return None; } if let Ok(ip) = line.parse::<ipnetwork::IpNetwork>() { Some(Rule::IpRule(IpRule { ip, allow })) } else { Some(Rule::Unknown) } } #[cfg(feature = "ssr")] fn parse_ip_network_list(contents: &str, allow: bool) -> Vec<RulePair> { parse_lines(contents, &|line| parse_ip_network_line(line, allow)) } #[cfg(feature = "ssr")] fn parse_regex(contents: &str) -> Vec<RulePair> { parse_lines(contents, &parse_regex_line) } #[cfg(feature = "ssr")] fn parse_unknown_lines(contents: &str) -> Vec<RulePair> { parse_lines(contents, &|_| Some(Rule::Unknown)) } #[cfg(feature = "ssr")] pub fn parse_list_contents(contents: &str, list_format: FilterListType) -> Vec<RulePair> { match list_format { FilterListType::Adblock => parse_adblock(contents), FilterListType::DomainBlocklist => parse_domain_list(contents, false, true), FilterListType::DomainBlocklistWithoutSubdomains => { parse_domain_list(contents, false, false) } FilterListType::DomainAllowlist => parse_domain_list(contents, true, false), FilterListType::IPBlocklist => parse_ip_network_list(contents, false), FilterListType::IPAllowlist => parse_ip_network_list(contents, true), FilterListType::IPNetBlocklist => parse_ip_network_list(contents, false), FilterListType::DenyHosts => parse_unknown_lines(contents), FilterListType::RegexAllowlist => parse_unknown_lines(contents), FilterListType::RegexBlocklist => parse_regex(contents), FilterListType::Hostfile => parse_domain_list(contents, false, true), } } #[server(ParseList)] pub async fn parse_list(url: FilterListUrl) -> Result<(), ServerFnError> { let start = std::time::Instant::now(); let pool = crate::server::get_db().await?; let url_str = url.as_str(); let record = sqlx::query!( "SELECT id, format, contents FROM filterLists WHERE url = $1", url_str ) .fetch_one(&pool) .await?; let list_format: FilterListType = record.format.parse()?; log::info!( "Parsing {} as format {}", url.as_str(), list_format.as_str() ); let list_id = record.id; let rules = { let contents = record .contents .ok_or_else(|| ServerFnError::new("No contents for list"))?; parse_list_contents(&contents, list_format) }; let (mut domain_src, mut domains, mut allow, mut subdomain) = (vec![], vec![], vec![], vec![]); let (mut ip_source, mut ips, mut allow_ips) = (vec![], vec![], vec![]); let mut other_rules_src = vec![]; for rule in &rules { let source = rule.get_source().as_ref(); let source = source[..source.len().min(2000)].to_string(); match rule.get_rule() { Rule::Domain(domain_rule) => { domain_src.push(source); domains.push(domain_rule.domain.clone()); allow.push(domain_rule.allow); subdomain.push(domain_rule.subdomain); } Rule::IpRule(ip_rule) => { ip_source.push(source); ips.push(ip_rule.ip); allow_ips.push(ip_rule.allow); } _ => { other_rules_src.push(source); } } } log::info!( "Inserting {} rules ({} domain rules, {} IP rules, {} unknown)", rules.len(), domain_src.len(), ip_source.len(), other_rules_src.len() ); sqlx::query!( "INSERT INTO domains (domain) SELECT domain FROM UNNEST($1::text[]) AS t(domain) ON CONFLICT DO NOTHING", &domains[..] as _ ) .execute(&pool) .await?; let mut tx = pool.begin().await?; sqlx::query! {"DELETE FROM list_rules WHERE list_id = $1", list_id} .execute(&mut *tx) .await?; sqlx::query!("INSERT INTO domain_rules (domain_id, allow, subdomain) SELECT domains.id, allow, subdomain FROM UNNEST($1::text[], $2::bool[], $3::bool[]) AS t(domain, allow, subdomain) INNER JOIN domains ON domains.domain = t.domain ON CONFLICT DO NOTHING", &domains[..] as _, &allow[..], &subdomain[..] ).execute(&mut *tx).await?; sqlx::query!("INSERT INTO Rules (domain_rule_id) SELECT domain_rules.id FROM UNNEST($1::text[], $2::bool[], $3::bool[]) AS t(domain, allow, subdomain) INNER JOIN domains ON domains.domain = t.domain INNER JOIN domain_rules ON domain_rules.domain_id = domains.id AND domain_rules.allow = t.allow AND domain_rules.subdomain = t.subdomain ON CONFLICT DO NOTHING", &domains[..] as _, &allow[..], &subdomain[..] ).execute(&mut *tx).await?; sqlx::query!("INSERT INTO rule_source (source, rule_id) SELECT source, Rules.id FROM UNNEST ($1::text[], $2::text[], $3::bool[], $4::bool[]) AS t(source, domain, allow, subdomain) INNER JOIN domains ON domains.domain = t.domain INNER JOIN domain_rules ON domain_rules.domain_id = domains.id AND domain_rules.allow = t.allow AND domain_rules.subdomain = t.subdomain INNER JOIN Rules ON Rules.domain_rule_id = domain_rules.id ON CONFLICT DO NOTHING", &domain_src[..], &domains[..] as _, &allow[..], &subdomain[..] ).execute(&mut *tx).await?; sqlx::query!( "INSERT INTO ip_rules (ip_network, allow) SELECT ip, allow FROM UNNEST($1::inet[], $2::bool[]) AS t(ip, allow) ON CONFLICT DO NOTHING", &ips[..], &allow_ips[..] ) .execute(&mut *tx) .await?; sqlx::query!( "INSERT INTO Rules (ip_rule_id) SELECT ip_rules.id FROM UNNEST($1::inet[], $2::bool[]) AS t(ip, allow) INNER JOIN ip_rules ON ip_rules.ip_network = t.ip AND ip_rules.allow = t.allow ON CONFLICT DO NOTHING", &ips[..], &allow_ips[..] ) .execute(&mut *tx) .await?; sqlx::query!( "INSERT INTO rule_source (source, rule_id) SELECT source, Rules.id FROM UNNEST ($1::text[], $2::inet[], $3::bool[]) AS t(source, ip, allow) INNER JOIN ip_rules ON ip_rules.ip_network = t.ip AND ip_rules.allow = t.allow INNER JOIN Rules ON Rules.ip_rule_id = ip_rules.id ON CONFLICT DO NOTHING", &ip_source[..], &ips[..], &allow_ips[..] ) .execute(&mut *tx) .await?; sqlx::query!("INSERT INTO list_rules (list_id, source_id) SELECT $1, rule_source.id FROM UNNEST ($2::text[], $3::inet[], $4::bool[]) AS t(source, ip, allow) INNER JOIN ip_rules ON ip_rules.ip_network = t.ip AND ip_rules.allow = t.allow INNER JOIN Rules ON Rules.ip_rule_id = ip_rules.id INNER JOIN rule_source ON rule_source.rule_id = Rules.id WHERE rule_source.source = t.source ON CONFLICT DO NOTHING", list_id, &ip_source[..], &ips[..], &allow_ips[..] ).execute(&mut *tx).await?; sqlx::query!( "INSERT INTO Rules (domain_rule_id, ip_rule_id) VALUES (NULL, NULL) ON CONFLICT DO NOTHING", ) .execute(&mut *tx) .await?; sqlx::query!( "INSERT INTO rule_source (source, rule_id) SELECT source, Rules.id FROM UNNEST ($1::text[]) AS t(source) INNER JOIN Rules ON Rules.domain_rule_id IS NULL AND Rules.ip_rule_id IS NULL ON CONFLICT DO NOTHING", &other_rules_src[..], ) .execute(&mut *tx) .await?; sqlx::query!( "INSERT INTO list_rules (list_id, source_id) SELECT $1, rule_source.id FROM UNNEST($2::text[], $3::text[], $4::bool[], $5::bool[]) AS t(source, domain, allow, subdomain) INNER JOIN domains ON domains.domain = t.domain INNER JOIN domain_rules ON domain_rules.domain_id = domains.id AND domain_rules.allow = t.allow AND domain_rules.subdomain = t.subdomain INNER JOIN Rules ON Rules.domain_rule_id = domain_rules.id INNER JOIN rule_source ON rule_source.rule_id = Rules.id WHERE rule_source.source = t.source ON CONFLICT DO NOTHING", list_id, &domain_src[..], &domains[..] as _, &allow[..], &subdomain[..] ).execute(&mut *tx).await?; sqlx::query!( "INSERT INTO list_rules (list_id, source_id) SELECT $1, rule_source.id FROM UNNEST ($2::text[]) AS t(source) INNER JOIN Rules ON Rules.domain_rule_id IS NULL AND Rules.ip_rule_id IS NULL INNER JOIN rule_source ON rule_source.rule_id = Rules.id WHERE rule_source.source = t.source ON CONFLICT DO NOTHING", list_id, &other_rules_src[..], ) .execute(&mut *tx) .await?; sqlx::query!( "UPDATE filterLists SET rule_count=$2 WHERE id = $1", list_id, rules.len() as i32 ) .execute(&mut *tx) .await?; log::info!("Inserted list rules"); tx.commit().await?; log::info!("Total time: {:?}", start.elapsed()); Ok(()) } #[derive(Clone, Debug, Serialize, Deserialize)] struct CsvRecord { pub name: String, pub url: FilterListUrl, pub author: String, pub license: String, pub expires: u64, pub list_type: FilterListType, } #[server] pub async fn load_filter_map() -> Result<(), ServerFnError> { dotenvy::dotenv()?; let filterlists_path: std::path::PathBuf = std::env::var("FILTERLISTS_PATH")?.parse()?; let contents = tokio::fs::read_to_string(filterlists_path).await?; let records = csv::Reader::from_reader(contents.as_bytes()) .deserialize::<CsvRecord>() .collect::<Result<Vec<CsvRecord>, _>>()?; let mut urls = Vec::new(); let mut names = Vec::new(); let mut formats = Vec::new(); let mut expires_list = Vec::new(); let mut authors = Vec::new(); let mut licenses = Vec::new(); for csv_record in &records { let url = csv_record.url.as_str().to_string(); let name = csv_record.name.clone(); let format = csv_record.list_type.as_str().to_string(); let expires = csv_record.expires as i32; let author = csv_record.author.clone(); let license = csv_record.license.clone(); urls.push(url); names.push(name); formats.push(format); expires_list.push(expires); authors.push(author); licenses.push(license); } let pool = crate::server::get_db().await?; sqlx::query!( "INSERT INTO filterLists (url, name, format, expires, author, license) SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::int[], $5::text[], $6::text[]) ON CONFLICT (url) DO UPDATE SET name = EXCLUDED.name, format = EXCLUDED.format, expires = EXCLUDED.expires, author = EXCLUDED.author, license = EXCLUDED.license ", &urls, &names, &formats, &expires_list, &authors, &licenses ).execute(&pool).await?; write_filter_map().await?; Ok(()) } #[server] pub async fn watch_filter_map() -> Result<(), ServerFnError> { dotenvy::dotenv()?; let filterlists_path: std::path::PathBuf = std::env::var("FILTERLISTS_PATH")?.parse()?; use notify::Watcher; let notify = std::sync::Arc::new(tokio::sync::Notify::new()); let notify2 = notify.clone(); load_filter_map().await?; let mut watcher = notify::recommended_watcher(move |_| { notify.notify_one(); })?; watcher.watch(&filterlists_path, notify::RecursiveMode::NonRecursive)?; let mut last_updated = std::time::Instant::now(); loop { notify2.notified().await; if last_updated.elapsed() > std::time::Duration::from_millis(200) { load_filter_map().await?; last_updated = std::time::Instant::now(); } } } #[server] pub async fn write_filter_map() -> Result<(), ServerFnError> { use csv::Writer; dotenvy::dotenv()?; let filterlists_path: std::path::PathBuf = std::env::var("FILTERLISTS_PATH")?.parse()?; let pool = crate::server::get_db().await?; let rows = sqlx::query!("SELECT url, name, format, expires, author, license FROM filterLists") .fetch_all(&pool) .await?; let mut records = Vec::new(); for record in rows { records.push(CsvRecord { name: record.name.unwrap_or(String::new()), url: record.url.parse()?, author: record.author.unwrap_or(String::new()), license: record.license.unwrap_or(String::new()), expires: record.expires as u64, list_type: FilterListType::from_str(&record.format)?, }); } records.sort_by_key(|record| (record.name.clone(), record.url.clone())); records.reverse(); let mut wtr = Writer::from_path(filterlists_path)?; for record in records { wtr.serialize(record)?; } Ok(()) } #[server] pub async fn get_filter_map() -> Result<FilterListMap, ServerFnError> { let pool = crate::server::get_db().await?; let rows = sqlx::query!( "SELECT url, name, format, expires, author, license, lastupdated, rule_count FROM filterLists" ) .fetch_all(&pool) .await?; let mut filter_list_map = Vec::new(); for record in rows { let url = record.url.parse()?; let record = FilterListRecord { name: record.name.unwrap_or(String::new()).into(), list_format: FilterListType::from_str(&record.format)?, author: record.author.unwrap_or(String::new()).into(), license: record.license.unwrap_or(String::new()).into(), expires: std::time::Duration::from_secs(record.expires as u64), last_updated: record.lastupdated, list_size: record.rule_count as usize, }; filter_list_map.push((url, record)); } Ok(FilterListMap(filter_list_map)) } #[cfg(feature = "ssr")] struct LastVersionData { last_updated: chrono::DateTime<chrono::Utc>, etag: Option<String>, } #[cfg(feature = "ssr")] async fn get_last_version_data( url: &FilterListUrl, ) -> Result<Option<LastVersionData>, ServerFnError> { let pool = crate::server::get_db().await?; let url_str = url.as_str(); #[allow(non_camel_case_types)] let last_version_data = sqlx::query!( r#"SELECT lastUpdated as "last_updated: chrono::DateTime<chrono::Utc>", etag FROM filterLists WHERE url = $1"#, url_str ) .fetch_one(&pool) .await .ok(); let last_version_data = last_version_data.and_then(|row| { Some(LastVersionData { last_updated: row.last_updated?, etag: row.etag, }) }); Ok(last_version_data) } #[server] pub async fn get_last_updated( url: FilterListUrl, ) -> Result<Option<chrono::DateTime<chrono::Utc>>, ServerFnError> { get_last_version_data(&url) .await .map(|data| data.map(|data| data.last_updated)) } #[cfg(feature = "ssr")] #[derive(thiserror::Error, Debug)] enum UpdateListError { #[error("Failed to fetch list")] FailedToFetch, } #[server(UpdateListFn)] pub async fn update_list(url: FilterListUrl) -> Result<(), ServerFnError> { let pool = crate::server::get_db().await?; let old_contents = sqlx::query!( "SELECT contents FROM filterLists WHERE url = $1", url.as_str() ) .fetch_one(&pool) .await? .contents; if let Some(internal_path) = url.to_internal_path() { let contents = tokio::fs::read_to_string(&internal_path).await?; let mut lines = contents.lines().collect::<Vec<_>>(); lines.sort_unstable(); lines.dedup(); let sorted_contents = lines.join("\n"); tokio::fs::write(internal_path, &sorted_contents).await?; let new_last_updated = chrono::Utc::now(); sqlx::query!( "UPDATE filterLists SET lastUpdated = $2, contents = $3 WHERE url = $1 ", url.as_str(), new_last_updated, sorted_contents ) .execute(&pool) .await?; if old_contents != Some(sorted_contents) { parse_list(url).await?; } return Ok(()); } log::info!("Updating {}", url.as_str()); let url_str = url.as_str(); let last_updated = get_last_version_data(&url).await?; let mut req = reqwest::Client::new().get(url_str); if let Some(last_updated) = last_updated { req = req.header( "if-modified-since", last_updated .last_updated .format("%a, %d %b %Y %H:%M:%S GMT") .to_string(), ); if let Some(etag) = last_updated.etag { req = req.header("if-none-match", etag); } } let response = req.send().await?; match response.status() { reqwest::StatusCode::NOT_MODIFIED => { log::info!("Not modified {:?}", url_str); sqlx::query!( "UPDATE filterLists SET lastUpdated = NOW() WHERE url = $1 ", url_str ) .execute(&pool) .await?; Ok(()) } reqwest::StatusCode::OK => { let headers = response.headers().clone(); let etag = headers.get("etag").and_then(|item| item.to_str().ok()); let body = response.text().await?; log::info!("Updated {} size ({})", url_str, body.len()); sqlx::query!( "UPDATE filterLists SET contents = $2, etag = $3 WHERE url = $1 ", url_str, body, etag ) .execute(&pool) .await?; if Some(body) == old_contents { log::info!("No change in contents for {}", url_str); } else { parse_list(url.clone()).await?; } sqlx::query!( "UPDATE filterLists SET lastUpdated = NOW() WHERE url = $1 ", url_str ) .execute(&pool) .await?; Ok(()) } status => { log::error!("Error fetching {}: {:?}", url_str, status); Err(UpdateListError::FailedToFetch.into()) } } } #[server(DeleteListFn)] pub async fn delete_list(url: FilterListUrl) -> Result<(), ServerFnError> { let pool = crate::server::get_db().await?; let url_str = url.as_str(); sqlx::query!( "DELETE FROM list_rules WHERE list_rules.list_id IN ( SELECT id FROM filterLists WHERE url = $1 )", url_str ) .execute(&pool) .await?; sqlx::query!("DELETE FROM filterLists WHERE url = $1", url_str) .execute(&pool) .await?; write_filter_map().await?; Ok(()) } #[component] pub fn FilterListLink(url: FilterListUrl) -> impl IntoView { let href = format!( "/list{}", params_map! { "url" => url.as_str(), } .to_query_string(), ); view! { <A href=href class="link link-neutral"> {url.as_str().to_string()} </A> } } #[server] async fn get_list_size(url: FilterListUrl) -> Result<Option<usize>, ServerFnError> { let pool = crate::server::get_db().await?; let url_str = url.as_str(); let record = sqlx::query!( "SELECT id, rule_count FROM filterLists WHERE url = $1", url_str ) .fetch_one(&pool) .await?; let list_id = record.id; let count = record.rule_count; if count == 0 { let count = sqlx::query!( "SELECT COUNT(*) FROM list_rules WHERE list_id = $1", list_id ) .fetch_one(&pool) .await? .count; if let Some(count) = count { sqlx::query!( "UPDATE filterLists SET rule_count = $1 WHERE id = $2", count as i32, list_id ) .execute(&pool) .await?; Ok(Some(count as usize)) } else { Ok(None) } } else { Ok(Some(count as usize)) } } #[component] pub fn ListSize(url: FilterListUrl, list_size: Option<usize>) -> impl IntoView { if let Some(size) = list_size { if size > 0 { return size.into_view(); } } view! { <Await future=move || { let url = url.clone(); async { get_list_size(url).await } } let:size > {match size { Err(err) => format!("{err:?}").into_view(), Ok(None) => "Never".into_view(), Ok(Some(size)) => size.into_view(), }} </Await> } .into_view() } #[server] async fn get_list_page( url: FilterListUrl, page: Option<usize>, page_size: usize, ) -> Result<Vec<(RuleId, SourceId, crate::filterlist::RulePair)>, ServerFnError> { let pool = crate::server::get_db().await?; let url_str = url.as_str(); let id = sqlx::query!("SELECT id FROM filterLists WHERE url = $1", url_str) .fetch_one(&pool) .await? .id; let start = page.unwrap_or(0) * page_size; let records = sqlx::query!( r#"SELECT Rules.id AS "rule_id: RuleId", rule_source.id AS "source_id: SourceId", rule_source.source, domain as "domain: Option<String>" , domain_rules.allow as "domain_allow: Option<bool>", subdomain as "subdomain: Option<bool>", ip_network as "ip_network: Option<ipnetwork::IpNetwork>", ip_rules.allow as "ip_allow: Option<bool>" FROM list_rules INNER JOIN rule_source ON rule_source.id = list_rules.source_id INNER JOIN Rules ON Rules.id = rule_source.rule_id LEFT JOIN domain_rules ON domain_rules.id = Rules.domain_rule_id LEFT JOIN domains ON domains.id = domain_rules.domain_id LEFT JOIN ip_rules ON ip_rules.id = Rules.ip_rule_id WHERE list_id = $1 ORDER BY list_rules.source_id LIMIT $2 OFFSET $3 "#r, id, page_size as i64 , start as i64 ) .fetch_all(&pool) .await?; let rules = records .iter() .map(|record| { let rule_data = RuleData { rule_id: record.rule_id, domain: record.domain.clone(), domain_allow: record.domain_allow, domain_subdomain: record.subdomain, ip_network: record.ip_network, ip_allow: record.ip_allow, }; let rule = rule_data.try_into()?; let source = record.source.clone(); let pair = crate::filterlist::RulePair::new(source.into(), rule); Ok((record.rule_id, record.source_id, pair)) }) .collect::<Result<Vec<(_, _, _)>, ServerFnError>>(); rules } #[component] fn LastUpdatedInner(last_updated: Option<chrono::DateTime<chrono::Utc>>) -> impl IntoView { view! { {match last_updated { Some(last_updated) => { view! { <div>{format!("{last_updated}")}</div> } } None => { view! { <div>"Never"</div> } } }} } } #[component] pub fn LastUpdated(url: FilterListUrl, record: Option<FilterListRecord>) -> impl IntoView { view! { {match record.clone() { Some(record) => { let last_updated = record.last_updated; view! { <LastUpdatedInner last_updated=last_updated/> } } None => { view! { <Await future={ let url = url.clone(); move || { let url = url.clone(); async move { crate::filterlist::get_last_updated(url.clone()).await } } } let:last_version_data > {match last_version_data { Ok(last_updated) => { view! { <LastUpdatedInner last_updated=*last_updated/> }.into_view() } Err(err) => view! { {format!("{err:?}")} }.into_view(), }} </Await> } } }} <FilterListUpdate url=url.clone()/> } } #[component] pub fn ParseList(url: FilterListUrl) -> impl IntoView { let parse_list_action = create_server_action::<crate::filterlist::ParseList>(); view! { <ActionForm action=parse_list_action> <button class="btn btn-primary" type="submit"> <input type="hidden" placeholder="url" id="url" name="url" value=url.to_string()/> "Parse" </button> </ActionForm> } } #[component] pub fn FilterListUpdate(url: FilterListUrl) -> impl IntoView { let update_list_action = create_server_action::<UpdateListFn>(); view! { <ActionForm action=update_list_action> <button class="btn btn-primary" type="submit"> <input type="hidden" placeholder="url" id="url" name="url" value=url.to_string()/> "Update" </button> </ActionForm> } } #[component] fn Contents(url: FilterListUrl, page: Option<usize>) -> impl IntoView { view! { <table class="table table-zebra"> <thead> <tr> <th>Source</th> <th>Rule</th> </tr> </thead> <Await future=move || { let url = url.clone(); async move { get_list_page(url, page, PAGE_SIZE).await } } let:contents > { let contents = contents.clone(); move || match contents.clone() { Ok(contents) => { let contents = contents.clone(); view! { <tbody> <For each=move || { contents.clone() } key=|(rule_id, source_id, _)| (*rule_id, *source_id) children=|(rule_id, _source_id, pair)| { let source = pair.get_source().to_string(); let rule = pair.get_rule().clone(); view! { <tr> <td>{source}</td> <td> <A href=rule_id.get_href() class="link link-neutral"> <DisplayRule rule=rule/> </A> </td> </tr> } } /> </tbody> } .into_view() } Err(err) => format!("{err:?}").into_view(), } } </Await> </table> } } #[component] fn FilterListInner(url: FilterListUrl, page: Option<usize>) -> impl IntoView { view! { <h1>"Filter List"</h1> <p>"URL: " {url.to_string()}</p> <p>"Last Updated: " <LastUpdated url=url.clone() record=None/></p> <p>"Rule count: " <ListSize url=url.clone() list_size=None/></p> <FilterListUpdate url=url.clone()/> <p> <ParseList url=url.clone()/> </p> <DeleteListButton url=url.clone()/> {if let Some(page) = page { view! { <p>"Page: " {page}</p> } } else { view! { <p>"Page: 0"</p> } }} {match page { None | Some(0) => view! {}.into_view(), Some(page) => { let params = params_map! { "url" => url.as_str(), "page" => (page.saturating_sub(1)).to_string() }; let href = format!("/list{}", params.to_query_string()); view! { <A href=href class="btn btn-neutral"> "Back" </A> } } }} { let params = params_map! { "url" => url.as_str(), "page" => (page.unwrap_or(0) + 1).to_string() }; let href = format!("/list{}", params.to_query_string()); view! { <A href=href class="btn btn-neutral"> "Next" </A> } } <p>"Contents: " <Contents url=url.clone() page=page/></p> } } #[derive(Params, PartialEq, Debug)] struct ViewListParams { url: Option<String>, page: Option<usize>, } #[derive(thiserror::Error, Debug)] enum ViewListError { #[error("Invalid URL")] ParseURL(#[from] url::ParseError), #[error("Invalid URL")] ParseParam(#[from] leptos_router::ParamsError), #[error("Invalid FilterListType")] InvalidFilterListType(#[from] InvalidFilterListTypeError), } impl ViewListParams { fn parse(&self) -> Result<FilterListUrl, ViewListError> { Ok(self .url .as_ref() .ok_or_else(|| ParamsError::MissingParam("Missing Param".into()))? .parse()?) } } #[component] fn DeleteListButton(url: FilterListUrl) -> impl IntoView { let delete_list_action = create_server_action::<DeleteListFn>(); view! { <ActionForm action=delete_list_action> <button class="btn btn-danger" type="submit"> <input type="hidden" placeholder="url" id="url" name="url" value=url.to_string()/> "Delete" </button> </ActionForm> } } #[component] pub fn FilterListPage() -> impl IntoView { let params = use_query::<ViewListParams>(); let get_url = move || { params.with(|param| { param .as_ref() .ok() .map(|param| param.parse().map(|url| (url, param.page))) }) }; view! { <div> {move || match get_url() { None => view! { <p>"No URL"</p> }.into_view(), Some(Err(err)) => view! { <p>"Error: " {format!("{err}")}</p> }.into_view(), Some(Ok((url, page))) => view! { <FilterListInner url=url page=page/> }.into_view(), }} </div> } } ================================================ FILE: src/home_page.rs ================================================ use crate::filterlist::{FilterListRecord, FilterListUrl, LastUpdated, ListSize}; use leptos::*; use leptos_router::*; #[component] fn FilterListSummary(url: FilterListUrl, record: FilterListRecord) -> impl IntoView { view! { <tr> <td class="break-normal break-words max-w-20"> { let name = if record.name.is_empty() { url.to_string() } else { record.name.to_string() }; let href = format!( "/list{}", params_map! { "url" => url.as_str(), } .to_query_string(), ); view! { <A href=href class="link link-neutral"> {name} </A> } } </td> <td class="break-normal break-words max-w-20">{record.author.to_string()}</td> <td class="max-w-xl break-normal break-words">{record.license.to_string()}</td> <td>{humantime::format_duration(record.expires).to_string()}</td> <td>{format!("{:?}", record.list_format)}</td> <td> <LastUpdated url=url.clone() record=Some(record.clone())/> </td> <td class="text-right"> <ListSize url=url.clone() list_size=Some(record.list_size)/> </td> </tr> } } /// Renders the home page of your application. #[component] pub fn HomePage() -> impl IntoView { view! { <Await future=|| async move { crate::filterlist::get_filter_map().await } let:filterlist_map > {match filterlist_map.clone() { Ok(data) => { view! { <table class="table table-zebra"> <thead> <td>"Name"</td> <td>"Author"</td> <td>"License"</td> <td>"Update frequency"</td> <td>"Format"</td> <td>"Last Updated"</td> <td>"Size"</td> </thead> <tbody> <For each=move || { let mut data = data.0.clone(); data.sort_unstable_by(|a, b| { b.1.last_updated.cmp(&a.1.last_updated) }); data } key=|(url, _)| url.as_str().to_string() children=|(url, record)| { view! { <FilterListSummary url=url.clone() record=record.clone()/> } } /> </tbody> </table> } .into_view() } Err(err) => view! { <p>"Error Loading " {format!("{err:?}")}</p> }.into_view(), }} </Await> } } ================================================ FILE: src/ip_view.rs ================================================ use leptos::*; use leptos_router::*; use std::{collections::BTreeSet, net::IpAddr}; use crate::{app::Loading, domain::DomainId}; #[server] async fn get_domans_which_resolve_to_ip( ip: IpAddr, ) -> Result<BTreeSet<(DomainId, String)>, ServerFnError> { let ip: ipnetwork::IpNetwork = ip.into(); let records = sqlx::query!( r#"SELECT domains.id as "domain_id: DomainId", domain from dns_ips INNER JOIN domains ON dns_ips.domain_id = domains.id WHERE dns_ips.ip_address = $1"#r, ip ) .fetch_all(&crate::server::get_db().await?) .await?; let domains = records .into_iter() .map(|record| (record.domain_id, record.domain)) .collect::<BTreeSet<_>>(); Ok(domains) } #[component] fn DomainsWhichResolveTo(get_ip: GetIp) -> impl IntoView { let domain_results = create_resource(get_ip, |ip| async move { let ip = ip?; let results = get_domans_which_resolve_to_ip(ip).await?; Ok::<_, ServerFnError>(results) }); view! { <Transition fallback=move || { view! { <p>"Loading" <Loading/></p> } }> {move || match domain_results.get() { Some(Ok(domains)) => { view! { <div> <p>"Domains which resolve to this IP Address"</p> <ul class="grid grid-cols-2 gap-2"> <For each=move || { domains.clone() } key=|(domain_id, _domain)| *domain_id children=|(_domain_id, domain)| { view! { <li> <A href=format!("/domain/{domain}") class="link link-neutral" > {domain} </A> </li> } } /> </ul> </div> } .into_view() } _ => view! { <p>"Error"</p> }.into_view(), }} </Transition> } } type GetIp = Box<dyn Fn() -> Result<IpAddr, ParamsError>>; #[derive(Params, PartialEq)] struct IpParam { ip: Option<IpAddr>, } #[component] pub fn IpView() -> impl IntoView { let params = use_params::<IpParam>(); let get_ip = move || { params.with(|param| { param.as_ref().map_err(Clone::clone).and_then(|param| { param .ip .ok_or_else(|| ParamsError::MissingParam("No domain".into())) }) }) }; view! { <div> <h1 class="text-2xl font-bold text-gray-800">{"IP Address: "} {get_ip}</h1> <DomainsWhichResolveTo get_ip=Box::new(get_ip)/> </div> } } ================================================ FILE: src/lib.rs ================================================ pub mod app; pub mod domain; pub mod error_template; pub mod filterlist; pub mod home_page; pub mod ip_view; pub mod rule; #[cfg(feature = "ssr")] pub mod server; pub mod stats_view; pub mod tasks; #[cfg(feature = "ssr")] use mimalloc::MiMalloc; use serde::*; #[cfg(feature = "ssr")] #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; const PAGE_SIZE: usize = 50; pub mod source { use super::*; #[cfg_attr(feature = "ssr", derive(sqlx::Encode, sqlx::Decode))] #[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct SourceId(i32); } #[derive(thiserror::Error, Debug)] pub enum DbInitError { #[error("Sqlx error {0}")] SqlxError(String), #[error("Missing DATABASE_URL")] MissingDatabaseUrl(String), } #[cfg(feature = "ssr")] impl From<sqlx::Error> for DbInitError { fn from(e: sqlx::Error) -> Self { Self::SqlxError(e.to_string()) } } #[cfg(feature = "ssr")] impl From<std::env::VarError> for DbInitError { fn from(e: std::env::VarError) -> Self { Self::MissingDatabaseUrl(e.to_string()) } } #[cfg(feature = "ssr")] pub mod fileserv; #[cfg(feature = "hydrate")] #[wasm_bindgen::prelude::wasm_bindgen] pub fn hydrate() { console_error_panic_hook::set_once(); let _ = console_log::init_with_level(log::Level::Debug); log::info!("Hydrating"); leptos::mount_to_body(crate::app::App); log::info!("Mounted"); // leptos::leptos_dom::HydrationCtx::stop_hydrating(); } ================================================ FILE: src/main.rs ================================================ #[cfg(feature = "ssr")] use clap::Parser; #[cfg(feature = "ssr")] #[derive(Debug, serde::Deserialize, serde::Serialize)] struct Config { listen_port: u16, peers: Vec<blockconvert::server::Peer>, } impl Default for Config { fn default() -> Self { Self { listen_port: 3000, peers: vec![Default::default(); 2], } } } #[cfg(feature = "ssr")] /// Simple program to greet a person #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Args { config_path: std::path::PathBuf, #[arg(short, long)] create_config: bool, #[arg(short, long)] run_tasks: bool, } #[cfg(feature = "ssr")] #[tokio::main] async fn main() { use axum::Router; use blockconvert::app::App; use blockconvert::fileserv::file_and_error_handler; use blockconvert::{filterlist, server}; use leptos::*; use leptos_axum::{generate_route_list, LeptosRoutes}; use tower_http::compression::CompressionLayer; use tower_http::compression::CompressionLevel; env_logger::init(); let args = Args::parse(); let config_path = args.config_path.clone(); let node_conf = if let Ok(conf) = tokio::fs::read_to_string(args.config_path).await { let Ok(conf): Result<Config, _> = toml::from_str(&conf) else { logging::warn!("Error parsing config file"); return; }; conf } else if args.create_config { let conf = Config::default(); let conf_str = toml::to_string(&conf).unwrap(); tokio::fs::write(config_path, conf_str).await.unwrap(); logging::log!("Created config file"); conf } else { logging::log!("Config file not found"); return; }; println!("Config: {:?}", node_conf); let peer_state = server::PeerState::new(&node_conf.peers); // Setting get_configuration(None) means we'll be using cargo-leptos's env values // For deployment these variables are: // <https://github.com/leptos-rs/start-axum#executing-a-server-on-a-remote-machine-without-the-toolchain> // Alternately a file can be specified such as Some("Cargo.toml") // The file would need to be included with the executable when moved to deployment let conf = get_configuration(Some("Cargo.toml")).await.unwrap(); let mut leptos_options = conf.leptos_options; let mut addr = leptos_options.site_addr; addr.set_port(node_conf.listen_port); leptos_options.site_addr = addr; let routes = generate_route_list(App); //let state = State { leptos_options }; // build our application with a route let app = Router::new() .leptos_routes(&leptos_options, routes, App) .nest("/peer/", server::get_peer_router(peer_state.clone())) .fallback(file_and_error_handler) .with_state(leptos_options) .layer(CompressionLayer::new().quality(CompressionLevel::Fastest)); let token = tokio_util::sync::CancellationToken::new(); let mut tasks = tokio::task::JoinSet::new(); if args.run_tasks { tasks.spawn(filterlist::watch_filter_map()); tasks.spawn(server::parse_missing_subdomains()); tasks.spawn(server::check_dns(token.clone())); tasks.spawn(server::import_pihole_logs()); tasks.spawn(blockconvert::rule::find_rule_matches()); tasks.spawn(server::build_list()); tasks.spawn(server::update_expired_lists()); tasks.spawn(server::garbage_collect()); tasks.spawn(server::run_cmd(token.clone())); tasks.spawn(server::certstream(token.clone())); } let listener = tokio::net::TcpListener::bind(&addr).await.unwrap(); let server = tasks.spawn(async move { logging::log!("listening on http://{}", &listener.local_addr()?); axum::serve(listener, app.into_make_service()).await?; Ok(()) }); { let token = token.clone(); tasks.spawn(async move { let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()).unwrap(); let mut hangup = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup()).unwrap(); let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap(); tokio::select! { _ = tokio::signal::ctrl_c() => logging::log!("Ctrl-C received, shutting down"), _ = interrupt.recv() => logging::log!("Interrupt received, shutting down"), _ = hangup.recv() => logging::log!("Hangup received, shutting down"), _ = terminate.recv() => logging::log!("Terminate received, shutting down"), } token.cancel(); Ok(()) }); } while let Some(task) = tasks.join_next().await { if let Err(e) = task.unwrap() { logging::log!("Error: {:?}", e); return; } logging::log!("Task completed"); if token.is_cancelled() { server.abort(); logging::log!("Shutting down"); tokio::select! { _ = async move {while tasks.join_next().await.is_some() {}} => {}, _ = tokio::time::sleep(std::time::Duration::from_secs(5)) => {} } break; } } logging::log!("Exiting"); } #[cfg(not(feature = "ssr"))] pub fn main() { // no client-side main function // unless we want this to work with e.g., Trunk for a purely client-side app // see lib.rs for hydration function instead } ================================================ FILE: src/rule.rs ================================================ use crate::app::Loading; use crate::filterlist::DomainRule; use crate::filterlist::FilterListLink; use crate::filterlist::FilterListUrl; use crate::filterlist::Rule; use crate::{domain::DomainId, filterlist::ListId, source::SourceId}; use leptos::*; use leptos_router::*; use serde::{Deserialize, Serialize}; #[cfg_attr(feature = "ssr", derive(sqlx::Encode, sqlx::Decode))] #[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct RuleId(i32); impl RuleId { pub fn get_href(&self) -> String { format!("/rule/{}", self.0) } } #[cfg(feature = "ssr")] pub async fn find_rule_matches() -> Result<(), ServerFnError> { use std::time::Duration; dotenvy::dotenv()?; let pool = crate::server::get_db().await?; let read_limit = std::env::var("READ_LIMIT")?.parse::<u32>()? as i64; let interval: u64 = std::env::var("RULE_MATCH_CHECK_INTERVAL")?.parse()?; let interval: Duration = Duration::from_secs(interval); let mut interval = tokio::time::interval(interval); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { interval.tick().await; let records = sqlx::query!( "SELECT id from Rules ORDER BY last_checked_matches ASC NULLS FIRST LIMIT $1", read_limit ) .fetch_all(&pool) .await?; let rule_ids = records .into_iter() .map(|record| record.id) .collect::<Vec<_>>(); let mut tx = pool.begin().await?; sqlx::query!( "DELETE FROM rule_matches WHERE rule_id = ANY($1::int[])", &rule_ids[..] ) .execute(&mut *tx) .await?; sqlx::query!(" INSERT INTO rule_matches(rule_id, domain_id) SELECT Rules.id AS rule_id, domains.id AS domain_id FROM Rules LEFT JOIN domain_rules ON Rules.domain_rule_id = domain_rules.id LEFT JOIN subdomains ON domain_rules.domain_id = subdomains.parent_domain_id AND domain_rules.subdomain = true LEFT JOIN ip_rules ON Rules.ip_rule_id = ip_rules.id AND ip_rules.allow=false LEFT JOIN dns_ips ON ip_rules.ip_network = dns_ips.ip_address LEFT JOIN dns_cnames ON (dns_cnames.cname_domain_id = domain_rules.domain_id OR dns_cnames.cname_domain_id = subdomains.domain_id) AND domain_rules.allow=false INNER JOIN domains ON domain_rules.domain_id = domains.id OR subdomains.domain_id = domains.id OR dns_ips.domain_id = domains.id OR dns_cnames.domain_id = domains.id INNER JOIN dns_ips AS dns_check ON dns_check.domain_id = domains.id AND dns_check.ip_address IS NOT NULL WHERE Rules.id = ANY($1::int[]) ON CONFLICT DO NOTHING", &rule_ids[..]).execute(&mut *tx).await?; let count = sqlx::query!( "SELECT COUNT(*) FROM rule_matches WHERE rule_id = ANY($1::int[])", &rule_ids[..] ) .fetch_one(&mut *tx) .await? .count .unwrap_or(0); log::info!( "Checked {} rules and found {} matches", rule_ids.len(), count ); sqlx::query!( "UPDATE rules SET last_checked_matches = now() WHERE id = ANY($1::int[])", &rule_ids[..] ) .execute(&mut *tx) .await?; tx.commit().await?; } } #[server] pub async fn get_rule(id: RuleId) -> Result<Rule, ServerFnError> { let record = sqlx::query!( "SELECT domain_rule_id, ip_rule_id FROM Rules WHERE Rules.id = $1", id.0 ) .fetch_one(&crate::server::get_db().await?) .await?; if let Some(domain_rule_id) = record.domain_rule_id { let record = sqlx::query!( "SELECT domain, allow, subdomain FROM domain_rules INNER JOIN domains ON domains.id = domain_rules.domain_id WHERE domain_rules.id = $1", domain_rule_id ) .fetch_one(&crate::server::get_db().await?) .await?; let domain_rule = DomainRule { domain: record.domain.parse()?, allow: record.allow, subdomain: record.subdomain, }; Ok(Rule::Domain(domain_rule)) } else if let Some(ip_rule_id) = record.ip_rule_id { let record = sqlx::query!( "SELECT ip_network, allow FROM ip_rules WHERE id = $1", ip_rule_id ) .fetch_one(&crate::server::get_db().await?) .await?; Ok(Rule::IpRule(crate::filterlist::IpRule { ip: record.ip_network, allow: record.allow, })) } else { Ok(Rule::Invalid) } } type GetId = Box<dyn Fn() -> Result<RuleId, ParamsError>>; #[server] async fn get_sources( id: RuleId, ) -> Result<Vec<(SourceId, String, ListId, FilterListUrl)>, ServerFnError> { let sources = sqlx::query!( r#"SELECT rule_source.id AS "source_id: SourceId", source, filterLists.id as "list_id: ListId", filterLists.url FROM rule_source INNER JOIN list_rules ON rule_source.id = list_rules.source_id INNER JOIN filterLists ON list_rules.list_id = filterLists.id WHERE rule_id = $1 ORDER BY (source) "#r, id.0 ) .fetch_all(&crate::server::get_db().await?) .await?; sources .into_iter() .map(|record| { Ok(( record.source_id, record.source, record.list_id, record.url.parse()?, )) }) .collect() } #[component] fn Sources(get_id: GetId) -> impl IntoView { let source_resource = create_resource(get_id, |id| async move { let sources = get_sources(id?).await?; Ok::<_, ServerFnError>(sources) }); view! { <Transition fallback=move || { view! { <p>"Loading" <Loading/></p> } }> {move || match source_resource.get() { Some(Ok(sources)) => { view! { <p> "Sources:" <table class="table table-zebra"> <thead> <tr> <th>Source</th> <th>List</th> </tr> </thead> <tbody> <For each=move || { sources.clone() } key=|(source_id, _, _, _)| *source_id children=|(_, source, _list_id, url)| { view! { <tr> <td>{source}</td> <td> <FilterListLink url=url/> </td> </tr> } } /> </tbody> </table> </p> } .into_view() } Some(Err(err)) => view! { <p>"Error: " {format!("{err}")}</p> }.into_view(), None => view! { "Invalid URL" }.into_view(), }} </Transition> } } #[component] pub fn DisplayRule(rule: Rule) -> impl IntoView { view! { {match rule { Rule::Domain(domain_rule) => { view! { {if domain_rule.allow { "ALLOW: " } else { "BLOCK: " }} {if domain_rule.subdomain { "*." } else { "" }} {domain_rule.domain.as_ref().to_owned()} } .into_view() } Rule::IpRule(ip_rule) => { view! { {if ip_rule.allow { "ALLOW: " } else { "BLOCK: " }} {ip_rule.ip.to_string()} } .into_view() } Rule::Unknown => "Unknown".into_view(), Rule::Invalid => "Invalid Rule".into_view(), }} } } #[component] fn RuleRawView( rule: Resource<Result<RuleId, ParamsError>, Result<Rule, ServerFnError>>, ) -> impl IntoView { view! { <Transition fallback=move || { view! { <p>"Loading" <Loading/></p> } }> {move || match rule.get() { Some(Ok(rule)) => view! { <DisplayRule rule=rule/> }.into_view(), Some(Err(err)) => view! { <p>"Error: " {format!("{err}")}</p> }.into_view(), None => view! { "Invalid URL" }.into_view(), }} </Transition> } } #[server] async fn get_rule_blocked_domains(id: RuleId) -> Result<Vec<(DomainId, String)>, ServerFnError> { let domains = sqlx::query!( r#"SELECT DISTINCT domains.id as "domain_id: DomainId", domain as "domain: String" FROM Rules INNER JOIN rule_matches ON Rules.id = rule_matches.rule_id INNER JOIN domains ON rule_matches.domain_id = domains.id WHERE Rules.id = $1"#r, id.0 ) .fetch_all(&crate::server::get_db().await?) .await?; Ok(domains .into_iter() .map(|record| (record.domain_id, record.domain)) .collect()) } #[component] fn RuleBlockedDomainsView(get_id: Box<dyn Fn() -> Result<RuleId, ParamsError>>) -> impl IntoView { let domains_resource = create_resource(get_id, |id| async move { let domains = get_rule_blocked_domains(id?).await?; Ok::<_, ServerFnError>(domains) }); view! { <Transition fallback=move || { view! { <p>"Loading" <Loading/></p> } }> {move || match domains_resource.get() { Some(Ok(domains)) => { view! { <p> "Matched Domains:" <table class="table table-zebra"> <thead> <tr> <th>Domain</th> </tr> </thead> <tbody> <For each=move || { domains.clone() } key=|(id, _)| *id children=|(_domain_id, domain)| { let domain_href = format!("/domain/{domain}"); view! { <tr> <td> <A href=domain_href class="link link-neutral"> {domain} </A> </td> </tr> } } /> </tbody> </table> </p> } .into_view() } Some(Err(err)) => view! { <p>"Error: " {format!("{err}")}</p> }.into_view(), None => "Invalid URL".into_view(), }} </Transition> } } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct RuleData { pub rule_id: RuleId, pub domain: Option<String>, pub domain_allow: Option<bool>, pub domain_subdomain: Option<bool>, pub ip_network: Option<ipnetwork::IpNetwork>, pub ip_allow: Option<bool>, } impl TryInto<Rule> for RuleData { type Error = ServerFnError; fn try_into(self) -> Result<Rule, Self::Error> { match ( self.domain, self.domain_allow, self.domain_subdomain, self.ip_network, self.ip_allow, ) { (Some(domain), Some(allow), Some(subdomain), None, None) => { Ok(Rule::Domain(DomainRule { domain: domain.parse()?, allow, subdomain, })) } (None, None, None, Some(ip_network), Some(allow)) => { Ok(Rule::IpRule(crate::filterlist::IpRule { ip: ip_network, allow, })) } _ => Ok(Rule::Invalid), } } } #[derive(Params, PartialEq)] struct RuleParam { id: Option<i32>, } #[component] pub fn RuleViewPage() -> impl IntoView { let params = use_params::<RuleParam>(); let get_id = move || { params.with(|param| { param.as_ref().map_err(Clone::clone).and_then(|param| { Ok(RuleId(param.id.ok_or_else(|| { ParamsError::MissingParam("No id".into()) })?)) }) }) }; let rule_resource = create_resource(get_id, |id| async move { let rule = get_rule(id?).await?; Ok::<_, ServerFnError>(rule) }); view! { <p>"Rule: " <RuleRawView rule=rule_resource/></p> <Sources get_id=Box::new(get_id)/> <RuleBlockedDomainsView get_id=Box::new(get_id)/> } } ================================================ FILE: src/server.rs ================================================ use crate::DbInitError; use crate::{domain::Domain, filterlist::FilterListUrl}; use addr::domain; use axum::body::Bytes; use axum::extract::{Path, State}; use leptos::*; use notify::Watcher; use std::collections::HashSet; use tokio::task::JoinSet; use tower_http::decompression::DecompressionLayer; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; use tokio_util::sync::CancellationToken; static SQLITE_POOL: tokio::sync::OnceCell<sqlx::PgPool> = tokio::sync::OnceCell::const_new(); pub async fn get_db() -> Result<sqlx::PgPool, DbInitError> { SQLITE_POOL .get_or_try_init(|| { let _ = dotenvy::dotenv(); let db_url = std::env::var("DATABASE_URL"); async { Ok::<_, DbInitError>(sqlx::PgPool::connect(&db_url?).await?) } }) .await .cloned() } pub async fn parse_missing_subdomains() -> Result<(), ServerFnError> { dotenvy::dotenv()?; let read_limit: usize = std::env::var("READ_LIMIT")?.parse()?; let pool = get_db().await?; loop { let records = sqlx::query!( "SELECT domain from domains WHERE processed_subdomains = false LIMIT $1", read_limit as i64 ) .fetch_all(&pool) .await?; if records.is_empty() { tokio::time::sleep(Duration::from_secs(30)).await; continue; } let mut checked_domains = Vec::new(); let mut all_domains = Vec::new(); let mut all_parents = Vec::new(); for record in records { checked_domains.push(record.domain.clone()); let Ok(domain) = record.domain.parse::<Domain>() else { continue; }; let parents = domain .as_ref() .match_indices('.') .map(|(i, _)| record.domain.split_at(i + 1).1) .filter_map(|parent| parent.parse::<Domain>().ok()); for parent in parents { all_domains.push(domain.clone()); all_parents.push(parent); } } let mut parent_set = all_parents .iter() .cloned() .collect::<std::collections::HashSet<_>>(); for domain in &all_domains { parent_set.remove(domain); } let parent_set = parent_set.into_iter().collect::<Vec<_>>(); sqlx::query!( "INSERT INTO domains (domain) SELECT domain FROM UNNEST($1::text[]) as t(domain) ON CONFLICT DO NOTHING", &parent_set[..] as _ ) .execute(&pool) .await?; sqlx::query!( "INSERT INTO subdomains (domain_id, parent_domain_id) SELECT domains_with_parents.id, parents.id FROM UNNEST($1::text[], $2::text[]) AS t(domain, parent) INNER JOIN domains AS domains_with_parents ON domains_with_parents.domain = t.domain INNER JOIN domains AS parents ON parents.domain = t.parent ON CONFLICT DO NOTHING", &all_domains[..] as _, &all_parents[..] as _, ) .execute(&pool) .await?; sqlx::query!( "INSERT INTO domains (domain) SELECT domain FROM UNNEST($1::text[]) as t(domain) ON CONFLICT(domain) DO UPDATE SET processed_subdomains = true", &checked_domains[..] ) .execute(&pool) .await?; } } pub async fn check_dns(token: CancellationToken) -> Result<(), ServerFnError> { let resolver = crate::domain::DomainResolver::new(token)?; resolver.run().await?; Ok(()) } pub async fn import_pihole_logs() -> Result<(), ServerFnError> { let _ = dotenvy::dotenv()?; let Ok(log_path) = std::env::var("PIHOLE_LOG_PATH") else { log::info!("No PIHOLE_LOG_PATH set, skipping"); return Ok(()); }; let log_path: std::path::PathBuf = log_path.parse()?; let write_frequency: u64 = std::env::var("WRITE_FREQUENCY")?.parse()?; let notify = std::sync::Arc::new(tokio::sync::Notify::new()); let notify2 = notify.clone(); let mut watcher = notify::recommended_watcher(move |_| { notify.notify_one(); })?; watcher.watch(&log_path, notify::RecursiveMode::NonRecursive)?; let pool: sqlx::Pool<sqlx::Postgres> = get_db().await?; let file = tokio::fs::File::open(log_path).await?; let buf = tokio::io::BufReader::new(file); let mut lines = buf.lines(); let mut domains = HashSet::new(); let mut last_wrote = std::time::Instant::now(); while let Ok(line) = lines.next_line().await { if let Some(line) = line { for segment in line.split_whitespace() { if let Ok(domain) = segment.parse() { let domain: Domain = domain; domains.insert(domain); } } } else { notify2.notified().await; } if last_wrote.elapsed().as_secs() > write_frequency { let domains_vec = domains.drain().collect::<Vec<_>>(); let record = sqlx::query!( "INSERT INTO domains (domain) SELECT domain FROM UNNEST($1::text[]) as t(domain) ON CONFLICT DO NOTHING", &domains_vec[..] as _ ) .execute(&pool) .await?; let inserted = record.rows_affected(); if inserted != 0 { log::info!("Inserted {} domains from Pihole logs", inserted); } last_wrote = std::time::Instant::now(); } } Ok(()) } pub async fn update_expired_lists() -> Result<(), ServerFnError> { let pool = get_db().await?; loop { let record = sqlx::query!( r#"SELECT url, lastupdated+expires * INTERVAL '1 seconds' AS nextupdate FROM filterLists ORDER BY nextupdate NULLS FIRST LIMIT 1"#r ) .fetch_one(&pool) .await?; if let Some(next_update) = record.nextupdate { let next_update = next_update - chrono::Utc::now(); if let Ok(next_update) = next_update.to_std() { tokio::time::sleep(next_update).await; } } let url: FilterListUrl = record.url.parse()?; if let Err(err) = crate::filterlist::update_list(url.clone()).await { log::warn!("Error updating list {}: {:?}", url.as_str(), err); } } } pub async fn build_list() -> Result<(), ServerFnError> { // return Ok(()); dotenvy::dotenv()?; let pool = get_db().await?; sqlx::query!("DELETE FROM allow_domains") .execute(&pool) .await?; sqlx::query!("DELETE FROM block_domains") .execute(&pool) .await?; let allow_record = sqlx::query!( "INSERT INTO allow_domains(domain_id) SELECT rule_matches.domain_id from rule_matches INNER JOIN rules ON rule_matches.rule_id = rules.id LEFT JOIN domain_rules ON rules.domain_rule_id = domain_rules.id LEFT JOIN ip_rules ON rules.ip_rule_id = ip_rules.id WHERE domain_rules.allow = true OR ip_rules.allow = true ON CONFLICT DO NOTHING", ) .execute(&pool) .await?; log::info!("Inserted {} allow rules", allow_record.rows_affected()); let record = sqlx::query!( "INSERT INTO block_domains(domain_id) SELECT rule_matches.domain_id from rule_matches INNER JOIN rules ON rule_matches.rule_id = rules.id LEFT JOIN domain_rules ON rules.domain_rule_id = domain_rules.id LEFT JOIN ip_rules ON rules.ip_rule_id = ip_rules.id WHERE domain_rules.allow = false OR ip_rules.allow = false ON CONFLICT DO NOTHING", ) .execute(&pool) .await?; log::info!("Inserted {} block rules", record.rows_affected()); { let records = sqlx::query!("select domain from block_domains INNER JOIN domains ON block_domains.domain_id = domains.id where not exists(select 1 from allow_domains where allow_domains.domain_id=block_domains.domain_id) ORDER BY domain").fetch_all(&pool).await?; let domain_file = tokio::fs::File::create("output/domains.txt").await?; let mut domain_buf = tokio::io::BufWriter::new(domain_file); let adblock_file = tokio::fs::File::create("output/adblock.txt").await?; let mut adblock_buf = tokio::io::BufWriter::new(adblock_file); let mut count = 0; for record in records { domain_buf.write_all(record.domain.as_bytes()).await?; domain_buf.write_all(b"\n").await?; adblock_buf.write_all(b"||").await?; adblock_buf.write_all(record.domain.as_bytes()).await?; adblock_buf.write_all(b"^\n").await?; count += 1; } domain_buf.flush().await?; log::info!("Wrote {} rules to output/domains.txt", count); adblock_buf.flush().await?; log::info!("Wrote {} rules to output/adblock.txt", count); } sqlx::query!("DELETE FROM allow_domains") .execute(&pool) .await?; sqlx::query!("DELETE FROM block_domains") .execute(&pool) .await?; Ok(()) } async fn garbage_collect_rule_source(pool: &sqlx::PgPool) -> Result<u64, ServerFnError> { let record = sqlx::query!( "delete from rule_source where not exists (select 1 from list_rules where source_id=rule_source.id)" ) .execute(pool) .await?; Ok(record.rows_affected()) } async fn garbage_collect_rules(pool: &sqlx::PgPool) -> Result<u64, ServerFnError> { let record = sqlx::query!( "delete from Rules where not exists (select 1 from rule_source where Rules.id=rule_source.rule_id)" ) .execute(pool) .await?; Ok(record.rows_affected()) } async fn garbage_collect_rule_matches(pool: &sqlx::PgPool) -> Result<u64, ServerFnError> { let record = sqlx::query!( "delete from rule_matches where not exists (select 1 from rules where Rules.id=rule_matches.rule_id)" ) .execute(pool) .await?; Ok(record.rows_affected()) } pub async fn garbage_collect() -> Result<(), ServerFnError> { let pool = get_db().await?; let gc_interval = std::env::var("GC_INTERVAL")?.parse::<u64>()?; let mut interval = tokio::time::interval(Duration::from_secs(gc_interval)); interval.tick().await; loop { interval.tick().await; let rows = garbage_collect_rule_source(&pool).await?; if rows > 0 { log::info!("Garbage collected {} rule sources", rows); } interval.tick().await; let rows = garbage_collect_rules(&pool).await?; if rows > 0 { log::info!("Garbage collected {} rules", rows); } interval.tick().await; let rows = garbage_collect_rule_matches(&pool).await?; if rows > 0 { log::info!("Garbage collected {} rule matches", rows); } } } pub async fn run_cmd(token: CancellationToken) -> Result<(), ServerFnError> { dotenvy::dotenv()?; let cmd = std::env::var("TASK_CMD")?; let mut interval = tokio::time::interval(Duration::from_secs(300)); loop { tokio::select! { _ = token.cancelled() => { log::info!("Shutting down run_cmd"); return Ok(());}, _ = interval.tick() => {}} let output = tokio::process::Command::new(&cmd).output().await; if let Err(err) = output { log::warn!("Error running command: {:?}", err); } } } const CERTSTREAM_URL: &str = "wss://certstream.calidog.io/domains-only"; #[derive(serde::Deserialize)] struct CertStreamMessage { data: Vec<String>, } async fn stream_certstream( domains: tokio::sync::mpsc::UnboundedSender<Domain>, ) -> Result<(), ServerFnError> { use futures::StreamExt; let (mut client, _) = tokio_tungstenite::connect_async(CERTSTREAM_URL).await?; while let Some(Ok(msg)) = client.next().await { if let tokio_tungstenite::tungstenite::protocol::Message::Text(msg) = msg { let msg: CertStreamMessage = serde_json::from_str(&msg)?; for domain in msg.data { let Ok(domain) = domain.parse::<Domain>() else { continue; }; domains.send(domain)?; } } } Ok(()) } async fn write_certstream( mut rx: tokio::sync::mpsc::UnboundedReceiver<Domain>, token: CancellationToken, ) -> Result<(), ServerFnError> { dotenvy::dotenv()?; let pool = get_db().await?; let interval = std::env::var("WRITE_FREQUENCY")?.parse::<u64>()?; let mut interval = tokio::time::interval(Duration::from_secs(interval)); interval.tick().await; loop { tokio::select! {_ = interval.tick() => {}, _ = token.cancelled() => log::info!("Shutting down certstream writer") } let mut domains = Vec::new(); while let Ok(domain) = rx.try_recv() { domains.push(domain.as_ref().to_string()); } if domains.is_empty() { continue; } let record = sqlx::query!( "INSERT INTO domains(domain) SELECT domain FROM UNNEST($1::text[]) as t(domain) ON CONFLICT DO NOTHING", &domains[..] ) .execute(&pool) .await?; log::info!( "Certstream inserted {} new domains (out of {} found)", record.rows_affected(), domains.len() ); if token.is_cancelled() { return Ok(()); } } } pub async fn certstream(token: CancellationToken) -> Result<(), ServerFnError> { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(async move { loop { if let Err(err) = stream_certstream(tx.clone()).await { log::warn!("Error streaming certstream: {:?}", err); tokio::time::sleep(Duration::from_secs(10)).await; } } }); write_certstream(rx, token).await?; Ok(()) } async fn dns_results(State(peer_state): State<PeerState>) {} #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] pub struct Peer { url: url::Url, get_dns_results: bool, } impl Default for Peer { fn default() -> Self { Self { url: "http://localhost:3000/".parse().unwrap(), get_dns_results: false, } } } #[derive(Clone)] pub struct PeerState { peers: Arc<[Peer]>, } #[derive(Debug, thiserror::Error)] enum PeerError { #[error("URL error: {0}")] Url(#[from] url::ParseError), #[error("Request error: {0}")] Request(#[from] reqwest::Error), #[error("Multiple errors: {0:?}")] Multiple(Vec<PeerError>), } impl PeerState { pub fn new(peers: &[Peer]) -> Self { Self { peers: peers.into() } } } pub fn get_peer_router(peer_state: PeerState) -> axum::Router<LeptosOptions> { axum::Router::new() .route("/dns-results", axum::routing::get(dns_results)) .with_state(peer_state) } ================================================ FILE: src/stats_view.rs ================================================ use leptos::*; #[server] async fn count_total_rules() -> Result<usize, ServerFnError> { let pool = crate::server::get_db().await?; let count = sqlx::query!( "SELECT reltuples::bigint AS count FROM pg_catalog.pg_class WHERE relname = 'rules'" ) .fetch_one(&pool) .await? .count .ok_or_else(|| ServerFnError::new("No count"))? as usize; Ok(count) } #[component] fn TotalRuleCount() -> impl IntoView { view! { <Await future=|| async { count_total_rules().await } let:total_rules> {match total_rules { Ok(count) => view! { {count} }.into_view(), Err(err) => view! { {format!("{err:?}")} }.into_view(), }} </Await> } } #[server] async fn get_total_rule_matches() -> Result<usize, ServerFnError> { let pool = crate::server::get_db().await?; let count = sqlx::query!( "SELECT reltuples::bigint AS count FROM pg_catalog.pg_class WHERE relname = 'rule_matches'" ) .fetch_one(&pool) .await? .count .ok_or_else(|| ServerFnError::new("No count"))? as usize; Ok(count) } #[component] fn TotalRuleMatches() -> impl IntoView { view! { <Await future=|| async { get_total_rule_matches().await } let:total_rule_matches> {match total_rule_matches { Ok(count) => view! { {count} }.into_view(), Err(err) => view! { {format!("{err:?}")} }.into_view(), }} </Await> } } #[server] async fn get_domain_count() -> Result<usize, ServerFnError> { let pool = crate::server::get_db().await?; let count = sqlx::query!( "SELECT reltuples::bigint AS count FROM pg_catalog.pg_class WHERE relname = 'domains'" ) .fetch_one(&pool) .await? .count .ok_or_else(|| ServerFnError::new("No count"))? as usize; Ok(count) } #[component] fn DomainCount() -> impl IntoView { view! { <Await future=|| async { get_domain_count().await } let:domain_count> {match domain_count { Ok(count) => view! { {count} }.into_view(), Err(err) => view! { {format!("{err:?}")} }.into_view(), }} </Await> } } #[server] async fn get_subdomains_count() -> Result<usize, ServerFnError> { let pool = crate::server::get_db().await?; let count = sqlx::query!( "SELECT reltuples::bigint AS count FROM pg_catalog.pg_class WHERE relname = 'subdomains'" ) .fetch_one(&pool) .await? .count .ok_or_else(|| ServerFnError::new("No count"))? as usize; Ok(count) } #[component] fn SubdomainCount() -> impl IntoView { view! { <Await future=|| async { get_subdomains_count().await } let:subdomain_count> {match subdomain_count { Ok(count) => view! { {count} }.into_view(), Err(err) => view! { {format!("{err:?}")} }.into_view(), }} </Await> } } #[server] async fn get_dns_ip_count() -> Result<usize, ServerFnError> { let pool = crate::server::get_db().await?; let count = sqlx::query!( "SELECT reltuples::bigint AS count FROM pg_catalog.pg_class WHERE relname = 'dns_ips'" ) .fetch_one(&pool) .await? .count .ok_or_else(|| ServerFnError::new("No count"))? as usize; Ok(count) } #[component] fn DnsIpCount() -> impl IntoView { view! { <Await future=|| async { get_dns_ip_count().await } let:dns_ip_count> {match dns_ip_count { Ok(count) => view! { {count} }.into_view(), Err(err) => view! { {format!("{err:?}")} }.into_view(), }} </Await> } } #[server] async fn get_dns_cname_count() -> Result<usize, ServerFnError> { let pool = crate::server::get_db().await?; let count = sqlx::query!( "SELECT reltuples::bigint AS count FROM pg_catalog.pg_class WHERE relname = 'dns_cnames'" ) .fetch_one(&pool) .await? .count .ok_or_else(|| ServerFnError::new("No count"))? as usize; Ok(count) } #[component] fn DnsCnameCount() -> impl IntoView { view! { <Await future=|| async { get_dns_cname_count().await } let:dns_ip_count> {match dns_ip_count { Ok(count) => view! { {count} }.into_view(), Err(err) => view! { {format!("{err:?}")} }.into_view(), }} </Await> } } #[component] pub fn StatsView() -> impl IntoView { view! { <div> <h1 class="mt-5 mb-5 text-4xl font-bold text-center text-indigo-600">Stats</h1> <table class="table max-w-fit"> <tr> <td>"Total Domains"</td> <td class="text-right"> <DomainCount/> </td> </tr> <tr> <td>"Total DNS IPs"</td> <td class="text-right"> <DnsIpCount/> </td> </tr> <tr> <td>"Total DNS CNAMES"</td> <td class="text-right"> <DnsCnameCount/> </td> </tr> <tr> <td>"Total Subdomains"</td> <td class="text-right"> <SubdomainCount/> </td> </tr> <tr> <td>"Total Rules"</td> <td class="text-right"> <TotalRuleCount/> </td> </tr> <tr> <td>"Total Rule Matches"</td> <td class="text-right"> <TotalRuleMatches/> </td> </tr> </table> </div> } } ================================================ FILE: src/tasks.rs ================================================ use leptos::*; trait Task { type Error; fn name(&self) -> &str; async fn run_once(&self) -> Result<String, Self::Error>; } #[cfg(feature = "ssr")] struct GarbageCollectRuleSource {} #[cfg(feature = "ssr")] impl Task for GarbageCollectRuleSource { type Error = ServerFnError; fn name(&self) -> &str { "Garbage collect rule_source" } async fn run_once(&self) -> Result<String, Self::Error> { let pool = crate::server::get_db().await?; let rows_removed = sqlx::query!( "delete from rule_source where not exists (select 1 from list_rules where source_id=rule_source.id)" ) .execute(&pool) .await? .rows_affected(); Ok(format!( "Garbage collected {} rows from rule_source", rows_removed )) } } #[cfg(feature = "ssr")] async fn register_task<T: Task>(_task: T) { let _pool = crate::server::get_db().await.unwrap(); } #[component] pub fn TaskView() -> impl IntoView { view! { <div> <h1>"Tasks"</h1> <p>"This is the tasks view"</p> </div> } } ================================================ FILE: style/main.scss ================================================ ================================================ FILE: style/tailwind.css ================================================ @tailwind base; @tailwind components; @tailwind utilities; ================================================ FILE: tailwind.config.js ================================================ /** @type {import('tailwindcss').Config} */ module.exports = { content: { relative: true, files: ["*.html", "./src/*", "./src/**/*.rs"], }, theme: { extend: {}, }, plugins: [require("daisyui")], } ================================================ FILE: tld_list.txt ================================================ # Version 2020071800, Last Updated Sat Jul 18 07:07:01 2020 UTC AAA AARP ABARTH ABB ABBOTT ABBVIE ABC ABLE ABOGADO ABUDHABI AC ACADEMY ACCENTURE ACCOUNTANT ACCOUNTANTS ACO ACTOR AD ADAC ADS ADULT AE AEG AERO AETNA AF AFAMILYCOMPANY AFL AFRICA AG AGAKHAN AGENCY AI AIG AIRBUS AIRFORCE AIRTEL AKDN AL ALFAROMEO ALIBABA ALIPAY ALLFINANZ ALLSTATE ALLY ALSACE ALSTOM AM AMAZON AMERICANEXPRESS AMERICANFAMILY AMEX AMFAM AMICA AMSTERDAM ANALYTICS ANDROID ANQUAN ANZ AO AOL APARTMENTS APP APPLE AQ AQUARELLE AR ARAB ARAMCO ARCHI ARMY ARPA ART ARTE AS ASDA ASIA ASSOCIATES AT ATHLETA ATTORNEY AU AUCTION AUDI AUDIBLE AUDIO AUSPOST AUTHOR AUTO AUTOS AVIANCA AW AWS AX AXA AZ AZURE BA BABY BAIDU BANAMEX BANANAREPUBLIC BAND BANK BAR BARCELONA BARCLAYCARD BARCLAYS BAREFOOT BARGAINS BASEBALL BASKETBALL BAUHAUS BAYERN BB BBC BBT BBVA BCG BCN BD BE BEATS BEAUTY BEER BENTLEY BERLIN BEST BESTBUY BET BF BG BH BHARTI BI BIBLE BID BIKE BING BINGO BIO BIZ BJ BLACK BLACKFRIDAY BLOCKBUSTER BLOG BLOOMBERG BLUE BM BMS BMW BN BNPPARIBAS BO BOATS BOEHRINGER BOFA BOM BOND BOO BOOK BOOKING BOSCH BOSTIK BOSTON BOT BOUTIQUE BOX BR BRADESCO BRIDGESTONE BROADWAY BROKER BROTHER BRUSSELS BS BT BUDAPEST BUGATTI BUILD BUILDERS BUSINESS BUY BUZZ BV BW BY BZ BZH CA CAB CAFE CAL CALL CALVINKLEIN CAM CAMERA CAMP CANCERRESEARCH CANON CAPETOWN CAPITAL CAPITALONE CAR CARAVAN CARDS CARE CAREER CAREERS CARS CASA CASE CASEIH CASH CASINO CAT CATERING CATHOLIC CBA CBN CBRE CBS CC CD CEB CENTER CEO CERN CF CFA CFD CG CH CHANEL CHANNEL CHARITY CHASE CHAT CHEAP CHINTAI CHRISTMAS CHROME CHURCH CI CIPRIANI CIRCLE CISCO CITADEL CITI CITIC CITY CITYEATS CK CL CLAIMS CLEANING CLICK CLINIC CLINIQUE CLOTHING CLOUD CLUB CLUBMED CM CN CO COACH CODES COFFEE COLLEGE COLOGNE COM COMCAST COMMBANK COMMUNITY COMPANY COMPARE COMPUTER COMSEC CONDOS CONSTRUCTION CONSULTING CONTACT CONTRACTORS COOKING COOKINGCHANNEL COOL COOP CORSICA COUNTRY COUPON COUPONS COURSES CPA CR CREDIT CREDITCARD CREDITUNION CRICKET CROWN CRS CRUISE CRUISES CSC CU CUISINELLA CV CW CX CY CYMRU CYOU CZ DABUR DAD DANCE DATA DATE DATING DATSUN DAY DCLK DDS DE DEAL DEALER DEALS DEGREE DELIVERY DELL DELOITTE DELTA DEMOCRAT DENTAL DENTIST DESI DESIGN DEV DHL DIAMONDS DIET DIGITAL DIRECT DIRECTORY DISCOUNT DISCOVER DISH DIY DJ DK DM DNP DO DOCS DOCTOR DOG DOMAINS DOT DOWNLOAD DRIVE DTV DUBAI DUCK DUNLOP DUPONT DURBAN DVAG DVR DZ EARTH EAT EC ECO EDEKA EDU EDUCATION EE EG EMAIL EMERCK ENERGY ENGINEER ENGINEERING ENTERPRISES EPSON EQUIPMENT ER ERICSSON ERNI ES ESQ ESTATE ET ETISALAT EU EUROVISION EUS EVENTS EXCHANGE EXPERT EXPOSED EXPRESS EXTRASPACE FAGE FAIL FAIRWINDS FAITH FAMILY FAN FANS FARM FARMERS FASHION FAST FEDEX FEEDBACK FERRARI FERRERO FI FIAT FIDELITY FIDO FILM FINAL FINANCE FINANCIAL FIRE FIRESTONE FIRMDALE FISH FISHING FIT FITNESS FJ FK FLICKR FLIGHTS FLIR FLORIST FLOWERS FLY FM FO FOO FOOD FOODNETWORK FOOTBALL FORD FOREX FORSALE FORUM FOUNDATION FOX FR FREE FRESENIUS FRL FROGANS FRONTDOOR FRONTIER FTR FUJITSU FUJIXEROX FUN FUND FURNITURE FUTBOL FYI GA GAL GALLERY GALLO GALLUP GAME GAMES GAP GARDEN GAY GB GBIZ GD GDN GE GEA GENT GENTING GEORGE GF GG GGEE GH GI GIFT GIFTS GIVES GIVING GL GLADE GLASS GLE GLOBAL GLOBO GM GMAIL GMBH GMO GMX GN GODADDY GOLD GOLDPOINT GOLF GOO GOODYEAR GOOG GOOGLE GOP GOT GOV GP GQ GR GRAINGER GRAPHICS GRATIS GREEN GRIPE GROCERY GROUP GS GT GU GUARDIAN GUCCI GUGE GUIDE GUITARS GURU GW GY HAIR HAMBURG HANGOUT HAUS HBO HDFC HDFCBANK HEALTH HEALTHCARE HELP HELSINKI HERE HERMES HGTV HIPHOP HISAMITSU HITACHI HIV HK HKT HM HN HOCKEY HOLDINGS HOLIDAY HOMEDEPOT HOMEGOODS HOMES HOMESENSE HONDA HORSE HOSPITAL HOST HOSTING HOT HOTELES HOTELS HOTMAIL HOUSE HOW HR HSBC HT HU HUGHES HYATT HYUNDAI IBM ICBC ICE ICU ID IE IEEE IFM IKANO IL IM IMAMAT IMDB IMMO IMMOBILIEN IN INC INDUSTRIES INFINITI INFO ING INK INSTITUTE INSURANCE INSURE INT INTEL INTERNATIONAL INTUIT INVESTMENTS IO IPIRANGA IQ IR IRISH IS ISMAILI IST ISTANBUL IT ITAU ITV IVECO JAGUAR JAVA JCB JCP JE JEEP JETZT JEWELRY JIO JLL JM JMP JNJ JO JOBS JOBURG JOT JOY JP JPMORGAN JPRS JUEGOS JUNIPER KAUFEN KDDI KE KERRYHOTELS KERRYLOGISTICS KERRYPROPERTIES KFH KG KH KI KIA KIM KINDER KINDLE KITCHEN KIWI KM KN KOELN KOMATSU KOSHER KP KPMG KPN KR KRD KRED KUOKGROUP KW KY KYOTO KZ LA LACAIXA LAMBORGHINI LAMER LANCASTER LANCIA LAND LANDROVER LANXESS LASALLE LAT LATINO LATROBE LAW LAWYER LB LC LDS LEASE LECLERC LEFRAK LEGAL LEGO LEXUS LGBT LI LIDL LIFE LIFEINSURANCE LIFESTYLE LIGHTING LIKE LILLY LIMITED LIMO LINCOLN LINDE LINK LIPSY LIVE LIVING LIXIL LK LLC LLP LOAN LOANS LOCKER LOCUS LOFT LOL LONDON LOTTE LOTTO LOVE LPL LPLFINANCIAL LR LS LT LTD LTDA LU LUNDBECK LUPIN LUXE LUXURY LV LY MA MACYS MADRID MAIF MAISON MAKEUP MAN MANAGEMENT MANGO MAP MARKET MARKETING MARKETS MARRIOTT MARSHALLS MASERATI MATTEL MBA MC MCKINSEY MD ME MED MEDIA MEET MELBOURNE MEME MEMORIAL MEN MENU MERCKMSD METLIFE MG MH MIAMI MICROSOFT MIL MINI MINT MIT MITSUBISHI MK ML MLB MLS MM MMA MN MO MOBI MOBILE MODA MOE MOI MOM MONASH MONEY MONSTER MORMON MORTGAGE MOSCOW MOTO MOTORCYCLES MOV MOVIE MP MQ MR MS MSD MT MTN MTR MU MUSEUM MUTUAL MV MW MX MY MZ NA NAB NAGOYA NAME NATIONWIDE NATURA NAVY NBA NC NE NEC NET NETBANK NETFLIX NETWORK NEUSTAR NEW NEWHOLLAND NEWS NEXT NEXTDIRECT NEXUS NF NFL NG NGO NHK NI NICO NIKE NIKON NINJA NISSAN NISSAY NL NO NOKIA NORTHWESTERNMUTUAL NORTON NOW NOWRUZ NOWTV NP NR NRA NRW NTT NU NYC NZ OBI OBSERVER OFF OFFICE OKINAWA OLAYAN OLAYANGROUP OLDNAVY OLLO OM OMEGA ONE ONG ONL ONLINE ONYOURSIDE OOO OPEN ORACLE ORANGE ORG ORGANIC ORIGINS OSAKA OTSUKA OTT OVH PA PAGE PANASONIC PARIS PARS PARTNERS PARTS PARTY PASSAGENS PAY PCCW PE PET PF PFIZER PG PH PHARMACY PHD PHILIPS PHONE PHOTO PHOTOGRAPHY PHOTOS PHYSIO PICS PICTET PICTURES PID PIN PING PINK PIONEER PIZZA PK PL PLACE PLAY PLAYSTATION PLUMBING PLUS PM PN PNC POHL POKER POLITIE PORN POST PR PRAMERICA PRAXI PRESS PRIME PRO PROD PRODUCTIONS PROF PROGRESSIVE PROMO PROPERTIES PROPERTY PROTECTION PRU PRUDENTIAL PS PT PUB PW PWC PY QA QPON QUEBEC QUEST QVC RACING RADIO RAID RE READ REALESTATE REALTOR REALTY RECIPES RED REDSTONE REDUMBRELLA REHAB REISE REISEN REIT RELIANCE REN RENT RENTALS REPAIR REPORT REPUBLICAN REST RESTAURANT REVIEW REVIEWS REXROTH RICH RICHARDLI RICOH RIGHTATHOME RIL RIO RIP RMIT RO ROCHER ROCKS RODEO ROGERS ROOM RS RSVP RU RUGBY RUHR RUN RW RWE RYUKYU SA SAARLAND SAFE SAFETY SAKURA SALE SALON SAMSCLUB SAMSUNG SANDVIK SANDVIKCOROMANT SANOFI SAP SARL SAS SAVE SAXO SB SBI SBS SC SCA SCB SCHAEFFLER SCHMIDT SCHOLARSHIPS SCHOOL SCHULE SCHWARZ SCIENCE SCJOHNSON SCOT SD SE SEARCH SEAT SECURE SECURITY SEEK SELECT SENER SERVICES SES SEVEN SEW SEX SEXY SFR SG SH SHANGRILA SHARP SHAW SHELL SHIA SHIKSHA SHOES SHOP SHOPPING SHOUJI SHOW SHOWTIME SHRIRAM SI SILK SINA SINGLES SITE SJ SK SKI SKIN SKY SKYPE SL SLING SM SMART SMILE SN SNCF SO SOCCER SOCIAL SOFTBANK SOFTWARE SOHU SOLAR SOLUTIONS SONG SONY SOY SPACE SPORT SPOT SPREADBETTING SR SRL SS ST STADA STAPLES STAR STATEBANK STATEFARM STC STCGROUP STOCKHOLM STORAGE STORE STREAM STUDIO STUDY STYLE SU SUCKS SUPPLIES SUPPLY SUPPORT SURF SURGERY SUZUKI SV SWATCH SWIFTCOVER SWISS SX SY SYDNEY SYSTEMS SZ TAB TAIPEI TALK TAOBAO TARGET TATAMOTORS TATAR TATTOO TAX TAXI TC TCI TD TDK TEAM TECH TECHNOLOGY TEL TEMASEK TENNIS TEVA TF TG TH THD THEATER THEATRE TIAA TICKETS TIENDA TIFFANY TIPS TIRES TIROL TJ TJMAXX TJX TK TKMAXX TL TM TMALL TN TO TODAY TOKYO TOOLS TOP TORAY TOSHIBA TOTAL TOURS TOWN TOYOTA TOYS TR TRADE TRADING TRAINING TRAVEL TRAVELCHANNEL TRAVELERS TRAVELERSINSURANCE TRUST TRV TT TUBE TUI TUNES TUSHU TV TVS TW TZ UA UBANK UBS UG UK UNICOM UNIVERSITY UNO UOL UPS US UY UZ VA VACATIONS VANA VANGUARD VC VE VEGAS VENTURES VERISIGN VERSICHERUNG VET VG VI VIAJES VIDEO VIG VIKING VILLAS VIN VIP VIRGIN VISA VISION VIVA VIVO VLAANDEREN VN VODKA VOLKSWAGEN VOLVO VOTE VOTING VOTO VOYAGE VU VUELOS WALES WALMART WALTER WANG WANGGOU WATCH WATCHES WEATHER WEATHERCHANNEL WEBCAM WEBER WEBSITE WED WEDDING WEIBO WEIR WF WHOSWHO WIEN WIKI WILLIAMHILL WIN WINDOWS WINE WINNERS WME WOLTERSKLUWER WOODSIDE WORK WORKS WORLD WOW WS WTC WTF XBOX XEROX XFINITY XIHUAN XIN XN--11B4C3D XN--1CK2E1B XN--1QQW23A XN--2SCRJ9C XN--30RR7Y XN--3BST00M XN--3DS443G XN--3E0B707E XN--3HCRJ9C XN--3OQ18VL8PN36A XN--3PXU8K XN--42C2D9A XN--45BR5CYL XN--45BRJ9C XN--45Q11C XN--4GBRIM XN--54B7FTA0CC XN--55QW42G XN--55QX5D XN--5SU34J936BGSG XN--5TZM5G XN--6FRZ82G XN--6QQ986B3XL XN--80ADXHKS XN--80AO21A XN--80AQECDR1A XN--80ASEHDB XN--80ASWG XN--8Y0A063A XN--90A3AC XN--90AE XN--90AIS XN--9DBQ2A XN--9ET52U XN--9KRT00A XN--B4W605FERD XN--BCK1B9A5DRE4C XN--C1AVG XN--C2BR7G XN--CCK2B3B XN--CCKWCXETD XN--CG4BKI XN--CLCHC0EA0B2G2A9GCD XN--CZR694B XN--CZRS0T XN--CZRU2D XN--D1ACJ3B XN--D1ALF XN--E1A4C XN--ECKVDTC9D XN--EFVY88H XN--FCT429K XN--FHBEI XN--FIQ228C5HS XN--FIQ64B XN--FIQS8S XN--FIQZ9S XN--FJQ720A XN--FLW351E XN--FPCRJ9C3D XN--FZC2C9E2C XN--FZYS8D69UVGM XN--G2XX48C XN--GCKR3F0F XN--GECRJ9C XN--GK3AT1E XN--H2BREG3EVE XN--H2BRJ9C XN--H2BRJ9C8C XN--HXT814E XN--I1B6B1A6A2E XN--IMR513N XN--IO0A7I XN--J1AEF XN--J1AMH XN--J6W193G XN--JLQ480N2RG XN--JLQ61U9W7B XN--JVR189M XN--KCRX77D1X4A XN--KPRW13D XN--KPRY57D XN--KPUT3I XN--L1ACC XN--LGBBAT1AD8J XN--MGB9AWBF XN--MGBA3A3EJT XN--MGBA3A4F16A XN--MGBA7C0BBN0A XN--MGBAAKC7DVF XN--MGBAAM7A8H XN--MGBAB2BD XN--MGBAH1A3HJKRD XN--MGBAI9AZGQP6J XN--MGBAYH7GPA XN--MGBBH1A XN--MGBBH1A71E XN--MGBC0A9AZCG XN--MGBCA7DZDO XN--MGBCPQ6GPA1A XN--MGBERP4A5D4AR XN--MGBGU82A XN--MGBI4ECEXP XN--MGBPL2FH XN--MGBT3DHD XN--MGBTX2B XN--MGBX4CD0AB XN--MIX891F XN--MK1BU44C XN--MXTQ1M XN--NGBC5AZD XN--NGBE9E0A XN--NGBRX XN--NODE XN--NQV7F XN--NQV7FS00EMA XN--NYQY26A XN--O3CW4H XN--OGBPF8FL XN--OTU796D XN--P1ACF XN--P1AI XN--PGBS0DH XN--PSSY2U XN--Q7CE6A XN--Q9JYB4C XN--QCKA1PMC XN--QXA6A XN--QXAM XN--RHQV96G XN--ROVU88B XN--RVC1E0AM3E XN--S9BRJ9C XN--SES554G XN--T60B56A XN--TCKWE XN--TIQ49XQYJ XN--UNUP4Y XN--VERMGENSBERATER-CTB XN--VERMGENSBERATUNG-PWB XN--VHQUV XN--VUQ861B XN--W4R85EL8FHU5DNRA XN--W4RS40L XN--WGBH1C XN--WGBL6A XN--XHQ521B XN--XKC2AL3HYE2A XN--XKC2DL3A5EE0H XN--Y9A3AQ XN--YFRO4I67O XN--YGBI2AMMX XN--ZFR164B XXX XYZ YACHTS YAHOO YAMAXUN YANDEX YE YODOBASHI YOGA YOKOHAMA YOU YOUTUBE YT YUN ZA ZAPPOS ZARA ZERO ZIP ZM ZONE ZUERICH ZW ================================================ FILE: update.sh ================================================ set -e git pull cargo run --release generate git commit output -m "Ran Autoupdate" git push