Showing preview only (232K chars total). Download the full file or copy to clipboard to get everything.
Repository: mkb2091/blockconvert
Branch: master
Commit: 4dcb5af3bc14
Files: 107
Total size: 84.6 MB
Directory structure:
gitextract_u7y2p1i3/
├── .cargo/
│ └── config.toml
├── .gitattributes
├── .gitignore
├── .vscode/
│ ├── launch.json
│ └── settings.json
├── CONTRIBUTING.md
├── Cargo.toml
├── LICENSE
├── README.md
├── _config.yml
├── build.rs
├── end2end/
│ ├── package.json
│ ├── playwright.config.ts
│ └── tests/
│ └── example.spec.ts
├── filterlists.csv
├── internal/
│ ├── adblock.txt
│ ├── allow_regex.txt
│ ├── allowlist.txt
│ ├── block_ipnets.txt
│ ├── block_ips.txt
│ ├── block_regex.txt
│ └── blocklist.txt
├── local.toml
├── local2.toml
├── migrations/
│ ├── 20240222234126_filterlists.sql
│ ├── 20240223010915_lastmodified.sql
│ ├── 20240223011106_lastmodified.sql
│ ├── 20240223011400_etag.sql
│ ├── 20240224194439_rules.sql
│ ├── 20240224195223_filter_list_contents.sql
│ ├── 20240224203224_list_rules.sql
│ ├── 20240225214254_list_source.sql
│ ├── 20240225230839_index.sql
│ ├── 20240225231841_index.sql
│ ├── 20240225232249_index.sql
│ ├── 20240226004619_change_date.sql
│ ├── 20240226013547_source.sql
│ ├── 20240226152939_temp.sql
│ ├── 20240226215547_remove_column.sql
│ ├── 20240226215929_remove_id.sql
│ ├── 20240226220711_remove_fkey.sql
│ ├── 20240226223817_domain_rules.sql
│ ├── 20240226230317_domain_block.sql
│ ├── 20240226230425_rename.sql
│ ├── 20240227191530_drop_column.sql
│ ├── 20240227194830_drop_column.sql
│ ├── 20240227200629_drop_table.sql
│ ├── 20240227201410_primary_key.sql
│ ├── 20240228001134_domain_rules.sql
│ ├── 20240228004601_extend_rules.sql
│ ├── 20240228005409_drop_rule.sql
│ ├── 20240228015906_change_unique.sql
│ ├── 20240228164553_ip.sql
│ ├── 20240228170646_ip.sql
│ ├── 20240228175807_remove_unknown.sql
│ ├── 20240228180753_index.sql
│ ├── 20240229210101_domains.sql
│ ├── 20240229212527_subdomains.sql
│ ├── 20240301000455_more_indexes.sql
│ ├── 20240301000900_subdomain_idx.sql
│ ├── 20240301030213_subdomain_inde.sql
│ ├── 20240302171950_expanded_subdomains.sql
│ ├── 20240302184040_processed_subdomains.sql
│ ├── 20240302192658_index.sql
│ ├── 20240302194037_domain_rule_id_idx.sql
│ ├── 20240302194733_not_null_parent.sql
│ ├── 20240302205633_not_null#.sql
│ ├── 20240302222401_dns.sql
│ ├── 20240304235746_filterlist.sql
│ ├── 20240305000257_filterlist.sql
│ ├── 20240305000612_filterlist.sql
│ ├── 20240305003411_filterlist.sql
│ ├── 20240305200551_rule_matches.sql
│ ├── 20240306123603_rule_count.sql
│ ├── 20240306214217_index.sql
│ ├── 20240307005702_lists.sql
│ ├── 20240307012450_index.sql
│ └── 20240307031445_idx.sql
├── output/
│ ├── adblock.txt
│ ├── allowed_ips.txt
│ ├── domains.rpz
│ ├── domains.txt
│ ├── hosts.txt
│ ├── ip_blocklist.txt
│ ├── whitelist_adblock.txt
│ └── whitelist_domains.txt
├── package.json
├── rust-toolchain.toml
├── rustfmt.toml
├── src/
│ ├── app.rs
│ ├── domain.rs
│ ├── error_template.rs
│ ├── fileserv.rs
│ ├── filterlist.rs
│ ├── home_page.rs
│ ├── ip_view.rs
│ ├── lib.rs
│ ├── main.rs
│ ├── rule.rs
│ ├── server.rs
│ ├── stats_view.rs
│ └── tasks.rs
├── style/
│ ├── main.scss
│ └── tailwind.css
├── tailwind.config.js
├── tld_list.txt
└── update.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .cargo/config.toml
================================================
[target.x86_64-unknown-linux-gnu]
rustflags = ["-Clink-arg=-fuse-ld=mold", "-Ctarget-cpu=native"]
================================================
FILE: .gitattributes
================================================
# Auto detect text files and perform LF normalization
* text eol=lf
================================================
FILE: .gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
dns_cache.txt
data/
db/
extracted/
passive_dns_db/
dns_db/
#Added by cargo
/target
Cargo.lock
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional stylelint cache
.stylelintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# vuepress v2.x temp and cache directory
.temp
.cache
# Docusaurus cache and generated files
.docusaurus
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
task_cmd
================================================
FILE: .vscode/launch.json
================================================
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug",
"program": "${workspaceFolder}/<executable file>",
"args": [],
"cwd": "${workspaceFolder}"
}
]
}
================================================
FILE: .vscode/settings.json
================================================
{
"rust-analyzer.linkedProjects": [
"./Cargo.toml",
],
"rust-analyzer.cargo.buildScripts.overrideCommand": null,
"emmet.includeLanguages": {
"rust": "html",
"*.rs": "html"
},
"tailwindCSS.includeLanguages": {
"rust": "html",
"*.rs": "html"
},
"files.associations": {
"*.rs": "rust"
},
"editor.quickSuggestions": {
"other": "on",
"comments": "on",
"strings": true
},
"css.validate": false,
"editor.fontWeight": "normal",
}
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing
If you are contibuting, then thank you, it is appreciated. Below is what requirements a change must have for it to be added, and what information would be useful to have.
## Adding to whitelist
Requirement(s):
- Adding domain to whitelist must fix broken functionality/blocked first-party websites, advert domains will not be whitelisted.
Information needed:
- What website is blocked/broken, for third-party domains, just the blocked domains that need to be whitelisted isn't enough.
## Adding a new filter list
Requirement(s):
- URL must be to original host(unless original no longer exists), not a mirror/processed version
- License must be compatible with GPLv3
Information needed:
- URL to filter list
- Whether filter list is blacklist or whitelist
- Whether it is a list of malicious paths(will result in base domains being added)
## Adding a domain to blacklist
Requirement(s):
- Must not break pages
Information needed:
- Reason for adding domain, eg for adverts, example website using it, for malicious domains, virustotal report or other proof that it is malicious
================================================
FILE: Cargo.toml
================================================
[package]
name = "blockconvert"
version = "0.1.0"
edition = "2021"
[lib]
crate-type = ["cdylib", "lib"]
[dependencies]
serde = { version = "1.0", features = ["derive", "rc"] }
url = { version = "2.5.0", features = ["serde"] }
axum = { version = "0.7", optional = true }
console_error_panic_hook = { version = "0.1", optional = true }
leptos = { git = "https://github.com/leptos-rs/leptos", version = "0.6", features = [
"nonce",
] }
leptos_axum = { git = "https://github.com/leptos-rs/leptos", version = "0.6", features = [
"nonce",
], optional = true }
leptos_meta = { git = "https://github.com/leptos-rs/leptos", version = "0.6", features = [
] }
leptos_router = { git = "https://github.com/leptos-rs/leptos", version = "0.6", features = [
] }
tokio = { version = "1", features = [
"rt-multi-thread",
"parking_lot",
"process",
"signal",
], optional = true }
tokio-util = { version = "0.7", optional = true }
tower = { version = "0.4", optional = true }
tower-http = { version = "0.5", features = [
"fs",
"compression-br",
"decompression-br",
], optional = true }
wasm-bindgen = { version = "=0.2.92", default-features = false, optional = true }
thiserror = "1"
http = "1"
csv = { version = "1.3.0", optional = true }
env_logger = { version = "0.11.3", optional = true }
log = "0.4.21"
console_log = { version = "1.0", features = ["color"], optional = true }
sqlx = { version = "0.7", features = [
"runtime-tokio",
"chrono",
"postgres",
"ipnetwork",
], optional = true }
reqwest = { version = "0.11.26", features = [
"native-tls",
"brotli",
], default-features = false, optional = true }
chrono = { version = "0.4", features = ["serde"] }
dotenvy = { version = "0.15.7", optional = true }
mimalloc = { version = "0.1.39", default-features = false, optional = true }
hickory-resolver = { version = "0.24.0", features = [
"tokio-runtime",
], optional = true }
addr = "0.15.6"
ipnetwork = "0.20.0"
futures = { version = "0.3.30", optional = true }
hickory-proto = { version = "0.24.0", default-features = false }
humantime = "2.1.0"
notify = { version = "6.1.1", optional = true }
async-channel = { version = "2.2.0", optional = true }
tokio-tungstenite = { version = "0.21.0", features = [
"native-tls",
], optional = true }
serde_json = { version = "1.0.114", optional = true }
metrics = { version = "0.22", optional = true }
metrics-exporter-prometheus = { version = "0.13.1", default-features = false, features = [
"push-gateway",
], optional = true }
clap = { version = "4.5.2", features = ["derive"], optional = true }
rand = {version = "0.8", optional = true}
toml = {version = "0.8", optional = true}
[features]
hydrate = [
"leptos/hydrate",
"leptos_meta/hydrate",
"leptos_router/hydrate",
"dep:console_log",
"dep:console_error_panic_hook",
"dep:wasm-bindgen",
]
ssr = [
"dep:axum",
"dep:tokio",
"dep:tokio-util",
"dep:tower",
"dep:tower-http",
"dep:leptos_axum",
"leptos/ssr",
"leptos_meta/ssr",
"leptos_router/ssr",
"dep:csv",
"dep:env_logger",
"dep:sqlx",
"dep:reqwest",
"dep:dotenvy",
"dep:mimalloc",
"dns_resolver",
"dep:notify",
"dep:async-channel",
"dep:tokio-tungstenite",
"dep:serde_json",
"dep:futures",
"dep:metrics",
"dep:metrics-exporter-prometheus",
"dep:clap",
"dep:rand",
"dep:toml",
]
dns_resolver = ["dep:hickory-resolver"]
default = ["ssr"]
# Defines a size-optimized profile for the WASM bundle in release mode
[profile.wasm-release]
inherits = "release"
opt-level = 'z'
lto = true
codegen-units = 1
panic = "abort"
[profile.wasm-dev]
inherits = "dev"
debug = 0
[profile.wasm-dev.package."*"]
opt-level = 's'
[profile.server-dev]
inherits = "dev"
opt-level = 1
debug = 0
lto = "off"
[profile.dev.package."*"]
opt-level = 3
[package.metadata.leptos]
# The name used by wasm-bindgen/cargo-leptos for the JS/WASM bundle. Defaults to the crate name
output-name = "blockconvert"
# The site root folder is where cargo-leptos generate all output. WARNING: all content of this folder will be erased on a rebuild. Use it in your server setup.
site-root = "target/blockconvert"
# The site-root relative folder where all compiled output (JS, WASM and CSS) is written
# Defaults to pkg
site-pkg-dir = "pkg"
# [Optional] The source CSS file. If it ends with .sass or .scss then it will be compiled by dart-sass into CSS. The CSS is optimized by Lightning CSS before being written to <site-root>/<site-pkg>/app.css
style-file = "style/main.scss"
# Assets source dir. All files found here will be copied and synchronized to site-root.
# The assets-dir cannot have a sub directory with the same name/path as site-pkg-dir.
#
# Optional. Env: LEPTOS_ASSETS_DIR.
assets-dir = "public"
# The IP and port (ex: 127.0.0.1:3000) where the server serves the content. Use it in your server setup.
site-addr = "127.0.0.1:3000"
# The port to use for automatic reload monitoring
reload-port = 3001
# [Optional] Command to use when running end2end tests. It will run in the end2end dir.
# [Windows] for non-WSL use "npx.cmd playwright test"
# This binary name can be checked in Powershell with Get-Command npx
end2end-cmd = "npx playwright test"
end2end-dir = "end2end"
# The browserlist query used for optimizing the CSS.
browserquery = "defaults"
# Set by cargo-leptos watch when building with that tool. Controls whether autoreload JS will be included in the head
watch = false
# The environment Leptos will run in, usually either "DEV" or "PROD"
env = "DEV"
# The features to use when compiling the bin target
#
# Optional. Can be over-ridden with the command line parameter --bin-features
bin-features = ["ssr"]
# If the --no-default-features flag should be used when compiling the bin target
#
# Optional. Defaults to false.
bin-default-features = false
bin-profile-dev = "server-dev"
# The features to use when compiling the lib target
#
# Optional. Can be over-ridden with the command line parameter --lib-features
lib-features = ["hydrate"]
# If the --no-default-features flag should be used when compiling the lib target
#
# Optional. Defaults to false.
lib-default-features = false
# The profile to use for the lib target when compiling for release
#
# Optional. Defaults to "release".
lib-profile-release = "wasm-release"
lib-profile-dev = "wasm-dev"
# The tailwind input file.
#
# Optional, Activates the tailwind build
tailwind-input-file = "style/tailwind.css"
# The tailwind config file.
#
# Optional, defaults to "tailwind.config.js" which if is not present
# is generated for you
tailwind-config-file = "tailwind.config.js"
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2022 henrik
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# BlockConvert
Malware, advert and tracking blocklist which consolidates and improves upon many other [blocklists](https://github.com/mkb2091/blockconvert/blob/master/filterlists.csv).
## What this blocks:
- Malware/Phishing: Many malware lists are used in building this list, including multiple malware IP lists, which are used to find many more malware domains.
- Adverts: Adblock syntax is partially supported, so this list is able to extract some advert domains. This list is pretty good at blocking adverts, but an in-browser adblocker such as uBlock Origin is recommended as well as relying on hosts/DNS blocking.
- Trackers: Many tracking domains are extracted from the lists used, including Privacy Badger data files which automatically identify trackers.
- Coin mining: A few coin mining blocklists are used to block browser-based coin mining from using cpu.
## Advantages of using this list:
- Conversion of list types. As well as supporting many common filter list formats, it also supports Privacy Badger data file, which uses algorithms to detect trackers allowing newly created trackers to be quickly detected and added to this blocklist without a human needing to spot the tracker.
- Reverse DNS and passive DNS on malware IP addresses. This allows finding all the domains which a malware IP blacklist suggests could be dangerous to be found and blocked. This allows blocking of malware domains that haven't yet been added to other malware domain lists.
- Use of a whitelist. Using a hosts file doesn't allow whitelisting, and many DNS-based blockers don't have great whitelist support. This list has it's own whitelist, as well as using a few others to try to reduce false positives. This list supports "*" in subdomain and TLD to aid in easily fixing many false positives at once. (If you do find a false positive(a domain that shouldn't be blocked), then please make an issue and I will remove it)
- Use of DNS to check if domains still exist. Many lists contain domains that have expired and no longer exist. This makes those lists larger than needed which wastes bandwidth, space and can slow blocking.
## How to use:
- Pi-hole: Go to the web interface. Log in. Settings -> Blocklists. Copy domain list URL(Pi-hole currently only supports domain lists) from below in the links section, and paste it in the textbox. Click Save.
- Blokada: Open Blokada. Click shield with black middle which says "{number} in blacklist". Click the plus in the circle at the bottom of the screen. Copy and paste hosts file from link sections. Click save. WARNING: This list is large and might slow down your phone
- uBlock Origin: Click the uBlock Origin logo/uBlock Origin extension. Click open dashboard(3 horizontal lines under the disable uBlock Origin button, on the right). Click Filter lists. Scroll to the bottom, and click Import(in custom section). Copy and paste the Adblock style blocklist from the link section below.
## Links
[Adblock Plus format](https://mkb2091.github.io/blockconvert/output/adblock.txt)
[Hosts file format](https://mkb2091.github.io/blockconvert/output/hosts.txt)
WARNING: Too large for Windows: https://github.com/mkb2091/blockconvert/issues/87
[Domain list](https://mkb2091.github.io/blockconvert/output/domains.txt)
[Blocked IP address list](https://mkb2091.github.io/blockconvert/output/ip_blocklist.txt)
[DNS Response Policy Zone(RPZ) format](https://mkb2091.github.io/blockconvert/output/domains.rpz)
As well as generating blocklists, this project also generates whitelists which are used in the process. If you maintain your own blocklist, you may find one of the following whitelists useful:
[Whitelisted domains](https://mkb2091.github.io/blockconvert/output/whitelist_domains.txt)
[Whitelisted ABP format](https://mkb2091.github.io/blockconvert/output/whitelist_adblock.txt)
## The Process
1. Download all expired filterlists
2. Combine and split all the filterlists based on their type. This splits the lines into seperate groups: Adblock rules, blocked domains, regexes of blocked domains, allowed domains, regex of allowed domains, ips which are blocked, ips which are allowed, subnets which are blocked, subnets which are allowed.
3. Apply a regex to all the filterlists to extract domains and combine with other domains found via other means.
4. For each of those domains, use DNS to check if the domain is still active. If the domain isn't in the allowed domains list, doesn't match any of the allowed regexes, isn't in allowed by an adblock exception rule and it is blocked, or one of its cnames/ips is blocked then add it to the output.
Sources: [Sources](https://github.com/mkb2091/blockconvert/blob/master/filterlists.csv)
================================================
FILE: _config.yml
================================================
theme: jekyll-theme-cayman
================================================
FILE: build.rs
================================================
// generated by `sqlx migrate build-script`
fn main() {
// trigger recompilation when a new migration is added
println!("cargo:rerun-if-changed=migrations");
}
================================================
FILE: end2end/package.json
================================================
{
"name": "end2end",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {},
"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
"@playwright/test": "^1.28.0"
}
}
================================================
FILE: end2end/playwright.config.ts
================================================
import type { PlaywrightTestConfig } from "@playwright/test";
import { devices } from "@playwright/test";
/**
* Read environment variables from file.
* https://github.com/motdotla/dotenv
*/
// require('dotenv').config();
/**
* See https://playwright.dev/docs/test-configuration.
*/
const config: PlaywrightTestConfig = {
testDir: "./tests",
/* Maximum time one test can run for. */
timeout: 30 * 1000,
expect: {
/**
* Maximum time expect() should wait for the condition to be met.
* For example in `await expect(locator).toHaveText();`
*/
timeout: 5000,
},
/* Run tests in files in parallel */
fullyParallel: true,
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,
/* Retry on CI only */
retries: process.env.CI ? 2 : 0,
/* Opt out of parallel tests on CI. */
workers: process.env.CI ? 1 : undefined,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: "html",
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
actionTimeout: 0,
/* Base URL to use in actions like `await page.goto('/')`. */
// baseURL: 'http://localhost:3000',
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: "on-first-retry",
},
/* Configure projects for major browsers */
projects: [
{
name: "chromium",
use: {
...devices["Desktop Chrome"],
},
},
{
name: "firefox",
use: {
...devices["Desktop Firefox"],
},
},
{
name: "webkit",
use: {
...devices["Desktop Safari"],
},
},
/* Test against mobile viewports. */
// {
// name: 'Mobile Chrome',
// use: {
// ...devices['Pixel 5'],
// },
// },
// {
// name: 'Mobile Safari',
// use: {
// ...devices['iPhone 12'],
// },
// },
/* Test against branded browsers. */
// {
// name: 'Microsoft Edge',
// use: {
// channel: 'msedge',
// },
// },
// {
// name: 'Google Chrome',
// use: {
// channel: 'chrome',
// },
// },
],
/* Folder for test artifacts such as screenshots, videos, traces, etc. */
// outputDir: 'test-results/',
/* Run your local dev server before starting the tests */
// webServer: {
// command: 'npm run start',
// port: 3000,
// },
};
export default config;
================================================
FILE: end2end/tests/example.spec.ts
================================================
import { test, expect } from "@playwright/test";
test("homepage has title and links to intro page", async ({ page }) => {
await page.goto("http://localhost:3000/");
await expect(page).toHaveTitle("Welcome to Leptos");
await expect(page.locator("h1")).toHaveText("Welcome to Leptos!");
});
================================================
FILE: filterlists.csv
================================================
name,url,author,license,expires,list_type
🎮 Game Console Adblock List,https://raw.githubusercontent.com/DandelionSprout/adfilt/master/GameConsoleAdblockList.txt,,Dandelicence,345600,Adblock
uBlock filters – Resource abuse,https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resource-abuse.txt,,GPLv3,345600,Adblock
uBlock filters – Privacy,https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt,,GPLv3,345600,Adblock
uBlock filters – Badware risks,https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt,,GPLv3,345600,Adblock
uBlock filters,https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt,,GPLv3,345600,Adblock
mobiletrackers,https://raw.githubusercontent.com/craiu/mobiletrackers/master/list.txt,,GPLv3,604800,DomainBlocklist
iOS Tracker Blocklist,https://raw.githubusercontent.com/jakejarvis/ios-trackers/master/blocklist.txt,jakejarvis,MIT,864000,DomainBlocklistWithoutSubdomains
hostsVN,https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts,,MIT,86400,Hostfile
abuse.ch URLhaus Response Policy Zones,https://urlhaus.abuse.ch/downloads/hostfile,,CC0,86400,Hostfile
abuse.ch SSLBL Botnet C2 IP Blacklist,https://sslbl.abuse.ch/blacklist/sslipblacklist.txt,,CC0,86400,IPBlocklist
abuse.ch Feodo Tracker Botnet C2 IP Blocklist,https://feodotracker.abuse.ch/downloads/ipblocklist.txt,,CC0,86400,IPBlocklist
Xiaomi DNS Blocklist,https://raw.githubusercontent.com/unknownFalleN/xiaomi-dns-blocklist/master/xiaomi_dns_block.lst,,GPLv3,604800,DomainBlocklist
WindowsSpyBlocker - Hosts spy rules,https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt,,MIT,86400,Hostfile
Whitelist,https://raw.githubusercontent.com/ookangzheng/blahdns/master/hosts/whitelist.txt,ookangzheng,CC BY-NC-SA 4.0(I received permission to add to this project),86400,DomainAllowlist
Whitelist,https://raw.githubusercontent.com/NoExitTV/whitelist/master/domains/whitelist.txt,,MIT,604800,DomainAllowlist
Trackers,https://git.herrbischoff.com/trackers/plain/trackers.txt,,The Unlicense,864000,DomainBlocklistWithoutSubdomains
The Ultimate Hosts Blacklist whitelist,https://raw.githubusercontent.com/Ultimate-Hosts-Blacklist/whitelist/master/domains.list,,MIT,86400,DomainAllowlist
The Hosts File Project,https://hblock.molinero.dev/hosts,Héctor Molinero Fernández <hector@molinero.dev>,MIT,86400,Hostfile
The Block List Project Tracking,https://raw.githubusercontent.com/blocklistproject/Lists/master/tracking.txt,,The Unlicense,86400,Hostfile
The Block List Project Phishing,https://raw.githubusercontent.com/blocklistproject/Lists/master/phishing.txt,,The Unlicense,86400,Hostfile
The Block List Project Malware,https://raw.githubusercontent.com/blocklistproject/Lists/master/malware.txt,,The Unlicense,86400,Hostfile
The Block List Project Ads,https://raw.githubusercontent.com/blocklistproject/Lists/master/ads.txt,,The Unlicense,86400,Hostfile
StevenBlack/hosts,https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts,,MIT,86400,Hostfile
Slovenian List,https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt,,The Unlicense,259200,Adblock
Scams and Phishing,https://raw.githubusercontent.com/infinitytec/blocklists/master/scams-and-phishing.txt,infinitytec,MIT,864000,Hostfile
Scam Blocklist by DurableNapkin,https://raw.githubusercontent.com/durablenapkin/scamblocklist/master/hosts.txt,DurableNapkin,MIT,86400,Hostfile
Regex Filters for Pi-hole Whitelist,https://raw.githubusercontent.com/mmotti/pihole-regex/master/whitelist.list,,With Permission from https://github.com/mmotti/pihole-regex/issues/30,864000,DomainAllowlist
Regex Filters for Pi-hole,https://raw.githubusercontent.com/mmotti/pihole-regex/master/regex.list,,With Permission from https://github.com/mmotti/pihole-regex/issues/30,864000,RegexBlocklist
Privacy filters,https://raw.githubusercontent.com/metaphoricgiraffe/tracking-filters/master/trackingfilters.txt,,The Unlicense,86400,Adblock
Possibilities,https://raw.githubusercontent.com/infinitytec/blocklists/master/possibilities.txt,infinitytec,MIT,864000,Hostfile
PiHoleBlocklist - SmartTV,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt,,MIT,86400,DomainBlocklist
PiHoleBlocklist - SessionReplay,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SessionReplay.txt,,MIT,86400,DomainBlocklist
PiHoleBlocklist - AndroidTracking,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt,,MIT,86400,DomainBlocklist
PiHoleBlocklist - AmazonFireTV,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt,,MIT,86400,DomainBlocklist
Pi-Hole blocklist,https://codeberg.org/spootle/blocklist/raw/branch/master/blocklist.txt,,GPL-3.0,86400,DomainBlocklist
Perflyst's SmartTV Blocklist for Pi-hole - RegEx extension,https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/regex.list,,MIT,86400,RegexBlocklist
Nopelist,https://raw.githubusercontent.com/genediazjr/nopelist/master/nopelist.txt,Gene Diaz,MIT,604800,DomainBlocklist
NoTrack Tracker Blocklist,https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt,QuidsUp,GPLv3,86400,DomainBlocklist
NoTrack Malware Blocklist,https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt,QuidsUp,GPLv3,86400,DomainBlocklist
NoCoin Filter List,https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt,,MIT,86400,Hostfile
MinerBlock Filters,https://raw.githubusercontent.com/xd4rker/MinerBlock/master/assets/filters.txt,,MIT,86400,Adblock
Malicious URL Blocklist,https://gitlab.com/curben/urlhaus-filter/raw/master/urlhaus-filter-agh.txt,,CC0,86400,Adblock
Magneto Malware Skanner - Burner Domains,https://raw.githubusercontent.com/gwillem/magento-malware-scanner/master/rules/burner-domains.txt,,GPLv3,864000,DomainBlocklist
Lightswitch05's Ads & Tracking,https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt,Daniel White,Apache2,172800,Hostfile
Latvian List,https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt,,CC-BY-SA-4.0,86400,Adblock
KADhosts,https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt,,CC-BY-SA-4,86400,Hostfile
International List,https://raw.githubusercontent.com/betterwebleon/international-list/master/filters.txt,,The Unlicense,259200,Adblock
I Hate Tracker,https://raw.githubusercontent.com/pirat28/IHateTracker/master/iHateTracker.txt,Pirat28,MIT,864000,DomainBlocklist
Herm's Ad Black List,https://raw.githubusercontent.com/hermanjustinm/Herms-Blacklist/master/HermsAdBlacklist.txt,,MPL-2.0,864000,Adblock
HackerList,https://pfblockerlists.smallbusinesstech.net/hackerlist.txt,Soren Stoutner,GPLv3,86400,IPBlocklist
Frellwit's Swedish Hosts File,https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt,,GPL-3.0,86400,Hostfile
First-party trackers host list,https://hostfiles.frogeye.fr/multiparty-trackers.txt,,MIT,604800,DomainBlocklist
Fanboy's Annoyance List,https://easylist.to/easylist/fanboy-annoyance.txt,,GPLv3,345600,Adblock
EasyPrivacy,https://easylist.to/easylist/easyprivacy.txt,,GPLv3,345600,Adblock
EasyList Italy,https://easylist-downloads.adblockplus.org/easylistitaly.txt,,GPLv3,86400,Adblock
EasyList Hebrew,https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt,,GPLv3,86400,Adblock
EasyList Germany,https://easylist.to/easylistgermany/easylistgermany.txt,,GPLv3,86400,Adblock
EasyList Dutch,https://easylist-downloads.adblockplus.org/easylistdutch.txt,,GPLv3,345600,Adblock
EasyList China,https://easylist-downloads.adblockplus.org/easylistchina.txt,,GPLv3,345600,Adblock
EasyList,https://easylist.to/easylist/easylist.txt,,GPLv3,345600,Adblock
Commonly white listed domains for Pi-Hole,https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/whitelist.txt,,MIT,86400,DomainAllowlist
CJX's Annoyance List,https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt,,LGPLv3,345600,Adblock
CB-Malicious-Domains,https://raw.githubusercontent.com/cb-software/CB-Malicious-Domains/master/block_lists/domains_only.txt,,MIT,86400,DomainBlocklist
BlockConvert Internal IP Blocklist,internal/block_ips.txt,mkb2091,MIT License,30,IPBlocklist
BlockConvert Internal Blocklist,internal/blocklist.txt,mkb2091,MIT License,30,DomainBlocklist
BlockConvert Internal Allowlist,internal/allowlist.txt,mkb2091,MIT License,30,DomainAllowlist
Basic tracking list by Disconnect,https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt,,GPLv3,86400,DomainBlocklist
BarbBlock,https://paulgb.github.io/BarbBlock/blacklists/hosts-file.txt,,MIT,86400,DomainBlocklist
BAR-list,https://raw.githubusercontent.com/zznidar/BAR/master/BAR-list,,GPL-3.0,864000,DomainBlocklist
Anudeep's Blacklist,https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt,Anudeep <anudeep@protonmail.com>,MIT,86400,Hostfile
Anti-WebMiner,https://raw.githubusercontent.com/greatis/Anti-WebMiner/master/blacklist.txt,,Apache2,86400,DomainBlocklist
Ads and Trackers,https://raw.githubusercontent.com/infinitytec/blocklists/master/ads-and-trackers.txt,infinitytec,MIT,864000,Hostfile
AdguardMobileAds,https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardMobileAds.txt,,GPLv3,86400,Hostfile
AdguardDNS,https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardDNS.txt,,GPLv3,86400,Hostfile
AdguardApps,https://raw.githubusercontent.com/r-a-y/mobile-hosts/master/AdguardApps.txt,,GPLv3,86400,Hostfile
Adblock List for Finland,https://raw.githubusercontent.com/finnish-easylist-addition/finnish-easylist-addition/master/Finland_adb.txt,,The Unlicense,432000,Adblock
AdGuard Simplified Domain Names filter,https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt,,GPLv3,86400,Adblock
AdBlock Farsi,https://raw.githubusercontent.com/SlashArash/adblockfa/master/adblockfa.txt,,The Beer-Ware License,432000,Adblock
AdAway default blocklist,https://adaway.org/hosts.txt,,CC-BY-3,86400,Hostfile
Ad filter list by Disconnect,https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt,,GPLv3,86400,DomainBlocklist
AZORult Tracker,https://azorult-tracker.net/api/list/domain?format=plain,,Open Database License,86400,DomainBlocklist
ABPindo,https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt,,GPLv3,345600,Adblock
,https://zerodot1.gitlab.io/CoinBlockerLists/list_browser.txt,,GPLv3,86400,DomainBlocklist
,https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt,,GPLv3,86400,DomainBlocklist
,https://raw.githubusercontent.com/xxcriticxx/.pl-host-file/master/hosts.txt,,GPLv3,86400,Hostfile
,https://raw.githubusercontent.com/mitchellkrogza/The-Big-List-of-Hacked-Malware-Web-Sites/master/hacked-domains.list,Mitchell Krog,MIT,86400,DomainBlocklist
,https://raw.githubusercontent.com/matomo-org/referrer-spam-blacklist/master/spammers.txt,,Public Domain,86400,DomainBlocklist
,https://raw.githubusercontent.com/ligyxy/Blocklist/master/BLOCKLIST,,MIT,86400,DomainBlocklist
,https://raw.githubusercontent.com/EFForg/privacybadger/master/src/data/yellowlist.txt,,GPLv3+,86400,DomainAllowlist
,https://cdn.jsdelivr.net/gh/realodix/AdBlockID@master/dist/adblockid.adfl.txt,,,86400,Adblock
,https://blocklists.kitsapcreator.com/scam-spam.txt,,The Unlicense,86400,DomainBlocklist
,https://blocklists.kitsapcreator.com/malware-malicious.txt,,The Unlicense,86400,DomainBlocklist
,https://blocklists.kitsapcreator.com/general.txt,,The Unlicense,86400,DomainBlocklist
,https://blocklists.kitsapcreator.com/ads.txt,,The Unlicense,86400,DomainBlocklist
================================================
FILE: internal/adblock.txt
================================================
||cedexis.net^$third-party,badfilter
||episerver.net^$third-party,badfilter
================================================
FILE: internal/allow_regex.txt
================================================
^gsp[0-9][0-9]-ssl\.ls\.apple\.com$
================================================
FILE: internal/allowlist.txt
================================================
*.a-msedge.net
*.adafruit.com
*.b.akamaiedge.net
*.bitcoin.it
*.burst-alliance.org
*.c-msedge.net
*.cloudflare-dns.com # Cloudflare dns server
*.coingeek.com
*.delivery.mp.microsoft.com
*.dl.delivery.mp.microsoft.com
*.dsx.mp.microsoft.com
*.e-msedge.net
*.g.akamai.net
*.g.akamaiedge.net
*.gab.com
*.md.mp.microsoft.com.*
*.mp.microsoft.com
*.prod.do.dsp.mp.microsoft.com
*.s-msedge.net
*.safelinks.protection.outlook.com
*.search.msn.com
*.simply.com
*.skype.com
*.smartscreen.microsoft.com
*.splunk.com
*.storage.live.com
*.telecommand.telemetry.microsoft.com.akadns.net
*.tlu.dl.delivery.mp.microsoft.com
*.ugc.bazaarvoice.com
*.update.microsoft.com
*.vk.com
*.windowsupdate.com
*.wns.windows.com
1.www.s81c.com
1drv.ms
2-01-2c3e-003d.cdx.cedexis.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-783295583 Breaks LinkedIn.com
24timezones.com
33dim-trikal.tri.sch.gr https://github.com/mkb2091/blockconvert/issues/117
3p.ampproject.net
79423.analytics.edgekey.net
91-cdn.com
91mobiles.com
a-msedge.net
a.espncdn.com # https://github.com/mkb2091/blockconvert/issues/58 Breaks ESPN
a248.e.akamai.net
aaa.com
aax-us-east.amazon-adsystem.com
accounts.eu1.gigya.com
accounts.us1.gigya.com # Needed for iRobot app https://github.com/mkb2091/blockconvert/issues/3#issuecomment-683157232
activation-v2.sls.microsoft.com
activation.sls.microsoft.com
ada.support # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Cricket cell phone customer support
adapools.org # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-809546196 Crypto staking
adaway.org
adblock.ee
add0n.com
addons.mozilla.org
adguard.com
adguard.com # Adblocking tool
adguardteam.github.io
adl.windows.com # https://github.com/mkb2091/blockconvert/issues/92 Breaks Windows Update
aeriagames.com
aetna.com # blocks www.aetna.com which is a health website
aftership.com
aka.ms
akismet.com
alexa.com
aliexpress.com
alive.github.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-867284255
allure.com
amazon.com
androidauthority.com
answers.com
answers.microsoft.com
api.bing.com
api.cdp.microsoft.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-739536391 Required for Microsoft Edge Updates
api.facebook.com
api.mobile.immobilienscout24.de # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-715523256 Breaks the immoscout24 app
api.mymonero.com
api.particle.io
api.qrserver.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-780859152 Breaks setting up Google 2FA
api.vanilla.futurecdn.net
api.videos.oovvuu.com
appleid.apple.com.akadns.net
appspot-preview.l.google.com
apt.newrelic.com
arc.msn.com
arc.msn.com.nsatc.net
arizona.edu
ars.smartscreen.microsoft.com
assets.bwbx.io # Breaks bloomberg.com layout
assets.micpn.com
assets.penny-arcade.com
assets.zendesk.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-867284255
att.ada.support # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Cricket cell phone customer support
attfeedback.uservoice.com
au.download.windowsupdate.com
auth.gfx.ms
awin1.com
ay.gy
az416426.vo.msecnd.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-744499191 Breaks Airplus Mastercard software
azorult-tracker.net
b-graph.facebook.com
b.akamaiedge.net
badges.twitch.tv # https://github.com/mkb2091/blockconvert/issues/94 Breaks Twitch TV badges
baidu.com
battle.net
bbci.co.uk
behance.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-703781874 Not malware
berush.com
bestbuy.ca
betfair.com
better.fyi
bgp.he.net # https://github.com/mkb2091/blockconvert/issues/86 Part of Hurricane Electric's Internet Toolkit
bilder.bild.de # https://github.com/mkb2091/blockconvert/issues/57 Breaks images on https://www.bild.de/
binarydefense.com
bing.com
bitbucket.org
bitcoin.com
bitcoin.it
bitlord.com
bitly.com
blackhatworld.com
blob.weather.microsoft.com
blog.hubspot.com
blog.tube8.com
blogs.sch.gr # https://github.com/mkb2091/blockconvert/issues/117
bountysource.com
brightcove.map.fastly.net # https://github.com/mkb2091/blockconvert/issues/78 Breaks Brightcove video
browser.pipe.aria.microsoft.com
c-msedge.net
c.footprint.net # https://github.com/mkb2091/blockconvert/issues/88 Blocks https://www.24ur.com/ on certain systems
c.paypal.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-939979052 Breaks PayPal on some websites
candycrushsoda.king.com
cdn.ampproject.org # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687829847 CDN for AMP
cdn.content.prod.cms.msn.com # Needed for Windows 10 News Images https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687662575
cdn.onenote.net
cdn.privacy-mgmt.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-701373707 Breaks closing of a popup
cdn2.editmysite.com
cdnjs.cloudflare.com
cdnp0.stackassets.com # https://github.com/mkb2091/blockconvert/issues/64 Static assets for stacksocial.com
cdnp1.stackassets.com # https://github.com/mkb2091/blockconvert/issues/64 Static assets for stacksocial.com
cdnp2.stackassets.com # https://github.com/mkb2091/blockconvert/issues/64 Static assets for stacksocial.com
cdnp3.stackassets.com # https://github.com/mkb2091/blockconvert/issues/64 Static assets for stacksocial.com
ce1.uicdn.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-769915219 Breaks login for many www.ionos.de and www.1und1.de services
cesanta.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-819106039 Not malware
change.org
checkappexec.microsoft.com
chng.it
christianpost.com
ci4.googleusercontent.com
ci5.googleusercontent.com
clickondetroit.com # https://github.com/mkb2091/blockconvert/issues/82 News site
client-office365-tas.msedge.net
client-s.gateway.messenger.live.com
cloudflare-dns.com
cloudfront.net
cmail19.com
cmail20.com
cn-geo1.uber.com # https://github.com/mkb2091/blockconvert/issues/71 Breaks Uber geolocation
cnet.com
code.jquery.com
code.tidio.co
code.visualstudio.com
codeberg.org
cognito-identity.us-east-1.amazonaws.com # https://github.com/mkb2091/blockconvert/issues/101 Breaks AWS app login
coinpot.co
cointraffic.io
completion.amazon.com
config.edge.skype.com
confirmit.com
content.invisioncic.com
countryflags.io
credit.finance.intuit.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-788266121 Breaks checking credit score on Mint Mobile App
cricket.att.ada.support # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Cricket cell phone customer support
crunchyroll.com
ct.sendgrid.net
ctldl.windowsupdate.com
cv.rdtcdn.com
cy2.displaycatalog.md.mp.microsoft.com.akadns.net
cy2.licensing.md.mp.microsoft.com.akadns.net
cy2.settings.data.microsoft.com.akadns.net
cybercrime-tracker.net
d2405b0jymm2dk.cloudfront.net
d25xi40x97liuc.cloudfront.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-798791030 Breaks Amazon Video
d2gatte9o95jao.cloudfront.net
dailymail.co.uk # Blocked by PrivacyBadger
datadoghq-browser-agent.com
dege.freeweb.hu # https://github.com/mkb2091/blockconvert/issues/81 Site that hosts tools for running old games
delivery.mp.microsoft.com
desktop.google.com
dev.maxmind.com # Used for MaxMind developer guide
developer.spotify.com # Breaks Spotify
digicert.com
digitalriver.com
dim-aigeir.ach.sch.gr # https://github.com/mkb2091/blockconvert/issues/117
discord.com
discuss.newrelic.com
displaycatalog.mp.microsoft.com
disq.us
distilnetworks.com
djvbdz1obemzo.cloudfront.net
dl.delivery.mp.microsoft.com
dlsrc.getmonero.org
dm3p.wns.notify.windows.com.akadns.net
dmd.metaservices.microsoft.com
dmd.metaservices.microsoft.com.akadns.net
dmqdd6hw24ucf.cloudfront.net # https://github.com/mkb2091/blockconvert/issues/63 Breaks Amazon Prime Subtitles
dns.google.com
do.co
donate.getmonero.org
download.db-ip.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-779885164 For Geo IP database download
download.electrum.org
download.maxmind.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-749884660 Used to download MaxMind GeoIP files
download.newrelic.com
download.visualware.com # https://github.com/mkb2091/blockconvert/issues/123
download.windowsupdate.com
download.windowsupdate.com.c.footprint.net
downloads.getmonero.org
dpreview.com
drh.img.digitalriver.com
drh1.img.digitalriver.com
drudgereport.com
dsx.mp.microsoft.com
duckdns.org
duckduckgo.com
duken.nl # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-869051876 A forum, not malware
e-msedge.net
e785s8hz.micpn.com
easylist-downloads.adblockplus.org
easylist.to
ecdinterface.philips.com
eec.crunchyroll.com
electrum.org
elinkeu.clickdimensions.com
eloqua.com
embed.spotify.com # Breaks Spotify
emdl.ws.microsoft.com
encrypted-tbn1.gstatic.com
encrypted-tbn2.gstatic.com
encrypted-tbn3.gstatic.com
entireweb.com
eu-central-1.protection.sophos.com # https://github.com/mkb2091/blockconvert/issues/93 Its a cloud security platform
ev.rdtcdn.com
everest.castbox.fm
evoke-windowsservices-tas.msedge.net
f.chtah.com
f1.media.brightcove.com # https://github.com/mkb2091/blockconvert/issues/78 Breaks Brightcove video
faast.schibsted.io
facebook.com
fandom.wikia.com
fast.fonts.net
fast.wistia.net
faucethub.io
fe2.update.microsoft.com
fe2.update.microsoft.com.akadns.net
fe3.delivery.dsp.mp.microsoft.com.nsatc.net
fe3.delivery.mp.microsoft.com
feodotracker.abuse.ch
fiddle.jshell.net
fide.com
files.catbox.moe
files.freedownloadmanager.org
files.slack.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-771638966 Breaks user uploaded content on Slack
files2.freedownloadmanager.org
filterlists.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-787428332 Website with list of filterlists
filtri-dns.ga
firebaseremoteconfig.googleapis.com # Breaks android applications internally updating
fiverr.com
flipboard.com
forms.office.com
forum.getmonero.org
forum.overclockers.co.uk # PC forum
forums.storagereview.com
fossbytes.com
freedownloadmanager.org
fs.microsoft.com
ftpcontent.worldnow.com
funk.eu
g.akamai.net
g.akamaiedge.net
g.live.com
g.msn.com.nsatc.net
gadgetsnow.com
gameofbitcoins.com
gamespot.com
gannettdigital.com
gardenstatehelicopters.com
gateway.reddit.com
gcs.sc-cdn.net
geo-prod.do.dsp.mp.microsoft.com
geo-prod.dodsp.mp.microsoft.com.nsatc.net
getmonero.org
github.com
github.developerdan.com
gitlab.com
gleam.io # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993
global.edge.bamgrid.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687908780 Blocks access to Disney Plus
global.ssl.fastly.net
go.microsoft.com
go2cloud.org
google.*
google.co.*
google.com.*
googletagmanager.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-780859152 Breaks setting up Google 2FA
gpticketshop.com # https://github.com/mkb2091/blockconvert/issues/111
gs-loc.apple.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-695044303 Needed for Apple geolocation
gsp-ssl-geomap.ls-apple.com.akadns.net # Breaks Apple Maps
gsp-ssl.ls-apple.com.akadns.net # Breaks Apple Maps
gsp-ssl.ls.apple.com # Break Apple Maps
guce.advertising.com
guce.huffpost.com
h-ebay.online-metrix.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks EBay external logins
hblock.molinero.dev
help.mint.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Mint help
heureka.cz
highcharts.com
homedepot.tt.omtrdc.net
hostsfile.mine.nu
howtogeek.com
hubpages.com
hwcdn.net
i-am3p-cor003.api.p001.1drv.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks OneDrive/OutLook file storage
i.ibb.co
ibm.com
ign.com
ignorelist.com
image.prntscr.com
images.gounlimited.to
imagizer.imageshack.com
img-prod-cms-rt-microsoft-com.akamaized.net
img-s-msn-com.akamaized.net # https://github.com/mkb2091/blockconvert/issues/67 Breaks msn.com images
in.appcenter.ms # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687908780 Blocks access to Outlook 365 mail
inc.com
inference.location.live.net
informer.com
infowars.com
infowarsstore.com
insight.rapid7.com
insights.eu.newrelic.com
instagram.com
int.whiteboard.microsoft.com
io9.com
ip-adress.com
ip-tracker.org
ip107307705.ahcdn.com # https://github.com/mkb2091/blockconvert/issues/75 Breaks videos on streaming XXX site
ip107316446.ahcdn.com # https://github.com/mkb2091/blockconvert/issues/75 Breaks videos on streaming XXX site
ipv4.login.msa.akadns6.net
it.toolbox.com
itch.io
jaleco.com # Breaks download link for GIMP
jarv.is
joblo.com
jpost.com # News website
js.t.sinajs.cn
kaspersky.com
kasperskycontenthub.com
knowyourmeme.com
ku6.com
larati.net
leagueoflegends.com
leanplum.com
licensing.mp.microsoft.com
linkedin.com
links.mint.com
linktr.ee
livejasmin.com
livejournal.com
location-inference-westus.cloudapp.net
location.services.mozilla.com
login.live.com
login.msa.akadns6.net
login.windows.net
logincdn.msauth.net
lowendbox.com
m.me
m.stripe.network
mail.com
mail.getmonero.org
mail.google.com
mailchi.mp
malwaredomainlist.com
manager-magazin-de.manager-magazin.de # https://github.com/mkb2091/blockconvert/issues/60 Breaks accepting cookie banner
manifest.prod.boltdns.net # https://github.com/mkb2091/blockconvert/issues/66 Breaks BrightCove player
maps.windows.com
mbl.is
md.mp.microsoft.com.*
media.pearsoncmg.com
mediafire.com
mediaredirect.microsoft.com
mine.nu
miro.medium.com # https://github.com/mkb2091/blockconvert/issues/77 and https://github.com/mkb2091/blockconvert/issues/3#issuecomment-748449378, Breaks images on medium.com
mirror.co.uk
mirrorservice.org # Breaks VLC update download
mobile.tube8.com
mobile.twitter.com
modern.watson.data.microsoft.com.akadns.net
monero.org
moonbit.co.in
motherboard.vice.com
movable-ink-397.com
mp.microsoft.com
mscrl.microsoft.com
msedge.api.cdp.microsoft.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-729147310 Needed for Microsoft Edge to check for updates
msedge.net
msftconnecttest.com
mvaali1.ws.byteoversea.net
mw1.wsj.net # https://github.com/mkb2091/blockconvert/issues/50
my.matterport.com # https://github.com/mkb2091/blockconvert/issues/72 Breaks a 3D space capture platform
mymonero.com
namecheap.com
narkive.com
naver.com
netfort.com
newrelic.com
news.bitcoin.com
newyorker.com # News website
next-services.apps.microsoft.com
nexus.ensighten.com
nexusrules.officeapps.live.com
ning.com
notabug.org
notifications.google.com
now.sh
ocation-inference-westus.cloudapp.net
ocos-office365-s2s.msedge.ne
ocos-office365-s2s.msedge.net
ocsp.digicert.com
ocsp.verisign.com # https://github.com/mkb2091/blockconvert/issues/94 Breaks verifying validity of certificates
od.lk
oem.twimg.com
officeclient.microsoft.com
ogs.google.com
oneclient.sfx.ms
onesignal.com
onion.to
online.jimmyjohns.com
open.spotify.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-788015015 Breaks Spotify
openwall.com
oracle.com # Breaks java download
orbitz.com
osce11-0-en.url.trendmicro.com
outlook.office365.com
overclockers.co.uk # PC forum
pastebin.com
pastebin.org
pasteboard.co
pbs.twimg.com
pcmag.com # Blocked by PrivacyBadger
pcmatic.com
phishtank.com
play.spotify.com # Breaks Spotify
play.spotify.edgekey.net # Breaks Spotify
polyfill.io
pool.ntp.org
pornhub.com
post.spmailtechnol.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Used for links in email
powr.io
privateinternetaccess.com
prod.do.dsp.mp.microsoft.com
prod.telemetry.ros.rockstargames.com
proofpoint.com
pti.store.microsoft.com
purchase.mp.microsoft.com
push.prod.netflix.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-809546196 Breaks Netflix DRM
pushbullet.com # https://github.com/mkb2091/blockconvert/issues/115
qq.com
query.prod.cms.rt.microsoft.com
quickenloans.com
quora.com
rakuten.com
ranker.com
ransomwaretracker.abuse.ch
raw.githubusercontent.com
readcomiconline.to
recruitics.com
reddit.app.link
reddit.com
redditstatic.s3.amazonaws.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687908780 Breaks something
redtube.com
refinery29.com # Blog website
report-uri.com
rest-prde.immedia-semi.com
returnpath.com
ris-prod-atm.trafficmanager.net
ris.api.iris.microsoft.com
ris.api.iris.microsoft.com.akadns.net
router.utorrent.com
rumble.com # https://github.com/mkb2091/blockconvert/issues/76 Video Streaming Platform
s-media-cache-ak0.pinimg.com
s-msedge.net
s.imgur.com
s.ndemiccreations.com
s.tradingview.com # https://github.com/mkb2091/blockconvert/issues/49
s0.ipstatp.com # https://github.com/mkb2091/blockconvert/issues/83 Breaks Captcha
s3-ap-southeast-1.amazonaws.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-793400847 Breaks AWS S3 in SEA
s3-eu-west-1.amazonaws.com
s3-us-west-1.amazonaws.com
s3.amazonaws.com
s3.tradingview.com # https://github.com/mkb2091/blockconvert/issues/49
s7.orientaltrading.com
safebrowsing-cache.google.com
safebrowsing.google.com
sat1.us1-1.nanorep.co # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks Mint help
scrapinghub.com
sdks.shopifycdn.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-707148021 Needed for webshops
search.msn.com
searchenginejournal.com
secure.indeed.com # https://github.com/mkb2091/blockconvert/issues/84 Breaks login page for indeed.com
secure.mbl.is
semrush.com
sendgrid.com # https://github.com/mkb2091/blockconvert/issues/70 Base domain is not used for tracking
serial.alcohol-soft.com # https://github.com/mkb2091/blockconvert/issues/80 CD and DVD burning software
service.weather.microsoft.com
settings-win.data.microsoft.com
settings.data.microsoft.com
sfdataservice.microsoft.com # Blocks Microsoft store payments
sh.st
shop.app # https://github.com/mkb2091/blockconvert/issues/98#issue-940336845 URL shortener for shopify
shop.gadgetsnow.com
si6ling.incestflix.com # https://github.com/mkb2091/blockconvert/issues/65 Static Assets
similarweb.com
sina.com.cn
sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo
sj14.mktossl.com # CNAME for safe.riskiq.com
skype.com
skypeecs-prod-usw-0-b.cloudapp.net
sls.update.microsoft.com
sls.update.microsoft.com.akadns.net
smallseotools.com
smartscreen-sn3p.smartscreen.microsoft.com
smartscreen.microsoft.com
socialize.us1.gigya.com # Needed for iRobot app https://github.com/mkb2091/blockconvert/issues/3#issuecomment-683157232
software.informer.com
sohu.com
som.aeroplan.com
soundcloud.com
sourceforge.net
spcweb.ch # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-808365475 Breaks swiss parcel tracking
speedcurve.com
spiegel-de.spiegel.de # https://github.com/mkb2091/blockconvert/issues/61
spotify.com # Breaks Spotify
squidblacklist.org
src.ebay-us.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks EBay external logins
ssl.bblck.me
ssl.marfeelcdn.com
sslbl.abuse.ch
stacksocial.com
staging-discuss.newrelic.com
star-mini.c10r.facebook.com
static-dot-virustotalcloud.appspot.com
static-exp1.licdn.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-783295583 Breaks LinkedIn.com
static.bbci.co.uk
static.getmonero.org
static.matterport.com # https://github.com/mkb2091/blockconvert/issues/72 Breaks a 3D space capture platform
static.tacdn.com # https://github.com/mkb2091/blockconvert/issues/52
static.tidiochat.com
static.wikia.nocookie.net # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-744090132 Breaks images on fandom.com wikis
static.zdassets.com
statics.streamable.com
stats.foldingathome.org # https://github.com/mkb2091/blockconvert/issues/74 Regex False Positive
stats.govt.nz # # https://github.com/mkb2091/blockconvert/issues/99 Government of New Zealand Statistics site
stats.pingdom.com
stats.stackexchange.com
statsfe2.update.microsoft.com.akadns.net
status.newrelic.com
status.roll20.net
statuscake-email.com
statuscake.com
storage.live.com # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-772293993 Breaks OneDrive/OutLook file storage
store-images.microsoft.com
store-images.s-microsoft.com
storecatalogrevocation.storequality.microsoft.com
storeedgefd.dsx.mp.microsoft.com
stripecdn.map.fastly.net
substrate.ms-acdc.office.com # CNAME for outlook.office.com
substrate.office.com # CNAME for outlook.office.com
supervisor.ext-twitch.tv # https://github.com/mkb2091/blockconvert/issues/100 Breaks twitch extensions
support.microsoft.com
survey.euro.confirmit.com
survey.pearsoncmg.com
techradar.com
tecmint.com
teespring.com
telecommand.telemetry.microsoft.com.akadns.net
texashomebase.com
thebot.net
theepochtimes.com # https://github.com/mkb2091/blockconvert/issues/73 News website
theoldnet.com
tiktok.com # Causes adblock to block www.tiktok.com
tile-service.weather.microsoft.com
time.samsungcloudsolution.com # Issues #3, blocks services like Plex, YouTube and Amazon Video on Samsung TV
time.windows.com
tlu.dl.delivery.mp.microsoft.com
tmx.td.com # https://github.com/mkb2091/blockconvert/issues/102 Breaks Canadian finance website
to-do.microsoft.com
toptenreviews.com
tracker.tvnihon.com
tracker2.itzmx.com # https://github.com/mkb2091/blockconvert/issues/124
tracking.epicgames.com
transfer.sh
travelocity.com
trial.alcohol-soft.com # https://github.com/mkb2091/blockconvert/issues/80 CD and DVD burning software
trk.klclick1.com
tsfe.trafficshaping.dsp.mp.microsoft.com
tube8.com
uk.com
umich.qualtrics.com
unitedstates.smartscreen-prod.microsoft.com
update.microsoft.com
urldefense.proofpoint.com
urlhaus.abuse.ch
us-east-1-a.route.herokuapp.com
us.configsvc1.live.com.akadns.net
use.typekit.net
users.telenet.be
uservoice.com
v.firebog.net
v10.events.data.microsoft.com
v10.vortex-win.data.microsoft.com
v20.events.data.microsoft.com
validation-v2.sls.microsoft.com
vanilla.futurecdn.net
vanityfair.com
version.hybrid.api.here.com # Needed for "Here We Go" map download https://github.com/mkb2091/blockconvert/issues/3#issuecomment-687661883
vgwort.de
videos.oovvuu.com
vignette.wikia.nocookie.net
vip5.afdorigin-prod-am02.afdogw.com
vip5.afdorigin-prod-ch02.afdogw.com
virustotalcloud.appspot.com
vivo.com.br
vmn.net
vortex.accuweather.com
vxvault.net
w.soundcloud.com # https://github.com/mkb2091/blockconvert/issues/51
wac.phicdn.net
wallet.microsoft.com
watchdog-basic.energized.pro # Used to check if user is using Energized Protection
watchdog-blu.energized.pro # Used to check if user is using Energized Protection
watchdog-blugo.energized.pro # Used to check if user is using Energized Protection
watchdog-dns.energized.pro # Used to check if user is using Energized Protection
watchdog-ext.energized.pro # Used to check if user is using Energized Protection
watchdog-porn.energized.pro # Used to check if user is using Energized Protection
watchdog-spark.energized.pro # Used to check if user is using Energized Protection
watchdog-ultimate.energized.pro # Used to check if user is using Energized Protection
watchdog-unified.energized.pro # Used to check if user is using Energized Protection
watchdog.energized.pro # Used to check if user is using Energized Protection
watson.telemetry.microsoft.com
wbd.ms
wd-prod-cp-us-west-1-fe.westus.cloudapp.azure.com
wd-prod-ss.trafficmanager.net
wdcp.microsoft.*
wdcp.microsoft.com
web.getmonero.org
web.tresorit.com
webmasters.stackexchange.com
webmd.com
whiteboard.microsoft.com
whiteboard.ms
widget-mediator.zopim.com
widgetdata.tradingview.com # https://github.com/mkb2091/blockconvert/issues/49
wikidot.com
wildcard.twimg.com
windowsupdate.com
wl.spotify.com # Breaks spotify account confirmation
wns.windows.com
worldstream.nl
wrapper-api.sp-prod.net
wscont.apps.microsoft.com
wscont1.apps.microsoft.com
wscont2.apps.microsoft.com
wsj.com
ww.youporn.com
ww2.sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo
www.pushbullet.com # https://github.com/mkb2091/blockconvert/issues/115
wwww.youporn.com
wx2.sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo
wx3.sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo
wx4.sinaimg.cn # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-758476709 Breaks images on Chinese social media Weibo
x1.com
xnxx.com # https://github.com/mkb2091/blockconvert/issues/56
yahoo.co.jp
yahoo.uservoice.com
yandex.ru # https://github.com/mkb2091/blockconvert/issues/3#issuecomment-786252939 Breaks yandex
yoast.com
youporn.com
zdnet.com
zerodot1.gitlab.io
zerohedge.com
zeustracker.abuse.ch
zoho.com
================================================
FILE: internal/block_ipnets.txt
================================================
0.0.0.0/8
10.0.0.0/8
100.64.0.0/10
127.0.0.0/8
169.254.0.0/16
172.16.0.0/12
192.0.0.0/24
192.0.2.0/24
192.88.99.0/24
192.168.0.0/16
198.18.0.0/15
198.51.100.0/24
203.0.113.0/24
224.0.0.0/4
240.0.0.0/4
255.255.255.255/32
================================================
FILE: internal/block_ips.txt
================================================
================================================
FILE: internal/block_regex.txt
================================================
\.(?:com|co\.uk|net)\.(?:login|account|payment|verif)
[_.-]analytics?[_.-]
================================================
FILE: internal/blocklist.txt
================================================
*.ct.sendgrid.net # Tracking https://github.com/mkb2091/blockconvert/issues/70
*.getemails.com # De-anonymization
af.congressosiv-isv2018.it
ag.bud-info.com.pl
all-prize-giveaway.life
an-geburtstag.fun # Spam
app0973.vnbue32.live
awfulcruelty9.live
bestprizesday4.life
bs.newlifecampania.it # Spam
cd.nestwood.it
cf.spettinatidautore.it # Spam
cl.folignobenecomune.it
competition4138.redirect-servers52.live
converseboom24.live
cp.milano2000.it
dxnp7918p5axx.cloudfront.net
ee.caleidoscopio-singoli-pisa.it
getemails.com # De-anonymization
installflash-s3.com
mobile-global-app-market1.life
noireblur2.live
nonamedvlp49.live
reward7978.nonamedvlp49.live
rododesk.pl
ysmk.pclatorre.it
zmusic-online.com
================================================
FILE: local.toml
================================================
listen_port = 3000
[[peers]]
url = "http://localhost/"
get_dns_results = false
================================================
FILE: local2.toml
================================================
listen_port = 3000
[[peers]]
url = "http://localhost:3000/"
get_dns_results = false
[[peers]]
url = "http://localhost:3000/"
get_dns_results = false
================================================
FILE: migrations/20240222234126_filterlists.sql
================================================
CREATE TABLE IF NOT EXISTS filterLists (
url TEXT PRIMARY KEY NOT NULL UNIQUE,
contents TEXT NOT NULL,
lastUpdated INTEGER NOT NULL
);
================================================
FILE: migrations/20240223010915_lastmodified.sql
================================================
-- Add migration script here
ALTER TABLE filterLists ADD COLUMN lastModified INTEGER NOT NULL DEFAULT 0;
================================================
FILE: migrations/20240223011106_lastmodified.sql
================================================
-- Add migration script here
alter table filterLists drop column lastmodified;
================================================
FILE: migrations/20240223011400_etag.sql
================================================
-- Add migration script here
-- Add migration script here
ALTER TABLE filterLists ADD COLUMN etag TEXT;
================================================
FILE: migrations/20240224194439_rules.sql
================================================
-- Add migration script here
CREATE TABLE IF NOT EXISTS Rules (
id SERIAL PRIMARY KEY,
rule TEXT NOT NULL UNIQUE
);
================================================
FILE: migrations/20240224195223_filter_list_contents.sql
================================================
-- Add migration script here
ALTER TABLE filterLists RENAME TO OldFilterListContents;
CREATE TABLE IF NOT EXISTS filterLists (
id SERIAL PRIMARY KEY,
url TEXT NOT NULL UNIQUE,
format TEXT NOT NULL,
contents TEXT NOT NULL,
lastUpdated INTEGER NOT NULL,
etag TEXT
);
INSERT INTO filterLists (url, format, contents, lastUpdated, etag) SELECT url, '', contents, lastUpdated, etag FROM filterLists;
DROP TABLE OldFilterListContents;
================================================
FILE: migrations/20240224203224_list_rules.sql
================================================
-- Add migration script here
CREATE table list_rules (
id SERIAL PRIMARY KEY,
list_id INTEGER NOT NULL,
rule_id INTEGER NOT NULL,
FOREIGN KEY (list_id) REFERENCES filterLists(id),
FOREIGN KEY (rule_id) REFERENCES rules(id)
);
================================================
FILE: migrations/20240225214254_list_source.sql
================================================
-- Add migration script here
ALTER TABLE list_rules ADD COLUMN source TEXT;
================================================
FILE: migrations/20240225230839_index.sql
================================================
-- Add migration script here
create unique index rule_index on Rules(rule);
================================================
FILE: migrations/20240225231841_index.sql
================================================
-- Add migration script here
create index list_rules_index on list_rules(list_id, rule_id);
================================================
FILE: migrations/20240225232249_index.sql
================================================
-- Add migration script here
drop index list_rules_index;
create index list_rules_index_list_id on list_rules(list_id);
create index list_rules_index_rule_id on list_rules(rule_id);
================================================
FILE: migrations/20240226004619_change_date.sql
================================================
-- Add migration script here
alter table filterLists alter column lastUpdated type timestamp with time zone using to_timestamp(lastUpdated);
================================================
FILE: migrations/20240226013547_source.sql
================================================
-- Add migration script here
CREATE TABLE rule_source (
id SERIAL PRIMARY KEY,
source TEXT NOT NULL UNIQUE
);
CREATE INDEX idx_rule_source_source ON rule_source (source);
ALTER TABLE list_rules DROP COLUMN source;
ALTER TABLE list_rules ADD COLUMN source_id INTEGER REFERENCES rule_source(id);
================================================
FILE: migrations/20240226152939_temp.sql
================================================
-- Add migration script here
CREATE TABLE temp_rule_source (
idx SERIAL PRIMARY KEY,
rule TEXT NOT NULL,
source TEXT NOT NULL
);
================================================
FILE: migrations/20240226215547_remove_column.sql
================================================
-- Add migration script here
ALTER TABLE temp_rule_source DROP COLUMN idx;
================================================
FILE: migrations/20240226215929_remove_id.sql
================================================
-- Add migration script here
ALTER TABLE list_rules DROP COLUMN id;
================================================
FILE: migrations/20240226220711_remove_fkey.sql
================================================
-- Add migration script here
ALTER TABLE list_rules DROP CONSTRAINT list_rules_list_id_fkey;
ALTER TABLE list_rules DROP CONSTRAINT list_rules_rule_id_fkey;
ALTER TABLE list_rules DROP CONSTRAINT list_rules_source_id_fkey;
================================================
FILE: migrations/20240226223817_domain_rules.sql
================================================
-- Add migration script here
CREATE TABLE domain_rules (
id SERIAL PRIMARY KEY,
domain TEXT NOT NULL
);
CREATE INDEX domain_rules_idx_domain ON domain_rules (domain);
================================================
FILE: migrations/20240226230317_domain_block.sql
================================================
-- Add migration script here
ALTER TABLE domain_rules ADD COLUMN block BOOLEAN NOT NULL;
================================================
FILE: migrations/20240226230425_rename.sql
================================================
-- Add migration script here
ALTER TABLE domain_rules RENAME COLUMN id TO rule_id;
================================================
FILE: migrations/20240227191530_drop_column.sql
================================================
-- Add migration script here
CREATE TEMPORARY TABLE temp_rule_source(source TEXT UNIQUE, rule_id INTEGER) ON COMMIT DROP;
INSERT INTO
temp_rule_source (source, rule_id)
SELECT
rule_source.source,
list_rules.rule_id
FROM
list_rules
INNER JOIN rule_source ON list_rules.source_id = rule_source.id ON CONFLICT DO NOTHING;
DELETE FROM
rule_source;
ALTER TABLE
rule_source
ADD
COLUMN rule_id INTEGER NOT NULL;
INSERT INTO
rule_source (source, rule_id)
SELECT
source,
rule_id
FROM
temp_rule_source;
================================================
FILE: migrations/20240227194830_drop_column.sql
================================================
-- Add migration script here
ALTER TABLE list_rules DROP COLUMN rule_id;
================================================
FILE: migrations/20240227200629_drop_table.sql
================================================
-- Add migration script here
DROP TABLE temp_rule_source;
================================================
FILE: migrations/20240227201410_primary_key.sql
================================================
-- Add migration script here
DELETE FROM list_rules;
ALTER TABLE
list_rules
ADD
PRIMARY KEY (list_id, source_id);
================================================
FILE: migrations/20240228001134_domain_rules.sql
================================================
-- Add migration script here
ALTER TABLE
domain_rules
ADD
COLUMN subdomain BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE
domain_rules
ALTER COLUMN
subdomain DROP DEFAULT;
CREATE TEMPORARY TABLE temp_domain_rules (
domain TEXT NOT NULL,
allow BOOLEAN NOT NULL,
subdomain BOOLEAN NOT NULL
);
INSERT INTO
temp_domain_rules (domain, allow, subdomain)
SELECT
domain,
block,
subdomain
FROM
domain_rules;
DELETE FROM
domain_rules;
ALTER TABLE
domain_rules
ADD
COLUMN id SERIAL NOT NULL UNIQUE;
ALTER TABLE
domain_rules RENAME COLUMN block TO allow;
ALTER TABLE
domain_rules DROP column rule_id;
ALTER TABLE
domain_rules
ADD
CONSTRAINT domain_rule_unique UNIQUE (domain, allow, subdomain);
INSERT INTO
domain_rules (domain, allow, subdomain)
SELECT
domain,
NOT allow,
subdomain
FROM
temp_domain_rules;
================================================
FILE: migrations/20240228004601_extend_rules.sql
================================================
-- Add migration script here
ALTER TABLE
Rules
ADD
COLUMN domain_rule_id INTEGER;
CREATE TABLE unknown_rules (
id SERIAL PRIMARY KEY,
rule TEXT NOT NULL UNIQUE
);
DELETE FROM
Rules;
ALTER TABLE
Rules
ADD
COLUMN unknown_rule_id INTEGER;
ALTER TABLE
Rules
ADD
CONSTRAINT unique_rules UNIQUE NULLS NOT DISTINCT (domain_rule_id, unknown_rule_id);
================================================
FILE: migrations/20240228005409_drop_rule.sql
================================================
-- Add migration script here
ALTER TABLE Rules DROP COLUMN rule;
================================================
FILE: migrations/20240228015906_change_unique.sql
================================================
-- Add migration script here
ALTER TABLE rule_source DROP CONSTRAINT rule_source_source_key;
ALTER TABLE rule_source ADD CONSTRAINT rule_source_unique UNIQUE (source, rule_id);
================================================
FILE: migrations/20240228164553_ip.sql
================================================
-- Add migration script here
CREATE TABLE ip_rules (
id SERIAL PRIMARY KEY,
ip_address inet NOT NULL,
allow boolean NOT NULL,
CONSTRAINT ip_rules_unique UNIQUE (ip_address, allow)
);
ALTER TABLE Rules ADD COLUMN ip_rule_id INTEGER;
ALTER TABLE Rules DROP CONSTRAINT unique_rules;
ALTER TABLE Rules ADD CONSTRAINT unique_rules UNIQUE NULLS NOT DISTINCT (domain_rule_id, ip_rule_id, unknown_rule_id);
================================================
FILE: migrations/20240228170646_ip.sql
================================================
-- Add migration script here
ALTER TABLE ip_rules RENAME COLUMN ip_address TO ip_network;
================================================
FILE: migrations/20240228175807_remove_unknown.sql
================================================
-- Add migration script here
DROP TABLE unknown_rules;
ALTER TABLE Rules DROP COLUMN unknown_rule_id;
================================================
FILE: migrations/20240228180753_index.sql
================================================
-- Add migration script here
DELETE FROM Rules;
ALTER TABLE Rules ADD CONSTRAINT rules_unique UNIQUE NULLS NOT DISTINCT (domain_rule_id, ip_rule_id);
================================================
FILE: migrations/20240229210101_domains.sql
================================================
-- Add migration script here
CREATE TABLE domains (
id SERIAL PRIMARY KEY,
domain TEXT NOT NULL UNIQUE
);
INSERT INTO
domains (domain)
SELECT
domain
FROM
domain_rules ON CONFLICT DO NOTHING;
ALTER TABLE
domain_rules RENAME TO domain_rules_old;
CREATE TABLE domain_rules (
id SERIAL PRIMARY KEY,
domain_id INTEGER NOT NULL,
allow BOOLEAN NOT NULL,
subdomain BOOLEAN NOT NULL,
CONSTRAINT domain_rules_unique UNIQUE (domain_id, allow, subdomain)
);
INSERT INTO
domain_rules (domain_id, allow, subdomain)
SELECT
domains.id,
domain_rules_old.allow,
domain_rules_old.subdomain
FROM
domain_rules_old
INNER JOIN domains ON domain_rules_old.domain = domains.domain;
DROP TABLE domain_rules_old;
================================================
FILE: migrations/20240229212527_subdomains.sql
================================================
-- Add migration script here
ALTER TABLE
domains
ALTER COLUMN
id TYPE bigint;
ALTER TABLE
domain_rules
ALTER COLUMN
domain_id TYPE bigint;
CREATE TABLE subdomains (
domain_id bigint NOT NULL UNIQUE,
parent_domain_id bigint
);
================================================
FILE: migrations/20240301000455_more_indexes.sql
================================================
-- Add migration script here
CREATE INDEX rule_source_rule_idx ON rule_source (rule_id);
CREATE INDEX domain_rules_domain_idx ON domain_rules (domain_id);
================================================
FILE: migrations/20240301000900_subdomain_idx.sql
================================================
-- Add migration script here
CREATE INDEX subdomain_domain_idx ON subdomains (domain_id);
CREATE INDEX subdomain_parent_idx ON subdomains (parent_domain_id);
================================================
FILE: migrations/20240301030213_subdomain_inde.sql
================================================
-- Add migration script here
CREATE INDEX domain_rules_subdomain_idx ON domain_rules (subdomain);
================================================
FILE: migrations/20240302171950_expanded_subdomains.sql
================================================
-- Add migration script here
DELETE FROM
subdomains
WHERE
parent_domain_id IS NOT NULL;
ALTER TABLE
subdomains DROP CONSTRAINT subdomains_domain_id_key;
ALTER TABLE
subdomains
ADD
CONSTRAINT subdomains_domain_id_key UNIQUE NULLS NOT DISTINCT (domain_id, parent_domain_id);
================================================
FILE: migrations/20240302184040_processed_subdomains.sql
================================================
DELETE FROM
domains;
DELETE FROM subdomains;
DELETE FROM domain_rules;
DELETE FROM Rules;
DELETE FROM rule_source;
ALTER TABLE
domains
ADD
COLUMN processed_subdomains boolean DEFAULT false;
================================================
FILE: migrations/20240302192658_index.sql
================================================
-- Add migration script here
CREATE INDEX domains_processed_subdomains_idx ON domains(processed_subdomains);
================================================
FILE: migrations/20240302194037_domain_rule_id_idx.sql
================================================
-- Add migration script here
CREATE INDEX rules_domain_rule_id_idx ON rules(domain_rule_id);
================================================
FILE: migrations/20240302194733_not_null_parent.sql
================================================
-- Add migration script here
DELETE FROM subdomains WHERE parent_domain_id IS NULL;
ALTER TABLE subdomains ALTER COLUMN parent_domain_id SET NOT NULL;
================================================
FILE: migrations/20240302205633_not_null#.sql
================================================
-- Add migration script here
ALTER TABLE domains ALTER COLUMN processed_subdomains SET NOT NULL;
================================================
FILE: migrations/20240302222401_dns.sql
================================================
-- Add migration script here
ALTER TABLE
domains
ADD
COLUMN last_checked_dns TIMESTAMP WITH TIME ZONE;
CREATE INDEX domains_last_checked_dns_idx ON domains(last_checked_dns);
CREATE TABLE dns_ips (
domain_id BIGINT NOT NULL,
ip_address INET NOT NULL
);
CREATE INDEX dns_ips_domain_id_idx ON dns_ips(domain_id);
CREATE INDEX dns_ips_ip_address_idx ON dns_ips(ip_address);
CREATE TABLE dns_cnames (
domain_id BIGINT NOT NULL,
cname_domain_id BIGINT NOT NULL
);
CREATE INDEX dns_cnames_domain_id_idx ON dns_cnames(domain_id);
CREATE INDEX dns_cnames_cname_domain_id_idx ON dns_cnames(cname_domain_id);
================================================
FILE: migrations/20240304235746_filterlist.sql
================================================
-- Add migration script here
ALTER TABLE filterlists ADD COLUMN name TEXT;
ALTER TABLE filterlists ADD COLUMN author TEXT;
ALTER TABLE filterlists ADD COLUMN expires INTEGER;
ALTER TABLE filterlists ADD COLUMN license TEXT;
================================================
FILE: migrations/20240305000257_filterlist.sql
================================================
-- Add migration script here
ALTER TABLE filterlists ALTER COLUMN contents DROP NOT NULL;
================================================
FILE: migrations/20240305000612_filterlist.sql
================================================
-- Add migration script here
ALTER TABLE filterlists ALTER COLUMN lastUpdated DROP NOT NULL;
================================================
FILE: migrations/20240305003411_filterlist.sql
================================================
-- Add migration script here
ALTER TABLE filterlists ALTER COLUMN expires SET NOT NULL;
================================================
FILE: migrations/20240305200551_rule_matches.sql
================================================
-- Add migration script here
CREATE TABLE rule_matches (
rule_id INTEGER NOT NULL,
domain_id BIGINT NOT NULL
);
ALTER TABLE rule_matches ADD CONSTRAINT rule_matches_pk UNIQUE (rule_id, domain_id);
CREATE INDEX rule_matches_rule_id_idx ON rule_matches (rule_id);
CREATE INDEX rule_matches_domain_id_idx ON rule_matches (domain_id);
ALTER TABLE Rules ADD COLUMN last_checked_matches TIMESTAMP;
================================================
FILE: migrations/20240306123603_rule_count.sql
================================================
-- Add migration script here
ALTER TABLE filterLists ADD COLUMN rule_count INT NOT NULL DEFAULT 0;
================================================
FILE: migrations/20240306214217_index.sql
================================================
-- Add migration script here
CREATE INDEX rules_last_checked_matches_idx ON rules (last_checked_matches);
================================================
FILE: migrations/20240307005702_lists.sql
================================================
-- Add migration script here
CREATE TABLE allow_domains (
domain_id BIGINT UNIQUE NOT NULL
);
CREATE TABLE block_domains (
domain_id BIGINT UNIQUE NOT NULL
);
================================================
FILE: migrations/20240307012450_index.sql
================================================
-- Add migration script here
CREATE INDEX domain_rules_allow_idx ON domain_rules (allow);
CREATE INDEX ip_rules_allow_idx ON ip_rules (allow);
================================================
FILE: migrations/20240307031445_idx.sql
================================================
-- Add migration script here
CREATE INDEX ip_rules_network_idx ON ip_rules (ip_network);
================================================
FILE: output/adblock.txt
================================================
[File too large to display: 44.7 MB]
================================================
FILE: output/allowed_ips.txt
================================================
================================================
FILE: output/domains.rpz
================================================
================================================
FILE: output/domains.txt
================================================
[File too large to display: 39.7 MB]
================================================
FILE: output/hosts.txt
================================================
================================================
FILE: output/ip_blocklist.txt
================================================
================================================
FILE: output/whitelist_adblock.txt
================================================
================================================
FILE: output/whitelist_domains.txt
================================================
================================================
FILE: package.json
================================================
{
"devDependencies": {
"@tailwindcss/typography": "^0.5.10",
"daisyui": "^4.7.2",
"tailwindcss": "^3.4.1"
}
}
================================================
FILE: rust-toolchain.toml
================================================
[toolchain]
channel = "stable"
================================================
FILE: rustfmt.toml
================================================
edition = "2021"
================================================
FILE: src/app.rs
================================================
use crate::{
domain::DomainViewPage,
error_template::{AppError, ErrorTemplate},
filterlist::FilterListPage,
home_page::HomePage,
ip_view::IpView,
rule::RuleViewPage,
stats_view::StatsView,
};
use leptos::*;
use leptos_meta::*;
use leptos_router::*;
#[component]
pub fn App() -> impl IntoView {
// Provides context that manages stylesheets, titles, meta tags, etc.
provide_meta_context();
view! {
<Stylesheet id="leptos" href="/pkg/blockconvert.css"/>
<Meta
http_equiv="Content-Security-Policy"
content=move || {
leptos::nonce::use_nonce()
.map(|nonce| {
format!(
"script-src 'strict-dynamic' 'nonce-{nonce}' \
'wasm-unsafe-eval';",
)
})
.unwrap_or_default()
}
/>
<Title text="BlockConvert"/>
// content for this welcome page
<Router fallback=|| {
let mut outside_errors = Errors::default();
outside_errors.insert_with_default_key(AppError::NotFound);
view! { <ErrorTemplate outside_errors/> }.into_view()
}>
<header class="p-4 text-white bg-indigo-600">
<nav class="container flex items-center justify-between mx-auto">
<A href="/" class="text-lg font-bold">
Home
</A>
<div class="space-x-4">
<A href="/tasks" class="hover:text-indigo-300">
Tasks
</A>
<A href="/stats" class="hover:text-indigo-300">
Stats
</A>
<A
href="/login"
class="px-4 py-2 text-indigo-600 bg-white rounded hover:bg-indigo-200"
>
Login
</A>
</div>
</nav>
</header>
<main>
<Routes>
<Route path="" view=HomePage ssr=SsrMode::Async/>
<Route path="tasks" view=crate::tasks::TaskView ssr=SsrMode::Async/>
<Route path="stats" view=StatsView ssr=SsrMode::InOrder/>
<Route path="list" view=FilterListPage ssr=SsrMode::InOrder/>
<Route path="rule/:id" view=RuleViewPage ssr=SsrMode::InOrder/>
<Route path="domain/:domain" view=DomainViewPage ssr=SsrMode::InOrder/>
<Route path="ip/:ip" view=IpView ssr=SsrMode::InOrder/>
</Routes>
</main>
</Router>
}
}
#[component]
pub fn Loading() -> impl IntoView {
view! { <span class="loading loading-spinner loading-sm"></span> }
}
================================================
FILE: src/domain.rs
================================================
#[cfg(feature = "ssr")]
use crate::rule::RuleData;
use crate::{
app::Loading,
filterlist::{FilterListLink, FilterListUrl},
rule::DisplayRule,
rule::RuleId,
source::SourceId,
};
use leptos::*;
use leptos_router::*;
#[cfg(feature = "ssr")]
pub use lookup_dns_task::DomainResolver;
use serde::{Deserialize, Serialize};
use std::{collections::BTreeSet, net::IpAddr, str::FromStr, sync::Arc};
#[cfg_attr(feature = "ssr", derive(sqlx::Encode, sqlx::Decode))]
#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct DomainId(i64);
#[derive(Debug, Clone, thiserror::Error)]
pub enum DomainParseError {
Addr,
HickoryProto,
Custom,
}
impl std::fmt::Display for DomainParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Invalid domain")
}
}
impl<'a> From<addr::error::Error<'a>> for DomainParseError {
fn from(_: addr::error::Error) -> Self {
DomainParseError::Addr
}
}
impl From<hickory_proto::error::ProtoError> for DomainParseError {
fn from(_: hickory_proto::error::ProtoError) -> Self {
DomainParseError::HickoryProto
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(transparent)]
pub struct Domain(Arc<str>);
#[cfg(feature = "ssr")]
impl sqlx::Type<sqlx::Postgres> for Domain {
fn type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::Type<sqlx::Postgres>>::type_info()
}
fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::Type<sqlx::Postgres>>::compatible(ty)
}
}
#[cfg(feature = "ssr")]
impl sqlx::postgres::PgHasArrayType for Domain {
fn array_type_info() -> sqlx::postgres::PgTypeInfo {
<&str as sqlx::postgres::PgHasArrayType>::array_type_info()
}
fn array_compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
<&str as sqlx::postgres::PgHasArrayType>::array_compatible(ty)
}
}
#[cfg(feature = "ssr")]
impl sqlx::Encode<'_, sqlx::Postgres> for Domain {
fn encode_by_ref(&self, buf: &mut sqlx::postgres::PgArgumentBuffer) -> sqlx::encode::IsNull {
<&str as sqlx::Encode<sqlx::Postgres>>::encode(self.as_ref(), buf)
}
}
impl AsRef<str> for Domain {
fn as_ref(&self) -> &str {
&self.0
}
}
impl FromStr for Domain {
type Err = DomainParseError;
fn from_str(domain: &str) -> Result<Domain, Self::Err> {
if domain.len() > 253 {
return Err(DomainParseError::Custom);
}
let mut domain: Arc<str> = domain.into();
Arc::get_mut(&mut domain).unwrap().make_ascii_lowercase();
if domain.starts_with('*') || domain.ends_with('.') {
return Err(DomainParseError::Custom);
}
if !addr::parse_dns_name(&domain)?.has_known_suffix() {
return Err(DomainParseError::Addr);
}
let name = hickory_proto::rr::Name::from_str_relaxed(&domain)?;
if name.num_labels() < 2 {
return Err(DomainParseError::Custom);
}
if domain.contains('/') {
log::warn!("Invalid domain: {:?}", domain);
return Err(DomainParseError::Custom);
}
Ok(Domain(domain))
}
}
#[cfg(test)]
mod tests {
use crate::domain::Domain;
#[test]
fn valid_domain() {
for domain in [
"amazonaws.com",
"s3-website.us-east-1.amazonaws.com",
"origin-mobile_mob.conduit.com",
] {
let domain: Result<Domain, _> = domain.parse();
assert!(domain.is_ok());
}
}
#[test]
fn invalid_domain() {
for domain_str in [
"com",
"@.amazonaws.com",
"1234",
"example.com,google.com",
"example.com.",
] {
let domain: Result<Domain, _> = domain_str.parse();
assert!(domain.is_err(), "{}", domain_str);
}
}
#[test]
fn makes_lowercase() {
let domain: Domain = "EXAMPLE.COM".parse().unwrap();
assert_eq!(domain.as_ref(), "example.com");
}
}
#[cfg(feature = "ssr")]
mod lookup_dns_task {
use std::{collections::HashSet, sync::Mutex, time::Duration};
use super::*;
use hickory_resolver::error::ResolveError;
use tokio_util::sync::CancellationToken;
fn parse_lookup_result(
result: Result<hickory_resolver::lookup_ip::LookupIp, ResolveError>,
) -> Result<(Vec<ipnetwork::IpNetwork>, Vec<Domain>), ResolveError> {
match result {
Ok(result) => {
let mut ips: Vec<ipnetwork::IpNetwork> = Vec::new();
let mut cnames = Vec::new();
let lookup = result.as_lookup();
for record in lookup.iter() {
if let Some(a) = record.as_a() {
let ip: IpAddr = a.0.into();
ips.push(ip.into());
} else if let Some(aaaa) = record.as_aaaa() {
let ip: IpAddr = aaaa.0.into();
ips.push(ip.into());
} else if let Some(cname) = record.as_cname() {
let mut cname = cname.0.to_ascii();
if cname.ends_with('.') {
cname.pop();
}
if let Ok(cname) = cname.parse() {
cnames.push(cname);
} else {
log::warn!("Invalid CNAME {}", cname);
}
} else {
log::info!("Unknown record type {:?}", record.record_type());
}
}
Ok((ips, cnames))
}
Err(err) => match err.kind() {
hickory_resolver::error::ResolveErrorKind::NoRecordsFound {
query: _,
soa: _,
negative_ttl: _,
response_code: _,
trusted: _,
} => Ok((vec![], vec![])),
hickory_resolver::error::ResolveErrorKind::Proto(_) => Ok((vec![], vec![])),
hickory_resolver::error::ResolveErrorKind::Timeout => Err(err),
_ => Err(err),
},
}
}
#[cfg(feature = "ssr")]
type Resolver = Arc<
hickory_resolver::AsyncResolver<
hickory_resolver::name_server::GenericConnector<
hickory_resolver::name_server::TokioRuntimeProvider,
>,
>,
>;
#[cfg(feature = "ssr")]
type Task = (DomainId, Domain);
#[cfg(feature = "ssr")]
#[derive(Clone)]
pub struct DomainResolver {
resolvers: Vec<(Arc<str>, Resolver)>,
tx: async_channel::Sender<Task>,
rx: async_channel::Receiver<Task>,
read_limit: i64,
failed_cache_size: usize,
failed_domains: Arc<Mutex<Vec<i64>>>,
written_domains: Arc<Mutex<Vec<i64>>>,
bad_domains: Arc<Mutex<Vec<i64>>>,
looked_up_domains: Arc<Mutex<Vec<i64>>>,
dns_ips: Arc<Mutex<(Vec<i64>, Vec<ipnetwork::IpNetwork>)>>,
dns_cnames: Arc<Mutex<(Vec<i64>, Vec<Domain>)>>,
token: CancellationToken,
}
#[cfg(feature = "ssr")]
impl DomainResolver {
pub fn new(token: CancellationToken) -> Result<Self, ServerFnError> {
let _ = dotenvy::dotenv()?;
let servers_str = std::env::var("DNS_SERVERS")?;
let read_limit = std::env::var("READ_LIMIT")?.parse::<u32>()? as i64;
let mut resolvers = Vec::new();
for server in servers_str.split(',') {
let server: Arc<str> = server.into();
let (addr, port) = server
.split_once(':')
.ok_or_else(|| ServerFnError::new("Bad DNS_SERVER env"))?;
let server_conf = hickory_resolver::config::NameServerConfigGroup::from_ips_clear(
&[addr.parse()?],
port.parse()?,
true,
);
let config =
hickory_resolver::config::ResolverConfig::from_parts(None, vec![], server_conf);
let mut opts = hickory_resolver::config::ResolverOpts::default();
opts.ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6;
opts.cache_size = 32;
opts.attempts = 3;
opts.timeout = std::time::Duration::from_secs_f32(5.0);
let resolver = Arc::new(hickory_resolver::AsyncResolver::tokio(config, opts));
resolvers.push((server, resolver));
}
if resolvers.is_empty() {
return Err(ServerFnError::new("Empty DNS server list"));
}
let (tx, rx) = async_channel::bounded(read_limit as usize);
let bad_domains = Arc::new(Mutex::new(Vec::new()));
let failed_cache_size = std::env::var("FAILED_CACHE_SIZE")?.parse()?;
Ok(Self {
resolvers,
bad_domains,
tx,
rx,
read_limit,
failed_cache_size,
failed_domains: Arc::new(Mutex::new(Vec::new())),
written_domains: Arc::new(Mutex::new(Vec::new())),
looked_up_domains: Arc::new(Mutex::new(Vec::new())),
dns_ips: Default::default(),
dns_cnames: Default::default(),
token,
})
}
pub async fn run(&self) -> Result<(), ServerFnError> {
dotenvy::dotenv()?;
let concurrent_lookups: usize = std::env::var("CONCURRENT_LOOKUPS")?.parse()?;
let mut tasks = tokio::task::JoinSet::new();
for resolver in &self.resolvers {
for _ in 0..concurrent_lookups {
let resolver_str = resolver.0.clone();
let resolver = resolver.1.clone();
let resolver_self = self.clone();
let task = async move {
let token = resolver_self.token.clone();
tokio::select! {
_ = token.cancelled() => {
log::info!("Shutting down DNS resolver");
Ok(())},
res =
resolver_self.run_task(resolver_str, resolver) => res}
};
tasks.spawn(task);
}
}
let selector = self.clone();
tasks.spawn(async move {
let token = selector.token.clone();
tokio::select! {
_ = token.cancelled() => {
log::info!("Shutting down DNS selector");
Ok(())}
res = selector.domain_selector() => res
}
});
let writer = self.clone();
tasks.spawn(async move { writer.write_to_db().await });
while let Some(result) = tasks.join_next().await {
let _ = result?;
}
Ok(())
}
async fn domain_selector(&self) -> Result<(), ServerFnError> {
let pool = crate::server::get_db().await?;
let mut started_domains = HashSet::new();
let mut failed_domains = std::collections::VecDeque::<i64>::new();
loop {
{
failed_domains.extend(std::mem::take(&mut *self.failed_domains.lock()?));
}
while failed_domains.len() > self.failed_cache_size {
if let Some(failed) = failed_domains.pop_front() {
started_domains.remove(&failed);
}
}
let written_domains = std::mem::take(&mut *self.written_domains.lock()?);
for domain_id in written_domains {
started_domains.remove(&domain_id);
}
let limit = self.read_limit + started_domains.len() as i64;
let records = sqlx::query!(
"SELECT id, domain
FROM Domains
WHERE last_checked_dns IS NULL
ORDER BY id DESC NULLS FIRST
LIMIT $1",
limit
)
.fetch_all(&pool)
.await?;
let recheck_domains = if records.len() < limit as usize {
sqlx::query!(
"SELECT id, domain
FROM Domains
ORDER BY last_checked_dns ASC NULLS FIRST
LIMIT $1",
limit
)
.fetch_all(&pool)
.await?
} else {
vec![]
};
let records = records.into_iter().map(|record| (record.id, record.domain));
let recheck_domains = recheck_domains
.into_iter()
.map(|record| (record.id, record.domain));
let mut has_domains = false;
for (domain_id, domain_str) in records.chain(recheck_domains) {
has_domains = true;
if !started_domains.insert(domain_id) {
continue;
}
if let Ok(domain) = domain_str.parse::<Domain>() {
if domain_str == domain.as_ref() {
self.tx.send((DomainId(domain_id), domain)).await?;
continue;
}
}
log::warn!("Invalid domain: {}", domain_str);
self.bad_domains.lock()?.push(domain_id);
}
if !has_domains {
log::info!("No domains to check, sleeping");
tokio::time::sleep(Duration::from_secs(30)).await;
}
}
}
async fn run_task(
&self,
resolver_str: Arc<str>,
resolver: Resolver,
) -> Result<(), ServerFnError> {
while let Ok(task) = self.rx.recv().await {
let (domain_id, domain) = task;
let mut domain_str = domain.as_ref().to_string();
domain_str.push('.');
let result = resolver.lookup_ip(&domain_str).await;
let result = parse_lookup_result(result);
match result {
Ok((ips, cnames)) => {
self.looked_up_domains.lock()?.push(domain_id.0);
{
let mut dns_ips = self.dns_ips.lock()?;
for ip in ips {
dns_ips.0.push(domain_id.0);
dns_ips.1.push(ip);
}
}
{
let mut dns_cnames = self.dns_cnames.lock()?;
for cname in cnames {
dns_cnames.0.push(domain_id.0);
dns_cnames.1.push(cname);
}
}
}
Err(err) => {
log::warn!(
"Server: {} Error looking up domain {}: {}",
resolver_str,
domain.as_ref(),
err
);
self.failed_domains.lock()?.push(domain_id.0);
}
}
}
Ok(())
}
async fn write_to_db(&self) -> Result<(), ServerFnError> {
let pool = crate::server::get_db().await?;
let write_frequency: u64 = std::env::var("WRITE_FREQUENCY")?.parse()?;
let mut interval = tokio::time::interval(Duration::from_secs(write_frequency));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
loop {
tokio::select! {
_ = interval.tick() => {},
_ = self.token.cancelled() => {
log::info!("Shutting down DNS writer")
}
}
let looked_up_domains = std::mem::take(&mut *self.looked_up_domains.lock()?);
let looked_up_domains_deduped =
looked_up_domains.iter().cloned().collect::<HashSet<_>>();
assert_eq!(looked_up_domains.len(), looked_up_domains_deduped.len());
let dns_ips = std::mem::take(&mut *self.dns_ips.lock()?);
let dns_ips_domain_ids = &dns_ips.0;
let dns_ips_ips = &dns_ips.1;
let dns_cnames = std::mem::take(&mut *self.dns_cnames.lock()?);
let dns_cnames_domain_ids = &dns_cnames.0;
let dns_cnames_cname = &dns_cnames.1;
let bad_domains = std::mem::take(&mut *self.bad_domains.lock()?);
let total_cnames = dns_cnames_cname
.iter()
.collect::<HashSet<_>>()
.into_iter()
.cloned()
.collect::<Vec<Domain>>();
let new_domains_from_cnames = sqlx::query!(
"INSERT INTO domains(domain)
SELECT domain FROM UNNEST($1::text[]) as t(domain)
ON CONFLICT DO NOTHING",
&total_cnames[..] as _
)
.execute(&pool)
.await?
.rows_affected();
let mut tx = pool.begin().await?;
sqlx::query!(
"DELETE FROM dns_ips WHERE domain_id = ANY($1::bigint[])",
&looked_up_domains[..]
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"DELETE FROM dns_cnames WHERE domain_id = ANY($1::bigint[])",
&looked_up_domains[..]
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"INSERT INTO dns_ips(domain_id, ip_address)
SELECT domain_id, ip FROM UNNEST($1::bigint[], $2::inet[]) as t(domain_id, ip)",
&dns_ips_domain_ids[..],
&dns_ips_ips[..]
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"INSERT INTO dns_cnames(domain_id, cname_domain_id)
SELECT domain_id, cname_domains.id FROM UNNEST($1::bigint[], $2::text[]) as t(domain_id, cname)
INNER JOIN domains AS cname_domains ON cname_domains.domain = t.cname
",
&dns_cnames_domain_ids[..],
&dns_cnames_cname[..] as _
)
.execute(&mut *tx)
.await?;
let updated_domains = sqlx::query!(
"UPDATE domains
SET last_checked_dns = now()
WHERE id = ANY($1::bigint[])",
&looked_up_domains[..]
)
.execute(&mut *tx)
.await?
.rows_affected();
assert_eq!(updated_domains, looked_up_domains.len() as u64);
tx.commit().await?;
self.written_domains.lock()?.extend(&looked_up_domains);
if !bad_domains.is_empty() {
log::info!("Removing {} bad domains", bad_domains.len());
sqlx::query!(
"DELETE FROM domains
WHERE id = ANY($1::bigint[])",
&bad_domains[..]
)
.execute(&pool)
.await?;
}
self.written_domains.lock()?.extend(&bad_domains);
log::info!(
"Looked up {} domains, got {} ips, {} cnames ({} new)",
looked_up_domains.len(),
dns_ips_domain_ids.len(),
dns_cnames_domain_ids.len(),
new_domains_from_cnames
);
if self.token.is_cancelled() {
return Ok(());
}
}
}
}
}
#[server]
async fn get_dns_result(
domain: Domain,
) -> Result<(BTreeSet<IpAddr>, BTreeSet<(DomainId, String)>), ServerFnError> {
let records = sqlx::query!(
r#"SELECT dns_ips.ip_address as "ip_address: Option<ipnetwork::IpNetwork>",
cname_domains.id as "cname_domain_id: Option<i64>",
cname_domains.domain as "cname_domain: Option<String>"
FROM domains
LEFT JOIN dns_ips ON dns_ips.domain_id=domains.id
LEFT JOIN dns_cnames on dns_cnames.domain_id=domains.id
LEFT JOIN domains AS cname_domains ON cname_domains.id=dns_cnames.cname_domain_id
WHERE domains.domain = $1
"#r,
domain.as_ref().to_string()
)
.fetch_all(&crate::server::get_db().await?)
.await?;
let mut ip_addresses = BTreeSet::new();
let mut cnames = BTreeSet::new();
for record in records {
if let Some(ip) = record.ip_address {
ip_addresses.insert(ip.ip());
}
if let (Some(id), Some(cname)) = (record.cname_domain_id, record.cname_domain) {
cnames.insert((DomainId(id), cname));
}
}
Ok((ip_addresses, cnames))
}
#[component]
fn DnsResultView(domain: Domain) -> impl IntoView {
view! {
<Await
future=move || {
let domain = domain.clone();
async move { get_dns_result(domain.clone()).await }
}
let:dns_results
>
{
let dns_results = dns_results.clone();
move || match dns_results.clone() {
Ok((ips, cnames)) => {
view! {
<div class="grid grid-cols-2 gap-4">
<div>
<h2 class="mb-2 text-lg font-bold">IP Addresses</h2>
<ul class="grid grid-cols-2">
<For
each=move || { ips.clone() }
key=|ip| *ip
children=|ip| {
let href = format!("/ip/{ip}");
view! {
<li>
<A href=href class="link link-neutral">
{ip.to_string()}
</A>
</li>
}
}
/>
</ul>
</div>
<div>
<h2 class="mb-2 text-lg font-bold">CNAMEs</h2>
<ul>
<For
each=move || { cnames.clone() }
key=|(id, _cname)| *id
children=|(_id, cname)| {
let href = format!("/domain/{cname}");
view! {
<li>
<A href=href class="link link-neutral">
{cname}
</A>
</li>
}
}
/>
</ul>
</div>
</div>
}
.into_view()
}
_ => view! { <p>"Error"</p> }.into_view(),
}
}
</Await>
}
}
#[server]
async fn get_blocked_by(
domain: String,
) -> Result<Vec<(FilterListUrl, RuleId, SourceId, crate::filterlist::RulePair)>, ServerFnError> {
let records = sqlx::query!(
r#"
SELECT Rules.id as "rule_id: RuleId",
domain_rules_domain.domain as "domain: Option<String>", domain_rules.allow as "domain_allow: Option<bool>", subdomain as "subdomain: Option<bool>",
ip_rules.ip_network as "ip_network: Option<ipnetwork::IpNetwork>", ip_rules.allow as "ip_allow: Option<bool>",
source_id AS "source_id: SourceId", source, url
FROM domains
INNER JOIN rule_matches ON domains.id = rule_matches.domain_id
INNER JOIN Rules on Rules.id = rule_matches.rule_id
INNER JOIN rule_source ON rules.id = rule_source.rule_id
INNER JOIN list_rules ON rule_source.id = list_rules.source_id
INNER JOIN filterLists ON list_rules.list_id = filterLists.id
LEFT JOIN domain_rules ON rules.domain_rule_id = domain_rules.id
LEFT JOIN domains AS domain_rules_domain ON domain_rules_domain.id = domain_rules.domain_id
LEFT JOIN ip_rules ON rules.ip_rule_id = ip_rules.id
WHERE domains.domain = $1
ORDER BY url
LIMIT 100
"#r,
domain
)
.fetch_all(&crate::server::get_db().await?)
.await?;
let rules = records
.into_iter()
.map(|record| {
let rule_data = RuleData {
rule_id: record.rule_id,
domain: record.domain.clone(),
domain_allow: record.domain_allow,
domain_subdomain: record.subdomain,
ip_network: record.ip_network,
ip_allow: record.ip_allow,
};
let rule = rule_data.try_into()?;
let source = record.source.clone();
let pair = crate::filterlist::RulePair::new(source.into(), rule);
let url = record.url.clone();
Ok((url.parse()?, record.rule_id, record.source_id, pair))
})
.collect::<Result<Vec<_>, ServerFnError>>()?;
Ok(rules)
}
#[component]
fn BlockedBy(get_domain: Box<dyn Fn() -> Result<String, ParamsError>>) -> impl IntoView {
let blocked_by = create_resource(get_domain, |domain| async move {
let rules = get_blocked_by(domain?).await?;
Ok::<_, ServerFnError>(rules)
});
view! {
<Transition fallback=move || {
view! { <p>"Loading" <Loading/></p> }
}>
{move || match blocked_by.get() {
Some(Ok(rules)) => {
view! {
<table class="table table-zebra">
<For
each=move || { rules.clone() }
key=|(_url, rule_id, source_id, _pair)| (*rule_id, *source_id)
children=|(url, rule_id, _source_id, pair)| {
let source = pair.get_source().to_string();
let rule = pair.get_rule().clone();
view! {
<tr>
<td>
<FilterListLink url=url/>
</td>
<td>{source}</td>
<td>
<A href=rule_id.get_href() class="link link-neutral">
<DisplayRule rule=rule/>
</A>
</td>
</tr>
}
}
/>
</table>
}
.into_view()
}
_ => view! { <p>"Error"</p> }.into_view(),
}}
</Transition>
}
}
#[server]
async fn get_subdomains(domain: String) -> Result<Vec<String>, ServerFnError> {
let records = sqlx::query!(
"SELECT subdomain_text.domain
FROM domains
INNER JOIN subdomains ON domains.id = subdomains.parent_domain_id
INNER JOIN domains AS subdomain_text ON subdomains.domain_id = subdomain_text.id
WHERE domains.domain = $1
",
domain
)
.fetch_all(&crate::server::get_db().await?)
.await?;
let subdomains = records.into_iter().map(|record| record.domain).collect();
Ok(subdomains)
}
#[component]
fn DisplaySubdomains(get_domain: Box<dyn Fn() -> Result<String, ParamsError>>) -> impl IntoView {
let subdomains = create_resource(get_domain, |domain| async move {
let subdomains = get_subdomains(domain?).await?;
Ok::<_, ServerFnError>(subdomains)
});
view! {
<Transition fallback=move || {
view! { <p>"Loading" <Loading/></p> }
}>
{move || match subdomains.get() {
Some(Ok(subdomains)) => {
view! {
<table class="table table-zebra">
<For
each=move || { subdomains.clone() }
key=std::clone::Clone::clone
children=|subdomain| {
let domain_href = format!("/domain/{subdomain}");
view! {
<tr>
<td>
<A href=domain_href class="link link-neutral">
{subdomain}
</A>
</td>
</tr>
}
}
/>
</table>
}
.into_view()
}
_ => view! { <p>"Error"</p> }.into_view(),
}}
</Transition>
}
}
#[derive(Params, PartialEq)]
struct DomainParam {
domain: Option<String>,
}
#[component]
pub fn DomainViewPage() -> impl IntoView {
let params = use_params::<DomainParam>();
let get_domain = move || {
params.with(|param| {
param.as_ref().map_err(Clone::clone).and_then(|param| {
param
.domain
.clone()
.ok_or_else(|| ParamsError::MissingParam("No domain".into()))
})
})
};
let get_domain_parsed = move || {
params.with(|param| {
Ok::<_, ServerFnError>(
param
.as_ref()?
.domain
.as_ref()
.ok_or_else(|| ParamsError::MissingParam("No domain".into()))?
.parse::<Domain>()?,
)
})
};
view! {
<div>
{move || {
let domain = get_domain_parsed();
match domain {
Ok(domain) => {
view! {
<h1 class="text-3xl">"Domain: " {domain.as_ref().to_string()}</h1>
<DnsResultView domain=domain.clone()/>
<p>"Filtered by"</p>
<BlockedBy get_domain=Box::new(get_domain)/>
<p>"Subdomains"</p>
<DisplaySubdomains get_domain=Box::new(get_domain)/>
}
.into_view()
}
Err(err) => view! { <p>"Error: " {format!("{err:?}")}</p> }.into_view(),
}
}}
</div>
}
}
================================================
FILE: src/error_template.rs
================================================
use http::status::StatusCode;
use leptos::*;
use thiserror::Error;
#[derive(Clone, Debug, Error)]
pub enum AppError {
#[error("Not Found")]
NotFound,
}
impl AppError {
pub fn status_code(&self) -> StatusCode {
match self {
AppError::NotFound => StatusCode::NOT_FOUND,
}
}
}
// A basic function to display errors served by the error boundaries.
// Feel free to do more complicated things here than just displaying the error.
#[component]
pub fn ErrorTemplate(
#[prop(optional)] outside_errors: Option<Errors>,
#[prop(optional)] errors: Option<RwSignal<Errors>>,
) -> impl IntoView {
let errors = match outside_errors {
Some(e) => create_rw_signal(e),
None => match errors {
Some(e) => e,
None => panic!("No Errors found and we expected errors!"),
},
};
// Get Errors from Signal
let errors = errors.get_untracked();
// Downcast lets us take a type that implements `std::error::Error`
let errors: Vec<AppError> = errors
.into_iter()
.filter_map(|(_k, v)| v.downcast_ref::<AppError>().cloned())
.collect();
println!("Errors: {errors:#?}");
// Only the response code for the first error is actually sent from the server
// this may be customized by the specific application
#[cfg(feature = "ssr")]
{
use leptos_axum::ResponseOptions;
let response = use_context::<ResponseOptions>();
if let Some(response) = response {
response.set_status(errors[0].status_code());
}
}
view! {
<h1>{if errors.len() > 1 { "Errors" } else { "Error" }}</h1>
<For
// a function that returns the items we're iterating over; a signal is fine
each=move || { errors.clone().into_iter().enumerate() }
// a unique key for each item as a reference
key=|(index, _error)| *index
// renders each item to a view
children=move |error| {
let error_string = error.1.to_string();
let error_code = error.1.status_code();
view! {
<h2>{error_code.to_string()}</h2>
<p>"Error: " {error_string}</p>
}
}
/>
}
}
================================================
FILE: src/fileserv.rs
================================================
use crate::app::App;
use axum::response::Response as AxumResponse;
use axum::{
body::Body,
extract::State,
http::{Request, Response, StatusCode, Uri},
response::IntoResponse,
};
use leptos::*;
use tower::ServiceExt;
use tower_http::services::ServeDir;
pub async fn file_and_error_handler(
uri: Uri,
State(options): State<LeptosOptions>,
req: Request<Body>,
) -> AxumResponse {
let root = options.site_root.clone();
let res = get_static_file(uri.clone(), &root).await.unwrap();
if res.status() == StatusCode::OK {
res.into_response()
} else {
let handler = leptos_axum::render_app_to_stream(options.clone(), App);
handler(req).await.into_response()
}
}
async fn get_static_file(uri: Uri, root: &str) -> Result<Response<Body>, (StatusCode, String)> {
let req = Request::builder()
.uri(uri.clone())
.body(Body::empty())
.unwrap();
// `ServeDir` implements `tower::Service` so we can call it with `tower::ServiceExt::oneshot`
// This path is relative to the cargo root
match ServeDir::new(root).oneshot(req).await {
Ok(res) => Ok(res.into_response()),
Err(err) => Err((
StatusCode::INTERNAL_SERVER_ERROR,
format!("Something went wrong: {err}"),
)),
}
}
================================================
FILE: src/filterlist.rs
================================================
pub use crate::domain::Domain;
use crate::PAGE_SIZE;
use crate::{rule::RuleId, source::SourceId};
use leptos::*;
use leptos::{server, ServerFnError};
use crate::rule::DisplayRule;
#[cfg(feature = "ssr")]
use crate::rule::RuleData;
use leptos_router::*;
use serde::{Deserialize, Serialize};
use std::str::FromStr;
use std::sync::Arc;
#[cfg_attr(feature = "ssr", derive(sqlx::Encode, sqlx::Decode))]
#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct ListId(i32);
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FilterListRecord {
pub name: Arc<str>,
pub list_format: FilterListType,
pub author: Arc<str>,
pub license: Arc<str>,
pub expires: std::time::Duration,
pub last_updated: Option<chrono::DateTime<chrono::Utc>>,
pub list_size: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord, Hash)]
#[serde(transparent)]
pub struct FilterListUrl {
url: Arc<str>,
}
impl FilterListUrl {
pub fn as_str(&self) -> &str {
self.as_ref()
}
pub fn to_internal_path(&self) -> Option<std::path::PathBuf> {
if self.as_str().starts_with("internal/") {
Some(std::path::PathBuf::from(self.as_str()))
} else {
None
}
}
}
impl std::ops::Deref for FilterListUrl {
type Target = str;
fn deref(&self) -> &Self::Target {
self.url.as_ref()
}
}
impl FromStr for FilterListUrl {
type Err = url::ParseError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"internal/blocklist.txt" | "internal/block_ips.txt" | "internal/allowlist.txt" => {
Ok(Self { url: s.into() })
}
s => Ok(Self {
url: url::Url::parse(s)?.as_str().into(),
}),
}
}
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum FilterListType {
Adblock,
DomainBlocklist,
DomainBlocklistWithoutSubdomains,
DomainAllowlist,
IPBlocklist,
IPAllowlist,
IPNetBlocklist,
DenyHosts,
RegexAllowlist,
RegexBlocklist,
Hostfile,
}
impl FilterListType {
pub fn as_str(&self) -> &'static str {
match self {
Self::Adblock => "Adblock",
Self::DomainBlocklist => "DomainBlocklist",
Self::DomainBlocklistWithoutSubdomains => "DomainBlocklistWithoutSubdomains",
Self::DomainAllowlist => "DomainAllowlist",
Self::IPBlocklist => "IPBlocklist",
Self::IPAllowlist => "IPAllowlist",
Self::IPNetBlocklist => "IPNetBlocklist",
Self::DenyHosts => "DenyHosts",
Self::RegexAllowlist => "RegexAllowlist",
Self::RegexBlocklist => "RegexBlocklist",
Self::Hostfile => "Hostfile",
}
}
}
#[derive(Debug, thiserror::Error)]
pub struct InvalidFilterListTypeError;
impl std::fmt::Display for InvalidFilterListTypeError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Invalid FilterListType")
}
}
impl std::str::FromStr for FilterListType {
type Err = InvalidFilterListTypeError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"Adblock" => Ok(Self::Adblock),
"DomainBlocklist" => Ok(Self::DomainBlocklist),
"DomainBlocklistWithoutSubdomains" => Ok(Self::DomainBlocklistWithoutSubdomains),
"DomainAllowlist" => Ok(Self::DomainAllowlist),
"IPBlocklist" => Ok(Self::IPBlocklist),
"IPAllowlist" => Ok(Self::IPAllowlist),
"IPNetBlocklist" => Ok(Self::IPNetBlocklist),
"DenyHosts" => Ok(Self::DenyHosts),
"RegexAllowlist" => Ok(Self::RegexAllowlist),
"RegexBlocklist" => Ok(Self::RegexBlocklist),
"Hostfile" => Ok(Self::Hostfile),
_ => Err(InvalidFilterListTypeError),
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct FilterListMap(
pub Vec<(FilterListUrl, FilterListRecord)>,
// Just so it is consistently ordered
);
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct DomainRule {
pub domain: Domain,
pub allow: bool,
pub subdomain: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct IpRule {
pub ip: ipnetwork::IpNetwork,
pub allow: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Rule {
#[serde(rename = "d")]
Domain(DomainRule),
IpRule(IpRule),
#[serde(rename = "u")]
Unknown,
#[serde(rename = "i")]
Invalid,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(into = "(Arc<str>, Rule)")]
#[serde(from = "(Arc<str>, Rule)")]
pub struct RulePair {
source: Arc<str>,
rule: Rule,
}
impl RulePair {
pub fn new(source: Arc<str>, rule: Rule) -> RulePair {
RulePair { source, rule }
}
pub fn get_rule(&self) -> &Rule {
&self.rule
}
pub fn get_source(&self) -> &Arc<str> {
&self.source
}
}
impl From<RulePair> for (Arc<str>, Rule) {
fn from(val: RulePair) -> Self {
(val.source, val.rule)
}
}
impl From<(Arc<str>, Rule)> for RulePair {
fn from((source, rule): (Arc<str>, Rule)) -> Self {
Self { source, rule }
}
}
#[cfg(feature = "ssr")]
fn parse_lines(contents: &str, parser: &dyn Fn(&str) -> Option<Rule>) -> Vec<RulePair> {
let mut rules = vec![];
for line in contents.lines() {
let source = line;
if line.is_empty() {
continue;
}
if let Some(rule) = parser(line) {
rules.push(RulePair {
source: source.into(),
rule,
});
}
}
rules
}
#[cfg(feature = "ssr")]
fn parse_domain_list_line(line: &str, allow: bool, subdomain: bool) -> Option<Rule> {
let line = line.split('#').next()?;
if line.is_empty() {
return None;
}
let line = line.trim();
let mut segments = line.split_whitespace();
match (segments.next(), segments.next(), segments.next()) {
(Some(domain), None, None) | (Some("127.0.0.1" | "0.0.0.0"), Some(domain), None) => {
let (subdomain, domain) = domain
.strip_prefix("*.")
.map_or((subdomain, domain), |domain| (true, domain));
if let Ok(domain) = domain.parse() {
let domain_rule = DomainRule {
domain,
allow,
subdomain,
};
Some(Rule::Domain(domain_rule))
} else {
Some(Rule::Invalid)
}
}
_ => Some(Rule::Invalid),
}
}
#[cfg(feature = "ssr")]
fn parse_domain_list(contents: &str, allow: bool, subdomain: bool) -> Vec<RulePair> {
parse_lines(contents, &|line| {
parse_domain_list_line(line, allow, subdomain)
})
}
#[cfg(feature = "ssr")]
fn parse_adblock_line(line: &str) -> Option<Rule> {
let rule = line;
if rule.starts_with('!') // Comment
|| rule.contains('#') // CSS selector
|| !rule.trim_matches('.').contains('.') // Not a domain
|| matches! {rule, "[Adblock Plus 2.0]" | "[Adblock Plus 1.1]"}
{
return None;
}
let mut match_end_domain = false;
let rule = if let Some((start, tags)) = rule.split_once('$') {
let mut block_site = false;
let mut has_specific_filters = false;
let mut has_unknown_tag = false;
for tag in tags.split(',') {
if tag.starts_with('~') // Can't partially block a site
|| tag.starts_with("rewrite=")
// Can't rewrite a site
{
return None;
} else if let Some(domain_tag) = tag.strip_prefix("domain=") {
for domain in domain_tag.split('|') {
if !start.contains(domain) {
return None;
}
}
} else {
match tag {
"3p" | "doc" | "document" | "all" => {
match_end_domain = true;
block_site = true;
}
"popup" | "ghide" | "generichide" | "genericblock" | "image" | "script"
| "third-party" | "xmlhttprequest" | "stylesheet" | "subdocument" | "media"
| "csp" => {
has_specific_filters = true;
}
"important" => {}
_ => {
has_unknown_tag = true;
}
}
}
}
if has_specific_filters && !block_site {
return None;
}
if has_unknown_tag {
return Some(Rule::Unknown);
}
start
} else {
rule
};
if let Some(rule) = rule.strip_prefix('/') {
if let Some(_rule) = rule.strip_suffix('/') {
// REGEX
} else {
return None; // Path selector
}
}
let (rule, exception) = if let Some(rule) = rule.strip_prefix("@@") {
(rule, true)
} else {
(rule, false)
};
let (rule, mut match_start_domain, match_exact_start) =
if let Some(rule) = rule.strip_prefix("||") {
(rule, true, false)
} else {
let (rule, match_exact_start) = rule
.strip_prefix('|')
.map_or((rule, false), |rule| (rule, true));
(rule, false, match_exact_start)
};
let (rule, match_end_domain_exact) = rule
.strip_suffix('|')
.map_or((rule, false), |rule| (rule, true));
let (mut rule, match_end_domain) = rule
.strip_suffix('^')
.map_or((rule, match_end_domain), |rule| (rule, true));
if rule.contains('/') {
return None; // Path selector
}
if rule.contains('*') {
return Some(Rule::Unknown);
}
if !match_start_domain {
(match_start_domain, rule) = rule
.strip_prefix('.')
.or_else(|| rule.strip_prefix("*."))
.map_or((false, rule), |rule| (true, rule));
}
if match_start_domain && (match_end_domain | match_end_domain_exact) {
if let Ok(domain) = rule.parse() {
let domain_rule = DomainRule {
domain,
allow: exception,
subdomain: !match_exact_start,
};
return Some(Rule::Domain(domain_rule));
} else if let Ok(ip) = rule.parse::<ipnetwork::IpNetwork>() {
return Some(Rule::IpRule(IpRule {
ip,
allow: exception,
}));
}
}
Some(Rule::Unknown)
}
#[cfg(feature = "ssr")]
fn parse_adblock(contents: &str) -> Vec<RulePair> {
parse_lines(contents, &parse_adblock_line)
}
#[cfg(feature = "ssr")]
fn parse_regex_line(line: &str) -> Option<Rule> {
if line.starts_with('#') {
return None;
}
if let Some(rule) = line.strip_prefix(r"(^|\.)") {
if let Some(rule) = rule.strip_suffix('$') {
let mut rule = rule.to_string();
rule.retain(|c| c != '/');
if let Ok(domain) = rule.parse() {
let domain_rule = DomainRule {
domain,
allow: false,
subdomain: true,
};
return Some(Rule::Domain(domain_rule));
}
}
}
Some(Rule::Unknown)
}
#[cfg(feature = "ssr")]
fn parse_ip_network_line(line: &str, allow: bool) -> Option<Rule> {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
return None;
}
if let Ok(ip) = line.parse::<ipnetwork::IpNetwork>() {
Some(Rule::IpRule(IpRule { ip, allow }))
} else {
Some(Rule::Unknown)
}
}
#[cfg(feature = "ssr")]
fn parse_ip_network_list(contents: &str, allow: bool) -> Vec<RulePair> {
parse_lines(contents, &|line| parse_ip_network_line(line, allow))
}
#[cfg(feature = "ssr")]
fn parse_regex(contents: &str) -> Vec<RulePair> {
parse_lines(contents, &parse_regex_line)
}
#[cfg(feature = "ssr")]
fn parse_unknown_lines(contents: &str) -> Vec<RulePair> {
parse_lines(contents, &|_| Some(Rule::Unknown))
}
#[cfg(feature = "ssr")]
pub fn parse_list_contents(contents: &str, list_format: FilterListType) -> Vec<RulePair> {
match list_format {
FilterListType::Adblock => parse_adblock(contents),
FilterListType::DomainBlocklist => parse_domain_list(contents, false, true),
FilterListType::DomainBlocklistWithoutSubdomains => {
parse_domain_list(contents, false, false)
}
FilterListType::DomainAllowlist => parse_domain_list(contents, true, false),
FilterListType::IPBlocklist => parse_ip_network_list(contents, false),
FilterListType::IPAllowlist => parse_ip_network_list(contents, true),
FilterListType::IPNetBlocklist => parse_ip_network_list(contents, false),
FilterListType::DenyHosts => parse_unknown_lines(contents),
FilterListType::RegexAllowlist => parse_unknown_lines(contents),
FilterListType::RegexBlocklist => parse_regex(contents),
FilterListType::Hostfile => parse_domain_list(contents, false, true),
}
}
#[server(ParseList)]
pub async fn parse_list(url: FilterListUrl) -> Result<(), ServerFnError> {
let start = std::time::Instant::now();
let pool = crate::server::get_db().await?;
let url_str = url.as_str();
let record = sqlx::query!(
"SELECT id, format, contents FROM filterLists WHERE url = $1",
url_str
)
.fetch_one(&pool)
.await?;
let list_format: FilterListType = record.format.parse()?;
log::info!(
"Parsing {} as format {}",
url.as_str(),
list_format.as_str()
);
let list_id = record.id;
let rules = {
let contents = record
.contents
.ok_or_else(|| ServerFnError::new("No contents for list"))?;
parse_list_contents(&contents, list_format)
};
let (mut domain_src, mut domains, mut allow, mut subdomain) = (vec![], vec![], vec![], vec![]);
let (mut ip_source, mut ips, mut allow_ips) = (vec![], vec![], vec![]);
let mut other_rules_src = vec![];
for rule in &rules {
let source = rule.get_source().as_ref();
let source = source[..source.len().min(2000)].to_string();
match rule.get_rule() {
Rule::Domain(domain_rule) => {
domain_src.push(source);
domains.push(domain_rule.domain.clone());
allow.push(domain_rule.allow);
subdomain.push(domain_rule.subdomain);
}
Rule::IpRule(ip_rule) => {
ip_source.push(source);
ips.push(ip_rule.ip);
allow_ips.push(ip_rule.allow);
}
_ => {
other_rules_src.push(source);
}
}
}
log::info!(
"Inserting {} rules ({} domain rules, {} IP rules, {} unknown)",
rules.len(),
domain_src.len(),
ip_source.len(),
other_rules_src.len()
);
sqlx::query!(
"INSERT INTO domains (domain)
SELECT domain FROM UNNEST($1::text[]) AS t(domain) ON CONFLICT DO NOTHING",
&domains[..] as _
)
.execute(&pool)
.await?;
let mut tx = pool.begin().await?;
sqlx::query! {"DELETE FROM list_rules WHERE list_id = $1", list_id}
.execute(&mut *tx)
.await?;
sqlx::query!("INSERT INTO domain_rules (domain_id, allow, subdomain)
SELECT domains.id, allow, subdomain FROM UNNEST($1::text[], $2::bool[], $3::bool[]) AS t(domain, allow, subdomain)
INNER JOIN domains ON domains.domain = t.domain
ON CONFLICT DO NOTHING",
&domains[..] as _,
&allow[..],
&subdomain[..]
).execute(&mut *tx).await?;
sqlx::query!("INSERT INTO Rules (domain_rule_id)
SELECT domain_rules.id FROM UNNEST($1::text[], $2::bool[], $3::bool[]) AS t(domain, allow, subdomain)
INNER JOIN domains ON domains.domain = t.domain
INNER JOIN domain_rules ON domain_rules.domain_id = domains.id AND domain_rules.allow = t.allow AND domain_rules.subdomain = t.subdomain
ON CONFLICT DO NOTHING",
&domains[..] as _,
&allow[..],
&subdomain[..]
).execute(&mut *tx).await?;
sqlx::query!("INSERT INTO rule_source (source, rule_id)
SELECT source, Rules.id FROM UNNEST ($1::text[], $2::text[], $3::bool[], $4::bool[]) AS t(source, domain, allow, subdomain)
INNER JOIN domains ON domains.domain = t.domain
INNER JOIN domain_rules ON domain_rules.domain_id = domains.id AND domain_rules.allow = t.allow AND domain_rules.subdomain = t.subdomain
INNER JOIN Rules ON Rules.domain_rule_id = domain_rules.id
ON CONFLICT DO NOTHING",
&domain_src[..],
&domains[..] as _,
&allow[..],
&subdomain[..]
).execute(&mut *tx).await?;
sqlx::query!(
"INSERT INTO ip_rules (ip_network, allow)
SELECT ip, allow FROM UNNEST($1::inet[], $2::bool[]) AS t(ip, allow)
ON CONFLICT DO NOTHING",
&ips[..],
&allow_ips[..]
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"INSERT INTO Rules (ip_rule_id)
SELECT ip_rules.id FROM UNNEST($1::inet[], $2::bool[]) AS t(ip, allow)
INNER JOIN ip_rules ON ip_rules.ip_network = t.ip AND ip_rules.allow = t.allow
ON CONFLICT DO NOTHING",
&ips[..],
&allow_ips[..]
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"INSERT INTO rule_source (source, rule_id)
SELECT source, Rules.id FROM UNNEST ($1::text[], $2::inet[], $3::bool[]) AS t(source, ip, allow)
INNER JOIN ip_rules ON ip_rules.ip_network = t.ip AND ip_rules.allow = t.allow
INNER JOIN Rules ON Rules.ip_rule_id = ip_rules.id
ON CONFLICT DO NOTHING",
&ip_source[..],
&ips[..],
&allow_ips[..]
)
.execute(&mut *tx)
.await?;
sqlx::query!("INSERT INTO list_rules (list_id, source_id)
SELECT $1, rule_source.id FROM UNNEST ($2::text[], $3::inet[], $4::bool[]) AS t(source, ip, allow)
INNER JOIN ip_rules ON ip_rules.ip_network = t.ip AND ip_rules.allow = t.allow
INNER JOIN Rules ON Rules.ip_rule_id = ip_rules.id
INNER JOIN rule_source ON rule_source.rule_id = Rules.id
WHERE rule_source.source = t.source
ON CONFLICT DO NOTHING",
list_id,
&ip_source[..],
&ips[..],
&allow_ips[..]
).execute(&mut *tx).await?;
sqlx::query!(
"INSERT INTO Rules (domain_rule_id, ip_rule_id) VALUES (NULL, NULL)
ON CONFLICT DO NOTHING",
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"INSERT INTO rule_source (source, rule_id)
SELECT source, Rules.id FROM UNNEST ($1::text[]) AS t(source)
INNER JOIN Rules ON Rules.domain_rule_id IS NULL AND Rules.ip_rule_id IS NULL
ON CONFLICT DO NOTHING",
&other_rules_src[..],
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"INSERT INTO list_rules (list_id, source_id)
SELECT $1, rule_source.id FROM UNNEST($2::text[], $3::text[], $4::bool[], $5::bool[]) AS t(source, domain, allow, subdomain)
INNER JOIN domains ON domains.domain = t.domain
INNER JOIN domain_rules ON domain_rules.domain_id = domains.id AND domain_rules.allow = t.allow AND domain_rules.subdomain = t.subdomain
INNER JOIN Rules ON Rules.domain_rule_id = domain_rules.id
INNER JOIN rule_source ON rule_source.rule_id = Rules.id
WHERE rule_source.source = t.source
ON CONFLICT DO NOTHING",
list_id,
&domain_src[..],
&domains[..] as _,
&allow[..],
&subdomain[..]
).execute(&mut *tx).await?;
sqlx::query!(
"INSERT INTO list_rules (list_id, source_id)
SELECT $1, rule_source.id FROM UNNEST ($2::text[]) AS t(source)
INNER JOIN Rules ON Rules.domain_rule_id IS NULL AND Rules.ip_rule_id IS NULL
INNER JOIN rule_source ON rule_source.rule_id = Rules.id
WHERE rule_source.source = t.source
ON CONFLICT DO NOTHING",
list_id,
&other_rules_src[..],
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"UPDATE filterLists SET rule_count=$2
WHERE id = $1",
list_id,
rules.len() as i32
)
.execute(&mut *tx)
.await?;
log::info!("Inserted list rules");
tx.commit().await?;
log::info!("Total time: {:?}", start.elapsed());
Ok(())
}
#[derive(Clone, Debug, Serialize, Deserialize)]
struct CsvRecord {
pub name: String,
pub url: FilterListUrl,
pub author: String,
pub license: String,
pub expires: u64,
pub list_type: FilterListType,
}
#[server]
pub async fn load_filter_map() -> Result<(), ServerFnError> {
dotenvy::dotenv()?;
let filterlists_path: std::path::PathBuf = std::env::var("FILTERLISTS_PATH")?.parse()?;
let contents = tokio::fs::read_to_string(filterlists_path).await?;
let records = csv::Reader::from_reader(contents.as_bytes())
.deserialize::<CsvRecord>()
.collect::<Result<Vec<CsvRecord>, _>>()?;
let mut urls = Vec::new();
let mut names = Vec::new();
let mut formats = Vec::new();
let mut expires_list = Vec::new();
let mut authors = Vec::new();
let mut licenses = Vec::new();
for csv_record in &records {
let url = csv_record.url.as_str().to_string();
let name = csv_record.name.clone();
let format = csv_record.list_type.as_str().to_string();
let expires = csv_record.expires as i32;
let author = csv_record.author.clone();
let license = csv_record.license.clone();
urls.push(url);
names.push(name);
formats.push(format);
expires_list.push(expires);
authors.push(author);
licenses.push(license);
}
let pool = crate::server::get_db().await?;
sqlx::query!(
"INSERT INTO filterLists (url, name, format, expires, author, license)
SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::int[], $5::text[], $6::text[])
ON CONFLICT (url) DO UPDATE
SET name = EXCLUDED.name, format = EXCLUDED.format, expires = EXCLUDED.expires, author = EXCLUDED.author, license = EXCLUDED.license
",
&urls,
&names,
&formats,
&expires_list,
&authors,
&licenses
).execute(&pool).await?;
write_filter_map().await?;
Ok(())
}
#[server]
pub async fn watch_filter_map() -> Result<(), ServerFnError> {
dotenvy::dotenv()?;
let filterlists_path: std::path::PathBuf = std::env::var("FILTERLISTS_PATH")?.parse()?;
use notify::Watcher;
let notify = std::sync::Arc::new(tokio::sync::Notify::new());
let notify2 = notify.clone();
load_filter_map().await?;
let mut watcher = notify::recommended_watcher(move |_| {
notify.notify_one();
})?;
watcher.watch(&filterlists_path, notify::RecursiveMode::NonRecursive)?;
let mut last_updated = std::time::Instant::now();
loop {
notify2.notified().await;
if last_updated.elapsed() > std::time::Duration::from_millis(200) {
load_filter_map().await?;
last_updated = std::time::Instant::now();
}
}
}
#[server]
pub async fn write_filter_map() -> Result<(), ServerFnError> {
use csv::Writer;
dotenvy::dotenv()?;
let filterlists_path: std::path::PathBuf = std::env::var("FILTERLISTS_PATH")?.parse()?;
let pool = crate::server::get_db().await?;
let rows = sqlx::query!("SELECT url, name, format, expires, author, license FROM filterLists")
.fetch_all(&pool)
.await?;
let mut records = Vec::new();
for record in rows {
records.push(CsvRecord {
name: record.name.unwrap_or(String::new()),
url: record.url.parse()?,
author: record.author.unwrap_or(String::new()),
license: record.license.unwrap_or(String::new()),
expires: record.expires as u64,
list_type: FilterListType::from_str(&record.format)?,
});
}
records.sort_by_key(|record| (record.name.clone(), record.url.clone()));
records.reverse();
let mut wtr = Writer::from_path(filterlists_path)?;
for record in records {
wtr.serialize(record)?;
}
Ok(())
}
#[server]
pub async fn get_filter_map() -> Result<FilterListMap, ServerFnError> {
let pool = crate::server::get_db().await?;
let rows = sqlx::query!(
"SELECT url, name, format, expires, author, license, lastupdated, rule_count
FROM filterLists"
)
.fetch_all(&pool)
.await?;
let mut filter_list_map = Vec::new();
for record in rows {
let url = record.url.parse()?;
let record = FilterListRecord {
name: record.name.unwrap_or(String::new()).into(),
list_format: FilterListType::from_str(&record.format)?,
author: record.author.unwrap_or(String::new()).into(),
license: record.license.unwrap_or(String::new()).into(),
expires: std::time::Duration::from_secs(record.expires as u64),
last_updated: record.lastupdated,
list_size: record.rule_count as usize,
};
filter_list_map.push((url, record));
}
Ok(FilterListMap(filter_list_map))
}
#[cfg(feature = "ssr")]
struct LastVersionData {
last_updated: chrono::DateTime<chrono::Utc>,
etag: Option<String>,
}
#[cfg(feature = "ssr")]
async fn get_last_version_data(
url: &FilterListUrl,
) -> Result<Option<LastVersionData>, ServerFnError> {
let pool = crate::server::get_db().await?;
let url_str = url.as_str();
#[allow(non_camel_case_types)]
let last_version_data = sqlx::query!(
r#"SELECT lastUpdated as "last_updated: chrono::DateTime<chrono::Utc>", etag FROM filterLists WHERE url = $1"#,
url_str
)
.fetch_one(&pool)
.await
.ok();
let last_version_data = last_version_data.and_then(|row| {
Some(LastVersionData {
last_updated: row.last_updated?,
etag: row.etag,
})
});
Ok(last_version_data)
}
#[server]
pub async fn get_last_updated(
url: FilterListUrl,
) -> Result<Option<chrono::DateTime<chrono::Utc>>, ServerFnError> {
get_last_version_data(&url)
.await
.map(|data| data.map(|data| data.last_updated))
}
#[cfg(feature = "ssr")]
#[derive(thiserror::Error, Debug)]
enum UpdateListError {
#[error("Failed to fetch list")]
FailedToFetch,
}
#[server(UpdateListFn)]
pub async fn update_list(url: FilterListUrl) -> Result<(), ServerFnError> {
let pool = crate::server::get_db().await?;
let old_contents = sqlx::query!(
"SELECT contents FROM filterLists WHERE url = $1",
url.as_str()
)
.fetch_one(&pool)
.await?
.contents;
if let Some(internal_path) = url.to_internal_path() {
let contents = tokio::fs::read_to_string(&internal_path).await?;
let mut lines = contents.lines().collect::<Vec<_>>();
lines.sort_unstable();
lines.dedup();
let sorted_contents = lines.join("\n");
tokio::fs::write(internal_path, &sorted_contents).await?;
let new_last_updated = chrono::Utc::now();
sqlx::query!(
"UPDATE filterLists
SET lastUpdated = $2, contents = $3
WHERE url = $1
",
url.as_str(),
new_last_updated,
sorted_contents
)
.execute(&pool)
.await?;
if old_contents != Some(sorted_contents) {
parse_list(url).await?;
}
return Ok(());
}
log::info!("Updating {}", url.as_str());
let url_str = url.as_str();
let last_updated = get_last_version_data(&url).await?;
let mut req = reqwest::Client::new().get(url_str);
if let Some(last_updated) = last_updated {
req = req.header(
"if-modified-since",
last_updated
.last_updated
.format("%a, %d %b %Y %H:%M:%S GMT")
.to_string(),
);
if let Some(etag) = last_updated.etag {
req = req.header("if-none-match", etag);
}
}
let response = req.send().await?;
match response.status() {
reqwest::StatusCode::NOT_MODIFIED => {
log::info!("Not modified {:?}", url_str);
sqlx::query!(
"UPDATE filterLists
SET lastUpdated = NOW()
WHERE url = $1
",
url_str
)
.execute(&pool)
.await?;
Ok(())
}
reqwest::StatusCode::OK => {
let headers = response.headers().clone();
let etag = headers.get("etag").and_then(|item| item.to_str().ok());
let body = response.text().await?;
log::info!("Updated {} size ({})", url_str, body.len());
sqlx::query!(
"UPDATE filterLists
SET contents = $2, etag = $3
WHERE url = $1
",
url_str,
body,
etag
)
.execute(&pool)
.await?;
if Some(body) == old_contents {
log::info!("No change in contents for {}", url_str);
} else {
parse_list(url.clone()).await?;
}
sqlx::query!(
"UPDATE filterLists
SET lastUpdated = NOW()
WHERE url = $1
",
url_str
)
.execute(&pool)
.await?;
Ok(())
}
status => {
log::error!("Error fetching {}: {:?}", url_str, status);
Err(UpdateListError::FailedToFetch.into())
}
}
}
#[server(DeleteListFn)]
pub async fn delete_list(url: FilterListUrl) -> Result<(), ServerFnError> {
let pool = crate::server::get_db().await?;
let url_str = url.as_str();
sqlx::query!(
"DELETE FROM list_rules
WHERE list_rules.list_id IN (
SELECT id FROM filterLists WHERE url = $1
)",
url_str
)
.execute(&pool)
.await?;
sqlx::query!("DELETE FROM filterLists WHERE url = $1", url_str)
.execute(&pool)
.await?;
write_filter_map().await?;
Ok(())
}
#[component]
pub fn FilterListLink(url: FilterListUrl) -> impl IntoView {
let href = format!(
"/list{}",
params_map! {
"url" => url.as_str(),
}
.to_query_string(),
);
view! {
<A href=href class="link link-neutral">
{url.as_str().to_string()}
</A>
}
}
#[server]
async fn get_list_size(url: FilterListUrl) -> Result<Option<usize>, ServerFnError> {
let pool = crate::server::get_db().await?;
let url_str = url.as_str();
let record = sqlx::query!(
"SELECT id, rule_count FROM filterLists WHERE url = $1",
url_str
)
.fetch_one(&pool)
.await?;
let list_id = record.id;
let count = record.rule_count;
if count == 0 {
let count = sqlx::query!(
"SELECT COUNT(*) FROM list_rules WHERE list_id = $1",
list_id
)
.fetch_one(&pool)
.await?
.count;
if let Some(count) = count {
sqlx::query!(
"UPDATE filterLists SET rule_count = $1 WHERE id = $2",
count as i32,
list_id
)
.execute(&pool)
.await?;
Ok(Some(count as usize))
} else {
Ok(None)
}
} else {
Ok(Some(count as usize))
}
}
#[component]
pub fn ListSize(url: FilterListUrl, list_size: Option<usize>) -> impl IntoView {
if let Some(size) = list_size {
if size > 0 {
return size.into_view();
}
}
view! {
<Await
future=move || {
let url = url.clone();
async { get_list_size(url).await }
}
let:size
>
{match size {
Err(err) => format!("{err:?}").into_view(),
Ok(None) => "Never".into_view(),
Ok(Some(size)) => size.into_view(),
}}
</Await>
}
.into_view()
}
#[server]
async fn get_list_page(
url: FilterListUrl,
page: Option<usize>,
page_size: usize,
) -> Result<Vec<(RuleId, SourceId, crate::filterlist::RulePair)>, ServerFnError> {
let pool = crate::server::get_db().await?;
let url_str = url.as_str();
let id = sqlx::query!("SELECT id FROM filterLists WHERE url = $1", url_str)
.fetch_one(&pool)
.await?
.id;
let start = page.unwrap_or(0) * page_size;
let records = sqlx::query!(
r#"SELECT Rules.id AS "rule_id: RuleId", rule_source.id AS "source_id: SourceId", rule_source.source,
domain as "domain: Option<String>" , domain_rules.allow as "domain_allow: Option<bool>", subdomain as "subdomain: Option<bool>",
ip_network as "ip_network: Option<ipnetwork::IpNetwork>", ip_rules.allow as "ip_allow: Option<bool>"
FROM list_rules
INNER JOIN rule_source ON rule_source.id = list_rules.source_id
INNER JOIN Rules ON Rules.id = rule_source.rule_id
LEFT JOIN domain_rules ON domain_rules.id = Rules.domain_rule_id
LEFT JOIN domains ON domains.id = domain_rules.domain_id
LEFT JOIN ip_rules ON ip_rules.id = Rules.ip_rule_id
WHERE list_id = $1
ORDER BY list_rules.source_id
LIMIT $2 OFFSET $3
"#r,
id,
page_size as i64 ,
start as i64
)
.fetch_all(&pool)
.await?;
let rules = records
.iter()
.map(|record| {
let rule_data = RuleData {
rule_id: record.rule_id,
domain: record.domain.clone(),
domain_allow: record.domain_allow,
domain_subdomain: record.subdomain,
ip_network: record.ip_network,
ip_allow: record.ip_allow,
};
let rule = rule_data.try_into()?;
let source = record.source.clone();
let pair = crate::filterlist::RulePair::new(source.into(), rule);
Ok((record.rule_id, record.source_id, pair))
})
.collect::<Result<Vec<(_, _, _)>, ServerFnError>>();
rules
}
#[component]
fn LastUpdatedInner(last_updated: Option<chrono::DateTime<chrono::Utc>>) -> impl IntoView {
view! {
{match last_updated {
Some(last_updated) => {
view! { <div>{format!("{last_updated}")}</div> }
}
None => {
view! { <div>"Never"</div> }
}
}}
}
}
#[component]
pub fn LastUpdated(url: FilterListUrl, record: Option<FilterListRecord>) -> impl IntoView {
view! {
{match record.clone() {
Some(record) => {
let last_updated = record.last_updated;
view! { <LastUpdatedInner last_updated=last_updated/> }
}
None => {
view! {
<Await
future={
let url = url.clone();
move || {
let url = url.clone();
async move {
crate::filterlist::get_last_updated(url.clone()).await
}
}
}
let:last_version_data
>
{match last_version_data {
Ok(last_updated) => {
view! { <LastUpdatedInner last_updated=*last_updated/> }.into_view()
}
Err(err) => view! { {format!("{err:?}")} }.into_view(),
}}
</Await>
}
}
}}
<FilterListUpdate url=url.clone()/>
}
}
#[component]
pub fn ParseList(url: FilterListUrl) -> impl IntoView {
let parse_list_action = create_server_action::<crate::filterlist::ParseList>();
view! {
<ActionForm action=parse_list_action>
<button class="btn btn-primary" type="submit">
<input type="hidden" placeholder="url" id="url" name="url" value=url.to_string()/>
"Parse"
</button>
</ActionForm>
}
}
#[component]
pub fn FilterListUpdate(url: FilterListUrl) -> impl IntoView {
let update_list_action = create_server_action::<UpdateListFn>();
view! {
<ActionForm action=update_list_action>
<button class="btn btn-primary" type="submit">
<input type="hidden" placeholder="url" id="url" name="url" value=url.to_string()/>
"Update"
</button>
</ActionForm>
}
}
#[component]
fn Contents(url: FilterListUrl, page: Option<usize>) -> impl IntoView {
view! {
<table class="table table-zebra">
<thead>
<tr>
<th>Source</th>
<th>Rule</th>
</tr>
</thead>
<Await
future=move || {
let url = url.clone();
async move { get_list_page(url, page, PAGE_SIZE).await }
}
let:contents
>
{
let contents = contents.clone();
move || match contents.clone() {
Ok(contents) => {
let contents = contents.clone();
view! {
<tbody>
<For
each=move || { contents.clone() }
key=|(rule_id, source_id, _)| (*rule_id, *source_id)
children=|(rule_id, _source_id, pair)| {
let source = pair.get_source().to_string();
let rule = pair.get_rule().clone();
view! {
<tr>
<td>{source}</td>
<td>
<A href=rule_id.get_href() class="link link-neutral">
<DisplayRule rule=rule/>
</A>
</td>
</tr>
}
}
/>
</tbody>
}
.into_view()
}
Err(err) => format!("{err:?}").into_view(),
}
}
</Await>
</table>
}
}
#[component]
fn FilterListInner(url: FilterListUrl, page: Option<usize>) -> impl IntoView {
view! {
<h1>"Filter List"</h1>
<p>"URL: " {url.to_string()}</p>
<p>"Last Updated: " <LastUpdated url=url.clone() record=None/></p>
<p>"Rule count: " <ListSize url=url.clone() list_size=None/></p>
<FilterListUpdate url=url.clone()/>
<p>
<ParseList url=url.clone()/>
</p>
<DeleteListButton url=url.clone()/>
{if let Some(page) = page {
view! { <p>"Page: " {page}</p> }
} else {
view! { <p>"Page: 0"</p> }
}}
{match page {
None | Some(0) => view! {}.into_view(),
Some(page) => {
let params = params_map! {
"url" => url.as_str(), "page" => (page.saturating_sub(1)).to_string()
};
let href = format!("/list{}", params.to_query_string());
view! {
<A href=href class="btn btn-neutral">
"Back"
</A>
}
}
}}
{
let params = params_map! {
"url" => url.as_str(), "page" => (page.unwrap_or(0) + 1).to_string()
};
let href = format!("/list{}", params.to_query_string());
view! {
<A href=href class="btn btn-neutral">
"Next"
</A>
}
}
<p>"Contents: " <Contents url=url.clone() page=page/></p>
}
}
#[derive(Params, PartialEq, Debug)]
struct ViewListParams {
url: Option<String>,
page: Option<usize>,
}
#[derive(thiserror::Error, Debug)]
enum ViewListError {
#[error("Invalid URL")]
ParseURL(#[from] url::ParseError),
#[error("Invalid URL")]
ParseParam(#[from] leptos_router::ParamsError),
#[error("Invalid FilterListType")]
InvalidFilterListType(#[from] InvalidFilterListTypeError),
}
impl ViewListParams {
fn parse(&self) -> Result<FilterListUrl, ViewListError> {
Ok(self
.url
.as_ref()
.ok_or_else(|| ParamsError::MissingParam("Missing Param".into()))?
.parse()?)
}
}
#[component]
fn DeleteListButton(url: FilterListUrl) -> impl IntoView {
let delete_list_action = create_server_action::<DeleteListFn>();
view! {
<ActionForm action=delete_list_action>
<button class="btn btn-danger" type="submit">
<input type="hidden" placeholder="url" id="url" name="url" value=url.to_string()/>
"Delete"
</button>
</ActionForm>
}
}
#[component]
pub fn FilterListPage() -> impl IntoView {
let params = use_query::<ViewListParams>();
let get_url = move || {
params.with(|param| {
param
.as_ref()
.ok()
.map(|param| param.parse().map(|url| (url, param.page)))
})
};
view! {
<div>
{move || match get_url() {
None => view! { <p>"No URL"</p> }.into_view(),
Some(Err(err)) => view! { <p>"Error: " {format!("{err}")}</p> }.into_view(),
Some(Ok((url, page))) => view! { <FilterListInner url=url page=page/> }.into_view(),
}}
</div>
}
}
================================================
FILE: src/home_page.rs
================================================
use crate::filterlist::{FilterListRecord, FilterListUrl, LastUpdated, ListSize};
use leptos::*;
use leptos_router::*;
#[component]
fn FilterListSummary(url: FilterListUrl, record: FilterListRecord) -> impl IntoView {
view! {
<tr>
<td class="break-normal break-words max-w-20">
{
let name = if record.name.is_empty() {
url.to_string()
} else {
record.name.to_string()
};
let href = format!(
"/list{}",
params_map! {
"url" => url.as_str(),
}
.to_query_string(),
);
view! {
<A href=href class="link link-neutral">
{name}
</A>
}
}
</td>
<td class="break-normal break-words max-w-20">{record.author.to_string()}</td>
<td class="max-w-xl break-normal break-words">{record.license.to_string()}</td>
<td>{humantime::format_duration(record.expires).to_string()}</td>
<td>{format!("{:?}", record.list_format)}</td>
<td>
<LastUpdated url=url.clone() record=Some(record.clone())/>
</td>
<td class="text-right">
<ListSize url=url.clone() list_size=Some(record.list_size)/>
</td>
</tr>
}
}
/// Renders the home page of your application.
#[component]
pub fn HomePage() -> impl IntoView {
view! {
<Await
future=|| async move { crate::filterlist::get_filter_map().await }
let:filterlist_map
>
{match filterlist_map.clone() {
Ok(data) => {
view! {
<table class="table table-zebra">
<thead>
<td>"Name"</td>
<td>"Author"</td>
<td>"License"</td>
<td>"Update frequency"</td>
<td>"Format"</td>
<td>"Last Updated"</td>
<td>"Size"</td>
</thead>
<tbody>
<For
each=move || {
let mut data = data.0.clone();
data.sort_unstable_by(|a, b| {
b.1.last_updated.cmp(&a.1.last_updated)
});
data
}
key=|(url, _)| url.as_str().to_string()
children=|(url, record)| {
view! {
<FilterListSummary url=url.clone() record=record.clone()/>
}
}
/>
</tbody>
</table>
}
.into_view()
}
Err(err) => view! { <p>"Error Loading " {format!("{err:?}")}</p> }.into_view(),
}}
</Await>
}
}
================================================
FILE: src/ip_view.rs
================================================
use leptos::*;
use leptos_router::*;
use std::{collections::BTreeSet, net::IpAddr};
use crate::{app::Loading, domain::DomainId};
#[server]
async fn get_domans_which_resolve_to_ip(
ip: IpAddr,
) -> Result<BTreeSet<(DomainId, String)>, ServerFnError> {
let ip: ipnetwork::IpNetwork = ip.into();
let records = sqlx::query!(
r#"SELECT domains.id as "domain_id: DomainId", domain
from dns_ips
INNER JOIN domains ON dns_ips.domain_id = domains.id
WHERE dns_ips.ip_address = $1"#r,
ip
)
.fetch_all(&crate::server::get_db().await?)
.await?;
let domains = records
.into_iter()
.map(|record| (record.domain_id, record.domain))
.collect::<BTreeSet<_>>();
Ok(domains)
}
#[component]
fn DomainsWhichResolveTo(get_ip: GetIp) -> impl IntoView {
let domain_results = create_resource(get_ip, |ip| async move {
let ip = ip?;
let results = get_domans_which_resolve_to_ip(ip).await?;
Ok::<_, ServerFnError>(results)
});
view! {
<Transition fallback=move || {
view! { <p>"Loading" <Loading/></p> }
}>
{move || match domain_results.get() {
Some(Ok(domains)) => {
view! {
<div>
<p>"Domains which resolve to this IP Address"</p>
<ul class="grid grid-cols-2 gap-2">
<For
each=move || { domains.clone() }
key=|(domain_id, _domain)| *domain_id
children=|(_domain_id, domain)| {
view! {
<li>
<A
href=format!("/domain/{domain}")
class="link link-neutral"
>
{domain}
</A>
</li>
}
}
/>
</ul>
</div>
}
.into_view()
}
_ => view! { <p>"Error"</p> }.into_view(),
}}
</Transition>
}
}
type GetIp = Box<dyn Fn() -> Result<IpAddr, ParamsError>>;
#[derive(Params, PartialEq)]
struct IpParam {
ip: Option<IpAddr>,
}
#[component]
pub fn IpView() -> impl IntoView {
let params = use_params::<IpParam>();
let get_ip = move || {
params.with(|param| {
param.as_ref().map_err(Clone::clone).and_then(|param| {
param
.ip
.ok_or_else(|| ParamsError::MissingParam("No domain".into()))
})
})
};
view! {
<div>
<h1 class="text-2xl font-bold text-gray-800">{"IP Address: "} {get_ip}</h1>
<DomainsWhichResolveTo get_ip=Box::new(get_ip)/>
</div>
}
}
================================================
FILE: src/lib.rs
================================================
pub mod app;
pub mod domain;
pub mod error_template;
pub mod filterlist;
pub mod home_page;
pub mod ip_view;
pub mod rule;
#[cfg(feature = "ssr")]
pub mod server;
pub mod stats_view;
pub mod tasks;
#[cfg(feature = "ssr")]
use mimalloc::MiMalloc;
use serde::*;
#[cfg(feature = "ssr")]
#[global_allocator]
static GLOBAL: MiMalloc = MiMalloc;
const PAGE_SIZE: usize = 50;
pub mod source {
use super::*;
#[cfg_attr(feature = "ssr", derive(sqlx::Encode, sqlx::Decode))]
#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct SourceId(i32);
}
#[derive(thiserror::Error, Debug)]
pub enum DbInitError {
#[error("Sqlx error {0}")]
SqlxError(String),
#[error("Missing DATABASE_URL")]
MissingDatabaseUrl(String),
}
#[cfg(feature = "ssr")]
impl From<sqlx::Error> for DbInitError {
fn from(e: sqlx::Error) -> Self {
Self::SqlxError(e.to_string())
}
}
#[cfg(feature = "ssr")]
impl From<std::env::VarError> for DbInitError {
fn from(e: std::env::VarError) -> Self {
Self::MissingDatabaseUrl(e.to_string())
}
}
#[cfg(feature = "ssr")]
pub mod fileserv;
#[cfg(feature = "hydrate")]
#[wasm_bindgen::prelude::wasm_bindgen]
pub fn hydrate() {
console_error_panic_hook::set_once();
let _ = console_log::init_with_level(log::Level::Debug);
log::info!("Hydrating");
leptos::mount_to_body(crate::app::App);
log::info!("Mounted");
// leptos::leptos_dom::HydrationCtx::stop_hydrating();
}
================================================
FILE: src/main.rs
================================================
#[cfg(feature = "ssr")]
use clap::Parser;
#[cfg(feature = "ssr")]
#[derive(Debug, serde::Deserialize, serde::Serialize)]
struct Config {
listen_port: u16,
peers: Vec<blockconvert::server::Peer>,
}
impl Default for Config {
fn default() -> Self {
Self {
listen_port: 3000,
peers: vec![Default::default(); 2],
}
}
}
#[cfg(feature = "ssr")]
/// Simple program to greet a person
#[derive(Parser, Debug)]
#[command(version, about, long_about = None)]
struct Args {
config_path: std::path::PathBuf,
#[arg(short, long)]
create_config: bool,
#[arg(short, long)]
run_tasks: bool,
}
#[cfg(feature = "ssr")]
#[tokio::main]
async fn main() {
use axum::Router;
use blockconvert::app::App;
use blockconvert::fileserv::file_and_error_handler;
use blockconvert::{filterlist, server};
use leptos::*;
use leptos_axum::{generate_route_list, LeptosRoutes};
use tower_http::compression::CompressionLayer;
use tower_http::compression::CompressionLevel;
env_logger::init();
let args = Args::parse();
let config_path = args.config_path.clone();
let node_conf = if let Ok(conf) = tokio::fs::read_to_string(args.config_path).await {
let Ok(conf): Result<Config, _> = toml::from_str(&conf) else {
logging::warn!("Error parsing config file");
return;
};
conf
} else if args.create_config {
let conf = Config::default();
let conf_str = toml::to_string(&conf).unwrap();
tokio::fs::write(config_path, conf_str).await.unwrap();
logging::log!("Created config file");
conf
} else {
logging::log!("Config file not found");
return;
};
println!("Config: {:?}", node_conf);
let peer_state = server::PeerState::new(&node_conf.peers);
// Setting get_configuration(None) means we'll be using cargo-leptos's env values
// For deployment these variables are:
// <https://github.com/leptos-rs/start-axum#executing-a-server-on-a-remote-machine-without-the-toolchain>
// Alternately a file can be specified such as Some("Cargo.toml")
// The file would need to be included with the executable when moved to deployment
let conf = get_configuration(Some("Cargo.toml")).await.unwrap();
let mut leptos_options = conf.leptos_options;
let mut addr = leptos_options.site_addr;
addr.set_port(node_conf.listen_port);
leptos_options.site_addr = addr;
let routes = generate_route_list(App);
//let state = State { leptos_options };
// build our application with a route
let app = Router::new()
.leptos_routes(&leptos_options, routes, App)
.nest("/peer/", server::get_peer_router(peer_state.clone()))
.fallback(file_and_error_handler)
.with_state(leptos_options)
.layer(CompressionLayer::new().quality(CompressionLevel::Fastest));
let token = tokio_util::sync::CancellationToken::new();
let mut tasks = tokio::task::JoinSet::new();
if args.run_tasks {
tasks.spawn(filterlist::watch_filter_map());
tasks.spawn(server::parse_missing_subdomains());
tasks.spawn(server::check_dns(token.clone()));
tasks.spawn(server::import_pihole_logs());
tasks.spawn(blockconvert::rule::find_rule_matches());
tasks.spawn(server::build_list());
tasks.spawn(server::update_expired_lists());
tasks.spawn(server::garbage_collect());
tasks.spawn(server::run_cmd(token.clone()));
tasks.spawn(server::certstream(token.clone()));
}
let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
let server = tasks.spawn(async move {
logging::log!("listening on http://{}", &listener.local_addr()?);
axum::serve(listener, app.into_make_service()).await?;
Ok(())
});
{
let token = token.clone();
tasks.spawn(async move {
let mut interrupt =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()).unwrap();
let mut hangup =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup()).unwrap();
let mut terminate =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap();
tokio::select! {
_ = tokio::signal::ctrl_c() =>
logging::log!("Ctrl-C received, shutting down"),
_ = interrupt.recv() =>
logging::log!("Interrupt received, shutting down"),
_ = hangup.recv() =>
logging::log!("Hangup received, shutting down"),
_ = terminate.recv() =>
logging::log!("Terminate received, shutting down"),
}
token.cancel();
Ok(())
});
}
while let Some(task) = tasks.join_next().await {
if let Err(e) = task.unwrap() {
logging::log!("Error: {:?}", e);
return;
}
logging::log!("Task completed");
if token.is_cancelled() {
server.abort();
logging::log!("Shutting down");
tokio::select! {
_ = async move {while tasks.join_next().await.is_some() {}} => {},
_ = tokio::time::sleep(std::time::Duration::from_secs(5)) => {}
}
break;
}
}
logging::log!("Exiting");
}
#[cfg(not(feature = "ssr"))]
pub fn main() {
// no client-side main function
// unless we want this to work with e.g., Trunk for a purely client-side app
// see lib.rs for hydration function instead
}
================================================
FILE: src/rule.rs
================================================
use crate::app::Loading;
use crate::filterlist::DomainRule;
use crate::filterlist::FilterListLink;
use crate::filterlist::FilterListUrl;
use crate::filterlist::Rule;
use crate::{domain::DomainId, filterlist::ListId, source::SourceId};
use leptos::*;
use leptos_router::*;
use serde::{Deserialize, Serialize};
#[cfg_attr(feature = "ssr", derive(sqlx::Encode, sqlx::Decode))]
#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct RuleId(i32);
impl RuleId {
pub fn get_href(&self) -> String {
format!("/rule/{}", self.0)
}
}
#[cfg(feature = "ssr")]
pub async fn find_rule_matches() -> Result<(), ServerFnError> {
use std::time::Duration;
dotenvy::dotenv()?;
let pool = crate::server::get_db().await?;
let read_limit = std::env::var("READ_LIMIT")?.parse::<u32>()? as i64;
let interval: u64 = std::env::var("RULE_MATCH_CHECK_INTERVAL")?.parse()?;
let interval: Duration = Duration::from_secs(interval);
let mut interval = tokio::time::interval(interval);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
interval.tick().await;
let records = sqlx::query!(
"SELECT id from Rules
ORDER BY last_checked_matches ASC NULLS FIRST
LIMIT $1",
read_limit
)
.fetch_all(&pool)
.await?;
let rule_ids = records
.into_iter()
.map(|record| record.id)
.collect::<Vec<_>>();
let mut tx = pool.begin().await?;
sqlx::query!(
"DELETE FROM rule_matches WHERE rule_id = ANY($1::int[])",
&rule_ids[..]
)
.execute(&mut *tx)
.await?;
sqlx::query!("
INSERT INTO rule_matches(rule_id, domain_id)
SELECT Rules.id AS rule_id, domains.id AS domain_id
FROM Rules
LEFT JOIN domain_rules ON Rules.domain_rule_id = domain_rules.id
LEFT JOIN subdomains ON domain_rules.domain_id = subdomains.parent_domain_id AND domain_rules.subdomain = true
LEFT JOIN ip_rules ON Rules.ip_rule_id = ip_rules.id AND ip_rules.allow=false
LEFT JOIN dns_ips ON ip_rules.ip_network = dns_ips.ip_address
LEFT JOIN dns_cnames ON (dns_cnames.cname_domain_id = domain_rules.domain_id
OR dns_cnames.cname_domain_id = subdomains.domain_id) AND domain_rules.allow=false
INNER JOIN domains ON domain_rules.domain_id = domains.id
OR subdomains.domain_id = domains.id
OR dns_ips.domain_id = domains.id
OR dns_cnames.domain_id = domains.id
INNER JOIN dns_ips AS dns_check ON dns_check.domain_id = domains.id AND dns_check.ip_address IS NOT NULL
WHERE Rules.id = ANY($1::int[])
ON CONFLICT DO NOTHING",
&rule_ids[..]).execute(&mut *tx).await?;
let count = sqlx::query!(
"SELECT COUNT(*) FROM rule_matches WHERE rule_id = ANY($1::int[])",
&rule_ids[..]
)
.fetch_one(&mut *tx)
.await?
.count
.unwrap_or(0);
log::info!(
"Checked {} rules and found {} matches",
rule_ids.len(),
count
);
sqlx::query!(
"UPDATE rules
SET last_checked_matches = now()
WHERE id = ANY($1::int[])",
&rule_ids[..]
)
.execute(&mut *tx)
.await?;
tx.commit().await?;
}
}
#[server]
pub async fn get_rule(id: RuleId) -> Result<Rule, ServerFnError> {
let record = sqlx::query!(
"SELECT domain_rule_id, ip_rule_id FROM Rules
WHERE Rules.id = $1",
id.0
)
.fetch_one(&crate::server::get_db().await?)
.await?;
if let Some(domain_rule_id) = record.domain_rule_id {
let record = sqlx::query!(
"SELECT domain, allow, subdomain FROM domain_rules
INNER JOIN domains ON domains.id = domain_rules.domain_id
WHERE domain_rules.id = $1",
domain_rule_id
)
.fetch_one(&crate::server::get_db().await?)
.await?;
let domain_rule = DomainRule {
domain: record.domain.parse()?,
allow: record.allow,
subdomain: record.subdomain,
};
Ok(Rule::Domain(domain_rule))
} else if let Some(ip_rule_id) = record.ip_rule_id {
let record = sqlx::query!(
"SELECT ip_network, allow FROM ip_rules WHERE id = $1",
ip_rule_id
)
.fetch_one(&crate::server::get_db().await?)
.await?;
Ok(Rule::IpRule(crate::filterlist::IpRule {
ip: record.ip_network,
allow: record.allow,
}))
} else {
Ok(Rule::Invalid)
}
}
type GetId = Box<dyn Fn() -> Result<RuleId, ParamsError>>;
#[server]
async fn get_sources(
id: RuleId,
) -> Result<Vec<(SourceId, String, ListId, FilterListUrl)>, ServerFnError> {
let sources = sqlx::query!(
r#"SELECT rule_source.id AS "source_id: SourceId", source, filterLists.id as "list_id: ListId", filterLists.url FROM rule_source
INNER JOIN list_rules ON rule_source.id = list_rules.source_id
INNER JOIN filterLists ON list_rules.list_id = filterLists.id
WHERE rule_id = $1
ORDER BY (source)
"#r,
id.0
)
.fetch_all(&crate::server::get_db().await?)
.await?;
sources
.into_iter()
.map(|record| {
Ok((
record.source_id,
record.source,
record.list_id,
record.url.parse()?,
))
})
.collect()
}
#[component]
fn Sources(get_id: GetId) -> impl IntoView {
let source_resource = create_resource(get_id, |id| async move {
let sources = get_sources(id?).await?;
Ok::<_, ServerFnError>(sources)
});
view! {
<Transition fallback=move || {
view! { <p>"Loading" <Loading/></p> }
}>
{move || match source_resource.get() {
Some(Ok(sources)) => {
view! {
<p>
"Sources:" <table class="table table-zebra">
<thead>
<tr>
<th>Source</th>
<th>List</th>
</tr>
</thead>
<tbody>
<For
each=move || { sources.clone() }
key=|(source_id, _, _, _)| *source_id
children=|(_, source, _list_id, url)| {
view! {
<tr>
<td>{source}</td>
<td>
<FilterListLink url=url/>
</td>
</tr>
}
}
/>
</tbody>
</table>
</p>
}
.into_view()
}
Some(Err(err)) => view! { <p>"Error: " {format!("{err}")}</p> }.into_view(),
None => view! { "Invalid URL" }.into_view(),
}}
</Transition>
}
}
#[component]
pub fn DisplayRule(rule: Rule) -> impl IntoView {
view! {
{match rule {
Rule::Domain(domain_rule) => {
view! {
{if domain_rule.allow { "ALLOW: " } else { "BLOCK: " }}
{if domain_rule.subdomain { "*." } else { "" }}
{domain_rule.domain.as_ref().to_owned()}
}
.into_view()
}
Rule::IpRule(ip_rule) => {
view! {
{if ip_rule.allow { "ALLOW: " } else { "BLOCK: " }}
{ip_rule.ip.to_string()}
}
.into_view()
}
Rule::Unknown => "Unknown".into_view(),
Rule::Invalid => "Invalid Rule".into_view(),
}}
}
}
#[component]
fn RuleRawView(
rule: Resource<Result<RuleId, ParamsError>, Result<Rule, ServerFnError>>,
) -> impl IntoView {
view! {
<Transition fallback=move || {
view! { <p>"Loading" <Loading/></p> }
}>
{move || match rule.get() {
Some(Ok(rule)) => view! { <DisplayRule rule=rule/> }.into_view(),
Some(Err(err)) => view! { <p>"Error: " {format!("{err}")}</p> }.into_view(),
None => view! { "Invalid URL" }.into_view(),
}}
</Transition>
}
}
#[server]
async fn get_rule_blocked_domains(id: RuleId) -> Result<Vec<(DomainId, String)>, ServerFnError> {
let domains = sqlx::query!(
r#"SELECT DISTINCT domains.id as "domain_id: DomainId", domain as "domain: String"
FROM Rules
INNER JOIN rule_matches ON Rules.id = rule_matches.rule_id
INNER JOIN domains ON rule_matches.domain_id = domains.id
WHERE Rules.id = $1"#r,
id.0
)
.fetch_all(&crate::server::get_db().await?)
.await?;
Ok(domains
.into_iter()
.map(|record| (record.domain_id, record.domain))
.collect())
}
#[component]
fn RuleBlockedDomainsView(get_id: Box<dyn Fn() -> Result<RuleId, ParamsError>>) -> impl IntoView {
let domains_resource = create_resource(get_id, |id| async move {
let domains = get_rule_blocked_domains(id?).await?;
Ok::<_, ServerFnError>(domains)
});
view! {
<Transition fallback=move || {
view! { <p>"Loading" <Loading/></p> }
}>
{move || match domains_resource.get() {
Some(Ok(domains)) => {
view! {
<p>
"Matched Domains:" <table class="table table-zebra">
<thead>
<tr>
<th>Domain</th>
</tr>
</thead>
<tbody>
<For
each=move || { domains.clone() }
key=|(id, _)| *id
children=|(_domain_id, domain)| {
let domain_href = format!("/domain/{domain}");
view! {
<tr>
<td>
<A href=domain_href class="link link-neutral">
{domain}
</A>
</td>
</tr>
}
}
/>
</tbody>
</table>
</p>
}
.into_view()
}
Some(Err(err)) => view! { <p>"Error: " {format!("{err}")}</p> }.into_view(),
None => "Invalid URL".into_view(),
}}
</Transition>
}
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub struct RuleData {
pub rule_id: RuleId,
pub domain: Option<String>,
pub domain_allow: Option<bool>,
pub domain_subdomain: Option<bool>,
pub ip_network: Option<ipnetwork::IpNetwork>,
pub ip_allow: Option<bool>,
}
impl TryInto<Rule> for RuleData {
type Error = ServerFnError;
fn try_into(self) -> Result<Rule, Self::Error> {
match (
self.domain,
self.domain_allow,
self.domain_subdomain,
self.ip_network,
self.ip_allow,
) {
(Some(domain), Some(allow), Some(subdomain), None, None) => {
Ok(Rule::Domain(DomainRule {
domain: domain.parse()?,
allow,
subdomain,
}))
}
(None, None, None, Some(ip_network), Some(allow)) => {
Ok(Rule::IpRule(crate::filterlist::IpRule {
ip: ip_network,
allow,
}))
}
_ => Ok(Rule::Invalid),
}
}
}
#[derive(Params, PartialEq)]
struct RuleParam {
id: Option<i32>,
}
#[component]
pub fn RuleViewPage() -> impl IntoView {
let params = use_params::<RuleParam>();
let get_id = move || {
params.with(|param| {
param.as_ref().map_err(Clone::clone).and_then(|param| {
Ok(RuleId(param.id.ok_or_else(|| {
ParamsError::MissingParam("No id".into())
})?))
})
})
};
let rule_resource = create_resource(get_id, |id| async move {
let rule = get_rule(id?).await?;
Ok::<_, ServerFnError>(rule)
});
view! {
<p>"Rule: " <RuleRawView rule=rule_resource/></p>
<Sources get_id=Box::new(get_id)/>
<RuleBlockedDomainsView get_id=Box::new(get_id)/>
}
}
================================================
FILE: src/server.rs
================================================
use crate::DbInitError;
use crate::{domain::Domain, filterlist::FilterListUrl};
use addr::domain;
use axum::body::Bytes;
use axum::extract::{Path, State};
use leptos::*;
use notify::Watcher;
use std::collections::HashSet;
use tokio::task::JoinSet;
use tower_http::decompression::DecompressionLayer;
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt};
use tokio_util::sync::CancellationToken;
static SQLITE_POOL: tokio::sync::OnceCell<sqlx::PgPool> = tokio::sync::OnceCell::const_new();
pub async fn get_db() -> Result<sqlx::PgPool, DbInitError> {
SQLITE_POOL
.get_or_try_init(|| {
let _ = dotenvy::dotenv();
let db_url = std::env::var("DATABASE_URL");
async { Ok::<_, DbInitError>(sqlx::PgPool::connect(&db_url?).await?) }
})
.await
.cloned()
}
pub async fn parse_missing_subdomains() -> Result<(), ServerFnError> {
dotenvy::dotenv()?;
let read_limit: usize = std::env::var("READ_LIMIT")?.parse()?;
let pool = get_db().await?;
loop {
let records = sqlx::query!(
"SELECT domain from domains
WHERE processed_subdomains = false
LIMIT $1",
read_limit as i64
)
.fetch_all(&pool)
.await?;
if records.is_empty() {
tokio::time::sleep(Duration::from_secs(30)).await;
continue;
}
let mut checked_domains = Vec::new();
let mut all_domains = Vec::new();
let mut all_parents = Vec::new();
for record in records {
checked_domains.push(record.domain.clone());
let Ok(domain) = record.domain.parse::<Domain>() else {
continue;
};
let parents = domain
.as_ref()
.match_indices('.')
.map(|(i, _)| record.domain.split_at(i + 1).1)
.filter_map(|parent| parent.parse::<Domain>().ok());
for parent in parents {
all_domains.push(domain.clone());
all_parents.push(parent);
}
}
let mut parent_set = all_parents
.iter()
.cloned()
.collect::<std::collections::HashSet<_>>();
for domain in &all_domains {
parent_set.remove(domain);
}
let parent_set = parent_set.into_iter().collect::<Vec<_>>();
sqlx::query!(
"INSERT INTO domains (domain)
SELECT domain FROM UNNEST($1::text[]) as t(domain)
ON CONFLICT DO NOTHING",
&parent_set[..] as _
)
.execute(&pool)
.await?;
sqlx::query!(
"INSERT INTO subdomains (domain_id, parent_domain_id)
SELECT domains_with_parents.id, parents.id
FROM UNNEST($1::text[], $2::text[]) AS t(domain, parent)
INNER JOIN domains AS domains_with_parents ON domains_with_parents.domain = t.domain
INNER JOIN domains AS parents ON parents.domain = t.parent
ON CONFLICT DO NOTHING",
&all_domains[..] as _,
&all_parents[..] as _,
)
.execute(&pool)
.await?;
sqlx::query!(
"INSERT INTO domains (domain)
SELECT domain FROM UNNEST($1::text[]) as t(domain)
ON CONFLICT(domain)
DO UPDATE SET processed_subdomains = true",
&checked_domains[..]
)
.execute(&pool)
.await?;
}
}
pub async fn check_dns(token: CancellationToken) -> Result<(), ServerFnError> {
let resolver = crate::domain::DomainResolver::new(token)?;
resolver.run().await?;
Ok(())
}
pub async fn import_pihole_logs() -> Result<(), ServerFnError> {
let _ = dotenvy::dotenv()?;
let Ok(log_path) = std::env::var("PIHOLE_LOG_PATH") else {
log::info!("No PIHOLE_LOG_PATH set, skipping");
return Ok(());
};
let log_path: std::path::PathBuf = log_path.parse()?;
let write_frequency: u64 = std::env::var("WRITE_FREQUENCY")?.parse()?;
let notify = std::sync::Arc::new(tokio::sync::Notify::new());
let notify2 = notify.clone();
let mut watcher = notify::recommended_watcher(move |_| {
notify.notify_one();
})?;
watcher.watch(&log_path, notify::RecursiveMode::NonRecursive)?;
let pool: sqlx::Pool<sqlx::Postgres> = get_db().await?;
let file = tokio::fs::File::open(log_path).await?;
let buf = tokio::io::BufReader::new(file);
let mut lines = buf.lines();
let mut domains = HashSet::new();
let mut last_wrote = std::time::Instant::now();
while let Ok(line) = lines.next_line().await {
if let Some(line) = line {
for segment in line.split_whitespace() {
if let Ok(domain) = segment.parse() {
let domain: Domain = domain;
domains.insert(domain);
}
}
} else {
notify2.notified().await;
}
if last_wrote.elapsed().as_secs() > write_frequency {
let domains_vec = domains.drain().collect::<Vec<_>>();
let record = sqlx::query!(
"INSERT INTO domains (domain)
SELECT domain FROM UNNEST($1::text[]) as t(domain)
ON CONFLICT DO NOTHING",
&domains_vec[..] as _
)
.execute(&pool)
.await?;
let inserted = record.rows_affected();
if inserted != 0 {
log::info!("Inserted {} domains from Pihole logs", inserte
gitextract_u7y2p1i3/ ├── .cargo/ │ └── config.toml ├── .gitattributes ├── .gitignore ├── .vscode/ │ ├── launch.json │ └── settings.json ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSE ├── README.md ├── _config.yml ├── build.rs ├── end2end/ │ ├── package.json │ ├── playwright.config.ts │ └── tests/ │ └── example.spec.ts ├── filterlists.csv ├── internal/ │ ├── adblock.txt │ ├── allow_regex.txt │ ├── allowlist.txt │ ├── block_ipnets.txt │ ├── block_ips.txt │ ├── block_regex.txt │ └── blocklist.txt ├── local.toml ├── local2.toml ├── migrations/ │ ├── 20240222234126_filterlists.sql │ ├── 20240223010915_lastmodified.sql │ ├── 20240223011106_lastmodified.sql │ ├── 20240223011400_etag.sql │ ├── 20240224194439_rules.sql │ ├── 20240224195223_filter_list_contents.sql │ ├── 20240224203224_list_rules.sql │ ├── 20240225214254_list_source.sql │ ├── 20240225230839_index.sql │ ├── 20240225231841_index.sql │ ├── 20240225232249_index.sql │ ├── 20240226004619_change_date.sql │ ├── 20240226013547_source.sql │ ├── 20240226152939_temp.sql │ ├── 20240226215547_remove_column.sql │ ├── 20240226215929_remove_id.sql │ ├── 20240226220711_remove_fkey.sql │ ├── 20240226223817_domain_rules.sql │ ├── 20240226230317_domain_block.sql │ ├── 20240226230425_rename.sql │ ├── 20240227191530_drop_column.sql │ ├── 20240227194830_drop_column.sql │ ├── 20240227200629_drop_table.sql │ ├── 20240227201410_primary_key.sql │ ├── 20240228001134_domain_rules.sql │ ├── 20240228004601_extend_rules.sql │ ├── 20240228005409_drop_rule.sql │ ├── 20240228015906_change_unique.sql │ ├── 20240228164553_ip.sql │ ├── 20240228170646_ip.sql │ ├── 20240228175807_remove_unknown.sql │ ├── 20240228180753_index.sql │ ├── 20240229210101_domains.sql │ ├── 20240229212527_subdomains.sql │ ├── 20240301000455_more_indexes.sql │ ├── 20240301000900_subdomain_idx.sql │ ├── 20240301030213_subdomain_inde.sql │ ├── 20240302171950_expanded_subdomains.sql │ ├── 20240302184040_processed_subdomains.sql │ ├── 20240302192658_index.sql │ ├── 20240302194037_domain_rule_id_idx.sql │ ├── 20240302194733_not_null_parent.sql │ ├── 20240302205633_not_null#.sql │ ├── 20240302222401_dns.sql │ ├── 20240304235746_filterlist.sql │ ├── 20240305000257_filterlist.sql │ ├── 20240305000612_filterlist.sql │ ├── 20240305003411_filterlist.sql │ ├── 20240305200551_rule_matches.sql │ ├── 20240306123603_rule_count.sql │ ├── 20240306214217_index.sql │ ├── 20240307005702_lists.sql │ ├── 20240307012450_index.sql │ └── 20240307031445_idx.sql ├── output/ │ ├── adblock.txt │ ├── allowed_ips.txt │ ├── domains.rpz │ ├── domains.txt │ ├── hosts.txt │ ├── ip_blocklist.txt │ ├── whitelist_adblock.txt │ └── whitelist_domains.txt ├── package.json ├── rust-toolchain.toml ├── rustfmt.toml ├── src/ │ ├── app.rs │ ├── domain.rs │ ├── error_template.rs │ ├── fileserv.rs │ ├── filterlist.rs │ ├── home_page.rs │ ├── ip_view.rs │ ├── lib.rs │ ├── main.rs │ ├── rule.rs │ ├── server.rs │ ├── stats_view.rs │ └── tasks.rs ├── style/ │ ├── main.scss │ └── tailwind.css ├── tailwind.config.js ├── tld_list.txt └── update.sh
SYMBOL INDEX (227 symbols across 41 files)
FILE: build.rs
function main (line 2) | fn main() {
FILE: migrations/20240222234126_filterlists.sql
type filterLists (line 1) | CREATE TABLE IF NOT EXISTS filterLists (
FILE: migrations/20240224194439_rules.sql
type Rules (line 2) | CREATE TABLE IF NOT EXISTS Rules (
FILE: migrations/20240224195223_filter_list_contents.sql
type filterLists (line 5) | CREATE TABLE IF NOT EXISTS filterLists (
FILE: migrations/20240224203224_list_rules.sql
type list_rules (line 2) | CREATE table list_rules (
FILE: migrations/20240225230839_index.sql
type rule_index (line 2) | create unique index rule_index on Rules(rule)
FILE: migrations/20240225231841_index.sql
type list_rules_index (line 2) | create index list_rules_index on list_rules(list_id, rule_id)
FILE: migrations/20240225232249_index.sql
type list_rules_index_list_id (line 4) | create index list_rules_index_list_id on list_rules(list_id)
type list_rules_index_rule_id (line 5) | create index list_rules_index_rule_id on list_rules(rule_id)
FILE: migrations/20240226013547_source.sql
type rule_source (line 2) | CREATE TABLE rule_source (
type idx_rule_source_source (line 7) | CREATE INDEX idx_rule_source_source ON rule_source (source)
FILE: migrations/20240226152939_temp.sql
type temp_rule_source (line 2) | CREATE TABLE temp_rule_source (
FILE: migrations/20240226223817_domain_rules.sql
type domain_rules (line 2) | CREATE TABLE domain_rules (
type domain_rules_idx_domain (line 6) | CREATE INDEX domain_rules_idx_domain ON domain_rules (domain)
FILE: migrations/20240227191530_drop_column.sql
type temp_rule_source (line 2) | CREATE TEMPORARY TABLE temp_rule_source(source TEXT UNIQUE, rule_id INTE...
FILE: migrations/20240228001134_domain_rules.sql
type temp_domain_rules (line 12) | CREATE TEMPORARY TABLE temp_domain_rules (
FILE: migrations/20240228004601_extend_rules.sql
type unknown_rules (line 7) | CREATE TABLE unknown_rules (
FILE: migrations/20240228164553_ip.sql
type ip_rules (line 2) | CREATE TABLE ip_rules (
FILE: migrations/20240229210101_domains.sql
type domains (line 2) | CREATE TABLE domains (
type domain_rules (line 17) | CREATE TABLE domain_rules (
FILE: migrations/20240229212527_subdomains.sql
type subdomains (line 12) | CREATE TABLE subdomains (
FILE: migrations/20240301000455_more_indexes.sql
type rule_source_rule_idx (line 2) | CREATE INDEX rule_source_rule_idx ON rule_source (rule_id)
type domain_rules_domain_idx (line 3) | CREATE INDEX domain_rules_domain_idx ON domain_rules (domain_id)
FILE: migrations/20240301000900_subdomain_idx.sql
type subdomain_domain_idx (line 2) | CREATE INDEX subdomain_domain_idx ON subdomains (domain_id)
type subdomain_parent_idx (line 4) | CREATE INDEX subdomain_parent_idx ON subdomains (parent_domain_id)
FILE: migrations/20240301030213_subdomain_inde.sql
type domain_rules_subdomain_idx (line 2) | CREATE INDEX domain_rules_subdomain_idx ON domain_rules (subdomain)
FILE: migrations/20240302192658_index.sql
type domains_processed_subdomains_idx (line 2) | CREATE INDEX domains_processed_subdomains_idx ON domains(processed_subdo...
FILE: migrations/20240302194037_domain_rule_id_idx.sql
type rules_domain_rule_id_idx (line 2) | CREATE INDEX rules_domain_rule_id_idx ON rules(domain_rule_id)
FILE: migrations/20240302222401_dns.sql
type domains_last_checked_dns_idx (line 7) | CREATE INDEX domains_last_checked_dns_idx ON domains(last_checked_dns)
type dns_ips (line 9) | CREATE TABLE dns_ips (
type dns_ips_domain_id_idx (line 14) | CREATE INDEX dns_ips_domain_id_idx ON dns_ips(domain_id)
type dns_ips_ip_address_idx (line 16) | CREATE INDEX dns_ips_ip_address_idx ON dns_ips(ip_address)
type dns_cnames (line 18) | CREATE TABLE dns_cnames (
type dns_cnames_domain_id_idx (line 23) | CREATE INDEX dns_cnames_domain_id_idx ON dns_cnames(domain_id)
type dns_cnames_cname_domain_id_idx (line 25) | CREATE INDEX dns_cnames_cname_domain_id_idx ON dns_cnames(cname_domain_id)
FILE: migrations/20240305200551_rule_matches.sql
type rule_matches (line 2) | CREATE TABLE rule_matches (
type rule_matches_rule_id_idx (line 8) | CREATE INDEX rule_matches_rule_id_idx ON rule_matches (rule_id)
type rule_matches_domain_id_idx (line 9) | CREATE INDEX rule_matches_domain_id_idx ON rule_matches (domain_id)
FILE: migrations/20240306214217_index.sql
type rules_last_checked_matches_idx (line 2) | CREATE INDEX rules_last_checked_matches_idx ON rules (last_checked_matches)
FILE: migrations/20240307005702_lists.sql
type allow_domains (line 2) | CREATE TABLE allow_domains (
type block_domains (line 6) | CREATE TABLE block_domains (
FILE: migrations/20240307012450_index.sql
type domain_rules_allow_idx (line 2) | CREATE INDEX domain_rules_allow_idx ON domain_rules (allow)
type ip_rules_allow_idx (line 3) | CREATE INDEX ip_rules_allow_idx ON ip_rules (allow)
FILE: migrations/20240307031445_idx.sql
type ip_rules_network_idx (line 2) | CREATE INDEX ip_rules_network_idx ON ip_rules (ip_network)
FILE: src/app.rs
function App (line 15) | pub fn App() -> impl IntoView {
function Loading (line 80) | pub fn Loading() -> impl IntoView {
FILE: src/domain.rs
type DomainId (line 19) | pub struct DomainId(i64);
type DomainParseError (line 22) | pub enum DomainParseError {
method fmt (line 29) | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
method from (line 35) | fn from(_: addr::error::Error) -> Self {
method from (line 41) | fn from(_: hickory_proto::error::ProtoError) -> Self {
type Domain (line 48) | pub struct Domain(Arc<str>);
method type_info (line 52) | fn type_info() -> sqlx::postgres::PgTypeInfo {
method compatible (line 56) | fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
method array_type_info (line 62) | fn array_type_info() -> sqlx::postgres::PgTypeInfo {
method array_compatible (line 66) | fn array_compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool {
method encode_by_ref (line 72) | fn encode_by_ref(&self, buf: &mut sqlx::postgres::PgArgumentBuffer) ->...
method as_ref (line 78) | fn as_ref(&self) -> &str {
type Err (line 84) | type Err = DomainParseError;
method from_str (line 85) | fn from_str(domain: &str) -> Result<Domain, Self::Err> {
function valid_domain (line 115) | fn valid_domain() {
function invalid_domain (line 127) | fn invalid_domain() {
function makes_lowercase (line 140) | fn makes_lowercase() {
function parse_lookup_result (line 154) | fn parse_lookup_result(
type Resolver (line 201) | type Resolver = Arc<
type Task (line 210) | type Task = (DomainId, Domain);
type DomainResolver (line 214) | pub struct DomainResolver {
method new (line 231) | pub fn new(token: CancellationToken) -> Result<Self, ServerFnError> {
method run (line 282) | pub async fn run(&self) -> Result<(), ServerFnError> {
method domain_selector (line 323) | async fn domain_selector(&self) -> Result<(), ServerFnError> {
method run_task (line 391) | async fn run_task(
method write_to_db (line 434) | async fn write_to_db(&self) -> Result<(), ServerFnError> {
function get_dns_result (line 549) | async fn get_dns_result(
function DnsResultView (line 580) | fn DnsResultView(domain: Domain) -> impl IntoView {
function get_blocked_by (line 652) | async fn get_blocked_by(
function BlockedBy (line 701) | fn BlockedBy(get_domain: Box<dyn Fn() -> Result<String, ParamsError>>) -...
function get_subdomains (line 748) | async fn get_subdomains(domain: String) -> Result<Vec<String>, ServerFnE...
function DisplaySubdomains (line 765) | fn DisplaySubdomains(get_domain: Box<dyn Fn() -> Result<String, ParamsEr...
type DomainParam (line 807) | struct DomainParam {
function DomainViewPage (line 812) | pub fn DomainViewPage() -> impl IntoView {
FILE: src/error_template.rs
type AppError (line 6) | pub enum AppError {
method status_code (line 12) | pub fn status_code(&self) -> StatusCode {
function ErrorTemplate (line 22) | pub fn ErrorTemplate(
FILE: src/fileserv.rs
function file_and_error_handler (line 13) | pub async fn file_and_error_handler(
function get_static_file (line 29) | async fn get_static_file(uri: Uri, root: &str) -> Result<Response<Body>,...
FILE: src/filterlist.rs
type ListId (line 18) | pub struct ListId(i32);
type FilterListRecord (line 21) | pub struct FilterListRecord {
type FilterListUrl (line 33) | pub struct FilterListUrl {
method as_str (line 38) | pub fn as_str(&self) -> &str {
method to_internal_path (line 41) | pub fn to_internal_path(&self) -> Option<std::path::PathBuf> {
type Target (line 51) | type Target = str;
method deref (line 52) | fn deref(&self) -> &Self::Target {
type Err (line 58) | type Err = url::ParseError;
method from_str (line 59) | fn from_str(s: &str) -> Result<Self, Self::Err> {
type FilterListType (line 72) | pub enum FilterListType {
method as_str (line 87) | pub fn as_str(&self) -> &'static str {
type Err (line 113) | type Err = InvalidFilterListTypeError;
method from_str (line 114) | fn from_str(s: &str) -> Result<Self, Self::Err> {
type InvalidFilterListTypeError (line 104) | pub struct InvalidFilterListTypeError;
method fmt (line 107) | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
type FilterListMap (line 133) | pub struct FilterListMap(
type DomainRule (line 139) | pub struct DomainRule {
type IpRule (line 146) | pub struct IpRule {
type Rule (line 152) | pub enum Rule {
type RulePair (line 165) | pub struct RulePair {
method new (line 171) | pub fn new(source: Arc<str>, rule: Rule) -> RulePair {
method get_rule (line 174) | pub fn get_rule(&self) -> &Rule {
method get_source (line 177) | pub fn get_source(&self) -> &Arc<str> {
method from (line 187) | fn from((source, rule): (Arc<str>, Rule)) -> Self {
function from (line 182) | fn from(val: RulePair) -> Self {
function parse_lines (line 193) | fn parse_lines(contents: &str, parser: &dyn Fn(&str) -> Option<Rule>) ->...
function parse_domain_list_line (line 211) | fn parse_domain_list_line(line: &str, allow: bool, subdomain: bool) -> O...
function parse_domain_list (line 239) | fn parse_domain_list(contents: &str, allow: bool, subdomain: bool) -> Ve...
function parse_adblock_line (line 246) | fn parse_adblock_line(line: &str) -> Option<Rule> {
function parse_adblock (line 362) | fn parse_adblock(contents: &str) -> Vec<RulePair> {
function parse_regex_line (line 367) | fn parse_regex_line(line: &str) -> Option<Rule> {
function parse_ip_network_line (line 389) | fn parse_ip_network_line(line: &str, allow: bool) -> Option<Rule> {
function parse_ip_network_list (line 402) | fn parse_ip_network_list(contents: &str, allow: bool) -> Vec<RulePair> {
function parse_regex (line 407) | fn parse_regex(contents: &str) -> Vec<RulePair> {
function parse_unknown_lines (line 412) | fn parse_unknown_lines(contents: &str) -> Vec<RulePair> {
function parse_list_contents (line 417) | pub fn parse_list_contents(contents: &str, list_format: FilterListType) ...
function parse_list (line 436) | pub async fn parse_list(url: FilterListUrl) -> Result<(), ServerFnError> {
type CsvRecord (line 644) | struct CsvRecord {
function load_filter_map (line 654) | pub async fn load_filter_map() -> Result<(), ServerFnError> {
function watch_filter_map (line 702) | pub async fn watch_filter_map() -> Result<(), ServerFnError> {
function write_filter_map (line 724) | pub async fn write_filter_map() -> Result<(), ServerFnError> {
function get_filter_map (line 753) | pub async fn get_filter_map() -> Result<FilterListMap, ServerFnError> {
type LastVersionData (line 781) | struct LastVersionData {
function get_last_version_data (line 787) | async fn get_last_version_data(
function get_last_updated (line 810) | pub async fn get_last_updated(
type UpdateListError (line 820) | enum UpdateListError {
function update_list (line 826) | pub async fn update_list(url: FilterListUrl) -> Result<(), ServerFnError> {
function delete_list (line 930) | pub async fn delete_list(url: FilterListUrl) -> Result<(), ServerFnError> {
function FilterListLink (line 950) | pub fn FilterListLink(url: FilterListUrl) -> impl IntoView {
function get_list_size (line 966) | async fn get_list_size(url: FilterListUrl) -> Result<Option<usize>, Serv...
function ListSize (line 1003) | pub fn ListSize(url: FilterListUrl, list_size: Option<usize>) -> impl In...
function get_list_page (line 1030) | async fn get_list_page(
function LastUpdatedInner (line 1084) | fn LastUpdatedInner(last_updated: Option<chrono::DateTime<chrono::Utc>>)...
function LastUpdated (line 1098) | pub fn LastUpdated(url: FilterListUrl, record: Option<FilterListRecord>)...
function ParseList (line 1137) | pub fn ParseList(url: FilterListUrl) -> impl IntoView {
function FilterListUpdate (line 1150) | pub fn FilterListUpdate(url: FilterListUrl) -> impl IntoView {
function Contents (line 1163) | fn Contents(url: FilterListUrl, page: Option<usize>) -> impl IntoView {
function FilterListInner (line 1223) | fn FilterListInner(url: FilterListUrl, page: Option<usize>) -> impl Into...
type ViewListParams (line 1273) | struct ViewListParams {
method parse (line 1289) | fn parse(&self) -> Result<FilterListUrl, ViewListError> {
type ViewListError (line 1279) | enum ViewListError {
function DeleteListButton (line 1299) | fn DeleteListButton(url: FilterListUrl) -> impl IntoView {
function FilterListPage (line 1312) | pub fn FilterListPage() -> impl IntoView {
FILE: src/home_page.rs
function FilterListSummary (line 6) | fn FilterListSummary(url: FilterListUrl, record: FilterListRecord) -> im...
function HomePage (line 48) | pub fn HomePage() -> impl IntoView {
FILE: src/ip_view.rs
function get_domans_which_resolve_to_ip (line 8) | async fn get_domans_which_resolve_to_ip(
function DomainsWhichResolveTo (line 29) | fn DomainsWhichResolveTo(get_ip: GetIp) -> impl IntoView {
type GetIp (line 74) | type GetIp = Box<dyn Fn() -> Result<IpAddr, ParamsError>>;
type IpParam (line 77) | struct IpParam {
function IpView (line 82) | pub fn IpView() -> impl IntoView {
FILE: src/lib.rs
constant PAGE_SIZE (line 21) | const PAGE_SIZE: usize = 50;
type SourceId (line 26) | pub struct SourceId(i32);
type DbInitError (line 30) | pub enum DbInitError {
method from (line 38) | fn from(e: sqlx::Error) -> Self {
method from (line 45) | fn from(e: std::env::VarError) -> Self {
function hydrate (line 55) | pub fn hydrate() {
FILE: src/main.rs
type Config (line 6) | struct Config {
method default (line 12) | fn default() -> Self {
type Args (line 24) | struct Args {
function main (line 34) | async fn main() {
function main (line 152) | pub fn main() {
FILE: src/rule.rs
type RuleId (line 13) | pub struct RuleId(i32);
method get_href (line 16) | pub fn get_href(&self) -> String {
function find_rule_matches (line 22) | pub async fn find_rule_matches() -> Result<(), ServerFnError> {
function get_rule (line 99) | pub async fn get_rule(id: RuleId) -> Result<Rule, ServerFnError> {
type GetId (line 138) | type GetId = Box<dyn Fn() -> Result<RuleId, ParamsError>>;
function get_sources (line 141) | async fn get_sources(
function Sources (line 169) | fn Sources(get_id: GetId) -> impl IntoView {
function DisplayRule (line 221) | pub fn DisplayRule(rule: Rule) -> impl IntoView {
function RuleRawView (line 246) | fn RuleRawView(
function get_rule_blocked_domains (line 264) | async fn get_rule_blocked_domains(id: RuleId) -> Result<Vec<(DomainId, S...
function RuleBlockedDomainsView (line 282) | fn RuleBlockedDomainsView(get_id: Box<dyn Fn() -> Result<RuleId, ParamsE...
type RuleData (line 335) | pub struct RuleData {
type Error (line 345) | type Error = ServerFnError;
method try_into (line 346) | fn try_into(self) -> Result<Rule, Self::Error> {
type RuleParam (line 373) | struct RuleParam {
function RuleViewPage (line 378) | pub fn RuleViewPage() -> impl IntoView {
FILE: src/server.rs
function get_db (line 20) | pub async fn get_db() -> Result<sqlx::PgPool, DbInitError> {
function parse_missing_subdomains (line 31) | pub async fn parse_missing_subdomains() -> Result<(), ServerFnError> {
function check_dns (line 108) | pub async fn check_dns(token: CancellationToken) -> Result<(), ServerFnE...
function import_pihole_logs (line 114) | pub async fn import_pihole_logs() -> Result<(), ServerFnError> {
function update_expired_lists (line 166) | pub async fn update_expired_lists() -> Result<(), ServerFnError> {
function build_list (line 189) | pub async fn build_list() -> Result<(), ServerFnError> {
function garbage_collect_rule_source (line 258) | async fn garbage_collect_rule_source(pool: &sqlx::PgPool) -> Result<u64,...
function garbage_collect_rules (line 268) | async fn garbage_collect_rules(pool: &sqlx::PgPool) -> Result<u64, Serve...
function garbage_collect_rule_matches (line 278) | async fn garbage_collect_rule_matches(pool: &sqlx::PgPool) -> Result<u64...
function garbage_collect (line 288) | pub async fn garbage_collect() -> Result<(), ServerFnError> {
function run_cmd (line 312) | pub async fn run_cmd(token: CancellationToken) -> Result<(), ServerFnErr...
constant CERTSTREAM_URL (line 329) | const CERTSTREAM_URL: &str = "wss://certstream.calidog.io/domains-only";
type CertStreamMessage (line 332) | struct CertStreamMessage {
function stream_certstream (line 336) | async fn stream_certstream(
function write_certstream (line 355) | async fn write_certstream(
function certstream (line 394) | pub async fn certstream(token: CancellationToken) -> Result<(), ServerFn...
function dns_results (line 408) | async fn dns_results(State(peer_state): State<PeerState>) {}
type Peer (line 411) | pub struct Peer {
method default (line 417) | fn default() -> Self {
type PeerState (line 426) | pub struct PeerState {
method new (line 440) | pub fn new(peers: &[Peer]) -> Self {
type PeerError (line 430) | enum PeerError {
function get_peer_router (line 445) | pub fn get_peer_router(peer_state: PeerState) -> axum::Router<LeptosOpti...
FILE: src/stats_view.rs
function count_total_rules (line 4) | async fn count_total_rules() -> Result<usize, ServerFnError> {
function TotalRuleCount (line 19) | fn TotalRuleCount() -> impl IntoView {
function get_total_rule_matches (line 32) | async fn get_total_rule_matches() -> Result<usize, ServerFnError> {
function TotalRuleMatches (line 47) | fn TotalRuleMatches() -> impl IntoView {
function get_domain_count (line 60) | async fn get_domain_count() -> Result<usize, ServerFnError> {
function DomainCount (line 75) | fn DomainCount() -> impl IntoView {
function get_subdomains_count (line 87) | async fn get_subdomains_count() -> Result<usize, ServerFnError> {
function SubdomainCount (line 102) | fn SubdomainCount() -> impl IntoView {
function get_dns_ip_count (line 115) | async fn get_dns_ip_count() -> Result<usize, ServerFnError> {
function DnsIpCount (line 130) | fn DnsIpCount() -> impl IntoView {
function get_dns_cname_count (line 143) | async fn get_dns_cname_count() -> Result<usize, ServerFnError> {
function DnsCnameCount (line 158) | fn DnsCnameCount() -> impl IntoView {
function StatsView (line 171) | pub fn StatsView() -> impl IntoView {
FILE: src/tasks.rs
type Task (line 3) | trait Task {
method name (line 5) | fn name(&self) -> &str;
method run_once (line 6) | async fn run_once(&self) -> Result<String, Self::Error>;
type Error (line 13) | type Error = ServerFnError;
method name (line 14) | fn name(&self) -> &str {
method run_once (line 17) | async fn run_once(&self) -> Result<String, Self::Error> {
type GarbageCollectRuleSource (line 9) | struct GarbageCollectRuleSource {}
function register_task (line 34) | async fn register_task<T: Task>(_task: T) {
function TaskView (line 39) | pub fn TaskView() -> impl IntoView {
Condensed preview — 107 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (229K chars).
[
{
"path": ".cargo/config.toml",
"chars": 97,
"preview": "[target.x86_64-unknown-linux-gnu]\nrustflags = [\"-Clink-arg=-fuse-ld=mold\", \"-Ctarget-cpu=native\"]"
},
{
"path": ".gitattributes",
"chars": 67,
"preview": "# Auto detect text files and perform LF normalization\n* text eol=lf"
},
{
"path": ".gitignore",
"chars": 3454,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": ".vscode/launch.json",
"chars": 495,
"preview": "{\n // Use IntelliSense to learn about possible attributes.\n // Hover to view descriptions of existing attributes.\n"
},
{
"path": ".vscode/settings.json",
"chars": 545,
"preview": "{\n \"rust-analyzer.linkedProjects\": [\n \"./Cargo.toml\",\n ],\n \"rust-analyzer.cargo.buildScripts.overrideCom"
},
{
"path": "CONTRIBUTING.md",
"chars": 1099,
"preview": "# Contributing\n\nIf you are contibuting, then thank you, it is appreciated. Below is what requirements a change must have"
},
{
"path": "Cargo.toml",
"chars": 6669,
"preview": "[package]\nname = \"blockconvert\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[lib]\ncrate-type = [\"cdylib\", \"lib\"]\n\n[dependencies]"
},
{
"path": "LICENSE",
"chars": 1063,
"preview": "MIT License\n\nCopyright (c) 2022 henrik\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof "
},
{
"path": "README.md",
"chars": 4706,
"preview": "# BlockConvert\n\nMalware, advert and tracking blocklist which consolidates and improves upon many other [blocklists](http"
},
{
"path": "_config.yml",
"chars": 26,
"preview": "theme: jekyll-theme-cayman"
},
{
"path": "build.rs",
"chars": 168,
"preview": "// generated by `sqlx migrate build-script`\nfn main() {\n // trigger recompilation when a new migration is added\n p"
},
{
"path": "end2end/package.json",
"chars": 222,
"preview": "{\n \"name\": \"end2end\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"main\": \"index.js\",\n \"scripts\": {},\n \"keywords\": ["
},
{
"path": "end2end/playwright.config.ts",
"chars": 2644,
"preview": "import type { PlaywrightTestConfig } from \"@playwright/test\";\nimport { devices } from \"@playwright/test\";\n\n/**\n * Read e"
},
{
"path": "end2end/tests/example.spec.ts",
"chars": 298,
"preview": "import { test, expect } from \"@playwright/test\";\n\ntest(\"homepage has title and links to intro page\", async ({ page }) =>"
},
{
"path": "filterlists.csv",
"chars": 11715,
"preview": "name,url,author,license,expires,list_type\n🎮 Game Console Adblock List,https://raw.githubusercontent.com/DandelionSprout/"
},
{
"path": "internal/adblock.txt",
"chars": 75,
"preview": "||cedexis.net^$third-party,badfilter\n||episerver.net^$third-party,badfilter"
},
{
"path": "internal/allow_regex.txt",
"chars": 36,
"preview": "^gsp[0-9][0-9]-ssl\\.ls\\.apple\\.com$\n"
},
{
"path": "internal/allowlist.txt",
"chars": 24755,
"preview": "*.a-msedge.net\n*.adafruit.com\n*.b.akamaiedge.net\n*.bitcoin.it\n*.burst-alliance.org\n*.c-msedge.net\n*.cloudflare-dns.com #"
},
{
"path": "internal/block_ipnets.txt",
"chars": 219,
"preview": "0.0.0.0/8\n10.0.0.0/8\n100.64.0.0/10\n127.0.0.0/8\n169.254.0.0/16\n172.16.0.0/12\n192.0.0.0/24\n192.0.2.0/24\n192.88.99.0/24\n192"
},
{
"path": "internal/block_ips.txt",
"chars": 1,
"preview": "\n"
},
{
"path": "internal/block_regex.txt",
"chars": 75,
"preview": "\\.(?:com|co\\.uk|net)\\.(?:login|account|payment|verif)\n[_.-]analytics?[_.-]\n"
},
{
"path": "internal/blocklist.txt",
"chars": 701,
"preview": "*.ct.sendgrid.net # Tracking https://github.com/mkb2091/blockconvert/issues/70\n*.getemails.com # De-anonymization\naf.con"
},
{
"path": "local.toml",
"chars": 80,
"preview": "listen_port = 3000\n\n[[peers]]\nurl = \"http://localhost/\"\nget_dns_results = false\n"
},
{
"path": "local2.toml",
"chars": 151,
"preview": "listen_port = 3000\n\n[[peers]]\nurl = \"http://localhost:3000/\"\nget_dns_results = false\n\n[[peers]]\nurl = \"http://localhost:"
},
{
"path": "migrations/20240222234126_filterlists.sql",
"chars": 146,
"preview": "CREATE TABLE IF NOT EXISTS filterLists (\n url TEXT PRIMARY KEY NOT NULL UNIQUE,\n contents TEXT NOT NULL,\n lastU"
},
{
"path": "migrations/20240223010915_lastmodified.sql",
"chars": 104,
"preview": "-- Add migration script here\nALTER TABLE filterLists ADD COLUMN lastModified INTEGER NOT NULL DEFAULT 0;"
},
{
"path": "migrations/20240223011106_lastmodified.sql",
"chars": 78,
"preview": "-- Add migration script here\nalter table filterLists drop column lastmodified;"
},
{
"path": "migrations/20240223011400_etag.sql",
"chars": 103,
"preview": "-- Add migration script here\n-- Add migration script here\nALTER TABLE filterLists ADD COLUMN etag TEXT;"
},
{
"path": "migrations/20240224194439_rules.sql",
"chars": 123,
"preview": "-- Add migration script here\nCREATE TABLE IF NOT EXISTS Rules (\n id SERIAL PRIMARY KEY,\n rule TEXT NOT NULL UNIQUE"
},
{
"path": "migrations/20240224195223_filter_list_contents.sql",
"chars": 455,
"preview": "-- Add migration script here\n\nALTER TABLE filterLists RENAME TO OldFilterListContents;\n\nCREATE TABLE IF NOT EXISTS filte"
},
{
"path": "migrations/20240224203224_list_rules.sql",
"chars": 245,
"preview": "-- Add migration script here\nCREATE table list_rules (\n id SERIAL PRIMARY KEY,\n list_id INTEGER NOT NULL,\n rule"
},
{
"path": "migrations/20240225214254_list_source.sql",
"chars": 75,
"preview": "-- Add migration script here\nALTER TABLE list_rules ADD COLUMN source TEXT;"
},
{
"path": "migrations/20240225230839_index.sql",
"chars": 75,
"preview": "-- Add migration script here\ncreate unique index rule_index on Rules(rule);"
},
{
"path": "migrations/20240225231841_index.sql",
"chars": 91,
"preview": "-- Add migration script here\ncreate index list_rules_index on list_rules(list_id, rule_id);"
},
{
"path": "migrations/20240225232249_index.sql",
"chars": 182,
"preview": "-- Add migration script here\ndrop index list_rules_index;\n\ncreate index list_rules_index_list_id on list_rules(list_id);"
},
{
"path": "migrations/20240226004619_change_date.sql",
"chars": 140,
"preview": "-- Add migration script here\nalter table filterLists alter column lastUpdated type timestamp with time zone using to_tim"
},
{
"path": "migrations/20240226013547_source.sql",
"chars": 305,
"preview": "-- Add migration script here\nCREATE TABLE rule_source (\n id SERIAL PRIMARY KEY,\n source TEXT NOT NULL UNIQUE\n);\n\nC"
},
{
"path": "migrations/20240226152939_temp.sql",
"chars": 140,
"preview": "-- Add migration script here\nCREATE TABLE temp_rule_source (\n idx SERIAL PRIMARY KEY,\n rule TEXT NOT NULL,\n sou"
},
{
"path": "migrations/20240226215547_remove_column.sql",
"chars": 74,
"preview": "-- Add migration script here\nALTER TABLE temp_rule_source DROP COLUMN idx;"
},
{
"path": "migrations/20240226215929_remove_id.sql",
"chars": 68,
"preview": "-- Add migration script here\nALTER TABLE list_rules DROP COLUMN id;\n"
},
{
"path": "migrations/20240226220711_remove_fkey.sql",
"chars": 222,
"preview": "-- Add migration script here\nALTER TABLE list_rules DROP CONSTRAINT list_rules_list_id_fkey;\nALTER TABLE list_rules DROP"
},
{
"path": "migrations/20240226223817_domain_rules.sql",
"chars": 170,
"preview": "-- Add migration script here\nCREATE TABLE domain_rules (\n id SERIAL PRIMARY KEY,\n domain TEXT NOT NULL\n);\nCREATE INDEX"
},
{
"path": "migrations/20240226230317_domain_block.sql",
"chars": 88,
"preview": "-- Add migration script here\nALTER TABLE domain_rules ADD COLUMN block BOOLEAN NOT NULL;"
},
{
"path": "migrations/20240226230425_rename.sql",
"chars": 82,
"preview": "-- Add migration script here\nALTER TABLE domain_rules RENAME COLUMN id TO rule_id;"
},
{
"path": "migrations/20240227191530_drop_column.sql",
"chars": 544,
"preview": "-- Add migration script here\nCREATE TEMPORARY TABLE temp_rule_source(source TEXT UNIQUE, rule_id INTEGER) ON COMMIT DROP"
},
{
"path": "migrations/20240227194830_drop_column.sql",
"chars": 72,
"preview": "-- Add migration script here\nALTER TABLE list_rules DROP COLUMN rule_id;"
},
{
"path": "migrations/20240227200629_drop_table.sql",
"chars": 57,
"preview": "-- Add migration script here\nDROP TABLE temp_rule_source;"
},
{
"path": "migrations/20240227201410_primary_key.sql",
"chars": 122,
"preview": "-- Add migration script here\nDELETE FROM list_rules;\n\nALTER TABLE\n list_rules\nADD\n PRIMARY KEY (list_id, source_id"
},
{
"path": "migrations/20240228001134_domain_rules.sql",
"chars": 896,
"preview": "-- Add migration script here\nALTER TABLE\n domain_rules\nADD\n COLUMN subdomain BOOLEAN NOT NULL DEFAULT FALSE;\n\nALTE"
},
{
"path": "migrations/20240228004601_extend_rules.sql",
"chars": 382,
"preview": "-- Add migration script here\nALTER TABLE\n Rules\nADD\n COLUMN domain_rule_id INTEGER;\n\nCREATE TABLE unknown_rules (\n"
},
{
"path": "migrations/20240228005409_drop_rule.sql",
"chars": 64,
"preview": "-- Add migration script here\nALTER TABLE Rules DROP COLUMN rule;"
},
{
"path": "migrations/20240228015906_change_unique.sql",
"chars": 178,
"preview": "-- Add migration script here\nALTER TABLE rule_source DROP CONSTRAINT rule_source_source_key;\n\nALTER TABLE rule_source AD"
},
{
"path": "migrations/20240228164553_ip.sql",
"chars": 415,
"preview": "-- Add migration script here\nCREATE TABLE ip_rules (\n id SERIAL PRIMARY KEY,\n ip_address inet NOT NULL,\n allow "
},
{
"path": "migrations/20240228170646_ip.sql",
"chars": 89,
"preview": "-- Add migration script here\nALTER TABLE ip_rules RENAME COLUMN ip_address TO ip_network;"
},
{
"path": "migrations/20240228175807_remove_unknown.sql",
"chars": 101,
"preview": "-- Add migration script here\nDROP TABLE unknown_rules;\nALTER TABLE Rules DROP COLUMN unknown_rule_id;"
},
{
"path": "migrations/20240228180753_index.sql",
"chars": 150,
"preview": "-- Add migration script here\nDELETE FROM Rules;\nALTER TABLE Rules ADD CONSTRAINT rules_unique UNIQUE NULLS NOT DISTINCT "
},
{
"path": "migrations/20240229210101_domains.sql",
"chars": 758,
"preview": "-- Add migration script here\nCREATE TABLE domains (\n id SERIAL PRIMARY KEY,\n domain TEXT NOT NULL UNIQUE\n);\n\nINSER"
},
{
"path": "migrations/20240229212527_subdomains.sql",
"chars": 251,
"preview": "-- Add migration script here\nALTER TABLE\n domains\nALTER COLUMN\n id TYPE bigint;\n\nALTER TABLE\n domain_rules\nALTE"
},
{
"path": "migrations/20240301000455_more_indexes.sql",
"chars": 155,
"preview": "-- Add migration script here\nCREATE INDEX rule_source_rule_idx ON rule_source (rule_id);\nCREATE INDEX domain_rules_domai"
},
{
"path": "migrations/20240301000900_subdomain_idx.sql",
"chars": 158,
"preview": "-- Add migration script here\nCREATE INDEX subdomain_domain_idx ON subdomains (domain_id);\n\nCREATE INDEX subdomain_parent"
},
{
"path": "migrations/20240301030213_subdomain_inde.sql",
"chars": 98,
"preview": "-- Add migration script here\nCREATE INDEX domain_rules_subdomain_idx ON domain_rules (subdomain);\n"
},
{
"path": "migrations/20240302171950_expanded_subdomains.sql",
"chars": 294,
"preview": "-- Add migration script here\nDELETE FROM\n subdomains\nWHERE\n parent_domain_id IS NOT NULL;\n\nALTER TABLE\n subdoma"
},
{
"path": "migrations/20240302184040_processed_subdomains.sql",
"chars": 202,
"preview": "DELETE FROM\n domains;\nDELETE FROM subdomains;\nDELETE FROM domain_rules;\nDELETE FROM Rules;\nDELETE FROM rule_source;\n\n"
},
{
"path": "migrations/20240302192658_index.sql",
"chars": 108,
"preview": "-- Add migration script here\nCREATE INDEX domains_processed_subdomains_idx ON domains(processed_subdomains);"
},
{
"path": "migrations/20240302194037_domain_rule_id_idx.sql",
"chars": 92,
"preview": "-- Add migration script here\nCREATE INDEX rules_domain_rule_id_idx ON rules(domain_rule_id);"
},
{
"path": "migrations/20240302194733_not_null_parent.sql",
"chars": 150,
"preview": "-- Add migration script here\nDELETE FROM subdomains WHERE parent_domain_id IS NULL;\nALTER TABLE subdomains ALTER COLUMN "
},
{
"path": "migrations/20240302205633_not_null#.sql",
"chars": 96,
"preview": "-- Add migration script here\nALTER TABLE domains ALTER COLUMN processed_subdomains SET NOT NULL;"
},
{
"path": "migrations/20240302222401_dns.sql",
"chars": 629,
"preview": "-- Add migration script here\nALTER TABLE\n domains\nADD\n COLUMN last_checked_dns TIMESTAMP WITH TIME ZONE;\n\nCREATE I"
},
{
"path": "migrations/20240304235746_filterlist.sql",
"chars": 223,
"preview": "-- Add migration script here\nALTER TABLE filterlists ADD COLUMN name TEXT;\nALTER TABLE filterlists ADD COLUMN author TEX"
},
{
"path": "migrations/20240305000257_filterlist.sql",
"chars": 89,
"preview": "-- Add migration script here\nALTER TABLE filterlists ALTER COLUMN contents DROP NOT NULL;"
},
{
"path": "migrations/20240305000612_filterlist.sql",
"chars": 92,
"preview": "-- Add migration script here\nALTER TABLE filterlists ALTER COLUMN lastUpdated DROP NOT NULL;"
},
{
"path": "migrations/20240305003411_filterlist.sql",
"chars": 87,
"preview": "-- Add migration script here\nALTER TABLE filterlists ALTER COLUMN expires SET NOT NULL;"
},
{
"path": "migrations/20240305200551_rule_matches.sql",
"chars": 402,
"preview": "-- Add migration script here\nCREATE TABLE rule_matches (\n rule_id INTEGER NOT NULL,\n domain_id BIGINT NOT NULL\n);\n"
},
{
"path": "migrations/20240306123603_rule_count.sql",
"chars": 99,
"preview": "-- Add migration script here\nALTER TABLE filterLists ADD COLUMN rule_count INT NOT NULL DEFAULT 0;\n"
},
{
"path": "migrations/20240306214217_index.sql",
"chars": 105,
"preview": "-- Add migration script here\nCREATE INDEX rules_last_checked_matches_idx ON rules (last_checked_matches);"
},
{
"path": "migrations/20240307005702_lists.sql",
"chars": 167,
"preview": "-- Add migration script here\nCREATE TABLE allow_domains (\n domain_id BIGINT UNIQUE NOT NULL\n);\n\nCREATE TABLE block_do"
},
{
"path": "migrations/20240307012450_index.sql",
"chars": 142,
"preview": "-- Add migration script here\nCREATE INDEX domain_rules_allow_idx ON domain_rules (allow);\nCREATE INDEX ip_rules_allow_id"
},
{
"path": "migrations/20240307031445_idx.sql",
"chars": 88,
"preview": "-- Add migration script here\nCREATE INDEX ip_rules_network_idx ON ip_rules (ip_network);"
},
{
"path": "output/allowed_ips.txt",
"chars": 0,
"preview": ""
},
{
"path": "output/domains.rpz",
"chars": 0,
"preview": ""
},
{
"path": "output/hosts.txt",
"chars": 0,
"preview": ""
},
{
"path": "output/ip_blocklist.txt",
"chars": 0,
"preview": ""
},
{
"path": "output/whitelist_adblock.txt",
"chars": 0,
"preview": ""
},
{
"path": "output/whitelist_domains.txt",
"chars": 0,
"preview": ""
},
{
"path": "package.json",
"chars": 126,
"preview": "{\n \"devDependencies\": {\n \"@tailwindcss/typography\": \"^0.5.10\",\n \"daisyui\": \"^4.7.2\",\n \"tailwindcss\": \"^3.4.1\"\n"
},
{
"path": "rust-toolchain.toml",
"chars": 32,
"preview": "\n[toolchain]\nchannel = \"stable\"\n"
},
{
"path": "rustfmt.toml",
"chars": 17,
"preview": "edition = \"2021\"\n"
},
{
"path": "src/app.rs",
"chars": 2966,
"preview": "use crate::{\n domain::DomainViewPage,\n error_template::{AppError, ErrorTemplate},\n filterlist::FilterListPage,\n"
},
{
"path": "src/domain.rs",
"chars": 33098,
"preview": "#[cfg(feature = \"ssr\")]\nuse crate::rule::RuleData;\nuse crate::{\n app::Loading,\n filterlist::{FilterListLink, Filte"
},
{
"path": "src/error_template.rs",
"chars": 2306,
"preview": "use http::status::StatusCode;\nuse leptos::*;\nuse thiserror::Error;\n\n#[derive(Clone, Debug, Error)]\npub enum AppError {\n "
},
{
"path": "src/fileserv.rs",
"chars": 1316,
"preview": "use crate::app::App;\nuse axum::response::Response as AxumResponse;\nuse axum::{\n body::Body,\n extract::State,\n h"
},
{
"path": "src/filterlist.rs",
"chars": 43022,
"preview": "pub use crate::domain::Domain;\nuse crate::PAGE_SIZE;\nuse crate::{rule::RuleId, source::SourceId};\nuse leptos::*;\nuse lep"
},
{
"path": "src/home_page.rs",
"chars": 3588,
"preview": "use crate::filterlist::{FilterListRecord, FilterListUrl, LastUpdated, ListSize};\nuse leptos::*;\nuse leptos_router::*;\n\n#"
},
{
"path": "src/ip_view.rs",
"chars": 3266,
"preview": "use leptos::*;\nuse leptos_router::*;\nuse std::{collections::BTreeSet, net::IpAddr};\n\nuse crate::{app::Loading, domain::D"
},
{
"path": "src/lib.rs",
"chars": 1489,
"preview": "pub mod app;\npub mod domain;\npub mod error_template;\npub mod filterlist;\npub mod home_page;\npub mod ip_view;\npub mod rul"
},
{
"path": "src/main.rs",
"chars": 5689,
"preview": "#[cfg(feature = \"ssr\")]\nuse clap::Parser;\n\n#[cfg(feature = \"ssr\")]\n#[derive(Debug, serde::Deserialize, serde::Serialize)"
},
{
"path": "src/rule.rs",
"chars": 13937,
"preview": "use crate::app::Loading;\nuse crate::filterlist::DomainRule;\nuse crate::filterlist::FilterListLink;\nuse crate::filterlist"
},
{
"path": "src/server.rs",
"chars": 15181,
"preview": "use crate::DbInitError;\nuse crate::{domain::Domain, filterlist::FilterListUrl};\n\nuse addr::domain;\nuse axum::body::Bytes"
},
{
"path": "src/stats_view.rs",
"chars": 5962,
"preview": "use leptos::*;\n\n#[server]\nasync fn count_total_rules() -> Result<usize, ServerFnError> {\n let pool = crate::server::g"
},
{
"path": "src/tasks.rs",
"chars": 1145,
"preview": "use leptos::*;\n\ntrait Task {\n type Error;\n fn name(&self) -> &str;\n async fn run_once(&self) -> Result<String, "
},
{
"path": "style/main.scss",
"chars": 0,
"preview": ""
},
{
"path": "style/tailwind.css",
"chars": 58,
"preview": "@tailwind base;\n@tailwind components;\n@tailwind utilities;"
},
{
"path": "tailwind.config.js",
"chars": 263,
"preview": "/** @type {import('tailwindcss').Config} */\n module.exports = {\n content: {\n relative: true,\n file"
},
{
"path": "tld_list.txt",
"chars": 10168,
"preview": "# Version 2020071800, Last Updated Sat Jul 18 07:07:01 2020 UTC\nAAA\nAARP\nABARTH\nABB\nABBOTT\nABBVIE\nABC\nABLE\nABOGADO\nABUDH"
},
{
"path": "update.sh",
"chars": 93,
"preview": "set -e\ngit pull\ncargo run --release generate\ngit commit output -m \"Ran Autoupdate\"\ngit push\n\n"
}
]
// ... and 2 more files (download for full content)
About this extraction
This page contains the full source code of the mkb2091/blockconvert GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 107 files (84.6 MB), approximately 56.9k tokens, and a symbol index with 227 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.