Repository: vla/BloomFilter.NetCore
Branch: main
Commit: c2885f904b7e
Files: 147
Total size: 1.1 MB
Directory structure:
gitextract_745aaz2k/
├── .gitignore
├── BloomFilter.NetCore.sln
├── Directory.Build.props
├── Directory.Packages.props
├── LICENSE
├── README.md
├── README.zh-CN.md
├── VERSION
├── build.cmd
├── build.sh
├── nuget_push.cmd
├── sign.snk
├── src/
│ ├── BloomFilter/
│ │ ├── Annotations.cs
│ │ ├── AsyncLock.cs
│ │ ├── BinaryHelper.cs
│ │ ├── BloomFilter.csproj
│ │ ├── BloomFilterConstValue.cs
│ │ ├── BloomFilterExtensions.cs
│ │ ├── Configurations/
│ │ │ ├── BloomFilterOptions.cs
│ │ │ ├── FilterMemoryOptions.cs
│ │ │ ├── FilterMemoryOptionsExtension.cs
│ │ │ └── IBloomFilterOptionsExtension.cs
│ │ ├── DefaultBloomFilterFactory.cs
│ │ ├── DefaultFilterMemorySerializer.cs
│ │ ├── Filter.cs
│ │ ├── FilterBuilder.cs
│ │ ├── FilterMemory.cs
│ │ ├── FilterMemorySerializerParam.cs
│ │ ├── FilterRedisBase.cs
│ │ ├── HashAlgorithms/
│ │ │ ├── Adler32.cs
│ │ │ ├── Crc32.cs
│ │ │ ├── Crc64.cs
│ │ │ ├── HashCrypto.cs
│ │ │ ├── Internal/
│ │ │ │ ├── Adler32.cs
│ │ │ │ ├── Crc32.Arm.cs
│ │ │ │ ├── Crc32.Table.cs
│ │ │ │ ├── Crc32.Vectorized.cs
│ │ │ │ ├── Crc32.cs
│ │ │ │ ├── Crc64.Table.cs
│ │ │ │ ├── Crc64.Vectorized.cs
│ │ │ │ ├── Crc64.cs
│ │ │ │ ├── FNV1.cs
│ │ │ │ ├── FNV1a.cs
│ │ │ │ ├── ModifiedFNV1.cs
│ │ │ │ ├── Murmur128BitsX64.State.cs
│ │ │ │ ├── Murmur128BitsX64.cs
│ │ │ │ ├── Murmur128BitsX86.State.cs
│ │ │ │ ├── Murmur128BitsX86.cs
│ │ │ │ ├── Murmur32BitsX86.State.cs
│ │ │ │ ├── Murmur32BitsX86.cs
│ │ │ │ ├── NonCryptoHashAlgorithm.cs
│ │ │ │ ├── ThrowHelper.cs
│ │ │ │ ├── VectorHelper.cs
│ │ │ │ ├── XxHash128.cs
│ │ │ │ ├── XxHash3.cs
│ │ │ │ ├── XxHash32.State.cs
│ │ │ │ ├── XxHash32.cs
│ │ │ │ ├── XxHash64.State.cs
│ │ │ │ ├── XxHash64.cs
│ │ │ │ └── XxHashShared.cs
│ │ │ ├── LCGWithFNV.cs
│ │ │ ├── Murmur128BitsX64.cs
│ │ │ ├── Murmur128BitsX86.cs
│ │ │ ├── Murmur32BitsX86.cs
│ │ │ ├── RNGWithFNV.cs
│ │ │ ├── XXHash128.cs
│ │ │ ├── XXHash3.cs
│ │ │ ├── XXHash32.cs
│ │ │ └── XXHash64.cs
│ │ ├── HashFunction.cs
│ │ ├── HashMethod.cs
│ │ ├── IBloomFilter.cs
│ │ ├── IBloomFilterFactory.cs
│ │ ├── IFilterMemorySerializer.cs
│ │ ├── Properties/
│ │ │ └── AssemblyInfo.cs
│ │ ├── ServiceCollectionExtensions.cs
│ │ └── StringSpanExtensions.cs
│ ├── BloomFilter.CSRedis/
│ │ ├── BloomFilter.CSRedis.csproj
│ │ ├── Configurations/
│ │ │ ├── FilterCSRedisOptions.cs
│ │ │ ├── FilterCSRedisOptionsExtension.cs
│ │ │ └── ServiceCollectionExtensions.cs
│ │ ├── FilterCSRedis.cs
│ │ └── FilterCSRedisBuilder.cs
│ ├── BloomFilter.EasyCaching/
│ │ ├── BloomFilter.EasyCaching.csproj
│ │ ├── Configurations/
│ │ │ ├── FilterEasyCachingRedisExtension.cs
│ │ │ ├── FilterEasyCachingRedisOptions.cs
│ │ │ └── ServiceCollectionExtensions.cs
│ │ ├── FilterEasyCachingBuilder.cs
│ │ └── FilterEasyCachingRedis.cs
│ ├── BloomFilter.FreeRedis/
│ │ ├── BloomFilter.FreeRedis.csproj
│ │ ├── Configurations/
│ │ │ ├── FilterFreeRedisOptions.cs
│ │ │ ├── FilterFreeRedisOptionsExtension.cs
│ │ │ └── ServiceCollectionExtensions.cs
│ │ ├── FilterFreeRedis.cs
│ │ └── FilterFreeRedisBuilder.cs
│ └── BloomFilter.Redis/
│ ├── BloomFilter.Redis.csproj
│ ├── Configurations/
│ │ ├── FilterRedisOptions.cs
│ │ ├── FilterRedisOptionsExtension.cs
│ │ └── ServiceCollectionExtensions.cs
│ ├── FilterRedis.cs
│ ├── FilterRedisBuilder.cs
│ ├── IRedisBitOperate.cs
│ └── RedisBitOperate.cs
└── test/
├── BenchmarkTest/
│ ├── BenchmarkTest.csproj
│ ├── FreeRedisBenchmark.cs
│ ├── Helper.cs
│ ├── MemoryBenchmark.cs
│ ├── Program.cs
│ └── RedisBenchmark.cs
├── BloomFilter.Redis.Test/
│ ├── BloomFilter.Redis.Test.csproj
│ ├── BloomFilterCSRedisTest.cs
│ ├── BloomFilterEasyCachingRedisTest.cs
│ ├── BloomFilterFreeRedisTest.cs
│ ├── BloomFilterRedisTest.cs
│ ├── ConfigurationsTest.cs
│ ├── RedisBitOperateTest.cs
│ └── Utilitiy.cs
├── BloomFilterTest/
│ ├── AsyncLockTests.cs
│ ├── BloomFilterTest.cs
│ ├── BloomFilterTest.csproj
│ ├── ConfigurationsTest.cs
│ ├── FluentFilterBuilderTest.cs
│ ├── HashAlgorithms/
│ │ ├── Adler32Test.cs
│ │ ├── CrcTest.cs
│ │ ├── FNVTest.cs
│ │ ├── Murmur3Test.cs
│ │ ├── TestPayloadParameter.cs
│ │ └── XxHashTest.cs
│ ├── ImportExportTest.cs
│ ├── IssuesTest.cs
│ ├── SerializerTest.cs
│ ├── Utilitiy.cs
│ ├── ValueTypeTest.cs
│ └── xunit.runner.json
├── Demo/
│ ├── BloomFilterMemory.cs
│ ├── BloomFilterRedis.cs
│ ├── Demo.csproj
│ ├── Program.cs
│ └── TestExcute.cs
└── PerformanceTest/
├── GeneralPerformance.cs
├── HashErrRate.cs
├── HashSpeed.cs
├── Helper.cs
├── Issues_2.cs
├── PerformanceTest.csproj
├── Program.cs
└── TestExcute.cs
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
# Build results
[Dd]ebug/
[Rr]elease/
x64/
app.publish/
[Bb]in/
[Oo]bj/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# Visual Studio
*.suo
*.user
*.sln.docstates
*.sln.ide/
project.lock.json
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
#GhostDoc
*.GhostDoc.xml
# NuGet Packages Directory
packages/
# Windows image file caches
Thumbs.db
ehthumbs.db
# Folder config file
Desktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Mac crap
.DS_Store
####################
#
# Current Project
#
####################
.vs/
artifacts/
*.log
/test/BenchmarkTest/BenchmarkDotNet.Artifacts
================================================
FILE: BloomFilter.NetCore.sln
================================================
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.0.31903.59
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{8D22B289-3A76-4399-BDA3-FD1B699B2F13}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{BFDF8E72-A7F7-4894-8782-74BB34438F99}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Demo", "test\Demo\Demo.csproj", "{42A8F6A2-7CE7-493F-B3A7-07A1753F5285}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BloomFilter", "src\BloomFilter\BloomFilter.csproj", "{E9FB2A60-8EFE-49E9-98FE-D75A076CAD0F}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BloomFilterTest", "test\BloomFilterTest\BloomFilterTest.csproj", "{24193FBE-88A5-4C26-9996-341DEBE0CA80}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerformanceTest", "test\PerformanceTest\PerformanceTest.csproj", "{D46EEAAF-36FD-4297-9381-1845C93D13C4}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BloomFilter.Redis", "src\BloomFilter.Redis\BloomFilter.Redis.csproj", "{4A858860-7405-43F4-9132-0DF40BADBDAD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BloomFilter.Redis.Test", "test\BloomFilter.Redis.Test\BloomFilter.Redis.Test.csproj", "{C02DC47A-7258-4B59-AAB4-DFA208DE9B36}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BenchmarkTest", "test\BenchmarkTest\BenchmarkTest.csproj", "{CFB47A7F-580A-4F4B-940D-03D99EE70C41}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BloomFilter.CSRedis", "src\BloomFilter.CSRedis\BloomFilter.CSRedis.csproj", "{903B0A3A-1753-47CA-ADBC-B4379FF8F372}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BloomFilter.EasyCaching", "src\BloomFilter.EasyCaching\BloomFilter.EasyCaching.csproj", "{C4AFDCB8-077D-463A-9EF9-D2F8723F01F4}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BloomFilter.FreeRedis", "src\BloomFilter.FreeRedis\BloomFilter.FreeRedis.csproj", "{311303FD-7ABA-4C7D-AA5C-90E265FA04A7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{42A8F6A2-7CE7-493F-B3A7-07A1753F5285}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{42A8F6A2-7CE7-493F-B3A7-07A1753F5285}.Debug|Any CPU.Build.0 = Debug|Any CPU
{42A8F6A2-7CE7-493F-B3A7-07A1753F5285}.Release|Any CPU.ActiveCfg = Release|Any CPU
{42A8F6A2-7CE7-493F-B3A7-07A1753F5285}.Release|Any CPU.Build.0 = Release|Any CPU
{E9FB2A60-8EFE-49E9-98FE-D75A076CAD0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E9FB2A60-8EFE-49E9-98FE-D75A076CAD0F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E9FB2A60-8EFE-49E9-98FE-D75A076CAD0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E9FB2A60-8EFE-49E9-98FE-D75A076CAD0F}.Release|Any CPU.Build.0 = Release|Any CPU
{24193FBE-88A5-4C26-9996-341DEBE0CA80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{24193FBE-88A5-4C26-9996-341DEBE0CA80}.Debug|Any CPU.Build.0 = Debug|Any CPU
{24193FBE-88A5-4C26-9996-341DEBE0CA80}.Release|Any CPU.ActiveCfg = Release|Any CPU
{24193FBE-88A5-4C26-9996-341DEBE0CA80}.Release|Any CPU.Build.0 = Release|Any CPU
{D46EEAAF-36FD-4297-9381-1845C93D13C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D46EEAAF-36FD-4297-9381-1845C93D13C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D46EEAAF-36FD-4297-9381-1845C93D13C4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D46EEAAF-36FD-4297-9381-1845C93D13C4}.Release|Any CPU.Build.0 = Release|Any CPU
{4A858860-7405-43F4-9132-0DF40BADBDAD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4A858860-7405-43F4-9132-0DF40BADBDAD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4A858860-7405-43F4-9132-0DF40BADBDAD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4A858860-7405-43F4-9132-0DF40BADBDAD}.Release|Any CPU.Build.0 = Release|Any CPU
{C02DC47A-7258-4B59-AAB4-DFA208DE9B36}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C02DC47A-7258-4B59-AAB4-DFA208DE9B36}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C02DC47A-7258-4B59-AAB4-DFA208DE9B36}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C02DC47A-7258-4B59-AAB4-DFA208DE9B36}.Release|Any CPU.Build.0 = Release|Any CPU
{CFB47A7F-580A-4F4B-940D-03D99EE70C41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CFB47A7F-580A-4F4B-940D-03D99EE70C41}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CFB47A7F-580A-4F4B-940D-03D99EE70C41}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CFB47A7F-580A-4F4B-940D-03D99EE70C41}.Release|Any CPU.Build.0 = Release|Any CPU
{903B0A3A-1753-47CA-ADBC-B4379FF8F372}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{903B0A3A-1753-47CA-ADBC-B4379FF8F372}.Debug|Any CPU.Build.0 = Debug|Any CPU
{903B0A3A-1753-47CA-ADBC-B4379FF8F372}.Release|Any CPU.ActiveCfg = Release|Any CPU
{903B0A3A-1753-47CA-ADBC-B4379FF8F372}.Release|Any CPU.Build.0 = Release|Any CPU
{C4AFDCB8-077D-463A-9EF9-D2F8723F01F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C4AFDCB8-077D-463A-9EF9-D2F8723F01F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C4AFDCB8-077D-463A-9EF9-D2F8723F01F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C4AFDCB8-077D-463A-9EF9-D2F8723F01F4}.Release|Any CPU.Build.0 = Release|Any CPU
{311303FD-7ABA-4C7D-AA5C-90E265FA04A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{311303FD-7ABA-4C7D-AA5C-90E265FA04A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{311303FD-7ABA-4C7D-AA5C-90E265FA04A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{311303FD-7ABA-4C7D-AA5C-90E265FA04A7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{42A8F6A2-7CE7-493F-B3A7-07A1753F5285} = {BFDF8E72-A7F7-4894-8782-74BB34438F99}
{E9FB2A60-8EFE-49E9-98FE-D75A076CAD0F} = {8D22B289-3A76-4399-BDA3-FD1B699B2F13}
{24193FBE-88A5-4C26-9996-341DEBE0CA80} = {BFDF8E72-A7F7-4894-8782-74BB34438F99}
{D46EEAAF-36FD-4297-9381-1845C93D13C4} = {BFDF8E72-A7F7-4894-8782-74BB34438F99}
{4A858860-7405-43F4-9132-0DF40BADBDAD} = {8D22B289-3A76-4399-BDA3-FD1B699B2F13}
{C02DC47A-7258-4B59-AAB4-DFA208DE9B36} = {BFDF8E72-A7F7-4894-8782-74BB34438F99}
{CFB47A7F-580A-4F4B-940D-03D99EE70C41} = {BFDF8E72-A7F7-4894-8782-74BB34438F99}
{903B0A3A-1753-47CA-ADBC-B4379FF8F372} = {8D22B289-3A76-4399-BDA3-FD1B699B2F13}
{C4AFDCB8-077D-463A-9EF9-D2F8723F01F4} = {8D22B289-3A76-4399-BDA3-FD1B699B2F13}
{311303FD-7ABA-4C7D-AA5C-90E265FA04A7} = {8D22B289-3A76-4399-BDA3-FD1B699B2F13}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {12D6ACDC-31C9-4A69-BE24-76E0F73E48A6}
EndGlobalSection
EndGlobal
================================================
FILE: Directory.Build.props
================================================
$(Ver)
$(Ver)
2019-2023 v.la
v.la@live.cn
A bloom filter implementation
bloom filter
$(MSBuildThisFileDirectory)/artifacts
Bloom;Filter;bloom-filter
latest
false
icon.png
https://github.com/vla/BloomFilter.NetCore
MIT
MIT
git://github.com/vla/BloomFilter.NetCore
git
master
true
true
True
False
$(MSBuildThisFileDirectory)/sign.snk
================================================
FILE: Directory.Packages.props
================================================
true
false
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2018 v.la@live.cn
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# BloomFilter.NetCore
[](https://opensource.org/licenses/MIT)
[](https://dotnet.microsoft.com/)
A high-performance, feature-complete Bloom filter library for .NET, supporting both in-memory and distributed Redis backends.
[中文文档](README.zh-CN.md)
## Table of Contents
- [Overview](#overview)
- [Key Features](#key-features)
- [Packages & Status](#packages--status)
- [Architecture](#architecture)
- [Core Functionality](#core-functionality)
- [Installation](#installation)
- [Quick Start](#quick-start)
- [Usage Examples](#usage-examples)
- [In-Memory Mode](#in-memory-mode)
- [Dependency Injection](#dependency-injection)
- [Redis Distributed Mode](#redis-distributed-mode)
- [Hash Algorithms](#hash-algorithms)
- [Performance Benchmarks](#performance-benchmarks)
- [Advanced Usage](#advanced-usage)
- [API Reference](#api-reference)
- [Contributing](#contributing)
- [License](#license)
## Overview
BloomFilter.NetCore is an enterprise-grade Bloom filter library designed for the .NET ecosystem. A Bloom filter is a space-efficient probabilistic data structure used to test whether an element is a member of a set. Its core characteristics are:
- **Space Efficient**: Extremely small memory footprint compared to traditional HashSets
- **O(1) Time Complexity**: Both add and query operations execute in constant time
- **Probabilistic**: May return false positives but never false negatives
This project provides two major implementation types:
1. **In-Memory Bloom Filter (FilterMemory)**: BitArray-based in-memory implementation, suitable for single-process scenarios
2. **Distributed Bloom Filter (FilterRedis series)**: Redis-backed distributed implementation, supports concurrent access from multiple applications
### Primary Use Cases
- **Cache Penetration Protection**: Prevent malicious queries for non-existent data from bypassing cache
- **Deduplication**: URL deduplication, email deduplication, user ID deduplication, etc.
- **Recommendation Systems**: Check if a user has seen specific content
- **Web Crawlers**: Check if URLs have been crawled
- **Distributed Systems**: Share state checks across multiple service instances
- **Big Data**: Existence checks for massive datasets
## Key Features
### 🎯 Flexible Configuration
- **Fully Configurable Parameters**: Bit array size (m), number of hash functions (k)
- **Automatic Parameter Calculation**: Automatically calculate optimal parameters based on tolerable false positive rate (p) and expected element count (n)
- **20+ Hash Algorithms**: Support for CRC, MD5, SHA, Murmur, LCGs, xxHash, or custom algorithms
### ⚡ High Performance
- **Fast Generation**: Bloom filter generation and operations are extremely fast
- **Optimized Implementation**: Uses Span, ReadOnlyMemory for zero-copy operations
- **Unsafe Code Optimization**: Uses unsafe code blocks in performance-critical paths
- **Rejection Sampling**: Implements rejection sampling and hash chaining, considering avalanche effect for improved hash quality
### 🔒 Concurrency Safe
- **Thread-Safe**: Uses AsyncLock mechanism for safe multi-threaded concurrent access
- **Async Support**: Comprehensive async/await support with async versions of all operations
- **Distributed Locking**: Redis implementations support concurrent access across applications
### 🌐 Multiple Backend Support
- **StackExchange.Redis**: Officially recommended Redis client
- **CSRedisCore**: High-performance Redis client
- **FreeRedis**: Lightweight Redis client
- **EasyCaching**: Supports EasyCaching abstraction layer, switchable cache providers
### 📦 Modern .NET Support
- **Multi-Framework Support**: net462, netstandard2.0, net6.0, net7.0, net8.0, net9.0, net10.0
- **Dependency Injection**: Native support for Microsoft.Extensions.DependencyInjection
- **Nullable Reference Types**: Enabled for improved code safety
## Packages & Status
| Package | NuGet | Description |
|---------|-------|-------------|
|**BloomFilter.NetCore**|[](https://www.nuget.org/packages/BloomFilter.NetCore)| Core package with in-memory Bloom filter |
|**BloomFilter.Redis.NetCore**|[](https://www.nuget.org/packages/BloomFilter.Redis.NetCore)| StackExchange.Redis implementation |
|**BloomFilter.CSRedis.NetCore**|[](https://www.nuget.org/packages/BloomFilter.CSRedis.NetCore)| CSRedisCore implementation |
|**BloomFilter.FreeRedis.NetCore**|[](https://www.nuget.org/packages/BloomFilter.FreeRedis.NetCore)| FreeRedis implementation |
|**BloomFilter.EasyCaching.NetCore**|[](https://www.nuget.org/packages/BloomFilter.EasyCaching.NetCore)| EasyCaching integration |
## Architecture
### Core Interface Layer
```
IBloomFilter (Interface)
├── Add / AddAsync - Add elements
├── Contains / ContainsAsync - Check elements
├── All / AllAsync - Batch check
├── Clear / ClearAsync - Clear filter
└── ComputeHash - Compute hash values
```
### Implementation Hierarchy
```
Filter (Abstract Base Class)
├── FilterMemory (In-Memory)
│ └── Uses BitArray storage
│
└── Redis Series (Distributed)
├── FilterRedis (StackExchange.Redis)
├── FilterCSRedis (CSRedisCore)
├── FilterFreeRedis (FreeRedis)
└── FilterEasyCachingRedis (EasyCaching)
```
### Configuration System
```
BloomFilterOptions
├── FilterMemoryOptions - In-memory mode configuration
├── FilterRedisOptions - StackExchange.Redis configuration
├── FilterCSRedisOptions - CSRedisCore configuration
├── FilterFreeRedisOptions - FreeRedis configuration
└── FilterEasyCachingOptions - EasyCaching configuration
```
## Core Functionality
### Mathematical Model
BloomFilter.NetCore implements the complete Bloom filter mathematical model:
#### 1. Optimal Bit Array Size (m)
Given expected element count `n` and false positive rate `p`, calculate optimal bit array size:
```
m = -(n * ln(p)) / (ln(2)^2)
```
#### 2. Optimal Number of Hash Functions (k)
Given element count `n` and bit array size `m`, calculate optimal number of hash functions:
```
k = (m / n) * ln(2)
```
#### 3. Actual False Positive Rate (p)
Given inserted element count, number of hash functions, and bit array size, calculate actual false positive rate:
```
p = (1 - e^(-k*n/m))^k
```
These calculations are provided by static methods in the `Filter` base class:
```csharp
// Calculate optimal bit array size
long m = Filter.BestM(expectedElements, errorRate);
// Calculate optimal number of hash functions
int k = Filter.BestK(expectedElements, capacity);
// Calculate optimal element count
long n = Filter.BestN(hashes, capacity);
// Calculate actual false positive rate
double p = Filter.BestP(hashes, capacity, insertedElements);
```
### Storage Mechanisms
#### In-Memory Storage
- **BitArray**: Uses .NET's BitArray as underlying storage
- **Bucketing Strategy**: Automatically splits into multiple BitArrays when capacity exceeds 2GB (MaxInt = 2,147,483,640)
- **Serialization Support**: Supports serialization/deserialization for persistence or transfer
#### Redis Storage
- **SETBIT/GETBIT**: Uses Redis bit operation commands
- **Distributed Access**: Multiple application instances can concurrently access the same filter
- **Persistence**: Leverages Redis persistence mechanisms for data safety
### Concurrency Control
```csharp
// AsyncLock ensures thread safety
public class AsyncLock
{
private readonly SemaphoreSlim _semaphore = new(1, 1);
public async ValueTask LockAsync()
{
await _semaphore.WaitAsync();
return new Release(_semaphore);
}
}
```
## Installation
### Install via NuGet
**In-Memory Mode (Core Package):**
```bash
dotnet add package BloomFilter.NetCore
```
**Redis Distributed Mode (Choose One):**
```bash
# StackExchange.Redis
dotnet add package BloomFilter.Redis.NetCore
# CSRedisCore
dotnet add package BloomFilter.CSRedis.NetCore
# FreeRedis
dotnet add package BloomFilter.FreeRedis.NetCore
# EasyCaching
dotnet add package BloomFilter.EasyCaching.NetCore
```
## Quick Start
### Simplest Example
```csharp
using BloomFilter;
// Create a Bloom filter: expect 10 million elements, 1% false positive rate
var bf = FilterBuilder.Build(10_000_000, 0.01);
// Add elements
bf.Add("user:123");
bf.Add("user:456");
// Check element existence
Console.WriteLine(bf.Contains("user:123")); // True
Console.WriteLine(bf.Contains("user:789")); // False (very small probability of True)
// Clear filter
bf.Clear();
```
### Async Operations
```csharp
// Async add
await bf.AddAsync(Encoding.UTF8.GetBytes("user:123"));
// Async check
bool exists = await bf.ContainsAsync(Encoding.UTF8.GetBytes("user:123"));
// Batch async operations
var users = new[] {
Encoding.UTF8.GetBytes("user:1"),
Encoding.UTF8.GetBytes("user:2"),
Encoding.UTF8.GetBytes("user:3")
};
await bf.AddAsync(users);
var results = await bf.ContainsAsync(users);
```
### Fluent API (New in v3.0)
v3.0 introduces a modern fluent API for building Bloom filters with improved discoverability and expressiveness:
```csharp
// In-Memory Fluent API
var filter = FilterBuilder.Create()
.WithName("UserFilter")
.ExpectingElements(10_000_000)
.WithErrorRate(0.001)
.UsingHashMethod(HashMethod.XXHash3)
.BuildInMemory();
// Redis Fluent API (StackExchange.Redis)
var redisFilter = FilterRedisBuilder.Create()
.WithRedisConnection("localhost:6379")
.WithRedisKey("bloom:users")
.WithName("UserFilter")
.ExpectingElements(10_000_000)
.WithErrorRate(0.001)
.BuildRedis();
// CSRedis Fluent API
var csredisFilter = FilterCSRedisBuilder.Create()
.WithRedisClient(csredisClient)
.WithRedisKey("bloom:users")
.ExpectingElements(10_000_000)
.BuildCSRedis();
// FreeRedis Fluent API
var freeRedisFilter = FilterFreeRedisBuilder.Create()
.WithRedisClient(redisClient)
.WithRedisKey("bloom:users")
.ExpectingElements(10_000_000)
.BuildFreeRedis();
// EasyCaching Fluent API
var easyCachingFilter = FilterEasyCachingBuilder.Create()
.WithRedisCachingProvider(provider)
.WithRedisKey("bloom:users")
.ExpectingElements(10_000_000)
.BuildEasyCaching();
// All common configuration methods:
// - WithName(string) - Set filter name
// - ExpectingElements(long) - Set expected element count
// - WithErrorRate(double) - Set false positive rate (0-1)
// - UsingHashMethod(HashMethod) - Use predefined hash algorithm
// - UsingCustomHash(HashFunction) - Use custom hash function
// - WithSerializer(IFilterMemorySerializer) - Set custom serializer (memory only)
```
**Why use Fluent API?**
- 🔍 Better discoverability with IntelliSense
- 📖 More readable and self-documenting code
- ⛓️ Chainable method calls
- 🎯 Type-safe configuration
- ✅ Backward compatible - old static methods still work!
## Usage Examples
### In-Memory Mode
#### Basic Usage
```csharp
using BloomFilter;
public class UserService
{
// Static shared Bloom filter
private static readonly IBloomFilter _bloomFilter =
FilterBuilder.Build(10_000_000, 0.01);
public void AddUser(string userId)
{
// Add user ID
_bloomFilter.Add(userId);
}
public bool MayExistUser(string userId)
{
// Check if user may exist
return _bloomFilter.Contains(userId);
}
}
```
#### Custom Configuration
```csharp
using BloomFilter;
// Method 1: Specify hash algorithm
var bf1 = FilterBuilder.Build(
expectedElements: 1_000_000,
errorRate: 0.001,
hashMethod: HashMethod.Murmur3
);
// Method 2: Use custom hash function
var hashFunction = new Murmur128BitsX64();
var bf2 = FilterBuilder.Build(
expectedElements: 1_000_000,
errorRate: 0.001,
hashFunction: hashFunction
);
// Method 3: Manually specify parameters (advanced usage)
var bf3 = FilterBuilder.Build(
capacity: 9585059, // Bit array size
hashes: 10, // Number of hash functions
hashMethod: HashMethod.XXHash3
);
// Method 4: Use configuration object
var options = new FilterMemoryOptions
{
Name = "MyFilter",
ExpectedElements = 5_000_000,
ErrorRate = 0.01,
Method = HashMethod.Murmur3
};
var bf4 = FilterBuilder.Build(options);
```
### Dependency Injection
#### ASP.NET Core Integration
```csharp
using BloomFilter;
using Microsoft.Extensions.DependencyInjection;
public class Startup
{
public void ConfigureServices(IServiceCollection services)
{
// Register Bloom filter service
services.AddBloomFilter(setupAction =>
{
setupAction.UseInMemory(options =>
{
options.Name = "MainFilter";
options.ExpectedElements = 10_000_000;
options.ErrorRate = 0.01;
options.Method = HashMethod.Murmur3;
});
});
services.AddControllers();
}
}
// Use in controller or service
public class UserController : ControllerBase
{
private readonly IBloomFilter _bloomFilter;
public UserController(IBloomFilter bloomFilter)
{
_bloomFilter = bloomFilter;
}
[HttpPost("users/{userId}")]
public IActionResult CheckUser(string userId)
{
if (_bloomFilter.Contains(userId))
{
// User may exist, continue to query database
return Ok("User may exist");
}
else
{
// User definitely doesn't exist, no need to query database
return NotFound("User doesn't exist");
}
}
}
```
#### Multiple Filter Instances
```csharp
services.AddBloomFilter(setupAction =>
{
// User filter
setupAction.UseInMemory(options =>
{
options.Name = "UserFilter";
options.ExpectedElements = 10_000_000;
options.ErrorRate = 0.01;
});
// Email filter
setupAction.UseInMemory(options =>
{
options.Name = "EmailFilter";
options.ExpectedElements = 5_000_000;
options.ErrorRate = 0.001;
});
});
// Use factory to get specific filter
public class MyService
{
private readonly IBloomFilter _userFilter;
private readonly IBloomFilter _emailFilter;
public MyService(IBloomFilterFactory factory)
{
_userFilter = factory.Get("UserFilter");
_emailFilter = factory.Get("EmailFilter");
}
}
```
### Redis Distributed Mode
#### StackExchange.Redis
```csharp
using BloomFilter;
// Method 1: Direct build
var bf = FilterRedisBuilder.Build(
redisHost: "localhost:6379",
name: "DistributedFilter",
expectedElements: 5_000_000,
errorRate: 0.001
);
bf.Add("item:123");
Console.WriteLine(bf.Contains("item:123")); // True
// Method 2: Dependency injection
services.AddBloomFilter(setupAction =>
{
setupAction.UseRedis(new FilterRedisOptions
{
Name = "UserFilter",
RedisKey = "BloomFilter:Users",
Endpoints = new List { "localhost:6379" },
Database = 0,
ExpectedElements = 10_000_000,
ErrorRate = 0.01,
Method = HashMethod.Murmur3
});
});
// Method 3: Advanced configuration (master-slave, sentinel, cluster)
services.AddBloomFilter(setupAction =>
{
setupAction.UseRedis(new FilterRedisOptions
{
Name = "ProductFilter",
RedisKey = "BloomFilter:Products",
Endpoints = new List
{
"redis-master:6379",
"redis-slave1:6379",
"redis-slave2:6379"
},
Password = "your-redis-password",
Ssl = true,
ConnectTimeout = 5000,
SyncTimeout = 3000,
ExpectedElements = 20_000_000,
ErrorRate = 0.001
});
});
```
#### CSRedisCore
```csharp
services.AddBloomFilter(setupAction =>
{
setupAction.UseCSRedis(new FilterCSRedisOptions
{
Name = "OrderFilter",
RedisKey = "BloomFilter:Orders",
ConnectionStrings = new List
{
"localhost:6379,password=123456,defaultDatabase=0,poolsize=50,prefix=myapp:"
},
ExpectedElements = 5_000_000,
ErrorRate = 0.01
});
});
```
#### FreeRedis
```csharp
services.AddBloomFilter(setupAction =>
{
setupAction.UseFreeRedis(new FilterFreeRedisOptions
{
Name = "CartFilter",
RedisKey = "BloomFilter:Carts",
ConnectionStrings = new List { "localhost:6379,password=123456" },
ExpectedElements = 1_000_000,
ErrorRate = 0.01
});
});
```
#### EasyCaching Integration
EasyCaching provides a unified caching abstraction layer, allowing you to easily switch underlying cache implementations:
```csharp
using EasyCaching.Core.Configurations;
using Microsoft.Extensions.DependencyInjection;
var services = new ServiceCollection();
// 1. Configure EasyCaching
services.AddEasyCaching(options =>
{
// Configure Redis provider
options.UseRedis(config =>
{
config.DBConfig.Endpoints.Add(new ServerEndPoint("127.0.0.1", 6379));
config.DBConfig.Database = 0;
}, "redis-provider-1");
// Can configure multiple providers
options.UseRedis(config =>
{
config.DBConfig.Endpoints.Add(new ServerEndPoint("127.0.0.1", 6379));
config.DBConfig.Database = 1;
}, "redis-provider-2");
});
// 2. Configure BloomFilter
services.AddBloomFilter(setupAction =>
{
// Use first Redis provider
setupAction.UseEasyCachingRedis(new FilterEasyCachingRedisOptions
{
Name = "BF1",
RedisKey = "BloomFilter1",
ProviderName = "redis-provider-1",
ExpectedElements = 10_000_000,
ErrorRate = 0.01
});
// Use second Redis provider
setupAction.UseEasyCachingRedis(new FilterEasyCachingRedisOptions
{
Name = "BF2",
RedisKey = "BloomFilter2",
ProviderName = "redis-provider-2",
ExpectedElements = 5_000_000,
ErrorRate = 0.001
});
});
var provider = services.BuildServiceProvider();
// Use default filter
var bf = provider.GetService();
bf.Add("value1");
// Use named filter
var factory = provider.GetService();
var bf1 = factory.Get("BF1");
var bf2 = factory.Get("BF2");
bf1.Add("item1");
bf2.Add("item2");
```
### Real-World Application Scenarios
#### 1. Cache Penetration Protection
```csharp
public class ProductService
{
private readonly IBloomFilter _bloomFilter;
private readonly ICache _cache;
private readonly IProductRepository _repository;
public ProductService(
IBloomFilter bloomFilter,
ICache cache,
IProductRepository repository)
{
_bloomFilter = bloomFilter;
_cache = cache;
_repository = repository;
}
public async Task GetProductAsync(string productId)
{
// First layer: Bloom filter
if (!_bloomFilter.Contains(productId))
{
// Product definitely doesn't exist, return null directly
return null;
}
// Second layer: Cache
var cached = await _cache.GetAsync(productId);
if (cached != null)
{
return cached;
}
// Third layer: Database
var product = await _repository.GetByIdAsync(productId);
if (product != null)
{
await _cache.SetAsync(productId, product);
}
return product;
}
public async Task CreateProductAsync(Product product)
{
// Save to database
await _repository.SaveAsync(product);
// Add to Bloom filter
_bloomFilter.Add(product.Id);
// Update cache
await _cache.SetAsync(product.Id, product);
}
}
```
#### 2. URL Deduplication (Web Crawler)
```csharp
public class WebCrawler
{
private readonly IBloomFilter _visitedUrls;
private readonly Queue _urlQueue;
public WebCrawler(IBloomFilter bloomFilter)
{
_visitedUrls = bloomFilter;
_urlQueue = new Queue();
}
public async Task CrawlAsync(string startUrl)
{
_urlQueue.Enqueue(startUrl);
while (_urlQueue.Count > 0)
{
var url = _urlQueue.Dequeue();
// Check if already visited
if (_visitedUrls.Contains(url))
{
continue; // Skip already visited URLs
}
// Mark as visited
_visitedUrls.Add(url);
// Download page
var page = await DownloadPageAsync(url);
// Process page
await ProcessPageAsync(page);
// Extract new URLs
var newUrls = ExtractUrls(page);
foreach (var newUrl in newUrls)
{
if (!_visitedUrls.Contains(newUrl))
{
_urlQueue.Enqueue(newUrl);
}
}
}
}
}
```
#### 3. Distributed Deduplication (Multiple Instances)
```csharp
// Configure distributed Bloom filter
services.AddBloomFilter(setupAction =>
{
setupAction.UseRedis(new FilterRedisOptions
{
Name = "GlobalDeduplication",
RedisKey = "BF:Dedup",
Endpoints = new List { "redis-cluster:6379" },
ExpectedElements = 100_000_000,
ErrorRate = 0.0001
});
});
// Use across multiple service instances
public class MessageProcessor
{
private readonly IBloomFilter _bloomFilter;
public async Task ProcessMessageAsync(Message message)
{
// All instances share the same Redis Bloom filter
if (await _bloomFilter.ContainsAsync(message.Id))
{
// Message already processed by another instance
return;
}
// Mark as processed
await _bloomFilter.AddAsync(message.Id);
// Process message
await HandleMessageAsync(message);
}
}
```
## Hash Algorithms
BloomFilter.NetCore supports 20+ hash algorithms, choose based on performance and accuracy requirements:
### Algorithm Categories
| Category | Algorithms | Characteristics | Use Cases |
|----------|-----------|-----------------|-----------|
| **LCG-based** | LCGWithFNV1
LCGWithFNV1a
LCGModifiedFNV1 | Extremely fast, lower quality | Extremely high performance requirements, can tolerate high false positive rates |
| **RNG-based** | RNGWithFNV1
RNGWithFNV1a
RNGModifiedFNV1 | High quality, slower | Scenarios requiring high accuracy |
| **Checksum** | CRC32
CRC64
Adler32 | Balanced performance and quality | General scenarios |
| **Murmur Family** | Murmur3
Murmur32BitsX86
Murmur128BitsX64
Murmur128BitsX86 | **Recommended**, good performance, high quality | Recommended for production |
| **Cryptographic** | SHA1
SHA256
SHA384
SHA512 | Highest quality, slowest | Scenarios requiring extreme security |
| **XXHash Family** | XXHash32
XXHash64
XXHash3
XXHash128 | **Fastest**, excellent quality | First choice for high performance |
### Selection Recommendations
```csharp
// Recommended: Default Murmur3 for production (balanced performance and quality)
var bf1 = FilterBuilder.Build(10_000_000, 0.01, HashMethod.Murmur3);
// High Performance: Choose XXHash3 for extreme performance requirements
var bf2 = FilterBuilder.Build(10_000_000, 0.01, HashMethod.XXHash3);
// High Precision: Choose SHA256 + lower errorRate for minimal false positive rate
var bf3 = FilterBuilder.Build(10_000_000, 0.0001, HashMethod.SHA256);
// Distributed: Recommend XXHash64 for Redis (fast and good cross-language support)
var bf4 = FilterRedisBuilder.Build(
"localhost:6379",
"MyFilter",
10_000_000,
0.01,
HashMethod.XXHash64
);
```
## Performance Benchmarks
### Test Environment
```
BenchmarkDotNet=v0.13.5
OS: Windows 11 (10.0.22621.1778/22H2)
CPU: AMD Ryzen 7 5800X, 1 CPU, 16 logical cores, 8 physical cores
.NET SDK: 7.0.304
Runtime: .NET 7.0.7 (7.0.723.27404), X64 RyuJIT AVX2
```
### Performance Rankings (64-byte data)
| Rank | Algorithm | Mean Time | Relative Speed |
|------|-----------|-----------|----------------|
| 🥇 1 | XXHash3 | 33.14 ns | Baseline (Fastest) |
| 🥈 2 | XXHash128 | 36.01 ns | 1.09x |
| 🥉 3 | CRC64 | 38.83 ns | 1.17x |
| 4 | XXHash64 | 50.62 ns | 1.53x |
| 5 | Murmur3 | 70.98 ns | 2.14x |
| ... | ... | ... | ... |
| 28 | SHA512 | 1,368.20 ns | 41.28x (Slowest) |
### Complete Performance Data
Click to expand full benchmark results
#### 64-byte Data
| Algorithm | Mean Time | Error | StdDev | Allocated |
|-----------|-----------|-------|--------|-----------|
| XXHash3 | 33.14 ns | 0.295 ns | 0.276 ns | 80 B |
| XXHash128 | 36.01 ns | 0.673 ns | 0.749 ns | 80 B |
| CRC64 | 38.83 ns | 0.399 ns | 0.333 ns | 80 B |
| XXHash64 | 50.62 ns | 0.756 ns | 0.670 ns | 80 B |
| Murmur3 | 70.98 ns | 1.108 ns | 1.036 ns | 80 B |
| XXHash32 | 73.15 ns | 0.526 ns | 0.466 ns | 80 B |
| Murmur128BitsX64 | 80.15 ns | 0.783 ns | 0.653 ns | 120 B |
| Murmur128BitsX86 | 82.73 ns | 1.211 ns | 1.011 ns | 120 B |
| LCGWithFNV1 | 91.27 ns | 1.792 ns | 2.134 ns | 80 B |
| CRC32 | 145.63 ns | 1.528 ns | 1.429 ns | 328 B |
| Adler32 | 150.07 ns | 0.664 ns | 0.589 ns | 336 B |
| RNGWithFNV1 | 445.32 ns | 8.463 ns | 9.747 ns | 384 B |
| SHA256 | 922.30 ns | 4.478 ns | 3.739 ns | 496 B |
| SHA1 | 1,045.67 ns | 6.411 ns | 5.997 ns | 464 B |
| SHA384 | 1,173.67 ns | 5.050 ns | 3.942 ns | 456 B |
| SHA512 | 1,368.20 ns | 10.967 ns | 9.722 ns | 504 B |
#### 1 MB Data
| Algorithm | Mean Time |
|-----------|-----------|
| XXHash3 | 30,258.92 ns (~30 μs) |
| XXHash128 | 33,778.68 ns (~34 μs) |
| CRC64 | 56,321.74 ns (~56 μs) |
| XXHash64 | 100,570.79 ns (~101 μs) |
| Murmur128BitsX64 | 163,915.44 ns (~164 μs) |
| ... | ... |
| SHA1 | 3,381,425.73 ns (~3.4 ms) |
### Performance Recommendations
1. **General Scenarios**: Use `Murmur3` (default), balanced performance and quality
2. **Extreme Performance**: Use `XXHash3`, 2x faster than Murmur3
3. **Large Data**: Use `XXHash128` or `Murmur128BitsX64`, 128-bit output reduces collisions
4. **Avoid**: LCG series (poor quality), SHA series (too slow)
## Advanced Usage
### Serialization and Deserialization
```csharp
// Export Bloom filter state
var bf = FilterBuilder.Build(1_000_000, 0.01);
bf.Add("item1");
bf.Add("item2");
// Get internal state (for persistence)
var memory = (FilterMemory)bf;
var buckets = memory.Buckets; // BitArray[]
var bucketBytes = memory.BucketBytes; // byte[][]
// Restore Bloom filter from state
var options = new FilterMemoryOptions
{
Name = "RestoredFilter",
ExpectedElements = 1_000_000,
ErrorRate = 0.01,
Buckets = buckets // Or use BucketBytes
};
var restoredBf = FilterBuilder.Build(options);
Console.WriteLine(restoredBf.Contains("item1")); // True
```
### Batch Operations
```csharp
// Batch add
var items = Enumerable.Range(1, 10000)
.Select(i => Encoding.UTF8.GetBytes($"user:{i}"))
.ToArray();
var addResults = bf.Add(items);
Console.WriteLine($"Successfully added: {addResults.Count(r => r)} elements");
// Batch check
var checkResults = bf.Contains(items);
Console.WriteLine($"Exist: {checkResults.Count(r => r)} elements");
// Check if all elements exist
bool allExist = bf.All(items);
// Async batch operations
var asyncAddResults = await bf.AddAsync(items);
var asyncCheckResults = await bf.ContainsAsync(items);
bool asyncAllExist = await bf.AllAsync(items);
```
### Custom Hash Function
```csharp
using BloomFilter.HashAlgorithms;
// Implement custom hash algorithm
public class MyCustomHash : HashFunction
{
public override long ComputeHash(ReadOnlySpan data)
{
// Custom hash logic
long hash = 0;
foreach (var b in data)
{
hash = hash * 31 + b;
}
return hash;
}
}
// Use custom hash
var customHash = new MyCustomHash();
var bf = FilterBuilder.Build(1_000_000, 0.01, customHash);
```
### Calculate Actual False Positive Rate
```csharp
var bf = FilterBuilder.Build(100_000, 0.01);
// Add 50,000 elements
for (int i = 0; i < 50_000; i++)
{
bf.Add($"item:{i}");
}
// Calculate theoretical false positive rate
var filter = (Filter)bf;
double theoreticalErrorRate = Filter.BestP(
filter.Hashes,
filter.Capacity,
50_000
);
Console.WriteLine($"Theoretical error rate: {theoreticalErrorRate:P4}");
// Test actual false positive rate
int falsePositives = 0;
int testCount = 100_000;
for (int i = 50_000; i < 50_000 + testCount; i++)
{
if (bf.Contains($"item:{i}"))
{
falsePositives++;
}
}
double actualErrorRate = (double)falsePositives / testCount;
Console.WriteLine($"Actual error rate: {actualErrorRate:P4}");
Console.WriteLine($"False positives: {falsePositives} / {testCount}");
```
### Monitoring and Statistics
```csharp
public class BloomFilterMonitor
{
private readonly IBloomFilter _filter;
private long _addCount;
private long _hitCount;
private long _missCount;
public BloomFilterMonitor(IBloomFilter filter)
{
_filter = filter;
}
public bool Add(string item)
{
Interlocked.Increment(ref _addCount);
return _filter.Add(item);
}
public bool Contains(string item)
{
var result = _filter.Contains(item);
if (result)
Interlocked.Increment(ref _hitCount);
else
Interlocked.Increment(ref _missCount);
return result;
}
public void PrintStats()
{
Console.WriteLine($"Total adds: {_addCount}");
Console.WriteLine($"Hits: {_hitCount}");
Console.WriteLine($"Misses: {_missCount}");
Console.WriteLine($"Hit rate: {(double)_hitCount / (_hitCount + _missCount):P2}");
}
}
```
## API Reference
### IBloomFilter Interface
```csharp
public interface IBloomFilter : IDisposable
{
// Properties
string Name { get; }
// Synchronous methods
bool Add(ReadOnlySpan data);
IList Add(IEnumerable elements);
bool Contains(ReadOnlySpan element);
IList Contains(IEnumerable elements);
bool All(IEnumerable elements);
void Clear();
long[] ComputeHash(ReadOnlySpan data);
// Asynchronous methods
ValueTask AddAsync(ReadOnlyMemory data);
ValueTask> AddAsync(IEnumerable elements);
ValueTask ContainsAsync(ReadOnlyMemory element);
ValueTask> ContainsAsync(IEnumerable elements);
ValueTask AllAsync(IEnumerable elements);
ValueTask ClearAsync();
}
```
### Filter Base Class
```csharp
public abstract class Filter : IBloomFilter
{
// Properties
public string Name { get; }
public HashFunction Hash { get; }
public long Capacity { get; }
public int Hashes { get; }
public long ExpectedElements { get; }
public double ErrorRate { get; }
// Static methods (mathematical calculations)
public static long BestM(long n, double p);
public static int BestK(long n, long m);
public static long BestN(int k, long m);
public static double BestP(int k, long m, long insertedElements);
}
```
### FilterBuilder
```csharp
public static class FilterBuilder
{
// Using expected elements and error rate
public static IBloomFilter Build(long expectedElements, double errorRate);
public static IBloomFilter Build(long expectedElements, double errorRate, HashMethod method);
public static IBloomFilter Build(long expectedElements, double errorRate, HashFunction hash);
// Using capacity and number of hash functions
public static IBloomFilter Build(long capacity, int hashes, HashMethod method);
public static IBloomFilter Build(long capacity, int hashes, HashFunction hash);
// Using configuration object
public static IBloomFilter Build(FilterMemoryOptions options);
}
```
### FilterRedisBuilder
```csharp
public static class FilterRedisBuilder
{
public static IBloomFilter Build(
string redisHost,
string name,
long expectedElements,
double errorRate,
HashMethod method = HashMethod.Murmur3);
}
```
### Extension Methods
```csharp
// Service registration
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddBloomFilter(
this IServiceCollection services,
Action setupAction);
}
// Configuration extensions
public static class BloomFilterOptionsExtensions
{
public static BloomFilterOptions UseInMemory(
this BloomFilterOptions options,
Action setup = null);
public static BloomFilterOptions UseRedis(
this BloomFilterOptions options,
FilterRedisOptions setup);
public static BloomFilterOptions UseCSRedis(
this BloomFilterOptions options,
FilterCSRedisOptions setup);
public static BloomFilterOptions UseFreeRedis(
this BloomFilterOptions options,
FilterFreeRedisOptions setup);
public static BloomFilterOptions UseEasyCachingRedis(
this BloomFilterOptions options,
FilterEasyCachingRedisOptions setup);
}
```
## Frequently Asked Questions (FAQ)
### 1. What is the false positive rate of a Bloom filter?
The false positive rate is determined by the `errorRate` parameter you specify when creating the filter. For example:
```csharp
// 1% false positive rate
var bf = FilterBuilder.Build(1_000_000, 0.01);
// 0.1% false positive rate (more accurate, but uses more memory)
var bf2 = FilterBuilder.Build(1_000_000, 0.001);
```
**Note**: Lower error rates require more memory space.
### 2. How to choose expectedElements?
`expectedElements` should be set to the number of elements you expect to add. If the actual number exceeds this, the false positive rate will increase.
Recommendations:
- Estimate actual element count
- Add 20%-50% buffer
- Monitor actual false positive rate regularly
### 3. In-Memory vs Redis Mode - How to Choose?
| Scenario | Recommended Mode | Reason |
|----------|-----------------|---------|
| Single-instance application | In-Memory | Highest performance, no network overhead |
| Multi-instance application | Redis | Shared state, distributed support |
| Persistence required | Redis | Redis provides persistence |
| Temporary deduplication | In-Memory | Simple and fast |
| Cross-service sharing | Redis | Multi-language access support |
### 4. How to clear a Bloom filter?
```csharp
// Synchronous clear
bf.Clear();
// Asynchronous clear
await bf.ClearAsync();
```
**Note**: Clear operation deletes all data, use with caution!
### 5. How much memory does a Bloom filter use?
Memory usage depends on capacity (m):
```
Memory (bytes) = m / 8
```
Example calculation:
```csharp
// 10 million elements, 1% false positive rate
var bf = FilterBuilder.Build(10_000_000, 0.01);
var filter = (Filter)bf;
// Calculate memory usage
long bits = filter.Capacity;
long bytes = bits / 8;
double mb = bytes / (1024.0 * 1024.0);
Console.WriteLine($"Bit array size: {bits:N0} bits");
Console.WriteLine($"Memory usage: {bytes:N0} bytes ({mb:F2} MB)");
// Output: approximately 11.4 MB
```
### 6. Can elements be deleted?
**No**. Standard Bloom filters do not support deletion because:
- Multiple elements may map to the same bits
- Deleting one element may affect detection of other elements
If deletion is needed, consider:
- Counting Bloom Filter
- Cuckoo Filter
### 7. Is it thread-safe?
Yes, BloomFilter.NetCore is thread-safe:
```csharp
// Multi-threaded concurrent access
var bf = FilterBuilder.Build(10_000_000, 0.01);
Parallel.For(0, 1000, i =>
{
bf.Add($"item:{i}"); // Thread-safe
});
Parallel.For(0, 1000, i =>
{
var exists = bf.Contains($"item:{i}"); // Thread-safe
});
```
### 8. How to monitor Redis connections?
```csharp
// Use StackExchange.Redis connection monitoring
services.AddBloomFilter(setupAction =>
{
setupAction.UseRedis(new FilterRedisOptions
{
Name = "MyFilter",
RedisKey = "BF:Key",
Endpoints = new List { "localhost:6379" },
// Enable connection logging
AbortOnConnectFail = false,
ConnectTimeout = 5000,
ConnectRetry = 3
});
});
// Get Redis connection information
var bf = serviceProvider.GetService();
if (bf is FilterRedis redisFilter)
{
var connection = redisFilter.Connection;
Console.WriteLine($"Connection status: {connection.IsConnected}");
Console.WriteLine($"Endpoints: {string.Join(", ", connection.GetEndPoints())}");
}
```
## Contributing
We welcome community contributions!
### How to Contribute
1. Fork this repository
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
3. Commit your changes (`git commit -m 'Add amazing feature'`)
4. Push to the branch (`git push origin feature/amazing-feature`)
5. Create a Pull Request
### Development Guidelines
```bash
# Clone repository
git clone https://github.com/vla/BloomFilter.NetCore.git
cd BloomFilter.NetCore
# Restore dependencies
dotnet restore
# Build project
dotnet build
# Run tests
dotnet test
# Run benchmarks
cd test/BenchmarkTest
dotnet run -c Release
```
### Code Standards
- Follow C# coding conventions
- Add XML documentation comments
- Write unit tests
- Update relevant documentation
## Acknowledgments
Thanks to all developers who contributed to this project!
Special thanks to:
- .NET Foundation
- StackExchange.Redis team
- All dependency library authors
If this project helps you, please give us a ⭐️ Star!
================================================
FILE: README.zh-CN.md
================================================
# BloomFilter.NetCore
[](https://opensource.org/licenses/MIT)
[](https://dotnet.microsoft.com/)
一个高性能、功能完整的 .NET 布隆过滤器实现库，支持内存存储和多种 Redis 分布式后端。
## 目录
- [项目概述](#项目概述)
- [主要特性](#主要特性)
- [包和状态](#包和状态)
- [整体架构](#整体架构)
- [核心功能](#核心功能)
- [安装](#安装)
- [快速开始](#快速开始)
- [使用示例](#使用示例)
- [内存模式](#内存模式)
- [依赖注入配置](#依赖注入配置)
- [Redis 分布式模式](#redis-分布式模式)
- [哈希算法](#哈希算法)
- [性能基准测试](#性能基准测试)
- [高级用法](#高级用法)
- [API 参考](#api-参考)
- [贡献指南](#贡献指南)
- [许可证](#许可证)
## 项目概述
BloomFilter.NetCore 是一个企业级的布隆过滤器库，专为 .NET 生态系统设计。布隆过滤器是一种空间效率极高的概率型数据结构,用于测试一个元素是否属于一个集合。它的核心特点是:
- **空间高效**: 相比传统的 HashSet,占用空间极小
- **时间复杂度 O(1)**: 添加和查询操作都是常数时间
- **允许一定误报率**: 可能返回假阳性(false positive),但绝不会出现假阴性(false negative)
本项目提供了两大类实现:
1. **内存布隆过滤器 (FilterMemory)**: 基于 BitArray 的内存实现,适用于单进程场景
2. **分布式布隆过滤器 (FilterRedis 系列)**: 基于 Redis 的分布式实现,支持多应用程序并发访问
### 主要用途
- **缓存穿透防护**: 防止恶意查询不存在的数据导致缓存击穿
- **去重场景**: URL 去重、邮箱去重、用户 ID 去重等
- **推荐系统**: 判断用户是否已看过某个内容
- **爬虫系统**: 判断 URL 是否已被爬取
- **分布式系统**: 多服务实例间共享状态判断
- **大数据场景**: 海量数据的存在性判断
## 主要特性
### 🎯 灵活配置
- **参数完全可配置**: 位数组大小 (m)、哈希函数数量 (k)
- **自动参数计算**: 根据容忍的误报率 (p) 和预期元素数量 (n) 自动计算最优参数
- **20+ 种哈希算法**: 支持 CRC、MD5、SHA、Murmur、LCGs、xxHash 等或自定义算法
### ⚡ 高性能
- **快速生成**: 布隆过滤器的生成和操作都极快
- **优化实现**: 使用 Span、ReadOnlyMemory 等零拷贝技术
- **不安全代码优化**: 在性能关键路径使用 unsafe 代码块
- **拒绝采样**: 实现了拒绝采样和哈希链,考虑雪崩效应以提高哈希质量
### 🔒 并发安全
- **线程安全**: 使用 AsyncLock 机制确保多线程并发访问安全
- **异步支持**: 全面的 async/await 支持,所有操作都有异步版本
- **分布式锁**: Redis 实现支持跨应用程序的并发访问
### 🌐 多后端支持
- **StackExchange.Redis**: 官方推荐的 Redis 客户端
- **CSRedisCore**: 高性能的 Redis 客户端
- **FreeRedis**: 轻量级 Redis 客户端
- **EasyCaching**: 支持 EasyCaching 抽象层,可切换多种缓存提供程序
### 📦 现代 .NET 支持
- **多框架支持**: net462, netstandard2.0, net6.0, net7.0, net8.0, net9.0, net10.0
- **依赖注入**: 原生支持 Microsoft.Extensions.DependencyInjection
- **可空引用类型**: 启用可空引用类型,提高代码安全性
## 包和状态
| 包名 | NuGet | 说明 |
|------|-------|------|
|**BloomFilter.NetCore**|[](https://www.nuget.org/packages/BloomFilter.NetCore)| 核心包,提供内存布隆过滤器 |
|**BloomFilter.Redis.NetCore**|[](https://www.nuget.org/packages/BloomFilter.Redis.NetCore)| StackExchange.Redis 实现 |
|**BloomFilter.CSRedis.NetCore**|[](https://www.nuget.org/packages/BloomFilter.CSRedis.NetCore)| CSRedisCore 实现 |
|**BloomFilter.FreeRedis.NetCore**|[](https://www.nuget.org/packages/BloomFilter.FreeRedis.NetCore)| FreeRedis 实现 |
|**BloomFilter.EasyCaching.NetCore**|[](https://www.nuget.org/packages/BloomFilter.EasyCaching.NetCore)| EasyCaching 集成 |
## 整体架构
### 核心接口层
```
IBloomFilter (接口)
├── Add / AddAsync - 添加元素
├── Contains / ContainsAsync - 检查元素
├── All / AllAsync - 批量检查
├── Clear / ClearAsync - 清空过滤器
└── ComputeHash - 计算哈希值
```
### 实现层次结构
```
Filter (抽象基类)
├── FilterMemory (内存实现)
│ └── 使用 BitArray 存储
│
└── Redis 系列 (分布式实现)
├── FilterRedis (StackExchange.Redis)
├── FilterCSRedis (CSRedisCore)
├── FilterFreeRedis (FreeRedis)
└── FilterEasyCachingRedis (EasyCaching)
```
### 配置系统
```
BloomFilterOptions
├── FilterMemoryOptions - 内存模式配置
├── FilterRedisOptions - StackExchange.Redis 配置
├── FilterCSRedisOptions - CSRedisCore 配置
├── FilterFreeRedisOptions - FreeRedis 配置
└── FilterEasyCachingOptions - EasyCaching 配置
```
## 核心功能
### 数学模型
BloomFilter.NetCore 实现了完整的布隆过滤器数学模型:
#### 1. 最优位数组大小 (m)
给定预期元素数 `n` 和误报率 `p`,计算最优的位数组大小:
```
m = -(n * ln(p)) / (ln(2)^2)
```
#### 2. 最优哈希函数数量 (k)
给定元素数 `n` 和位数组大小 `m`,计算最优的哈希函数数量:
```
k = (m / n) * ln(2)
```
#### 3. 实际误报率 (p)
给定已插入元素数、哈希函数数量和位数组大小,计算实际误报率:
```
p = (1 - e^(-k*n/m))^k
```
这些计算由 `Filter` 基类提供的静态方法实现:
```csharp
// 计算最优位数组大小
long m = Filter.BestM(expectedElements, errorRate);
// 计算最优哈希函数数量
int k = Filter.BestK(expectedElements, capacity);
// 计算最优元素数量
long n = Filter.BestN(hashes, capacity);
// 计算实际误报率
double p = Filter.BestP(hashes, capacity, insertedElements);
```
### 存储机制
#### 内存存储
- **BitArray**: 使用 .NET 的 BitArray 作为底层存储
- **分桶策略**: 当容量超过 2GB (MaxInt = 2,147,483,640) 时,自动分成多个 BitArray
- **序列化支持**: 支持序列化/反序列化以持久化或传输过滤器状态
#### Redis 存储
- **SETBIT/GETBIT**: 使用 Redis 的位操作命令
- **分布式访问**: 多个应用实例可以并发访问同一个过滤器
- **持久化**: 利用 Redis 的持久化机制保证数据安全
### 并发控制
```csharp
// AsyncLock 确保线程安全
public class AsyncLock
{
private readonly SemaphoreSlim _semaphore = new(1, 1);
public async ValueTask LockAsync()
{
await _semaphore.WaitAsync();
return new Release(_semaphore);
}
}
```
## 安装
### 通过 NuGet 安装
**内存模式 (核心包):**
```bash
dotnet add package BloomFilter.NetCore
```
**Redis 分布式模式 (选择一个):**
```bash
# StackExchange.Redis
dotnet add package BloomFilter.Redis.NetCore
# CSRedisCore
dotnet add package BloomFilter.CSRedis.NetCore
# FreeRedis
dotnet add package BloomFilter.FreeRedis.NetCore
# EasyCaching
dotnet add package BloomFilter.EasyCaching.NetCore
```
## 快速开始
### 最简单的示例
```csharp
using BloomFilter;
// 创建一个布隆过滤器:预期 1000 万元素,1% 误报率
var bf = FilterBuilder.Build(10_000_000, 0.01);
// 添加元素
bf.Add("user:123");
bf.Add("user:456");
// 检查元素是否存在
Console.WriteLine(bf.Contains("user:123")); // True
Console.WriteLine(bf.Contains("user:789")); // False (可能极小概率为 True)
// 清空过滤器
bf.Clear();
```
### 异步操作
```csharp
// 异步添加
await bf.AddAsync(Encoding.UTF8.GetBytes("user:123"));
// 异步检查
bool exists = await bf.ContainsAsync(Encoding.UTF8.GetBytes("user:123"));
// 批量异步操作
var users = new[] {
Encoding.UTF8.GetBytes("user:1"),
Encoding.UTF8.GetBytes("user:2"),
Encoding.UTF8.GetBytes("user:3")
};
await bf.AddAsync(users);
var results = await bf.ContainsAsync(users);
```
### Fluent API（v3.0 新增）
v3.0 引入了现代化的流式 API，提供更好的可发现性和表达力：
```csharp
// 内存模式 Fluent API
var filter = FilterBuilder.Create()
.WithName("UserFilter")
.ExpectingElements(10_000_000)
.WithErrorRate(0.001)
.UsingHashMethod(HashMethod.XXHash3)
.BuildInMemory();
// Redis Fluent API (StackExchange.Redis)
var redisFilter = FilterRedisBuilder.Create()
.WithRedisConnection("localhost:6379")
.WithRedisKey("bloom:users")
.WithName("UserFilter")
.ExpectingElements(10_000_000)
.WithErrorRate(0.001)
.BuildRedis();
// CSRedis Fluent API
var csredisFilter = FilterCSRedisBuilder.Create()
.WithRedisClient(csredisClient)
.WithRedisKey("bloom:users")
.ExpectingElements(10_000_000)
.BuildCSRedis();
// FreeRedis Fluent API
var freeRedisFilter = FilterFreeRedisBuilder.Create()
.WithRedisClient(redisClient)
.WithRedisKey("bloom:users")
.ExpectingElements(10_000_000)
.BuildFreeRedis();
// EasyCaching Fluent API
var easyCachingFilter = FilterEasyCachingBuilder.Create()
.WithRedisCachingProvider(provider)
.WithRedisKey("bloom:users")
.ExpectingElements(10_000_000)
.BuildEasyCaching();
// 所有通用配置方法：
// - WithName(string) - 设置过滤器名称
// - ExpectingElements(long) - 设置预期元素数量
// - WithErrorRate(double) - 设置误报率 (0-1)
// - UsingHashMethod(HashMethod) - 使用预定义哈希算法
// - UsingCustomHash(HashFunction) - 使用自定义哈希函数
// - WithSerializer(IFilterMemorySerializer) - 设置自定义序列化器（仅内存模式）
```
**为什么使用 Fluent API？**
- 🔍 通过 IntelliSense 更好的可发现性
- 📖 代码更易读、更自文档化
- ⛓️ 可链式调用方法
- 🎯 类型安全的配置
- ✅ 向后兼容 - 旧的静态方法仍然可用！
## 使用示例
### 内存模式
#### 基本用法
```csharp
using BloomFilter;
public class UserService
{
// 静态共享的布隆过滤器
private static readonly IBloomFilter _bloomFilter =
FilterBuilder.Build(10_000_000, 0.01);
public void AddUser(string userId)
{
// 添加用户 ID
_bloomFilter.Add(userId);
}
public bool MayExistUser(string userId)
{
// 检查用户是否可能存在
return _bloomFilter.Contains(userId);
}
}
```
#### 自定义配置
```csharp
using BloomFilter;
// 方式 1: 指定哈希算法
var bf1 = FilterBuilder.Build(
expectedElements: 1_000_000,
errorRate: 0.001,
hashMethod: HashMethod.Murmur3
);
// 方式 2: 使用自定义哈希函数
var hashFunction = new Murmur128BitsX64();
var bf2 = FilterBuilder.Build(
expectedElements: 1_000_000,
errorRate: 0.001,
hashFunction: hashFunction
);
// 方式 3: 手动指定参数 (高级用法)
var bf3 = FilterBuilder.Build(
capacity: 9585059, // 位数组大小
hashes: 10, // 哈希函数数量
hashMethod: HashMethod.XXHash3
);
// 方式 4: 使用配置对象
var options = new FilterMemoryOptions
{
Name = "MyFilter",
ExpectedElements = 5_000_000,
ErrorRate = 0.01,
Method = HashMethod.Murmur3
};
var bf4 = FilterBuilder.Build(options);
```
### 依赖注入配置
#### ASP.NET Core 集成
```csharp
using BloomFilter;
using Microsoft.Extensions.DependencyInjection;
public class Startup
{
public void ConfigureServices(IServiceCollection services)
{
// 注册布隆过滤器服务
services.AddBloomFilter(setupAction =>
{
setupAction.UseInMemory(options =>
{
options.Name = "MainFilter";
options.ExpectedElements = 10_000_000;
options.ErrorRate = 0.01;
options.Method = HashMethod.Murmur3;
});
});
services.AddControllers();
}
}
// 在控制器或服务中使用
public class UserController : ControllerBase
{
private readonly IBloomFilter _bloomFilter;
public UserController(IBloomFilter bloomFilter)
{
_bloomFilter = bloomFilter;
}
[HttpPost("users/{userId}")]
public IActionResult CheckUser(string userId)
{
if (_bloomFilter.Contains(userId))
{
// 用户可能存在,继续查询数据库
return Ok("用户可能存在");
}
else
{
// 用户一定不存在,无需查询数据库
return NotFound("用户不存在");
}
}
}
```
#### 多个过滤器实例
```csharp
services.AddBloomFilter(setupAction =>
{
// 用户过滤器
setupAction.UseInMemory(options =>
{
options.Name = "UserFilter";
options.ExpectedElements = 10_000_000;
options.ErrorRate = 0.01;
});
// 邮箱过滤器
setupAction.UseInMemory(options =>
{
options.Name = "EmailFilter";
options.ExpectedElements = 5_000_000;
options.ErrorRate = 0.001;
});
});
// 使用工厂获取指定过滤器
public class MyService
{
private readonly IBloomFilter _userFilter;
private readonly IBloomFilter _emailFilter;
public MyService(IBloomFilterFactory factory)
{
_userFilter = factory.Get("UserFilter");
_emailFilter = factory.Get("EmailFilter");
}
}
```
### Redis 分布式模式
#### StackExchange.Redis
```csharp
using BloomFilter;
// 方式 1: 直接构建
var bf = FilterRedisBuilder.Build(
redisHost: "localhost:6379",
name: "DistributedFilter",
expectedElements: 5_000_000,
errorRate: 0.001
);
bf.Add("item:123");
Console.WriteLine(bf.Contains("item:123")); // True
// 方式 2: 依赖注入
services.AddBloomFilter(setupAction =>
{
setupAction.UseRedis(new FilterRedisOptions
{
Name = "UserFilter",
RedisKey = "BloomFilter:Users",
Endpoints = new List { "localhost:6379" },
Database = 0,
ExpectedElements = 10_000_000,
ErrorRate = 0.01,
Method = HashMethod.Murmur3
});
});
// 方式 3: 高级配置 (主从、哨兵、集群)
services.AddBloomFilter(setupAction =>
{
setupAction.UseRedis(new FilterRedisOptions
{
Name = "ProductFilter",
RedisKey = "BloomFilter:Products",
Endpoints = new List
{
"redis-master:6379",
"redis-slave1:6379",
"redis-slave2:6379"
},
Password = "your-redis-password",
Ssl = true,
ConnectTimeout = 5000,
SyncTimeout = 3000,
ExpectedElements = 20_000_000,
ErrorRate = 0.001
});
});
```
#### CSRedisCore
```csharp
services.AddBloomFilter(setupAction =>
{
setupAction.UseCSRedis(new FilterCSRedisOptions
{
Name = "OrderFilter",
RedisKey = "BloomFilter:Orders",
ConnectionStrings = new List
{
"localhost:6379,password=123456,defaultDatabase=0,poolsize=50,prefix=myapp:"
},
ExpectedElements = 5_000_000,
ErrorRate = 0.01
});
});
```
#### FreeRedis
```csharp
services.AddBloomFilter(setupAction =>
{
setupAction.UseFreeRedis(new FilterFreeRedisOptions
{
Name = "CartFilter",
RedisKey = "BloomFilter:Carts",
ConnectionStrings = new List { "localhost:6379,password=123456" },
ExpectedElements = 1_000_000,
ErrorRate = 0.01
});
});
```
#### EasyCaching 集成
EasyCaching 提供了统一的缓存抽象层,允许您轻松切换底层缓存实现:
```csharp
using EasyCaching.Core.Configurations;
using Microsoft.Extensions.DependencyInjection;
var services = new ServiceCollection();
// 1. 配置 EasyCaching
services.AddEasyCaching(options =>
{
// 配置 Redis 提供程序
options.UseRedis(config =>
{
config.DBConfig.Endpoints.Add(new ServerEndPoint("127.0.0.1", 6379));
config.DBConfig.Database = 0;
}, "redis-provider-1");
// 可以配置多个提供程序
options.UseRedis(config =>
{
config.DBConfig.Endpoints.Add(new ServerEndPoint("127.0.0.1", 6379));
config.DBConfig.Database = 1;
}, "redis-provider-2");
});
// 2. 配置 BloomFilter
services.AddBloomFilter(setupAction =>
{
// 使用第一个 Redis 提供程序
setupAction.UseEasyCachingRedis(new FilterEasyCachingRedisOptions
{
Name = "BF1",
RedisKey = "BloomFilter1",
ProviderName = "redis-provider-1",
ExpectedElements = 10_000_000,
ErrorRate = 0.01
});
// 使用第二个 Redis 提供程序
setupAction.UseEasyCachingRedis(new FilterEasyCachingRedisOptions
{
Name = "BF2",
RedisKey = "BloomFilter2",
ProviderName = "redis-provider-2",
ExpectedElements = 5_000_000,
ErrorRate = 0.001
});
});
var provider = services.BuildServiceProvider();
// 使用默认过滤器
var bf = provider.GetService();
bf.Add("value1");
// 使用指定名称的过滤器
var factory = provider.GetService();
var bf1 = factory.Get("BF1");
var bf2 = factory.Get("BF2");
bf1.Add("item1");
bf2.Add("item2");
```
### 实际应用场景
#### 1. 防止缓存穿透
```csharp
public class ProductService
{
private readonly IBloomFilter _bloomFilter;
private readonly ICache _cache;
private readonly IProductRepository _repository;
public ProductService(
IBloomFilter bloomFilter,
ICache cache,
IProductRepository repository)
{
_bloomFilter = bloomFilter;
_cache = cache;
_repository = repository;
}
public async Task GetProductAsync(string productId)
{
// 第一层防护: 布隆过滤器
if (!_bloomFilter.Contains(productId))
{
// 商品一定不存在,直接返回 null
return null;
}
// 第二层: 缓存
var cached = await _cache.GetAsync(productId);
if (cached != null)
{
return cached;
}
// 第三层: 数据库
var product = await _repository.GetByIdAsync(productId);
if (product != null)
{
await _cache.SetAsync(productId, product);
}
return product;
}
public async Task CreateProductAsync(Product product)
{
// 保存到数据库
await _repository.SaveAsync(product);
// 添加到布隆过滤器
_bloomFilter.Add(product.Id);
// 更新缓存
await _cache.SetAsync(product.Id, product);
}
}
```
#### 2. URL 去重 (爬虫系统)
```csharp
public class WebCrawler
{
private readonly IBloomFilter _visitedUrls;
private readonly Queue _urlQueue;
public WebCrawler(IBloomFilter bloomFilter)
{
_visitedUrls = bloomFilter;
_urlQueue = new Queue();
}
public async Task CrawlAsync(string startUrl)
{
_urlQueue.Enqueue(startUrl);
while (_urlQueue.Count > 0)
{
var url = _urlQueue.Dequeue();
// 检查是否已访问
if (_visitedUrls.Contains(url))
{
continue; // 跳过已访问的 URL
}
// 标记为已访问
_visitedUrls.Add(url);
// 抓取页面
var page = await DownloadPageAsync(url);
// 处理页面
await ProcessPageAsync(page);
// 提取新的 URL
var newUrls = ExtractUrls(page);
foreach (var newUrl in newUrls)
{
if (!_visitedUrls.Contains(newUrl))
{
_urlQueue.Enqueue(newUrl);
}
}
}
}
}
```
#### 3. 分布式去重 (多实例)
```csharp
// 配置分布式布隆过滤器
services.AddBloomFilter(setupAction =>
{
setupAction.UseRedis(new FilterRedisOptions
{
Name = "GlobalDeduplication",
RedisKey = "BF:Dedup",
Endpoints = new List { "redis-cluster:6379" },
ExpectedElements = 100_000_000,
ErrorRate = 0.0001
});
});
// 在多个服务实例中使用
public class MessageProcessor
{
private readonly IBloomFilter _bloomFilter;
public async Task ProcessMessageAsync(Message message)
{
// 所有实例共享同一个 Redis 布隆过滤器
if (await _bloomFilter.ContainsAsync(message.Id))
{
// 消息已被其他实例处理
return;
}
// 标记为已处理
await _bloomFilter.AddAsync(message.Id);
// 处理消息
await HandleMessageAsync(message);
}
}
```
## 哈希算法
BloomFilter.NetCore 支持 20+ 种哈希算法,可根据性能和准确性需求选择:
### 算法分类
| 类别 | 算法 | 特点 | 适用场景 |
|------|------|------|----------|
| **LCG 类** | LCGWithFNV1
LCGWithFNV1a
LCGModifiedFNV1 | 极快,但质量较低 | 性能要求极高,可容忍高误报率 |
| **RNG 类** | RNGWithFNV1
RNGWithFNV1a
RNGModifiedFNV1 | 质量高,但较慢 | 对准确性要求高的场景 |
| **校验和** | CRC32
CRC64
Adler32 | 平衡性能和质量 | 通用场景 |
| **Murmur 系列** | Murmur3
Murmur32BitsX86
Murmur128BitsX64
Murmur128BitsX86 | **推荐**,性能好,质量高 | 生产环境推荐 |
| **加密哈希** | SHA1
SHA256
SHA384
SHA512 | 质量最高,但最慢 | 安全性要求极高的场景 |
| **XXHash 系列** | XXHash32
XXHash64
XXHash3
XXHash128 | **最快**,质量优秀 | 高性能场景首选 |
### 选择建议
```csharp
// 推荐: 生产环境默认选择 Murmur3 (平衡性能和质量)
var bf1 = FilterBuilder.Build(10_000_000, 0.01, HashMethod.Murmur3);
// 高性能: 对性能要求极高,选择 XXHash3
var bf2 = FilterBuilder.Build(10_000_000, 0.01, HashMethod.XXHash3);
// 高精度: 对误报率要求极低,选择 SHA256 + 更低的 errorRate
var bf3 = FilterBuilder.Build(10_000_000, 0.0001, HashMethod.SHA256);
// 分布式: Redis 场景推荐 XXHash64 (速度快且跨语言支持好)
var bf4 = FilterRedisBuilder.Build(
"localhost:6379",
"MyFilter",
10_000_000,
0.01,
HashMethod.XXHash64
);
```
## 性能基准测试
### 测试环境
```
BenchmarkDotNet=v0.13.5
OS: Windows 11 (10.0.22621.1778/22H2)
CPU: AMD Ryzen 7 5800X, 1 CPU, 16 logical cores, 8 physical cores
.NET SDK: 7.0.304
Runtime: .NET 7.0.7 (7.0.723.27404), X64 RyuJIT AVX2
```
### 性能排名 (64 字节数据)
| 排名 | 算法 | 平均时间 | 相对速度 |
|------|------|----------|----------|
| 🥇 1 | XXHash3 | 33.14 ns | 基准 (最快) |
| 🥈 2 | XXHash128 | 36.01 ns | 1.09x |
| 🥉 3 | CRC64 | 38.83 ns | 1.17x |
| 4 | XXHash64 | 50.62 ns | 1.53x |
| 5 | Murmur3 | 70.98 ns | 2.14x |
| ... | ... | ... | ... |
| 28 | SHA512 | 1,368.20 ns | 41.28x (最慢) |
### 完整性能数据
点击展开完整基准测试结果
#### 64 字节数据
| 算法 | 平均时间 | 误差 | 标准差 | 内存分配 |
|------|---------|------|--------|---------|
| XXHash3 | 33.14 ns | 0.295 ns | 0.276 ns | 80 B |
| XXHash128 | 36.01 ns | 0.673 ns | 0.749 ns | 80 B |
| CRC64 | 38.83 ns | 0.399 ns | 0.333 ns | 80 B |
| XXHash64 | 50.62 ns | 0.756 ns | 0.670 ns | 80 B |
| Murmur3 | 70.98 ns | 1.108 ns | 1.036 ns | 80 B |
| XXHash32 | 73.15 ns | 0.526 ns | 0.466 ns | 80 B |
| Murmur128BitsX64 | 80.15 ns | 0.783 ns | 0.653 ns | 120 B |
| Murmur128BitsX86 | 82.73 ns | 1.211 ns | 1.011 ns | 120 B |
| LCGWithFNV1 | 91.27 ns | 1.792 ns | 2.134 ns | 80 B |
| CRC32 | 145.63 ns | 1.528 ns | 1.429 ns | 328 B |
| Adler32 | 150.07 ns | 0.664 ns | 0.589 ns | 336 B |
| RNGWithFNV1 | 445.32 ns | 8.463 ns | 9.747 ns | 384 B |
| SHA256 | 922.30 ns | 4.478 ns | 3.739 ns | 496 B |
| SHA1 | 1,045.67 ns | 6.411 ns | 5.997 ns | 464 B |
| SHA384 | 1,173.67 ns | 5.050 ns | 3.942 ns | 456 B |
| SHA512 | 1,368.20 ns | 10.967 ns | 9.722 ns | 504 B |
#### 1 MB 数据
| 算法 | 平均时间 |
|------|---------|
| XXHash3 | 30,258.92 ns (~30 μs) |
| XXHash128 | 33,778.68 ns (~34 μs) |
| CRC64 | 56,321.74 ns (~56 μs) |
| XXHash64 | 100,570.79 ns (~101 μs) |
| Murmur128BitsX64 | 163,915.44 ns (~164 μs) |
| ... | ... |
| SHA1 | 3,381,425.73 ns (~3.4 ms) |
### 性能建议
1. **通用场景**: 使用 `Murmur3` (默认),性能和质量平衡
2. **极限性能**: 使用 `XXHash3`,比 Murmur3 快 2 倍
3. **大数据**: 使用 `XXHash128` 或 `Murmur128BitsX64`,128 位输出减少碰撞
4. **避免使用**: LCG 系列 (质量差)、SHA 系列 (太慢)
## 高级用法
### 序列化和反序列化
```csharp
// 导出布隆过滤器状态
var bf = FilterBuilder.Build(1_000_000, 0.01);
bf.Add("item1");
bf.Add("item2");
// 获取内部状态 (用于持久化)
var memory = (FilterMemory)bf;
var buckets = memory.Buckets; // BitArray[]
var bucketBytes = memory.BucketBytes; // byte[][]
// 从状态恢复布隆过滤器
var options = new FilterMemoryOptions
{
Name = "RestoredFilter",
ExpectedElements = 1_000_000,
ErrorRate = 0.01,
Buckets = buckets // 或使用 BucketBytes
};
var restoredBf = FilterBuilder.Build(options);
Console.WriteLine(restoredBf.Contains("item1")); // True
```
### 批量操作
```csharp
// 批量添加
var items = Enumerable.Range(1, 10000)
.Select(i => Encoding.UTF8.GetBytes($"user:{i}"))
.ToArray();
var addResults = bf.Add(items);
Console.WriteLine($"成功添加: {addResults.Count(r => r)} 个元素");
// 批量检查
var checkResults = bf.Contains(items);
Console.WriteLine($"存在: {checkResults.Count(r => r)} 个元素");
// 检查所有元素是否都存在
bool allExist = bf.All(items);
// 异步批量操作
var asyncAddResults = await bf.AddAsync(items);
var asyncCheckResults = await bf.ContainsAsync(items);
bool asyncAllExist = await bf.AllAsync(items);
```
### 自定义哈希函数
```csharp
using BloomFilter.HashAlgorithms;
// 实现自定义哈希算法
public class MyCustomHash : HashFunction
{
public override long ComputeHash(ReadOnlySpan data)
{
// 自定义哈希逻辑
long hash = 0;
foreach (var b in data)
{
hash = hash * 31 + b;
}
return hash;
}
}
// 使用自定义哈希
var customHash = new MyCustomHash();
var bf = FilterBuilder.Build(1_000_000, 0.01, customHash);
```
### 计算实际误报率
```csharp
var bf = FilterBuilder.Build(100_000, 0.01);
// 添加 50,000 个元素
for (int i = 0; i < 50_000; i++)
{
bf.Add($"item:{i}");
}
// 计算理论误报率
var filter = (Filter)bf;
double theoreticalErrorRate = Filter.BestP(
filter.Hashes,
filter.Capacity,
50_000
);
Console.WriteLine($"理论误报率: {theoreticalErrorRate:P4}");
// 测试实际误报率
int falsePositives = 0;
int testCount = 100_000;
for (int i = 50_000; i < 50_000 + testCount; i++)
{
if (bf.Contains($"item:{i}"))
{
falsePositives++;
}
}
double actualErrorRate = (double)falsePositives / testCount;
Console.WriteLine($"实际误报率: {actualErrorRate:P4}");
Console.WriteLine($"误报数量: {falsePositives} / {testCount}");
```
### 监控和统计
```csharp
public class BloomFilterMonitor
{
private readonly IBloomFilter _filter;
private long _addCount;
private long _hitCount;
private long _missCount;
public BloomFilterMonitor(IBloomFilter filter)
{
_filter = filter;
}
public bool Add(string item)
{
Interlocked.Increment(ref _addCount);
return _filter.Add(item);
}
public bool Contains(string item)
{
var result = _filter.Contains(item);
if (result)
Interlocked.Increment(ref _hitCount);
else
Interlocked.Increment(ref _missCount);
return result;
}
public void PrintStats()
{
Console.WriteLine($"总添加: {_addCount}");
Console.WriteLine($"命中: {_hitCount}");
Console.WriteLine($"未命中: {_missCount}");
Console.WriteLine($"命中率: {(double)_hitCount / (_hitCount + _missCount):P2}");
}
}
```
## API 参考
### IBloomFilter 接口
```csharp
public interface IBloomFilter : IDisposable
{
// 属性
string Name { get; }
// 同步方法
bool Add(ReadOnlySpan data);
IList Add(IEnumerable elements);
bool Contains(ReadOnlySpan element);
IList Contains(IEnumerable elements);
bool All(IEnumerable elements);
void Clear();
long[] ComputeHash(ReadOnlySpan data);
// 异步方法
ValueTask AddAsync(ReadOnlyMemory data);
ValueTask> AddAsync(IEnumerable elements);
ValueTask ContainsAsync(ReadOnlyMemory element);
ValueTask> ContainsAsync(IEnumerable elements);
ValueTask AllAsync(IEnumerable elements);
ValueTask ClearAsync();
}
```
### Filter 基类
```csharp
public abstract class Filter : IBloomFilter
{
// 属性
public string Name { get; }
public HashFunction Hash { get; }
public long Capacity { get; }
public int Hashes { get; }
public long ExpectedElements { get; }
public double ErrorRate { get; }
// 静态方法 (数学计算)
public static long BestM(long n, double p);
public static int BestK(long n, long m);
public static long BestN(int k, long m);
public static double BestP(int k, long m, long insertedElements);
}
```
### FilterBuilder
```csharp
public static class FilterBuilder
{
// 使用预期元素数和误报率
public static IBloomFilter Build(long expectedElements, double errorRate);
public static IBloomFilter Build(long expectedElements, double errorRate, HashMethod method);
public static IBloomFilter Build(long expectedElements, double errorRate, HashFunction hash);
// 使用容量和哈希函数数量
public static IBloomFilter Build(long capacity, int hashes, HashMethod method);
public static IBloomFilter Build(long capacity, int hashes, HashFunction hash);
// 使用配置对象
public static IBloomFilter Build(FilterMemoryOptions options);
}
```
### FilterRedisBuilder
```csharp
public static class FilterRedisBuilder
{
public static IBloomFilter Build(
string redisHost,
string name,
long expectedElements,
double errorRate,
HashMethod method = HashMethod.Murmur3);
}
```
### 扩展方法
```csharp
// 服务注册
public static class ServiceCollectionExtensions
{
public static IServiceCollection AddBloomFilter(
this IServiceCollection services,
Action setupAction);
}
// 配置扩展
public static class BloomFilterOptionsExtensions
{
public static BloomFilterOptions UseInMemory(
this BloomFilterOptions options,
Action setup = null);
public static BloomFilterOptions UseRedis(
this BloomFilterOptions options,
FilterRedisOptions setup);
public static BloomFilterOptions UseCSRedis(
this BloomFilterOptions options,
FilterCSRedisOptions setup);
public static BloomFilterOptions UseFreeRedis(
this BloomFilterOptions options,
FilterFreeRedisOptions setup);
public static BloomFilterOptions UseEasyCachingRedis(
this BloomFilterOptions options,
FilterEasyCachingRedisOptions setup);
}
```
## 常见问题 (FAQ)
### 1. 布隆过滤器的误报率是多少?
误报率由您在创建时指定的 `errorRate` 参数决定。例如:
```csharp
// 1% 误报率
var bf = FilterBuilder.Build(1_000_000, 0.01);
// 0.1% 误报率 (更准确,但占用更多内存)
var bf2 = FilterBuilder.Build(1_000_000, 0.001);
```
**注意**: 误报率越低,需要的内存空间越大。
### 2. 如何选择 expectedElements?
`expectedElements` 应设置为您预期要添加的元素数量。如果实际添加的元素超过这个数量,误报率会增加。
建议:
- 估算实际元素数量
- 留出 20%-50% 的冗余
- 定期监控实际误报率
### 3. 内存模式 vs Redis 模式如何选择?
| 场景 | 推荐模式 | 原因 |
|------|---------|------|
| 单实例应用 | 内存模式 | 性能最高,无网络开销 |
| 多实例应用 | Redis 模式 | 共享状态,支持分布式 |
| 需要持久化 | Redis 模式 | Redis 提供持久化 |
| 临时去重 | 内存模式 | 简单快速 |
| 跨服务共享 | Redis 模式 | 支持多语言访问 |
### 4. 如何清空布隆过滤器?
```csharp
// 同步清空
bf.Clear();
// 异步清空
await bf.ClearAsync();
```
**注意**: 清空操作会删除所有数据,谨慎使用!
### 5. 布隆过滤器占用多少内存?
内存占用取决于容量 (m):
```
内存 (字节) = m / 8
```
示例计算:
```csharp
// 1000 万元素, 1% 误报率
var bf = FilterBuilder.Build(10_000_000, 0.01);
var filter = (Filter)bf;
// 计算内存占用
long bits = filter.Capacity;
long bytes = bits / 8;
double mb = bytes / (1024.0 * 1024.0);
Console.WriteLine($"位数组大小: {bits:N0} bits");
Console.WriteLine($"内存占用: {bytes:N0} bytes ({mb:F2} MB)");
// 输出: 约 11.4 MB
```
### 6. 可以删除元素吗?
**不可以**。标准布隆过滤器不支持删除操作,因为:
- 多个元素可能映射到相同的位
- 删除一个元素可能影响其他元素的检测
如果需要删除功能,考虑使用:
- Counting Bloom Filter (计数布隆过滤器)
- Cuckoo Filter (布谷鸟过滤器)
### 7. 线程安全吗?
是的,BloomFilter.NetCore 是线程安全的:
```csharp
// 多线程并发访问
var bf = FilterBuilder.Build(10_000_000, 0.01);
Parallel.For(0, 1000, i =>
{
bf.Add($"item:{i}"); // 线程安全
});
Parallel.For(0, 1000, i =>
{
var exists = bf.Contains($"item:{i}"); // 线程安全
});
```
### 8. 如何监控 Redis 连接?
```csharp
// 使用 StackExchange.Redis 的连接监控
services.AddBloomFilter(setupAction =>
{
setupAction.UseRedis(new FilterRedisOptions
{
Name = "MyFilter",
RedisKey = "BF:Key",
Endpoints = new List { "localhost:6379" },
// 启用连接日志
AbortOnConnectFail = false,
ConnectTimeout = 5000,
ConnectRetry = 3
});
});
// 获取 Redis 连接信息
var bf = serviceProvider.GetService();
if (bf is FilterRedis redisFilter)
{
var connection = redisFilter.Connection;
Console.WriteLine($"连接状态: {connection.IsConnected}");
Console.WriteLine($"端点: {string.Join(", ", connection.GetEndPoints())}");
}
```
## 贡献指南
我们欢迎社区贡献!
### 如何贡献
1. Fork 本仓库
2. 创建特性分支 (`git checkout -b feature/amazing-feature`)
3. 提交更改 (`git commit -m 'Add amazing feature'`)
4. 推送到分支 (`git push origin feature/amazing-feature`)
5. 创建 Pull Request
### 开发指南
```bash
# 克隆仓库
git clone https://github.com/vla/BloomFilter.NetCore.git
cd BloomFilter.NetCore
# 还原依赖
dotnet restore
# 构建项目
dotnet build
# 运行测试
dotnet test
# 运行基准测试
cd test/BenchmarkTest
dotnet run -c Release
```
### 代码规范
- 遵循 C# 编码规范
- 添加 XML 文档注释
- 编写单元测试
- 更新相关文档
## 致谢
感谢所有为本项目做出贡献的开发者!
特别感谢:
- .NET Foundation
- StackExchange.Redis 团队
- 所有依赖库的作者
如果这个项目对您有帮助,请给我们一个 ⭐️ Star!
================================================
FILE: VERSION
================================================
3.0.0
================================================
FILE: build.cmd
================================================
set artifacts=%~dp0artifacts
if exist %artifacts% rd /q /s %artifacts%
set /p ver=Specifies that null is allowed as an input even if the corresponding type disallows it.
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)]
internal sealed class AllowNullAttribute : Attribute { }
/// Specifies that null is disallowed as an input even if the corresponding type allows it.
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)]
internal sealed class DisallowNullAttribute : Attribute { }
/// Specifies that an output may be null even if the corresponding type disallows it.
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)]
internal sealed class MaybeNullAttribute : Attribute { }
/// Specifies that an output will not be null even if the corresponding type allows it.
[AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)]
internal sealed class NotNullAttribute : Attribute { }
/// Specifies that when a method returns , the parameter may be null even if the corresponding type disallows it.
[AttributeUsage(AttributeTargets.Parameter, Inherited = false)]
internal sealed class MaybeNullWhenAttribute : Attribute
{
/// Initializes the attribute with the specified return value condition.
///
/// The return value condition. If the method returns this value, the associated parameter may be null.
///
public MaybeNullWhenAttribute(bool returnValue) => ReturnValue = returnValue;
/// Gets the return value condition.
public bool ReturnValue { get; }
}
/// Specifies that when a method returns , the parameter will not be null even if the corresponding type allows it.
[AttributeUsage(AttributeTargets.Parameter, Inherited = false)]
internal sealed class NotNullWhenAttribute : Attribute
{
/// Initializes the attribute with the specified return value condition.
///
/// The return value condition. If the method returns this value, the associated parameter will not be null.
///
public NotNullWhenAttribute(bool returnValue) => ReturnValue = returnValue;
/// Gets the return value condition.
public bool ReturnValue { get; }
}
/// Specifies that the output will be non-null if the named parameter is non-null.
[AttributeUsage(AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, AllowMultiple = true, Inherited = false)]
internal sealed class NotNullIfNotNullAttribute : Attribute
{
/// Initializes the attribute with the associated parameter name.
///
/// The associated parameter name. The output will be non-null if the argument to the parameter specified is non-null.
///
public NotNullIfNotNullAttribute(string parameterName) => ParameterName = parameterName;
/// Gets the associated parameter name.
public string ParameterName { get; }
}
/// Applied to a method that will never return under any circumstance.
[AttributeUsage(AttributeTargets.Method, Inherited = false)]
internal sealed class DoesNotReturnAttribute : Attribute { }
/// Specifies that the method will not return if the associated Boolean parameter is passed the specified value.
[AttributeUsage(AttributeTargets.Parameter, Inherited = false)]
internal sealed class DoesNotReturnIfAttribute : Attribute
{
/// Initializes the attribute with the specified parameter value.
///
/// The condition parameter value. Code after the method will be considered unreachable by diagnostics if the argument to
/// the associated parameter matches this value.
///
public DoesNotReturnIfAttribute(bool parameterValue) => ParameterValue = parameterValue;
/// Gets the condition parameter value.
public bool ParameterValue { get; }
}
/// Specifies that the method or property will ensure that the listed field and property members have not-null values.
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)]
internal sealed class MemberNotNullAttribute : Attribute
{
/// Initializes the attribute with a field or property member.
///
/// The field or property member that is promised to be not-null.
///
public MemberNotNullAttribute(string member) => Members = new[] { member };
/// Initializes the attribute with the list of field and property members.
///
/// The list of field and property members that are promised to be not-null.
///
public MemberNotNullAttribute(params string[] members) => Members = members;
/// Gets field or property member names.
public string[] Members { get; }
}
/// Specifies that the method or property will ensure that the listed field and property members have not-null values when returning with the specified return value condition.
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)]
internal sealed class MemberNotNullWhenAttribute : Attribute
{
/// Initializes the attribute with the specified return value condition and a field or property member.
///
/// The return value condition. If the method returns this value, the associated parameter will not be null.
///
///
/// The field or property member that is promised to be not-null.
///
public MemberNotNullWhenAttribute(bool returnValue, string member)
{
ReturnValue = returnValue;
Members = new[] { member };
}
/// Initializes the attribute with the specified return value condition and list of field and property members.
///
/// The return value condition. If the method returns this value, the associated parameter will not be null.
///
///
/// The list of field and property members that are promised to be not-null.
///
public MemberNotNullWhenAttribute(bool returnValue, params string[] members)
{
ReturnValue = returnValue;
Members = members;
}
/// Gets the return value condition.
public bool ReturnValue { get; }
/// Gets field or property member names.
public string[] Members { get; }
}
#endif
#if NETSTANDARD2_1
namespace System.Diagnostics.CodeAnalysis;
/// Specifies that the method or property will ensure that the listed field and property members have not-null values.
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)]
internal sealed class MemberNotNullAttribute : Attribute
{
/// Initializes the attribute with a field or property member.
///
/// The field or property member that is promised to be not-null.
///
public MemberNotNullAttribute(string member) => Members = new[] { member };
/// Initializes the attribute with the list of field and property members.
///
/// The list of field and property members that are promised to be not-null.
///
public MemberNotNullAttribute(params string[] members) => Members = members;
/// Gets field or property member names.
public string[] Members { get; }
}
/// Specifies that the method or property will ensure that the listed field and property members have not-null values when returning with the specified return value condition.
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)]
internal sealed class MemberNotNullWhenAttribute : Attribute
{
/// Initializes the attribute with the specified return value condition and a field or property member.
///
/// The return value condition. If the method returns this value, the associated parameter will not be null.
///
///
/// The field or property member that is promised to be not-null.
///
public MemberNotNullWhenAttribute(bool returnValue, string member)
{
ReturnValue = returnValue;
Members = new[] { member };
}
/// Initializes the attribute with the specified return value condition and list of field and property members.
///
/// The return value condition. If the method returns this value, the associated parameter will not be null.
///
///
/// The list of field and property members that are promised to be not-null.
///
public MemberNotNullWhenAttribute(bool returnValue, params string[] members)
{
ReturnValue = returnValue;
Members = members;
}
/// Gets the return value condition.
public bool ReturnValue { get; }
/// Gets field or property member names.
public string[] Members { get; }
}
#endif
================================================
FILE: src/BloomFilter/AsyncLock.cs
================================================
using System;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace BloomFilter;
///
/// AsyncLock
///
internal readonly struct AsyncLock : IDisposable
{
private readonly SemaphoreSlim _semaphore;
private readonly Releaser _releaser;
private readonly Task _releaserTask;
public int MaxCount { get; }
public AsyncLock() : this(1)
{
}
public AsyncLock(int maxCount = 1)
{
MaxCount = maxCount;
_semaphore = new SemaphoreSlim(maxCount);
_releaser = new Releaser(_semaphore);
_releaserTask = Task.FromResult(_releaser);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Releaser Acquire()
{
_semaphore.Wait();
return _releaser;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Task AcquireAsync(bool continueOnCapturedContext = false)
{
var acquireAsync = _semaphore.WaitAsync();
return Return(acquireAsync, continueOnCapturedContext);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Dispose() => _semaphore.Dispose();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private Task Return(Task acquireAsync, bool continueOnCapturedContext)
{
return acquireAsync.Status == TaskStatus.RanToCompletion
? _releaserTask
: WaitForAcquireAsync(acquireAsync, continueOnCapturedContext);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private async Task WaitForAcquireAsync(Task acquireAsync, bool continueOnCapturedContext)
{
await acquireAsync.ConfigureAwait(continueOnCapturedContext);
return _releaser;
}
public readonly struct Releaser(SemaphoreSlim? semaphore) : IDisposable
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Dispose() => semaphore?.Release();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int GetRemainingCount()
{
return MaxCount - _semaphore.CurrentCount;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int GetCurrentCount()
{
return _semaphore.CurrentCount;
}
}
================================================
FILE: src/BloomFilter/BinaryHelper.cs
================================================
using System;
using System.Collections;
using System.Numerics;
using System.Runtime.CompilerServices;
#if NET6_0_OR_GREATER
using System.Buffers.Binary;
#endif
namespace BloomFilter;
internal class BinaryHelper
{
public static int BitToIntOne(BitArray bit, int from, int to)
{
const int size = 32;
int len = to - from;
int bitCount = bit.Count;
int result = 0;
for (int i = 0; i < len && i < bitCount && i < size; i++)
{
result = bit[i + from] ? result + (1 << i) : result;
}
return result;
}
///
/// Perform rejection sampling on a 32-bit,
/// https://en.wikipedia.org/wiki/Rejection_sampling
///
/// The random.
/// integer output range.
///
public static long Rejection(long random, long m)
{
var intMax = (long)uint.MaxValue;
random = Math.Abs(random);
if (random > (intMax - intMax % m) || random == uint.MinValue)
return -1;
return random % m;
}
public static uint NumberOfTrailingZeros(uint i)
{
// HD, Figure 5-14
uint y;
if (i == 0) return 32;
uint n = 31;
y = i << 16; if (y != 0) { n -= 16; i = y; }
y = i << 8; if (y != 0) { n -= 8; i = y; }
y = i << 4; if (y != 0) { n -= 4; i = y; }
y = i << 2; if (y != 0) { n -= 2; i = y; }
return n - ((i << 1) >> 31);
}
public static uint NumberOfLeadingZeros(uint i)
{
// HD, Figure 5-6
if (i == 0)
return 32;
uint n = 1;
if (i >> 16 == 0) { n += 16; i <<= 16; }
if (i >> 24 == 0) { n += 8; i <<= 8; }
if (i >> 28 == 0) { n += 4; i <<= 4; }
if (i >> 30 == 0) { n += 2; i <<= 2; }
n -= i >> 31;
return n;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int RotateLeft(int i, int distance)
{
#if NET6_0_OR_GREATER
return (int)BitOperations.RotateLeft((uint)i, distance);
#else
return (i << distance) | (int)((uint)i >> -distance);
#endif
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint RotateLeft(uint i, int distance)
{
#if NET6_0_OR_GREATER
return BitOperations.RotateLeft(i, distance);
#else
return (i << distance) | (i >> -distance);
#endif
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int RotateRight(int i, int distance)
{
#if NET6_0_OR_GREATER
return (int)BitOperations.RotateRight((uint)i, distance);
#else
return (int)((uint)i >> distance) | (i << -distance);
#endif
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint RotateRight(uint i, int distance)
{
#if NET6_0_OR_GREATER
return BitOperations.RotateRight(i, distance);
#else
return (i >> distance) | (i << -distance);
#endif
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static long RotateLeft(long i, int distance)
{
#if NET6_0_OR_GREATER
return (long)BitOperations.RotateLeft((ulong)i, distance);
#else
return (i << distance) | (long)((ulong)i >> -distance);
#endif
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong RotateLeft(ulong i, int distance)
{
#if NET6_0_OR_GREATER
return BitOperations.RotateLeft(i, distance);
#else
return (i << distance) | (i >> -distance);
#endif
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static long RotateRight(long i, int distance)
{
#if NET6_0_OR_GREATER
return (long)BitOperations.RotateRight((ulong)i, distance);
#else
return (long)((ulong)i >> distance) | (i << -distance);
#endif
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong RotateRight(ulong i, int distance)
{
#if NET6_0_OR_GREATER
return BitOperations.RotateRight(i, distance);
#else
return (i >> distance) | (i << -distance);
#endif
}
public static int RightMove(int value, int pos)
{
if (pos != 0)
{
var mask = 0x7fffffff;
value >>= 1;
value &= mask;
value >>= pos - 1;
}
return value;
}
public static long RightMove(long value, int pos)
{
if (pos != 0)
{
var mask = 0x7fffffff;
value >>= 1;
value &= mask;
value >>= pos - 1;
}
return value;
}
public static uint RightMove(uint value, int pos)
{
if (pos != 0)
{
uint mask = 0x7fffffff;
value >>= 1;
value &= mask;
value >>= pos - 1;
}
return value;
}
}
================================================
FILE: src/BloomFilter/BloomFilter.csproj
================================================
BloomFilter
BloomFilter
BloomFilter.NetCore
net462;netstandard2.0;net6.0;net7.0;net8.0;net9.0;net10.0
true
true
enable
true
$(NoWarn);CS1591
================================================
FILE: src/BloomFilter/BloomFilterConstValue.cs
================================================
namespace BloomFilter;
public class BloomFilterConstValue
{
///
/// The default name of the in-memory.
///
public const string DefaultInMemoryName = "DefaultInMemory";
///
/// The default name of the redis.
///
public const string DefaultRedisName = "DefaultRedis";
}
================================================
FILE: src/BloomFilter/BloomFilterExtensions.cs
================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BloomFilter;
///
/// BloomFilterExtensions
///
public static class BloomFilterExtensions
{
#region Byte
public static bool Add(this IBloomFilter bloomFilter, byte data) => bloomFilter.Add(new[] { data });
public static ValueTask AddAsync(this IBloomFilter bloomFilter, byte data) => bloomFilter.AddAsync(new[] { data });
public static bool Contains(this IBloomFilter bloomFilter, byte data) => bloomFilter.Contains(new[] { data });
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, byte data) => bloomFilter.ContainsAsync(new[] { data });
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => new byte[] { data }));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => new byte[] { data }));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => new byte[] { data }));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => new byte[] { data }));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => new byte[] { data }));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => new byte[] { data }));
#endregion Byte
#region String
public static bool Add(this IBloomFilter bloomFilter, string data) => bloomFilter.Add(Encoding.UTF8.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, string data) => bloomFilter.AddAsync(Encoding.UTF8.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, string data) => bloomFilter.Contains(Encoding.UTF8.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, string data) => bloomFilter.ContainsAsync(Encoding.UTF8.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => Encoding.UTF8.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => Encoding.UTF8.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => Encoding.UTF8.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => Encoding.UTF8.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => Encoding.UTF8.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => Encoding.UTF8.GetBytes(data)));
#endregion String
#region ReadOnlySpan
public static bool Add(this IBloomFilter bloomFilter, ReadOnlySpan data) => bloomFilter.Add(data.ToUtf8().Span);
public static ValueTask AddAsync(this IBloomFilter bloomFilter, ReadOnlyMemory data) => bloomFilter.AddAsync(data.Span.ToUtf8());
public static bool Contains(this IBloomFilter bloomFilter, ReadOnlySpan data) => bloomFilter.Contains(data.ToUtf8().Span);
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, ReadOnlyMemory data) => bloomFilter.ContainsAsync(data.Span.ToUtf8());
#endregion String
#region Double
public static bool Add(this IBloomFilter bloomFilter, double data) => bloomFilter.Add(BitConverter.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, double data) => bloomFilter.AddAsync(BitConverter.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, double data) => bloomFilter.Contains(BitConverter.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, double data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data)));
#endregion Double
#region Single
public static bool Add(this IBloomFilter bloomFilter, float data) => bloomFilter.Add(BitConverter.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, float data) => bloomFilter.AddAsync(BitConverter.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, float data) => bloomFilter.Contains(BitConverter.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, float data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data)));
#endregion Single
#region Int16
public static bool Add(this IBloomFilter bloomFilter, short data) => bloomFilter.Add(BitConverter.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, short data) => bloomFilter.AddAsync(BitConverter.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, short data) => bloomFilter.Contains(BitConverter.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, short data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data)));
#endregion Int16
#region Int32
public static bool Add(this IBloomFilter bloomFilter, int data) => bloomFilter.Add(BitConverter.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, int data) => bloomFilter.AddAsync(BitConverter.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, int data) => bloomFilter.Contains(BitConverter.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, int data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data)));
#endregion Int32
#region Int64
public static bool Add(this IBloomFilter bloomFilter, long data) => bloomFilter.Add(BitConverter.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, long data) => bloomFilter.AddAsync(BitConverter.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, long data) => bloomFilter.Contains(BitConverter.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, long data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data)));
#endregion Int64
#region UInt16
public static bool Add(this IBloomFilter bloomFilter, ushort data) => bloomFilter.Add(BitConverter.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, ushort data) => bloomFilter.AddAsync(BitConverter.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, ushort data) => bloomFilter.Contains(BitConverter.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, ushort data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data)));
#endregion UInt16
#region UInt32
public static bool Add(this IBloomFilter bloomFilter, uint data) => bloomFilter.Add(BitConverter.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, uint data) => bloomFilter.AddAsync(BitConverter.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, uint data) => bloomFilter.Contains(BitConverter.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, uint data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data)));
#endregion UInt32
#region UInt64
public static bool Add(this IBloomFilter bloomFilter, ulong data) => bloomFilter.Add(BitConverter.GetBytes(data));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, ulong data) => bloomFilter.AddAsync(BitConverter.GetBytes(data));
public static bool Contains(this IBloomFilter bloomFilter, ulong data) => bloomFilter.Contains(BitConverter.GetBytes(data));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, ulong data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data)));
#endregion UInt64
#region DateTime
public static bool Add(this IBloomFilter bloomFilter, DateTime data) => bloomFilter.Add(BitConverter.GetBytes(data.Ticks));
public static ValueTask AddAsync(this IBloomFilter bloomFilter, DateTime data) => bloomFilter.AddAsync(BitConverter.GetBytes(data.Ticks));
public static bool Contains(this IBloomFilter bloomFilter, DateTime data) => bloomFilter.Contains(BitConverter.GetBytes(data.Ticks));
public static ValueTask ContainsAsync(this IBloomFilter bloomFilter, DateTime data) => bloomFilter.ContainsAsync(BitConverter.GetBytes(data.Ticks));
public static IList Add(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Add(elements.Select(data => BitConverter.GetBytes(data.Ticks)));
public static ValueTask> AddAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AddAsync(elements.Select(data => BitConverter.GetBytes(data.Ticks)));
public static IList Contains(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.Contains(elements.Select(data => BitConverter.GetBytes(data.Ticks)));
public static ValueTask> ContainsAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.ContainsAsync(elements.Select(data => BitConverter.GetBytes(data.Ticks)));
public static bool All(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.All(elements.Select(data => BitConverter.GetBytes(data.Ticks)));
public static ValueTask AllAsync(this IBloomFilter bloomFilter, IEnumerable elements) => bloomFilter.AllAsync(elements.Select(data => BitConverter.GetBytes(data.Ticks)));
#endregion DateTime
}
================================================
FILE: src/BloomFilter/Configurations/BloomFilterOptions.cs
================================================
using System;
using System.Collections.Generic;
namespace BloomFilter.Configurations;
///
/// BloomFilterOptions
///
public class BloomFilterOptions
{
///
/// Gets the extensions.
///
/// The extensions.
internal IList Extensions { get; } = new List();
///
/// Registers the extension.
///
/// Extension.
public void RegisterExtension(IBloomFilterOptionsExtension extension)
{
#if NET6_0_OR_GREATER
ArgumentNullException.ThrowIfNull(extension);
#else
if (extension == null) throw new ArgumentNullException(nameof(extension));
#endif
Extensions.Add(extension);
}
}
================================================
FILE: src/BloomFilter/Configurations/FilterMemoryOptions.cs
================================================
using System.Collections;
using System.Collections.Generic;
namespace BloomFilter.Configurations;
public class FilterMemoryOptions
{
///
/// The Name
///
public string Name { get; set; } = BloomFilterConstValue.DefaultInMemoryName;
///
/// The expected elements
///
public long ExpectedElements { get; set; } = 1_000_000;
///
/// The error rate
///
public double ErrorRate { get; set; } = 0.01;
///
/// The Hash Method
///
public HashMethod Method { get; set; } = HashMethod.Murmur3;
///
/// Multiple bitmap
///
public BitArray[]? Buckets { get; set; }
///
/// Multiple bitmap from bytes
///
public IList? BucketBytes { get; set; }
}
================================================
FILE: src/BloomFilter/Configurations/FilterMemoryOptionsExtension.cs
================================================
using System;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
namespace BloomFilter.Configurations;
public class FilterMemoryOptionsExtension : IBloomFilterOptionsExtension
{
private readonly FilterMemoryOptions _options;
private readonly Type? _serializerType;
///
/// Initializes a new instance of the class.
///
/// Configure.
///
public FilterMemoryOptionsExtension(FilterMemoryOptions options, Type? serializerType = null)
{
_options = options;
_serializerType = serializerType;
}
public void AddServices(IServiceCollection services)
{
services.TryAddSingleton();
if (_serializerType is null)
{
services.TryAddSingleton();
}
else
{
services.TryAddSingleton(typeof(IFilterMemorySerializer), _serializerType);
}
services.AddSingleton(x =>
{
return new FilterMemory(_options, x.GetRequiredService());
});
}
}
================================================
FILE: src/BloomFilter/Configurations/IBloomFilterOptionsExtension.cs
================================================
using Microsoft.Extensions.DependencyInjection;
namespace BloomFilter.Configurations;
///
/// BloomFilter options extension.
///
public interface IBloomFilterOptionsExtension
{
///
/// Adds the services.
///
/// Services.
void AddServices(IServiceCollection services);
}
================================================
FILE: src/BloomFilter/DefaultBloomFilterFactory.cs
================================================
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
namespace BloomFilter;
public class DefaultBloomFilterFactory : IBloomFilterFactory
{
private readonly IEnumerable _bloomFilters;
public DefaultBloomFilterFactory(IEnumerable bloomFilters)
{
_bloomFilters = bloomFilters;
}
public IBloomFilter Get(string name)
{
#if NET6_0_OR_GREATER
ArgumentNullException.ThrowIfNull(name);
#else
if (name == null) throw new ArgumentNullException(nameof(name));
#endif
var filter = _bloomFilters.FirstOrDefault(x => x.Name.Equals(name, StringComparison.OrdinalIgnoreCase));
if (filter == null) throw new ArgumentException("can not find a match BloomFilter");
return filter;
}
public bool TryGet(string name,
#if NET5_0_OR_GREATER
[MaybeNullWhen(false)]
#endif
out IBloomFilter bloomFilter)
{
bloomFilter = _bloomFilters.FirstOrDefault(x => x.Name.Equals(name, StringComparison.OrdinalIgnoreCase));
return bloomFilter != null;
}
}
================================================
FILE: src/BloomFilter/DefaultFilterMemorySerializer.cs
================================================
using System;
using System.Collections;
using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace BloomFilter;
public class DefaultFilterMemorySerializer : IFilterMemorySerializer
{
public async ValueTask SerializeAsync(FilterMemorySerializerParam param, Stream stream)
{
byte[] nameBytes = Encoding.UTF8.GetBytes(param.Name ?? string.Empty);
await stream.WriteAsync(BitConverter.GetBytes(nameBytes.Length), 0, 4);
await stream.WriteAsync(nameBytes, 0, nameBytes.Length);
await stream.WriteAsync(BitConverter.GetBytes(param.ExpectedElements), 0, 8);
await stream.WriteAsync(BitConverter.GetBytes(param.ErrorRate), 0, 8);
await stream.WriteAsync(BitConverter.GetBytes((int)param.Method), 0, 4);
await stream.WriteAsync(BitConverter.GetBytes(param.Buckets.Length), 0, 4);
foreach (var bucket in param.Buckets)
{
byte[] bucketBytes = new byte[bucket.Length / 8 + Mod(bucket.Length)];
bucket.CopyTo(bucketBytes, 0);
await stream.WriteAsync(BitConverter.GetBytes(bucketBytes.Length), 0, 4).ConfigureAwait(false);
await stream.WriteAsync(bucketBytes, 0, bucketBytes.Length).ConfigureAwait(false);
}
}
public async ValueTask DeserializeAsync(Stream stream)
{
byte[] lengthBytes = new byte[4];
byte[] int64Bytes = new byte[8];
// Read name
await ReadExactlyAsync(stream, lengthBytes);
int nameLength = BitConverter.ToInt32(lengthBytes, 0);
byte[] nameBytes = new byte[nameLength];
await ReadExactlyAsync(stream, nameBytes);
string name = Encoding.UTF8.GetString(nameBytes);
// Read expected elements
await ReadExactlyAsync(stream, int64Bytes);
long expectedElements = BitConverter.ToInt64(int64Bytes, 0);
// Read error rate
await ReadExactlyAsync(stream, int64Bytes);
double errorRate = BitConverter.ToDouble(int64Bytes, 0);
// Read method
await ReadExactlyAsync(stream, lengthBytes);
HashMethod method = (HashMethod)BitConverter.ToInt32(lengthBytes, 0);
// Read buckets
await ReadExactlyAsync(stream, lengthBytes);
int bucketsLength = BitConverter.ToInt32(lengthBytes, 0);
var buckets = new BitArray[bucketsLength];
for (int i = 0; i < bucketsLength; i++)
{
await ReadExactlyAsync(stream, lengthBytes);
int bitArrayLength = BitConverter.ToInt32(lengthBytes, 0);
byte[] bucketBytes = new byte[bitArrayLength];
await ReadExactlyAsync(stream, bucketBytes);
buckets[i] = new BitArray(bucketBytes);
}
// Create param with object initializer (compatible with init properties)
return new FilterMemorySerializerParam
{
Name = name,
ExpectedElements = expectedElements,
ErrorRate = errorRate,
Method = method,
Buckets = buckets
};
}
private async Task ReadExactlyAsync(Stream stream, byte[] data)
{
#if NET7_0_OR_GREATER
await stream.ReadExactlyAsync(data, 0, data.Length).ConfigureAwait(false);
#else
await stream.ReadAsync(data, 0, data.Length).ConfigureAwait(false);
#endif
}
private int Mod(int len) => len % 8 > 0 ? 1 : 0;
}
================================================
FILE: src/BloomFilter/Filter.cs
================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace BloomFilter;
///
/// Represents a Bloom filter and provides
///
///
public abstract class Filter : IBloomFilter
{
//256MB
protected const int MaxInt = 2_147_483_640;
///
/// Gets the name specified by BloomFilter.
///
public string Name { get; }
///
///
///
public HashFunction Hash { get; private set; }
///
/// the Capacity of the Bloom filter
///
public long Capacity { get; private set; }
///
/// number of hash functions
///
public int Hashes { get; private set; }
///
/// the expected elements.
///
public long ExpectedElements { get; private set; }
///
/// the number of expected elements
///
public double ErrorRate { get; private set; }
///
/// Initializes a new instance of the class.
///
///
/// The expected elements.
/// The error rate.
/// The hash function.
///
/// expectedElements - expectedElements must be > 0
/// or
/// errorRate
///
/// hashFunction
public Filter(string name, long expectedElements, double errorRate, HashFunction hashFunction)
{
if (expectedElements < 1)
throw new ArgumentOutOfRangeException("expectedElements", expectedElements, "expectedElements must be > 0");
if (errorRate >= 1 || errorRate <= 0)
throw new ArgumentOutOfRangeException("errorRate", errorRate, string.Format("errorRate must be between 0 and 1, exclusive. Was {0}", errorRate));
#if NET6_0_OR_GREATER
ArgumentNullException.ThrowIfNull(hashFunction);
#else
if (hashFunction == null)
throw new ArgumentNullException(nameof(hashFunction));
#endif
Name = name;
ExpectedElements = expectedElements;
ErrorRate = errorRate;
Hash = hashFunction;
Capacity = BestM(expectedElements, errorRate);
Hashes = BestK(expectedElements, Capacity);
}
///
/// Initializes a new instance of the class.
///
///
/// The capacity.
/// The hashes.
/// The hash function.
///
/// capacity - capacity must be > 0
/// or
/// hashes - hashes must be > 0
///
/// hashFunction
public Filter(string name, long capacity, int hashes, HashFunction hashFunction)
{
if (capacity < 1)
throw new ArgumentOutOfRangeException("capacity", capacity, "capacity must be > 0");
if (hashes < 1)
throw new ArgumentOutOfRangeException("hashes", hashes, "hashes must be > 0");
#if NET6_0_OR_GREATER
ArgumentNullException.ThrowIfNull(hashFunction);
#else
if (hashFunction == null)
throw new ArgumentNullException(nameof(hashFunction));
#endif
Name = name;
Capacity = capacity;
Hashes = hashes;
Hash = hashFunction;
ExpectedElements = BestN(hashes, capacity);
ErrorRate = BestP(hashes, capacity, ExpectedElements);
}
protected void SetFilterParam(long expectedElements, double errorRate, HashMethod method)
{
if (expectedElements < 1)
throw new ArgumentOutOfRangeException("expectedElements", expectedElements, "expectedElements must be > 0");
if (errorRate >= 1 || errorRate <= 0)
throw new ArgumentOutOfRangeException("errorRate", errorRate, string.Format("errorRate must be between 0 and 1, exclusive. Was {0}", errorRate));
ExpectedElements = expectedElements;
ErrorRate = errorRate;
Hash = HashFunction.Functions[method];
Capacity = BestM(expectedElements, errorRate);
Hashes = BestK(expectedElements, Capacity);
}
///
/// Adds the passed value to the filter.
///
///
///
public abstract bool Add(ReadOnlySpan data);
///
/// Async Adds the passed value to the filter.
///
///
///
public abstract ValueTask AddAsync(ReadOnlyMemory data);
///
/// Adds the specified elements.
///
/// The elements.
///
public abstract IList Add(IEnumerable elements);
///
/// Async Adds the specified elements.
///
/// The elements.
///
public abstract ValueTask> AddAsync(IEnumerable elements);
///
/// Removes all elements from the filter
///
public abstract void Clear();
///
/// Async Removes all elements from the filter
///
public abstract ValueTask ClearAsync();
///
/// Tests whether an element is present in the filter
///
///
///
public abstract bool Contains(ReadOnlySpan element);
///
/// Async Tests whether an element is present in the filter
///
///
///
public abstract ValueTask ContainsAsync(ReadOnlyMemory element);
///
/// Tests whether an elements is present in the filter
///
///
///
public abstract IList Contains(IEnumerable elements);
///
/// Async Tests whether an elements is present in the filter
///
///
///
public abstract ValueTask> ContainsAsync(IEnumerable elements);
///