Repository: lionsoul2014/ip2region Branch: master Commit: 1209b72452ad Files: 272 Total size: 149.0 MB Directory structure: gitextract_eppf0yhb/ ├── .gitattributes ├── .gitignore ├── LICENSE.md ├── README.md ├── README_zh.md ├── binding/ │ ├── c/ │ │ ├── Makefile │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── main.c │ │ ├── test_util.c │ │ ├── xdb_api.h │ │ ├── xdb_searcher.c │ │ └── xdb_util.c │ ├── cpp/ │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── src/ │ │ │ ├── base.cc │ │ │ ├── base.h │ │ │ ├── bench.cc │ │ │ ├── bench.h │ │ │ ├── edit.cc │ │ │ ├── edit.h │ │ │ ├── header.cc │ │ │ ├── header.h │ │ │ ├── ip.cc │ │ │ ├── ip.h │ │ │ ├── make.cc │ │ │ ├── make.h │ │ │ ├── search.cc │ │ │ └── search.h │ │ └── test/ │ │ ├── bench.cc │ │ ├── edit_v4.cc │ │ ├── edit_v6.cc │ │ ├── header.cc │ │ ├── make.cc │ │ └── search.cc │ ├── csharp/ │ │ ├── .editorconfig │ │ ├── .gitignore │ │ ├── CHANGELOG.md │ │ ├── IP2Region.Net/ │ │ │ ├── Abstractions/ │ │ │ │ └── ISearcher.cs │ │ │ ├── Extensions/ │ │ │ │ └── ServiceCollectionExtensions.cs │ │ │ ├── IP2Region.Net.csproj │ │ │ ├── Internal/ │ │ │ │ ├── CacheStrategyFactory.cs │ │ │ │ ├── ContentCacheStrategy.cs │ │ │ │ ├── FileCacheStrategy.cs │ │ │ │ ├── ICacheStrategy.cs │ │ │ │ └── VectorIndexCacheStrategy.cs │ │ │ └── XDB/ │ │ │ ├── CachePolicy.cs │ │ │ ├── Searcher.cs │ │ │ ├── Util.cs │ │ │ └── XdbVersion.cs │ │ ├── IP2Region.Net.BenchMark/ │ │ │ ├── Benmarks.cs │ │ │ ├── IP2Region.Net.BenchMark.csproj │ │ │ └── Program.cs │ │ ├── IP2Region.Net.Test/ │ │ │ ├── IP2Region.Net.Test.csproj │ │ │ ├── SearcherTest.cs │ │ │ ├── UtilTest.cs │ │ │ └── XdbTest.cs │ │ ├── IP2Region.Net.slnx │ │ └── README.md │ ├── erlang/ │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── benchmarks/ │ │ │ └── xdb-benchmark.sh │ │ ├── include/ │ │ │ └── ip2region.hrl │ │ ├── priv/ │ │ │ └── dummy │ │ ├── rebar.config │ │ ├── src/ │ │ │ ├── ip2region.app.src │ │ │ ├── ip2region_app.erl │ │ │ ├── ip2region_sup.erl │ │ │ ├── ip2region_util.erl │ │ │ ├── ip2region_worker.erl │ │ │ ├── xdb.erl │ │ │ └── xdb_benchmark.erl │ │ └── test/ │ │ └── xdb_test.erl │ ├── golang/ │ │ ├── Makefile │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── go.mod │ │ ├── go.sum │ │ ├── main.go │ │ ├── service/ │ │ │ ├── config.go │ │ │ ├── config_test.go │ │ │ ├── ip2region.go │ │ │ ├── ip2region_test.go │ │ │ ├── searcher_pool.go │ │ │ └── searcher_pool_test.go │ │ └── xdb/ │ │ ├── header.go │ │ ├── searcher.go │ │ ├── util.go │ │ ├── util_test.go │ │ └── version.go │ ├── java/ │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── pom.xml │ │ └── src/ │ │ ├── main/ │ │ │ └── java/ │ │ │ └── org/ │ │ │ └── lionsoul/ │ │ │ └── ip2region/ │ │ │ ├── SearcherTest.java │ │ │ ├── service/ │ │ │ │ ├── Config.java │ │ │ │ ├── ConfigBuilder.java │ │ │ │ ├── InvalidConfigException.java │ │ │ │ ├── Ip2Region.java │ │ │ │ └── SearcherPool.java │ │ │ └── xdb/ │ │ │ ├── Header.java │ │ │ ├── IPv4.java │ │ │ ├── IPv6.java │ │ │ ├── InetAddressException.java │ │ │ ├── LittleEndian.java │ │ │ ├── Log.java │ │ │ ├── LongByteArray.java │ │ │ ├── Searcher.java │ │ │ ├── Util.java │ │ │ ├── Version.java │ │ │ └── XdbException.java │ │ └── test/ │ │ └── java/ │ │ └── org/ │ │ └── lionsoul/ │ │ └── ip2region/ │ │ ├── service/ │ │ │ ├── ConfigTest.java │ │ │ ├── Ip2RegionTest.java │ │ │ └── SearcherPoolTest.java │ │ └── xdb/ │ │ ├── BufferTest.java │ │ ├── IPv4Test.java │ │ ├── LittleEndianTest.java │ │ ├── UtilTest.java │ │ └── VersionTest.java │ ├── javascript/ │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── index.d.ts │ │ ├── index.js │ │ ├── package.json │ │ ├── searcher.js │ │ ├── tests/ │ │ │ ├── bench.app.js │ │ │ ├── search.app.js │ │ │ ├── searcher.test.js │ │ │ └── util.test.js │ │ ├── tsconfig.json │ │ └── util.js │ ├── lua/ │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── bench_test.lua │ │ ├── search_test.lua │ │ ├── util_test.lua │ │ └── xdb_searcher.lua │ ├── lua_c/ │ │ ├── Makefile │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── bench_test.lua │ │ ├── search_test.lua │ │ ├── util_test.lua │ │ └── xdb_searcher.c │ ├── nginx/ │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── config │ │ ├── src/ │ │ │ ├── ngx_http_ip2region_module.c │ │ │ └── ngx_http_ip2region_module.h │ │ └── t/ │ │ └── http_ip2region.t │ ├── nodejs/ │ │ └── README.md │ ├── php/ │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── batch_test.php │ │ ├── bench_test.php │ │ ├── search_test.php │ │ └── xdb/ │ │ ├── Searcher.class.php │ │ └── util_test.php │ ├── python/ │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── MANIFEST.in │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── bench_test.py │ │ ├── ip2region/ │ │ │ ├── __init__.py │ │ │ ├── searcher.py │ │ │ └── util.py │ │ ├── search_test.py │ │ ├── setup.py │ │ └── util_test.py │ ├── rust/ │ │ ├── Cargo.toml │ │ ├── README.md │ │ ├── README_zh.md │ │ ├── example/ │ │ │ ├── Cargo.toml │ │ │ └── src/ │ │ │ ├── cmd.rs │ │ │ └── main.rs │ │ └── ip2region/ │ │ ├── Cargo.toml │ │ ├── benches/ │ │ │ └── search.rs │ │ └── src/ │ │ ├── error.rs │ │ ├── ip_value.rs │ │ ├── lib.rs │ │ └── searcher.rs │ └── typescript/ │ └── README.md ├── data/ │ ├── ip2region_v4.xdb │ ├── ip2region_v6.xdb │ ├── ipv4_source.txt │ ├── ipv6_source.txt │ └── sample/ │ ├── github-issue-196.fix │ ├── github-issue-200.fix │ ├── github-issue-243.fix │ ├── github-issue-287.bug │ ├── ip.test.txt │ ├── segments.tests │ └── segments.tests.mixed └── maker/ ├── c/ │ └── ReadMe.md ├── cpp/ │ └── README.md ├── csharp/ │ ├── .gitignore │ ├── IP2RegionMaker/ │ │ ├── IP2RegionMaker.csproj │ │ ├── Program.cs │ │ ├── Properties/ │ │ │ └── PublishProfiles/ │ │ │ └── FolderProfile.pubxml │ │ └── XDB/ │ │ ├── IndexPolicy.cs │ │ ├── Maker.cs │ │ ├── Segment.cs │ │ └── Util.cs │ ├── IP2RegionMaker.Test/ │ │ ├── IP2RegionMaker.Test.csproj │ │ ├── Usings.cs │ │ └── UtilTest.cs │ ├── README.md │ └── README_zh.md ├── golang/ │ ├── Dockerfile │ ├── Makefile │ ├── README.md │ ├── README_zh.md │ ├── cmd/ │ │ ├── bench.go │ │ ├── edit.go │ │ ├── generate.go │ │ ├── process.go │ │ ├── search.go │ │ └── util.go │ ├── go.mod │ ├── go.sum │ ├── main.go │ ├── make.bat │ └── xdb/ │ ├── editor.go │ ├── index.go │ ├── maker.go │ ├── processor.go │ ├── searcher.go │ ├── segment.go │ ├── util.go │ ├── util_test.go │ └── version.go ├── java/ │ ├── README.md │ ├── README_zh.md │ ├── pom.xml │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── org/ │ │ └── lionsoul/ │ │ └── ip2region/ │ │ ├── MakerApp.java │ │ └── xdb/ │ │ ├── IPv4.java │ │ ├── IPv6.java │ │ ├── IndexPolicy.java │ │ ├── InvalidInetAddressException.java │ │ ├── LittleEndian.java │ │ ├── Log.java │ │ ├── Maker.java │ │ ├── Segment.java │ │ ├── Util.java │ │ └── Version.java │ └── test/ │ └── java/ │ └── org/ │ └── lionsoul/ │ └── ip2region/ │ └── xdb/ │ ├── IndexPolicyTest.java │ ├── LittleEndianTest.java │ ├── MakerTest.java │ ├── SegmentTest.java │ ├── UtilTest.java │ └── VersionTest.java ├── python/ │ ├── README.md │ ├── README_zh.md │ ├── main.py │ └── xdb/ │ ├── __init__.py │ ├── index.py │ ├── maker.py │ ├── segment.py │ └── util.py └── rust/ ├── README.md ├── README_zh.md └── maker/ ├── Cargo.toml └── src/ ├── command.rs ├── error.rs ├── header.rs ├── lib.rs ├── main.rs ├── maker.rs └── segment.rs ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ ================================================ FILE: .gitignore ================================================ *.class *.out *.o *.pyc *~ *.log *.la *.so *.iml META-INF/ .DS_Store # Binary Files # *.jar !dbMaker-*.jar # ignore all xdb except the one in ./data/ *.xdb !/data/*.xdb # Package Files # .settings/ .classpath .project # vim swp file # *.swp .idea .vscode # binding /binding/java/classes/ /binding/java/doc/ /binding/java/target/ /binding/java/*.jar # python /**/__pycache__ # clang /binding/c/xdb_searcher /binding/c/test_util /binding/c/cmake-build-debug # lua/luc_c /binding/lua_c/cmake-build-debug # golang /binding/golang/searcher /binding/golang/xdb_searcher /binding/golang/golang # rust Cargo.lock target # VS ignore cases /**/*.sln /binding/c#/**/.vs/ /binding/c#/**/packages /binding/c#/**/bin /binding/c#/**/obj # Nodejs /binding/nodejs/tests/unitTests/__snapshots__ /binding/nodejs/coverage /binding/nodejs/node_modules /binding/nodejs/.nyc_output /binging/nodejs/package-lock.json # Javascript /binding/javascript/tests/unitTests/__snapshots__ /binding/javascript/coverage /binding/javascript/node_modules /binding/javascript/.nyc_output /binding/javascript/package-lock.json # maker ## golang /maker/golang/dbmaker /maker/golang/xdb_maker /maker/golang/golang #erlang /binding/erlang/_build /binding/erlang/doc #vscode .vscode build ================================================ FILE: LICENSE.md ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ========================================================================== The following license applies to the ip2region library -------------------------------------------------------------------------- Copyright (c) 2015 Lionsoul Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region [ip2region](https://ip2region.net) - is an offline IP address localization library and IP localization data management framework. It supports both `IPv4` and `IPv6` with query efficiency at the 10-microsecond level. It provides `xdb` data generation and query client implementations for many mainstream programming languages. # Features ### 1. Offline Localization Library The project itself provides raw data for both IPv4 (`data/ipv4_source.txt`) and IPv6 (`data/ipv6_source.txt`), along with corresponding xdb files (`data/ip2region_v4.xdb` and `data/ip2region_v6.xdb`) to achieve city-level query localization. The field format is: `Country|Province|City|ISP|iso-alpha2-code`. Localization information for China is entirely in Chinese, while regional information for non-China areas is entirely in English. ### 2. Data Management Framework `xdb` supports hundreds of millions of IP data segment rows. Region information supports full customization. The region information of the built-in data is fixed in the format: `Country|Province|City|ISP|iso-alpha2-Code`. You can append data for specific business needs to the region, such as: GPS information/International Standard Regional Codes/Zip codes, etc. In other words, you can fully use ip2region to manage your own IP localization data. ### 3. Data Deduplication and Compression The `xdb` format generation program automatically processes the input raw data, checks and completes the merging of adjacent IP segments, and performs deduplication and compression of identical regional information. ### 4. High-Speed Query Response Even for queries based entirely on the `xdb` file, the single query response time is at the 10-microsecond level. Memory-accelerated queries can be enabled through the following two methods: 1. `vIndex` Index Caching: Uses a fixed `512KiB` of memory to cache vector index data, reducing one disk IO operation and maintaining average query efficiency within 100 microseconds. 2. Entire `xdb` File Caching: Loads the entire `xdb` file into memory. Memory usage is equal to the `xdb` file size. There is no disk IO operation, maintaining 10-microsecond level query efficiency. ### 5. Unified Query Interface `xdb` provides version-compatible query implementations. A unified API can simultaneously provide queries for both IPv4 and IPv6 data and return unified data. # `xdb` Query For API introductions, usage documentation, and test programs, please refer to the README introduction under the corresponding `searcher` query client. All query binding implementations are as follows: | Language | Description | IPv4 Support | IPv6 Support | | --- | --- | --- | --- | | [Golang](binding/golang/README.md) | golang query client | :white_check_mark: | :white_check_mark: | | [PHP](binding/php/README.md) | php query client | :white_check_mark: | :white_check_mark: | | [Java](binding/java/README.md) | java query client | :white_check_mark: | :white_check_mark: | | [C](binding/c/README.md) | C[std=c99] query client | :white_check_mark: | :white_check_mark: | | [Lua_c](binding/lua_c/README.md) | lua c extension query client | :white_check_mark: | :white_check_mark: | | [Lua](binding/lua/README.md) | lua query client | :white_check_mark: | :white_check_mark: | | [Rust](binding/rust/README.md) | rust query client | :white_check_mark: | :white_check_mark: | | [Python](binding/python/README.md) | python query client | :white_check_mark: | :white_check_mark: | | [Javascript](binding/javascript/README.md) | javascript query client | :white_check_mark: | :white_check_mark: | | [Csharp](binding/csharp) | csharp query client | :white_check_mark: | :white_check_mark: | | [Erlang](binding/erlang/README.md) | erlang query client | :white_check_mark: | :x: | | [Nginx](binding/nginx) | nginx extension query client | :white_check_mark: | :white_check_mark: | | [C++](binding/cpp/README.md) | C++ query client | :white_check_mark: | :white_check_mark: | The following toolchain implementations are contributed by community developers via third-party repositories: | Language | Description | | --- | --- | | [ip2region-composer](https://github.com/zoujingli/ip2region) | php composer management client | | [ip2region-ts](https://github.com/Steven-Qiang/ts-ip2region2) | node.js addon management client | | [ruby-ip2region](https://github.com/jicheng1014/ruby-ip2region) | ruby xdb query client implementation | | [Ip2regionTool](https://github.com/orestonce/Ip2regionTool) | ip2region data conversion tool | # `xdb` Generation For API introductions, usage documentation, and test programs, please refer to the README documents under the following `maker` generation programs: | Language | Description | IPv4 Support | IPv6 Support | | --- | --- | --- | --- | | [Golang](maker/golang/README.md) | golang xdb generation program | :white_check_mark: | :white_check_mark: | | [Java](maker/java/README.md) | java xdb generation program | :white_check_mark: | :white_check_mark: | | [Python](maker/python/README.md) | python xdb generation program | :white_check_mark: | :x: | | [Csharp](maker/csharp/README.md) | csharp xdb generation program | :white_check_mark: | :x: | | [Rust](maker/rust/README.md) | rust xdb generation program | :white_check_mark: | :white_check_mark: | | [C++](maker/cpp) | C++ xdb generation program | :white_check_mark: | :white_check_mark: | # `xdb` Update The core of the ip2region project lies in **researching the design and implementation of IP data storage and fast querying**. The raw data `./data/ipv4_source.txt` and `./data/ipv6_source.txt` included in the project are updated irregularly. For scenarios with high requirements for data accuracy and update frequency, it is recommended to purchase commercial offline data from the [Ip2Region Community](https://ip2region.net/products/offline) or third-party vendors. You can try to update the data yourself using the following methods: ### Manual Editing and Updating You can modify the data yourself based on the raw IP data provided by ip2region in `./data/ipv4_source.txt` and `./data/ipv6_source.txt` using the editing tools provided by ip2region. Currently, the data sources include: 1. Data provided by the ip2region community (please refer to the official account at the bottom for community notifications) 2. Project Issues tagged with `[Data_Updates]` 3. Other custom data: e.g., data provided by customers, data obtained through GPS and WIFI positioning, or legal and compliant data from other platforms. For instructions on using the raw IP data editing tools, please refer to the README documents under the following `maker` generation programs: | Language | Description | IPv4 Support | IPv6 Support | | --- | --- | --- | --- | | [Golang](maker/golang/README.md#xdb-data-editing) | golang IP raw data editor | :white_check_mark: | :white_check_mark: | | [C++](maker/cpp/README.md) | C++ IP raw data editor | :white_check_mark: | :white_check_mark: | ### Detection Automatic Update If you want to update data via your own API or data source, you can refer to the update algorithm based on the "Detection Algorithm" shared in the following videos to write your own update program: 1. [Data Update Implementation Video Sharing - part1](https://www.bilibili.com/video/BV1934y1E7Q5/) 2. [Data Update Implementation Video Sharing - part2](https://www.bilibili.com/video/BV1pF411j7Aw/) # Official Community The Ip2Region official community was officially launched on `2025/06/12`. On one hand, it provides stable [commercial offline data](https://ip2region.net/products/offline) services. On the other hand, it facilitates the strengthening of the IP toolchain and data services outside the core code, such as [usage documentation](https://ip2region.net/doc/), [query testing](https://ip2region.net/search/demo), and data correction. For more information and services regarding the community, please visit the [Ip2Region Official Community](https://ip2region.net/). # Related Remarks ### 1. xdb Technical Documents: 1. xdb Data Structure Analysis: ["ip2region xdb - Data Structure Description"](https://ip2region.net/doc/xdb/structure) 2. xdb Query Process Analysis: ["ip2region xdb - Query Process Description"](https://ip2region.net/doc/xdb/search) 3. xdb Generation Process Analysis: ["ip2region xdb - Generation Process Description"](https://ip2region.net/doc/xdb/generate) 4. xdb File Generation Tutorial: ["ip2region xdb - File Generation Tutorial"](https://ip2region.net/doc/data/xdb_make) 5. xdb Concurrent Safety Query: ["ip2region xdb - Concurrent Safety Query"](https://ip2region.net/doc/xdb/concurrent) 6. xdb Data Update Method: ["ip2region Data Update and Use of xdb Data Editor"](https://mp.weixin.qq.com/s/cZH5qIn4E5rQFy6N32RCzA) ### 3. Technical Information Blogs 1. WeChat Official Account - lionsoul-org, the author's active technical sharing channel 2. [Ip2Region Official Community](https://ip2region.net) ================================================ FILE: README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region [ip2region](https://ip2region.net) - 是一个离线IP地址定位库和IP定位数据管理框架，同时支持 `IPv4` 和 `IPv6` ，10微秒级别的查询效率，提供了众多主流编程语言的 `xdb` 数据生成和查询客户端实现。 # 项目特性 ### 1、离线定位库 项目本身同时了提供了一份 IPv4 (`data/ipv4_source.txt`) 和 IPv6 (`data/ipv6_source.txt`) 的原始数据和对应的 xdb 文件(`data/ip2region_v4.xdb` 和 `data/ip2region_v6.xdb`) 用于实现精确到城市的查询定位功能，字段格式为：`国家|省份|城市|ISP|iso-alpha2-code(国家两字母简称)`，中国的定位信息全部为中文，非中国地区的地域信息全部为英文。 ### 2、数据管理框架 `xdb` 支持亿级别的 IP 数据段行数，region 信息支持完全自定义，自带数据的 region 信息固定了格式为：`国家|省份|城市|ISP|iso-alpha2-Code`，你可以在 region 中追加特定业务需求的数据，例如：GPS信息/国际统一地域信息编码/邮编等。也就是你完全可以使用 ip2region 来管理你自己的 IP 定位数据。 ### 3、数据去重和压缩 `xdb` 格式生成程序会自动处理输入的原始数据，检查并且完成相连 IP 段的的合并以及相同地域信息的去重和压缩。 ### 4、极速查询响应 即使是完全基于 `xdb` 文件的查询，单次查询响应时间在十微秒级别，可通过如下两种方式开启内存加速查询： 1. `vIndex` 索引缓存 ：使用固定的 `512KiB` 的内存空间缓存 vector index 数据，减少一次 IO 磁盘操作，保持平均查询效率稳定在100微秒之内。 2. `xdb` 整个文件缓存：将整个 `xdb` 文件全部加载到内存，内存占用等同于 `xdb` 文件大小，无磁盘 IO 操作，保持10微秒级别的查询效率。 ### 5、统一的查询接口 `xdb` 提供了版本兼容的查询实现，一个统一的 API 可以同时提供对 IPv4 和 IPv6 数据的查询并且返回统一的数据。 # `xdb` 查询 API 介绍，使用文档和测试程序请参考对应 `searcher` 查询客户端下的 README 介绍，全部查询 binding 实现情况如下： | 编程语言 | 描述 | IPv4 支持 | IPv6 支持 | | --- | --- | --- | --- | | [Golang](binding/golang/README_zh.md) | golang 查询客户端 | :white_check_mark: | :white_check_mark: | | [PHP](binding/php/README_zh.md) | php 查询客户端 | :white_check_mark: | :white_check_mark: | | [Java](binding/java/README_zh.md) | java 查询客户端 | :white_check_mark: | :white_check_mark: | | [C](binding/c/README_zh.md) | C[std=c99] 查询客户端 | :white_check_mark: | :white_check_mark: | | [Lua_c](binding/lua_c/README_zh.md) | lua c 扩展查询客户端 | :white_check_mark: | :white_check_mark: | | [Lua](binding/lua/README_zh.md) | lua 查询客户端 | :white_check_mark: | :white_check_mark: | | [Rust](binding/rust/README_zh.md) | rust 查询客户端 | :white_check_mark: | :white_check_mark: | | [Python](binding/python/README_zh.md) | python 查询客户端 | :white_check_mark: | :white_check_mark: | | [Javascript](binding/javascript/README_zh.md) | javascript 查询客户端 | :white_check_mark: | :white_check_mark: | | [Csharp](binding/csharp) | csharp 查询客户端 | :white_check_mark: | :white_check_mark: | | [Erlang](binding/erlang/README_zh.md) | erlang 查询客户端 | :white_check_mark: | :x: | | [Nginx](binding/nginx) | nginx 扩展查询客户端 | :white_check_mark: | :white_check_mark: | | [C++](binding/cpp/README_zh.md) | C++ 查询客户端 | :white_check_mark: | :white_check_mark: | 以下工具链实现由社区开发者通过第三方仓库贡献： | 编程语言 | 描述 | | --- | --- | | [ip2region-composer](https://github.com/zoujingli/ip2region) | php composer 管理客户端 | | [ip2region-ts](https://github.com/Steven-Qiang/ts-ip2region2) | node.js addon 管理客户端| | [ruby-ip2region](https://github.com/jicheng1014/ruby-ip2region) | ruby xdb 查询客户端实现 | | [Ip2regionTool](https://github.com/orestonce/Ip2regionTool) | ip2region 数据转换工具 | # `xdb` 生成 API 介绍，使用文档和测试程序请参考如下 `maker` 生成程序下的 README 文档： | 编程语言 | 描述 | IPv4 支持 | IPv6 支持 | | --- | --- | --- | --- | | [Golang](maker/golang/README_zh.md) | golang xdb 生成程序 | :white_check_mark: | :white_check_mark: | | [Java](maker/java/README_zh.md) | java xdb 生成程序 | :white_check_mark: | :white_check_mark: | | [Python](maker/python/README_zh.md) | python xdb 生成程序 | :white_check_mark: | :x: | | [Csharp](maker/csharp/README_zh.md) | csharp xdb 生成程序 | :white_check_mark: | :x: | | [Rust](maker/rust/README_zh.md) | rust xdb 生成程序 | :white_check_mark: | :white_check_mark: | | [C++](maker/cpp) | C++ xdb 生成程序 | :white_check_mark: | :white_check_mark: | # `xdb` 更新 ip2region 项目的核心在于 研究 IP 数据的存储和快速查询的设计和实现， 项目自带的 `./data/ipv4_source.txt` 和 `./data/ipv6_source.txt` 原始数据不定期更新，对于数据精度和更新频率要求很高的使用场景建议到 [Ip2Region社区](https://ip2region.net/products/offline) 或者第三方购买商用离线数据，你可以使用如下几种方式来尝试自己更新数据： ### 手动编辑更新 你可以基于 ip2region 自带的 `./data/ipv4_source.txt` 和 `./data/ipv6_source.txt` 原始 IP 数据用 ip2region 提供的编辑工具来自己修改，目前数据源有如下几种方式： 1. ip2region 社区提供的数据（请参考地底部的公众号关注社区通知） 2. ip2region Github/Gitee 中带有 `[数据源补充]` 标签的 Issue 3. 其他自定义数据：例如客户提供的数据，或者通过 GPS 和 WIFI 定位得到的数据，或者来自其他平台的合法合规的数据 原始 IP 数据编辑工具使用方法请参考如下的 `maker` 生成程序下的 README 文档： | 编程语言 | 描述 | IPv4 支持 | IPv6 支持 | | --- | --- | --- | --- | | [Golang](maker/golang/README_zh.md#xdb-数据编辑) | golang IP 原始数据编辑器 | :white_check_mark: | :white_check_mark: | | [C++](maker/cpp) | C++ IP 原始数据编辑器 | :white_check_mark: | :white_check_mark: | ### 检测自动更新 如果你想通过你自己的 API 或数据源来更新数据，你可以参考以下视频分享的 `基于检测算法` 的更新算法来自己编写一个更新程序： 1. [数据更新实现视频分享 - part1](https://www.bilibili.com/video/BV1934y1E7Q5/) 2. [数据更新实现视频分享 - part2](https://www.bilibili.com/video/BV1pF411j7Aw/) # 官方社区 Ip2Region 官方社区正式上线于 `2025/06/12` 日，一方面提供了稳定的 [商用离线数据](https://ip2region.net/products/offline) 服务，另一方面便于在核心代码外强化 IP 工具链和数据服务，例如 [使用文档](https://ip2region.net/doc/)，[查询测试](https://ip2region.net/search/demo)，数据纠错等，更多关于社区的信息和服务请访问 [Ip2Region 官方社区](https://ip2region.net/)。 # 相关备注 ### 1、xdb 技术文档： 1. xdb 数据结构分析：[“ip2region xdb-数据结构描述“](https://ip2region.net/doc/xdb/structure) 2. xdb 查询过程分析：[“ip2region xdb-查询过程描述”](https://ip2region.net/doc/xdb/search) 3. xdb 生成过程分析：[“ip2region xdb-生成过程描述”](https://ip2region.net/doc/xdb/generate) 4. xdb 文件生成教程：[“ip2region xdb-文件生成教程”](https://ip2region.net/doc/data/xdb_make) 5. xdb 并发安全查询：[“ip2region xdb-并发安全查询”](https://ip2region.net/doc/xdb/concurrent) 6. xdb 数据更新方法：[“ip2region 数据更新和 xdb 数据编辑器的使用”](https://mp.weixin.qq.com/s/cZH5qIn4E5rQFy6N32RCzA) ### 3、技术信息博客 1. 微信公众号 - lionsoul-org，作者活跃的技术分享渠道 2. [Ip2Region 官方社区](https://ip2region.net) ================================================ FILE: binding/c/Makefile ================================================ all: xdb_searcher test_util xdb_searcher: xdb_api.h xdb_util.c xdb_searcher.c main.c gcc -std=c99 -Wall -O2 -I./ xdb_util.c xdb_searcher.c main.c -o xdb_searcher test_util: xdb_api.h xdb_util.c test_util.c gcc -std=c99 -Wall -O2 -I./ xdb_util.c test_util.c -o test_util xdb_searcher.o: xdb_searcher.c gcc -std=c99 -Wall -c xdb_searcher.c xdb_util.o: xdb_util.c gcc -std=c99 -Wall -c xdb_util.c xdb_searcher_lib: xdb_util.o xdb_searcher.o mkdir -p build/lib mkdir -p build/include ar -rc build/lib/libxdb.a `find . -name "*.o"` cp xdb_api.h build/include clean: find ./ -name \*.o | xargs rm -f find ./ -name test_util | xargs rm -f find ./ -name xdb_searcher | xargs rm -f rm -rf build .PHONY: all clean xdb_searcher test_util ================================================ FILE: binding/c/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region c Query Client # Usage ### About Query API The prototype of the Query API is as follows: ```c // Query via string IP int xdb_search_by_string(xdb_searcher_t *, const string_ip_t *, xdb_region_buffer_t *); // Query via binary IP returned by xdb_parse_ip int xdb_search(xdb_searcher_t *, const bytes_ip_t *, int, xdb_region_buffer_t *); ``` If the query fails, a non-`0` error code will be returned. If the query is successful, the `region` information string can be obtained from `xdb_region_buffer_t`. If the input IP cannot be found, `xdb_region_buffer_t` will receive an empty string `""`. ### About IPv4 and IPv6 This xdb query client implementation supports both IPv4 and IPv6 queries. The usage is as follows: ```c #include "xdb_api.h"; // For IPv4: Set xdb path to the v4 xdb file, specify IP version as IPv4 const char *db_path = "../../data/ip2region_v4.xdb"; // or your ipv4 xdb path xdb_version_t *version = XDB_IPv4; // For IPv6: Set xdb path to the v6 xdb file, specify IP version as IPv6 const char *db_path = "../../data/ip2region_v6.xdb"; // or your ipv6 xdb path xdb_version_t *version = XDB_IPv6; // The IP version of the xdb specified by db_path must be consistent with the version, otherwise an error will occur during query execution // Note: The following demonstration directly uses db_path and version variables ``` ### XDB File Verification It is recommended that you proactively verify the applicability of the xdb file, as some future new features may cause the current Searcher version to be incompatible with the xdb file you are using. Verification can avoid unpredictable errors during runtime. You do not need to verify every time; for example, verify when the service starts or manually call a command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios. ```c #include "xdb_api.h"; int errcode = xdb_verify(db_path); if ($err != 0) { // Applicability verification failed!!! // The current query client implementation is not suitable for querying the xdb file specified by db_path. // You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with db_path. printf("failed to verify xdb file `%s`, errcode: %d\n", db_path, errcode); return; } // Verification passed, the current Searcher can be safely used for query operations on the xdb pointed to by dbPath ``` ### File-Based Query ```c #include #include "xdb_api.h" int main(int argc, char *argv[]) { xdb_searcher_t searcher; char region_buffer[512] = {'\0'}; xdb_region_buffer_t region; // Initialize region_buffer_t using region_buffer from stack space int err = xdb_region_buffer_init(®ion, region_buffer, sizeof(region_buffer)); if (err != 0) { printf("failed to init the region buffer with errcode=%d\n", err); return 1; } // Initialize winsock when the service starts; no need to call repeatedly, only needed on Windows systems err = xdb_init_winsock(); if (err != 0) { printf("failed to init the winsock with errno=%d\n", err); return 1; } // 1. Initialize xdb query object from db_path. // @Note: Use the db_path and version described above to create the searcher err = xdb_new_with_file_only(version, &searcher, db_path); if (err != 0) { printf("failed to create xdb searcher from `%s` with errno=%d\n", db_path, err); return 1; } // 2. Call search API to query, both IPv4 and IPv6 are supported. const char *ip_string = "1.2.3.4"; // ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long cost_time = 0, s_time = xdb_now(); err = xdb_search_by_string(&searcher, ip_string, ®ion); cost_time = (int) (xdb_now() - s_time); if (err != 0) { printf("failed search(%s) with errno=%d\n", ip_string, err); } else { printf("{region: %s, took: %d μs}", region.value, cost_time); } // Clean up memory resources for region info; must be called after every search xdb_region_buffer_free(®ion); // Note: For concurrent use, each thread needs to define and initialize its own searcher query object independently. // 3. Close xdb searcher xdb_close(&searcher); xdb_clean_winsock(); // Call on Windows return 0; } ``` ### Caching `VectorIndex` We can pre-load VectorIndex data from the xdb file and cache it globally. Using the global VectorIndex cache every time a Searcher object is created can reduce a fixed IO operation, thereby accelerating queries and reducing IO pressure. ```c #include #include "xdb_api.h" int main(int argc, char *argv[]) { xdb_vector_index_t *v_index; xdb_searcher_t searcher; xdb_region_buffer_t region; // Initialize region_buffer with NULL to let it manage memory allocation automatically int err = xdb_region_buffer_init(®ion, NULL, 0); if (err != 0) { printf("failed to init the region buffer with errcode=%d\n", err); return 0; } // Initialize winsock when the service starts; no need to call repeatedly, only needed on Windows systems err = xdb_init_winsock(); if (err != 0) { printf("failed to init the winsock with errno=%d\n", err); return 1; } // 1. Load VectorIndex from the db_path described above. // Obtain v_index to create a global cache for subsequent repeated use. // Note: v_index does not need to be loaded every time; it is recommended to load it once at service startup as a global resource. v_index = xdb_load_vector_index_from_file(db_path); if (v_index == NULL) { printf("failed to load vector index from `%s`\n", db_path); return 1; } // 2. Use the global VectorIndex variable to create an xdb searcher with VectorIndex cache. // @Note: Use the db_path and version described above to create the searcher err = xdb_new_with_vector_index(version, &searcher, db_path, v_index); if (err != 0) { printf("failed to create vector index cached searcher with errcode=%d\n", err); return 2; } // 3. Call search API to query, both IPv4 and IPv6 are supported const char *ip_string = "1.2.3.4"; // ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long cost_time = 0, s_time = xdb_now(); err = xdb_search_by_string(&searcher, ip_string, ®ion); cost_time = (int) (xdb_now() - s_time); if (err != 0) { printf("failed search(%s) with errno=%d\n", ip_string, err); } else { printf("{region: %s, took: %d μs}", region.value, cost_time); } // Clean up memory resources for region info; must be called after every search xdb_region_buffer_free(®ion); // Note: For concurrent use, each thread needs to define and initialize its own searcher query object independently. // 4. Close xdb searcher; if the service is being shut down, the memory for v_index also needs to be freed. xdb_close(&searcher); xdb_close_vector_index(v_index); xdb_clean_winsock(); return 0; } ``` ### Caching the Entire `xdb` File We can also pre-load the entire xdb file into memory and then create a query object based on this data to achieve fully memory-based queries, similar to the previous memory search. ```c #include #include "xdb_api.h" int main(int argc, char *argv[]) { xdb_content_t *c_buffer; xdb_searcher_t searcher; xdb_region_buffer_t region; // Initialize region_buffer with NULL to let it manage memory allocation automatically int err = xdb_region_buffer_init(®ion, NULL, 0); if (err != 0) { printf("failed to init the region buffer with errcode=%d\n", err); return 0; } // Initialize winsock when the service starts; no need to call repeatedly, only needed on Windows systems err = xdb_init_winsock(); if (err != 0) { printf("failed to init the winsock with errno=%d\n", err); return 1; } // 1. Load the entire xdb data from the db_path described above. c_buffer = xdb_load_content_from_file(db_path); if (v_index == NULL) { printf("failed to load xdb content from `%s`\n", db_path); return 1; } // 2. Use the global c_buffer variable to create a fully memory-based xdb query object. // @Note: Use the version described above to create the searcher. err = xdb_new_with_buffer(version, &searcher, c_buffer); if (err != 0) { printf("failed to create content cached searcher with errcode=%d\n", err); return 2; } // 3. Call search API to query, both IPv4 and IPv6 are supported const char *ip_string = "1.2.3.4"; // ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long cost_time = 0, s_time = xdb_now(); err = xdb_search_by_string(&searcher, ip_string, ®ion); cost_time = (int) (xdb_now() - s_time); if (err != 0) { printf("failed search(%s) with errno=%d\n", ip_string, err); } else { printf("{region: %s, took: %d μs}", region.value, cost_time); } // Clean up memory resources for region info; must be called after every search xdb_region_buffer_free(®ion); // Note: For concurrent use, xdb query objects created this way can be safely used for concurrency. // It is recommended to create them when the service starts and then use them safely in parallel until the service shuts down. // 4. Close xdb searcher; memory for c_buffer needs to be freed when shutting down the service. xdb_close(&searcher); xdb_close_content(c_buffer); xdb_clean_winsock(); return 0; } ``` ### About Storage of Location Information In older implementations, search-related functions relied on a specified `region_buffer` memory to store location information, which had significant limitations. The new implementation provides an `xdb_region_buffer_t` object to manage these memory allocations. You can still specify a fixed `region_buffer` to create memory management for the region; this is suitable when the maximum length of your location information is known, which helps reduce memory fragmentation during runtime. If the length of the location information is uncertain or if your program is not suited for pre-allocating a block of memory, you can initialize `xdb_region_buffer_t` by specifying `NULL`. In this case, the object will automatically manage memory allocation, making it suitable for storing location information of any length, though this approach will certainly increase memory fragmentation over long-term operation. ```c // 1. Create region_buffer by specifying a memory block char buffer[512]; xdb_region_buffer_t region; int err = xdb_region_buffer_init(®ion, buffer, sizeof(buffer)); if (err != 0) { // Initialization failed printf("failed to init region buffer width errcode=%d", err); return; } // 2. Create region_buffer by specifying NULL to let it allocate memory as needed automatically xdb_region_buffer_t region; int err = xdb_region_buffer_init(®ion, NULL, 0); if (err != 0) { // Initialization failed printf("failed to init region buffer width errcode=%d", err); return; } // Note: After each query call, you must manually call the function to free memory. // The search function will report an error if used with uncleaned region info. xdb_region_buffer_free(®ion); ``` # Compiling the Test Program Compile and obtain the `xdb_searcher` executable as follows: ```bash # cd to the c binding root directory ➜ c git:(master) ✗ make gcc -std=c99 -Wall -O2 -I./ xdb_util.c xdb_searcher.c main.c -o xdb_searcher gcc -std=c99 -Wall -O2 -I./ xdb_util.c test_util.c -o test_util ``` # Query Testing Test queries against xdb via the `xdb_searcher search` command: ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search ./xdb_searcher search [command options] options: --db string ip2region binary xdb file path --cache-policy string cache policy: file/vectorIndex/content ``` Example: performing IPv4 query testing using the default data/ip2region_v4.xdb: ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 39 μs} ip2region>> 120.229.45.2 {region: 中国|广东省|深圳市|移动|CN, io_count: 3, took: 13 μs} ``` Example: performing IPv6 query testing using the default data/ip2region_v6.xdb: ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> :: {region: , io_count: 1, took: 38 μs} ip2region>> 2604:bc80:8001:11a4:ffff:ffff:ffff:ffff {region: United States|Florida|Miami|velia.net Internetdienste GmbH|US, io_count: 14, took: 76 μs} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 42 μs} ``` Enter an IP to perform a query; enter `quit` to exit the test program. You can also set `cache-policy` to file/vectorIndex/content respectively to test the efficiency of the three different cache implementations. # bench Testing Perform bench testing via the `xdb_searcher bench` command. This ensures there are no errors in the query program and the `xdb` file, while also providing average query performance through a large number of queries: ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench ./xdb_searcher bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` Example: performing IPv4 bench testing via the default data/ip2region_v4.xdb and data/ipv4_source.txt: ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt Bench finished, {cache_policy: vectorIndex, total: 1367686, took: 7.640s, cost: 5 μs/op} ``` Example: performing IPv6 bench testing via the default data/ip2region_v6.xdb and data/ipv6_source.txt: ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt Bench finished, {cache_policy: vectorIndex, total: 34159862, took: 857.750s, cost: 24 μs/op} ``` You can set the `cache-policy` parameter to test the efficiency of different cache mechanisms (file/vectorIndex/content). @Note: Please ensure that the `src` file used for benching is the same source file used to generate the corresponding `xdb` file. ================================================ FILE: binding/c/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region c 查询客户端 # 使用方式 ### 关于查询 API 查询 API 的原型如下： ```c // 通过字符串 IP 进行查询 int xdb_search_by_string(xdb_searcher_t *, const string_ip_t *, xdb_region_buffer_t *); // 通过 xdb_parse_ip 返回的二进制 IP 进行查询 int xdb_search(xdb_searcher_t *, const bytes_ip_t *, int, xdb_region_buffer_t *); ``` 如果查询失败将会返回非 `0` 的错误代码，如果查询成功 xdb_region_buffer_t 可以获取到字符串的 `region` 信息，如果输入的 IP 找不到相关的信息，xdb_region_buffer_t 将会得到一个空的字符串 `""`。 ### 关于 IPv4 和 IPv6 该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询，使用方式如下： ```c #include "xdb_api.h"; // 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 IPv4 const char *db_path = "../../data/ip2region_v4.xdb"; // 或者你的 ipv4 xdb 的路径 xdb_version_t *version = XDB_IPv4; // 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件，IP版本指定为 IPv6 const char *db_path = "../../data/ip2region_v6.xdb"; // 或者你的 ipv6 xdb 路径 xdb_version_t *version = XDB_IPv6; // db_path 指定的 xdb 的 IP 版本必须和 version 指定的一致，不然查询执行的时候会报错 // 备注：以下演示直接使用 db_path 和 version 变量 ``` ### XDB 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 ```c #include "xdb_api.h"; int errcode = xdb_verify(db_path); if ($err != 0) { // 适用性验证失败！！！ // 当前查询客户端实现不适用于 db_path 指定的 xdb 文件的查询. // 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 db_path 的 Searcher 实现。 printf("failed to verify xdb file `%s`, errcode: %d\n", db_path, errcode); return; } // 验证通过，当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作 ``` ### 完全基于文件的查询 ```c #include #include "xdb_api.h" int main(int argc, char *argv[]) { xdb_searcher_t searcher; char region_buffer[512] = {'\0'}; xdb_region_buffer_t region; // 使用栈空间的 region_buffer 初始化 region_buffer_t int err = xdb_region_buffer_init(®ion, region_buffer, sizeof(region_buffer)); if (err != 0) { printf("failed to init the region buffer with errcode=%d\n", err); return 1; } // 在服务启动的时候初始化 winsock，不需要重复调用，只需要在 windows 系统下调用 err = xdb_init_winsock(); if (err != 0) { printf("failed to init the winsock with errno=%d\n", err); return 1; } // 1、从 db_path 初始化 xdb 查询对象. // @Note: 使用顶部描述的 db_path 和 version 来创建 searcher err = xdb_new_with_file_only(version, &searcher, db_path); if (err != 0) { printf("failed to create xdb searcher from `%s` with errno=%d\n", db_path, err); return 1; } // 2、调用 search API 查询，IPv4 和 IPv6 都支持. const char *ip_string = "1.2.3.4"; // ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long cost_time = 0, s_time = xdb_now(); err = xdb_search_by_string(&searcher, ip_string, ®ion); cost_time = (int) (xdb_now() - s_time); if (err != 0) { printf("failed search(%s) with errno=%d\n", ip_string, err); } else { printf("{region: %s, took: %d μs}", region.value, cost_time); } // 清理 region 信息的内存资源，每次 search 之后都得调用 xdb_region_buffer_free(®ion); // 备注：并发使用，每一个线程需要单独定义并且初始化一个 searcher 查询对象。 // 3、关闭 xdb 查询器 xdb_close(&searcher); xdb_clean_winsock(); // windows 下调用 return 0; } ``` ### 缓存 `VectorIndex` 索引 我们可以提前从 xdb 文件中加载出来 VectorIndex 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。 ```c #include #include "xdb_api.h" int main(int argc, char *argv[]) { xdb_vector_index_t *v_index; xdb_searcher_t searcher; xdb_region_buffer_t region; // 使用 NULL 初始化 region_buffer，让其自动管理内存的分配 int err = xdb_region_buffer_init(®ion, NULL, 0); if (err != 0) { printf("failed to init the region buffer with errcode=%d\n", err); return 0; } // 在服务启动的时候初始化 winsock，不需要重复调用，只需要在 windows 系统下调用 err = xdb_init_winsock(); if (err != 0) { printf("failed to init the winsock with errno=%d\n", err); return 1; } // 1、从顶部描述的 db_path 加载 VectorIndex 索引。 // 得到 v_index 做成全局缓存，便于后续反复使用。 // 注意：v_index 不需要每次都加载，建议在服务启动的时候加载一次，然后做成全局资源。 v_index = xdb_load_vector_index_from_file(db_path); if (v_index == NULL) { printf("failed to load vector index from `%s`\n", db_path); return 1; } // 2、使用全局的 VectorIndex 变量创建带 VectorIndex 缓存的 xdb 查询对象. // @Note: 使用顶部描述的 db_path 和 version 来创建 searcher err = xdb_new_with_vector_index(version, &searcher, db_path, v_index); if (err != 0) { printf("failed to create vector index cached searcher with errcode=%d\n", err); return 2; } // 3、调用 search API 查询，IPv4 和 IPv6 都支持 const char *ip_string = "1.2.3.4"; // ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long cost_time = 0, s_time = xdb_now(); err = xdb_search_by_string(&searcher, ip_string, ®ion); cost_time = (int) (xdb_now() - s_time); if (err != 0) { printf("failed search(%s) with errno=%d\n", ip_string, err); } else { printf("{region: %s, took: %d μs}", region.value, cost_time); } // 清理 region 信息的内存资源，每次 search 之后都得调用 xdb_region_buffer_free(®ion); // 备注：并发使用，每一个线程需要单独定义并且初始化一个 searcher 查询对象。 // 4、关闭 xdb 查询器，如果是要关闭服务，也需要释放 v_index 的内存。 xdb_close(&searcher); xdb_close_vector_index(v_index); xdb_clean_winsock(); return 0; } ``` ### 缓存整个 `xdb` 文件 我们也可以预先加载整个 xdb 文件到内存，然后基于这个数据创建查询对象来实现完全基于内存的查询，类似之前的 memory search。 ```c #include #include "xdb_api.h" int main(int argc, char *argv[]) { xdb_content_t *c_buffer; xdb_searcher_t searcher; xdb_region_buffer_t region; // 使用 NULL 初始化 region_buffer，让其自动管理内存的分配 int err = xdb_region_buffer_init(®ion, NULL, 0); if (err != 0) { printf("failed to init the region buffer with errcode=%d\n", err); return 0; } // 在服务启动的时候初始化 winsock，不需要重复调用，只需要在 windows 系统下调用 err = xdb_init_winsock(); if (err != 0) { printf("failed to init the winsock with errno=%d\n", err); return 1; } // 1、从 顶部描述的 db_path 加载整个 xdb 的数据。 c_buffer = xdb_load_content_from_file(db_path); if (v_index == NULL) { printf("failed to load xdb content from `%s`\n", db_path); return 1; } // 2、使用全局的 c_buffer 变量创建一个完全基于内存的 xdb 查询对象. // @Note: 使用顶部描述的 version 来创建 searcher. err = xdb_new_with_buffer(version, &searcher, c_buffer); if (err != 0) { printf("failed to create content cached searcher with errcode=%d\n", err); return 2; } // 3、调用 search API 查询，IPv4 和 IPv6 都支持 const char *ip_string = "1.2.3.4"; // ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long cost_time = 0, s_time = xdb_now(); err = xdb_search_by_string(&searcher, ip_string, ®ion); cost_time = (int) (xdb_now() - s_time); if (err != 0) { printf("failed search(%s) with errno=%d\n", ip_string, err); } else { printf("{region: %s, took: %d μs}", region.value, cost_time); } // 清理 region 信息的内存资源，每次 search 之后都得调用 xdb_region_buffer_free(®ion); // 备注：并发使用，使用这种方式创建的 xdb 查询对象可以安全用于并发。 // 建议在服务启动的时候创建好，然后一直安全并发使用，直到服务关闭。 // 4、关闭 xdb 查询器，关闭服务的时候需要释放 c_buffer 的内存。 xdb_close(&searcher); xdb_close_content(c_buffer); xdb_clean_winsock(); return 0; } ``` ### 关于定位信息的存储 在旧版本的实现中，search相关的函数都是依靠指定一个 `region_buffer` 内存来用于存储地域信息，这种方式还是有很大的局限性。 新的实现提供了一个 `xdb_region_buffer_t` 对象来管理这些内存的分配，你依然可以指定一个固定的 `region_buffer` 来创建 region 的内存管理，这个情况适合当你的地域信息的最大长度是可知的，这种方式可以减少运行过程中内存的碎片堆积。如果地域信息的长度不确定或者你的程序不适合提前分配一块内存来管理，你可以通过指定 NULL 的方式来初始化 `xdb_region_buffer_t`，这样对象会自动管理内存的分配，也适合任意长度的地域信息的存储，不过这种方式在长期的运行过程中肯定会增加内存碎片的堆积。 ```c // 1, 通过指定一块内存来创建 region_buffer char buffer[512]; xdb_region_buffer_t region; int err = xdb_region_buffer_init(®ion, buffer, sizeof(buffer)); if (err != 0) { // 初始化失败 printf("failed to init region buffer width errcode=%d", err); return; } // 2，通过指定 NULL 来创建 region_buffer，让其自动按需分配内存 xdb_region_buffer_t region; int err = xdb_region_buffer_init(®ion, NULL, 0); if (err != 0) { // 初始化失败 printf("failed to init region buffer width errcode=%d", err); return; } // 备注：在每次调用 search 完成 IP 定位信息的查询后，你需要手动调用函数来释放内存 . // search 函数使用未经清理的 region 信息会报错。 xdb_region_buffer_free(®ion); ``` # 测试程序编译 通过如下方式编译得到 xdb_searcher 可执行程序： ```bash # cd 到 c binding 根目录 ➜ c git:(master) ✗ make gcc -std=c99 -Wall -O2 -I./ xdb_util.c xdb_searcher.c main.c -o xdb_searcher gcc -std=c99 -Wall -O2 -I./ xdb_util.c test_util.c -o test_util ``` # 查询测试 通过 `xdb_searcher search` 命令来测试对 xdb 的查询： ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search ./xdb_searcher search [command options] options: --db string ip2region binary xdb file path --cache-policy string cache policy: file/vectorIndex/content ``` 例如：使用默认的 data/ip2region_v4.xdb 进行 IPv4 查询测试： ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 39 μs} ip2region>> 120.229.45.2 {region: 中国|广东省|深圳市|移动|CN, io_count: 3, took: 13 μs} ``` 例如：使用默认的 data/ip2region_v6.xdb 进行 IPv6 查询测试： ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> :: {region: , io_count: 1, took: 38 μs} ip2region>> 2604:bc80:8001:11a4:ffff:ffff:ffff:ffff {region: United States|Florida|Miami|velia.net Internetdienste GmbH|US, io_count: 14, took: 76 μs} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 42 μs} ``` 输入 ip 即可进行查询，输入 quit 即可退出测试程序。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同的缓存实现的效率。 # bench 测试 通过 `xdb_searcher bench` 命令来进行 bench 测试，一方面确保查询程序和 `xdb` 文件没有错误，另一方面可以通过大量的查询得到平均的查询性能： ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench ./xdb_searcher bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` 例如：通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 来进行 IPv4 的 bench 测试： ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt Bench finished, {cache_policy: vectorIndex, total: 1367686, took: 7.640s, cost: 5 μs/op} ``` 例如：通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 来进行 IPv6 的 bench 测试： ```bash ➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt Bench finished, {cache_policy: vectorIndex, total: 34159862, took: 857.750s, cost: 24 μs/op} ``` 可以设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 不同缓存实现机制的效率。 @Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件相同的源文件。 ================================================ FILE: binding/c/main.c ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/28 #include "stdio.h" #include "xdb_api.h" struct searcher_test_entry { xdb_searcher_t searcher; xdb_vector_index_t *v_index; xdb_content_t *c_buffer; // xdb region buffer // char region_buffer[256]; xdb_region_buffer_t region; }; typedef struct searcher_test_entry searcher_test_t; int init_searcher_test(searcher_test_t *test, char *db_path, char *cache_policy) { int err, errcode = 0; FILE *handle = fopen(db_path, "rb"); if (handle == NULL) { return -1; } // auto detect the version from the xdb header xdb_header_t *header = xdb_load_header(handle); if (header == NULL) { printf("failed to load header from `%s`\n", db_path); errcode = 1; goto defer; } // verify the current xdb err = xdb_verify_from_header(handle, header); if (err != 0) { printf("failed to verify xdb file `%s` with errno=%d\n", db_path, err); errcode = 2; goto defer; } xdb_version_t *version = xdb_version_from_header(header); if (version == NULL) { printf("failed to load version from header\n"); errcode = 3; goto defer; } test->v_index = NULL; test->c_buffer = NULL; if (strcmp(cache_policy, "file") == 0) { err = xdb_new_with_file_only(version, &test->searcher, db_path); if (err != 0) { printf("failed to create searcher with errcode=%d\n", err); errcode = 4; goto defer; } } else if (strcmp(cache_policy, "vectorIndex") == 0) { test->v_index = xdb_load_vector_index_from_file(db_path); if (test->v_index == NULL) { printf("failed to load vector index from `%s`\n", db_path); errcode = 4; goto defer; } err = xdb_new_with_vector_index(version, &test->searcher, db_path, test->v_index); if (err != 0) { printf("failed to create vector index cached searcher with errcode=%d\n", err); errcode = 5; goto defer; } } else if (strcmp(cache_policy, "content") == 0) { test->c_buffer = xdb_load_content_from_file(db_path); if (test->c_buffer == NULL) { printf("failed to load xdb content from `%s`\n", db_path); errcode = 4; goto defer; } err = xdb_new_with_buffer(version, &test->searcher, test->c_buffer); if (err != 0) { printf("failed to create content cached searcher with errcode=%d\n", err); errcode = 5; goto defer; } } else { printf("invalid cache policy `%s`, options: file/vectorIndex/content\n", cache_policy); errcode = 6; goto defer; } // init the region buffer // err = xdb_region_buffer_init(&test->region, test->region_buffer, sizeof(test->region_buffer)); err = xdb_region_buffer_init(&test->region, NULL, 0); if (err != 0) { printf("failed to init the region buffer with err=%d\n", err); errcode = 7; goto defer; } defer: if (header != NULL) { xdb_free_header(header); } if (handle != NULL) { fclose(handle); } return errcode; } void destroy_searcher_test(searcher_test_t *test) { xdb_close(&test->searcher); // check and free the vector index if (test->v_index != NULL) { xdb_free_vector_index(test->v_index); test->v_index = NULL; } // check and free the content buffer if (test->c_buffer != NULL) { xdb_free_content(test->c_buffer); test->c_buffer = NULL; } } //read a line from a command line. static char *get_line(FILE *fp, char *__dst) { register int c; register char *cs; cs = __dst; while ( ( c = getc( fp ) ) != EOF ) { if ( c == '\n' ) break; *cs++ = c; } *cs = '\0'; return ( c == EOF && cs == __dst ) ? NULL : __dst; } void print_help(char *argv[]) { printf("ip2region xdb searcher\n"); printf("%s [command] [command options]\n", argv[0]); printf("Command: \n"); printf(" search search input test\n"); printf(" bench search bench test\n"); } void test_search(int argc, char *argv[]) { int i, n, err; // for args parse char *r, key[33] = {'\0'}, val[256] = {'\0'}; char db_file[256] = {'\0'}, cache_policy[16] = {"vectorIndex"}; // for search long s_time, c_time; char line[512] = {'\0'}; // ip parse xdb_version_t *version; bytes_ip_t ip_bytes[16] = {'\0'}; searcher_test_t test; for (i = 2; i < argc; i++) { r = argv[i]; if (strlen(r) < 5) { continue; } if (r[0] != '-' || r[1] != '-') { continue; } if (strchr(r, '=') == NULL) { printf("missing = for args pair '%s'\n", r); return; } n = sscanf(r+2, "%32[^=]=%255[^\n]", key, val); if (n != 2) { printf("invalid option flag `%s`\n", r); return; } // printf("key=%s, val=%s\n", key, val); if (strcmp(key, "db") == 0) { snprintf(db_file, sizeof(db_file), "%s", val); } else if (strcmp(key, "cache-policy") == 0) { memcpy(cache_policy, val, sizeof(cache_policy) - 1); // snprintf(cache_policy, sizeof(cache_policy), "%s", val); } else { printf("undefined option `%s`\n", r); return; } } if (strlen(db_file) < 1) { printf("%s search [command options]\n", argv[0]); printf("options:\n"); printf(" --db string ip2region binary xdb file path\n"); printf(" --cache-policy string cache policy: file/vectorIndex/content\n"); return; } // init the win sock err = xdb_init_winsock(); if (err != 0) { printf("failed to init the winsock with errno=%d\n", err); return; } // printf("db_file=%s, cache_policy=%s\n", db_file, cache_policy); err = init_searcher_test(&test, db_file, cache_policy); if (err != 0) { // init program will print the error reasons; return; } printf("ip2region xdb searcher test program\n" "source xdb: %s (%s, %s)\n" "type 'quit' to exit\n", db_file, xdb_get_version(&test.searcher)->name, cache_policy); while ( 1 ) { printf("ip2region>> "); get_line(stdin, line); if ( strlen(line) < 2 ) { continue; } if (strcmp(line, "quit") == 0 ) { break; } version = xdb_parse_ip(line, ip_bytes, sizeof(ip_bytes)); if (version == NULL) { printf("invalid ip address `%s`\n", line); continue; } s_time = xdb_now(); err = xdb_search(&test.searcher, ip_bytes, version->bytes, &test.region); if (err != 0) { printf("{err: %d, io_count: %d}\n", err, xdb_get_io_count(&test.searcher)); } else { c_time = xdb_now() - s_time; printf("{region: %s, io_count: %d, took: %ld μs}\n", test.region.value, xdb_get_io_count(&test.searcher), c_time); } // free the region xdb_region_buffer_free(&test.region); } destroy_searcher_test(&test); xdb_clean_winsock(); printf("searcher test program exited, thanks for trying\n"); } void test_bench(int argc, char *argv[]) { int i, n, err; char *r, key[33] = {'\0'}, val[256] = {'\0'}; char db_file[256] = {'\0'}, src_file[256] = {'\0'}, cache_policy[16] = {"vectorIndex"}; FILE *handle; char line[1024] = {'\0'}, sip_str[INET6_ADDRSTRLEN+1] = {'\0'}, eip_str[INET6_ADDRSTRLEN+1] = {'\0'}; char src_region[512] = {'\0'}; int count = 0, took; long s_time, t_time, c_time = 0; // ip parse xdb_version_t *s_version, *e_version; bytes_ip_t sip_bytes[16] = {'\0'}, eip_bytes[16] = {'\0'}; string_ip_t ip_string[INET6_ADDRSTRLEN] = {'\0'}; bytes_ip_t *ip_list[2]; searcher_test_t test; for (i = 2; i < argc; i++) { r = argv[i]; if (strlen(r) < 5) { continue; } if (r[0] != '-' || r[1] != '-') { continue; } if (strchr(r, '=') == NULL) { printf("missing = for args pair '%s'\n", r); return; } n = sscanf(r+2, "%32[^=]=%255[^\n]", key, val); if (n != 2) { printf("invalid option flag `%s`\n", r); return; } if (strcmp(key, "db") == 0) { snprintf(db_file, sizeof(db_file), "%s", val); } else if (strcmp(key, "src") == 0) { snprintf(src_file, sizeof(src_file), "%s", val); } else if (strcmp(key, "cache-policy") == 0) { memcpy(cache_policy, val, sizeof(cache_policy) - 1); } else { printf("undefined option `%s`\n", r); return; } } if (strlen(db_file) < 1 || strlen(src_file) < 1) { printf("%s bench [command options]\n", argv[0]); printf("options:\n"); printf(" --db string ip2region binary xdb file path\n"); printf(" --src string source ip text file path\n"); printf(" --cache-policy string cache policy: file/vectorIndex/content\n"); return; } // init the win sock err = xdb_init_winsock(); if (err != 0) { printf("failed to init the winsock with errno=%d\n", err); return; } // printf("db_file=%s, src_file=%s, cache_policy=%s\n", db_file, src_file, cache_policy); s_time = xdb_now(); err = init_searcher_test(&test, db_file, cache_policy); if (err != 0) { // the init function will print the details; return; } // open the source file handle = fopen(src_file, "r"); if (handle == NULL) { printf("failed to open source text file `%s`\n", src_file); return; } while(fgets(line, sizeof(line), handle) != NULL) { n = sscanf(line, "%46[^|]|%46[^|]|%511[^\n]", sip_str, eip_str, src_region); if (n != 3) { printf("invalid ip segment line `%s`\n", line); return; } s_version = xdb_parse_ip(sip_str, sip_bytes, sizeof(sip_bytes)); if (s_version == NULL) { printf("invalid start ip `%s`\n", sip_str); return; } e_version = xdb_parse_ip(eip_str, eip_bytes, sizeof(eip_bytes)); if (e_version == NULL) { printf("invalid end ip `%s`\n", sip_str); return; } if (s_version->id != e_version->id) { printf("start ip and end ip version not match for line `%s`\n", line); return; } if (xdb_ip_sub_compare(sip_bytes, s_version->bytes, (string_ip_t *) eip_bytes, 0) > 0) { printf("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str); return; } ip_list[0] = sip_bytes; ip_list[1] = eip_bytes; for (i = 0; i < 2; i++) { t_time = xdb_now(); err = xdb_search(&test.searcher, ip_list[i], s_version->bytes, &test.region); c_time += xdb_now() - t_time; if (err != 0) { xdb_ip_to_string(ip_list[i], s_version->bytes, ip_string, sizeof(ip_string)); printf("failed to search ip `%s` with errno=%d\n", ip_string, err); return; } // check the region info if (strcmp(test.region.value, src_region) != 0) { xdb_ip_to_string(ip_list[i], s_version->bytes, ip_string, sizeof(ip_string)); printf("failed to search(%s) with (%s != %s)\n", ip_string, test.region.value, src_region); return; } // free the region buffer xdb_region_buffer_free(&test.region); count++; } }; took = xdb_now() - s_time; destroy_searcher_test(&test); xdb_clean_winsock(); fclose(handle); printf("Bench finished, {cache_policy: %s, total: %d, took: %.3fs, cost: %d μs/op}\n", cache_policy, count, took/1e6, count == 0 ? 0 : (int)(c_time/count)); } int main(int argc, char *argv[]) { if (argc < 2) { print_help(argv); return 0; } char *opt = argv[1]; if (strcmp(opt, "search") == 0) { test_search(argc, argv); } else if (strcmp(opt, "bench") == 0) { test_bench(argc, argv); } else { print_help(argv); } return 0; } ================================================ FILE: binding/c/test_util.c ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/27 #include "stdio.h" #include "xdb_api.h" typedef void (* test_func_ptr) (); struct test_func_entry { char *name; test_func_ptr func; }; typedef struct test_func_entry test_func_t; void test_load_header() { xdb_header_t *header = xdb_load_header_from_file("../../data/ip2region_v4.xdb"); if (header == NULL) { printf("failed to load header"); } else { printf("header loaded: {\n" " version: %d, \n" " index_policy: %d, \n" " created_at: %u, \n" " start_index_ptr: %d, \n" " end_index_ptr: %d\n" " ip_version: %d\n" " runtime_ptr_bytes: %d\n" " length: %d\n" "}\n", header->version, header->index_policy, header->created_at, header->start_index_ptr, header->end_index_ptr, header->ip_version, header->runtime_ptr_bytes, header->length ); } xdb_free_header(header); } void test_load_vector_index() { xdb_vector_index_t *v_index = xdb_load_vector_index_from_file("../../data/ip2region_v4.xdb"); if (v_index == NULL) { printf("failed to load vector index from file\n"); } else { printf("vector index loaded from file, length=%d\n", v_index->length); } xdb_free_vector_index(v_index); } void test_load_content() { xdb_content_t *content = xdb_load_content_from_file("../../data/ip2region_v4.xdb"); if (content == NULL) { printf("failed to load content from file\n"); } else { printf("content loaded from file, length=%d\n", content->length); } xdb_free_content(content); } void test_parse_ip() { const char *ip_list[] = { "1.0.0.0", "58.251.30.115", "192.168.1.100", "::", "2c0f:fff0::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "240e:982:e617:ffff:ffff:ffff:ffff:ffff", "219.xx.xx.11", "::xx:ffff", NULL }; int errcode; xdb_version_t *version; bytes_ip_t ip_bytes[16] = {'\0'}; string_ip_t ip_string[INET6_ADDRSTRLEN] = {'\0'}; // init the sock env (for windows) if ((errcode = xdb_init_winsock()) != 0) { printf("failed to init the winsock"); return; } for (int i = 0;; i++) { if (ip_list[i] == NULL) { break; } version = xdb_parse_ip(ip_list[i], ip_bytes, sizeof(ip_bytes)); if (version == NULL) { printf("failed to parse ip `%s`\n", ip_list[i]); continue; } xdb_ip_to_string(ip_bytes, version->bytes, ip_string, sizeof(ip_string)); printf("ip: %s (version=v%d), toString: %s\n", ip_list[i], version->id, ip_string); } // clean up the winsock xdb_clean_winsock(); } struct ip_pair { const char *sip; const char *eip; }; void test_ip_compare() { struct ip_pair ip_pair_list[] = { {"1.0.0.0", "1.0.0.1"}, {"192.168.1.101", "192.168.1.90"}, {"219.133.111.87", "114.114.114.114"}, {"1.0.4.0", "1.0.1.0"}, {"1.0.4.0", "1.0.3.255"}, {"2000::", "2000:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, {"2001:4:112::", "2001:4:112:ffff:ffff:ffff:ffff:ffff"}, {"ffff::", "2001:4:ffff:ffff:ffff:ffff:ffff:ffff"}, {NULL, NULL} }; struct ip_pair *pair_ptr = NULL; bytes_ip_t sip_bytes[16] = {'\0'}; bytes_ip_t eip_bytes[16] = {'\0'}; xdb_version_t *s_version, *e_version; int errcode; // init the sock env (for windows) if ((errcode = xdb_init_winsock()) != 0) { printf("failed to init the winsock"); return; } for (int i = 0; ;i++) { pair_ptr = &ip_pair_list[i]; if (pair_ptr->sip == NULL) { break; } s_version = xdb_parse_ip(pair_ptr->sip, sip_bytes, sizeof(sip_bytes)); if (s_version == NULL) { printf("failed to parse sip `%s`", pair_ptr->sip); continue; } e_version = xdb_parse_ip(pair_ptr->eip, eip_bytes, sizeof(eip_bytes)); if (e_version == NULL) { printf("failed to parse eip `%s`", pair_ptr->eip); continue; } if (s_version->id != e_version->id) { printf("sip and eip version not match `%s` != `%s`\n", s_version->name, e_version->name); continue; } printf( "ip_sub_compare(%s, %s): %d\n", pair_ptr->sip, pair_ptr->eip, xdb_ip_sub_compare(sip_bytes, s_version->bytes, (string_ip_t *) eip_bytes, 0) ); } // clean up the winsock xdb_clean_winsock(); } // please register your function heare static test_func_t _test_function_list[] = { // xdb buffer {"test_load_header", test_load_header}, {"test_load_vector_index", test_load_vector_index}, {"test_load_content", test_load_content}, // ip utils {"test_parse_ip", test_parse_ip}, {"test_ip_compare", test_ip_compare}, {NULL, NULL} }; // valgrind --tool=memcheck --leak-check=full ./a.out int main(int argc, char *argv[]) { int i; char *name; // check and call the function if (argc < 2) { printf("please specified the function name to call\n"); return 1; } name = argv[1]; test_func_ptr func = NULL; for (i = 0; ; i++) { if (_test_function_list[i].name == NULL) { break; } if (strcmp(name, _test_function_list[i].name) == 0) { func = _test_function_list[i].func; break; } } if (func == NULL) { printf("can't find test function `%s`\n", name); return 1; } // call the function func(); return 0; } ================================================ FILE: binding/c/xdb_api.h ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/27 #ifndef C_IP2REGION_XDB_H #define C_IP2REGION_XDB_H // @Note: // this define must be put before any header include // force the LFS for ftell #define _FILE_OFFSET_BITS 64 #include #include #include #if ( defined(WIN32) || defined(_WIN32) || defined(__WINDOWS_) || defined(WINNT) ) # define XDB_PUBLIC(type) extern __declspec(dllexport) type # define XDB_PRIVATE(type) static type # define XDB_WINDOWS #include #include #include #pragma comment(lib, "ws2_32.lib") #elif (defined(linux) || defined(_UNIX) || defined(__APPLE__) || defined(unix) || defined(__unix) || defined(__unix__) || defined(__linux__) || defined(linux) || defined(__linux)) # define XDB_PUBLIC(type) extern type # define XDB_PRIVATE(type) static inline type # define XDB_LINUX #include #include #include #else # define XDB_PUBLIC(type) type # define XDB_PRIVATE(type) static type #endif #define xdb_calloc( _blocks, _bytes ) calloc( _blocks, _bytes ) #define xdb_malloc( _bytes ) malloc( _bytes ) #define xdb_free( _ptr ) free( _ptr ) // public constants define #define xdb_structure_20 2 #define xdb_structure_30 3 #define xdb_header_info_length 256 #define xdb_vector_index_rows 256 #define xdb_vector_index_cols 256 #define xdb_vector_index_size 8 #define xdb_v4_index_size 14 // 4 + 4 + 2 + 4 #define xdb_v6_index_size 38 // 16 + 16 + 2 + 4 // --- ip version info #define xdb_ipv4_id 4 #define xdb_ipv6_id 6 #define xdb_ipv4_bytes 4 #define xdb_ipv6_bytes 16 // cache of vector_index_row × vector_index_rows × vector_index_size #define xdb_vector_index_length 524288 // --- xdb buffer functions // use the following buffer struct to wrap the binary buffer data // since the buffer data could not be operated with the string API. struct xdb_header { unsigned short version; unsigned short index_policy; unsigned int created_at; unsigned int start_index_ptr; unsigned int end_index_ptr; // since 3.0+ with IPv6 supporting unsigned short ip_version; unsigned short runtime_ptr_bytes; // the original buffer unsigned int length; char buffer[xdb_header_info_length]; }; typedef struct xdb_header xdb_header_t; XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *); XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *); XDB_PUBLIC(void) xdb_free_header(void *); // --- vector index buffer struct xdb_vector_index { unsigned int length; char buffer[xdb_vector_index_length]; }; typedef struct xdb_vector_index xdb_vector_index_t; XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *); XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *); XDB_PUBLIC(void) xdb_free_vector_index(void *); // --- content buffer struct xdb_content { unsigned int length; char *buffer; }; typedef struct xdb_content xdb_content_t; XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *); XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *); XDB_PUBLIC(void) xdb_free_content(void *); // --- xdb verify // Verify if the current Searcher could be used to search the specified xdb file. // Why do we need this check ? // The future features of the xdb impl may cause the current searcher not able to work properly. // // @Note: You Just need to check this ONCE when the service starts // Or use another process (eg, A command) to check once Just to confirm the suitability. XDB_PUBLIC(int) xdb_verify(FILE *); XDB_PUBLIC(int) xdb_verify_from_header(FILE *handle, xdb_header_t *); XDB_PUBLIC(int) xdb_verify_from_file(const char *); // --- End xdb buffer // types type define typedef char string_ip_t; typedef unsigned char bytes_ip_t; // --- ip version #define XDB_IPv4 (xdb_version_v4()) #define XDB_IPv6 (xdb_version_v6()) typedef int (* ip_compare_fn_t) (const bytes_ip_t *, int, const char *, int); struct xdb_ip_version_entry { int id; // version id char *name; // version name int bytes; // ip bytes number int segment_index_size; // segment index size in bytes // function to compare two ips ip_compare_fn_t ip_compare; }; typedef struct xdb_ip_version_entry xdb_version_t; XDB_PUBLIC(xdb_version_t *) xdb_version_v4(); XDB_PUBLIC(xdb_version_t *) xdb_version_v6(); XDB_PUBLIC(int) xdb_version_is_v4(const xdb_version_t *); XDB_PUBLIC(int) xdb_version_is_v6(const xdb_version_t *); XDB_PUBLIC(xdb_version_t *) xdb_version_from_name(char *); XDB_PUBLIC(xdb_version_t *) xdb_version_from_header(xdb_header_t *); // --- END ip version // --- xdb util functions // to compatiable with the windows // returns: 0 for ok and -1 for failed XDB_PUBLIC(int) xdb_init_winsock(); XDB_PUBLIC(void) xdb_clean_winsock(); // get the current time in microseconds XDB_PUBLIC(long) xdb_now(); // get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian XDB_PUBLIC(unsigned int) xdb_le_get_uint32(const char *, int); // get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian XDB_PUBLIC(int) xdb_le_get_uint16(const char *, int); // parse the specified IP address to byte array. // returns: xdb_version_t for valid ipv4 / ipv6, or NULL for failed XDB_PUBLIC(xdb_version_t *) xdb_parse_ip(const string_ip_t *, bytes_ip_t *, size_t); // parse the specified IPv4 address to byte array // returns: xdb_version_t for valid ipv4, or NULL for failed XDB_PUBLIC(xdb_version_t *) xdb_parse_v4_ip(const string_ip_t *, bytes_ip_t *, size_t); // parse the specified IPv6 address to byte array // returns: xdb_version_t for valid ipv6, or NULL for failed XDB_PUBLIC(xdb_version_t *) xdb_parse_v6_ip(const string_ip_t *, bytes_ip_t *, size_t); // convert a specified ip bytes to humen-readable string. // returns: 0 for success or -1 for failed. XDB_PUBLIC(int) xdb_ip_to_string(const bytes_ip_t *, int, char *, size_t); // ipv4 bytes to string XDB_PUBLIC(int) xdb_v4_ip_to_string(const bytes_ip_t *, char *, size_t); // ipv6 bytes to string XDB_PUBLIC(int) xdb_v6_ip_to_string(const bytes_ip_t *, char *, size_t); // compare the specified ip bytes with another ip bytes in the specified buff from offset. // ip args must be the return value from #xdb_parse_ip. // returns: -1 if ip1 < ip2, 1 if ip1 > ip2 or 0 XDB_PUBLIC(int) xdb_ip_sub_compare(const bytes_ip_t *, int, const char *, int); // large file seek and tell XDB_PUBLIC(int) xdb_fseek(FILE *, long long, int); XDB_PUBLIC(long long) xdb_ftell(FILE *); // --- END xdb utils // --- xdb searcher api // xdb region info structure #define xdb_region_buffer_wrapper 1 #define xdb_region_buffer_auto 2 struct xdb_region_buffer_entry { int type; // buffer type char *value; // region value size_t length; // buffer length }; typedef struct xdb_region_buffer_entry xdb_region_buffer_t; // wrapper the region from a local stack buffer. // returns: 0 for succeed or failed XDB_PUBLIC(int) xdb_region_buffer_init(xdb_region_buffer_t *, char *, size_t); // do the buffer alloc. // returns: 0 for ok or failed XDB_PUBLIC(int) xdb_region_buffer_alloc(xdb_region_buffer_t *, int); // empty alloc - empty string // returns: 0 - always XDB_PUBLIC(int) xdb_region_buffer_empty(xdb_region_buffer_t *); XDB_PUBLIC(void) xdb_region_buffer_free(xdb_region_buffer_t *); // xdb searcher structure struct xdb_searcher_entry { // ip version xdb_version_t *version; // xdb file handle FILE *handle; // header info const char *header; int io_count; // vector index buffer cache. // preload the vector index will reduce the number of IO operations // thus speedup the search process. const xdb_vector_index_t *v_index; // content buffer. // cache the whole xdb content. const xdb_content_t *content; }; typedef struct xdb_searcher_entry xdb_searcher_t; // xdb searcher new api define XDB_PUBLIC(int) xdb_new_with_file_only(xdb_version_t *, xdb_searcher_t *, const char *); XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_version_t *, xdb_searcher_t *, const char *, const xdb_vector_index_t *); XDB_PUBLIC(int) xdb_new_with_buffer(xdb_version_t *, xdb_searcher_t *, const xdb_content_t *); XDB_PUBLIC(void) xdb_close(void *); // xdb searcher search api define XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *, const string_ip_t *, xdb_region_buffer_t *); XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, const bytes_ip_t *, int, xdb_region_buffer_t *); XDB_PUBLIC(xdb_version_t *) xdb_get_version(xdb_searcher_t *); XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *); // --- END xdb searcher api #endif // C_IP2REGION_XDB_H ================================================ FILE: binding/c/xdb_searcher.c ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/27 #include "xdb_api.h" // --- region buffer XDB_PUBLIC(int) xdb_region_buffer_init(xdb_region_buffer_t *region, char *buffer, size_t length) { if (buffer == NULL) { region->type = xdb_region_buffer_auto; region->length = 0; } else if (length <= 0) { return 1; } else { region->type = xdb_region_buffer_wrapper; region->length = length; memset(buffer, 0x00, length); // zero-fill the buffer } region->value = buffer; return 0; } XDB_PUBLIC(int) xdb_region_buffer_alloc(xdb_region_buffer_t *region, int length) { if (length <= 0) { return 1; } // no allocation supports for the buffer wapper if (region->type == xdb_region_buffer_wrapper) { if (length >= region->length) { return 2; } region->value[length] = '\0'; return 0; } // ensure that the value were freed // by calling #xdb_region_buffer_free if (region->value != NULL) { return 3; } char *ptr = (char *) xdb_malloc(length + 1); if (ptr == NULL) { return 4; } ptr[length] = '\0'; // NULL-end region->value = ptr; region->length = length; return 0; } // fixed internal empty string ptr static char * _empty_region_string = "\0"; XDB_PUBLIC(int) xdb_region_buffer_empty(xdb_region_buffer_t *region) { // no allocation supports for the buffer wapper if (region->type == xdb_region_buffer_wrapper) { region->value[0] = '\0'; return 0; } // ensure that the value were freed // by calling #xdb_region_buffer_free if (region->value != NULL) { return 3; } region->value = _empty_region_string; region->length = 0; return 0; } XDB_PUBLIC(void) xdb_region_buffer_free(xdb_region_buffer_t *region) { if (region->type == xdb_region_buffer_auto) { // empty string interception if (region->length == 0 || region->value == _empty_region_string) { // do nothing for empty string } else { xdb_free(region->value); } // reset the value region->value = NULL; } } // --- END region buffer // internal function prototype define XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length); XDB_PRIVATE(int) xdb_new_base(xdb_version_t *version, xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { memset(xdb, 0x00, sizeof(xdb_searcher_t)); // set the version xdb->version = version; // check the content buffer first if (c_buffer != NULL) { xdb->v_index = NULL; xdb->content = c_buffer; return 0; } // open the xdb binary file FILE *handle = fopen(db_path, "rb"); if (handle == NULL) { return 1; } xdb->handle = handle; xdb->v_index = v_index; return 0; } // xdb searcher new api define XDB_PUBLIC(int) xdb_new_with_file_only(xdb_version_t *version, xdb_searcher_t *xdb, const char *db_path) { return xdb_new_base(version, xdb, db_path, NULL, NULL); } XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_version_t *version, xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) { return xdb_new_base(version, xdb, db_path, v_index, NULL); } XDB_PUBLIC(int) xdb_new_with_buffer(xdb_version_t *version, xdb_searcher_t *xdb, const xdb_content_t *c_buffer) { return xdb_new_base(version, xdb, NULL, NULL, c_buffer); } XDB_PUBLIC(void) xdb_close(void *ptr) { xdb_searcher_t *xdb = (xdb_searcher_t *) ptr; if (xdb->handle != NULL) { fclose(xdb->handle); xdb->handle = NULL; } } // --- xdb searcher search api define XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *xdb, const string_ip_t *ip_string, xdb_region_buffer_t *region) { bytes_ip_t ip_bytes[16] = {'\0'}; xdb_version_t *version = xdb_parse_ip(ip_string, ip_bytes, sizeof(ip_bytes)); if (version == NULL) { return 10; } else { return xdb_search(xdb, ip_bytes, version->bytes, region); } } XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, const bytes_ip_t *ip_bytes, int ip_len, xdb_region_buffer_t *region) { int il0, il1, idx, err, bytes, d_bytes; register int seg_index_size, l, h, m, p; unsigned int s_ptr, e_ptr, data_ptr, data_len; char vector_buffer[xdb_vector_index_size]; char segment_buffer[xdb_v6_index_size]; // ip version check if (ip_len != xdb->version->bytes) { return -1; } // some resets err = 0; data_len = 0; bytes = xdb->version->bytes; d_bytes = xdb->version->bytes << 1; xdb->io_count = 0; // locate the segment index block based on the vector index il0 = (int) (ip_bytes[0]); il1 = (int) (ip_bytes[1]); idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; if (xdb->v_index != NULL) { s_ptr = xdb_le_get_uint32(xdb->v_index->buffer, idx); e_ptr = xdb_le_get_uint32(xdb->v_index->buffer, idx + 4); } else if (xdb->content != NULL) { s_ptr = xdb_le_get_uint32(xdb->content->buffer, xdb_header_info_length + idx); e_ptr = xdb_le_get_uint32(xdb->content->buffer, xdb_header_info_length + idx + 4); } else { err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); if (err != 0) { return 10 + err; } s_ptr = xdb_le_get_uint32(vector_buffer, 0); e_ptr = xdb_le_get_uint32(vector_buffer, 4); } // printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr); // @Note: ptr validate, zero ptr means source data missing // so we could just stop here and return an empty string. if (s_ptr == 0 || e_ptr == 0) { xdb_region_buffer_empty(region); return err; } // binary search to get the final region info // segment_buffer = xdb_malloc(seg_index_size); seg_index_size = xdb->version->segment_index_size; data_len = 0, data_ptr = 0; l = 0, h = ((int) (e_ptr - s_ptr)) / seg_index_size; while (l <= h) { m = (l + h) >> 1; p = s_ptr + m * seg_index_size; // read the segment index item err = read(xdb, p, segment_buffer, seg_index_size); if (err != 0) { return 20 + err; } // decode the data fields as needed if (xdb->version->ip_compare(ip_bytes, bytes, segment_buffer, 0) < 0) { h = m - 1; } else if (xdb->version->ip_compare(ip_bytes, bytes, segment_buffer, bytes) > 0) { l = m + 1; } else { data_len = xdb_le_get_uint16(segment_buffer, d_bytes); data_ptr = xdb_le_get_uint32(segment_buffer, d_bytes + 2); break; } } // printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr); if (data_len == 0) { // return 100; xdb_region_buffer_empty(region); return err; } // buffer alloc checking err = xdb_region_buffer_alloc(region, data_len); if (err != 0) { return 100 + err; } err = read(xdb, data_ptr, region->value, data_len); if (err != 0) { return 30 + err; } return err; } XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) { // check the xdb content cache first if (xdb->content != NULL) { memcpy(buffer, xdb->content->buffer + offset, length); return 0; } // seek to the offset if (fseek(xdb->handle, offset, SEEK_SET) == -1) { return 1; } xdb->io_count++; if (fread(buffer, 1, length, xdb->handle) != length) { return 2; } return 0; } XDB_PUBLIC(xdb_version_t *) xdb_get_version(xdb_searcher_t *xdb) { return xdb->version; } XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) { return xdb->io_count; } ================================================ FILE: binding/c/xdb_util.c ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/27 #include "xdb_api.h" #include // for Linux #ifdef XDB_LINUX #include "sys/time.h" #endif #ifdef XDB_WINDOWS #include #endif // @Note: since 2023/10/13 to compatible with the windows system #ifdef XDB_WINDOWS static int winsock_initialized = 0; XDB_PUBLIC(int) xdb_init_winsock() { if (winsock_initialized == 1) { return 0; } WSADATA wsaData; if (WSAStartup(MAKEWORD(2,2), &wsaData) != 0) { return -1; } winsock_initialized = 1; return 0; } XDB_PUBLIC(void) xdb_clean_winsock() { if (winsock_initialized == 1) { WSACleanup(); winsock_initialized = 0; } } XDB_PRIVATE(int) gettimeofday(struct timeval* tp, void* tzp) { time_t clock; struct tm tm; SYSTEMTIME wtm; GetLocalTime(&wtm); tm.tm_year = wtm.wYear - 1900; tm.tm_mon = wtm.wMonth - 1; tm.tm_mday = wtm.wDay; tm.tm_hour = wtm.wHour; tm.tm_min = wtm.wMinute; tm.tm_sec = wtm.wSecond; tm.tm_isdst = -1; clock = mktime(&tm); tp->tv_sec = clock; tp->tv_usec = wtm.wMilliseconds * 1000; return (0); } #else XDB_PUBLIC(int) xdb_init_winsock() {return 0;} XDB_PUBLIC(void) xdb_clean_winsock() {} #endif // --- xdb buffer function implementations XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *handle) { xdb_header_t *header; unsigned int size = xdb_header_info_length; // entry alloc header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t)); if (header == NULL) { return NULL; } if (fseek(handle, 0, SEEK_SET) == -1) { xdb_free(header); return NULL; } if (fread(header->buffer, 1,size, handle) != size) { xdb_free(header); return NULL; } // fill the fields header->length = size; header->version = (unsigned short) xdb_le_get_uint16(header->buffer, 0); header->index_policy = (unsigned short) xdb_le_get_uint16(header->buffer, 2); header->created_at = xdb_le_get_uint32(header->buffer, 4); header->start_index_ptr = xdb_le_get_uint32(header->buffer, 8); header->end_index_ptr = xdb_le_get_uint32(header->buffer,12); // since IPv6 supporting header->ip_version = xdb_le_get_uint16(header->buffer, 16); header->runtime_ptr_bytes = xdb_le_get_uint16(header->buffer, 18); return header; } XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *db_path) { xdb_header_t *header; FILE *handle = fopen(db_path, "rb"); if (handle == NULL) { return NULL; } header = xdb_load_header(handle); fclose(handle); return header; } XDB_PUBLIC(void) xdb_free_header(void *ptr) { xdb_header_t *header = (xdb_header_t *) ptr; if (header->length > 0) { header->length = 0; xdb_free(header); } } // --- vector index XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *handle) { xdb_vector_index_t *v_index; unsigned int size = xdb_vector_index_length; // seek to the vector index offset if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) { return NULL; } // do the buffer read v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t)); if (v_index == NULL) { return NULL; } v_index->length = size; if (fread(v_index->buffer, 1, size, handle) != size) { xdb_free(v_index); return NULL; } return v_index; } XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *db_path) { xdb_vector_index_t *v_index; FILE *handle = fopen(db_path, "rb"); if (handle == NULL) { return NULL; } v_index = xdb_load_vector_index(handle); fclose(handle); return v_index; } XDB_PUBLIC(void) xdb_free_vector_index(void *ptr) { xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr; if (v_index->length > 0) { v_index->length = 0; xdb_free(v_index); } } // --- content buffer XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *handle) { unsigned int size; xdb_content_t *content; // determine the file size if (fseek(handle, 0, SEEK_END) == -1) { return NULL; } size = (unsigned int) ftell(handle); if (fseek(handle, 0, SEEK_SET) == -1) { return NULL; } // do the file read content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t)); if (content == NULL) { return NULL; } // do the buffer alloc content->buffer = (char *) xdb_malloc(size); if (content->buffer == NULL) { xdb_free(content); return NULL; } // read the content into the buffer content->length = size; if (fread(content->buffer, 1, size, handle) != size) { xdb_free(content); return NULL; } return content; } XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *db_path) { xdb_content_t *content; FILE *handle = fopen(db_path, "rb"); if (handle == NULL) { return NULL; } content = xdb_load_content(handle); fclose(handle); return content; } XDB_PUBLIC(void) xdb_free_content(void *ptr) { xdb_content_t *content = (xdb_content_t *) ptr; if (content->length > 0) { content->length = 0; xdb_free(content->buffer); content->buffer = NULL; xdb_free(content); } } XDB_PUBLIC(int) xdb_verify_from_header(FILE *handle, xdb_header_t *header) { unsigned int runtime_ptr_bytes = 0; // runtime ptr bytes if (header->version == xdb_structure_20) { runtime_ptr_bytes = 4; } else if (header->version == xdb_structure_30) { runtime_ptr_bytes = header->runtime_ptr_bytes; } else { return 2; } // 1, confirm the xdb file size. // to ensure that the maximum file pointer does not overflow. int err = fseek(handle, 0L, SEEK_END); if (err != 0) { return 3; } long long fileBytes = xdb_ftell(handle); long long maxFilePtr = (1LL << (runtime_ptr_bytes * 8)) - 1; // printf("fileBytes: %lld, maxFilePtr: %lld\n", fileBytes, maxFilePtr); if (fileBytes > maxFilePtr) { return 4; } return 0; } XDB_PUBLIC(int) xdb_verify(FILE *handle) { xdb_header_t *header = xdb_load_header(handle); if (header == NULL) { return 1; } int errcode = xdb_verify_from_header(handle, header); if (errcode != 0) { goto done; } // what next ? done: xdb_free_header(header); return errcode; } XDB_PUBLIC(int) xdb_verify_from_file(const char *db_path) { FILE *handle = fopen(db_path, "rb"); if (handle == NULL) { return -1; } int r = xdb_verify(handle); fclose(handle); return r; } // --- End content buffer // --- ip version // ip compare for IPv4 // ip1 - with Big endian byte order parsed from an input // ip2 - with Little endian byte order read from the xdb index. // to compatiable with the Little Endian encoded IPv4 on xdb 2.0. XDB_PRIVATE(int) _ipv4_sub_compare(const bytes_ip_t *ip_bytes, int bytes, const char *buffer, int offset) { register int i0, i1; for (int i = 0, j = offset + bytes - 1; i < bytes; i++, j--) { i0 = ip_bytes[i]; i1 = buffer[j] & 0xFF; if (i0 > i1) { return 1; } else if (i0 < i1) { return -1; } } return 0; } static xdb_version_t _ip_version_list[] = { // 14 = 4 + 4 + 2 + 4 {xdb_ipv4_id, "IPv4", xdb_ipv4_bytes, xdb_v4_index_size, _ipv4_sub_compare}, // 38 = 16 + 16 + 2 + 4 {xdb_ipv6_id, "IPv6", xdb_ipv6_bytes, xdb_v6_index_size, xdb_ip_sub_compare}, // END {0, NULL, 0, 0, NULL} }; XDB_PUBLIC(xdb_version_t *) xdb_version_v4() { return &_ip_version_list[0]; } XDB_PUBLIC(xdb_version_t *) xdb_version_v6() { return &_ip_version_list[1]; } XDB_PUBLIC(int) xdb_version_is_v4(const xdb_version_t *version) { return version->id == xdb_ipv4_id; } XDB_PUBLIC(int) xdb_version_is_v6(const xdb_version_t *version) { return version->id == xdb_ipv6_id; } XDB_PUBLIC(xdb_version_t *) xdb_version_from_name(char *name) { // to upper case the name for (int i = 0; name[i] != '\0'; i++) { name[i] = toupper((unsigned char) name[i]); } if (strcmp(name, "V4") == 0 || strcmp(name, "IPV4") == 0) { return xdb_version_v4(); } else if (strcmp(name, "V6") == 0 || strcmp(name, "IPV6") == 0) { return xdb_version_v6(); } else { return NULL; } } XDB_PUBLIC(xdb_version_t *) xdb_version_from_header(xdb_header_t *header) { // Old structure with ONLY IPv4 supports if (header->version == xdb_structure_20) { return xdb_version_v4(); } // structure 3.0 with IPv6 supporting if (header->version != xdb_structure_30) { return NULL; } if (header->ip_version == xdb_ipv4_id) { return xdb_version_v4(); } else if (header->ip_version == xdb_ipv6_id) { return xdb_version_v6(); } else { return NULL; } } // --- END ip version XDB_PUBLIC(long) xdb_now() { struct timeval c_time; gettimeofday(&c_time, NULL); return c_time.tv_sec * (int)1e6 + c_time.tv_usec; } XDB_PUBLIC(unsigned int) xdb_le_get_uint32(const char *buffer, int offset) { return ( ((buffer[offset ]) & 0x000000FF) | ((buffer[offset+1] << 8) & 0x0000FF00) | ((buffer[offset+2] << 16) & 0x00FF0000) | ((buffer[offset+3] << 24) & 0xFF000000) ); } XDB_PUBLIC(int) xdb_le_get_uint16(const char *buffer, int offset) { return ( ((buffer[offset ]) & 0x000000FF) | ((buffer[offset+1] << 8) & 0x0000FF00) ); } XDB_PUBLIC(xdb_version_t *) xdb_parse_ip(const string_ip_t *ip_string, bytes_ip_t *buffer, size_t length) { char *d_ptr = strchr(ip_string, '.'); char *c_ptr = strchr(ip_string, ':'); // version check if (d_ptr != NULL && c_ptr == NULL) { return xdb_parse_v4_ip(ip_string, buffer, length); } else if (c_ptr != NULL) { return xdb_parse_v6_ip(ip_string, buffer, length); } return NULL; } XDB_PUBLIC(xdb_version_t *) xdb_parse_v4_ip(const string_ip_t *ip_string, bytes_ip_t *buffer, size_t length) { struct in_addr addr; // buffer length checking if (length < xdb_ipv4_bytes) { return NULL; } if (inet_pton(AF_INET, ip_string, &addr) != 1) { return NULL; } // encode the address to buffer with big endian byte bufffer. buffer[0] = (addr.s_addr) & 0xFF; buffer[1] = (addr.s_addr >> 8) & 0xFF; buffer[2] = (addr.s_addr >> 16) & 0xFF; buffer[3] = (addr.s_addr >> 24) & 0xFF; return XDB_IPv4; } XDB_PUBLIC(xdb_version_t *) xdb_parse_v6_ip(const string_ip_t *ip_string, bytes_ip_t *buffer, size_t length) { struct in6_addr addr; // buffer length checking if (length < xdb_ipv6_bytes) { return NULL; } if (inet_pton(AF_INET6, ip_string, &addr) != 1) { return NULL; } memcpy(buffer, addr.s6_addr, xdb_ipv6_bytes); return XDB_IPv6; } XDB_PUBLIC(int) xdb_ip_to_string(const bytes_ip_t *ip_bytes, int bytes, char *ip_string, size_t length) { if (bytes == xdb_ipv4_bytes) { return xdb_v4_ip_to_string(ip_bytes, ip_string, length); } else if (bytes == xdb_ipv6_bytes) { return xdb_v6_ip_to_string(ip_bytes, ip_string, length); } return -1; } XDB_PUBLIC(int) xdb_v4_ip_to_string(const bytes_ip_t *ip_bytes, char *ip_string, size_t length) { if (!ip_bytes || !ip_string || length == 0) { return -1; } // buffer length checking if (length < INET_ADDRSTRLEN) { return -1; } if (inet_ntop(AF_INET, ip_bytes, ip_string, length) == NULL) { return -1; } return 0; } XDB_PUBLIC(int) xdb_v6_ip_to_string(const bytes_ip_t *ip_bytes, char *ip_string, size_t length) { if (!ip_bytes || !ip_string || length == 0) { return -1; } if (length < INET6_ADDRSTRLEN) { return -1; } if (inet_ntop(AF_INET6, ip_bytes, ip_string, length) == NULL) { return -1; } return 0; } XDB_PUBLIC(int) xdb_ip_sub_compare(const bytes_ip_t *ip1, int bytes, const char *buffer, int offset) { register int i, i1, i2; for (i = 0; i < bytes; i++) { i1 = ip1[i]; i2 = buffer[offset + i] & 0xFF; if (i1 > i2) { return 1; } else if (i1 < i2) { return -1; } } return 0; } XDB_PUBLIC(int) xdb_fseek(FILE *handle, long long offset, int whence) { // we may have to use the large file solution later // #if defined(XDB_LINUX) // return fseeko(handle, (off_t) offset, whence); // #elif defined(XDB_WINDOWS) // return _fseeki64(handle, (__int64) offset, whence) // #else // return fseek(handle, (long) offset, whence); // #endif return fseek(handle, (long) offset, whence); } XDB_PUBLIC(long long) xdb_ftell(FILE *handle) { // we may have to use the large file solution later // #if defined(XDB_LINUX) // return (long long) ftello(handle); // #elif defined(XDB_WINDOWS) // return (long long) _ftelli64(handle); // #else // // report error ? // return (long long) ftell(handle); // #endif return (long long) ftell(handle); } ================================================ FILE: binding/cpp/.gitignore ================================================ bin/ ================================================ FILE: binding/cpp/Makefile ================================================ all: bin header search bench make edit FILES=$(wildcard src/*.cc) bin: mkdir -p bin header: $(FILES) test/header.cc g++ -std=c++11 -O2 $^ -o bin/$@ search: $(FILES) test/search.cc g++ -std=c++11 -O2 $^ -o bin/$@ bench: $(FILES) test/bench.cc g++ -std=c++11 -O2 $^ -o bin/$@ make: $(FILES) test/make.cc g++ -std=c++11 -O2 $^ -o bin/$@ edit: $(FILES) g++ -std=c++11 -O2 $^ test/edit_v4.cc -o bin/edit_v4 g++ -std=c++11 -O2 $^ test/edit_v6.cc -o bin/edit_v6 clean: rm -rf bin ================================================ FILE: binding/cpp/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region C++ query client ## 0. File Description ``` Makefile --------- Build src ------------------ Source directory src/base.* ----------- Constants and utility functions src/ip.* ------------- IP processing implementation src/header.* --------- xdb header parsing implementation src/search.* --------- xdb search implementation src/bench.* ---------- Search benchmarking implementation src/make.* ----------- xdb file generation implementation src/edit.* ----------- Raw data editing implementation test ---------------- Test directory test/header.cc ------ Test header test/search.cc ------ Test search test/bench.cc ------- Benchmarking test/make.cc -------- Generate xdb file test/edit_v4.cc ----- Test raw data editing (ipv4) test/edit_v6.cc ----- Test raw data editing (ipv6) bin --------------- Executable directory (generated via make) bin/header -------- Test header bin/search -------- Test search bin/bench --------- Benchmarking bin/make ---------- Generate xdb file bin/edit_v4 ------- Test raw data editing (ipv4) bin/edit_v6 ------- Test raw data editing (ipv6) readme.md --------- readme ``` ## 1. Compilation ``` $ make ``` ## 2. Search ### 2.1 Example ```cpp #include "src/search.h" // IP Version: xdb::ipv4 xdb::ipv6 // Policy: xdb::policy_file xdb::policy_vector xdb::policy_content // No cache Partial cache Full cache int main() { std::string xdb_name = "../../data/ip2region_v6.xdb"; int version = xdb::ipv6; int policy = xdb::policy_content; std::string ip = "2001:200:124::"; xdb::search_t s(xdb_name, version, policy); std::cout << s.search(ip) << std::endl; return 0; } // $ g++ src/*.cc 1.cc --- Compile // $ ./a.out ------------- Test // Japan|Tokyo|Asagaya-minami|WIDE Project|JP ``` ### 2.2 Test xdb Header ``` $ ./bin/header Test IPv4 Version: 3 Cache Policy: 1 File Generation Time: 2025-09-06 02:24:16 Index Start Address: 955933 Index End Address: 11042415 IP Version: 4 Pointer Bytes: 4 Test IPv6 Version: 3 Cache Policy: 1 File Generation Time: 2025-10-17 04:41:04 Index Start Address: 3094259 Index End Address: 36258303 IP Version: 6 Pointer Bytes: 4 ``` ### 2.3 Test Search ``` $ ./bin/search Test IPv4 No cache: Success Test IPv4 Partial cache: Success Test IPv4 Full cache: Success Test IPv6 No cache: Success Test IPv6 Partial cache: Success Test IPv6 Full cache: Success ``` ## 3. Benchmarking and Correctness Verification ``` ./bin/bench Test IPv4, No cache, total: 3910284, took: 27.60s, cost: 6.59μs/op, io count: 28227147 Test IPv4, Partial cache, total: 3910284, took: 21.85s, cost: 5.15μs/op, io count: 24316863 Test IPv4, Full cache, total: 3910284, took: 2.26s, cost: 0.25μs/op, io count: 0 Test IPv6, No cache, total: 4792520, took: 100.40s, cost: 20.22μs/op, io count: 80758866 Test IPv6, Partial cache, total: 4792520, took: 93.06s, cost: 18.71μs/op, io count: 75966346 Test IPv6, Full cache, total: 4792520, took: 6.24s, cost: 0.81μs/op, io count: 0 ``` ## 4. Generate xdb File ### 4.1 Generate xdb File ``` $ ./bin/make Generate ipv4 xdb file, took: 0.57s Generate ipv6 xdb file, took: 1.24s ``` ## 5. Raw Data Editing ### 5.1. Instructions for Use * New IP attribution files can contain empty lines * New IP attribution files can be out of order; the program will automatically sort them * New IP attribution files can overlap; as long as there is no ambiguity, the program will automatically merge them * The final result will automatically merge adjacent lines with the same attribution * For the following tests, the original file uses the data file provided in the repository, and the new file uses 1.txt in the current directory ================================================ FILE: binding/cpp/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region C++ 查询客户端 ## 0. 文件说明 ``` Makefile --------- 构建 src ------------------ 源文件目录 src/base.* ----------- 常量及工具函数 src/ip.* ------------- 实现 IP 处理 src/header.* --------- 实现 xdb 头部解析 src/search.* --------- 实现 xdb 查找 src/bench.* ---------- 实现 查找 测速 src/make.* ----------- 实现 生成 xdb 文件 src/edit.* ----------- 实现 原始数据编辑 test ---------------- 测试目录 test/header.cc ------ 测试 头部 test/search.cc ------ 测试 查找 test/bench.cc ------- 测速 test/make.cc -------- 生成 xdb 文件 test/edit_v4.cc ----- 测试 原始数据编辑(ipv4) test/edit_v6.cc ----- 测试 原始数据编辑(ipv6) bin --------------- 可执行文件目录(通过 make 生成) bin/header -------- 测试 头部 bin/search -------- 测试 查找 bin/bench --------- 测速 bin/make ---------- 生成 xdb 文件 bin/edit_v4 ------- 测试 原始数据编辑(ipv4) bin/edit_v6 ------- 测试 原始数据编辑(ipv6) readme.md --------- readme ``` ## 1. 编译 ``` $ make ``` ## 2. 查找 ### 2.1 示例 ```cpp #include "src/search.h" // IP 版本: xdb::ipv4 xdb::ipv6 // 策略: xdb::policy_file xdb::policy_vector xdb::policy_content // 不缓存 部分缓存 全部缓存 int main() { std::string xdb_name = "../../data/ip2region_v6.xdb"; int version = xdb::ipv6; int policy = xdb::policy_content; std::string ip = "2001:200:124::"; xdb::search_t s(xdb_name, version, policy); std::cout << s.search(ip) << std::endl; return 0; } // $ g++ src/*.cc 1.cc --- 编译 // $ ./a.out ------------- 测试 // Japan|Tokyo|Asagaya-minami|WIDE Project|JP ``` ### 2.2 测试 xdb 头部 ``` $ ./bin/header 测试 IPv4 版本号: 3 缓存策略: 1 文件生成时间: 2025-09-06 02:24:16 索引起始地址: 955933 索引结束地址: 11042415 IP版本: 4 指针字节数: 4 测试 IPv6 版本号: 3 缓存策略: 1 文件生成时间: 2025-10-17 04:41:04 索引起始地址: 3094259 索引结束地址: 36258303 IP版本: 6 指针字节数: 4 ``` ### 2.3 测试查找 ``` $ ./bin/search 测试 IPv4 不缓存: 成功 测试 IPv4 部分缓存: 成功 测试 IPv4 全部缓存: 成功 测试 IPv6 不缓存: 成功 测试 IPv6 部分缓存: 成功 测试 IPv6 全部缓存: 成功 ``` ## 3. 测速以及检验正确性 ``` ./bin/bench 测试 IPv4, 不缓存, total: 3910284, took: 27.60s, cost: 6.59μs/op, io count: 28227147 测试 IPv4, 部分缓存, total: 3910284, took: 21.85s, cost: 5.15μs/op, io count: 24316863 测试 IPv4, 全部缓存, total: 3910284, took: 2.26s, cost: 0.25μs/op, io count: 0 测试 IPv6, 不缓存, total: 4792520, took: 100.40s, cost: 20.22μs/op, io count: 80758866 测试 IPv6, 部分缓存, total: 4792520, took: 93.06s, cost: 18.71μs/op, io count: 75966346 测试 IPv6, 全部缓存, total: 4792520, took: 6.24s, cost: 0.81μs/op, io count: 0 ``` ## 4. 生成 xdb 文件 ### 4.1 生成 xdb 文件 ``` $ ./bin/make 生成 ipv4 的 xdb 文件, took: 0.57s 生成 ipv6 的 xdb 文件, took: 1.24s ``` ## 5. 原始数据编辑 ### 5.1. 使用说明 * 新的IP归属地文件可以包含空行 * 新的IP归属地文件顺序可以乱序, 程序会自动排序 * 新的IP归属地文件顺序可以重叠, 只要无二义性, 程序会自动合并 * 最终的结果会将相邻的且归属地相同的行自动合并 * 以下测试, 原文件使用仓库自带的数据文件, 新文件使用当前目录下的 1.txt ================================================ FILE: binding/cpp/src/base.cc ================================================ #include "base.h" namespace xdb { int ip_version; // ip 版本 int ip_size; // ip 占的字节数 int content_size; void init_xdb(int version) { ip_version = version; ip_size = version == ipv4 ? 4 : 16; content_size = ip_size * 2 + 2 + 4; } void log_exit(const string &msg) { std::cout << msg << std::endl; exit(-1); } void read_bin(int index, char *buf, size_t len, FILE *db) { fseek(db, index, SEEK_SET); if (fread(buf, 1, len, db) != len) log_exit(__func__); } unsigned to_uint(const char *buf) { return ((buf[0]) & 0x000000FF) | ((buf[1] << 8) & 0x0000FF00) | ((buf[2] << 16) & 0x00FF0000) | ((buf[3] << 24) & 0xFF000000); } unsigned to_ushort(const char *buf) { return ((buf[0]) & 0x000000FF) | ((buf[1] << 8) & 0x0000FF00); } unsigned to_int(const char *buf, int n) { return n == 2 ? to_ushort(buf) : to_uint(buf); } void write_uint(unsigned data, char buf[]) { buf[0] = (data >> 0) & 0xFF; buf[1] = (data >> 8) & 0xFF; buf[2] = (data >> 16) & 0xFF; buf[3] = (data >> 24) & 0xFF; } void write_uint(unsigned data, FILE *dst) { char buf[4]; write_uint(data, buf); fwrite(buf, 1, sizeof(buf), dst); } void write_ushort(unsigned data, char buf[]) { buf[0] = (data >> 0) & 0xFF; buf[1] = (data >> 8) & 0xFF; } void write_ushort(unsigned data, FILE *dst) { char buf[2]; write_ushort(data, buf); fwrite(buf, 1, sizeof(buf), dst); } void write_string(const char *buf, unsigned len, FILE *dst) { fwrite(buf, 1, len, dst); } unsigned long long get_time() { struct timeval tv1; gettimeofday(&tv1, NULL); return (unsigned long long)tv1.tv_sec * 1000 * 1000 + tv1.tv_usec; } } // namespace xdb ================================================ FILE: binding/cpp/src/base.h ================================================ #ifndef BASE_H #define BASE_H #include #include #include #include #include #include #include #include #include #include #include #include namespace xdb { using std::string; constexpr int ipv4 = 4; constexpr int ipv6 = 6; constexpr int policy_file = 0; constexpr int policy_vector = 1; constexpr int policy_content = 2; constexpr int length_header = 256; constexpr int length_vector = 256 * 256 * 8; extern int ip_version; // ip 版本 extern int ip_size; // ip 占的字节数 extern int content_size; void init_xdb(int version); void log_exit(const string &msg); void read_bin(int index, char *buf, size_t len, FILE *db); unsigned to_uint(const char *buf); unsigned to_ushort(const char *buf); unsigned to_int(const char *buf, int n); void write_uint(unsigned data, char buf[]); void write_uint(unsigned data, FILE *dst); void write_ushort(unsigned data, char buf[]); void write_ushort(unsigned data, FILE *dst); void write_string(const char *buf, unsigned len, FILE *dst); unsigned long long get_time(); } // namespace xdb #endif ================================================ FILE: binding/cpp/src/bench.cc ================================================ #include "bench.h" namespace xdb { bench_t::bench_t(const std::string &file_name, int version, int policy) : search(file_name, version, policy) { } void bench_t::test_one(const ip_t &ip, const string region) { if (search.search(ip.to_string()) != region) xdb::log_exit("failed: " + ip.to_string() + " " + region); sum_io_count += search.get_io_count(); sum_cost_time += search.get_cost_time(); sum_count++; } void bench_t::test_line(char *buf) { size_t buf_len = strlen(buf); if (buf_len == 0) return; buf[buf_len - 1] = '\0'; // 去掉换行符 node_t node(buf); // 只测五个 for (int i = 0; i < 5 && node.ip1 < node.ip2; ++i) { test_one(node.ip1, node.region); node.ip1 = node.ip1 + 1; } test_one(node.ip2, node.region); } void bench_t::test_file(const std::string &file_name) { FILE *f = fopen(file_name.data(), "r"); if (f == NULL) xdb::log_exit("can't open " + file_name); char buf[1024]; while (fgets(buf, sizeof(buf), f) != NULL) test_line(buf); } void bench_t::test(const string &file_name) { sum_io_count = 0; sum_cost_time = 0; sum_count = 0; unsigned long long tv1 = xdb::get_time(); test_file(file_name); unsigned long long tv2 = xdb::get_time(); double took = (tv2 - tv1) * 1.0 / 1000 / 1000; double cost = sum_cost_time * 1.0 / sum_count; printf( "total: %llu, took: %8.2fs, cost: %6.2fμs/op, io " "count: " "%llu\n", sum_count, took, cost, sum_io_count); } } // namespace xdb ================================================ FILE: binding/cpp/src/bench.h ================================================ #ifndef BENCH_H #define BENCH_H #include "search.h" namespace xdb { class bench_t { public: bench_t(const string &file_name, int version, int policy); void test(const string &file_name); private: void test_one(const ip_t &ip, const string region); void test_line(char *buf); void test_file(const std::string &file_name); search_t search; unsigned long long sum_io_count; unsigned long long sum_cost_time; unsigned long long sum_count; }; } // namespace xdb #endif ================================================ FILE: binding/cpp/src/edit.cc ================================================ #include "edit.h" namespace xdb { void handle_ip_txt(const string& name, std::list& regions) { FILE* f = fopen(name.data(), "r"); if (f == NULL) log_exit("can't open " + name); char buf[1024]; while (fgets(buf, sizeof(buf), f) != NULL) { unsigned int buf_len = strlen(buf); // 去掉多余的空 while (buf_len > 0 && isspace(buf[buf_len - 1])) --buf_len; if (buf_len == 0) continue; buf[buf_len] = '\0'; regions.push_back(node_t(buf)); } fclose(f); } void edit_t::handle_new_file(const std::string& file_name) { handle_ip_txt(file_name, new_regions); // 输入 new_regions.sort(); // 排序 // 检验及其去重 auto it = new_regions.begin(); for (;;) { if (it == new_regions.end()) break; auto next = it; ++next; if (next == new_regions.end()) break; if (it->ip1 > it->ip2) it = new_regions.erase(it); // 非法, 直接跳过 else if (it->ip1 == next->ip1 || next->ip1 <= it->ip2) { // 数据重叠 if (it->region != next->region) log_exit("数据有二义性: " + it->to_string() + ", " + next->to_string()); it->ip2 = std::max(it->ip2, next->ip2); new_regions.erase(next); } else if (it->ip2 + 1 == next->ip1 && it->region == next->region) { // 数据连接 it->ip2 = next->ip2; new_regions.erase(next); } else { ++it; } } } void edit_t::handle_old_file(const std::string& file_name) { handle_ip_txt(file_name, old_regions); } void edit_t::merge() { auto it1 = old_regions.begin(); auto it2 = new_regions.begin(); for (;;) { if (it2 == new_regions.end()) break; if (it2->ip1 > it2->ip2) { ++it2; continue; } // it1->ip1 it1->ip2 it2->ip1 it2->ip2 while (it1->ip2 < it2->ip1) ++it1; if (it1->ip2 <= it2->ip2) { // it1->ip1 it2->ip1 it1->ip2 it2->ip2 node_t node; node.ip1 = it2->ip1; node.ip2 = it1->ip2; node.region = it2->region; it1->ip2 = node.ip1 - 1; it2->ip1 = node.ip2 + 1; ++it1; it1 = old_regions.insert(it1, node); ++it1; } else { // it1->ip1 it2->ip1 it2->ip2 it1->ip2 node_t node; node.ip1 = it2->ip2 + 1; node.ip2 = it1->ip2; node.region = it1->region; it1->ip2 = it2->ip1 - 1; ++it1; it1 = old_regions.insert(it1, *it2); ++it1; it1 = old_regions.insert(it1, node); ++it2; } } } void edit_t::write_old_file(const std::string& file_name) { FILE* f = fopen(file_name.data(), "w"); if (f == NULL) log_exit("can't open " + file_name); auto it = old_regions.begin(); // 删除非法的数据 for (;;) { if (it == old_regions.end()) break; if (it->ip1 > it->ip2) it = old_regions.erase(it); else ++it; } // 合并数据域相同的相邻数据 it = old_regions.begin(); for (;;) { if (it == old_regions.end()) break; auto next = it; ++next; if (next == old_regions.end()) break; if (it->region == next->region) { it->ip2 = next->ip2; old_regions.erase(next); } else { ++it; } } for (auto& d : old_regions) { string res = d.ip1.to_string() + "|" + d.ip2.to_string() + "|" + d.region + "\n"; fputs(res.data(), f); } fclose(f); } edit_t::edit_t(const string& name_old, const string& name_new, int version) { unsigned long long tv1 = get_time(); init_xdb(version); handle_new_file(name_new); handle_old_file(name_old); merge(); write_old_file(name_old); unsigned long long tv2 = get_time(); double took = (tv2 - tv1) * 1.0 / 1000 / 1000; printf("took: %.2fs\n", took); } } // namespace xdb ================================================ FILE: binding/cpp/src/edit.h ================================================ #ifndef EDIT_H #define EDIT_H #include "ip.h" namespace xdb { class edit_t { public: edit_t(const string& old_name, const string& new_name, int version); private: void handle_new_file(const string& file_name); void handle_old_file(const string& file_name); void merge(); void write_old_file(const string& file_name); std::list old_regions; std::list new_regions; }; } // namespace xdb #endif ================================================ FILE: binding/cpp/src/header.cc ================================================ #include "header.h" namespace xdb { header_t::header_t(FILE* db) { read_bin(0, header, sizeof(header), db); } header_t::~header_t() { } int header_t::version() { return to_int(header, 2); // 版本号(2) } int header_t::index_policy() { return to_int(header + 2, 2); // 缓存策略(2) } int header_t::create_at() { return to_int(header + 4, 4); // 文件生成时间(4) } int header_t::index_start() { return to_int(header + 8, 4); // 索引起始地址(4) } int header_t::index_end() { return to_int(header + 12, 4); // 索引结束地址(4) } int header_t::ip_version() { return to_int(header + 16, 2); // IP 版本(2) } int header_t::ptr() { return to_int(header + 18, 2); // 指针字节数(2) } } // namespace xdb ================================================ FILE: binding/cpp/src/header.h ================================================ #ifndef HEADER_H #define HEADER_H #include "base.h" namespace xdb { class header_t { public: header_t(FILE* db); virtual ~header_t(); int version(); // 版本号 int index_policy(); // 缓存策略 int create_at(); // 文件生成时间 int index_start(); // 索引起始地址 int index_end(); // 索引结束地址 int ip_version(); // IP 版本 int ptr(); // 指针字节数 protected: char header[length_header]; }; } // namespace xdb #endif ================================================ FILE: binding/cpp/src/ip.cc ================================================ #include "ip.h" namespace xdb { ip_t::ip_t() { memset(p, '\0', sizeof(p)); } ip_t::ip_t(const ip_t& rhs, int val) { memcpy(p, rhs.p, ip_size); if (val == 0 || val == 255) for (int i = 2; i < ip_size; ++i) p[i] = val; } ip_t::ip_t(const char* p) { from_xdb(p); } bool ip_t::from_str(const string& str) { int af_inet = ip_version == ipv4 ? AF_INET : AF_INET6; return inet_pton(af_inet, str.data(), p) == 1; } void ip_t::from_xdb(const char str[16]) { for (int i = 0; i < ip_size; ++i) if (ip_version == ipv6) p[i] = str[i]; else p[i] = str[ip_size - 1 - i]; } ip_t& ip_t::operator=(const ip_t& rhs) { memcpy(p, rhs.p, ip_size); return *this; } int ip_t::compare(const ip_t& rhs) const { for (int i = 0; i < ip_size; ++i) { if ((unsigned char)p[i] > (unsigned char)rhs.p[i]) return 1; if ((unsigned char)p[i] < (unsigned char)rhs.p[i]) return -1; } return 0; } bool ip_t::operator<(const ip_t& rhs) const { return compare(rhs) < 0; } bool ip_t::operator<=(const ip_t& rhs) const { return compare(rhs) <= 0; } bool ip_t::operator>(const ip_t& rhs) const { return compare(rhs) > 0; } bool ip_t::operator>=(const ip_t& rhs) const { return compare(rhs) >= 0; } bool ip_t::operator==(const ip_t& rhs) const { return compare(rhs) == 0; } bool ip_t::operator!=(const ip_t& rhs) const { return compare(rhs) != 0; } string ip_t::to_string() const { char buf[INET6_ADDRSTRLEN + 1]; int af_inet = ip_version == ipv4 ? AF_INET : AF_INET6; inet_ntop(af_inet, p, buf, sizeof(buf)); return string(buf); } string ip_t::to_bit() const { string str; for (int i = 0; i < ip_size; ++i) if (ip_version == ipv6) str.push_back(p[i]); else str.push_back(p[ip_size - 1 - i]); return str; } ip_t operator+(const ip_t& lhs, int v) { ip_t ip; int i = ip_size; while (--i >= 0) { v += lhs.p[i]; ip.p[i] = v % 256; v /= 256; } return ip; } ip_t operator-(const ip_t& lhs, int v) { ip_t ip; int i = ip_size; v = -v; while (--i >= 0) { v += lhs.p[i]; if (v == -1) ip.p[i] = 255; else { ip.p[i] = v; v = 0; } } return ip; } // node_t node_t::node_t() { } node_t::node_t(char* buf) { char* pos1 = strchr(buf, '|'); if (pos1 == NULL) log_exit("invalid data: " + std::string(buf)); char* pos2 = strchr(pos1 + 1, '|'); if (pos2 == NULL) log_exit("invalid data: " + std::string(buf)); *pos1 = '\0'; *pos2 = '\0'; region = pos2 + 1; if (!ip1.from_str(buf) || !ip2.from_str(pos1 + 1) || ip2 < ip1 || region.empty()) { *pos1 = *pos2 = '|'; log_exit(string("invalid data: ") + buf); } } bool node_t::operator<(const node_t& rhs) const { if (ip1 < rhs.ip1) return true; return ip2 < rhs.ip2; } string node_t::to_string() const { return ip1.to_string() + "|" + ip2.to_string() + "|" + region; } string node_t::to_bit() const { return ip1.to_bit() + ip2.to_bit(); } } // namespace xdb ================================================ FILE: binding/cpp/src/ip.h ================================================ #ifndef IP_H #define IP_H #include "base.h" namespace xdb { struct ip_t { unsigned char p[16]; ip_t(); ip_t(const char* p); // val 为 0 或 255 时, 将 ip 的后几位置为 val ip_t(const ip_t& rhs, int val = -1); bool from_str(const string& str); void from_xdb(const char str[16]); ip_t& operator=(const ip_t& rhs); int compare(const ip_t& rhs) const; bool operator<(const ip_t& rhs) const; bool operator<=(const ip_t& rhs) const; bool operator>(const ip_t& rhs) const; bool operator>=(const ip_t& rhs) const; bool operator==(const ip_t& rhs) const; bool operator!=(const ip_t& rhs) const; string to_string() const; string to_bit() const; }; ip_t operator+(const ip_t& lhs, int v); ip_t operator-(const ip_t& lhs, int v); struct node_t { ip_t ip1; ip_t ip2; string region; node_t(); node_t(char* buf); bool operator<(const node_t& rhs) const; string to_string() const; string to_bit() const; }; } // namespace xdb #endif ================================================ FILE: binding/cpp/src/make.cc ================================================ #include "make.h" namespace xdb { void make_t::vector_index_push_back(int row, int col, const node_t &node) { vector_index[row][col].push_back( std::make_pair(node.to_bit(), string(node.region))); } void make_t::vector_index_push_back(node_t &node) { ip_t ip1 = node.ip1; ip_t ip2 = node.ip2; unsigned ip1_1 = ip1.p[0]; unsigned ip1_2 = ip1.p[1]; unsigned ip2_1 = ip2.p[0]; unsigned ip2_2 = ip2.p[1]; if (ip1_1 == ip2_1 && ip1_2 == ip2_2) { vector_index_push_back(ip1_1, ip1_2, node); return; } node.ip1 = ip1; node.ip2 = ip_t(ip1, 255); vector_index_push_back(ip1_1, ip1_2, node); node.ip1 = ip_t(ip2, 0); node.ip2 = ip2; vector_index_push_back(ip2_1, ip2_2, node); for (;;) { ++ip1_2; if (ip1_2 == 256) { ++ip1_1; ip1_2 = 0; } if (ip1_1 == ip2_1 && ip1_2 == ip2_2) break; ip1.p[0] = ip1_1; ip1.p[1] = ip1_2; node.ip1 = ip_t(ip1, 0); node.ip2 = ip_t(ip1, 255); vector_index_push_back(ip1_1, ip1_2, node); } } void make_t::handle_input_help(char *buf) { // 去掉多余的空 unsigned int buf_len = strlen(buf); while (buf_len > 0 && isspace(buf[buf_len - 1])) --buf_len; if (buf_len == 0) return; buf[buf_len] = '\0'; node_t node(buf); if (node.ip1 < next_ip) { log_exit("ip 未排序: " + node.ip1.to_string() + ", " + next_ip.to_string()); } next_ip = node.ip2 + 1; if (region.find(node.region) == region.end()) { region[node.region] = region_index; region_index += node.region.size(); } vector_index_push_back(node); } void make_t::handle_input(const std::string &file_name) { FILE *src = fopen(file_name.data(), "r"); if (src == NULL) log_exit("can't open " + file_name); char buf[1024]; while (fgets(buf, sizeof(buf), src) != NULL) handle_input_help(buf); fclose(src); } void make_t::handle_header() { char buf[length_header]; memset(buf, 0, length_header); write_ushort(3, buf); // 版本号 write_ushort(1, buf + 2); // 缓存策略 write_uint(time(NULL), buf + 4); // 时间 // 索引 unsigned int content_left = length_header + length_vector; for (auto &d : region) content_left += d.first.size(); unsigned int content_right = content_left; for (int i = 0; i < 256; ++i) for (int j = 0; j < 256; ++j) content_right += vector_index[i][j].size() * content_size; content_right -= content_size; write_uint(content_left, buf + 8); write_uint(content_right, buf + 12); write_ushort(ip_version, buf + 16); // IP write_ushort(4, buf + 18); // 指针数 write_string(buf, length_header, db); } void make_t::handle_vector_index() { unsigned index = length_header + length_vector; for (auto &d : region) index += d.first.size(); for (unsigned i = 0; i < 256; ++i) for (unsigned j = 0; j < 256; ++j) if (vector_index[i][j].size() == 0) { write_uint(0, db); write_uint(0, db); } else { write_uint(index, db); index += content_size * vector_index[i][j].size(); write_uint(index, db); } } void make_t::handle_region() { for (auto &d : region) { fseek(db, d.second, SEEK_SET); write_string(d.first.data(), d.first.size(), db); } } void make_t::handle_content() { fseek(db, 0, SEEK_END); for (unsigned i = 0; i < 256; ++i) for (unsigned j = 0; j < 256; ++j) for (auto d : vector_index[i][j]) { write_string(d.first.data(), d.first.size(), db); write_ushort(d.second.size(), db); write_uint(region[d.second], db); } } make_t::make_t(const string &src, const string &dst, int version) : region_index(length_vector + length_header) { unsigned long long tv1 = get_time(); init_xdb(version); handle_input(src); db = fopen(dst.data(), "w"); if (db == NULL) log_exit("can't open " + dst); handle_header(); handle_vector_index(); handle_region(); handle_content(); fclose(db); unsigned long long tv2 = get_time(); printf("took: %.2fs\n", (tv2 - tv1) * 1.0 / 1000 / 1000); } } // namespace xdb ================================================ FILE: binding/cpp/src/make.h ================================================ #ifndef MAKE_H #define MAKE_H #include "ip.h" namespace xdb { class make_t { public: make_t(const string &src, const string &dst, int version); private: void vector_index_push_back(int row, int col, const node_t &node); void vector_index_push_back(node_t &node); void handle_input_help(char buf[]); void handle_input(const std::string &file_name); void handle_header(); void handle_vector_index(); void handle_region(); void handle_content(); FILE *db = NULL; std::vector> vector_index[256][256]; std::unordered_map region; unsigned region_index; ip_t next_ip; }; } // namespace xdb #endif ================================================ FILE: binding/cpp/src/search.cc ================================================ #include "search.h" namespace xdb { search_t::search_t(const string &file, int version, int p) : db(fopen(file.data(), "r")), header(db), policy(p) { init_xdb(version); if (db == NULL) log_exit("can't open " + file); if (header.ip_version() != version) log_exit("ip 版本不匹配"); if (policy != policy_file) { read_bin(length_header, vector, length_vector, db); if (policy == policy_content) { fseek(db, 0, SEEK_END); int size = ftell(db) - length_vector - length_header; content = (char *)malloc(size); read_bin(length_vector + length_header, content, size, db); } } } search_t::~search_t() { fclose(db); if (policy == policy_content) free(content); } int search_t::get_io_count() { return io_count; } int search_t::get_cost_time() { return cost_time; } char const *search_t::get_content_index_help(int index) { if (policy != policy_file) return vector + index; ++io_count; static char v[8]; read_bin(length_header + index, v, sizeof(v), db); return v; } void search_t::get_content_index(const ip_t &ip, int &left, int &right) { int index = ((unsigned char)ip.p[0] * 256 + (unsigned char)ip.p[1]) * 8; const char *p = get_content_index_help(index); left = to_uint(p); right = to_uint(p + 4); } char const *search_t::get_content_help(int index) { if (policy == policy_content) return content + index - length_header - length_vector; ++io_count; static char v[16 + 16 + 2 + 4]; read_bin(index, v, content_size, db); return v; } string search_t::get_region(int index, int len) { if (policy == policy_content) return string(content + index - length_header - length_vector, len); ++io_count; char *p = (char *)malloc(sizeof(char) * len); read_bin(index, p, len, db); string res(p, len); free(p); return res; } void search_t::get_content(int index, ip_t &ip_left, ip_t &ip_right, int ®ion_len, int ®ion_index) { const char *p = get_content_help(index); ip_left.from_xdb(p); ip_right.from_xdb(p + ip_size); region_len = to_ushort(p + ip_size * 2); region_index = to_uint(p + ip_size * 2 + 2); } string search_t::search(const ip_t &ip) { io_count = 0; int content_left, content_right; get_content_index(ip, content_left, content_right); if (content_left == 0 || content_right == 0) return ""; ip_t ip_left, ip_right; int region_len; int region_index; int left = 0; int right = (content_right - content_left) / content_size; for (;;) { int mid = left + (right - left) / 2; int mid_index = content_left + mid * content_size; get_content(mid_index, ip_left, ip_right, region_len, region_index); // ip ip_left ip_right if (ip < ip_left) right = mid - 1; // ip_left ip_right ip else if (ip_right < ip) left = mid + 1; else return get_region(region_index, region_len); } } string search_t::search(const string &str) { unsigned long long t1 = get_time(); ip_t ip; if (ip.from_str(str) == false) return "invalid ipv" + std::to_string(ip_version) + ": " + str; string region = search(ip); unsigned long long t2 = get_time(); cost_time = t2 - t1; return region; } } // namespace xdb ================================================ FILE: binding/cpp/src/search.h ================================================ #ifndef SEARCH_H #define SEARCH_H #include "header.h" #include "ip.h" namespace xdb { class search_t { protected: FILE *db; header_t header; int policy; int io_count; int cost_time; char vector[length_vector]; char *content; public: search_t(const string &file_name, int version, int policy); virtual ~search_t(); int get_io_count(); int get_cost_time(); string search(const string &str); protected: string search(const ip_t &ip); void get_content_index(const ip_t &ip1, int &left, int &right); void get_content(int index, ip_t &left, ip_t &right, int ®ion_len, int ®ion_index); char const *get_content_index_help(int index); char const *get_content_help(int index); string get_region(int index, int len); }; } // namespace xdb #endif ================================================ FILE: binding/cpp/test/bench.cc ================================================ #include "../src/bench.h" std::map prompt; void test_ipv4(int policy) { std::cout << "测试 IPv4, " << prompt[policy]; xdb::bench_t("../../data/ip2region_v4.xdb", xdb::ipv4, policy) .test("../../data/ipv4_source.txt"); } void test_ipv6(int policy) { std::cout << "测试 IPv6, " << prompt[policy]; xdb::bench_t("../../data/ip2region_v6.xdb", xdb::ipv6, policy) .test("../../data/ipv6_source.txt"); } int main() { prompt[xdb::policy_file] = " 不缓存, "; prompt[xdb::policy_vector] = "部分缓存, "; prompt[xdb::policy_content] = "全部缓存, "; test_ipv4(xdb::policy_file); test_ipv4(xdb::policy_vector); test_ipv4(xdb::policy_content); test_ipv6(xdb::policy_file); test_ipv6(xdb::policy_vector); test_ipv6(xdb::policy_content); return 0; } ================================================ FILE: binding/cpp/test/edit_v4.cc ================================================ #include "../src/edit.h" int main() { std::string file_name_old = "../../data/ipv4_source.txt"; std::string file_name_new = "./1.txt"; xdb::edit_t xdb(file_name_old, file_name_new, xdb::ipv4); return 0; } ================================================ FILE: binding/cpp/test/edit_v6.cc ================================================ #include "../src/edit.h" int main() { std::string file_name_old = "../../data/ipv6_source.txt"; std::string file_name_new = "./1.txt"; xdb::edit_t xdb(file_name_old, file_name_new, xdb::ipv6); return 0; } ================================================ FILE: binding/cpp/test/header.cc ================================================ #include "../src/header.h" void test(const std::string& prompt, const std::string& file_name) { std::cout << prompt << std::endl; xdb::header_t head(fopen(file_name.data(), "r")); std::cout << "版本号: " << head.version() << std::endl; std::cout << "缓存策略: " << head.index_policy() << std::endl; time_t rawtime = head.create_at(); struct tm* info = localtime(&rawtime); char buf[80]; strftime(buf, 80, "%Y-%m-%d %H:%M:%S", info); std::cout << "文件生成时间: " << buf << std::endl; std::cout << "索引起始地址: " << head.index_start() << std::endl; std::cout << "索引结束地址: " << head.index_end() << std::endl; std::cout << "IP版本: " << head.ip_version() << std::endl; std::cout << "指针字节数: " << head.ptr() << std::endl; std::cout << std::endl; } int main() { test("测试 IPv4", "../../data/ip2region_v4.xdb"); test("测试 IPv6", "../../data/ip2region_v6.xdb"); return 0; } ================================================ FILE: binding/cpp/test/make.cc ================================================ #include "../src/make.h" void test(const std::string& prompt, const std::string& filename_xdb, const std::string& filename_src, int version ) { std::cout << prompt; xdb::make_t(filename_xdb, filename_src, version); } int main() { test("生成 ipv4 的 xdb 文件, ", "../../data/ipv4_source.txt", "./ip2region_v4.xdb", xdb::ipv4); test("生成 ipv6 的 xdb 文件, ", "../../data/ipv6_source.txt", "./ip2region_v6.xdb", xdb::ipv6); return 0; } ================================================ FILE: binding/cpp/test/search.cc ================================================ #include "../src/search.h" std::map prompt; void test(xdb::search_t& s, const std::string& ip, const std::string& region) { if (s.search(ip) != region) xdb::log_exit("测试失败, ip " + ip + ", region " + region); } void test_ipv4(int policy) { std::cout << "测试 IPv4 " << prompt[policy]; xdb::search_t s("../../data/ip2region_v4.xdb", xdb::ipv4, policy); test(s, "0.0.0.0", "Reserved|Reserved|Reserved|0|0"); test(s, "1.2.3.4", "Australia|Queensland|Brisbane|0|AU"); std::cout << " 成功" << std::endl; } void test_ipv6(int policy) { std::cout << "测试 IPv6 " << prompt[policy]; xdb::search_t s("../../data/ip2region_v6.xdb", xdb::ipv6, policy); test(s, "::1", ""); test(s, "2001:200:124::", "Japan|Tokyo|Asagaya-minami|WIDE Project|JP"); test(s, "2001:200:124::", "Japan|Tokyo|Asagaya-minami|WIDE Project|JP"); test(s, "240e:3b7:3273:51d0:cd38:8ae1:e3c0:b708", "中国|广东省|深圳市|电信|CN"); std::cout << " 成功" << std::endl; } int main() { prompt[xdb::policy_file] = " 不缓存:"; prompt[xdb::policy_vector] = "部分缓存:"; prompt[xdb::policy_content] = "全部缓存:"; test_ipv4(xdb::policy_file); test_ipv4(xdb::policy_vector); test_ipv4(xdb::policy_content); test_ipv6(xdb::policy_file); test_ipv6(xdb::policy_vector); test_ipv6(xdb::policy_content); return 0; } ================================================ FILE: binding/csharp/.editorconfig ================================================ root = true # To learn more about .editorconfig see https://aka.ms/editorconfigdocs [*] charset = utf-8 indent_style = space trim_trailing_whitespace = true insert_final_newline = true spelling_exclusion_path = .\exclusion.dic [*.csproj] charset = utf-8 insert_final_newline = true [*.{xml,config,csproj,nuspec,props,resx,targets,yml,tasks,json}] indent_size = 2 [*.sh] end_of_line = lf [*.cs] csharp_style_namespace_declarations = file_scoped:silent [*.cs] file_header_template = Copyright 2025 The Ip2Region Authors. All rights reserved.\nUse of this source code is governed by a Apache2.0-style\nlicense that can be found in the LICENSE file.\n@Author Alan \n@Date 2023/07/25\nUpdated by Argo Zhang at 2025/11/21 ================================================ FILE: binding/csharp/.gitignore ================================================ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore # User-specific files *.rsuser *.suo *.user *.userosscache *.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ x64/ x86/ [Aa][Rr][Mm]/ [Aa][Rr][Mm]64/ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ # Visual Studio 2015/2017 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # Visual Studio 2017 auto generated files Generated\ Files/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* # NUNIT *.VisualState.xml TestResult.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ # .NET Core project.lock.json project.fragment.lock.json artifacts/ # StyleCop StyleCopReport.xml # Files built by Visual Studio *_i.c *_p.c *_h.h *.ilk *.meta *.obj *.iobj *.pch *.pdb *.ipdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj *_wpftmp.csproj *.log *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opendb *.opensdf *.sdf *.cachefile *.VC.db *.VC.VC.opendb # Visual Studio profiler *.psess *.vsp *.vspx *.sap # Visual Studio Trace Files *.e2e # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # JustCode is a .NET coding add-in .JustCode # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # AxoCover is a Code Coverage Tool .axoCover/* !.axoCover/settings.json # Visual Studio code coverage results *.coverage *.coveragexml # NCrunch _NCrunch_* .*crunch*.local.xml nCrunchTemp_* # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted *.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted PublishScripts/ # NuGet Packages *.nupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* # except build/, which is used as an MSBuild target. !**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets # Microsoft Azure Build Output csx/ *.build.csdef # Microsoft Azure Emulator ecf/ rcf/ # Windows Store app package directories and files AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt *.appx # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache !?*.[Cc]ache/ # Others ClientBin/ ~$* *~ *.dbmdl *.dbproj.schemaview *.jfm *.pfx *.publishsettings orleans.codegen.cs # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) #*.snk # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ *.rptproj.bak # SQL Server files *.mdf *.ldf *.ndf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings *.rptproj.rsuser *- Backup*.rdl # Microsoft Fakes FakesAssemblies/ # GhostDoc plugin setting file *.GhostDoc.xml # Node.js Tools for Visual Studio .ntvs_analysis.dat node_modules/ # Visual Studio 6 build log *.plg # Visual Studio 6 workspace options file *.opt # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) *.vbw # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts **/*.DesktopClient/ModelManifest.xml **/*.Server/GeneratedArtifacts **/*.Server/ModelManifest.xml _Pvt_Extensions # Paket dependency manager .paket/paket.exe paket-files/ # FAKE - F# Make .fake/ # JetBrains Rider .idea/ *.sln.iml # CodeRush personal settings .cr/personal # Python Tools for Visual Studio (PTVS) __pycache__/ *.pyc # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio *.tss # Telerik's JustMock configuration file *.jmconfig # BizTalk build output *.btp.cs *.btm.cs *.odx.cs *.xsd.cs # OpenCover UI analysis results OpenCover/ # Azure Stream Analytics local run output ASALocalRun/ # MSBuild Binary and Structured Log *.binlog # NVidia Nsight GPU debugger configuration file *.nvuser # MFractors (Xamarin productivity tool) working folder .mfractor/ # Local History for Visual Studio .localhistory/ ================================================ FILE: binding/csharp/CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. ## [3.0.0] - 2025-11-22 - 支持 .NET 10.0 - 增加 IPv6 支持 - 修复若干 bug ## [2.0.1] - 2023-07-30 ### Added - Support netstandard2.0 ## [2.0.0] - 2023-07-26 ### Removed - Remove nuget include xdb file - Searcher cache policy default parameters - Searcher xdb file path default parameters ### Added - Dependent file query policies CachePolicy.VectorIndex, CachePolicy.File support thread-safe concurrent queries - Dramatically optimizes overall performance ================================================ FILE: binding/csharp/IP2Region.Net/Abstractions/ISearcher.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using System.Net; namespace IP2Region.Net.Abstractions; /// /// IP 转化为地理位置搜索器接口 /// public interface ISearcher : IDisposable { /// /// 搜索方法 /// /// IP 地址字符串 如 192.168.0.1 /// string? Search(string ipStr); /// /// 搜索方法 /// string? Search(IPAddress ipAddress); /// /// 搜索方法 仅限 IPv4 使用 /// /// IPv4 地址字节数组小端读取 uint 数值 [Obsolete("已弃用，请改用其他方法；Deprecated; please use Search(string) or Search(IPAddress) method.")] string? Search(uint ipAddress); /// /// 获得 内部 IO 访问次数 /// int IoCount { get; } } ================================================ FILE: binding/csharp/IP2Region.Net/Extensions/ServiceCollectionExtensions.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using IP2Region.Net.Abstractions; using IP2Region.Net.XDB; using Microsoft.Extensions.DependencyInjection.Extensions; namespace Microsoft.Extensions.DependencyInjection; /// /// IP2Region 服务扩展类 /// public static class IP2RegionExtensions { /// /// 添加 IP2RegionService 服务。 /// /// 集合 /// IP2Region 数据库文件的路径。 /// 缓存策略，默认为 。 public static IServiceCollection AddIP2RegionService(this IServiceCollection services, string path, CachePolicy cachePolicy = CachePolicy.Content) { services.TryAddSingleton(provider => { return new Searcher(cachePolicy, path); }); #if NET8_0_OR_GREATER services.TryAddKeyedSingleton("IP2Region.Net", (provider, _) => { return provider.GetRequiredService(); }); #endif return services; } } ================================================ FILE: binding/csharp/IP2Region.Net/IP2Region.Net.csproj ================================================ IP2Region.Net 3.0.2 IP2Region.Net Alan Lee;Argo Zhang(argo@live.ca) Apache-2.0 README.md https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp Please refer to CHANGELOG.md for details .NET client library for ip2region IP2Region GeoIP IPSearch git enable enable netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0;net9.0;net10.0 latest c2f07fe1-a300-4de3-8200-1278ed8cb5b7 CHANGELOG.md ================================================ FILE: binding/csharp/IP2Region.Net/Internal/CacheStrategyFactory.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using IP2Region.Net.XDB; namespace IP2Region.Net.Internal; static class CacheStrategyFactory { public static ICacheStrategy CreateCacheStrategy(CachePolicy cachePolicy, string xdbPath) => cachePolicy switch { CachePolicy.Content => new ContentCacheStrategy(xdbPath), CachePolicy.VectorIndex => new VectorIndexCacheStrategy(xdbPath), _ => new FileCacheStrategy(xdbPath), }; } ================================================ FILE: binding/csharp/IP2Region.Net/Internal/ContentCacheStrategy.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Wong at 2025/12/31 namespace IP2Region.Net.Internal; class ContentCacheStrategy(string xdbPath) : ICacheStrategy { // TODO: these constants can be moved to the interface as defaults when using .NET 10 private const int HeaderInfoLength = 256; private const int VectorIndexSize = 8; private readonly ReadOnlyMemory _cacheData = File.ReadAllBytes(xdbPath); public int IoCount => 0; public void ResetIoCount() { // Do nothing } public ReadOnlyMemory GetVectorIndex(int offset) => _cacheData.Slice(HeaderInfoLength + offset, VectorIndexSize); public ReadOnlyMemory GetData(long offset, int length) => _cacheData.Slice((int)offset, length); public void Dispose() { // Do nothing } } ================================================ FILE: binding/csharp/IP2Region.Net/Internal/FileCacheStrategy.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using System.Buffers; namespace IP2Region.Net.Internal; class FileCacheStrategy(string xdbPath) : ICacheStrategy { protected const int HeaderInfoLength = 256; protected const int VectorIndexSize = 8; protected const int BufferSize = 64 * 1024; protected FileStream XdbFileStream = new(xdbPath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, FileOptions.RandomAccess); public int IoCount { get; set; } public void ResetIoCount() { IoCount = 0; } public virtual ReadOnlyMemory GetVectorIndex(int offset) => GetData(HeaderInfoLength + offset, VectorIndexSize); public virtual ReadOnlyMemory GetData(long offset, int length) { var buffer = ArrayPool.Shared.Rent(length); try { int totalBytesRead = 0; XdbFileStream.Seek(offset, SeekOrigin.Begin); int bytesRead; while (totalBytesRead < length) { bytesRead = XdbFileStream.Read(buffer, totalBytesRead, length - totalBytesRead); if (bytesRead == 0) { break; } totalBytesRead += bytesRead; IoCount++; } var ret = new byte[totalBytesRead]; if (totalBytesRead > 0) { Array.Copy(buffer, 0, ret, 0, totalBytesRead); } return ret; } finally { ArrayPool.Shared.Return(buffer); } } /// /// 释放文件句柄 /// /// protected virtual void Dispose(bool disposing) { if (disposing) { XdbFileStream.Close(); XdbFileStream.Dispose(); } } /// /// /// public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } } ================================================ FILE: binding/csharp/IP2Region.Net/Internal/ICacheStrategy.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 namespace IP2Region.Net.Internal; internal interface ICacheStrategy : IDisposable { int IoCount { get; } void ResetIoCount(); ReadOnlyMemory GetVectorIndex(int offset); ReadOnlyMemory GetData(long offset, int length); } ================================================ FILE: binding/csharp/IP2Region.Net/Internal/VectorIndexCacheStrategy.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 namespace IP2Region.Net.Internal; class VectorIndexCacheStrategy : FileCacheStrategy { private const int VectorIndexRows = 256; private const int VectorIndexCols = 256; private readonly ReadOnlyMemory _vectorCache; public VectorIndexCacheStrategy(string xdbPath) : base(xdbPath) { _vectorCache = GetData(HeaderInfoLength, VectorIndexRows * VectorIndexCols * VectorIndexSize); } public override ReadOnlyMemory GetVectorIndex(int offset) => _vectorCache.Slice(offset, VectorIndexSize); } ================================================ FILE: binding/csharp/IP2Region.Net/XDB/CachePolicy.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 namespace IP2Region.Net.XDB; /// /// 缓存策略枚举 /// public enum CachePolicy { /// /// no cache /// File, /// /// cache vector index , reduce the number of IO operations /// VectorIndex, /// /// default cache policy , cache whole xdb file /// Content } ================================================ FILE: binding/csharp/IP2Region.Net/XDB/Searcher.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using IP2Region.Net.Abstractions; using IP2Region.Net.Internal; using System.Buffers.Binary; using System.Diagnostics.CodeAnalysis; using System.Net; using System.Text; namespace IP2Region.Net.XDB; /// /// 实现类 /// /// /// /// public class Searcher(CachePolicy cachePolicy, string xdbPath) : ISearcher { private readonly ICacheStrategy _cacheStrategy = CacheStrategyFactory.CreateCacheStrategy(cachePolicy, xdbPath); /// /// /// public int IoCount => _cacheStrategy.IoCount; /// /// /// public string? Search(string ipStr) { var ipAddress = IPAddress.Parse(ipStr); return SearchCore(ipAddress.GetAddressBytes()); } /// /// /// public string? Search(IPAddress ipAddress) => SearchCore(ipAddress.GetAddressBytes()); /// /// /// [Obsolete("已弃用，请改用其他方法；Deprecated; please use Search(string) or Search(IPAddress) method.")] [ExcludeFromCodeCoverage] public string? Search(uint ipAddress) { var bytes = BitConverter.GetBytes(ipAddress); Array.Reverse(bytes); return SearchCore(bytes); } string? SearchCore(byte[] ipBytes) { // 重置 IO 计数器 _cacheStrategy.ResetIoCount(); // 每个 vector 索引项的字节数 var vectorIndexSize = 8; // vector 索引的列数 var vectorIndexCols = 256; // 计算得到 vector 索引项的开始地址。 var il0 = ipBytes[0]; var il1 = ipBytes[1]; var idx = il0 * vectorIndexCols * vectorIndexSize + il1 * vectorIndexSize; var vector = _cacheStrategy.GetVectorIndex(idx); var sPtr = BinaryPrimitives.ReadUInt32LittleEndian(vector.Span); var ePtr = BinaryPrimitives.ReadUInt32LittleEndian(vector.Span.Slice(4)); // @Note: ptr validate, zero ptr means source data missing // so we could just stop here and return an empty string. if (sPtr == 0 || ePtr == 0) { return ""; } var length = ipBytes.Length; var indexSize = length * 2 + 6; var l = 0; var h = (ePtr - sPtr) / indexSize; var dataLen = 0; long dataPtr = 0; while (l <= h) { int m = (int)(l + h) >> 1; var p = sPtr + m * indexSize; var buff = _cacheStrategy.GetData(p, indexSize); var s = buff.Span.Slice(0, length); var e = buff.Span.Slice(length, length); if (ByteCompare(ipBytes, s) < 0) { h = m - 1; } else if (ByteCompare(ipBytes, e) > 0) { l = m + 1; } else { dataLen = BinaryPrimitives.ReadUInt16LittleEndian(buff.Span.Slice(length * 2, 2)); dataPtr = BinaryPrimitives.ReadUInt32LittleEndian(buff.Span.Slice(length * 2 + 2, 4)); break; } } var regionBuff = _cacheStrategy.GetData(dataPtr, dataLen); return Encoding.UTF8.GetString(regionBuff.Span.ToArray()); } static int ByteCompare(byte[] ip1, ReadOnlySpan ip2) => ip1.Length == 4 ? IPv4Compare(ip1, ip2) : IPv6Compare(ip1, ip2); static int IPv4Compare(byte[] ip1, ReadOnlySpan ip2) { var ret = 0; for (int i = 0; i < ip1.Length; i++) { var ip2Index = ip1.Length - 1 - i; if (ip1[i] < ip2[ip2Index]) { return -1; } else if (ip1[i] > ip2[ip2Index]) { return 1; } } return ret; } static int IPv6Compare(byte[] ip1, ReadOnlySpan ip2) { var ret = 0; for (int i = 0; i < ip1.Length; i++) { if (ip1[i] < ip2[i]) { return -1; } else if (ip1[i] > ip2[i]) { return 1; } } return ret; } /// /// /// public void Dispose() { _cacheStrategy.Dispose(); GC.SuppressFinalize(this); } } ================================================ FILE: binding/csharp/IP2Region.Net/XDB/Util.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using System.Buffers; using System.Buffers.Binary; using System.Net; using System.Runtime.InteropServices; namespace IP2Region.Net.XDB; /// /// 工具类 /// public static class Util { public static uint IpAddressToUInt32(string ipAddress) { var address = IPAddress.Parse(ipAddress); return IpAddressToUInt32(address); } public static uint IpAddressToUInt32(IPAddress ipAddress) { byte[] bytes = ipAddress.GetAddressBytes(); Array.Reverse(bytes); return MemoryMarshal.Read(bytes); } public static uint GetMidIp(uint x, uint y) => (x & y) + ((x ^ y) >> 1); public static async Task GetVersionAsync(string dbPath, CancellationToken token = default) { if (string.IsNullOrEmpty(dbPath)) { throw new ArgumentNullException(nameof(dbPath)); } if (!File.Exists(dbPath)) { throw new FileNotFoundException("xdb file not found.", dbPath); } using var reader = File.OpenRead(dbPath); return await GetVersionAsync(reader, token); } internal static async Task GetVersionAsync(FileStream reader, CancellationToken token = default) { XdbVersion ret = default; var buffer = ArrayPool.Shared.Rent(256); try { var length = await reader.ReadAsync(buffer, 0, 256, token); if (length == 256) { ret = Parse(buffer); } } finally { ArrayPool.Shared.Return(buffer); } return ret; } private static XdbVersion Parse(ReadOnlySpan buffer) { var ret = new XdbVersion { Ver = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(0, 2)), CachePolice = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(2, 2)), StartIndex = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(8, 4)), EndIndex = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(12, 4)), IPVer = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(16, 2)), BytesCount = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(18, 2)) }; var createdAt = BinaryPrimitives.ReadUInt32LittleEndian(buffer.Slice(4, 4)); var dtm = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.FromHours(0)); ret.CreatedTime = dtm.AddSeconds(createdAt); return ret; } } ================================================ FILE: binding/csharp/IP2Region.Net/XDB/XdbVersion.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 namespace IP2Region.Net.XDB; /// /// XdbVersion 结构体 /// public struct XdbVersion { /// /// 获得/设置 版本号 /// public ushort Ver { get; set; } /// /// 获得/设置 缓存策略 /// public ushort CachePolice { get; set; } /// /// 获得/设置 文件生成时间 /// public DateTimeOffset CreatedTime { get; set; } /// /// 获得/设置 索引起始地址 /// public uint StartIndex { get; set; } /// /// 获得/设置 索引结束地址 /// public uint EndIndex { get; set; } /// /// 获得/设置 IP版本 /// public ushort IPVer { get; set; } /// /// 获得/设置 指针字节数 /// public ushort BytesCount { get; set; } } ================================================ FILE: binding/csharp/IP2Region.Net.BenchMark/Benmarks.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using BenchmarkDotNet.Attributes; using IP2Region.Net.XDB; namespace IP2Region.Net.BenchMark; [MemoryDiagnoser] public class Benchmarks { private static readonly string XdbPathV4 = Path.Combine(AppContext.BaseDirectory, "IP2Region", "ip2region_v4.xdb"); private static readonly string XdbPathV6 = Path.Combine(AppContext.BaseDirectory, "IP2Region", "ip2region_v6.xdb"); private static readonly Searcher _contentV4Searcher = new(CachePolicy.Content, XdbPathV4); private static readonly Searcher _vectorV4Searcher = new(CachePolicy.VectorIndex, XdbPathV4); private static readonly Searcher _fileV4Searcher = new(CachePolicy.File, XdbPathV4); private static readonly Searcher _contentV6Searcher = new(CachePolicy.Content, XdbPathV6); private static readonly Searcher _vectorV6Searcher = new(CachePolicy.VectorIndex, XdbPathV6); private static readonly Searcher _fileV6Searcher = new(CachePolicy.File, XdbPathV6); private readonly string _testIPv4Address = "114.114.114.114"; private readonly string _testIPv6Address = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; public Benchmarks() { _contentV4Searcher.Search(_testIPv4Address); _vectorV4Searcher.Search(_testIPv4Address); _fileV4Searcher.Search(_testIPv4Address); _contentV6Searcher.Search(_testIPv6Address); _vectorV6Searcher.Search(_testIPv6Address); _fileV6Searcher.Search(_testIPv6Address); } [Benchmark] [BenchmarkCategory("IPv4")] public void ContentIPv4() => _contentV4Searcher.Search(_testIPv4Address); [Benchmark] [BenchmarkCategory("IPv4")] public void VectorIPv4() => _vectorV4Searcher.Search(_testIPv4Address); [Benchmark] [BenchmarkCategory("IPv4")] public void FileIPv4() => _fileV4Searcher.Search(_testIPv4Address); [Benchmark] [BenchmarkCategory("IPv6")] public void ContentIPv6() => _contentV6Searcher.Search(_testIPv6Address); [Benchmark] [BenchmarkCategory("IPv6")] public void VectorIPv6() => _vectorV6Searcher.Search(_testIPv6Address); [Benchmark] [BenchmarkCategory("IPv6")] public void FileIPv6() => _fileV6Searcher.Search(_testIPv6Address); } ================================================ FILE: binding/csharp/IP2Region.Net.BenchMark/IP2Region.Net.BenchMark.csproj ================================================ Exe net10.0 enable enable IP2Region/ip2region_v4.xdb PreserveNewest IP2Region/ip2region_v6.xdb PreserveNewest ================================================ FILE: binding/csharp/IP2Region.Net.BenchMark/Program.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using BenchmarkDotNet.Running; using IP2Region.Net.BenchMark; BenchmarkRunner.Run(); ================================================ FILE: binding/csharp/IP2Region.Net.Test/IP2Region.Net.Test.csproj ================================================  net10.0 enable enable false runtime; build; native; contentfiles; analyzers; buildtransitive all all runtime; build; native; contentfiles; analyzers; buildtransitive TestData/ipv4_source.txt PreserveNewest TestData/ip2region_v4.xdb PreserveNewest TestData/ipv6_source.txt PreserveNewest TestData/ip2region_v6.xdb PreserveNewest ================================================ FILE: binding/csharp/IP2Region.Net.Test/SearcherTest.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using IP2Region.Net.Abstractions; using IP2Region.Net.XDB; using Microsoft.Extensions.DependencyInjection; using System.Net; using Xunit; namespace IP2Region.Net.Test; public class SearcherTest { private readonly string _xdbPathV4 = Path.Combine(AppContext.BaseDirectory, "TestData", "ip2region_v4.xdb"); private readonly string _xdbPathV6 = Path.Combine(AppContext.BaseDirectory, "TestData", "ip2region_v6.xdb"); [Theory] [InlineData("58.251.27.201", "中国|广东省|深圳市|联通|CN", "v4")] [InlineData("114.114.114.114", "中国|江苏省|南京市|0|CN", "v4")] [InlineData("119.29.29.29", "中国|北京|北京市|腾讯|CN", "v4")] [InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里|CN", "v4")] [InlineData("180.76.76.76", "中国|北京|北京市|百度|CN", "v4")] [InlineData("8.8.8.8", "United States|California|0|Google LLC|US", "v4")] [InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|电信|CN", "v6")] public void TestSearchCacheContent(string ip, string expected, string version) { var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6; var contentSearcher = new Searcher(CachePolicy.Content, _xdbPath); var region = contentSearcher.Search(ip); Assert.Equal(expected, region); } [Theory] [InlineData("58.251.27.201", "中国|广东省|深圳市|联通|CN", "v4")] [InlineData("114.114.114.114", "中国|江苏省|南京市|0|CN", "v4")] [InlineData("119.29.29.29", "中国|北京|北京市|腾讯|CN", "v4")] [InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里|CN", "v4")] [InlineData("180.76.76.76", "中国|北京|北京市|百度|CN", "v4")] [InlineData("8.8.8.8", "United States|California|0|Google LLC|US", "v4")] [InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|电信|CN", "v6")] public void TestSearchCacheVector(string ip, string expected, string version) { var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6; var vectorSearcher = new Searcher(CachePolicy.VectorIndex, _xdbPath); var region = vectorSearcher.Search(ip); Assert.Equal(expected, region); } [Theory] [InlineData("58.251.0.0", "中国|广东省|深圳市|联通|CN", "v4")] [InlineData("58.251.255.255", "中国|广东省|深圳市|联通|CN", "v4")] [InlineData("58.251.27.201", "中国|广东省|深圳市|联通|CN", "v4")] [InlineData("114.114.114.114", "中国|江苏省|南京市|0|CN", "v4")] [InlineData("119.29.29.29", "中国|北京|北京市|腾讯|CN", "v4")] [InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里|CN", "v4")] [InlineData("180.76.76.76", "中国|北京|北京市|百度|CN", "v4")] [InlineData("8.8.8.8", "United States|California|0|Google LLC|US", "v4")] [InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|电信|CN", "v6")] [InlineData("240e:044d:2d00:0000:0000:0000:0000:0000", "中国|云南|楚雄|电信|CN", "v6")] public void TestSearchCacheFile(string ip, string expected, string version) { var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6; var fileSearcher = new Searcher(CachePolicy.File, _xdbPath); var region = fileSearcher.Search(ip); Assert.Equal(expected, region); } [Fact] public void IoCount_File_Ok() { var searcher = new Searcher(CachePolicy.File, _xdbPathV4); searcher.Search("58.251.27.201"); Assert.Equal(3, searcher.IoCount); searcher.Search("58.251.27.201"); Assert.Equal(3, searcher.IoCount); searcher.Dispose(); } [Fact] public void IoCount_Vector_Ok() { var searcher = new Searcher(CachePolicy.VectorIndex, _xdbPathV4); searcher.Search("58.251.27.201"); Assert.Equal(2, searcher.IoCount); searcher.Search("58.251.27.201"); Assert.Equal(2, searcher.IoCount); searcher.Dispose(); } [Fact] public void IoCount_Content_Ok() { var searcher = new Searcher(CachePolicy.Content, _xdbPathV4); searcher.Search("58.251.27.201"); Assert.Equal(0, searcher.IoCount); searcher.Search("58.251.27.201"); Assert.Equal(0, searcher.IoCount); searcher.Dispose(); } [Theory] [InlineData("58.251.255.255", "中国|广东省|深圳市|联通|CN")] public void Search_Ip_Ok(string ipStr, string expected) { var fileSearcher = new Searcher(CachePolicy.File, _xdbPathV4); var ipAddress = IPAddress.Parse(ipStr); var region = fileSearcher.Search(ipAddress); Assert.Equal(expected, region); } [Theory] [InlineData("58.251.255.255", "中国|广东省|深圳市|联通|CN")] public void AddIP2RegionService_Ok(string ipStr, string expected) { var services = new ServiceCollection(); services.AddIP2RegionService(_xdbPathV4, CachePolicy.File); var provider = services.BuildServiceProvider(); var searcher = provider.GetRequiredService(); var region = searcher.Search(ipStr); Assert.Equal(expected, region); searcher = provider.GetRequiredKeyedService("IP2Region.Net"); region = searcher.Search(ipStr); Assert.Equal(expected, region); } [Theory] [InlineData(CachePolicy.Content, "v4")] [InlineData(CachePolicy.VectorIndex, "v4")] [InlineData(CachePolicy.File, "v4")] [InlineData(CachePolicy.Content, "v6")] [InlineData(CachePolicy.VectorIndex, "v6")] [InlineData(CachePolicy.File, "v6")] public void TestBenchSearch(CachePolicy cachePolicy, string version) { var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6; var searcher = new Searcher(cachePolicy, _xdbPath); var srcPath = Path.Combine(AppContext.BaseDirectory, "TestData", $"ip{version}_source.txt"); foreach (var line in File.ReadLines(srcPath)) { var ps = line.Trim().Split("|", 3); var sip = ps[0]; var eip = ps[1]; var s1 = searcher.Search(sip); var s2 = searcher.Search(eip); Assert.Equal(s1, ps[2]); Assert.Equal(s2, ps[2]); } } } ================================================ FILE: binding/csharp/IP2Region.Net.Test/UtilTest.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using Xunit; namespace IP2Region.Net.Test; public class UtilTest { [Fact] public void IpAddressToUInt32_Ok() { var uintIp = XDB.Util.IpAddressToUInt32("114.114.114.114"); Assert.Equal((uint)1920103026, uintIp); } [Fact] public void GetMidIp_Ok() { var uintIp = XDB.Util.GetMidIp(1, 10); Assert.Equal((uint)5, uintIp); } } ================================================ FILE: binding/csharp/IP2Region.Net.Test/XdbTest.cs ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan // @Date 2023/07/25 // Updated by Argo Zhang at 2025/11/21 using Xunit; namespace IP2Region.Net.Test; public class XdbTest { [Fact] public async Task VersionIPV4_Ok() { var db = Path.Combine(AppContext.BaseDirectory, "TestData", $"ip2region_v4.xdb"); var version = await XDB.Util.GetVersionAsync(db); Assert.Equal(3, version.Ver); Assert.Equal(1, version.CachePolice); //Assert.Equal("2025-09-06 02:24:16", version.CreatedTime.ToString("yyyy-MM-dd HH:mm:ss")); //Assert.Equal((uint)955933, version.StartIndex); //Assert.Equal((uint)11042415, version.EndIndex); Assert.Equal(4, version.IPVer); Assert.Equal(4, version.BytesCount); } [Fact] public async Task VersionIPV6_Ok() { var db = Path.Combine(AppContext.BaseDirectory, "TestData", $"ip2region_v6.xdb"); var version = await XDB.Util.GetVersionAsync(db); Assert.Equal(3, version.Ver); Assert.Equal(1, version.CachePolice); //Assert.Equal("2025-10-17 04:41:04", version.CreatedTime.ToString("yyyy-MM-dd HH:mm:ss")); //Assert.Equal((uint)3094259, version.StartIndex); //Assert.Equal((uint)36258303, version.EndIndex); Assert.Equal(6, version.IPVer); Assert.Equal(4, version.BytesCount); } [Fact] public async Task GetVersionAsync_Error() { await Assert.ThrowsAsync(async () => await XDB.Util.GetVersionAsync(null!)); await Assert.ThrowsAsync(async () => await XDB.Util.GetVersionAsync(Path.Combine(AppContext.BaseDirectory, "test.xdb"))); } } ================================================ FILE: binding/csharp/IP2Region.Net.slnx ================================================ ================================================ FILE: binding/csharp/README.md ================================================ # IP2Region.Net .NET client library for IP2Region ## Installation Install the package with [NuGet](https://www.nuget.org/packages/IP2Region.Net) ```bash Install-Package IP2Region.Net ``` ## Usage ```csharp using IP2Region.Net.Abstractions; using IP2Region.Net.XDB; ISearcher searcher = new Searcher(CachePolicy , "your xdb file path"); ``` ### Cache Policy Description | Cache Policy | Description | Thread Safe | |-------------------------|------------------------------------------------------------------------------------------------------------|-------------| | CachePolicy.Content | Cache the entire `xdb` data. | Yes | | CachePolicy.VectorIndex | Cache `vecotorIndex` to speed up queries and reduce system io pressure by reducing one fixed IO operation. | Yes | | CachePolicy.File | Completely file-based queries | Yes | ### XDB File Description Generate using [maker](https://github.com/lionsoul2014/ip2region/tree/master/maker/csharp), or [download](https://github.com/lionsoul2014/ip2region/blob/master/data/ip2region.xdb) pre-generated xdb files ## ASP.NET Core Usage ```csharp services.AddIP2RegionService("your xdb file path", cachePolicy: CachePolicy.Content); ``` NET6/7 ```csharp provider.GetRequiredService() ``` NET8+ support keyed service ```csharp provider.GetRequiredKeyedService("IP2Region.Net"); ``` ## TargetFrameworks netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0;net9.0;net10.0 ## Performance // * Summary * BenchmarkDotNet v0.15.6, Windows 11 (10.0.26200.7171) 13th Gen Intel Core i7-13700 2.10GHz, 1 CPU, 24 logical and 16 physical cores .NET SDK 10.0.100 [Host] : .NET 10.0.0 (10.0.0, 10.0.25.52411), X64 RyuJIT x86-64-v3 DefaultJob : .NET 10.0.0 (10.0.0, 10.0.25.52411), X64 RyuJIT x86-64-v3 | Method | Mean | Error | StdDev | Gen0 | Allocated | |------------ |-------------:|-----------:|-----------:|-------:|----------:| | ContentIPv4 | 53.70 ns | 0.296 ns | 0.277 ns | 0.0086 | 136 B | | VectorIPv4 | 4,446.04 ns | 18.673 ns | 15.593 ns | 0.0076 | 232 B | | FileIPv4 | 6,712.40 ns | 15.718 ns | 13.934 ns | 0.0153 | 264 B | | ContentIPv6 | 145.53 ns | 0.331 ns | 0.277 ns | 0.0126 | 200 B | | VectorIPv6 | 7,058.39 ns | 125.505 ns | 117.398 ns | 0.0381 | 712 B | | FileIPv6 | 10,657.97 ns | 53.907 ns | 50.425 ns | 0.0458 | 744 B | // * Hints * Outliers Benchmarks.VectorIPv4: Default -> 2 outliers were removed (4.55 us, 4.58 us) Benchmarks.FileIPv4: Default -> 1 outlier was removed (6.79 us) Benchmarks.ContentIPv6: Default -> 2 outliers were removed (148.08 ns, 152.27 ns) // * Legends * Mean : Arithmetic mean of all measurements Error : Half of 99.9% confidence interval StdDev : Standard deviation of all measurements Gen0 : GC Generation 0 collects per 1000 operations Allocated : Allocated memory per single operation (managed only, inclusive, 1KB = 1024B) 1 ns : 1 Nanosecond (0.000000001 sec) // * Diagnostic Output - MemoryDiagnoser * // ***** BenchmarkRunner: End ***** Run time: 00:02:06 (126.09 sec), executed benchmarks: 6 Global total time: 00:02:13 (133.47 sec), executed benchmarks: 6 // * Artifacts cleanup * Artifacts cleanup is finished ## Contributing Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change. Please make sure to update tests as appropriate. ## License [Apache License 2.0](https://github.com/lionsoul2014/ip2region/blob/master/LICENSE.md) ================================================ FILE: binding/erlang/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region erlang query client ### Introduction This binding implements the xdb query client in Erlang, based on the Erlang OTP Application. The query logic is implemented by the `ip2region_worker` worker process, supporting multiple worker processes for load balancing. ### Application Configuration The configurable parameters for this application are in `ip2region.app.src`, as follows: ```erlang {env,[ {poolargs, [ {size, 1}, %% Default number of worker processes {max_overflow, 5} %% Maximum number of worker processes ]} ]} ``` ### Compile ``` $ rebar3 compile ``` ### Run Place the xdb file in the `priv` directory, then start the Erlang node: ``` $ rebar3 shell ``` Call the `xdb:search/1` interface in the Erlang shell to query IP address information. This interface supports IP addresses represented as list strings, binary strings, tuples, and integers, as follows: ``` 1> xdb:search("1.0.8.0"). [20013,22269,124,48,124,24191,19996,30465,124,24191,24030, 24066,124,30005,20449] 2> 3> io:format("~ts~n", [xdb:search("1.0.8.0")]). 中国|0|广东省|广州市|电信 io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]). 中国|0|广东省|广州市|电信 4> io:format("~ts~n", [xdb:search({1,0,8,0})]). 中国|0|广东省|广州市|电信 6> io:format("~ts~n", [xdb:search(16779264)]). 中国|0|广东省|广州市|电信 ``` ### Usage * Add the dependency in `rebar.config` ``` {deps, [ ip2region ]}. ``` * Start the ip2region Application ``` ...... application:ensure_started(ip2region), ...... ``` * Call the `xdb:search/1` interface to query IP information ``` ...... ip2region:search("1.0.8.0"), ...... ``` ### Unit Test ``` $ rebar3 eunit ===> Verifying dependencies... ===> Analyzing applications... ===> Compiling ip2region ===> Performing EUnit tests... =INFO REPORT==== 17-Jan-2023::11:52:59.920155 === XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb .... Finished in 0.074 seconds 4 tests, 0 failures ``` ### Benchmark ``` $ cd benchmarks/ $ sh xdb-benchmark.sh ===> Verifying dependencies... ===> Analyzing applications... ===> Compiling ip2region Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] Eshell V12.3.2.2 (abort with ^G) 1> =INFO REPORT==== 17-Jan-2023::11:37:35.631095 === XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/default/lib/ip2region/priv/ip2region.xdb ===> Booted ip2region ===> Evaluating: "xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()." CPU info: model name : AMD EPYC 7K62 48-Core Processor cache size : 512 KB cpu MHz : 2595.124 bogomips : 5190.24 cores/threads : 2 Erlang info: system_version:Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] load test data use 4.835593s start run benchmark tests search from file: ip count:683844, total time: 28.201699s, search 24248.326315375536 times per second, use 41.23995969841075 micro second per search search from cache: ip count:683844, total time: 0.671801s, search 1017926.4395259906 times per second, use 0.9823892583688677 micro second per search benchmark test finish ``` ================================================ FILE: binding/erlang/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region erlang 查询客户端 ### 简介 该bingding以erlang语言实现xdb查询客户端，基于Erlang OTP Application，查询逻辑由ip2region_worker工作进程实现，支持配多个工作进程来进行负载均衡。 ### 应用配置 该应用可配置的参数在ip2region.app.src中,如下： ``` erlang {env,[ {poolargs, [ {size, 1}, %% 工作进程默认数量 {max_overflow, 5} %% 工作进程最大数量 ]} ]} ``` ### 编译 ``` $ rebar3 compile ``` ### 运行 将xdb文件放到priv目录下，然后启动erlang节点： ``` $ rebar3 shell ``` 在erlang shell中调用xdb:search/1接口查询Ip地址信息, 该接口支持以list格式字符串、binary格式字符串、tuple和整数表示的IP地址，如下： ``` 1> xdb:search("1.0.8.0"). [20013,22269,124,48,124,24191,19996,30465,124,24191,24030, 24066,124,30005,20449] 2> 3> io:format("~ts~n", [xdb:search("1.0.8.0")]). 中国|0|广东省|广州市|电信 io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]). 中国|0|广东省|广州市|电信 4> io:format("~ts~n", [xdb:search({1,0,8,0})]). 中国|0|广东省|广州市|电信 6> io:format("~ts~n", [xdb:search(16779264)]). 中国|0|广东省|广州市|电信 ``` ### 使用方法 * 在rebar.config中引入依赖 ``` {deps, [ ip2region ]}. ``` * 启动ip2region Application ``` ...... application:ensure_started(ip2region), ...... ``` * 调用xdb:search/1接口查询IP信息 ``` ...... ip2region:search("1.0.8.0"), ...... ``` ### 单元测试 ``` $ rebar3 eunit ===> Verifying dependencies... ===> Analyzing applications... ===> Compiling ip2region ===> Performing EUnit tests... =INFO REPORT==== 17-Jan-2023::11:52:59.920155 === XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb .... Finished in 0.074 seconds 4 tests, 0 failures ``` ### 基准测试 ``` $ cd benchmarks/ $ sh xdb-benchmark.sh ===> Verifying dependencies... ===> Analyzing applications... ===> Compiling ip2region Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] Eshell V12.3.2.2 (abort with ^G) 1> =INFO REPORT==== 17-Jan-2023::11:37:35.631095 === XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/default/lib/ip2region/priv/ip2region.xdb ===> Booted ip2region ===> Evaluating: "xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()." CPU info: model name : AMD EPYC 7K62 48-Core Processor cache size : 512 KB cpu MHz : 2595.124 bogomips : 5190.24 cores/threads : 2 Erlang info: system_version:Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] load test data use 4.835593s start run benchmark tests search from file: ip count:683844, total time: 28.201699s, search 24248.326315375536 times per second, use 41.23995969841075 micro second per search search from cache: ip count:683844, total time: 0.671801s, search 1017926.4395259906 times per second, use 0.9823892583688677 micro second per search benchmark test finish ``` ================================================ FILE: binding/erlang/benchmarks/xdb-benchmark.sh ================================================ #!/bin/bash cd .. rebar3 shell --eval="xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()." ================================================ FILE: binding/erlang/include/ip2region.hrl ================================================ -ifndef(IP2REGION_HRL). -define(IP2REGION_HRL, true). -define(NONE, none). -define(APP_NAME, ip2region). -define(XDB_VECTOR_INDEX, ets_xdb_vector_index). -define(XDB_SEGMENT_INDEX, ets_xdb_segement_index). -define(IP2REGION_CACHE, ets_ip2region_cache). -define(XDB_HEADER_SIZE, 256). -define(XDB_VECTOR_COLS, 256). -define(XDB_VECTOR_INDEX_SIZE, 8). -define(XDB_VECTOR_INDEX_COUNT, (16#10000)). %% 256*256 -define(XDB_SEGMENT_INDEX_SIZE, 14). -define(IP2REGION_POOL, ip2region_pool). -ifndef(IF). -define(IF(C, T, F), case (C) of true -> (T); false -> (F) end). -define(IF(C, T), ?IF(C, T, skip)). -endif. -endif. ================================================ FILE: binding/erlang/priv/dummy ================================================ ================================================ FILE: binding/erlang/rebar.config ================================================ {erl_opts, [ debug_info, export_all, nowarn_export_all ]}. {plugins, [rebar3_hex, rebar3_ex_doc]}. {deps, [ poolboy ]}. {shell, [ % {config, "config/sys.config"}, {apps, [ip2region]} ]}. {ex_doc, [ {extras, ["README.md"]}, {main, "README.md"}, {source_url, "https://github.com/leihua996/ip2region/tree/master/binding/erlang"} ]}. {hex, [{doc, ex_doc}]}. ================================================ FILE: binding/erlang/src/ip2region.app.src ================================================ {application, ip2region, [{description, "ip2region xdb client application"}, {vsn, "0.1.0"}, {registered, []}, {mod, {ip2region_app, []}}, {applications, [kernel, stdlib ]}, {env,[ {poolargs, [ {size, 1}, {max_overflow, 5} ]} ]}, {modules, []}, {licenses, ["Apache-2.0"]}, {links, [{"Github", "https://github.com/leihua996/ip2region/tree/master/binding/erlang"}]} ]}. ================================================ FILE: binding/erlang/src/ip2region_app.erl ================================================ %%%------------------------------------------------------------------- %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. %% %% @doc %% @end %%%------------------------------------------------------------------- -module(ip2region_app). -behaviour(application). -export([start/2, stop/1]). start(_StartType, _StartArgs) -> ip2region_sup:start_link(). stop(_State) -> ok. %% internal functions ================================================ FILE: binding/erlang/src/ip2region_sup.erl ================================================ %%%------------------------------------------------------------------- %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. %% %% @doc ip2region top level supervisor. %% @end %%%------------------------------------------------------------------- -module(ip2region_sup). -behaviour(supervisor). -include("ip2region.hrl"). -export([start_link/0]). -export([init/1, create_table/0]). -define(SERVER, ?MODULE). start_link() -> {ok, SupPid} = supervisor:start_link({local, ?SERVER}, ?MODULE, []), {ok, _PoolPid} = start_ip2region_pool(SupPid), {ok, SupPid}. %% sup_flags() = #{strategy => strategy(), % optional %% intensity => non_neg_integer(), % optional %% period => pos_integer()} % optional %% child_spec() = #{id => child_id(), % mandatory %% start => mfargs(), % mandatory %% restart => restart(), % optional %% shutdown => shutdown(), % optional %% type => worker(), % optional %% modules => modules()} % optional init([]) -> create_table(), SupFlags = #{strategy => one_for_one, intensity => 10, period => 5}, ChildSpecs = [], {ok, {SupFlags, ChildSpecs}}. %% internal functions %% create_table() -> Opts = [named_table, set, public, {read_concurrency, true}, {keypos, 1}], ets:new(?XDB_VECTOR_INDEX, Opts), ets:new(?XDB_SEGMENT_INDEX, Opts), ets:new(?IP2REGION_CACHE, Opts). start_ip2region_pool(Sup) -> {ok, PoolArgsCfg} = application:get_env(poolargs), PoolName = ?IP2REGION_POOL, PoolArgs = [{strategy, fifo}, {name, {local, PoolName}}, {worker_module, ip2region_worker} | PoolArgsCfg], WorkerArgs = [], ChildSpecs = poolboy:child_spec(PoolName, PoolArgs, WorkerArgs), supervisor:start_child(Sup, ChildSpecs). ================================================ FILE: binding/erlang/src/ip2region_util.erl ================================================ %%%------------------------------------------------------------------- %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. %% %% @doc %% ip2region utils %% @end %%%------------------------------------------------------------------- -module(ip2region_util). -export([ipv4_to_n/1]). ipv4_to_n(IntIp) when is_integer(IntIp) -> IntIp; ipv4_to_n({A, B, C, D}) -> <> = <>, N; ipv4_to_n(Ip) when is_binary(Ip) -> ipv4_to_n(binary_to_list(Ip)); ipv4_to_n(Ip) when is_list(Ip) -> case inet_parse:address(Ip) of {ok, Addr} -> ipv4_to_n(Addr); _ -> {error, bad_ip_format} end. ================================================ FILE: binding/erlang/src/ip2region_worker.erl ================================================ %%%------------------------------------------------------------------- %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. %% %% @doc %% ip2region xdb client worker %% @end %%%------------------------------------------------------------------- -module(ip2region_worker). -behaviour(gen_server). -include("ip2region.hrl"). %% API -export([start/1, stop/1, start_link/1]). -export([search/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -record(state, {xdb_fd}). %%========================================== %% API %% ========================================= start(Args) -> Opts = [{spawn_opt, [{min_heap_size, 6000}]}], gen_server:start(?MODULE, Args, Opts). start_link(Args) -> Opts = [{spawn_opt, [{min_heap_size, 6000}]}], gen_server:start_link(?MODULE, Args, Opts). stop(Pid) -> gen_server:call(Pid, stop). search(Pid, Ip) -> gen_server:call(Pid, {search, Ip}). %%========================================== %% gen_server callbacks %% ========================================= init(_Args) -> process_flag(trap_exit, true), AppName = case application:get_application() of {ok, AName} -> AName; _ -> ?APP_NAME end, PrivDir = code:priv_dir(AppName), XdbFileName = filename:join([PrivDir, "ip2region.xdb"]), error_logger:info_report(io_lib:format("XdbFile:~s~n", [XdbFileName])), {ok, IoDevice} = file:open(XdbFileName, [read, binary]), load_vector_index(IoDevice), {ok, #state{xdb_fd = IoDevice}}. handle_call(Request, From, State) -> try do_call(Request, From, State) catch Class:Error:Stacktrace -> error_logger:error_report(io_lib:format("~p handle call error, Req:~p ~p, stacktrace:~p~n", [?MODULE, Request, {Class, Error}, Stacktrace])), {reply, {error, {Class, Error}}, State} end. handle_cast(Msg, State) -> try do_cast(Msg, State) catch Class:Error:Stacktrace -> error_logger:error_report(io_lib:format("~p handle cast error, Msg:~p, ~p, stacktrace:~w~n", [?MODULE, Msg, {Class, Error}, Stacktrace])), {noreply, State} end. handle_info(Info, State) -> try do_info(Info, State) catch Class:Error:Stacktrace -> error_logger:error_report(io_lib:format("~p handle info error, Info:~p, ~p, stacktrace:~p~n", [?MODULE, Info, {Class, Error}, Stacktrace])), {noreply, State} end. terminate(_Reason, State) -> #state{xdb_fd = XdbFd} = State, case is_pid(XdbFd) of true -> file:close(XdbFd); _ -> skip end, ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. %%========================================== %% Internal function %% ========================================= do_call({search, Ip}, _From, #state{xdb_fd = IoDevice} = State) -> Reply = search_ip(IoDevice, Ip), {reply, Reply, State}; do_call(stop, _From, State) -> {stop, normal, stopped, State}; do_call(Request, From, State) -> error_logger:error_report(io_lib:format("unknown request: ~p, from:~p", [Request, From])), {noreply, State}. do_cast(Msg, State) -> error_logger:error_report(io_lib:format("unknown msg: ~p", [Msg])), {noreply, State}. do_info(Info, State) -> error_logger:error_report(io:format("unknown info: ~p", [Info])), {noreply, State}. load_vector_index(IoDevice) -> Key = ip2region_header_loaded, case persistent_term:get(Key, false) of true -> ok; _ -> {ok, <<_Header:?XDB_HEADER_SIZE/binary, VectorIndexBin/binary>> } = file:read(IoDevice, ?XDB_HEADER_SIZE + ?XDB_VECTOR_INDEX_COUNT*8), load_vector_index_aux(VectorIndexBin, 0), persistent_term:put(Key, true) end. load_vector_index_aux(<<>>, _Index) -> ok; load_vector_index_aux(<>, Index) -> Term = {Index, SPtr, EPtr}, ets:insert(?XDB_VECTOR_INDEX, Term), load_vector_index_aux(VectorIndexBin, Index + 1). search_ip(IoDevice, Ip) -> IntIp = ip2region_util:ipv4_to_n(Ip), case ets:lookup(?IP2REGION_CACHE, IntIp) of [{_IntIp, RegionInfo}] -> RegionInfo; _ -> <> = <>, VectorIdx = A * ?XDB_VECTOR_COLS + B, [{_, SPtr, EPtr}] = ets:lookup(?XDB_VECTOR_INDEX, VectorIdx), RegionInfo = search_ip(IoDevice, IntIp, SPtr, EPtr, 0, (EPtr - SPtr) div ?XDB_SEGMENT_INDEX_SIZE), ets:insert_new(?IP2REGION_CACHE, {IntIp, RegionInfo}), RegionInfo end. search_ip(IoDevice, IntIp, SPtr, EPtr, Low, High) when Low =< High -> Middle = (Low + High) bsr 1, SPtr2 = SPtr + Middle * ?XDB_SEGMENT_INDEX_SIZE, {SIp, EIp, DataLen, DataPtr} = read_segement_index(IoDevice, SPtr2), if IntIp < SIp -> search_ip(IoDevice, IntIp, SPtr, EPtr, Low, Middle - 1); IntIp > EIp -> search_ip(IoDevice, IntIp, SPtr, EPtr, Middle + 1, High); true -> {ok, DataBin} = read_file(IoDevice, DataPtr, DataLen), unicode:characters_to_nfc_list(DataBin) end; search_ip(_IoDevice, _IntIp, _SPtr, _EPtr, _Low, _High) -> {error, unknown}. read_file(IoDevice, Position, DataLength) -> file:position(IoDevice, {bof, Position}), file:read(IoDevice, DataLength). read_segement_index(IoDevice, SPtr) -> case ets:lookup(?XDB_SEGMENT_INDEX, SPtr) of [{_SPtr, SIp, EIp, DataLen, DataPtr}] -> {SIp, EIp, DataLen, DataPtr}; _ -> {ok, <>} = read_file(IoDevice, SPtr, ?XDB_SEGMENT_INDEX_SIZE), ets:insert_new(?XDB_SEGMENT_INDEX, {SPtr, SIp, EIp, DataLen, DataPtr}), {SIp, EIp, DataLen, DataPtr} end. ================================================ FILE: binding/erlang/src/xdb.erl ================================================ %%%------------------------------------------------------------------- %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. %% %% @doc %% ip2region xdb client search api %% @end %%%------------------------------------------------------------------- -module(xdb). -include("ip2region.hrl"). -export([search/1]). -spec search(Ip :: tuple() | list() | binary()) -> Result :: binary | {error, Reason::atom()}. search(Ip) when is_integer(Ip); is_list(Ip); is_tuple(Ip); is_binary(Ip) -> case ip2region_util:ipv4_to_n(Ip) of IntIp when is_integer(IntIp) -> case ets:lookup(?IP2REGION_CACHE, IntIp) of [{_IntIp, Region}] -> Region; _ -> Worker = poolboy:checkout(?IP2REGION_POOL, true, infinity), try ip2region_worker:search(Worker, IntIp) after poolboy:checkin(?IP2REGION_POOL, Worker) end end; Ret -> Ret end. ================================================ FILE: binding/erlang/src/xdb_benchmark.erl ================================================ %%%------------------------------------------------------------------- %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. %% %% @doc %% ip2region xdb client benchmark test %% @end %%%------------------------------------------------------------------- -module(xdb_benchmark). -export([main/1]). main(DataFile) -> application:ensure_started(ip2region), show_hw_sw_info(), IpList = load_test_data(DataFile), run(IpList). show_hw_sw_info() -> io:format("CPU info:~n", []), io:format("~s", [os:cmd("egrep '^model name' /proc/cpuinfo | head -1")]), io:format("~s", [os:cmd("egrep '^cache' /proc/cpuinfo | head -1")]), io:format("~s", [os:cmd("egrep '^cpu MHz' /proc/cpuinfo | head -1")]), io:format("~s", [os:cmd("egrep '^bogomips' /proc/cpuinfo | head -1")]), io:format("cores/threads : ~s~n", [os:cmd("egrep -c '^processor' /proc/cpuinfo")]), io:format("Erlang info:~n", []), io:format("system_version:~s", [erlang:system_info(system_version)]), ok. load_test_data(DataFile) -> {ok, Fd} = file:open(DataFile, [read]), T0 = os:timestamp(), IpList = load_test_data(Fd, []), T1 = os:timestamp(), Sec = timer:now_diff(T1, T0) / 1000000, io:format("load test data use ~ps~n", [Sec]), IpList. load_test_data(Fd, IpList) -> case file:read_line(Fd) of {ok, Ip} -> case string:tokens(unicode:characters_to_list(Ip), "|") of [SIp | _Tail] -> load_test_data(Fd, [string:trim(SIp)| IpList]); _ -> load_test_data(Fd, IpList) end; _ -> file:close(Fd), IpList end. run(IpList) -> garbage_collect(), io:format("~nstart run benchmark tests~n", []), io:format("~nsearch from file:~n", []), run_test(IpList), io:format("~nsearch from cache:~n", []), run_test(IpList), io:format("~nbenchmark test finish~n", []). run_test(IpList) -> T0 = os:timestamp(), run_test_aux(IpList), T1 = os:timestamp(), Sec = timer:now_diff(T1, T0) / 1000000, IpCount = length(IpList), io:format("ip count:~p,~ntotal time: ~ps,~nsearch ~p times per second,~nuse ~p micro second per search~n", [IpCount, Sec, IpCount / Sec, Sec * 1000000/IpCount]). run_test_aux([]) -> ok; run_test_aux([Ip | Tail]) -> xdb:search(Ip), run_test_aux(Tail). ================================================ FILE: binding/erlang/test/xdb_test.erl ================================================ -module(xdb_test). -include_lib("eunit/include/eunit.hrl"). search_test_() -> application:ensure_started(ip2region), A = "中国|0|广东省|广州市|电信", Region0 = xdb:search("1.0.8.0"), Region1 = xdb:search(<<"1.0.8.0">>), Region2 = xdb:search({1,0,8,0}), Region3 = xdb:search("xxx.0.8.0"), [ ?_assert(A =:= Region0), ?_assert(A =:= Region1), ?_assert(A =:= Region2), ?_assert({error, bad_ip_format} =:= Region3) ]. ================================================ FILE: binding/golang/Makefile ================================================ # ip2region golang binding makefile all: build .PHONY: all build: go build -o xdb_searcher test: go test -v ./... clean: find ./ -name xdb_searcher | xargs rm -f ================================================ FILE: binding/golang/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region golang query client # Usage ### package get ```bash go get github.com/lionsoul2014/ip2region/binding/golang ``` ### About the query service Starting from version `3.11.0`, a dual-protocol compatible and concurrency-safe `Ip2Region` query service is provided. **It is recommended to prioritize this method for query calls.** The specific usage is as follows: ```go import "github.com/lionsoul2014/ip2region/binding/golang/service" // 1. Create v4 configuration: specify the cache policy and the v4 xdb file path // Parameter 1: Cache policy, options: service.NoCache / service.VIndexCache / service.BufferCache // Parameter 2: xdb file path // Parameter 3: Number of initialized searchers v4Config, err := service.NewV4Config(service.VIndexCache, "ip2region v4 xdb path", 20) if err != nil { return fmt.Errorf("failed to create v4 config: %s", err) } // 2. Create v6 configuration: specify the cache policy and the v6 xdb file path v6Config, err := service.NewV6Config(service.VIndexCache, "ip2region v6 xdb path", 20) if err != nil { return fmt.Errorf("failed to create v6 config: %s", err) } // 3. Create the Ip2Region query service using the above configurations ip2region, err := service.NewIp2Region(v4Config, v6Config) if err != nil { return fmt.Errorf("failed to create ip2region service: %s", err) } // 4. Export the ip2region service for concurrent dual-version IP address queries, for example: v4Region, err := ip2region.SearchByStr("113.92.157.29") // Perform IPv4 query v6Region, err := ip2region.SearchByStr("240e:3b7:3272:d8d0:db09:c067:8d59:539e") // Perform IPv6 query // 5. When the parent service needs to be closed, close the ip2region query service as well ip2region.Close() ``` ##### `Ip2Region` Query Notes: 1. The API of this query service is concurrency-safe and supports both IPv4 and IPv6 addresses; the internal implementation handles identification automatically. 2. v4 and v6 configurations need to be created separately. You can set different cache policies for v4 and v6, or specify one as `nil`, which will cause queries for that IP version to return `""`. 3. Please set an appropriate number of searchers based on your project's concurrency. This value is fixed during runtime; each query borrows a searcher from the pool to complete the operation and returns it afterward. If the pool is empty during borrowing, it will wait until a searcher becomes available. 4. If the cache policy is set to `service.BufferCache` (Full Memory Cache), a single-instance memory searcher is used by default. This implementation is natively concurrency-safe, and the specified number of searchers will be ignored. 5. If `Close` is called while the `Ip2Region` service is running, it will wait up to 10 seconds by default for searchers to be returned. You can also call `CloseTimeout` to define a custom maximum wait time. ### About the Query API The location information query API prototypes are: ```go SearchByStr(string) (string, error) Search([]byte) (string, error) ``` If a query fails, the `error` will contain specific error details. If successful, it returns the `region` string. If the specified IP cannot be found, it returns an empty string `""`. ### About IPv4 / IPv6 This xdb query client implementation supports both IPv4 and IPv6 queries. Usage is as follows: ```go // For IPv4: Set the xdb path to the v4 xdb file and specify the IP version as xdb.IPv4 dbPath := "../../data/ip2region_v4.xdb" // Or your ipv4 xdb path version := xdb.IPv4 // For IPv6: Set the xdb path to the v6 xdb file and specify the IP version as xdb.IPv6 dbPath = "../../data/ip2region_v6.xdb" // Or your ipv6 xdb path version = xdb.IPv6 // The IP version of the xdb specified by dbPath must match the version specified, otherwise the query will throw an error. // Note: The following demonstrations directly use the dbPath and version variables. ``` ### File Verification It is recommended to actively verify the applicability of the xdb file. New features in later versions may make the current Searcher version incompatible with your xdb file. Verification helps avoid unpredictable errors during runtime. You do not need to verify every time; for example, run it during service startup or manually via command line to confirm version matching. Do not run verification every time a Searcher is created, as this will impact query response speed, especially in high-concurrency scenarios. ```go err := xdb.VerifyFromFile(dbPath) if err != nil { // err contains the verification error return fmt.Errorf("xdb file verify: %w", err) } // The current Searcher can safely be used for query operations on the xdb specified by dbPath. ``` ### Pure File-Based Query ```go import ( "fmt" "github.com/lionsoul2014/ip2region/binding/golang/xdb" "time" ) func main() { // Create a pure file-based query object using version and dbPath searcher, err := xdb.NewWithFileOnly(version, dbPath) if err != nil { fmt.Printf("failed to create searcher: %s\n", err.Error()) return } defer searcher.Close() // Location info query: both IPv4 and IPv6 addresses are supported var ip = "1.2.3.4" // IPv4 // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 var tStart = time.Now() region, err := searcher.SearchByStr(ip) if err != nil { fmt.Printf("failed to SearchIP(%s): %s\n", ip, err) return } // IPv4 or IPv6 location information fmt.Printf("{region: %s, took: %s}\n", region, time.Since(tStart)) // Note: For concurrent use, each goroutine needs to create an independent searcher object. } ``` ### Caching `VectorIndex` You can pre-load the `vectorIndex` cache and store it as a global variable. Using the global `vectorIndex` whenever creating a searcher reduces a fixed IO operation, accelerating queries and reducing system IO pressure. ```go // 1. Load VectorIndex cache from dbPath and store the vIndex variable globally in memory. vIndex, err := xdb.LoadVectorIndexFromFile(dbPath) if err != nil { fmt.Printf("failed to load vector index from `%s`: %s\n", dbPath, err) return } // 2. Create a query object with VectorIndex cache using the global vIndex. searcher, err := xdb.NewWithVectorIndex(version, dbPath, vIndex) if err != nil { fmt.Printf("failed to create searcher with vector index: %s\n", err) return } // Note: For concurrent use, all goroutines share the global read-only vIndex cache, while each goroutine creates an independent searcher object. ``` ### Caching the entire `xdb` file You can pre-load the entire xdb file into memory for full memory-based queries, similar to the previous memory search. ```go // 1. Load the entire xdb from dbPath into memory cBuff, err := xdb.LoadContentFromFile(dbPath) if err != nil { fmt.Printf("failed to load content from `%s`: %s\n", dbPath, err) return } // 2. Create a fully memory-based query object using the global cBuff. searcher, err := xdb.NewWithBuffer(version, cBuff) if err != nil { fmt.Printf("failed to create searcher with content: %s\n", err) return } // Note: For concurrent use, searcher objects created with the entire xdb cache can be safely used for concurrency. ``` # Compile the test program Compile to get the xdb_searcher executable through the following method: ```bash # cd to the golang binding root directory first make ``` # Query test ### Query command Test xdb queries using the `./xdb_searcher search` command: ```bash ➜ golang git:(master) ✗ ./xdb_searcher search --help ./xdb_searcher search [command options] options: --v4-db string ip2region v4 binary xdb file path --v4-cache-policy string v4 cache policy, default vectorIndex, options: file/vectorIndex/content --v6-db string ip2region v6 binary xdb file path --v6-cache-policy string v6 cache policy, default vectorIndex, options: file/vectorIndex/content --help print this help menu ``` ### Parameter parsing 1. `v4-xdb`: IPv4 xdb file path, defaults to data/ip2region_v4.xdb in the repository 2. `v6-xdb`: IPv6 xdb file path, defaults to data/ip2region_v6.xdb in the repository 3. `v4-cache-policy`: Cache policy used for v4 queries, defaults to `vectorIndex`, options: file/vectorIndex/content 4. `v6-cache-policy`: Cache policy used for v6 queries, defaults to `vectorIndex`, options: file/vectorIndex/content ### Test Demo Example: Perform query tests using the default data/ip2region_v4.xdb and data/ip2region_v6.xdb: ```bash ➜ golang git:(master) ✗ ./xdb_searcher search ip2region search service test program +-v4 db: /data01/code/c/ip2region/data/ip2region_v4.xdb (vectorIndex) +-v6 db: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, took: 50.216µs} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, took: 100.606µs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, took: 99.078µs} ``` Enter an IPv4 or IPv6 address to perform query tests. You can also set `cache-policy` to file/vectorIndex/content respectively to test the performance of the three different cache implementations. # bench test ### Test command Perform automatic bench testing using the `xdb_searcher bench` command. This ensures that both the program and the `xdb` file are free of errors, and provides average query performance through a large number of queries: ```bash ➜ golang git:(fr_xdb_ipv6) ./xdb_searcher bench ./xdb_searcher bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` ### v4 bench Example: Perform ipv4 bench testing using data/ip2region_v4.xdb and data/ipv4_source.txt: ```bash ./xdb_searcher bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` ### v6 bench Example: Perform ipv6 bench testing using data/ip2region_v6.xdb and data/ipv6_source.txt: ```bash ./xdb_searcher bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` You can set the `cache-policy` parameter to test the efficiency of file/vectorIndex/content cache mechanisms respectively. *Please note that the src file used for bench needs to be the same source file used to generate the corresponding xdb file*. The bench program will read the source IP file specified by `src` line by line, then select the start and end IPs from each IP segment for testing to ensure that the queried region information matches the original region information. There is no debug information output during the test; if an error occurs, the error message will be printed and execution will terminate. Seeing `Bench finished` indicates the bench was successful. Cost represents the average time for each query operation (ns). ================================================ FILE: binding/golang/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region golang 查询客户端 # 使用方式 ### package 获取 ```bash go get github.com/lionsoul2014/ip2region/binding/golang ``` ### 关于查询服务 从 `3.11.0` 版本开始提供了一个双协议兼容且并发安全的 `Ip2Region` 查询服务，**建议优先使用该方式来进行查询调用**，具体使用方式如下： ```go import "github.com/lionsoul2014/ip2region/binding/golang/service" // 1, 创建 v4 的配置：指定缓存策略和 v4 的 xdb 文件路径 // 参数1： 缓存策略, options: service.NoCache / service.VIndexCache / service.BufferCache // 参数2: xdb 文件路径 // 参数3: 初始化的查询器数量 v4Config, err := service.NewV4Config(service.VIndexCache, "ip2region v4 xdb path", 20) if err != nil { return fmt.Errorf("failed to create v4 config: %s", err) } // 2, 创建 v6 的配置：指定缓存策略和 v6 的 xdb 文件路径 v6Config, err := service.NewV6Config(service.VIndexCache, "ip2region v6 xdb path", 20) if err != nil { return fmt.Errorf("failed to create v6 config: %s", err) } // 3，通过上述配置创建 Ip2Region 查询服务 ip2region, err := service.NewIp2Region(v4Config, v6Config) if err != nil { return fmt.Errorf("failed to create ip2region service: %s", err) } // 4，导出 ip2region 服务进行双版本的IP地址的并发查询，例如： v4Region, err := ip2region.SearchByStr("113.92.157.29") // 进行 IPv4 查询 v6Region, err := ip2region.SearchByStr("240e:3b7:3272:d8d0:db09:c067:8d59:539e") // 进行 IPv6 查询 // 5，在服务需要关闭的时候，同时关闭 ip2region 查询服务 ip2region.Close() ``` ##### `Ip2Region` 查询备注: 1. 该查询服务的 API 并发安全且同时支持 IPv4 和 Ipv6 的地址，内部实现会自动判断。 2. v4 和 v6 的配置需要单独创建，可以给 v4 和 v6 设置使用不同的缓存策略，也可以指定其中一个为 `nil` 则该版本的 IP 地址查询都会返回 `""`。 3. 请结合您的项目的并发数设置一个合适的查询器数量，这个值在运行过程中是固定的，每次查询会从池子里租借一个查询器来完成查询操作，查询完成后再归还回去，如果租借的时候池子已经空了则等待直到有可用的查询器来完成查询服务。 4. 如果配置设置的缓存策略为 `service.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器，该实现天生并发安全，此时指定的查询器数量无效。 5. 如果 `Ip2Region` 查询器在提供服务期间，调用 Close 默认会最大等待 10 秒钟来等待尽量多的查询器归还，也可以调用 `CloseTimeout` 来自定义最长等待时间。 ### 关于查询 API 定位信息查询 API 原型为： ```go SearchByStr(string) (string, error) Search([]byte) (string, error) ``` 查询出错则 error 会包含具体的错误信息，查询成功会返回字符串的 `region` 信息，如果指定的 IP 查询不到则会返回空字符串 `""`。 ### 关于 IPv4 / IPv6 该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询，使用方式如下： ```go // 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 xdb.IPv4 dbPath := "../../data/ip2region_v4.xdb" // 或者你的 ipv4 xdb 的路径 version := xdb.IPv4 // 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件，IP版本指定为 xdb.IPv6 dbPath = "../../data/ip2region_v6.xdb" // 或者你的 ipv6 xdb 路径 version = xdb.IPv6 // dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致，不然查询执行的时候会报错 // 备注：以下演示直接使用 dbPath 和 version 变量 ``` ### 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 ```go err := xdb.VerifyFromFile(dbPath) if err != nil { // err 包含的验证的错误 return fmt.Errorf("xdb file verify: %w", err) } // 当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作 ``` ### 完全基于文件的查询 ```go import ( "fmt" "github.com/lionsoul2014/ip2region/binding/golang/xdb" "time" ) func main() { // 通过 version 和 dbPath 创建完全基于文件的查询对象 searcher, err := xdb.NewWithFileOnly(version, dbPath) if err != nil { fmt.Printf("failed to create searcher: %s\n", err.Error()) return } defer searcher.Close() // 定位信息查询：IPv4 或者 IPv6 的地址都支持 var ip = "1.2.3.4" // IPv4 // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 var tStart = time.Now() region, err := searcher.SearchByStr(ip) if err != nil { fmt.Printf("failed to SearchIP(%s): %s\n", ip, err) return } // IPv4 或者 IPv6 的定位信息 fmt.Printf("{region: %s, took: %s}\n", region, time.Since(tStart)) // 备注：并发使用，每个 goroutine 需要创建一个独立的 searcher 对象。 } ``` ### 缓存 `VectorIndex` 索引 可以预先加载 `vectorIndex` 缓存，然后做成全局变量，每次创建 searcher 的时候使用全局的 `vectorIndex`，可以减少一次固定的 IO 操作从而加速查询，减少系统 io 压力。 ```go // 1、从 dbPath 加载 VectorIndex 缓存，把下述 vIndex 变量全局到内存里面。 vIndex, err := xdb.LoadVectorIndexFromFile(dbPath) if err != nil { fmt.Printf("failed to load vector index from `%s`: %s\n", dbPath, err) return } // 2、用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。 searcher, err := xdb.NewWithVectorIndex(version, dbPath, vIndex) if err != nil { fmt.Printf("failed to create searcher with vector index: %s\n", err) return } // 备注：并发使用，全部 goroutine 共享全局的只读 vIndex 缓存，每个 goroutine 创建一个独立的 searcher 对象 ``` ### 缓存整个 `xdb` 文件 可以预先加载整个 xdb 文件到内存，完全基于内存查询，类似于之前的 memory search 查询。 ```go // 1、从 dbPath 加载整个 xdb 到内存 cBuff, err := xdb.LoadContentFromFile(dbPath) if err != nil { fmt.Printf("failed to load content from `%s`: %s\n", dbPath, err) return } // 2、用全局的 cBuff 创建完全基于内存的查询对象。 searcher, err := xdb.NewWithBuffer(version, cBuff) if err != nil { fmt.Printf("failed to create searcher with content: %s\n", err) return } // 备注：并发使用，用整个 xdb 缓存创建的 searcher 对象可以安全用于并发。 ``` # 编译测试程序 通过如下方式编译得到 xdb_searcher 可执行程序： ```bash # 切换到 golang binding 根目录 make ``` # 查询测试 ### 查询命令 通过 `./xdb_searcher search` 命令来测试 xdb 的查询： ```bash ➜ golang git:(master) ✗ ./xdb_searcher search --help ./xdb_searcher search [command options] options: --v4-db string ip2region v4 binary xdb file path --v4-cache-policy string v4 cache policy, default vectorIndex, options: file/vectorIndex/content --v6-db string ip2region v6 binary xdb file path --v6-cache-policy string v6 cache policy, default vectorIndex, options: file/vectorIndex/content --help print this help menu ``` ### 参数解析 1. `v4-xdb`: IPv4 的 xdb 文件路径，默认为仓库中的 data/ip2region_v4.xdb 2. `v6-xdb`: IPv6 的 xdb 文件路径，默认为仓库中的 data/ip2region_v6.xdb 3. `v4-cache-policy`: v4 查询使用的缓存策略，默认为 `vectorIndex`，可选：file/vectorIndex/content 4. `v6-cache-policy`: v6 查询使用的缓存策略，默认为 `vectorIndex`，可选：file/vectorIndex/content ### 测试 Demo 例如：使用默认的 data/ip2region_v4.xdb 和 data/ip2region_v6.xdb 进行查询测试： ```bash ➜ golang git:(master) ✗ ./xdb_searcher search ip2region search service test program +-v4 db: /data01/code/c/ip2region/data/ip2region_v4.xdb (vectorIndex) +-v6 db: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, took: 50.216µs} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, took: 100.606µs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, took: 99.078µs} ``` 输入 v4 或者 v6 的 IP 地址即可进行查询测试，也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。 # bench 测试 ### 测试命令 通过 `xdb_searcher bench` 命令来进行自动 bench 测试，一方面确保程序和 `xdb` 文件都没有错误，另一方面通过大量的查询得到平均查询性能： ```bash ➜ golang git:(fr_xdb_ipv6) ./xdb_searcher bench ./xdb_searcher bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` ### v4 bench 例如：通过 data/ip2region_v4.xdb 和 data/ipv4_source.txt 进行 ipv4 的 bench 测试： ```bash ./xdb_searcher bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` ### v6 bench 例如：通过 data/ip2region_v6.xdb 和 data/ipv6_source.txt 进行 ipv6 的 bench 测试： ```bash ./xdb_searcher bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` 可以设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 不同缓存实现机制的效率。 *请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件*。 bench 程序会逐行读取 `src` 指定的源IP文件，然后每个 IP 段选取开始和结束的 IP 进行测试，以确保查询的 region 信息和原始的 region 信息是相同。测试途中没有调试信息的输出，有错误会打印错误信息并且终止运行，所以看到 `Bench finished` 就表示 bench 成功了，cost 是表示每次查询操作的平均时间(ns)。 ================================================ FILE: binding/golang/go.mod ================================================ module github.com/lionsoul2014/ip2region/binding/golang go 1.17 require github.com/mitchellh/go-homedir v1.1.0 ================================================ FILE: binding/golang/go.sum ================================================ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= ================================================ FILE: binding/golang/main.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/16 package main import ( "bufio" "fmt" "log" "os" "path/filepath" "strings" "time" "github.com/lionsoul2014/ip2region/binding/golang/service" "github.com/lionsoul2014/ip2region/binding/golang/xdb" "github.com/mitchellh/go-homedir" ) func getXdbPath(fileName string) (string, error) { binPath, err := os.Executable() if err != nil { return "", fmt.Errorf("failed to get executale: %w", err) } xdbPath := filepath.Join(filepath.Dir(filepath.Dir(filepath.Dir(binPath))), "/data/", fileName) _, err = os.Stat(xdbPath) if err != nil { return "", nil } // fmt.Printf("xdbPath=%s\n", xdbPath) return xdbPath, nil } func createService(v4XdbPath string, v4CachePolicy string, v6XdbPath string, v6CachePolicy string) (*service.Ip2Region, error) { // try to create v4 config v4CPolicy, err := service.CachePolicyFromName(v4CachePolicy) if err != nil { return nil, fmt.Errorf("parse v4 cache policy: %w", err) } v4DbPath, err := homedir.Expand(v4XdbPath) if err != nil { return nil, fmt.Errorf("Expand(`%s`): %s", v4XdbPath, err) } v4Config, err := service.NewV4Config(v4CPolicy, v4DbPath, 1) if err != nil { return nil, fmt.Errorf("NewV4Config: %w", err) } // try to create v6 config v6Policy, err := service.CachePolicyFromName(v6CachePolicy) if err != nil { return nil, fmt.Errorf("parse v6 cache policy: %w", err) } v6DbPath, err := homedir.Expand(v6XdbPath) if err != nil { return nil, fmt.Errorf("Expand(`%s`): %s", v6XdbPath, err) } v6Config, err := service.NewV6Config(v6Policy, v6DbPath, 1) if err != nil { return nil, fmt.Errorf("NewV6Config: %w", err) } return service.NewIp2Region(v4Config, v6Config) } func createSearcher(dbPath string, cachePolicy string) (*xdb.Searcher, error) { handle, err := os.OpenFile(dbPath, os.O_RDONLY, 0600) if err != nil { return nil, fmt.Errorf("open xdb file `%s`: %w", dbPath, err) } defer handle.Close() // verify the xdb file // @Note: do NOT call it every time you create a searcher since this will slow down the search response. // @see the util.Verify function for details. err = xdb.Verify(handle) if err != nil { return nil, fmt.Errorf("xdb verify: %w", err) } // auto-detect the ip version from the xdb header header, err := xdb.LoadHeader(handle) if err != nil { return nil, fmt.Errorf("failed to load header from `%s`: %s", dbPath, err) } version, err := xdb.VersionFromHeader(header) if err != nil { return nil, fmt.Errorf("failed to detect IP version from `%s`: %s", dbPath, err) } switch cachePolicy { case "nil", "file": return xdb.NewWithFileOnly(version, dbPath) case "vectorIndex": vIndex, err := xdb.LoadVectorIndexFromFile(dbPath) if err != nil { return nil, fmt.Errorf("failed to load vector index from `%s`: %w", dbPath, err) } return xdb.NewWithVectorIndex(version, dbPath, vIndex) case "content": cBuff, err := xdb.LoadContentFromFile(dbPath) if err != nil { return nil, fmt.Errorf("failed to load content from '%s': %w", dbPath, err) } return xdb.NewWithBuffer(version, cBuff) default: return nil, fmt.Errorf("invalid cache policy `%s`, options: file/vectorIndex/content", cachePolicy) } } func printHelp() { fmt.Printf("ip2region xdb searcher\n") fmt.Printf("%s [command] [command options]\n", os.Args[0]) fmt.Printf("Command: \n") fmt.Printf(" search search input test\n") fmt.Printf(" bench search bench test\n") } func testSearch() { var err error var help = "" var v4DbPath, v4CachePolicy = "", "vectorIndex" var v6DbPath, v6CachePolicy = "", "vectorIndex" for i := 2; i < len(os.Args); i++ { r := os.Args[i] if len(r) < 5 { continue } if strings.Index(r, "--") != 0 { continue } var key, val = "", "" var sIdx = strings.Index(r, "=") if sIdx < 0 { // fmt.Printf("missing = for args pair '%s'\n", r) // return key = r[2:] } else { key = r[2:sIdx] val = r[sIdx+1:] } switch key { case "help": if val == "" { help = "true" } else { help = val } case "v4-db": v4DbPath = val case "v4-cache-policy": v4CachePolicy = val case "v6-db": v6DbPath = val case "v6-cache-policy": v6CachePolicy = val default: fmt.Printf("undefined option `%s`\n", r) return } } // check and get the get the default v4 xdb path if v4DbPath == "" { v4DbPath, err = getXdbPath("ip2region_v4.xdb") if err != nil { fmt.Printf("failed to get v4 xdb path: %s", err) return } } // check and get the get the default v6 xdb path if v6DbPath == "" { v6DbPath, err = getXdbPath("ip2region_v6.xdb") if err != nil { fmt.Printf("failed to get v6 xdb path: %s", err) return } } if v4DbPath == "" || v6DbPath == "" || help == "true" { fmt.Printf("%s search [command options]\n", os.Args[0]) fmt.Printf("options:\n") fmt.Printf(" --v4-db string ip2region v4 binary xdb file path\n") fmt.Printf(" --v4-cache-policy string v4 cache policy, default vectorIndex, options: file/vectorIndex/content\n") fmt.Printf(" --v6-db string ip2region v6 binary xdb file path\n") fmt.Printf(" --v6-cache-policy string v6 cache policy, default vectorIndex, options: file/vectorIndex/content\n") fmt.Printf(" --help print this help menu\n") return } // create the search service with the xdb paths and cache policies ip2region, err := createService(v4DbPath, v4CachePolicy, v6DbPath, v6CachePolicy) if err != nil { fmt.Printf("failed to create ip2region service: %s\n", err.Error()) return } defer func() { ip2region.Close() fmt.Printf("searcher test program exited, thanks for trying\n") }() fmt.Printf(`ip2region search service test program +-v4 db: %s (%s) +-v6 db: %s (%s) type 'quit' to exit `, v4DbPath, v4CachePolicy, v6DbPath, v6CachePolicy) reader := bufio.NewReader(os.Stdin) for { fmt.Print("ip2region>> ") str, err := reader.ReadString('\n') if err != nil { log.Fatalf("failed to read string: %s", err) } line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) if len(line) == 0 { continue } if line == "quit" { break } tStart := time.Now() region, err := ip2region.SearchByStr(line) if err != nil { fmt.Printf("\x1b[0;31m{err: %s}\x1b[0m\n", err.Error()) } else { fmt.Printf("\x1b[0;32m{region: %s, took: %s}\x1b[0m\n", region, time.Since(tStart)) } } } func testBench() { var err error var dbFile, srcFile, cachePolicy = "", "", "vectorIndex" for i := 2; i < len(os.Args); i++ { r := os.Args[i] if len(r) < 5 { continue } if strings.Index(r, "--") != 0 { continue } var sIdx = strings.Index(r, "=") if sIdx < 0 { fmt.Printf("missing = for args pair '%s'\n", r) return } switch r[2:sIdx] { case "db": dbFile = r[sIdx+1:] case "src": srcFile = r[sIdx+1:] case "cache-policy": cachePolicy = r[sIdx+1:] default: fmt.Printf("undefined option `%s`\n", r) return } } if dbFile == "" || srcFile == "" { fmt.Printf("%s bench [command options]\n", os.Args[0]) fmt.Printf("options:\n") fmt.Printf(" --db string ip2region binary xdb file path\n") fmt.Printf(" --src string source ip text file path\n") fmt.Printf(" --cache-policy string cache policy: file/vectorIndex/content\n") return } dbPath, err := homedir.Expand(dbFile) if err != nil { fmt.Printf("invalid xdb file path `%s`: %s", dbFile, err) return } searcher, err := createSearcher(dbPath, cachePolicy) if err != nil { fmt.Printf("failed to create searcher: %s\n", err.Error()) return } defer func() { searcher.Close() }() handle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) if err != nil { fmt.Printf("failed to open source text file: %s\n", err) return } defer handle.Close() var count, tStart, costs = int64(0), time.Now(), int64(0) var scanner = bufio.NewScanner(handle) scanner.Split(bufio.ScanLines) for scanner.Scan() { var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n")) var ps = strings.SplitN(l, "|", 3) if len(ps) != 3 { fmt.Printf("invalid ip segment line `%s`\n", l) return } sip, err := xdb.ParseIP(ps[0]) if err != nil { fmt.Printf("check start ip `%s`: %s\n", ps[0], err) return } eip, err := xdb.ParseIP(ps[1]) if err != nil { fmt.Printf("check end ip `%s`: %s\n", ps[1], err) return } if xdb.IPCompare(sip, eip) > 0 { fmt.Printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]) return } for _, ip := range [][]byte{sip, eip} { sTime := time.Now() region, err := searcher.Search(ip) if err != nil { fmt.Printf("failed to search ip '%s': %s\n", xdb.IP2String(ip), err) return } costs += time.Since(sTime).Nanoseconds() // check the region info if region != ps[2] { fmt.Printf("failed Search(%s) with (%s != %s)\n", xdb.IP2String(ip), region, ps[2]) return } count++ } } cost := time.Since(tStart) fmt.Printf("Bench finished, {cachePolicy: %s, total: %d, took: %s, cost: %d μs/op}\n", cachePolicy, count, cost, costs/count/1000) } func main() { if len(os.Args) < 2 { printHelp() return } // set the log flag log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) switch strings.ToLower(os.Args[1]) { case "search": testSearch() case "bench": testBench() default: printHelp() } } ================================================ FILE: binding/golang/service/config.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package service import ( "fmt" "os" "strings" "github.com/lionsoul2014/ip2region/binding/golang/xdb" ) // --- // Ip2Region service config // // @Author Lion // @Date 2025/12/03 const ( NoCache = 0 VIndexCache = 1 BufferCache = 2 ) func CachePolicyFromName(name string) (int, error) { switch strings.ToLower(name) { case "file", "nocache": return NoCache, nil case "vectorindex", "vindex", "vindexcache": return VIndexCache, nil case "content", "buffercache": return BufferCache, nil default: return NoCache, fmt.Errorf("invalid cache policy name `%s`", name) } } type Config struct { cachePolicy int ipVersion *xdb.Version // xdb file path xdbPath string header *xdb.Header // buffers vIndex []byte cBuffer []byte searchers int } func NewV4Config(cachePolicy int, xdbPath string, searchers int) (*Config, error) { return newConfig(cachePolicy, xdb.IPv4, xdbPath, searchers) } func NewV6Config(cachePolicy int, xdbPath string, searchers int) (*Config, error) { return newConfig(cachePolicy, xdb.IPv6, xdbPath, searchers) } func newConfig(cachePolicy int, ipVersion *xdb.Version, xdbPath string, searchers int) (*Config, error) { if searchers < 1 { return nil, fmt.Errorf("searchers=%d, > 0 expected", searchers) } // open the xdb binary file handle, err := os.OpenFile(xdbPath, os.O_RDONLY, 0600) if err != nil { return nil, err } // 1, verify the xdb err = xdb.Verify(handle) if err != nil { return nil, err } // 2, load the header header, err := xdb.LoadHeader(handle) if err != nil { return nil, err } // verify the ip version xIpVersion, err := xdb.VersionFromHeader(header) if err != nil { return nil, err } if xIpVersion.Id != ipVersion.Id { return nil, fmt.Errorf("ip verison not match: xdb file %s with ip version=%s, as %s expected", xdbPath, xIpVersion.Name, ipVersion.Name) } // 3, check and load the vector index buffer var vIndex []byte = nil if cachePolicy == VIndexCache { vIndex, err = xdb.LoadVectorIndex(handle) if err != nil { return nil, err } } // 4, check and load the content buffer var cBuffer []byte = nil if cachePolicy == BufferCache { cBuffer, err = xdb.LoadContent(handle) if err != nil { return nil, err } } return &Config{ cachePolicy: cachePolicy, ipVersion: ipVersion, xdbPath: xdbPath, header: header, vIndex: vIndex, cBuffer: cBuffer, searchers: searchers, }, nil } func (c *Config) String() string { vIndex := "null" if c.vIndex != nil { vIndex = fmt.Sprintf("{bytes:%d}", len(c.vIndex)) } cBuffer := "null" if c.cBuffer != nil { cBuffer = fmt.Sprintf("{bytes:%d}", len(c.cBuffer)) } return fmt.Sprintf( "{cache_policy:%d, version:%s, xdb_path:%s, header:%s, v_index:%s, c_buffer:%s}", c.cachePolicy, c.ipVersion.String(), c.xdbPath, c.header.String(), vIndex, cBuffer, ) } func (c *Config) CachePolicy() int { return c.cachePolicy } func (c *Config) IPVersion() *xdb.Version { return c.ipVersion } func (c *Config) Header() *xdb.Header { return c.header } func (c *Config) VIndex() []byte { return c.vIndex } func (c *Config) CBuffer() []byte { return c.cBuffer } func (c *Config) Searchers() int { return c.searchers } ================================================ FILE: binding/golang/service/config_test.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package service import ( "fmt" "testing" ) func TestV4Config(t *testing.T) { v4Config, err := NewV4Config(VIndexCache, "../../../data/ip2region_v4.xdb", 10) if err != nil { t.Errorf("failed to new v4 config: %s", err) return } v4BufferConfig, err := NewV4Config(BufferCache, "../../../data/ip2region_v4.xdb", 10) if err != nil { t.Errorf("failed to new v4 config: %s", err) return } fmt.Printf("v4Config: %s\n", v4Config) fmt.Printf("v4BufferConfig: %s\n", v4BufferConfig) } func TestV6Config(t *testing.T) { v6Config, err := NewV6Config(NoCache, "../../../data/ip2region_v6.xdb", 10) if err != nil { t.Errorf("failed to new v6 config: %s", err) return } fmt.Printf("v6Config: %s\n", v6Config) } ================================================ FILE: binding/golang/service/ip2region.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package service import ( "fmt" "time" "github.com/lionsoul2014/ip2region/binding/golang/xdb" ) // --- // Ip2Region service // 1. Unified query interface to IPv4 and IPv6 address. // 2. Concurrency search support. // // @Author Lion // @Date 2025/12/05 type Ip2Region struct { // v4 pool for cache policy vIndex or NoCache v4Pool *SearcherPool // v4 xdb searcher for full in-memeory search v4InMemSearcher *xdb.Searcher // v6 pool for cache policy vIndex or NoCache:w v6Pool *SearcherPool // v6 xdb searcher for full in-memeory search v6InMemSearcher *xdb.Searcher } // create a new Ip2Region service with specified v4 and v6 config. // set it to nil to disabled the specified search for the specified version. func NewIp2Region(v4Config *Config, v6Config *Config) (*Ip2Region, error) { var err error // check and init the v4 pool or in-memory searcher var v4Pool *SearcherPool var v4InMemSearcher *xdb.Searcher if v4Config == nil { // with IPv4 disabled ? v4Pool = nil v4InMemSearcher = nil } else if v4Config.cachePolicy == BufferCache { v4Pool = nil v4InMemSearcher, err = xdb.NewWithBuffer(v4Config.ipVersion, v4Config.cBuffer) if err != nil { return nil, fmt.Errorf("failed to create v4 in-memory searcher: %w", err) } } else { v4InMemSearcher = nil v4Pool, err = NewSearcherPool(v4Config) if err != nil { return nil, fmt.Errorf("failed to create v4 searcher pool: %w", err) } } // check and init the v6 pool or in-memory searcher var v6Pool *SearcherPool var v6InMemSearcher *xdb.Searcher if v6Config == nil { v6Pool = nil v6InMemSearcher = nil } else if v6Config.cachePolicy == BufferCache { v6Pool = nil v6InMemSearcher, err = xdb.NewWithBuffer(v6Config.ipVersion, v6Config.cBuffer) if err != nil { return nil, fmt.Errorf("failed to create v6 in-memory searcher: %w", err) } } else { v6InMemSearcher = nil v6Pool, err = NewSearcherPool(v6Config) if err != nil { return nil, fmt.Errorf("failed to create v6 in-memeory searcher pool: %w", err) } } return &Ip2Region{ v4Pool: v4Pool, v4InMemSearcher: v4InMemSearcher, v6Pool: v6Pool, v6InMemSearcher: v6InMemSearcher, }, nil } // create the ip2region search service with the specified v4 & v6 xdb path. // with default cache policy VIndexCache and default searchers = 20 func NewIp2RegionWithPath(v4XdbPath string, v6XdbPath string) (*Ip2Region, error) { var err error // create v4 config with default config items var v4Config *Config if v4XdbPath == "" { v4Config = nil } else { v4Config, err = NewV4Config(VIndexCache, v4XdbPath, 20) if err != nil { return nil, fmt.Errorf("failed to create v4 config: %w", err) } } // create v6 config with default config items var v6Config *Config if v6XdbPath == "" { v6Config = nil } else { v6Config, err = NewV6Config(VIndexCache, v6XdbPath, 20) if err != nil { return nil, fmt.Errorf("failed to create v6 config: %w", err) } } return NewIp2Region(v4Config, v6Config) } func (ip2r *Ip2Region) SearchByStr(ipStr string) (string, error) { ipBytes, err := xdb.ParseIP(ipStr) if err != nil { return "", err } return ip2r.Search(ipBytes) } func (ip2r *Ip2Region) Search(ipBytes []byte) (string, error) { if l := len(ipBytes); l == 4 { return ip2r.v4Search(ipBytes) } else if l == 16 { return ip2r.v6Search(ipBytes) } else { return "", fmt.Errorf("invalid byte ip address with len=%d", l) } } func (ip2r *Ip2Region) v4Search(ipBytes []byte) (string, error) { if ip2r.v4InMemSearcher != nil { return ip2r.v4InMemSearcher.Search(ipBytes) } // v4 search is disabled if ip2r.v4Pool == nil { return "", nil } v4Searcher := ip2r.v4Pool.BorrowSearcher() defer ip2r.v4Pool.ReturnSearcher(v4Searcher) return v4Searcher.Search(ipBytes) } func (ip2r *Ip2Region) v6Search(ipBytes []byte) (string, error) { if ip2r.v6InMemSearcher != nil { return ip2r.v6InMemSearcher.Search(ipBytes) } // v6 search is disabled if ip2r.v6Pool == nil { return "", nil } v6Searcher := ip2r.v6Pool.BorrowSearcher() defer ip2r.v6Pool.ReturnSearcher(v6Searcher) return v6Searcher.Search(ipBytes) } func (ip2r *Ip2Region) Close() { ip2r.CloseTimeout(time.Second * 10) } func (ip2r *Ip2Region) CloseTimeout(d time.Duration) { if ip2r.v4Pool != nil { ip2r.v4Pool.CloseTimeout(d) } if ip2r.v6Pool != nil { ip2r.v6Pool.CloseTimeout(d) } } ================================================ FILE: binding/golang/service/ip2region_test.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package service import ( "fmt" "sync" "sync/atomic" "testing" "time" "github.com/lionsoul2014/ip2region/binding/golang/xdb" ) func TestConfigCreate(t *testing.T) { v4Config, err := NewV4Config(VIndexCache, "../../../data/ip2region_v4.xdb", 10) if err != nil { t.Fatalf("failed to create v4 config: %s", err) } v6Config, err := NewV6Config(VIndexCache, "../../../data/ip2region_v6.xdb", 10) if err != nil { t.Fatalf("failed to create v6 config: %s", err) } ip2region, err := NewIp2Region(v4Config, v6Config) if err != nil { t.Fatalf("failed to create ip2region service: %s", err) } v4Bytes, err := xdb.ParseIP("219.133.110.197") if err != nil { t.Fatal("invalid ipv4 address") } v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") if err != nil { t.Fatalf("invalid ipv6 address") } for i := 0; i < 20; i++ { v4Bytes = xdb.IPAddOne(v4Bytes) v6Bytes = xdb.IPAddOne(v6Bytes) v4Region, err := ip2region.Search(v4Bytes) if err != nil { t.Fatalf("failed to search(%s): %s", xdb.IP2String(v4Bytes), err) } v6Region, err := ip2region.Search(v6Bytes) if err != nil { t.Fatalf("failed to search(%s): %s", xdb.IP2String(v6Bytes), err) } fmt.Printf( "%2d->search(%s)=%s, search(%s)=%s\n", i, xdb.IP2String(v4Bytes), v4Region, xdb.IP2String(v6Bytes), v6Region, ) } ip2region.Close() fmt.Print("ip2region closed gracefully") } func TestPathCreate(t *testing.T) { ip2region, err := NewIp2RegionWithPath("../../../data/ip2region_v4.xdb", "../../../data/ip2region_v6.xdb") if err != nil { t.Fatalf("failed to create ip2region with path: %s", err) } v4Bytes, err := xdb.ParseIP("219.133.110.197") if err != nil { t.Fatal("invalid ipv4 address") } v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") if err != nil { t.Fatalf("invalid ipv6 address") } for i := 0; i < 20; i++ { v4Bytes = xdb.IPAddOne(v4Bytes) v6Bytes = xdb.IPAddOne(v6Bytes) v4Region, err := ip2region.Search(v4Bytes) if err != nil { t.Fatalf("failed to search(%s): %s", xdb.IP2String(v4Bytes), err) } v6Region, err := ip2region.Search(v6Bytes) if err != nil { t.Fatalf("failed to search(%s): %s", xdb.IP2String(v6Bytes), err) } fmt.Printf( "%2d->search(%s)=%s, search(%s)=%s\n", i, xdb.IP2String(v4Bytes), v4Region, xdb.IP2String(v6Bytes), v6Region, ) } ip2region.Close() fmt.Print("ip2region closed gracefully") } func TestInMemSearch(t *testing.T) { v4Config, err := NewV4Config(BufferCache, "../../../data/ip2region_v4.xdb", 10) if err != nil { t.Fatalf("failed to create v4 config: %s", err) } v6Config, err := NewV6Config(BufferCache, "../../../data/ip2region_v6.xdb", 10) if err != nil { t.Fatalf("failed to create v6 config: %s", err) } ip2region, err := NewIp2Region(v4Config, v6Config) if err != nil { t.Fatalf("failed to create ip2region service: %s", err) } v4Bytes, err := xdb.ParseIP("219.133.110.197") if err != nil { t.Fatal("invalid ipv4 address") } v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") if err != nil { t.Fatalf("invalid ipv6 address") } for i := 0; i < 20; i++ { v4Bytes = xdb.IPAddOne(v4Bytes) v6Bytes = xdb.IPAddOne(v6Bytes) v4Region, err := ip2region.Search(v4Bytes) if err != nil { t.Fatalf("failed to search(%s): %s", xdb.IP2String(v4Bytes), err) } v6Region, err := ip2region.Search(v6Bytes) if err != nil { t.Fatalf("failed to search(%s): %s", xdb.IP2String(v6Bytes), err) } fmt.Printf( "%2d->search(%s)=%s, search(%s)=%s\n", i, xdb.IP2String(v4Bytes), v4Region, xdb.IP2String(v6Bytes), v6Region, ) } ip2region.Close() fmt.Print("ip2region closed gracefully") } func TestV4Only(t *testing.T) { v4Config, err := NewV4Config(NoCache, "../../../data/ip2region_v4.xdb", 10) if err != nil { t.Fatalf("failed to create v4 config: %s", err) } ip2region, err := NewIp2Region(v4Config, nil) if err != nil { t.Fatalf("failed to create ip2region service: %s", err) } v4Bytes, err := xdb.ParseIP("219.133.110.197") if err != nil { t.Fatal("invalid ipv4 address") } v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") if err != nil { t.Fatalf("invalid ipv6 address") } for i := 0; i < 10; i++ { v4Bytes = xdb.IPAddOne(v4Bytes) v6Bytes = xdb.IPAddOne(v6Bytes) v4Region, err := ip2region.Search(v4Bytes) if err != nil { t.Fatalf("failed to search(%s): %s", xdb.IP2String(v4Bytes), err) } v6Region, err := ip2region.Search(v6Bytes) if err != nil { t.Fatalf("failed to search(%s): %s", xdb.IP2String(v6Bytes), err) } fmt.Printf( "%2d->search(%s)=%s, search(%s)=%s\n", i, xdb.IP2String(v4Bytes), v4Region, xdb.IP2String(v6Bytes), v6Region, ) } ip2region.Close() fmt.Print("ip2region closed gracefully") } func TestConcurrentCall(t *testing.T) { v4Config, err := NewV4Config(VIndexCache, "../../../data/ip2region_v4.xdb", 15) if err != nil { t.Fatalf("failed to create v4 config: %s", err) } v6Config, err := NewV6Config(VIndexCache, "../../../data/ip2region_v6.xdb", 15) if err != nil { t.Fatalf("failed to create v6 config: %s", err) } v4Bytes, err := xdb.ParseIP("219.133.110.197") if err != nil { t.Fatal("invalid ipv4 address") } v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") if err != nil { t.Fatalf("invalid ipv6 address") } fmt.Printf("v4Config: %s\n", v4Config) fmt.Printf("v6Config: %s\n", v6Config) ip2region, err := NewIp2Region(v4Config, v6Config) if err != nil { t.Fatalf("failed to create ip2region service: %s", err) } coroutines := 100 var wg sync.WaitGroup var count int64 = 0 tStart := time.Now() for i := 0; i < coroutines; i++ { wg.Add(1) go func() { var ipBytes []byte for i := 0; i < 5000; i++ { if i%2 == 0 { ipBytes = v4Bytes } else { ipBytes = v6Bytes } region, err := ip2region.Search(ipBytes) if err != nil { fmt.Printf("Error: failed to search(%s): %s", xdb.IP2String(ipBytes), err) break } if l := len(ipBytes); l == 4 { if region != "中国|广东省|深圳市|电信" { fmt.Print("Error: region not equals") break } } else { if region != "中国|广东省|深圳市|家庭宽带" { fmt.Print("Error: region not equals") break } } atomic.AddInt64(&count, 1) } // mark all searches finished for this coroutine wg.Done() }() } // wait for all the searches to finished wg.Wait() costs := time.Since(tStart) fmt.Printf("%d searches finished in %s, avg took: %s\n", count, costs, costs/time.Duration(count)) ip2region.Close() fmt.Print("ip2region closed gracefully") } ================================================ FILE: binding/golang/service/searcher_pool.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package service import ( "fmt" "sync/atomic" "time" "github.com/lionsoul2014/ip2region/binding/golang/xdb" ) // --- // ip2region searcher pool // // @Author Lion // @Date 2025/12/03 type SearcherPool struct { // config config *Config // searcher pool pool chan *xdb.Searcher // for pool close closing chan struct{} // searcher number that was loaned out loanCount int32 } func NewSearcherPool(config *Config) (*SearcherPool, error) { if config.searchers < 1 { return nil, fmt.Errorf("config.searchers must > 0") } pool := make(chan *xdb.Searcher, config.searchers+1) // check and create all the searchers for i := 0; i < config.searchers; i++ { searcher, err := xdb.NewSearcher(config.ipVersion, config.xdbPath, config.vIndex, config.cBuffer) if err != nil { return nil, fmt.Errorf("failed to create the %dth searcher: %w", i+1, err) } // push the search to the pool pool <- searcher } return &SearcherPool{ config: config, pool: pool, closing: make(chan struct{}, 1), loanCount: 0, }, nil } // get the loaned count func (sp *SearcherPool) LoanCount() int { return int(atomic.LoadInt32(&sp.loanCount)) } func (sp *SearcherPool) BorrowSearcher() *xdb.Searcher { // @Note: still accept searcher borrow while closing s := <-sp.pool atomic.AddInt32(&sp.loanCount, 1) return s } func (sp *SearcherPool) ReturnSearcher(searcher *xdb.Searcher) { select { case <-sp.closing: // manually close the searcher searcher.Close() // decrease the loan count atomic.AddInt32(&sp.loanCount, -1) default: // return the searcher sp.pool <- searcher atomic.AddInt32(&sp.loanCount, -1) } } func (sp *SearcherPool) Close() { sp.CloseTimeout(time.Second * 10) } func (sp *SearcherPool) CloseTimeout(d time.Duration) { close(sp.closing) for { timeout := false select { case s := <-sp.pool: s.Close() case <-time.After(d): // check if all the loaned searchers was closed timeout = true } lc, left := sp.LoanCount(), len(sp.pool) if left == 0 && lc == 0 { break } if timeout { break } } } ================================================ FILE: binding/golang/service/searcher_pool_test.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package service import ( "fmt" "testing" ) func TestV4SearcherPool(t *testing.T) { v4Config, err := NewV4Config(VIndexCache, "../../../data/ip2region_v4.xdb", 5) if err != nil { t.Fatalf("failed to new v4 config: %s", err) } searcherPool, err := NewSearcherPool(v4Config) if err != nil { t.Fatalf("failed to create searcher pool: %s", err) } ipString := "219.133.110.197" for i := 0; i < 20; i++ { searcher := searcherPool.BorrowSearcher() region, err := searcher.SearchByStr(ipString) if err != nil { t.Fatalf("failed to search(%s): %s", ipString, err) } fmt.Printf("%2d->search(%s)=%s\n", i, ipString, region) searcherPool.ReturnSearcher(searcher) } // borrow one at last for Close timeout wait testing ONLY // searcherPool.BorrowSearcher() // close the searcher pool searcherPool.Close() } func TestV6SearcherPool(t *testing.T) { v6Config, err := NewV6Config(VIndexCache, "../../../data/ip2region_v6.xdb", 5) if err != nil { t.Fatalf("failed to new v6 config: %s", err) } searcherPool, err := NewSearcherPool(v6Config) if err != nil { t.Fatalf("failed to create searcher pool: %s", err) } ipString := "240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6" for i := 0; i < 20; i++ { searcher := searcherPool.BorrowSearcher() region, err := searcher.SearchByStr(ipString) if err != nil { t.Fatalf("failed to search(%s): %s", ipString, err) } fmt.Printf("%2d->search(%s)=%s\n", i, ipString, region) searcherPool.ReturnSearcher(searcher) } // borrow one at last for Close timeout wait testing ONLY // searcherPool.BorrowSearcher() // close the searcher pool searcherPool.Close() } ================================================ FILE: binding/golang/xdb/header.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // Ip2Region database v2.0 searcher. // @Note this is a Not thread safe implementation. // // @Author Lion // @Date 2025/12/03 package xdb import ( "encoding/binary" "fmt" ) const ( Structure20 = 2 Structure30 = 3 HeaderInfoLength = 256 VectorIndexRows = 256 VectorIndexCols = 256 VectorIndexSize = 8 ) // --- Index policy define type IndexPolicy int const ( VectorIndexPolicy IndexPolicy = 1 BTreeIndexPolicy IndexPolicy = 2 ) func (i IndexPolicy) String() string { switch i { case VectorIndexPolicy: return "VectorIndex" case BTreeIndexPolicy: return "BtreeIndex" default: return "unknown" } } // --- Header define type Header struct { // data []byte Version uint16 IndexPolicy IndexPolicy CreatedAt uint32 StartIndexPtr uint32 EndIndexPtr uint32 // since IPv6 supporting IPVersion int RuntimePtrBytes int } func NewHeader(input []byte) (*Header, error) { if len(input) < 16 { return nil, fmt.Errorf("invalid input buffer") } return &Header{ Version: binary.LittleEndian.Uint16(input[0:]), IndexPolicy: IndexPolicy(binary.LittleEndian.Uint16(input[2:])), CreatedAt: binary.LittleEndian.Uint32(input[4:]), StartIndexPtr: binary.LittleEndian.Uint32(input[8:]), EndIndexPtr: binary.LittleEndian.Uint32(input[12:]), IPVersion: int(binary.LittleEndian.Uint16(input[16:])), RuntimePtrBytes: int(binary.LittleEndian.Uint16(input[18:])), }, nil } func (h *Header) String() string { return fmt.Sprintf( "{version:%d, index_policy:%d, created_at:%d, start_index_ptr:%d, end_index_ptr:%d, ip_version:%d, runtime_ptr_bytes:%d}", h.Version, h.IndexPolicy, h.CreatedAt, h.StartIndexPtr, h.EndIndexPtr, h.IPVersion, h.RuntimePtrBytes, ) } ================================================ FILE: binding/golang/xdb/searcher.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // Ip2Region database v2.0 searcher. // @Note this is a Not thread safe implementation. // // @Author Lion // @Date 2022/06/16 package xdb import ( "encoding/binary" "fmt" "os" ) type Searcher struct { version *Version handle *os.File ioCount int // use it only when this feature enabled. // Preload the vector index will reduce the number of IO operations // thus speedup the search process vectorIndex []byte // content buffer. // running with the whole xdb file cached contentBuff []byte } func NewWithFileOnly(version *Version, dbFile string) (*Searcher, error) { return NewSearcher(version, dbFile, nil, nil) } func NewWithVectorIndex(version *Version, dbFile string, vIndex []byte) (*Searcher, error) { return NewSearcher(version, dbFile, vIndex, nil) } func NewWithBuffer(version *Version, cBuff []byte) (*Searcher, error) { return NewSearcher(version, "", nil, cBuff) } func NewSearcher(version *Version, dbFile string, vIndex []byte, cBuff []byte) (*Searcher, error) { var err error // content buff first if cBuff != nil { return &Searcher{ version: version, vectorIndex: nil, contentBuff: cBuff, }, nil } // open the xdb binary file handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) if err != nil { return nil, err } return &Searcher{ version: version, handle: handle, vectorIndex: vIndex, }, nil } func (s *Searcher) Close() { if s.handle != nil { err := s.handle.Close() if err != nil { // do error log here ? } } } // IPVersion return the ip version func (s *Searcher) IPVersion() *Version { return s.version } // GetIOCount return the global io count for the last search func (s *Searcher) GetIOCount() int { return s.ioCount } // SearchByStr find the region for the specified ip string func (s *Searcher) SearchByStr(str string) (string, error) { ip, err := ParseIP(str) if err != nil { return "", err } return s.Search(ip) } // Search find the region for the specified long ip func (s *Searcher) Search(ip []byte) (string, error) { // ip version check if len(ip) != s.version.Bytes { return "", fmt.Errorf("invalid ip address(%s expected)", s.version.Name) } // reset the global ioCount s.ioCount = 0 // locate the segment index block based on the vector index var il0, il1 = int(ip[0]), int(ip[1]) var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize var sPtr, ePtr = uint32(0), uint32(0) if s.vectorIndex != nil { sPtr = binary.LittleEndian.Uint32(s.vectorIndex[idx:]) ePtr = binary.LittleEndian.Uint32(s.vectorIndex[idx+4:]) } else if s.contentBuff != nil { sPtr = binary.LittleEndian.Uint32(s.contentBuff[HeaderInfoLength+idx:]) ePtr = binary.LittleEndian.Uint32(s.contentBuff[HeaderInfoLength+idx+4:]) } else { // read the vector index block var buff = make([]byte, VectorIndexSize) err := s.read(int64(HeaderInfoLength+idx), buff) if err != nil { return "", fmt.Errorf("read vector index block at %d: %w", HeaderInfoLength+idx, err) } sPtr = binary.LittleEndian.Uint32(buff) ePtr = binary.LittleEndian.Uint32(buff[4:]) } // fmt.Printf("sPtr=%d, ePtr=%d\n", sPtr, ePtr) // @Note: ptr validate, zero ptr means source data missing // so we could just stop here and return an empty string. if sPtr == 0 || ePtr == 0 { return "", nil } // binary search the segment index to get the region var bytes, dBytes = len(ip), len(ip) << 1 var segIndexSize = uint32(s.version.SegmentIndexSize) var dataLen, dataPtr = 0, uint32(0) var buff = make([]byte, segIndexSize) var l, h = 0, int((ePtr - sPtr) / segIndexSize) for l <= h { m := (l + h) >> 1 p := sPtr + uint32(m)*segIndexSize err := s.read(int64(p), buff) if err != nil { return "", fmt.Errorf("read segment index at %d: %w", p, err) } // decode the data step by step to reduce the unnecessary operations if s.version.IPCompare(ip, buff[0:bytes]) < 0 { h = m - 1 } else if s.version.IPCompare(ip, buff[bytes:dBytes]) > 0 { l = m + 1 } else { dataLen = int(binary.LittleEndian.Uint16(buff[dBytes:])) dataPtr = binary.LittleEndian.Uint32(buff[dBytes+2:]) break } } // fmt.Printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr) if dataLen == 0 { return "", nil } // load and return the region data var regionBuff = make([]byte, dataLen) err := s.read(int64(dataPtr), regionBuff) if err != nil { return "", fmt.Errorf("read region at %d: %w", dataPtr, err) } return string(regionBuff), nil } // do the data read operation based on the setting. // content buffer first or will read from the file. // this operation will invoke the Seek for file based read. func (s *Searcher) read(offset int64, buff []byte) error { if s.contentBuff != nil { cLen := copy(buff, s.contentBuff[offset:]) if cLen != len(buff) { return fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } } else { _, err := s.handle.Seek(offset, 0) if err != nil { return fmt.Errorf("seek to %d: %w", offset, err) } s.ioCount++ rLen, err := s.handle.Read(buff) if err != nil { return fmt.Errorf("handle read: %w", err) } if rLen != len(buff) { return fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } } return nil } ================================================ FILE: binding/golang/xdb/util.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/16 package xdb import ( "bytes" "embed" "fmt" "io" "net" "os" ) func ParseIP(ip string) ([]byte, error) { parsedIP := net.ParseIP(ip) if parsedIP == nil { return nil, fmt.Errorf("invalid ip address: %s", ip) } v4 := parsedIP.To4() if v4 != nil { return v4, nil } v6 := parsedIP.To16() if v6 != nil { return v6, nil } return nil, fmt.Errorf("invalid ip address: %s", ip) } func IP2String(ip []byte) string { return net.IP(ip[:]).String() } // IPCompare compares two IP addresses // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 func IPCompare(ip1, ip2 []byte) int { // for i := 0; i < len(ip1); i++ { // if ip1[i] < ip2[i] { // return -1 // } // if ip1[i] > ip2[i] { // return 1 // } // } // return 0 return bytes.Compare(ip1, ip2) } func IPAddOne(ip []byte) []byte { var r = make([]byte, len(ip)) copy(r, ip) for i := len(ip) - 1; i >= 0; i-- { r[i]++ if r[i] != 0 { // No overflow break } } return r } func IPSubOne(ip []byte) []byte { var r = make([]byte, len(ip)) copy(r, ip) for i := len(ip) - 1; i >= 0; i-- { if r[i] != 0 { // No borrow needed r[i]-- break } r[i] = 0xFF // borrow from the next byte } return r } // Verify if the current Searcher could be used to search the specified xdb file. // Why do we need this check ? // The future features of the xdb impl may cause the current searcher not able to work properly. // // @Note: You Just need to check this ONCE when the service starts // Or use another process (eg, A command) to check once Just to confirm the suitability. func Verify(handle *os.File) error { header, err := LoadHeader(handle) if err != nil { return fmt.Errorf("loading header: %w", err) } // get the runtime ptr bytes runtimePtrBytes := 0 switch header.Version { case Structure20: runtimePtrBytes = 4 case Structure30: runtimePtrBytes = header.RuntimePtrBytes default: return fmt.Errorf("invalid version: %d", header.Version) } // 1, confirm the xdb file size. // to sure that the MaxFilePointer does no overflow stat, err := handle.Stat() if err != nil { return fmt.Errorf("file stat: %w", err) } maxFilePtr := int64(1<<(runtimePtrBytes*8) - 1) if stat.Size() > maxFilePtr { return fmt.Errorf("xdb file exceeds the maximum supported bytes: %d", maxFilePtr) } return nil } // VerifyFromFile check Verify for details func VerifyFromFile(dbFile string) error { handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) if err != nil { return fmt.Errorf("open xdb file `%s`: %w", dbFile, err) } defer handle.Close() return Verify(handle) } // LoadHeader load the header info from the specified handle func LoadHeader(handle *os.File) (*Header, error) { _, err := handle.Seek(0, 0) if err != nil { return nil, fmt.Errorf("seek to the header: %w", err) } var buff = make([]byte, HeaderInfoLength) rLen, err := handle.Read(buff) if err != nil { return nil, err } if rLen != len(buff) { return nil, fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } return NewHeader(buff) } // LoadHeaderFromFile load header info from the specified db file path func LoadHeaderFromFile(dbFile string) (*Header, error) { handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) if err != nil { return nil, fmt.Errorf("open xdb file `%s`: %w", dbFile, err) } defer handle.Close() header, err := LoadHeader(handle) if err != nil { return nil, err } return header, nil } // LoadHeaderFromBuff wrap the header info from the content buffer func LoadHeaderFromBuff(cBuff []byte) (*Header, error) { return NewHeader(cBuff[0:HeaderInfoLength]) } // LoadVectorIndex util function to load the vector index from the specified file handle func LoadVectorIndex(handle *os.File) ([]byte, error) { // load all the vector index block _, err := handle.Seek(HeaderInfoLength, 0) if err != nil { return nil, fmt.Errorf("seek to vector index: %w", err) } var buff = make([]byte, VectorIndexRows*VectorIndexCols*VectorIndexSize) rLen, err := handle.Read(buff) if err != nil { return nil, err } if rLen != len(buff) { return nil, fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } return buff, nil } // LoadVectorIndexFromFile load vector index from a specified file path func LoadVectorIndexFromFile(dbFile string) ([]byte, error) { handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) if err != nil { return nil, fmt.Errorf("open xdb file `%s`: %w", dbFile, err) } defer handle.Close() vIndex, err := LoadVectorIndex(handle) if err != nil { return nil, err } return vIndex, nil } // LoadContent load the whole xdb content from the specified file handle func LoadContent(handle *os.File) ([]byte, error) { // get file size fi, err := handle.Stat() if err != nil { return nil, fmt.Errorf("stat: %w", err) } size := fi.Size() // seek to the head of the file _, err = handle.Seek(0, 0) if err != nil { return nil, fmt.Errorf("seek to get xdb file length: %w", err) } var buff = make([]byte, size) rLen, err := handle.Read(buff) if err != nil { return nil, err } if rLen != len(buff) { return nil, fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } return buff, nil } // LoadContentFromFile load the whole xdb content from the specified db file path func LoadContentFromFile(dbFile string) ([]byte, error) { handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) if err != nil { return nil, fmt.Errorf("open xdb file `%s`: %w", dbFile, err) } defer handle.Close() cBuff, err := LoadContent(handle) if err != nil { return nil, err } return cBuff, nil } // LoadContentFromFS load the whole xdb binary from embed.FS func LoadContentFromFS(fs embed.FS, filePath string) ([]byte, error) { file, err := fs.Open(filePath) if err != nil { return nil, fmt.Errorf("failed to open embedded file `%s`: %w", filePath, err) } defer file.Close() var cBuff []byte cBuff, err = io.ReadAll(file) if err != nil { return nil, fmt.Errorf("failed to read embedded file `%s`: %w", filePath, err) } return cBuff, nil } ================================================ FILE: binding/golang/xdb/util_test.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/16 package xdb import ( "fmt" "testing" "time" ) func TestParseIP(t *testing.T) { var ips = []string{"29.34.191.255", "2c0f:fff0::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} for _, ip := range ips { bytes, err := ParseIP(ip) if err != nil { t.Errorf("check ip `%s`: %s\n", IP2String(bytes), err) } nip := IP2String(bytes) fmt.Printf("checkip: (%s / %s), isEqual: %v\n", ip, nip, ip == nip) } } func TestIPCompare(t *testing.T) { var ipPairs = [][]string{ {"1.2.3.4", "1.2.3.5"}, {"58.250.36.41", "58.250.30.41"}, {"2c10::", "2e00::"}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, {"fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, } for _, pairs := range ipPairs { fmt.Printf("IPCompare(%s, %s): %d\n", pairs[0], pairs[1], IPCompare([]byte(pairs[0]), []byte(pairs[1]))) } } func TestLoadVectorIndex(t *testing.T) { vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region_v4.xdb") if err != nil { fmt.Printf("failed to load vector index: %s\n", err) return } fmt.Printf("vIndex length: %d\n", len(vIndex)) } func TestLoadContent(t *testing.T) { buff, err := LoadContentFromFile("../../../data/ip2region_v4.xdb") if err != nil { fmt.Printf("failed to load xdb content: %s\n", err) return } fmt.Printf("buff length: %d\n", len(buff)) } func TestLoadHeader(t *testing.T) { header, err := LoadHeaderFromFile("../../../data/ip2region_v4.xdb") if err != nil { fmt.Printf("failed to load xdb header info: %s\n", err) return } fmt.Printf("Version : %d\n", header.Version) fmt.Printf("IndexPolicy : %s\n", header.IndexPolicy.String()) fmt.Printf("CreatedAt : %d(%s)\n", header.CreatedAt, time.Unix(int64(header.CreatedAt), 0).Format(time.RFC3339)) fmt.Printf("StartIndexPtr : %d\n", header.StartIndexPtr) fmt.Printf("EndIndexPtr : %d\n", header.EndIndexPtr) fmt.Printf("IPVersion : %d\n", header.IPVersion) fmt.Printf("RuntimePtrBytes : %d\n", header.RuntimePtrBytes) } ================================================ FILE: binding/golang/xdb/version.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package xdb import ( "bytes" "fmt" "strings" ) type Version struct { Id int Name string Bytes int SegmentIndexSize int // function to compare two ips IPCompare func([]byte, []byte) int } func (v *Version) String() string { return fmt.Sprintf( "{id:%d, name:%s, bytes:%d, segment_index_size:%d}", v.Id, v.Name, v.Bytes, v.SegmentIndexSize, ) } const ( IPv4VersionNo = 4 IPv6VersionNo = 6 ) var ( IPvx = &Version{} IPv4 = &Version{ Id: IPv4VersionNo, Name: "IPv4", Bytes: 4, SegmentIndexSize: 14, // 4 + 4 + 2 + 4, IPCompare: func(ip1, ip2 []byte) int { // ip1 - with Big endian byte order parsed from an input // ip2 - with Little endian byte order read from the xdb index ip2[0], ip2[3] = ip2[3], ip2[0] ip2[1], ip2[2] = ip2[2], ip2[1] return bytes.Compare(ip1, ip2) }, } IPv6 = &Version{ Id: IPv6VersionNo, Name: "IPv6", Bytes: 16, SegmentIndexSize: 38, // 16 + 16 + 2 + 4, IPCompare: func(ip1, ip2 []byte) int { return bytes.Compare(ip1, ip2) }, } ) func VersionFromIP(ip string) (*Version, error) { r, err := ParseIP(ip) if err != nil { return IPvx, fmt.Errorf("parse ip fail: %w", err) } if len(r) == 4 { return IPv4, nil } return IPv6, nil } func VersionFromName(name string) (*Version, error) { switch strings.ToUpper(name) { case "V4", "IPV4": return IPv4, nil case "V6", "IPV6": return IPv6, nil default: return IPvx, fmt.Errorf("invalid version name `%s`", name) } } func VersionFromHeader(header *Header) (*Version, error) { // old structure with IPv4 supports ONLY if header.Version == Structure20 { return IPv4, nil } // structure 3.0 after IPv6 supporting if header.Version != Structure30 { return IPvx, fmt.Errorf("invalid version `%d`", header.IPVersion) } switch header.IPVersion { case IPv4VersionNo: return IPv4, nil case IPv6VersionNo: return IPv6, nil default: return IPvx, fmt.Errorf("invalid version `%d`", header.Version) } } ================================================ FILE: binding/java/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region java Query Client # Usage ### Maven Repository: ```xml org.lionsoul ip2region 3.3.6 ``` ### About Query Service Starting from version `3.2.0`, a dual-protocol compatible and concurrency-safe `Ip2Region` query service is provided. **It is recommended to prioritize this method for query calls.** The specific usage is as follows: ```java import org.lionsoul.ip2region.service.Config; import org.lionsoul.ip2region.service.Ip2Region; // 1. Create v4 configuration: specify cache policy and v4 xdb file path final Config v4Config = Config.custom() .setCachePolicy(Config.VIndexCache) // Specify cache policy: NoCache / VIndexCache / BufferCache .setSearchers(15) // Set the number of initialized searchers // .setCacheSliceBytes(int) // Set cache slice bytes, default is 50MiB // .setXdbInputStream(InputStream) // Set v4 xdb file inputstream object // .setXdbFile(File) // Set v4 xdb File object .setXdbPath("ip2region v4 xdb path") // Set the path of v4 xdb file .asV4(); // Specify as v4 configuration // 2. Create v6 configuration: specify cache policy and v6 xdb file path final Config v6Config = Config.custom() .setCachePolicy(Config.VIndexCache) // Specify cache policy: NoCache / VIndexCache / BufferCache .setSearchers(15) // Set the number of initialized searchers // .setCacheSliceBytes(int) // Set cache slice bytes, default is 50MiB // .setXdbInputStream(InputStream) // Set v6 xdb file inputstream object // .setXdbFile(File) // Set v6 xdb File object .setXdbPath("ip2region v6 xdb path") // Set the path of v6 xdb file .asV6(); // Specify as v6 configuration // Note: Priority for the three types of Xdb initialization inputs: XdbInputStream -> XdbFile -> XdbPath // setXdbInputStream is only for the convenience of users to load xdb file content from jar packages, in which case cachePolicy can only be set to Config.BufferCache // 3. Create Ip2Region query service through the above configurations final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config); // 4. Export the ip2region service as a global variable to perform concurrent queries for both versions of IP addresses, for example: final String v4Region = ip2Region.search("113.92.157.29"); // Perform IPv4 query final String v6Region = ip2Region.search("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); // Perform IPv6 query // 5. When the service needs to be shut down, close the ip2region query service at the same time // Note: The close method only needs to be called when the entire service is shut down; no operation is needed during queries ip2Region.close(); ``` ##### `Ip2Region` Query Notes: 1. The API of this query service is concurrency-safe and supports both `IPv4` and `IPv6` addresses; the internal implementation will automatically distinguish them. 2. v4 and v6 configurations need to be created separately. You can set different cache policies for v4 and v6, or specify one of them as `null`, in which case IP address queries for that version will return `null`. 3. Please set a suitable number of searchers for `setSearchers` based on your project's concurrency. The default is 20. This value is fixed during runtime. Each query will borrow a searcher from the pool and return it after the query is completed. If the pool is empty when borrowing, it will wait until a searcher becomes available. The borrow lock is managed using `ReentrantLock`. You can also set the `Ip2Region` query service to use a fair lock as follows: ```java final Ip2Region ip2region = Ip2Region.create(v4Config, v6Config, true); ``` 4. If the cache policy in the configuration is set to `Config.BufferCache` (i.e., `Full Memory Cache`), a single-instance memory searcher will be used by default. This implementation is natively concurrency-safe, and the number of searchers specified via `setSearchers` will be ignored. 5. If `close` is called while the `ip2region` searcher is providing service, it will wait for a maximum of 10 seconds by default to allow as many searchers as possible to be returned. ### About Query API The prototype of the location information query API is: ```java String search(String ipStr) throw Exception; String search(byte[] ip) throw Exception; ``` An exception will be thrown if the query fails. If the query is successful, the `region` information string will be returned. If the specified IP cannot be found, an empty string `""` will be returned, which may occur for custom data or incomplete data. ### About IPv4 and IPv6 This xdb query client implementation supports both IPv4 and IPv6 queries. The usage is as follows: ```java import org.lionsoul.ip2region.xdb.Version; // For IPv4: Set xdb path to the v4 xdb file, specify IP version as Version.IPv4 final String dbPath = "../../data/ip2region_v4.xdb"; // or your ipv4 xdb path final Version version = Version.IPv4; // For IPv6: Set xdb path to the v6 xdb file, specify IP version as Version.IPv6 final String dbPath = "../../data/ip2region_v6.xdb"; // or your ipv6 xdb path final Version version = Version.IPv6; // The IP version of the xdb specified by dbPath must be consistent with version, otherwise an error will occur during query execution // Note: The following demonstration directly uses the dbPath and version variables ``` ### File Verification It is recommended that you proactively verify the applicability of the xdb file, as some future new features may cause the current Searcher version to be incompatible with the xdb file you are using. Verification can avoid unpredictable errors during runtime. You do not need to verify every time; for example, verify when the service starts or manually call a command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios. ```java try { Searcher.verifyFromFile(dbPath); } catch (Exception e) { // Applicability verification failed!!! // The current query client implementation is not suitable for querying the xdb file specified by dbPath. // You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with dbPath. return; } // Verification passed, the current Searcher can be safely used for query operations on the xdb pointed to by dbPath ``` ### File-Based Query ```java import org.lionsoul.ip2region.xdb.Searcher; import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { // 1. Create a searcher object using the version and dbPath mentioned above Searcher searcher = null; try { searcher = Searcher.newWithFileOnly(version, dbPath); } catch (IOException e) { System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e); return; } // 2. Query, both IPv4 and IPv6 addresses are supported try { String ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long sTime = System.nanoTime(); String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } // 3. Close resources searcher.close(); // Note: For concurrent use, each thread needs to create an independent searcher object for separate use. } } ``` ### Caching `VectorIndex` We can pre-load `VectorIndex` data from the `xdb` file and cache it globally. Using the global VectorIndex cache every time a Searcher object is created can reduce a fixed IO operation, thereby accelerating queries and reducing IO pressure. ```java import org.lionsoul.ip2region.xdb.Searcher; import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { // Note: For version and dbPath sources, please see the version description above // 1. Pre-load VectorIndex cache from dbPath and use the obtained data as a global variable for subsequent repeated use. byte[] vIndex; try { vIndex = Searcher.loadVectorIndexFromFile(dbPath); } catch (Exception e) { System.out.printf("failed to load vector index from `%s`: %s\n", dbPath, e); return; } // 2. Use the global vIndex to create a query object with VectorIndex cache. Searcher searcher; try { searcher = Searcher.newWithVectorIndex(version, dbPath, vIndex); } catch (Exception e) { System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e); return; } // 3. Query, both IPv4 and IPv6 addresses are supported try { String ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long sTime = System.nanoTime(); String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } // 4. Close resources searcher.close(); // Note: Each thread needs to create an independent Searcher object, but they all share the same read-only global vIndex cache. } } ``` ### Caching the Entire `xdb` File We can also pre-load the entire xdb file data into memory and then create a query object based on this data to achieve a fully memory-based query, similar to the previous memory search. ```java import org.lionsoul.ip2region.xdb.Searcher; import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { // Note: For version and dbPath sources, please see the version description above // 1. Load the entire xdb from dbPath into memory. // Starting from this release version, the xdb buffer uses LongByteArray for storage to avoid int type overflow when the xdb file is too large LongByteArray cBuff; try { cBuff = Searcher.loadContentFromFile(dbPath); } catch (Exception e) { System.out.printf("failed to load content from `%s`: %s\n", dbPath, e); return; } // 2. Use the above cBuff to create a fully memory-based query object. Searcher searcher; try { searcher = Searcher.newWithBuffer(version, cBuff); } catch (Exception e) { System.out.printf("failed to create content cached searcher: %s\n", e); return; } // 3. Query, both IPv4 and IPv6 are supported try { String ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long sTime = System.nanoTime(); String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } // 4. Close resources - this searcher object can be safely used for concurrency; close it when the entire service is shut down // searcher.close(); // Note: For concurrent use, the query object created with the entire xdb data cache can be safely used for concurrency, // meaning you can make this searcher object a global object for cross-thread access. } } ``` If an OOM error occurs while calling the `loadContentXXX` method to load the xdb buffer, please refer to the [sliceBytes setting](#slicebytes) below and choose the `loadContentXXX` method with the sliceBytes parameter. ### sliceBytes sliceBytes represents the size of the partitioned memory for the `List buffs` collection maintained inside the `LongByteArray` class during full memory caching. The default value is `Searcher.DEFAULT_SLICE_BYTES` = `50MiB`. The maximum allowed value is `Searcher.MAX_WRITE_BYTES` = `0x7ffff000`. For the source of this value, please refer to the author's blog post: [https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA). 1. Starting from version `3.3.3`, `LongByteArray` implements fixed partition size support, which allows for fast `offset` positioning through simple calculation to perform `slice` or `copy` operations. 2. In terms of calculation speed, the larger the sliceBytes, the smaller the length of buffs and the lower the calculation time. However, with the fixed sliceBytes implementation, this gap is completely negligible. Therefore, it is recommended to keep the default value of `50MiB`, which also avoids the OOM issues that could be caused by elastic partition sizes previously. # Compiling the Test Program Compile the test program via Maven. ```bash # cd to the root directory of java binding cd binding/java/ mvn compile package ``` Then a packaged file named ip2region-{version}.jar will be generated in the target directory under the current folder. # Query Testing ### Test Command You can test queries via the `java -jar target/ip2region-{version}.jar search` command: ```bash ➜ java git:(master) ✗ java -jar target/ip2region-3.3.4.jar search --help java -jar ip2region-{version}.jar search [command options] options: --v4-db string ip2region ipv4 binary xdb file path --v4-cache-policy string v4 cache policy, default vectorIndex, options: file/vectorIndex/content --v6-db string ip2region ipv6 binary xdb file path --v6-cache-policy string v6 cache policy, default vectorIndex, options: file/vectorIndex/content --help print this help menu ``` ### Parameter Parsing 1. `v4-xdb`: IPv4 xdb file path, defaults to data/ip2region_v4.xdb in the repository. 2. `v6-xdb`: IPv6 xdb file path, defaults to data/ip2region_v6.xdb in the repository. 3. `v4-cache-policy`: Cache policy used for v4 queries, default is `vectorIndex`, options: file/vectorIndex/content. 4. `v6-cache-policy`: Cache policy used for v6 queries, default is `vectorIndex`, options: file/vectorIndex/content. ### Test Demo Example: performing query testing using default data/ip2region_v4.xdb and data/ip2region_v6.xdb: ```bash ➜ java git:(java_app_with_ip2region_service) ✗ java -jar target/ip2region-3.3.4.jar search ip2region search service test program +-v4 xdb: /data01/code/c/ip2region/data/ip2region_v4.xdb (vectorIndex) +-v6 xdb: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, took: 140 μs} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, took: 391 μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, took: 503 μs} ``` Enter a v4 or v6 IP address to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the effects of the three different cache implementations. # bench Testing ### Test Command You can perform bench testing via the `java -jar ip2region-{version}.jar bench` command to ensure the `xdb` file is error-free and to evaluate query performance: ```bash ➜ java git:(fr_java_ipv6) ✗ java -jar target/ip2region-3.3.4.jar bench java -jar ip2region-{version}.jar bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` ### v4 bench Example: IPv4 bench testing using default data/ip2region_v4.xdb and data/ipv4_source.txt files: ```bash java -jar target/ip2region-3.3.4.jar bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` ### v6 bench Example: IPv6 bench testing using default data/ip2region_v6.xdb and data/ipv6_source.txt files: ```bash java -jar target/ip2region-3.3.4.jar bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` You can test the effects of the three different cache implementations by setting `cache-policy` to file/vectorIndex/content. @Note: Please ensure that the src file used for benching is the same source file used to generate the corresponding xdb file. ================================================ FILE: binding/java/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region java 查询客户端 # 使用方式 ### maven 仓库： ```xml org.lionsoul ip2region 3.3.6 ``` ### 关于查询服务 从 `3.2.0` 版本开始提供了一个双协议兼容且并发安全的 `Ip2Region` 查询服务，**建议优先使用该方式来进行查询调用**，具体使用方式如下： ```java import org.lionsoul.ip2region.service.Config; import org.lionsoul.ip2region.service.Ip2Region; // 1, 创建 v4 的配置：指定缓存策略和 v4 的 xdb 文件路径 final Config v4Config = Config.custom() .setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache .setSearchers(15) // 设置初始化的查询器数量 // .setCacheSliceBytes(int) // 设置缓存的分片字节数，默认为 50MiB // .setXdbInputStream(InputStream) // 设置 v4 xdb 文件的 inputstream 对象 // .setXdbFile(File) // 设置 v4 xdb File 对象 .setXdbPath("ip2region v4 xdb path") // 设置 v4 xdb 文件的路径 .asV4(); // 指定为 v4 配置 // 2, 创建 v6 的配置：指定缓存策略和 v6 的 xdb 文件路径 final Config v6Config = Config.custom() .setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache .setSearchers(15) // 设置初始化的查询器数量 // .setCacheSliceBytes(int) // 设置缓存的分片字节数，默认为 50MiB // .setXdbInputStream(InputStream) // 设置 v6 xdb 文件的 inputstream 对象 // .setXdbFile(File) // 设置 v6 xdb File 对象 .setXdbPath("ip2region v6 xdb path") // 设置 v6 xdb 文件的路径 .asV6(); // 指定为 v6 配置 // 备注：Xdb 三种初始化输入的优先级：XdbInputStream -> XdbFile -> XdbPath // setXdbInputStream 仅方便使用者从 jar 包中加载 xdb 文件内容，这时 cachePolicy 只能设置为 Config.BufferCache // 3，通过上述配置创建 Ip2Region 查询服务 final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config); // 4，导出 ip2region 服务作为全局变量，进行双版本的IP地址的并发查询，例如： final String v4Region = ip2Region.search("113.92.157.29"); // 进行 IPv4 查询 final String v6Region = ip2Region.search("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); // 进行 IPv6 查询 // 5，在服务需要关闭的时候，同时关闭 ip2region 查询服务 // 备注：close 方法只需要在整个服务关闭的时候关闭，查询途中不需要操作 ip2Region.close(); ``` ##### `Ip2Region` 查询备注： 1. 该查询服务的 API 并发安全且同时支持 `IPv4` 和 `IPv6` 的地址，内部实现会自动判断。 2. v4 和 v6 的配置需要单独创建，可以给 v4 和 v6 设置使用不同的缓存策略，也可以指定其中一个为 `null` 则该版本的 IP 地址查询都会返回 `null`。 3. 请结合您项目的并发数给 `setSearchers` 一个合适的查询器数量，默认为 20 个，这个值在运行过程中是固定的，每次查询会从池子里租借一个查询器来完成查询操作，查询完成后再归还回去，如果租借的时候池子已经空了则等待直到有可用的查询器来完成查询服务，租借的锁是使用的 `ReentrantLock` 来管理，也可以通过如下方式来设置 `Ip2Region` 查询服务使用公平锁： ```java final Ip2Region ip2region = Ip2Region.create(v4Config, v6Config, true); ``` 4. 如果配置设置的缓存策略为 `Config.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器，该实现天生并发安全，此时通过 `setSearchers` 指定的查询器数量无效。 5. 如果 `ip2region` 查询器在提供服务期间，调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。 ### 关于查询 API 定位信息查询 API 的原型为： ```java String search(String ipStr) throw Exception; String search(byte[] ip) throw Exception; ``` 查询出错会抛出异常，如果查询成功会返回字符串的 `region` 信息，如果指定的 ip 查询不到会返回空字符串 `""`，这对于自定义数据或者数据不完整的情况会出现。 ### 关于 IPv4 和 IPv6 该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询，使用方式如下： ```java import org.lionsoul.ip2region.xdb.Version; // 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 Version.IPv4 final String dbPath = "../../data/ip2region_v4.xdb"; // 或者你的 ipv4 xdb 的路径 final Version version = Version.IPv4; // 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件，IP版本指定为 Version.IPv6 final String dbPath = "../../data/ip2region_v6.xdb"; // 或者你的 ipv6 xdb 路径 final Version version = Version.IPv6; // dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致，不然查询执行的时候会报错 // 备注：以下演示直接使用 dbPath 和 version 变量 ``` ### 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 ```java try { Searcher.verifyFromFile(dbPath); } catch (Exception e) { // 适用性验证失败！！！ // 当前查询客户端实现不适用于 dbPath 指定的 xdb 文件的查询. // 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 dbPath 的 Searcher 实现。 return; } // 验证通过，当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作 ``` ### 完全基于文件的查询 ```java import org.lionsoul.ip2region.xdb.Searcher; import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { // 1、使用上述的 version 和 dbPath 创建 searcher 对象 Searcher searcher = null; try { searcher = Searcher.newWithFileOnly(version, dbPath); } catch (IOException e) { System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e); return; } // 2、查询，IPv4 或者 IPv6 的地址都支持 try { String ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long sTime = System.nanoTime(); String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } // 3、关闭资源 searcher.close(); // 备注：并发使用，每个线程需要创建一个独立的 searcher 对象单独使用。 } } ``` ### 缓存 `VectorIndex` 索引 我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。 ```java import org.lionsoul.ip2region.xdb.Searcher; import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { // 备注：version 和 dbPath 来源，请看上面的版本描述 // 1、从 dbPath 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。 byte[] vIndex; try { vIndex = Searcher.loadVectorIndexFromFile(dbPath); } catch (Exception e) { System.out.printf("failed to load vector index from `%s`: %s\n", dbPath, e); return; } // 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。 Searcher searcher; try { searcher = Searcher.newWithVectorIndex(version, dbPath, vIndex); } catch (Exception e) { System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e); return; } // 3、查询，IPv4 或者 IPv6 地址都支持 try { String ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long sTime = System.nanoTime(); String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } // 4、关闭资源 searcher.close(); // 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的只读 vIndex 缓存。 } } ``` ### 缓存整个 `xdb` 文件 我们也可以预先加载整个 xdb 文件的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。 ```java import org.lionsoul.ip2region.xdb.Searcher; import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { // 备注：version 和 dbPath 来源，请看上面的版本描述 // 1、从 dbPath 加载整个 xdb 到内存。 // 从这个 release 版本开始，xdb 的 buffer 使用 LongByteArray 来存储，避免 xdb 文件过大的时候 int 类型的溢出 LongByteArray cBuff; try { cBuff = Searcher.loadContentFromFile(dbPath); } catch (Exception e) { System.out.printf("failed to load content from `%s`: %s\n", dbPath, e); return; } // 2、使用上述的 cBuff 创建一个完全基于内存的查询对象。 Searcher searcher; try { searcher = Searcher.newWithBuffer(version, cBuff); } catch (Exception e) { System.out.printf("failed to create content cached searcher: %s\n", e); return; } // 3、查询，IPv4 和 IPv6 都支持 try { String ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 long sTime = System.nanoTime(); String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } // 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher // searcher.close(); // 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。 } } ``` 如果调用 `loadContentXXX` 方法来加载 xdb buffer 的过程中出现了 OOM 错误，请参考以下的 [sliceBytes 设置](#slicebytes)，选择使用带 sliceBytes 参数的 `loadContentXXX` 方法来加载 。 ### sliceBytes sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List buffs` 集合的分片内存的大小，默认值为 `Searcher.DEFAULT_SLICE_BYTES` = `50MiB`，这个值的最大允许值为 `Searcher.MAX_WRITE_BYTES` = `0x7ffff000`，关于该取值的来源可以参考作者博客文章：[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。 1. 从 `3.3.3` 版本开始 `LongByteArray` 实现了固定分片尺寸支持，可以通过简单的计算快速的完成 `offset` 定位的从而实现 `slice` 或者 `copy` 操作。 2. 从计算速度来说 sliceBytes 越大 buffs 的长度越小，计算耗时越小，不过有了固定 sliceBytes 实现这个差距完全可以忽略，所以建议保持默认值为 `50MiB` 即可，也不会出现之前弹性分片尺寸可能导致的 OOM 问题。 # 编译测试程序 通过 maven 来编译测试程序。 ```bash # cd 到 java binding 的根目录 cd binding/java/ mvn compile package ``` 然后会在当前目录的 target 目录下得到一个 ip2region-{version}.jar 的打包文件。 # 查询测试 ### 测试命令 可以通过 `java -jar target/ip2region-{version}.jar search` 命令来测试查询： ```bash ➜ java git:(master) ✗ java -jar target/ip2region-3.3.0.jar search --help java -jar ip2region-{version}.jar search [command options] options: --v4-db string ip2region ipv4 binary xdb file path --v4-cache-policy string v4 cache policy, default vectorIndex, options: file/vectorIndex/content --v6-db string ip2region ipv6 binary xdb file path --v6-cache-policy string v6 cache policy, default vectorIndex, options: file/vectorIndex/content --help print this help menu ``` ### 参数解析 1. `v4-xdb`: IPv4 的 xdb 文件路径，默认为仓库中的 data/ip2region_v4.xdb 2. `v6-xdb`: IPv6 的 xdb 文件路径，默认为仓库中的 data/ip2region_v6.xdb 3. `v4-cache-policy`: v4 查询使用的缓存策略，默认为 `vectorIndex`，可选：file/vectorIndex/content 4. `v6-cache-policy`: v6 查询使用的缓存策略，默认为 `vectorIndex`，可选：file/vectorIndex/content ### 测试 Demo 例如：使用默认的 data/ip2region_v4.xdb 和 data/ip2region_v6.xdb 进行查询测试： ```bash ➜ java git:(java_app_with_ip2region_service) ✗ java -jar target/ip2region-3.3.0.jar search ip2region search service test program +-v4 xdb: /data01/code/c/ip2region/data/ip2region_v4.xdb (vectorIndex) +-v6 xdb: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, took: 140 μs} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, took: 391 μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, took: 503 μs} ``` 输入 v4 或者 v6 的 IP 地址即可进行查询测试，也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。 # bench 测试 ### 测试命令 可以通过 `java -jar ip2region-{version}.jar bench` 命令来进行 bench 测试，一方面确保 `xdb` 文件没有错误，一方面可以评估查询性能： ```bash ➜ java git:(fr_java_ipv6) ✗ java -jar target/ip2region-3.1.0.jar bench java -jar ip2region-{version}.jar bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` ### v4 bench 例如：通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 文件进行 IPv4 的 bench 测试： ```bash java -jar target/ip2region-3.1.0.jar bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` ### v6 bench 例如：通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 文件进行 IPv6 的 bench 测试： ```bash java -jar target/ip2region-3.1.0.jar bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` 可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。 @Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。 ================================================ FILE: binding/java/pom.xml ================================================ 4.0.0 org.lionsoul ip2region 3.3.6 jar ip2region https://github.com/lionsoul2014/ip2region An open source offline IP address data manager framework and locator with both IPv4 and IPv6 suppported The Apache Software License, Version 2.0 https://www.apache.org/licenses/LICENSE-2.0.txt repo git@github.com:lionsoul2014/ip2region.git scm:git:git@github.com:lionsoul2014/ip2region.git scm:git:git@github.com:lionsoul2014/ip2region.git lionsoul chenxin chenxin619315@gmail.com lionsoul https://oss.sonatype.org/content/repositories/snapshots/ lionsoul https://oss.sonatype.org/service/local/staging/deploy/maven2/ https://github.com/lionsoul2014/ip2region/issues Github issues UTF-8 UTF-8 1.8 1.8 junit junit 4.13.1 test org.apache.maven.plugins maven-source-plugin 2.1.2 attach-sources package jar org.apache.maven.plugins maven-javadoc-plugin 2.9 attach-javadocs package jar ${javadoc.opts} false org.apache.maven.plugins maven-shade-plugin 1.4 package shade org.lionsoul.ip2region.SearcherTest java8-doclint-disabled [1.8,) -Xdoclint:none release org.apache.maven.plugins maven-source-plugin 2.2.1 package jar-no-fork org.apache.maven.plugins maven-javadoc-plugin 2.9.1 package jar ${javadoc.opts} org.apache.maven.plugins maven-gpg-plugin 1.5 verify sign org.sonatype.central central-publishing-maven-plugin 0.7.0 true lionsoul true ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/SearcherTest.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Lion // @Date 2022/06/23 package org.lionsoul.ip2region; import org.lionsoul.ip2region.service.Config; import org.lionsoul.ip2region.service.InvalidConfigException; import org.lionsoul.ip2region.service.Ip2Region; import org.lionsoul.ip2region.xdb.InetAddressException; import org.lionsoul.ip2region.xdb.XdbException; import org.lionsoul.ip2region.xdb.LongByteArray; import org.lionsoul.ip2region.xdb.Searcher; import org.lionsoul.ip2region.xdb.Util; import org.lionsoul.ip2region.xdb.Version; import java.io.*; import java.nio.charset.Charset; import java.nio.file.Path; import java.nio.file.Paths; import java.security.CodeSource; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void printHelp(String[] args) { System.out.print("ip2region xdb searcher\n"); System.out.print("java -jar ip2region-{version}.jar [command] [command options]\n"); System.out.print("Command: \n"); System.out.print(" search search input test\n"); System.out.print(" bench search bench test\n"); } public static final String getXdbPath(String fileName) throws IOException { String xdbPath; final CodeSource cs = SearcherTest.class.getProtectionDomain().getCodeSource(); if (cs != null) { // log.debugf("code path: %s", cs.getLocation().getPath().concat("../../../../data/")); final Path jarPath = Paths.get(cs.getLocation().getPath()); xdbPath = jarPath.getParent().toString().concat("/../../../data/").concat(fileName); } else { xdbPath = "../../../data/".concat(fileName); } final File xdbFile = new File(xdbPath); return xdbFile.exists() ? xdbFile.getCanonicalPath() : ""; } public static final Ip2Region createService( String v4XdbPath, String v4CachePolicy, String v6XdbPath, String v6CachePolicy) throws IOException, XdbException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.cachePolicyFromName(v4CachePolicy)) .setSearchers(1) .setXdbPath(v4XdbPath).asV4(); final Config v6Config = Config.custom() .setCachePolicy(Config.cachePolicyFromName(v6CachePolicy)) .setSearchers(1) .setXdbPath(v6XdbPath).asV6(); return Ip2Region.create(v4Config, v6Config); } public static Searcher createSearcher(String dbPath, String cachePolicy) throws IOException, XdbException { final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); // verify the xdb file // @Note: do NOT call it every time you create a searcher since this will slow // down the search response. // @see the util.Verify function for details. Searcher.verify(handle); // get the ip version from header final Version version = Version.fromHeader(Searcher.loadHeader(handle)); // create the final searcher if ("file".equals(cachePolicy)) { return Searcher.newWithFileOnly(version, dbPath); } else if ("vectorIndex".equals(cachePolicy)) { final byte[] vIndex = Searcher.loadVectorIndexFromFile(dbPath); return Searcher.newWithVectorIndex(version, dbPath, vIndex); } else if ("content".equals(cachePolicy)) { final LongByteArray cBuff = Searcher.loadContentFromFile(dbPath); return Searcher.newWithBuffer(version, cBuff); } else { throw new IOException("invalid cache policy `" + cachePolicy + "`, options: file/vectorIndex/content"); } } public static void searchTest(String[] args) throws IOException, XdbException, InterruptedException, InvalidConfigException { String help = ""; String v4DbPath = "", v4CachePolicy = "vectorIndex"; String v6DbPath = "", v6CachePolicy = "vectorIndex"; for (final String r : args) { if (r.length() < 5) { continue; } if (r.indexOf("--") != 0) { continue; } String key = "", val = ""; int sIdx = r.indexOf('='); if (sIdx < 0) { key = r.substring(2); // System.out.printf("missing = for args pair `%s`\n", r); // return; } else { key = r.substring(2, sIdx); val = r.substring(sIdx + 1); } // System.out.printf("key=%s, val=%s\n", key, val); if ("help".equals(key)) { help = val == "" ? "true" : val; } else if ("v4-db".equals(key)) { v4DbPath = val; } else if ("v4-cache-policy".equals(key)) { v4CachePolicy = val; } else if ("v6-db".equals(key)) { v6DbPath = val; } else if ("v6-cache-policy".equals(key)) { v6CachePolicy = val; } else { System.out.printf("undefined option `%s`\n", r); return; } } // check and set the default path for v4 if (v4DbPath.isEmpty()) { v4DbPath = getXdbPath("ip2region_v4.xdb"); } // check and set the default path for 6 if (v6DbPath.isEmpty()) { v6DbPath = getXdbPath("ip2region_v6.xdb"); } if (v4DbPath.isEmpty() || v6DbPath.isEmpty() || help.equals("true")) { System.out.print("java -jar ip2region-{version}.jar search [command options]\n"); System.out.print("options:\n"); System.out.print(" --v4-db string ip2region ipv4 binary xdb file path\n"); System.out.print(" --v4-cache-policy string v4 cache policy, default vectorIndex, options: file/vectorIndex/content\n"); System.out.print(" --v6-db string ip2region ipv6 binary xdb file path\n"); System.out.print(" --v6-cache-policy string v6 cache policy, default vectorIndex, options: file/vectorIndex/content\n"); System.out.print(" --help print this help menu\n"); return; } final Ip2Region ip2region = createService(v4DbPath, v4CachePolicy, v6DbPath, v6CachePolicy); final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); System.out.printf("ip2region search service test program\n" + "+-v4 xdb: %s (%s)\n" + "+-v6 xdb: %s (%s)\n" + "type 'quit' to exit\n", v4DbPath, v4CachePolicy, v6DbPath, v6CachePolicy); while ( true ) { System.out.print("ip2region>> "); String line = reader.readLine().trim(); if ( line.length() < 2 ) { continue; } if ( line.equalsIgnoreCase("quit") ) { break; } try { double sTime = System.nanoTime(); String region = ip2region.search(line); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, took: %d μs}\n", region, cost); } catch (Exception e) { System.out.printf("{region: , err: %s}\n", e); } } reader.close(); ip2region.close(); System.out.println("ip2region test program exited, thanks for trying"); } public static void benchTest(String[] args) throws IOException, XdbException, InetAddressException { String dbPath = "", srcPath = "", cachePolicy = "vectorIndex"; for (final String r : args) { if (r.length() < 5) { continue; } if (r.indexOf("--") != 0) { continue; } int sIdx = r.indexOf('='); if (sIdx < 0) { System.out.printf("missing = for args pair `%s`\n", r); return; } String key = r.substring(2, sIdx); String val = r.substring(sIdx + 1); if ("db".equals(key)) { dbPath = val; } else if ("src".equals(key)) { srcPath = val; } else if ("cache-policy".equals(key)) { cachePolicy = val; } else { System.out.printf("undefined option `%s`\n", r); return; } } if (dbPath.length() < 1 || srcPath.length() < 1) { System.out.print("java -jar ip2region-{version}.jar bench [command options]\n"); System.out.print("options:\n"); System.out.print(" --db string ip2region binary xdb file path\n"); System.out.print(" --src string source ip text file path\n"); System.out.print(" --cache-policy string cache policy: file/vectorIndex/content\n"); return; } Searcher searcher = createSearcher(dbPath, cachePolicy); long count = 0, costs = 0, tStart = System.nanoTime(); String line; final Charset charset = Charset.forName("utf-8"); final FileInputStream fis = new FileInputStream(srcPath); final BufferedReader reader = new BufferedReader(new InputStreamReader(fis, charset)); while ((line = reader.readLine()) != null) { String l = line.trim(); String[] ps = l.split("\\|", 3); if (ps.length != 3) { reader.close(); System.out.printf("invalid ip segment `%s`\n", l); return; } // mark the start time long sTime = System.nanoTime(); byte[] sip; try { sip = Util.parseIP(ps[0]); } catch (Exception e) { reader.close(); System.out.printf("check start ip `%s`: %s\n", ps[0], e); return; } byte[] eip; try { eip = Util.parseIP(ps[1]); } catch (Exception e) { reader.close(); System.out.printf("check end ip `%s`: %s\n", ps[1], e); return; } if (Util.ipCompare(sip, eip) > 0) { reader.close(); System.out.printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]); return; } for (final byte[] ip : new byte[][]{sip, eip}) { String region = searcher.search(ip); // check the region info if (!ps[2].equals(region)) { System.out.printf("failed search(%s) with (%s != %s)\n", Util.ipToString(ip), region, ps[2]); reader.close(); return; } count++; } costs += System.nanoTime() - sTime; } reader.close(); searcher.close(); long took = System.nanoTime() - tStart; System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n", cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took), count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs/count)); } public static void main(String[] args) { if (args.length < 1) { printHelp(args); return; } if ("search".equals(args[0])) { try { searchTest(args); } catch (Exception e) { System.out.printf("failed running search test: %s\n", e); } } else if ("bench".equals(args[0])) { try { benchTest(args); } catch (Exception e) { System.out.printf("fwailed running bench test: %s\n", e); } } else { printHelp(args); } } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/service/Config.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.service; import java.io.File; import java.io.IOException; import org.lionsoul.ip2region.xdb.Header; import org.lionsoul.ip2region.xdb.LongByteArray; import org.lionsoul.ip2region.xdb.Version; import org.lionsoul.ip2region.xdb.XdbException; /** * ip2region config class * @Author Lion * @Date 2025/11/20 */ public class Config { // cache policy consts public static final int NoCache = 0; public static final int VIndexCache = 1; public static final int BufferCache = 2; // alias of BufferCache but easier to understand or Remember public static final int FullCache = 2; // search cache policy public final int cachePolicy; public final Version ipVersion; // xdb file path public final File xdbFile; public final Header header; public final byte[] vIndex; public final LongByteArray cBuffer; public final int searchers; // config builder public static ConfigBuilder custom() { return new ConfigBuilder(); } protected Config(int cachePolicy, Version ipVersion, File xdbFile, Header header, byte[] vIndex, LongByteArray cBuffer, int searchers) throws IOException, XdbException { this.cachePolicy = cachePolicy; this.ipVersion = ipVersion; this.xdbFile = xdbFile; this.header = header; this.vIndex = vIndex; this.cBuffer = cBuffer; final Version xVersion = Version.fromHeader(header); // verify the ip version (ipVersion and the version of the xdb file should be the same) if (header.ipVersion != ipVersion.id) { throw new XdbException("ip verison not match: xdb file " + xdbFile.getAbsolutePath() + " (" + xVersion.name + "), as " + ipVersion.name + " expected"); } this.searchers = searchers; } @Override public String toString() { final StringBuffer sb = new StringBuffer(); sb.append('{'); sb.append("cache_policy:").append(cachePolicy).append(','); sb.append("version:").append(ipVersion.toString()).append(','); sb.append("xdb_path:").append(xdbFile == null ? "null" : xdbFile.getAbsolutePath()).append(','); sb.append("header:").append(header.toString()).append(','); if (vIndex == null) { sb.append("v_index: null, "); } else { sb.append("v_index: {bytes: ").append(vIndex.length).append("},"); } if (cBuffer == null) { sb.append("c_buffer: null, "); } else { sb.append("c_buffer: {bytes: ").append(cBuffer.length()).append("},"); } sb.append("searchers:").append(searchers); sb.append('}'); return sb.toString(); } public static final int cachePolicyFromName(String name) throws InvalidConfigException { final String lName = name.toLowerCase(); if (lName.equals("file") || lName.equals("nocache")) { return NoCache; } else if (lName.equals("vectorindex") || lName.equals("vindex") || lName.equals("vindexcache")) { return VIndexCache; } else if (lName.equals("content") || lName.equals("buffercache")) { return BufferCache; } else { throw new InvalidConfigException("invalid cache policy `" + name + "`"); } } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/service/ConfigBuilder.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.service; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; import org.lionsoul.ip2region.xdb.Header; import org.lionsoul.ip2region.xdb.LongByteArray; import org.lionsoul.ip2region.xdb.Searcher; import org.lionsoul.ip2region.xdb.Version; import org.lionsoul.ip2region.xdb.XdbException; /** * ip2region config builder * @Author Lion * @Date 2025/11/20 */ public class ConfigBuilder { // cache policy private int cachePolicy = Config.VIndexCache; // xdb file path / Object / InputStream. // Priority: InputStream -> File Object -> String path private String xdbPath = null; private File xdbFile = null; private InputStream xdbInputStream = null; // slice bytes for in-memory xdb content private int cacheSliceBytes = Searcher.DEFAULT_SLICE_BYTES; // searchers private int searchers = 20; public ConfigBuilder() {} public ConfigBuilder(String xdbPath) { this.xdbPath = xdbPath; } public ConfigBuilder setCachePolicy(int cachePolicy) { this.cachePolicy = cachePolicy; return this; } public ConfigBuilder setXdbPath(String xdbPath) { assert xdbPath != null && xdbPath.length() > 0; this.xdbPath = xdbPath; return this; } public ConfigBuilder setXdbFile(File xdbFile) { assert xdbFile != null; this.xdbFile = xdbFile; return this; } public ConfigBuilder setXdbInputStream(InputStream xdbInputStream) { assert xdbInputStream != null; this.xdbInputStream = xdbInputStream; return this; } public ConfigBuilder setCacheSliceBytes(int cacheSliceBytes) { this.cacheSliceBytes = cacheSliceBytes; return this; } public ConfigBuilder setSearchers(int searchers) { this.searchers = searchers; return this; } private Config build(Version ipVersion) throws IOException, XdbException, InvalidConfigException { if (xdbInputStream == null) { // everyting is fine } else if (cachePolicy != Config.BufferCache) { // @Note: we can't directly rewrite the cachePolicy to Config.BufferCache. // you must know what you are doing. throw new InvalidConfigException("SetXdbInputStream could ONLY be used with cachePolicy = Config.BufferCache"); } else { // 1, load the content buffer final LongByteArray cBuffer = Searcher.loadContentFromInputStream(xdbInputStream, cacheSliceBytes); // 2, verify the xdb from the buffer Searcher.verify(Searcher.loadHeaderFromBuffer(cBuffer), cBuffer.length()); // 3, load the header final Header header = Searcher.loadHeaderFromBuffer(cBuffer); // create the config without xdbFile and vIndex return new Config(cachePolicy, ipVersion, null, header, null, cBuffer, searchers); } // load the header and the cache buffer final File xdbFile; if (this.xdbFile != null) { xdbFile = this.xdbFile; } else if (this.xdbPath != null) { xdbFile = new File(this.xdbPath); } else { throw new InvalidConfigException("Both xdbFile and xdbPath is null"); } final RandomAccessFile raf = new RandomAccessFile(xdbFile, "r"); // 1, verify the xdb Searcher.verify(raf); // 2, load the header final Header header = Searcher.loadHeader(raf); // 3, check and load the vector index buffer final byte[] vIndex = cachePolicy == Config.VIndexCache ? Searcher.loadVectorIndex(raf) : null; // 4, check and load the content buffer final LongByteArray cBuffer = cachePolicy == Config.BufferCache ? Searcher.loadContent(raf, cacheSliceBytes) : null; raf.close(); return new Config(cachePolicy, ipVersion, xdbFile, header, vIndex, cBuffer, searchers); } // build the final #Config instance for IPv4 public Config asV4() throws IOException, XdbException, InvalidConfigException { return build(Version.IPv4); } // build the final #Config instance for IPv6 public Config asV6() throws IOException, XdbException, InvalidConfigException { return build(Version.IPv6); } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/service/InvalidConfigException.java ================================================ package org.lionsoul.ip2region.service; public class InvalidConfigException extends Exception { public InvalidConfigException(String str) { super(str); } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/service/Ip2Region.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.service; import java.io.File; import java.io.IOException; import org.lionsoul.ip2region.xdb.InetAddressException; import org.lionsoul.ip2region.xdb.Searcher; import org.lionsoul.ip2region.xdb.Util; import org.lionsoul.ip2region.xdb.XdbException; /** * ip2region searcher manager service to provider: * 1. Unified query interface for IPv4 and IPv6 address. * 2. Concurrency search support. * * @Author Lion * Date 2025/11/21 */ public class Ip2Region { /* v4 pool for cache policy vIndex or NoCache */ private final SearcherPool v4Pool; /* v4 xdb searcher for cache policy cBuffer */ private final Searcher v4InMemSearcher; /* v6 pool for cache policy vIndex or NoCache*/ private final SearcherPool v6Pool; /* v6 xdb searcher for cache policy cBuffer */ private final Searcher v6InMemSearcher; public static final Ip2Region create(final Config v4Config, final Config v6Config) throws IOException { return new Ip2Region(v4Config, v6Config).init(); } public static final Ip2Region create(final String v4XdbPath, final String v6XdbPath) throws IOException, XdbException, InvalidConfigException { return new Ip2Region(new File(v4XdbPath), new File(v6XdbPath)).init(); } public static final Ip2Region create(final File v4XdbFile, final File v6XdbFile) throws IOException, XdbException, InvalidConfigException { return new Ip2Region(v4XdbFile, v6XdbFile).init(); } /** * init the ip2reigon with two xdb file path and default cachePolicy vIndex. * set it to null to disabled the search for specified version * * @param v4XdbFile * @param v6XdbFile * @throws XdbException * @throws IOException * @throws InvalidConfigException */ protected Ip2Region(File v4XdbFile, File v6XdbFile) throws IOException, XdbException, InvalidConfigException { this( v4XdbFile == null ? null : Config.custom().setXdbFile(v4XdbFile).asV4(), v6XdbFile == null ? null : Config.custom().setXdbFile(v6XdbFile).asV6() ); } /** * init the ip2region with specified config. * set it to null for disabled the search for specified version * * @param v4Config * @param v6Config * @throws IOException */ protected Ip2Region(Config v4Config, Config v6Config) throws IOException { if (v4Config == null) { // @Note: with IPv4 disabled ? this.v4InMemSearcher = null; this.v4Pool = null; } else if (v4Config.cachePolicy == Config.BufferCache) { this.v4InMemSearcher = Searcher.newWithBuffer(v4Config.ipVersion, v4Config.cBuffer); this.v4Pool = null; } else { this.v4InMemSearcher = null; this.v4Pool = new SearcherPool(v4Config); } if (v6Config == null) { // @Note: with IPv6 disabled ? this.v6InMemSearcher = null; this.v6Pool = null; } else if (v6Config.cachePolicy == Config.BufferCache) { this.v6InMemSearcher = Searcher.newWithBuffer(v6Config.ipVersion, v6Config.cBuffer); this.v6Pool = null; } else { this.v6InMemSearcher = null; this.v6Pool = new SearcherPool(v6Config); } } // init the current ip2region service protected Ip2Region init() throws IOException { if (v4Pool != null) { v4Pool.init(); } if (v6Pool != null) { v6Pool.init(); } return this; } public String search(String ipString) throws InetAddressException, IOException, InterruptedException { return search(Util.parseIP(ipString)); } public String search(byte[] ipBytes) throws InetAddressException, IOException, InterruptedException { if (ipBytes.length == 4) { return v4Search(ipBytes); } else if (ipBytes.length == 16) { return v6Search(ipBytes); } else { throw new InetAddressException("invalid byte ip address with length=" + ipBytes.length); } } protected String v4Search(final byte[] ipBytes) throws IOException, InetAddressException, InterruptedException { if (v4InMemSearcher != null) { return v4InMemSearcher.search(ipBytes); } // IPv4 search is disabled if (v4Pool == null) { return null; } final Searcher searcher = v4Pool.borrowSearcher(); try { return searcher.search(ipBytes); } finally { v4Pool.returnSearcher(searcher); } } protected String v6Search(final byte[] ipBytes) throws IOException, InetAddressException, InterruptedException { if (v6InMemSearcher != null) { return v6InMemSearcher.search(ipBytes); } // IPv6 search is disabled if (v6Pool == null) { return null; } final Searcher searcher = v6Pool.borrowSearcher(); try { return searcher.search(ipBytes); } finally { v6Pool.returnSearcher(searcher); } } public void close() throws InterruptedException { close(10000); } public void close(long timeoutMillis) throws InterruptedException { if (v4Pool != null) { v4Pool.close(timeoutMillis); } if (v6Pool != null) { v6Pool.close(timeoutMillis); } } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/service/SearcherPool.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.service; import java.io.IOException; import java.util.Iterator; import java.util.LinkedList; import java.util.Queue; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import org.lionsoul.ip2region.xdb.Searcher; /** * ip2region searcher pool manager to provider Concurrency search support. * * @author Leon * Date 2025/11/21 */ public class SearcherPool { // config instance public final Config config; // searcher pool private final Queue pool; // lock & conditions private final ReentrantLock lock; private final Condition emptyCondition; private final Condition fullCondition; // searcher numbers that was loaned out private int loanCount; // static method to create and init the searcher pool public static final SearcherPool create(final Config config) throws IOException { return new SearcherPool(config).init(); } public static final SearcherPool create(final Config config, boolean fair) throws IOException { return new SearcherPool(config, fair).init(); } protected SearcherPool(Config config) throws IOException { this(config, false); } protected SearcherPool(Config config, boolean fair) { assert config.searchers > 0; this.config = config; this.pool = new LinkedList<>(); this.lock = new ReentrantLock(fair); this.emptyCondition = this.lock.newCondition(); this.fullCondition = this.lock.newCondition(); this.loanCount = 0; } protected SearcherPool init() throws IOException { // create the searchers for (int i = pool.size(); i < config.searchers; i++) { final Searcher searcher = new Searcher(config.ipVersion, config.xdbFile, config.vIndex, config.cBuffer); pool.add(searcher); } return this; } public int getLoanCount() { lock.lock(); int lc = this.loanCount; lock.unlock(); return lc; } public Searcher borrowSearcher() throws InterruptedException { lock.lock(); try { while (pool.isEmpty()) { emptyCondition.await(); } loanCount++; return pool.poll(); } finally { lock.unlock(); } } public void returnSearcher(final Searcher searcher) { lock.lock(); try { pool.add(searcher); loanCount--; emptyCondition.signal(); // check and signal the full condition. // pool close if (loanCount == 0) { fullCondition.signal(); } } finally { lock.unlock(); } } // close the searcher pool public void close() throws InterruptedException { close(10000); } public void close(long timeoutMillis) throws InterruptedException { lock.lock(); try { while (loanCount > 0) { fullCondition.wait(timeoutMillis); } final Iterator it = pool.iterator(); while (it.hasNext()) { final Searcher searcher = it.next(); try {searcher.close();} catch (IOException e) {} it.remove(); } } finally { lock.unlock(); } } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Lion // @Date 2022/06/23 package org.lionsoul.ip2region.xdb; public class Header { public final int version; public final int indexPolicy; public final long createdAt; public final long startIndexPtr; public final long endIndexPtr; // since xdb 3.0 with IPv6 supporting public final int ipVersion; public final int runtimePtrBytes; public final byte[] buffer; public Header(byte[] buff) { assert buff.length >= 16; version = LittleEndian.getUint16(buff, 0); indexPolicy = LittleEndian.getUint16(buff, 2); createdAt = LittleEndian.getUint32(buff, 4); startIndexPtr = LittleEndian.getUint32(buff, 8); endIndexPtr = LittleEndian.getUint32(buff, 12); ipVersion = LittleEndian.getUint16(buff, 16); runtimePtrBytes = LittleEndian.getUint16(buff, 18); buffer = buff; } @Override public String toString() { return "{" + "Version:" + version + ',' + "IndexPolicy:" + indexPolicy + ',' + "CreatedAt:" + createdAt + ',' + "StartIndexPtr:" + startIndexPtr + ',' + "EndIndexPtr:" + endIndexPtr + ',' + "IPVersion:" + ipVersion + ',' + "RuntimePtrBytes:" + runtimePtrBytes + '}'; } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/IPv4.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; // IPv4 version implementation // @Author Lion // @Date 2025/09/10 public class IPv4 extends Version { public IPv4() { // segmentIndex: 4 + 4 + 2 + 4 super(4, "IPv4", 4, 14); } @Override public int putBytes(byte[] buff, int offset, byte[] ip) { // use the Little endian byte order to compatible with the old searcher implementation buff[offset++] = ip[3]; buff[offset++] = ip[2]; buff[offset++] = ip[1]; buff[offset ] = ip[0]; return ip.length; } @Override public int ipSubCompare(byte[] ip1, byte[] buff, int offset) { // ip1: Big endian byte order parsed from input // ip2: Little endian byte order read from xdb index. // @Note: to compatible with the old Litten endian index encode implementation. int j = offset + ip1.length - 1; for (int i = 0; i < ip1.length; i++, j--) { final int i1 = (int) (ip1[i] & 0xFF); final int i2 = (int) (buff[j] & 0xFF); if (i1 < i2) { return -1; } if (i1 > i2) { return 1; } } return 0; } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/IPv6.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; // IPv4 version implementation // @Author Lion // @Date 2025/09/10 public class IPv6 extends Version { public IPv6() { // segmentIndex: 16 + 16 + 2 + 4 super(6, "IPv6", 16, 38); } @Override public int putBytes(byte[] buff, int offset, byte[] ip) { System.arraycopy(ip, 0, buff, offset, ip.length); return ip.length; } @Override public int ipSubCompare(byte[] ip1, byte[] buff, int offset) { return Util.ipSubCompare(ip1, buff, offset); } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/InetAddressException.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; public class InetAddressException extends Exception { public InetAddressException(String str) { super(str); } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/LittleEndian.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; // Little Endian basic data type decode and encode. // @Author Lion // @Date 2025/09/10 public class LittleEndian { public final static int[] shiftIndex = {0, 8, 16, 24, 32, 40, 48, 56}; // put specified bytes to the buffer started from the offset public static void put(final byte[] buff, int offset, long value, int bytes) { if (bytes > 8) { throw new IndexOutOfBoundsException("bytes should be <= 8"); } for (int i = 0; i < bytes; i++) { buff[offset++] = (byte)((value >>> shiftIndex[i]) & 0xFF); } } // put an uint32 (4 bytes long) to the buffer from the offset public static void putUint32(final byte[] buff, int offset, long value) { buff[offset++] = (byte) (value & 0xFF); buff[offset++] = (byte) ((value >> 8) & 0xFF); buff[offset++] = (byte) ((value >> 16) & 0xFF); buff[offset ] = (byte) ((value >> 24) & 0xFF); } // put a 2-bytes int to the buffer from the specified offset public static void putUint16(final byte[] buff, int offset, int value) { buff[offset++] = (byte) (value & 0xFF); buff[offset ] = (byte) ((value >> 8) & 0xFF); } // get an uint32 from a byte array from the specified offset public static long getUint32(final byte[] buff, int offset) { return ( ((buff[offset++] & 0x000000FFL)) | ((buff[offset++] << 8) & 0x0000FF00L) | ((buff[offset++] << 16) & 0x00FF0000L) | ((buff[offset ] << 24) & 0xFF000000L) ); } // get an 2 bytes int from a byte array from the specified offset public static int getUint16(final byte[] buff, int offset) { return ( ((buff[offset++]) & 0x000000FF) | ((buff[offset ] << 8) & 0x0000FF00) ); } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Lion // @Date 2022/07/14 package org.lionsoul.ip2region.xdb; import java.text.SimpleDateFormat; import java.util.Date; // simple log implementation public class Log { /* Log level constants define */ public static final int DEBUG = 0; public static final int INFO = 1; public static final int WARN = 2; public static final int ERROR = 3; // level name public static final String[] level_string = new String[] { "DEBUG", "INFO", "WARN", "ERROR" }; public final Class baseClass; private int level = INFO; public Log(Class baseClass) { this.baseClass = baseClass; } public static Log getLogger(Class baseClass) { return new Log(baseClass); } public String format(int level, String format, Object... args) { // append the datetime final StringBuilder sb = new StringBuilder(); final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); sb.append(String.format("%s %-5s ", sdf.format(new Date()), level_string[level])); // append the class name sb.append(baseClass.getName()).append(' '); sb.append(String.format(format, args)); return sb.toString(); } public void printf(int level, String format, Object... args) { if (level < DEBUG || level > ERROR) { throw new IndexOutOfBoundsException("invalid level index " + level); } // level filter if (level < this.level) { return; } System.out.println(format(level, format, args)); System.out.flush(); } public String getDebugf(String format, Object... args) { return format(DEBUG, format, args); } public void debugf(String format, Object... args) { printf(DEBUG, format, args); } public String getInfof(String format, Object... args) { return format(INFO, format, args); } public void infof(String format, Object... args) { printf(INFO, format, args); } public String getWarnf(String format, Object... args) { return format(WARN, format, args); } public void warnf(String format, Object... args) { printf(WARN, format, args); } public String getErrorf(String format, Object... args) { return format(ERROR, format, args); } public void errorf(String format, Object... args) { printf(ERROR, format, args); } public Log setLevel(int level) { this.level = level; return this; } public Log setLevel(String level) { String v = level.toLowerCase(); if ("debug".equals(v)) { this.level = DEBUG; } else if ("info".equals(v)) { this.level = INFO; } else if ("warn".equals(v)) { this.level = WARN; } else if ("error".equals(v)) { this.level = ERROR; } return this; } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/LongByteArray.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; import java.io.IOException; // xdb byte buffer which used to instead of the byte array // when the size of the xdb file is greater than 2^32 << 2; // xdb file v4 is designed to be a maximum of 2^32 bytes in size. // @Author Leon // @Date 2025/08/22 import java.util.ArrayList; import java.util.List; public class LongByteArray { // slice bytes // if it is greater than the 0 we will use the fixed slice bytes // or we use the dynamic slice bytes. private final int sliceBytes; // when EOF is true means we cannot call the #append anymore. // for fixed slice bytes only. private boolean _eof = false; // byte buffer list private final List buffs = new ArrayList(); private long length; public LongByteArray() { this.length = 0; this.sliceBytes = -1; } public LongByteArray(int sliceBytes) { assert sliceBytes != 0; assert sliceBytes <= Searcher.MAX_WRITE_BYTES; this.sliceBytes = sliceBytes; } // append new buffer public void append(final byte[] buffer) throws IOException{ // check and assert the slice bytes if (sliceBytes > 0) { if (_eof) { throw new IOException("buffer array closed (EOF=true)"); } else if (buffer.length != sliceBytes) { // mark the buffer array as closed // since the last buffer block bytes is not equal to the expected #sliceBytes _eof = true; } } buffs.add(buffer); length += buffer.length; } public long length() { return length; } public int size() { return buffs.size(); } // internal method to determine the position of the specified offset private Position determinate(final long offset) { int index = 0, position = 0, buffLen = buffs.size(); if (sliceBytes > 0) { // simply some math calcs to determine the offset index = (int) (offset / sliceBytes); position = (int) (offset - (index * sliceBytes)); // position = (int) (offset % sliceBytes); } else { // loop the buffer to determine the offset long curIndex = 0; for (index = 0; index < buffLen; index++) { final byte[] buff = buffs.get(index); if (curIndex + buff.length < offset) { curIndex += buff.length; continue; } // matched and calc the position position = (int) (offset - curIndex); break; } } return new Position(index, position); } // Copy from the current buffers to a specified buffer // from the specified offset with a specified length public byte[] copy(final long srcPos, final byte[] dest, final int destPos, final int length) { if (srcPos >= this.length) { throw new IndexOutOfBoundsException("srcPos exceed the maximum array length `" + this.length + "`"); } if (destPos + length > dest.length) { throw new IndexOutOfBoundsException("destPost+length exceed the maximum dest buffer length `" + dest.length + "`"); } final Position pos = determinate(srcPos); // copy from the current buffer final byte[] hBuff = buffs.get(pos.index++); final int copyLen = Math.min(hBuff.length - pos.offset, length); System.arraycopy(hBuff, pos.offset, dest, destPos, copyLen); // check and copy from the rest buffer? int sPos = destPos + copyLen; int left = length - copyLen; while (left > 0) { final byte[] tBuff = buffs.get(pos.index++); final int buffLen = tBuff.length; if (left <= buffLen) { System.arraycopy(tBuff, 0, dest, sPos, left); break; } System.arraycopy(tBuff, 0, dest, sPos, buffLen); sPos += buffLen; left -= buffLen; } return dest; } // get a byte-buffer from the specified index with a specified length. // this method will allocate a new byte buffer with length = $length. public byte[] slice(long offset, int length) { if (offset + length > this.length) { throw new IndexOutOfBoundsException("offset+length exceed the maximum array length `" + this.length + "`"); } final byte[] buffer = new byte[length]; return copy(offset, buffer, 0, length); } // get a 4-bytes uint32 integer from the specified index public long getUint32(long offset) { final byte[] b = new byte[4]; copy(offset, b, 0, 4); return LittleEndian.getUint32(b, 0); } public int getInt2(long offset) { final byte[] b = new byte[4]; copy(offset, b, 0, 4); return LittleEndian.getUint16(b, 0); } // position entry class public static class Position { public int index; public int offset; public Position(int index, int offset) { this.index = index; this.offset = offset; } } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; import java.io.File; // xdb searcher (Not thread safe implementation) // @Author Lion // @Date 2022/06/23 import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; public class Searcher { // xdb structure version no public static final int STRUCTURE_20 = 2; public static final int STRUCTURE_30 = 3; // constant defined copied from the xdb maker public static final int HeaderInfoLength = 256; public static final int VectorIndexRows = 256; public static final int VectorIndexCols = 256; public static final int VectorIndexSize = 8; // maximum slice bytes for dynamic buffer array. // Linux max write / read bytes. // Check https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA // to get to know why we default to this value. public static final int MAX_WRITE_BYTES = 0x7ffff000; // default slice bytes (50 MiB) for fixed buffer array. public static final int DEFAULT_SLICE_BYTES = 50 * 1024 * 1024; // ip version private final Version version; // random access file handle for file-based search private final File xdbFile; private final RandomAccessFile handle; private int ioCount = 0; // vector index. // use the byte[] instead of VectorIndex entry array to keep // the minimal memory allocation. private final byte[] vectorIndex; // xdb content buffer, used for in-memory search. // @Note: use the LongByteArray instead since 2025/08/22 // private final byte[] contentBuff; private final LongByteArray contentBuff; // --- static method to create searchers public static Searcher newWithFileOnly(Version version, String xdbPath) throws IOException { return new Searcher(version, new File(xdbPath), null, null); } public static Searcher newWithFileOnly(Version version, File xdbFile) throws IOException { return new Searcher(version, xdbFile, null, null); } public static Searcher newWithVectorIndex(Version version, String xdbPath, byte[] vectorIndex) throws IOException { return new Searcher(version, new File(xdbPath), vectorIndex, null); } public static Searcher newWithVectorIndex(Version version, File xdbFile, byte[] vectorIndex) throws IOException { return new Searcher(version, xdbFile, vectorIndex, null); } public static Searcher newWithBuffer(Version version, LongByteArray cBuff) throws IOException { return new Searcher(version, null, null, cBuff); } // --- End of creator public Searcher(Version version, File xdbFile, byte[] vectorIndex, LongByteArray cBuff) throws IOException { this.version = version; this.xdbFile = xdbFile; if (cBuff != null) { this.handle = null; this.vectorIndex = null; this.contentBuff = cBuff; } else { this.handle = new RandomAccessFile(xdbFile, "r"); this.vectorIndex = vectorIndex; this.contentBuff = null; } } public void close() throws IOException { if (this.handle != null) { this.handle.close(); } } public Version getIPVersion() { return version; } public int getIOCount() { return ioCount; } public String search(String ipStr) throws Exception { return search(Util.parseIP(ipStr)); } public String search(byte[] ip) throws IOException, InetAddressException { // ip version check if (ip.length != version.bytes) { throw new InetAddressException("invalid ip address ("+version.name+" expected)"); } // reset the global counter this.ioCount = 0; // locate the segment index block based on the vector index long sPtr = 0, ePtr = 0; int il0 = (int) (ip[0] & 0xFF); int il1 = (int) (ip[1] & 0xFF); int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; // System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx); if (vectorIndex != null) { sPtr = LittleEndian.getUint32(vectorIndex, idx); ePtr = LittleEndian.getUint32(vectorIndex, idx + 4); } else if (contentBuff != null) { sPtr = contentBuff.getUint32(HeaderInfoLength + idx); ePtr = contentBuff.getUint32(HeaderInfoLength + idx + 4); } else { final byte[] buff = new byte[VectorIndexSize]; read(HeaderInfoLength + idx, buff); sPtr = LittleEndian.getUint32(buff, 0); ePtr = LittleEndian.getUint32(buff, 4); } // System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr); // @Note: ptr validate, zero ptr means source data missing // so we could just stop here and return an empty string. if (sPtr == 0 || ePtr == 0) { return ""; } // binary search the segment index block to get the region info final int bytes = ip.length, dBytes = ip.length << 1; final int segIndexSize = version.segmentIndexSize; final byte[] buff = new byte[segIndexSize]; int dataLen = 0; long dataPtr = 0, l = 0, h = (ePtr - sPtr) / segIndexSize; while (l <= h) { long m = (l + h) >> 1; long p = sPtr + m * segIndexSize; // read the segment index read(p, buff); if (version.ipSubCompare(ip, buff, 0) < 0) { h = m - 1; } else if (version.ipSubCompare(ip, buff, bytes) > 0) { l = m + 1; } else { dataLen = LittleEndian.getUint16(buff, dBytes); dataPtr = LittleEndian.getUint32(buff, dBytes + 2); break; } } // empty match interception // System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr); if (dataLen == 0) { return ""; } // load and return the region data final byte[] regionBuff = new byte[dataLen]; read(dataPtr, regionBuff); return new String(regionBuff, "utf-8"); } protected void read(long offset, byte[] buffer) throws IOException { // check the in-memory buffer first if (contentBuff != null) { contentBuff.copy(offset, buffer, 0, buffer.length); return; } // read from the file handle assert handle != null; handle.seek(offset); this.ioCount++; int rLen = handle.read(buffer); if (rLen != buffer.length) { throw new IOException("incomplete read: read bytes should be " + buffer.length); } } @Override public String toString() { return String.format( "%s->{version:%s, xdb:%s, vIndex:%s, cBuffer:%s}", super.toString(), version.name, xdbFile == null ? "null" : xdbFile.getAbsolutePath(), vectorIndex == null ? "null" : String.valueOf(vectorIndex.length), contentBuff == null ? "null" : String.valueOf(contentBuff.length()) ); } // --- // --- static util function // --- read xdb header public static Header loadHeader(RandomAccessFile handle) throws IOException { handle.seek(0); final byte[] buff = new byte[HeaderInfoLength]; handle.read(buff); return new Header(buff); } public static Header loadHeaderFromFile(File xdbFile) throws IOException { final RandomAccessFile handle = new RandomAccessFile(xdbFile, "r"); final Header header = loadHeader(handle); handle.close(); return header; } public static Header loadHeaderFromFile(String xdbPath) throws IOException { return loadHeaderFromFile(new File(xdbPath)); } public static Header loadHeaderFromBuffer(LongByteArray cBuffer) throws IOException { return new Header(cBuffer.slice(0, HeaderInfoLength)); } // --- read xdb vector index public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException { handle.seek(HeaderInfoLength); int len = VectorIndexRows * VectorIndexCols * VectorIndexSize; final byte[] buff = new byte[len]; int rLen = handle.read(buff); if (rLen != len) { throw new IOException("incomplete read: read bytes should be " + len); } return buff; } public static byte[] loadVectorIndexFromFile(File xdbFile) throws IOException { final RandomAccessFile handle = new RandomAccessFile(xdbFile, "r"); final byte[] vIndex = loadVectorIndex(handle); handle.close(); return vIndex; } public static byte[] loadVectorIndexFromFile(String xdbPath) throws IOException { return loadVectorIndexFromFile(new File(xdbPath)); } public static byte[] loadVectorIndexFromBuffer(LongByteArray cBuffer) throws IOException { final int len = VectorIndexRows * VectorIndexCols * VectorIndexSize; return cBuffer.slice(HeaderInfoLength, len); } // --- read xdb content // -- load xdb buffer with random access file handle public static LongByteArray loadContent(RandomAccessFile handle) throws IOException { return loadContent(handle, DEFAULT_SLICE_BYTES); } public static LongByteArray loadContent(RandomAccessFile handle, final int sliceBytes) throws IOException { handle.seek(0); // check the length and do the buff load long toRead = handle.length(); final LongByteArray byteArray = new LongByteArray(sliceBytes); while (toRead > 0) { final byte[] buff = new byte[(int) Math.min(toRead, sliceBytes)]; final int rLen = handle.read(buff); if (rLen != buff.length) { throw new IOException("incomplete read: read bytes should be " + buff.length + ", got `" + rLen + "`"); } byteArray.append(buff); toRead -= rLen; } return byteArray; } // -- load xdb buffer with xdb file object public static LongByteArray loadContentFromFile(File xdbFile) throws IOException { return loadContentFromFile(xdbFile, DEFAULT_SLICE_BYTES); } public static LongByteArray loadContentFromFile(File xdbFile, final int sliceBytes) throws IOException { final RandomAccessFile handle = new RandomAccessFile(xdbFile, "r"); final LongByteArray content = loadContent(handle, sliceBytes); handle.close(); return content; } // -- load xdb buffer with xdb file path public static LongByteArray loadContentFromFile(String xdbPath) throws IOException { return loadContentFromFile(xdbPath, DEFAULT_SLICE_BYTES); } public static LongByteArray loadContentFromFile(String xdbPath, final int sliceBytes) throws IOException { return loadContentFromFile(new File(xdbPath), sliceBytes); } // load xdb buffer from input stream public static LongByteArray loadContentFromInputStream(InputStream is) throws IOException { return loadContentFromInputStream(is, DEFAULT_SLICE_BYTES); } public static LongByteArray loadContentFromInputStream(InputStream is, final int sliceBytes) throws IOException { final LongByteArray byteArray = new LongByteArray(sliceBytes); while (true) { boolean done = false; // read at most MAX_WRITE_BYTES bytes int rLen, tBytes = 0; final byte[] buff = new byte[sliceBytes]; while (true) { rLen = is.read(buff, tBytes, buff.length - tBytes); if (rLen == -1) { // reach the end of the stream done = true; break; } else if (rLen == 0) { // the entire buff was filled break; } tBytes += rLen; } // check and copy the buffer with its actual filled bytes if (tBytes == buff.length) { byteArray.append(buff); } else { final byte[] nBuff = new byte[tBytes]; System.arraycopy(buff, 0, nBuff, 0, tBytes); byteArray.append(nBuff); } if (done) { break; } } return byteArray; } // --- verify util function // Verify if the current Searcher could be used to search the specified xdb file. // Why do we need this check ? // The future features of the xdb impl may cause the current searcher not able to work properly. // // @Note: You Just need to check this ONCE when the service starts // Or use another process (eg, A command) to check once Just to confirm the suitability. public static void verify(Header header, long fileBytes) throws IOException, XdbException { // get the runtime ptr bytes int runtimePtrBytes = 0; if (header.version == STRUCTURE_20) { runtimePtrBytes = 4; } else if (header.version == STRUCTURE_30) { runtimePtrBytes = header.runtimePtrBytes; } else { throw new XdbException("invalid structure version `" + header.version + "`"); } // 1, confirm the xdb file size // to ensure that the maximum file pointer does not overflow final long maxFilePtr = (1L << (runtimePtrBytes * 8)) - 1; if (fileBytes > maxFilePtr) { throw new XdbException("xdb file exceeds the maximum supported bytes: "+maxFilePtr+""); } } public static void verify(RandomAccessFile handle) throws IOException, XdbException { verify(loadHeader(handle), handle.length()); } public static void verifyFromFile(File xdbFile) throws IOException, XdbException { final RandomAccessFile handle = new RandomAccessFile(xdbFile, "r"); verify(handle); handle.close(); } public static void verifyFromFile(String xdbPath) throws IOException, XdbException { verifyFromFile(new File(xdbPath)); } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Lion // @Date 2022/07/14 package org.lionsoul.ip2region.xdb; import java.net.InetAddress; import java.net.UnknownHostException; public class Util { // parse the specified IP address and return its bytes. // returns: byte[4] for IPv4 and byte[16] for IPv6 and the bytes should be in Big endian order. public static byte[] parseIP(String ip) throws InetAddressException { try { return InetAddress.getByName(ip).getAddress(); } catch (UnknownHostException e) { throw new InetAddressException("invalid ip address `"+ip+"`"); } } // convert the byte[] ip to string ip address public static String ipToString(final byte[] ip) { if (ip.length != 4 && ip.length != 16) { return String.format("invalid-ip-address-length: %d", ip.length); } try { return InetAddress.getByAddress(ip).getHostAddress(); } catch (UnknownHostException e) { return String.format("invalid-ip-address `%s`", ipJoin(ip)); } } // implode the byte[] ip with its byte value. public static String ipJoin(byte[] ip) { return bytesToString(ip, 0, ip.length); } public static String bytesToString(byte[] buff, int offset, int length) { final StringBuffer sb = new StringBuffer(); sb.append("["); for (int i = 0; i < length; i++) { if (i > 0) { sb.append(','); } sb.append((buff[offset+i] & 0xFF)); } sb.append("]"); return sb.toString(); } // compare two byte ip // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 public static int ipCompare(byte[] ip1, byte[] ip2) { return ipSubCompare(ip1, ip2, 0); } // compare the ip with the ip in the buffer start from offset // Returns: -1 if ip < buff[offset], 0 if ip == buff[offset], 1 if ip > buff[offset] public static int ipSubCompare(byte[] ip, byte[] buff, int offset) { for (int i = 0; i < ip.length; i++) { // covert the byte to int to sure the uint8 attribute final int i1 = (int)(ip[i] & 0xFF); final int i2 = (int)(buff[offset+i] & 0xFF); if (i1 < i2) { return -1; } if (i1 > i2) { return 1; } } return 0; } public static byte[] ipAddOne(byte[] ip) { final byte[] r = new byte[ip.length]; System.arraycopy(ip, 0, r, 0, ip.length); for (int i = ip.length - 1; i >= 0; i--) { final int v = (int)(r[i] & 0xFF); if (v < 255) { // No overflow r[i]++; break; } r[i] = 0; } return r; } public static byte[] ipSubOne(byte[] ip) { final byte[] r = new byte[ip.length]; System.arraycopy(ip, 0, r, 0, ip.length); for (int i = ip.length - 1; i >= 0; i--) { final int v = (int)(r[i] & 0xFF); if (v > 0) { // No borrow needed r[i]--; break; } r[i] = (byte) 0xFF; // borrow from the next byte } return r; } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/Version.java ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; // IP version abstract manager (IPv4 & IPv6) // @Author Lion // @Date 2025/09/10 public abstract class Version { public static final int IPv4VersionNo = 4; public static final int IPv6VersionNo = 6; public static final IPv4 IPv4 = new IPv4(); public static final IPv6 IPv6 = new IPv6(); // version id and name public final int id; public final String name; // the numbers of bytes for one IP public final int bytes; // segment index size (bytes) public final int segmentIndexSize; public Version(int id, String name, int bytes, int segmentIndexSize) { this.id = id; this.name = name; this.bytes = bytes; this.segmentIndexSize = segmentIndexSize; } // encode the specified IP bytes to the specified buffer public abstract int putBytes(byte[] buff, int offset, byte[] ip); // compare the two IPs with the current version. // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 public int ipCompare(byte[] ip1, byte[] ip2) { return ipSubCompare(ip1, ip2, 0); } // @see ipCompare public abstract int ipSubCompare(byte[] ip1, byte[] buff, int offset); // parse the version from an name public static final Version fromName(String name) throws Exception { final String n = name.toUpperCase(); if (n.equals("V4") || n.equals("IPV4")) { return IPv4; } else if (n.equals("V6") || n.equals("IPV6")) { return IPv6; } else { throw new Exception("invalid version name `"+name+"`"); } } // parse the version from header public static final Version fromHeader(Header header) throws XdbException { // Old 2.0 structure with IPv4 supports ONLY. if (header.version == Searcher.STRUCTURE_20) { return IPv4; } // structure 3.0 after IPv6 supporting if (header.version != Searcher.STRUCTURE_30) { throw new XdbException("invalid xdb structure version `"+header.version+"`"); } if (header.ipVersion == IPv4VersionNo) { return IPv4; } else if (header.ipVersion == IPv6VersionNo) { return IPv6; } else { throw new XdbException("invalid ip version number `" + header.ipVersion + "`"); } } @Override public String toString() { return String.format("{Id:%d, Name:%s, Bytes:%d, IndexSize: %d}", id, name, bytes, segmentIndexSize); } } ================================================ FILE: binding/java/src/main/java/org/lionsoul/ip2region/xdb/XdbException.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; public class XdbException extends Exception { public XdbException(String str) { super(str); } } ================================================ FILE: binding/java/src/test/java/org/lionsoul/ip2region/service/ConfigTest.java ================================================ package org.lionsoul.ip2region.service; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.security.CodeSource; import org.junit.Test; import org.lionsoul.ip2region.xdb.Log; import org.lionsoul.ip2region.xdb.XdbException; public class ConfigTest { private static final Log log = Log.getLogger(ConfigTest.class).setLevel(Log.DEBUG); public static final String getDataPath(String xdbFile) { final CodeSource cs = ConfigTest.class.getProtectionDomain().getCodeSource(); if (cs != null) { // log.debugf("code path: %s", cs.getLocation().getPath().concat("../../../../data/")); return cs.getLocation().getPath().concat("../../../../data/").concat(xdbFile); } else { return "../../../../data/".concat(xdbFile); } } @Test public void testBuildV4Config() throws IOException, XdbException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.VIndexCache) .setXdbPath(getDataPath("ip2region_v4.xdb")) .setSearchers(20) .asV4(); log.debugf("builded config: %s", v4Config); } @Test public void testBuildV4ConfigFromFile() throws IOException, XdbException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.BufferCache) .setXdbFile(new File(getDataPath("ip2region_v4.xdb"))) .setSearchers(20) .asV4(); log.debugf("builded config: %s", v4Config); } @Test public void testBuildV4ConfigFromInputStream() throws IOException, XdbException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.BufferCache) .setXdbInputStream(new FileInputStream(getDataPath("ip2region_v4.xdb"))) .setSearchers(20) .asV4(); log.debugf("builded config: buffs.size=%d, %s", v4Config.cBuffer.size(), v4Config); } @Test public void testBuildV4SliceBytes() throws IOException, XdbException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.FullCache) .setCacheSliceBytes(1024 * 1024) // 1MiB .setXdbPath(getDataPath("ip2region_v4.xdb")) .setSearchers(20) .asV4(); log.debugf("builded config: buffs.size=%d, %s", v4Config.cBuffer.size(), v4Config); } // --- IPv6 @Test public void testBuildV6Config() throws IOException, XdbException, InvalidConfigException { final Config v6Config = Config.custom() .setCachePolicy(Config.VIndexCache) .setXdbPath(getDataPath("ip2region_v6.xdb")) .setSearchers(20) .asV6(); log.debugf("builded config: %s", v6Config); } @Test public void testBuildV6ConfigFromFile() throws IOException, XdbException, InvalidConfigException { final Config v6Config = Config.custom() .setCachePolicy(Config.BufferCache) .setXdbFile(new File(getDataPath("ip2region_v6.xdb"))) .setSearchers(20) .asV6(); log.debugf("builded config: %s", v6Config); } @Test public void testBuildV6ConfigFromInputStream() throws IOException, XdbException, InvalidConfigException { final Config v6Config = Config.custom() .setCachePolicy(Config.BufferCache) .setXdbInputStream(new FileInputStream(getDataPath("ip2region_v6.xdb"))) .setSearchers(20) .asV6(); log.debugf("builded config: buffs.size=%d, %s", v6Config.cBuffer.size(), v6Config); } @Test public void testBuildV6SliceBytes() throws IOException, XdbException, InvalidConfigException { final Config v6Config = Config.custom() .setCachePolicy(Config.BufferCache) .setCacheSliceBytes(1024 * 1024 * 4) // 4 MiB .setXdbInputStream(new FileInputStream(getDataPath("ip2region_v6.xdb"))) .setSearchers(20) .asV6(); log.debugf("builded config: buffs.size=%d, %s", v6Config.cBuffer.size(), v6Config); } } ================================================ FILE: binding/java/src/test/java/org/lionsoul/ip2region/service/Ip2RegionTest.java ================================================ package org.lionsoul.ip2region.service; import static org.junit.Assert.assertEquals; import java.io.IOException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; import org.lionsoul.ip2region.xdb.InetAddressException; import org.lionsoul.ip2region.xdb.Log; import org.lionsoul.ip2region.xdb.Util; import org.lionsoul.ip2region.xdb.XdbException; public class Ip2RegionTest { private static final Log log = Log.getLogger(Ip2RegionTest.class).setLevel(Log.DEBUG); @Test public void TestConfigCreate() throws IOException, XdbException, InetAddressException, InterruptedException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.NoCache) .setSearchers(10) .setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb")) .asV4(); final Config v6Config = Config.custom() .setCachePolicy(Config.VIndexCache) .setSearchers(10) .setXdbPath(ConfigTest.getDataPath("ip2region_v6.xdb")) .asV6(); byte[] v4Bytes = Util.parseIP("113.92.157.29"); byte[] v6Bytes = Util.parseIP("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config); for (int i = 0; i < 50; i++) { v4Bytes = Util.ipAddOne(v4Bytes); v6Bytes = Util.ipAddOne(v6Bytes); final String v4Region = ip2Region.search(v4Bytes); final String v6Region = ip2Region.search(v6Bytes); log.debugf("search(%s)=%s, search(%s)=%s", Util.ipToString(v4Bytes), v4Region, Util.ipToString(v6Bytes), v6Region); } ip2Region.close(); log.debugf("ip2region closed gracefully"); } @Test public void TestPathCreate() throws InetAddressException, IOException, XdbException, InterruptedException, InvalidConfigException { byte[] v4Bytes = Util.parseIP("113.92.157.29"); byte[] v6Bytes = Util.parseIP("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); final Ip2Region ip2Region = Ip2Region.create(ConfigTest.getDataPath("ip2region_v4.xdb"), ConfigTest.getDataPath("ip2region_v6.xdb")); for (int i = 0; i < 50; i++) { v4Bytes = Util.ipAddOne(v4Bytes); v6Bytes = Util.ipAddOne(v6Bytes); final String v4Region = ip2Region.search(v4Bytes); final String v6Region = ip2Region.search(v6Bytes); log.debugf("search(%s)=%s, search(%s)=%s", Util.ipToString(v4Bytes), v4Region, Util.ipToString(v6Bytes), v6Region); } ip2Region.close(); log.debugf("ip2region closed gracefully"); } @Test public void TestInMemSearch() throws IOException, XdbException, InetAddressException, InterruptedException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.BufferCache) .setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb")) .asV4(); final Config v6Config = Config.custom() .setCachePolicy(Config.BufferCache) .setXdbPath(ConfigTest.getDataPath("ip2region_v6.xdb")) .asV6(); byte[] v4Bytes = Util.parseIP("113.92.157.29"); byte[] v6Bytes = Util.parseIP("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config); for (int i = 0; i < 50; i++) { v4Bytes = Util.ipAddOne(v4Bytes); v6Bytes = Util.ipAddOne(v6Bytes); final String v4Region = ip2Region.search(v4Bytes); final String v6Region = ip2Region.search(v6Bytes); log.debugf("search(%s)=%s, search(%s)=%s", Util.ipToString(v4Bytes), v4Region, Util.ipToString(v6Bytes), v6Region); } ip2Region.close(); log.debugf("ip2region closed gracefully"); } @Test public void TestConcurrentCall() throws IOException, XdbException, InetAddressException, InterruptedException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.VIndexCache) .setSearchers(15) .setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb")) .asV4(); final Config v6Config = Config.custom() .setCachePolicy(Config.VIndexCache) .setSearchers(15) .setXdbPath(ConfigTest.getDataPath("ip2region_v6.xdb")) .asV6(); byte[] v4Bytes = Util.parseIP("113.92.157.29"); byte[] v6Bytes = Util.parseIP("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); final int threads = 100; final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config); final CountDownLatch latch = new CountDownLatch(threads); final AtomicInteger count = new AtomicInteger(0); final long tStart = System.nanoTime(); for (int i = 0; i < threads; i++) { final Runnable t = new Runnable() { @Override public void run() { for (int i = 0; i < 5000; i++) { final byte[] ipBytes = i % 2 == 0 ? v4Bytes : v6Bytes; try { final String region = ip2Region.search(ipBytes); if (ipBytes.length == 4) { assertEquals("v4 region not equals", region, "中国|广东省|深圳市|电信|CN"); } else { assertEquals("v6 region not equals", region, "中国|广东省|深圳市|电信|CN"); } } catch (InetAddressException | IOException | InterruptedException e) { log.errorf("failed to search(%s): %s", Util.ipToString(ipBytes), e.getMessage()); } count.incrementAndGet(); } latch.countDown(); } }; t.run(); } latch.await(); final long costs = System.nanoTime() - tStart; log.debugf("%d searches finished in %dms, avg took: %dµs", count.get(), costs / 1000_000, costs / count.get() / 1000); ip2Region.close(); log.debugf("ip2region closed gracefully"); } @Test public void TestV4Only() throws IOException, XdbException, InetAddressException, InterruptedException, InvalidConfigException { final Config v4Config = Config.custom() .setCachePolicy(Config.NoCache) .setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb")) .asV4(); byte[] v4Bytes = Util.parseIP("113.92.157.29"); byte[] v6Bytes = Util.parseIP("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); final Ip2Region ip2Region = Ip2Region.create(v4Config, null); for (int i = 0; i < 10; i++) { v4Bytes = Util.ipAddOne(v4Bytes); final String v4Region = ip2Region.search(v4Bytes); final String v6Region = ip2Region.search(v6Bytes); log.debugf("search(%s)=%s, search(%s)=%s", Util.ipToString(v4Bytes), v4Region, Util.ipToString(v6Bytes), v6Region); } ip2Region.close(); log.debugf("ip2region closed gracefully"); } } ================================================ FILE: binding/java/src/test/java/org/lionsoul/ip2region/service/SearcherPoolTest.java ================================================ package org.lionsoul.ip2region.service; import org.junit.Test; import org.lionsoul.ip2region.xdb.Log; import org.lionsoul.ip2region.xdb.Searcher; public class SearcherPoolTest { private static final Log log = Log.getLogger(SearcherPoolTest.class).setLevel(Log.DEBUG); @Test public void testV4SearcherPool() throws Exception { final Config v4Config = Config.custom() .setCachePolicy(Config.VIndexCache) .setSearchers(5) .setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb")) .asV4(); final String ipStr = "58.250.36.41"; final SearcherPool v4Pool = SearcherPool.create(v4Config); for (int i = 0; i < 20; i++) { final Searcher searcher = v4Pool.borrowSearcher(); log.debugf("borrowed searcher %d: %s", i, searcher.toString()); final String region = searcher.search(ipStr); log.debugf("search(%s)=%s", ipStr, region); v4Pool.returnSearcher(searcher); log.debugf("return searcher %d", i); } v4Pool.close(); log.debugf("v4 searcher pool closed gracefully"); } @Test public void testInMemV4SearcherPool() throws Exception { final Config v4Config = Config.custom() .setCachePolicy(Config.FullCache) .setSearchers(5) .setXdbPath(ConfigTest.getDataPath("ip2region_v4.xdb")) .asV4(); final String ipStr = "58.250.36.41"; final SearcherPool v4Pool = SearcherPool.create(v4Config); for (int i = 0; i < 20; i++) { final Searcher searcher = v4Pool.borrowSearcher(); log.debugf("borrowed searcher %d: %s", i, searcher.toString()); final String region = searcher.search(ipStr); log.debugf("search(%s)=%s", ipStr, region); v4Pool.returnSearcher(searcher); log.debugf("return searcher %d", i); } v4Pool.close(); log.debugf("v4 searcher pool closed gracefully"); } @Test public void testV6SearcherPool() throws Exception { final Config v6Config = Config.custom() .setCachePolicy(Config.VIndexCache) .setSearchers(5) .setXdbPath(ConfigTest.getDataPath("ip2region_v6.xdb")) .asV6(); final String ipStr = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; final SearcherPool v4Pool = SearcherPool.create(v6Config); for (int i = 0; i < 20; i++) { final Searcher searcher = v4Pool.borrowSearcher(); log.debugf("borrowed searcher %d: %s", i, searcher.toString()); final String region = searcher.search(ipStr); log.debugf("search(%s)=%s", ipStr, region); v4Pool.returnSearcher(searcher); log.debugf("return searcher %d", i); } v4Pool.close(); log.debugf("v6 searcher pool closed gracefully"); } } ================================================ FILE: binding/java/src/test/java/org/lionsoul/ip2region/xdb/BufferTest.java ================================================ package org.lionsoul.ip2region.xdb; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.security.CodeSource; import org.junit.Test; public class BufferTest { private static final Log log = Log.getLogger(VersionTest.class).setLevel(Log.DEBUG); public static final String getDataPath(String xdbFile) { final CodeSource cs = BufferTest.class.getProtectionDomain().getCodeSource(); if (cs != null) { // log.debugf("code path: %s", cs.getLocation().getPath().concat("../../../../data/")); return cs.getLocation().getPath().concat("../../../../data/").concat(xdbFile); } else { return "../../../../data/".concat(xdbFile); } } // --- v4 @Test public void testV4InputStreamBuffer() throws Exception { final LongByteArray cBuffer = Searcher.loadContentFromInputStream( new FileInputStream(getDataPath("ip2region_v4.xdb")) ); log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size()); } @Test public void testV4FixedBuffer() throws Exception { final LongByteArray cBuffer = Searcher.loadContentFromFile( new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024 ); final Header header = Searcher.loadHeaderFromBuffer(cBuffer); log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size()); log.debugf("Header->%s", header); } @Test public void testV4BufferAssert() throws Exception { final LongByteArray m2Bufer = Searcher.loadContentFromFile( new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024 ); final LongByteArray m5Bufer = Searcher.loadContentFromFile( new File(getDataPath("ip2region_v4.xdb")), 5 * 1024 * 1024 ); final int[] offsets = new int[]{0, 10, 512, 1024, 39672, 1024 * 1024 * 2}; for (int idx : offsets) { final long m2Val = m2Bufer.getUint32(idx); final long m5Val = m5Bufer.getUint32(idx); log.debugf("m2Buffer[%8d:4]: %10d, m5Buffer[%8d:4]: %10d, equals ? %s", idx, m2Val, idx, m5Val, m2Val == m5Val ? "true" : "false"); } } @Test public void testV4BufferEOF() throws IOException { final LongByteArray buffer = Searcher.loadContentFromFile( new File(getDataPath("ip2region_v4.xdb")), 2 * 1024 * 1024 ); try { buffer.append(new byte[1024]); } catch (IOException e) { log.debugf("failed to append: %s", e.getMessage()); } } // --- v6 @Test public void testV6InputStreamBuffer() throws Exception { final LongByteArray cBuffer = Searcher.loadContentFromInputStream( new FileInputStream(getDataPath("ip2region_v6.xdb")) ); log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size()); } @Test public void testV6FixedBuffer() throws Exception { final LongByteArray cBuffer = Searcher.loadContentFromFile( new File(getDataPath("ip2region_v6.xdb")), 5 * 1024 * 1024 ); final Header header = Searcher.loadHeaderFromBuffer(cBuffer); log.debugf("cBuffer->{length:%d, size:%d}", cBuffer.length(), cBuffer.size()); log.debugf("Header->%s", header); } } ================================================ FILE: binding/java/src/test/java/org/lionsoul/ip2region/xdb/IPv4Test.java ================================================ package org.lionsoul.ip2region.xdb; import org.junit.Test; public class IPv4Test { private static final Log log = Log.getLogger(IPv4Test.class); @Test public void testIpSubCompare() throws InetAddressException { final byte[] sip = Util.parseIP("0.255.255.255"); final byte[] eip = Util.parseIP("1.0.0.2"); final byte[] buff = new byte[Version.IPv4.segmentIndexSize]; Version.IPv4.putBytes(buff, 0, sip); Version.IPv4.putBytes(buff, 4, eip); log.infof("bytesToString(buff): %s", Util.bytesToString(buff, 0, 8)); final byte[] ip = Util.parseIP("1.0.0.0"); // compare the sip log.infof("ipSubCompare(%s, %s): %d", Util.ipToString(ip), Util.bytesToString(buff, 0, 4), Util.ipSubCompare(ip, buff, 0) ); log.infof("IPv4.ipSubCompare(%s, %s): %d", Util.ipToString(ip), Util.bytesToString(buff, 0, 4), Version.IPv4.ipSubCompare(ip, buff, 0) ); // compare the eip log.infof("ipSubCompare(%s, %s): %d", Util.ipToString(ip), Util.bytesToString(buff, 4, 4), Util.ipSubCompare(ip, buff, 4) ); log.infof("IPv4.ipSubCompare(%s, %s): %d", Util.ipToString(ip), Util.bytesToString(buff, 4, 4), Version.IPv4.ipSubCompare(ip, buff, 4) ); } } ================================================ FILE: binding/java/src/test/java/org/lionsoul/ip2region/xdb/LittleEndianTest.java ================================================ package org.lionsoul.ip2region.xdb; import static org.junit.Assert.assertEquals; import org.junit.Test; public class LittleEndianTest { private static final Log log = Log.getLogger(LittleEndianTest.class).setLevel(Log.DEBUG); @Test public void testAll() { final byte[] buff = new byte[14]; // encode // do the put LittleEndian.put(buff, 0, 1L, 4); LittleEndian.put(buff, 4, 2L, 4); // putUint32 LittleEndian.putUint16(buff, 8, 24); LittleEndian.putUint32(buff, 10, 1024L); // decode assertEquals(LittleEndian.getUint32(buff, 0), 1); assertEquals(LittleEndian.getUint32(buff, 4), 2); assertEquals(LittleEndian.getUint16(buff, 8), 24); assertEquals(LittleEndian.getUint32(buff, 10), 1024); log.debugf("uint32(buff, 0): %d", LittleEndian.getUint32(buff, 0)); log.debugf("uint32(buff, 4): %d", LittleEndian.getUint32(buff, 4)); log.debugf("int2(buff, 8): %d", LittleEndian.getUint16(buff, 8)); log.debugf("uint32(buff, 10): %d", LittleEndian.getUint32(buff, 10)); } } ================================================ FILE: binding/java/src/test/java/org/lionsoul/ip2region/xdb/UtilTest.java ================================================ package org.lionsoul.ip2region.xdb; import org.junit.Test; public class UtilTest { private static final Log log = Log.getLogger(UtilTest.class).setLevel(Log.DEBUG); @Test public void testCheckIP() throws InetAddressException { final String[] ips = new String[]{ "192.168.1.102", "219.133.111.87", "::", "3000::", "::1001:ffff", "2001:2:0:ffff:ffff:ffff:ffff:ffff", "::ffff:114.114.114.114" }; for (String ip : ips) { final byte[] ipBytes = Util.parseIP(ip); log.debugf("%s(v=%s) => %s", ip, Util.ipJoin(ipBytes), Util.ipToString(ipBytes)); } } @Test public void testIpCompare() throws InetAddressException { final String[][] ipPairs = new String[][]{ {"1.0.0.0", "1.0.0.1"}, {"192.168.1.101", "192.168.1.90"}, {"219.133.111.87", "114.114.114.114"}, {"2000::", "2000:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, {"2001:4:112::", "2001:4:112:ffff:ffff:ffff:ffff:ffff"}, {"ffff::", "2001:4:ffff:ffff:ffff:ffff:ffff:ffff"} }; for (String[] ips : ipPairs) { final byte[] ip1 = Util.parseIP(ips[0]); final byte[] ip2 = Util.parseIP(ips[1]); log.debugf("compare(%s, %s): %d", ips[0], ips[1], Util.ipCompare(ip1, ip2)); } } @Test public void testIpSubCompare() throws InetAddressException { final String[][] ipPairs = new String[][] { {"1.0.0.0", "1.0.0.1"}, {"192.168.1.100", "192.168.2.100"}, {"10.100.1.10", "11.100.2.10"}, {"11.100.1.10", "10.100.2.10"} }; for (final String[] ips : ipPairs) { final byte[] ip1 = Util.parseIP(ips[0]); final byte[] ip2 = Util.parseIP(ips[1]); log.debugf("ipSubCompare(%s, %s): %d", Util.ipToString(ip1), Util.ipToString(ip2), Util.ipSubCompare(ip1, ip2, 0)); } } } ================================================ FILE: binding/java/src/test/java/org/lionsoul/ip2region/xdb/VersionTest.java ================================================ package org.lionsoul.ip2region.xdb; import static org.junit.Assert.assertEquals; import org.junit.Test; public class VersionTest { private static final Log log = Log.getLogger(VersionTest.class).setLevel(Log.DEBUG); @Test public void testFromName() throws Exception { final String[] vers = new String[]{"IPv4", "IPv6"}; final Version v4 = Version.fromName(vers[0]); assertEquals(v4.name, vers[0]); final Version v6 = Version.fromName(vers[1]); assertEquals(v6.name, vers[1]); log.debugf("v4: %s", v4); log.debugf("v6: %s", v6); } @Test public void testFromHeader() throws XdbException { // create the header buffer final byte[] buff1 = new byte[Searcher.HeaderInfoLength]; // structure version LittleEndian.put(buff1, 0, 2, 2); LittleEndian.put(buff1, 16, Version.IPv4VersionNo, 2); final byte[] buff2 = new byte[Searcher.HeaderInfoLength]; LittleEndian.put(buff2, 0, 3, 2); LittleEndian.put(buff2, 16, Version.IPv6VersionNo, 2); final Version ver1 = Version.fromHeader(new Header(buff1)); final Version ver2 = Version.fromHeader(new Header(buff2)); log.debugf("ver1: %s", ver1.toString()); log.debugf("ver2: %s", ver2.toString()); } } ================================================ FILE: binding/javascript/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region JavaScript Query Client # Usage ### Install `ip2region.js` ```bash npm install ip2region.js --save ``` ### About Query API The prototype of the Query API is: ```javascript // Query via a string IP or a binary IP (Buffer type) parsed by parseIP search(ip: string | Buffer): string; ``` If an error occurs during the query, an exception will be thrown. If the query is successful, the `region` information string will be returned. If the specified IP cannot be found, an empty string `""` will be returned. ### About IPv4 and IPv6 This xdb query client implementation supports both IPv4 and IPv6 queries. The usage is as follows: ```javascript import {IPv4, IPv6} from 'ip2region.js'; // For IPv4: Set xdb path to the v4 xdb file, and set IP version to Version.IPv4 let dbPath = "../../data/ip2region_v4.xdb"; // or your ipv4 xdb path let version = IPv4; // For IPv6: Set xdb path to the v6 xdb file, and set IP version to Version.IPv6 let dbPath = "../../data/ip2region_v6.xdb"; // or your ipv6 xdb path let version = IPv6; // The IP version of the xdb specified by dbPath must match the version specified; otherwise, an error will occur during execution. // Note: The following demonstrations directly use the dbPath and version variables. ``` ### File Verification It is recommended that you proactively verify the applicability of the xdb file. Some new features in the future may cause the current Searcher version to be incompatible with the xdb file you are using. Verification helps avoid unpredictable errors during runtime. You do not need to verify every time; for example, verify once when the service starts or manually call the command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios. ```javascript import {verifyFromFile} from 'ip2region.js'; try { verifyFromFile(dbPath); } catch (e) { // Applicability verification failed!!! // The current query client implementation is not applicable for queries on the xdb file specified by dbPath. // You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation that fits dbPath. console.log(`binding is not applicable for xdb file '${dbPath}': ${e.message}`); return; } // Verification passed. The current Searcher can be safely used for query operations on the xdb pointed to by dbPath. ``` ### File-Only Query ```javascript import {newWithFileOnly} from 'ip2region.js'; // 1. Create a file-only query object using the version and dbPath mentioned above let searcher; try { searcher = newWithFileOnly(version, dbPath); } catch(e) { console.log(`failed to newWithFileOnly: ${err.message}`); return; } // 2. Query; the interface is the same for both IPv4 and IPv6 addresses let ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 try { let region = await searcher.search(ip); console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`); } catch(e) { console.log(`${err.message}`); } // 3. Close resources searcher.close(); // Note: Each thread needs to create an independent Searcher object separately ``` ### Caching `VectorIndex` We can pre-load the `VectorIndex` data from the `xdb` file and cache it globally. Using a global VectorIndex cache every time a Searcher object is created can reduce one fixed IO operation, thereby accelerating queries and reducing IO pressure. ```javascript import {loadVectorIndexFromFile, newWithVectorIndex} from 'ip2region.js'; // 1. Pre-load VectorIndex cache from dbPath and keep this data as a global variable for subsequent repeated use. let vIndex; try { vIndex = loadVectorIndexFromFile(dbPath); } catch (e) { console.log(`failed to load vector index from ${dbPath}: ${e.message}`); return; } // 2. Create a query object with VectorIndex cache using the global vIndex. let searcher; try { searcher = newWithVectorIndex(version, dbPath, vIndex); } catch(e) { console.log(`failed to newWithVectorIndex: ${err.message}`); return; } // 3. Query; the interface is the same for both IPv4 and IPv6 addresses let ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 try { let region = await searcher.search(ip); console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`); } catch(e) { console.log(`${err.message}`); } // 4. Close resources searcher.close(); // Note: Each thread needs to create a separate independent Searcher object, but they all share the global read-only vIndex cache. ``` ### Caching the entire `xdb` file We can also pre-load the data of the entire xdb file into memory and then create a query object based on this data to achieve a completely memory-based query, similar to the previous memory search. ```javascript import {loadContentFromFile, newWithBuffer} from 'ip2region.js'; // 1. Load the entire xdb from dbPath into memory. let cBuffer; try { cBuffer = loadContentFromFile(dbPath); } catch (e) { console.log(`failed to load content from ${dbPath}: ${e.message}`); return; } // 2. Use the cBuff above to create a completely memory-based query object. let searcher; try { searcher = newWithBuffer(version, cBuffer); } catch(e) { console.log(`failed to newWithBuffer: ${err.message}`); return; } // 3. Query; the interface is the same for both IPv4 and IPv6 addresses let ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 try { let region = await searcher.search(ip); console.log(`search(${ip}): {region: ${region}`); } catch(e) { console.log(`${err.message}`); } // 4. Close resources - This searcher object is safe for concurrent use; close the searcher only when the entire service shuts down. // searcher.close(); // Note: For concurrent use, the query object created with the entire xdb data cache can be safely used concurrently, meaning you can make this searcher object a global object for cross-thread access. ``` # Query Test You can test queries using the `node tests/search.app.js` command: ```bash ➜ javascript git:(fr_javascript_ipv6) node tests/search.app.js usage: Usage node tests/search.app.js [command options] ip2region search script optional arguments: -h, --help show this help message and exit --db DB ip2region binary xdb file path --cache-policy CACHE_POLICY cache policy: file/vectorIndex/content, default: vectorIndex ``` Example: Using the default data/ip2region_v4.xdb file for IPv4 query testing: ```bash ➜ javascript git:(fr_javascript_ipv6) ✗ node tests/search.app.js --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 657.035 μs} ip2region>> 113.118.113.77 {region: 中国|广东省|深圳市|电信|CN, ioCount: 2, took: 169.927 μs} ``` Example: Using the default data/ip2region_v6.xdb file for IPv6 query testing: ```bash ➜ javascript git:(fr_javascript_ipv6) ✗ node tests/search.app.js --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 98.953 μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, ioCount: 13, took: 287.703 μs} ``` Enter an IP to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the effects of the three different cache implementations. # bench Test You can perform a bench test via the `node tests/bench.app.js` command, which ensures the `xdb` file is error-free and evaluates query performance: ```bash ➜ javascript git:(fr_javascript_ipv6) ✗ node tests/bench.app.js usage: Usage node tests/bench.app.js [command options] ip2region bench script optional arguments: -h, --help show this help message and exit --db DB ip2region binary xdb file path --src SRC source ip text file path --cache-policy CACHE_POLICY cache policy: file/vectorIndex/content, default: vectorIndex ``` Example: Perform an IPv4 bench test using the default data/ip2region_v4.xdb and data/ipv4_source.txt files: ```bash node tests/bench.app.js --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` Example: Perform an IPv6 bench test using the default data/ip2region_v6.xdb and data/ipv6_source.txt files: ```bash node tests/bench.app.js --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` You can test the effects of the three different cache implementations by setting `cache-policy` to file/vectorIndex/content. @Note: Ensure the src file used for bench is the same source file used to generate the corresponding xdb file. ### Third-party Library Support: 1. [ts-ip2region2](https://github.com/Steven-Qiang/ts-ip2region2) - Based on the official C extension, providing higher execution efficiency than pure JS. ================================================ FILE: binding/javascript/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region javascript 查询客户端 # 使用方式 ### 安装 `ip2region.js` ```bash npm install ip2region.js --save ``` ### 关于查询 API 查询 API 的原型为： ```javascript // 通过字符串 IP 或者 parseIP 解析得到的二进制 IP (Buffer类型) 进行查询 search(ip: string | Buffer): string; ``` 如果查询出错会抛异常，查询成功则会返回字符的 `region` 信息，如果指定的 IP 查询不到则会返回空字符串 `""`。 ### 关于 IPv4 和 IPv6 该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询，使用方式如下： ```javascript import {IPv4, IPv6} from 'ip2region.js'; // 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 Version.IPv4 let dbPath = "../../data/ip2region_v4.xdb"; // 或者你的 ipv4 xdb 的路径 let version = IPv4; // 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件，IP版本指定为 Version.IPv6 let dbPath = "../../data/ip2region_v6.xdb"; // 或者你的 ipv6 xdb 路径 let version = IPv6; // dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致，不然查询执行的时候会报错 // 备注：以下演示直接使用 dbPath 和 version 变量 ``` ### 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 ```javascript import {verifyFromFile} from 'ip2region.js'; try { verifyFromFile(dbPath); } catch (e) { // 适用性验证失败！！！ // 当前查询客户端实现不适用于 dbPath 指定的 xdb 文件的查询. // 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 dbPath 的 Searcher 实现。 console.log(`binding is not applicable for xdb file '${dbPath}': ${e.message}`); return; } // 验证通过，当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作 ``` ### 完全基于文件的查询 ```javascript import {newWithFileOnly} from 'ip2region.js'; // 1，使用上述的 version 和 dbPath 创建完全基于文件的查询对象 let searcher; try { searcher = newWithFileOnly(version, dbPath); } catch(e) { console.log(`failed to newWithFileOnly: ${err.message}`); return; } // 2、查询，IPv4 或者 IPv6 的地址都是同一个接口 let ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 try { let region = await searcher.search(ip); console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`); } catch(e) { console.log(`${err.message}`); } // 3、关闭资源 searcher.close(); // 备注：每个线程需要单独创建一个独立的 Searcher 对象 ``` ### 缓存 `VectorIndex` 索引 我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。 ```javascript import {loadVectorIndexFromFile, newWithVectorIndex} from 'ip2region.js'; // 1、从 dbPath 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。 let vIndex; try { vIndex = loadVectorIndexFromFile(dbPath); } catch (e) { console.log(`failed to load vector index from ${dbPath}: ${e.message}`); return; } // 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。 let searcher; try { searcher = newWithVectorIndex(version, dbPath, vIndex); } catch(e) { console.log(`failed to newWithVectorIndex: ${err.message}`); return; } // 3、查询，IPv4 或者 IPv6 的地址都是同一个接口 let ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 try { let region = await searcher.search(ip); console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`); } catch(e) { console.log(`${err.message}`); } // 4、关闭资源 searcher.close(); // 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的只读 vIndex 缓存。 ``` ### 缓存整个 `xdb` 文件 我们也可以预先加载整个 xdb 文件的数据到内存，然后基于这个数据创建查询对象来实现完全基于内存的查询，类似之前的 memory search。 ```javascript import {loadContentFromFile, newWithBuffer} from 'ip2region.js'; // 1、从 dbPath 加载整个 xdb 到内存。 let cBuffer; try { cBuffer = loadContentFromFile(dbPath); } catch (e) { console.log(`failed to load content from ${dbPath}: ${e.message}`); return; } // 2、使用上述的 cBuff 创建一个完全基于内存的查询对象。 let searcher; try { searcher = newWithBuffer(version, cBuffer); } catch(e) { console.log(`failed to newWithBuffer: ${err.message}`); return; } // 3、查询，IPv4 或者 IPv6 的地址都是同一个接口 let ip = "1.2.3.4"; // ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 try { let region = await searcher.search(ip); console.log(`search(${ip}): {region: ${region}`); } catch(e) { console.log(`${err.message}`); } // 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher // searcher.close(); // 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。 ``` # 查询测试 可以通过 `node tests/search.app.js` 命令来测试查询： ```bash ➜ javascript git:(fr_javascript_ipv6) node tests/search.app.js usage: Usage node tests/search.app.js [command options] ip2region search script optional arguments: -h, --help show this help message and exit --db DB ip2region binary xdb file path --cache-policy CACHE_POLICY cache policy: file/vectorIndex/content, default: vectorIndex ``` 例如：使用默认的 data/ip2region_v4.xdb 文件进行 IPv4 的查询测试： ```bash ➜ javascript git:(fr_javascript_ipv6) ✗ node tests/search.app.js --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 657.035 μs} ip2region>> 113.118.113.77 {region: 中国|广东省|深圳市|电信|CN, ioCount: 2, took: 169.927 μs} ``` 例如：使用默认的 data/ip2region_v6.xdb 文件进行 IPv6 的查询测试： ```bash ➜ javascript git:(fr_javascript_ipv6) ✗ node tests/search.app.js --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 98.953 μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, ioCount: 13, took: 287.703 μs} ``` 输入 ip 即可进行查询测试，也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。 # bench 测试 可以通过 `node tests/bench.app.js` 命令来进行 bench 测试，一方面确保 `xdb` 文件没有错误，一方面可以评估查询性能： ```bash ➜ javascript git:(fr_javascript_ipv6) ✗ node tests/bench.app.js usage: Usage node tests/bench.app.js [command options] ip2region bench script optional arguments: -h, --help show this help message and exit --db DB ip2region binary xdb file path --src SRC source ip text file path --cache-policy CACHE_POLICY cache policy: file/vectorIndex/content, default: vectorIndex ``` 例如：通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 文件进行 IPv4 的 bench 测试： ```bash node tests/bench.app.js --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` 例如：通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 文件进行 IPv6 的 bench 测试： ```bash node tests/bench.app.js --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` 可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。 @Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。 ### 第三方库支持： 1. [ts-ip2region2](https://github.com/Steven-Qiang/ts-ip2region2) - 基于官方的 C 扩展，比纯 JS 有更高的运行效率。 ================================================ FILE: binding/javascript/index.d.ts ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // Type definitions for ip2region // @Author Lion export declare class Header { version: number; indexPolicy: number; createdAt: number; startIndexPtr: number; endIndexPtr: number; ipVersion: number; runtimePtrBytes: number; buff: Buffer; constructor(buff: Buffer); toString(): string; } export declare class Version { id: number; name: string; bytes: number; indexSize: number; ipCompareFunc: (ip1: Buffer, ip2: Buffer, offset: number) => number; constructor(id: number, name: string, bytes: number, indexSize: number, ipCompareFunc: (ip1: Buffer, ip2: Buffer, offset: number) => number); ipCompare(ip1: Buffer, ip2: Buffer): number; ipSubCompare(ip1: Buffer, ip2: Buffer, offset: number): number; toString(): string; } export declare class Searcher { ioCount: number; version: Version; handle: number | null; vectorIndex: Buffer | null; cBuffer: Buffer | null; constructor(version: Version, dbPath: string | null, vectorIndex: Buffer | null, cBuffer: Buffer | null); getIPVersion(): Version; getIOCount(): number; search(ip: string | Buffer): string; read(offset: number, buff: Buffer, stats?: any): void; close(): void; } // Constants export declare const XdbStructure20: 2; export declare const XdbStructure30: 3; export declare const XdbIPv4Id: 4; export declare const XdbIPv6Id: 6; export declare const HeaderInfoLength: 256; export declare const VectorIndexRows: 256; export declare const VectorIndexCols: 256; export declare const VectorIndexSize: 8; // Version instances export declare const IPv4: Version; export declare const IPv6: Version; // Utility functions export declare function parseIP(ipString: string): Buffer; export declare function ipToString(ipBytes: Buffer, compress?: boolean): string; export declare function ipBytesString(ipBytes: Buffer): string; export declare function ipSubCompare(ip1: Buffer, buff: Buffer, offset: number): number; export declare function ipCompare(ip1: Buffer, ip2: Buffer): number; export declare function versionFromName(name: string): Version | null; export declare function versionFromHeader(h: Header): Version | null; // File operations export declare function loadHeader(fd: number): Header; export declare function loadHeaderFromFile(dbPath: string): Header; export declare function loadVectorIndex(fd: number): Buffer; export declare function loadVectorIndexFromFile(dbPath: string): Buffer; export declare function loadContent(fd: number): Buffer; export declare function loadContentFromFile(dbPath: string): Buffer; // Searcher factory functions export declare function newWithFileOnly(version: Version, dbPath: string): Searcher; export declare function newWithVectorIndex(version: Version, dbPath: string, vectorIndex: Buffer): Searcher; export declare function newWithBuffer(version: Version, cBuffer: Buffer): Searcher; ================================================ FILE: binding/javascript/index.js ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // ip2region JavaScript binding with IPv4 and IPv6 support. // @Author Lion export { Header, Version, IPv4, IPv6, XdbStructure20, XdbStructure30, XdbIPv4Id, XdbIPv6Id, HeaderInfoLength, VectorIndexRows, VectorIndexCols, VectorIndexSize, parseIP, ipToString, ipBytesString, ipSubCompare, ipCompare, versionFromName, versionFromHeader, loadHeader, loadHeaderFromFile, loadVectorIndex, loadVectorIndexFromFile, loadContent, loadContentFromFile, verify, verifyFromFile } from './util.js'; export { Searcher, newWithFileOnly, newWithVectorIndex, newWithBuffer } from './searcher.js'; ================================================ FILE: binding/javascript/package.json ================================================ { "name": "ip2region.js", "version": "3.1.8", "description": "official javascript binding for ip2region with both IPv4 and IPv6 supported ", "type": "module", "main": "index.js", "types": "index.d.ts", "scripts": { "test": "NODE_OPTIONS='--experimental-vm-modules --no-warnings' jest" }, "repository": { "type": "git", "url": "git+https://github.com/lionsoul2014/ip2region.git" }, "keywords": [ "ip2region", "ip-address", "ip-region", "ip-location", "ip-lookup", "ip-search", "ipv4-address", "ipv4-region", "ipv4-location", "ipv4-lookup", "ipv4-search", "ipv6-address", "ipv6-region", "ipv6-location", "ipv6-lookup", "ipv6-search" ], "author": "lionsoul2014", "license": "ISC", "bugs": { "url": "https://github.com/lionsoul2014/ip2region/issues" }, "homepage": "https://github.com/lionsoul2014/ip2region#readme", "devDependencies": { "@types/jest": "^30.0.0", "argparse": "^2.0.1", "jest": "^30.2.0", "n-readlines": "^1.0.1", "ts-jest": "^29.4.5", "typescript": "^5.9.3" } } ================================================ FILE: binding/javascript/searcher.js ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // searcher implementation // @Author Lion import fs from 'fs'; import { parseIP, HeaderInfoLength, VectorIndexCols, VectorIndexSize, ipToString } from './util.js'; export class Searcher { constructor(version, dbPath, vectorIndex, cBuffer) { this.ioCount = 0; this.dbPath = dbPath; this.version = version; if (cBuffer != null) { this.handle = null; this.vectorIndex = null; this.cBuffer = cBuffer; } else { this.handle = fs.openSync(dbPath, 'r'); this.vectorIndex = vectorIndex; this.cBuffer = null; } } getIPVersion() { return this.version; } getIOCount() { return this.ioCount; } async search(ip) { // check and parse the string ip const ipBytes = Buffer.isBuffer(ip) ? ip : parseIP(ip); // ip version check if (ipBytes.length != this.version.bytes) { throw new Error(`invalid ip address '${ipToString(ipBytes)}' (${this.version.name} expected)`); } // reset the global counter this.ioCount = 0; // located the segment index block based on the vector index let sPtr = 0, ePtr = 0; let il0 = ipBytes[0], il1 = ipBytes[1]; let idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; if (this.vectorIndex != null) { sPtr = this.vectorIndex.readUint32LE(idx); ePtr = this.vectorIndex.readUint32LE(idx + 4); } else if (this.cBuffer != null) { sPtr = this.cBuffer.readUint32LE(HeaderInfoLength + idx); ePtr = this.cBuffer.readUint32LE(HeaderInfoLength + idx + 4); } else { const buff = Buffer.alloc(VectorIndexSize); this.read(HeaderInfoLength + idx, buff); sPtr = buff.readUInt32LE(0); ePtr = buff.readUInt32LE(4); } // console.log(`sPtr: ${sPtr}, ePtr: ${ePtr}`); // @Note: ptr validate, zero ptr means source data missing // so we could just stop here and return an empty string. if (sPtr == 0 || ePtr == 0) { return ""; } // binary search the segment index block to get the region info const bytes = ipBytes.length, dBytes = ipBytes.length << 1; const indexSize = this.version.indexSize; const buff = Buffer.alloc(indexSize); let dLen = 0, dPtr = 0, l = 0, h = (ePtr - sPtr) / indexSize; while (l <= h) { const m = (l + h) >> 1; const p = sPtr + m * indexSize; // read the segment index block this.read(p, buff); if (this.version.ipSubCompare(ipBytes, buff, 0) < 0) { h = m - 1; } else if (this.version.ipSubCompare(ipBytes, buff, bytes) > 0) { l = m + 1; } else { dLen = buff.readUint16LE(dBytes); dPtr = buff.readUint32LE(dBytes + 2); break; } } // empty match interception. // and this could be a case. if (dLen == 0) { return ""; } // console.log(`dLen: ${dLen}, dPtr: ${dPtr}`); const region = Buffer.alloc(dLen); this.read(dPtr, region); return region.toString('utf-8'); } read(offset, buff, stats) { // check the in-memory buffer first if (this.cBuffer != null) { this.cBuffer.copy(buff, 0, offset, offset + buff.length); return; } // increase the io counts this.ioCount++; // read the data let rBytes = fs.readSync(this.handle, buff, 0, buff.length, offset); if (rBytes != buff.length) { throw new Error(`incomplete read: read bytes should be ${buff.length}`); } } // close the searcher close() { if (this.handle != null) { fs.close(this.handle); } } toString() { const vn = this.version.name; const vi = this.vectorIndex == null ? 'null' : this.vectorIndex.length; const cf = this.cBuffer == null ? 'null' : this.cBuffer.length; return `{"version": ${vn}, "dbPath": ${this.dbPath}, "handle": ${this.handle}, "vectorIndex": ${vi} "cBuffer": ${cf}}`; } } export function newWithFileOnly(version, dbPath) { return new Searcher(version, dbPath, null, null); } export function newWithVectorIndex(version, dbPath, vectorIndex) { return new Searcher(version, dbPath, vectorIndex, null); } export function newWithBuffer(version, cBuffer) { return new Searcher(version, null, null, cBuffer); } ================================================ FILE: binding/javascript/tests/bench.app.js ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // app to do the xdb bench // @Author Lion import * as xdb from '../index.js'; import {ArgumentParser} from 'argparse'; import LineByLine from 'n-readlines'; import fs from 'fs'; const parser = new ArgumentParser({ add_help: true, description: 'ip2region bench script', prog: 'node tests/bench.app.js', usage: 'Usage %(prog)s [command options]' }); parser.add_argument('--db', {help: 'ip2region binary xdb file path'}); parser.add_argument('--src', {help: 'source ip text file path'}); parser.add_argument('--cache-policy', {help: 'cache policy: file/vectorIndex/content, default: vectorIndex'}); const args = parser.parse_args(); const dbPath = args.db || ''; const srcPath = args.src || ''; const cachePolicy = args.cache_policy || 'vectorIndex'; // create the searcher const createSearcher = () => { const handle = fs.openSync(dbPath, 'r'); // verify the xdb file // @Note: do NOT call it every time you create a searcher since this will slow // down the search response. // @see the verify function for details. xdb.verify(handle); // get the ip version from the header const version = xdb.versionFromHeader(xdb.loadHeader(handle)); let searcher = null; switch(cachePolicy) { case 'file': searcher = xdb.newWithFileOnly(version, dbPath); break; case 'vectorIndex': const vIndex = xdb.loadVectorIndexFromFile(dbPath); searcher = xdb.newWithVectorIndex(version, dbPath, vIndex); break; case 'content': const cBuffer = xdb.loadContentFromFile(dbPath); searcher = xdb.newWithBuffer(version, cBuffer); break; default: fs.closeSync(handle); throw new Error(`invalid cache-policy '${cachePolicy}'`); } fs.closeSync(handle); return searcher; } const _split = (line) => { const ps = []; const s1 = line.indexOf('|'); if (s1 === -1) { ps.push(line); return ps; } ps.push(line.substring(0, s1)); const s2 = line.indexOf('|', s1 + 1); if (s2 === -1) { ps.push(line.substring(s1+1)); return ps; } ps.push(line.substring(s1 + 1, s2)); ps.push(line.substring(s2 + 1)); return ps; } const main = async () => { if (dbPath.length < 1 || srcPath.length < 1) { parser.print_help(); return; } const searcher = createSearcher(); console.log(`Searcher: ${searcher.toString()}`); // read the source line and do the search bench let totalMicroSecs = 0, count = 0, line = null; const rl = new LineByLine(srcPath); while (line = rl.next()) { const ps = _split(line.toString('utf-8')); const sTime = process.hrtime(); const sip = xdb.parseIP(ps[0]); const eip = xdb.parseIP(ps[1]); if (xdb.ipCompare(sip, eip) > 0) { throw new Error(`start ip(${ps[0]}) should not be greater than end ip(${ps[1]})`); } const test_list = [sip, eip]; for (let i = 0; i < test_list.length; i++) { const region = await searcher.search(test_list[i]); if (region != ps[2]) { throw new Error(`failed to search(${xdb.ipToString(test_list[i])}) with (${region} != ${ps[2]})`); } count++; } const diff = process.hrtime(sTime); const took = diff[0] * 1_000_000 + diff[1] / 1e3; totalMicroSecs += took; } const tookSec = totalMicroSecs / 1e6; const _eachUs = count == 0 ? 0 : totalMicroSecs / count; console.log(`Bench finished, {cachePolicy: ${cachePolicy}, total: ${count}, took: ${tookSec} s, cost: ${_eachUs} μs/op}`); } main(); ================================================ FILE: binding/javascript/tests/search.app.js ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // app to do the xdb search // @Author Lion import * as xdb from '../index.js'; import {ArgumentParser} from 'argparse'; import fs from 'fs'; const parser = new ArgumentParser({ add_help: true, description: 'ip2region search script', prog: 'node tests/search.app.js', usage: 'Usage %(prog)s [command options]' }); parser.add_argument('--db', {help: 'ip2region binary xdb file path'}); parser.add_argument('--cache-policy', {help: 'cache policy: file/vectorIndex/content, default: vectorIndex'}); const args = parser.parse_args(); const dbPath = args.db || ''; const cachePolicy = args.cache_policy || 'vectorIndex'; // create the searcher const createSearcher = () => { const handle = fs.openSync(dbPath, 'r'); // verify the xdb file // @Note: do NOT call it every time you create a searcher since this will slow // down the search response. // @see the verify function for details. xdb.verify(handle); // get the ip version from the header const version = xdb.versionFromHeader(xdb.loadHeader(handle)); let searcher = null; switch(cachePolicy) { case 'file': searcher = xdb.newWithFileOnly(version, dbPath); break; case 'vectorIndex': const vIndex = xdb.loadVectorIndexFromFile(dbPath); searcher = xdb.newWithVectorIndex(version, dbPath, vIndex); break; case 'content': const cBuffer = xdb.loadContentFromFile(dbPath); searcher = xdb.newWithBuffer(version, cBuffer); break; default: fs.closeSync(handle); throw new Error(`invalid cache-policy '${cachePolicy}'`); } fs.closeSync(handle); return searcher; } const readlineSync = () => { return new Promise((resolve, reject) => { process.stdin.resume(); process.stdin.on('data', (buff) => { process.stdin.pause(); resolve(buff.toString('utf-8')); }); }); } const main = async () => { if (dbPath.length < 1) { parser.print_help(); return; } const searcher = createSearcher(); console.log(`ip2region xdb searcher test program source xdb: ${dbPath} (${searcher.getIPVersion().name}, ${cachePolicy}) type 'quit' to exit`); while (true) { process.stdout.write('ip2region>> '); // get the input ip const ipString = (await readlineSync()).trim(); if (ipString.length == 0) { continue; } if (ipString == 'quit') { break; } // parse the ip address let ipBytes = null; try { ipBytes = xdb.parseIP(ipString); } catch (e) { console.log(`failed to parse ip: ${e.message}`); continue; } // do the search const sTime = process.hrtime(); let region = null; try { region = await searcher.search(ipBytes); } catch (e) { console.log(`{err: ${e.message}, ioCount: ${searcher.getIOCount()}}`); continue; } const diff = process.hrtime(sTime); const took = diff[0] * 1_000_000 + diff[1] / 1e3; console.log(`{region: ${region}, ioCount: ${searcher.getIOCount()}, took: ${took} μs}`); } } main(); ================================================ FILE: binding/javascript/tests/searcher.test.js ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // searcher new tester // @Author Lion import path from 'path'; import { fileURLToPath } from 'url'; import { IPv4, IPv6, XdbIPv4Id, parseIP, ipToString, verifyFromFile, loadVectorIndexFromFile, loadContentFromFile, newWithFileOnly, newWithVectorIndex, newWithBuffer } from '../index.js'; import { fail } from 'assert'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const dbPath = { v4: path.join(__dirname, '..', '..', '..', 'data', 'ip2region_v4.xdb'), v6: path.join(__dirname, '..', '..', '..', 'data', 'ip2region_v6.xdb') } test('xdb file verify', () => { // verify the xdb file // @Note: do NOT call it every time you create a searcher since this will slow // down the search response. // @see the verify function for details. for (k in dbPath) { if (!dbPath.hasOwnProperty(k)) { continue; } try { verifyFromFile(dbPath[k]); console.log(`xdb file '${dbPath[k]}' verified`); } catch (e) { throw new Error(`binding is not applicable for xdb file '${dbPath[k]}': ${e.message}`); } } }); // --- // search api testing test('ipv4 search test', async () => { let searcher = newWithFileOnly(IPv4, dbPath.v4); let ip_list = [ '1.0.0.0', parseIP('113.118.112.93'), '240e:3b7::' ]; for (var i = 0; i < ip_list.length; i++) { let ip = ip_list[i]; searcher.search(ip).then((region)=>{ let ipStr = Buffer.isBuffer(ip) ? ipToString(ip) : ip; console.log(`search(${ipStr}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`); }).catch((err) => { console.log(`${err.message}`); }); } // close searcher searcher.close(); }); test('ipv6 search test', async () => { let searcher = newWithFileOnly(IPv6, dbPath.v6); let ip_list = [ '2a02:26f7:c409:4001::', parseIP('2a11:8080:200::a:a05c'), '240e:3b7::', '120.229.45.92' ]; for (var i = 0; i < ip_list.length; i++) { let ip = ip_list[i]; searcher.search(ip).then((region)=>{ let ipStr = Buffer.isBuffer(ip) ? ipToString(ip) : ip; console.log(`search(${ipStr}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`); }).catch((err) => { console.log(`${err.message}`); }); } // close searcher searcher.close(); }); // --- // searcher with different cache policy testing function _get_creater_list(version) { let dbFile = version.id == XdbIPv4Id ? dbPath.v4 : dbPath.v6; return [function(){ return ["newWithFileOnly", newWithFileOnly(version, dbFile)]; }, function(){ const vIndex = loadVectorIndexFromFile(dbFile); return ["newWithVectorIndex", newWithVectorIndex(version, dbFile, vIndex)]; }, function(){ const cBuffer = loadContentFromFile(dbFile); return ["newWithBuffer", newWithBuffer(version, cBuffer)]; }]; } test('ipv4 searcher test', async () => { const ip_Str = '120.229.45.92'; const c_list = _get_creater_list(IPv4); try { let bRegion = null; for (var i = 0; i < c_list.length; i++) { const meta = c_list[i](); const region = await meta[1].search(ip_Str); if (bRegion != null) { expect(region).toBe(region); } bRegion = region; console.log(`${meta[0]}.search(${ip_Str}): ${region}`); // searcher close meta[1].close(); } } catch (e) { console.error(`${e.message}`); } }); test('ipv6 searcher test', async () => { const ip_Str = '240e:57f:32ff:ffff:ffff:ffff:ffff:ffff'; const c_list = _get_creater_list(IPv6); try { let bRegion = null; for (var i = 0; i < c_list.length; i++) { const meta = c_list[i](); const region = await meta[1].search(ip_Str); if (bRegion != null) { expect(region).toBe(region); } bRegion = region; console.log(`${meta[0]}.search(${ip_Str}): ${region}`); // searcher close meta[1].close(); } } catch (e) { console.error(`${e.message}`); } }); ================================================ FILE: binding/javascript/tests/util.test.js ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // util test script // @Author Lion import * as util from '../util.js'; import path from 'node:path'; import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const dbPath = path.join(__dirname, '..', '..', '..', 'data', 'ip2region_v4.xdb') test('const print', () => { console.log("IPv4: ", util.IPv4.toString()); console.log("IPv6: ", util.IPv6.toString()); }); test("test version from name", () => { let vs = ["v4", "ipv4", "v4x", "v6", "ipv6", "v6x"]; vs.forEach(ele => { let v = util.versionFromName(ele); if (v == null) { console.log(`invalid version name ${ele}`); return; } console.log(`versionFrom(${ele}): ${v.toString()}, id=${v.id}, name=${v.name}`); }); }); test("test version ip compare", () => { let ip_list = [ ["1.0.0.0", "0.0.1.2", 1], ["192.168.1.101", "192.168.1.90", 1], ["219.133.111.87", "114.114.114.114", 1], ["2000::", "2000:ffff:ffff:ffff:ffff:ffff:ffff:ffff", -1], ["2001:4:112::", "2001:4:112:ffff:ffff:ffff:ffff:ffff", -1], ["ffff::", "2001:4:ffff:ffff:ffff:ffff:ffff:ffff", 1] ]; ip_list.forEach(ips => { const ip1 = util.parseIP(ips[0]); const ip2 = util.parseIP(ips[1]); if (ip1.length != ip2.length) { fail(`ip1 and ip2 are not the same ip type`); } const version = ip1.length == 4 ? util.IPv4 : util.IPv6; const cmp = version.ipSubCompare(ip1, ip1.length == 4 ? ip2.reverse() : ip2, 0); expect(cmp).toBe(ips[2]); console.log(`compare(${ips[0]}, ${ips[1]}): ${cmp}`); }); }); test('parse ip address', () => { let ip_list = [ "1.0.0.0", "58.251.30.115", "192.168.1.100", "126.255.32.255", "219.xx.xx.11", "::", "::1", "fffe::", "2c0f:fff0::", "2c0f:fff0::1", "2a02:26f7:c409:4001::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "240e:982:e617:ffff:ffff:ffff:ffff:ffff", "::xx:ffff" ]; ip_list.forEach(ipString => { let ipBytes = null; try { ipBytes = util.parseIP(ipString); } catch (e) { console.log(`failed to parse ip '${ipString}': ${e.message}`); return; } let to_Str = util.ipToString(ipBytes, true); let toByte = util.ipBytesString(ipBytes); console.log(`parseIP(${ipString}): {Bytes: ${toByte}, String: ${to_Str}}`); expect(ipString).toBe(to_Str); }); }); test('ip compare', () => { let ip_list = [ ["1.0.0.0", "1.0.0.1", -1], ["192.168.1.101", "192.168.1.90", 1], ["219.133.111.87", "114.114.114.114", 1], ["2000::", "2000:ffff:ffff:ffff:ffff:ffff:ffff:ffff", -1], ["2001:4:112::", "2001:4:112:ffff:ffff:ffff:ffff:ffff", -1], ["ffff::", "2001:4:ffff:ffff:ffff:ffff:ffff:ffff", 1] ]; ip_list.forEach(ips => { const ip1 = util.parseIP(ips[0]); const ip2 = util.parseIP(ips[1]); const cmp = util.ipCompare(ip1, ip2); expect(cmp).toBe(ips[2]); console.log(`compare(${ips[0]}, ${ips[1]}): ${cmp}`); }); }); test('test load header', () => { let header = util.loadHeaderFromFile(dbPath); console.log(`dbPath: ${dbPath}, header: ${header.toString()}}`); }); test('test load vector index', () => { let vIndex = util.loadVectorIndexFromFile(dbPath); console.log(`dbPath: ${dbPath}, vIndex: ${vIndex.length}}`); }); test('test load content', () => { let content = util.loadContentFromFile(dbPath); console.log(`dbPath: ${dbPath}, content: ${content.length}}`); }); ================================================ FILE: binding/javascript/tsconfig.json ================================================ { "compilerOptions": { "target": "ES2020", "module": "ES2020", "moduleResolution": "node", "allowJs": true, "checkJs": false, "declaration": true, "outDir": "./dist", "strict": true, "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, "resolveJsonModule": true, "allowSyntheticDefaultImports": true }, "include": [ "*.js", "*.d.ts", "tests/**/*" ], "exclude": [ "node_modules", "dist" ] } ================================================ FILE: binding/javascript/util.js ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // util functions // @Author Lion import fs from 'fs'; export const XdbStructure20 = 2; export const XdbStructure30 = 3; export const XdbIPv4Id = 4; export const XdbIPv6Id = 6; export const HeaderInfoLength = 256; export const VectorIndexRows = 256; export const VectorIndexCols = 256; export const VectorIndexSize = 8; export class Header { constructor(buff) { this.version = buff.readUInt16LE(0); this.indexPolicy = buff.readUInt16LE(2); this.createdAt = buff.readUInt32LE(4); this.startIndexPtr = buff.readUInt32LE(8); this.endIndexPtr = buff.readUInt32LE(12); // since IPv6 supporting this.ipVersion = buff.readUInt16LE(16); this.runtimePtrBytes = buff.readUInt16LE(18); // keep the raw data this.buff = buff; } toString() { return `{ "version":${this.version}, "index_policy":${this.indexPolicy}, "start_index_ptr": ${this.startIndexPtr}, "end_index_ptr": ${this.endIndexPtr}, "ipVersion": ${this.ipVersion}, "runtime_ptr_bytes": ${this.runtimePtrBytes} }`; } } // --- // parse ipv4 address function _parse_ipv4_addr(v4String) { let ps = v4String.split('.', 4); if (ps.length != 4) { throw new Error('invalid ipv4 address'); } var v; const ipBytes = Buffer.alloc(4); for (var i = 0; i < ps.length; i++) { v = parseInt(ps[i], 10); if (isNaN(v)) { throw new Error(`invalid ipv4 part '${ps[i]}', a valid number expected`); } if (v < 0 || v > 255) { throw new Error(`invalid ipv4 part '${ps[i]}' should >= 0 and <= 255`); } ipBytes[i] = (v & 0xFF); } return ipBytes; } // parse ipv6 address function _parse_ipv6_addr(v6String) { let ps = v6String.split(':', 8); if (ps.length < 3) { throw new Error('invalid ipv6 address'); } let dc_num = 0, offset = 0; const ipBytes = Buffer.alloc(16); for (var i = 0; i < ps.length; i++) { let s = ps[i].trim(); // Double colon check and auto padding if (s.length == 0) { // ONLY one double colon allow if (dc_num > 0) { throw new Error('invalid ipv6 address: multi double colon detected'); } let start = i, mi = ps.length - 1; // clear all the consecutive spaces for (i++;;) { s = ps[i].trim(); if (s.length > 0) { i--; break; } if (i >= mi) { break; } i++; } dc_num = 1; // padding = 8 - start - left let padding = 8 - start - (mi - i); offset += 2 * padding; continue; } let v = parseInt(s, 16); if (isNaN(v)) { throw new Error(`invalid ipv6 part '${ps[i]}', a valid hex number expected`); } if (v < 0 || v > 0xFFFF) { throw new Error(`invalid ipv6 part '${ps[i]}' should >= 0 and <= 65534`); } ipBytes.writeUint16BE(v, offset); offset += 2; } return ipBytes; } // parse the specified string ip and return its bytes // @param ip string // @return Buffer export function parseIP(ipString) { let sDot = ipString.indexOf('.'); let cDot = ipString.indexOf(':'); if (sDot > -1 && cDot == -1) { return _parse_ipv4_addr(ipString); } else if (cDot > -1) { return _parse_ipv6_addr(ipString); } else { throw new Error(`invalid ip address '${ipString}'`); } } // --- // ipv4 bytes to string function _ipv4_to_string(v4Bytes) { return v4Bytes.join('.'); } // ipv6 bytes to string function _ipv6_to_string(v6Bytes, compress) { let ps = [], needCompress = false; let lastHex = -1, hex = 0; for (var i = 0; i < v6Bytes.length; i += 2) { hex = v6Bytes.readUint16BE(i).toString(16); ps.push(hex); // check the necessity for compress if (lastHex > -1 && hex == 0 && lastHex == 0) { needCompress = true; } // reset the last hex lastHex = hex; } if (needCompress == false || compress === false) { return ps.join(':'); } // auto compression of consecutive zero let _ = [], mi = ps.length - 1; for (i = 0; i < ps.length; i++) { if (i >= mi) { _.push(ps[i]); continue; } if (ps[i] != '0' || ps[i+1] != '0') { _.push(ps[i]); continue; } // find the first two zero part // and keep find all the zero part for (i += 2; i < ps.length;) { if (ps[i] != '0') { i--; break; } i++; } // make sure there is an empty head. if (_.length == 0) { _.push(''); } _.push(''); // empty for double colon // make sure there is an empty tail if (i == ps.length && _.length < ps.length) { _.push(''); } } return _.join(':'); } // bytes ip to humen-readable string ip export function ipToString(ipBytes, compress) { if (!Buffer.isBuffer(ipBytes)) { throw new Error('invalid bytes ip, not a Buffer'); } if (ipBytes.length == 4) { return _ipv4_to_string(ipBytes, compress); } else if (ipBytes.length == 16) { return _ipv6_to_string(ipBytes, compress); } else { throw new Error('invalid bytes ip with length not 4 or 16'); } } export function ipBytesString(ipBytes) { if (!Buffer.isBuffer(ipBytes)) { throw new Error('invalid bytes ip, not a Buffer'); } let ps = []; for (var i = 0; i < ipBytes.length; i++) { ps.push(ipBytes[i] & 0xFF); } return ps.join('.'); } // compare two byte ips // ip2 = buff[offset:ip1.length] // returns: -1 if ip1 < ip2, 1 if ip1 > ip2 or 0 export function ipSubCompare(ip1, buff, offset) { return ip1.compare(buff, offset, offset + ip1.length); } export function ipCompare(ip1, ip2) { return ipSubCompare(ip1, ip2, 0); } // --- export class Version { constructor(id, name, bytes, indexSize, ipCompareFunc) { this.id = id; this.name = name; this.bytes = bytes; this.indexSize = indexSize; this.ipCompareFunc = ipCompareFunc; } ipCompare(ip1, ip2) { return this.ipCompareFunc(ip1, ip2, 0); } ipSubCompare(ip1, ip2, offset) { return this.ipCompareFunc(ip1, ip2, offset); } toString() { return `{"id": ${this.id}, "name": "${this.name}", "bytes":${this.bytes}, "index_size": ${this.indexSize}}`; } } // 14 = 4 + 4 + 2 + 4 export const IPv4 = new Version(XdbIPv4Id, "IPv4", 4, 14, function(ip1, buff, offset){ // ip1: Big endian byte order parsed from input // ip2: Little endian byte order read from xdb index. // @Note: to compatible with the old Litten endian index encode implementation. let i, j = offset + ip1.length - 1; for (i = 0; i < ip1.length; i++, j--) { const i1 = ip1[i] & 0xFF; const i2 = buff[j] & 0xFF; if (i1 < i2) { return -1; } if (i1 > i2) { return 1; } } return 0; }); // 38 = 16 + 16 + 2 + 4 export const IPv6 = new Version(XdbIPv6Id, "IPv6", 16, 38, ipSubCompare); export function versionFromName(name) { let n = name.toUpperCase(); if (n == "V4" || n == "IPV4") { return IPv4; } else if (n == "V6" || n == "IPV6") { return IPv6; } else { return null; } } export function versionFromHeader(h) { // old structure with ONLY IPv4 supporting if (h.version == XdbStructure20) { return IPv4; } // structure 3.0 with IPv6 supporting if (h.version != XdbStructure30) { return null; } let ipVer = h.ipVersion; if (ipVer == XdbIPv4Id) { return IPv4; } else if (ipVer == XdbIPv6Id) { return IPv6; } else { return null; } } // --- export function loadHeader(fd) { const buffer = Buffer.alloc(HeaderInfoLength); const rBytes = fs.readSync(fd, buffer, 0, HeaderInfoLength, 0); if (rBytes != HeaderInfoLength) { throw new Error(`incomplete read (${rBytes} read, ${header.HeaderInfoLength} expected)`); } return new Header(buffer); } export function loadHeaderFromFile(dbPath) { const fd = fs.openSync(dbPath, "r"); const header = loadHeader(fd); fs.closeSync(fd); return header; } export function loadVectorIndex(fd) { const vBytes = VectorIndexCols * VectorIndexRows * VectorIndexSize; const buffer = Buffer.alloc(vBytes); const rBytes = fs.readSync(fd, buffer, 0, vBytes, HeaderInfoLength); if (rBytes != vBytes) { throw new Error(`incomplete read (${rBytes} read, ${vBytes} expected)`); } return buffer; } export function loadVectorIndexFromFile(dbPath) { const fd = fs.openSync(dbPath, "r"); const vIndex = loadVectorIndex(fd); fs.closeSync(fd); return vIndex; } export function loadContent(fd) { const stats = fs.fstatSync(fd); const buffer = Buffer.alloc(stats.size); const rBytes = fs.readSync(fd, buffer, 0, buffer.length, 0); if (rBytes != stats.size) { throw new Error(`incomplete read (${rBytes} read, ${stats.size} expected)`); } return buffer; } export function loadContentFromFile(dbPath) { const fd = fs.openSync(dbPath, "r"); const content = loadContent(fd); fs.closeSync(fd); return content; } // --- // Verify if the current Searcher could be used to search the specified xdb file. // Why do we need this check ? // The future features of the xdb impl may cause the current searcher not able to work properly. // // @Note: You Just need to check this ONCE when the service starts // Or use another process (eg, A command) to check once Just to confirm the suitability. export function verify(fd) { const header = loadHeader(fd); // get the runtime ptr bytes let runtimePtrBytes = 0; if (header.version == XdbStructure20) { runtimePtrBytes = 4; } else if (header.version == XdbStructure30) { runtimePtrBytes = header.runtimePtrBytes; } else { throw new Error(`invalid structure version ${header.version}`); } // 1, confirm the xdb file size // to ensure that the maximum file pointer does not overflow const maxFilePtr = (1n << BigInt(runtimePtrBytes * 8)) - 1n; const _fileBytes = BigInt(fs.fstatSync(fd).size); if (_fileBytes > maxFilePtr) { throw new Error(`xdb file exceeds the maximum supported bytes: ${maxFilePtr}`); } } export function verifyFromFile(dbPath) { const fd = fs.openSync(dbPath, "r"); verify(fd); fs.closeSync(fd); } ================================================ FILE: binding/lua/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region lua query client #### Note: Please prioritize the use of the lua_c extension query client, as its performance is much faster than the pure lua implementation!!! # Version Compatibility This implementation is compatible with lua `5.3` and `5.4`, and no longer provides compatibility for lower versions. If you need to use it under lower versions of Lua, please consider using the `[lua_c](../lua_c/)` extension. # Usage ### About Query API The prototype of the query API is as follows: ```lua -- Query via IP string search_by_string(ip_string) (region, error) -- Query via bytes IP returned by parse_ip search(ip_bytes) (region, error) ``` If the query fails, it returns a non-`nil` error string. If successful, it returns the `region` information as a string. If the IP address cannot be found, it returns an empty string `""`. ### About IPv4 and IPv6 ```lua local xdb = require("xdb_searcher") -- For IPv4: Set xdb path to the v4 xdb file, specify IP version as Version.IPv4 local dbPath = "../../data/ip2region_v4.xdb" -- or your ipv4 xdb path local version = xdb.IPv4 -- For IPv6: Set xdb path to the v6 xdb file, specify IP version as Version.IPv6 local dbPath = "../../data/ip2region_v6.xdb" -- or your ipv6 xdb path local version = xdb.IPv6 -- The IP version of the xdb specified by dbPath must match the version specified, otherwise an error will occur during query execution -- Note: The following demonstration directly uses the dbPath and version variables ``` ### File Verification It is recommended that you actively verify the suitability of the xdb file, as some new features in the future may cause the current Searcher version to be incompatible with the xdb file you are using. Verification can avoid unpredictable errors during runtime. You don't need to verify every time; for example, verify when the service starts or by manually calling the verification command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios. ```lua local xdb = require('xdb_searcher') local err = xdb.verify(dbPath); if err ~= nil then -- Suitability verification failed!!! -- The current query client implementation is not suitable for the xdb file specified by dbPath. -- You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with dbPath. print(string.format("binding is not applicable for xdb file '%s': %s", dbPath, err)) return end -- Verification passed, the current Searcher can safely be used for query operations on the xdb pointed to by dbPath ``` ### Entirely File-Based Query ```lua local xdb = require("xdb_searcher") -- 1. Create an entirely file-based query object using the version and dbPath mentioned above local searcher, err = xdb.new_with_file_only(version, db_path) if err ~= nil then print(string.format("failed to create searcher: %s", err)) return end -- 2. Call the query API; with both IPv4 and IPv6 addresses supported local ip_str = "1.2.3.4" -- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6 local s_time = xdb.now() region, err = searcher:search_by_string(ip_str) if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time)) -- 3. Close resources searcher:close() -- Note: For concurrent use, each coroutine needs to create a separate xdb query object ``` ### Caching `VectorIndex` If supported by your `lua` environment, you can pre-load the vectorIndex cache and make it a global variable. Using the global vectorIndex every time a Searcher is created can reduce one fixed IO operation, thereby accelerating queries and reducing IO pressure. ```lua local xdb = require("xdb_searcher") -- 1. Load vectorIndex cache from the specified db_path and make the v_index object below a global variable. -- vectorIndex only needs to be loaded once; it is recommended to load it as a global object when the service starts. v_index, err = xdb.load_vector_index(dbPath) if err ~= nil then print(string.format("failed to load vector index from '%s'", db_path)) return end -- 2. Use the global v_index to create a query object with vectorIndex cache. searcher, err = xdb.new_with_vector_index(version, dbPath, v_index) if err ~= nil then print(string.format("failed to create vector index searcher: %s", err)) return end -- 3. Call the query API; the same interface is used for both IPv4 and IPv6 local ip_str = "1.2.3.4" -- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6 local s_time = xdb.now() region, err = searcher:search_by_string(ip_str) if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time)) -- 4. Close resources searcher:close() -- Note: For concurrent use, each coroutine needs to create a separate xdb query object, but they share the global v_index object ``` ### Caching the Entire `xdb` File If supported by your `lua` environment, you can pre-load the entire xdb data into memory to achieve completely memory-based queries, similar to the previous memory search. ```lua local xdb = require("xdb_searcher") -- 1. Load the entire xdb into memory from the specified dbPath. -- xdb content only needs to be loaded once; it is recommended to load it as a global object when the service starts. content, err = xdb.load_content(dbPath) if err ~= nil then print(string.format("failed to load xdb content: %s", err)) return end -- 2. Use the global content to create an entirely memory-based query object. searcher, err = xdb.new_with_buffer(version, content) if err ~= nil then print(string.format("failed to create content buffer searcher: %s", err)) return end -- 3. Call the query API; the same interface is used for both IPv4 and IPv6 local ip_str = "1.2.3.4" -- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6 local s_time = xdb.now() region, err = searcher:search_by_string(ip_str) if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time)) -- 4. Close resources - This searcher object can be safely used for concurrency; close the searcher only when the entire service is shut down -- searcher:close() -- Note: For concurrent use, query objects created with the entire xdb cache can be safely used concurrently. -- It is recommended to create a global searcher object when the service starts and then use it globally and concurrently. ``` # Query Testing Perform query tests via the `lua search_test.lua` script: ```bash ➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua lua search_test.lua [command options] options: --db string ip2region binary xdb file path --cache-policy string cache policy: file/vectorIndex/content ``` For example: using the default data/ip2region_v4.xdb file for IPv4 query testing: ```bash ➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 0μs} ip2region>> 113.118.113.77 {region: 中国|广东省|深圳市|电信|CN, io_count: 2, took: 0μs} ``` For example: using the default data/ip2region_v6.xdb file for IPv6 query testing: ```bash ➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 0μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, io_count: 13, took: 0μs} ``` Enter an IP to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the efficiency of the three different cache implementations. # Bench Testing Perform automatic bench testing via the `lua bench_test.lua` script. This ensures that the `xdb` file has no errors and tests average query performance through a large number of queries: ```bash ➜ lua git:(fr_lua_ipv6) ✗ lua bench_test.lua lua bench_test.lua [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` For example: perform IPv4 bench testing using default data/ip2region_v4.xdb and data/ipv4_source.txt files: ```bash lua bench_test.lua --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` For example: perform IPv6 bench testing using default data/ip2region_v6.xdb and data/ipv6_source.txt files: ```bash lua bench_test.lua --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` You can test the performance of the three different cache implementations (file/vectorIndex/content) by setting the `cache-policy` parameter. @Note: Please note that the src file used for the bench must be the same source file used to generate the corresponding xdb file. ================================================ FILE: binding/lua/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region lua 查询客户端 #### 备注：请优先使用 lua_c 扩展 xdb 查询客户端，性能比纯 lua 实现的要快很多！！！ # 版本兼容 该实现兼容 lua `5.3`, `5.4`，且不再提供更低版本的兼容，如果需要在更低版本的 Lua 下使用请考虑使用 `[lua_c](../lua_c/)` 扩展。 # 使用方式 ### 关于查询 API 查询 API 的原型如下： ```lua -- 通过字符串 IP 查询 search_by_string(ip_string) (region, error) -- 通过 parse_ip 返回的 bytes IP 查询 search(ip_bytes) (region, error) ``` 如果查询出错则会返回非 `nil` 的 error 字符串信息，如果查询成功则会返回字符串的 `region` 信息，如果查询的 IP 地址找不到则会返回空字符串 `""`。 ### 关于 IPv4 和 IPv6 ```lua local xdb = require("xdb_searcher") -- 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 Version.IPv4 local dbPath = "../../data/ip2region_v4.xdb" -- 或者你的 ipv4 xdb 的路径 local version = xdb.IPv4 -- 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件，IP版本指定为 Version.IPv6 local dbPath = "../../data/ip2region_v6.xdb" -- 或者你的 ipv6 xdb 路径 local version = xdb.IPv6 -- dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致，不然查询执行的时候会报错 -- 备注：以下演示直接使用 dbPath 和 version 变量 ``` ### 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 ```lua local xdb = require('xdb_searcher') local err = xdb.verify(dbPath); if err ~= nil then -- 适用性验证失败！！！ -- 当前查询客户端实现不适用于 dbPath 指定的 xdb 文件的查询. -- 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 dbPath 的 Searcher 实现。 print(string.format("binding is not applicable for xdb file '%s': %s", dbPath, err)) return end -- 验证通过，当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作 ``` ### 完全基于文件的查询 ```lua local xdb = require("xdb_searcher") -- 1，使用上述的 version 和 dbPath 创建完全基于文件的查询对象 local searcher, err = xdb.new_with_file_only(version, db_path) if err ~= nil then print(string.format("failed to create searcher: %s", err)) return end -- 2、调用查询 API 进行查询，IPv4 或者 IPv6 的地址都是同一个接口 local ip_str = "1.2.3.4" -- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6 local s_time = xdb.now() region, err = searcher:search_by_string(ip_str) if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time)) -- 3、关闭资源 searcher:close() -- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象 ``` ### 缓存 `VectorIndex` 索引 如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。 ```lua local xdb = require("xdb_searcher") -- 1、从指定的 db_path 加载 vectorIndex 缓存，把下述的 v_index 对象做成全局变量。 -- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。 v_index, err = xdb.load_vector_index(dbPath) if err ~= nil then print(string.format("failed to load vector index from '%s'", db_path)) return end -- 2、使用全局的 v_index 创建带 vectorIndex 缓存的查询对象。 searcher, err = xdb.new_with_vector_index(version, dbPath, v_index) if err ~= nil then print(string.format("failed to create vector index searcher: %s", err)) return end -- 3、调用查询 API，IPv4 或者 IPv6 都是同一个接口 local ip_str = "1.2.3.4" -- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6 local s_time = xdb.now() region, err = searcher:search_by_string(ip_str) if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time)) -- 4、关闭资源 searcher:close() -- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象 ``` ### 缓存整个 `xdb` 文件 如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。 ```lua local xdb = require("xdb_searcher") -- 1、从上述的 dbPath 加载整个 xdb 到内存。 -- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。 content, err = xdb.load_content(dbPath) if err ~= nil then print(string.format("failed to load xdb content: %s", err)) return end -- 2、使用全局的 content 创建带完全基于内存的查询对象。 searcher, err = xdb.new_with_buffer(version, content) if err ~= nil then print(string.format("failed to create content buffer searcher: %s", err)) return end -- 3、调用查询 API，IPv4 或者 IPv6 都是同一个接口 local ip_str = "1.2.3.4" -- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6 local s_time = xdb.now() region, err = searcher:search_by_string(ip_str) if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time)) -- 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher -- searcher:close() -- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。 -- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。 ``` # 查询测试 通过 `lua search_test.lua` 脚本来进行查询测试： ```bash ➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua lua search_test.lua [command options] options: --db string ip2region binary xdb file path --cache-policy string cache policy: file/vectorIndex/content ``` 例如：使用默认的 data/ip2region_v4.xdb 文件进行 IPv4 的查询测试： ```bash ➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 0μs} ip2region>> 113.118.113.77 {region: 中国|广东省|深圳市|电信|CN, io_count: 2, took: 0μs} ``` 例如：使用默认的 data/ip2region_v6.xdb 文件进行 IPv6 的查询测试： ```bash ➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 0μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, io_count: 13, took: 0μs} ``` 输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。 # bench 测试 通过 `lua bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能： ```bash ➜ lua git:(fr_lua_ipv6) ✗ lua bench_test.lua lua bench_test.lua [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` 例如：通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 文件进行 IPv4 的 bench 测试： ```bash lua bench_test.lua --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` 例如：通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 文件进行 IPv6 的 bench 测试： ```bash lua bench_test.lua --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` 可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。 @Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。 ================================================ FILE: binding/lua/bench_test.lua ================================================ -- Copyright 2022 The Ip2Region Authors. All rights reserved. -- Use of this source code is governed by a Apache2.0-style -- license that can be found in the LICENSE file. -- -- --- -- @Author Lion -- @Date 2022/06/30 -- set the package to load the current xdb_searcher.so package.path = "./?.lua" .. package.path package.cpath = "./?.so" .. package.cpath local xdb = require("xdb_searcher") function printHelp() print("lua bench_test.lua [command options]") print("options: ") print(" --db string ip2region binary xdb file path") print(" --src string source ip text file path") print(" --cache-policy string cache policy: file/vectorIndex/content") end if #arg < 2 then printHelp() return end -- parser the command line args local dbFile, srcFile = "", "" local cachePolicy = "vectorIndex" for _, r in ipairs(arg) do if string.len(r) < 5 then goto continue end if string.sub(r, 1, 2) ~= "--" then goto continue end for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do if k == "db" then dbFile = v elseif k == "src" then srcFile = v elseif k == "cache-policy" then cachePolicy = v else print(string.format("undefined option `%s`", r)) return end -- break the match iterate break end -- continue this loop ::continue:: end -- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy)) if string.len(dbFile) < 2 or string.len(srcFile) < 2 then printHelp() return end -- open the dbFile local handle, closer, err = xdb.open_file(dbFile, "rb") if err ~= nil then print(string.format("failed to open %s: %s", dbFile, err)) return end -- verify the xdb err = xdb.verify(handle) if err ~= nil then closer() print(string.format("verify(%s): %s", dbFile, err)) return end -- load the header and define the ip version local header, err = xdb.load_header(handle) if err ~= nil then closer() print(string.format("failed to load the header: %s", err)) return end local version, err = xdb.version_from_header(header) if err ~= nil then closer() print(string.format("failed to detect version from header: %s", err)) return end -- file close closer("bench_test") -- create the searcher based on the cache-policy local searcher, v_index, content if cachePolicy == "file" then searcher, err = xdb.new_with_file_only(version, dbFile) if err ~= nil then print(string.format("failed to create searcher: %s", err)) return end elseif cachePolicy == "vectorIndex" then v_index, err = xdb.load_vector_index(dbFile) if err ~= nil then print(string.format("failed to load vector index: %s", err)) return end searcher, err = xdb.new_with_vector_index(version, dbFile, v_index) if err ~= nil then print(string.format("failed to create vector index searcher: %s", err)) return end elseif cachePolicy == "content" then content, err = xdb.load_content(dbFile) if err ~= nil then print(string.format("failed to load xdb content: %s", err)) return end searcher, err = xdb.new_with_buffer(version, content) if err ~= nil then print(string.format("failed to create content buffer searcher: %s", err)) return end else print(string.format("undefined cache-policy `%s`", cachePolicy)) return end -- do the bench test local handle = io.open(srcFile, "r") if handle == nil then print(string.format("failed to open src text file `%s`", handle)) return end local lines = handle:lines() local sip_str, eip_str, s_region, region = "", "", "", nil local sip, eip, err = nil, nil, nil local count, c_time = 0, 0 local s_time = xdb.now() for l in lines do if string.len(l) < 1 then goto continue end for v1, v2, v3 in string.gmatch(l, "([^|]+)|([^|]+)|([^\n]+)") do sip_str = v1 eip_str = v2 s_region = v3 break end -- print('sip', sip_str, 'eip', eip_str, 'region', s_region) sip, err = xdb.parse_ip(sip_str) if err ~= nil then print(string.format("invalid start ip `%s`", sip_str)) return end eip, err = xdb.parse_ip(eip_str) if err ~= nil then print(string.format("invalid end ip `%s`", sip_str)) return end if xdb.ip_compare(sip, eip) > 0 then print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str)) return end local t_time = xdb.now() for _, ip in ipairs({sip, eip}) do region, err = searcher:search(ip) if err ~= nil then print(string.format("failed to search ip `%s`", xdb.ip_to_string(ip))) return end -- check the region if region ~= s_region then print(string.format("failed search(%s) with (%s != %s)\n", xdb.ip_to_string(ip), region, s_region)) return end count = count + 1 end c_time = c_time + xdb.now() - t_time ::continue:: end -- resource cleanup searcher:close() -- print the stats local total_costs = (xdb.now() - s_time) / 1e6 local avg_costs = 0 if count > 0 then avg_costs = c_time / count end print(string.format( "Bench finished, {cachePolicy: %s, total: %d, took: {total: %0.3fs, search: %0.3fs}, cost: %.3f μs/op}", cachePolicy, count, total_costs, c_time/1e6, avg_costs )) ================================================ FILE: binding/lua/search_test.lua ================================================ -- Copyright 2022 The Ip2Region Authors. All rights reserved. -- Use of this source code is governed by a Apache2.0-style -- license that can be found in the LICENSE file. -- -- --- -- @Author Lion -- @Date 2022/06/30 -- set the package to load the current xdb_searcher.so package.path = "./?.lua" .. package.path package.cpath = "./?.so" .. package.cpath local xdb = require("xdb_searcher") function printHelp() print("lua search_test.lua [command options]") print("options: ") print(" --db string ip2region binary xdb file path") print(" --cache-policy string cache policy: file/vectorIndex/content") end if #arg < 1 then printHelp() return end -- parser the command line args local dbFile = "" local cachePolicy = "vectorIndex" for _, r in ipairs(arg) do if string.len(r) < 5 then goto continue end if string.sub(r, 1, 2) ~= "--" then goto continue end for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do if k == "db" then dbFile = v elseif k == "cache-policy" then cachePolicy = v else print(string.format("undefined option `%s`", r)) return end -- break the match iterate break end -- continue this loop ::continue:: end -- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy)) if string.len(dbFile) < 2 then printHelp() return end -- open the dbFile local handle, closer, err = xdb.open_file(dbFile, "rb") if err ~= nil then print(string.format("failed to open %s: %s", dbFile, err)) return end -- verify the xdb err = xdb.verify(handle) if err ~= nil then closer("xdb_verify") print(string.format("verify(%s): %s", dbFile, err)) return end -- load the header and define the ip version local header, err = xdb.load_header(handle) if err ~= nil then closer() print(string.format("failed to load the header: %s", err)) return end local version, err = xdb.version_from_header(header) if err ~= nil then closer() print(string.format("failed to detect version from header: %s", err)) return end -- file close closer("search_test") -- create the searcher based on the cache-policy local searcher, v_index, content if cachePolicy == "file" then searcher, err = xdb.new_with_file_only(version, dbFile) if err ~= nil then print(string.format("failed to create searcher: %s", err)) return end elseif cachePolicy == "vectorIndex" then v_index, err = xdb.load_vector_index(dbFile) if err ~= nil then print(string.format("failed to load vector index: %s", err)) return end searcher, err = xdb.new_with_vector_index(version, dbFile, v_index) if err ~= nil then print(string.format("failed to create vector index searcher: %s", err)) return end elseif cachePolicy == "content" then content, err = xdb.load_content(dbFile) if err ~= nil then print(string.format("failed to load xdb content: %s", err)) return end searcher, err = xdb.new_with_buffer(version, content) if err ~= nil then print(string.format("failed to create content buffer searcher: %s", err)) return end else print(string.format("undefined cache-policy `%s`", cachePolicy)) return end -- do the search print(string.format([[ ip2region xdb searcher test program source xdb: %s (%s, %s) type 'quit' to exit]], dbFile, version.name, cachePolicy)) local region, err = nil, nil local ip_bytes, s_time, c_time = nil, 0, 0 while true do io.write("ip2region>> "); io.input(io.stdin); local line = io.read(); if line == nil then break end if #line < 1 then goto continue end if line == "quit" then break end s_time = xdb.now() ip_bytes, err = xdb.parse_ip(line) if err ~= nil then print(string.format("failed to parse ip `%s`: %s", line, err)) goto continue end -- do the search region, err = searcher:search(ip_bytes) if err ~= nil then print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count())) else c_time = xdb.now() - s_time print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time)) end ::continue:: end -- resource cleanup searcher:close() ================================================ FILE: binding/lua/util_test.lua ================================================ -- Copyright 2022 The Ip2Region Authors. All rights reserved. -- Use of this source code is governed by a Apache2.0-style -- license that can be found in the LICENSE file. -- -- --- -- @Author Lion -- @Date 2022/07/05 package.path = "./?.lua" .. package.path package.cpath = "./?.so" .. package.cpath local xdb = require("xdb_searcher") function test_version_parse() print("IPv4", xdb.IPv4) print("IPv6", xdb.IPv6) -- version name parse local v_list = {"v4", "ipv4", "v4x", "v6", "ipv6", "v6x"} for _, name in ipairs(v_list) do local version, err = xdb.version_from_name(name) if err ~= nil then print(string.format("version_from_name(%s): %s", name, err)) else print(string.format("version_from_name(%s): %s", name, version)) end end end function test_parse_ip() local ip_list = { "1.0.0.0", "58.251.30.115", "192.168.1.100", "126.255.32.255", "219.xx.xx.11", "::", "::1", "fffe::", "2c0f:fff0::", "2c0f:fff0::1", "2a02:26f7:c409:4001::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "240e:982:e617:ffff:ffff:ffff:ffff:ffff", "::xx:ffff" } for _, ip_str in ipairs(ip_list) do ip_bytes, err = xdb.parse_ip(ip_str) if err ~= nil then print(string.format("failed to parse ip address `%s`: %s", ip_str, err)) else local ip_to_str = xdb.ip_to_string(ip_bytes) print(string.format( "parse_ip(`%s`)->{bytes:%d, to_string:%s, equal:%s}", ip_str, #ip_bytes, ip_to_str, tostring(ip_str == ip_to_str) )) end end end function test_ip_compare() local ip_list = { {"1.0.0.0", "1.0.0.1", -1}, {"192.168.1.101", "192.168.1.90", 1}, {"219.133.111.87", "114.114.114.114", 1}, {"2000::", "2000:ffff:ffff:ffff:ffff:ffff:ffff:ffff", -1}, {"2001:4:112::", "2001:4:112:ffff:ffff:ffff:ffff:ffff", -1}, {"ffff::", "2001:4:ffff:ffff:ffff:ffff:ffff:ffff", 1} } for _,ip_pair in ipairs(ip_list) do local ip1 = xdb.parse_ip(ip_pair[1]) local ip2 = xdb.parse_ip(ip_pair[2]) local cmp = xdb.ip_compare(ip1, ip2) print(string.format("compare(%s, %s): %d ? %s", ip_pair[1], ip_pair[2], cmp, tostring(cmp == ip_pair[3]))) end end ---- buffer loading test function test_load_header() header, err = xdb.load_header("../../data/ip2region_v4.xdb") if err ~= nil then print("failed to load header: ", err) else print("xdb header buffer loaded") local tpl = [[ header: { version: %d index_policy: %d created_at: %d start_index_ptr: %d end_index_ptr: %d ip_version: %d runtime_ptr_bytes: %d }]] print(string.format(tpl, header["version"], header["index_policy"], header["created_at"], header["start_index_ptr"], header["end_index_ptr"], header["ip_version"], header["runtime_ptr_bytes"] )) end end function test_load_vector_index() v_index, err = xdb.load_vector_index("../../data/ip2region_v4.xdb") if err ~= nil then print("failed to load vector index: ", err) else print("xdb vector index buffer loaded, length=", #v_index) end end function test_load_content() c_buffer, err = xdb.load_content("../../data/ip2region_v4.xdb") if err ~= nil then print("failed to load content: ", err) else print("xdb content buffer loaded, length=", #c_buffer) end end function test_verify() local xdb_files = { "../../data/ip2region_v4.xdb", "../../data/ip2region_v6.xdb" } for _, path in ipairs(xdb_files) do local err = xdb.verify(path) if err ~= nil then print(string.format("verify(%s): %s", path, err)) else print(string.format("verify(%s): Ok", path)) end end end function test_ip_search() local test_list = { -- ipv4 {"1.2.3.4", xdb.IPv4, "../../data/ip2region_v4.xdb"}, -- ipv6 {"240e:3b7:3272:d8d0:db09:c067:8d59:539e", xdb.IPv6, "../../data/ip2region_v6.xdb"} } for _, test in ipairs(test_list) do -- ipv6 local ip_str = test[1] searcher, err = xdb.new_with_file_only(test[2], test[3]) t_start = xdb.now() region, err = searcher:search_by_string(ip_str) if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) else local c_time = xdb.now() - t_start print(string.format( "search(%s): {region:%s, io_count:%d, took:%dμs}", ip_str, region, searcher:get_io_count(), c_time )) end searcher:close() end end -- check and call the function local func_name = arg[1] if func_name == nil then print("please specified the function to test") return end if (_G[func_name] == nil) then print(string.format("undefined function `%s` to call", func_name)) return end local s_time = xdb.now(); print(string.format("+---calling test function %s ...", func_name)) _G[func_name](); local cost_time = xdb.now() - s_time print(string.format("|---done, elapsed %.3fμs", cost_time)) ================================================ FILE: binding/lua/xdb_searcher.lua ================================================ -- Copyright 2022 The Ip2Region Authors. All rights reserved. -- Use of this source code is governed by a Apache2.0-style -- license that can be found in the LICENSE file. -- -- --- -- @Author Lion -- @Date 2022/07/05 -- constants define local header_info_length = 256 local vector_index_rows = 256 local vector_index_cols = 256 local vector_index_size = 8 local vector_index_length = 524288 -- cols x rows * 8 local xdb_structure_20 = 2 local xdb_structure_30 = 3 local xdb_ipv4_id = 4 local xdb_ipv6_id = 6 local xdb = { -- ip version version = nil, -- xdb file handle handle = nil, -- header info header = nil, io_count = 0, -- vector index vector_index = nil, -- xdb content buffer content_buff = nil } -- index and to string attribute set xdb.__index = xdb xdb.__tostring = function(self) return "xdb searcher object (lua)" end -- construct functions function new_base(version, db_path, v_index, c_buffer) local obj = setmetatable({}, xdb) obj.version = version if c_buffer ~= nil then obj.io_count = 0 obj.vector_index = nil obj.content_buff = c_buffer else obj.io_count = 0 obj.vector_index = v_index obj.handle = io.open(db_path, "r") if obj.handle == nil then return nil, string.format("failed to open xdb file `%s`", db_path) end end return obj, nil end function xdb.new_with_file_only(version, db_path) return new_base(version, db_path, nil, nil) end function xdb.new_with_vector_index(version, db_path, v_index) return new_base(version, db_path, v_index, nil) end function xdb.new_with_buffer(version, c_buffer) return new_base(version, nil, nil, c_buffer) end -- End of constructors -- object api impl, must call via ':' function xdb:search_by_string(ip_str) local ip_bytes, err = xdb.parse_ip(ip_str) if err ~= nil then return "", string.format("failed to parse string ip `%s`: %s", ip_str, err) end return self:search(ip_bytes) end function xdb:search(ip_bytes) -- check the bytes ip if type(ip_bytes) ~= "string" then return "", string.format("invalid bytes ip `%s`", ip_bytes) end -- ip version check local version = self.version if #ip_bytes ~= version.bytes then return "", string.format("invalid ip address `%s` (%s expected)", xdb.ip_to_string(ip_bytes), version.name); end -- reset the global counter -- and global resource local cache self.io_count = 0 local vector_index = self.vector_index local content_buff = self.content_buff local read_data = self.read -- locate the segment index based on the vector index local il0 = string.byte(ip_bytes, 1) & 0xFF local il1 = string.byte(ip_bytes, 2) & 0xFF local idx = il0 * vector_index_cols * vector_index_size + il1 * vector_index_size local s_ptr, e_ptr = 0, 0 if vector_index ~= nil then s_ptr = le_get_uint32(vector_index, idx + 1) e_ptr = le_get_uint32(vector_index, idx + 5) elseif content_buff ~= nil then s_ptr = le_get_uint32(content_buff, header_info_length + idx + 1) e_ptr = le_get_uint32(content_buff, header_info_length + idx + 5) else -- load from the file buff, err = read_data(self, header_info_length + idx, vector_index_size) if err ~= nil then return "", string.format("read buffer: %s", err) end s_ptr = le_get_uint32(buff, 1) e_ptr = le_get_uint32(buff, 5) end -- print(string.format("s_ptr: %d, e_ptr: %d", s_ptr, e_ptr)) -- @Note: ptr validate, zero ptr means source data missing -- so we could just stop here and return an empty string. if s_ptr == 0 or e_ptr == 0 then return "", nil end -- binary search to get the data local index_size, ip_sub_compare = version.index_size, version.ip_sub_compare local bytes, d_bytes = version.bytes, version.bytes << 1 local data_ptr, data_len, p = 0, 0, 0 local buff, err = nil, nil local l, m, h = 0, 0, (e_ptr - s_ptr) / index_size while l <= h do m = (l + h) >> 1 p = s_ptr + m * index_size -- read the segment index buff, err = read_data(self, p, index_size) if err ~= nil then return "", string.format("read segment index at %d", p) end -- check the index if ip_sub_compare(ip_bytes, buff, 1) < 0 then h = m - 1 elseif ip_sub_compare(ip_bytes, buff, bytes + 1) > 0 then l = m + 1 else data_len = le_get_uint16(buff, d_bytes + 1) data_ptr = le_get_uint32(buff, d_bytes + 3) break end end -- matching nothing interception -- print(string.format("data_len=%d, data_ptr=%d", data_len, data_ptr)) if data_len == 0 then return "", nil end -- load and return the region data buff, err = read_data(self, data_ptr, data_len) if err ~= nil then return "", string.format("read data at %d:%d", data_ptr, data_len) end return buff, nil end -- read specified bytes from the specified index function xdb:read(offset, length) -- local cache local content_buff = self.content_buff local handle = self.handle -- check the in-memory buffer first if content_buff ~= nil then return string.sub(content_buff, offset + 1, offset + length), nil end -- read from the file local r = handle:seek("set", offset) if r == nil then return nil, string.format("seek to offset %d", offset) end self.io_count = self.io_count + 1 local buff = handle:read(length) if buff == nil then return nil, string.format("read %d bytes", length) end return buff, nil end function xdb:get_ip_version() return self.version end function xdb:get_io_count() return self.io_count end function xdb:close() if self.handle ~= nil then self.handle:close() end end --- -- internal function to decode buffer function le_get_uint32(buff, idx) local i1 = (string.byte(buff, idx)) local i2 = (string.byte(buff, idx+1) << 8) local i3 = (string.byte(buff, idx+2) << 16) local i4 = (string.byte(buff, idx+3) << 24) return (i1 | i2 | i3 | i4) end function le_get_uint16(buff, idx) local i1 = (string.byte(buff, idx)) local i2 = (string.byte(buff, idx+1) << 8) return (i1 | i2) end -- static util functions function xdb.open_file(db_path, mode) local t, handle = type(db_path), nil local _closer = nil if t == "userdata" then handle = db_path -- file handle _closer = function(caller) end elseif t == "string" then handle = io.open(db_path, mode) if handle == nil then return nil, nil, string.format("failed to open xdb file `%s`", db_path) end _closer = function(caller) handle:close() end end return handle, _closer, nil end function xdb.load_header(db_path) local handle, closer, err = xdb.open_file(db_path, "rb") if err ~= nil then return nil, err end local r = handle:seek("set", 0) if r == nil then closer("load_header") return nil, "failed to seek to 0" end local c = handle:read(header_info_length) if c == nil then closer("load_header") return nil, string.format("failed to read %d bytes", header_info_length) end closer("load_header") return { ["version"] = le_get_uint16(c, 1), ["index_policy"] = le_get_uint16(c, 3), ["created_at"] = le_get_uint32(c, 5), ["start_index_ptr"] = le_get_uint32(c, 9), ["end_index_ptr"] = le_get_uint32(c, 13), -- xdb 3.0 since IPv6 supporting ["ip_version"] = le_get_uint16(c, 17), ["runtime_ptr_bytes"] = le_get_uint16(c, 19), ["raw_data"] = c }, nil end function xdb.load_vector_index(db_path) local handle, closer, err = xdb.open_file(db_path, "rb") if err ~= nil then return nil, err end local r = handle:seek("set", header_info_length) if r == nil then closer("load_vector_index") return nil, string.format("failed to seek to %d", header_info_length) end local c = handle:read(vector_index_length) if c == nil then closer("load_vector_index") return nil, string.format("failed to read %d bytes", vector_index_length) end closer("load_vector_index") return c, nil end function xdb.load_content(db_path) local handle, closer, err = xdb.open_file(db_path, "rb") local c = handle:read("*a") if c == nil then closer("load_content") return nil, string.format("failed to read xdb content") end closer("load_content") return c, nil end -- Verify if the current Searcher could be used to search the specified xdb file. -- Why do we need this check ? -- The future features of the xdb impl may cause the current searcher not able to work properly. -- -- @Note: You Just need to check this ONCE when the service starts -- Or use another process (eg, A command) to check once Just to confirm the suitability. function xdb.verify(db_path) local handle, closer, err = xdb.open_file(db_path, "rb") if err ~= nil then return err end -- load the header from handle local header, err = xdb.load_header(handle) if err ~= nil then closer() return string.format("failed to load header: %s", err) end -- get the runtime ptr bytes local runtime_ptr_bytes = 0 if header.version == xdb_structure_20 then runtime_ptr_bytes = 4 elseif header.version == xdb_structure_30 then runtime_ptr_bytes = header.runtime_ptr_bytes else closer() return string.format("invalid structure version %d", header.version); end -- 1, confirm the xdb file size -- to ensure that the maximum file pointer does not overflow local max_file_ptr = ((1 << (runtime_ptr_bytes * 8)) & 0xFFFFFFFFFFFFFFFF) - 1 local _file_bytes = handle:seek("end", 0) -- print("max_file_ptr=", max_file_ptr, "_file_bytes", _file_bytes) if _file_bytes > max_file_ptr then closer() return string.format("xdb file exceeds the maximum supported bytes: %d", max_file_ptr); end closer() return nil end -- -- parse ip string function split(str, sep) local ps, sIndex, length = {}, 1, #str -- loop to find all parts while true do local mi = string.find(str, sep, sIndex, true) if mi == nil then table.insert(ps, string.sub(str, sIndex)) break end if sIndex == mi then table.insert(ps, "") else table.insert(ps, string.sub(str, sIndex, mi - 1)) end -- reset the start index sIndex = mi + 1 end return ps end function _parse_ipv4_addr(v4_str) local ps = split(v4_str, ".") if #ps ~= 4 then return nil, string.format("invalid ipv4 address `%s`", v4_str) end local bytes = {0x00, 0x00, 0x00, 0x00} for i, s in ipairs(ps) do local v = tonumber(s) if v == nil then return nil, string.format("invalid ipv4 part `%s`, a valid number expected", s) end if v < 0 or v > 255 then return nil, string.format("invalid ipv4 part `%s`, should <=0 and <= 255", s) end bytes[i] = v end return string.char(table.unpack(bytes)), nil end function _parse_ipv6_addr(v6_str) local ps = split(v6_str, ':') if #ps < 3 or #ps > 8 then return nil, string.format("invalid ipv6 address `%s`", v6_str) end local bytes = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } local i, v, dc_num, offset, length = 1, 0, 0, 1, #ps -- process the v6 parts while i <= length do local s = ps[i]:match("^%s*(.-)%s*$") -- Double colon check and auto padding if #s == 0 then -- ONLY one double colon allow if dc_num > 0 then return nil, "invalid ipv6 address: multi double colon detected" end -- clear all the consecutive spaces local start = i i = i + 1 while true do s = ps[i]:match("^%s*(.-)%s*$") if #s > 0 then i = i - 1 break end if i >= length then break end i = i + 1 end dc_num = 1 -- padding = 9 - start - left local padding = 9 - start - (length - i) offset = offset + 2 * padding -- print("-> i ", i, "start", start, "padding: ", padding, "offset", offset) goto continue end v = tonumber(s, 16); if v == nil then return nil, string.format("invalid ipv6 part `%s`, a valid hex number expected", ps[i]) end if v < 0 or v > 0xFFFF then return nil, string.format("invalid ipv6 part `%s` should >= 0 and <= 65534", ps[i]) end bytes[offset] = (v >> 8) & 0xFF bytes[offset+1] = (v & 0xFF) offset = offset + 2 ::continue:: i = i + 1 end return string.char(table.unpack(bytes)) end function xdb.parse_ip(ip_str) local s_dot = string.find(ip_str, ".", 1, true) local c_dot = string.find(ip_str, ":", 1, true) if s_dot ~= nil and c_dot == nil then return _parse_ipv4_addr(ip_str) elseif c_dot ~= nil then return _parse_ipv6_addr(ip_str) else return nil, string.format("invalid ip address `%s`", ip_str) end end -- -- ip to string function _ipv4_to_string(ip_bytes) return string.format( "%d.%d.%d.%d", string.byte(ip_bytes, 1), string.byte(ip_bytes, 2), string.byte(ip_bytes, 3), string.byte(ip_bytes, 4) ), nil end function _ipv6_to_string(ip_bytes, compress) local ps, i, hex = {}, 0, 0 local last_hex, need_compress = -1, false for i = 1, #ip_bytes, 2 do hex = (string.byte(ip_bytes, i) << 8) | string.byte(ip_bytes, i + 1) table.insert(ps, string.format("%x", hex)) -- check the necessity for compress if last_hex > -1 and hex == 0 and last_hex == 0 then need_compress = true end -- reset the last hex last_hex = hex end -- print('need_compress', need_compress) if need_compress == false or (compress ~= nil and compress == false) then return table.concat(ps, ':'), nil end -- auto compression of consecutive zero local i, j, length, mi = 1, 0, #ps, #ps + 1 local _ = {} while i <= length do if i >= length or j > 0 then table.insert(_, ps[i]) goto continue end if ps[i] ~= '0' or ps[i+1] ~= '0' then table.insert(_, ps[i]) goto continue end -- find the first two zero part -- and keep find all the zero part i = i + 2 while i <= length do if ps[i] ~= '0' then i = i - 1 break end i = i + 1 end -- make sure there is an empty head. if #_ == 0 then table.insert(_, '') end table.insert(_, '') -- empty for double colon -- make sure there is an empty tail if i == mi and #_ < length then table.insert(_, '') end --continue ::continue:: i = i + 1 end return table.concat(_, ':') end function xdb.ip_to_string(ip_bytes, compress) local l = #ip_bytes if l == 4 then return _ipv4_to_string(ip_bytes) elseif l == 16 then return _ipv6_to_string(ip_bytes, compress) else return nil, string.format("invalid bytes ip with length not 4 or 6") end end -- -- ip bytes compare function xdb.ip_sub_compare(ip1, buff, offset) local ip2 = string.sub(buff, offset, offset + #ip1 - 1) if ip1 > ip2 then return 1 elseif ip1 < ip2 then return -1 else return 0 end end function xdb.ip_compare(ip1, ip2) return xdb.ip_sub_compare(ip1, ip2, 1) end -- this is a bit weird -- but we have no better choice for now function xdb.now() return os.time() * 1e6 end --- -- ip version local Version = { __tostring = function(t) return string.format( '{id:%d, name:%s, bytes:%d, index_size:%d}', t.id, t.name, t.bytes, t.index_size ) end } local IPv4 = { id = xdb_ipv4_id, name = "IPv4", bytes = 4, index_size = 14, -- 14 = 4 + 4 + 2 + 4 ip_sub_compare = function(ip1, buff, offset) -- ip1: Big endian byte order parsed from input -- ip2: Little endian byte order read from xdb index. -- @Note: to compatible with the old Litten endian index encode implementation. local l = #ip1 local j = offset + l - 1 for i = 1, l, 1 do local i1 = string.byte(ip1, i) local i2 = string.byte(buff, j) if i1 > i2 then return 1 end if i1 < i2 then return -1 end j = j - 1 end return 0 end } local IPv6 = { id = xdb_ipv6_id, name = "IPv6", bytes = 16, index_size = 38, -- 38 = 16 + 16 + 2 + 4 ip_sub_compare = xdb.ip_sub_compare } setmetatable(IPv4, Version) setmetatable(IPv6, Version) xdb.version_from_name = function(name) local n = string.upper(name) if n == "V4" or n == "IPV4" then return IPv4, nil elseif n == "V6" or n == "IPV6" then return IPv6, nil else return nil, string.format("invalid version name `%s`", name) end end xdb.version_from_header = function(header) -- old structure with ONLY IPv4 supporting if header.version == xdb_structure_20 then return IPv4, nil end -- structure 3.0 with IPv6 supporting if header.version ~= xdb_structure_30 then return nil, string.format("unsupported structure version `%d`", header.version) end local ip_ver = header.ip_version if ip_ver == xdb_ipv4_id then return IPv4, nil elseif ip_ver == xdb_ipv6_id then return IPv6, nil else return nil, string.format("unkown ip version id `%d`", ip_ver) end end -- constants register xdb.ipv4_id = xdb_ipv4_id xdb.ipv6_id = xdb_ipv6_id xdb.structure_20 = xdb_structure_20 xdb.structure_30 = xdb_structure_30 xdb.IPv4 = IPv4 xdb.IPv6 = IPv6 return xdb ================================================ FILE: binding/lua_c/Makefile ================================================ LuaVersion ?= 5.4 LIB_DIR = /usr/local/share/lua/$(LuaVersion) all: ../c/xdb_api.h ../c/xdb_util.c ../c/xdb_searcher.c xdb_searcher.c gcc -std=c99 -Wall -O2 -I../c/ -I/usr/include/lua$(LuaVersion) ../c/xdb_util.c ../c/xdb_searcher.c xdb_searcher.c -fPIC -shared -o xdb_searcher.so install: sudo mkdir -p $(LIB_DIR); \ sudo cp xdb_searcher.so $(LIB_DIR);\ echo "install xdb searcher to $(LIB_DIR) successfully.";\ clean: find . -name \*.so | xargs rm -f find . -name \*.o | xargs rm -f .PHONY: clean ================================================ FILE: binding/lua_c/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region lua c extension query client # Version Compatibility This implementation is compatible with lua `5.1`, `5.2`, `5.3`, and `5.4`. # Compilation and Installation ### Default Compilation Use the following commands to compile and install the default `Lua5.4` version of the extension: ```bash # cd to the root directory of lua_c binding make sudo make install ``` ### Specify Lua Version Specify the Lua version for compilation using the `LuaVersion` parameter, for example: `5.1` / `5.2` / `5.3` / `5.4` ```bash # cd to the root directory of lua_c binding # For example, compile the extension compatible with version 5.1 make LuaVersion=5.1 sudo make install ``` Note: Please use the same version of `lua` to run the following tests as the one used to compile the extension. For example: ```bash # Compile extension using lua 5.1 make LuaVersion=5.1 # Run query test using lua5.1 lua5.1 search_test.py --db=../../data/ip2region_v4.xdb ``` # Usage ### About Query API The prototype of the query API is as follows: ```lua -- Query via IP string or binary IP parsed by xdb.parse_ip search(ip_string | ip_bytes) (region, error) ``` If the query fails, `error` will be a non-`nil` error description string. If successful, it returns the `region` information as a string. If the IP address is not found, it returns an empty string `""`. ### About IPv4 and IPv6 This xdb query client implementation supports both IPv4 and IPv6 queries. Usage is as follows: ```lua -- Import xdb searcher extension local xdb = require("xdb_searcher") -- For IPv4: Set xdb path to v4 xdb file, specify IP version as IPv4 local db_path = "../../data/ip2region_v4.xdb" -- or your ipv4 xdb path local version = xdb.IPv4 -- For IPv6: Set xdb path to v6 xdb file, specify IP version as IPv6 local db_path = "../../data/ip2region_v6.xdb"; -- or your ipv6 xdb path local version = xdb.IPv6 -- The IP version of the xdb specified by db_path must match the version specified, otherwise an error will occur during query execution -- Note: The following demonstration directly uses the db_path and version variables ``` ### XDB File Verification It is recommended to actively verify the suitability of the xdb file. New features in the future may cause the current Searcher version to be incompatible with the xdb file you are using. Verification helps avoid unpredictable errors during runtime. You don't need to verify every time; for example, verify when the service starts or by manually calling the verification command. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios. ```lua local xdb = require("xdb_searcher") -- verify the xdb if xdb.verify(db_path) == false then -- Suitability verification failed!!! -- The current query client implementation is not suitable for the xdb file specified by db_path. -- You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with db_path. print(string.format("failed to verify the xdb file: %s", db_path)) return end -- Verification passed, the current Searcher can safely be used for query operations on the xdb pointed to by db_path ``` ### Entirely File-Based Query ```lua local xdb = require("xdb_searcher") -- 1. Create a file-based xdb query object from db_path using version local searcher, err = xdb.new_with_file_only(version, db_path) if err ~= nil then print(string.format("failed to create searcher: %s", err)) return end -- 2. Call the query API; both IPv4 and IPv6 are supported local ip_str = "1.2.3.4" -- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 local s_time = xdb.now() region, err = searcher:search(ip_str) local c_time = xdb.now() - s_time if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, took: %.5f μs}", region, c_time)) -- Note: For concurrent use, each coroutine needs to create a separate xdb query object -- 3. Close the xdb searcher searcher:close() -- -- 4. Module resource cleanup, only call before the entire service is completely shut down xdb.cleanup() ``` ### Caching `VectorIndex` If supported by your `lua` environment, you can pre-load the `vectorIndex` cache and make it a global variable. Using the global `vectorIndex` every time a Searcher is created can reduce one fixed IO operation, thereby accelerating queries and reducing IO pressure. ```lua local xdb = require("xdb_searcher") -- 1. Load VectorIndex cache from the specified db_path and make the v_index object below a global variable. -- vectorIndex only needs to be loaded once; it is recommended to load it as a global object when the service starts. v_index, err = xdb.load_vector_index(db_path) if err ~= nil then print(string.format("failed to load vector index from '%s'", db_path)) return end -- 2. Use the global v_index to create a query object with VectorIndex cache. searcher, err = xdb.new_with_vector_index(version, db_path, v_index) if err ~= nil then print(string.format("failed to create vector index searcher: %s", err)) return end -- 3. Call the query API; both IPv4 and IPv6 are supported local ip_str = "1.2.3.4" -- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 local s_time = xdb.now() region, err = searcher:search(ip_str) local c_time = xdb.now() = s_time if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, took: %.5f μs}", region, c_time)) -- Note: For concurrent use, each coroutine needs to create a separate xdb query object, but they share the global v_index object -- 4. Close the xdb searcher searcher:close() -- -- 5. Module resource cleanup, only call before the entire service is completely shut down xdb.cleanup() ``` ### Caching the Entire `xdb` File If supported by your `lua` environment, you can pre-load the entire xdb data into memory to achieve completely memory-based queries, similar to the previous memory search. ```lua local xdb = require("xdb_searcher") -- 1. Load the entire xdb into memory from the specified db_path. -- xdb content only needs to be loaded once; it is recommended to load it as a global object when the service starts. local content = xdb.load_content(db_path) if content == nil then print(string.format("failed to load xdb content from '%s'", db_path)) return end -- 2. Use the global content to create a query object based entirely on memory. searcher, err = xdb.new_with_buffer(version, content) if err ~= nil then print(string.format("failed to create content buffer searcher: %s", err)) return end -- 3. Call the query API; both IPv4 and IPv6 are supported local ip_str = "1.2.3.4" -- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 local s_time = xdb.now() region, err = searcher:search(ip_str) local c_time = xdb.now() - s_time if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, took: %.5f μs}", region, c_time)) -- Note: For concurrent use, query objects created with the entire xdb cache can be safely used concurrently. -- It is recommended to create a global searcher object when the service starts and then use it globally and concurrently. -- 4. Close the xdb searcher searcher:close() -- -- 5. Module resource cleanup, only call before the entire service is completely shut down xdb.cleanup() ``` # Query Testing Perform query tests via the `search_test.lua` script: ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./search_test.lua lua search_test.lua [command options] options: --db string ip2region binary xdb file path --cache-policy string cache policy: file/vectorIndex/content ``` For example: using the default `data/ip2region_v4.xdb` for IPv4 query testing: ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./search_test.lua --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 17μs} ip2region>> 120.229.45.2 {region: 中国|广东省|深圳市|移动|CN, io_count: 3, took: 40μs} ``` For example: using the default `data/ip2region_v6.xdb` for IPv6 query testing: ```bash ➜ lua_c git:(master) lua ./search_test.lua --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> :: {region: , io_count: 1, took: 48μs} ip2region>> 240e:3b7:3276:33b0:958f:f34c:d04f:f6a {region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 52μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, io_count: 13, took: 35μs} ``` Enter an IP to perform a query test. You can also set `cache-policy` to `file`/`vectorIndex`/`content` respectively to test the efficiency of the three different cache implementations. # Bench Testing Perform automatic bench testing via the `bench_test.lua` script. This ensures that the `xdb` file has no errors and tests average query performance through a large number of queries: ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua lua bench_test.lua [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` For example: perform IPv4 bench testing using default `data/ip2region_v4.xdb` and `data/ipv4_source.txt`: ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt Bench finished, {cachePolicy: vectorIndex, total: 1367686, took: 8.593 s, cost: 5.433 μs/op} ``` For example: perform IPv6 bench testing using default `data/ip2region_v6.xdb` and `data/ipv6_source.txt`: ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt Bench finished, {cachePolicy: vectorIndex, total: 34159862, took: 829.008 s, cost: 23.176 μs/op} ``` You can test the performance of the three different cache implementations (`file`/`vectorIndex`/`content`) by setting the `cache-policy` parameter. @Note: Please ensure that the `src` file used for the bench is the same source file used to generate the corresponding `xdb` file. ================================================ FILE: binding/lua_c/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region lua c 扩展查询客户端 # 版本兼容 该实现兼容 lua `5.1`，`5.2`，`5.3`, `5.4` # 编译安装 ### 默认编译 通过如下方式来编译安装默认的 `Lua5.4` 版本的扩展： ```bash # cd 到 lua_c binding 的根目录 make sudo make install ``` ### 指定 Lua 版本 通过如下的 `LuaVersion` 参数指定 Lua 版本编译，例如：`5.1` / `5.2` / `5.3` / `5.4` ```bash # cd 到 lua_c binding 的根目录 # 例如，编译 5.1 版本兼容的扩展 make LuaVersion=5.1 sudo make install ``` 备注：使用了指定的版本的 lua 编译的扩展就请使用相同版本的`lua`去运行以下的测试，例如： ```bash # 使用 lua 5.1 编译扩展 make LuaVersion=5.1 # 使用 lua5.1 运行查询测试 lua5.1 search_test.py --db=../../data/ip2region_v4.xdb ``` # 使用方式 ### 关于查询 API 查询 API 的原型如下： ```lua -- 通过字符串 IP 或者 xdb.parse_ip 解析得到的二进制 IP 进行查询 search(ip_string | ip_bytes) (region, error) ``` 如果查询失败则 error 将会为一个非 `nil` 的错误描述字符串，查询成功将会返回字符串的 `region` 信息，如果查询的 IP 地址没找到则会返回一个空字符 `""`。 ### 关于 IPv4 和 IPv6 该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询，使用方式如下： ```lua -- 引入 xdb searcher 扩展 local xdb = require("xdb_searcher") -- 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 IPv4 local db_path = "../../data/ip2region_v4.xdb" -- 或者你的 ipv4 xdb 的路径 local version = xdb.IPv4 -- 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件，IP版本指定为 IPv6 local db_path = "../../data/ip2region_v6.xdb"; -- 或者你的 ipv6 xdb 路径 local version = xdb.IPv6 -- db_path 指定的 xdb 的 IP 版本必须和 version 指定的一致，不然查询执行的时候会报错 -- 备注：以下演示直接使用 db_path 和 version 变量 ``` ### XDB 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 ```lua local xdb = require("xdb_searcher") -- verify the xdb if xdb.verify(db_path) == false then -- 适用性验证失败！！！ -- 当前查询客户端实现不适用于 db_path 指定的 xdb 文件的查询. -- 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 db_path 的 Searcher 实现。 print(string.format("failed to verify the xdb file: %s", db_path)) return end -- 验证通过，当前使用的 Searcher 可以安全的用于对 db_path 指向的 xdb 的查询操作 ``` ### 完全基于文件的查询 ```lua local xdb = require("xdb_searcher") -- 1、使用 version 从 db_path 创建基于文件的 xdb 查询对象 local searcher, err = xdb.new_with_file_only(version, db_path) if err ~= nil then print(string.format("failed to create searcher: %s", err)) return end -- 2、调用查询 API 进行查询，IPv4 和 IPv6 都支持 local ip_str = "1.2.3.4" -- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 local s_time = xdb.now() region, err = searcher:search(ip_str) local c_time = xdb.now() - s_time if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, took: %.5f μs}", region, c_time)) -- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象 -- 3，关闭 xdb 查询器 searcher:close() -- -- 4，模块资源清理，仅在需要将整个服务完全关闭前调用 xdb.cleanup() ``` ### 缓存 `VectorIndex` 索引 如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。 ```lua local xdb = require("xdb_searcher") -- 1、从指定的 db_path 加载 VectorIndex 缓存，把下述的 v_index 对象做成全局变量。 -- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。 v_index, err = xdb.load_vector_index(db_path) if err ~= nil then print(string.format("failed to load vector index from '%s'", db_path)) return end -- 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。 searcher, err = xdb.new_with_vector_index(version, db_path, v_index) if err ~= nil then print(string.format("failed to create vector index searcher: %s", err)) return end -- 3、调用查询 API ，IPv4 和 IPv6 都支持 local ip_str = "1.2.3.4" -- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 local s_time = xdb.now() region, err = searcher:search(ip_str) local c_time = xdb.now() = s_time if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, took: %.5f μs}", region, c_time)) -- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象 -- 4，关闭 xdb 查询器 searcher:close() -- -- 5，模块资源清理，仅在需要将整个服务完全关闭前调用 xdb.cleanup() ``` ### 缓存整个 `xdb` 文件 如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。 ```lua local xdb = require("xdb_searcher") -- 1、从指定的 db_path 加载整个 xdb 到内存。 -- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。 local content = xdb.load_content(db_path) if content == nil then print(string.format("failed to load xdb content from '%s'", db_path)) return end -- 2、使用全局的 content 创建带完全基于内存的查询对象。 searcher, err = xdb.new_with_buffer(version, content) if err ~= nil then print(string.format("failed to create content buffer searcher: %s", err)) return end -- 3、调用查询 API ，IPv4 和 IPv6 都支持 local ip_str = "1.2.3.4" -- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 local s_time = xdb.now() region, err = searcher:search(ip_str) local c_time = xdb.now() - s_time if err ~= nil then print(string.format("failed to search(%s): %s", ip_str, err)) return end print(string.format("{region: %s, took: %.5f μs}", region, c_time)) -- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。 -- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。 -- 4，关闭 xdb 查询器 searcher:close() -- -- 5，模块资源清理，仅在需要将整个服务完全关闭前调用 xdb.cleanup() ``` # 查询测试 通过 `search_test.lua` 脚本来进行查询测试： ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./search_test.lua lua search_test.lua [command options] options: --db string ip2region binary xdb file path --cache-policy string cache policy: file/vectorIndex/content ``` 例如：使用默认的 data/ip2region_v4.xdb 进行 IPv4 查询测试： ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./search_test.lua --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 17μs} ip2region>> 120.229.45.2 {region: 中国|广东省|深圳市|移动|CN, io_count: 3, took: 40μs} ``` 例如：使用默认的 data/ip2region_v6.xdb 进行 IPv6 查询测试： ```bash ➜ lua_c git:(master) lua ./search_test.lua --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> :: {region: , io_count: 1, took: 48μs} ip2region>> 240e:3b7:3276:33b0:958f:f34c:d04f:f6a {region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 52μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, io_count: 13, took: 35μs} ``` 输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。 # bench 测试 通过 `bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能： ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua lua bench_test.lua [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` 例如：通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 来进行 IPv4 的 bench 测试： ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt Bench finished, {cachePolicy: vectorIndex, total: 1367686, took: 8.593 s, cost: 5.433 μs/op} ``` 例如：通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 来进行 IPv6 的 bench 测试： ```bash ➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt Bench finished, {cachePolicy: vectorIndex, total: 34159862, took: 829.008 s, cost: 23.176 μs/op} ``` 可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。 @Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。 ================================================ FILE: binding/lua_c/bench_test.lua ================================================ -- Copyright 2022 The Ip2Region Authors. All rights reserved. -- Use of this source code is governed by a Apache2.0-style -- license that can be found in the LICENSE file. -- -- --- -- @Author Lion -- @Date 2022/06/30 -- set the package to load the current xdb_searcher.so package.path = "./?.lua;" .. package.path package.cpath = "./?.so;" .. package.cpath local xdb = require("xdb_searcher") function printHelp() print("lua bench_test.lua [command options]") print("options: ") print(" --db string ip2region binary xdb file path") print(" --src string source ip text file path") print(" --cache-policy string cache policy: file/vectorIndex/content") end if #arg < 2 then printHelp() return end -- parser the command line args local dbFile, srcFile = "", "" local cachePolicy = "vectorIndex" for _, r in ipairs(arg) do if string.len(r) < 5 then -- continue and do nothing here elseif string.sub(r, 1, 2) ~= "--" then -- continue and do nothing here else for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do if k == "db" then dbFile = v elseif k == "src" then srcFile = v elseif k == "cache-policy" then cachePolicy = v else print(string.format("undefined option `%s`", r)) return end -- break the match iterate break end end end -- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy)) if string.len(dbFile) < 2 or string.len(srcFile) < 2 then printHelp() return end -- verify the xdb from header if xdb.verify(dbFile) == false then print(string.format("failed to verify the xdb file: %s", dbFile)) return end -- detect the version from the xdb header header, err = xdb.load_header(dbFile) if err ~= nil then print(string.format("failed to load header: %s", err)) return end version, err = xdb.version_from_header(header); if err ~= nil then print(string.format("failed to detect version from header: %s", err)) return end -- create the searcher based on the cache-policy local searcher, v_index, content if cachePolicy == "file" then searcher, err = xdb.new_with_file_only(version, dbFile) if err ~= nil then print(string.format("failed to create searcher: %s", err)) return end elseif cachePolicy == "vectorIndex" then v_index, err = xdb.load_vector_index(dbFile) if err ~= nil then print(string.format("failed to load vector index: %s", err)) return end searcher, err = xdb.new_with_vector_index(version, dbFile, v_index) if err ~= nil then print(string.format("failed to create vector index searcher: %s", err)) return end elseif cachePolicy == "content" then content, err = xdb.load_content(dbFile) if err ~= nil then print(string.format("failed to load xdb content from '%s'", dbFile)) return end searcher, err = xdb.new_with_buffer(version, content) if err ~= nil then print(string.format("failed to create content buffer searcher: %s", err)) return end else print(string.format("undefined cache-policy `%s`", cachePolicy)) return end -- do the bench test local handle = io.open(srcFile, "r") if handle == nil then print(string.format("failed to open src text file `%s`", handle)) return end local lines = handle:lines() local sip_str, eip_str, s_region, region, err = "", "", "", "", 0 local count, t_time, c_time = 0, 0, 0 local s_time = xdb.now() for l in lines do if string.len(l) < 1 then -- continue and do nothing here else for v1, v2, v3 in string.gmatch(l, "([^|]+)|([^|]+)|([^\n]+)") do -- print(sip_str, eip_str, region) sip_str = v1 eip_str = v2 s_region = v3 break end t_time = xdb.now() sip_bytes, err = xdb.parse_ip(sip_str) if err ~= nil then print(string.format("invalid start ip `%s`", sip_str)) return end eip_bytes, err = xdb.parse_ip(eip_str) if err ~= nil then print(string.format("invalid end ip `%s`", sip_str)) return end if xdb.ip_compare(sip_bytes, eip_bytes) > 0 then print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str)) return end for _, ip_bytes in ipairs({sip_bytes, eip_bytes}) do region, err = searcher:search(ip_bytes) if err ~= nil then print(string.format("failed to search ip `%s`", xdb.ip_to_string(ip_bytes))) return end -- check the region if region ~= s_region then print(string.format("failed search(%s) with (%s != %s)\n", xdb.ip_to_string(ip_bytes), region, s_region)) return end count = count + 1 end -- increase the time costs c_time = c_time + xdb.now() - t_time end end -- resource cleanup searcher:close() if v_index ~= nil then v_index:close() end if content ~= nil then content:close() end xdb.cleanup() -- print the stats local avg_costs = 0 if count > 0 then avg_costs = c_time / count end print(string.format("Bench finished, {cachePolicy: %s, total: %d, took: %.3f s, cost: %.3f μs/op}", cachePolicy, count, (xdb.now() - s_time)/1e6, c_time / count)) ================================================ FILE: binding/lua_c/search_test.lua ================================================ -- Copyright 2022 The Ip2Region Authors. All rights reserved. -- Use of this source code is governed by a Apache2.0-style -- license that can be found in the LICENSE file. -- -- --- -- @Author Lion -- @Date 2022/06/30 -- set the package to load the current xdb_searcher.so package.path = "./?.lua;" .. package.path package.cpath = "./?.so;" .. package.cpath local xdb = require("xdb_searcher") function printHelp() print("lua search_test.lua [command options]") print("options: ") print(" --db string ip2region binary xdb file path") print(" --cache-policy string cache policy: file/vectorIndex/content") end if #arg < 1 then printHelp() return end -- parser the command line args local dbFile = "" local cachePolicy = "vectorIndex" for _, r in ipairs(arg) do if string.len(r) < 5 then -- continue and do nothing here elseif string.sub(r, 1, 2) ~= "--" then -- continue and do nothing here else for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do if k == "db" then dbFile = v elseif k == "cache-policy" then cachePolicy = v else print(string.format("undefined option `%s`", r)) return end -- break the match iterate break end end end -- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy)) if string.len(dbFile) < 2 then printHelp() return end -- verify the xdb if xdb.verify(dbFile) == false then print(string.format("failed to verify the xdb file: %s", dbFile)) return end -- detect the version from the xdb header header, err = xdb.load_header(dbFile) if err ~= nil then print(string.format("failed to load header: %s", err)) return end version, err = xdb.version_from_header(header); if err ~= nil then print(string.format("failed to detect version from header: %s", err)) return end -- create the searcher based on the cache-policy local searcher, v_index, content if cachePolicy == "file" then searcher, err = xdb.new_with_file_only(version, dbFile) if err ~= nil then print(string.format("failed to create searcher: %s", err)) return end elseif cachePolicy == "vectorIndex" then v_index, err = xdb.load_vector_index(dbFile) if err ~= nil then print(string.format("failed to load vector index: %s", err)) return end searcher, err = xdb.new_with_vector_index(version, dbFile, v_index) if err ~= nil then print(string.format("failed to create vector index searcher: %s", err)) return end elseif cachePolicy == "content" then content, err = xdb.load_content(dbFile) if err ~= nil then print(string.format("failed to load xdb content from '%s'", dbFile)) return end searcher, err = xdb.new_with_buffer(version, content) if err ~= nil then print(string.format("failed to create content buffer searcher: %s", err)) return end else print(string.format("undefined cache-policy `%s`", cachePolicy)) return end -- do the search print(string.format([[ ip2region xdb searcher test program source xdb: %s (%s, %s) type 'quit' to exit]], dbFile, xdb.version_info(version).name, cachePolicy)) local region, err = "", nil local s_time, c_time = 0, 0 while ( true ) do io.write("ip2region>> "); io.input(io.stdin); local line = io.read(); if (line == nil) then break end if ( line == "quit" ) then break end -- empty string ignore line = line:gsub("^%s*(.-)%s*$", "%1") if string.len(line) < 1 then -- continue and do nothing here else s_time = xdb.now() ip_bytes, err = xdb.parse_ip(line) -- print(string.format("parse(%s): %s, err: %s", line, xdb.ip_to_string(ip_bytes), err)) if err ~= nil then print(string.format("invalid ip address `%s`", line)) else -- do the search region, err = searcher:search(ip_bytes) c_time = xdb.now() - s_time if err ~= nil then print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count())) else print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time)) end end end end -- resource cleanup searcher:close() if v_index ~= nil then v_index:close() end if content ~= nil then content:close() end xdb.cleanup(); ================================================ FILE: binding/lua_c/util_test.lua ================================================ -- Copyright 2022 The Ip2Region Authors. All rights reserved. -- Use of this source code is governed by a Apache2.0-style -- license that can be found in the LICENSE file. -- -- --- -- @Author Lion -- @Date 2022/06/30 -- set the package to load the current xdb_searcher.so package.path = "./?.lua;" .. package.path package.cpath = "./?.so;" .. package.cpath local xdb = require("xdb_searcher") ---- ip checking testing function test_parse_ip() local ip_list = { "1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0", "invalid-ipv.4", "::", "3000::", "240e:3b7:3276:33b0:4844:6f28:f69c:1eee", "2001:4:112::", "invalid-ipv::6" } local s_time = xdb.now() for _, ip_src in ipairs(ip_list) do ip_bytes, err = xdb.parse_ip(ip_src) if err ~= nil then print(string.format("invalid ip address `%s`: %s", ip_src, err)) else local ip_string = xdb.ip_to_string(ip_bytes); print(string.format("parse_ip(%s)->%s ? %s", ip_src, ip_string, tostring(ip_src==ip_string))) end end end function test_print_const() print("ipv4: ", xdb.IPv4); print("ipv6: ", xdb.IPv6); print("header_buffer: ", xdb.header_buffer); print("v_index_buffer: ", xdb.v_index_buffer); print("content_buffer: ", xdb.content_buffer); end ---- buffer loading test function test_load_header() header, err = xdb.load_header("../../data/ip2region_v4.xdb") if err ~= nil then print("failed to load header: ", err) else print(string.format("xdb header buffer `%s` loaded", tostring(header))) local tpl = [[ header: { version: %d index_policy: %d created_at: %d start_index_ptr: %d end_index_ptr: %d ip_version: %d runtime_ptr_bytes: %d }]] local t = header:to_table() print(string.format(tpl, t["version"], t["index_policy"], t["created_at"], t["start_index_ptr"], t["end_index_ptr"], t["ip_version"], t["runtime_ptr_bytes"]) ) end end function test_version_info() local v4 = xdb.version_info(xdb.IPv4) print(string.format("{id:%d, name: %s, bytes: %d, segment_index_size: %d}", v4.id, v4.name, v4.bytes, v4.segment_index_size)) local v6 = xdb.version_info(xdb.IPv6) print(string.format("{id:%d, name: %s, bytes: %d, segment_index_size: %d}", v6.id, v6.name, v6.bytes, v6.segment_index_size)) local vx = xdb.version_info(3) end function test_load_vector_index() v_index, err = xdb.load_vector_index("../../data/ip2region_v4.xdb") if err ~= nil then print("failed to load vector index: ", err) else print(string.format("xdb vector index buffer `%s` loaded, info={name=%s, type=%d, length=%d}", tostring(v_index), v_index:name(), v_index:type(), v_index:length())) v_index:close() end end function test_load_content() c_buffer, err = xdb.load_content("../../data/ip2region_v4.xdb") if err ~= nil then print("failed to load content: ", err) else print(string.format("xdb content buffer `%s` loaded, info={name=%s, type=%d, length=%d}", tostring(c_buffer), c_buffer:name(), c_buffer:type(), c_buffer:length())) c_buffer:close(); end end function test_search() -- ipv4 local ip_str = "1.2.3.4" searcher, err = xdb.new_with_file_only(xdb.IPv4, "../../data/ip2region_v4.xdb") print(string.format("searcher.tostring=%s", tostring(searcher))) local t_start = xdb.now() region, err = searcher:search(ip_str) local c_time = xdb.now() - t_start print(string.format("search(%s): {region=%s, io_count: %d, took: %dμs, err=%s}", ip_str, region, searcher:get_io_count(), c_time, tostring(err))) searcher:close() -- IPv6 ip_str = "240e:3b7:3276:33b0:958f:f34c:d04f:f6a" searcher, err = xdb.new_with_file_only(xdb.IPv6, "../../data/ip2region_v6.xdb") print(string.format("searcher.tostring=%s", tostring(searcher))) t_start = xdb.now() region, err = searcher:search(ip_str) c_time = xdb.now() - t_start print(string.format("search(%s): {region=%s, io_count: %d, took: %dμs, err=%s}", ip_str, region, searcher:get_io_count(), c_time, tostring(err))) searcher:close() end local func_name = arg[1] if func_name == nil then print("please specified the function to test") return end if (_G[func_name] == nil) then print(string.format("undefined function `%s` to call", func_name)) return end local s_time = xdb.now(); print(string.format("+---calling test function %s ...", func_name)) _G[func_name]() local cost_time = xdb.now() - s_time xdb.cleanup(); print(string.format("|---done, took: %.3fμs", cost_time)) ================================================ FILE: binding/lua_c/xdb_searcher.c ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // @Author Lion // @Date 2022/06/30 #include "stdio.h" #include "lua.h" #include "lauxlib.h" #include "../c/xdb_api.h" #define XDB_BUFFER_METATABLE_NAME "xdb_buffer_mt" #define XDB_METATABLE_NAME "xdb_metatable_name" #define xdb_header_buffer 1 #define xdb_vector_index_buffer 2 #define xdb_content_buffer 3 // --- xdb buffer interface impl struct xdb_buffer_entry { int type; // buffer type char *name; // buffer name void *ptr; // buffer ptr void (*closer) (void *); }; typedef struct xdb_buffer_entry xdb_buffer_t; static int lua_xdb_buffer_name(lua_State *L) { xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); lua_pushstring(L, buffer->name); return 1; } static int lua_xdb_buffer_type(lua_State *L) { xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); lua_pushinteger(L, buffer->type); return 1; } static int lua_xdb_buffer_to_table(lua_State *L) { xdb_buffer_t *buffer; xdb_header_t *header; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); lua_newtable(L); if (buffer->type == xdb_header_buffer) { header = (xdb_header_t *) buffer->ptr; lua_pushinteger(L, header->version); lua_setfield(L, -2, "version"); lua_pushinteger(L, header->index_policy); lua_setfield(L, -2, "index_policy"); lua_pushinteger(L, header->created_at); lua_setfield(L, -2, "created_at"); lua_pushinteger(L, header->start_index_ptr); lua_setfield(L, -2, "start_index_ptr"); lua_pushinteger(L, header->end_index_ptr); lua_setfield(L, -2, "end_index_ptr"); lua_pushinteger(L, header->ip_version); lua_setfield(L, -2, "ip_version"); lua_pushinteger(L, header->runtime_ptr_bytes); lua_setfield(L, -2, "runtime_ptr_bytes"); } else { // do nothing for now } return 1; } static int lua_xdb_buffer_length(lua_State *L) { xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); if (buffer->type == xdb_header_buffer) { lua_pushinteger(L, ((xdb_header_t *) buffer->ptr)->length); } else if (buffer->type == xdb_vector_index_buffer) { lua_pushinteger(L, ((xdb_vector_index_t *) buffer->ptr)->length); } else if (buffer->type == xdb_content_buffer) { lua_pushinteger(L, ((xdb_content_t *) buffer->ptr)->length); } else { lua_pushinteger(L, -1); } return 1; } static int lua_xdb_buffer_tostring(lua_State *L) { xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); lua_pushfstring(L, "xdb %s buffer object {name: %s, type: %d}", buffer->name, buffer->name, buffer->type); return 1; } static int lua_xdb_buffer_close(lua_State *L) { xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); // check and call the closer if (buffer->closer != NULL) { buffer->closer(buffer->ptr); buffer->closer = NULL; } return 0; } // module method define, should be access via ':' static const struct luaL_Reg xdb_buffer_methods[] = { {"name", lua_xdb_buffer_name}, {"type", lua_xdb_buffer_type}, {"length", lua_xdb_buffer_length}, {"to_table", lua_xdb_buffer_to_table}, {"close", lua_xdb_buffer_close}, {"__gc", lua_xdb_buffer_close}, {"__tostring", lua_xdb_buffer_tostring}, {NULL, NULL} }; // --- End of xdb buffer // --- xdb util function static int lua_xdb_load_header_from_file(lua_State *L) { const char *db_path; xdb_header_t *header; xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and the xdb file path expected"); db_path = luaL_checkstring(L, 1); header = xdb_load_header_from_file(db_path); if (header == NULL) { lua_pushnil(L); lua_pushfstring(L, "load header from `%s`", db_path); return 2; } // alloc the buffer. buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t)); if (buffer == NULL) { lua_pushnil(L); lua_pushfstring(L, "failed to alloc xdb buffer entry"); return 2; } // init the buffer buffer->type = xdb_header_buffer; buffer->name = "header"; buffer->ptr = header; buffer->closer = xdb_free_header; // set the metatable of the header buffer object and push onto the stack luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME); lua_setmetatable(L, -2); lua_pushnil(L); return 2; } static int lua_xdb_load_vector_index_from_file(lua_State *L) { const char *db_path; xdb_vector_index_t *v_index; xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and the xdb path expected"); db_path = luaL_checkstring(L, 1); v_index = xdb_load_vector_index_from_file(db_path); if (v_index == NULL) { lua_pushnil(L); lua_pushfstring(L, "load vector index from `%s`", db_path); return 2; } // alloc the buffer. buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t)); if (buffer == NULL) { lua_pushnil(L); lua_pushstring(L, "failed to alloc xdb buffer entry"); return 2; } // init the buffer buffer->type = xdb_vector_index_buffer; buffer->name = "v_index"; buffer->ptr = v_index; buffer->closer = xdb_free_vector_index; // set the metatable of the header buffer object and push onto the stack luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME); lua_setmetatable(L, -2); lua_pushnil(L); return 2; } static int lua_xdb_load_content_from_file(lua_State *L) { const char *db_path; xdb_content_t *content; xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and xdb path expected"); db_path = luaL_checkstring(L, 1); content = xdb_load_content_from_file(db_path); if (content == NULL) { lua_pushnil(L); lua_pushfstring(L, "load xdb content from `%s`", db_path); return 2; } // alloc the buffer. buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t)); if (buffer == NULL) { lua_pushnil(L); lua_pushstring(L, "failed to alloc xdb buffer entry"); return 2; } // init the buffer buffer->type = xdb_content_buffer; buffer->name = "content"; buffer->ptr = content; buffer->closer = xdb_free_content; // set the metatable of the header buffer object and push onto the stack luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME); lua_setmetatable(L, -2); lua_pushnil(L); return 2; } static int lua_xdb_verify_from_file(lua_State *L) { const char *db_path; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and the xdb file path expected"); db_path = luaL_checkstring(L, 1); lua_pushboolean(L, xdb_verify_from_file(db_path) == 0 ? 1 : 0); return 1; } static int lua_xdb_version_from_header(lua_State *L) { xdb_version_t *version; xdb_buffer_t *buffer; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and xdb header expected"); // header buffer checking buffer = luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); if (buffer->type != xdb_header_buffer) { return luaL_error(L, "invalid xdb header buffer"); } version = xdb_version_from_header((xdb_header_t *) buffer->ptr); if (version == NULL) { lua_pushnil(L); lua_pushstring(L, "failed to detect version from header"); } else { lua_pushinteger(L, version->id); lua_pushnil(L); } return 2; } static xdb_version_t *_get_version(lua_State *L, int arg) { int vid = luaL_checkinteger(L, arg); if (vid == xdb_ipv4_id) { return XDB_IPv4; } else if (vid == xdb_ipv6_id) { return XDB_IPv6; } else { return NULL; } } static int lua_xdb_version_info(lua_State *L) { xdb_version_t *version; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and version id expected"); // check the ip version version = _get_version(L, 1); if (version == NULL) { return luaL_error(L, "invalid verison id specified"); } lua_newtable(L); lua_pushinteger(L, version->id); lua_setfield(L, -2, "id"); lua_pushstring(L, version->name); lua_setfield(L, -2, "name"); lua_pushinteger(L, version->bytes); lua_setfield(L, -2, "bytes"); lua_pushinteger(L, version->segment_index_size); lua_setfield(L, -2, "segment_index_size"); return 1; } static int lua_xdb_parse_ip(lua_State *L) { const char *ip_str; bytes_ip_t ip_bytes[19] = {'\0'}; xdb_version_t *version; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and string ip expected, eg: 1.2.3.4 / 3000::"); ip_str = luaL_checkstring(L, 1); version = xdb_parse_ip(ip_str, ip_bytes + 2, sizeof(ip_bytes) - 2); if (version == NULL) { lua_pushnil(L); lua_pushfstring(L, "failed to parse the `%s`", ip_str); return 2; } // append the magic char for later analysis // printf("ip:%s, version->id: %d\n", ip_str, version->id); ip_bytes[0] = '&'; ip_bytes[1] = (bytes_ip_t) version->id; lua_pushlstring(L, (string_ip_t *) ip_bytes, version->bytes + 2); lua_pushnil(L); return 2; } static int lua_xdb_ip_to_string(lua_State *L) { int err, vid, bytes; const string_ip_t *ip_bytes; char ip_string[INET6_ADDRSTRLEN + 1] = {'\0'}; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and bytes ip expected"); ip_bytes = luaL_checkstring(L, 1); if (strlen(ip_bytes) < 2) { lua_pushnil(L); lua_pushstring(L, "invalid binary ip bytes specified"); return 2; } if (ip_bytes[0] != '&') { lua_pushnil(L); lua_pushstring(L, "invalid binary ip bytes specified"); return 2; } vid = ip_bytes[1] & 0xFF; if (vid == xdb_ipv4_id) { // IPv4 bytes = xdb_ipv4_bytes; } else if (vid == xdb_ipv6_id) { // IPv6 bytes = xdb_ipv6_bytes; } else { lua_pushnil(L); lua_pushstring(L, "invalid binary ip bytes specified"); return 2; } err = xdb_ip_to_string(((bytes_ip_t *) ip_bytes) + 2, bytes, ip_string, sizeof(ip_string)); if (err != 0) { lua_pushnil(L); lua_pushstring(L, "failed to conver the ip bytes to string"); return 2; } lua_pushstring(L, ip_string); lua_pushnil(L); return 2; } static int _validate_bytes_ip(const string_ip_t *ip_bytes) { if (strlen(ip_bytes) < 2) { return 1; } if (ip_bytes[0] != '&') { return 2; } int vid = ip_bytes[1] & 0xFF; if (vid != xdb_ipv4_id && vid != xdb_ipv6_id) { return 3; } return 0; } static int lua_xdb_ip_compare(lua_State *L) { int err; const string_ip_t *ip1_bytes, *ip2_bytes; luaL_argcheck(L, lua_gettop(L) == 2, 1, "call via '.' bytes ip1 and ip2 expected"); ip1_bytes = luaL_checkstring(L, 1); ip2_bytes = luaL_checkstring(L, 2); // validate the ip1 err = _validate_bytes_ip(ip1_bytes); if (err != 0) { lua_pushnil(L); lua_pushfstring(L, "failed to validate ip1 with errcode=%d", err); return 2; } // validate the ip2 err = _validate_bytes_ip(ip2_bytes); if (err != 0) { lua_pushnil(L); lua_pushfstring(L, "failed to validate ip2 with errcode=%d", err); return 2; } if (ip1_bytes[1] != ip2_bytes[1]) { lua_pushnil(L); lua_pushstring(L, "ip version of ip1 and ip2 are not the same"); return 2; } err = xdb_ip_sub_compare(((bytes_ip_t *)ip1_bytes) + 2, (ip1_bytes[1] & 0xFF), ip2_bytes, 2); lua_pushinteger(L, err); lua_pushnil(L); return 2; } static int lua_xdb_now(lua_State *L) { lua_pushinteger(L, xdb_now()); return 1; } // --- End of xdb util api // --- xdb searcher api static int lua_xdb_new_with_file_only(lua_State *L) { int err; xdb_version_t *version; xdb_searcher_t *searcher; const char *db_path = NULL; luaL_argcheck(L, lua_gettop(L) == 2, 1, "call via '.' and ip version / xdb file path expected"); // check the ip version version = _get_version(L, 1); if (version == NULL) { return luaL_error(L, "invalid verison id specified"); } // check the db path db_path = luaL_checkstring(L, 2); // alloc for the searcher searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t)); if (searcher == NULL) { return luaL_error(L, "failed to alloc xdb searcher entry"); } // init the xdb searcher err = xdb_new_with_file_only(version, searcher, db_path); if (err != 0) { lua_pushnil(L); lua_pushfstring(L, "init xdb searcher on `%s`: errcode=%d", db_path, err); return 2; } // push the metatable onto the stack and // set it as the metatable of the current searcher luaL_getmetatable(L, XDB_METATABLE_NAME); lua_setmetatable(L, -2); lua_pushnil(L); return 2; } static int lua_xdb_new_with_vector_index(lua_State *L) { xdb_version_t *version; xdb_searcher_t *searcher; xdb_buffer_t *xBuffer; const char *db_path; int err; luaL_argcheck(L, lua_gettop(L) == 3, 1, "call via '.', ip version / xdb file path / vector index buffer expected"); // check the ip version version = _get_version(L, 1); if (version == NULL) { return luaL_error(L, "invalid verison id specified"); } // db_path checking db_path = luaL_checkstring(L, 2); // vector index buffer checking xBuffer = luaL_checkudata(L, 3, XDB_BUFFER_METATABLE_NAME); if (xBuffer->type != xdb_vector_index_buffer) { return luaL_error(L, "invalid vector index buffer"); } // alloc the searcher searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t)); if (searcher == NULL) { return luaL_error(L, "failed to alloc xdb searcher entry"); } // init the xdb searcher err = xdb_new_with_vector_index(version, searcher, db_path, (xdb_vector_index_t *) xBuffer->ptr); if (err != 0) { lua_pushnil(L); lua_pushfstring(L, "init vector index cached xdb searcher on `%s` with errcode=%d", db_path, err); return 2; } // push the metatable onto the stack and // set it as the metatable of the current searcher luaL_getmetatable(L, XDB_METATABLE_NAME); lua_setmetatable(L, -2); lua_pushnil(L); return 2; } static int lua_xdb_new_with_buffer(lua_State *L) { xdb_version_t *version; xdb_searcher_t *searcher; xdb_buffer_t *xBuffer; int err; luaL_argcheck(L, lua_gettop(L) == 2, 1, "call via '.' and ip version / xdb content buffer expected"); // check the ip version version = _get_version(L, 1); if (version == NULL) { return luaL_error(L, "invalid verison id specified"); } // content buffer checking xBuffer = (xdb_buffer_t *) luaL_checkudata(L, 2, XDB_BUFFER_METATABLE_NAME); if (xBuffer->type != xdb_content_buffer) { return luaL_error(L, "invalid xdb content buffer"); } // alloc the searcher searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t)); if (searcher == NULL) { return luaL_error(L, "failed to alloc xdb searcher entry"); } // init the xdb searcher err = xdb_new_with_buffer(version, searcher, (xdb_content_t *) xBuffer->ptr); if (err != 0) { lua_pushnil(L); lua_pushfstring(L, "init content cached xdb searcher with errcode=%d", err); return 2; } // push the metatable onto the stack and // set it as the metatable of the current searcher luaL_getmetatable(L, XDB_METATABLE_NAME); lua_setmetatable(L, -2); lua_pushnil(L); return 2; } static int lua_xdb_close(lua_State *L) { xdb_searcher_t *searcher; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_METATABLE_NAME); if (searcher == NULL) { return luaL_error(L, "broken xdb searcher instance"); } xdb_close(searcher); return 0; } static int lua_xdb_search(lua_State *L) { int err, vid, ip_len; const char *ip_string; bytes_ip_t ip_buffer[INET6_ADDRSTRLEN] = {'\0'}; const bytes_ip_t *ip_bytes; xdb_version_t *version; xdb_region_buffer_t region; xdb_searcher_t *searcher; luaL_argcheck(L, lua_gettop(L) == 2, 2, "call via ':' and string ip address expected"); // get the searcher searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_METATABLE_NAME); ip_string = luaL_checkstring(L, 2); // ip string type checking if (strlen(ip_string) < 2) { lua_pushnil(L); lua_pushfstring(L, "invalid ip address `%s`", ip_string); return 2; } if (ip_string[0] == '&') { vid = ip_string[1] & 0xFF; if (vid == xdb_ipv4_id) { ip_len = xdb_ipv4_bytes; } else if (vid == xdb_ipv6_id) { ip_len = xdb_ipv6_bytes; } else { lua_pushnil(L); lua_pushstring(L, "invalid binary ip bytes specified"); return 2; } ip_bytes = (bytes_ip_t *)ip_string + 2; // printf("ip_len: %d, vid: %d\n", ip_len, vid); } else { version = xdb_parse_ip(ip_string, ip_buffer, sizeof(ip_buffer)); if (version == NULL) { lua_pushnil(L); lua_pushfstring(L, "failed to parse string ip `%s`", ip_string); return 2; } ip_len = version->bytes; ip_bytes = ip_buffer; } // init the region buffer err = xdb_region_buffer_init(®ion, NULL, 0); if (err != 0) { return luaL_error(L, "failed to init the region buffer with errcode=%d", err); } // do the search err = xdb_search(searcher, ip_bytes, ip_len, ®ion); if (err != 0) { lua_pushinteger(L, err); lua_pushfstring(L, "err=%d", err); } else { lua_pushstring(L, region.value); lua_pushnil(L); } // clean up the region buffer xdb_region_buffer_free(®ion); return 2; } static int lua_xdb_get_io_count(lua_State *L) { xdb_searcher_t *searcher; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':' or xdb searcher was broken"); searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_METATABLE_NAME); lua_pushinteger(L, xdb_get_io_count(searcher)); return 1; } static int lua_xdb_tostring(lua_State *L) { xdb_searcher_t *searcher; luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':' or xdb searcher was broken"); searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_METATABLE_NAME); lua_pushfstring(L, "xdb %s searcher object", xdb_get_version(searcher)->name); return 1; } // cleanup the current module static int lua_xdb_cleanup(lua_State *L) { xdb_clean_winsock(); return 0; } // module method define, should be access via ':' static const struct luaL_Reg xdb_searcher_methods[] = { {"search", lua_xdb_search}, {"get_io_count",lua_xdb_get_io_count}, {"close", lua_xdb_close}, {"__gc", lua_xdb_close}, {"__tostring", lua_xdb_tostring}, {NULL, NULL}, }; // module function define, should be access via '.' static const struct luaL_Reg xdb_searcher_functions[] = { {"new_with_file_only", lua_xdb_new_with_file_only}, {"new_with_vector_index", lua_xdb_new_with_vector_index}, {"new_with_buffer", lua_xdb_new_with_buffer}, {"load_header", lua_xdb_load_header_from_file}, {"load_vector_index", lua_xdb_load_vector_index_from_file}, {"load_content", lua_xdb_load_content_from_file}, {"verify", lua_xdb_verify_from_file}, {"version_from_header", lua_xdb_version_from_header}, {"version_info", lua_xdb_version_info}, {"cleanup", lua_xdb_cleanup}, {"parse_ip", lua_xdb_parse_ip}, {"ip_to_string", lua_xdb_ip_to_string}, {"ip_compare", lua_xdb_ip_compare}, {"now", lua_xdb_now}, {NULL, NULL} }; // module register function int luaopen_xdb_searcher(lua_State *L) { int err = xdb_init_winsock(); if (err != 0) { luaL_error(L, "failed to init the winsock with errno=%d\n", err); return 1; } // create a metatable for xdb buffer object luaL_newmetatable(L, XDB_BUFFER_METATABLE_NAME); lua_pushvalue(L, -1); lua_setfield(L, -2, "__index"); #if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM == 501 // lua 5.1 luaL_register(L, NULL, xdb_buffer_methods); #elif defined(LUA_VERSION_NUM) && LUA_VERSION_NUM >= 502 // lua version 5.2, 5.3, 5.4 ... luaL_setfuncs(L, xdb_buffer_methods, 0); #endif // create a metatable for xdb searcher object luaL_newmetatable(L, XDB_METATABLE_NAME); lua_pushvalue(L, -1); lua_setfield(L, -2, "__index"); #if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM == 501 // lua 5.1 luaL_register(L, NULL, xdb_searcher_methods); luaL_register(L, NULL, xdb_searcher_functions); #elif defined(LUA_VERSION_NUM) && LUA_VERSION_NUM >= 502 // lua version 5.2, 5.3, 5.4 ... luaL_setfuncs(L, xdb_searcher_methods, 0); luaL_setfuncs(L, xdb_searcher_functions, 0); #endif // register the constants attributes lua_pushinteger(L, xdb_ipv4_id); lua_setfield(L, -2, "IPv4"); lua_pushinteger(L, xdb_ipv6_id); lua_setfield(L, -2, "IPv6"); lua_pushinteger(L, xdb_header_buffer); lua_setfield(L, -2, "header_buffer"); lua_pushinteger(L, xdb_vector_index_buffer); lua_setfield(L, -2, "v_index_buffer"); lua_pushinteger(L, xdb_content_buffer); lua_setfield(L, -2, "content_buffer"); return 1; } ================================================ FILE: binding/nginx/Dockerfile ================================================ ARG NGINX_VERSION=1.29.6 FROM nginx:${NGINX_VERSION} AS build ARG NGINX_VERSION # prepare the build environment RUN apt-get update && \ apt-get install -y build-essential libpcre2-dev zlib1g-dev libssl-dev git WORKDIR /usr/src RUN curl -LO https://github.com/nginx/nginx/releases/download/release-${NGINX_VERSION}/nginx-${NGINX_VERSION}.tar.gz && \ tar -zxf nginx-$NGINX_VERSION.tar.gz COPY . /usr/src/ip2region WORKDIR /usr/src/ip2region/binding/c RUN make xdb_searcher_lib # parameters for building dynamic modules WORKDIR /usr/src RUN nginx -V 2>&1 | grep 'configure arguments' | sed 's/ --/ \\\n --/g' | sed "s/pie'/pie' \\\/g" | grep -v 'configure arguments' >> /tmp/conf_arg RUN echo \ ' --add-dynamic-module=$(pwd)/../ip2region/binding/nginx \\\n' \ ' --with-cc-opt="-I $(pwd)/../ip2region/binding/c/build/include" \\\n' \ ' --with-ld-opt="-L $(pwd)/../ip2region/binding/c/build/lib"' >> /tmp/conf_arg RUN cat /tmp/conf_arg WORKDIR /usr/src/nginx-$NGINX_VERSION RUN eval "./configure $(cat /tmp/conf_arg)" RUN make modules && \ cp objs/ngx_http_ip2region_module.so /etc/nginx/modules # for buildx export FROM scratch AS export_so ARG NGINX_VERSION COPY --from=build /etc/nginx/modules/ngx_http_ip2region_module.so /ngx_http_ip2region_module.so ================================================ FILE: binding/nginx/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # nginx-ip2region ## build ```shell $ mkdir -p workspace $ cd workspace $ wget https://nginx.org/download/nginx-1.23.6.tar.gz $ tar -zxf nginx-1.23.6.tar.gz && rm -rf nginx-1.23.6.tar.gz $ git clone https://github.com/lionsoul2014/ip2region.git $ cd ip2region/binding/c $ make xdb_searcher_lib $ cd ../../../nginx-1.23.6 $ ./configure \ --add-module=$(PWD)/../ip2region/binding/nginx \ --with-cc-opt="-I $(PWD)/../ip2region/binding/c/build/include" \ --with-ld-opt="-L $(PWD)/../ip2region/binding/c/build/lib" $ make $ make install ``` ## nginx conf > Syntax: `ip2region_db xdb_file_path [cache_policy Optional]`; > Context: http cache_policy: `file` or `vectorIndex` or `content`, default: `content` Edit `nginx.conf` add `ip2region_db` directive ```nginx ... http { log_format main escape=json '{' '"remote_addr": "$remote_addr", ' '"region": "$ip2region", ' '"http_x_forwarded_for": "$http_x_forwarded_for"' '}'; access_log logs/access.log main; # set xdb file path ip2region_db ip2region.xdb; # ip2region_db ip2region.xdb vectorIndex; # ip2region_db ip2region.xdb file; # ip2region_db ip2region.xdb content; server { listen 80; server_name localhost; location / { root html; index index.html index.htm; } } } ``` Copy `ip2region_v4.xdb` to `nginx/config` folder (rename name it to ip2region.xdb), then restart nginx, the `region` data stored in `ip2region` variable nginx access log sample ```log {"remote_addr": "127.0.0.1", "region": "Reserved|Reserved|Reserved|0|0", "http_x_forwarded_for": ""} {"remote_addr": "127.0.0.1", "region": "Reserved|Reserved|Reserved|0|0", "http_x_forwarded_for": ""} ``` Additionally, you can build the nginx dynamic module using the Dockerfile in the current directory. > * The [buildx](https://github.com/docker/buildx) plugin is required to enable export functionality. ```shell docker build -t export_so -o type=tar,dest=./so.tar . # The final result is a dynamic module named ngx_http_ip2region_module.so. tar xf so.tar && rm so.tar ``` usage of dynamic modules ``` # nginx.conf load_module /etc/nginx/my-modules/ngx_http_ip2region_module.so; http { # ... ip2region_db /etc/nginx/conf.d/ip2region_v4.xdb content; ip2region_db6 /etc/nginx/conf.d/ip2region_v6.xdb content; # ... } ``` ================================================ FILE: binding/nginx/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # nginx-ip2region ## build ```shell $ mkdir -p workspace $ cd workspace $ wget https://nginx.org/download/nginx-1.23.6.tar.gz $ tar -zxf nginx-1.23.6.tar.gz && rm -rf nginx-1.23.6.tar.gz $ git clone https://github.com/lionsoul2014/ip2region.git $ cd ip2region/binding/c $ make xdb_searcher_lib $ cd ../../../nginx-1.23.6 $ ./configure \ --add-module=$(PWD)/../ip2region/binding/nginx \ --with-cc-opt="-I $(PWD)/../ip2region/binding/c/build/include" \ --with-ld-opt="-L$(PWD)/../ip2region/binding/c/build/lib" $ make $ make install ``` ## nginx conf > Syntax: `ip2region_db xdb_file_path [cache_policy Optional]`; > Context: http cache_policy: `file` or `vectorIndex` or `content`, default: `content` Edit `nginx.conf` add `ip2region_db` directive ```nginx ... http { log_format main escape=json '{' '"remote_addr": "$remote_addr", ' '"region": "$ip2region", ' '"http_x_forwarded_for": "$http_x_forwarded_for"' '}'; access_log logs/access.log main; # set xdb file path ip2region_db ip2region.xdb; # ip2region_db ip2region.xdb vectorIndex; # ip2region_db ip2region.xdb file; # ip2region_db ip2region.xdb content; server { listen 80; server_name localhost; location / { root html; index index.html index.htm; } } } ``` Copy `ip2region_v4.xdb` to `nginx/config` folder (rename name it to ip2region.xdb), then restart nginx, the `region` data stored in `ip2region` variable nginx access log sample ```log {"remote_addr": "127.0.0.1", "region": "Reserved|Reserved|Reserved|0|0", "http_x_forwarded_for": ""} {"remote_addr": "127.0.0.1", "region": "Reserved|Reserved|Reserved|0|0", "http_x_forwarded_for": ""} ``` 另外，也可以使用当前路径的 Dockerfile 构建 nginx 动态模块 > *需要安装 [buildx](https://github.com/docker/buildx) 插件以支持导出* ```shell docker build -t export_so -o type=tar,dest=./so.tar . # 最终得到一个叫 ngx_http_ip2region_module.so 的动态模块 tar xf so.tar && rm so.tar ``` 动态模块使用方式 ``` # nginx.conf load_module /etc/nginx/my-modules/ngx_http_ip2region_module.so; http { # ... ip2region_db /etc/nginx/conf.d/ip2region_v4.xdb content; ip2region_db6 /etc/nginx/conf.d/ip2region_v6.xdb content; # ... } ``` ================================================ FILE: binding/nginx/config ================================================ ngx_addon_name=ngx_http_ip2region_module NGX_HTTP_IP2REGION_SRCS=" \ $ngx_addon_dir/src/ngx_http_ip2region_module.c \ " NGX_HTTP_IP2REGION_DEPS=" \ " if test -n "$ngx_module_link"; then ngx_module_type=HTTP ngx_module_name=$ngx_addon_name ngx_module_deps="$NGX_HTTP_IP2REGION_DEPS" ngx_module_srcs="$NGX_HTTP_IP2REGION_SRCS" ngx_module_libs="-lxdb" . auto/module else HTTP_MODULES="$HTTP_MODULES $ngx_addon_name" NGX_ADDON_DEPS="$NGX_ADDON_DEPS $NGX_HTTP_IP2REGION_DEPS" NGX_ADDON_SRCS="$NGX_ADDON_SRCS $NGX_HTTP_IP2REGION_SRCS" fi ================================================ FILE: binding/nginx/src/ngx_http_ip2region_module.c ================================================ /* * Created by Wu Jian Ping on - 2023/03/30. */ #include "ngx_http_ip2region_module.h" static ngx_int_t ngx_http_ip2region_is_absolute_path(char *name); static char *ngx_http_ip2region_init_searcher(ngx_conf_t *cf, char *db_name, char *cache_policy, xdb_version_t *expected_version, const char *directive_name); static ngx_int_t ngx_http_ip2region_add_variables(ngx_conf_t *cf); static void *ngx_http_ip2region_create_conf(ngx_conf_t *cf); static void ngx_http_ip2region_cleanup(void *data); static char *ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static ngx_int_t ngx_http_ip2region_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); static ngx_http_module_t ngx_http_ip2region_ctx = { ngx_http_ip2region_add_variables, /* pre configuration */ NULL, /* post configuration */ ngx_http_ip2region_create_conf, /* create main configuration */ NULL, /* init main configuration */ NULL, /* create server configuration */ NULL, /* merge server configuration */ NULL, /* create location configuration */ NULL /* merge location configuration */ }; static char *ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_ip2region_init_v6(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static ngx_command_t ngx_http_ip2region_commands[] = { { ngx_string("ip2region_db"), NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE12, ngx_http_ip2region_init, NGX_HTTP_MAIN_CONF_OFFSET, 0, NULL }, { ngx_string("ip2region_db6"), NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE12, ngx_http_ip2region_init_v6, NGX_HTTP_MAIN_CONF_OFFSET, 0, NULL }, ngx_null_command }; /* ngx_module_t is required, otherwise failed at complie time */ ngx_module_t ngx_http_ip2region_module = { NGX_MODULE_V1, &ngx_http_ip2region_ctx, /* module context */ ngx_http_ip2region_commands, /* module directives */ NGX_HTTP_MODULE, /* module type */ NULL, /* init master */ NULL, /* init module */ NULL, /* init process */ NULL, /* init thread */ NULL, /* exit thread */ NULL, /* exit process */ NULL, /* exit master */ NGX_MODULE_V1_PADDING }; static ngx_http_variable_t ngx_http_ip2region_vars[] = { { ngx_string("ip2region"), NULL, ngx_http_ip2region_variable, 0, 0, 0 }, ngx_http_null_variable }; // 用于初始化带版本校验的搜索器的辅助函数 static char * ngx_http_ip2region_init_searcher(ngx_conf_t *cf, char *db_name, char *cache_policy, xdb_version_t *expected_version, const char *directive_name) { ip2region_searcher_t *ip2region_searcher; int err; char *db_path; size_t len; if(ngx_http_ip2region_is_absolute_path(db_name) == NGX_OK) { db_path = db_name; } else { // relative path to conf directory len = ngx_cycle->conf_prefix.len + strlen(db_name) + 1; db_path = malloc(len); if (db_path == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to allocate memory for db_path"); return NGX_CONF_ERROR; } memset(db_path, '\0', len); memcpy(db_path, ngx_cycle->conf_prefix.data, ngx_cycle->conf_prefix.len); strcat(db_path, db_name); } ip2region_searcher = ngx_palloc(cf->pool, sizeof(ip2region_searcher_t)); if(ip2region_searcher == NULL) { if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } ip2region_searcher->v_index = NULL; ip2region_searcher->c_buffer = NULL; // 检查XDB文件的版本信息以确定IP类型 xdb_header_t *header = xdb_load_header_from_file(db_path); if (header == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to load xdb header from: %s", db_path); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } xdb_version_t *xdb_version = xdb_version_from_header(header); if (xdb_version == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to determine xdb version from header: %s", db_path); xdb_free_header((void *)header); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } // 验证XDB文件版本是否匹配 if (xdb_version->id != expected_version->id) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "%s expects %s xdb file, but got %s: %s", directive_name, expected_version->name, xdb_version->name, db_path); xdb_free_header((void *)header); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } if (strcmp(cache_policy, "file") == 0) { err = xdb_new_with_file_only(xdb_version, &ip2region_searcher->searcher, db_path); xdb_free_header((void *)header); if (err != 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to create searcher: %s", db_path); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } } else if (strcmp(cache_policy, "vectorIndex") == 0) { ip2region_searcher->v_index = xdb_load_vector_index_from_file(db_path); if (ip2region_searcher->v_index == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to load vector index from: %s", db_path); xdb_free_header((void *)header); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } err = xdb_new_with_vector_index(xdb_version, &ip2region_searcher->searcher, db_path, ip2region_searcher->v_index); xdb_free_header((void *)header); if (err != 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to create vector index cached searcher: %s", db_path); xdb_free_vector_index((void *)ip2region_searcher->v_index); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } } else if (strcmp(cache_policy, "content") == 0) { ip2region_searcher->c_buffer = xdb_load_content_from_file(db_path); if (ip2region_searcher->c_buffer == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to load xdb content: %s", db_path); xdb_free_header((void *)header); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } err = xdb_new_with_buffer(xdb_version, &ip2region_searcher->searcher, ip2region_searcher->c_buffer); xdb_free_header((void *)header); if (err != 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "failed to create content cached searcher: %s", db_path); xdb_free_content((void *)ip2region_searcher->c_buffer); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } } else { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid cache policy, options: file/vectorIndex/content"); xdb_free_header((void *)header); if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return NGX_CONF_ERROR; } if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) { free(db_path); } return (char *)ip2region_searcher; } static char * ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_ip2region_conf_t *ip2region_cf; char *db_name, *cache_policy; ngx_str_t *value; char *result; ip2region_cf = conf; if (ip2region_cf->v4_searcher) { return "ip2region_db is duplicate"; } value = cf->args->elts; db_name = (char *)value[1].data; // default cache_policy: content if(cf->args->nelts == 2) { cache_policy = "content"; } else { cache_policy = (char *)value[2].data; } result = ngx_http_ip2region_init_searcher(cf, db_name, cache_policy, xdb_version_v4(), "ip2region_db"); if (result == NGX_CONF_ERROR) { return NGX_CONF_ERROR; } ip2region_cf->v4_searcher = (ip2region_searcher_t *)result; return NGX_CONF_OK; } static char * ngx_http_ip2region_init_v6(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_ip2region_conf_t *ip2region_cf; char *db_name, *cache_policy; ngx_str_t *value; char *result; ip2region_cf = conf; if (ip2region_cf->v6_searcher) { return "ip2region_db6 is duplicate"; } value = cf->args->elts; db_name = (char *)value[1].data; // default cache_policy: content if(cf->args->nelts == 2) { cache_policy = "content"; } else { cache_policy = (char *)value[2].data; } result = ngx_http_ip2region_init_searcher(cf, db_name, cache_policy, xdb_version_v6(), "ip2region_db6"); if (result == NGX_CONF_ERROR) { return NGX_CONF_ERROR; } ip2region_cf->v6_searcher = (ip2region_searcher_t *)result; return NGX_CONF_OK; } static void * ngx_http_ip2region_create_conf(ngx_conf_t *cf) { ngx_pool_cleanup_t *cln; ngx_http_ip2region_conf_t *conf; conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_ip2region_conf_t)); if (conf == NULL) { return NULL; } cln = ngx_pool_cleanup_add(cf->pool, 0); if (cln == NULL) { return NULL; } cln->handler = ngx_http_ip2region_cleanup; cln->data = conf; return conf; } static ngx_int_t ngx_http_ip2region_add_variables(ngx_conf_t *cf) { ngx_http_variable_t *var; ngx_http_variable_t *v; for (v = ngx_http_ip2region_vars; v->name.len; v++) { var = ngx_http_add_variable(cf, &v->name, v->flags); if (var == NULL) { return NGX_ERROR; } var->get_handler = v->get_handler; var->data = v->data; } return NGX_OK; } static ngx_int_t ngx_http_ip2region_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { ngx_http_ip2region_conf_t *ip2region_conf; struct sockaddr_in *sin; char region_buffer[512] = {'\0'}; xdb_region_buffer_t region; int err = 1; unsigned int ip; xdb_searcher_t *searcher_ptr = NULL; #if (NGX_HAVE_INET6) u_char *p; in_addr_t addr; struct sockaddr_in6 *sin6; #endif ip2region_conf = ngx_http_get_module_main_conf(r, ngx_http_ip2region_module); if (ip2region_conf->v4_searcher == NULL && ip2region_conf->v6_searcher == NULL) { v->not_found = 1; return NGX_OK; } // 初始化 region buffer err = xdb_region_buffer_init(®ion, region_buffer, sizeof(region_buffer)); if (err != 0) { v->not_found = 1; return NGX_OK; } switch (r->connection->sockaddr->sa_family) { case AF_INET: sin = (struct sockaddr_in *) r->connection->sockaddr; // 检查是否有IPv4 searcher if (ip2region_conf->v4_searcher == NULL) { ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, "no IPv4 searcher available for IPv4 address"); break; } searcher_ptr = &ip2region_conf->v4_searcher->searcher; // 正确转换IP地址字节序，按ip2region期望的格式 ip = ntohl(sin->sin_addr.s_addr); // 将网络字节序转换为主机字节序 // 按照xdb_parse_v4_ip中的格式重新组织字节 { bytes_ip_t ip_bytes[4]; ip_bytes[0] = (ip >> 24) & 0xFF; ip_bytes[1] = (ip >> 16) & 0xFF; ip_bytes[2] = (ip >> 8) & 0xFF; ip_bytes[3] = ip & 0xFF; err = xdb_search(searcher_ptr, ip_bytes, 4, ®ion); } if (err == 0) { v->data = (unsigned char *)region.value; v->len = strlen(region.value); xdb_region_buffer_free(®ion); return NGX_OK; } else { ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, "ip2region search failed for IPv4 address"); } break; #if (NGX_HAVE_INET6) case AF_INET6: sin6 = (struct sockaddr_in6 *) r->connection->sockaddr; p = sin6->sin6_addr.s6_addr; if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) { // 处理IPv4映射的IPv6地址 if (ip2region_conf->v4_searcher != NULL) { searcher_ptr = &ip2region_conf->v4_searcher->searcher; addr = p[12] << 24; addr += p[13] << 16; addr += p[14] << 8; addr += p[15]; // 按照xdb_parse_v4_ip中的格式重新组织字节 { bytes_ip_t ip_bytes[4]; ip_bytes[0] = (addr >> 24) & 0xFF; ip_bytes[1] = (addr >> 16) & 0xFF; ip_bytes[2] = (addr >> 8) & 0xFF; ip_bytes[3] = addr & 0xFF; err = xdb_search(searcher_ptr, ip_bytes, 4, ®ion); } if (err == 0) { v->data = (unsigned char *)region.value; v->len = strlen(region.value); xdb_region_buffer_free(®ion); return NGX_OK; } else { ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, "ip2region search failed for IPv4-mapped IPv6 address"); } } } else { // 处理纯IPv6地址 if (ip2region_conf->v6_searcher != NULL) { searcher_ptr = &ip2region_conf->v6_searcher->searcher; bytes_ip_t ip6_bytes[16]; if (p != NULL) { memcpy(ip6_bytes, p, 16); err = xdb_search(searcher_ptr, ip6_bytes, 16, ®ion); if (err == 0) { v->data = (unsigned char *)region.value; v->len = strlen(region.value); xdb_region_buffer_free(®ion); return NGX_OK; } else { ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, "ip2region search failed for IPv6 address"); } } } else { ngx_log_error(NGX_LOG_WARN, r->connection->log, 0, "no IPv6 searcher available for IPv6 address"); } } break; #endif } // 如果搜索失败，释放 region buffer xdb_region_buffer_free(®ion); ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, "ip2region: no region found for IP address"); v->not_found = 1; return NGX_OK; } static void ngx_http_ip2region_cleanup(void *data) { ngx_http_ip2region_conf_t *ip2region_conf = data; // 清理 IPv4 searcher if(ip2region_conf->v4_searcher != NULL) { xdb_close(&ip2region_conf->v4_searcher->searcher); // check and free the vector index if (ip2region_conf->v4_searcher->v_index != NULL) { xdb_free_vector_index((void *)ip2region_conf->v4_searcher->v_index); ip2region_conf->v4_searcher->v_index = NULL; } // check and free the content buffer if (ip2region_conf->v4_searcher->c_buffer != NULL) { xdb_free_content((void *)ip2region_conf->v4_searcher->c_buffer); ip2region_conf->v4_searcher->c_buffer = NULL; } ip2region_conf->v4_searcher = NULL; } // 清理 IPv6 searcher if(ip2region_conf->v6_searcher != NULL) { xdb_close(&ip2region_conf->v6_searcher->searcher); // check and free the vector index if (ip2region_conf->v6_searcher->v_index != NULL) { xdb_free_vector_index((void *)ip2region_conf->v6_searcher->v_index); ip2region_conf->v6_searcher->v_index = NULL; } // check and free the content buffer if (ip2region_conf->v6_searcher->c_buffer != NULL) { xdb_free_content((void *)ip2region_conf->v6_searcher->c_buffer); ip2region_conf->v6_searcher->c_buffer = NULL; } ip2region_conf->v6_searcher = NULL; } } static ngx_int_t ngx_http_ip2region_is_absolute_path(char *name) { #if (NGX_WIN32) u_char c0, c1; c0 = name[0]; if (strlen(name) < 2) { if (c0 == '/') { return 2; } return NGX_DECLINED; } c1 = name[1]; if (c1 == ':') { c0 |= 0x20; if ((c0 >= 'a' && c0 <= 'z')) { return NGX_OK; } return NGX_DECLINED; } if (c1 == '/') { return NGX_OK; } if (c0 == '/') { return 2; } return NGX_DECLINED; #else if (name[0] == '/') { return NGX_OK; } return NGX_DECLINED; #endif } ================================================ FILE: binding/nginx/src/ngx_http_ip2region_module.h ================================================ /* * Created by Wu Jian Ping on - 2023/03/30. */ #ifndef __NGX_HTTP_IP2REGION_MODULE_H_INCLUDED__ #define __NGX_HTTP_IP2REGION_MODULE_H_INCLUDED__ #include #include #include #include #if (NGX_HAVE_INET6) #include #include #include #endif typedef struct { xdb_searcher_t searcher; xdb_vector_index_t *v_index; xdb_content_t *c_buffer; } ip2region_searcher_t; typedef struct { ip2region_searcher_t *v4_searcher; // IPv4 searcher for ip2region_db ip2region_searcher_t *v6_searcher; // IPv6 searcher for ip2region_db6 } ngx_http_ip2region_conf_t; #endif ================================================ FILE: binding/nginx/t/http_ip2region.t ================================================ use lib 'lib'; use Test::Nginx::Socket; # 'no_plan'; repeat_each(2); plan tests => repeat_each() * 124; no_long_string(); #no_diff; run_tests(); __DATA__ === TEST 1: set request header at client side --- config location /foo { echo $http_x_foo; } --- request GET /foo --- more_headers X-Foo: blah --- response_headers ! X-Foo --- response_body blah ================================================ FILE: binding/nodejs/README.md ================================================ # :cn: [中文简体] # ip2region nodejs 查询客户端 请使用最新的 IPv6 兼容的 javascript binding：[javascript binding](../javascript/) --- # :globe_with_meridians: [English] # ip2region nodejs query client Please use the latest IPv6-compliant JavaScript binding: [javascript binding](../javascript/) ================================================ FILE: binding/php/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region PHP Query Client # Usage ### About Query API The prototype of the Query API is as follows: ```php // Query via string IP // @throw Exception search($ip_string) string // Query via binary IP returned by Util.parseIP // @throw Exception searchByBytes($ip_bytes) string ``` If the query fails, an exception will be thrown; if the query is successful, the `region` information string will be returned; if the IP being queried cannot be found, an empty string `""` will be returned. ### About IPv4 and IPv6 This xdb query client implementation supports both IPv4 and IPv6 queries. The usage is as follows: ```php use \ip2region\xdb\{IPv4, IPv6}; // For IPv4: Set xdb path to the v4 xdb file, specify IP version as IPv4 $dbFile = "../../data/ip2region_v4.xdb"; // or your ipv4 xdb path $version = IPv4::default(); // For IPv6: Set xdb path to the v6 xdb file, specify IP version as IPv6 $dbFile = "../../data/ip2region_v6.xdb"; // or your ipv6 xdb path $version = IPv6::default(); // The IP version of the xdb specified by dbPath must be consistent with the version specified, otherwise an error will occur during query execution // Note: The following demonstration directly uses $dbFile and $version variables ``` ### XDB File Verification It is recommended that you proactively verify the applicability of the xdb file, as some new features in the future may cause the current Searcher version to be incompatible with the xdb file you are using. Verification can avoid unpredictable errors during runtime. You do not need to verify every time; for example, verify when the service starts or manually call the command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios. ```php use \ip2region\xdb\Util; $err = Util::verify($dbFile); if ($err != null) { // Applicability verification failed!!! // The current query client implementation is not suitable for querying the xdb file specified by dbPath. // You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with dbPath. printf("failed to verify xdb file `%s`: %s\n", $dbFile, $err); return; } // Verification passed, the current Searcher can be safely used for query operations on the xdb pointed to by dbPath ``` ### File-Based Query ```php // require or autoload the xdb\Searcher require 'xdb\Searcher.php'; use \ip2region\xdb\Util; use \ip2region\xdb\Searcher; // 1. Create a Searcher object using the $version and $dbFile mentioned above try { $searcher = Searcher::newWithFileOnly($version, $dbFile); } catch (Exception $e) { printf("failed to create searcher with '%s': %s\n", $dbFile, $e->getMessage()); return; } // 2. Query, both IPv4 or IPv6 addresses are supported try { $ip = '1.2.3.4'; // $ip = ""240e:3b7:3272:d8d0:db09:c067:8d59:539e; // IPv6 $sTime = Util::now(); $region = $searcher->search($ip); $costMs = Util::now() - $sTime; printf("{region: %s, took: %.5f ms}\n", $region, $costMs); } catch (Exception $e) { printf("failed to search(%s): %s", $ip, $e->getMessage()); } // 3. Close resources $searcher->close(); // Note: For concurrent use, each thread or coroutine needs to create an independent searcher object. ``` ### Caching `VectorIndex` If your PHP environment supports it, you can pre-load the vectorIndex cache and make it a global variable. Using the global vectorIndex every time you create a Searcher can reduce a fixed IO operation, thereby accelerating queries and reducing IO pressure. ```php // require or autoload the xdb\Searcher require 'xdb\Searcher.php'; use \ip2region\xdb\Util; use \ip2region\xdb\Searcher; // 1. Load VectorIndex cache from $dbFile and cache the following vIndex variable in memory. $vIndex = Util::loadVectorIndexFromFile($dbFile); if ($vIndex === null) { printf("failed to load vector index from '%s'\n", $dbFile); return; } // 2. Use the global vIndex to create a query object with VectorIndex cache. try { $searcher = Searcher::newWithVectorIndex($version, $dbFile, $vIndex); } catch (Exception $e) { printf("failed to create vectorIndex searcher with '%s': %s\n", $dbFile, $e->getMessage()); return; } // 3. Query, both IPv4 or IPv6 are supported try { $ip = '1.2.3.4'; // $ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 $sTime = Util::now(); $region = $searcher->search($ip); $costMs = Util::now() - $sTime; printf("{region: %s, took: %.5f ms}\n", $region, $costMs); } catch (Exception $e) { printf("failed to search(%s): %s", $ip, $e->getMessage()); } // 4. Close resources $searcher->close(); // Note: For concurrent use, each thread or coroutine needs to create an independent searcher object, but they all share the same read-only global vectorIndex. ``` ### Caching the Entire `xdb` File If your PHP environment supports it, you can pre-load the entire `xdb` file into memory. This allows for fully memory-based queries, similar to the previous memory search. ```php // require or autoload the xdb\Searcher require 'xdb\Searcher.php'; use \ip2region\xdb\Util; use \ip2region\xdb\Searcher; // 1. Load the entire xdb from $dbFile into memory. $cBuff = Util::loadContentFromFile($dbFile); if ($cBuff === null) { printf("failed to load content buffer from '%s'\n", $dbFile); return; } // 2. Use the global cBuff to create a query object that is fully based on memory. try { $searcher = Searcher::newWithBuffer($version, $cBuff); } catch (Exception $e) { printf("failed to create buffer cached searcher: %s\n", $dbFile, $e->getMessage()); return; } // 3. Query, both IPv4 or IPv6 are supported try { $ip = '1.2.3.4'; // $ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 $sTime = Util::now(); $region = $searcher->search($ip); $costMs = Util::now() - $sTime; printf("{region: %s, took: %.5f ms}\n", $region, $costMs); } catch (Exception $e) { printf("failed to search(%s): %s", $ip, $e->getMessage()); } // 4. Close resources // This searcher object can be safely used for concurrency; close it when the entire service is shut down // $searcher->close(); // Note: For concurrent use, the searcher object created with the entire xdb cache can be safely used for concurrency. ``` # Query Testing Run query tests via the `search_test.php` script: ```bash ➜ php git:(fr_php_ipv6) ✗ php search_test.php php search_test.php [command options] options: --db string ip2region binary xdb file path --cache-policy string cache policy: file/vectorIndex/content ``` For example: using the default data/ip2region_v4.xdb for IPv4 query testing: ```bash ➜ php git:(fr_php_ipv6) ✗ php search_test.php --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb file: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 0.12695 ms} ip2region>> 120.229.45.2 {region: 中国|广东省|深圳市|移动|CN, ioCount: 3, took: 0.07397 ms} ``` For example: using the default data/ip2region_v6.xdb for IPv6 query testing: ```bash ➜ php git:(master) ✗ php ./search_test.php --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb file: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> :: {region: , ioCount: 1, took: 0.08887 ms} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 0.10303 ms} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, ioCount: 13, took: 0.04614 ms} ``` Enter an IP to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the efficiency of the three different cache implementations. # Bench Testing Run automatic bench testing via the `bench_test.php` script. On one hand, this ensures that there are no errors in the `xdb` file; on the other hand, it tests average query performance through a large number of queries: ```bash ➜ php git:(fr_php_ipv6) ✗ php bench_test.php php bench_test.php [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` For example: perform an IPv4 bench test using the default data/ip2region_v4.xdb and data/ipv4_source.txt files: ```bash php bench_test.php --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` For example: perform an IPv6 bench test using the default data/ip2region_v6.xdb and data/ipv6_source.txt files: ```bash php bench_test.php --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` You can test the performance of the three different cache implementations (file/vectorIndex/content) by setting the `cache-policy` parameter. @Note: Please ensure that the src file used for benching is the same source file used to generate the corresponding xdb file. ### Third-party Repository Support 1. Composer supported [zoujingli/ip2region](https://github.com/zoujingli/ip2region) - IPv6 supported. ================================================ FILE: binding/php/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region php 查询客户端 # 使用方式 ### 关于查询 API 查询 API 的原型如下： ```php // 通过字符串 IP 进行查询 // @throw Exception search($ip_string) string // 通过 Util.parseIP 返回的二进制 IP 进行查询 // @throw Exception searchByBytes($ip_bytes) string ``` 如果查询失败则会抛出异常，如果查询成功则会返回字符串的 `region` 信息，如果查询的 IP 找不到则会返回空字符串 `""`。 ### 关于 IPv4 和 IPv6 该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询，使用方式如下： ```php use \ip2region\xdb\{IPv4, IPv6}; // 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 IPv4 $dbFile = "../../data/ip2region_v4.xdb"; // 或者你的 ipv4 xdb 的路径 $version = IPv4::default(); // 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件，IP版本指定为 IPv6 $dbFile = "../../data/ip2region_v6.xdb"; // 或者你的 ipv6 xdb 路径 $version = IPv6::default(); // dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致，不然查询执行的时候会报错 // 备注：以下演示直接使用 $dbFile 和 $version 变量 ``` ### XDB 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 ```php use \ip2region\xdb\Util; $err = Util::verify($dbFile); if ($err != null) { // 适用性验证失败！！！ // 当前查询客户端实现不适用于 dbPath 指定的 xdb 文件的查询. // 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 dbPath 的 Searcher 实现。 printf("failed to verify xdb file `%s`: %s\n", $dbFile, $err); return; } // 验证通过，当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作 ``` ### 完全基于文件的查询 ```php // require or autoload the xdb\Searcher require 'xdb\Searcher.php'; use \ip2region\xdb\Util; use \ip2region\xdb\Searcher; // 1, 使用上述的 $version 和 $dbFile 创建 Searcher 对象 try { $searcher = Searcher::newWithFileOnly($version, $dbFile); } catch (Exception $e) { printf("failed to create searcher with '%s': %s\n", $dbFile, $e->getMessage()); return; } // 2, 查询，IPv4 或者 IPv6 的地址都支持 try { $ip = '1.2.3.4'; // $ip = ""240e:3b7:3272:d8d0:db09:c067:8d59:539e; // IPv6 $sTime = Util::now(); $region = $searcher->search($ip); $costMs = Util::now() - $sTime; printf("{region: %s, took: %.5f ms}\n", $region, $costMs); } catch (Exception $e) { printf("failed to search(%s): %s", $ip, $e->getMessage()); } // 3，关闭资源 $searcher->close(); // 备注：并发使用，每个线程或者协程需要创建一个独立的 searcher 对象。 ``` ### 缓存 `VectorIndex` 索引 如果你的 php 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。 ```php // require or autoload the xdb\Searcher require 'xdb\Searcher.php'; use \ip2region\xdb\Util; use \ip2region\xdb\Searcher; // 1、从 $dbFile 加载 VectorIndex 缓存，把下述的 vIndex 变量缓存到内存里面。 $vIndex = Util::loadVectorIndexFromFile($dbFile); if ($vIndex === null) { printf("failed to load vector index from '%s'\n", $dbFile); return; } // 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。 try { $searcher = Searcher::newWithVectorIndex($version, $dbFile, $vIndex); } catch (Exception $e) { printf("failed to create vectorIndex searcher with '%s': %s\n", $dbFile, $e->getMessage()); return; } // 3、查询，IPv4 或者 IPv6 都支持 try { $ip = '1.2.3.4'; // $ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 $sTime = Util::now(); $region = $searcher->search($ip); $costMs = Util::now() - $sTime; printf("{region: %s, took: %.5f ms}\n", $region, $costMs); } catch (Exception $e) { printf("failed to search(%s): %s", $ip, $e->getMessage()); } // 4, 关闭资源 $searcher->close(); // 备注：并发使用，每个线程或者协程需要创建一个独立的 searcher 对象，但是都共享统一的只读全局 vectorIndex。。 ``` ### 缓存整个 `xdb` 文件 如果你的 PHP 母环境支持，可以预先加载整个 `xdb` 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。 ```php // require or autoload the xdb\Searcher require 'xdb\Searcher.php'; use \ip2region\xdb\Util; use \ip2region\xdb\Searcher; // 1、从 $dbFile 加载整个 xdb 到内存。 $cBuff = Util::loadContentFromFile($dbFile); if ($cBuff === null) { printf("failed to load content buffer from '%s'\n", $dbFile); return; } // 2、使用全局的 cBuff 创建带完全基于内存的查询对象。 try { $searcher = Searcher::newWithBuffer($version, $cBuff); } catch (Exception $e) { printf("failed to create buffer cached searcher: %s\n", $dbFile, $e->getMessage()); return; } // 3、查询，IPv4 或者 IPv6 都支持 try { $ip = '1.2.3.4'; // $ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6 $sTime = Util::now(); $region = $searcher->search($ip); $costMs = Util::now() - $sTime; printf("{region: %s, took: %.5f ms}\n", $region, $costMs); } catch (Exception $e) { printf("failed to search(%s): %s", $ip, $e->getMessage()); } // 4，关闭资源 // 该 searcher 对象可以安全的用于并发，等整个服务都关闭的时候再关闭 searcher // $searcher->close(); // 备注：并发使用，用整个 xdb 缓存创建的 searcher 对象可以安全用于并发。 ``` # 查询测试 通过 `search_test.php` 脚本来进行查询测试： ```bash ➜ php git:(fr_php_ipv6) ✗ php search_test.php php search_test.php [command options] options: --db string ip2region binary xdb file path --cache-policy string cache policy: file/vectorIndex/content ``` 例如：使用默认的 data/ip2region_v4.xdb 进行 IPv4 的查询测试： ```bash ➜ php git:(fr_php_ipv6) ✗ php search_test.php --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb file: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 0.12695 ms} ip2region>> 120.229.45.2 {region: 中国|广东省|深圳市|移动|CN, ioCount: 3, took: 0.07397 ms} ``` 例如：使用默认的 data/ip2region_v6.xdb 进行 IPv6 的查询测试： ```bash ➜ php git:(master) ✗ php ./search_test.php --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb file: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> :: {region: , ioCount: 1, took: 0.08887 ms} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 0.10303 ms} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, ioCount: 13, took: 0.04614 ms} ``` 输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。 # bench 测试 通过 `bench_test.php` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能： ```bash ➜ php git:(fr_php_ipv6) ✗ php bench_test.php php bench_test.php [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --cache-policy string cache policy: file/vectorIndex/content ``` 例如：通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 文件进行 IPv4 的 bench 测试： ```bash php bench_test.php --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` 例如：通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 文件进行 IPv6 的 bench 测试： ```bash php bench_test.php --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` 可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。 @Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。 ### 第三方仓库支持 1. composer 支持的 [zoujingli/ip2region](https://github.com/zoujingli/ip2region) - 已提供 IPv6 支持。 ================================================ FILE: binding/php/batch_test.php ================================================ // @Date 2022/06/22 require dirname(__FILE__) . '/xdb/Searcher.class.php'; use \ip2region\xdb\Util; use \ip2region\xdb\{IPv4, IPv6}; use \ip2region\xdb\Searcher; function printHelp($argv) { printf("php %s [command options]\n", $argv[0]); printf("options: \n"); printf(" --db string ip2region binary xdb file path\n"); printf(" --src string source ip text file path\n"); printf(" --cache-policy string cache policy: file/vectorIndex/content\n"); } if($argc < 2) { printHelp($argv); return; } $dbFile = ""; $srcFile = ""; $cachePolicy = 'vectorIndex'; array_shift($argv); foreach ($argv as $r) { if (strlen($r) < 5) { continue; } if (strpos($r, '--') != 0) { continue; } $sIdx = strpos($r, "="); if ($sIdx < 0) { printf("missing = for args pair %s\n", $r); return; } $key = substr($r, 2, $sIdx - 2); $val = substr($r, $sIdx + 1); if ($key == 'db') { $dbFile = $val; } else if ($key == 'src') { $srcFile = $val; } else if ($key == 'cache-policy') { $cachePolicy = $val; } else { printf("undefined option `%s`\n", $r); return; } } if (strlen($dbFile) < 1 || strlen($srcFile) < 1) { printHelp($argv); return; } // printf("debug: dbFile: %s, cachePolicy: %s\n", $dbFile, $cachePolicy); $handle = fopen($dbFile, 'r'); if ($handle === false) { printf("failed to open the xdb file `{$dbFile}`\n"); return; } // load header $header = Util::loadHeader($handle); if ($header == null) { printf("failed to load the header\n"); return; } // get the version number from the xdb header try { $version = Util::versionFromHeader($header); } catch (Exception $e) { printf("failed to detect version from header: {$e->getMessage()}\n"); return; } // create the xdb searcher by the cache-policy switch ( $cachePolicy ) { case 'file': try { $searcher = Searcher::newWithFileOnly($version, $dbFile); } catch (Exception $e) { printf("failed to create searcher with '%s': %s\n", $dbFile, $e); return; } break; case 'vectorIndex': $vIndex = Util::loadVectorIndex($handle); if ($vIndex == null) { printf("failed to load vector index from '%s'\n", $dbFile); return; } try { $searcher = Searcher::newWithVectorIndex($version, $dbFile, $vIndex); } catch (Exception $e) { printf("failed to create vector index cached searcher with '%s': %s\n", $dbFile, $e); return; } break; case 'content': $cBuff = Util::loadContent($handle); if ($cBuff == null) { printf("failed to load xdb content from '%s'\n", $dbFile); return; } try { $searcher = Searcher::newWithBuffer($version, $cBuff); } catch (Exception $e) { printf("failed to create content cached searcher: %s", $e); return; } break; default: printf("undefined cache-policy `%s`\n", $cachePolicy); return; } // do the bench test $handle = fopen($srcFile, "r"); if ($handle === false) { printf("failed to open source text file `%s`\n", $srcFile); return null; } $count = 0; $qx_count = 0; while (!feof($handle)) { $line = trim(fgets($handle, 1024)); if (strlen($line) < 1) { continue; } //@Note extract the ip address from line with special chars // if (preg_match('/^([0-9\.]+)/', $line, $m) != 1) { // continue; // } // $line = $m[1]; // print("ip: {$m[1]}\n"); try { $ipBytes = Util::parseIP($line); } catch (Exception $e) { printf("failed to parse ip `%s`: %s\n", $line, $e->getMessage()); continue; } $count++; $region = $searcher->searchByBytes($ipBytes); $ss = explode('|', $region); echo $line, ",", str_replace('|', ',', $region), "\n"; } fclose($handle); echo "Done, with {$count} IPs\n"; ================================================ FILE: binding/php/bench_test.php ================================================ // @Date 2022/06/22 require dirname(__FILE__) . '/xdb/Searcher.class.php'; use \ip2region\xdb\Util; use \ip2region\xdb\{IPv4, IPv6}; use \ip2region\xdb\Searcher; function printHelp($argv) { printf("php %s [command options]\n", $argv[0]); printf("options: \n"); printf(" --db string ip2region binary xdb file path\n"); printf(" --src string source ip text file path\n"); printf(" --cache-policy string cache policy: file/vectorIndex/content\n"); } if($argc < 2) { printHelp($argv); return; } $dbFile = ""; $srcFile = ""; $cachePolicy = 'vectorIndex'; array_shift($argv); foreach ($argv as $r) { if (strlen($r) < 5) { continue; } if (strpos($r, '--') != 0) { continue; } $sIdx = strpos($r, "="); if ($sIdx < 0) { printf("missing = for args pair %s\n", $r); return; } $key = substr($r, 2, $sIdx - 2); $val = substr($r, $sIdx + 1); if ($key == 'db') { $dbFile = $val; } else if ($key == 'src') { $srcFile = $val; } else if ($key == 'cache-policy') { $cachePolicy = $val; } else { printf("undefined option `%s`\n", $r); return; } } if (strlen($dbFile) < 1 || strlen($srcFile) < 1) { printHelp($argv); return; } // printf("debug: dbFile: %s, cachePolicy: %s\n", $dbFile, $cachePolicy); $handle = fopen($dbFile, 'r'); if ($handle === false) { printf("failed to open the xdb file `{$dbFile}`\n"); return; } // verify the xdb file // @Note: do NOT call it every time you create a searcher since this will slow // down the search response. // @see the Util.verify function for details. $err = Util::verify($handle); if ($err != null) { printf("failed to verify xdb file `%s`: %s\n", $dbFile, $err); return; } // load header $header = Util::loadHeader($handle); if ($header == null) { printf("failed to load the header\n"); return; } // get the version number from the xdb header try { $version = Util::versionFromHeader($header); } catch (Exception $e) { printf("failed to detect version from header: {$e->getMessage()}\n"); return; } // create the xdb searcher by the cache-policy switch ( $cachePolicy ) { case 'file': try { $searcher = Searcher::newWithFileOnly($version, $dbFile); } catch (Exception $e) { printf("failed to create searcher with '%s': %s\n", $dbFile, $e); return; } break; case 'vectorIndex': $vIndex = Util::loadVectorIndex($handle); if ($vIndex == null) { printf("failed to load vector index from '%s'\n", $dbFile); return; } try { $searcher = Searcher::newWithVectorIndex($version, $dbFile, $vIndex); } catch (Exception $e) { printf("failed to create vector index cached searcher with '%s': %s\n", $dbFile, $e); return; } break; case 'content': $cBuff = Util::loadContent($handle); if ($cBuff == null) { printf("failed to load xdb content from '%s'\n", $dbFile); return; } try { $searcher = Searcher::newWithBuffer($version, $cBuff); } catch (Exception $e) { printf("failed to create content cached searcher: %s", $e); return; } break; default: printf("undefined cache-policy `%s`\n", $cachePolicy); return; } // do the bench test $handle = fopen($srcFile, "r"); if ($handle === false) { printf("failed to open source text file `%s`\n", $srcFile); return null; } $count = 0; $costs = 0; $sTime = Util::now(); while (!feof($handle)) { $line = trim(fgets($handle, 1024)); if (strlen($line) < 1) { continue; } // ignore the comment if ($line[0] == '#') { continue; } $ps = explode('|', $line, 3); if (count($ps) != 3) { printf("invalid ip segment line `${line}`\n"); return; } $sip = Util::parseIP($ps[0]); if ($sip === null) { printf("invalid start ip `%s`\n", $ps[0]); return; } $eip = Util::parseIP($ps[1]); if ($eip === null) { printf("invalid end ip `%s`\n", $ps[1]); return; } if (Util::ipCompare($sip, $eip) > 0) { printf( "start ip(%s) should not be greater than end ip(%s)\n", Util::ipToString($ps[0]), Util::ipToString($ps[1]) ); return; } foreach ([$sip, $eip] as $ip) { try { $cTime = Util::now(); $region = $searcher->searchByBytes($ip); $costs += Util::now() - $cTime; } catch (Exception $e) { printf("failed to search ip `%s`: %s\n", Util::ipToString($ip), $e->getMessage()); return; } if ($region == null) { printf("failed to search ip `%s`: empty region info\n", Util::ipToString($ip)); return; } // check the region info if ($region != $ps[2]) { printf("failed search(%s) with (%s != %s)\n", Util::ipToString($ip), $region, $ps[2]); return; } $count++; } } // close the searcher at last fclose($handle); $searcher->close(); printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %.3f ms/op}\n", $cachePolicy, $count, (Util::now() - $sTime)/1000, $count == 0 ? 0 : $costs/$count); ================================================ FILE: binding/php/search_test.php ================================================ // @Date 2022/06/21 require dirname(__FILE__) . '/xdb/Searcher.class.php'; use \ip2region\xdb\Util; use \ip2region\xdb\{IPv4, IPv6}; use \ip2region\xdb\Searcher; function printHelp($argv) { printf("php %s [command options]\n", $argv[0]); printf("options: \n"); printf(" --db string ip2region binary xdb file path\n"); printf(" --cache-policy string cache policy: file/vectorIndex/content\n"); } if($argc < 2) { printHelp($argv); return; } $dbFile = ""; $cachePolicy = 'vectorIndex'; array_shift($argv); foreach ($argv as $r) { if (strlen($r) < 5) { continue; } if (strpos($r, '--') != 0) { continue; } $sIdx = strpos($r, "="); if ($sIdx < 0) { printf("missing = for args pair %s\n", $r); return; } $key = substr($r, 2, $sIdx - 2); $val = substr($r, $sIdx + 1); if ($key == 'db') { $dbFile = $val; } else if ($key == 'cache-policy') { $cachePolicy = $val; } else { printf("undefined option `%s`\n", $r); return; } } if (strlen($dbFile) < 1) { printHelp($argv); return; } // printf("debug: dbFile: %s, cachePolicy: %s\n", $dbFile, $cachePolicy); $handle = fopen($dbFile, 'r'); if ($handle === false) { printf("failed to open the xdb file `{$dbFile}`\n"); return; } // verify the xdb file // @Note: do NOT call it every time you create a searcher since this will slow // down the search response. // @see the Util.verify function for details. $err = Util::verify($handle); if ($err != null) { printf("failed to verify xdb file `%s`: %s\n", $dbFile, $err); return; } // load header $header = Util::loadHeader($handle); if ($header == null) { printf("failed to load the header\n"); return; } // get the version number from the xdb header try { $version = Util::versionFromHeader($header); } catch (Exception $e) { printf("failed to detect version from header: {$e->getMessage()}\n"); return; } // create the xdb searcher by the cache-policy switch ( $cachePolicy ) { case 'file': try { $searcher = Searcher::newWithFileOnly($version, $dbFile); } catch (Exception $e) { printf("failed to create searcher with '%s': %s\n", $dbFile, $e); return; } break; case 'vectorIndex': $vIndex = Util::loadVectorIndex($handle); if ($vIndex == null) { printf("failed to load vector index from '%s'\n", $dbFile); return; } try { $searcher = Searcher::newWithVectorIndex($version, $dbFile, $vIndex); } catch (Exception $e) { printf("failed to create vector index cached searcher with '%s': %s\n", $dbFile, $e); return; } break; case 'content': $cBuff = Util::loadContent($handle); if ($cBuff == null) { printf("failed to load xdb content from '%s'\n", $dbFile); return; } try { $searcher = Searcher::newWithBuffer($version, $cBuff); } catch (Exception $e) { printf("failed to create content cached searcher: %s", $e); return; } break; default: printf("undefined cache-policy `%s`\n", $cachePolicy); return; } printf(<<name}, ${cachePolicy}) type 'quit' to exit\n EOF); while ( true ) { echo "ip2region>> "; $line = trim(fgets(STDIN)); if (strlen($line) < 2) { continue; } if ($line == 'quit') { break; } $cost = -1; try { $sTime = Util::now(); $region = $searcher->search($line); $cost = Util::now() - $sTime; } catch (Exception $e) { printf("search call failed: %s\n", $e->getMessage()); continue; } printf( "{region: %s, ioCount: %d, took: %.5f ms}\n", $region, $searcher->getIOCount(), $cost ); } // close the searcher at last $searcher->close(); printf("searcher test program exited, thanks for trying\n"); ================================================ FILE: binding/php/xdb/Searcher.class.php ================================================ // @Date 2022/06/21 namespace ip2region\xdb; use \Exception; // global constants const Structure_20 = 2; const Structure_30 = 3; const IPv4VersionNo = 4; const IPv6VersionNo = 6; const HeaderInfoLength = 256; const VectorIndexRows = 256; const VectorIndexCols = 256; const VectorIndexSize = 8; // Util class class Util { // parse the specified IP address and return its bytes. // returns: NULL for failed or the packed bytes public static function parseIP($ipString) { $flag = FILTER_FLAG_IPV4 | FILTER_FLAG_IPV6; if (!filter_var($ipString, FILTER_VALIDATE_IP, $flag)) { return null; } return inet_pton($ipString); } // IP bytes to string public static function ipToString($ipBytes) { $l = strlen($ipBytes); return ($l == 4 || $l == 16) ? inet_ntop($ipBytes) : ''; } // compare two ip bytes (packed string return by parsedIP) // returns: -1 if ip1 < ip2, 0 if ip1 == ip2 or 1 if ip1 > ip2 public static function ipSubCompare($ip1, $buff, $offset) { // $r = substr_compare($ip1, $buff, $offset, strlen($ip1)); // @Note: substr_compare is not working, use the substr + strcmp instead $r = strcmp($ip1, substr($buff, $offset, strlen($ip1))); if ($r < 0) { return -1; } else if ($r > 0) { return 1; } else { return 0; } } // returns: -1 if ip1 < ip2, 0 if ip1 == ip2 or 1 if ip1 > ip2 public static function ipCompare($ip1, $ip2) { $r = strcmp($ip1, $ip2); if ($r < 0) { return -1; } else if ($r > 0) { return 1; } else { return 0; } } // version parse public static function versionFromName($ver_name) { $name = strtoupper($ver_name); if ($name == "V4" || $name == "IPv4") { return IPv4::default(); } else if ($name == "V6" || $name == "IPv6") { return IPv6::default(); } else { throw new Exception("invalid verstion name `{$ver_name}`"); } } // version parse from header public static function versionFromHeader($header) { // Old structure 2.0 with IPv4 supports ONLY if ($header['version'] == Structure_20) { return IPv4::default(); } // structure 3.0 after IPv6 supporting if ($header['version'] != Structure_30) { throw new Exception("invalid xdb structure version `{$header['version']}`"); } if ($header['ipVersion'] == IPv4VersionNo) { return IPv4::default(); } else if ($header['ipVersion'] == IPv6VersionNo) { return IPv6::default(); } else { throw new Exception("invalid ip version number `{$header['ipVersion']}`"); } } // binary string chars implode with space public static function bytesToString($buff, $offset, $length) { $sb = []; for ($i = 0; $i < $length; $i++) { $sb[] = ord($buff[$offset+$i]) & 0xFF; } return '['.implode(' ', $sb).']'; } // decode a 4bytes long with Little endian byte order from a byte buffer public static function le_getUint32($b, $idx) { $val = (ord($b[$idx])) | (ord($b[$idx+1]) << 8) | (ord($b[$idx+2]) << 16) | (ord($b[$idx+3]) << 24); // convert signed int to unsigned int if on 32 bit operating system if ($val < 0 && PHP_INT_SIZE == 4) { $val = sprintf("%u", $val); } return $val; } // read a 2bytes int with litten endian byte order from a byte buffer public static function le_getUint16($b, $idx) { return ((ord($b[$idx])) | (ord($b[$idx+1]) << 8)); } // Verify if the current Searcher could be used to search the specified xdb file. // Why do we need this check ? // The future features of the xdb impl may cause the current searcher not able to work properly. // // @Note: You Just need to check this ONCE when the service starts // Or use another process (eg, A command) to check once Just to confirm the suitability. // returns: null for everything is ok or the error string. public static function verify($handle) { // load the header $header = self::loadHeader($handle); if ($header == null) { return 'failed to load the header'; } // get the runtime ptr bytes $runtimePtrBytes = 0; if ($header['version'] == Structure_20) { $runtimePtrBytes = 4; } else if ($header['version'] == Structure_30) { $runtimePtrBytes = $header['runtimePtrBytes']; } else { return "invalid structure version `{$header['version']}`"; } // 1, confirm the xdb file size // to ensure that the maximum file pointer does not overflow $stat = fstat($handle); if ($stat == false) { return 'failed to stat the xdb file'; } $maxFilePtr = (1 << ($runtimePtrBytes * 8)) - 1; // print_r([$stat['size'], $maxFilePtr]); if ($stat['size'] > $maxFilePtr) { return "xdb file exceeds the maximum supported bytes: {$maxFilePtr}"; } return null; } public static function verifyFromFile($dbFile) { $handle = fopen($dbFile, 'r'); if ($handle === false) { return null; } $r = self::verify($handle); fclose($handle); return $r; } // load header info from a specified file handle public static function loadHeader($handle) { if (fseek($handle, 0) == -1) { return null; } $buff = fread($handle, HeaderInfoLength); if ($buff === false) { return null; } // read bytes length checking if (strlen($buff) != HeaderInfoLength) { return null; } // return the decoded header info return array( 'version' => self::le_getUint16($buff, 0), 'indexPolicy' => self::le_getUint16($buff, 2), 'createdAt' => self::le_getUint32($buff, 4), 'startIndexPtr' => self::le_getUint32($buff, 8), 'endIndexPtr' => self::le_getUint32($buff, 12), 'ipVersion' => self::le_getUint16($buff, 16), 'runtimePtrBytes' => self::le_getUint16($buff, 18) ); } // load header info from the specified xdb file path public static function loadHeaderFromFile($dbFile) { $handle = fopen($dbFile, 'r'); if ($handle === false) { return null; } $header = self::loadHeader($handle); fclose($handle); return $header; } // load vector index from a file handle public static function loadVectorIndex($handle) { if (fseek($handle, HeaderInfoLength) == -1) { return null; } $rLen = VectorIndexRows * VectorIndexCols * VectorIndexSize; $buff = fread($handle, $rLen); if ($buff === false) { return null; } if (strlen($buff) != $rLen) { return null; } return $buff; } // load vector index from a specified xdb file path public static function loadVectorIndexFromFile($dbFile) { $handle = fopen($dbFile, 'r'); if ($handle === false) { return null; } $vIndex = self::loadVectorIndex($handle); fclose($handle); return $vIndex; } // load the xdb content from a file handle public static function loadContent($handle) { if (fseek($handle, 0, SEEK_END) == -1) { return null; } $size = ftell($handle); if ($size === false) { return null; } // seek to the head for reading if (fseek($handle, 0) == -1) { return null; } $buff = fread($handle, $size); if ($buff === false) { return null; } // read length checking if (strlen($buff) != $size) { return null; } return $buff; } // load the xdb content from a file path public static function loadContentFromFile($dbFile) { $str = file_get_contents($dbFile, false); if ($str === false) { return null; } else { return $str; } } public static function now() { return (microtime(true) * 1000); } } // IPv4 version class class IPv4 { public $id; public $name; public $bytes; public $segmentIndexSize; private static $C = null; public static function default() { if (self::$C == null) { // 14 = 4 + 4 + 2 + 4 self::$C = new self(IPv4VersionNo, 'IPv4', 4, 14); } return self::$C; } public function __construct($id, $name, $bytes, $segmentIndexSize) { $this->id = $id; $this->name = $name; $this->bytes = $bytes; $this->segmentIndexSize = $segmentIndexSize; } // compare the two ip bytes with the current version public function ipSubCompare($ip1, $buff, $offset) { // ip1: Little endian byte order encoded long from searcher. // ip2: Little endian byte order read from xdb index. $len = strlen($ip1); $eIdx = $offset + $len; for ($i = 0, $j = $eIdx - 1; $i < $len; $i++, $j--) { $i1 = ord($ip1[$i]) & 0xFF; $i2 = ord($buff[$j]) & 0xFF; // printf("i:%d, j:%d, i1:%d, i2:%d\n", $i, $j, $i1, $i2); if ($i1 > $i2) { return 1; } else if ($i1 < $i2) { return -1; } } return 0; } public function __toString() { return sprintf( "{id:%d, name:%s, bytes:%d, segmentIndexSize:%d}", $this->id, $this->name, $this->bytes, $this->segmentIndexSize ); } } class IPv6 { public $id; public $name; public $bytes; public $segmentIndexSize; private static $C = null; public static function default() { if (self::$C == null) { // 38 = 16 + 16 + 2 + 4 self::$C = new self(IPv6VersionNo, 'IPv6', 16, 38); } return self::$C; } public function __construct($id, $name, $bytes, $segmentIndexSize) { $this->id = $id; $this->name = $name; $this->bytes = $bytes; $this->segmentIndexSize = $segmentIndexSize; } public function ipSubCompare($ip, $buff, $offset) { // return Util::ipCompare($ip, substr($buff, $offset, strlen($ip))); return Util::ipSubCompare($ip, $buff, $offset); } public function __toString() { return sprintf( "{id:%d, name:%s, bytes:%d, segmentIndexSize:%d}", $this->id, $this->name, $this->bytes, $this->segmentIndexSize ); } } // Xdb searcher implementation class Searcher { // ip version private $version; // xdb file handle private $handle = null; private $ioCount = 0; // vector index in binary string. // string decode will be faster than the map based Array. private $vectorIndex = null; // xdb content buffer private $contentBuff = null; // --- // static function to create searcher /** * @throws Exception */ public static function newWithFileOnly($version, $dbFile) { return new self($version, $dbFile, null, null); } /** * @throws Exception */ public static function newWithVectorIndex($version, $dbFile, $vIndex) { return new self($version, $dbFile, $vIndex, null); } /** * @throws Exception */ public static function newWithBuffer($version, $cBuff) { return new self($version, null, null, $cBuff); } // --- End of static creator /** * initialize the xdb searcher * @throws Exception */ function __construct($version, $dbFile, $vectorIndex=null, $cBuff=null) { $this->version = $version; // check the content buffer first if ($cBuff != null) { $this->vectorIndex = null; $this->contentBuff = $cBuff; } else { // open the xdb binary file $this->handle = fopen($dbFile, "r"); if ($this->handle === false) { throw new Exception("failed to open xdb file '%s'", $dbFile); } $this->vectorIndex = $vectorIndex; } } public function close() { if ($this->handle != null) { fclose($this->handle); } } public function getIPVersion() { return $this->version; } public function getIOCount() { return $this->ioCount; } /** * find the region info for the specified ip address. * @Note: the ip address couldO ONLY be a human-readable IP address string, * DO not use the packed binary string returned by #parseIP * * @throws Exception */ public function search($ip) { $ipBytes = Util::parseIP($ip); if ($ipBytes == null) { throw new Exception("invalid ip address `{$ip}`"); } return $this->searchByBytes($ipBytes); } /** * find the region info for the specified binary ip bytes returned by #parseIP. * * @throws Exception */ public function searchByBytes($ipBytes) { // ip version check if (strlen($ipBytes) != $this->version->bytes) { throw new Exception("invalid ip address ({$this->version->name} expected)"); } // reset the global counter $this->ioCount = 0; // locate the segment index block based on the vector index $il0 = ord($ipBytes[0]) & 0xFF; $il1 = ord($ipBytes[1]) & 0xFF; $idx = $il0 * VectorIndexCols * VectorIndexSize + $il1 * VectorIndexSize; if ($this->vectorIndex != null) { $sPtr = Util::le_getUint32($this->vectorIndex, $idx); $ePtr = Util::le_getUint32($this->vectorIndex, $idx + 4); } else if ($this->contentBuff != null) { $sPtr = Util::le_getUint32($this->contentBuff, HeaderInfoLength + $idx); $ePtr = Util::le_getUint32($this->contentBuff, HeaderInfoLength + $idx + 4); } else { // read the vector index block $buff = $this->read(HeaderInfoLength + $idx, 8); $sPtr = Util::le_getUint32($buff, 0); $ePtr = Util::le_getUint32($buff, 4); } // printf("sPtr: %d, ePtr: %d\n", $sPtr, $ePtr); // @Note: ptr validate, zero ptr means source data missing // so we could just stop here and return an empty string. if ($sPtr == 0 || $ePtr == 0) { return ""; } [$bytes, $dBytes] = [strlen($ipBytes), strlen($ipBytes) << 1]; // binary search the segment index to get the region info $idxSize = $this->version->segmentIndexSize; [$dataLen, $dataPtr, $l, $h] = [0, 0, 0, ($ePtr - $sPtr) / $idxSize]; while ($l <= $h) { $m = ($l + $h) >> 1; $p = $sPtr + $m * $idxSize; // read the segment index $buff = $this->read($p, $idxSize); // compare the segment index if ($this->version->ipSubCompare($ipBytes, $buff, 0) < 0) { $h = $m - 1; } else if ($this->version->ipSubCompare($ipBytes, $buff, $bytes) > 0) { $l = $m + 1; } else { $dataLen = Util::le_getUint16($buff, $dBytes); $dataPtr = Util::le_getUint32($buff, $dBytes + 2); break; } } // empty match interception. // printf("dataLen: %d, dataPtr: %d\n", $dataLen, $dataPtr); if ($dataLen == 0) { return ""; } // load and return the region data return $this->read($dataPtr, $dataLen); } // read specified bytes from the specified index private function read($offset, $len) { // check the in-memory buffer first if ($this->contentBuff != null) { return substr($this->contentBuff, $offset, $len); } // read from the file $r = fseek($this->handle, $offset); if ($r == -1) { throw new Exception("failed to fseek to {$offset}"); } $this->ioCount++; $buff = fread($this->handle, $len); if ($buff === false) { throw new Exception("failed to fread from {$len}"); } if (strlen($buff) != $len) { throw new Exception("incomplete read: read bytes should be {$len}"); } return $buff; } } ================================================ FILE: binding/php/xdb/util_test.php ================================================ // @Date 2022/06/22 require 'Searcher.class.php'; use \ip2region\xdb\Util; use \ip2region\xdb\Searcher; use \ip2region\xdb\IPv4; use \ip2region\xdb\IPv6; // check and get the function to run if($argc < 2) { printf("please specified the function name\n"); return; } else { $func_name = trim($argv[1]); } $basePath = dirname(dirname(dirname(dirname(__FILE__)))); function testLoadHeader() { global $basePath; $header = Util::loadHeaderFromFile("{$basePath}/data/ip2region_v4.xdb"); if ($header == null) { printf("failed to load header from file\n"); return; } printf("header loaded: "); print_r($header); } function testLoadVectorIndex() { global $basePath; $vIndex = Util::loadVectorIndexFromFile("{$basePath}/data/ip2region_v4.xdb"); if ($vIndex == null) { printf("failed to load vector index from file\n"); return; } printf("vector index loaded: length=%d\n", strlen($vIndex)); } function testLoadContent() { global $basePath; $cBuff = Util::loadContentFromFile("{$basePath}/data/ip2region_v4.xdb"); if ($cBuff == null) { printf("failed to load content from file\n"); return; } printf("content loaded, length=%d\n", strlen($cBuff)); } function testParseIP() { $ips = [ // IPv4 "1.0.0.1", "192.168.1.100", "121.35.184.170", "xx.xx.1.100", // IPv6 "3000::", "240e:87c:71a:639a:3dff:ffff:ffff:ffff", "240e:87c:71a:c877:900::" ]; foreach ($ips as $ip) { $bytes = Util::parseIP($ip); if ($bytes == NULL) { printf("invalid ip address: `%s`\n", $ip); continue; } // for ($i = 0; $i < strlen($bytes); $i++) { // printf("%d: []: %s, ord: %d\n", $i, $bytes[$i], ord($bytes[$i])); // } printf("bytes: %s (%s), address: %s\n", bin2hex($bytes), gettype($bytes), Util::ipToString($bytes)); } } function testIPCompare() { $ipPairs = [ ["1.0.0.0", "1.0.0.1"], ["192.168.1.101", "192.168.1.90"], ["219.133.111.87", "114.114.114.114"], ["1.0.4.0", "1.0.1.0"], ["1.0.4.0", "1.0.3.255"], ["2000::", "2000:ffff:ffff:ffff:ffff:ffff:ffff:ffff"], ["2001:4:112::", "2001:4:112:ffff:ffff:ffff:ffff:ffff"], ["ffff::", "2001:4:ffff:ffff:ffff:ffff:ffff:ffff"] ]; foreach ($ipPairs as $ips) { $ip1 = Util::parseIP($ips[0]); $ip2 = Util::parseIP($ips[1]); printf("ipCompare(%s, %s): %d\n", Util::ipToString($ip1), Util::ipToString($ip2), Util::ipSubCompare($ip1, $ip2, 0)); } } function testAttributes() { printf("IPv4VersioNo: %d\n", \ip2region\xdb\IPv4VersionNo); printf("IPv6VersioNo: %d\n", \ip2region\xdb\IPv6VersionNo); printf("IPv4 Object: %s\n", IPv4::default()); printf("IPv6 Object: %s\n", IPv6::default()); } if (!function_exists($func_name)) { printf("function {$func_name} not found\n"); } else { printf("calling {$func_name} ... \n"); $now = Util::now(); $func_name(); $cost = Util::now() - $now; printf("done, cost: %0.5f ms\n", $cost); } ================================================ FILE: binding/python/.gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg # Environments .env .venv env/ venv/ ENV/ # PyPI configuration file .pypirc ================================================ FILE: binding/python/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ========================================================================== The following license applies to the ip2region library -------------------------------------------------------------------------- Copyright (c) 2015 Lionsoul Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: binding/python/MANIFEST.in ================================================ include LICENSE include ReadMe.md include search_test.py include bench_test.py recursive-include tests ================================================ FILE: binding/python/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region python query client # Version Compatibility This implementation is compatible with Python `>=` **`3.7`** # Usage ### Install `py-ip2region` ```bash pip3 install py-ip2region ``` ### About Query API The prototype of the Query API is: ```python # Query via string IP or binary IP (bytes type) parsed by util.parse_ip search(ip: str | bytes) ``` An exception will be thrown if the query fails. If successful, the `region` information in string format will be returned. If the specified IP cannot be found, an empty string `""` will be returned. ### About IPv4 and IPv6 This xdb query client implementation supports both IPv4 and IPv6 queries. Use it as follows: ```python import ip2region.util as util # For IPv4: Set xdb path to the v4 xdb file, and specify the IP version as util.IPv4 db_path = "../../data/ip2region_v4.xdb" # Or your ipv4 xdb path version = util.IPv4 # For IPv6: Set xdb path to the v6 xdb file, and specify the IP version as util.IPv6 db_path = "../../data/ip2region_v6.xdb" # Or your ipv6 xdb path version = util.IPv6 # The IP version of the xdb specified by db_path must match the version specified by version, otherwise an error will occur during query execution # Note: The following demonstrations directly use the db_path and version variables ``` ### File Verification It is recommended that you proactively verify the applicability of the xdb file. Future new features may cause the current Searcher version to be incompatible with the xdb file you are using; verification can prevent unpredictable errors during runtime. You do not need to verify every time. For example, perform verification when the service starts or manually call the command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios. ```python import ip2region.util as util try: util.verify_from_file(db_path) except Exception e: # Applicability verification failed!!! # The current query client implementation is not applicable for the xdb file specified by db_path. # You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation suitable for db_path. print(f"binding is not applicable for xdb file '{db_path}': {str(e)}") return # Verification passed, the current Searcher can be safely used for query operations on the xdb pointed to by db_path ``` ### File-Only Query ```python import ip2region.util as util import ip2region.searcher as xdb # 1. Use the version and db_path mentioned above to create a file-only query object try: searcher = xdb.new_with_file_only(version, db_path) except Exception as e: print(f"failed to new_with_file_only: {str(e)}") return # 2. Query, it supports both IPv4 and IPv6 addresses ip = "1.2.3.4" # ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 try: region = searcher.search(ip) print(f"search({ip}): {{region: {region}, io_count: {searcher.get_io_count()}}}") except Exception as e: print(f"failed to search: {str(e)}") # 3. Close resources searcher.close() # Note: Each thread needs to create an independent Searcher object ``` ### Caching `VectorIndex` We can pre-load the `VectorIndex` data from the `xdb` file and cache it globally. Using the global VectorIndex cache every time a Searcher object is created can reduce a fixed IO operation, thereby accelerating queries and reducing IO pressure. ```python import ip2region.util as util import ip2region.searcher as xdb # 1. Pre-load VectorIndex cache from db_path and use this data as a global variable for subsequent repeated use. try: v_index = util.load_vector_index_from_file(db_path) except Exception as e: print(f"failed to load vector index from {db_path}: {str(e)}") return # 2. Use the global v_index to create a query object with VectorIndex cache. try: searcher = xdb.new_with_vector_index(version, db_path, v_index) except Exception as e: print(f"failed to new_with_vector_index: {str(e)}") return # 3. Query; the interface is the same for both IPv4 and IPv6 addresses ip = "1.2.3.4" # ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 try: region = searcher.search(ip) print(f"search({ip}): {{region: {region}, io_count: {searcher.get_io_count()}}}") except Exception as e: print(f"failed to search: {str(e)}"); # 4. Close resources searcher.close() # Note: Each thread needs to create an independent Searcher object, but they all share the global read-only v_index cache. ``` ### Caching the Entire `xdb` File We can also pre-load the data of the entire xdb file into memory and create a query object based on this data to achieve fully memory-based queries, similar to the previous memory search. ```python import ip2region.util as util import ip2region.searcher as xdb # 1. Load the entire xdb into memory from db_path. try: c_buffer = util.load_content_from_file(db_path) except Exception as e: print(f"failed to load content from {db_path}: {str(e)}") return # 2. Use the c_buffer mentioned above to create a fully memory-based query object. try: searcher = xdb.new_with_buffer(version, c_buffer) except Exception e: print(f"failed to new_with_buffer: {str(e)}") return # 3. Query; the interface is the same for both IPv4 and IPv6 addresses ip = "1.2.3.4" # ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" # IPv6 try: region = searcher.search(ip) print(f"search({ip}): {{region: {region}, io_count: 0}}") except Exception as e: print(f"failed to search: {str(e)}") # 4. Close resources - This searcher object can be safely used for concurrency; close the searcher only when the entire service is going to shut down # searcher.close() # Note: For concurrent use, query objects created with the entire xdb data cache can be safely used for concurrency, meaning you can make this searcher object a global object for cross-thread access. ``` # Query Test You can test queries via the `python3 search_test.py` command: ```bash ➜ python git:(fr_python_ipv6) ✗ python3 search_test.py usage: python search_test.py [command option] ip2region search test script options: -h, --help show this help message and exit --db DB ip2region binary xdb file path --cache-policy CACHE_POLICY cache policy: file/vectorIndex/content, default: vectorIndex ``` For example: Use the default data/ip2region_v4.xdb file for IPv4 query testing: ```bash ➜ python git:(fr_python_ipv6) ✗ python3 search_test.py --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 188 μs} ip2region>> 113.118.113.77 {region: 中国|广东省|深圳市|电信|CN, ioCount: 2, took: 143 μs} ``` For example: Use the default data/ip2region_v6.xdb file for IPv6 query testing: ```bash ➜ python git:(fr_python_ipv6) ✗ python3 search_test.py --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> :: {region: , ioCount: 1, took: 166 μs} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 197 μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, ioCount: 13, took: 240 μs} ``` Input an IP to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the effects of the three different cache implementations. # bench Test Bench testing can be performed via the `python3 bench_test.py` command, which ensures the `xdb` file is error-free and allows for performance evaluation: ```bash ➜ python git:(fr_python_ipv6) ✗ python3 bench_test.py usage: python bench_test.py [command option] ip2region bench test script options: -h, --help show this help message and exit --db DB ip2region binary xdb file path --src SRC source ip text file path --cache-policy CACHE_POLICY cache policy: file/vectorIndex/content, default: vectorIndex ``` For example: Perform IPv4 bench testing via the default data/ip2region_v4.xdb and data/ipv4_source.txt files: ```bash python3 bench_test.py --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` For example: Perform IPv6 bench testing via the default data/ip2region_v6.xdb and data/ipv6_source.txt files: ```bash python3 bench_test.py --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` You can test the effects of the three different cache implementations by setting `cache-policy` to file/vectorIndex/content respectively. @Note: Ensure the src file used for bench is the same source file used to generate the corresponding xdb file. ================================================ FILE: binding/python/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region python 查询客户端 # 版本兼容 该实现兼容 Python `>=` **`3.7`** # 使用方式 ### 安装 `py-ip2region` ```bash pip3 install py-ip2region ``` ### 关于查询 API 查询 API 的原型为： ```python # 通过字符串 IP 或者 util.parse_ip 解析得到的二进制 IP (bytes类型) 进行查询 search(ip: str | bytes) ``` 如果查询出错会抛异常，查询成功则会返回字符的 `region` 信息，如果指定的 IP 查询不到则会返回空字符串 `""`。 ### 关于 IPv4 和 IPv6 该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询，使用方式如下： ```python import ip2region.util as util # 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 util.IPv4 db_path = "../../data/ip2region_v4.xdb" # 或者你的 ipv4 xdb 的路径 version = util.IPv4 # 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件，IP版本指定为 util.IPv6 db_path = "../../data/ip2region_v6.xdb" # 或者你的 ipv6 xdb 路径 version = util.IPv6 # db_path 指定的 xdb 的 IP 版本必须和 version 指定的一致，不然查询执行的时候会报错 # 备注：以下演示直接使用 db_path 和 version 变量 ``` ### 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 ```python import ip2region.util as util try: util.verify_from_file(db_path) except Exception e: # 适用性验证失败！！！ # 当前查询客户端实现不适用于 db_path 指定的 xdb 文件的查询. # 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 db_path 的 Searcher 实现。 print(f"binding is not applicable for xdb file '{db_path}': {str(e)}") return # 验证通过，当前使用的 Searcher 可以安全的用于对 db_path 指向的 xdb 的查询操作 ``` ### 完全基于文件的查询 ```python import ip2region.util as util import ip2region.searcher as xdb # 1，使用上述的 version 和 db_path 创建完全基于文件的查询对象 try: searcher = xdb.new_with_file_only(version, db_path) except Exception as e: print(f"failed to new_with_file_only: {str(e)}") return # 2、查询，IPv4 或者 IPv6 的地址都是同一个接口 ip = "1.2.3.4" # ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 try: region = searcher.search(ip) print(f"search({ip}): {{region: {region}, io_count: {searcher.get_io_count()}}}") except Exception as e: print(f"failed to search: {str(e)}") # 3、关闭资源 searcher.close() # 备注：每个线程需要单独创建一个独立的 Searcher 对象 ``` ### 缓存 `VectorIndex` 索引 我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。 ```python import ip2region.util as util import ip2region.searcher as xdb # 1、从 db_path 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。 try: v_index = util.load_vector_index_from_file(db_path) except Exception as e: print(f"failed to load vector index from {db_path}: {str(e)}") return # 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。 try: searcher = xdb.new_with_vector_index(version, db_path, v_index) except Exception as e: print(f"failed to new_with_vector_index: {str(e))}") return # 3、查询，IPv4 或者 IPv6 的地址都是同一个接口 ip = "1.2.3.4" # ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6 try: region = searcher.search(ip) print(f"search({ip}): {{region: {region}, io_count: {searcher.get_io_count()}}}") except Exception as e: print(f"failed to search: {str(e)}"); # 4、关闭资源 searcher.close() # 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的只读 v_index 缓存。 ``` ### 缓存整个 `xdb` 文件 我们也可以预先加载整个 xdb 文件的数据到内存，然后基于这个数据创建查询对象来实现完全基于内存的查询，类似之前的 memory search。 ```python import ip2region.util as util import ip2region.searcher as xdb # 1、从 db_path 加载整个 xdb 到内存。 try: c_buffer = util.load_content_from_file(db_path) except Exception as e: print(f"failed to load content from {db_path}: {str(e)}") return # 2、使用上述的 c_buff 创建一个完全基于内存的查询对象。 try: searcher = xdb.new_with_buffer(version, c_buffer) except Exception e: print(f"failed to new_with_buffer: {str(e)}") return # 3、查询，IPv4 或者 IPv6 的地址都是同一个接口 ip = "1.2.3.4" # ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" # IPv6 try: region = searcher.search(ip) print(f"search({ip}): {{region: {region}, io_count: 0}}") except Exception as e: print(f"failed to search: {str(e)}") # 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher # searcher.close() # 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。 ``` # 查询测试 可以通过 `python3 search_test.py` 命令来测试查询： ```bash ➜ python git:(fr_python_ipv6) ✗ python3 search_test.py usage: python search_test.py [command option] ip2region search test script options: -h, --help show this help message and exit --db DB ip2region binary xdb file path --cache-policy CACHE_POLICY cache policy: file/vectorIndex/content, default: vectorIndex ``` 例如：使用默认的 data/ip2region_v4.xdb 文件进行 IPv4 的查询测试： ```bash ➜ python git:(fr_python_ipv6) ✗ python3 search_test.py --db=../../data/ip2region_v4.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex) type 'quit' to exit ip2region>> 1.2.3.4 {region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 188 μs} ip2region>> 113.118.113.77 {region: 中国|广东省|深圳市|电信|CN, ioCount: 2, took: 143 μs} ``` 例如：使用默认的 data/ip2region_v6.xdb 文件进行 IPv6 的查询测试： ```bash ➜ python git:(fr_python_ipv6) ✗ python3 search_test.py --db=../../data/ip2region_v6.xdb ip2region xdb searcher test program source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex) type 'quit' to exit ip2region>> :: {region: , ioCount: 1, took: 166 μs} ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e {region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 197 μs} ip2region>> 2604:a840:3::a04d {region: United States|California|San Jose|xTom|US, ioCount: 13, took: 240 μs} ``` 输入 ip 即可进行查询测试，也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。 # bench 测试 可以通过 `python3 bench_test.py` 命令来进行 bench 测试，一方面确保 `xdb` 文件没有错误，一方面可以评估查询性能： ```bash ➜ python git:(fr_python_ipv6) ✗ python3 bench_test.py usage: python bench_test.py [command option] ip2region bench test script options: -h, --help show this help message and exit --db DB ip2region binary xdb file path --src SRC source ip text file path --cache-policy CACHE_POLICY cache policy: file/vectorIndex/content, default: vectorIndex ``` 例如：通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 文件进行 IPv4 的 bench 测试： ```bash python3 bench_test.py --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt ``` 例如：通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 文件进行 IPv6 的 bench 测试： ```bash python3 bench_test.py --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt ``` 可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。 @Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。 ================================================ FILE: binding/python/bench_test.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # xdb searcher bench test on 2025/10/30 # Author Leon import io import sys import argparse import time import ip2region.util as util import ip2region.searcher as xdb def create_searcher(db_path, cache_policy): # open the source xdb file handle = io.open(db_path, "rb") # verify the xdb file # @Note: do NOT call it every time you create a searcher since this will slow # down the search response. # @see the verify function for details. util.verify(handle) # get the ip version from header header = util.load_header(handle) version = util.version_from_header(header) if version is None: handle.close() raise Exception("failed to get version from header") searcher = None if cache_policy == "file": searcher = xdb.new_with_file_only(version, db_path) elif cache_policy == "vectorIndex": v_index = util.load_vector_index(handle) searcher = xdb.new_with_vector_index(version, db_path, v_index) elif cache_policy == "content": c_buffer = util.load_content(handle) searcher = xdb.new_with_buffer(version, c_buffer) else: raise ValueError("invalid cache_policy `{}`".format(cache_policy)) handle.close() return searcher def run_bench_test(db_path: str, src_path: str, cache_policy: str): # create the searcher searcher = None try: searcher = create_searcher(args.db, args.cache_policy) except Exception as e: print("failed to create searcher: {}".format(str(e))) return print("searcher ->", searcher) # read the source lines and do the search test count, total_secs = 0, 0 try: handle = io.open(src_path, "rb") while True: line = handle.readline().decode("utf-8").strip() # ignore empty or comment line if len(line) < 1: break if line[0] == "#": continue # line splits ps = line.split("|", 2) if len(ps) != 3: raise Exception(f"invalid ip segment line `{line}`") # ip parse and compare start_time = time.time() sip_bytes = util.parse_ip(ps[0]) eip_bytes = util.parse_ip(ps[1]) if util.ip_compare(sip_bytes, eip_bytes) > 0: raise Exception(f"start ip({ps[0]}) should not be greater than end ip({ps[1]})") # do the search test for ip_bytes in [sip_bytes, eip_bytes]: count = count + 1 region = searcher.search(ip_bytes) if region != ps[2]: raise Exception(f"failed to search({util.ip_to_string(ip_bytes)}) with {region} != {ps[2]}") total_secs = total_secs + (time.time() - start_time) # resource cleanup handle.close() except Exception as e: print(f"bench failed: {str(e)}") return # print the stats info each_us = total_secs * 1000_000 / count print(f"Bench finished, {{cachePolicy: {cache_policy}, total: {count}, took: {total_secs:.3f} s, cost: {each_us:.0f} μs/op}}"); # resource cleanup searcher.close() if __name__ == "__main__": parser = argparse.ArgumentParser( add_help=True, prog="python bench_test.py", description="ip2region bench test script", usage="%(prog)s [command option]" ) # check the args parser.add_argument('--db', help='ip2region binary xdb file path') parser.add_argument('--src', help='source ip text file path') parser.add_argument('--cache-policy', help='cache policy: file/vectorIndex/content, default: vectorIndex', default="vectorIndex") args = parser.parse_args() if (args.db is None) or (args.src is None): parser.print_help() sys.exit() # run the search test run_bench_test(args.db, args.src, args.cache_policy) ================================================ FILE: binding/python/ip2region/__init__.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # xdb package ================================================ FILE: binding/python/ip2region/searcher.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # xdb searcher on 2025/10/30 # Author Leon import io import ip2region.util as util from typing import Union class Searcher(object): ''' xdb searcher class with Both IPv4 and IPv6 supported. three kinds of cache policy: file / vectorIndex / content ''' def __init__(self, version: util.Version, db_path: str, vector_index: bytes, c_buffer: bytes): self.version = version self.__db_path = db_path self.__io_count = 0 if c_buffer != None: self.__handle = None self.vector_index = None self.c_buffer = c_buffer else: self.__handle = io.open(db_path, "rb") self.vector_index = vector_index self.c_buffer = None def get_ip_version(self): return self.version def get_io_count(self): return self.__io_count def search(self, ip: Union[bytes, str]): # check and parse the string ip ip_bytes = None if isinstance(ip, str): ip_bytes = util.parse_ip(ip) elif isinstance(ip, bytes): ip_bytes = ip else: raise ValueError("invalid ip address `{}`".format(ip)) # ip version check if len(ip_bytes) != self.version.byte_num: raise ValueError("invalid ip address `{}` ({} expected)".format( util.ip_to_string(ip_bytes), self.version.name)) # reset the global io_count self.__io_count = 0 # located the segment index block based on the vector index s_ptr, e_ptr, i0, i1 = 0, 0, ip_bytes[0], ip_bytes[1] idx = i0 * util.VectorIndexCols * util.VectorIndexSize + i1 * util.VectorIndexSize if self.vector_index != None: s_ptr = util.le_get_uint32(self.vector_index, idx) e_ptr = util.le_get_uint32(self.vector_index, idx + 4) elif self.c_buffer != None: offset = util.HeaderInfoLength + idx s_ptr = util.le_get_uint32(self.c_buffer, offset) e_ptr = util.le_get_uint32(self.c_buffer, offset + 4) else: buff = self.read(util.HeaderInfoLength + idx, util.VectorIndexSize) s_ptr = util.le_get_uint32(buff, 0) e_ptr = util.le_get_uint32(buff, 4) # print("s_ptr: {}, e_ptr: {}".format(s_ptr, e_ptr)) # @Note: ptr validate, zero ptr means source data missing # so we could just stop here and return an empty string. if s_ptr == 0 or e_ptr == 0: return "" # binary search the segment index block to get the region info _bytes, _d_bytes = len(ip_bytes), len(ip_bytes) << 1 index_size = self.version.index_size d_len, d_ptr, l, h = 0, 0, int(0), int((e_ptr - s_ptr) / index_size) while l <= h: m = (l + h) >> 1 p = int(s_ptr + m * index_size) # read the segment index buff = self.read(p, index_size) if self.version.ip_sub_compare(ip_bytes, buff, 0) < 0: h = m - 1 elif self.version.ip_sub_compare(ip_bytes, buff, _bytes) > 0: l = m + 1 else: d_len = util.le_get_uint16(buff, _d_bytes) d_ptr = util.le_get_uint32(buff, _d_bytes + 2) break # print("d_len: {}, d_ptr: {}".format(d_len, d_ptr)) # empty match interception. # and this could be a case. if d_len == 0: return "" # read and return the region info return self.read(d_ptr, d_len).decode("utf-8") def read(self, offset: int, length: int): # check the content buffer first if self.c_buffer != None: return self.c_buffer[offset:offset+length] # load the buffer from file self.__handle.seek(offset) self.__io_count += 1 return self.__handle.read(length) def close(self): if self.__handle != None: self.__handle.close() def __str__(self): return '{{"version": {}, "db_path": "{}", "v_index": {}, "c_buffer": {}}}'.format( self.version.name, self.__db_path, None if self.vector_index is None else len(self.vector_index), None if self.c_buffer is None else len(self.c_buffer) ) # --- # functions to create Searcher with different cache policy def new_with_file_only(version: util.Version, db_path: str): return Searcher(version, db_path, None, None) def new_with_vector_index(version: util.Version, db_path: str, vector_index: bytes): return Searcher(version, db_path, vector_index, None) def new_with_buffer(version: util.Version, c_buffer: bytes): return Searcher(version, None, None, c_buffer) ================================================ FILE: binding/python/ip2region/util.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # xdb utils on 2025/10/29 # Author Leon import io import os import ipaddress from typing import Callable # global constants XdbStructure20 = 2 XdbStructure30 = 3 XdbIPv4Id = 4 XdbIPv6Id = 6 HeaderInfoLength = 256 VectorIndexRows = 256 VectorIndexCols = 256 VectorIndexSize = 8 # cache of VectorIndexCols × VectorIndexRows × VectorIndexSize VectorIndexLength = 524288 class Header(object): def __init__(self, buff: bytes): self.version = le_get_uint16(buff, 0) self.indexPolicy = le_get_uint16(buff, 2) self.createdAt = le_get_uint32(buff, 4) self.startIndexPtr = le_get_uint32(buff, 8) self.endIndexPtr = le_get_uint32(buff, 12) # since IPv6 supporting self.ipVersion = le_get_uint16(buff, 16) self.runtimePtrBytes = le_get_uint16(buff, 18) # keep the raw data self.buff = buff def __str__(self): return '''{{ "version": {}, "indexPolicy": {}, "createdAt": {}, "startIndexPtr": {}, "endIndexPtr": {}, "ipVersion": {}, "runtimePtrBytes": {} }}'''.format( self.version, self.indexPolicy, self.createdAt, self.startIndexPtr, self.endIndexPtr, self.ipVersion, self.runtimePtrBytes ) # --- # ip parse and convert functions def parse_ip(ip_string: str): try: return ipaddress.ip_address(ip_string).packed except: raise ValueError("invalid ip address `{}`".format(ip_string)) def ip_to_string(ip_bytes: bytes): if isinstance(ip_bytes, bytes): return str(ipaddress.ip_address(ip_bytes)) else: raise ValueError("invalid bytes ip `{}`".format(ip_bytes)) def ip_compare(ip1: bytes, ip2: bytes): if ip1 > ip2: return 1 elif ip1 < ip2: return -1 else: return 0 def ip_sub_compare(ip1: bytes, buff: bytes, offset: int): ip2 = buff[offset:offset+len(ip1)] if ip1 > ip2: return 1 elif ip1 < ip2: return -1 else: return 0 # --- # ip version class and functions class Version(object): def __init__(self, id: int, name: str, byte_num: int, index_size: int, ip_compare: Callable[[bytes, bytes, int], int]): self.id = id self.name = name self.byte_num = byte_num self.index_size = index_size self.ip_compare = ip_compare def ip_compare(self, ip1: bytes, ip2: bytes): return self.ip_compare(ip1, ip2, 0) def ip_sub_compare(self, ip1: bytes, buff: bytes, offset: int): return self.ip_compare(ip1, buff, offset) def __str__(self): return '{{"id": {}, "name": "{}", "bytes": {}, "index_size": {}}}'.format( self.id, self.name, self.byte_num, self.index_size ) def _v4_sub_compare(ip1: bytes, buff: bytes, offset: int): # ip1: Big endian byte order parsed from input # ip2: Little endian byte order read from xdb index. # @Note: to compatible with the old Litten endian index encode implementation. j = offset + len(ip1) - 1 for i in range(len(ip1)): i1 = ip1[i] i2 = buff[j] if i1 < i2: return -1 if i1 > i2: return 1 # increase the j j = j - 1 return 0 # --- # IPv4 and IPv6 version constants # 14 = 4 + 4 + 2 + 4 IPv4 = Version(XdbIPv4Id, "IPv4", 4, 14, _v4_sub_compare) # 38 = 16 + 16 + 2 + 4 IPv6 = Version(XdbIPv6Id, "IPv6", 16, 38, ip_sub_compare) def version_from_name(name: str): u_name = name.upper() if u_name == "IPV4" or u_name == "V4": return IPv4 elif u_name == "IPV6" or u_name == "V6": return IPv6 else: return None def version_from_header(header: bytes): # old xdb 2.0 with IPv4 supports ONLY if header.version < XdbStructure30: return IPv4 # xdb 3.0 or later version ip_version = header.ipVersion if ip_version == XdbIPv4Id: return IPv4 elif ip_version == XdbIPv6Id: return IPv6 else: return None # --- # buffer decode functions def le_get_uint32(buff: bytes, offset: int): ''' decode an unsinged 4-bytes int from a buffer started from offset with little byte endian ''' return ( ((buff[offset ]) & 0x000000FF) | ((buff[offset+1] << 8) & 0x0000FF00) | ((buff[offset+2] << 16) & 0x00FF0000) | ((buff[offset+3] << 24) & 0xFF000000) ) def le_get_uint16(buff: bytes, offset: int): ''' decode an unsinged 2-bytes short from a buffer started from offset with little byte endian ''' return ( ((buff[offset ]) & 0x000000FF) | ((buff[offset+1] << 8) & 0x0000FF00) ) # --- # xdb buffer load functions def load_header(handle): ''' load xdb header from a specified file handle ''' handle.seek(0) return Header(handle.read(HeaderInfoLength)) def load_header_from_file(db_file: str): handle = io.open(db_file, "rb") header = load_header(handle) handle.close() return header def load_vector_index(handle): ''' load xdb vector index from a specified file handle ''' handle.seek(HeaderInfoLength) return handle.read(VectorIndexLength) def load_vector_index_from_file(db_file: str): handle = io.open(db_file, "rb") v_index = load_vector_index(handle) handle.close() return v_index def load_content(handle): ''' load the whole xdb content from a specified file handle ''' handle.seek(0) return handle.read() def load_content_from_file(db_file: str): handle = io.open(db_file, "rb") c_buff = load_content(handle) handle.close() return c_buff # --- # Verify if the current Searcher could be used to search the specified xdb file. # Why do we need this check ? # The future features of the xdb impl may cause the current searcher not able to work properly. # # @Note: You Just need to check this ONCE when the service starts # Or use another process (eg, A command) to check once Just to confirm the suitability. def verify(handle): header = load_header(handle) # get the runtime ptr bytes runtime_ptr_bytes = 0 if header.version == XdbStructure20: runtime_ptr_bytes = 4 elif header.version == XdbStructure30: runtime_ptr_bytes = header.runtimePtrBytes else: # Higher versions of the structure are usually incompatible. raise ValueError("invalid structure version {}".format(header.version)) # 1, confirm the xdb file size # to ensure that the maximum file pointer does not overflow max_file_ptr = (1 << (runtime_ptr_bytes * 8)) - 1 __file_bytes = os.stat(handle.fileno()).st_size # print("max_file_ptr: {}, file_bytes: {}".format(max_file_ptr, __file_bytes)) if __file_bytes > max_file_ptr: raise Exception("xdb file exceeds the maximum supported bytes: {}".format(max_file_ptr)) def verify_from_file(db_file: str): handle = io.open(db_file, "rb") verify(handle) handle.close() ================================================ FILE: binding/python/search_test.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # xdb searcher test on 2025/10/30 # Author Leon import io import sys import argparse import time import ip2region.util as util import ip2region.searcher as xdb def create_searcher(db_path, cache_policy): # open the source xdb file handle = io.open(db_path, "rb") # verify the xdb file # @Note: do NOT call it every time you create a searcher since this will slow # down the search response. # @see the verify function for details. util.verify(handle) # get the ip version from header header = util.load_header(handle) version = util.version_from_header(header) if version is None: handle.close() raise Exception("failed to get version from header") searcher = None if cache_policy == "file": searcher = xdb.new_with_file_only(version, db_path) elif cache_policy == "vectorIndex": v_index = util.load_vector_index(handle) searcher = xdb.new_with_vector_index(version, db_path, v_index) elif cache_policy == "content": c_buffer = util.load_content(handle) searcher = xdb.new_with_buffer(version, c_buffer) else: raise ValueError("invalid cache_policy `{}`".format(cache_policy)) handle.close() return searcher def run_search_test(db_path: str, cache_policy: str): # create the searcher searcher = None try: searcher = create_searcher(args.db, args.cache_policy) except Exception as e: print("failed to create searcher: {}".format(str(e))) return # print the searcher for debug # print("searcher -> ", searcher) print('''ip2region xdb searcher test program source xdb: {} ({}, {}) type 'quit' to exit'''.format(db_path, searcher.get_ip_version().name, cache_policy)) # get input ip address and do the search while True: ip_str = input("ip2region>> ").strip() if len(ip_str) < 2: continue if ip_str == "quit": break s_time = time.time() try: ip_bytes = util.parse_ip(ip_str) except Exception as e: print(f"invalid ip address `{ip_str}`") continue try: region = searcher.search(ip_bytes) except Exception as e: print("failed to search({}): {}".format(util.ip_to_string(ip_bytes), str(e))) continue took = (time.time() - s_time) * 1000_000 print(f"{{region: {region}, ioCount: {searcher.get_io_count()}, took: {took:.0f} μs}}"); # close the searcher searcher.close() if __name__ == "__main__": parser = argparse.ArgumentParser( add_help=True, prog="python search_test.py", description="ip2region search test script", usage="%(prog)s [command option]" ) # check the args parser.add_argument('--db', help='ip2region binary xdb file path') parser.add_argument('--cache-policy', help='cache policy: file/vectorIndex/content, default: vectorIndex', default="vectorIndex") args = parser.parse_args() if args.db is None: parser.print_help() sys.exit() # run the search test run_search_test(args.db, args.cache_policy) ================================================ FILE: binding/python/setup.py ================================================ import setuptools setuptools.setup( name="py-ip2region", version="3.0.4", description="ip2region official python binding with both IPv4 and IPv6 supported", long_description=open("README.md", encoding='utf-8').read(), long_description_content_type='text/markdown', url="https://github.com/lionsoul2014/ip2region", license="Apache-2.0 License", author_email="chenxin619315@gmail.com", author="lionsoul2014", packages=setuptools.find_packages(), include_package_data=True, install_requires=[], classifiers=( "Programming Language :: Python :: 3", "Operating System :: OS Independent", ), keywords=( "ip2region", "ip-address", "ip-region", "ip-location", "ip-lookup", "ip-search", "ipv4-address", "ipv4-region", "ipv4-location", "ipv4-lookup", "ipv4-search", "ipv6-address", "ipv6-region", "ipv6-location", "ipv6-lookup", "ipv6-search" ) ) ================================================ FILE: binding/python/util_test.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # util test script on 2025/10/29 # Author Leon import os import sys import time import ip2region.util as util import ip2region.searcher as xdb script_dir = os.path.dirname(__file__) data_dir = os.path.join(script_dir, '../../data/') xdb_v4_path = os.path.join(data_dir, "ip2region_v4.xdb") xdb_v6_path = os.path.join(data_dir, "ip2region_v6.xdb") # print(script_dir, data_dir, xdb_v4_path, xdb_v6_path) def test_version(): print("1, version contants: ") print("IPv4 -> ", util.IPv4) print("IPv6 -> ", util.IPv6) # version from name print("2, version from name: ") for name in ["v4", "IPv4", "v4x", "v6", "IPv6", "v6x"]: print("version_from_name({}) -> ".format(name), util.version_from_name(name)) # version from header print("3, version from header: ") v4_header = util.load_header_from_file(xdb_v4_path) v6_header = util.load_header_from_file(xdb_v6_path) print("version_from_header(v4_header) -> ", util.version_from_header(v4_header)) print("version_from_header(v6_header) -> ", util.version_from_header(v6_header)) def test_verify(): # v4 xdb verify try: util.verify_from_file(xdb_v4_path) except Exception as e: print("failed to verify the xdb file `{}`: {}".format(xdb_v4_path, str(e))) else: print("xdb file `{}` verified".format(xdb_v4_path)) # v6 xdb verify try: util.verify_from_file(xdb_v6_path) except Exception as e: print("failed to verify the xdb file `{}`: {}".format(xdb_v6_path, str(e))) else: print("xdb file `{}` verified".format(xdb_v6_path)) def test_load_header(): v4_header = util.load_header_from_file(xdb_v4_path) v6_header = util.load_header_from_file(xdb_v6_path) print("v4_header -> ", v4_header) print("v6_header -> ", v6_header) def test_load_vector_index(): v4_v_index = util.load_vector_index_from_file(xdb_v4_path) v6_v_index = util.load_vector_index_from_file(xdb_v6_path) print("v4_v_index.length={}".format(len(v4_v_index))) print("v6_v_index.length={}".format(len(v6_v_index))) def test_load_content(): v4_content = util.load_content_from_file(xdb_v4_path) v6_content = util.load_content_from_file(xdb_v6_path) print("v4_content.length={}".format(len(v4_content))) print("v6_content.length={}".format(len(v6_content))) def test_parse_ip(): ip_list = [ "1.0.0.0", "58.251.30.115", "192.168.1.100", "126.255.32.255", "219.xx.xx.11", "::", "::1", "fffe::", "2c0f:fff0::", "2c0f:fff0::1", "2a02:26f7:c409:4001::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "240e:982:e617:ffff:ffff:ffff:ffff:ffff", "::xx:ffff" ] for ip in ip_list: try : ip_bytes = util.parse_ip(ip) ip_string = util.ip_to_string(ip_bytes) print("parse_ip({}) -> {{addr:{}, equal:{}}}".format(ip, ip_string, ip_string == ip)) except ValueError as e: print("failed to parse ip `{}`: {}".format(ip, e)) def test_ip_compare(): ip_list = [ ["1.0.0.0", "1.0.0.1", -1], ["192.168.1.101", "192.168.1.90", 1], ["219.133.111.87", "114.114.114.114", 1], ["2000::", "2000:ffff:ffff:ffff:ffff:ffff:ffff:ffff", -1], ["2001:4:112::", "2001:4:112:ffff:ffff:ffff:ffff:ffff", -1], ["ffff::", "2001:4:ffff:ffff:ffff:ffff:ffff:ffff", 1] ] for ip_pair in ip_list: ip1 = util.parse_ip(ip_pair[0]) ip2 = util.parse_ip(ip_pair[1]) cmp = util.ip_compare(ip1, ip2) print("compare({}, {}) -> {} ? {}".format(util.ip_to_string(ip1), util.ip_to_string(ip2), cmp, cmp == ip_pair[2])) def _get_searcher_list(version: util.Version): db_path = xdb_v4_path if version.id == util.XdbIPv4Id else xdb_v6_path return [ ["new_with_file_only", lambda: xdb.new_with_file_only(version, db_path)], ["new_with_vector_index", lambda: xdb.new_with_vector_index(version, db_path, util.load_vector_index_from_file(db_path))], ["new_with_buffer", lambda: xdb.new_with_buffer(version, util.load_content_from_file(db_path))] ] def test_ip_search(): # ipv4 search test print("---ipv4 search test:") ip_str = "120.229.45.92" s_list = _get_searcher_list(util.IPv4) try: b_region = None for meta in s_list: searcher = meta[1]() region = searcher.search(ip_str) if b_region != None: assert b_region == region, f"region and b_region is not the same" print(f"{meta[0]}.search({ip_str}): {region}") # searcher close searcher.close() except Exception as e: print(f"failed to search({ip_str}): {str(e)}") # ipv6 search test print("---ipv6 search test:") ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" s_list = _get_searcher_list(util.IPv6) try: b_region = None for meta in s_list: searcher = meta[1]() region = searcher.search(ip_str) if b_region != None: assert b_region == region, f"region and b_region is not the same" print(f"{meta[0]}.search({ip_str}): {region}") # searcher close searcher.close() except Exception as e: print(f"failed to search({ip_str}): {str(e)}") if __name__ == "__main__": # check and call the specified function if len(sys.argv) < 2: sys.exit("please specified the function to test") func = sys.argv[1] all_ids = globals() if func in all_ids and callable(all_ids[func]): print("+---calling test function {} ...".format(func)) s_time = time.time() all_ids[func]() c_time = time.time() - s_time print(f"|---Done, elapsed {c_time:.6f}s") else: sys.exit("unable to call function {}".format(func)) ================================================ FILE: binding/rust/Cargo.toml ================================================ [workspace] resolver = "2" members = ["example", "ip2region"] ================================================ FILE: binding/rust/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) ## `ip2region rust` Query Client ## Features * Supports queries using both `ip` strings and `u32`/`u28` numeric types * Supports IPv4 and IPv6 * Supports three modes: No Cache, Vector Index Cache, and Full Data Cache ## Cache Policy Comparison and Description | Cache Mode | IPv4 Memory Usage | IPv6 Memory Usage | IPv4 benchmark query time | IPv6 benchmark query time | | --- | --- | --- | --- | --- | | No Cache | 1-2MB | 1-2MB | 54 us | 122us | | vector index | 1-2MB | 1-2MB | 27 us | 100us | | Full Cache | 20 MB | 200 MB | 120 ns | 178 ns | * During the initialization of `ip2region::Searcher`, an IO operation occurs to read the `xdb` header information to initialize the `Searcher`. The header information mainly includes the IP version of the `xdb`. This operation does not affect the performance or time consumption of subsequent IP queries and occupies approximately 20 additional bytes of memory. * In No Cache mode and `vector index` cache mode, all `xdb` IO reads are performed on-demand (based on bytes offset and bytes length) for small amounts of information. Both are thread-safe, as verified by benchmark testing. * In Full Cache mode, the `xdb` file is read and loaded into memory at once. Testing shows the `IPv6 xdb` file occupies about 200MB of memory. If queries are infrequent, memory usage will gradually decrease. * In all cache modes, including during the initialization of `ip2region::Searcher`, the program is thread-safe. There are no globally modifiable intermediate variables. After `ip2region::Searcher` initialization is complete, calling the `search` function uses immutable references. Meanwhile, `ip2region::Searcher` can also be passed to different threads using `Arc`. ## Usage Create a new project using `cargo`, such as `cargo new ip-test` Configure `[dependencies]` in `Cargo.toml` as follows: ```toml [dependencies] ip2region = { git = "https://github.com/lionsoul2014/ip2region.git", branch = "master" } ``` ### Basic Usage Example Write `main.rs` ```rust use ip2region::{CachePolicy, Searcher}; fn main() { for cache_policy in [ CachePolicy::NoCache, CachePolicy::FullMemory, CachePolicy::VectorIndex, ] { // Create an IPv4 searcher let ipv4_seacher = Searcher::new("../ip2region/data/ip2region_v4.xdb".to_owned(), cache_policy).unwrap(); for ip in [1_u32, 2, 3] { let result = ipv4_seacher.search(ip).unwrap(); println!("CachePolicy: {cache_policy:?}, IP: {ip}, Result: {result}"); } for ip in ["1.1.1.1", "2.2.2.2"] { let result = ipv4_seacher.search(ip).unwrap(); println!("CachePolicy: {cache_policy:?}, IP: {ip}, Result: {result}"); } // Create an IPv6 searcher let ipv6_seacher = Searcher::new("../ip2region/data/ip2region_v6.xdb".to_owned(), cache_policy).unwrap(); for ip in ["2001::", "2001:4:112::"] { let result = ipv6_seacher.search(ip).unwrap(); println!("CachePolicy: {cache_policy:?}, IP: {ip}, Result: {result}"); } for ip in [1_u128, 2, 3<<125] { let result = ipv6_seacher.search(ip).unwrap(); println!("CachePolicy: {cache_policy:?}, IP: {ip}, Result: {result}"); } } } ``` ## Cache policy benchmark ```bash $ cd binding/rust/ip2region $ cargo test $ cargo bench // --snip--- ipv4_no_memory_bench time: [54.699 µs 57.401 µs 61.062 µs] Found 16 outliers among 100 measurements (16.00%) 10 (10.00%) high mild 6 (6.00%) high severe ipv4_vector_index_cache_bench time: [25.972 µs 26.151 µs 26.360 µs] Found 9 outliers among 100 measurements (9.00%) 1 (1.00%) low severe 6 (6.00%) high mild 2 (2.00%) high severe ipv4_full_memory_cache_bench time: [132.04 ns 139.48 ns 149.20 ns] Found 10 outliers among 100 measurements (10.00%) 4 (4.00%) high mild 6 (6.00%) high severe ipv6_no_memory_bench time: [121.00 µs 122.14 µs 123.40 µs] Found 5 outliers among 100 measurements (5.00%) 2 (2.00%) high mild 3 (3.00%) high severe ipv6_vector_index_cache_bench time: [96.830 µs 100.23 µs 104.81 µs] Found 8 outliers among 100 measurements (8.00%) 2 (2.00%) high mild 6 (6.00%) high severe ipv6_full_memory_cache_bench time: [175.29 ns 178.82 ns 183.77 ns] Found 6 outliers among 100 measurements (6.00%) 2 (2.00%) high mild 4 (4.00%) high severe // --snip-- ``` ## Testing, Result Verification, and Benchmark ```bash $ cd binding/rust/example $ cargo build -r ``` The location of the built executable is `binding/rust/target/release/searcher` Testing IPv6 and IPv4 requires verifying query results against the contents of `ipv6_source.txt` and `ipv4_source.txt`. **The query results shown here represent data at the current time; subsequent results may differ due to updates and corrections in the IP region segments of `ip_source.txt` and `xdb` binary data.** #### Test IPv6 ```bash $ cd binding/rust $ cargo build -r $ ./target/release/searcher --xdb='../../data/ip2region_v6.xdb' query ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit ip2region>> :: region: Ok(""), took: 79.651412ms ip2region>> 240e:3b7:3273:51d0:cd38:8ae1:e3c0:b708 region: Ok("中国|广东省|深圳市|电信|CN"), took: 7.575µs ip2region>> 2001:: region: Ok("0|0|Reserved|Reserved|Reserved"), took: 7.256µs ip2region>> 2001:268:9a02:8888:: region: Ok("Japan|Aichi|Nagoya|KDDI CORPORATION|JP"), took: 7.921µs ip2region>> 2a02:26f7:b408:a6c2:: region: Ok("United States|Virginia|Emporia|Akamai Technologies, Inc.|US"), took: 8.461µs ip2region>> 2c99:: region: Ok("0|0|Reserved|Reserved|Reserved"), took: 5.33µs ``` #### Test IPv4 ```bash $ cd binding/rust $ cargo build -r $ ./target/release/searcher --xdb='../../data/ip2region_v4.xdb' query ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit ip2region>> 1.2.3.4 region: Ok("Australia|Queensland|Brisbane|0|AU"), took: 6.07µs ip2region>> 1.1.2.1 region: Ok("中国|福建省|福州市|0|CN"), took: 5.653µs ip2region>> 2.2.21.1 region: Ok("United States|Texas|0|Oracle Svenska AB|US"), took: 4.556µs ``` #### Benchmark and Result Verification Test performance via the `searcher` program while comparing query results against `ip sources` files to check for errors. ```bash $ cd binding/rust/example $ cargo build -r ## Perform IPv4 bench test using data/ip2region_v4.xdb and data/ipv4_source.txt: $ RUST_LOG=debug ../target/release/searcher --xdb='../../../data/ip2region_v4.xdb' bench '../../../data/ipv4_source.txt' 2025-09-24T07:02:07.840535Z DEBUG ip2region::searcher: Load xdb file with header header=Header { version: 3, index_policy: VectorIndex, create_time: 1757125456, start_index_ptr: 955933, end_index_ptr: 11042415, ip_version: V4, runtime_ptr_bytes: 4 } 2025-09-24T07:02:07.840894Z DEBUG ip2region::searcher: Load vector index cache 2025-09-24T07:02:07.840905Z DEBUG ip2region::searcher: Load full cache filepath="../../../data/ip2region_v4.xdb" 2025-09-24T07:02:08.409990Z INFO searcher: Benchmark finished count=3404406 took=569.388667ms avg_took=167ns ## Perform IPv6 bench test using data/ip2region_v6.xdb and data/ipv6_source.txt: $ RUST_LOG=debug ../target/release/searcher --xdb='../../../data/ip2region_v6.xdb' bench '../../../data/ipv6_source.txt' 2025-09-24T07:01:48.991835Z DEBUG ip2region::searcher: Load xdb file with header header=Header { version: 3, index_policy: VectorIndex, create_time: 1756970508, start_index_ptr: 6585371, end_index_ptr: 647078145, ip_version: V6, runtime_ptr_bytes: 4 } 2025-09-24T07:01:48.992557Z DEBUG ip2region::searcher: Load vector index cache 2025-09-24T07:01:48.992563Z DEBUG ip2region::searcher: Load full cache filepath="../../../data/ip2region_v6.xdb" 2025-09-24T07:01:59.775879Z INFO searcher: Benchmark finished count=38335905 took=10.784124584s avg_took=281ns ``` ================================================ FILE: binding/rust/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) ## `ip2region rust` 查询客户端 ## Features - 支持`ip`字符串和`u32`/`u28` 数字两种类型的查询 - 支持 IPv4 和 IPv6 - 支持无缓存，Vector 索引缓存，全部数据缓存三种模式 ## 缓存策略对比与说明 | 缓存模式 | IPv4 数据内存占用 | IPv6 数据内存占用 | IPv4 benchmark 查询耗时 | IPv6 benchmark 查询耗时 | | ------------ | ----------- | ----------- | ------------------- |---------------------| | 无缓存 | 1-2MB | 1-2MB | 54 us | 122us | | vector index | 1-2MB | 1-2MB | 27 us | 100us | | 全部缓存 | 20 MB | 200 MB | 120 ns | 178 ns | - 在 `ip2region::Searcher` 初始化的时候会产生一次 IO, 读取 `xdb` 的 header 信息以初始化 `Searcher`，header 信息主要包含了 `xdb` 的 IP 版本，该操作对后续 IP 的查询不产生性能，耗时影响，多占用约 20 Byte 的内存 - 在无缓存模式与 `vector index` 缓存模式下，所有 `xdb` 的 IO 读取都是按需（按照 bytes offset, bytes length）读取少量信息, 都是线程安全的，可以 benchmark 测试验证 - 在全部缓存模式下，`xdb` 文件会一次读取，加载到内存中，测试 `IPv6 xdb` 文件大约占用内存 200MB 左右，查询不频繁的话，占用内存会逐渐降低 - 所有缓存模式下，包括初始化 `ip2region::Searcher` 过程当中，程序都是线程安全的，不存在某个全局可修改的中间变量，`ip2region::Searcher` 初始化完成以后，调用函数`search`都是使用不可变引用，同时 `ip2region::Searcher` 也可以通过 `Arc` 方式传递给不同线程使用 ## 使用方式 使用`cargo`新建一个项目，比如`cargo new ip-test` 配置`Cargo.toml`的`[dependencies]`如下 ```toml [dependencies] ip2region = { git = "https://github.com/lionsoul2014/ip2region.git", branch = "master" } ``` ### 基本使用示例 编写`main.rs` ```rust use ip2region::{CachePolicy, Searcher}; fn main() { for cache_policy in [ CachePolicy::NoCache, CachePolicy::FullMemory, CachePolicy::VectorIndex, ] { let ipv4_seacher = Searcher::new("../ip2region/data/ip2region_v4.xdb".to_owned(), cache_policy).unwrap(); for ip in [1_u32, 2, 3] { let result = ipv4_seacher.search(ip).unwrap(); println!("CachePolicy: {cache_policy:?}, IP: {ip}, Result: {result}"); } for ip in ["1.1.1.1", "2.2.2.2"] { let result = ipv4_seacher.search(ip).unwrap(); println!("CachePolicy: {cache_policy:?}, IP: {ip}, Result: {result}"); } let ipv6_seacher = Searcher::new("../ip2region/data/ip2region_v6.xdb".to_owned(), cache_policy).unwrap(); for ip in ["2001::", "2001:4:112::"] { let result = ipv6_seacher.search(ip).unwrap(); println!("CachePolicy: {cache_policy:?}, IP: {ip}, Result: {result}"); } for ip in [1_u128, 2, 3<<125] { let result = ipv6_seacher.search(ip).unwrap(); println!("CachePolicy: {cache_policy:?}, IP: {ip}, Result: {result}"); } } } ``` ## Cache policy benchmark ```bash $ cd binding/rust/ip2region $ cargo test $ cargo bench // --snip--- ipv4_no_memory_bench time: [54.699 µs 57.401 µs 61.062 µs] Found 16 outliers among 100 measurements (16.00%) 10 (10.00%) high mild 6 (6.00%) high severe ipv4_vector_index_cache_bench time: [25.972 µs 26.151 µs 26.360 µs] Found 9 outliers among 100 measurements (9.00%) 1 (1.00%) low severe 6 (6.00%) high mild 2 (2.00%) high severe ipv4_full_memory_cache_bench time: [132.04 ns 139.48 ns 149.20 ns] Found 10 outliers among 100 measurements (10.00%) 4 (4.00%) high mild 6 (6.00%) high severe ipv6_no_memory_bench time: [121.00 µs 122.14 µs 123.40 µs] Found 5 outliers among 100 measurements (5.00%) 2 (2.00%) high mild 3 (3.00%) high severe ipv6_vector_index_cache_bench time: [96.830 µs 100.23 µs 104.81 µs] Found 8 outliers among 100 measurements (8.00%) 2 (2.00%) high mild 6 (6.00%) high severe ipv6_full_memory_cache_bench time: [175.29 ns 178.82 ns 183.77 ns] Found 6 outliers among 100 measurements (6.00%) 2 (2.00%) high mild 4 (4.00%) high severe // --snip-- ``` ## 测试与结果验证，benchmark ```bash $ cd binding/rust/example $ cargo build -r ``` 构建的执行程序位置 `binding/rust/target/release/searcher` 测试 IPv6 以及 IPv4 需要结合 ipv6_source.txt 以及 ipv4_source.txt 的内容进行查询结果校验 **此处展示的查询结果只表示当前时间数据的查询，后续查询结果可能会由于 ip_source.txt 以及 xdb 二进制数据的 IP region 段更新修正导致不同** #### 测试 IPv6 ```bash $ cd binding/rust $ cargo build -r $ ./target/release/searcher --xdb='../../data/ip2region_v6.xdb' query ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit ip2region>> :: region: Ok(""), took: 79.651412ms ip2region>> 240e:3b7:3273:51d0:cd38:8ae1:e3c0:b708 region: Ok("中国|广东省|深圳市|电信|CN"), took: 7.575µs ip2region>> 2001:: region: Ok("0|0|Reserved|Reserved|Reserved"), took: 7.256µs ip2region>> 2001:268:9a02:8888:: region: Ok("Japan|Aichi|Nagoya|KDDI CORPORATION|JP"), took: 7.921µs ip2region>> 2a02:26f7:b408:a6c2:: region: Ok("United States|Virginia|Emporia|Akamai Technologies, Inc.|US"), took: 8.461µs ip2region>> 2c99:: region: Ok("0|0|Reserved|Reserved|Reserved"), took: 5.33µs ``` #### 测试 IPv4 ```bash $ cd binding/rust $ cargo build -r $ ./target/release/searcher --xdb='../../data/ip2region_v4.xdb' query ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit ip2region>> 1.2.3.4 region: Ok("Australia|Queensland|Brisbane|0|AU"), took: 6.07µs ip2region>> 1.1.2.1 region: Ok("中国|福建省|福州市|0|CN"), took: 5.653µs ip2region>> 2.2.21.1 region: Ok("United States|Texas|0|Oracle Svenska AB|US"), took: 4.556µs ``` #### Benchmark 与验证结果 通过 searcher 程序来测试性能，同时依据 ip sources 文件对比查询结果，检测是否存在错误 ```bash $ cd binding/rust/example $ cargo build -r ## 通过 data/ip2region_v4.xdb 和 data/ipv4_source.txt 进行 ipv4 的 bench 测试： $ RUST_LOG=debug ../target/release/searcher --xdb='../../../data/ip2region_v4.xdb' bench '../../../data/ipv4_source.txt' 2025-09-24T07:02:07.840535Z DEBUG ip2region::searcher: Load xdb file with header header=Header { version: 3, index_policy: VectorIndex, create_time: 1757125456, start_index_ptr: 955933, end_index_ptr: 11042415, ip_version: V4, runtime_ptr_bytes: 4 } 2025-09-24T07:02:07.840894Z DEBUG ip2region::searcher: Load vector index cache 2025-09-24T07:02:07.840905Z DEBUG ip2region::searcher: Load full cache filepath="../../../data/ip2region_v4.xdb" 2025-09-24T07:02:08.409990Z INFO searcher: Benchmark finished count=3404406 took=569.388667ms avg_took=167ns ## 通过 data/ip2region_v6.xdb 和 data/ipv6_source.txt 进行 ipv6 的 bench 测试： $ RUST_LOG=debug ../target/release/searcher --xdb='../../../data/ip2region_v6.xdb' bench '../../../data/ipv6_source.txt' 2025-09-24T07:01:48.991835Z DEBUG ip2region::searcher: Load xdb file with header header=Header { version: 3, index_policy: VectorIndex, create_time: 1756970508, start_index_ptr: 6585371, end_index_ptr: 647078145, ip_version: V6, runtime_ptr_bytes: 4 } 2025-09-24T07:01:48.992557Z DEBUG ip2region::searcher: Load vector index cache 2025-09-24T07:01:48.992563Z DEBUG ip2region::searcher: Load full cache filepath="../../../data/ip2region_v6.xdb" 2025-09-24T07:01:59.775879Z INFO searcher: Benchmark finished count=38335905 took=10.784124584s avg_took=281ns ``` ================================================ FILE: binding/rust/example/Cargo.toml ================================================ [package] name = "searcher" default-run = "searcher" version = "0.2.0" edition = "2024" rust-version = "1.89.0" description = "Rust binding example for ip2region" license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] ip2region = { path = "../ip2region" } clap = { version = "4.5", features = ["derive", "env"] } tracing-subscriber = "0.3" tracing = "0.1" ================================================ FILE: binding/rust/example/src/cmd.rs ================================================ use clap::{Parser, Subcommand, ValueEnum}; /// Rust binding example for ip2region /// /// e.g /// /// ``` /// /// export XDB='../../../data/ip2region_v4.xdb' ## or export XDB='../../../data/ip2region_v6.xdb' /// /// export CHECK='../../../data/ipv4_source.txt' ## or export CHECK='../../../data/ipv6_source.txt' /// /// cd binding/rust/example /// /// ./searcher --xdb=$XDB bench $CHECK /// /// ./searcher --xdb=$XDB query /// /// ``` #[derive(Parser)] pub struct Command { /// xdb filepath, e.g. `../../../data/ip2region_v4.xdb` or `../../../data/ip2region_v6.xdb` #[arg(long, env = "XDB")] pub xdb: String, #[arg(long, value_enum, default_value_t = CmdCachePolicy::FullMemory)] pub cache_policy: CmdCachePolicy, #[clap(subcommand)] pub action: Action, } #[derive(Subcommand)] pub enum Action { /// Bench the ip search and output performance info Bench { check_file: String }, /// Interactive input and output, querying one IP and get result at a time Query, } #[derive(Debug, PartialEq, ValueEnum, Clone, Copy, Default)] pub enum CmdCachePolicy { #[default] FullMemory, NoCache, VectorIndex, } ================================================ FILE: binding/rust/example/src/main.rs ================================================ use std::fs::File; use std::io::Write; use std::io::{BufRead, BufReader}; use std::net::IpAddr; use std::str::FromStr; use std::time::Instant; use clap::Parser; use ip2region::{CachePolicy, Searcher}; use tracing::info; use crate::cmd::{Action, CmdCachePolicy, Command}; mod cmd; macro_rules! perform_check { ($searcher:expr, $start_ip:expr, $end_ip:expr, $check:expr) => {{ let start_ip = $start_ip; let end_ip = $end_ip; let mid_ip = (start_ip >> 1) + (end_ip >> 1); let mut checked = 0; let checks = [ start_ip, (start_ip >> 1) + (mid_ip >> 1), mid_ip, (mid_ip >> 1) + (end_ip >> 1), end_ip, ]; for ip in checks.iter() { if *ip < start_ip || *ip > end_ip { // IP not in start - end ip // This happens when start ip equals end ip continue; } let result = $searcher.search(*ip).unwrap(); assert_eq!(result.as_str(), $check); checked += 1; } checked }}; } fn check(searcher: &Searcher, start_ip: IpAddr, end_ip: IpAddr, check: &str) -> usize { match (start_ip, end_ip) { (IpAddr::V4(original_start_ip), IpAddr::V4(original_end_ip)) => { let start_ip = u32::from(original_start_ip); let end_ip = u32::from(original_end_ip); perform_check!(searcher, start_ip, end_ip, check) } (IpAddr::V6(original_start_ip), IpAddr::V6(original_end_ip)) => { let start_ip = u128::from(original_start_ip); let end_ip = u128::from(original_end_ip); perform_check!(searcher, start_ip, end_ip, check) } _ => panic!("invalid start ip and end ip"), } } fn bench(searcher: &Searcher, check_filepath: &str) { let file = File::open(check_filepath).unwrap(); let reader = BufReader::new(file); let now = Instant::now(); let mut count = 0; for line in reader.lines().map_while(Result::ok) { let ip_test_line = line.splitn(3, '|').collect::>(); if ip_test_line.len() == 3 { let start_ip = IpAddr::from_str(ip_test_line[0]).unwrap(); let end_ip = IpAddr::from_str(ip_test_line[1]).unwrap(); count += check(searcher, start_ip, end_ip, ip_test_line[2]); } } info!(count, took=?now.elapsed(), avg_took=?(now.elapsed() / (count as u32)), "Benchmark finished"); } fn query(searcher: &Searcher) { println!("ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit"); loop { print!("ip2region>> "); std::io::stdout().flush().unwrap(); let mut line = String::new(); std::io::stdin().read_line(&mut line).unwrap(); if line.contains("quit") { break; } let line = line.trim(); let now = Instant::now(); let result = searcher.search(line); let cost = now.elapsed(); println!("region: {result:?}, took: {cost:?}",); } } fn main() { tracing_subscriber::fmt::init(); let cmd = Command::parse(); let cache_policy = match cmd.cache_policy { CmdCachePolicy::FullMemory => CachePolicy::FullMemory, CmdCachePolicy::VectorIndex => CachePolicy::VectorIndex, CmdCachePolicy::NoCache => CachePolicy::NoCache, }; let searcher = Searcher::new(cmd.xdb, cache_policy).unwrap(); match cmd.action { Action::Bench { check_file } => bench(&searcher, &check_file), Action::Query => query(&searcher), } } ================================================ FILE: binding/rust/ip2region/Cargo.toml ================================================ [package] name = "ip2region" version = "0.2.1" edition = "2024" rust-version = "1.89.0" description = "The rust binding for ip2region" license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] tracing = "0.1" thiserror = "2" maker = { path = "../../../maker/rust/maker"} [dev-dependencies] criterion = "0.7" rand = "0.9" [[bench]] name = "search" harness = false ================================================ FILE: binding/rust/ip2region/benches/search.rs ================================================ use std::net::Ipv6Addr; use std::ops::Range; use std::str::FromStr; use criterion::{Criterion, criterion_group, criterion_main}; use ip2region::{CachePolicy, Searcher}; macro_rules! bench_search { ($name:ident, $xdb:expr, $cache_policy:expr, $range:ident) => { fn $name(c: &mut Criterion) { let searcher = Searcher::new($xdb.to_owned(), $cache_policy).unwrap(); let range = $range(); c.bench_function(stringify!($name), |b| { b.iter(|| { searcher.search(rand::random_range(range.clone())).unwrap(); }) }); } }; } fn ipv4_range() -> Range { 0..((1_u64 << 32) - 1) as u32 } /// The range of IPv6 is too large, and the value range needs to be limited to /// make the benchmark test results closer to the production environment fn ipv6_range() -> Range { let start = u128::from(Ipv6Addr::from_str("2000::").unwrap()); let end = u128::from(Ipv6Addr::from_str("2004::").unwrap()); start..end } const IPV4_XDB: &str = "../../../data/ip2region_v4.xdb"; const IPV6_XDB: &str = "../../../data/ip2region_v6.xdb"; bench_search!( ipv4_no_memory_bench, IPV4_XDB, CachePolicy::NoCache, ipv4_range ); bench_search!( ipv4_vector_index_cache_bench, IPV4_XDB, CachePolicy::VectorIndex, ipv4_range ); bench_search!( ipv4_full_memory_cache_bench, IPV4_XDB, CachePolicy::FullMemory, ipv4_range ); bench_search!( ipv6_no_memory_bench, IPV6_XDB, CachePolicy::NoCache, ipv6_range ); bench_search!( ipv6_vector_index_cache_bench, IPV6_XDB, CachePolicy::VectorIndex, ipv6_range ); bench_search!( ipv6_full_memory_cache_bench, IPV6_XDB, CachePolicy::FullMemory, ipv6_range ); criterion_group!( benches, ipv4_no_memory_bench, ipv4_vector_index_cache_bench, ipv4_full_memory_cache_bench, ipv6_no_memory_bench, ipv6_vector_index_cache_bench, ipv6_full_memory_cache_bench ); criterion_main!(benches); ================================================ FILE: binding/rust/ip2region/src/error.rs ================================================ #[derive(Debug, thiserror::Error)] pub enum Ip2RegionError { #[error("Io error: {0}")] IoError(#[from] std::io::Error), #[error("From UTF-8 error: {0}")] Utf8Error(#[from] std::string::FromUtf8Error), #[error("Parse invalid IP address")] ParseIpaddressFailed, #[error("No matched Ipaddress")] NoMatchedIP, #[error("Searcher load IPv4 data, couldn't search IPv6 data")] OnlyIPv4Version, #[error("Searcher load IPv6 data, couldn't search IPv4 data")] OnlyIPv6Version, #[error("Try from slice failed")] TryFromSliceFailed(#[from] std::array::TryFromSliceError), #[error("Maker crate error: {0}")] MakerError(#[from] maker::MakerError), } pub type Result = std::result::Result; ================================================ FILE: binding/rust/ip2region/src/ip_value.rs ================================================ use std::borrow::Cow; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::str::FromStr; use crate::error::{Ip2RegionError, Result}; pub trait IpValueExt { fn to_ipaddr(self) -> Result; } impl IpValueExt for &str { fn to_ipaddr(self) -> Result { IpAddr::from_str(self).map_err(|_| Ip2RegionError::ParseIpaddressFailed) } } impl IpValueExt for u32 { fn to_ipaddr(self) -> Result { Ok(IpAddr::V4(Ipv4Addr::from(self))) } } impl IpValueExt for Ipv4Addr { fn to_ipaddr(self) -> Result { Ok(IpAddr::V4(self)) } } impl IpValueExt for Ipv6Addr { fn to_ipaddr(self) -> Result { Ok(IpAddr::V6(self)) } } impl IpValueExt for u128 { fn to_ipaddr(self) -> Result { Ok(IpAddr::V6(Ipv6Addr::from(self))) } } pub trait CompareExt { fn ip_lt(&self, other: Cow<'_, [u8]>) -> bool; fn ip_gt(&self, other: Cow<'_, [u8]>) -> bool; } impl CompareExt for IpAddr { fn ip_lt(&self, other: Cow<'_, [u8]>) -> bool { match self { IpAddr::V4(ip) => ip.octets() < [other[3], other[2], other[1], other[0]], IpAddr::V6(ip) => ip.octets() < other[0..16].try_into().unwrap(), } } fn ip_gt(&self, other: Cow<'_, [u8]>) -> bool { match self { IpAddr::V4(ip) => ip.octets() > [other[3], other[2], other[1], other[0]], IpAddr::V6(ip) => ip.octets() > other[0..16].try_into().unwrap(), } } } ================================================ FILE: binding/rust/ip2region/src/lib.rs ================================================ mod error; mod ip_value; mod searcher; pub use ip_value::IpValueExt; pub use searcher::{CachePolicy, Searcher}; ================================================ FILE: binding/rust/ip2region/src/searcher.rs ================================================ use std::borrow::Cow; use std::fmt::Display; use std::fs::File; use std::io::{Read, Seek, SeekFrom}; use std::net::IpAddr; use std::path::Path; use std::sync::OnceLock; use maker::{ HEADER_INFO_LENGTH, Header, IpVersion, VECTOR_INDEX_COLS, VECTOR_INDEX_LENGTH, VECTOR_INDEX_SIZE, }; use tracing::{debug, trace, warn}; use crate::error::{Ip2RegionError, Result}; use crate::ip_value::{CompareExt, IpValueExt}; pub struct Searcher { pub filepath: String, pub cache_policy: CachePolicy, pub header: Header, vector_cache: OnceLock>, full_cache: OnceLock>, } #[derive(PartialEq, Debug, Copy, Clone)] pub enum CachePolicy { NoCache, VectorIndex, FullMemory, } impl Searcher { pub fn new(filepath: String, cache_policy: CachePolicy) -> Result { let mut file = File::open(Path::new(&filepath))?; let mut buf = [0; HEADER_INFO_LENGTH]; file.read_exact(&mut buf)?; let header = Header::try_from(&buf)?; debug!(?header, "Load xdb file with header"); Ok(Self { filepath, cache_policy, header, vector_cache: OnceLock::new(), full_cache: OnceLock::new(), }) } pub fn search(&self, ip: T) -> Result where T: IpValueExt + Display, { let ip = ip.to_ipaddr()?; let (il0, il1) = match (ip, self.header.ip_version()) { (IpAddr::V6(ip), IpVersion::V6) => (ip.octets()[0], ip.octets()[1]), (IpAddr::V4(ip), IpVersion::V4) => (ip.octets()[0], ip.octets()[1]), (_, IpVersion::V4) => return Err(Ip2RegionError::OnlyIPv4Version), (_, IpVersion::V6) => return Err(Ip2RegionError::OnlyIPv6Version), }; let start_point = VECTOR_INDEX_SIZE * ((il0 as usize) * VECTOR_INDEX_COLS + (il1 as usize)); let vector_index = self.vector_index()?; let start_ptr = u32::from_le_bytes(vector_index[start_point..start_point + 4].try_into()?) as usize; let end_ptr = u32::from_le_bytes(vector_index[start_point + 4..start_point + 8].try_into()?) as usize; // @Note: ptr validate, zero ptr means source data missing // so we could just stop here and return an empty string. if start_ptr == 0 || end_ptr == 0 { return Ok(String::new()) } // Binary search the segment index to get the region let segment_index_size = self.header.segment_index_size(); let ip_bytes_len = self.header.ip_bytes_len(); let ip_end_offset = ip_bytes_len * 2; let mut left: usize = 0; let mut right: usize = (end_ptr - start_ptr) / segment_index_size; while left <= right { let mid = (left + right) >> 1; let offset = start_ptr + mid * segment_index_size; let buffer_ip_value = self.read_buf(offset, segment_index_size)?; if ip.ip_lt(Cow::Borrowed(&buffer_ip_value[0..ip_bytes_len])) { let Some(m) = mid.checked_sub(1) else { break }; right = m; } else if ip.ip_gt(Cow::Borrowed(&buffer_ip_value[ip_bytes_len..ip_end_offset])) { left = mid + 1; } else { let data_length = u16::from_le_bytes([ buffer_ip_value[ip_end_offset], buffer_ip_value[ip_end_offset + 1], ]); let data_offset = u32::from_le_bytes( buffer_ip_value[ip_end_offset + 2..ip_end_offset + 6].try_into()?, ); let result = String::from_utf8( self.read_buf(data_offset as usize, data_length as usize)? .to_vec(), )?; return Ok(result); } } // From xdb 3.0 version no matched IP result change to empty string, // so users should check string is empty. // // Err(Ip2RegionError::NoMatchedIP) Ok(String::new()) } pub fn vector_index(&self) -> Result> { if self.cache_policy.eq(&CachePolicy::NoCache) { return self.read_buf(HEADER_INFO_LENGTH, VECTOR_INDEX_LENGTH); } match self.vector_cache.get() { None => { debug!("Load vector index cache"); let data = self .read_buf(HEADER_INFO_LENGTH, VECTOR_INDEX_LENGTH)? .to_vec(); let _ = self .vector_cache .set(data) .inspect_err(|_| warn!("Vector index cache already initialized")); // Safety: vector cache checked and set for empty before let cache = self.vector_cache.get().unwrap(); Ok(Cow::Borrowed(cache)) } Some(cache) => Ok(Cow::Borrowed(cache)), } } pub fn read_buf(&self, offset: usize, size: usize) -> Result> { trace!(offset, size = size, "Read buffer"); if self.cache_policy.ne(&CachePolicy::FullMemory) { debug!(filepath=?self.filepath, offset=offset, size=size, "Read buf without cache"); let mut file = File::open(&self.filepath)?; file.seek(SeekFrom::Start(offset as u64))?; let mut buf = vec![0u8; size]; file.take(size as u64).read_exact(&mut buf)?; return Ok(Cow::from(buf)); } match self.full_cache.get() { None => { debug!(filepath=?self.filepath, "Load full cache"); let mut file = File::open(&self.filepath)?; let mut buf = Vec::new(); file.read_to_end(&mut buf)?; let _ = self .full_cache .set(buf) .inspect_err(|_| warn!("Full cache already initialized")); // Safety: FULL_CACHE checked and set for empty before let cache = self.full_cache.get().unwrap(); Ok(Cow::from(&cache[offset..offset + size])) } Some(cache) => { let data = Cow::from(&cache[offset..offset + size]); Ok(data) } } } } #[cfg(test)] mod tests { use std::fs::File; use std::io::{BufRead, BufReader}; use std::str::FromStr; use super::*; // Test ipv6 need after run command `git lfs pull` const IPV4_XDB_PATH: &str = "../../../data/ip2region_v4.xdb"; const IPV4_CHECK_PATH: &str = "../../../data/ipv4_source.txt"; const IPV6_XDB_PATH: &str = "../../../data/ip2region_v6.xdb"; const IPV6_CHECK_PATH: &str = "../../../data/ipv6_source.txt"; ///test all types find correct #[test] fn test_multi_type_ip() { for cache_policy in [ CachePolicy::NoCache, CachePolicy::FullMemory, CachePolicy::VectorIndex, ] { let searcher = Searcher::new(IPV4_XDB_PATH.to_owned(), cache_policy).unwrap(); searcher.search("1.0.1.0").unwrap(); searcher.search("1.0.1.2").unwrap(); searcher.search(0u32).unwrap(); let searcher = Searcher::new(IPV6_XDB_PATH.to_owned(), cache_policy).unwrap(); searcher.search("2c0f:fff1::").unwrap(); searcher.search("2c0f:fff1::1").unwrap(); searcher.search(111u128).unwrap(); } } fn match_ip_correct(xdb_filepath: &str, check_path: &str, cache_policy: CachePolicy) { let searcher = Searcher::new(xdb_filepath.to_owned(), cache_policy).unwrap(); let file = File::open(check_path).unwrap(); let reader = BufReader::new(file); for line in reader.lines().take(10_000) { let line = line.unwrap(); if !line.contains("|") { continue; } let ip_test_line = line.splitn(3, "|").collect::>(); let start_ip = IpAddr::from_str(ip_test_line[0]).unwrap(); let end_ip = IpAddr::from_str(ip_test_line[1]).unwrap(); for _ in 0..3 { let result = match (start_ip, end_ip) { (IpAddr::V4(start), IpAddr::V4(end)) => { let value = rand::random_range(u32::from(start)..u32::from(end) + 1); searcher.search(value).unwrap() } (IpAddr::V6(start), IpAddr::V6(end)) => { let value = rand::random_range(u128::from(start)..u128::from(end) + 1); searcher.search(value).unwrap() } _ => panic!("invalid ip address"), }; assert_eq!(result.as_str(), ip_test_line[2]) } } } #[test] fn test_match_ip_correct() { for cache_policy in [ CachePolicy::NoCache, CachePolicy::FullMemory, CachePolicy::VectorIndex, ] { match_ip_correct(IPV4_XDB_PATH, IPV4_CHECK_PATH, cache_policy); match_ip_correct(IPV6_XDB_PATH, IPV6_CHECK_PATH, cache_policy); } } } ================================================ FILE: binding/typescript/README.md ================================================ # :cn: [中文简体] # ip2region typescript xdb 查询客户端 请使用最新的 IPv6 兼容的 javascript binding：[javascript binding](../javascript/) --- # :globe_with_meridians: [English] # ip2region typescript query client Please use the latest IPv6-compliant JavaScript binding: [javascript binding](../javascript/) ================================================ FILE: data/ip2region_v4.xdb ================================================ [File too large to display: 10.2 MB] ================================================ FILE: data/ip2region_v6.xdb ================================================ [File too large to display: 34.4 MB] ================================================ FILE: data/ipv4_source.txt ================================================ [File too large to display: 33.0 MB] ================================================ FILE: data/ipv6_source.txt ================================================ [File too large to display: 70.6 MB] ================================================ FILE: data/sample/github-issue-196.fix ================================================ 39.144.0.0|39.144.0.255|中国|山东省|菏泽市|移动 39.144.1.0|39.144.9.255|中国|0|0|移动 39.144.10.0|39.144.12.255|中国|北京|北京|移动 39.144.13.0|39.144.16.255|中国|广东|汕头|移动 39.144.17.0|39.144.18.255|中国|重庆|重庆|移动 39.144.19.0|39.144.19.255|中国|北京|北京|移动 39.144.20.0|39.144.20.255|中国|重庆|重庆|移动 39.144.21.0|39.144.29.255|中国|河南省|0|移动 39.144.22.0|39.144.22.255|中国|河南|郑州|移动 39.144.23.0|39.144.24.255|中国|河南|南阳|移动 39.144.25.0|39.144.29.255|中国|河南|0|移动 39.144.34.0|39.144.34.255|中国|安徽省|0|移动 39.144.37.0|39.144.37.255|中国|安徽|蚌埠|移动 39.144.38.0|39.144.38.255|中国|安徽省|合肥市|移动 39.144.39.0|39.144.40.255|中国|上海|上海|移动 39.144.41.0|39.144.42.255|中国|贵州|毕节|移动 39.144.43.0|39.144.47.255|中国|上海|上海|移动 39.144.48.0|39.144.48.255|中国|北京|北京|移动 39.144.49.0|39.144.49.255|中国|河北|廊坊|移动 39.144.50.0|39.144.50.255|中国|河北|石家庄|移动 39.144.51.0|39.144.52.255|中国|北京|北京|移动 39.144.53.0|39.144.53.255|中国|辽宁|朝阳|移动 39.144.54.0|39.144.54.255|中国|辽宁|盘锦|移动 39.144.55.0|39.144.55.255|中国|辽宁|大连|移动 39.144.56.0|39.144.56.255|中国|辽宁|本溪|移动 39.144.57.0|39.144.58.255|中国|辽宁|沈阳|移动 39.144.64.0|39.144.64.255|中国|广西|0|移动 39.144.66.0|39.144.66.255|中国|广西|桂林|移动 39.144.67.0|39.144.68.255|中国|北京|北京|移动 39.144.99.0|39.144.99.255|中国|山西|临汾|移动 39.144.100.0|39.144.100.255|中国|吉林|长春|移动 39.144.137.0|39.144.137.255|中国|四川省|成都市|移动 39.144.138.0|39.144.138.255|中国|四川省|阿坝藏族羌族自治州|移动 39.144.145.0|39.144.145.255|中国|云南省|昆明市|移动 39.144.147.0|39.144.147.255|中国|云南省|0|移动 39.144.151.0|39.144.151.255|中国|江苏|淮安|移动 39.144.153.0|39.144.153.255|中国|江苏|苏州|移动 39.144.154.0|39.144.154.127|中国|江苏省|无锡市|移动 39.144.154.128|39.144.154.255|中国|江苏省|常州市|移动 39.144.169.0|39.144.169.63|中国|江西省|宜春市|移动 39.144.169.64|39.144.169.127|中国|江西省|吉安市|移动 39.144.169.128|39.144.169.191|中国|江西省|赣州市|移动 39.144.169.192|39.144.169.255|中国|江西省|南昌市|移动 39.144.177.0|39.144.177.255|中国|河南省|郑州市|移动 39.144.179.0|39.144.182.255|中国|河南|0|移动 39.144.218.0|39.144.222.255|中国|重庆|重庆市|移动 39.144.219.0|39.144.219.255|中国|重庆|重庆|移动 ================================================ FILE: data/sample/github-issue-200.fix ================================================ 112.224.0.0|112.224.63.255|中国|山东省|济南市|联通 112.224.64.0|112.224.64.255|中国|山东省|青岛市|联通 112.224.65.0|112.224.65.255|中国|山东省|0|联通 112.224.66.0|112.224.66.255|中国|山东省|青岛市|联通 112.224.67.0|112.224.67.255|中国|山东省|0|联通 112.224.68.0|112.224.71.255|中国|山东省|青岛市|联通 112.224.72.0|112.224.73.255|中国|山东省|0|联通 112.224.74.0|112.224.127.255|中国|山东省|青岛市|联通 112.224.128.0|112.224.128.255|中国|山东省|0|联通 112.224.129.0|112.224.130.255|中国|山东省|泰安市|联通 112.224.131.0|112.224.132.255|中国|山东省|0|联通 112.224.133.0|112.224.136.255|中国|山东省|烟台市|联通 112.224.137.0|112.224.138.255|中国|山东省|潍坊市|联通 112.224.139.0|112.224.140.255|中国|山东省|烟台市|联通 112.224.141.0|112.224.144.255|中国|山东省|0|联通 112.224.145.0|112.224.146.255|中国|山东省|临沂市|联通 112.224.147.0|112.224.148.255|中国|山东省|0|联通 112.224.149.0|112.224.152.255|中国|河北省|0|联通 112.224.153.0|112.224.153.255|中国|山东省|青岛市|联通 112.224.154.0|112.224.154.255|中国|山东省|0|联通 112.224.155.0|112.224.156.255|中国|山东省|青岛市|联通 112.224.157.0|112.224.157.255|中国|山东省|济南市|联通 112.224.158.0|112.224.159.255|中国|山东省|临沂市|联通 112.224.160.0|112.224.163.255|中国|山东省|0|联通 112.224.164.0|112.224.164.255|中国|山东省|青岛市|联通 112.224.165.0|112.224.167.255|中国|山东省|0|联通 112.224.168.0|112.224.255.255|中国|山东省|济南市|联通 112.225.0.0|112.226.255.255|中国|山东省|青岛市|联通 ================================================ FILE: data/sample/github-issue-243.fix ================================================ 36.132.128.0|36.132.147.255|中国|黑龙江省|哈尔滨市|移动 36.132.148.0|36.132.150.255|中国|黑龙江省|齐齐哈尔市|移动 36.132.151.0|36.132.255.255|中国|黑龙江省|哈尔滨市|移动 36.133.0.0|36.133.11.255|中国|广东省|广州市|移动 36.133.12.0|36.133.23.255|中国|湖南省|长沙市|移动 36.133.24.0|36.133.35.255|中国|江苏省|南京市|移动 36.133.36.0|36.133.47.255|中国|河南省|郑州市|移动 36.133.48.0|36.133.59.255|中国|北京|北京市|移动 36.133.60.0|36.133.71.255|中国|四川省|成都市|移动 36.133.72.0|36.133.83.255|中国|上海|上海市|移动 36.133.84.0|36.133.95.255|中国|浙江省|0|移动 36.133.96.0|36.133.107.255|中国|陕西省|西安市|移动 36.133.108.0|36.133.119.255|中国|重庆|重庆市|移动 36.133.120.0|36.133.131.255|中国|山东省|济南市|移动 36.133.132.0|36.133.143.255|中国|江苏省|苏州市|移动 36.133.144.0|36.133.151.255|中国|广东省|广州市|移动 36.133.152.0|36.133.159.255|中国|湖南省|长沙市|移动 36.133.160.0|36.133.175.255|中国|江苏省|南京市|移动 36.133.176.0|36.133.191.255|中国|广东省|广州市|移动 36.133.192.0|36.133.199.255|中国|北京|北京市|移动 36.133.200.0|36.133.207.255|中国|四川省|成都市|移动 36.133.208.0|36.133.223.255|中国|河南省|0|移动 36.133.224.0|36.133.231.255|中国|广东省|广州市|移动 36.133.232.0|36.133.239.255|中国|重庆|重庆市|移动 36.133.240.0|36.133.247.255|中国|北京|北京市|移动 36.133.248.0|36.133.255.255|中国|陕西省|西安市|移动 36.134.0.0|36.134.15.255|中国|浙江省|杭州市|移动 36.134.16.0|36.134.23.255|中国|江苏省|南京市|移动 36.134.24.0|36.134.31.255|中国|重庆|重庆市|移动 36.134.32.0|36.134.39.255|中国|上海|上海市|移动 36.134.40.0|36.134.47.255|中国|山东省|0|移动 36.134.48.0|36.134.63.255|中国|浙江省|杭州市|移动 36.134.64.0|36.134.65.255|中国|天津|天津市|移动 36.134.66.0|36.134.67.255|中国|吉林省|长春市|移动 36.134.68.0|36.134.69.255|中国|辽宁省|0|移动 36.134.70.0|36.134.71.255|中国|福建省|福州市|移动 36.134.72.0|36.134.73.255|中国|甘肃省|兰州市|移动 36.134.74.0|36.134.75.255|中国|云南省|昆明市|移动 36.134.76.0|36.134.77.255|中国|山西省|太原市|移动 36.134.78.0|36.134.79.255|中国|湖北省|武汉市|移动 36.134.80.0|36.134.81.255|中国|江西省|南昌市|移动 36.134.82.0|36.134.83.255|中国|0|0|移动 36.134.84.0|36.134.85.255|中国|安徽省|合肥市|移动 36.134.86.0|36.134.87.255|中国|广西|南宁市|移动 36.134.88.0|36.134.89.255|中国|内蒙古|呼和浩特市|移动 36.142.0.0|36.142.1.255|中国|四川省|成都市|移动 36.142.2.0|36.142.31.255|中国|甘肃省|兰州市|移动 36.142.32.0|36.142.127.255|中国|甘肃省|0|移动 36.142.128.0|36.142.131.255|中国|甘肃省|白银市|移动 36.142.132.0|36.142.135.255|中国|甘肃省|张掖市|移动 36.142.136.0|36.142.139.255|中国|甘肃省|武威市|移动 36.142.140.0|36.142.145.255|中国|甘肃省|定西市|移动 36.142.146.0|36.142.153.255|中国|甘肃省|天水市|移动 36.142.154.0|36.142.157.255|中国|甘肃省|平凉市|移动 36.142.158.0|36.142.163.255|中国|甘肃省|庆阳市|移动 36.142.164.0|36.142.165.255|中国|甘肃省|甘南市|移动 36.142.166.0|36.142.171.255|中国|甘肃省|陇南市|移动 36.142.172.0|36.142.175.255|中国|甘肃省|临夏市|移动 36.142.176.0|36.142.187.255|中国|甘肃省|兰州市|移动 36.142.188.0|36.142.189.255|中国|甘肃省|金昌市|移动 36.142.190.0|36.142.191.255|中国|甘肃省|嘉峪关市|移动 36.142.192.0|36.142.195.255|中国|甘肃省|酒泉市|移动 36.142.196.0|36.142.197.255|中国|甘肃省|白银市|移动 36.142.198.0|36.142.199.255|中国|甘肃省|张掖市|移动 36.142.200.0|36.142.201.255|中国|甘肃省|武威市|移动 36.142.202.0|36.142.203.255|中国|甘肃省|定西市|移动 36.142.204.0|36.142.205.255|中国|甘肃省|天水市|移动 36.142.206.0|36.142.207.255|中国|甘肃省|平凉市|移动 36.142.208.0|36.142.209.255|中国|甘肃省|庆阳市|移动 36.142.210.0|36.142.211.255|中国|甘肃省|甘南市|移动 36.142.212.0|36.142.213.255|中国|甘肃省|陇南市|移动 36.142.214.0|36.142.215.255|中国|甘肃省|临夏市|移动 36.142.216.0|36.142.217.255|中国|甘肃省|兰州市|移动 36.142.218.0|36.142.219.255|中国|甘肃省|金昌市|移动 36.142.220.0|36.142.221.255|中国|甘肃省|嘉峪关市|移动 36.142.222.0|36.142.223.255|中国|甘肃省|酒泉市|移动 36.142.224.0|36.142.255.255|中国|甘肃省|0|移动 36.143.0.0|36.143.0.255|中国|河北省|张家口市|移动 36.143.1.0|36.143.2.255|中国|河北省|廊坊市|移动 36.143.3.0|36.143.4.255|中国|河北省|邯郸市|移动 36.143.5.0|36.143.12.255|中国|河北省|张家口市|移动 36.143.13.0|36.143.14.255|中国|河北省|保定市|移动 36.143.15.0|36.143.17.255|中国|河北省|石家庄市|移动 36.143.18.0|36.143.22.255|中国|河北省|唐山市|移动 36.143.23.0|36.143.29.255|中国|河北省|保定市|移动 36.143.30.0|36.143.31.255|中国|河北省|石家庄市|移动 36.143.32.0|36.143.35.255|中国|河北省|保定市|移动 36.143.36.0|36.143.39.255|中国|河北省|唐山市|移动 36.143.40.0|36.143.43.255|中国|河北省|邯郸市|移动 36.143.44.0|36.143.47.255|中国|河北省|保定市|移动 36.143.48.0|36.143.51.255|中国|河北省|张家口市|移动 36.143.52.0|36.143.55.255|中国|河北省|0|移动 36.143.56.0|36.143.59.255|中国|河北省|邯郸市|移动 36.143.60.0|36.143.63.255|中国|河北省|廊坊市|移动 36.143.64.0|36.143.67.255|中国|河北省|石家庄市|移动 36.143.68.0|36.143.71.255|中国|河北省|保定市|移动 36.143.72.0|36.143.75.255|中国|河北省|张家口市|移动 36.143.76.0|36.143.79.255|中国|河北省|保定市|移动 36.143.80.0|36.143.95.255|中国|河北省|0|移动 36.143.96.0|36.143.99.255|中国|河北省|唐山市|移动 36.143.100.0|36.143.107.255|中国|河北省|石家庄市|移动 36.143.108.0|36.143.115.255|中国|河北省|廊坊市|移动 36.143.116.0|36.143.119.255|中国|河北省|唐山市|移动 36.143.120.0|36.143.123.255|中国|河北省|张家口市|移动 36.143.124.0|36.143.127.255|中国|河北省|保定市|移动 36.143.128.0|36.143.255.255|中国|北京|北京市|移动 36.148.0.0|36.148.31.255|中国|湖南省|长沙市|移动 36.148.32.0|36.148.49.255|中国|湖南省|常德市|移动 36.148.50.0|36.148.63.255|中国|湖南省|娄底市|移动 36.148.64.0|36.148.79.255|中国|湖南省|株洲市|移动 36.148.80.0|36.148.87.255|中国|湖南省|郴州市|移动 36.148.88.0|36.148.95.255|中国|湖南省|衡阳市|移动 36.148.96.0|36.148.103.255|中国|湖南省|怀化市|移动 36.148.104.0|36.148.107.255|中国|湖南省|永州市|移动 36.148.108.0|36.148.111.255|中国|湖南省|益阳市|移动 36.148.112.0|36.148.123.255|中国|湖南省|张家界市|移动 36.148.124.0|36.148.124.255|中国|湖南省|长沙市|移动 36.148.125.0|36.148.127.255|中国|湖南省|湘潭市|移动 36.148.128.0|36.148.143.255|中国|湖南省|常德市|移动 36.148.144.0|36.148.155.255|中国|湖南省|郴州市|移动 36.148.156.0|36.148.159.255|中国|湖南省|衡阳市|移动 36.148.160.0|36.148.163.255|中国|湖南省|娄底市|移动 36.148.164.0|36.148.168.255|中国|湖南省|永州市|移动 36.148.169.0|36.148.175.255|中国|湖南省|益阳市|移动 36.148.176.0|36.148.177.255|中国|湖南省|岳阳市|移动 36.148.178.0|36.148.189.255|中国|湖南省|长沙市|移动 36.148.190.0|36.148.255.255|中国|湖南省|0|移动 36.161.0.0|36.161.15.255|中国|安徽省|六安市|移动 36.161.16.0|36.161.31.255|中国|安徽省|蚌埠市|移动 36.161.32.0|36.161.47.255|中国|安徽省|淮南市|移动 36.161.48.0|36.161.63.255|中国|安徽省|铜陵市|移动 36.161.64.0|36.161.79.255|中国|安徽省|池州市|移动 36.161.80.0|36.161.95.255|中国|安徽省|黄山市|移动 36.161.96.0|36.161.127.255|中国|安徽省|合肥市|移动 36.161.128.0|36.161.131.255|中国|安徽省|淮北市|移动 36.161.132.0|36.161.135.255|中国|安徽省|合肥市|移动 36.161.136.0|36.161.143.255|中国|安徽省|宣城市|移动 36.161.144.0|36.161.151.255|中国|安徽省|马鞍山市|移动 36.161.152.0|36.161.157.255|中国|安徽省|铜陵市|移动 36.161.158.0|36.161.159.255|中国|安徽省|合肥市|移动 36.161.160.0|36.161.167.255|中国|安徽省|池州市|移动 36.161.168.0|36.161.183.255|中国|安徽省|黄山市|移动 36.161.184.0|36.161.191.255|中国|安徽省|宿州市|移动 36.161.192.0|36.161.207.255|中国|安徽省|亳州市|移动 36.161.208.0|36.161.223.255|中国|安徽省|六安市|移动 36.161.224.0|36.161.231.255|中国|安徽省|宣城市|移动 36.161.232.0|36.161.239.255|中国|安徽省|合肥市|移动 36.161.240.0|36.161.240.255|中国|安徽省|芜湖市|移动 36.161.241.0|36.161.247.255|中国|安徽省|安庆市|移动 36.161.248.0|36.161.255.255|中国|安徽省|黄山市|移动 120.231.0.0|120.231.4.255|中国|广东省|0|移动 120.231.5.0|120.231.38.255|中国|广东省|湛江市|移动 120.231.39.0|120.231.68.255|中国|广东省|茂名市|移动 120.231.69.0|120.231.89.255|中国|广东省|肇庆市|移动 120.231.90.0|120.231.94.255|中国|广东省|0|移动 120.231.95.0|120.231.134.255|中国|广东省|中山市|移动 120.231.135.0|120.231.141.255|中国|广东省|广州市|移动 120.231.142.0|120.231.161.255|中国|广东省|清远市|移动 120.231.162.0|120.231.186.255|中国|广东省|珠海市|移动 120.231.187.0|120.231.201.255|中国|广东省|江门市|移动 120.231.202.0|120.231.249.255|中国|广东省|深圳市|移动 120.231.250.0|120.231.255.255|中国|广东省|清远市|移动 ================================================ FILE: data/sample/github-issue-287.bug ================================================ # report by https://github.com/lionsoul2014/ip2region/issues/287 # triggered the bug of getInt2 193.150.116.0|193.150.116.255|俄罗斯|克拉斯诺亚尔斯克边疆区|克拉斯诺亚尔斯克|Federal State Budgetary Educational Institution of Higher Education Krasnoyarsk State Medical University named after Professor V.F. Voino-Yasenetsky of the Ministry of Health of the Russian Federation ================================================ FILE: data/sample/ip.test.txt ================================================ 1.0.0.0|1.0.0.255|澳大利亚|0|0|0 1.0.1.0|1.0.3.255|中国|福建省|福州市|电信 1.0.4.0|1.0.7.255|澳大利亚|维多利亚|墨尔本|0 1.0.8.0|1.0.15.255|中国|广东省|广州市|电信 1.0.16.0|1.0.31.255|日本|0|0|0 1.0.32.0|1.0.63.255|中国|广东省|广州市|电信 1.0.64.0|1.0.79.255|日本|广岛县|0|0 1.0.80.0|1.0.127.255|日本|冈山县|0|0 1.0.128.0|1.0.128.255|泰国|清莱府|0|TOT 1.0.129.0|1.0.132.191|泰国|曼谷|曼谷|TOT 1.0.132.192|1.0.132.255|泰国|Nakhon-Ratchasima|0|TOT 1.0.133.0|1.0.133.255|泰国|素攀武里府|0|TOT 1.0.134.0|1.0.134.255|泰国|曼谷|曼谷|TOT 1.0.135.0|1.0.135.127|泰国|华富里府|0|TOT 1.0.135.128|1.0.135.255|泰国|素攀武里府|0|TOT 1.0.136.0|1.0.136.255|泰国|龙仔厝府|0|TOT 1.0.137.0|1.0.137.255|泰国|大城府|0|TOT 1.0.138.0|1.0.143.255|泰国|曼谷|曼谷|TOT 1.0.144.0|1.0.159.255|泰国|春蓬府|0|TOT 1.0.160.0|1.0.162.255|泰国|洛坤府|0|TOT 1.0.163.0|1.0.163.255|泰国|春蓬府|0|TOT 1.0.164.0|1.0.164.63|泰国|0|0|TOT 1.0.164.64|1.0.164.127|泰国|普吉府|0|TOT 1.0.164.128|1.0.170.255|泰国|0|0|TOT 1.0.171.0|1.0.175.255|泰国|攀牙府|0|TOT ================================================ FILE: data/sample/segments.tests ================================================ 192.168.2.1|192.168.2.20|0|0|内网IP|办公室A 192.168.2.21|192.168.2.30|0|0|内网IP|办公室A 192.168.2.31|192.168.2.60|0|0|内网IP|办公室B 192.168.2.61|192.168.2.91|0|0|内网IP|办公室B 223.255.236.0|223.255.239.255|中国|上海|上海市|电信 ================================================ FILE: data/sample/segments.tests.mixed ================================================ 192.168.2.1|192.168.2.20|0|0|内网IP|办公室A 192.168.2.21|192.168.2.30|0|0|内网IP|办公室A 192.168.2.31|192.168.2.60|0|0|内网IP|办公室B 192.168.2.61|192.168.2.91|0|0|内网IP|办公室B 223.255.236.0|223.255.239.255|中国|上海|上海市|电信 2c0f:fff1::|2c0f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|毛里求斯|威廉平原区|卡特勒博尔纳|专线用户 2e00::|2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|德国|黑森|美因河畔法兰克福|专线用户 3000::|fbff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户 fe00::|fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户 fe80::|febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户 ================================================ FILE: maker/c/ReadMe.md ================================================ # ip2region xdb c语言生成实现 # 数据生成 # 数据查询 # bench 测试 ================================================ FILE: maker/cpp/README.md ================================================ # :cn: [中文简体] 1. [生成 xdb 文件](../../binding/cpp#4-生成-xdb-文件) 2. [原始数据编辑](../../binding/cpp#5-原始数据编辑) # :globe_with_meridians: [English] 1. [make xdb file](../../binding/cpp##4-generate-xdb-file) 2. [source data editor](../../binding/cpp#5-raw-data-editing) ================================================ FILE: maker/csharp/.gitignore ================================================ *.swp *.*~ project.lock.json .DS_Store *.pyc nupkg/ # Visual Studio Code .vscode # Rider .idea # User-specific files *.suo *.user *.userosscache *.sln.docstates # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ x64/ x86/ build/ bld/ [Bb]in/ [Oo]bj/ [Oo]ut/ msbuild.log msbuild.err msbuild.wrn # Visual Studio 2015 .vs/ ================================================ FILE: maker/csharp/IP2RegionMaker/IP2RegionMaker.csproj ================================================ Exe net6.0 enable enable ================================================ FILE: maker/csharp/IP2RegionMaker/Program.cs ================================================ using IP2RegionMaker.XDB; using System.Diagnostics; string srcFile = "", dstFile = ""; IndexPolicy indexPolicy = IndexPolicy.VectorIndexPolicy; if (args.Length < 2) { PrintHelp(); } string[] aliases = { "--src", "--dst", "--index" }; for (int i = 0; i < args.Length; i++) { var arg = args[i]; var key = aliases.FirstOrDefault(x => arg.StartsWith($"{x}=")); if (string.IsNullOrEmpty(key)) { continue; } var value = arg.Split("=", 2).LastOrDefault()?.Trim(); if (string.IsNullOrEmpty(value)) { continue; } switch (key) { case "--src": srcFile = value; break; case "--dst": dstFile = value; break; case "--index": var flag = Enum.TryParse(value, out indexPolicy); Console.WriteLine("parse policy failed {arg}", arg); break; } } Console.WriteLine(srcFile); if (string.IsNullOrEmpty(srcFile)||string.IsNullOrEmpty(dstFile)) { PrintHelp(); return; } Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); Maker maker = new Maker(IndexPolicy.VectorIndexPolicy, srcFile, dstFile); maker.Init(); maker.Build(); stopwatch.Stop(); Console.WriteLine($"Done, elapsed:{stopwatch.Elapsed.TotalMinutes}m"); void PrintHelp() { Console.WriteLine($"ip2region xdb maker"); Console.WriteLine("dotnet IP2RegionMaker.dll [command options]"); Console.WriteLine("--src string source ip text file path"); Console.WriteLine("--dst string destination binary xdb file path"); } ================================================ FILE: maker/csharp/IP2RegionMaker/Properties/PublishProfiles/FolderProfile.pubxml ================================================  Release Any CPU bin\Release\net6.0\publish\ FileSystem <_TargetId>Folder ================================================ FILE: maker/csharp/IP2RegionMaker/XDB/IndexPolicy.cs ================================================ using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; namespace IP2RegionMaker.XDB { public enum IndexPolicy { VectorIndexPolicy = 1, BTreeIndexPolicy = 2, } } ================================================ FILE: maker/csharp/IP2RegionMaker/XDB/Maker.cs ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Alan Lee // @Date 2022/8/8 // --- Ip2Region v2.0 data structure // // +----------------+--------------------------+---------------+--------------+ // | header space | vector speed up index | data payload | block index | // +----------------+--------------------------+---------------+--------------+ // | 256 bytes | 512 KiB (fixed) | dynamic size | dynamic size | // +----------------+--------------------------+---------------+--------------+ // // 1. padding space : for header info like block index ptr, version, release date eg ... or any other temporary needs. // -- 2bytes: version number, different version means structure update, it fixed to 2 for now // -- 2bytes: index algorithm code. // -- 4bytes: generate unix timestamp (version) // -- 4bytes: index block start ptr // -- 4bytes: index block end ptr // // // 2. data block : region or whatever data info. // 3. segment index block : binary index block. // 4. vector index block : fixed index info for block index search speed up. // space structure table: // -- 0 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- 1 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- 2 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- ... // -- 255 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // // // super block structure: // +-----------------------+----------------------+ // | first index block ptr | last index block ptr | // +-----------------------+----------------------+ // // data entry structure: // +--------------------+-----------------------+ // | 2bytes (for desc) | dynamic length | // +--------------------+-----------------------+ // data length whatever in bytes // // index entry structure // +------------+-----------+---------------+------------+ // | 4bytes | 4bytes | 2bytes | 4 bytes | // +------------+-----------+---------------+------------+ // start ip end ip data length data ptr using System.Text; namespace IP2RegionMaker.XDB { public class Maker { const ushort VersionNo = 2; const int HeaderInfoLength = 256; const int VectorIndexRows = 256; const int VectorIndexCols = 256; const int VectorIndexSize = 8; const int SegmentIndexSize = 14; const int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize; private readonly Stream _srcHandle; private readonly Stream _dstHandle; private readonly IndexPolicy _indexPolicy; private readonly List _segments; private readonly Dictionary _regionPool; private readonly byte[] _vectorIndex; public Maker(IndexPolicy indexPolicy,string srcFile, string dstFile) { _indexPolicy = indexPolicy; _srcHandle = File.Open(@srcFile, FileMode.Open); _dstHandle = File.Open(@dstFile, FileMode.Create); _segments = new List(); _regionPool = new Dictionary(); _vectorIndex = new byte[VectorIndexLength]; } ~Maker() { _srcHandle.Close(); _dstHandle.Close(); } private void InitDbHeader() { _srcHandle.Seek(0, SeekOrigin.Begin); var header = new byte[HeaderInfoLength]; BitConverter.GetBytes(VersionNo).CopyTo(header, 0); BitConverter.GetBytes((ushort)_indexPolicy).CopyTo(header, 2); BitConverter.GetBytes(DateTimeOffset.UtcNow.ToUnixTimeSeconds()).CopyTo(header, 4); BitConverter.GetBytes((uint)0).CopyTo(header, 8); BitConverter.GetBytes((uint)0).CopyTo(header, 12); using var writer = new BinaryWriter(_dstHandle, Encoding.UTF8, true); writer.Write(header); } private void LoadSegments() { Console.WriteLine("try to load the segments ... "); using var reader = new StreamReader(_srcHandle); while (true) { var line = reader.ReadLine(); if (line == null) break; var seg=Util.GetSegment(line); _segments?.Add(seg); } if (_segments!=null) { Util.CheckSegments(_segments); } Console.WriteLine($"all segments loaded, length: {_segments?.Count}"); } public void Init() { InitDbHeader(); LoadSegments(); } public void Build() { _dstHandle.Seek(HeaderInfoLength + VectorIndexLength, SeekOrigin.Begin); using var writer = new BinaryWriter(_dstHandle, Encoding.UTF8, false); Console.WriteLine("try to write the data block ... "); foreach (var seg in _segments) { Console.WriteLine($"try to write region {seg.Region}"); if (_regionPool.TryGetValue(seg.Region, out var value)) { Console.WriteLine($"--[Cached] with ptr={value}"); continue; } var region = Encoding.UTF8.GetBytes(seg.Region); if (region.Length > 0xFFFF) { throw new ArgumentException($"too long region info `{seg.Region}`: should be less than {0xFFFF} bytes"); } var pos = _dstHandle.Seek(0, SeekOrigin.Current); writer.Write(region); _regionPool[seg.Region] = (uint)pos; } Console.WriteLine("try to write the segment index block ... "); var indexBuff = new byte[SegmentIndexSize]; var counter = 0; long startPtr = -1; long endPtr = -1; foreach (var seg in _segments) { var dataPtr = _regionPool[seg.Region]; if (!_regionPool.ContainsKey(seg.Region)) { throw new Exception($"missing ptr cache for region `{seg.Region}`"); } var datalen = Encoding.UTF8.GetBytes(seg.Region).Length; if (datalen < 1) { throw new ArgumentNullException(nameof(seg.Region)); } var segList = seg.Split(); Console.WriteLine($"try to index segment({segList.Count}) {seg} ..."); foreach (var item in segList) { var pos = _dstHandle.Seek(0, SeekOrigin.Current); BitConverter.GetBytes(item.StartIP).CopyTo(indexBuff, 0); BitConverter.GetBytes(item.EndIP).CopyTo(indexBuff, 4); BitConverter.GetBytes((ushort)datalen).CopyTo(indexBuff, 8); BitConverter.GetBytes(dataPtr).CopyTo(indexBuff, 10); writer.Write(indexBuff); Console.WriteLine($"|-segment index: {counter}, ptr: {pos}, segment: {seg}"); SetVectorIndex(item.StartIP, (uint)pos); counter++; if (startPtr == -1) { startPtr = pos; } endPtr = pos; } } Console.WriteLine($"try to write the vector index block ... "); _dstHandle.Seek(HeaderInfoLength, SeekOrigin.Begin); writer.Write(_vectorIndex); Console.WriteLine("try to write the segment index ptr ... "); BitConverter.GetBytes((uint)startPtr).CopyTo(indexBuff, 0); BitConverter.GetBytes((uint)endPtr).CopyTo(indexBuff, 4); _dstHandle.Seek(0, SeekOrigin.Begin); writer.Write(indexBuff[..8]); Console.WriteLine($"write done, dataBlocks: {_regionPool.Count}, indexBlocks: ({_segments.Count}, {counter}), indexPtr: ({startPtr}, {endPtr})"); } private void SetVectorIndex(uint ip, uint ptr) { var il0 = (ip >> 24) & 0xFF; var il1 = (ip >> 16) & 0xFF; var idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; ArraySegment bytes = new(_vectorIndex, (int)idx, _vectorIndex.Length - 1 - (int)idx); var sPtr = BitConverter.ToUInt32(bytes); if (sPtr == 0) { BitConverter.GetBytes(ptr).CopyTo(_vectorIndex, idx); BitConverter.GetBytes(ptr + SegmentIndexSize).CopyTo(_vectorIndex, idx + 4); } else { BitConverter.GetBytes(ptr + SegmentIndexSize).CopyTo(_vectorIndex, idx + 4); } } } } ================================================ FILE: maker/csharp/IP2RegionMaker/XDB/Segment.cs ================================================ using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; namespace IP2RegionMaker.XDB { public class Segment { public uint StartIP { get; set; } public uint EndIP { get; set; } public string Region { get; set; } public List Split() { var tList = new List(); var sByte = (StartIP >> 24) & 0xFF; var eByte = (EndIP >> 24) & 0xFF; var nSip = StartIP; for (var i = sByte; i <= eByte; i++) { var sip = (i << 24) | (nSip & 0xFFFFFF); var eip = (i << 24) | 0xFFFFFF; if (eip < EndIP) { nSip = (i + 1) << 24; } else { eip = EndIP; } tList.Add(new Segment { StartIP = sip, EndIP = eip, }); } var segList = new List(); foreach (var seg in tList) { var temp = seg.StartIP & 0xFF000000; nSip = seg.StartIP; sByte = (seg.StartIP >> 16) & 0xFF; eByte = (seg.EndIP >> 16) & 0xFF; for (var i = sByte; i <= eByte; i++) { var sip = temp | (i << 16) | (nSip & 0xFFFF); var eip = temp | (i << 16) | 0xFFFF; if (eip < seg.EndIP) { nSip = 0; } else { eip = seg.EndIP; } segList.Add(new Segment { StartIP = sip, EndIP = eip, Region = Region, }); } } return segList; } public override string ToString() { return $"{Util.UInt32ToIpAddress(StartIP)}|{Util.UInt32ToIpAddress(EndIP)}|{Region}"; } } } ================================================ FILE: maker/csharp/IP2RegionMaker/XDB/Util.cs ================================================ using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Text; using System.Threading.Tasks; namespace IP2RegionMaker.XDB { public static class Util { public static uint IpAddressToUInt32(string ipAddress) { var address = IPAddress.Parse(ipAddress); byte[] bytes = address.GetAddressBytes(); Array.Reverse(bytes); return BitConverter.ToUInt32(bytes, 0); } public static string UInt32ToIpAddress(uint ipAddress) { byte[] bytes = BitConverter.GetBytes(ipAddress); Array.Reverse(bytes); return new IPAddress(bytes).ToString(); } public static Segment GetSegment(string line) { var ps = line.Split("|", 3); if (ps.Length != 3) { throw new ArgumentException($"invalid ip segment line {line}"); } var sip = Util.IpAddressToUInt32(ps[0]); var eip = Util.IpAddressToUInt32(ps[1]); if (sip > eip) { throw new ArgumentException($"start ip {ps[0]} should not be greater than end ip {ps[1]}"); } if (string.IsNullOrEmpty(ps[2])) { throw new ArgumentException($"empty region info in segment line {line}"); } return new Segment { StartIP = sip, EndIP = eip, Region = ps[2], }; } public static void CheckSegments(List segments) { Segment? last = null; foreach (var seg in segments) { if (seg.StartIP > seg.EndIP) { throw new ArgumentException($"segment `{seg}`: start ip should not be greater than end ip"); } if (last != null && last.EndIP + 1 != seg.StartIP) { throw new ArgumentException($"discontinuous data segment: last.eip+1({seg.StartIP}) != seg.sip({seg.EndIP},#{seg})"); } last = seg; } } } } ================================================ FILE: maker/csharp/IP2RegionMaker.Test/IP2RegionMaker.Test.csproj ================================================ net6.0 enable enable false ================================================ FILE: maker/csharp/IP2RegionMaker.Test/Usings.cs ================================================ global using NUnit.Framework; ================================================ FILE: maker/csharp/IP2RegionMaker.Test/UtilTest.cs ================================================ using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; namespace IP2RegionMaker.Test { [TestFixture] internal class UtilTest { [TestCase("114.114.114.114")] public void TestIpAddressToUInt32(string value) { Assert.DoesNotThrow(() => XDB.Util.IpAddressToUInt32(value)); } [TestCase(1920103026)] public void TestUInt32ToIpAddress(int value) { Assert.DoesNotThrow(() => XDB.Util.UInt32ToIpAddress((uint)value)); } [TestCase("28.201.224.0|29.34.191.255|美国|0|0|0|0")] public void TestSplitSegment(string value) { Assert.DoesNotThrow(() => { var seg=XDB.Util.GetSegment(value); var segList= seg.Split(); XDB.Util.CheckSegments(segList); foreach (var item in segList) { Console.WriteLine(item); } }); } } } ================================================ FILE: maker/csharp/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb csharp generation implementation # Compilation and Installation Compilation environment: [dotnet6.0](https://dotnet.microsoft.com/zh-cn/download/dotnet/6.0) ```bash # cd to maker/csharp/IP2RegionMaker directory dotnet publish -o ./bin ``` Then you will get a packaged file named IP2RegionMaker.dll in the bin directory of the current folder. # `xdb` Data Generation Generate the xdb binary file via `dotnet IP2RegionMaker.dll`: ```bash ➜ csharp git:(master) ✗ dotnet IP2RegionMaker.dll ip2region xdb maker dotnet IP2RegionMaker.dll [command options] --src string source ip text file path --dst string destination binary xdb file path ``` For example, using the default data/ipv4_source.txt source data to generate an ip2region_v4.xdb binary file in the current directory: ```bash ➜ csharp git:(master) ✗ dotnet ./IP2RegionMaker/bin/IP2RegionMaker.dll --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb # You will see a lot of output; eventually, you will see the following output indicating the run was successful ... ... ... write done, dataBlocks: 13804, indexBlocks: (683591, 720221), indexPtr: (982904, 11065984) Done, elapsed:2.1966620833333335m ``` # Data Query / bench Test All [bindings](../../binding/) come with query and bench test programs as well as usage documentation. You can use the searcher of your familiar language for query testing or bench testing to confirm the correctness and integrity of the data. ================================================ FILE: maker/csharp/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb csharp 生成实现 ## 编译安装 编译环境：[dotnet6.0](https://dotnet.microsoft.com/zh-cn/download/dotnet/6.0) ```bash # cd 到 maker/csharp/IP2RegionMaker目录 dotnet publish -o ./bin ``` 然后会在当前目录的 bin 目录下得到一个 IP2RegionMaker.dll 的打包文件。 # 数据生成 通过 `dotnet IP2RegionMaker.dll` 来生成 xdb 二进制文件： ```bash ➜ csharp git:(master) ✗ dotnet IP2RegionMaker.dll ip2region xdb maker dotnet IP2RegionMaker.dll [command options] --src string source ip text file path --dst string destination binary xdb file path ``` 例如，通过默认的 data/ipv4_source.txt 原数据，在当前目录生成一个 ip2region_v4.xdb 二进制文件： ```bash ➜ csharp git:(master) ✗ dotnet ./IP2RegionMaker/bin/IP2RegionMaker.dll --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb # 会看到一堆输出，最终会看到如下输出表示运行成功 ... ... ... write done, dataBlocks: 13804, indexBlocks: (683591, 720221), indexPtr: (982904, 11065984) Done, elapsed:2.1966620833333335m ``` # 数据 查询/bench 测试 已经完成开发的 [binding](../../binding/) 都有查询和 bench 测试程序以及使用文档，你可以使用你熟悉的语言的 searcher 进行查询测试或者bench测试，来确认数据的正确性和完整性。 ================================================ FILE: maker/golang/Dockerfile ================================================ # ============================================ # ip2region xdb maker - Golang version # Multi-stage build, resulting in an image of approximately 10 MB. # ============================================ FROM golang:1.22-alpine AS builder WORKDIR /build COPY . ./ # Compile a static binary (disable CGO for fully static linking) RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o xdb_maker FROM alpine:3.19 RUN apk --no-cache add ca-certificates tzdata && \ cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ echo "Asia/Shanghai" > /etc/timezone # Create a non-root user. RUN adduser -D -g '' appuser WORKDIR /app # Copy binaries from the build stage. COPY --from=builder /build/xdb_maker /app/ RUN mkdir -p /app/data && chown -R appuser:appuser /app USER appuser ENTRYPOINT ["/app/xdb_maker"] CMD ["--help"] ================================================ FILE: maker/golang/Makefile ================================================ # ip2region golang maker makefile all: build .PHONY: all build: go build -o xdb_maker test: go test -v ./... clean: find ./ -name xdb_maker | xargs rm -f ================================================ FILE: maker/golang/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb golang generation implementation # Program Compilation Compile to get the xdb_maker executable through the following method: ``` # cd to the golang maker root directory make ``` After successful compilation, a xdb_maker executable file will be generated in the current directory. # `xdb` Data Generation Generate the ip2region.xdb binary file via the `xdb_maker gen` command: ``` ./xdb_maker gen [command options] options: --src string source ip text file path --dst string destination binary xdb file path --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7 --log-level string set the log level, options: debug/info/warn/error ``` For example, generate the xdb file to the current directory using the default source data under the repository's data/ directory: ```bash # ipv4 ./xdb_maker gen --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb --version=ipv4 # ipv6 ./xdb_maker gen --src=../../data/ipv6_source.txt --dst=./ip2region_v6.xdb --version=ipv6 ``` For custom data fields during the generation process, please refer to [xdb-文件生成#自定义数据字段](https://ip2region.net/doc/data/xdb_make#field-list) # `xdb` Data Search Test the input IP via the `xdb_maker search` command: ``` ➜ golang git:(v2.0_xdb) ✗ ./xdb_maker search ./xdb_maker search [command options] options: --db string ip2region binary xdb file path ``` For example, run a search test using the built-in xdb file: ```bash # ipv4 ./xdb_maker search --db=../../data/ip2region_v4.xdb ip2region xdb search test program, source xdb: ../../data/ip2region_v4.xdb (IPv4) commands: loadIndex : load the vector index for search speedup. clearIndex: clear the vector index. quit : exit the test program ip2region>> 58.251.30.115 {region:中国|广东省|深圳市|联通|CN, iocount:2, took:27.893µs} ip2region>> 1.2.3.4 {region:Australia|Queensland|Brisbane|0|AU, iocount:5, took:58.746µs} # ipv6 ./xdb_maker search --db=../../data/ip2region_v6.xdb ip2region xdb search test program, source xdb: ../../data/ip2region_v6.xdb (IPv6) commands: loadIndex : load the vector index for search speedup. clearIndex: clear the vector index. quit : exit the test program ip2region>> 2604:bc80:8001:11a4:ffff:ffff:ffff:ffff {region:United States|Florida|Miami|velia.net Internetdienste GmbH|US, iocount:15, took:140.942µs} ip2region>> 240e:3b7:3273:51d0:13f9:bf0:3db1:aa3f {region:中国|广东省|深圳市|电信|CN, iocount:9, took:67058µs} ``` # `xdb` Data Editing Edit the raw IP data via the `xdb_maker edit` command: ``` ./xdb_maker edit [command options] options: --src string source ip text file path --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused ``` For example, opening `./data/ipv4_source.txt` with the editor will show the following operation panel: ```bash ./xdb_maker edit --src=../../data/ipv4_source.txt --version=ipv4 init the editor from source @ `../../data/ipv4_source.txt` ... all segments loaded, length: 683591, elapsed: 479.73743ms command list: put [segment] : put the specifield $segment put_file [file] : put all the segments from the specified $file list [offset] [size] : list the first $size segments start from $offset save : save all the changes to the destination source file quit : exit the program help : print this help menu editor>> ``` Modify the location information of a specified IP segment using the `put` command, for example: ```bash editor>> put 36.132.128.0|36.132.147.255|中国|黑龙江省|哈尔滨市|移动|CN Put(36.132.128.0|36.132.147.255|中国|黑龙江省|哈尔滨市|移动|CN): Ok, with 1 deletes and 2 additions *editor>> ``` Batch load modifications from a file using the `put_file` command. The IP segments in the file do not need to be as strict as the data in `./data/ipvx_source.txt`; they do not need to be continuous, and it does not matter if different IP segments overlap. The editor will automatically analyze and process them, for example: ```bash *editor>> put_file ../../data/sample/ip.test.txt PutFile(../../data/sample/ip.test.txt): Ok, with 25 deletes and 25 additions *editor>> ``` Save modifications using the `save` command. After saving successfully, you can re-generate the xdb from the modified raw IP file using the commands mentioned above: ```bash *editor>> save all segments saved to ../../data/ipv4_source.txt editor>> ``` # bench Test If you have generated the `xdb` file yourself, please ensure you run the following `xdb_maker bench` command to verify the correctness of the generated `xdb` file: ``` ./xdb_maker bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused --log-level string set the log level, options: debug/info/warn/error --ignore-error bool keep going if bench failed ``` For example: use the source files under data to bench test the xdb files in data: ```bash # ipv4 ./xdb_maker bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt --version=ipv4 #ipv6 ./xdb_maker bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt --version=ipv6 ``` *Please note that the `src` file used for the bench test must be the same as the source file used to generate the xdb*. If an error occurs during execution, it will stop immediately. You can also execute with the `--ignore-error=true` parameter to ignore errors and view the failed statistics at the end. # Docker ```bash # Build the image (run in the maker/golang directory). cd ip2region/maker/golang docker build -t ip2region-maker . # Generate IPv4 xdb docker run --rm -v $(pwd)/../../data:/app/data ip2region-maker \ gen --src=/app/data/ipv4_source.txt \ --dst=/app/data/ip2region_v4.xdb \ --version=ipv4 # Generate IPv6 xdb docker run --rm -v $(pwd)/../../data:/app/data ip2region-maker \ gen --src=/app/data/ipv6_source.txt \ --dst=/app/data/ip2region_v6.xdb \ --version=ipv6 # Interactive IPv4 Query docker run -it --rm -v $(pwd)/../../data:/app/data ip2region-maker \ search --db=/app/data/ip2region_v4.xdb # Bench test IPv4 docker run --rm -v $(pwd)/../../data:/app/data ip2region-maker \ bench --db=/app/data/ip2region_v4.xdb \ --src=/app/data/ipv4_source.txt \ --version=ipv4 ``` ================================================ FILE: maker/golang/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb golang 生成实现 # 程序编译 通过如下方式编译得到 xdb_maker 可执行程序: ``` # 切换到 golang maker 根目录 make ``` 编译成功后会在当前目录生成一个 xdb_maker 的可执行文件 # `xdb` 数据生成 通过 `xdb_maker gen` 命令生成 ip2region.xdb 二进制文件: ``` ./xdb_maker gen [command options] options: --src string source ip text file path --dst string destination binary xdb file path --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7 --log-level string set the log level, options: debug/info/warn/error ``` 例如，使用默认的仓库 data/ 下默认的原始数据生成生成 xdb 文件到当前目录： ```bash # ipv4 ./xdb_maker gen --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb --version=ipv4 # ipv6 ./xdb_maker gen --src=../../data/ipv6_source.txt --dst=./ip2region_v6.xdb --version=ipv6 ``` 生成过程中数据字段自定义请参考 [xdb-文件生成#自定义数据字段](https://ip2region.net/doc/data/xdb_make#field-list) # `xdb` 数据查询 通过 `xdb_maker search` 命令来测试查询输入的 ip： ``` ➜ golang git:(v2.0_xdb) ✗ ./xdb_maker search ./xdb_maker search [command options] options: --db string ip2region binary xdb file path ``` 例如，使用自带的 xdb 文件来运行查询测试： ```bash # ipv4 ./xdb_maker search --db=../../data/ip2region_v4.xdb ip2region xdb search test program, source xdb: ../../data/ip2region_v4.xdb (IPv4) commands: loadIndex : load the vector index for search speedup. clearIndex: clear the vector index. quit : exit the test program ip2region>> 58.251.30.115 {region:中国|广东省|深圳市|联通|CN, iocount:2, took:27.893µs} ip2region>> 1.2.3.4 {region:Australia|Queensland|Brisbane|0|AU, iocount:5, took:58.746µs} # ipv6 ./xdb_maker search --db=../../data/ip2region_v6.xdb ip2region xdb search test program, source xdb: ../../data/ip2region_v6.xdb (IPv6) commands: loadIndex : load the vector index for search speedup. clearIndex: clear the vector index. quit : exit the test program ip2region>> 2604:bc80:8001:11a4:ffff:ffff:ffff:ffff {region:United States|Florida|Miami|velia.net Internetdienste GmbH|US, iocount:15, took:140.942µs} ip2region>> 240e:3b7:3273:51d0:13f9:bf0:3db1:aa3f {region:中国|广东省|深圳市|电信|CN, iocount:9, took:67.058µs} ``` # `xdb` 数据编辑 通过 `xdb_maker edit` 命令来编辑原始的　IP 数据： ``` ./xdb_maker edit [command options] options: --src string source ip text file path --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused ``` 例如，使用编辑器打开 `./data/ipv4_source.txt` 会看到如下的操作面板： ```bash ./xdb_maker edit --src=../../data/ipv4_source.txt --version=ipv4 init the editor from source @ `../../data/ipv4_source.txt` ... all segments loaded, length: 683591, elapsed: 479.73743ms command list: put [segment] : put the specifield $segment put_file [file] : put all the segments from the specified $file list [offset] [size] : list the first $size segments start from $offset save : save all the changes to the destination source file quit : exit the program help : print this help menu editor>> ``` 通过 `put` 命令修改指定 IP 段的定位信息，例如： ```bash editor>> put 36.132.128.0|36.132.147.255|中国|黑龙江省|哈尔滨市|移动|CN Put(36.132.128.0|36.132.147.255|中国|黑龙江省|哈尔滨市|移动): Ok, with 1 deletes and 2 additions *editor>> ``` 通过 `put_file` 命令从文件中批量载入修改，文件中的 IP 段不需要像　./data/ipvx_source.txt 中的数据那么严格，不需要前后连续，不同 IP　段有重叠也没关系，编辑器会自动分析处理，例如： ```bash *editor>> put_file ../../data/sample/ip.test.txt PutFile(../../data/sample/ip.test.txt): Ok, with 25 deletes and 25 additions *editor>> ``` 通过 `save` 命令保存修改，保存成功后，再通过上面的命令从修改后的原始 IP 文件重新生成 xdb 即可： ```bash *editor>> save all segments saved to ../../data/ipv4_source.txt editor>> ``` # bench 测试 如果你自主生成了 `xdb` 文件，请确保运行如下的 `xdb_maker bench` 命令来确保生成的的 `xdb` 文件的正确性： ``` ./xdb_maker bench [command options] options: --db string ip2region binary xdb file path --src string source ip text file path --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused --log-level string set the log level, options: debug/info/warn/error --ignore-error bool keep going if bench failed ``` 例如：使用 data 下的源文件来 bench 测试 data 的 xdb 文件： ```bash # ipv4 ./xdb_maker bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt --version=ipv4 #ipv6 ./xdb_maker bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt --version=ipv6 ``` *请注意 bench 测试使用的 `src` 文件需要是对应的生成 xdb 的源文件相同*。 如果运行过程中有错误会立马停止运行，也可以执行 --ignore-error=true 参数来忽略错误，在最后看 failed 的统计结果。 # Docker ```bash # 构建镜像（在 maker/golang 目录下执行） cd ip2region/maker/golang docker build -t ip2region-maker . # 生成 IPv4 xdb docker run --rm -v $(pwd)/../../data:/app/data ip2region-maker \ gen --src=/app/data/ipv4_source.txt \ --dst=/app/data/ip2region_v4.xdb \ --version=ipv4 # 生成 IPv6 xdb docker run --rm -v $(pwd)/../../data:/app/data ip2region-maker \ gen --src=/app/data/ipv6_source.txt \ --dst=/app/data/ip2region_v6.xdb \ --version=ipv6 # 交互式查询 IPv4 docker run -it --rm -v $(pwd)/../../data:/app/data ip2region-maker \ search --db=/app/data/ip2region_v4.xdb # Bench 测试 IPv4 docker run --rm -v $(pwd)/../../data:/app/data ip2region-maker \ bench --db=/app/data/ip2region_v4.xdb \ --src=/app/data/ipv4_source.txt \ --version=ipv4 ``` ================================================ FILE: maker/golang/cmd/bench.go ================================================ package cmd import ( "fmt" "log/slog" "os" "time" "github.com/lionsoul2014/ip2region/maker/golang/xdb" ) func Bench() { var err error var dbFile, srcFile, ipVersion, logLevel = "", "", "", "" var ignoreError = false var fErr = iterateFlags(func(key string, val string) error { switch key { case "db": dbFile = val case "src": srcFile = val case "version": ipVersion = val case "log-level": logLevel = val case "ignore-error": if val == "true" || val == "1" { ignoreError = true } else if val == "false" || val == "0" { ignoreError = false } else { return fmt.Errorf("invalid value for ignore-error option, could be false/0 or true/1") } default: return fmt.Errorf("undefined option '%s=%s'", key, val) } return nil }) if fErr != nil { fmt.Printf("failed to parse flags: %s", fErr) return } if dbFile == "" || srcFile == "" { fmt.Printf("%s bench [command options]\n", os.Args[0]) fmt.Printf("options:\n") fmt.Printf(" --db string ip2region binary xdb file path\n") fmt.Printf(" --src string source ip text file path\n") fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") fmt.Printf(" --ignore-error bool keep going if bench failed\n") return } // check and define the IP version var version *xdb.Version = nil if len(ipVersion) < 2 { slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") return } else if v, err := xdb.VersionFromName(ipVersion); err != nil { slog.Error("failed to parse version name", "error", err) return } else { version = v } // check and apply the log level err = applyLogLevel(logLevel) if err != nil { slog.Error("failed to apply log level", "error", err) return } searcher, err := xdb.NewSearcher(version, dbFile) if err != nil { fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err) return } defer func() { searcher.Close() }() handle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) if err != nil { fmt.Printf("failed to open source text file: %s\n", err) return } defer handle.Close() var count, errCount, tStart = 0, 0, time.Now() slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile) _, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error { var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) slog.Debug("try to bench", "segment", l) // mip := xdb.IPMiddle(seg.StartIP, seg.EndIP) // for _, ip := range [][]byte{seg.StartIP, xdb.IPMiddle(seg.EndIP, mip), mip, xdb.IPMiddle(mip, seg.EndIP), seg.EndIP} { for _, ip := range [][]byte{seg.StartIP, seg.EndIP} { slog.Debug("|-try to bench", "ip", xdb.IP2String(ip)) r, _, err := searcher.Search(ip) if err != nil { return fmt.Errorf("failed to search ip '%s': %s", xdb.IP2Long(ip), err) } // check the region info count++ if r != seg.Region { errCount++ slog.Error(" --[Failed] region not match", "src", r, "dst", seg.Region) if !ignoreError { return fmt.Errorf("") } } else { slog.Debug(" --[Ok]") } } return nil }) if iErr != nil { fmt.Printf("%s", err) return } slog.Info("Bench finished", "count", count, "failed", errCount, "elapsed", time.Since(tStart)) } ================================================ FILE: maker/golang/cmd/edit.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package cmd import ( "bufio" "fmt" "log/slog" "os" "regexp" "strings" "time" "github.com/lionsoul2014/ip2region/maker/golang/xdb" ) // source ip data editor func Edit() { var err error var srcFile, ipVersion = "", "" var fErr = iterateFlags(func(key string, val string) error { switch key { case "src": srcFile = val case "version": ipVersion = val default: return fmt.Errorf("undefined option '%s=%s'", key, val) } return nil }) if fErr != nil { fmt.Printf("failed to parse flags: %s", fErr) return } if srcFile == "" { fmt.Printf("%s edit [command options]\n", os.Args[0]) fmt.Printf("options:\n") fmt.Printf(" --src string source ip text file path\n") fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") return } // check and define the IP version var version *xdb.Version = nil if len(ipVersion) < 2 { slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") return } else if v, err := xdb.VersionFromName(ipVersion); err != nil { slog.Error("failed to parse version name", "error", err) return } else { version = v } rExp, err := regexp.Compile(`\s+`) if err != nil { fmt.Printf("failed to compile regexp: %s\n", err) return } fmt.Printf("init the editor from source @ `%s` ... \n", srcFile) var tStart = time.Now() editor, err := xdb.NewEditor(version, srcFile) if err != nil { fmt.Printf("failed to init editor: %s", err) return } fmt.Printf("all segments loaded, length: %d, elapsed: %s\n", editor.SegLen(), time.Since(tStart)) var help = func() { fmt.Printf("command list: \n") fmt.Printf(" put [segment] : put the specifield $segment\n") fmt.Printf(" put_file [file] : put all the segments from the specified $file\n") fmt.Printf(" list [offset] [size] : list the first $size segments start from $offset\n") fmt.Printf(" save : save all the changes to the destination source file\n") fmt.Printf(" quit : exit the program\n") fmt.Printf(" help : print this help menu\n") } help() var sTip = "" var reader = bufio.NewReader(os.Stdin) for { if editor.NeedSave() { sTip = "*" } else { sTip = "" } fmt.Printf("%seditor>> ", sTip) line, err := reader.ReadString('\n') if err != nil { fmt.Printf("failed to read line from cli: %s\n", err) break } cmd := strings.TrimSpace(line) if cmd == "help" { help() } else if cmd == "quit" { if editor.NeedSave() { fmt.Printf("there are changes that need to save, type 'quit!' to force quit\n") } else { break } } else if cmd == "quit!" { // quit directly break } else if cmd == "save" { err = editor.Save() if err != nil { fmt.Printf("failed to save the changes: %s\n", err) continue } fmt.Printf("all segments saved to %s\n", srcFile) } else if strings.HasPrefix(cmd, "list") { var sErr error off, size, l := 0, 10, len("list") str := strings.TrimSpace(cmd) if len(str) > l { sets := rExp.Split(cmd, 3) switch len(sets) { case 2: _, sErr = fmt.Sscanf(cmd, "%s %d", &str, &off) case 3: _, sErr = fmt.Sscanf(cmd, "%s %d %d", &str, &off, &size) } } if sErr != nil { fmt.Printf("failed to parse the offset and size: %s\n", sErr) continue } fmt.Printf("+-slice(%d,%d): \n", off, size) for _, s := range editor.Slice(off, size) { fmt.Printf("%s\n", s) } } else if strings.HasPrefix(cmd, "put ") { seg := strings.TrimSpace(cmd[len("put "):]) o, n, err := editor.Put(seg) if err != nil { fmt.Printf("failed to Put(%s): %s\n", seg, err) continue } fmt.Printf("Put(%s): Ok, with %d deletes and %d additions\n", seg, o, n) } else if strings.HasPrefix(cmd, "put_file ") { file := strings.TrimSpace(cmd[len("put_file "):]) o, n, err := editor.PutFile(file) if err != nil { fmt.Printf("failed to PutFile(%s): %s\n", file, err) continue } fmt.Printf("PutFile(%s): Ok, with %d deletes and %d additions\n", file, o, n) } else if len(cmd) > 0 { help() } } } ================================================ FILE: maker/golang/cmd/generate.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package cmd import ( "fmt" "log/slog" "os" "time" "github.com/lionsoul2014/ip2region/maker/golang/xdb" ) // script to do the xdb generate func Generate() { var err error var srcFile, dstFile = "", "" var ipVersion, fieldList, logLevel = "", "", "info" var indexPolicy = xdb.VectorIndexPolicy var fErr = iterateFlags(func(key string, val string) error { switch key { case "src": srcFile = val case "dst": dstFile = val case "version": ipVersion = val case "log-level": logLevel = val case "field-list": fieldList = val case "index": indexPolicy, err = xdb.IndexPolicyFromString(val) if err != nil { return fmt.Errorf("parse policy: %w", err) } default: return fmt.Errorf("undefine option `%s=%s`", key, val) } return nil }) if fErr != nil { fmt.Printf("failed to parse flags: %s", fErr) return } if srcFile == "" || dstFile == "" { fmt.Printf("%s gen [command options]\n", os.Args[0]) fmt.Printf("options:\n") fmt.Printf(" --src string source ip text file path\n") fmt.Printf(" --dst string destination binary xdb file path\n") fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") return } // check and define the IP version var version *xdb.Version = nil if len(ipVersion) < 2 { slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") return } else if v, err := xdb.VersionFromName(ipVersion); err != nil { slog.Error("failed to parse version name", "error", err) return } else { version = v } // check and apply the log level err = applyLogLevel(logLevel) if err != nil { slog.Error("failed to apply log level", "error", err) return } fields, err := getFilterFields(fieldList) if err != nil { slog.Error("failed to get filter fields", "error", err) return } // make the binary file tStart := time.Now() maker, err := xdb.NewMaker(version, indexPolicy, srcFile, dstFile, fields) if err != nil { fmt.Printf("failed to create %s\n", err) return } slog.Info("Generating xdb with", "src", srcFile, "dst", dstFile, "logLevel", logLevel) err = maker.Init() if err != nil { fmt.Printf("failed Init: %s\n", err) return } err = maker.Start() if err != nil { fmt.Printf("failed Start: %s\n", err) return } err = maker.End() if err != nil { fmt.Printf("failed End: %s\n", err) } slog.Info("make done", "elapsed", time.Since(tStart)) } ================================================ FILE: maker/golang/cmd/process.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package cmd import ( "fmt" "log/slog" "os" "strconv" "time" "github.com/lionsoul2014/ip2region/maker/golang/xdb" ) // source data process, sort, de-duplicate, merge func Process() { var err error var srcFile, dstFile = "", "" var fieldList, logLevel = "", "" var clearBasedIndex = -1 var clearValueEqual, clearValueExcept = "", "" var fErr = iterateFlags(func(key string, val string) error { switch key { case "src": srcFile = val case "dst": dstFile = val case "field-list": fieldList = val case "clear-based-index": num, err := strconv.Atoi(val) if err != nil { return fmt.Errorf("invalid clear-based-index '%s=%s', integer expected", key, val) } clearBasedIndex = num case "clear-value-equal": clearValueEqual = val case "clear-value-except": clearValueExcept = val case "log-level": logLevel = val default: return fmt.Errorf("undefined option '%s=%s'", key, val) } return nil }) if fErr != nil { fmt.Printf("failed to parse flags: %s", fErr) return } if srcFile == "" || dstFile == "" { fmt.Printf("%s process [command options]\n", os.Args[0]) fmt.Printf("options:\n") fmt.Printf(" --src string source ip text file path\n") fmt.Printf(" --dst string target ip text file path\n") fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") fmt.Printf(" --clear-based-index integer clear based index eg: 3\n") fmt.Printf(" --clear-value-equal string clear value equal to the specified one\n") fmt.Printf(" --clear-value-except string clear value except the specified one\n") fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") return } if clearBasedIndex > -1 { if len(clearValueEqual) > 0 && len(clearValueExcept) > 0 { fmt.Print("Only one can be specified besides clear-value-equal and clear-value-except") return } if len(clearValueEqual) == 0 && len(clearValueExcept) == 0 { fmt.Print("At least one must be specified for clear-value-equal and clear-value-except") return } } // check and apply the log level err = applyLogLevel(logLevel) if err != nil { slog.Error("failed to apply log level", "error", err) return } fields, err := getFilterFields(fieldList) if err != nil { slog.Error("failed to get filter fields", "error", err) return } // make the binary file tStart := time.Now() processor, err := xdb.NewProcessor(srcFile, dstFile, fields, clearBasedIndex, clearValueEqual, clearValueExcept) if err != nil { fmt.Printf("failed to create %s\n", err) return } err = processor.Init() if err != nil { fmt.Printf("failed Init: %s\n", err) return } slog.Info("Processing", "src", srcFile, "dst", dstFile, "fields", fields, "clearBasedIndex", clearBasedIndex, "clearValueEqual", clearValueEqual, "clearValueExcept", clearValueExcept, "logLevel", logLevel) err = processor.Start() if err != nil { fmt.Printf("failed Start: %s\n", err) return } err = processor.End() if err != nil { fmt.Printf("failed End: %s\n", err) } slog.Info("processor done", "elapsed", time.Since(tStart)) } ================================================ FILE: maker/golang/cmd/search.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package cmd import ( "bufio" "encoding/binary" "fmt" "log/slog" "os" "strings" "time" "github.com/lionsoul2014/ip2region/maker/golang/xdb" ) // xdb searcher test func Search() { var err error var dbFile = "" var fErr = iterateFlags(func(key string, val string) error { if key == "db" { dbFile = val } else { return fmt.Errorf("undefined option '%s=%s'", key, val) } return nil }) if fErr != nil { fmt.Printf("failed to parse flags: %s", fErr) return } if dbFile == "" { fmt.Printf("%s search [command options]\n", os.Args[0]) fmt.Printf("options:\n") fmt.Printf(" --db string ip2region binary xdb file path\n") return } // detect the version from the xdb header header, err := xdb.LoadXdbHeaderFromFile(dbFile) if err != nil { slog.Error("failed to load xdb header", "error", err) return } var version *xdb.Version = nil versionNo := binary.LittleEndian.Uint16(header[0:]) if versionNo == 2 { // old xdb file version = xdb.IPv4 } else if versionNo == 3 { ipNo := int(binary.LittleEndian.Uint16(header[16:])) if ipNo == xdb.IPv4.Id { version = xdb.IPv4 } else if ipNo == xdb.IPv6.Id { version = xdb.IPv6 } else { slog.Error("invalid ip version", "id", ipNo) return } } else { slog.Error("invalid xdb version", "versionNo", versionNo, "xdbFile", dbFile) return } searcher, err := xdb.NewSearcher(version, dbFile) if err != nil { fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err.Error()) return } defer func() { searcher.Close() fmt.Printf("test program exited, thanks for trying\n") }() fmt.Printf(`ip2region xdb search test program, source xdb: %s (%s) commands: loadIndex : load the vector index for search speedup. clearIndex: clear the vector index. quit : exit the test program `, dbFile, version.Name) reader := bufio.NewReader(os.Stdin) for { fmt.Print("ip2region>> ") str, err := reader.ReadString('\n') if err != nil { slog.Error("failed to read string", "error", err) return } line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) if len(line) == 0 { continue } // command interception and execution if line == "loadIndex" { err = searcher.LoadVectorIndex() if err != nil { slog.Error("failed to load vector index", "error", err) return } fmt.Printf("vector index cached\n") continue } else if line == "clearIndex" { searcher.ClearVectorIndex() fmt.Printf("vector index cleared\n") continue } else if line == "quit" { break } ip, err := xdb.ParseIP(line) if err != nil { fmt.Printf("invalid ip address `%s`\n", line) continue } tStart := time.Now() region, ioCount, err := searcher.Search(ip) if err != nil { fmt.Printf("\x1b[0;31m{err:%s, iocount:%d}\x1b[0m\n", err.Error(), ioCount) } else { fmt.Printf("\x1b[0;32m{region:%s, iocount:%d, took:%s}\x1b[0m\n", region, ioCount, time.Since(tStart)) } } } ================================================ FILE: maker/golang/cmd/util.go ================================================ package cmd import ( "fmt" "log/slog" "os" "regexp" "sort" "strconv" "strings" ) func PrintHelp() { fmt.Printf("ip2region xdb maker\n") fmt.Printf("%s [command] [command options]\n", os.Args[0]) fmt.Printf("Command: \n") fmt.Printf(" gen generate the binary xdb file\n") fmt.Printf(" search binary xdb search test\n") fmt.Printf(" bench binary xdb bench test\n") fmt.Printf(" edit edit the source ip data\n") fmt.Printf(" process process the source ip data\n") } // Iterate the cli flags func iterateFlags(cb func(key string, val string) error) error { for i := 2; i < len(os.Args); i++ { r := os.Args[i] if len(r) < 5 { continue } if strings.Index(r, "--") != 0 { continue } var sIdx = strings.Index(r, "=") if sIdx < 0 { return fmt.Errorf("missing = for args pair '%s'", r) } if err := cb(r[2:sIdx], r[sIdx+1:]); err != nil { return err } } return nil } func applyLogLevel(logLevel string) error { // check and apply the log level var levelLog = slog.LevelInfo switch strings.ToLower(logLevel) { case "debug": levelLog = slog.LevelDebug case "info": levelLog = slog.LevelInfo case "warn": levelLog = slog.LevelWarn case "error": levelLog = slog.LevelError case "": // ignore the empty value // and default it to LevelInfo default: return fmt.Errorf("invalid log level %s", logLevel) } slog.SetLogLoggerLevel(levelLog) return nil } var pattern = regexp.MustCompile(`^(\d+(-\d+)?)$`) func getFilterFields(fieldList string) ([]int, error) { if len(fieldList) == 0 { return []int{}, nil } var fields []int var mapping = make(map[string]string) fList := strings.Split(fieldList, ",") for _, f := range fList { f = strings.TrimSpace(f) if len(f) == 0 { return nil, fmt.Errorf("empty field index value `%s`", f) } ms := pattern.FindString(f) if len(ms) == 0 { return nil, fmt.Errorf("field `%s` is not a number or number range", f) } // if strings.Index(ms, "-") == -1 { if !strings.Contains(ms, "-") { if _, ok := mapping[ms]; ok { return nil, fmt.Errorf("duplicate option `%s`", f) } idx, err := strconv.Atoi(ms) if err != nil { return nil, fmt.Errorf("field index `%s` not an integer", f) } mapping[ms] = ms fields = append(fields, idx) continue } ra := strings.Split(ms, "-") if len(ra) != 2 { return nil, fmt.Errorf("invalid field index range `%s`", ms) } start, err := strconv.Atoi(ra[0]) if err != nil { return nil, fmt.Errorf("range start `%s` not an integer", ra[0]) } end, err := strconv.Atoi(ra[1]) if err != nil { return nil, fmt.Errorf("range end `%s` not an integer", ra[1]) } if start > end { return nil, fmt.Errorf("index range start(%d) should <= end(%d)", start, end) } for i := start; i <= end; i++ { s := strconv.Itoa(i) if _, ok := mapping[s]; ok { return nil, fmt.Errorf("duplicate option `%s`", s) } mapping[s] = s fields = append(fields, i) } } // sort the fields sort.Ints(fields) // fmt.Printf("%+v\n", fields) return fields, nil } ================================================ FILE: maker/golang/go.mod ================================================ module github.com/lionsoul2014/ip2region/maker/golang go 1.17 ================================================ FILE: maker/golang/go.sum ================================================ ================================================ FILE: maker/golang/main.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package main import ( "log" "os" "strings" "github.com/lionsoul2014/ip2region/maker/golang/cmd" ) func main() { if len(os.Args) < 2 { cmd.PrintHelp() return } log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) switch strings.ToLower(os.Args[1]) { case "gen": cmd.Generate() case "search": cmd.Search() case "bench": cmd.Bench() case "edit": cmd.Edit() case "process": cmd.Process() default: cmd.PrintHelp() } } ================================================ FILE: maker/golang/make.bat ================================================ ::ip2region golang maker makefile in windows @echo off if [%1] == [] goto:build if %1==clean ( call:clean ) else if %1==build ( call:build ) exit /b 0 :build go build -o xdb_maker.exe exit /b 0 :clean del/f/s/q xdb_maker.exe ================================================ FILE: maker/golang/xdb/editor.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // original source ip editor package xdb import ( "container/list" "fmt" "os" "path/filepath" "sort" ) type Editor struct { verison *Version // source ip file srcPath string srcHandle *os.File toSave bool // segments list segments *list.List } func NewEditor(version *Version, srcFile string) (*Editor, error) { // check the src and dst file srcPath, err := filepath.Abs(srcFile) if err != nil { return nil, err } srcHandle, err := os.OpenFile(srcPath, os.O_RDONLY, 0600) if err != nil { return nil, err } e := &Editor{ verison: version, srcPath: srcPath, srcHandle: srcHandle, toSave: false, segments: list.New(), } // load the segments if err = e.loadSegments(); err != nil { return nil, fmt.Errorf("failed to load segments: %s", err) } return e, nil } // Load all the segments from the source file func (e *Editor) loadSegments() error { var last *Segment = nil var segments []*Segment var sorting = false _, _, iErr := IterateSegments(e.srcHandle, true, func(l string) { // do nothing here }, nil, func(seg *Segment) error { // version check if len(seg.StartIP) != e.verison.Bytes { return fmt.Errorf("invalid ip segment(%s expected)", e.verison.Name) } // check the order of the data segment // if err := seg.RightBehind(last); err != nil { if err := seg.After(last); err != nil { // return err // @Note: If the continuity is disrupted, // we will sort all these segments later. sorting = true } // e.segments.PushBack(seg) segments = append(segments, seg) last = seg return nil }) if iErr != nil { return iErr } // check and do the sorting if sorting { sort.Slice(segments, func(i, j int) bool { return IPCompare(segments[i].StartIP, segments[j].StartIP) < 0 }) // open the to save e.toSave = true } // check and fill in the discontinuous segments // to Keep the entire data continuous. last = nil for _, seg := range segments { if err := seg.After(last); err != nil { } if last == nil { if IPCompare(seg.StartIP, e.verison.Min) > 0 { e.segments.PushBack(&Segment{ StartIP: e.verison.Min, EndIP: IPSubOne(seg.StartIP), Region: "", }) } } else if err := seg.RightBehind(last); err == nil { // Do nothing here since it just right behind the last } else if err := seg.After(last); err != nil { // segments overlap return fmt.Errorf("overlap checking: %w", err) } else { // push the padding segments e.segments.PushBack(&Segment{ StartIP: IPAddOne(last.EndIP), EndIP: IPSubOne(seg.StartIP), Region: "", }) } // push the current segment e.segments.PushBack(seg) // reset the last last = seg } // check and padding the tailing segmnet if back := e.segments.Back(); back != nil { if IPCompare(e.verison.Max, back.Value.(*Segment).EndIP) > 0 { e.segments.PushBack(&Segment{ StartIP: IPAddOne(back.Value.(*Segment).EndIP), EndIP: e.verison.Max, Region: "", }) } } segments = nil // let GC do it work return nil } func (e *Editor) NeedSave() bool { return e.toSave } func (e *Editor) SegLen() int { return e.segments.Len() } func (e *Editor) Slice(offset int, size int) []*Segment { var index = -1 var out []*Segment var next *list.Element for ele := e.segments.Front(); ele != nil; ele = next { next = ele.Next() s, ok := ele.Value.(*Segment) if !ok { continue } // offset match index++ if index < offset { continue } out = append(out, s) if len(out) >= size { break } } return out } func (e *Editor) Put(ip string) (int, int, error) { seg, err := SegmentFrom(ip) if err != nil { return 0, 0, err } return e.PutSegment(seg) } // PutSegment put the specified segment into the current segment list with // the following position relationships. // 1, A - fully contained like: // StartIP------seg.StartIP--------seg.EndIP----EndIP // // |------------------| // // 2, B - intersect like: // StartIP------seg.StartIP------EndIP------| // // |---------------------seg.EndIP func (e *Editor) PutSegment(seg *Segment) (int, int, error) { var next *list.Element var eList []*list.Element var found, counter = false, 0 for ele := e.segments.Front(); ele != nil; ele = next { next = ele.Next() s, ok := ele.Value.(*Segment) if !ok { // could this even be a case ? return 0, 0, fmt.Errorf("type error: ele not a Segment ptr") } counter++ // found the related segment if found { // just keep going } else if IPCompare(seg.StartIP, s.StartIP) >= 0 && IPCompare(seg.StartIP, s.EndIP) <= 0 { found = true } else { continue } eList = append(eList, ele) if IPCompare(seg.EndIP, s.EndIP) <= 0 { break } } if len(eList) == 0 { // could this even be a case ? return 0, 0, fmt.Errorf("failed to find the related segment") } // print for debug // for i, s := range eList { // fmt.Printf("ele %d: %s\n", i, s.Value.(*Segment)) // } // segment split var sList []*Segment var head = eList[0].Value.(*Segment) if IPCompare(seg.StartIP, head.StartIP) > 0 { sList = append(sList, &Segment{ StartIP: head.StartIP, EndIP: IPSubOne(seg.StartIP), Region: head.Region, }) } // append the new segment sList = append(sList, seg) // check and do the tailing segment append var tail = eList[len(eList)-1].Value.(*Segment) if IPCompare(seg.EndIP, tail.EndIP) < 0 { sList = append(sList, &Segment{ StartIP: IPAddOne(seg.EndIP), EndIP: tail.EndIP, Region: tail.Region, }) } // print for debug // for i, s := range sList { // fmt.Printf("%d: %s\n", i, s) // } // delete all the in-range segments and var base *list.Element var oldRows, newRows = len(eList), len(sList) for _, ele := range eList { base = ele.Next() e.segments.Remove(ele) } // add all the new segments if base == nil { for _, s := range sList { e.segments.PushBack(s) } } else { for _, s := range sList { e.segments.InsertBefore(s, base) } } // open the to save flag e.toSave = true return oldRows, newRows, nil } func (e *Editor) PutFile(src string) (int, int, error) { handle, err := os.OpenFile(src, os.O_RDONLY, 0600) if err != nil { return 0, 0, err } var oldRows, newRows = 0, 0 _, _, iErr := IterateSegments(handle, true, func(l string) { // do nothing here }, nil, func(seg *Segment) error { o, n, err := e.PutSegment(seg) if err == nil { oldRows += o newRows += n } return err }) if iErr != nil { return oldRows, newRows, iErr } _ = handle.Close() return oldRows, newRows, nil } func (e *Editor) Save() error { // check the to-save flag if !e.toSave { return fmt.Errorf("nothing changed") } dstHandle, err := os.OpenFile(e.srcPath, os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return err } // loop and flush all the segments to the dstHandle var next *list.Element for ele := e.segments.Front(); ele != nil; ele = next { next = ele.Next() s, ok := ele.Value.(*Segment) if !ok { // could this even be a case ? continue } // ignore the padded or empty segment if s.Region == "" { continue } // var l = s.String() // _, err = dstHandle.WriteString(fmt.Sprintf("%s\n", l)) _, err = fmt.Fprintln(dstHandle, s.String()) if err != nil { return err } } // close the handle // and close the to-save flag _ = dstHandle.Close() e.toSave = false return nil } func (e *Editor) Close() { _ = e.srcHandle.Close() } ================================================ FILE: maker/golang/xdb/index.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package xdb import ( "fmt" "strings" ) type IndexPolicy int const ( VectorIndexPolicy IndexPolicy = 1 BTreeIndexPolicy IndexPolicy = 2 ) func IndexPolicyFromString(str string) (IndexPolicy, error) { switch strings.ToLower(str) { case "vector": return VectorIndexPolicy, nil case "btree": return BTreeIndexPolicy, nil default: return VectorIndexPolicy, fmt.Errorf("invalid policy '%s'", str) } } ================================================ FILE: maker/golang/xdb/maker.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // ---- // Ip2Region database v2.0 structure // // +----------------+-------------------+---------------+--------------+ // | header space | speed up index | data payload | block index | // +----------------+-------------------+---------------+--------------+ // | 256 bytes | 512 KiB (fixed) | dynamic size | dynamic size | // +----------------+-------------------+---------------+--------------+ // // 1. padding space : for header info like block index ptr, version, release date eg ... or any other temporary needs. // -- 2bytes: version number, different version means structure update, // it fixed to 2 before IPv6 supporting, then updated to 3 since IPv6 supporting // -- 2bytes: index algorithm code. // -- 4bytes: generate unix timestamp (version) // -- 4bytes: index block start ptr // -- 4bytes: index block end ptr // -- 2bytes: ip version number (4/6 since IPv6 supporting) // -- 2bytes: runtime ptr bytes // // // 2. data block: region or whatever data info. // 3. segment index block: binary index block. // 4. vector index block: fixed index info for block index search speedup. // space structure table: // -- 0 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- 1 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- 2 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- ... // -- 255 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // // // super block structure: // +-----------------------+----------------------+ // | first index block ptr | last index block ptr | // +-----------------------+----------------------+ // // data entry structure: // +--------------------+-----------------------+ // | 2bytes (for desc) | dynamic length | // +--------------------+-----------------------+ // data length whatever in bytes // // index entry structure // +------------+-----------+---------------+------------+ // | 4bytes | 4bytes | 2bytes | 4 bytes | // +------------+-----------+---------------+------------+ // start ip end ip data length data ptr package xdb import ( "encoding/binary" "fmt" "log/slog" "math" "os" "sort" "time" ) const ( VersionNo = 3 // since 2025/09/01 (IPv6 supporting) HeaderInfoLength = 256 VectorIndexRows = 256 VectorIndexCols = 256 VectorIndexSize = 8 // in bytes RuntimePtrSize = 4 // in bytes VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize ) type Maker struct { version *Version srcHandle *os.File dstHandle *os.File // self-define field index fields []int indexPolicy IndexPolicy segments []*Segment regionPool map[string]uint32 vectorIndex []byte } func NewMaker(version *Version, policy IndexPolicy, srcFile string, dstFile string, fields []int) (*Maker, error) { // open the source file with READONLY mode var err error var srcHandle *os.File if srcFile == "" { srcHandle = nil } else { srcHandle, err = os.OpenFile(srcFile, os.O_RDONLY, 0600) if err != nil { return nil, fmt.Errorf("open source file `%s`: %w", srcFile, err) } } // open the destination file with Read/Write mode dstHandle, err := os.OpenFile(dstFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { return nil, fmt.Errorf("open target file `%s`: %w", dstFile, err) } return &Maker{ version: version, srcHandle: srcHandle, dstHandle: dstHandle, // fields filter index fields: fields, indexPolicy: policy, segments: []*Segment{}, regionPool: map[string]uint32{}, vectorIndex: make([]byte, VectorIndexLength), }, nil } func (m *Maker) initDbHeader() error { slog.Info("try to init the db header ... ") _, err := m.dstHandle.Seek(0, 0) if err != nil { return err } // make and write the header space var header = make([]byte, 256) // 1, data version number binary.LittleEndian.PutUint16(header, uint16(VersionNo)) // 2, index policy code binary.LittleEndian.PutUint16(header[2:], uint16(m.indexPolicy)) // 3, generate unix timestamp binary.LittleEndian.PutUint32(header[4:], uint32(time.Now().Unix())) // 4, index block start ptr binary.LittleEndian.PutUint32(header[8:], uint32(0)) // 5, index block end ptr binary.LittleEndian.PutUint32(header[12:], uint32(0)) // 6, ip version binary.LittleEndian.PutUint16(header[16:], uint16(m.version.Id)) // 7, runtime ptr bytes binary.LittleEndian.PutUint16(header[18:], uint16(RuntimePtrSize)) _, err = m.dstHandle.Write(header) if err != nil { return err } return nil } func (m *Maker) loadSegments() error { slog.Info("try to load the segments ... ") var last *Segment = nil var tStart = time.Now() var sorting = false _, mergeCount, iErr := IterateSegments(m.srcHandle, true, func(l string) { slog.Debug("loaded", "segment", l) }, func(region string) (string, error) { // apply the field filter return RegionFiltering(region, m.fields) }, func(seg *Segment) error { // ip version check if len(seg.StartIP) != m.version.Bytes { return fmt.Errorf("invalid ip segment(%s expected)", m.version.Name) } // check the order of the data segment if sorting { // just keep going } else if err := seg.After(last); err != nil { // return err // @Note: If the continuity is disrupted, // we will sort all these segments later. sorting = true } m.segments = append(m.segments, seg) last = seg return nil }) if iErr != nil { return fmt.Errorf("failed to load segments: %s", iErr) } // check and do the sorting if sorting { slog.Info("try to sort all the segments based on its start ip ...") sort.Slice(m.segments, func(i, j int) bool { return IPCompare(m.segments[i].StartIP, m.segments[j].StartIP) < 0 }) slog.Info("try to check if there is overlap in the segments ...") last = nil for _, seg := range m.segments { // check the order of the data segment if err := seg.After(last); err != nil { return fmt.Errorf("overlap checking: %w", err) } // reset the last last = seg } } slog.Info("all segments loaded", "length", len(m.segments), "merged", mergeCount, "sorting", sorting, "elapsed", time.Since(tStart)) return nil } // Init the db binary file func (m *Maker) Init() error { // init the db header err := m.initDbHeader() if err != nil { return fmt.Errorf("init db header: %w", err) } // load all the segments if m.srcHandle == nil { // do nothing here } else { err = m.loadSegments() if err != nil { return fmt.Errorf("load segments: %w", err) } } return nil } // Append a new segment func (m *Maker) Append(seg *Segment) { m.segments = append(m.segments, seg) } // refresh the vector index of the specified ip func (m *Maker) setVectorIndex(ip []byte, ptr uint32) { var segIdxSize = uint32(m.version.SegmentIndexSize) var il0, il1 = int(ip[0]), int(ip[1]) var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize var sPtr = binary.LittleEndian.Uint32(m.vectorIndex[idx:]) if sPtr == 0 { binary.LittleEndian.PutUint32(m.vectorIndex[idx:], ptr) binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+segIdxSize) } else { binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+segIdxSize) } } // Start to make the binary file func (m *Maker) Start() error { if len(m.segments) < 1 { return fmt.Errorf("empty segment list") } // 1, write all the region/data to the binary file _, err := m.dstHandle.Seek(int64(HeaderInfoLength+VectorIndexLength), 0) if err != nil { return fmt.Errorf("seek to data first ptr: %w", err) } slog.Info("try to write the data block ... ") for _, seg := range m.segments { slog.Debug("try to write", "region", seg.Region) ptr, has := m.regionPool[seg.Region] if has { slog.Debug(" --[Cached]", "ptr=", ptr) continue } var region = []byte(seg.Region) if len(region) > 0xFFFF { return fmt.Errorf("too long region info `%s`: should be less than %d bytes", seg.Region, 0xFFFF) } // get the first ptr of the next region pos, err := m.dstHandle.Seek(0, 1) if err != nil { return fmt.Errorf("seek to current ptr: %w", err) } // @TODO: remove this if the long ptr operation were supported if pos >= math.MaxUint32 { return fmt.Errorf("region ptr exceed the max length of %d", math.MaxUint32) } _, err = m.dstHandle.Write(region) if err != nil { return fmt.Errorf("write region '%s': %w", seg.Region, err) } m.regionPool[seg.Region] = uint32(pos) slog.Debug(" --[Added] with", "ptr", pos) } // 2, write the index block and cache the super index block slog.Info("try to write the segment index block ... ") var indexBuff = make([]byte, m.version.SegmentIndexSize) var counter, startIndexPtr, endIndexPtr = 0, int64(-1), int64(-1) for _, seg := range m.segments { dataPtr, has := m.regionPool[seg.Region] if !has { return fmt.Errorf("missing ptr cache for region `%s`", seg.Region) } // @Note: data length should be the length of bytes. // this works fine because of the string feature (byte sequence) of golang. var dataLen = len(seg.Region) if dataLen < 1 { // @TODO: could this even be a case ? // return fmt.Errorf("empty region info for segment '%s'", seg) // Allow empty region info since 2024/09/24 } var _offset = 0 var segList = seg.Split() slog.Debug("try to index segment", "length", len(segList), "splits", seg.String()) for _, s := range segList { pos, err := m.dstHandle.Seek(0, 1) if err != nil { return fmt.Errorf("seek to segment index block: %w", err) } // @TODO: remove this if the long ptr operation were supported if pos >= math.MaxUint32 { return fmt.Errorf("segment index ptr exceed the max length of %d", math.MaxUint32) } // encode the segment index. // @Note by Leon at 2025/09/05: // This is a tough decision since the directly copy of the bytes will make everything simpler. // But in order to compatible with the old searcher implementation we had to keep encoding the IPv4 bytes with little endian. // @TODO: we may choose to use the big-endian byte order in the future. // But now compatibility is the most important !!! m.version.PutBytes(indexBuff[0:], s.StartIP) m.version.PutBytes(indexBuff[len(s.StartIP):], s.EndIP) _offset = len(s.StartIP) + len(s.EndIP) binary.LittleEndian.PutUint16(indexBuff[_offset:], uint16(dataLen)) binary.LittleEndian.PutUint32(indexBuff[_offset+2:], dataPtr) _, err = m.dstHandle.Write(indexBuff) if err != nil { return fmt.Errorf("write segment index for '%s': %w", s.String(), err) } slog.Debug("|-segment index", "counter", counter, "ptr", pos, "segment", s.String()) m.setVectorIndex(s.StartIP, uint32(pos)) counter++ // check and record the start index ptr if startIndexPtr == -1 { startIndexPtr = pos } endIndexPtr = pos } } // synchronized the vector index block slog.Info("try to write the vector index block ... ") _, err = m.dstHandle.Seek(int64(HeaderInfoLength), 0) if err != nil { return fmt.Errorf("seek vector index first ptr: %w", err) } _, err = m.dstHandle.Write(m.vectorIndex) if err != nil { return fmt.Errorf("write vector index: %w", err) } // synchronized the segment index info slog.Info("try to write the segment index ptr ... ") binary.LittleEndian.PutUint32(indexBuff, uint32(startIndexPtr)) binary.LittleEndian.PutUint32(indexBuff[4:], uint32(endIndexPtr)) _, err = m.dstHandle.Seek(8, 0) if err != nil { return fmt.Errorf("seek segment index ptr: %w", err) } _, err = m.dstHandle.Write(indexBuff[:8]) if err != nil { return fmt.Errorf("write segment index ptr: %w", err) } slog.Info("write done", "dataBlocks", len(m.regionPool), "indexBlocks", len(m.segments), "counter", counter, "startIndexPtr", startIndexPtr, "endIndexPtr", endIndexPtr) return nil } func (m *Maker) End() error { err := m.dstHandle.Close() if err != nil { return err } err = m.srcHandle.Close() if err != nil { return err } return nil } ================================================ FILE: maker/golang/xdb/processor.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // original source ip processor package xdb import ( "fmt" "log/slog" "os" "sort" "strings" "time" ) type Processor struct { srcHandle *os.File dstHandle *os.File // value clear clearBasedIndex int clearValueEqual string clearValueExcept string fields []int segments []*Segment } func NewProcessor(srcFile string, dstFile string, fields []int, clearBasedIndex int, clearValueEqual string, clearValueExcept string) (*Processor, error) { // open the source file with READONLY mode srcHandle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) if err != nil { return nil, fmt.Errorf("open source file `%s`: %w", srcFile, err) } // open the destination file with Read/Write mode dstHandle, err := os.OpenFile(dstFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { return nil, fmt.Errorf("open target file `%s`: %w", dstFile, err) } return &Processor{ srcHandle: srcHandle, dstHandle: dstHandle, // clear clearBasedIndex: clearBasedIndex, clearValueEqual: clearValueEqual, clearValueExcept: clearValueExcept, // filter fields index fields: fields, segments: []*Segment{}, }, nil } func (p *Processor) loadSegments() error { slog.Info("try to load the segments ... ") var tStart = time.Now() _, mergeCount, iErr := IterateSegments(p.srcHandle, true, func(l string) { slog.Debug("loaded", "segment", l) }, func(region string) (string, error) { if p.clearBasedIndex > -1 { var ps = strings.Split(region, "|") var pl = len(ps) if p.clearBasedIndex >= pl { return region, fmt.Errorf("clearBasedIndex(%d) >= fields length(%d)", p.clearBasedIndex, pl) } clear := false if len(p.clearValueEqual) > 0 { if ps[p.clearBasedIndex] == p.clearValueEqual { clear = true } } else if len(p.clearValueExcept) > 0 { if ps[p.clearBasedIndex] != p.clearValueExcept { clear = true } } if clear { for i := 0; i < pl; i++ { ps[i] = "" } // reset the region region = strings.Join(ps, "|") } } return RegionFiltering(region, p.fields) }, func(seg *Segment) error { // check the continuity of the data segment // if err := seg.AfterCheck(last); err != nil { // return err // } // slog.Info("filtered", "source", seg.Region, "filtered", region) p.segments = append(p.segments, seg) return nil }) if iErr != nil { return fmt.Errorf("failed to load segments: %s", iErr) } slog.Info("all segments loaded", "length", len(p.segments), "merged", mergeCount, "elapsed", time.Since(tStart)) return nil } // Init the db binary file func (p *Processor) Init() error { // load all the segments err := p.loadSegments() if err != nil { return fmt.Errorf("load segments: %w", err) } return nil } func (p *Processor) Start() error { slog.Info("try to sort all the segments based on its start ip ...") sort.Slice(p.segments, func(i, j int) bool { return IPCompare(p.segments[i].StartIP, p.segments[j].StartIP) < 0 }) slog.Info("try to write all segments to target file ...") for _, seg := range p.segments { _, err := fmt.Fprintln(p.dstHandle, seg.String()) if err != nil { return fmt.Errorf("write segment index for '%s': %w", seg.String(), err) } } slog.Info("process done", "segments", len(p.segments)) return nil } func (p *Processor) End() error { err := p.dstHandle.Close() if err != nil { return err } err = p.srcHandle.Close() if err != nil { return err } return nil } ================================================ FILE: maker/golang/xdb/searcher.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // --- // Ip2Region database v2.0 searcher. // this is part of the maker for testing and validate. // please use the searcher in binding/golang for production use. // And this is a Not thread safe implementation. package xdb import ( "encoding/binary" "fmt" "os" ) type Searcher struct { version *Version handle *os.File // header info header []byte // use it only when this feature enabled. // Preload the vector index will reduce the number of IO operations // thus speedup the search process vectorIndex []byte } func NewSearcher(version *Version, dbFile string) (*Searcher, error) { handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) if err != nil { return nil, err } return &Searcher{ version: version, handle: handle, header: nil, vectorIndex: nil, }, nil } func (s *Searcher) Close() { if s.handle != nil { err := s.handle.Close() if err != nil { return } } } // LoadVectorIndex load and cache the vector index for search speedup. // this will take up VectorIndexRows x VectorIndexCols x VectorIndexSize bytes memory. func (s *Searcher) LoadVectorIndex() error { // loaded already if s.vectorIndex != nil { return nil } // load all the vector index block _, err := s.handle.Seek(HeaderInfoLength, 0) if err != nil { return fmt.Errorf("seek to vector index: %w", err) } var buff = make([]byte, VectorIndexLength) rLen, err := s.handle.Read(buff) if err != nil { return err } if rLen != len(buff) { return fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } s.vectorIndex = buff return nil } // ClearVectorIndex clear preloaded vector index cache func (s *Searcher) ClearVectorIndex() { s.vectorIndex = nil } // Search find the region for the specified ip address func (s *Searcher) Search(ip []byte) (string, int, error) { // version check if len(ip) != s.version.Bytes { return "", 0, fmt.Errorf("invalid ip address(%s expected)", s.version.Name) } // locate the segment index block based on the vector index var ioCount = 0 var il0, il1, bytes, tBytes = int(ip[0]), int(ip[1]), len(ip), len(ip) << 1 var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize var sPtr, ePtr = uint32(0), uint32(0) if s.vectorIndex != nil { sPtr = binary.LittleEndian.Uint32(s.vectorIndex[idx:]) ePtr = binary.LittleEndian.Uint32(s.vectorIndex[idx+4:]) } else { pos, err := s.handle.Seek(int64(HeaderInfoLength+idx), 0) if err != nil { return "", ioCount, fmt.Errorf("seek to vector index %d: %w", HeaderInfoLength+idx, err) } ioCount++ var buff = make([]byte, VectorIndexSize) rLen, err := s.handle.Read(buff) if err != nil { return "", ioCount, fmt.Errorf("read vector index at %d: %w", pos, err) } if rLen != len(buff) { return "", ioCount, fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } sPtr = binary.LittleEndian.Uint32(buff) ePtr = binary.LittleEndian.Uint32(buff[4:]) } //log.Printf("vIndex=%s", vIndex) if sPtr == 0 || ePtr == 0 { return "", ioCount, nil } // binary search the segment index to get the region var segIndexSize = uint32(s.version.SegmentIndexSize) var dataLen, dataPtr = 0, uint32(0) var buff = make([]byte, segIndexSize) var l, h = 0, int((ePtr - sPtr) / segIndexSize) for l <= h { // log.Printf("l=%d, h=%d", l, h) m := (l + h) >> 1 p := sPtr + uint32(m)*segIndexSize // log.Printf("m=%d, p=%d", m, p) _, err := s.handle.Seek(int64(p), 0) if err != nil { return "", ioCount, fmt.Errorf("seek to segment block at %d: %w", p, err) } ioCount++ rLen, err := s.handle.Read(buff) if err != nil { return "", ioCount, fmt.Errorf("read segment index at %d: %w", p, err) } if rLen != len(buff) { return "", ioCount, fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } // decode the data step by step to reduce the unnecessary calculations if s.version.IPCompare(ip, buff[0:bytes]) < 0 { h = m - 1 } else if s.version.IPCompare(ip, buff[bytes:tBytes]) > 0 { l = m + 1 } else { dataLen = int(binary.LittleEndian.Uint16(buff[tBytes:])) dataPtr = binary.LittleEndian.Uint32(buff[tBytes+2:]) break } } if dataLen == 0 { return "", ioCount, nil } // load and return the region data _, err := s.handle.Seek(int64(dataPtr), 0) if err != nil { return "", ioCount, fmt.Errorf("seek to data block at %d: %w", dataPtr, err) } ioCount++ var regionBuff = make([]byte, dataLen) rLen, err := s.handle.Read(regionBuff) if err != nil { return "", ioCount, fmt.Errorf("read region data at %d: %w", dataPtr, err) } if rLen != dataLen { return "", ioCount, fmt.Errorf("incomplete read: readed bytes should be %d", dataLen) } return string(regionBuff), ioCount, nil } // LoadXdbHeader load the header info from the specified handle func LoadXdbHeader(handle *os.File) ([]byte, error) { _, err := handle.Seek(0, 0) if err != nil { return nil, fmt.Errorf("seek to the header: %w", err) } var buff = make([]byte, HeaderInfoLength) rLen, err := handle.Read(buff) if err != nil { return nil, err } if rLen != len(buff) { return nil, fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) } return buff, nil } // LoadXdbHeaderFromFile load header info from the specified db file path func LoadXdbHeaderFromFile(dbFile string) ([]byte, error) { handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) if err != nil { return nil, fmt.Errorf("open xdb file `%s`: %w", dbFile, err) } defer func(handle *os.File) { _ = handle.Close() }(handle) header, err := LoadXdbHeader(handle) if err != nil { return nil, err } return header, nil } ================================================ FILE: maker/golang/xdb/segment.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package xdb import ( "fmt" "strings" ) type Segment struct { StartIP []byte EndIP []byte Region string } func SegmentFrom(seg string) (*Segment, error) { var ps = strings.SplitN(strings.TrimSpace(seg), "|", 3) if len(ps) != 3 { return nil, fmt.Errorf("invalid ip segment `%s`", seg) } sip, err := ParseIP(ps[0]) if err != nil { return nil, fmt.Errorf("check start ip `%s`: %s", ps[0], err) } eip, err := ParseIP(ps[1]) if err != nil { return nil, fmt.Errorf("check end ip `%s`: %s", ps[1], err) } if IPCompare(sip, eip) > 0 { return nil, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) } return &Segment{ StartIP: sip, EndIP: eip, Region: ps[2], }, nil } // RightBehind check the current segment is just right behind the specified one // which mean last.EndIP + 1 = s.startIP func (s *Segment) RightBehind(last *Segment) error { if last != nil { if IPCompare(IPAddOne(last.EndIP), s.StartIP) != 0 { return fmt.Errorf( "discontinuous data segment: last.eip(%s)+1 != seg.sip(%s, %s)", IP2String(last.EndIP), IP2String(s.StartIP), s.Region, ) } } return nil } // After check the current segment is after the specified one // which means last.EndIP < s.startIP func (s *Segment) After(last *Segment) error { if last != nil { if IPCompare(last.EndIP, s.StartIP) >= 0 { return fmt.Errorf( "disorder data segment: last.eip(%s) >= seg.sip(%s, %s)", IP2String(last.EndIP), IP2String(s.StartIP), s.Region, ) } } return nil } // Split the segment based on the pre-two bytes func (s *Segment) Split() []*Segment { // 1, split the segment with the first byte var tList []*Segment var sByte1, eByte1 = int(s.StartIP[0]), int(s.EndIP[0]) // var nSip = s.StartIP for i := sByte1; i <= eByte1; i++ { // Make and init the new start & end IP sip := make([]byte, len(s.StartIP)) eip := make([]byte, len(s.StartIP)) if i == sByte1 { sip = s.StartIP } else { sip[0] = byte(i) } if i == eByte1 { eip = s.EndIP } else { // set the first byte eip[0] = byte(i) // fill the buffer with 0xFF for j := 1; j < len(eip); j++ { eip[j] = 0xFF } } // sip := (i << 24) | (nSip & 0xFFFFFF) // eip := (i << 24) | 0xFFFFFF // if eip < s.EndIP { // nSip = (i + 1) << 24 // } else { // eip = s.EndIP // } // fmt.Printf("sip:%+v, eip: %+v\n", sip, eip) // append the new segment (maybe) tList = append(tList, &Segment{ StartIP: sip, EndIP: eip, // @Note: don't bother to copy the region /// Region: s.Region, }) } // 2, split the segments with the second byte var segList []*Segment for _, seg := range tList { // base := seg.StartIP & 0xFF000000 // nSip := seg.StartIP // sb2, eb2 := (seg.StartIP>>16)&0xFF, (seg.EndIP>>16)&0xFF sb2, eb2 := int(seg.StartIP[1]), int(seg.EndIP[1]) // fmt.Printf("seg: %s, sb2: %d, eb2: %d\n", seg.String(), sb2, eb2) for i := sb2; i <= eb2; i++ { // sip := base | (i << 16) | (nSip & 0xFFFF) // eip := base | (i << 16) | 0xFFFF // if eip < seg.EndIP { // nSip = 0 // } else { // eip = seg.EndIP // } sip := make([]byte, len(s.StartIP)) eip := make([]byte, len(s.StartIP)) sip[0] = seg.StartIP[0] eip[0] = seg.StartIP[0] if i == sb2 { sip = seg.StartIP } else { sip[1] = byte(i) } if i == eb2 { eip = seg.EndIP } else { eip[1] = byte(i) for j := 2; j < len(eip); j++ { eip[j] = 0xFF } } // fmt.Printf("i=%d, sip:%+v, eip: %+v\n", i, sip, eip) segList = append(segList, &Segment{ StartIP: sip, EndIP: eip, Region: s.Region, }) } } return segList } func (s *Segment) String() string { return fmt.Sprintf("%s|%s|%s", IP2String(s.StartIP), IP2String(s.EndIP), s.Region) } // Contains checks if an IP address is within this segment func (s *Segment) Contains(ip []byte) bool { return IPCompare(s.StartIP, ip) <= 0 && IPCompare(ip, s.EndIP) <= 0 } ================================================ FILE: maker/golang/xdb/util.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package xdb import ( "bufio" "bytes" "fmt" "math/big" "net" "os" "strings" ) // Util function func ParseIP(ip string) ([]byte, error) { parsedIP := net.ParseIP(ip) if parsedIP == nil { return nil, fmt.Errorf("invalid ip address: %s", ip) } v4 := parsedIP.To4() if v4 != nil { return v4, nil } v6 := parsedIP.To16() if v6 != nil { return v6, nil } return nil, fmt.Errorf("invalid ip address: %s", ip) } func IP2String(ip []byte) string { return net.IP(ip[:]).String() } func IP2Long(ip []byte) *big.Int { return big.NewInt(0).SetBytes(ip) } // IPCompare compares two IP addresses // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 func IPCompare(ip1, ip2 []byte) int { // for i := 0; i < len(ip1); i++ { // if ip1[i] < ip2[i] { // return -1 // } // if ip1[i] > ip2[i] { // return 1 // } // } // return 0 return bytes.Compare(ip1, ip2) } func IPAddOne(ip []byte) []byte { var r = make([]byte, len(ip)) copy(r, ip) for i := len(ip) - 1; i >= 0; i-- { r[i]++ if r[i] != 0 { // No overflow break } } return r } func IPSubOne(ip []byte) []byte { var r = make([]byte, len(ip)) copy(r, ip) for i := len(ip) - 1; i >= 0; i-- { if r[i] != 0 { // No borrow needed r[i]-- break } r[i] = 0xFF // borrow from the next byte } return r } // IPSub Sub the spcecified two byte ip func IPSub(sip, eip []byte) ([]byte, error) { if len(sip) != len(eip) { return []byte{}, fmt.Errorf("length of the two ips are not the same") } var carry uint16 = 0 var result = make([]byte, len(sip)+1) for i := len(sip) - 1; i >= 0; i-- { sum := uint16(sip[i]) + uint16(eip[i]) + carry result[i+1] = byte(sum) // Store standard 8-bit result carry = sum >> 8 // Extract the 1-bit carry for the next byte } // check and append the carry if carry > 0 { result[0] = byte(carry) return result, nil } else { return result[1:], nil } } // IPHalf get the half value of an input byte ip func IPHalf(ip []byte) []byte { var length = len(ip) var result = make([]byte, length) // Tracks the bit falling off from the previous byte var carry byte = 0 for i := 0; i < length; i++ { // 1. Shift current byte right by 1 // 2. Or (|) with the carry from the previous byte (shifted to the MSB position) result[i] = (ip[i] >> 1) | (carry << 7) // 3. Capture the Least Significant Bit (LSB) to use as carry for the next byte carry = ip[i] & 1 } return result } // IPMiddle get the middle value of two input ip address func IPMiddle(sip, eip []byte) ([]byte, error) { buf, err := IPSub(sip, eip) if err != nil { return []byte{}, fmt.Errorf("IPSub(%s, %s): %w", IP2String(sip), IP2String(eip), err) } return IPHalf(buf), nil } func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) { var last *Segment = nil var totalCount, mergeCount = 0, 0 var scanner = bufio.NewScanner(handle) scanner.Split(bufio.ScanLines) for scanner.Scan() { var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n")) if len(l) < 1 { // ignore empty line continue } if l[0] == '#' { // ignore the comment line continue } totalCount++ if before != nil { before(l) } var ps = strings.SplitN(l, "|", 3) if len(ps) != 3 { return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l) } sip, err := ParseIP(ps[0]) if err != nil { return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err) } eip, err := ParseIP(ps[1]) if err != nil { return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err) } if len(sip) != len(eip) { return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l) } if IPCompare(sip, eip) > 0 { return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) } // Allow empty region info since 2024/09/24 // if len(ps[2]) < 1 { // return fmt.Errorf("empty region info in segment line `%s`", l) // } // check and do the region filter var region = ps[2] if filter != nil { region, err = filter(ps[2]) if err != nil { return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err) } } var seg = &Segment{ StartIP: sip, EndIP: eip, Region: region, } // check and automatic merging the Consecutive Segments, which means: // 1, region info is the same // 2, last.eip+1 = cur.sip if last == nil { last = seg continue } else if autoMerge && last.Region == seg.Region { if err = seg.RightBehind(last); err == nil { mergeCount++ last.EndIP = seg.EndIP continue } } if err = done(last); err != nil { return totalCount, mergeCount, err } // reset the last last = seg } // process the last segment if last != nil { return totalCount, mergeCount, done(last) } return totalCount, mergeCount, nil } func CheckSegments(segList []*Segment) error { var last *Segment for _, seg := range segList { // sip must <= eip if IPCompare(seg.StartIP, seg.EndIP) > 0 { return fmt.Errorf("segment `%s`: start ip should not be greater than end ip", seg.String()) } // check the continuity of the data segment if last != nil { if IPCompare(IPAddOne(last.EndIP), seg.StartIP) != 0 { return fmt.Errorf("discontinuous segment `%s`: last.eip+1 != cur.sip", seg.String()) } } last = seg } return nil } func RegionFiltering(region string, fields []int) (string, error) { if len(fields) == 0 { return region, nil } fs := strings.Split(region, "|") var sb []string for _, idx := range fields { if idx < 0 { return "", fmt.Errorf("negative filter index %d", idx) } if idx >= len(fs) { return "", fmt.Errorf("field index %d exceeded the max length of %d", idx, len(fs)) } sb = append(sb, fs[idx]) } return strings.Join(sb, "|"), nil } ================================================ FILE: maker/golang/xdb/util_test.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package xdb import ( "encoding/binary" "fmt" "os" "testing" ) func TestParseIP(t *testing.T) { var ips = []string{"29.34.191.255", "2c0f:fff0::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} for _, ip := range ips { bytes, err := ParseIP(ip) if err != nil { t.Errorf("check ip `%s`: %s\n", IP2String(bytes), err) } nip := IP2String(bytes) fmt.Printf("checkip: (%s / %s), isEqual: %v\n", ip, nip, ip == nip) } } func TestIPCompare(t *testing.T) { var ipPairs = [][]string{ {"1.2.3.4", "1.2.3.5"}, {"58.250.36.41", "58.250.30.41"}, {"2c10::", "2e00::"}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, {"fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, } for _, pairs := range ipPairs { fmt.Printf("IPCompare(%s, %s): %d\n", pairs[0], pairs[1], IPCompare([]byte(pairs[0]), []byte(pairs[1]))) } } func TestIPAddOne(t *testing.T) { var ipPairs = [][]string{ {"1.2.3.4", "1.2.3.5"}, {"2.3.4.5", "2.3.4.6"}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"}, {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"}, } for _, pairs := range ipPairs { sip, err := ParseIP(pairs[0]) if err != nil { t.Errorf("parse ip `%s`: %s\n", pairs[0], err) } eip, err := ParseIP(pairs[1]) if err != nil { t.Errorf("parse ip `%s`: %s\n", pairs[1], err) } fmt.Printf("IPAddOne(%s) = %s ? %d\n", pairs[0], pairs[1], IPCompare(IPAddOne(sip), eip)) } } func TestIPAddOne2(t *testing.T) { var ip = []byte{0, 1, 2, 3} nip := IPAddOne(ip) fmt.Printf("nip: %+v, ip:%+v", ip, nip) } func TestIPSubOne(t *testing.T) { var ipPairs = [][]string{ {"1.2.3.4", "1.2.3.5"}, {"2.3.4.5", "2.3.4.6"}, {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"}, {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"}, } for _, pairs := range ipPairs { sip, err := ParseIP(pairs[0]) if err != nil { t.Errorf("parse ip `%s`: %s\n", pairs[0], err) } eip, err := ParseIP(pairs[1]) if err != nil { t.Errorf("parse ip `%s`: %s\n", pairs[1], err) } fmt.Printf("IPSubOne(%s) = %s ? %d\n", pairs[1], pairs[0], IPCompare(IPSubOne(eip), sip)) } } func TestIPSubOne2(t *testing.T) { var ip = []byte{0, 1, 2, 3} nip := IPSubOne(ip) fmt.Printf("nip: %+v, ip:%+v", ip, nip) } func TestIPSub(t *testing.T) { var strToSub = "1.2.3.4" bytesToSub, err := ParseIP(strToSub) if err != nil { t.Fatalf("failed to parse ip %s", strToSub) } var intToSub = int(binary.BigEndian.Uint32(bytesToSub)) t.Logf("to sub ip: %d -> %s", intToSub, strToSub) counter := 0 buf := make([]byte, 4) for i := 0; i < 0x2FFFFFFF; i++ { binary.BigEndian.PutUint32(buf, uint32(i)) subVal, err := IPSub(buf, bytesToSub) if err != nil { t.Fatalf("failed to IPSub(%s,%s): %s", IP2String(buf), strToSub, err) } // do it as two integers byteSub := int(binary.BigEndian.Uint32(subVal)) intSub := i + intToSub if byteSub != intSub { t.Fatal("byte and int sub value are not the same") } counter++ } t.Logf("test done with %d ips", counter) } func TestIPHalf(t *testing.T) { var buf = make([]byte, 4) for i := 0; i < 0xFFFFFFFF; i++ { binary.BigEndian.PutUint32(buf, uint32(i)) half := IPHalf(buf) // do it as two integers byteMiddle := binary.BigEndian.Uint32(half) intMidle := i >> 1 if byteMiddle != uint32(intMidle) { t.Fatal("byte middle and int middle are not the same") } } } func TestSubOverflow(t *testing.T) { var ip1Str = "255.255.255.250" ip1Bytes, err := ParseIP(ip1Str) if err != nil { t.Fatalf("failed to ParseIP(%s): %s", ip1Str, err) } var buff = make([]byte, 4) for i := 0; i < 10; i++ { binary.BigEndian.PutUint32(buff, uint32(i)) ipSub, err := IPSub(ip1Bytes, buff) if err != nil { t.Fatalf("failed to IPSub(%s, %s): %s", ip1Str, IP2String(buff), err) } t.Logf("IPSub(%s, %s) = %+v", ip1Str, IP2String(buff), ipSub) } } func TestIPMiddle(t *testing.T) { var sIPStr = "0.0.0.0" sBytes, err := ParseIP(sIPStr) if err != nil { t.Fatalf("failed to parse ip %s", sIPStr) } var sInt = int(binary.BigEndian.Uint32(sBytes)) t.Logf("start ip: %d -> %s", sInt, sIPStr) counter := 0 buf := make([]byte, 4) for i := 0; i < 0x0FFFFFFF; i++ { binary.BigEndian.PutUint32(buf, uint32(i)) midVal, err := IPMiddle(sBytes, buf) if err != nil { t.Fatalf("failed to IPMiddle(%s,%s): %s", sIPStr, IP2String(buf), err) } // do it as two integers byteMid := int(binary.BigEndian.Uint32(midVal)) intMid := (sInt + i) >> 1 if byteMid != intMid { t.Fatal("byte and int middle value are not the same") } counter++ } t.Logf("test done with %d ips", counter) } func TestSplitSegmentV4(t *testing.T) { // var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信" // var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP" // var str = "29.0.0.0|29.34.191.255|美国|0|0|0|0" var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0" seg, err := SegmentFrom(str) if err != nil { t.Fatalf("failed to parser segment '%s': %s", str, err) } fmt.Printf("idx: src, seg: %s\n", seg.String()) var segList = seg.Split() err = CheckSegments(segList) if err != nil { t.Fatalf("check segments: %s", err.Error()) } for i, s := range segList { fmt.Printf("idx: %3d, seg: %s\n", i, s.String()) } } func TestRegionFiltering(t *testing.T) { var line = "2001:1203:31:8000::|2001:1203:31:bfff:ffff:ffff:ffff:ffff||墨西哥|瓜纳华托州||||专线用户|" seg, err := SegmentFrom(line) if err != nil { t.Fatalf("failed to parse segment '%s': %s", line, err) } fReg, err := RegionFiltering(seg.Region, []int{1, 2, 4, 6}) if err != nil { t.Fatalf("failed to filter region '%s': %s", seg.Region, err) } fmt.Printf("region: %s, filtered: %s\n", seg.Region, fReg) } func TestSplitSegmentV6(t *testing.T) { var str = "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff||瑞士|弗里堡州||||专线用户|IANA" seg, err := SegmentFrom(str) if err != nil { t.Fatalf("failed to parser segment '%s': %s", str, err) } fmt.Printf("idx: src, seg: %s\n", seg.String()) var segList = seg.Split() err = CheckSegments(segList) if err != nil { t.Fatalf("check segments: %s", err.Error()) } for i, s := range segList { fmt.Printf("idx: %3d, seg: %s\n", i, s.String()) } } func TestIterateSegments(t *testing.T) { handle, err := os.OpenFile("../../../data/sample/segments.tests.mixed", os.O_RDONLY, 0600) if err != nil { t.Fatalf("failed to open tests file: %s", err) } _, _, _ = IterateSegments(handle, true, func(l string) { // fmt.Printf("load segment: `%s`\n", l) }, nil, func(seg *Segment) error { fmt.Printf("get segment: `%s`\n", seg) return nil }) } ================================================ FILE: maker/golang/xdb/version.go ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package xdb import ( "bytes" "fmt" "strings" ) const ( IPv4VersionNo = 4 IPv6VersionNo = 6 ) type Version struct { Id int Name string Bytes int SegmentIndexSize int // bytes encode PutBytes func([]byte, []byte) int // ip compares IPCompare func([]byte, []byte) int Min []byte Max []byte } func (v *Version) String() string { return fmt.Sprintf("{Id:%d, Name:%s, Bytes:%d, IndexSize: %d}", v.Id, v.Name, v.Bytes, v.SegmentIndexSize) } var ( IPvx = &Version{} IPv4 = &Version{ Id: 4, Name: "IPv4", Bytes: 4, SegmentIndexSize: 14, // 4 + 4 + 2 + 4, PutBytes: func(buff []byte, ip []byte) int { // binary.LittleEndian.PutUint32(buff, binary.BigEndian.Uint32(ip)) // Little Endian byte order for compatible with the old searcher implementation buff[0] = ip[3] buff[1] = ip[2] buff[2] = ip[1] buff[3] = ip[0] return len(ip) }, IPCompare: func(ip1 []byte, ip2 []byte) int { // ip1 - with Bit endian parsed from an input // ip2 - with Little endian read from the xdb index ip2[0], ip2[3] = ip2[3], ip2[0] ip2[1], ip2[2] = ip2[2], ip2[1] return bytes.Compare(ip1, ip2) }, Min: []byte{0x00, 0x00, 0x00, 0x00}, Max: []byte{0xff, 0xff, 0xff, 0xff}, } IPv6 = &Version{ Id: 6, Name: "IPv6", Bytes: 16, SegmentIndexSize: 38, // 16 + 16 + 2 + 4, // Big Endian byte order to follow the network byte order PutBytes: func(buff []byte, ip []byte) int { return copy(buff, ip) }, IPCompare: func(ip1, ip2 []byte) int { return bytes.Compare(ip1, ip2) }, Min: []byte{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }, Max: []byte{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, }, } ) func VersionFromIP(ip string) (*Version, error) { r, err := ParseIP(ip) if err != nil { return IPvx, fmt.Errorf("parse ip fail: %w", err) } if len(r) == 4 { return IPv4, nil } return IPv6, nil } func VersionFromName(name string) (*Version, error) { switch strings.ToUpper(name) { case "V4", "IPV4": return IPv4, nil case "V6", "IPV6": return IPv6, nil default: return IPvx, fmt.Errorf("invalid version name `%s`", name) } } ================================================ FILE: maker/java/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb java generation implementation # Compilation and Installation Compile the executable jar program via maven: ```bash # cd to the maker/java root directory mvn clean compile package ``` Then you will get an ip2region-maker-{version}.jar package file in the target directory of the current directory. # Data Generation Generate the xdb binary file via `java -jar ip2region-maker-{version}.jar`: ```bash ➜ java git:(master) java -jar target/ip2region-maker-3.0.0.jar ip2region xdb maker java -jar ip2region-maker-{version}.jar [command options] options: --src string source ip text file path --dst string destination binary xdb file path --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7 --log-level string set the log level, options: debug/info/warn/error ``` For example, generate an IPv4 ip2region_v4.xdb binary file in the current directory using the default data/ipv4_source.txt raw data: ```bash java -jar target/ip2region-maker-3.0.0.jar --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb --version=ipv4 ... 2025-09-13 00:33:06 INFO org.lionsoul.ip2region.xdb.Maker write done, dataBlocks: 13827, indexBlocks: (683843, 720464), indexPtr: (955933, 11042415) 2025-09-13 00:33:06 INFO org.lionsoul.ip2region.MakerApp Done, elapsed: 2 s ``` For example, generate an IPv6 ip2region_v6.xdb binary file in the current directory using the default data/ipv6_source.txt raw data: ```bash java -jar target/ip2region-maker-3.0.0.jar --src=../../data/ipv6_source.txt --dst=./ip2region_v6.xdb --version=ipv6 ... 2025-09-13 00:35:34 INFO org.lionsoul.ip2region.xdb.Maker write done, dataBlocks: 120446, indexBlocks: (16789611, 16855074), indexPtr: (6585371, 647078145) 2025-09-13 00:35:34 INFO org.lionsoul.ip2region.MakerApp Done, elapsed: 67 s ``` For custom data fields during the generation process, please refer to [xdb-文件生成#自定义数据字段](https://ip2region.net/doc/data/xdb_make#field-list) # Data Search/bench Test All [bindings](../../binding/) come with search and bench test programs as well as usage documentation. You can use the searcher of your familiar language for query testing or bench testing to confirm the correctness and integrity of the data. ================================================ FILE: maker/java/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb java 生成实现 # 编译安装 通过 maven 来编译可运行 jar 程序： ```bash # cd 到 maker/java 根目录 mvn clean compile package ``` 然会会在当前目录的 target 目录下得到一个 ip2region-maker-{version}.jar 的打包文件。 # 数据生成 通过 `java -jar ip2region-maker-{version}.jar` 来生成 xdb 二进制文件： ```bash ➜ java git:(master) java -jar target/ip2region-maker-3.0.0.jar ip2region xdb maker java -jar ip2region-maker-{version}.jar [command options] options: --src string source ip text file path --dst string destination binary xdb file path --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7 --log-level string set the log level, options: debug/info/warn/error ``` 例如，通过默认的 data/ipv4_source.txt 原始数据，在当前目录生成一个 IPv4 的 ip2region_v4.xdb 二进制文件： ```bash java -jar target/ip2region-maker-3.0.0.jar --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb --version=ipv4 ... 2025-09-13 00:33:06 INFO org.lionsoul.ip2region.xdb.Maker write done, dataBlocks: 13827, indexBlocks: (683843, 720464), indexPtr: (955933, 11042415) 2025-09-13 00:33:06 INFO org.lionsoul.ip2region.MakerApp Done, elapsed: 2 s ``` 例如，通过默认的 data/ipv6_source.txt 有原始据，在当前目录生成一个 IPv6 的 ip2region_v6.xdb 二进制文件： ```bash java -jar target/ip2region-maker-3.0.0.jar --src=../../data/ipv6_source.txt --dst=./ip2region_v6.xdb --version=ipv6 ... 2025-09-13 00:35:34 INFO org.lionsoul.ip2region.xdb.Maker write done, dataBlocks: 120446, indexBlocks: (16789611, 16855074), indexPtr: (6585371, 647078145) 2025-09-13 00:35:34 INFO org.lionsoul.ip2region.MakerApp Done, elapsed: 67 s ``` 生成过程中数据字段自定义请参考 [xdb-文件生成#自定义数据字段](https://ip2region.net/doc/data/xdb_make#field-list) # 数据 查询/bench 测试 已经完成开发的 [binding](../../binding/) 都有查询和 bench 测试程序以及使用文档，你可以使用你熟悉的语言的 searcher 进行查询测试或者bench测试，来确认数据的正确性和完整性。 ================================================ FILE: maker/java/pom.xml ================================================ 4.0.0 org.lionsoul ip2region-maker 3.1.0 jar ip2region https://github.com/lionsoul2014/ip2region Open source offline internet address db manager framework and locator The Apache Software License, Version 2.0 https://www.apache.org/licenses/LICENSE-2.0.txt repo git@github.com:lionsoul2014/ip2region.git scm:git:git@github.com:lionsoul2014/ip2region.git scm:git:git@github.com:lionsoul2014/ip2region.git lionsoul chenxin chenxin619315@gmail.com https://github.com/lionsoul2014/ip2region/issues Github issues UTF-8 UTF-8 1.8 1.8 junit junit 4.13.2 test org.apache.maven.plugins maven-source-plugin 2.1.2 attach-sources package jar org.apache.maven.plugins maven-javadoc-plugin 2.9 attach-javadocs package jar ${javadoc.opts} false org.apache.maven.plugins maven-shade-plugin 1.4 package shade org.lionsoul.ip2region.MakerApp org.apache.maven.plugins maven-compiler-plugin 1.8 1.8 java8-doclint-disabled [1.8,) -Xdoclint:none release org.apache.maven.plugins maven-source-plugin 2.2.1 package jar-no-fork org.apache.maven.plugins maven-javadoc-plugin 2.9.1 package jar ${javadoc.opts} org.apache.maven.plugins maven-gpg-plugin 1.5 verify sign oss https://oss.sonatype.org/content/repositories/snapshots/ oss https://oss.sonatype.org/service/local/staging/deploy/maven2/ ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/MakerApp.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Lion // @Date 2022/07/12 package org.lionsoul.ip2region; import org.lionsoul.ip2region.xdb.IndexPolicy; import org.lionsoul.ip2region.xdb.Log; import org.lionsoul.ip2region.xdb.Maker; import org.lionsoul.ip2region.xdb.Version; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; public class MakerApp { public final static Log log = Log.getLogger(MakerApp.class); public final static Pattern p = Pattern.compile("^(\\d+(-\\d+)?)$"); public static void printHelp(String[] args) { System.out.println("ip2region xdb maker"); System.out.println("java -jar ip2region-maker-{version}.jar [command options]"); System.out.println("options:"); System.out.println(" --src string source ip text file path"); System.out.println(" --dst string destination binary xdb file path"); System.out.println(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused"); System.out.println(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7"); System.out.println(" --log-level string set the log level, options: debug/info/warn/error"); } private static int[] getFieldList(String fieldList) { final ArrayList list = new ArrayList(); final Map map = new HashMap(); if (!fieldList.isEmpty()) { String[] fList = fieldList.split(","); for (String f : fList) { final String s = f.trim(); if (s.isEmpty()) { log.errorf("undefined option `%s`", f); return null; } final Matcher m = p.matcher(s); if (!m.matches()) { log.errorf("field `%s` is not a number", f); return null; } final String ms = m.group(1); if (ms.indexOf('-') == -1) { if (map.containsKey(s)) { log.errorf("duplicate field index `%s`", s); return null; } map.put(s, s); final int idx = Integer.parseInt(s); if (idx < 0) { log.errorf("field index `%s` is negative", s); return null; } list.add(idx); continue; } // index range parse final String[] ra = ms.split("-"); if (ra.length != 2) { log.errorf("field `%s` is not a valid range", ms); return null; } final int start = Integer.parseInt(ra[0]); final int end = Integer.parseInt(ra[1]); if (start > end) { log.errorf("index range start(%d) should <= end(%d)", start, end); return null; } for (int i = start; i <= end; i++) { final String _s = String.valueOf(i); if (map.containsKey(_s)) { log.errorf("duplicate field index `%s`", _s); return null; } map.put(_s, _s); list.add(i); } } } // let's keep it a complex way so the old JDK could run these pieces. final int[] fields = new int[list.size()]; for (int i = 0; i < list.size(); i++) { fields[i] = list.get(i); } // sort the fields to make sure the fields follow the original index order Arrays.sort(fields); return fields; } public static void genDb(String[] args) throws Exception { String srcFile = "", dstFile = "", ipVersion = ""; String fieldList = "", logLevel = "info"; int indexPolicy = IndexPolicy.Vector; for (final String r : args) { if (r.length() < 5) { continue; } if (r.indexOf("--") != 0) { continue; } int sIdx = r.indexOf('='); if (sIdx < 0) { System.out.printf("missing = for args pair `%s`\n", r); return; } String key = r.substring(2, sIdx); String val = r.substring(sIdx + 1); // System.out.printf("key=%s, val=%s\n", key, val); if ("src".equals(key)) { srcFile = val; } else if ("dst".equals(key)) { dstFile = val; } else if ("version".equals(key)) { ipVersion = val; } else if ("field-list".equals(key)) { fieldList = val; } else if ("log-level".equals(key)) { logLevel = val; } else { System.out.printf("undefined option `%s`\n", r); return; } } if (srcFile.isEmpty() || dstFile.isEmpty()) { printHelp(args); return; } // IP version final Version version = Version.fromName(ipVersion); final int[] fields = getFieldList(fieldList); if (fields == null) { return; } // check and make the field list long tStart = System.currentTimeMillis(); final Maker maker = new Maker(version, indexPolicy, srcFile, dstFile, fields); log.infof("Generating xdb with src=%s, dst=%s, logLevel=%s", srcFile, dstFile, logLevel); MakerApp.log.setLevel(logLevel); maker.init(); maker.start(); maker.end(); log.infof("Done, elapsed: %d s", (System.currentTimeMillis() - tStart) / 1000); } public static void main(String[] args) { if (args.length < 1) { printHelp(args); return; } try { genDb(args); } catch (Exception e) { System.out.printf("failed running genDb: %s\n", e); } } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/IPv4.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; // IPv4 version implementation // @Author Lion // @Date 2025/09/10 public class IPv4 extends Version { public IPv4() { // segmentIndex: 4 + 4 + 2 + 4 super(4, "IPv4", 4, 14); } @Override public int putBytes(byte[] buff, int offset, byte[] ip) { // use the Little endian byte order to compatible with the old searcher implementation buff[offset++] = ip[3]; buff[offset++] = ip[2]; buff[offset++] = ip[1]; buff[offset ] = ip[0]; return ip.length; } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/IPv6.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; // IPv4 version implementation // @Author Lion // @Date 2025/09/10 public class IPv6 extends Version { public IPv6() { // segmentIndex: 16 + 16 + 2 + 4 super(6, "IPv6", 16, 38); } @Override public int putBytes(byte[] buff, int offset, byte[] ip) { System.arraycopy(ip, 0, buff, offset, ip.length); return ip.length; } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Lion // @Date 2022/07/14 package org.lionsoul.ip2region.xdb; public class IndexPolicy { public static final int Vector = 1; public static final int BTree = 2; // parser the index policy from string public static int parse(String policy) throws Exception { String v = policy.toLowerCase(); if ("vector".equals(v)) { return Vector; } else if ("btree".equals(v)) { return BTree; } else { throw new Exception("unknown index policy `"+policy+"`"); } } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/InvalidInetAddressException.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; public class InvalidInetAddressException extends Exception { public InvalidInetAddressException(String str) { super(str); } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/LittleEndian.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; // Little Endian basic data type decode and encode. // @Author Lion // @Date 2025/09/10 public class LittleEndian { public final static int[] shiftIndex = {0, 8, 16, 24, 32, 40, 48, 56}; // put specified bytes to the buffer started from the offset public static void put(final byte[] buff, int offset, long value, int bytes) { if (bytes > 8) { throw new IndexOutOfBoundsException("bytes should be <= 8"); } for (int i = 0; i < bytes; i++) { buff[offset++] = (byte)((value >>> shiftIndex[i]) & 0xFF); } } // put an uint32 (4 bytes long) to the buffer from the offset public static void putUint32(final byte[] buff, int offset, long value) { buff[offset++] = (byte) (value & 0xFF); buff[offset++] = (byte) ((value >> 8) & 0xFF); buff[offset++] = (byte) ((value >> 16) & 0xFF); buff[offset ] = (byte) ((value >> 24) & 0xFF); } // put a 2-bytes int to the buffer from the specified offset public static void putInt2(final byte[] buff, int offset, int value) { buff[offset++] = (byte) (value & 0xFF); buff[offset ] = (byte) ((value >> 8) & 0xFF); } // get an uint32 from a byte array from the specified offset public static long getUint32(final byte[] buff, int offset) { return ( ((buff[offset++] & 0x000000FFL)) | ((buff[offset++] << 8) & 0x0000FF00L) | ((buff[offset++] << 16) & 0x00FF0000L) | ((buff[offset ] << 24) & 0xFF000000L) ); } // get an 2 bytes int from a byte array from the specified offset public static int getInt2(final byte[] buff, int offset) { return ( ((buff[offset++]) & 0x000000FF) | ((buff[offset ] << 8) & 0x0000FF00) ); } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Lion // @Date 2022/07/14 package org.lionsoul.ip2region.xdb; import java.text.SimpleDateFormat; import java.util.Date; // simple log implementation public class Log { /* Log level constants define */ public static final int DEBUG = 0; public static final int INFO = 1; public static final int WARN = 2; public static final int ERROR = 3; // level name public static final String[] level_string = new String[] { "DEBUG", "INFO", "WARN", "ERROR" }; public final Class baseClass; private int level = INFO; public Log(Class baseClass) { this.baseClass = baseClass; } public static Log getLogger(Class baseClass) { return new Log(baseClass); } public String format(int level, String format, Object... args) { // append the datetime final StringBuilder sb = new StringBuilder(); final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); sb.append(String.format("%s %-5s ", sdf.format(new Date()), level_string[level])); // append the class name sb.append(baseClass.getName()).append(' '); sb.append(String.format(format, args)); return sb.toString(); } public void printf(int level, String format, Object... args) { if (level < DEBUG || level > ERROR) { throw new IndexOutOfBoundsException("invalid level index " + level); } // level filter if (level < this.level) { return; } System.out.println(format(level, format, args)); System.out.flush(); } public String getDebugf(String format, Object... args) { return format(DEBUG, format, args); } public void debugf(String format, Object... args) { printf(DEBUG, format, args); } public String getInfof(String format, Object... args) { return format(INFO, format, args); } public void infof(String format, Object... args) { printf(INFO, format, args); } public String getWarnf(String format, Object... args) { return format(WARN, format, args); } public void warnf(String format, Object... args) { printf(WARN, format, args); } public String getErrorf(String format, Object... args) { return format(ERROR, format, args); } public void errorf(String format, Object... args) { printf(ERROR, format, args); } public Log setLevel(int level) { this.level = level; return this; } public Log setLevel(String level) { String v = level.toLowerCase(); if ("debug".equals(v)) { this.level = DEBUG; } else if ("info".equals(v)) { this.level = INFO; } else if ("warn".equals(v)) { this.level = WARN; } else if ("error".equals(v)) { this.level = ERROR; } return this; } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Lion // @Date 2022/07/12 // --- Ip2Region v2.0 data structure // // +----------------+--------------------------+---------------+--------------+ // | header space | vector speed up index | data payload | block index | // +----------------+--------------------------+---------------+--------------+ // | 256 bytes | 512 KiB (fixed) | dynamic size | dynamic size | // +----------------+--------------------------+---------------+--------------+ // // 1. padding space : for header info like block index ptr, version, release date eg ... or any other temporary needs. // -- 2bytes: version number, different version means structure update, it fixed to 2 for now // -- 2bytes: index algorithm code. // -- 4bytes: generate unix timestamp (version) // -- 4bytes: index block start ptr // -- 4bytes: index block end ptr // -- 2bytes: ip version number (4/6 since IPv6 supporting) // -- 2bytes: runtime ptr bytes // // // 2. data block : region or whatever data info. // 3. segment index block : binary index block. // 4. vector index block : fixed index info for block index search speed up. // space structure table: // -- 0 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- 1 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- 2 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // -- ... // -- 255 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block // // // super block structure: // +-----------------------+----------------------+ // | first index block ptr | last index block ptr | // +-----------------------+----------------------+ // // data entry structure: // +--------------------+-----------------------+ // | 2bytes (for desc) | dynamic length | // +--------------------+-----------------------+ // data length whatever in bytes // // index entry structure // +------------+-----------+---------------+------------+ // | 4bytes | 4bytes | 2bytes | 4 bytes | // +------------+-----------+---------------+------------+ // start ip end ip data length data ptr package org.lionsoul.ip2region.xdb; import java.io.*; import java.nio.charset.Charset; import java.util.*; import org.lionsoul.ip2region.xdb.Segment.IterateAction; public class Maker { // constants define public static final int VersionNo = 3; // 2 for XDB 2.0, 3 for XDB 3.0 public static final int HeaderInfoLength = 256; public static final int VectorIndexRows = 256; public static final int VectorIndexCols = 256; public static final int VectorIndexSize = 8; // in bytes public static final int RuntimePtrSize = 4; // in bytes public static final long MaxFilePointer = (1L << (RuntimePtrSize * 8)) - 1; public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize; public static final Log log = Log.getLogger(Maker.class); // IP version since xdb 3.0 private final Version version; // source text file handle private final File srcFile; private final int[] fields; private final List segments; private final Charset bytesCharset; // destination binary file handle private final RandomAccessFile dstHandle; // index policy private final int indexPolicy; // region pool private final Map regionPool; // vector index raw bytes private final byte[] vectorIndex; public Maker(Version version, int policy, String srcPath, String dstPath, int[] fields) throws IOException { this.srcFile = new File(srcPath); if (!this.srcFile.exists()) { throw new FileNotFoundException("source text file `" + srcPath + "` not found"); } this.version = version; this.fields = fields; /// check and delete the target xdb file if it exists /// final File dstFile = new File(dstPath); /// if (dstFile.exists() && !dstFile.delete()) { /// log.warnf("failed to delete the dest xdb file `%s`", dstPath); /// } this.bytesCharset = Charset.forName("utf-8"); this.segments = new ArrayList(); this.dstHandle = new RandomAccessFile(dstPath, "rw"); this.indexPolicy = policy; this.regionPool = new HashMap(); this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0 // truncate the original xdb file this.dstHandle.setLength(0); } // init the header of the target xdb binary file private void initHeader() throws IOException { log.infof("try to init the db header ... "); dstHandle.seek(0); // make and write the header space final byte[] header = new byte[HeaderInfoLength]; // encode the data // 1, data version number LittleEndian.put(header, 0, VersionNo, 2); // 2, index policy code LittleEndian.put(header, 2, indexPolicy, 2); // 3, generate unix timestamp LittleEndian.put(header, 4, System.currentTimeMillis() / 1000, 4); // 4, index block start ptr LittleEndian.put(header, 8, 0, 4); // start index ptr // 5, index block end ptr LittleEndian.put(header, 12, 0, 4); // end index ptr // since xdb 3.0 // 6, IP version LittleEndian.put(header, 16, version.id, 2); // IP version // 7, runtime ptr bytes LittleEndian.put(header, 18, RuntimePtrSize, 2); // runtime ptr bytes dstHandle.write(header); } // load all the segments. private void loadSegments() throws Exception { log.infof("try to load the segments ... "); final long tStart = System.currentTimeMillis(); final IterateAction itAct = new Segment.IterateAction() { private Segment last = null; private boolean sorting = false; @Override public void before(String line) { log.debugf("load segment: `%s`", line); } @Override public String filter(String region) { return Util.regionFiltering(region, fields); } @Override public void handle(Segment seg) throws Exception { // ip version check if (seg.startIP.length != version.bytes) { throw new Exception("invalid ip segment("+version.name+" expected)"); } if (sorting) { // just keep going } else if (last != null && !seg.after(last)) { // throw new Exception("discontinuous data segment: last.eip(" // + Util.ipToString(last.endIP)+")+1 != seg.sip("+ Util.ipToString(seg.startIP) + ", "+ seg.region +")"); // @Note: If the continuity is disrupted, // we will sort all these segments later. sorting = true; } segments.add(seg); last = seg; } public boolean sorting() { return sorting; } }; // load iterate all the segments Segment.iterate(srcFile, itAct); final boolean sorting = itAct.sorting(); // check and sort all the segments if (sorting) { log.infof("try to sort all the segments based on its start ip ..."); segments.sort((o1, o2) -> {return Util.ipCompare(o1.startIP, o2.startIP);}); log.infof("try to check if there is overlap in the segments ..."); Segment last = null; for (final Segment seg : segments) { // check the order of the data segment if (last != null && !seg.after(last)) { throw new Exception("overlap checking: last.eip(" + Util.ipToString(last.endIP)+") >= seg.sip("+ Util.ipToString(seg.startIP) + ", "+ seg.region +")"); } // reset the last last = seg; } } log.infof( "all segments loaded, length: %d, sorting: %s, elapsed: %d ms", segments.size(), sorting ? "true" : "false", System.currentTimeMillis() - tStart ); } // init the maker public void init() throws Exception { // init the db header initHeader(); // load all the segments loadSegments(); } // set the vector index info of the specified ip private void setVectorIndex(final byte[] ip, long ptr) { final int il0 = (int) (ip[0] & 0xFF); final int il1 = (int) (ip[1] & 0xFF); final int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; final long sPtr = LittleEndian.getUint32(vectorIndex, idx); if (sPtr == 0) { LittleEndian.put(vectorIndex, idx, ptr, 4); LittleEndian.put(vectorIndex, idx + 4, ptr + version.segmentIndexSize, 4); } else { LittleEndian.put(vectorIndex, idx + 4, ptr + version.segmentIndexSize, 4); } } // start to make the binary file public void start() throws Exception { if (segments.isEmpty()) { throw new Exception("empty segment list"); } // 1, write all the region/data to the binary file dstHandle.seek(HeaderInfoLength + VectorIndexLength); log.infof("try to write the data block ... "); for (final Segment seg : segments) { log.debugf("try to write region `%s` ... ", seg.region); final DataEntry e = regionPool.get(seg.region); if (e != null) { log.debugf(" --[Cached] with ptr=%d", e.ptr); continue; } // get the utf-8 bytes of the region info final byte[] regionBuff = seg.region.getBytes(bytesCharset); if (regionBuff.length < 1) { // allow empty region info // throw new Exception("empty region info for segment `"+seg+"`"); } else if (regionBuff.length > 0xFFFF) { throw new Exception("too long region info `"+seg.region+"`: should be less than 65535 bytes"); } // record the current ptr final long pos = dstHandle.getFilePointer(); dstHandle.write(regionBuff); // @TODO: remove this if the long ptr operation were supported if (pos >= MaxFilePointer) { throw new IOException("region ptr exceed the max length of '" + MaxFilePointer + "'"); } // record the mapping regionPool.put(seg.region, new DataEntry(regionBuff.length, pos)); log.debugf(" --[Added] with ptr=%d", pos); } // 2, write the index block cache the super index block log.infof("try to write the segment index block ... "); int counter = 0; long startIndexPtr = -1, endIndexPtr = -1; final byte[] indexBuff = new byte[version.segmentIndexSize]; for (Segment seg : segments) { // we need the region ptr final DataEntry e = regionPool.get(seg.region); if (e == null) { throw new Exception("missing ptr cache for region `"+seg.region+"`"); } int _offset = 0; List segList = seg.split(); log.debugf("try to index segment(%d splits) %s ... ", segList.size(), seg); for (final Segment s : segList) { long pos = dstHandle.getFilePointer(); // @TODO: remove this if the long ptr operation were supported if (pos >= MaxFilePointer) { throw new IOException("region ptr exceed the max length of '" + MaxFilePointer + "'"); } // encode the segment index info. // @Note: in order to compatible with the old searcher implementation we choose to keep // encode the IPv4 bytes with little endian. // for IPv6 we choose the Network byte order (Big endian). version.putBytes(indexBuff, 0, s.startIP); version.putBytes(indexBuff, s.startIP.length, s.endIP); _offset = s.startIP.length + s.endIP.length; LittleEndian.put(indexBuff, _offset, e.length, 2); LittleEndian.put(indexBuff, _offset + 2, e.ptr, 4); dstHandle.write(indexBuff); log.debugf("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s); setVectorIndex(s.startIP, pos); counter++; // check and record the start index ptr if (startIndexPtr == -1) { startIndexPtr = pos; } endIndexPtr = pos; } } // 3, synchronize the vector index block log.infof("try to write the vector index block ... "); dstHandle.seek(HeaderInfoLength); dstHandle.write(vectorIndex); // 4, synchronize the segment index info log.infof("try to write the segment index ptr ... "); LittleEndian.put(indexBuff, 0, startIndexPtr, 4); LittleEndian.put(indexBuff, 4, endIndexPtr, 4); dstHandle.seek(8); dstHandle.write(indexBuff, 0, 8); log.infof("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)", regionPool.size(), segments.size(), counter, startIndexPtr, endIndexPtr); } // end the make, do the resource clean up public void end() throws IOException { this.dstHandle.close(); } private static class DataEntry { final long ptr; final int length; // in bytes DataEntry(int length, long ptr) { this.length = length; this.ptr = ptr; } } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Lion // @Date 2022/07/14 package org.lionsoul.ip2region.xdb; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; public class Segment { public final byte[] startIP; public final byte[] endIP; public final String region; public Segment(final byte[] startIP, final byte[] endIP, String region) { this.startIP = startIP; this.endIP = endIP; this.region = region; } // split the current segment for vector index public List split() { final int sByte1 = (int) (startIP[0] & 0xFF); final int eByte1 = (int) (endIP[0] & 0xFF); final List tList = new ArrayList(); for (int i = sByte1; i <= eByte1; i++) { final byte[] sip = new byte[startIP.length]; final byte[] eip = new byte[startIP.length]; if (i == sByte1) { System.arraycopy(startIP, 0, sip, 0, sip.length); } else { sip[0] = (byte) (i & 0xFF); } if (i == eByte1) { System.arraycopy(endIP, 0, eip, 0, eip.length); } else { eip[0] = (byte)(i & 0xFF); // fill the rest buffer with 0xFF for (int j = 1; j < eip.length; j++) { eip[j] = (byte) (0xFF); } } // append the new segment: // @Note: Don't bother to copy the region. tList.add(new Segment(sip, eip, null)); } // 2, split the segments with the second byte final List segList = new ArrayList(); for (Segment seg : tList) { final int sByte2 = (int) (seg.startIP[1] & 0xFF); final int eByte2 = (int) (seg.endIP[1] & 0xFF); for (int i = sByte2; i <= eByte2; i++) { final byte[] sip = new byte[seg.startIP.length]; final byte[] eip = new byte[seg.startIP.length]; sip[0] = seg.startIP[0]; eip[0] = seg.startIP[0]; if (i == sByte2) { System.arraycopy(seg.startIP, 0, sip, 0, seg.startIP.length); } else { sip[1] = (byte) (i & 0xFF); } if (i == eByte2) { System.arraycopy(seg.endIP, 0, eip, 0, seg.endIP.length); } else { eip[1] = (byte) (i & 0xFF); for (int j = 2; j < seg.endIP.length; j++) { eip[j] = (byte) 0xFF; } } // append the new segment segList.add(new Segment(sip, eip, region)); } } return segList; } @Override public String toString() { return Util.ipToString(startIP) + "|" + Util.ipToString(endIP) + "|" + region; } // check if the an IP address in within this segment public boolean contains(final byte[] ip) { return Util.ipCompare(ip, startIP) >= 0 && Util.ipCompare(ip, endIP) <= 0; } // check if the current segment just right behind the specified one. // which mean last.endIP + 1 = this.startIP public boolean rightBehind(final Segment last) { return Util.ipCompare(Util.ipAddOne(last.endIP), startIP) == 0; } // check if the current segment is after the specified one. // which means last.endIP < this.startIP public boolean after(final Segment last) { return Util.ipCompare(last.endIP, startIP) < 0; } // parser the Segment from an input string public static Segment parse(String input) throws Exception { final String[] ps = input.trim().split("\\|", 3); if (ps.length != 3) { throw new Exception("invalid ip segment `"+input+"`"); } final byte[] sip = Util.parseIP(ps[0]); final byte[] eip = Util.parseIP(ps[1]); if (Util.ipCompare(sip, eip) > 0) { throw new Exception("start ip `"+ps[0]+"` should not be greater than end ip `"+ps[1]+"`"); } return new Segment(sip, eip, ps[2]); } // static class to handler the iterate callback public static interface IterateAction { // need sort all the iterated segments ? default boolean sorting() { return false; } public void before(final String line); public String filter(final String region); public void handle(final Segment seg) throws Exception; } // iterate the segments from the specified ip source file and call the handler public static void iterate(final String srcFile, IterateAction action) throws Exception { iterate(new File(srcFile), action); } public static void iterate(final File srcFile, IterateAction action) throws Exception { Segment last = null; String line = null; final FileInputStream fis = new FileInputStream(srcFile); final BufferedReader br = new BufferedReader(new InputStreamReader(fis, "utf-8")); while ((line = br.readLine()) != null) { final String l = line.trim(); // ignore empty line if (l.length() < 1) { continue; } // ignore comment line if (l.charAt(0) == '#') { continue; } // call the action.before action.before(l); // split the line to create the segment final String[] ps = line.split("\\|", 3); if (ps.length != 3) { br.close(); throw new Exception("invalid ip segment line `"+ps[0]+"`"); } final byte[] sip = Util.parseIP(ps[0]); final byte[] eip = Util.parseIP(ps[1]); if (sip.length != eip.length) { br.close(); throw new Exception("invalid ip segment line `" + line + "`: sip/eip version not match"); } if (Util.ipCompare(sip, eip) > 0) { br.close(); throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")"); } // allow empty region info // if (ps[2].isEmpty()) { // br.close(); // throw new Exception("empty region info in segment line `"+ps[2]+"`"); // } final Segment seg = new Segment(sip, eip, action.filter(ps[2])); // check and set the last segment if (last == null) { last = seg; continue; } // check and automatic merging the Consecutive Segments, which means: // 1, region info is the same // 2, last.eip+1 = cur.sip if (last.region.equals(seg.region) && seg.rightBehind(last)) { // last.endIP = seg.endIP; System.arraycopy(seg.endIP, 0, last.endIP, 0, seg.endIP.length); continue; } // pass the segment to the aciton.handle try { action.handle(last); } catch (Exception e) { // break the loop if the handle return false br.close(); throw new Exception(e.getMessage()); } // reset the last last = seg; } // process the last segment if (last != null) { action.handle(last); } br.close(); } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java ================================================ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // // @Author Lion // @Date 2022/07/14 package org.lionsoul.ip2region.xdb; import java.net.InetAddress; import java.net.UnknownHostException; public class Util { // parse the specified IP address and return its bytes. // returns: byte[4] for IPv4 and byte[16] for IPv6 and the bytes should be in Big endian order. public static byte[] parseIP(String ip) throws InvalidInetAddressException { try { return InetAddress.getByName(ip).getAddress(); } catch (UnknownHostException e) { throw new InvalidInetAddressException("invalid ip address `"+ip+"`"); } } // convert the byte[] ip to string ip address public static String ipToString(final byte[] ip) { if (ip.length != 4 && ip.length != 16) { return String.format("invalid-ip-address-length: %d", ip.length); } try { return InetAddress.getByAddress(ip).getHostAddress(); } catch (UnknownHostException e) { return String.format("invalid-ip-address `%s`", ipArrayString(ip)); } } // implode the byte[] ip with its byte value. public static String ipArrayString(byte[] ip) { final StringBuffer sb = new StringBuffer(); sb.append("["); for (int i = 0; i < ip.length; i++) { if (i > 0) { sb.append(','); } sb.append((ip[i] & 0xFF)); } sb.append("]"); return sb.toString(); } // compare two byte ip // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 public static int ipCompare(byte[] ip1, byte[] ip2) { for (int i = 0; i < ip1.length; i++) { // covert the byte to int to sure the uint8 attribute final int i1 = (int)(ip1[i] & 0xFF); final int i2 = (int)(ip2[i] & 0xFF); if (i1 < i2) { return -1; } if (i1 > i2) { return 1; } } return 0; } public static byte[] ipAddOne(byte[] ip) { final byte[] r = new byte[ip.length]; System.arraycopy(ip, 0, r, 0, ip.length); for (int i = ip.length - 1; i >= 0; i--) { final int v = (int)(r[i] & 0xFF); if (v < 255) { // No overflow r[i]++; break; } r[i] = 0; } return r; } public static byte[] ipSubOne(byte[] ip) { final byte[] r = new byte[ip.length]; System.arraycopy(ip, 0, r, 0, ip.length); for (int i = ip.length - 1; i >= 0; i--) { final int v = (int)(r[i] & 0xFF); if (v > 0) { // No borrow needed r[i]--; break; } r[i] = (byte) 0xFF; // borrow from the next byte } return r; } // region filtering public static String regionFiltering(String region, int[] fields) { if (fields.length == 0) { return region; } final String[] fs = region.split("\\|", -1); final StringBuilder sb = new StringBuilder(); final int tailing = fields.length - 1; for (int i = 0; i < fields.length; i++) { final int idx = fields[i]; if (idx >= fs.length) { throw new IllegalArgumentException("field index `" + idx + "` exceeded the max length `" + fs.length + "`"); } sb.append(fs[idx]); if (i < tailing) { sb.append("|"); } } return sb.toString(); } } ================================================ FILE: maker/java/src/main/java/org/lionsoul/ip2region/xdb/Version.java ================================================ // Copyright 2025 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. package org.lionsoul.ip2region.xdb; // IP version abstract manager (IPv4 & IPv6) // @Author Lion // @Date 2025/09/10 public abstract class Version { public static final int IPv4VersionNo = 4; public static final int IPv6VersionNo = 6; public static final IPv4 IPv4 = new IPv4(); public static final IPv6 IPv6 = new IPv6(); // version id and name public final int id; public final String name; // the numbers of bytes for one IP public final int bytes; // segment index size (bytes) public final int segmentIndexSize; public Version(int id, String name, int bytes, int segmentIndexSize) { this.id = id; this.name = name; this.bytes = bytes; this.segmentIndexSize = segmentIndexSize; } // encode the specified IP bytes to the specified buffer public abstract int putBytes(byte[] buff, int offset, byte[] ip); // compare the two IPs with the current version. // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 public int ipCompare(byte[] ip1, byte[] ip2) { return Util.ipCompare(ip1, ip2); } // parse the version from an name public static final Version fromName(String name) throws Exception { final String n = name.toUpperCase(); if (n.equals("V4") || n.equals("IPV4")) { return IPv4; } else if (n.equals("V6") || n.equals("IPV6")) { return IPv6; } else { throw new Exception("invalid version name `"+name+"`"); } } @Override public String toString() { return String.format("{Id:%d, Name:%s, Bytes:%d, IndexSize: %d}", id, name, bytes, segmentIndexSize); } } ================================================ FILE: maker/java/src/test/java/org/lionsoul/ip2region/xdb/IndexPolicyTest.java ================================================ package org.lionsoul.ip2region.xdb; import org.junit.Test; public class IndexPolicyTest { private static final Log log = Log.getLogger(IndexPolicyTest.class).setLevel(Log.DEBUG); @Test public void testParse() { final String[] inputs = {"vector", "btree", "VecTor", "BTree", "abc"}; for (String str : inputs) { try { int policy = IndexPolicy.parse(str); log.infof("parse(%s)=%d", str, policy); } catch (Exception e) { log.errorf("parse index policy `%s`: %s", str, e.getMessage()); } } } } ================================================ FILE: maker/java/src/test/java/org/lionsoul/ip2region/xdb/LittleEndianTest.java ================================================ package org.lionsoul.ip2region.xdb; import static org.junit.Assert.assertEquals; import org.junit.Test; public class LittleEndianTest { private static final Log log = Log.getLogger(LittleEndianTest.class).setLevel(Log.DEBUG); @Test public void testAll() { final byte[] buff = new byte[14]; // encode // do the put LittleEndian.put(buff, 0, 1L, 4); LittleEndian.put(buff, 4, 2L, 4); // putUint32 LittleEndian.putInt2(buff, 8, 24); LittleEndian.putUint32(buff, 10, 1024L); // decode assertEquals(LittleEndian.getUint32(buff, 0), 1); assertEquals(LittleEndian.getUint32(buff, 4), 2); assertEquals(LittleEndian.getInt2(buff, 8), 24); assertEquals(LittleEndian.getUint32(buff, 10), 1024); log.debugf("uint32(buff, 0): %d", LittleEndian.getUint32(buff, 0)); log.debugf("uint32(buff, 4): %d", LittleEndian.getUint32(buff, 4)); log.debugf("int2(buff, 8): %d", LittleEndian.getInt2(buff, 8)); log.debugf("uint32(buff, 10): %d", LittleEndian.getUint32(buff, 10)); } } ================================================ FILE: maker/java/src/test/java/org/lionsoul/ip2region/xdb/MakerTest.java ================================================ package org.lionsoul.ip2region.xdb; import org.junit.Test; public class MakerTest { private static final Log log = Log.getLogger(MakerTest.class); @Test public void testInit() { log.infof("MaxFilePointer(%d bytes): %d", Maker.RuntimePtrSize, Maker.MaxFilePointer); log.infof("MaxFilePoiner(5 bytes): %d", ((1L << (5 * 8)) - 1)); log.infof("MaxFilePoiner(6 bytes): %d", ((1L << (6 * 8)) - 1)); } } ================================================ FILE: maker/java/src/test/java/org/lionsoul/ip2region/xdb/SegmentTest.java ================================================ package org.lionsoul.ip2region.xdb; import java.io.File; import java.net.URL; import org.junit.Test; public class SegmentTest { private final static Log log = Log.getLogger(SegmentTest.class).setLevel(Log.DEBUG); @Test public void testParse() throws Exception { final String[] strs = { "1.1.0.0|1.3.3.24|中国|广东|深圳|电信", "28.201.224.0|29.34.191.255|美国|0|0|0|0", "2001:4:112::|2001:4:112:ffff:ffff:ffff:ffff:ffff|德国|黑森|美因河畔法兰克福|专线用户" }; for (final String str : strs) { final Segment seg = Segment.parse(str); log.debugf("seg: %s", seg.toString()); } } @Test public void testSplit() throws Exception { final String[] t_segs = { "1.1.0.0|1.3.3.24|中国|广东|深圳|电信", "28.201.224.0|29.34.191.255|美国|0|0|0|0", "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户|IANA" }; for (String str : t_segs) { final Segment seg = Segment.parse(str); log.infof("segment(%s)->split: ", seg.toString()); for (final Segment s : seg.split()) { log.debugf(s.toString()); } } } @Test public void testIterate() throws Exception { final URL res = getClass().getClassLoader().getResource(""); if (res == null) { throw new Exception("unable to get the resource path"); } final String base = new File(res.getPath()).getParentFile().getParentFile().getParentFile().getParent(); Segment.iterate(base+"/data/sample/segments.tests.mixed", new Segment.IterateAction() { @Override public void before(String line) { // log.debugf("load segment: `%s`", line); } @Override public String filter(String region) { return region; } @Override public void handle(Segment seg) throws Exception { log.infof("handle segment: `%s`", seg.toString()); } }); } } ================================================ FILE: maker/java/src/test/java/org/lionsoul/ip2region/xdb/UtilTest.java ================================================ package org.lionsoul.ip2region.xdb; import org.junit.Test; public class UtilTest { private static final Log log = Log.getLogger(UtilTest.class).setLevel(Log.DEBUG); @Test public void testCheckIP() throws InvalidInetAddressException { final String[] ips = new String[]{ "192.168.1.102", "219.133.111.87", "::", "3000::", "::1001:ffff", "2001:2:0:ffff:ffff:ffff:ffff:ffff", "::ffff:114.114.114.114" }; for (String ip : ips) { final byte[] ipBytes = Util.parseIP(ip); log.debugf("%s(v=%s) => %s", ip, Util.ipArrayString(ipBytes), Util.ipToString(ipBytes)); } } @Test public void testIpCompare() throws InvalidInetAddressException { final String[][] ipPairs = new String[][]{ {"1.0.0.0", "1.0.0.1"}, {"192.168.1.101", "192.168.1.90"}, {"219.133.111.87", "114.114.114.114"}, {"2000::", "2000:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, {"2001:4:112::", "2001:4:112:ffff:ffff:ffff:ffff:ffff"}, {"ffff::", "2001:4:ffff:ffff:ffff:ffff:ffff:ffff"} }; for (String[] ips : ipPairs) { final byte[] ip1 = Util.parseIP(ips[0]); final byte[] ip2 = Util.parseIP(ips[1]); log.debugf("compare(%s, %s): %d", ips[0], ips[1], Util.ipCompare(ip1, ip2)); } } @Test public void testIpAddOne() throws InvalidInetAddressException { final String[] ips = new String[] { "1.0.0.0", "192.168.1.255", "2000::", "255.255.255.254", "0.0.0.255", "0.255.255.255", "1.1.255.255" }; for (String ip : ips) { final byte[] ipBytes = Util.parseIP(ip); log.debugf("ipAddOne(%s): %s", ip, Util.ipToString(Util.ipAddOne(ipBytes))); } } @Test public void testIpSubOne() throws InvalidInetAddressException { final String[] ips = new String[] { "192.168.1.255", "1.0.0.1", "1.0.0.0", "2.0.0.0", "2000::", "ffff::", "1::1", }; for (String ip : ips) { final byte[] ipBytes = Util.parseIP(ip); log.debugf("ipSubOne(%s): %s", ip, Util.ipToString(Util.ipSubOne(ipBytes))); } } @Test public void testRegionFiltering() { final String[] regions = new String[]{ "亚洲|中国|广东|深圳|宝安|电信|113.88311|22.55371|440306|0755|518100|Asia/Shanghai|CNY|11|CHXX0120", "大洲|国家|省份|城市|区县|ISP|经度|纬度|0|0|0|0|0", "||||||||||||", "亚洲|中国|||||||||||", "美国|加利福尼亚州|洛杉矶|专线用户|||||||||" }; final int[] fields = new int[] {1,2,3,4,6,7}; for (String region : regions) { log.infof("filtering: %s", Util.regionFiltering(region, fields)); } } } ================================================ FILE: maker/java/src/test/java/org/lionsoul/ip2region/xdb/VersionTest.java ================================================ package org.lionsoul.ip2region.xdb; import static org.junit.Assert.assertEquals; import org.junit.Test; public class VersionTest { private static final Log log = Log.getLogger(VersionTest.class).setLevel(Log.DEBUG); @Test public void testFromName() throws Exception { final String[] vers = new String[]{"IPv4", "IPv6"}; final Version v4 = Version.fromName(vers[0]); assertEquals(v4.name, vers[0]); final Version v6 = Version.fromName(vers[1]); assertEquals(v6.name, vers[1]); log.debugf("v4: %s", v4); log.debugf("v6: %s", v6); } } ================================================ FILE: maker/python/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb python generation implementation # Cli Command ``` # cd to the python maker root directory > python main.py ip2region xdb maker main.py [command] [command options] Command: gen generate the binary db file ``` # `xdb` Data Generation Generate the xdb binary file via the `python main.py gen` command: ``` ➜ python git:(v2.0_xdb) ✗ python main.py gen main.py gen [command options] options: --src string source ip text file path --dst string destination binary xdb file path ``` For example, using the default data/ipv4_source.txt as the source data to generate an ip2region_v4.xdb in the current directory: ``` ➜ python git:(v2.0_xdb) ✗ python main.py gen --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb # You will see a lot of output; eventually, you will see something like the following output indicating the end of the execution ... 2022-07-13 19:58:00,540-root-238-INFO - write done, dataBlocks: 13804, indexBlocks: (683591, 720221), indexPtr: (982904, 11065984) 2022-07-13 19:58:00,540-root-63-INFO - Done, elapsed: 3m3s ``` # `xdb` Data Query and bench Test For query functions and testing based on the xdb format, see ip2region [bindings](../../binding) ================================================ FILE: maker/python/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb python 生成实现 # cli 命令 ``` # 切换到python maker 根目录 > python main.py ip2region xdb maker main.py [command] [command options] Command: gen generate the binary db file ``` # `xdb` 数据生成 通过 `python main.py gen` 命令生成 xdb 二进制文件: ``` ➜ python git:(v2.0_xdb) ✗ python main.py gen main.py gen [command options] options: --src string source ip text file path --dst string destination binary xdb file path ``` 例如，使用默认的 data/ipv4_source.txt 作为源数据，生成一个 ip2region_v4.xdb 到当前目录： ``` ➜ python git:(v2.0_xdb) ✗ python main.py gen --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb # 会看到一堆输出，最终会看到类似如下输出表示运行结束 ... 2022-07-13 19:58:00,540-root-238-INFO - write done, dataBlocks: 13804, indexBlocks: (683591, 720221), indexPtr: (982904, 11065984) 2022-07-13 19:58:00,540-root-63-INFO - Done, elapsed: 3m3s ``` # `xdb` 数据查询 和 bench 测试 基于xdb 格式的查询功能和测试见 ip2region [binding](../../binding) ================================================ FILE: maker/python/main.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # # Author: linyufeng # Date : 2022/7/14 17:00 # import logging import sys import time import xdb.maker as mk import xdb.index as idx # Format log logging.basicConfig( level=logging.INFO, format="%(asctime)s-%(name)s-%(lineno)s-%(levelname)s - %(message)s", ) log = logging.getLogger(__name__) def print_help(): print("ip2region xdb python maker") print("{} [command] [command options]".format(sys.argv[0])) print("Command: ") print(" gen generate the binary db file") def gen_db(): src_file, dst_file = "", "" index_policy = idx.Vector_Index_Policy # Check input parameters for i in range(2, len(sys.argv)): r = sys.argv[i] if len(r) < 5: continue if not r.startswith("--"): continue s_idx = r.index("=") if s_idx < 0: print("missing = for args pair '{}'".format(r)) return if r[2:s_idx] == "src": src_file = r[s_idx + 1:] elif r[2:s_idx] == "dst": dst_file = r[s_idx + 1:] elif r[2:s_idx] == "index": index_policy = idx.index_policy_from_string(r[s_idx + 1:]) else: print("undefined option `{}`".format(r)) return if src_file == "" or dst_file == "": print("{} gen [command options]".format(sys.argv[0])) print("options:") print(" --src string source ip text file path") print(" --dst string destination binary xdb file path") return start_time = time.time() # Make the binary file maker = mk.new_maker(index_policy, src_file, dst_file) maker.init() maker.start() maker.end() logging.info( "Done, elapsed: {:.0f}m{:.0f}s".format( (time.time() - start_time) / 60, (time.time() - start_time) % 60 ) ) def main(): if len(sys.argv) < 2: print_help() return cmd = sys.argv[1].lower() if cmd == "gen": gen_db() else: print_help() if __name__ == "__main__": main() ================================================ FILE: maker/python/xdb/__init__.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # # Author: linyufeng # Date : 2022/7/14 17:00 # ================================================ FILE: maker/python/xdb/index.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # # Author: linyufeng # Date : 2022/7/14 17:00 # import struct Vector_Index_Policy = 1 BTree_Index_Policy = 2 def index_policy_from_string(s: str) -> int: sl = s.lower() if sl == "vector": return Vector_Index_Policy elif sl == "btree": return BTree_Index_Policy else: print("invalid policy `{}`, used default vector index".format(s)) return Vector_Index_Policy class VectorIndexBlock: first_ptr = 0 last_ptr = 0 def __init__(self, fp=0, lp=0): self.first_ptr = fp self.last_ptr = lp def __str__(self): return "FirstPtr: {}, LastPrt: {}".format(self.first_ptr, self.last_ptr) def encode(self) -> bytes: return struct.pack(" bytes: return struct.pack( " # Date : 2022/7/14 17:00 # # ---- # ip2region database v2.0 structure # # +----------------+-------------------+---------------+--------------+ # | header space | speed up index | data payload | block index | # +----------------+-------------------+---------------+--------------+ # | 256 bytes | 512 KiB (fixed) | dynamic size | dynamic size | # +----------------+-------------------+---------------+--------------+ # # 1. padding space : for header info like block index ptr, version, release date eg ... or any other temporary needs. # -- 2bytes: version number, different version means structure update, it fixed to 2 for now # -- 2bytes: index algorithm code. # -- 4bytes: generate unix timestamp (version) # -- 4bytes: index block start ptr # -- 4bytes: index block end ptr # # # 2. data block : region or whatever data info. # 3. segment index block : binary index block. # 4. vector index block : fixed index info for block index search speed up. # space structure table: # -- 0 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block # -- 1 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block # -- 2 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block # -- ... # -- 255 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block # # # super block structure: # +-----------------------+----------------------+ # | first index block ptr | last index block ptr | # +-----------------------+----------------------+ # # data entry structure: # +--------------------+-----------------------+ # | 2bytes (for desc) | dynamic length | # +--------------------+-----------------------+ # data length whatever in bytes # # index entry structure # +------------+-----------+---------------+------------+ # | 4bytes | 4bytes | 2bytes | 4 bytes | # +------------+-----------+---------------+------------+ # start ip end ip data length data ptr import logging import struct import time import sys import xdb.segment as seg import xdb.index as idx import xdb.util as util Version_No = 2 Header_Info_Length = 256 Vector_Index_Rows = 256 Vector_Index_Cols = 256 Vector_Index_Size = 8 Vector_Index_Length = Vector_Index_Rows * Vector_Index_Cols * Vector_Index_Size class Maker: src_handle = None dst_handle = None index_policy = idx.Vector_Index_Policy segments = None region_pool = None vector_index = None def __init__(self, sh, dh, ip, sg, rp, vi): self.src_handle = sh self.dst_handle = dh self.index_policy = ip self.segments = sg self.region_pool = rp self.vector_index = vi def init(self): """ Init the `xdb` binary file. 1. Init the file header 2. Load all the segments """ self.init_db_header() self.load_segments() def init_db_header(self): """ Init and write the file header to the destination xdb file. """ logging.info("try to init the db header ... ") self.src_handle.seek(0, 0) # Make and write the header space header = bytearray([0] * 256) # 1. Version number header[0:2] = Version_No.to_bytes(2, byteorder="little") # 2. Index policy code header[2:4] = int(self.index_policy).to_bytes(2, byteorder="little") # 3. Generate unix timestamp header[4:8] = int(time.time()).to_bytes(4, byteorder="little") # 4. Index block start ptr header[8:12] = int(0).to_bytes(4, byteorder="little") # 5. Index block end ptr header[12:16] = int(0).to_bytes(4, byteorder="little") # Write header buffer to file self.dst_handle.write(header) def load_segments(self): """ Load the segments [start ip|end ip|region] from source ip text file. :return: the list of Segment """ logging.info("try to load the segments ... ") last = None s_tm = time.time() lines = self.src_handle.read().splitlines() for line in lines: logging.info("load segment: `{}`".format(line)) ps = line.split("|", maxsplit=2) if len(ps) != 3: raise Exception("invalid ip segment line `{}`".format(line)) sip = util.check_ip(ps[0]) if sip == -1: raise Exception( "invalid ip address `{}` in line `{}`".format(ps[0], line) ) eip = util.check_ip(ps[1]) if eip == -1: raise Exception( "invalid ip address `{}` in line `{}`".format(ps[1], line) ) if sip > eip: raise Exception( "start ip({}) should not be greater than end ip({})".format( ps[0], ps[1] ) ) if len(ps[2]) < 1: raise Exception("empty region info in segment line `{}`".format(line)) segment = seg.Segment(sip=sip, eip=eip, reg=ps[2]) # Check the continuity of data segment if last is not None: if last.end_ip + 1 != segment.start_ip: raise Exception( "discontinuous data segment: last.eip+1({})!=seg.sip({}, {})".format( sip, eip, ps[0] ) ) self.segments.append(segment) last = segment logging.info( "all segments loaded, length: {}, elapsed: {}".format( len(self.segments), time.time() - s_tm ) ) def set_vector_index(self, ip, ptr): """ Init and refresh the vector index based on the IP pre-two bytes. """ row, col = (ip >> 24) & 0xFF, (ip >> 16) & 0xFF vi_block = self.vector_index[row][col] if vi_block.first_ptr == 0: vi_block.first_ptr = ptr vi_block.last_ptr = ptr + idx.Segment_Index_Block_Size else: vi_block.last_ptr = ptr + idx.Segment_Index_Block_Size self.vector_index[row][col] = vi_block def start(self): """ Start to make the 'xdb' binary file. """ if len(self.segments) < 1: raise Exception("empty segment list") # 1. Write all the region/data to the binary file self.dst_handle.seek(Header_Info_Length + Vector_Index_Length, 0) logging.info("try to write the data block ... ") for s in self.segments: logging.info("try to write region '{}'...".format(s.region)) if s.region in self.region_pool: logging.info( " --[Cached] with ptr={}".format(self.region_pool[s.region]) ) continue region = bytes(s.region, encoding="utf-8") if len(region) > 0xFFFF: raise Exception( "too long region info `{}`: should be less than {} bytes".format( s.region, 0xFFFF ) ) # Get the first ptr of the next region pos = self.dst_handle.seek(0, 1) logging.info("{} {} {}".format(pos, region, s.region)) self.dst_handle.write(region) self.region_pool[s.region] = pos logging.info(" --[Added] with ptr={}".format(pos)) # 2. Write the index block and cache the super index block logging.info("try to write the segment index block ... ") counter, start_index_ptr, end_index_ptr = 0, -1, -1 for sg in self.segments: if sg.region not in self.region_pool: raise Exception("missing ptr cache for region `{}`".format(sg.region)) data_len = len(bytes(sg.region, encoding="utf-8")) if data_len < 1: raise Exception("empty region info for segment '{}'".format(sg.region)) seg_list = sg.split() logging.info( "try to index segment({} split) {} ...".format(len(seg_list), sg) ) for s in seg_list: pos = self.dst_handle.seek(0, 1) s_index = idx.SegmentIndexBlock( sip=s.start_ip, eip=s.end_ip, dl=data_len, dp=self.region_pool[sg.region], ) self.dst_handle.write(s_index.encode()) logging.info( "|-segment index: {}, ptr: {}, segment: {}".format(counter, pos, s) ) self.set_vector_index(s.start_ip, pos) counter += 1 # Check and record the start index ptr if start_index_ptr == -1: start_index_ptr = pos end_index_ptr = pos # 3. Synchronized the vector index block logging.info("try to write the vector index block ... ") self.dst_handle.seek(Header_Info_Length, 0) for i in range(0, len(self.vector_index)): for j in range(0, len(self.vector_index[i])): vi = self.vector_index[i][j] self.dst_handle.write(vi.encode()) # 4. Synchronized the segment index info logging.info("try to write the segment index ptr ... ") buff = struct.pack(" Maker: """ Create a xdb Maker to make the xdb binary file :param policy: index algorithm code 1:vector, 2:b-tree :param srcfile: source ip text file path :param dstfile: destination binary xdb file path :return: the 'xdb' Maker """ try: sh = open(srcfile, mode="r", encoding="utf-8") dh = open(dstfile, mode="wb") return Maker( sh=sh, dh=dh, ip=policy, sg=[], rp={}, vi=[ [idx.VectorIndexBlock() for _ in range(Vector_Index_Rows)] for _ in range(Vector_Index_Cols) ], ) except IOError as e: logging.error(e) sys.exit() ================================================ FILE: maker/python/xdb/segment.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # # Author: linyufeng # Date : 2022/7/14 17:00 # import xdb.util as util class Segment: start_ip = 0 end_ip = 0 region = "" def __init__(self, sip=0, eip=0, reg=""): self.start_ip, self.end_ip = sip, eip self.region = reg def __str__(self): return "{}|{}|{}".format( util.long2ip(self.start_ip), util.long2ip(self.end_ip), self.region ) def split(self) -> list: """ Split the segment based on the pre-two bytes. :return: the list of segment ofter split """ # Example: # split the segment "116.31.76.0|117.21.79.49|region" # # Return the list with segments: # 116.31.76.0 | 116.31.255.255 | region # 116.32.0.0 | 116.32.255.255 | region # ... | ... | region # 116.255.0.0 | 116.255.255.255 | region # 117.0.0.0 | 117.0.255.255 | region # 117.1.0.0 | 117.1.255.255 | region # ... | ... | region # 117.21.0.0 | 117.21.79.49 | region # 1. Split the segment with the first byte t_list_1 = [] s_byte_1, e_byte_1 = (self.start_ip >> 24) & 0xFF, (self.end_ip >> 24) & 0xFF n_sip = self.start_ip for i in range(s_byte_1, e_byte_1 + 1): sip = (i << 24) | (n_sip & 0xFFFFFF) eip = (i << 24) | 0xFFFFFF if eip < self.end_ip: n_sip = (i + 1) << 24 else: eip = self.end_ip # Append the new segment (maybe) t_list_1.append(Segment(sip, eip)) # 2. Split the segments with the second byte t_list_2 = [] for s in t_list_1: base = s.start_ip & 0xFF000000 n_sip = s.start_ip s_byte_2, e_byte_2 = (s.start_ip >> 16) & 0xFF, (s.end_ip >> 16) & 0xFF for i in range(s_byte_2, e_byte_2 + 1): sip = base | (i << 16) | (n_sip & 0xFFFF) eip = base | (i << 16) | 0xFFFF if eip < self.end_ip: n_sip = 0 else: eip = self.end_ip t_list_2.append(Segment(sip, eip, self.region)) return t_list_2 ================================================ FILE: maker/python/xdb/util.py ================================================ # Copyright 2022 The Ip2Region Authors. All rights reserved. # Use of this source code is governed by a Apache2.0-style # license that can be found in the LICENSE file. # # Author: linyufeng # Date : 2022/7/14 17:00 # _SHIFT_INDEX = (24, 16, 8, 0) def check_ip(ip: str) -> int: """ Convert ip string to integer. Return -1 if ip is not the correct ipv4 address. """ if not is_ipv4(ip): return -1 ps = ip.split(".") val = 0 for i in range(len(ps)): d = int(ps[i]) val |= d << _SHIFT_INDEX[i] return val def long2ip(num: int) -> str: """ Convert integer to ip string. Return empty string if the num greater than UINT32_MAX or less than 0. """ if num < 0 or num > 0xFFFFFFFF: return "" return "{}.{}.{}.{}".format( (num >> 24) & 0xFF, (num >> 16) & 0xFF, (num >> 8) & 0xFF, num & 0xFF ) def is_ipv4(ip: str) -> bool: """ Determine whether it is an ipv4 address. """ ps = ip.split(".") if len(ps) != 4: return False for p in ps: if not p.isdigit() or len(p) > 3 or (int(p) < 0 or int(p) > 255): return False return True ================================================ FILE: maker/rust/README.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb rust generation implementation ## Program Compilation ```bash $ cd maker/rust/maker $ cargo build -r ``` After successful compilation, the executable file is located at `./target/release/maker` ## `xdb` Data Generation ```bash # CWD ip2region/maker/rust/maker $ ./target/release/maker --help Usage: maker [OPTIONS] --src --dst --ip-version Options: --src ip source region txt filepath --dst generated xdb filepath --ip-version Possible values: - v4: IPv4 - v6: Ipv6 --index-policy index cache policy [default: vector-index] [possible values: vector-index, b-tree-index] --filter-fields region filter fields, the index of the fields, e.g. `1,2,3,5` -h, --help Print help (see a summary with '-h') ``` For example, use the default raw data under the repository's data/ directory to generate the xdb file to the current directory (`ip2region/maker/rust/maker`): ```bash # ipv6 ./target/release/maker --src=../../../data/ipv6_source.txt --dst=./target/ipv6.xdb --ip-version v6 # ipv4 ./target/release/maker --src=../../../data/ipv4_source.txt --dst=./target/ipv4.xdb --ip-version v4 ``` ## `xdb` Data Search and bench Test For search functions and testing based on the xdb format, see [ip2region bindings](../../binding) ## Comparison with xdb files generated by other makers It is recommended to use `vbindiff`. The only difference from other files should be the create time information; all other data must be identical. Build xdb using the golang version maker ```bash $ cd maker golang $ make $ ./xdb_maker gen --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb --version=ipv4 $ ./xdb_maker gen --src=../../data/ipv6_source.txt --dst=./ip2region_v6.xdb --version=ipv6 ``` Compare xdb differences ```bash $ cd maker/rust/maker # Generate xdb files using rust maker $ ./target/release/maker --src=../../../data/ipv4_source.txt --dst=./target/ipv4.xdb --ip-version v4 $ ./target/release/maker --src=../../../data/ipv6_source.txt --dst=./target/ipv6.xdb --ip-version v6 # Compare with golang generated files $ vbindiff ./ipv4.xdb ../../golang/ip2region_v4.xdb $ vbindiff ./ipv6.xdb ../../golang/ip2region_v6.xdb ``` ================================================ FILE: maker/rust/README_zh.md ================================================ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) # ip2region xdb rust 生成实现 ## 程序编译 ```bash $ cd maker/rust/maker $ cargo build -r ``` 编译成功以后，执行文件位置 `./target/release/maker` ## `xdb` 数据生成 ```bash # CWD ip2region/maker/rust/maker $ ./target/release/maker --help Usage: maker [OPTIONS] --src --dst --ip-version Options: --src ip source region txt filepath --dst generated xdb filepath --ip-version Possible values: - v4: IPv4 - v6: Ipv6 --index-policy index cache policy [default: vector-index] [possible values: vector-index, b-tree-index] --filter-fields region filter fields, the index of the fields, e.g. `1,2,3,5` -h, --help Print help (see a summary with '-h') ``` 例如，使用默认的仓库 data/ 下默认的原始数据生成生成 xdb 文件到当前目录(`ip2region/maker/rust/maker`)： ```bash # ipv6 ./target/release/maker --src=../../../data/ipv6_source.txt --dst=./target/ipv6.xdb --ip-version v6 # ipv4 ./target/release/maker --src=../../../data/ipv4_source.txt --dst=./target/ipv4.xdb --ip-version v4 ``` ## `xdb` 数据查询 和 bench 测试 基于xdb 格式的查询功能和测试见 [ip2region binding](../../binding) ## 对比其他 maker 生成的 xdb 文件 推荐使用 `vbindiff`, 与其他文件的差异只有 create time 信息上有差异，其他数据都需要是一样的 golang 版本 maker 构建 xdb ```bash $ cd maker golang $ make $ ./xdb_maker gen --src=../../data/ipv4_source.txt --dst=./ip2region_v4.xdb --version=ipv4 $ ./xdb_maker gen --src=../../data/ipv6_source.txt --dst=./ip2region_v6.xdb --version=ipv6 ``` 对比 xdb 差异 ```bash $ cd maker/rust/maker $ ./target/release/maker --src=../../../data/ipv4_source.txt --dst=./target/ipv4.xdb --ip-version v4 $ ./target/release/maker --src=../../../data/ipv6_source.txt --dst=./target/ipv6.xdb --ip-version v6 $ vbindiff ./ipv4.xdb ../../golang/ip2region_v4.xdb $ vbindiff ./ipv6.xdb ../../golang/ip2region_v6.xdb ``` ================================================ FILE: maker/rust/maker/Cargo.toml ================================================ [package] name = "maker" version = "0.2.0" edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] clap = { version = "4.5", features = ["derive"] } thiserror = "2" bytes = "1" num-derive = "0.4.2" num-traits = "0.2.19" chrono = "0.4" itertools = "0.14" tracing = "0.1" tracing-subscriber = "0.3" ================================================ FILE: maker/rust/maker/src/command.rs ================================================ use clap::Parser; use crate::IpVersion; use crate::header::IndexPolicy; #[derive(Parser, Debug)] pub struct Command { /// ip source region txt filepath #[arg(long)] pub src: String, /// generated xdb filepath #[clap(long)] pub dst: String, #[clap(long, value_enum)] pub ip_version: IpVersion, /// index cache policy #[clap(long, value_enum, default_value_t = IndexPolicy::VectorIndex)] pub index_policy: IndexPolicy, /// region filter fields, the index of the fields, e.g. `1,2,3,5` #[clap(long, value_delimiter = ',')] pub filter_fields: Vec, } ================================================ FILE: maker/rust/maker/src/error.rs ================================================ #[derive(Debug, thiserror::Error)] pub enum MakerError { #[error("Io error: {0}")] IoError(#[from] std::io::Error), #[error("Header parse error: {0}")] HeaderParsed(String), #[error("Parse line src ip, dst ip, region failed for line: {0}")] ParseIPRegion(String), #[error("Invalid sip/eip version")] InvalidIPVersion, #[error("Ipaddr parse error: {0}")] IpaddrParseError(#[from] std::net::AddrParseError), #[error("Region filter fields value too big, limit: {limit}, actual: {actual}")] RegionFilterFieldsTooBig { limit: usize, actual: usize }, #[error("Empty segments")] EmptySegments, #[error("Try from int failed")] TryFromIntError(#[from] std::num::TryFromIntError), #[error("Try from slice failed")] TryFromSliceFailed(#[from] std::array::TryFromSliceError), #[error("Region could not found")] RegionNotFound, } pub type Result = std::result::Result; ================================================ FILE: maker/rust/maker/src/header.rs ================================================ use std::fmt::Display; use std::net::IpAddr; use bytes::{BufMut, Bytes, BytesMut}; use clap::ValueEnum; use itertools::Itertools; use num_derive::FromPrimitive; use num_traits::FromPrimitive; use crate::error::{MakerError, Result}; pub const VERSION_NO: u16 = 3; // since 2025/09/01 (IPv6 supporting) pub const HEADER_INFO_LENGTH: usize = 256; pub const VECTOR_INDEX_COLS: usize = 256; pub const VECTOR_INDEX_ROWS: usize = 256; pub const VECTOR_INDEX_SIZE: usize = 8; pub const VECTOR_INDEX_LENGTH: usize = VECTOR_INDEX_COLS * VECTOR_INDEX_ROWS * VECTOR_INDEX_SIZE; pub const RUNTIME_PTR_SIZE: u16 = 4; pub const REGION_START: u64 = (HEADER_INFO_LENGTH + VECTOR_INDEX_LENGTH) as u64; #[allow(dead_code)] #[derive(Debug)] pub struct Header { version: u16, index_policy: IndexPolicy, create_time: u32, start_index_ptr: u32, end_index_ptr: u32, ip_version: IpVersion, runtime_ptr_bytes: u16, } impl TryFrom<&[u8; 256]> for Header { type Error = MakerError; fn try_from(value: &[u8; 256]) -> Result { if value.len() < 20 { return Err(MakerError::HeaderParsed("Header bytes too short".into())); } let index_policy_value = u16::from_le_bytes([value[2], value[3]]); let ip_version_value = u16::from_le_bytes([value[16], value[17]]); Ok(Header { version: u16::from_le_bytes([value[0], value[1]]), index_policy: IndexPolicy::from_u16(index_policy_value).ok_or_else(|| { MakerError::HeaderParsed(format!( "Header index policy invalid: {index_policy_value}" )) })?, create_time: u32::from_le_bytes([value[4], value[5], value[6], value[7]]), start_index_ptr: u32::from_le_bytes([value[8], value[9], value[10], value[11]]), end_index_ptr: u32::from_le_bytes([value[12], value[13], value[14], value[15]]), ip_version: IpVersion::from_u16(ip_version_value).ok_or_else(|| { MakerError::HeaderParsed(format!("Header ip version invalid: {ip_version_value}")) })?, runtime_ptr_bytes: u16::from_le_bytes([value[18], value[19]]), }) } } impl Header { pub fn new(index_policy: IndexPolicy, ip_version: IpVersion) -> Header { Header { version: VERSION_NO, index_policy, create_time: chrono::Utc::now().timestamp() as u32, start_index_ptr: 0, end_index_ptr: 0, ip_version, runtime_ptr_bytes: RUNTIME_PTR_SIZE, } } pub fn encode_bytes(&self, start_index_ptr: u32, end_index_ptr: u32) -> Bytes { let mut buf = BytesMut::with_capacity(HEADER_INFO_LENGTH); buf.put_u16_le(VERSION_NO); buf.put_u16_le(self.index_policy as u16); buf.put_u32_le(self.create_time); buf.put_u32_le(start_index_ptr); // index block end ptr buf.put_u32_le(end_index_ptr); buf.put_u16_le(self.ip_version as u16); buf.put_u16_le(self.runtime_ptr_bytes); buf.freeze() } } #[derive(FromPrimitive, Debug, Copy, Clone, ValueEnum)] #[repr(u16)] pub enum IndexPolicy { VectorIndex = 1, BTreeIndex = 2, } impl Display for IndexPolicy { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { IndexPolicy::VectorIndex => write!(f, "VectorIndex"), IndexPolicy::BTreeIndex => write!(f, "BTreeIndex"), } } } #[derive(FromPrimitive, Debug, Copy, Clone, ValueEnum, PartialEq)] #[repr(u16)] pub enum IpVersion { /// IPv4 V4 = 4, /// Ipv6 V6 = 6, } impl IpVersion { pub fn ip_bytes_len(&self) -> usize { match &self { IpVersion::V4 => 4, IpVersion::V6 => 16, } } pub fn segment_index_size(&self) -> usize { match &self { IpVersion::V4 => 14, IpVersion::V6 => 38, } } } impl Header { pub fn ip_bytes_len(&self) -> usize { self.ip_version.ip_bytes_len() } pub fn segment_index_size(&self) -> usize { self.ip_version.segment_index_size() } pub fn ip_version(&self) -> &IpVersion { &self.ip_version } } pub trait IPAddrExt { fn ipaddr_bytes(&self) -> Vec; fn encode_ipaddr_bytes(&self) -> Vec; } impl IPAddrExt for IpAddr { fn ipaddr_bytes(&self) -> Vec { match self { IpAddr::V4(addr) => addr.octets().to_vec(), IpAddr::V6(addr) => addr.octets().to_vec(), } } fn encode_ipaddr_bytes(&self) -> Vec { match self { IpAddr::V4(addr) => addr.octets().into_iter().rev().collect_vec(), IpAddr::V6(addr) => addr.octets().to_vec(), } } } ================================================ FILE: maker/rust/maker/src/lib.rs ================================================ mod command; mod error; mod header; mod maker; mod segment; pub use command::Command; pub use error::{MakerError, Result}; pub use header::{ HEADER_INFO_LENGTH, Header, IpVersion, REGION_START, VECTOR_INDEX_COLS, VECTOR_INDEX_LENGTH, VECTOR_INDEX_SIZE, }; pub use maker::Maker; ================================================ FILE: maker/rust/maker/src/main.rs ================================================ use std::time::Instant; use clap::Parser; use maker::{Command, Maker, Result}; use tracing::info; /// Ip2Region database structure /// See https://github.com/lionsoul2014/ip2region/blob/master/maker/golang/xdb/maker.go fn main() -> Result<()> { tracing_subscriber::fmt::init(); let now = Instant::now(); let cmd = Command::parse(); info!(?cmd, "Generate xdb"); let mut maker = Maker::new( cmd.ip_version, cmd.index_policy, &cmd.src, &cmd.dst, cmd.filter_fields, )?; maker.start()?; info!(cost_time=?now.elapsed(), "Make completed"); Ok(()) } ================================================ FILE: maker/rust/maker/src/maker.rs ================================================ use std::collections::HashMap; use std::fs::File; use std::io::{Seek, SeekFrom, Write}; use std::sync::Arc; use bytes::{BufMut, BytesMut}; use itertools::Itertools; use tracing::{info, trace}; use crate::error::{MakerError, Result}; use crate::header::{IPAddrExt, IndexPolicy, IpVersion, VECTOR_INDEX_ROWS}; use crate::segment::Segment; use crate::{HEADER_INFO_LENGTH, Header, REGION_START, VECTOR_INDEX_COLS, VECTOR_INDEX_SIZE}; pub struct Maker { ip_version: IpVersion, dst_file: File, region_pool: HashMap, u32>, vector_index: [[[u8; VECTOR_INDEX_SIZE]; VECTOR_INDEX_ROWS]; VECTOR_INDEX_COLS], segments: Vec, header: Header, } impl Maker { pub fn new( ip_version: IpVersion, index_policy: IndexPolicy, src_filepath: &str, end_filepath: &str, filter_fields: Vec, ) -> Result { let header = Header::new(index_policy, ip_version); let segments = Segment::from_file(src_filepath, ip_version, &filter_fields)?; if segments.is_empty() { return Err(MakerError::EmptySegments); } let mut region_pool = HashMap::with_capacity(segments.len()); let mut dst_file = File::create(end_filepath)?; let mut region_buf = BytesMut::new(); let mut current = u32::try_from(REGION_START)?; for region in segments.iter().map(|s| s.region.clone()).unique() { region_buf.extend_from_slice(region.as_bytes()); let region_len = region.len() as u32; region_pool.insert(region, current); current += region_len; } dst_file.seek(SeekFrom::Start(REGION_START))?; dst_file.write_all(region_buf.as_ref())?; info!("Load region pool successfully"); Ok(Self { ip_version, dst_file, region_pool, vector_index: [[[0; VECTOR_INDEX_SIZE]; VECTOR_INDEX_ROWS]; VECTOR_INDEX_COLS], segments, header, }) } fn set_vector_index(&mut self, ip: &[u8], ptr: u32) -> Result<()> { let (l0, l1) = (ip[0] as usize, ip[1] as usize); let block = &mut self.vector_index[l0][l1]; if block[0..4].eq(&[0; 4]) { block[0..4].copy_from_slice(&ptr.to_le_bytes()); } let end_value = ptr + self.ip_version.segment_index_size() as u32; block[4..].copy_from_slice(&end_value.to_le_bytes()); Ok(()) } pub fn start(&mut self) -> Result<()> { let start_index_ptr = u32::try_from(self.dst_file.stream_position()?)?; let mut segment_count = 0; let mut buf = BytesMut::with_capacity(self.ip_version.segment_index_size() * self.segments.len()); for segment in std::mem::take(&mut self.segments) { let region_ptr = *self .region_pool .get(&segment.region) .ok_or(MakerError::RegionNotFound)?; let region_len = u16::try_from(segment.region.len())?; trace!(?segment, "before segment split"); for seg in segment.split()? { self.set_vector_index( &seg.start_ip.ipaddr_bytes(), start_index_ptr + buf.len() as u32, )?; buf.put_slice(&seg.start_ip.encode_ipaddr_bytes()); buf.put_slice(&seg.end_ip.encode_ipaddr_bytes()); buf.put_u16_le(region_len); buf.put_u32_le(region_ptr); segment_count += 1; } } info!( region_pool_len = self.region_pool.len(), segment_count, "Write segment index buffer" ); self.dst_file .seek(SeekFrom::Start(start_index_ptr as u64))?; self.dst_file.write_all(buf.as_ref())?; info!("Write header buffer"); let header_buf = self.header.encode_bytes( start_index_ptr, start_index_ptr + (buf.len() as u32) - (self.ip_version.segment_index_size() as u32), ); self.dst_file.seek(SeekFrom::Start(0))?; self.dst_file.write_all(header_buf.as_ref())?; info!("Write vector index buffer"); self.dst_file .seek(SeekFrom::Start(HEADER_INFO_LENGTH as u64))?; self.dst_file .write_all(self.vector_index.as_flattened().as_flattened())?; Ok(()) } } ================================================ FILE: maker/rust/maker/src/segment.rs ================================================ use std::fs::File; use std::io::{BufRead, BufReader}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::str::FromStr; use std::sync::Arc; use itertools::Itertools; use tracing::{debug, info, trace}; use crate::IpVersion; use crate::error::{MakerError, Result}; use crate::header::IPAddrExt; pub trait IpPlusEq { fn ip_plus_eq(&self, other: &Self) -> bool; } impl IpPlusEq for IpAddr { fn ip_plus_eq(&self, other: &Self) -> bool { match (self, other) { (IpAddr::V4(start), IpAddr::V4(end)) => Ipv4Addr::from(u32::from(*start) + 1).eq(end), (IpAddr::V6(start), IpAddr::V6(end)) => Ipv6Addr::from(u128::from(*start) + 1).eq(end), _ => false, } } } #[derive(Debug)] pub struct Segment { pub start_ip: IpAddr, pub end_ip: IpAddr, pub region: Arc, } fn region_filter(region: &str, filter_fields: &[usize]) -> Result { if filter_fields.is_empty() { return Ok(region.to_owned()); } let fields = region.split('|').collect::>(); let filtered = filter_fields .iter() .map(|idx| { fields .get(*idx) .ok_or(MakerError::RegionFilterFieldsTooBig { limit: fields.len(), actual: *idx, }) }) .collect::>>()?; Ok(filtered.into_iter().join("|")) } impl Segment { pub fn from_file( src_filepath: &str, ip_version: IpVersion, filter_fields: &[usize], ) -> Result> { info!("Read src file"); let mut last = None; let mut segments = vec![]; let reader = BufReader::new(File::open(src_filepath)?); for original_line in reader.lines() { let original_line = original_line?; let line = original_line.trim(); if line.is_empty() || line.starts_with('#') { continue; } trace!(?line, "Processing line"); let v = line.splitn(3, '|').collect::>(); if v.len() != 3 { return Err(MakerError::ParseIPRegion(line.to_owned())); } let (start_ip, end_ip, region) = (v[0], v[1], v[2]); let (start_ip, end_ip) = if ip_version.eq(&IpVersion::V4) { ( IpAddr::V4(Ipv4Addr::from_str(start_ip)?), IpAddr::V4(Ipv4Addr::from_str(end_ip)?), ) } else { ( IpAddr::V6(Ipv6Addr::from_str(start_ip)?), IpAddr::V6(Ipv6Addr::from_str(end_ip)?), ) }; if start_ip.gt(&end_ip) { return Err(MakerError::ParseIPRegion(line.to_owned())); } let segment = Segment { start_ip, end_ip, region: Arc::new(region_filter(region, filter_fields)?), }; match last.take() { None => { last = Some(segment); } Some(mut l) if segment.region.eq(&l.region) && l.end_ip.ip_plus_eq(&segment.start_ip) => { l.end_ip = segment.end_ip; last = Some(l); } Some(seg) => { segments.push(seg); last = Some(segment); } } } if let Some(last) = last { segments.push(last); } info!(length = segments.len(), "load segments"); Ok(segments) } pub fn split(self) -> Result> { let start_bytes = self.start_ip.ipaddr_bytes(); let end_bytes = self.end_ip.ipaddr_bytes(); let start_byte = u16::from_be_bytes([start_bytes[0], start_bytes[1]]); let end_byte = u16::from_be_bytes([end_bytes[0], end_bytes[1]]); let segments = (start_byte..=end_byte) .map(|index| { let sip = if index == start_byte { self.start_ip } else if self.start_ip.is_ipv4() { IpAddr::from(Ipv4Addr::from((index as u32) << 16)) } else { IpAddr::from(Ipv6Addr::from((index as u128) << 112)) }; let eip = if index == end_byte { self.end_ip } else if self.start_ip.is_ipv4() { let mask = (1 << 16) - 1; let v = (index as u32) << 16; IpAddr::from(Ipv4Addr::from(v | mask)) } else { let mask = (1 << 112) - 1; let v = (index as u128) << 112; IpAddr::from(Ipv6Addr::from(v | mask)) }; trace!(?index, ?sip, ?eip, ?self.region, "in split segment"); Segment { start_ip: sip, end_ip: eip, region: self.region.clone(), } }) .collect_vec(); debug!(?self, length = segments.len(), "Try to index segment"); Ok(segments) } }