[ { "path": ".gitignore", "content": "# Binaries for programs and plugins\n*.exe\n*.exe~\n*.dll\n*.so\n*.dylib\n\n# Test binary, built with `go test -c`\n*.test\n\n# Output of the go coverage tool, specifically when used with LiteIDE\n*.out\n\n# Dependency directories (remove the comment below to include it)\n# vendor/\n.idea" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "\n

\n\"GoDoc\"\n\"GoDoc\"\n\"Go\n

\n\n\n## XDS - eXtensible Data Structure Sets
(第三方可扩展的 Golang 高性能数据结构和数据类型合集)\n\nA third-party extensible collection of high-performance data structures and data types in Go\n\n- [XDS - eXtensible Data Structure
（第三方可扩展的 Go 语言中高性能数据结构和数据类型合集）](#xds---extensible-data-structure-第三方可扩展的-go-语言中高性能数据结构和数据类型合集)\n- [XDS 介绍：（什么是 Xds）](#xds-介绍什么是-xds)\n- [XDS - XMap 概要介绍](#xds---xmap-概要介绍)\n - [为什么要设计 XMap？](#为什么要设计-xmap)\n - [XMap 设计目标是什么？](#xmap-设计目标是什么)\n - [XMap 的技术特点](#xmap-的技术特点)\n - [XMap 性能数据和实现对比](#xmap-性能数据和实现对比)\n - [XMap 与 Go 官方数据结构特点对比：(20% 写入，80% 读场景)](#xmap-与-go-官方数据结构特点对比20-写入80-读场景)\n- [如何使用 XMap？](#如何使用-xmap)\n - [XMap 各类 API 使用案例](#xmap-各类-api-使用案例)\n - [- XMap 使用示例](#--xmap-使用示例)\n- [XMap 内部是如何实现的？](#xmap-内部是如何实现的)\n- [XDS 项目开发者](#xds-项目开发者)\n- [XDS 技术交流](#xds-技术交流)\n\n
\n\n## XDS 介绍：（什么是 Xds）\n\nXDS - eXtensible Data Structure（第三方可扩展的 Go 语言中高性能数据结构和数据类型合集）\n\nXDS 主要是为了解决现有 Go 语言官方内置的各类数据结构性能在高并发场景中不尽如人意的情况而开发，核心主要是依赖于 [XMM](https://github.com/heiyeluren/xmm) 内存管理库基础之上开发，保证了高性能和内存可控。\n\nXDS 集合目前主要包含：\n- XMap - 高性能的类似 map/sync.map 的 Map 型数据结构类型（已开源）\n- XSlice - 高性能类似 slice 的数组型数据结构类型（开发中）\n- XChannel - 高性能的 channel 管道类型结构（调研中）\n- 更多...\n\n
\n\n
\n\n
\n\n## XDS - XMap 概要介绍\n\nXMap 是属于高性能开源 Go 数据结构 Xds 中的 map 数据结构类型的实现，主要是基于高性能内存管理库 [XMM](https://github.com/heiyeluren/xmm) 基础之上进行的开发，主要弥补了 Go 内置 map 的无法并发读写，并且总体读写性能比较差的问题而开发。\n\n
\n\n### 为什么要设计 XMap？\n\n现有 Golang 中的 map 数据结构无法解决并发读写问题，Sync.map 并发性能偏差，针对这个情况，各种高性能底层服务需要一个高性能、大容量、高并发、无 GC 的 Map，所以开发实现 XMap。\n针对我们需求调研了市场上主要的 hashmap 结构，不能满足我们性能和功能要求。\n\n
\n\n### XMap 设计目标是什么？\n\n要求设计一个可以并发读写不会出现 panic，要求并发读写 200w+ OPS/s 的并发 map 结构。\n（写 20%，读 80% 场景；说明：go 自带 map 读写性能在 80w ops/s，大量并发读写下可能 panic；sync.map 写入性能在 100w OPS/s）\n\n
\n\n### XMap 的技术特点\n\n- 绝对高性能的 map 数据结构（map 的 3 倍，sync.map 的 2 倍并发性能）\n- 内部实现机制对比 Go 原生 map/sync.map 技术设计上要更细致，更考虑性能，使用包括开地址法，红黑树等等结构提升性能；\n- 为了节约内存，初始的值比较低，但是依赖于 XMM 高性能内存扩容方式，能够快速进行内存扩容，保证并发写入性能\n- 底层采用 XMM 内存管理，不会受到 Go 系统本身 GC 机制的卡顿影响，保证高性能；\n- 提供 API 更具备扩展性，在一些高性能场景提供更多调用定制设置，并且能够同时支持 map 类型操作和底层 hash 表类型操作，适用场景更多；\n- 其他特性\n\n
\n\n### XMap 性能数据和实现对比\n\nXMap 目前并发读写场景下性能可以达到 200 万 op/s，对比原生 map 单机性能 80 万 op/s，提升了 3 倍 +，对比 Go 官方扩展库 sync.Map 性能有 2 倍的提升。\n\n
\n\n#### XMap 与 Go 官方数据结构特点对比：(20% 写入，80% 读场景)\n\n| map 模块 | 性能数据
| 加锁机制 | 底层数据结构 | 内存机制 |\n|------|------|------|------|------|\n|map | 80w+ read/s
并发读写会 panic | 无 | Hashtable + Array | Go gc |\n|sync.Map | 100w+ op/s | RWMutex | map | Go gc |\n| Xds.XMap | 200w+ op/s | CAS + RWMutex | Hashtable + Array + RBTree | XMM |\n\n
\n
\n\n## 如何使用 XMap？\n\n快速使用：\n\n1. 下载对应包\n\n```shell\ngo get -u github.com/heiyeluren/xds\ngo get -u github.com/heiyeluren/xmm\n```\n\n2. 快速包含调用库：\n\n```go\n//注意：本代码只是伪代码，大家最好看这个使用测试案例更充分一些\n//详细使用案例：https://github.com/heiyeluren/xds/blob/main/example/xmap_test0.go\n//快速使用入门：https://github.com/heiyeluren/xds/blob/main/example/xmap_test1.go\n\nimport (\n xmm \"github.com/heiyeluren/xmm\"\n xds \"github.com/heiyeluren/xds\"\n xmap \"github.com/heiyeluren/xds/xmap\"\n)\n\n// 创建一个 XMM 内存块\nf := &xmm.Factory{}\nmm, err := f.CreateMemory(0.75)\n\n// 构建一个 map[string]string 的 xmap\nm, err := xmap.NewMap(mm, xds.String, xds.String)\n\n// 写入、读取、删除一个元素\nerr = m.Set(\"name\", \"heiyeluren\")\nret, key_exists, err := m.Get(\"name\")\nerr = m.Remove(\"name\")\n\n// 遍历整个map\nm.Each(func(key, val interface{}) error {\n fmt.Printf(\"For each XMap all key:[%s] value:[%s] \\n\", key, val)\n return nil\n})\n\n//...\n\n```\n\n3. 执行对应代码\n\n```shell\ngo run map-test.go\n```\n\n
\n\n### XMap 各类 API 使用案例\n\n#### - [Xmap 快速使用入门](https://github.com/heiyeluren/xds/blob/main/example/xmap_test1.go)\n\n#### - [XMap 详细使用示例](https://github.com/heiyeluren/xds/blob/main/example/xmap_test0.go) - \n\n - 更多案例（期待）\n\n以上代码案例执行输出：\n
\n\n\n
\n\n## XMap 内部是如何实现的？\n\n#### - [《Xds-XMap技术设计与实现》](https://github.com/heiyeluren/xds/blob/main/docs/Xmap-Implement.md) -\n\n
\n\n- 参考：[《Go map 内部实现机制一》](https://www.jianshu.com/p/aa0d4808cbb8) | [《Go map 内部实现二》](https://zhuanlan.zhihu.com/p/406751292) | [《Golang sync.Map 性能及原理分析》](https://blog.csdn.net/u010853261/article/details/103848666)\n- 其他\n\n
\n\n
\n\n
\n
\n
\n\n## XDS 项目开发者\n\n| 项目角色 | 项目成员 |\n| ----------- | ----------- |\n| 项目发起人/负责人 | 黑夜路人 ( @heiyeluren )
老张 ( @Zhang-Jun-tao ) |\n| 项目开发者 | 老张 ( @Zhang-Jun-tao )
黑夜路人 ( @heiyeluren )
Viktor ( @guojun1992 ) |\n\n

\n\n## XDS 技术交流\n\nXDS 还在早期，当然也少不了一些问题和 bug，欢迎大家一起共创，或者直接提交 PR 等等。\n\n\n欢迎加入 XDS 技术交流微信群，要加群，可以先关注添加如下公众号：
\n（如无法看到图片，请直接微信里搜索公众号“黑夜路人技术”，关注发送“加群”字样信息即可 ）\n\n\n\n\n\n
\n
\n" }, { "path": "common.go", "content": "// Copyright (c) 2022 XDS project Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// XDS Project Site: https://github.com/heiyeluren\n// XDS URL: https://github.com/heiyeluren/xds\n//\n\npackage xds\n\nimport (\n\t\"errors\"\n\t\"reflect\"\n\t\"unsafe\"\n\t// \"github.com/heiyeluren/xmm\"\n)\n\n// 定义XDS共用的常量和一些基础函数\n//调用方法：\n/*\n\t//包含\n\timport(\n\t\t\"github.com/heiyeluren/xds\"\n\t)\n\n\t//结构体中使用\n\ttype ConcurrentHashMap struct {\n\t\tkeyKind xds.Kind\n\t\tvalKind xds.Kind\n\t\tdata *ConcurrentRawHashMap\n\t}\n\n\t//一般数据结构中使用\n\tm, err := xds.NewMap(mm, xds.String, xds.Int)\n\n\t//序列化处理 (其他类型到 []byte，两个函数效果一样)\n\tdata, err := xds.Marshal(xds.String, \"heiyeluren\")\n\tdata, err := xds.RawToByte(xds.String, \"heiyeluren\")\n\n\t// 反序列化处理 (从 []byte到原始类型)\n\tstr, err := xds.UnMarshal(xds.String, data)\n\tdata, err := xds.ByteToRaw(xds.String, \"heiyeluren\")\n\n*/\n\n\n\n//========================================================\n//\n// XDS 常量和数据结构定义区\n// XDS constant and data structure definition\n//\n//========================================================\n\n// XDS 中数据类型的定义\n// map[keyKind]valKind == xds.NewMap(mm, xds.keyKind, xds.valKind)\n// make([]typeKind, len) == xds.NewSlice(mm, xds.dataKind)\n\ntype Kind uint\n\nconst (\n\tInvalid Kind = iota\n\tBool\n\tInt\n\tInt8\n\tInt16\n\tInt32\n\tInt64\n\tUint\n\tUint8\n\tUint16\n\tUint32\n\tUint64\n\tUintptr\n\tFloat32\n\tFloat64\n\tComplex64\n\tComplex128\n\tArray\n\tChan\n\tFunc\n\tInterface\n\tMap\n\tPtr\n\tByteSlice\n\tString\n\tStruct\n\tUnsafePointer\n)\n\n// 类型错误\n// type error\nvar InvalidType = errors.New(\"Kind type Error\")\n\n// type ConcurrentHashMap struct {\n// \tkeyKind Kind\n// \tvalKind Kind\n// \tdata *ConcurrentRawHashMap\n// }\n\n\n\n//========================================================\n//\n// XDS 常用函数定义区\n// XDS common function definition area\n//\n//========================================================\n\n// 针对一些Kind数据类型的序列化\n// Serialization for some kind data types\nfunc Marshal(kind Kind, content interface{}) (data []byte, err error) {\n\tswitch kind {\n\tcase String:\n\t\tdata, ok := content.(string)\n\t\tif !ok {\n\t\t\treturn nil, InvalidType\n\t\t}\n\t\tsh := (*reflect.StringHeader)(unsafe.Pointer(&data))\n\t\treturn *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: sh.Data, Len: sh.Len, Cap: sh.Len})), nil\n\tcase ByteSlice:\n\t\tdata, ok := content.([]byte)\n\t\tif !ok {\n\t\t\treturn nil, InvalidType\n\t\t}\n\t\treturn data, nil\n\tcase Int:\n\t\th, ok := content.(int)\n\t\tif !ok {\n\t\t\treturn nil, InvalidType\n\t\t}\n\t\treturn (*[8]byte)(unsafe.Pointer(&h))[:], nil\n\tcase Uintptr:\n\t\th, ok := content.(uintptr)\n\t\tif !ok {\n\t\t\treturn nil, InvalidType\n\t\t}\n\t\treturn (*[8]byte)(unsafe.Pointer(&h))[:], nil\n\t}\n\treturn\n}\n\n// 针对一些Kind数据类型的反序列化\n// Deserialization for some kind data types\nfunc UnMarshal(kind Kind, data []byte) (content interface{}, err error) {\n\tswitch kind {\n\tcase String:\n\t\treturn *(*string)(unsafe.Pointer(&data)), nil\n\tcase Uintptr:\n\t\tsh := (*reflect.SliceHeader)(unsafe.Pointer(&data))\n\t\treturn *(*uintptr)(unsafe.Pointer(sh.Data)), nil\n\tcase Int:\n\t\tsh := (*reflect.SliceHeader)(unsafe.Pointer(&data))\n\t\treturn *(*int)(unsafe.Pointer(sh.Data)), nil\n\tcase ByteSlice:\n\t\treturn data, nil\n\t}\n\treturn\n}\n\n// Marshal 函数的别名\n// Marshal function link name\nfunc RawToByte(kind Kind, content interface{}) (data []byte, err error) {\n\treturn Marshal(kind, content)\n}\n\n// UnMarshal 函数的别名\n// UnMarshal function link name\nfunc ByteToRaw(kind Kind, data []byte) (content interface{}, err error) {\n\treturn UnMarshal(kind, data)\n}\n\n" }, { "path": "docs/README.md", "content": "\n" }, { "path": "docs/Xmap-Implement.md", "content": "\n# Xds - XMap技术设计与实现\n\n## 一、XMap 背景目标\n\n- 名词解释：\n```\nXMap - eXtensible Map Struct（高性能的第三方Map数据结构）\nMap - Golang原生map数据结构\nsync.Map - Golang原生并发map结构\n```\n\n#### XMap设计背景：\n现有Golang中的map数据结构无法解决并发读写问题，Sync.map 并发性能偏差，针对这个情况，为XCache服务需要一个高性能、大容量、高并发、无GC的Map，所以开发实现 XMap。\n针对我们需求调研了市场上主要的 hashmap 结构，不能满足我们性能和功能要求。\n\n
\n\n#### XMap设计目标：\n要求设计一个可以并发读写不会出现panic，要求并发读写 200w+ OPS/s 的并发map结构。\n（写20%，读80%场景；说明：go自带map读写性能在80w ops/s，大量并发读写下可能panic；sync.map 写入性能在 100w OPS/s）\n\n
\n
\n\n## 二、XMap数据结构设计\n
\n\n为了保证读写性能的要求，同时保证内存的友好。我们采用了开链法方式来做为存储结构，如图为基本数据结构：\n
\n\n```go\ntype ConcurrentHashMap struct {\n size uint64 //当前key的数量\n threshold uint64 // 扩容阈值\n initCap uint64 //初始化bulks大小\n \n //off-heap\n bulks *[]uintptr // 保存链表头结点指针的数组\n mm xmm.XMemory // 非堆的对象初始化器\n lock sync.RWMutex \n\n treeSize uint64 // 由链表转为红黑树的阈值\n\n //resize\n sizeCtl int64 // 状态： -1 正在扩容\n\n nextBulks *[]uintptr //正在扩容的目标Bulks \n transferIndex uint64 //扩容的索引\n}\n```\n\n
\n\n核心数据结构示例图：\n\n\n\n- 核心说明：使用key的hash值计算出数组索引，每个数组中保存一个链表或者红黑树头结点，找到所处的索引，利用尾插发追加节点。每个节点中有key、value等数据。\n\n
\n
\n\n## 三、XMap内部实现流程图\n\n#### 3.1、Set 存储\n\n\n
\n\n#### 3.2、Get 查询\n\n
\n\n#### 3.3、Resize 扩容\n\n
\n\n\n\n" }, { "path": "docs/img/README", "content": "\n" }, { "path": "example/README.md", "content": "\n" }, { "path": "example/xmap_test0.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"github.com/spf13/cast\"\n\t\"github.com/heiyeluren/xmm\"\n\t\"github.com/heiyeluren/xds\"\n\t\"github.com/heiyeluren/xds/xmap\"\n)\n\n// TestMap testing\n// -----------------------------------\n// 把Xmap当做普通map来使用\n// 说明：类型不限制，初始化必须设定好数据类型，写入数据必须与这个数据类型一致，类似于 map[KeyType]ValType 必须相互符合\n// -----------------------------------\n/*\n目前支持的类似于 map[keyType][valType] key value 类型如下：\n就是调用：m, err := xds.NewMap(mm, xmap.String, xmap.Int) 后面的两个 keyType 和 valType 类型定义如下：\nconst (\n\tInvalid Kind = iota\n\tBool\n\tInt\n\tInt8\n\tInt16\n\tInt32\n\tInt64\n\tUint\n\tUint8\n\tUint16\n\tUint32\n\tUint64\n\tUintptr\n\tFloat32\n\tFloat64\n\tComplex64\n\tComplex128\n\tArray\n\tChan\n\tFunc\n\tInterface\n\tMap\n\tPtr\n\tByteSlice\n\tString\n\tStruct\n\tUnsafePointer\n)\n*/\nfunc TestMap(mm xmm.XMemory) {\n\n\t// -------------------\n\t// 常规的map操作\n\t// -------------------\n\n\tfmt.Println(\"\\n--[ XMap NewMap() API example]--\")\n\t// 初始化xmap的时候必须制定key和val的数据类型，数据类型是在xmap中定义的\n\t// 构建一个 map[string]int 的xmap\n\tm, err := xmap.NewMap(mm, xds.String, xds.Int)\n\tif err != nil {\n\t\tpanic(\"call NewMap() fail\")\n\t}\n\n\tvar (\n\t\tk11 string = \"id\"\n\t\tv11 int = 9527\n\t)\n\n\t// set的时候需要关注类型是否跟初始化对象的时候一致\n\terr = m.Set(k11, v11)\n\tif err != nil {\n\t\tpanic(\"XMap.Set fail\")\n\t}\n\tfmt.Println(\"XMap.Set key: [\", k11, \"] success\")\n\n\t// get数据不用关心类型，也不用做类型转换\n\tret, exists, err := m.Get(k11)\n\tif err != nil {\n\t\tpanic(\"XMap.Get fail\")\n\t}\n\tfmt.Println(\"XMap.Get key: [\", k11, \"] , value: [\", ret, \"]\")\n\n\t// Remove数据\n\terr = m.Remove(k11)\n\tif err != nil {\n\t\tpanic(\"XMap.Remove fail\")\n\t}\n\tfmt.Println(\"XMap.Remove key: [\", k11, \"] succes\")\n\tret, exists, err = m.Get(k11)\n\tif !exists {\n\t\tfmt.Println(\"XMap.Get key: [\", k11, \"] not found\")\n\t}\n\n\n\t// -------------------\n\t// 调用扩展的Map函数使用方法(可以获得更多定制性和更高性能)\n\t// -------------------\n\n\tfmt.Println(\"\\n--[ XMap NewMapEx() API example]--\")\n\n\n\t// 生成KV数据\n\tvar (\n\t\tk22 = \"name\"\n\t\tv22 = \"heiyeluren\"\n\t)\n\t// 生成一个 map[string]string 数据结构，默认大小256个元素，占用了75%后进行map扩容(这个初始化函数可以获得更好性能，看个人使用场景）\n\tm1, err := xmap.NewMapEx(mm, xds.String, xds.String, uintptr(256), 0.75)\n\t// set数据\n\tm1.Set(k22, v22)\n\t// get数据\n\tret, exists, err = m1.Get(k22)\n\tfmt.Println(\"XMap.Get key: [\", k22, \"] , value: [\", ret, \"]\")\n\n\n\n\t// -------------------\n\t// 遍历所有map数据\n\t// -------------------\n\n\tfmt.Println(\"\\n--[ XMap ForEach all Key ]--\")\n\n\t// 写入数据\n\terr = m.Set(\"k1\", 1)\n\terr = m.Set(\"k2\", 2)\n\terr = m.Set(\"k3\", 3)\n\terr = m.Set(\"k4\", 4)\n\terr = m.Set(\"k5\", 5)\n\n\t//全局变量可以在匿名函数中访问（如果需要使用外部变量，可以像这样）\n\tgi := 1\n\tfmt.Printf(\"for each itam start, gi: [%s] \\n\", gi)\n\n\t// 遍历xmap中所有元素\n\tm.Each(func(key, val interface{}) error {\n\t\t//针对每个KV进行操作，比如打印出来\n\t\tfmt.Printf(\"for each XMap all key:[%s] value:[%s] \\n\", key, val)\n\t\t//外部变量使用操作\n\t\tgi++\n\t\treturn nil\n\t})\n\tfmt.Printf(\"for each itam done, gi: [%s] \\n\", gi)\n\n\t//读取map长度\n\tlen := m.Len()\n\tfmt.Printf(\"\\nMap length(size): [%s] \\n\", len)\n\n}\n\n\n// TestHashMap testing\n// -----------------------------------\n// 把Xmap当做普通hashmap来使用\n// 说明：Key/Value 都必须是 []byte\n// -----------------------------------\nfunc TestHashMap(mm xmm.XMemory) {\n\n\tfmt.Println(\"\\n\\n===== XMap X(eXtensible) Raw Map (HashMap) example ======\\n\")\n\n\thm, err := xmap.NewHashMap(mm)\n\tif err != nil {\n\t\tpanic(\"call NewHashMap() fail\")\n\t}\n\n\tvar (\n\t\tk1 string = \"name\"\n\t\tv1 string = \"heiyeluren\"\n\t\tk2 string = \"id\"\n\t\tv2 uint32 = 9527\n\t)\n\n\t// 新增Key\n\tfmt.Println(\"\\n--[ Raw Map Set Key ]--\")\n\terr = hm.Set([]byte(k1), []byte(v1))\n\tif err != nil {\n\t\tpanic(\"xmap.Set fail\")\n\t}\n\tfmt.Println(\"Xmap.Set key: [\", k1, \"] success\")\n\terr = hm.Set([]byte(k2), []byte(cast.ToString(v2)))\n\tif err != nil {\n\t\tpanic(\"xmap.Set fail\")\n\t}\n\tfmt.Println(\"Xmap.Set key: [\", k2, \"] success\")\n\n\t// 读取Key\n\tfmt.Println(\"\\n--[ Raw Map Get Key ]--\")\n\ts1, exists, err := hm.Get([]byte(k1))\n\tif err != nil {\n\t\tpanic(\"xmap.Get fail\")\n\t}\n\tfmt.Println(\"Xmap.Get key: [\", k1, \"], value: [\", cast.ToString(s1), \"]\")\n\ts2, exists, err := hm.Get([]byte(k2))\n\tif err != nil {\n\t\tpanic(\"xmap.Get fail\")\n\t}\n\tfmt.Println(\"Xmap.Get key: [\", k2, \"], value: [\", cast.ToString(s2), \"]\")\n\n\t// 删除Key\n\tfmt.Println(\"\\n--[ Raw Map Remove Key ]--\")\n\terr = hm.Remove([]byte(k1))\n\tif err != nil {\n\t\tpanic(\"xmap.Remove fail\")\n\t}\n\tfmt.Println(\"Xmap.Remove key: [\", k1, \"]\")\n\ts1, exists, err = hm.Get([]byte(k1))\n\t// fmt.Println(s1, exists, err)\n\tif !exists {\n\t\tfmt.Println(\"Xmap.Get key: [\", k1, \"] Not Found\")\n\t}\n\ts2, exists, err = hm.Get([]byte(k2))\n\tif err != nil {\n\t\tpanic(\"xmap.Get fail\")\n\t}\n\tfmt.Println(\"Xmap.Get key: [\", k2, \"], value: [\", cast.ToString(s2), \"]\")\n\terr = hm.Remove([]byte(k2))\n\tif err != nil {\n\t\tpanic(\"xmap.Remove fail\")\n\t}\n\tfmt.Println(\"Xmap.Remove key: [\", k2, \"]\")\n\ts2, exists, err = hm.Get([]byte(k2))\n\t// fmt.Println(s1, exists, err)\n\tif !exists {\n\t\tfmt.Println(\"Xmap.Get key: [\", k2, \"] Not Found\")\n\t}\n\ts1, exists, err = hm.Get([]byte(k1))\n\t// fmt.Println(s1, exists, err)\n\tif !exists {\n\t\tfmt.Println(\"Xmap.Get key: [\", k1, \"] Not Found\")\n\t}\n\n\t//--------------------\n\t// 遍历RawMap\n\t//--------------------\n\tfmt.Println(\"\\n--[ Raw Map for each all Key ]--\")\n\thm1, err := xmap.NewHashMap(mm)\n\n\t// 写入数据\n\thm1.Set([]byte(\"K1\"), []byte(\"V1\"))\n\thm1.Set([]byte(\"K2\"), []byte(\"V2\"))\n\thm1.Set([]byte(\"K3\"), []byte(\"V3\"))\n\thm1.Set([]byte(\"K4\"), []byte(\"V4\"))\n\n\t//hm1.Each(func(key, val []byte)(error) {\n\t//\tfmt.Println(\"for each raw map key:[\", key, \"], val[\", val, \"]\")\n\t//})\n\n\n\t//全局变量可以在匿名函数中访问（如果需要使用外部变量，可以像这样）\n\tgi := 1\n\tfmt.Printf(\"for each itam start, gi: [%s] \\n\", gi)\n\n\t// 遍历xmap中所有元素\n\thm1.Each(func(key, val []byte) error {\n\t\t//针对每个KV进行操作，比如打印出来\n\t\tfmt.Printf(\"for each XMap all key:[%s] value:[%s] \\n\", key, val)\n\t\t//外部变量使用操作\n\t\tgi++\n\t\treturn nil\n\t})\n\t//读取map长度\n\tlen := hm1.Len()\n\tfmt.Printf(\"\\nMap length(size): [%s] \\n\", len)\n\n}\n\n// xmap测试代码\nfunc main() {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tpanic(\"xmm.CreateConcurrentHashMapMemory fail\")\n\t}\n\tfmt.Println(\"\\n===== XMap X(eXtensible) Map example ======\\n\")\n\n\t// var NotFound = errors.New(\"not found\")\n\n\t// 把Xmap当做普通map来使用\n\tTestMap(mm)\n\n\t// 把Xmap当做普通hashmap来使用\n\tTestHashMap(mm)\n\n\tfmt.Println(\"\\nXmap test case done.\\n\\n\")\n\n}\n" }, { "path": "example/xmap_test1.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"github.com/heiyeluren/xmm\"\n\t\"github.com/heiyeluren/xds\"\n\t\"github.com/heiyeluren/xds/xmap\"\n)\n\n//------------------\n// xmap快速使用案例\n//------------------\n\nfunc main() {\n\n\t//创建XMM内存块\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tpanic(\"error\")\n\t}\n\n\t//用xmap创建一个map，结构类似于 map[string]uint，操作类似于 m: = make(map[string]uint)\n\tm, err := xmap.NewMap(mm, xds.String, xds.Int)\n\tif err != nil {\n\t\tpanic(\"error\")\n\t}\n\t// 调用Set()给xmap写入数据\n\terr = m.Set(\"k1\", 1)\n\terr = m.Set(\"k2\", 2)\n\terr = m.Set(\"k3\", 3)\n\terr = m.Set(\"k4\", 4)\n\terr = m.Set(\"k5\", 5)\n\n\t// 调用Get()读取单个数据\n\tret, exists, err := m.Get(\"k1\")\n\tif exists == false {\n\t\tpanic(\"key not found\")\n\t}\n\tfmt.Printf(\"Get data key:[%s] value:[%s] \\n\", \"k1\", ret)\n\n\t// 使用Each()访问所有xmap元素\n\t// 在遍历操作中，让全局变量可以在匿名函数中访问（如果需要使用外部变量，可以像这样）\n\tgi := 1\n\t//调用 Each 遍历函数\n\tm.Each(func(key, val interface{}) error {\n\t\t//针对每个KV进行操作，比如打印出来\n\t\tfmt.Printf(\"For each XMap all key:[%s] value:[%s] \\n\", key, val)\n\n\t\t//外部变量使用操作\n\t\tgi++\n\t\treturn nil\n\t})\n\tfmt.Printf(\"For each success, var[gi], val[%s]\\n\", gi);\n\n\t//使用Len()获取xmap元素总数量\n\tlen := m.Len()\n\tfmt.Printf(\"Xmap item size:[%s]\\n\", len);\n\n}\n" }, { "path": "example/xslice_test0.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t//\t\"github.com/heiyeluren/xmm\"\n\t\"github.com/heiyeluren/xds\"\n\t\"github.com/heiyeluren/xds/xslice\"\n)\n\n//---------------------\n// xslice快速使用案例\n//---------------------\n\nfunc main() {\n \n fmt.Println(\"\\n--[ Xslice API example]--\\n\")\n\n\n\t//创建XSlice对象, 类似于 s0 []Int64 = make([]Int64, 1)\n\ts0 := xslice.NewXslice(xds.Int,1)\n\t\n\tfmt.Println(\"\\n### Xslice - Set/Get OP ###\\n\")\n\n\t//Set压入数据, 类似于 s0 :=[] int { 11, 22, 33 }\n\ts0.Set(1, 11)\n\ts0.Set(2, 22)\n\ts0.Set(3, 33)\n\n\t//Get读取一个数据\n\tvi,nil := s0.Get(1)\n\tfmt.Println(\"XSlice get data 1: \", vi)\n\tvi,nil = s0.Get(2)\n\tfmt.Println(\"XSlice get data 2: \", vi)\n\tvi,nil = s0.Get(3)\n\tfmt.Println(\"XSlice get data 3: \", vi)\n\n\t//读取整个slice长度, 类似于 len()\n\tfmt.Println(\"XSlice data size: \", s0.Len(), \"\\n\")\n\n\t//释放资源\n\ts0.Free()\n\t\n\tfmt.Println(\"\\n### Xslice - Append/ForEach OP ###\\n\")\n\n\t//批量压入Int ( 类似于 s1 []Int64 = make([]Int64, 1) )\n\ts1 := xslice.NewXslice(xds.Int, 1)\n\tfor i:=50; i<=55; i++ {\n\t\ts1.Append(i)\n\t}\n\n\t//fmt.Println(s1.Get(2))\n\n\t//使用ForEach读取所有数据\n\ts1.ForEach(func(i int, v interface{}) error {\n\t\tfmt.Println(\"XSlice foreach data i: \",i, \" v: \", v)\n\t\treturn nil\n\t})\n\t\n\tfmt.Println(\"\\n### Xslice - Append/ForEach Data struct ###\\n\")\n\tfmt.Println(s1, \"\\n\")\n\n\t////读取整个slice长度, 类似于 len()\n\tfmt.Println(\"XSlice data size: \", s1.Len(), \"\\n\")\n}\n" }, { "path": "go.mod", "content": "module github.com/heiyeluren/xds\n\ngo 1.12\n\nrequire (\n\tgithub.com/heiyeluren/xmm v0.2.7\n\tgithub.com/spf13/cast v1.4.1\t\n)\n" }, { "path": "go.sum", "content": "github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=\ngithub.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=\ngithub.com/heiyeluren/xmm v0.2.7 h1:jC3LX4bd7VezczzCJwDLHFQTUAfW9uVNzQZL0jNQNKk=\ngithub.com/heiyeluren/xmm v0.2.7/go.mod h1:l/H95AVDlcr0eGIbyGb7T7RVUPTKrGGK5knl75nsWk8=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=\ngithub.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=\ngithub.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=\ngithub.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=\ngithub.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=\n" }, { "path": "xmap/README", "content": "\n" }, { "path": "xmap/concurrent_hash_map_benchmark_test.go", "content": "// Copyright (c) 2022 XDS project Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// XDS Project Site: https://github.com/heiyeluren\n// XDS URL: https://github.com/heiyeluren/xds\n//\n\npackage xmap\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\t\"unsafe\"\n\t// \"xds/xmap/entry\"\n\t// \"github.com/spf13/cast\"\n\t\"github.com/heiyeluren/xmm\"\n\t\"github.com/heiyeluren/xds/xmap/entry\"\n)\n\n// 1000000000\nfunc BenchmarkCHM_Put(b *testing.B) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tchm, err := NewConcurrentRawHashMap(mm, 16, 2, 8)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tkeys := make([]string, 10000000)\n\tfor i := 0; i < 10000000; i++ {\n\t\tkeys[i] = strconv.Itoa(rand.Int())\n\t}\n\tlength := len(keys)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tkey := []byte(keys[rand.Int()%length])\n\t\tif err := chm.Put(key, key); err != nil {\n\t\t\tb.Error(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkCHM_Get(b *testing.B) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tchm, err := NewConcurrentRawHashMap(mm, 16, 2, 8)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tkeys := make([]string, 8000000)\n\tfor i := 0; i < 8000000; i++ {\n\t\tkeys[i] = strconv.Itoa(rand.Int())\n\t}\n\tlength := len(keys)\n\tfor _, key := range keys {\n\n\t\tif err := chm.Put([]byte(key), []byte(key)); err != nil {\n\t\t\tb.Error(err)\n\t\t}\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tkey := keys[rand.Int()%length]\n\t\tif _, exist, err := chm.Get([]byte(key)); err != nil || !exist {\n\t\t\tb.Error(err)\n\t\t}\n\t}\n}\n\nfunc Test_CHM_Concurrent_Get(t *testing.T) {\n\t// Init()\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tchm, err := NewConcurrentRawHashMap(mm, 16, 2, 8)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tkeys := make([]string, 8000000)\n\tfor i := 0; i < 8000000; i++ {\n\t\tkeys[i] = strconv.Itoa(rand.Int() % 800000)\n\t}\n\tfor _, key := range keys {\n\t\tif err := chm.Put([]byte(key), []byte(key)); err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\tvar wait sync.WaitGroup\n\twait.Add(10)\n\tfmt.Println(\"开始\")\n\tt1 := time.Now()\n\tfor j := 0; j < 10; j++ {\n\t\tgo func(z int) {\n\t\t\tdefer wait.Done()\n\t\t\tstart, end := z*800000, (z+1)*800000\n\t\t\tfor _, s := range keys[start:end] {\n\t\t\t\tif err := chm.Put([]byte(s), []byte(s)); err != nil {\n\t\t\t\t\tt.Error(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}(j)\n\t}\n\twait.Wait()\n\tfmt.Println(len(keys), time.Now().Sub(t1), len(keys))\n\t<-time.After(time.Minute)\n}\n\nfunc TestGoCreateEntry(t *testing.T) {\n\tvar wait sync.WaitGroup\n\twait.Add(10)\n\tnode := &entry.NodeEntry{}\n\t// nodes := make([]*entry.NodeEntry, 8000000)\n\ttt := time.Now()\n\tfor j := 0; j < 10; j++ {\n\t\tgo func(z int) {\n\t\t\tdefer wait.Done()\n\t\t\tfor i := 0; i < 800000; i++ {\n\t\t\t\tnode.Key = []byte(\"keyPtr\")\n\t\t\t\tnode.Value = []byte(\"valPtr\")\n\t\t\t\tnode.Hash = 12121\n\t\t\t\t// nodes[z*800000+i] = node\n\t\t\t}\n\t\t}(j)\n\t}\n\twait.Wait()\n\tfmt.Println(time.Now().Sub(tt))\n}\n\nfunc TestCreateEntry(t *testing.T) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tentryPtr, err := mm.Alloc(_NodeEntrySize)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t/* 80445440 \\ 283598848 \\ 228630528 \\ 267530240 \\97157120 \\ 129908736\n\t keyPtr, valPtr, err := mm.From2(\"hjjhj\", \"jjjshjfhsdf\")\n\t if err != nil {\n\t \tt.Fatal(err)\n\t }*/\n\tpageNum := float64(uintptr(entryPtr)) / 4096.0\n\tfmt.Println(uintptr(entryPtr), pageNum, (pageNum+1)*4096 > float64(uintptr(entryPtr)+_NodeEntrySize))\n\tnode := (*entry.NodeEntry)(entryPtr)\n\ttt := time.Now()\n\tvar wait sync.WaitGroup\n\twait.Add(10)\n\tfor j := 0; j < 10; j++ {\n\t\tgo func(z int) {\n\t\t\tdefer wait.Done()\n\t\t\tfor i := 0; i < 8000000; i++ {\n\t\t\t\tnode.Key = []byte(\"keyPtr\")\n\t\t\t\tnode.Value = []byte(\"valPtr\")\n\t\t\t\tnode.Hash = 12121\n\t\t\t}\n\t\t}(j)\n\t}\n\twait.Wait()\n\tfmt.Println(time.Now().Sub(tt))\n}\n\n// todo 优秀一些，使用这种方式\nfunc TestFieldCopy(t *testing.T) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvar wait sync.WaitGroup\n\twait.Add(10)\n\tfor j := 0; j < 10; j++ {\n\t\tgo func(z int) {\n\t\t\tdefer wait.Done()\n\t\t\tfor i := 0; i < 800000; i++ {\n\t\t\t\tentryPtr, err := mm.Alloc(_NodeEntrySize)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t\tkeyPtr, valPtr, err := mm.Copy2([]byte(\"hjjhj\"), []byte(\"jjjshjfhsdf\"))\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t\tsource := entry.NodeEntry{Value: keyPtr, Key: valPtr, Hash: 12312}\n\t\t\t\toffset := unsafe.Offsetof(source.Next) // 40\n\t\t\t\tsrcData := (*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&source)), Len: int(offset), Cap: int(offset)}))\n\t\t\t\tdstData := (*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(entryPtr), Len: int(offset), Cap: int(offset)}))\n\t\t\t\tcopy(*dstData, *srcData)\n\t\t\t}\n\t\t}(j)\n\t}\n\twait.Wait()\n}\n" }, { "path": "xmap/concurrent_hash_map_test.go", "content": "// Copyright (c) 2022 XDS project Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// XDS Project Site: https://github.com/heiyeluren\n// XDS URL: https://github.com/heiyeluren/xds\n//\n\npackage xmap\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\t\"unsafe\"\n\t// \"xds/xmap/entry\"\n\t\"github.com/heiyeluren/xds/xmap/entry\"\n\t\"github.com/heiyeluren/xmm\"\n\t\"github.com/spf13/cast\"\n)\n\nfunc TestMap(t *testing.T) {\n\tt1 := time.Now()\n\tvar data sync.Map\n\tvar wait sync.WaitGroup\n\twait.Add(10)\n\tfor h := 0; h < 10; h++ {\n\t\tgo func() {\n\t\t\tdefer wait.Done()\n\t\t\tfor i := 0; i < 1000000; i++ {\n\t\t\t\tkey := cast.ToString(i)\n\t\t\t\tdata.Store(key, key)\n\t\t\t}\n\t\t}()\n\t}\n\twait.Wait()\n\tfmt.Println(time.Now().Sub(t1))\n}\n\nfunc TestConcurrentRawHashMap_Performance(t *testing.T) {\n\tInit()\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.6)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tchm, err := NewConcurrentRawHashMap(mm, 16, 0.75, 8)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfmt.Println(\"http://localhost:6060/debug/pprof/profile\")\n\t// http://localhost:6060/debug/pprof/profile\n\tfor i := 0; i < 1000000000; i++ {\n\t\tkey := cast.ToString(i)\n\t\tif err := chm.Put([]byte(key), []byte(key)); err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n}\n\nfunc TestConcurrentRawHashMap_Function_Second(t *testing.T) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tchm, err := NewConcurrentRawHashMap(mm, 16, 4, 8)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar wait sync.WaitGroup\n\twait.Add(10)\n\tkeys := make(chan string, 1000000)\n\tquit := make(chan bool, 1)\n\tgo func() {\n\t\t<-time.After(time.Second * 22)\n\t\tt2 := time.Now()\n\t\tfor {\n\t\t\tkey, ok := <-keys\n\t\t\tif !ok {\n\t\t\t\tfmt.Println(\"read time:\", time.Now().Sub(t2))\n\t\t\t\tquit <- true\n\t\t\t\treturn\n\t\t\t}\n\t\t\tval, exist, err := chm.Get([]byte(cast.ToString(key)))\n\t\t\tif bytes.Compare(val, []byte(cast.ToString(key))) != 0 || !exist {\n\t\t\t\tt.Error(err, key)\n\t\t\t}\n\t\t}\n\t}()\n\tt1 := time.Now()\n\tfor j := 0; j < 10; j++ {\n\t\tgo func(z int) {\n\t\t\tdefer wait.Done()\n\t\t\tfor i := 0; i < 100000; i++ {\n\t\t\t\tkey := cast.ToString(i + (z * 10000))\n\t\t\t\tif err := chm.Put([]byte(key), []byte(key)); err != nil {\n\t\t\t\t\tt.Error(err)\n\t\t\t\t} else {\n\t\t\t\t\tkeys <- key\n\t\t\t\t}\n\t\t\t}\n\t\t}(j)\n\t}\n\twait.Wait()\n\tfmt.Println(len(keys), time.Now().Sub(t1), len(keys))\n\tclose(keys)\n\tf.PrintStatus()\n\n\t<-quit\n\t<-time.After(time.Second * 10)\n}\n\nfunc TestConcurrentRawHashMap_Function1(t *testing.T) {\n\tInit()\n\t// runtime.GOMAXPROCS(16)\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tchm, err := NewConcurrentRawHashMap(mm, 16, 4, 8)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar wait sync.WaitGroup\n\twait.Add(10)\n\tkeys := make(chan string, 8000000)\n\tquit := make(chan bool, 1)\n\tgo func() {\n\t\t<-time.After(time.Second * 10)\n\t\tt2 := time.Now()\n\t\tfor {\n\t\t\tkey, ok := <-keys\n\t\t\tif !ok {\n\t\t\t\tfmt.Println(\"read time:\", time.Now().Sub(t2))\n\t\t\t\tquit <- true\n\t\t\t\treturn\n\t\t\t}\n\t\t\tval, exist, err := chm.Get([]byte(key))\n\t\t\tif bytes.Compare(val, []byte(key)) != 0 || !exist {\n\t\t\t\tt.Fatal(err, key)\n\t\t\t}\n\t\t}\n\t}()\n\tt1 := time.Now()\n\tfor j := 0; j < 10; j++ {\n\t\tgo func(z int) {\n\t\t\tdefer wait.Done()\n\t\t\tfor i := 0; i < 800000; i++ {\n\t\t\t\tkey := cast.ToString(i + (z * 10000))\n\t\t\t\tif err := chm.Put([]byte(key), []byte(key)); err != nil {\n\t\t\t\t\tt.Error(err)\n\t\t\t\t} else {\n\t\t\t\t\tkeys <- key\n\t\t\t\t}\n\t\t\t}\n\t\t}(j)\n\t}\n\twait.Wait()\n\tfmt.Println(len(keys), time.Now().Sub(t1), len(keys))\n\tclose(keys)\n\t<-quit\n}\n\nfunc TestMMM(t *testing.T) {\n\t// /usr/local/go/src/runtime\n\tfilepath.Walk(\"/usr/local/go/src/runtime\", func(path string, info os.FileInfo, err error) error {\n\t\tif info.IsDir() {\n\t\t\treturn nil\n\t\t}\n\t\tif path[len(path)-3:] != \".go\" {\n\t\t\treturn nil\n\t\t}\n\t\tbytes, err := ioutil.ReadFile(path)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, s1 := range strings.Split(string(bytes), \"\\n\") {\n\t\t\ts1 = strings.TrimSpace(s1)\n\t\t\tif len(s1) < 3 || s1[:len(\"//\")] == \"//\" || strings.Contains(s1, \" = \") || strings.Contains(info.Name(), \"_test.go\") {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tfor _, s2 := range strings.Split(s1, \" \") {\n\t\t\t\ts := strings.TrimSpace(s2)\n\t\t\t\tif len(s) < 1 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif int(s[0]) <= 64 || int(s[0]) >= 91 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfmt.Println( /*path, info.Name(),*/ s, s1)\n\t\t\t}\n\t\t}\n\t\treturn nil\n\t})\n}\n\ntype A struct {\n\tlock sync.RWMutex\n\tage int\n}\n\nfunc TestSizeOf(t *testing.T) {\n\tfmt.Println(unsafe.Sizeof(sync.RWMutex{}), unsafe.Sizeof(A{}))\n\tfmt.Println(unsafe.Sizeof(Bucket2{}))\n}\n\ntype Bucket2 struct {\n\tforwarding bool // 已经迁移完成\n\trwlock sync.RWMutex\n\tindex uint64\n\tnewBuckets *[]uintptr\n\tHead *entry.NodeEntry\n\t/*\n\t\tTree *entry.Tree\n\t\tisTree bool\n\t\tsize uint64\n\t*/\n}\n\nfunc TestMMCopyString(t *testing.T) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar item1, item2 string\n\tfor i := 0; i < 10000000; i++ {\n\t\titem1, item2, err = mm.From2(\"sdsddsds\", \"sdsddsdssdsds\")\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\tfmt.Println(item1, item2)\n}\n\nfunc TestGoCopyString(t *testing.T) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.75)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar item1, item2 string\n\tfor i := 0; i < 10000000; i++ {\n\t\titem1, item2, err = mm.From2(\"sdsddsds\", \"sdsddsdssdsds\")\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\tfmt.Println(item1, item2)\n}\n\ntype KV struct {\n\tK string\n\tV string\n}\n\nfunc TestDataSet(t *testing.T) {\n\tfmt.Println(string(make([]byte, 100)))\n\tnum, maxLen := 10000000, 1000\n\tkvs := make([]*KV, num)\n\tfor i := 0; i < num; i++ {\n\t\tr := rand.New(rand.NewSource(time.Now().UnixNano())).Int()\n\t\tk := RandString(r % maxLen)\n\t\tkvs[i] = &KV{\n\t\t\tK: cast.ToString(r % num),\n\t\t\tV: k,\n\t\t}\n\t}\n\tfmt.Println(RandString(1000))\n}\n\nfunc RandString(len int) string {\n\tr := rand.New(rand.NewSource(time.Now().UnixNano()))\n\tbytes := make([]byte, len)\n\tfor i := 0; i < len; i++ {\n\t\tb := r.Intn(26) + 65\n\t\tbytes[i] = byte(b)\n\t}\n\treturn string(bytes)\n}\n\n/*func TestDecode(t *testing.T) {\n\tnode := entry.NodeEntry{}\n\tsss := node.Int64Encode(212121)\n\tfmt.Println(node.Int64Decode(sss))\n\n}*/\n" }, { "path": "xmap/concurrent_raw_hash_map.go", "content": "// Copyright (c) 2022 XDS project Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// XDS Project Site: https://github.com/heiyeluren\n// XDS URL: https://github.com/heiyeluren/xds\n//\n\npackage xmap\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"github.com/heiyeluren/xds/xmap/entry\"\n\t\"github.com/heiyeluren/xmm\"\n\t\"log\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"unsafe\"\n)\n\n// MinTransferStride is the minimum number of entries to transfer between\n// 1、步长resize\n// 2、sizeCtl增加cap的cas，不允许提前resize。\n// 考虑 数组+ 链表方式\nconst MinTransferStride = 16\nconst maximumCapacity = 1 << 30\n\n// KeyExists 标识key已经存在\nconst KeyExists = true\n\n// KeyNotExists 标识key不存在\nconst KeyNotExists = false\n\nvar NotFound = errors.New(\"not found\")\nvar _BucketSize = unsafe.Sizeof(Bucket{})\nvar _ForwardingBucketSize = unsafe.Sizeof(ForwardingBucket{})\nvar _NodeEntrySize = unsafe.Sizeof(entry.NodeEntry{})\nvar _TreeSize = unsafe.Sizeof(entry.Tree{})\nvar uintPtrSize = uintptr(8)\n\n// ConcurrentRawHashMap is a concurrent hash map with a fixed number of buckets.\n// 清理方式：1、del清除entry(简单)\n// \t\t2、buckets清除旧的（resize结束，并无get读引用【引入重试来解决该问题】）\n// \t\t3、Bucket清除\n// \t\t4、ForwardingBucket清除\n// \t\t5、Tree 清除（spliceEntry2时候）\ntype ConcurrentRawHashMap struct {\n\tsize uint64\n\tthreshold uint64\n\tinitCap uint64\n\t// off-heap\n\tbuckets *[]uintptr\n\tmm xmm.XMemory\n\tlock sync.RWMutex\n\n\ttreeSize uint64\n\n\t// resize\n\tsizeCtl int64 // -1 正在扩容\n\treSizeGen uint64\n\n\tnextBuckets *[]uintptr\n\ttransferIndex uint64\n\n\tdestroyed uint32 // 0:未销毁 1:已经销毁\n\n\tdestroyLock sync.RWMutex\n}\n\n// Bucket is a hash bucket.\ntype Bucket struct {\n\tforwarding bool // 已经迁移完成\n\trwLock sync.RWMutex\n\tindex uint64\n\tnewBuckets *[]uintptr\n\tHead *entry.NodeEntry\n\tTree *entry.Tree\n\tisTree bool\n\tsize uint64\n}\n\n// ForwardingBucket is a hash bucket that has been forwarded to a new table.\ntype ForwardingBucket struct {\n\tforwarding bool // 已经迁移完成\n\trwLock sync.RWMutex\n\tindex uint64\n\tnewBuckets *[]uintptr\n}\n\n// Snapshot 利用快照比对产生\ntype Snapshot struct {\n\tbuckets *[]uintptr\n\tsizeCtl int64 // -1 正在扩容\n\tnextBuckets *[]uintptr\n}\n\n// NewDefaultConcurrentRawHashMap returns a new ConcurrentRawHashMap with the default\n// mm: xmm\nfunc NewDefaultConcurrentRawHashMap(mm xmm.XMemory) (*ConcurrentRawHashMap, error) {\n\treturn NewConcurrentRawHashMap(mm, 16, 0.75, 8)\n}\n\n// NewConcurrentRawHashMap will create a new ConcurrentRawHashMap with the given\n// mm: xmm 内存对象\n// cap:初始化bucket长度 （可以理解为 map 元素预计最大个数~，如果知道这个值可以提前传递）\n// fact:负责因子，当存放的元素超过该百分比，就会触发扩容。\n// treeSize：bucket中的链表长度达到该值后，会转换为红黑树。\nfunc NewConcurrentRawHashMap(mm xmm.XMemory, cap uintptr, fact float64, treeSize uint64) (*ConcurrentRawHashMap, error) {\n\tif cap < 1 {\n\t\treturn nil, errors.New(\"cap < 1\")\n\t}\n\tvar alignCap uintptr\n\tfor i := 1; i < 64 && cap > alignCap; i++ {\n\t\talignCap = 1 << uint(i)\n\t}\n\tcap = alignCap\n\tbucketsPtr, err := mm.AllocSlice(uintPtrSize, cap, cap)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tbuckets := (*[]uintptr)(bucketsPtr)\n\treturn &ConcurrentRawHashMap{buckets: buckets, initCap: uint64(cap), threshold: uint64(float64(cap) * fact),\n\t\tmm: mm, treeSize: treeSize}, nil\n}\n\nfunc (chm *ConcurrentRawHashMap) getBucket(h uint64, tab *[]uintptr) *Bucket {\n\tmask := uint64(cap(*tab) - 1)\n\tidx := h & mask\n\t_, _, bucket := chm.tabAt(tab, idx)\n\tif bucket != nil && bucket.forwarding && chm.transferIndex >= 0 {\n\t\treturn chm.getBucket(h, bucket.newBuckets)\n\t}\n\treturn bucket\n}\n\n// Get Fetch key from hashmap\nfunc (chm *ConcurrentRawHashMap) Get(key []byte) (val []byte, keyExists bool, err error) {\n\th := BKDRHashWithSpread(key)\n\tbucket := chm.getBucket(h, chm.buckets)\n\tif bucket == nil {\n\t\treturn nil, KeyNotExists, NotFound\n\t}\n\tif bucket.isTree {\n\t\texist, value := bucket.Tree.Get(key)\n\t\tif !exist {\n\t\t\treturn nil, KeyNotExists, NotFound\n\t\t}\n\t\treturn value, KeyExists, nil\n\t}\n\tkeySize := len(key)\n\tfor cNode := bucket.Head; cNode != nil; cNode = cNode.Next {\n\t\tif keySize == len(cNode.Key) && bytes.Compare(key, cNode.Key) == 0 {\n\t\t\treturn cNode.Value, KeyExists, nil\n\t\t}\n\t}\n\treturn nil, KeyNotExists, NotFound\n}\nfunc (chm *ConcurrentRawHashMap) initForwardingEntries(newBuckets *[]uintptr, index uint64) (*ForwardingBucket, error) {\n\tentriesPtr, err := chm.mm.Alloc(_ForwardingBucketSize)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tentries := (*ForwardingBucket)(entriesPtr)\n\tchm.assignmentForwardingEntries(newBuckets, entries, index)\n\treturn entries, nil\n}\n\nfunc (chm *ConcurrentRawHashMap) assignmentForwardingEntries(newBuckets *[]uintptr, entries *ForwardingBucket, index uint64) {\n\tentries.newBuckets = newBuckets\n\tentries.index = index\n\tentries.forwarding = true\n}\n\nfunc (chm *ConcurrentRawHashMap) initEntries(entry *entry.NodeEntry, idx uint64) (*Bucket, error) {\n\tptr, err := chm.mm.Alloc(_BucketSize)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tentries := (*Bucket)(ptr)\n\tchm.assignmentEntries(entries, entry, idx)\n\treturn entries, nil\n}\n\nfunc (chm *ConcurrentRawHashMap) assignmentEntries(entries *Bucket, entry *entry.NodeEntry, index uint64) {\n\tentries.index = index\n\tentries.Head = entry\n}\n\nfunc (chm *ConcurrentRawHashMap) getStride(length uint64) (stride uint64) {\n\tcpuNum := uint64(runtime.NumCPU())\n\tif cpuNum > 1 {\n\t\tstride = (length >> 3) / cpuNum\n\t} else {\n\t\tstride = length\n\t}\n\tif stride < MinTransferStride {\n\t\tstride = MinTransferStride\n\t}\n\treturn stride\n}\n\nfunc (chm *ConcurrentRawHashMap) resizeStamp(length uint64) (stride int64) {\n\treturn -10000 - int64(length)\n}\n\nfunc (chm *ConcurrentRawHashMap) helpTransform(entry *entry.NodeEntry, bucket *Bucket, tab *[]uintptr) (currentBucket *Bucket, init bool, currentTab *[]uintptr, err error) {\n\tif bucket != nil && bucket.forwarding {\n\t\tif err := chm.reHashSize(tab); err != nil {\n\t\t\treturn nil, init, nil, err\n\t\t}\n\t\ttabPtr := bucket.newBuckets\n\t\tbucket, swapped, err := chm.getAndInitBucket(entry, tabPtr)\n\t\tif err != nil {\n\t\t\treturn nil, init, nil, err\n\t\t}\n\t\tif swapped {\n\t\t\treturn nil, true, nil, err\n\t\t}\n\t\treturn bucket, init, tabPtr, nil\n\t}\n\treturn bucket, init, tab, nil\n}\n\nfunc (chm *ConcurrentRawHashMap) indexAndInitBucket(entry *entry.NodeEntry) (entries *Bucket, init bool, tabPtr *[]uintptr, err error) {\n\ttabPtr = chm.buckets\n\tbucket, swapped, err := chm.getAndInitBucket(entry, tabPtr)\n\tif err != nil {\n\t\treturn nil, init, nil, err\n\t}\n\tif swapped {\n\t\treturn bucket, true, tabPtr, nil\n\t}\n\tif bucket != nil && bucket.forwarding && chm.transferIndex >= 0 {\n\t\tbucket, swapped, tabPtr, err = chm.helpTransform(entry, bucket, tabPtr)\n\t\tif err != nil {\n\t\t\treturn nil, init, nil, err\n\t\t}\n\t\tif swapped {\n\t\t\treturn bucket, true, tabPtr, nil\n\t\t}\n\t}\n\treturn bucket, init, tabPtr, nil\n}\n\nfunc (chm *ConcurrentRawHashMap) index(h uint64, length int) uint64 {\n\tidx := h & uint64(length-1)\n\treturn idx\n}\n\nfunc (chm *ConcurrentRawHashMap) tabAt(buckets *[]uintptr, idx uint64) (*uintptr, uintptr, *Bucket) {\n\taddr := &((*buckets)[idx])\n\tptr := atomic.LoadUintptr(addr)\n\tif ptr == 0 {\n\t\treturn addr, ptr, nil\n\t}\n\tbucket := (*Bucket)(unsafe.Pointer(ptr))\n\treturn addr, ptr, bucket\n}\n\n// cas 设置bucket\nfunc (chm *ConcurrentRawHashMap) getAndInitBucket(entry *entry.NodeEntry, tabPtr *[]uintptr) (bucket *Bucket, swapped bool, err error) {\n\th := entry.Hash\n\tidx := chm.index(h, cap(*tabPtr))\n\t// retry := 10改小后，出现该问题。\n\taddr, _, bucket := chm.tabAt(tabPtr, idx)\n\tif bucket != nil {\n\t\treturn bucket, false, nil\n\t}\n\tentity, err := chm.initEntries(entry, idx)\n\tif err != nil {\n\t\treturn nil, false, err\n\t}\n\tptr := uintptr(unsafe.Pointer(entity))\n\tatomic.CompareAndSwapUintptr(addr, 0, ptr)\n\tbucket = (*Bucket)(unsafe.Pointer(atomic.LoadUintptr(addr)))\n\treturn bucket, swapped, nil\n}\n\nfunc (chm *ConcurrentRawHashMap) increaseSize() (newSize uint64) {\n\tretry := 30\n\tvar swapped bool\n\tfor !swapped && retry > 0 {\n\t\tretry--\n\t\toldVal := atomic.LoadUint64(&chm.size)\n\t\tnewVal := oldVal + 1\n\t\tswapped = atomic.CompareAndSwapUint64(&chm.size, oldVal, newVal)\n\t\tif swapped {\n\t\t\treturn newVal\n\t\t}\n\t}\n\tchm.lock.Lock()\n\tdefer chm.lock.Unlock()\n\tsize := chm.size + 1\n\tchm.size = size\n\treturn size\n}\n\nfunc (chm *ConcurrentRawHashMap) createEntry(key []byte, val []byte, hash uint64) (*entry.NodeEntry, error) {\n\tentryPtr, err := chm.mm.Alloc(_NodeEntrySize)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnode := (*entry.NodeEntry)(entryPtr)\n\tkeyPtr, valPtr, err := chm.mm.Copy2(key, val)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tchm.entryAssignment(keyPtr, valPtr, hash, node)\n\treturn node, nil\n}\n\nfunc (chm *ConcurrentRawHashMap) entryAssignment(keyPtr []byte, valPtr []byte, hash uint64, entry *entry.NodeEntry) {\n\tentry.Key = keyPtr\n\tentry.Value = valPtr\n\tentry.Hash = hash\n}\n\nfunc (chm *ConcurrentRawHashMap) entryAssignmentCpy(keyPtr []byte, valPtr []byte, hash uint64, nodePtr unsafe.Pointer) error {\n\tsource := entry.NodeEntry{Value: keyPtr, Key: valPtr, Hash: hash}\n\toffset := 40 // unsafe.Offsetof(source.Next) //40\n\tsrcData := (*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&source)), Len: offset, Cap: offset}))\n\tdstData := (*[]byte)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(nodePtr), Len: offset, Cap: offset}))\n\tif offset != copy(*dstData, *srcData) {\n\t\treturn errors.New(\"incorrect copy length\") // 拷贝长度不正确\n\t}\n\treturn nil\n}\n\nfunc (chm *ConcurrentRawHashMap) putVal(key []byte, val []byte, h uint64) (*[]uintptr, error) {\n\tnode, err := chm.createEntry(key, val, h)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tloop := true\n\tvar tabPtr *[]uintptr\n\tfor loop {\n\t\tbucket, init, newTabPtr, err := chm.indexAndInitBucket(node)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ttabPtr = newTabPtr\n\t\tif init {\n\t\t\tbreak\n\t\t}\n\t\tif loop, err = chm.PutBucketValue(bucket, node, tabPtr); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn tabPtr, nil\n}\n\n// Put 将键值对添加到map中\nfunc (chm *ConcurrentRawHashMap) Put(key []byte, val []byte) error {\n\th := BKDRHashWithSpread(key)\n\ttabPtr, err := chm.putVal(key, val, h)\n\tif err != nil {\n\t\treturn err\n\t}\n\tsize := chm.increaseSize()\n\tthreshold := atomic.LoadUint64(&chm.threshold)\n\tif size >= threshold && size < maximumCapacity {\n\t\treturn chm.reHashSize(tabPtr)\n\t}\n\treturn nil\n}\n\n// Del 删除键值对\nfunc (chm *ConcurrentRawHashMap) Del(key []byte) error {\n\th := BKDRHashWithSpread(key)\n\treturn chm.delVal(key, h)\n}\n\nfunc (chm *ConcurrentRawHashMap) delVal(key []byte, h uint64) error {\n\tloop := true\n\tvar tabPtr *[]uintptr\n\ttabPtr = chm.buckets\n\tfor loop {\n\t\tidx := chm.index(h, cap(*tabPtr))\n\t\t_, _, bucket := chm.tabAt(tabPtr, idx)\n\t\tif bucket == nil {\n\t\t\treturn nil\n\t\t}\n\t\tif bucket != nil && bucket.forwarding {\n\t\t\tif err := chm.reHashSize(tabPtr); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ttabPtr = bucket.newBuckets\n\t\t\tidx = chm.index(h, cap(*tabPtr))\n\t\t\t_, _, bucket = chm.tabAt(tabPtr, idx)\n\t\t\tcontinue\n\t\t}\n\t\tvar removeNode *entry.NodeEntry\n\t\tfunc() {\n\t\t\t// 删除bucket中的数目\n\t\t\tbucket.rwLock.Lock()\n\t\t\tdefer bucket.rwLock.Unlock()\n\t\t\t_, _, newBucket := chm.tabAt(tabPtr, chm.index(h, cap(*tabPtr)))\n\t\t\tif newBucket != bucket || (bucket != nil && bucket.forwarding) {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif bucket.isTree {\n\t\t\t\tif node := bucket.Tree.Delete(key); node != nil {\n\t\t\t\t\tremoveNode = node\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tkeySize := len(key)\n\t\t\t\tvar pre *entry.NodeEntry\n\t\t\t\tfor cNode := bucket.Head; cNode != nil; cNode = cNode.Next {\n\t\t\t\t\tif keySize == len(cNode.Key) && bytes.Compare(key, cNode.Key) == 0 {\n\t\t\t\t\t\tremoveNode = cNode\n\t\t\t\t\t\tif pre == nil {\n\t\t\t\t\t\t\tbucket.Head = cNode.Next\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tpre.Next = cNode.Next\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tpre = cNode\n\t\t\t\t}\n\t\t\t}\n\t\t\tloop = false\n\t\t\treturn\n\t\t}()\n\t\tif err := chm.freeEntry(removeNode); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// freeEntry todo 异步free\nfunc (chm *ConcurrentRawHashMap) freeEntry(removeNode *entry.NodeEntry) error {\n\tif removeNode == nil {\n\t\treturn nil\n\t}\n\tif err := chm.mm.Free(uintptr(unsafe.Pointer(removeNode))); err != nil {\n\t\treturn err\n\t}\n\tkeySlice, valSlice := (*reflect.SliceHeader)(unsafe.Pointer(&removeNode.Key)), (*reflect.SliceHeader)(unsafe.Pointer(&removeNode.Value))\n\tif err := chm.mm.Free(keySlice.Data); err != nil {\n\t\treturn err\n\t}\n\tif err := chm.mm.Free(valSlice.Data); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (chm *ConcurrentRawHashMap) growTree(bucket *Bucket) error {\n\tif bucket.isTree || bucket.size < chm.treeSize {\n\t\treturn nil\n\t}\n\ttreePtr, err := chm.mm.Alloc(_TreeSize)\n\tif err != nil {\n\t\treturn err\n\t}\n\tbucket.Tree = (*entry.Tree)(treePtr)\n\tbucket.Tree.SetComparator(entry.BytesAscSort)\n\tfor node := bucket.Head; node != nil; {\n\t\tnext := node.Next\n\t\tif err := bucket.Tree.Put(node); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tnode = next\n\t}\n\tbucket.Head = nil\n\tbucket.isTree = true\n\treturn nil\n}\n\n// PutBucketValue table的可见性问题，并发问题。\nfunc (chm *ConcurrentRawHashMap) PutBucketValue(bucket *Bucket, node *entry.NodeEntry, tab *[]uintptr) (loop bool, err error) {\n\tbucket.rwLock.Lock()\n\tdefer bucket.rwLock.Unlock()\n\tidx := chm.index(node.Hash, cap(*tab))\n\t_, _, newBucket := chm.tabAt(tab, idx)\n\tif newBucket != bucket || newBucket.forwarding {\n\t\treturn true, nil\n\t}\n\tbucket = newBucket\n\t// 树化\n\tif err := chm.growTree(bucket); err != nil {\n\t\treturn false, err\n\t}\n\tif bucket.isTree {\n\t\tif err := bucket.Tree.Put(node); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\treturn false, nil\n\t}\n\tkey, val := node.Key, node.Value\n\tvar last *entry.NodeEntry\n\tfor node := bucket.Head; node != nil; node = node.Next {\n\t\tif len(key) == len(node.Key) && bytes.Compare(node.Key, key) == 0 {\n\t\t\tnode.Value = val\n\t\t\treturn false, nil\n\t\t}\n\t\tif node.Next == nil {\n\t\t\tlast = node\n\t\t}\n\t}\n\tbucket.size += 1\n\t// 加入\n\tif last == nil {\n\t\tbucket.Head = node\n\t\treturn false, nil\n\t}\n\tlast.Next = node\n\treturn false, nil\n}\n\nfunc (chm *ConcurrentRawHashMap) expandCap(tab *[]uintptr) (s *Snapshot, need bool) {\n\told, size, transferIndex, threshold := uint64(cap(*chm.buckets)), atomic.LoadUint64(&chm.size),\n\t\tatomic.LoadUint64(&chm.transferIndex), atomic.LoadUint64(&chm.threshold)\n\tif size < threshold || transferIndex >= old {\n\t\treturn nil, false\n\t}\n\tnextBuckets, buckets, sizeCtl := chm.nextBuckets, chm.buckets, atomic.LoadInt64(&chm.sizeCtl)\n\t// 正在扩容\n\tif sizeCtl < 0 && nextBuckets != nil && cap(*nextBuckets) == int(old)*2 {\n\t\tif sizeCtl >= 0 {\n\t\t\treturn nil, false\n\t\t}\n\t\t// 当前扩容状态正确，开始扩容\n\t\tif unsafe.Pointer(tab) == unsafe.Pointer(buckets) && nextBuckets != nil {\n\t\t\tif atomic.CompareAndSwapInt64(&chm.sizeCtl, sizeCtl, sizeCtl+1) {\n\t\t\t\treturn &Snapshot{buckets: buckets, sizeCtl: sizeCtl, nextBuckets: nextBuckets}, true\n\t\t\t}\n\t\t}\n\t\treturn nil, false\n\t}\n\t// 未开始扩容的判断\n\tif sizeCtl >= 0 && (old<<1) > uint64(sizeCtl) && unsafe.Pointer(tab) == unsafe.Pointer(buckets) {\n\t\tnewSizeCtl := chm.resizeStamp(old) + 2\n\t\tswapped := atomic.CompareAndSwapInt64(&chm.sizeCtl, sizeCtl, newSizeCtl)\n\t\t// 开始扩容\n\t\tif swapped {\n\t\t\tnewCap := old << 1\n\t\t\tbucketsPtr, err := chm.mm.AllocSlice(uintPtrSize, uintptr(newCap), uintptr(newCap))\n\t\t\tif err != nil {\n\t\t\t\tlog.Printf(\"ConcurrentRawHashMap chm.mm.Alloc newCap:%d err:%s \\n\", newCap, err)\n\t\t\t\treturn nil, false\n\t\t\t}\n\t\t\tnextBuckets = (*[]uintptr)(bucketsPtr)\n\t\t\tchm.nextBuckets = nextBuckets\n\t\t\tchm.transferIndex = 0\n\t\t\treturn &Snapshot{buckets: buckets, sizeCtl: sizeCtl, nextBuckets: nextBuckets}, true\n\t\t}\n\t\treturn nil, false\n\t}\n\treturn nil, false\n}\n\nfunc (chm *ConcurrentRawHashMap) reHashSize(tab *[]uintptr) error {\n\t// cas锁\n\tsnapshot, need := chm.expandCap(tab)\n\tif !need {\n\t\treturn nil\n\t}\n\tcurrentCap := uint64(cap(*snapshot.buckets))\n\tbuckets := snapshot.nextBuckets\n\t// 取当前内容oldBuckets，在oldBuckets中利用bucket的lock来避免同时操作。\n\terr := chm.reSizeBuckets(snapshot)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor {\n\t\tsizeCtl := atomic.LoadInt64(&chm.sizeCtl)\n\t\tif sizeCtl >= 0 {\n\t\t\treturn nil\n\t\t}\n\t\tif atomic.CompareAndSwapInt64(&chm.sizeCtl, sizeCtl, sizeCtl-1) {\n\t\t\tif sizeCtl != chm.resizeStamp(currentCap)+2 {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\treturn func() error {\n\t\toldBuckets, tabPtr := chm.buckets, unsafe.Pointer(&chm.buckets)\n\t\tbucketsAddr := (*unsafe.Pointer)(tabPtr)\n\t\told := (*reflect.SliceHeader)(atomic.LoadPointer(bucketsAddr))\n\t\tif old.Cap != int(currentCap) {\n\t\t\treturn nil\n\t\t}\n\t\tif atomic.CompareAndSwapPointer(bucketsAddr, unsafe.Pointer(old), unsafe.Pointer(buckets)) {\n\t\t\t// 更换内容赋值\n\t\t\tchm.nextBuckets = nil\n\t\t\tnewCap := currentCap << 1\n\t\t\tatomic.StoreInt64(&chm.sizeCtl, int64(newCap))\n\t\t\tchm.threshold = chm.threshold << 1\n\t\t\tchm.reSizeGen += 1\n\t\t\tif err := chm.freeBuckets(oldBuckets); err != nil {\n\t\t\t\tlog.Printf(\"freeBuckets err:%s\\n\", err)\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}()\n}\n\n/*\ntodo 异步free 清除内存\nbuckets清除旧的（resize结束，并无get读引用【引入重试来解决该问题】）\n3、Bucket清除\n4、元素清除*/\nfunc (chm *ConcurrentRawHashMap) freeBuckets(buckets *[]uintptr) error {\n\tfor _, ptr := range *buckets {\n\t\tif ptr < 1 {\n\t\t\tcontinue\n\t\t}\n\t\tif err := chm.mm.Free(ptr); err != nil {\n\t\t\tlog.Printf(\"freeBuckets Free element(%d) err:%s\\n\", ptr, err)\n\t\t}\n\t}\n\tif err := chm.mm.Free(uintptr(unsafe.Pointer(buckets))); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (chm *ConcurrentRawHashMap) increaseTransferIndex(cap uint64, stride uint64) (offset uint64, over bool) {\n\tfor {\n\t\ttransferIndex := atomic.LoadUint64(&chm.transferIndex)\n\t\tif transferIndex >= cap {\n\t\t\treturn 0, true\n\t\t}\n\t\tswapped := atomic.CompareAndSwapUint64(&chm.transferIndex, transferIndex, transferIndex+stride)\n\t\tif swapped {\n\t\t\treturn transferIndex, false\n\t\t}\n\t}\n}\n\nfunc (chm *ConcurrentRawHashMap) reSizeBuckets(s *Snapshot) error {\n\tnewBuckets, oldBuckets, currentCap := s.nextBuckets, s.buckets, uint64(cap(*s.buckets))\n\tfor {\n\t\tstride := chm.getStride(currentCap)\n\t\toffset, over := chm.increaseTransferIndex(currentCap, stride)\n\t\tif over {\n\t\t\treturn nil\n\t\t}\n\t\tmaxIndex := offset + stride\n\t\tif maxIndex > currentCap {\n\t\t\tmaxIndex = currentCap\n\t\t}\n\t\tfor index := offset; index < maxIndex; index++ {\n\t\t\tvar err error\n\t\t\tloop := true\n\t\t\tfor loop {\n\t\t\t\taddr := &((*oldBuckets)[index])\n\t\t\t\tentries := (*Bucket)(unsafe.Pointer(atomic.LoadUintptr(addr)))\n\t\t\t\tif entries == nil {\n\t\t\t\t\tforwardingEntries, err := chm.initForwardingEntries(newBuckets, index)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Printf(\"ERROR reSizeBucket initForwardingEntries err:%s\\n\", err)\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tptr := uintptr(unsafe.Pointer(forwardingEntries))\n\t\t\t\t\tif atomic.CompareAndSwapUintptr(addr, 0, ptr) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t} else {\n\t\t\t\t\t\tentries = (*Bucket)(unsafe.Pointer(atomic.LoadUintptr(addr)))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif loop, err = chm.reSizeBucket(entries, s); err != nil {\n\t\t\t\t\tlog.Printf(\"ERROR reSizeBucket oldEntries err:%s\\n\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Chain .链接\ntype Chain struct {\n\tHead *entry.NodeEntry\n\tTail *entry.NodeEntry\n}\n\n// Add 添加节点\nfunc (c *Chain) Add(node *entry.NodeEntry) {\n\tif c.Head == nil {\n\t\tc.Head = node\n\t}\n\tif c.Tail != nil {\n\t\tc.Tail.Next = node\n\t}\n\tc.Tail = node\n}\n\n// GetHead 获取head节点\nfunc (c *Chain) GetHead() *entry.NodeEntry {\n\tif c.Tail != nil {\n\t\tc.Tail.Next = nil\n\t}\n\treturn c.Head\n}\n\nfunc (chm *ConcurrentRawHashMap) reSizeBucket(entries *Bucket, s *Snapshot) (loop bool, err error) {\n\tentries.rwLock.Lock()\n\tdefer entries.rwLock.Unlock()\n\toldBuckets, newBuckets := s.buckets, s.nextBuckets\n\tcurrentCap := uint64(cap(*s.buckets))\n\ttabPtr, nextTabPtr := unsafe.Pointer(s.buckets), unsafe.Pointer(s.nextBuckets)\n\t_, _, currentEntries := chm.tabAt(oldBuckets, entries.index)\n\t// 判断newBuckets为chm.newBuckets,判断newBuckets为chm.newBuckets\n\tif entries.forwarding || tabPtr != unsafe.Pointer(chm.buckets) ||\n\t\tnextTabPtr != unsafe.Pointer(chm.nextBuckets) || entries != currentEntries {\n\t\treturn false, nil\n\t}\n\tif entries != nil && ((!entries.isTree && entries.Head == nil) || (entries.isTree && entries.Tree == nil)) {\n\t\tentries.newBuckets = newBuckets\n\t\tentries.forwarding = true\n\t\treturn false, nil\n\t}\n\tmask := (currentCap << 1) - 1\n\tindex := entries.index\n\tvar oldIndex, newIndex = index, currentCap + index\n\toldChains, newChains, err := chm.spliceEntry2(entries, mask)\n\tif err != nil {\n\t\treturn true, err\n\t}\n\toldBucket, newBucket, err := chm.spliceBucket2(oldChains, newChains)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tif oldBucket != nil {\n\t\toldBucket.index = oldIndex\n\t\t(*newBuckets)[oldIndex] = uintptr(unsafe.Pointer(oldBucket))\n\t}\n\tif newBucket != nil {\n\t\tnewBucket.index = newIndex\n\t\t(*newBuckets)[newIndex] = uintptr(unsafe.Pointer(newBucket))\n\t}\n\tentries.newBuckets = newBuckets\n\tentries.forwarding = true\n\tentries.Head = nil\n\treturn false, nil\n}\n\n// TreeSplice .\nfunc (chm *ConcurrentRawHashMap) TreeSplice(node *entry.NodeEntry, index uint64, mask uint64, oldChains, newChains *Chain) {\n\tif node == nil {\n\t\treturn\n\t}\n\tif node.Hash&mask == index {\n\t\toldChains.Add(node)\n\t} else {\n\t\tnewChains.Add(node)\n\t}\n\tchm.TreeSplice(node.Left(), index, mask, oldChains, newChains)\n\tchm.TreeSplice(node.Right(), index, mask, oldChains, newChains)\n}\n\n// spliceEntry2 将Entry拆分为两个列表\nfunc (chm *ConcurrentRawHashMap) spliceEntry2(entries *Bucket, mask uint64) (old *entry.NodeEntry, new *entry.NodeEntry, err error) {\n\tvar oldHead, oldTail, newHead, newTail *entry.NodeEntry\n\tindex := entries.index\n\tif entries.isTree {\n\t\tvar oldChains, newChains Chain\n\t\tchm.TreeSplice(entries.Tree.GetRoot(), index, mask, &oldChains, &newChains)\n\t\toldHead, newHead = oldChains.GetHead(), newChains.GetHead()\n\t\t// 删除现在的tree内容\n\t\tif err := chm.mm.Free(uintptr(unsafe.Pointer(entries.Tree))); err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t} else {\n\t\tfor nodeEntry := entries.Head; nodeEntry != nil; nodeEntry = nodeEntry.Next {\n\t\t\tidx := nodeEntry.Hash & mask\n\t\t\tif idx == index {\n\t\t\t\tif oldHead == nil {\n\t\t\t\t\toldHead = nodeEntry\n\t\t\t\t} else if oldTail != nil {\n\t\t\t\t\toldTail.Next = nodeEntry\n\t\t\t\t}\n\t\t\t\toldTail = nodeEntry\n\t\t\t} else {\n\t\t\t\tif newHead == nil {\n\t\t\t\t\tnewHead = nodeEntry\n\t\t\t\t} else if newTail != nil {\n\t\t\t\t\tnewTail.Next = nodeEntry\n\t\t\t\t}\n\t\t\t\tnewTail = nodeEntry\n\t\t\t}\n\t\t}\n\t}\n\tif newTail != nil {\n\t\tnewTail.Next = nil\n\t}\n\tif oldTail != nil {\n\t\toldTail.Next = nil\n\t}\n\treturn oldHead, newHead, nil\n}\n\n// 性能更好\nfunc (chm *ConcurrentRawHashMap) spliceBucket2(old *entry.NodeEntry, new *entry.NodeEntry) (oldBucket *Bucket, newBucket *Bucket, err error) {\n\tif old != nil {\n\t\thead, index := old, uint64(0)\n\t\titem, err := chm.initEntries(head, index)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\toldBucket = item\n\t}\n\n\tif new != nil {\n\t\thead, index := new, uint64(0)\n\t\titem, err := chm.initEntries(head, index)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tnewBucket = item\n\t}\n\treturn\n}\n\nfunc (chm *ConcurrentRawHashMap) indexBucket(idx int, tab *[]uintptr) (bucket *Bucket, outIndex bool) {\n\tif idx >= cap(*tab) {\n\t\treturn nil, false\n\t}\n\t_, _, bucket = chm.tabAt(tab, uint64(idx))\n\tif bucket != nil && bucket.forwarding && chm.transferIndex >= 0 {\n\t\treturn chm.indexBucket(idx, bucket.newBuckets)\n\t}\n\treturn bucket, true\n}\n\nfunc (chm *ConcurrentRawHashMap) ForEach(fun func(key, val []byte) error) error {\n\tbucket, exist := chm.indexBucket(0, chm.buckets)\n\tfor i := 1; exist; i++ {\n\t\tif bucket != nil {\n\t\t\tif tree := bucket.Tree; bucket.isTree && tree != nil {\n\t\t\t\tif err := chm.TreeForEach(fun, tree.GetRoot()); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor cNode := bucket.Head; cNode != nil; cNode = cNode.Next {\n\t\t\t\t\tif err := fun(cNode.Key, cNode.Value); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tbucket, exist = chm.indexBucket(i, chm.buckets)\n\t}\n\treturn nil\n}\n\nfunc (chm *ConcurrentRawHashMap) TreeForEach(fun func(key, val []byte) error, node *entry.NodeEntry) error {\n\tif node == nil {\n\t\treturn nil\n\t}\n\tif err := fun(node.Key, node.Value); err != nil {\n\t\treturn err\n\t}\n\tchm.TreeForEach(fun, node.Left())\n\tchm.TreeForEach(fun, node.Right())\n\treturn nil\n}\n\n// BKDRHashWithSpread . 31 131 1313 13131 131313 etc..\nfunc BKDRHashWithSpread(str []byte) uint64 {\n\tseed := uint64(131)\n\thash := uint64(0)\n\tfor i := 0; i < len(str); i++ {\n\t\thash = (hash * seed) + uint64(str[i])\n\t}\n\treturn hash ^ (hash>>16)&0x7FFFFFFF\n}\n" }, { "path": "xmap/entry/rbtree.go", "content": "/*\nCopyright 2014 Gavin Bong.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\nhttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,\neither express or implied. See the License for the specific\nlanguage governing permissions and limitations under the\nLicense.\n*/\n\n// Package entry Package redblacktree provides a pure Golang implementation\n// of a red-black tree as described by Thomas H. Cormen's et al.\n// in their seminal Algorithms book (3rd ed). This data structure\n// is not multi-goroutine safe.\npackage entry\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"log\"\n\t\"os\"\n\t\"reflect\"\n\t\"strings\"\n\t\"sync\"\n)\n\n// NodeEntry interface defines the methods that a node must implement.\ntype NodeEntry struct {\n\tKey []byte\n\tValue []byte\n\tHash uint64\n\tNext *NodeEntry\n\n\tright *NodeEntry\n\tleft *NodeEntry\n\tparent *NodeEntry\n\tcolor Color // 比二叉查找树要多出一个颜色属性\n}\n\nfunc round(n, a uint64) uint64 {\n\treturn (n + a - 1) &^ (a - 1)\n}\n\ntype encodedNodeEntry struct {\n\tTotalLen uint64\n\tKeyLen uint64\n\tValueLen uint64\n\tHash uint64\n\tNext *NodeEntry\n\n\tright *NodeEntry\n\tleft *NodeEntry\n\tparent *NodeEntry\n\tcolor Color // 比二叉查找树要多出一个颜色属性\n\n\t// 追加string key、value 要不然就内存泄露了\n\n}\n\n/*// total'len(64) + key'len(64) + key'content +\nfunc (n *NodeEntry) encode() []byte {\n\tvar keyLen, valLen = round(uint64(len(n.Key)), 8), round(uint64(len(n.Value)), 8)\n\ttotal := keyLen + valLen + 8*8\n\tencoded := encodedNodeEntry{\n\t\tTotalLen: total,\n\t\tKeyLen: keyLen,\n\t\tValueLen: valLen,\n\t\tHash: 0,\n\t\tNext: nil,\n\t\tright: nil,\n\t\tleft: nil,\n\t\tparent: nil,\n\t\tcolor: false,\n\t}\n\n}*/\n\n// BytesAscSort is a helper function that sorts a slice of byte slices\nvar BytesAscSort Comparator = func(o1, o2 interface{}) int {\n\tkey1, key2 := o1.([]byte), o2.([]byte)\n\treturn bytes.Compare(key1, key2)\n}\n\n// Color of a redblack tree node is either\n// `Black` (true) & `Red` (false)\ntype Color bool\n\n// Direction points to either the Left or Right subtree\ntype Direction byte\n\nfunc (c Color) String() string {\n\tswitch c {\n\tcase true:\n\t\treturn \"Black\"\n\tdefault:\n\t\treturn \"Red\"\n\t}\n}\n\nfunc (d Direction) String() string {\n\tswitch d {\n\tcase LEFT:\n\t\treturn \"left\"\n\tcase RIGHT:\n\t\treturn \"right\"\n\tcase NODIR:\n\t\treturn \"center\"\n\tdefault:\n\t\treturn \"not recognized\"\n\t}\n}\n\nconst (\n\tBLACK, RED Color = true, false\n)\nconst (\n\tLEFT Direction = iota\n\tRIGHT\n\tNODIR\n)\n\n// A node needs to be able to answer the query:\n// (i) Who is my parent node ?\n// (ii) Who is my grandparent node ?\n// The zero value for Node has color Red.\n\nfunc (n *NodeEntry) String() string {\n\tif n == nil {\n\t\treturn \"\"\n\t}\n\treturn fmt.Sprintf(\"(%#v : %s)\", n.Key, n.Color())\n}\n\n// Parent node will be nil if the node is the root node.\nfunc (n *NodeEntry) Parent() *NodeEntry {\n\treturn n.parent\n}\n\n// SetColor nodes color to the given color.\nfunc (n *NodeEntry) SetColor(color Color) {\n\tn.color = color\n}\n\n// Color returns the color of the node.\nfunc (n *NodeEntry) Color() Color {\n\treturn n.color\n}\n\n// Left indicates the left child of the node.\nfunc (n *NodeEntry) Left() *NodeEntry {\n\treturn n.left\n}\n\n// Right indicates the right child of the node.\nfunc (n *NodeEntry) Right() *NodeEntry {\n\treturn n.right\n}\n\n// Visitor is a function that is called on each node of the tree.\ntype Visitor interface {\n\tVisit(*NodeEntry)\n}\n\n// Visitable A redblack tree is `Visitable` by a `Visitor`.\ntype Visitable interface {\n\tWalk(Visitor)\n}\n\n// Comparator is a function that compares two objects.\n// Keys must be comparable. It's mandatory to provide a Comparator,\n// which returns zero if o1 == o2, -1 if o1 < o2, 1 if o1 > o2\ntype Comparator func(o1, o2 interface{}) int\n\n// IntComparator is a Comparator that compares ints.\n// Default comparator expects keys to be of type `int`.\n// Warning: if either one of `o1` or `o2` cannot be asserted to `int`, it panics.\nfunc IntComparator(o1, o2 interface{}) int {\n\ti1 := o1.(int)\n\ti2 := o2.(int)\n\tswitch {\n\tcase i1 > i2:\n\t\treturn 1\n\tcase i1 < i2:\n\t\treturn -1\n\tdefault:\n\t\treturn 0\n\t}\n}\n\n// StringComparator ...\n// Keys of type `string`.\n// Warning: if either one of `o1` or `o2` cannot be asserted to `string`, it panics.\nfunc StringComparator(o1, o2 interface{}) int {\n\ts1 := o1.(string)\n\ts2 := o2.(string)\n\treturn bytes.Compare([]byte(s1), []byte(s2))\n}\n\n// Tree encapsulates the data structure.\ntype Tree struct {\n\troot *NodeEntry // tip of the tree\n\tcmp Comparator // required function to order keys\n\tlock sync.RWMutex\n}\n\n// `lock` protects `logger`\nvar loggerlock sync.Mutex\nvar logger *log.Logger\n\nfunc init() {\n\tlogger = log.New(ioutil.Discard, \"\", log.LstdFlags)\n}\n\n// TraceOn turns on logging output to Stderr\nfunc TraceOn() {\n\tSetOutput(os.Stderr)\n}\n\n// TraceOff turns off logging.\n// By default logging is turned off.\nfunc TraceOff() {\n\tSetOutput(ioutil.Discard)\n}\n\n// SetOutput redirects log output\nfunc SetOutput(w io.Writer) {\n\tloggerlock.Lock()\n\tdefer loggerlock.Unlock()\n\tlogger = log.New(w, \"\", log.LstdFlags)\n}\n\n// NewTree returns an empty Tree with default comparator `IntComparator`.\n// `IntComparator` expects keys to be type-assertable to `int`.\nfunc NewTree() *Tree {\n\treturn &Tree{root: nil, cmp: IntComparator}\n}\n\n// NewTreeWith returns an empty Tree with a supplied `Comparator`.\nfunc NewTreeWith(c Comparator) *Tree {\n\treturn &Tree{root: nil, cmp: c}\n}\n\n// SetComparator in the Tree of the supplied `Comparator`.\nfunc (t *Tree) SetComparator(c Comparator) {\n\tt.cmp = c\n}\n\n// Get looks for the node with supplied key and returns its mapped payload.\n// Return value in 1st position indicates whether any payload was found.\nfunc (t *Tree) Get(key []byte) (bool, []byte) {\n\tif err := mustBeValidKey(key); err != nil {\n\t\t// logger.Printf(\"Get was prematurely aborted: %s\\n\", err.Error())\n\t\treturn false, nil\n\t}\n\tok, node := t.getNode(key)\n\tif ok {\n\t\treturn true, node.Value\n\t}\n\treturn false, nil\n}\n\nfunc (t *Tree) getNode(key interface{}) (bool, *NodeEntry) {\n\tfound, parent, dir := t.GetParent(key)\n\tif found {\n\t\tif parent == nil {\n\t\t\treturn true, t.root\n\t\t}\n\t\tvar node *NodeEntry\n\t\tswitch dir {\n\t\tcase LEFT:\n\t\t\tnode = parent.left\n\t\tcase RIGHT:\n\t\t\tnode = parent.right\n\t\t}\n\n\t\tif node != nil {\n\t\t\treturn true, node\n\t\t}\n\t}\n\treturn false, nil\n}\n\n// getMinimum returns the node with minimum key starting\n// at the subtree rooted at node x. Assume x is not nil.\nfunc (t *Tree) getMinimum(x *NodeEntry) *NodeEntry {\n\tfor {\n\t\tif x.left != nil {\n\t\t\tx = x.left\n\t\t} else {\n\t\t\treturn x\n\t\t}\n\t}\n}\n\n// GetParent looks for the node with supplied key and returns the parent node.\nfunc (t *Tree) GetParent(key interface{}) (found bool, parent *NodeEntry, dir Direction) {\n\tif err := mustBeValidKey(key); err != nil {\n\t\t// logger.Printf(\"GetParent was prematurely aborted: %s\\n\", err.Error())\n\t\treturn false, nil, NODIR\n\t}\n\n\tif t.root == nil {\n\t\treturn false, nil, NODIR\n\t}\n\tt.lock.RLock()\n\tdefer t.lock.RUnlock()\n\treturn t.internalLookup(nil, t.root, key, NODIR)\n}\n\nfunc (t *Tree) internalLookup(parent *NodeEntry, this *NodeEntry, key interface{}, dir Direction) (bool, *NodeEntry, Direction) {\n\tswitch {\n\tcase this == nil:\n\t\treturn false, parent, dir\n\tcase t.cmp(key, this.Key) == 0:\n\t\treturn true, parent, dir\n\tcase t.cmp(key, this.Key) < 0:\n\t\treturn t.internalLookup(this, this.left, key, LEFT)\n\tcase t.cmp(key, this.Key) > 0:\n\t\treturn t.internalLookup(this, this.right, key, RIGHT)\n\tdefault:\n\t\treturn false, parent, NODIR\n\t}\n}\n\n// RotateRight Reverses actions of RotateLeft\nfunc (t *Tree) RotateRight(y *NodeEntry) {\n\tif y == nil {\n\t\t// logger.Printf(\"RotateRight: nil arg cannot be rotated. Noop\\n\")\n\t\treturn\n\t}\n\tif y.left == nil {\n\t\t// logger.Printf(\"RotateRight: y has nil left subtree. Noop\\n\")\n\t\treturn\n\t}\n\t// logger.Printf(\"\\t\\t\\trotate right of %s\\n\", y)\n\tx := y.left\n\ty.left = x.right\n\tif x.right != nil {\n\t\tx.right.parent = y\n\t}\n\tx.parent = y.parent\n\tif y.parent == nil {\n\t\tt.root = x\n\t} else {\n\t\tif y == y.parent.left {\n\t\t\ty.parent.left = x\n\t\t} else {\n\t\t\ty.parent.right = x\n\t\t}\n\t}\n\tx.right = y\n\ty.parent = x\n}\n\n// RotateLeft Reverses actions of RotateRight\n// Side-effect: red-black tree properties is maintained.\nfunc (t *Tree) RotateLeft(x *NodeEntry) {\n\tif x == nil {\n\t\t// logger.Printf(\"RotateLeft: nil arg cannot be rotated. Noop\\n\")\n\t\treturn\n\t}\n\tif x.right == nil {\n\t\t// logger.Printf(\"RotateLeft: x has nil right subtree. Noop\\n\")\n\t\treturn\n\t}\n\t// logger.Printf(\"\\t\\t\\trotate left of %s\\n\", x)\n\n\ty := x.right\n\tx.right = y.left\n\tif y.left != nil {\n\t\ty.left.parent = x\n\t}\n\ty.parent = x.parent\n\tif x.parent == nil {\n\t\tt.root = y\n\t} else {\n\t\tif x == x.parent.left {\n\t\t\tx.parent.left = y\n\t\t} else {\n\t\t\tx.parent.right = y\n\t\t}\n\t}\n\ty.left = x\n\tx.parent = y\n}\n\n// Put saves the mapping (key, data) into the tree.\n// If a mapping identified by `key` already exists, it is overwritten.\n// Constraint: Not everything can be a key.\nfunc (t *Tree) Put(node *NodeEntry) error {\n\tnode.parent, node.left, node.Next, node.right = nil, nil, nil, nil\n\tkey, data := node.Key, node.Value\n\tif err := mustBeValidKey(key); err != nil {\n\t\t// logger.Printf(\"Put was prematurely aborted: %s\\n\", err.Error())\n\t\treturn err\n\t}\n\tt.lock.Lock()\n\tdefer t.lock.Unlock()\n\tif t.root == nil {\n\t\tnode.color = BLACK\n\t\tt.root = node\n\t\t// logger.Printf(\"Added %s as root node\\n\", t.root.String())\n\t\treturn nil\n\t}\n\n\tfound, parent, dir := t.internalLookup(nil, t.root, key, NODIR)\n\tif found {\n\t\tif parent == nil {\n\t\t\t// logger.Printf(\"Put: parent=nil & found. Overwrite ROOT node\\n\")\n\t\t\tt.root.Value = data\n\t\t} else {\n\t\t\t// logger.Printf(\"Put: parent!=nil & found. Overwriting\\n\")\n\t\t\tswitch dir {\n\t\t\tcase LEFT:\n\t\t\t\tparent.left.Value = data\n\t\t\tcase RIGHT:\n\t\t\t\tparent.right.Value = data\n\t\t\t}\n\t\t}\n\n\t} else {\n\t\tif parent != nil {\n\t\t\tnode.parent = parent\n\t\t\tnewNode := node\n\t\t\tswitch dir {\n\t\t\tcase LEFT:\n\t\t\t\tparent.left = newNode\n\t\t\tcase RIGHT:\n\t\t\t\tparent.right = newNode\n\t\t\t}\n\t\t\t// logger.Printf(\"Added %s to %s node of parent %s\\n\", newNode.String(), dir, parent.String())\n\t\t\tt.fixupPut(newNode)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc isRed(n *NodeEntry) bool {\n\tkey := reflect.ValueOf(n)\n\tif key.IsNil() {\n\t\treturn false\n\t}\n\treturn n.color == RED\n}\n\n// fix possible violations of red-black-tree properties\n// with combinations of:\n// 1. recoloring\n// 2. rotations\n//\n// Preconditions:\n// P1) z is not nil\n//\n// @param z - the newly added Node to the tree.\nfunc (t *Tree) fixupPut(z *NodeEntry) {\n\t// logger.Printf(\"\\tfixup new node z %s\\n\", z.String())\nloop:\n\tfor {\n\t\t// logger.Printf(\"\\tcurrent z %s\\n\", z.String())\n\t\tswitch {\n\t\tcase z.parent == nil:\n\t\t\tfallthrough\n\t\tcase z.parent.color == BLACK:\n\t\t\tfallthrough\n\t\tdefault:\n\t\t\t// When the loop terminates, it does so because p[z] is black.\n\t\t\t// logger.Printf(\"\\t\\t=> bye\\n\")\n\t\t\tbreak loop\n\t\tcase z.parent.color == RED:\n\t\t\tgrandparent := z.parent.parent\n\t\t\t// logger.Printf(\"\\t\\tgrandparent is nil %t addr:%d\\n\", grandparent == nil, unsafe.Pointer(t))\n\t\t\tif z.parent == grandparent.left {\n\t\t\t\t// logger.Printf(\"\\t\\t%s is the left child of %s\\n\", z.parent, grandparent)\n\t\t\t\ty := grandparent.right\n\t\t\t\t// //logger.Printf(\"\\t\\ty (right) %s\\n\", y)\n\t\t\t\tif isRed(y) {\n\t\t\t\t\t// case 1 - y is RED\n\t\t\t\t\t// logger.Printf(\"\\t\\t(*) case 1\\n\")\n\t\t\t\t\tz.parent.color = BLACK\n\t\t\t\t\ty.color = BLACK\n\t\t\t\t\tgrandparent.color = RED\n\t\t\t\t\tz = grandparent\n\n\t\t\t\t} else {\n\t\t\t\t\tif z == z.parent.right {\n\t\t\t\t\t\t// case 2\n\t\t\t\t\t\t// logger.Printf(\"\\t\\t(*) case 2\\n\")\n\t\t\t\t\t\tz = z.parent\n\t\t\t\t\t\tt.RotateLeft(z)\n\t\t\t\t\t}\n\n\t\t\t\t\t// case 3\n\t\t\t\t\t// logger.Printf(\"\\t\\t(*) case 3\\n\")\n\t\t\t\t\tz.parent.color = BLACK\n\t\t\t\t\tgrandparent.color = RED\n\t\t\t\t\tt.RotateRight(grandparent)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// logger.Printf(\"\\t\\t%s is the right child of %s\\n\", z.parent, grandparent)\n\t\t\t\ty := grandparent.left\n\t\t\t\t// logger.Printf(\"\\t\\ty (left) %s\\n\", y)\n\t\t\t\tif isRed(y) {\n\t\t\t\t\t// case 1 - y is RED\n\t\t\t\t\t// logger.Printf(\"\\t\\t..(*) case 1\\n\")\n\t\t\t\t\tz.parent.color = BLACK\n\t\t\t\t\ty.color = BLACK\n\t\t\t\t\tgrandparent.color = RED\n\t\t\t\t\tz = grandparent\n\n\t\t\t\t} else {\n\t\t\t\t\t// logger.Printf(\"\\t\\t## %s\\n\", z.parent.left)\n\t\t\t\t\tif z == z.parent.left {\n\t\t\t\t\t\t// case 2\n\t\t\t\t\t\t// logger.Printf(\"\\t\\t..(*) case 2\\n\")\n\t\t\t\t\t\tz = z.parent\n\t\t\t\t\t\tt.RotateRight(z)\n\t\t\t\t\t}\n\n\t\t\t\t\t// case 3\n\t\t\t\t\t// logger.Printf(\"\\t\\t..(*) case 3\\n\")\n\t\t\t\t\tz.parent.color = BLACK\n\t\t\t\t\tgrandparent.color = RED\n\t\t\t\t\tt.RotateLeft(grandparent)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tt.root.color = BLACK\n}\n\n// Size returns the number of items in the tree.\nfunc (t *Tree) Size() uint64 {\n\tvisitor := &countingVisitor{}\n\tt.Walk(visitor)\n\treturn visitor.Count\n}\n\n// Has checks for existence of a item identified by supplied key.\nfunc (t *Tree) Has(key interface{}) bool {\n\tif err := mustBeValidKey(key); err != nil {\n\t\t// logger.Printf(\"Has was prematurely aborted: %s\\n\", err.Error())\n\t\treturn false\n\t}\n\tfound, _, _ := t.internalLookup(nil, t.root, key, NODIR)\n\treturn found\n}\n\nfunc (t *Tree) transplant(u *NodeEntry, v *NodeEntry) {\n\tif u.parent == nil {\n\t\tt.root = v\n\t} else if u == u.parent.left {\n\t\tu.parent.left = v\n\t} else {\n\t\tu.parent.right = v\n\t}\n\tif v != nil && u != nil {\n\t\tv.parent = u.parent\n\t}\n}\n\n// Delete removes the item identified by the supplied key.\n// Delete is a noop if the supplied key doesn't exist.\nfunc (t *Tree) Delete(key []byte) *NodeEntry {\n\tif !t.Has(key) {\n\t\t// logger.Printf(\"Delete: bail as no node exists for key %d\\n\", key)\n\t\treturn nil\n\t}\n\t_, z := t.getNode(key)\n\ty := z\n\tyOriginalColor := y.color\n\tvar x *NodeEntry\n\n\tif z.left == nil {\n\t\t// one child (RIGHT)\n\t\t// logger.Printf(\"\\t\\tDelete: case (a)\\n\")\n\t\tx = z.right\n\t\t// logger.Printf(\"\\t\\t\\t--- x is right of z\")\n\t\tt.transplant(z, z.right)\n\n\t} else if z.right == nil {\n\t\t// one child (LEFT)\n\t\t// logger.Printf(\"\\t\\tDelete: case (b)\\n\")\n\t\tx = z.left\n\t\t// logger.Printf(\"\\t\\t\\t--- x is left of z\")\n\t\tt.transplant(z, z.left)\n\n\t} else {\n\t\t// two children\n\t\t// logger.Printf(\"\\t\\tDelete: case (c) & (d)\\n\")\n\t\ty = t.getMinimum(z.right)\n\t\t// logger.Printf(\"\\t\\t\\tminimum of z.right is %s (color=%s)\\n\", y, y.color)\n\t\tyOriginalColor = y.color\n\t\tx = y.right\n\t\t// logger.Printf(\"\\t\\t\\t--- x is right of minimum\")\n\n\t\tif y.parent == z {\n\t\t\tif x != nil {\n\t\t\t\tx.parent = y\n\t\t\t}\n\t\t} else {\n\t\t\tt.transplant(y, y.right)\n\t\t\ty.right = z.right\n\t\t\ty.right.parent = y\n\t\t}\n\t\tt.transplant(z, y)\n\t\ty.left = z.left\n\t\ty.left.parent = y\n\t\ty.color = z.color\n\t}\n\tif yOriginalColor == BLACK {\n\t\tt.fixupDelete(x)\n\t}\n\treturn z\n}\n\nfunc (t *Tree) fixupDelete(x *NodeEntry) {\n\t// logger.Printf(\"\\t\\t\\tfixupDelete of node %s\\n\", x)\n\tif x == nil {\n\t\treturn\n\t}\nloop:\n\tfor {\n\t\tswitch {\n\t\tcase x == t.root:\n\t\t\t// logger.Printf(\"\\t\\t\\t=> bye .. is root\\n\")\n\t\t\tbreak loop\n\t\tcase x.color == RED:\n\t\t\t// logger.Printf(\"\\t\\t\\t=> bye .. RED\\n\")\n\t\t\tbreak loop\n\t\tcase x == x.parent.right:\n\t\t\t// logger.Printf(\"\\t\\tBRANCH: x is right child of parent\\n\")\n\t\t\tw := x.parent.left // is nillable\n\t\t\tif isRed(w) {\n\t\t\t\t// Convert case 1 into case 2, 3, or 4\n\t\t\t\t// logger.Printf(\"\\t\\t\\tR> case 1\\n\")\n\t\t\t\tw.color = BLACK\n\t\t\t\tx.parent.color = RED\n\t\t\t\tt.RotateRight(x.parent)\n\t\t\t\tw = x.parent.left\n\t\t\t}\n\t\t\tif w != nil {\n\t\t\t\tswitch {\n\t\t\t\tcase !isRed(w.left) && !isRed(w.right):\n\t\t\t\t\t// case 2 - both children of w are BLACK\n\t\t\t\t\t// logger.Printf(\"\\t\\t\\tR> case 2\\n\")\n\t\t\t\t\tw.color = RED\n\t\t\t\t\tx = x.parent // recurse up tree\n\t\t\t\tcase isRed(w.right) && !isRed(w.left):\n\t\t\t\t\t// case 3 - right child RED & left child BLACK\n\t\t\t\t\t// convert to case 4\n\t\t\t\t\t// logger.Printf(\"\\t\\t\\tR> case 3\\n\")\n\t\t\t\t\tw.right.color = BLACK\n\t\t\t\t\tw.color = RED\n\t\t\t\t\tt.RotateLeft(w)\n\t\t\t\t\tw = x.parent.left\n\t\t\t\t}\n\t\t\t\tif isRed(w.left) {\n\t\t\t\t\t// case 4 - left child is RED\n\t\t\t\t\t// logger.Printf(\"\\t\\t\\tR> case 4\\n\")\n\t\t\t\t\tw.color = x.parent.color\n\t\t\t\t\tx.parent.color = BLACK\n\t\t\t\t\tw.left.color = BLACK\n\t\t\t\t\tt.RotateRight(x.parent)\n\t\t\t\t\tx = t.root\n\t\t\t\t}\n\t\t\t}\n\t\tcase x == x.parent.left:\n\t\t\t// logger.Printf(\"\\t\\tBRANCH: x is left child of parent\\n\")\n\t\t\tw := x.parent.right // is nillable\n\t\t\tif isRed(w) {\n\t\t\t\t// Convert case 1 into case 2, 3, or 4\n\t\t\t\t// logger.Printf(\"\\t\\t\\tL> case 1\\n\")\n\t\t\t\tw.color = BLACK\n\t\t\t\tx.parent.color = RED\n\t\t\t\tt.RotateLeft(x.parent)\n\t\t\t\tw = x.parent.right\n\t\t\t}\n\t\t\tif w != nil {\n\t\t\t\tswitch {\n\t\t\t\tcase !isRed(w.left) && !isRed(w.right):\n\t\t\t\t\t// case 2 - both children of w are BLACK\n\t\t\t\t\t// logger.Printf(\"\\t\\t\\tL> case 2\\n\")\n\t\t\t\t\tw.color = RED\n\t\t\t\t\tx = x.parent // recurse up tree\n\t\t\t\tcase isRed(w.left) && !isRed(w.right):\n\t\t\t\t\t// case 3 - left child RED & right child BLACK\n\t\t\t\t\t// convert to case 4\n\t\t\t\t\t// logger.Printf(\"\\t\\t\\tL> case 3\\n\")\n\t\t\t\t\tw.left.color = BLACK\n\t\t\t\t\tw.color = RED\n\t\t\t\t\tt.RotateRight(w)\n\t\t\t\t\tw = x.parent.right\n\t\t\t\t}\n\t\t\t\tif isRed(w.right) {\n\t\t\t\t\t// case 4 - right child is RED\n\t\t\t\t\t// logger.Printf(\"\\t\\t\\tL> case 4\\n\")\n\t\t\t\t\tw.color = x.parent.color\n\t\t\t\t\tx.parent.color = BLACK\n\t\t\t\t\tw.right.color = BLACK\n\t\t\t\t\tt.RotateLeft(x.parent)\n\t\t\t\t\tx = t.root\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tx.color = BLACK\n}\n\n// Walk accepts a Visitor\nfunc (t *Tree) Walk(visitor Visitor) {\n\tvisitor.Visit(t.root)\n}\n\n// GetRoot get root node\nfunc (t *Tree) GetRoot() *NodeEntry {\n\treturn t.root\n}\n\n// countingVisitor counts the number\n// of nodes in the tree.\ntype countingVisitor struct {\n\tCount uint64\n}\n\n// Visit .\nfunc (v *countingVisitor) Visit(node *NodeEntry) {\n\tif node == nil {\n\t\treturn\n\t}\n\n\tv.Visit(node.left)\n\tv.Count = v.Count + 1\n\tv.Visit(node.right)\n}\n\n// InorderVisitor walks the tree in inorder fashion.\n// This visitor maintains internal state; thus do not\n// reuse after the completion of a walk.\ntype InorderVisitor struct {\n\tbuffer bytes.Buffer\n}\n\n// Eq 比较\nfunc (v *InorderVisitor) Eq(other *InorderVisitor) bool {\n\tif other == nil {\n\t\treturn false\n\t}\n\treturn v.String() == other.String()\n}\n\nfunc (v *InorderVisitor) trim(s string) string {\n\treturn strings.TrimRight(strings.TrimRight(s, \"ed\"), \"lack\")\n}\n\n// String returns string\nfunc (v *InorderVisitor) String() string {\n\treturn v.buffer.String()\n}\n\n// Visit .\nfunc (v *InorderVisitor) Visit(node *NodeEntry) {\n\tif node == nil {\n\t\tv.buffer.Write([]byte(\".\"))\n\t\treturn\n\t}\n\tv.buffer.Write([]byte(\"(\"))\n\tv.Visit(node.left)\n\tv.buffer.Write([]byte(fmt.Sprintf(\"%s\", node.Key))) // @TODO\n\t// v.buffer.Write([]byte(fmt.Sprintf(\"%d{%s}\", node.Key, v.trim(node.color.String()))))\n\tv.Visit(node.right)\n\tv.buffer.Write([]byte(\")\"))\n}\n\n// HookVisitor .\ntype HookVisitor struct {\n\tHook func(node *NodeEntry)\n}\n\n// Visit .\nfunc (v *HookVisitor) Visit(node *NodeEntry) {\n\tif node == nil {\n\t\treturn\n\t}\n\tv.Hook(node)\n\tv.Visit(node.left)\n\tv.Visit(node.right)\n}\n\nvar (\n\t// ErrorKeyIsNil the literal nil not allowed as keys\n\tErrorKeyIsNil = errors.New(\"the literal nil not allowed as keys\")\n\t// ErrorKeyDisallowed disallowed key type\n\tErrorKeyDisallowed = errors.New(\"disallowed key type\")\n)\n\n// Allowed key types are: Boolean, Integer, Floating point, Complex, String values\n// And structs containing these.\n// @TODO Should pointer type be allowed ?\nfunc mustBeValidKey(key interface{}) error {\n\tif key == nil {\n\t\treturn ErrorKeyIsNil\n\t}\n\n\t/*keyValue := reflect.ValueOf(key)\n\t switch keyValue.Kind() {\n\t case reflect.Chan:\n\t \tfallthrough\n\t case reflect.Func:\n\t \tfallthrough\n\t case reflect.Interface:\n\t \tfallthrough\n\t case reflect.Map:\n\t \tfallthrough\n\t case reflect.Ptr:\n\t \treturn ErrorKeyDisallowed\n\t default:\n\t \treturn nil\n\t }*/\n\treturn nil\n}\n" }, { "path": "xmap/map_test.go", "content": "// Copyright (c) 2022 XDS project Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// XDS Project Site: https://github.com/heiyeluren\n// XDS URL: https://github.com/heiyeluren/xds\n//\n\npackage xmap\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"github.com/heiyeluren/xds\"\n\t\"github.com/heiyeluren/xds/xmap/entry\"\n\t\"github.com/heiyeluren/xmm\"\n\t\"log\"\n\t\"math/rand\"\n\t\"net/http\"\n\t_ \"net/http/pprof\"\n\t\"os\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\t\"unsafe\"\n\t// \"xds/xmap/entry\"\n\t\"github.com/spf13/cast\"\n)\n\ntype User struct {\n\tName int\n\tAge int\n\tAddr string\n}\n\n// mudi : 1001 -> 10000\n// 1001 << 1 -> 10010\n// 10010 & (1001 & 0)\nfunc round(n uintptr) uintptr {\n\treturn (n << 1) & (0 & (n))\n}\n\ntype mmanClass uint8\n\nfunc makemmanClass(sizeclass uint8, noscan bool) mmanClass {\n\treturn mmanClass(sizeclass<<1) | mmanClass(bool2int(noscan))\n}\n\nfunc (sc mmanClass) sizeclass() int8 {\n\treturn int8(sc >> 1)\n}\n\nfunc (sc mmanClass) noscan() bool {\n\treturn sc&1 != 0\n}\n\nfunc bool2int(x bool) int {\n\t// Avoid branches. In the SSA compiler, this compiles to\n\t// exactly what you would want it to.\n\treturn int(uint8(*(*uint8)(unsafe.Pointer(&x))))\n}\n\nfunc Init() {\n\t// 略\n\truntime.GOMAXPROCS(6) // 限制 CPU 使用数，避免过载\n\truntime.SetMutexProfileFraction(1) // 开启对锁调用的跟踪\n\truntime.SetBlockProfileRate(1) // 开启对阻塞操作的跟踪\n\n\tgo func() {\n\t\t// 启动一个 http server，注意 pprof 相关的 handler 已经自动注册过了\n\t\tif err := http.ListenAndServe(\":6060\", nil); err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t\tos.Exit(0)\n\t}()\n\t<-time.After(time.Second * 10)\n}\n\nfunc Test_xmmanPool222(t *testing.T) {\n\t// 同步扩容 680 异步扩容 551\n\tfmt.Println(100000 / (4096 / 48))\n\tf := &xmm.Factory{}\n\n\tmm, err := f.CreateMemory(0.6)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar s unsafe.Pointer\n\tsl, err := mm.AllocSlice(unsafe.Sizeof(s), 100000+1, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tssss := *(*[]unsafe.Pointer)(sl)\n\tvar start unsafe.Pointer\n\tdata := (*reflect.SliceHeader)(unsafe.Pointer(&ssss)).Data\n\tvar us []unsafe.Pointer\n\tfor i := 0; i < 100000; i++ {\n\t\tp, err := mm.Alloc(unsafe.Sizeof(User{}))\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tuser := (*User)(p)\n\t\tuser.Age = i\n\t\tuser.Name = rand.Int()\n\t\tus = append(us, p)\n\t\tssss = append(ssss, p)\n\t\tif start == nil {\n\t\t\tstart = ssss[0]\n\t\t}\n\t}\n\tif (*reflect.SliceHeader)(unsafe.Pointer(&ssss)).Data != data {\n\t\tt.Fatal(\"扩容了\")\n\t}\n\tfmt.Printf(\"sssssss %d %d\\n \", start, ssss[0])\n\tif ssss[0] != start {\n\t\tt.Fatal(\"-\")\n\t}\n\tfor i, pointer := range us {\n\t\ttep := (*User)(ssss[i])\n\t\tif sss := (*User)(pointer); sss.Age != i || tep.Age != sss.Age {\n\t\t\tt.Fatalf(\"%+v\\n\", pointer)\n\t\t}\n\t}\n}\n\nfunc TestPointer2(t *testing.T) {\n\ttmp := make([]*User, 10000000)\n\tus := &tmp\n\tvar wait sync.WaitGroup\n\twait.Add(80)\n\tvar sm sync.Map\n\tfor j := 0; j < 80; j++ {\n\t\tgo func(z int) {\n\t\t\tdefer wait.Done()\n\t\t\tfor i := 0; i < 10000000; i++ {\n\t\t\t\tkey := cast.ToString(i + (z * 1000))\n\t\t\t\taddr := (*unsafe.Pointer)(unsafe.Pointer(&(*us)[i]))\n\t\t\t\tuser := atomic.LoadPointer(addr)\n\t\t\t\tif user == nil {\n\t\t\t\t\tut := &User{\n\t\t\t\t\t\tName: i,\n\t\t\t\t\t\tAge: i,\n\t\t\t\t\t\tAddr: key,\n\t\t\t\t\t}\n\t\t\t\t\tptr := unsafe.Pointer(ut)\n\t\t\t\t\tif atomic.CompareAndSwapPointer(addr, nil, ptr) {\n\t\t\t\t\t\tsm.Store(i, uintptr(ptr))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}(j)\n\t}\n\twait.Wait()\n\n\tfor i := 0; i < 10000000; i++ {\n\t\taddr := (*unsafe.Pointer)(unsafe.Pointer(&(*us)[i]))\n\t\tuser := atomic.LoadPointer(addr)\n\t\tu := (*User)(user)\n\t\tif val, ok := sm.Load(i); !ok || val != uintptr(unsafe.Pointer(u)) {\n\t\t\tt.Fatal(val, user, i, u)\n\t\t}\n\t}\n}\n\nfunc TestPointer(t *testing.T) {\n\ttmp := make([]uintptr, 1000000)\n\tvar users []*User\n\tus := &tmp\n\tvar wait sync.WaitGroup\n\twait.Add(80)\n\tvar sm sync.Map\n\tfor j := 0; j < 80; j++ {\n\t\tgo func(z int) {\n\t\t\tdefer wait.Done()\n\t\t\tfor i := 0; i < 100000; i++ {\n\t\t\t\tkey := cast.ToString(i + (z * 1000))\n\t\t\t\taddr := &((*us)[i])\n\t\t\t\tuser := atomic.LoadUintptr(addr)\n\t\t\t\tif user == 0 {\n\t\t\t\t\tut := &User{\n\t\t\t\t\t\tName: i,\n\t\t\t\t\t\tAge: i,\n\t\t\t\t\t\tAddr: key,\n\t\t\t\t\t}\n\t\t\t\t\tptr := uintptr(unsafe.Pointer(ut))\n\t\t\t\t\tif atomic.CompareAndSwapUintptr(addr, 0, ptr) {\n\t\t\t\t\t\tusers = append(users, ut)\n\t\t\t\t\t\tsm.Store(i, ptr)\n\t\t\t\t\t\t// fmt.Printf(\"i:%d ptr:%d\\n\", i, ptr)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}(j)\n\t}\n\twait.Wait()\n\n\tfor i := 0; i < 100000; i++ {\n\t\taddr := &((*us)[i])\n\t\tuser := atomic.LoadUintptr(addr)\n\t\tu := (*User)(unsafe.Pointer(user))\n\t\tif val, ok := sm.Load(i); !ok || val != user {\n\t\t\tt.Fatal(val, user, i, u)\n\t\t}\n\t}\n}\n\n// todo CompareAndSwapPointer xuexi\n\nfunc TestRBTree(t *testing.T) {\n\trbt := new(entry.Tree)\n\trbt.SetComparator(func(o1, o2 interface{}) int {\n\t\tkey1, key2 := o1.(string), o2.(string)\n\t\treturn strings.Compare(key1, key2)\n\t})\n\tnum := 10000000\n\tst := time.Now()\n\tfor i := 0; i < num/10; i++ {\n\t\tkey := cast.ToString(i)\n\t\tce := &entry.NodeEntry{\n\t\t\tValue: []byte(key),\n\t\t\tKey: []byte(key),\n\t\t\tHash: uint64(rand.Int()),\n\t\t}\n\t\trbt.Put(ce)\n\t}\n\tfor i := 0; i < num/10; i++ {\n\t\texist, node := rbt.Get([]byte(strconv.Itoa(i)))\n\t\tif !exist || bytes.Compare(node, []byte(strconv.Itoa(i))) != 0 {\n\t\t\tpanic(i)\n\t\t}\n\t}\n\tfmt.Println(rbt.Get([]byte(strconv.Itoa(5))))\n\tfmt.Println(time.Now().Sub(st).Seconds())\n\trbt.Walk(&entry.HookVisitor{Hook: func(node *entry.NodeEntry) {\n\t\tfmt.Println(node)\n\t}})\n}\n\nfunc Test_NewDefaultConcurrentHashMap(t *testing.T) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.6)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tchmp, err := NewDefaultConcurrentHashMap(mm, xds.Uintptr, xds.Uintptr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor i := 0; i < 10000000; i++ {\n\t\ts := uintptr(i)\n\t\tif err := chmp.Put(s, s); err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\tfor i := 0; i < 10000000; i++ {\n\t\ts := uintptr(i)\n\t\tif val, exist, err := chmp.Get(s); err != nil || val != s || !exist {\n\t\t\tt.Error(\"sss\", err)\n\t\t}\n\t}\n}\n\nfunc TestXmap_ForEach(t *testing.T) {\n\tf := &xmm.Factory{}\n\tmm, err := f.CreateMemory(0.6)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tchmp, err := NewDefaultConcurrentHashMap(mm, xds.Uintptr, xds.Uintptr)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tfor i := 0; i < 10000; i++ {\n\t\ts := uintptr(i)\n\t\tif err := chmp.Put(s, s); err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\tfor i := 0; i < 10000; i++ {\n\t\ts := uintptr(i)\n\t\tif val, exist, err := chmp.Get(s); err != nil || val != s || !exist {\n\t\t\tt.Error(\"sss\", err)\n\t\t}\n\t}\n\ti := 0\n\terr = chmp.ForEach(func(key, val interface{}) error {\n\t\t//fmt.Printf(\"ForEach key:%s value:%s \\n\", key, val)\n\t\ti++\n\t\treturn nil\n\t})\n\tfmt.Println(i, chmp.Len())\n\tif err != nil {\n\t\tt.Error(\"sss\", err)\n\t}\n}\n" }, { "path": "xmap/xmap.go", "content": "// Copyright (c) 2022 XDS project Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n//\n// XDS Project Site: https://github.com/heiyeluren\n// XDS URL: https://github.com/heiyeluren/xds\n//\n\npackage xmap\n\nimport (\n\t\"github.com/heiyeluren/xds\"\n\t// \"xds\"\n\t\"github.com/heiyeluren/xmm\"\n)\n\n// XMap is a map of maps.\n// ------------------------------------------------\n// 当做map[]来使用的场景\n// 本套API主要是提供给把Xmap当做map来使用的场景\n// ------------------------------------------------\n\n// Xmap struct\ntype XMap struct {\n\tchm *ConcurrentHashMap\n}\n\n// NewMap returns a new XMap.\n// 初始化调用xmap生成对象\n//\tmm 是XMM的内存池对象\n//\tkeyKind 是要生成 map[keyType]valType 中的 keyType\n//\tvalKind 是要生成 map[keyType]valType 中的 valType\n//\t说明：keyKind / valKind 都是直接调用xmap中对应的kind类型，必须提前初始化写入\n//\n// 本函数调用参考：\n// 生成一个 map[int]string 数据结构，默认大小16个元素，占用了75%后进行map扩容\n//\tm, err := xds.NewMapEx(mm, xmap.Int, xmap.String)\nfunc NewMap(mm xmm.XMemory, keyKind xds.Kind, valKind xds.Kind) (*XMap, error) {\n\tchm, err := NewDefaultConcurrentHashMap(mm, keyKind, valKind)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &XMap{chm: chm}, nil\n}\n\n// NewMapEx returns a new XMap.\n// 初始化调用xmap生成对象 - 可扩展方法\n// \tmm 是XMM的内存池对象\n// \tkeyKind 是要生成 map[keyType]valType 中的 keyType\n// \tvalKind 是要生成 map[keyType]valType 中的 valType\n// \t说明：keyKind / valKind 都是直接调用xmap中对应的kind类型，必须提前初始化写入\n// \tcapSize 是默认初始化整个map的大小，本值最好是 2^N 的数字比较合适（2的N次方）；\n// \t\t\t这个值主要提升性能，比如如果你预估到最终会有128个元素，可以初始化时候设置好，这样Xmap不会随意的动态扩容（默认值是16），提升性能\n// \tfactSize 负载因子，当存放的元素超过该百分比，就会触发扩容；建议值是0.75 (75%)，这就是当存储数据容量达到75%会触发扩容机制；\n//\n// 本函数调用参考：\n// \t//生成一个 map[int]string 数据结构，默认大小256个元素，占用了75%后进行map扩容\n// \tm, err := xds.NewMapEx(mm, xmap.Int, xmap.String, (uintptr)256, 0.75)\nfunc NewMapEx(mm xmm.XMemory, keyKind xds.Kind, valKind xds.Kind, capSize uintptr, factSize float64) (*XMap, error) {\n\tchm, err := NewConcurrentHashMap(mm, capSize, factSize, 8, keyKind, valKind)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &XMap{chm: chm}, nil\n}\n\n// Set 写入数据\nfunc (xm *XMap) Set(key interface{}, val interface{}) (err error) {\n\treturn xm.chm.Put(key, val)\n}\n\n// Remove 删除数据\nfunc (xm *XMap) Remove(key interface{}) (err error) {\n\treturn xm.chm.Del(key)\n}\n\n// Get 数据\nfunc (xm *XMap) Get(key interface{}) (val interface{}, keyExists bool, err error) {\n\treturn xm.chm.Get(key)\n}\n\n// Each 遍历所有数据\nfunc (xm *XMap) Each(f func(key, val interface{}) error) error {\n\treturn xm.chm.ForEach(f)\n}\n\n// Len 获取整个map的元素数量\nfunc (xm *XMap) Len() uint64 {\n\treturn xm.chm.Len()\n}\n\n\n\n// RawMap - Hashmap\n// ------------------------------------------------\n// 当做原生Hash Map来使用场景\n// 本套API主要是提供给把Xmap当做Hash表来使用的场景\n// ------------------------------------------------\n// 定义xmap的入口主结构\ntype RawMap struct {\n\tchm *ConcurrentRawHashMap\n}\n\n// NewHashMap returns a new RawMap.\n// 初始化调用xmap - hashmap 生成对象 mm 是XMM的内存池对象\nfunc NewHashMap(mm xmm.XMemory) (*RawMap, error) {\n\tchm, err := NewDefaultConcurrentRawHashMap(mm)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &RawMap{chm: chm}, nil\n}\n\n// Set 写入数据\nfunc (xm *RawMap) Set(key []byte, val []byte) error {\n\treturn xm.chm.Put(key, val)\n}\n\n// Remove 删除数据\nfunc (xm *RawMap) Remove(key []byte) error {\n\treturn xm.chm.Del(key)\n}\n\n// Get 数据\nfunc (xm *RawMap) Get(key []byte) ([]byte, bool, error) {\n\treturn xm.chm.Get(key)\n}\n\n// Each 遍历所有数据\nfunc (xm *RawMap) Each(f func(key, val []byte) error) error {\n\treturn xm.chm.ForEach(f)\n}\n\n// Len 获取整个map的元素数量\nfunc (xm *RawMap) Len() uint64 {\n\treturn xm.chm.size\n}\n\n\n\n\n\n// The data structure and interface between xMAP and the bottom layer\n// ------------------------------------------------\n// xmap与底层交互的数据结构和接口\n//\n// 主要提供给上层xmap api调用，完成一些转换工作\n// ------------------------------------------------\n\n//底层交互数据结构（带有传入数据类型保存）\ntype ConcurrentHashMap struct {\n\tkeyKind xds.Kind\n\tvalKind xds.Kind\n\tdata *ConcurrentRawHashMap\n}\n\n// NewDefaultConcurrentHashMap 类似于make(map[keyKind]valKind)\n// mm 内存分配模块\n// keyKind: map中key的类型\n// valKind: map中value的类型\nfunc NewDefaultConcurrentHashMap(mm xmm.XMemory, keyKind, valKind xds.Kind) (*ConcurrentHashMap, error) {\n\treturn NewConcurrentHashMap(mm, 16, 0.75, 8, keyKind, valKind)\n}\n\n// NewConcurrentHashMap 类似于make(map[keyKind]valKind)\n// mm 内存分配模块\n// keyKind: map中key的类型\n// cap:初始化bucket长度\n// fact:负载因子，当存放的元素超过该百分比，就会触发扩容。\n// treeSize：bucket中的链表长度达到该值后，会转换为红黑树。\n// valKind: map中value的类型\nfunc NewConcurrentHashMap(mm xmm.XMemory, cap uintptr, fact float64, treeSize uint64, keyKind, valKind xds.Kind) (*ConcurrentHashMap, error) {\n\tchm, err := NewConcurrentRawHashMap(mm, cap, fact, treeSize)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &ConcurrentHashMap{keyKind: keyKind, valKind: valKind, data: chm}, nil\n}\n\n//Get\nfunc (chm *ConcurrentHashMap) Get(key interface{}) (val interface{}, keyExists bool, err error) {\n\t// k, err := chm.Marshal(chm.keyKind, key)\n\tk, err := xds.RawToByte(chm.keyKind, key)\n\tif err != nil {\n\t\treturn nil, false, err\n\t}\n\tvalBytes, exists, err := chm.data.Get(k)\n\tif err != nil {\n\t\treturn nil, exists, err\n\t}\n\t// ret, err := chm.UnMarshal(chm.valKind, valBytes)\n\tret, err := xds.ByteToRaw(chm.valKind, valBytes)\n\tif err != nil {\n\t\treturn nil, false, err\n\t}\n\treturn ret, true, nil\n}\n\n//Put\nfunc (chm *ConcurrentHashMap) Put(key interface{}, val interface{}) (err error) {\n\t// k, err := chm.Marshal(chm.keyKind, key)\n\tk, err := xds.RawToByte(chm.keyKind, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\t// v, err := chm.Marshal(chm.valKind, val)\n\tv, err := xds.RawToByte(chm.valKind, val)\n\treturn chm.data.Put(k, v)\n}\n\n//Del\nfunc (chm *ConcurrentHashMap) Del(key interface{}) (err error) {\n\t// k, err := chm.Marshal(chm.keyKind, key)\n\tk, err := xds.RawToByte(chm.keyKind, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn chm.data.Del(k)\n}\n\n// ForEach 遍历所有key-vel对\nfunc (chm *ConcurrentHashMap) ForEach(fun func(key, val interface{}) error) error {\n\treturn chm.data.ForEach(func(keyRaw, valRaw []byte) error {\n\t\tkey, err := xds.ByteToRaw(chm.keyKind, keyRaw)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tval, err := xds.ByteToRaw(chm.valKind, valRaw)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn fun(key, val)\n\t})\n}\n\n// Len 存储元素个数\nfunc (chm *ConcurrentHashMap) Len() uint64 {\n\tif chm.data == nil {\n\t\treturn 0\n\t}\n\treturn chm.data.size\n}\n" }, { "path": "xslice/xslice.go", "content": "package xslice\n\nimport (\n\t\"errors\"\n\t\"math\"\n\t\"sync\"\n\n\t\"github.com/heiyeluren/xds\"\n)\n\nvar InvalidType = errors.New(\"type Error\") // 类型错误\n\ntype _gap []byte\nconst _blockSize int = 8\n\ntype Xslice struct{\n\ts [][_blockSize]_gap //[]*uintptr 改为[]数组。数组内存放基本类型结构体和指针。string的话，存放指针。\n\n\t//s []uintptr uintptr地址指向一个连续内存(数组)的头。\n\t// s 的创建和扩容，通过xmm的alloc(元素个数*8)。迁移完后销毁。\n\t// s 中的元素创建：通过xmm的alloc(元素个数*元素单位长度)。append到s中。\n\t// 基本类型存放：int、uint、直接放入到s的元素中。\n\t// 非基本类型存放：string、[]byte,先拷贝内存到xmm中，然后，将指针append到s的元素中。\n\n\tlock *sync.RWMutex\n\n\tsizeCtl int64 // -1 正在扩容\n\n\t//游标位置\n\tcurRPos int //读光标\n\tcurWGapPos int //写gap光标\n\tcurWSlotPos int //写slot光标\n\n\t_sizeof int //每个元素大小\n\t_stype xds.Kind //类型\n\t_len int //元素个数计数\n\n\t_scap int //容量\n}\n\n//func NewXslice(mm xmm.XMemory,_type Kind,_cap int) * Xslice {\nfunc NewXslice(_type xds.Kind,_cap int) *Xslice {\n\n\t//计算分配slot数量，有余数会多分配一个\n\tslotCap := float64(_cap % _blockSize)\n\tvar slotNum int64 = int64(math.Ceil(slotCap))\n\n\tif slotNum <= 0{\n\t\tslotNum = 1\n\t}\n\n\tslot := make([][8]_gap,slotNum,slotNum)\n\treturn &Xslice{\n\t\ts: slot,\n\t\tlock: new(sync.RWMutex),\n\t\t_scap: _cap,\n\t\t_stype: _type,\n\t}\n}\n\nfunc(xs *Xslice) Append(v interface{}) (*Xslice,error) {\n\txs.lock.Lock()\n\tdefer xs.lock.Unlock()\n\n\n\t//判断类型：基本copy进去。\n\t// s中将要存放的uintptr拿到，uintptr为一个起始地址，加上一个index偏移量为写入地址。\n\n\n\tvar gapPos int64 = 0\n\n\tslotCap := float64(xs.curWGapPos % _blockSize)\n\n\tgapPos = int64(slotCap)\n\t// 没有空位，扩容\n\tif slotCap == 0 && xs.curWGapPos >= _blockSize{\n\t\tnewSlot := [_blockSize]_gap{}\n\t\txs.s = append(xs.s,newSlot)\n\t\txs.curWSlotPos += 1\n\t}\n\tb,err := xds.Marshal(xs._stype,v)\n\tif err != nil{\n\t\treturn xs,err\n\t}\n\n\txs.s[xs.curWSlotPos][gapPos] = b\n\n\txs.curWGapPos += 1\n\txs._len += 1\n\n\treturn xs,nil\n}\n\nfunc (xs *Xslice) Set(n int,v interface{}) error {\n\txs.lock.Lock()\n\tdefer xs.lock.Unlock()\n\n\tvar gapPos int64 = 0\n\tslotCap := float64(n % _blockSize)\n\tgapPos = int64(slotCap)\n\n\t// 没有空位，扩容\n\tif slotCap == 0 && xs.curWGapPos >= _blockSize{\n\t\tnewSlot := [_blockSize]_gap{}\n\t\txs.s = append(xs.s,newSlot)\n\t\txs.curWSlotPos += 1\n\t}\n\n\tb,err := xds.Marshal(xs._stype,v)\n\tif err != nil{\n\t\treturn err\n\t}\n\n\txs.s[xs.curWSlotPos][gapPos] = b\n\txs.curWGapPos += 1\n\txs._len += 1\n\n\treturn nil\n}\n\nfunc (xs *Xslice) Get(n int) (interface{}, error) {\n\txs.lock.Lock()\n\tdefer xs.lock.Unlock()\n\n\tvar gapPos int64 = 0\n\tslotCap := float64(n % _blockSize)\n\tgapPos = int64(slotCap)\n\tb := xs.s[xs.curWSlotPos][gapPos]\n\tv,err := xds.UnMarshal(xs._stype,b)\n\tif err != nil{\n\t\treturn nil,err\n\t}\n\n\treturn v,nil\n}\n\nfunc (xs *Xslice) Free () {\n\txs.lock.Lock()\n\txs.curWGapPos = 0\n\txs.curRPos = 0\n\txs._scap = 0\n\txs._sizeof = 0\n\txs._len = 0\n\txs.curWSlotPos = 0\n\txs.s = nil\n\txs.lock.Unlock()\n}\n\nfunc(xs *Xslice) ForEach(f func(i int, v interface{}) error) error{\n\txs.lock.Lock()\n\tdefer xs.lock.Unlock()\n\n\tvar c int = 0\n\tfor i,gaps := range xs.s {\n\t\tfor _, b := range gaps {\n\t\t\t//判断坐标是否到头\n\t\t\tif xs.curWSlotPos == i && xs.curWGapPos == c{\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tc = c + 1\n\n\t\t\tv,err := xds.UnMarshal(xs._stype,b)\n\t\t\tif err != nil{\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tif err := f(c,v);err != nil{\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//返回xslice长度\nfunc (xs *Xslice) Len() int {\n\txs.lock.Lock()\n\tdefer xs.lock.Unlock()\n\treturn xs._len\n}\n\n//@todo 把扩容逻辑封装起来\nfunc (xs *Xslice) increase() *Xslice {\n\treturn nil\n}" }, { "path": "xslice/xslice_test.go", "content": "package xslice\n\nimport (\n\t\"log\"\n\t\"testing\"\n\n\t\"github.com/heiyeluren/xds\"\n\n)\n\nfunc TestNewXslice(t *testing.T) {\n\txs := NewXslice(xds.String,10)\n\tlog.Println(xs.s)\n}\n\nfunc TestXslice_Append(t *testing.T) {\n\txs2 := NewXslice(xds.Int,1)\n\tlog.Println(xs2.s)\n\n\tfor i:=0;i<=30;i++ {\n\t\txs2.Append(i)\n\t}\n\n\tlog.Println(xs2.s)\n}\n\nfunc TestXslice_Set(t *testing.T) {\n\txs3 := NewXslice(xds.Int,1)\n\tlog.Println(xs3.s)\n\n\terr := xs3.Set(1,123)\n\tif err != nil{\n\t\tpanic(err)\n\t}\n\n\tlog.Println(xs3.s)\n\n}\n\nfunc TestXslice_Get(t *testing.T) {\n\txs4 := NewXslice(xds.Int,1)\n\terr := xs4.Set(1,123)\n\tif err != nil{\n\t\tpanic(err)\n\t}\n\n\tv,err := xs4.Get(1)\n\n\tif err !=nil{\n\t\tpanic(err)\n\t}\n\n\tlog.Println(v)\n}\n\nfunc TestXslice_Free(t *testing.T) {\n\txs5 := NewXslice(xds.Int,1)\n\terr := xs5.Set(1,123)\n\tif err != nil{\n\t\tpanic(err)\n\t}\n\n\tlog.Println(xs5)\n\n\txs5.Free()\n\n\tlog.Println(xs5)\n}\n\nfunc TestXslice_ForEach(t *testing.T) {\n\txs6 := NewXslice(xds.String,1)\n\txs6.Append(\"aaaa\")\n\txs6.Append(\"bbb\")\n\txs6.Append(\"cccc\")\n\txs6.Append(\"cccc\")\n\txs6.Append(\"cccc\")\n\txs6.Append(\"cccc\")\n\txs6.Append(\"cccc\")\n\txs6.Append(\"cccc\")\n\txs6.Append(\"cccc\")\n\txs6.Append(\"eee\")\n\n\n\txs6.ForEach(func(i int, v []byte) error {\n\t\tlog.Println(\"i: \",i, \"v: \", string(v))\n\t\treturn nil\n\t})\n}\n\nfunc TestXslice_Len(t *testing.T) {\n\txs8 := NewXslice(xds.Int,1)\n\tlog.Println(xs8.s)\n\n\terr := xs8.Set(1,123)\n\tif err != nil{\n\t\tpanic(err)\n\t}\n\n\tlog.Println(xs8.s)\n\txs8.Set(2,123)\n\txs8.Set(3,123)\n\n\tlog.Println(\"len: \",xs8.Len())\n}\n" } ]