[ { "path": "AUTHORS", "content": "# Names should be added to this file like so:\n# Name or Organization \n\nTencent Inc. 2016\n\n# Initial version authors:\nJack Liang\t\nRobin Tang\t\nArthur Zou\t\n\n# Partial list of contributors:\n" }, { "path": "LICENSE.TXT", "content": "Tencent is pleased to support the open source community by making libwxfreq available. \n\nCopyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n\nIf you have downloaded a copy of the libwxfreq binary from Tencent, please\nnote that the libwxfreq binary is licensed under the BSD 3-Clause License.\nIf you have downloaded a copy of the libwxfreq source code from Tencent,\nplease note that libwxfreq source code is licensed under the BSD 3-Clause\nLicense, except for the third-party components listed below which are subject\nto different license terms. Your integration of libwxfreq into your own\nprojects may require compliance with the BSD 3-Clause License, as well as the\nother licenses applicable to the third-party components included within libwxfreq.\n\nA copy of the BSD 3-Clause License is included in this file.\n\nOther dependencies and licenses:\n=====================================================================================\nOpen Source Software Licensed Under the BSD 3-Clause License: \nThe below software in this distribution may have been modified by THL A29 Limited \n(Tencent Modifications). All Tencent Modifications are Copyright (C) 2017 THL A29\nLimited.\n-------------------------------------------------------------------------------------\n1. LevelDB 1.20\nCopyright (c) 2011 The LevelDB Authors. All rights reserved\n\n\nTerms of the BSD 3-Clause License:\n--------------------------------------------------------------------------------------\n\nRedistribution and use in source and binary forms, with or without modification, are\npermitted provided that the following conditions are met:\n* Redistributions of source code must retain the above copyright notice, this list of\n conditions and the following disclaimer.\n* Redistributions in binary form must reproduce the above copyright notice, this list of\n conditions and the following disclaimer in the documentation and/or other materials\n provided with the distribution.\n* Neither the name of [copyright holder] nor the names of its contributors may be used to\n endorse or promote products derived from this software without specific prior written\n permission.\n \n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY\nEXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\nMERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL\nTHE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT\nOF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\nHOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR\nTORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,\nEVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." }, { "path": "Makefile", "content": "NAME = libwxfreq\nSERVERNAME = WxFreq\nLIBNAME = libwxfreq.a\nOUTPUTDIR = ./output\nINCLUDE = $(OUTPUTDIR)/include/$(NAME)\nLIB64 = $(OUTPUTDIR)/lib64\nLIBINSTALLDIR = /usr/local/lib64/\nHEADERINSTALLDIR = /usr/local/include/\nDIRLIBEVENT=/data1/mm64/arthurzou/archive\n\nCC = g++\nLDFLAGS = -lwxfreq -pthread -L$(LIB64) -I $(dir $(INCLUDE))\nCPPFLAGS = $(CFLAGS)\nCPPFLAGS += -g -O2 -Wall -std=c++11\nINC = -I ./util -I ./freq -I ./net --std=c++11\n\nCPPFILE += $(wildcard freq/*.cpp)\nCPPFILE += $(wildcard util/*.cpp)\nOBJ = $(patsubst %.cpp, %.o, $(CPPFILE))\nDFILE = $(patsubst %.o, %.d, $(OBJ))\n\nNETCPPFILE += main.cpp\nNETCPPFILE += $(wildcard net/*.cpp)\nNETOBJ = $(patsubst %.cpp, %.o, $(NETCPPFILE))\nNETDFILE = $(patsubst %.o, %.d, $(NETOBJ))\n\n\n.PHONY : all clean test\n\n$(SERVERNAME) : $(OBJ) $(NETOBJ) main.o\n\t@rm -rf $(DFILE)\n\t@rm -rf $(NETDFILE)\n\t$(CC) $(CPPFLAGS) $^ -o $@ -pthread -levent -lrt -static -L $(DIRLIBEVENT)\n\n$(LIBNAME) : $(OBJ)\n\t@rm -rf $(OUTPUTDIR)\n\t@rm -rf $(DFILE)\n\t@mkdir -p $(LIB64)\n\t@mkdir -p $(INCLUDE)\n\tar scr $@ $(OBJ)\n\tmv $@ $(LIB64)\n\tcp freq/libwxfreq.h $(INCLUDE)\n\t@$(CC) -MM freq/libwxfreq.h $(INC) | \\\n\t\ttr \" \" \"\\n\" | grep \".h\" | xargs -I{} cp {} $(INCLUDE)\n\t@find $(INCLUDE) -name \"*.h\" | xargs sed -i 's;#include \";#include \"$(NAME)/;'\n\n%: test/%.cpp $(LIBNAME)\n\t$(CC) $(CPPFLAGS) $< -o $@.out $(LDFLAGS)\n\n\n-include $(DFILE)\n-include $(NETDFILE)\n\n%.d: %.cpp\n\t@$(CC) $(INC) -MT $*.o -MM $< > $@ && \\\n\t\techo -e '\\t$$(CC) $$(CPPFLAGS) $(INC) -c $$< -o $$@' >> $@\n\nclean:\n\t@echo -n \"clean workspace ...\"\n\t@rm -rf $(OUTPUTDIR) $(OBJ) $(NETOBJ) $(NETDFILE) $(DFILE) *.out $(SERVERNAME)\n\t@echo -e \"\\t\\t\\033[31m[done]\\033[0m\"\n\ninstall: $(LIBNAME)\n\tcp -r $(INCLUDE) $(HEADERINSTALLDIR)\n\tcp -r $(LIB64)/$(LIBNAME) $(LIBINSTALLDIR)\n" }, { "path": "README.md", "content": "# libwxfreq\n\n## 编译\n 1. 编译lib: make libwxfreq.a\n 2. 编译server: 依赖libevent库，所以要修改DIRLIBEVENT变量为libevent.a所在目录，并确定libevent相关头文件在头文件搜索路径中， 运行make\n\n## 安装\n 1. 修改makefile文件中的HEADERINSTALLDIR和LIBINSTALLDIR变量\n 2. make install\n\n## 单元测试\n 1. 以测试文件为目标make. 例如: make demo\n\n## 简介\n libwxfreq是一个高性能频率限制库。通过简洁的几个接口便可以实现通用的可配置的频率\n限制功能。配合其他网络框架，便可以实现一个通用的高性能频率控制服务。\n\n## 特点：\n 1. 高度可配置。统计时长可配，规则可配。\n 2. 支持任意key类型。\n \n## 配置文件\n 1. 格式\n \n appid的元信息，指明apppid的统计时长。默认第一个统计时长是60s，第二个统计时 \n 时长是3600s， 第三个是86400s。可以手动修改任何一个统计时长为[0, 86400] \n\n [appid]\n \n 5 = 30, 60, 90 // appid为5的统计时长分别是30s 60s 90s \n 25 = 10, 60 // appid为25的统计时长分别是10s 60s 86400s \n \n 频率规则， 用于说明某个appid的阈值， 拦截等级等 \n [match_rule_name] // 规则名字，用户指明中了哪条规则 \n match_appid = 5 // 匹配的appid \n block_level = 1 // 拦截等级， 大于0 \n rule_type = user // 规则对应的key类型 \n item = min_interval >= 4 // 具体的阈值表达式， 支持大于和等于\n ```\n min_interval 表示第一个统计时长的统计值， \n mid_interval 表示第二个统计时长的统计值， \n max_interval 表示第三个统计时长的统计值.\n ```\n\n 2. 例子\n ```\n [appid]\n 100 = 30, 60, 90\n\n [rule1]\n match_appid = 100\n item = min_interval >= 10\n block_level = 1\n rule_type = user\n\n [rule2]\n match_appid = 100\n item = mid_interval >= 15 \n block_level = 1\n rule_type = user\n\n [rule3]\n match_appid = 100\n item = max_interval >= 20\n block_level = 1\n rule_type = user\n\n [rule4]\n match_appid = 100\n item = min_interval >= 100\n block_level = 1\n rule_type = ip \n\n [rule5]\n match_appid = 100\n item = mid_interval >= 150\n block_level = 1\n rule_type = ip \n\n [rule6]\n match_appid = 100\n item = max_interval >= 200 \n block_level = 1\n rule_type = ip \n ```\n\n## API接口\n\n 设置规则配置文件\n > void SetRuleConfFile(const char *filename);\n\n 注册一个新的统计维度。type_name 指明类型名称， zero_init标志初始化时是否需要\n 初始化为0， key 是共享内存key，item_cnt 表示共享内存大小，最大能统计的item个\n 数。\n > int RegisterNewShmStat(const char* type_name, bool zero_init, key_t key,\n > unsigned int item_cnt);\n\n 系统初始化\n > bool InitFreq();\n\n 上报一次频率到系统中。type_name 是统计类型，说明key的含义。 key是主体标识，\n appid 是业务标识， cnt 表示本次需要累加的次数。 例如，统计QQ号为3402393864的\n 用户在appid为5的服务上的频率，使用ReportAndCheck(\"user\", 3402393864, 5, 1);\n 同样统计ip为202.204.105.8访问appid的5的频率，使用ReportAndCheck( \"IP\", \n 3402393864, 5, 1); 因为202.204.105.8的整形表示也是3402393864。type_name指明了\n key的含义。\n > struct BlockResult ReportAndCheck(const char* type_name, const char *key,\n > const uint32_t appid, const uint32_t cnt);\n\n 只检查当前是否中频率\n > struct BlockResult OnlyCheck(const char* type_name, const char *key,\n > const uint32_t appid);\n\n 只上报，不检查\n > int OnlyReport(const char* type_name, const char *key, const uint32_t appid,\n > const uint32_t cnt);\n\n 获取频率统计值, cnt为0获取当前统计值，不为零表示先累计再获取\n > struct FreqCache GetCache(const char* type_name, const char *key,\n > const uint32_t appid, const uint32_t cnt = 0);\n\n 加白操作，linger_time表示加白有效期\n > int AddWhite(const char* type_name, const char *key, const uint32_t appid,\n > const uint32_t linger_time);\n\n 删除白名单\n > int DeleteWhite(const char* type_name, const char *key, const uint32_t appid);\n\n 手动加拦截\n > int AddBlock(const char* type_name, const char *key, const uint32_t appid,\n > const uint32_t linger_time, const uint32_t block_level);\n\n 手动删除拦截\n > int DeleteBlock(const char* type_name, const char *key, const uint32_t appid);\n\n 设置写日志方法，默认写本地文件/tmp/libwxfreq.log\n > void SetLogFunc(LogFunction logfunc);\n\n 设置重启时恢复白名单/拦截方法，默认从本地文件/tmp/libwxfreq_db.$type_name读取\n > void SetLoadFunc(LoadFunc func);\n\n 设置白名单/加拦截数据持久化方法。默认写本地文件/tmp/libwxfreq_db.$type_name\n > void SetDumpFunc(DumpFunc func);\n" }, { "path": "TODO", "content": "1. 表达式支持逻辑运算\n" }, { "path": "freq/appid.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_APPID_H_\n#define FREQLIB_FREQ_APPID_H_\n\n#include \n#include \n#include \n\nnamespace libwxfreq {\n\nclass AppidMeta {\n public:\n AppidMeta()\n : min_interval_(60),\n mid_interval_(3600),\n max_interval_(86400) {\n }\n\n inline void set_min_interval(const uint32_t& min_interval) {\n min_interval_ = min_interval;\n }\n\n inline void set_mid_interval(const uint32_t& mid_interval) {\n mid_interval_ = mid_interval;\n }\n\n inline void set_max_interval(const uint32_t& max_interval) {\n max_interval_ = max_interval;\n }\n\n inline uint16_t min_interval() const {\n return min_interval_;\n }\n\n inline uint16_t mid_interval() const {\n return mid_interval_;\n }\n inline uint16_t max_interval() const {\n return max_interval_;\n }\n\n inline std::string DebugString() const {\n char buf[128];\n snprintf(buf, sizeof(buf), \"min_interval=%d, mid_interval=%d,\"\n \" max_interval=%d\", min_interval_, mid_interval_, max_interval_);\n return buf;\n }\n\n private:\n uint16_t min_interval_;\n uint16_t mid_interval_;\n uint32_t max_interval_;\n};\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_FREQ_APPID_H_\n" }, { "path": "freq/appid_manager.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"appid_manager.h\"\n#include \n#include \n#include \n#include \n#include \"config.h\"\n#include \"log.h\"\n\n\nnamespace libwxfreq {\n\nstatic AppidMeta dummymeta;\nAppidMapVecAppidMeta AppidManager::appid_map_[2];\nunsigned char AppidManager::index_ = 0;\n\nvoid AppidManager::UpdateAppidMeta(const Config& config) {\n int ret = 0;\n unsigned int appid = 0; \n std::string interval_value; //60, 3600, 86400\n\n unsigned char new_index = (index_ + 1) % 2;\n AppidMapVecAppidMeta& tmp_appid = appid_map_[new_index];\n tmp_appid.clear();\n\n std::vector keys;\n config.GetKeysBySection(\"appid\", keys);\n\n for (std::vector::iterator it = keys.begin();\n it != keys.end(); ++it) {\n appid = atoi(it->c_str());\n ret = config.ReadItem(\"appid\", *it, \"\", interval_value);\n if (ret != 0 || appid == 0) {\n gLog(\"[%s][%d]: read %s appid %d failed %d\\n\", __FILE__, __LINE__,\n it->c_str(), appid, ret);\n continue;\n }\n\n int32_t min = 0, mid = 0, max = 0;\n char *save_ptr = NULL;\n char *p = NULL;\n char buf[128];\n snprintf(buf, sizeof(buf), \"%s\", interval_value.c_str());\n p = strtok_r(buf, \",\", &save_ptr);\n if (p != NULL) {\n min = atoi(p);\n if (min > 0 && min <= 86400) {\n tmp_appid[appid].set_min_interval(min);\n }\n }\n\n p = strtok_r(NULL, \",\", &save_ptr);\n if (p != NULL) {\n mid = atoi(p);\n if (mid > 0 && mid <= 86400) {\n tmp_appid[appid].set_mid_interval(mid);\n }\n }\n\n p = strtok_r(NULL, \",\", &save_ptr);\n if (p != NULL) {\n max = atoi(p);\n if (max > 0 && max <= 86400) {\n tmp_appid[appid].set_max_interval(max);\n }\n }\n }\n\n index_ = new_index;\n gLog(\"[%s][%d]: index = %d\\n\", __FILE__, __LINE__, index_);\n for (AppidMapVecAppidMeta::iterator it = appid_map_[index_].begin();\n it != appid_map_[index_].end(); ++it) {\n gLog(\"[%s][%d]: appid = %u, meta = %s\\n\", __FILE__, __LINE__,\n it->first, it->second.DebugString().c_str());\n }\n}\n\nconst AppidMeta* AppidManager::GetAppidMeta(const uint32_t appid) {\n AppidMapVecAppidMeta::iterator it = appid_map_[index_].find(appid);\n if (it == appid_map_[index_].end()) {\n gLog(\"[%s][%d]: can't find appid %d meta\\n\", __FILE__, __LINE__, appid);\n return NULL;\n }\n return &(it->second);\n}\n\n} // idspace libwxfreq\n" }, { "path": "freq/appid_manager.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_APPID_MANAGER_H_\n#define FREQLIB_FREQ_APPID_MANAGER_H_\n#include \n#include \n#include \n#include \"appid.h\"\n\nnamespace libwxfreq {\nclass Config;\n\ntypedef std::map AppidMapVecAppidMeta;\n\nclass AppidManager {\n public:\n static const AppidMeta* GetAppidMeta(const uint32_t appid);\n static void UpdateAppidMeta(const Config& config);\n\n private:\n static AppidMapVecAppidMeta appid_map_[2];\n static uint8_t index_;\n};\n\n} // namespace libwxfreq\n#endif // FREQLIB_FREQ_APPID_MANAGER_H_\n" }, { "path": "freq/appname.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_APPNAME_H_\n#define FREQLIB_FREQ_APPNAME_H_\n\n#include \n#include \n#include \n\nnamespace libwxfreq {\n\nclass AppNameMeta {\n public:\n AppNameMeta()\n : appname_(),\n appid_(0),\n min_interval_(60),\n mid_interval_(3600),\n max_interval_(86400) {\n }\n\n inline void set_min_interval(const uint32_t& min_interval) {\n min_interval_ = min_interval;\n }\n\n inline void set_mid_interval(const uint32_t& mid_interval) {\n mid_interval_ = mid_interval;\n }\n\n inline void set_max_interval(const uint32_t& max_interval) {\n max_interval_ = max_interval;\n }\n\n inline void set_appid(const uint32_t& appid) {\n appid_ = appid;\n }\n\n inline void set_appname(const std::string& appname) {\n appname_ = appname;\n }\n\n inline std::string appname() const {\n return appname_;\n }\n\n inline uint16_t min_interval() const {\n return min_interval_;\n }\n\n inline uint16_t mid_interval() const {\n return mid_interval_;\n }\n inline uint16_t max_interval() const {\n return max_interval_;\n }\n\n inline uint32_t appid() const {\n return appid_;\n }\n\n inline std::string DebugString() const {\n char buf[128];\n snprintf(buf, sizeof(buf), \"appname=%s, appid=%d, min_interval=%d, \"\n \"mid_interval=%d, max_interval=%d\", appname_.c_str(), appid_,\n min_interval_, mid_interval_, max_interval_);\n return buf;\n }\n\n private:\n std::string appname_;\n uint32_t appid_;\n uint16_t min_interval_;\n uint16_t mid_interval_;\n uint32_t max_interval_;\n};\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_FREQ_APPNAME_H_\n" }, { "path": "freq/appname_manager.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"appname_manager.h\"\n#include \n#include \n#include \n#include \"config.h\"\n#include \"log.h\"\n\n\nnamespace libwxfreq {\n\nstatic AppNameMeta dummymeta;\nAppanmeMapVecAppnameMeta AppnameManager::appname_map_[2];\nunsigned char AppnameManager::index_ = 0;\n\nvoid AppnameManager::UpdateAppNameMeta(const Config& config) {\n int ret = 0;\n unsigned int appid = 0, interval = 0;\n std::string appname, interval_key;\n\n unsigned char new_index = (index_ + 1) % 2;\n AppanmeMapVecAppnameMeta& tmp_appname = appname_map_[new_index];\n tmp_appname.clear();\n\n std::vector keys;\n config.GetKeysBySection(\"appname\", keys);\n\n for (std::vector::iterator it = keys.begin();\n it != keys.end(); ++it) {\n appid = 0;\n std::size_t pos = it->find(\":\");\n if (pos == std::string::npos) {\n ret = config.ReadItem(\"appname\", *it, 0, appid);\n if (ret != 0 || appid == 0) {\n gLog(\"[%s][%d]: read %s appid %d failed\\n\", __FILE__, __LINE__,\n it->c_str(), appid);\n continue;\n }\n tmp_appname[*it].set_appname(*it);\n tmp_appname[*it].set_appid(appid);\n } else {\n appname = it->substr(0, pos);\n interval_key = it->substr(pos + 1);\n AppNameMeta& meta = tmp_appname[appname];\n\n ret = config.ReadItem(\"appname\", *it, 0, interval);\n if (ret != 0) {\n gLog(\"[%s][%d]: read %s %s failed\\n\", __FILE__, __LINE__,\n appname.c_str(), interval_key.c_str());\n continue;\n }\n if (interval_key == \"min_interval\") {\n meta.set_min_interval(interval);\n } else if (interval_key == \"mid_interval\") {\n meta.set_mid_interval(interval);\n } else if (interval_key == \"max_interval\") {\n meta.set_max_interval(interval);\n }\n }\n }\n index_ = new_index;\n gLog(\"[%s][%d]: index = %d\\n\", __FILE__, __LINE__, index_);\n for (AppanmeMapVecAppnameMeta::iterator it = appname_map_[index_].begin();\n it != appname_map_[index_].end(); ++it) {\n gLog(\"[%s][%d]: appname = %s, meta = %s\\n\", __FILE__, __LINE__,\n it->first.c_str(), it->second.DebugString().c_str());\n }\n}\n\nconst AppNameMeta* AppnameManager::GetAppNameMeta(const std::string& appname) {\n AppanmeMapVecAppnameMeta::iterator it = appname_map_[index_].find(appname);\n if (it == appname_map_[index_].end()) {\n gLog(\"[%s][%d]: can't find appname %s meta\\n\", __FILE__, __LINE__,\n appname.c_str());\n return NULL;\n }\n return &(it->second);\n}\n\n} // namespace libwxfreq\n" }, { "path": "freq/appname_manager.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_APPNAME_MANAGER_H_\n#define FREQLIB_FREQ_APPNAME_MANAGER_H_\n#include \n#include \n#include \n#include \"appname.h\"\n\nnamespace libwxfreq {\nclass Config;\n\ntypedef std::map AppanmeMapVecAppnameMeta;\n\nclass AppnameManager {\n public:\n static const AppNameMeta* GetAppNameMeta(const std::string& appname);\n static void UpdateAppNameMeta(const Config& config);\n\n private:\n static AppanmeMapVecAppnameMeta appname_map_[2];\n static uint8_t index_;\n};\n\n} // namespace libwxfreq\n#endif // FREQLIB_FREQ_APPNAME_MANAGER_H_\n" }, { "path": "freq/expression.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_EXPRESSION_H_\n#define FREQLIB_FREQ_EXPRESSION_H_\n#include \n#include \"freq_item.h\"\n\nnamespace libwxfreq {\n\nclass Expression {\n public:\n virtual ~Expression() {}\n virtual bool ParseItem(const std::string& item) = 0;\n virtual bool IsMatch(const FreqItem& freqitem) const = 0;\n virtual std::string DebugString() const = 0;\n virtual Expression* Clone() const = 0;\n};\n\n\n} // namespace libwxfreq\n\n\n#endif // FREQLIB_FREQ_EXPRESSION_H_\n" }, { "path": "freq/freq_item.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_FREQ_ITEM_H_\n#define FREQLIB_FREQ_FREQ_ITEM_H_\n#include \n#include \n\n#pragma pack(1)\nnamespace libwxfreq {\n\nstruct FreqKey {\n char key[32];\n uint32_t appid;\n};\n\nstruct FreqItem {\n unsigned int level1_cnt;\n unsigned int level2_cnt;\n unsigned int level3_cnt;\n};\n\n} // namespace libwxfreq\n#pragma pack()\n#endif // FREQLIB_FREQ_FREQ_ITEM_H_\n" }, { "path": "freq/freq_manager.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"freq_manager.h\"\n#include \"freq_item.h\"\n#include \"appid.h\"\n#include \"freq_stat.h\"\n#include \"log.h\"\n#include \"appid_manager.h\"\n\nnamespace libwxfreq {\n\nTypeName2FreqStatMap FreqManager::type_name_map_;\nconst uint32_t FreqManager::kWhiteFlag = 0xffffffff;\nconst uint32_t FreqManager::kAddBlockFlag = 0xfffffffe;\nstatic AppidMeta dummy_appid_meta;\n\ninline uint32_t FreqManager::TimeSplit(time_t timestamp, uint32_t scale) {\n return timestamp/scale;\n}\n\nbool FreqManager::Init(const TypeName2FreqStatMap& type_name_map) {\n TypeName2FreqStatMap::iterator it = type_name_map_.begin();\n while (it != type_name_map_.end()) {\n delete it->second;\n it->second = NULL;\n ++it;\n }\n type_name_map_ = type_name_map;\n it = type_name_map_.begin();\n\n while (it != type_name_map_.end()) {\n if (it->second->TryReload() == false ) {\n gLog(\"[%s][%d]: ERROR type name %s TryReload failed\\n\",\n __FILE__, __LINE__, it->first.c_str());\n }\n ++it;\n }\n return true;\n}\n\nconst FreqItem* FreqManager::CommAdd(FreqStat *freq_stat,\n const std::string& key,\n const AppidMeta& appid_meta,\n const uint32_t appid,\n const uint32_t cnt) {\n FreqItem *freq_item = NULL;\n uint32_t& last_time = freq_stat->GetItem(key, appid, cnt, freq_item);\n if (freq_item == NULL) {\n gLog(\"[%s][%d]: appid %u key %s GetItem empty\\n\", __FILE__, __LINE__,\n appid, key.c_str());\n return NULL;\n }\n\n if (freq_item->level1_cnt == kWhiteFlag) {\n gLog(\"[%s][%d]: appid %u key %s white\\n\", __FILE__, __LINE__,\n appid, key.c_str());\n return freq_item;\n } else if (freq_item->level1_cnt == kAddBlockFlag) {\n gLog(\"[%s][%d]: appid %u key %s blocked\\n\", __FILE__, __LINE__,\n appid, key.c_str());\n return freq_item;\n }\n\n time_t current_time = time(NULL);\n uint32_t current_interval_level3 = TimeSplit(current_time,\n appid_meta.max_interval());\n uint32_t current_interval_level2 = TimeSplit(current_time,\n appid_meta.mid_interval());\n uint32_t current_interval_level1 = TimeSplit(current_time,\n appid_meta.min_interval());\n\n uint32_t last_interval_level3 = TimeSplit(last_time,\n appid_meta.max_interval());\n uint32_t last_interval_level2 = TimeSplit(last_time,\n appid_meta.mid_interval());\n uint32_t last_interval_level1 = TimeSplit(last_time,\n appid_meta.min_interval());\n\n if (last_interval_level1 != current_interval_level1) {\n freq_item->level1_cnt = cnt;\n } else {\n __sync_fetch_and_add(&freq_item->level1_cnt, cnt);\n }\n\n if (last_interval_level2 != current_interval_level2) {\n freq_item->level2_cnt = cnt;\n } else {\n __sync_fetch_and_add(&freq_item->level2_cnt, cnt);\n }\n\n if (last_interval_level3 != current_interval_level3) {\n freq_item->level3_cnt = cnt;\n } else {\n __sync_fetch_and_add(&freq_item->level3_cnt, cnt);\n }\n last_time = current_time;\n return freq_item;\n}\n\nconst FreqItem* FreqManager::Add(const char* type_name,\n const std::string& key,\n const uint32_t appid,\n const uint32_t cnt) {\n if (type_name == NULL) {\n gLog(\"[%s][%d]: type name empty\\n\", __FILE__, __LINE__);\n return NULL;\n }\n if (appid == 0) {\n gLog(\"[%s][%d]: appid = 0\\n\", __FILE__, __LINE__);\n return NULL;\n }\n TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);\n if (it == type_name_map_.end()) {\n gLog(\"[%s][%d]: type name %s not exists\\n\", __FILE__, __LINE__, type_name);\n return NULL;\n }\n\n const AppidMeta *meta = AppidManager::GetAppidMeta(appid);\n if (meta == NULL) {\n return CommAdd(it->second, key, dummy_appid_meta, appid, cnt);\n } else {\n return CommAdd(it->second, key, *meta, appid, cnt);\n }\n}\n\nbool FreqManager::AddWhite(const char* type_name, const std::string& key,\n const uint32_t appid, const uint32_t linger_time) {\n if (type_name == NULL) {\n gLog(\"[%s][%d]: type name empty\\n\", __FILE__, __LINE__);\n return false;\n }\n if (appid == 0) {\n gLog(\"[%s][%d]: appid = 0\\n\", __FILE__, __LINE__);\n return false;\n }\n TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);\n if (it == type_name_map_.end()) {\n gLog(\"[%s][%d]: type name %s not exists\\n\", __FILE__, __LINE__, type_name);\n return false;\n }\n FreqStat *freq_stat = it->second;\n\n if (freq_stat == NULL) {\n gLog(\"[%s][%d]: type name %s freq_stat NULL\\n\",\n __FILE__, __LINE__, type_name);\n return false;\n }\n\n FreqItem *freq_item = NULL;\n uint32_t& last_time = freq_stat->GetItem(key, appid, 1, freq_item);\n if (freq_item == NULL) {\n gLog(\"[%s][%d]: appid %u key %s GetItem empty\\n\",\n __FILE__, __LINE__, appid, key.c_str());\n return false;\n }\n\n unsigned int expired_time = time(NULL) + linger_time;\n freq_item->level1_cnt = kWhiteFlag;\n last_time = expired_time - 90000;\n return true;\n}\n\nbool FreqManager::DeleteWhite(const char* type_name, const std::string& key,\n const uint32_t appid) {\n if (type_name == NULL) {\n gLog(\"[%s][%d]: type name empty\\n\", __FILE__, __LINE__);\n return false;\n }\n\n TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);\n if (it == type_name_map_.end()) {\n gLog(\"[%s][%d]: type name %s not exists\\n\", __FILE__, __LINE__, type_name);\n return false;\n }\n FreqStat *freq_stat = it->second;\n\n if (freq_stat == NULL) {\n gLog(\"[%s][%d]: type name %s freq_stat NULL\\n\",\n __FILE__, __LINE__, type_name);\n return false;\n }\n\n FreqItem *freq_item = NULL;\n uint32_t& last_time = freq_stat->GetItem(key, appid, 0, freq_item);\n if (freq_item == NULL) {\n return true;\n }\n\n if (GetWhiteLevel(freq_item) > 0) {\n last_time = 1; //expired\n }\n return true;\n}\n\nbool FreqManager::AddBlock(const char* type_name, const std::string& key,\n const uint32_t appid, const uint32_t linger_time,\n const uint32_t block_level) {\n if (type_name == NULL) {\n gLog(\"[%s][%d]: type name empty\\n\", __FILE__, __LINE__);\n return false;\n }\n\n TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);\n if (it == type_name_map_.end()) {\n gLog(\"[%s][%d]: type name %s not exists\\n\", __FILE__, __LINE__, type_name);\n return false;\n }\n FreqStat *freq_stat = it->second;\n\n\n FreqItem *freq_item = NULL;\n uint32_t& last_time = freq_stat->GetItem(key, appid, 1, freq_item);\n if (freq_item == NULL) {\n gLog(\"[%s][%d]: appid %d key %s GetItem empty\\n\",\n __FILE__, __LINE__, appid, key.c_str());\n return false;\n }\n\n if (freq_stat == NULL) {\n gLog(\"[%s][%d]: type name %s freq_stat NULL\\n\",\n __FILE__, __LINE__, type_name);\n return false;\n }\n\n if (GetWhiteLevel(freq_item) > 0) {\n return true;\n }\n\n unsigned int expired_time = time(NULL) + linger_time;\n freq_item->level1_cnt = kAddBlockFlag;\n freq_item->level2_cnt = block_level;\n last_time = expired_time - 90000;\n return true;\n}\n\nbool FreqManager::DeleteBlock(const char* type_name, const std::string& key,\n const uint32_t appid) {\n if (type_name == NULL) {\n gLog(\"[%s][%d]: type name empty\\n\", __FILE__, __LINE__);\n return false;\n }\n\n TypeName2FreqStatMap::iterator it = type_name_map_.find(type_name);\n if (it == type_name_map_.end()) {\n gLog(\"[%s][%d]: type name %s not exists\\n\", __FILE__, __LINE__, type_name);\n return false;\n }\n FreqStat *freq_stat = it->second;\n\n if (freq_stat == NULL) {\n gLog(\"[%s][%d]: type name %s freq_stat NULL\\n\",\n __FILE__, __LINE__, type_name);\n return false;\n }\n\n\n FreqItem *freq_item = NULL;\n uint32_t& last_time = freq_stat->GetItem(key, appid, 0, freq_item);\n if (freq_item == NULL) {\n return true;\n }\n\n if (GetWhiteLevel(freq_item) > 0) {\n return true;\n }\n\n last_time = 1;\n return true;\n}\n\nuint32_t FreqManager::GetWhiteLevel(const FreqItem* item) {\n return item->level1_cnt == kWhiteFlag ? 1 : 0;\n}\n\nuint32_t FreqManager::GetBlackLevel(const FreqItem* item) {\n if (item->level1_cnt == kAddBlockFlag) {\n return item->level2_cnt;\n }\n return 0;\n}\n\n} // namespace libwxfreq\n" }, { "path": "freq/freq_manager.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_FREQ_MANAGER_H_\n#define FREQLIB_FREQ_FREQ_MANAGER_H_\n#include \n#include \n#include \n\nnamespace libwxfreq {\n\nclass FreqStat;\nclass AppidMeta;\nclass FreqItem;\ntypedef std::map TypeName2FreqStatMap;\n\nclass FreqManager {\n public:\n static bool Init(const TypeName2FreqStatMap& type_name_map);\n static const FreqItem* Add(const char* type_name, const std::string& key,\n const uint32_t appid, const uint32_t cnt);\n static bool AddWhite(const char* type_name, const std::string& key,\n const uint32_t appid, const uint32_t linger_time);\n static bool DeleteWhite(const char* type_name, const std::string& key,\n const uint32_t appid);\n static bool AddBlock(const char* type_name, const std::string& key,\n const uint32_t appid, const uint32_t linger_time,\n const uint32_t block_level);\n static bool DeleteBlock(const char* type_name, const std::string& key,\n const uint32_t appid);\n static uint32_t GetWhiteLevel(const FreqItem* item);\n static uint32_t GetBlackLevel(const FreqItem* item);\n\n private:\n static const uint32_t kWhiteFlag;\n static const uint32_t kAddBlockFlag;\n static const FreqItem* CommAdd(FreqStat *freq_stat, const std::string& key,\n const AppidMeta& appid_meta,\n const uint32_t appid, const uint32_t cnt);\n static uint32_t TimeSplit(time_t timestamp, uint32_t scale);\n static TypeName2FreqStatMap type_name_map_;\n};\n\n} // namespace libwxfreq\n#endif // FREQLIB_FREQ_FREQ_MANAGER_H_\n" }, { "path": "freq/freqlib.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"libwxfreq.h\"\n#include \n#include \n#include \"appid_manager.h\"\n#include \"freq_manager.h\"\n#include \"rule_manager.h\"\n#include \"map_freq_stat.h\"\n#include \"shm_freq_stat.h\"\n#include \"options.h\"\n#include \"freq_stat.h\"\n#include \"log.h\"\n#include \"reload.h\"\n\nusing namespace libwxfreq;\n\nconst static uint32_t kDeleteTime = 86400;\nTypeName2FreqStatMap type_name_map;\n\nint RegisterNewMapStat(const char* type_name, bool /*zero_init*/,\n key_t /*key*/, unsigned int /*item_cnt*/) {\n if (type_name == NULL) {\n gLog(\"[%s][%d]: type name empty\\n\", __FILE__, __LINE__);\n return kErrorInput;\n }\n TypeName2FreqStatMap::iterator it = type_name_map.find(type_name);\n if (it != type_name_map.end()) delete it->second;\n FreqStat * tmp_map = new MapFreqStat();\n if (tmp_map == NULL) {\n gLog(\"[%s][%d]: alloc stat memory error, type name:%s\\n\",\n __FILE__, __LINE__, type_name);\n return kErrorSystem;\n }\n tmp_map->set_type_name(type_name);\n type_name_map[type_name] = tmp_map;\n return kOK;\n}\n\nint RegisterNewShmStat(const char* type_name, bool zero_init,\n key_t key, unsigned int item_cnt) {\n if (type_name == NULL) {\n gLog(\"[%s][%d]: type name empty\\n\", __FILE__, __LINE__);\n return kErrorInput;\n }\n TypeName2FreqStatMap::iterator it = type_name_map.find(type_name);\n if (it != type_name_map.end()) delete it->second;\n FreqStat * tmp_map = new ShmFreqStat(zero_init, key, item_cnt);\n if (tmp_map == NULL) {\n gLog(\"[%s][%d]: alloc stat memory error, type name:%s\\n\",\n __FILE__, __LINE__, type_name);\n return kErrorSystem;\n }\n tmp_map->set_type_name(type_name);\n type_name_map[type_name] = tmp_map;\n return kOK;\n}\n\nvoid SetRuleConfFile(const char *filename) {\n RuleManager::set_rule_conf_file(filename);\n}\n\nbool InitFreq() {\n RuleManager::Init();\n return FreqManager::Init(type_name_map);\n}\n\nstatic BlockResult MakeBlockResult(unsigned int block_level,\n const char* match_rule) {\n BlockResult result;\n result.block_level = block_level;\n strncpy(result.match_rule, match_rule, sizeof(result.match_rule));\n return result;\n}\n\nBlockResult ReportAndCheck(const char* type_name, const char *key,\n const uint32_t appid, const uint32_t cnt) {\n const FreqItem* p = NULL;\n p = FreqManager::Add(type_name, key, appid, cnt);\n if (p == NULL) return MakeBlockResult(kErrorOutOfMemory, \"\");\n\n RuleManager::Result res = RuleManager::IsMatchRule(type_name, appid, *p);\n return MakeBlockResult(res.block_level, res.match_rule.c_str());\n}\n\nstruct BlockResult OnlyCheck(const char* type_name, const char *key,\n const uint32_t appid) {\n return ReportAndCheck(type_name, key, appid, 0);\n}\n\nint OnlyReport(const char* type_name, const char *key, const uint32_t appid,\n const uint32_t cnt) {\n if (FreqManager::Add(type_name, key, appid, cnt) == NULL)\n return kErrorOutOfMemory;\n return 0;\n}\n\n\nstatic FreqCache MakeFreqCache(const uint32_t level1, const uint32_t level2,\n const uint32_t level3) {\n FreqCache freqcache = {level1, level2, level3};\n return freqcache;\n}\n\n\nFreqCache GetCache(const char* type_name, const char *key,\n const uint32_t appid, const uint32_t cnt) {\n const FreqItem* p = NULL;\n p = FreqManager::Add(type_name, key, appid, cnt);\n if (p == NULL) return MakeFreqCache(0, 0, 0);\n return MakeFreqCache(p->level1_cnt, p->level2_cnt, p->level3_cnt);\n}\n\nvoid SetLogFunc(LogFunction logfunc) {\n SetLog(logfunc);\n}\n\nint AddWhite(const char* type_name, const char *key, const uint32_t appid,\n const uint32_t linger_time) {\n if (gDumpFunc(\"AddWhite\", type_name, key, appid, linger_time, 1) != 0)\n return kErrorSystem;\n if (FreqManager::AddWhite(type_name, key, appid, linger_time) == false)\n return kErrorOutOfMemory;\n return 0;\n}\n\nint DeleteWhite(const char* type_name, const char *key, const uint32_t appid) {\n if (gDumpFunc(\"DeleteWhite\", type_name, key, appid, kDeleteTime, 1) != 0)\n return kErrorSystem;\n if (FreqManager::DeleteWhite(type_name, key, appid) == false)\n return kErrorOutOfMemory;\n return 0;\n}\n\nint AddBlock(const char* type_name, const char *key, const uint32_t appid,\n const uint32_t linger_time, const uint32_t block_level) {\n if (gDumpFunc(\"AddBlock\", type_name, key, appid,\n linger_time, block_level) != 0) return kErrorSystem;\n if (FreqManager::AddBlock(type_name, key, appid, linger_time,\n block_level) == false) return kErrorOutOfMemory;\n return 0;\n}\n\nint DeleteBlock(const char* type_name, const char *key, const uint32_t appid) {\n if (gDumpFunc(\"DeleteBlock\", type_name, key, appid, kDeleteTime, 1) != 0)\n return kErrorSystem;\n if (FreqManager::DeleteBlock(type_name, key, appid) == false)\n return kErrorOutOfMemory;\n return 0;\n}\n\nvoid SetLoadFunc(LoadFunc func) {\n libwxfreq::SetLoadFunc(func);\n}\n\nvoid SetDumpFunc(DumpFunc func) {\n libwxfreq::SetDumpFunc(func);\n}\n" }, { "path": "freq/libwxfreq.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_FREQLIB_H_\n#define FREQLIB_FREQ_FREQLIB_H_\n#include \n#include \n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nenum {\n kOK = 0,\n kErrorOutOfMemory = -1,\n kErrorNoAppName = -2,\n kErrorSystem = -3,\n kErrorInput = -4,\n};\n\nstruct BlockResult {\n int block_level;\n char match_rule[32];\n};\n\nstruct FreqCache {\n uint32_t level1_cnt;\n uint32_t level2_cnt;\n uint32_t level3_cnt;\n};\n\nint RegisterNewShmStat(const char* type_name, bool zero_init, key_t key,\n unsigned int item_cnt);\nint RegisterNewMapStat(const char* type_name, bool zero_init, key_t key,\n unsigned int item_cnt);\n\nvoid SetRuleConfFile(const char *filename);\n\nbool InitFreq();\n\nstruct BlockResult ReportAndCheck(const char* type_name, const char *key,\n const uint32_t appid, const uint32_t cnt);\nstruct BlockResult OnlyCheck(const char* type_name, const char *key,\n const uint32_t appid);\nint OnlyReport(const char* type_name, const char *key, const uint32_t appid,\n const uint32_t cnt);\nstruct FreqCache GetCache(const char* type_name, const char *key,\n const uint32_t appid, const uint32_t cnt = 0);\n\nint AddWhite(const char* type_name, const char *key, const uint32_t appid,\n const uint32_t linger_time);\nint DeleteWhite(const char* type_name, const char *key, const uint32_t appid);\nint AddBlock(const char* type_name, const char *key, const uint32_t appid,\n const uint32_t linger_time, const uint32_t block_level);\nint DeleteBlock(const char* type_name, const char *key, const uint32_t appid);\n\ntypedef int (*LogFunction)(const char* format, ...);\nvoid SetLogFunc(LogFunction logfunc);\n\ntypedef int (*LoadFunc)(const char* type_name);\nvoid SetLoadFunc(LoadFunc func);\n\ntypedef int (*DumpFunc)(const char* opname, const char* type_name,\n const char *key, const uint32_t appid,\n const unsigned int linger_time,\n const uint32_t block_level);\nvoid SetDumpFunc(DumpFunc func);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif // FREQLIB_FREQ_FREQLIB_H_\n" }, { "path": "freq/plain_expression.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"plain_expression.h\"\n#include \n#include \n#include \n#include \"slice.h\"\n#include \"log.h\"\n\nnamespace libwxfreq {\n\nstatic bool Greater(const FreqItem& freqitem, const unsigned int index,\n const unsigned int threshold) {\n if (index >= 3) return false;\n const unsigned int *p = (const unsigned int *)&freqitem;\n return p[index] > threshold;\n}\n\nstatic bool GreaterOrEqual(const FreqItem& freqitem, const unsigned int index,\n const unsigned int threshold) {\n if (index >= 3) return false;\n const unsigned int *p = (const unsigned int *)&freqitem;\n return p[index] >= threshold;\n}\n\nbool PlainExpression::ParseItem(const std::string& item) {\n const char *ptr = NULL;\n ptr = strstr(item.c_str(), \">\");\n if (ptr == NULL) return false;\n Slice keyword(item.c_str(), ptr);\n keyword.StrTrim(\" \");\n if (keyword == \"min_interval\") {\n key_index_ = 0;\n } else if (keyword == \"mid_interval\") {\n key_index_ = 1;\n } else if (keyword == \"max_interval\") {\n key_index_ = 2;\n } else {\n gLog(\"[%s][%d]: wrong keyword %s, item %s\\n\",\n __FILE__, __LINE__, keyword.ToStr().c_str(), item.c_str());\n return false;\n }\n ptr++;\n if (*ptr == '=') {\n ptr++;\n cmp_ = GreaterOrEqual;\n } else {\n cmp_ = Greater;\n }\n\n if (*ptr == '\\0') {\n gLog(\"[%s][%d]: wrong syntax item %s\\n\", __FILE__, __LINE__, item.c_str());\n return false;\n }\n threshold_ = strtoul(ptr, NULL, 10);\n return threshold_ > 0;\n}\n\nbool PlainExpression::IsMatch(const FreqItem& freqitem) const {\n return cmp_(freqitem, key_index_, threshold_);\n}\n\nstd::string PlainExpression::DebugString() const {\n std::string op = \"NULL\";\n if (cmp_ == GreaterOrEqual) op = \"GreaterOrEqual\";\n else if (cmp_ == Greater) op = \"Greater\";\n\n char buf[128];\n snprintf(buf, sizeof(buf), \"key_index:%u\\tthreshold:%u\\tcmp:%s\\taddr:%lx\",\n key_index_, threshold_, op.c_str(), (unsigned long int)cmp_);\n\n return std::string(buf);\n}\n\nExpression* PlainExpression::Clone() const {\n return new PlainExpression(cmp_, threshold_, key_index_);\n}\n\n} // namespace libwxfreq\n" }, { "path": "freq/plain_expression.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_PLAIN_EXPRESSION_H_\n#define FREQLIB_FREQ_PLAIN_EXPRESSION_H_\n\n#include \n#include \"expression.h\"\n#include \"log.h\"\n\nnamespace libwxfreq {\n\nclass PlainExpression: public Expression {\n public:\n PlainExpression() { }\n\n bool ParseItem(const std::string& item);\n bool IsMatch(const FreqItem& freqitem) const;\n std::string DebugString() const;\n Expression* Clone() const;\n\n private:\n typedef bool (*cmp)(const FreqItem&, const unsigned int, const unsigned int);\n\n PlainExpression(const cmp func, const unsigned int threshold,\n const unsigned char key_index)\n : cmp_(func),\n threshold_(threshold),\n key_index_(key_index) { }\n\n // parse item\n cmp cmp_;\n unsigned int threshold_;\n uint8_t key_index_;\n};\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_FREQ_PLAIN_EXPRESSION_H_\n" }, { "path": "freq/rule_item.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_RULE_ITEM_H_\n#define FREQLIB_FREQ_RULE_ITEM_H_\n\n#include \n#include \n#include \"expression.h\"\n\nnamespace libwxfreq {\nclass FreqItem;\n\nclass RuleItem {\n public:\n RuleItem(const std::string& rulename, uint16_t blocklevel)\n : match_rulename_(rulename),\n block_level_(blocklevel),\n expression_(NULL) { }\n\n RuleItem(const RuleItem& other) {\n *this = other;\n expression_ = other.expression_->Clone();\n }\n\n ~RuleItem() {\n if (expression_ != NULL) delete expression_;\n }\n\n inline uint16_t block_level() const {\n return block_level_;\n }\n\n inline const std::string& match_rulename() const {\n return match_rulename_;\n }\n\n inline void set_match_rulename(const std::string& match_rulename) {\n match_rulename_ = match_rulename;\n }\n\n inline void set_block_level(uint16_t block_level) {\n block_level_ = block_level;\n }\n\n inline void set_expression(Expression* p) {\n if (expression_ != NULL) delete expression_;\n expression_ = p;\n }\n\n inline Expression* expression() {\n return expression_;\n }\n\n inline bool ParseItem(const std::string& item) {\n return expression_->ParseItem(item);\n }\n\n inline bool IsMatch(const FreqItem& freqitem) const {\n return expression_->IsMatch(freqitem);\n }\n\n inline std::string DebugString() {\n std::string expression = expression_->DebugString();\n char buf[512];\n snprintf(buf, sizeof(buf), \"match_rulename:%s\\tblock_level:%u\\t\"\n \"expression:%s\", match_rulename_.c_str(), block_level_,\n expression.c_str());\n return std::string(buf);\n }\n\n private:\n std::string match_rulename_;\n uint16_t block_level_;\n Expression* expression_;\n};\n\n} // namespace libwxfreq\n#endif // FREQLIB_FREQ_RULE_ITEM_H_\n" }, { "path": "freq/rule_manager.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"rule_manager.h\"\n#include \n#include \n#include \n#include \"plain_expression.h\"\n#include \"config.h\"\n#include \"log.h\"\n#include \"appid_manager.h\"\n#include \"freq_item.h\"\n#include \"rule_item.h\"\n#include \"freq_manager.h\"\n\nnamespace libwxfreq {\n\nstd::string RuleManager::rule_conf_file_;\nbool RuleManager::is_loading_ = false;\ntime_t RuleManager::last_modify_time_ = 0;\nunsigned char RuleManager::index_ = 0;\nTypeNameMap RuleManager::map_[2];\n\nint RuleManager::Init() {\n return TryLoadFile();\n}\n\nint RuleManager::TryLoadFile() {\n if (rule_conf_file_.empty()) {\n gLog(\"[%s][%d]: rule confile empty\\n\", __FILE__, __LINE__);\n return -1;\n }\n\n struct stat _filestat;\n if (stat(rule_conf_file_.c_str(), &_filestat) < 0) {\n gLog(\"[%s][%d]: stat confile %s failed\\n\",\n __FILE__, __LINE__, rule_conf_file_.c_str());\n return -2;\n }\n if (_filestat.st_mtime > last_modify_time_ &&\n time(NULL) - _filestat.st_mtime >= 2) {\n if (__sync_bool_compare_and_swap(&is_loading_, false, true)) {\n last_modify_time_ = _filestat.st_mtime;\n LoadFile();\n is_loading_ = false;\n }\n }\n return 0;\n}\n\nint RuleManager::LoadFile() {\n gLog(\"[%s][%d]: Load confile %s\\n\",\n __FILE__, __LINE__, rule_conf_file_.c_str());\n Config cfg(rule_conf_file_);\n if (cfg.Init() != 0) {\n gLog(\"[%s][%d]: int confile %s failed\\n\",\n __FILE__, __LINE__, rule_conf_file_.c_str());\n return -1;\n }\n\n unsigned char new_index = (index_ + 1) % 2;\n map_[new_index].clear();\n\n std::vector sectionList;\n cfg.section_list(sectionList);\n for (std::vector::iterator it = sectionList.begin();\n it != sectionList.end(); ++it) {\n if (*it != \"appid\") InsertRule(cfg, it->c_str());\n }\n\n AppidManager::UpdateAppidMeta(cfg);\n\n index_ = new_index;\n gLog(\"[%s][%d]: new index %d\\n\", __FILE__, __LINE__, index_);\n return 0;\n}\n\nint RuleManager::InsertRule(const Config& config, const std::string& section) {\n std::string item;\n std::string rule_type;\n unsigned int block_level = 0;\n unsigned int match_appid;\n unsigned int new_index = (index_ + 1) % 2;\n\n int ret1 = config.ReadItem(section, \"item\", \"\", item);\n int ret2 = config.ReadItem(section, \"match_appid\", 0, match_appid);\n int ret3 = config.ReadItem(section, \"block_level\", 0, block_level);\n int ret4 = config.ReadItem(section, \"rule_type\", \"user\", rule_type);\n\n if (ret1 != 0 || ret2 != 0 || ret3 != 0 || ret4 < 0) {\n gLog(\"[%s][%d]: insert rule section %s failed %d|%d|%d|%d\\n\",\n __FILE__, __LINE__, section.c_str(), ret1, ret2, ret3, ret4);\n return -1;\n }\n\n if (block_level <= 0) {\n gLog(\"[%s][%d]: insert rule section %s block_level %d, skip\\n\",\n __FILE__, __LINE__, section.c_str(), block_level);\n return -2;\n }\n\n RuleItem rule_item(section, block_level);\n rule_item.set_expression(new PlainExpression());\n if (rule_item.ParseItem(item) == false) {\n gLog(\"[%s][%d]: insert rule section %s parse failed skip\\n\",\n __FILE__, __LINE__, section.c_str());\n return -3;\n }\n\n map_[new_index][rule_type][match_appid].push_back(rule_item);\n\n gLog(\"[%s][%d]: insert rule section %s match_appid %u block_level %d\"\n \" rule_type %s item %s rule_item %s\\n\", __FILE__, __LINE__,\n section.c_str(), match_appid, block_level, rule_type.c_str(),\n item.c_str(), rule_item.DebugString().c_str());\n return 0;\n}\n\nstd::string RuleManager::DebugString() {\n TryLoadFile();\n\n TypeNameMap& current_map = map_[index_];\n std::string rule_string;\n for (TypeNameMap::iterator cit = current_map.begin();\n cit != current_map.end(); ++cit) {\n for (AppidMapVecRuleItem::iterator it = cit->second.begin();\n it != cit->second.end(); ++it) {\n for (std::vector::iterator vit = it->second.begin();\n vit != it->second.end(); ++vit) {\n rule_string.append(\"match_appid:\\t\");\n rule_string.append(std::to_string(it->first));\n rule_string.append(\"\\n\");\n rule_string.append(vit->DebugString());\n }\n }\n }\n return rule_string;\n}\n\nRuleManager::Result RuleManager::IsMatch(const uint32_t appid,\n const FreqItem& freq_item,\n const AppidMapVecRuleItem& map) {\n TryLoadFile();\n\n AppidMapVecRuleItem::const_iterator it = map.find(appid);\n if (it == map.end()) {\n return RuleManager::Result(0, \"\");\n }\n\n for (std::vector::const_iterator vit = it->second.begin();\n vit != it->second.end(); ++vit) {\n if (vit->IsMatch(freq_item)) {\n RuleManager::Result res(vit->block_level(), vit->match_rulename());\n return res;\n }\n }\n return RuleManager::Result(0, \"\");\n}\n\nRuleManager::Result RuleManager::IsMatchRule(const char * type_name,\n const uint32_t appid,\n const FreqItem& freq_item) {\n if (type_name == NULL || FreqManager::GetWhiteLevel(&freq_item) > 0) {\n return RuleManager::Result(0, \"\");\n }\n if (FreqManager::GetBlackLevel(&freq_item) > 0) {\n return RuleManager::Result(FreqManager::GetBlackLevel(&freq_item), \"black\");\n }\n TypeNameMap::iterator it = map_[index_].find(type_name);\n if (it == map_[index_].end()) {\n return RuleManager::Result(0, \"\");\n }\n return IsMatch(appid, freq_item, it->second);\n}\n\n} // namespace libwxfreq\n" }, { "path": "freq/rule_manager.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_FREQ_RULE_MANAGER_H_\n#define FREQLIB_FREQ_RULE_MANAGER_H_\n#include \n#include \n#include \n#include \n\nnamespace libwxfreq {\n\nclass Config;\nclass FreqItem;\nclass RuleItem;\n\ntypedef std::map > AppidMapVecRuleItem;\ntypedef std::map TypeNameMap;\n\nclass RuleManager {\n public:\n struct Result {\n int block_level;\n std::string match_rule;\n\n Result(): block_level(0), match_rule() { }\n Result(int b, const std::string& matchrule)\n : block_level(b),\n match_rule(matchrule) { }\n };\n\n public:\n static inline void set_rule_conf_file(const std::string& filename) {\n rule_conf_file_ = filename;\n }\n static int Init();\n\n static std::string DebugString();\n static Result IsMatchRule(const char * type_name, const uint32_t appid,\n const FreqItem& freq_item);\n\n private:\n static int LoadFile();\n static int TryLoadFile();\n static int InsertRule(const Config& config, const std::string& section);\n\n static Result IsMatch(const uint32_t appid, const FreqItem& freq_item,\n const AppidMapVecRuleItem& map);\n\n private:\n static std::string rule_conf_file_;\n static bool is_loading_;\n static time_t last_modify_time_;\n\n static TypeNameMap map_[2];\n static unsigned char index_;\n};\n\n} // namespace libwxfreq\n#endif // FREQLIB_FREQ_RULE_MANAGER_H_\n" }, { "path": "main.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \n#include \n#include \n#include \n#include \"server.h\"\n#include \"worker.h\"\n#include \"libwxfreq.h\"\n\nusing std::cout;\nusing std::endl;\n\nstatic inline int Log(const char* format, ...) {\n return 0;\n}\n\nint main(int argc, char* argv[]) {\n libwxfreq::ServerConf& conf = libwxfreq::ServerConf::GetInstance();\n conf.LoadFile(\"./server.conf\");\n SetLogFunc(Log);\n\n SetRuleConfFile(conf.libwxfreq_conf_path().c_str());\n const std::vector& type = conf.stat_vec();\n for (std::vector::const_iterator it = type.cbegin();\n it != type.cend(); ++it) {\n int ret = RegisterNewShmStat(it->GetTypeName().c_str(), it->init() > 0,\n it->shm_key(), it->item_count());\n if (ret != 0) {\n cout << \"RegisterNewShmStat failed, ret = \" << ret << endl;\n return -1;\n }\n }\n\n if (InitFreq() == false) {\n cout << \"InitFreq failed\" << endl;\n return -2;\n }\n\n libwxfreq::Server server(libwxfreq::TxtFreqReq);\n // libwxfreq::Server server(ip, port, libwxfreq::Echo, worker_num);\n server.Run();\n return 0;\n}\n" }, { "path": "net/acceptor.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"acceptor.h\"\n#include \"server.h\"\nnamespace libwxfreq {\nAcceptor::Acceptor(Server* s)\n :s_(s) {\n}\n\nvoid Acceptor::LoopAccept(const char *ip, const int port) {\n int listen_fd = socket(AF_INET, SOCK_STREAM, 0);\n struct sockaddr_in addr;\n memset(&addr, 0, sizeof(addr));\n addr.sin_family = AF_INET;\n addr.sin_port = htons(port);\n addr.sin_addr.s_addr = inet_addr(ip);\n if (bind(listen_fd, (struct sockaddr*) &addr, sizeof(addr)) < 0) {\n close(listen_fd);\n exit(-1);\n }\n if (listen(listen_fd, 1024) < 0) {\n close(listen_fd);\n exit(-1);\n }\n while (true) {\n struct sockaddr_in addr_in;\n socklen_t socklen = sizeof(addr_in);\n int accepted_fd = accept(listen_fd, (struct sockaddr*) &addr_in, &socklen);\n if (accepted_fd > 0) {\n s_->pool().AddNewConn(accepted_fd);\n }\n }\n close(listen_fd);\n}\n\n} // namespace libwxfreq\n" }, { "path": "net/acceptor.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef LIBFEQ_NET_ACCEPTOR_H_\n#define LIBFEQ_NET_ACCEPTOR_H_\n\nnamespace libwxfreq {\nclass Server;\nclass Acceptor {\n public:\n explicit Acceptor(Server *s);\n void LoopAccept(const char *ip, const int port);\n private:\n Server* s_;\n};\n\n} // namespace libwxfreq\n\n#endif // LIBFEQ_NET_ACCEPTOR_H_\n" }, { "path": "net/iothread.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"iothread.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"util.h\"\n#include \"server_conf.h\"\n\nnamespace libwxfreq {\nusing std::string;\nusing std::max;\n\nstatic NotifySessionStat notify_stat;\nstatic CloseSessionStat close_stat;\nstatic ReadSessionStat read_stat;\nstatic CheckSessionStat check_stat;\nstatic WriteSessionStat write_stat;\nstatic ProcessSessionStat process_stat;\n\nstatic size_t ValidatePkg(const char* buf, size_t len) {\n const char *pn = static_cast(memchr(buf, '\\n', len));\n const char *pr = static_cast(memchr(buf, '\\r', len));\n if (pn == NULL && pr == NULL) {\n return -1;\n }\n const char *p = pn != NULL ? pn : pr;\n if (pr != NULL && pr < p) p = pr;\n while ((*p == '\\0' || *p == '\\r' || *p == '\\n' ||\n *p == ' ' || *p == '\\t') && p < buf + len) p++;\n return p - buf;\n}\n\n\nSKBuff::SKBuff():buffer_(array_), write_pos_(0), read_pos_(0),\n capacity_(LIBFREQ_BUFFER_SIZE) {\n buffer_[0] ='\\0';\n}\n\nSKBuff::~SKBuff() {\n if (buffer_ != array_) {\n delete[] buffer_;\n }\n}\n\nchar* SKBuff::GetWritePos() {\n return buffer_ + write_pos_;\n}\n\nchar* SKBuff::GetReadPos() {\n return buffer_ + read_pos_;\n}\n\nsize_t SKBuff::AddWritePos(size_t size) {\n write_pos_ += size;\n return write_pos_;\n}\n\nsize_t SKBuff::AddReadPos(size_t size) {\n read_pos_ += size;\n return read_pos_;\n}\n\nsize_t SKBuff::GetUseableSize() {\n if (capacity_ - write_pos_ < 32) {\n ExpandSize(capacity_);\n }\n return capacity_ - write_pos_;\n}\n\nvoid SKBuff::Attach(SKBuff& buff) {\n Reset();\n write_pos_ = buff.write_pos_;\n read_pos_ = buff.read_pos_;\n capacity_ = buff.capacity_;\n if (buff.array_ == buff.buffer_) {\n buffer_ = array_;\n memcpy(buffer_, buff.GetReadPos(), buff.GetUnReadSize());\n read_pos_ = 0;\n write_pos_ = buff.GetUnReadSize();\n } else {\n buffer_ = buff.buffer_;\n buff.buffer_ = NULL;\n }\n}\nvoid SKBuff::Attach(const char* buff, size_t len) {\n Reset();\n ExpandSize(len);\n memcpy(buffer_, buff, len);\n write_pos_ = len;\n}\n\nchar* SKBuff::GetBuffer() {\n return buffer_;\n}\n\nsize_t SKBuff::GetBufferLen() {\n return write_pos_;\n}\n\nsize_t SKBuff::GetUnReadSize() {\n return write_pos_ - read_pos_;\n}\n\nvoid SKBuff::Reset() {\n if (buffer_ != array_) delete[] buffer_;\n buffer_ = array_;\n write_pos_ = 0;\n read_pos_ = 0;\n capacity_ = LIBFREQ_BUFFER_SIZE;\n buffer_[0] = '\\0';\n}\n\nvoid SKBuff::Shrink() {\n memmove(buffer_, buffer_ + read_pos_, GetUnReadSize());\n write_pos_ -= read_pos_;\n read_pos_ = 0;\n}\n\nvoid SKBuff::ExpandSize(size_t len) {\n if (len > capacity_ - write_pos_) {\n capacity_ = max(capacity_ * 2, len + write_pos_);\n char * tmp = new char[capacity_];\n memcpy(tmp, buffer_, write_pos_);\n if (buffer_ != array_) delete[] buffer_;\n buffer_ = tmp;\n }\n}\n\nvoid IOThread::run() {\n thread_->base = event_base_new();\n ConnSession *session = new ConnSession();\n session->fd = thread_->receive_fd;\n session->iothread = this;\n session->stat = ¬ify_stat;\n SetSockNonBlock(session->fd);\n event_set(&session->event, thread_->receive_fd,\n EV_READ | EV_PERSIST, ConnSession::Handler, session);\n\n event_base_set(thread_->base, &session->event);\n\n if (event_add(&session->event, 0) != 0) {\n exit(-1);\n }\n\n // run thread\n if (pthread_create(&thread_->tid, NULL, IOThread::ThreadEntry, thread_) != 0) {\n exit(-1);\n }\n}\n\nIOThread::IOThread(Thread * t)\n :thread_(t) {\n}\n\nThread& IOThread::thread() {\n return *thread_;\n}\n\nvoid* IOThread::ThreadEntry(void *args) {\n Thread *thread = static_cast(args);\n event_base_loop(thread->base, 0);\n event_base_free(thread->base);\n return NULL;\n}\n\nvoid ConnSession::Handler(int fd, int16_t event, void* args) {\n ConnSession *session = static_cast(args);\n if (event & EV_TIMEOUT) {\n session->stat = &close_stat;\n }\n while (session->stat->Handler(event, session) > 0) {\n continue;\n }\n}\n\nConnSession::~ConnSession() {\n event_del(&event);\n event_del(&timeout_event);\n close(fd);\n}\n\nconst std::string& NotifySessionStat::GetName() const {\n static std::string name = \"notify\";\n return name;\n}\n\nint NotifySessionStat::Handler(int16_t event, ConnSession* session) {\n char buf[1];\n int count = read(session->fd, buf, sizeof(buf));\n if (count <= 0) return kDone;\n\n ConnItem *item = session->iothread->thread().queue.DeQueue();\n if (item == NULL) return kDone;\n\n ConnSession *new_conn_session = new ConnSession();\n new_conn_session->fd = item->fd;\n SetSockNonBlock(item->fd);\n new_conn_session->iothread = session->iothread;\n new_conn_session->stat = &read_stat;\n new_conn_session->keep_alive = false;\n\n event_set(&new_conn_session->event, new_conn_session->fd,\n EV_READ | EV_PERSIST, ConnSession::Handler, new_conn_session);\n event_base_set(session->iothread->thread().base, &new_conn_session->event);\n\n event_set(&new_conn_session->timeout_event, -1, 0,\n ConnSession::Handler, new_conn_session);\n event_base_set(session->iothread->thread().base, &new_conn_session->timeout_event);\n\n ServerConf& conf = ServerConf::GetInstance();\n struct timeval tv = {conf.timeout(), 0};\n if (event_add(&new_conn_session->event, 0) != 0 ||\n event_add(&new_conn_session->timeout_event, &tv) != 0) {\n new_conn_session->stat = &close_stat;\n return kContinue;\n }\n return kDone;\n}\n\nconst std::string& CloseSessionStat::GetName() const {\n static std::string name = \"close\";\n return name;\n}\n\nint CloseSessionStat::Handler(int16_t event, ConnSession* session) {\n delete session;\n return kDone;\n}\n\nconst std::string& ReadSessionStat::GetName() const {\n static std::string name = \"read\";\n return name;\n}\n\nint ReadSessionStat::Handler(int16_t event, ConnSession* session) {\n int read_size = read(session->fd, session->recv.GetWritePos(), session->recv.GetUseableSize());\n if (read_size <= 0) {\n session->stat = &close_stat;\n return kContinue;\n }\n session->recv.AddWritePos(read_size);\n session->stat = &check_stat;\n return kContinue;\n}\n\nconst std::string& CheckSessionStat::GetName() const {\n static std::string name = \"check\";\n return name;\n}\n\nint CheckSessionStat::Handler(int16_t event, ConnSession* session) {\n int pkglen = ValidatePkg(session->recv.GetBuffer(), session->recv.GetBufferLen());\n if (pkglen <= 0) {\n session->stat = &read_stat;\n return kDone; // continue read more data\n }\n session->in.Attach(session->recv.GetReadPos(), pkglen);\n session->recv.AddReadPos(pkglen);\n session->recv.Shrink();\n session->stat = &process_stat;\n return kContinue;\n}\n\nconst std::string& WriteSessionStat::GetName() const {\n static std::string name = \"write\";\n return name;\n}\n\nint WriteSessionStat::Handler(int16_t event, ConnSession* session) {\n size_t size = write(session->fd, session->out.GetReadPos(), session->out.GetUnReadSize());\n if (size < 0) {\n session->stat = &close_stat;\n return kContinue;\n }\n if (size < session->out.GetUnReadSize()) {\n session->out.AddReadPos(size);\n return kDone; // continue to write;\n }\n if (session->keep_alive) {\n event_del(&session->event);\n event_set(&session->event, session->fd,\n EV_READ | EV_PERSIST, ConnSession::Handler, session);\n\n event_base_set(session->iothread->thread().base, &session->event);\n if (event_add(&session->event, 0) != 0) {\n session->stat = &close_stat;\n return kContinue;\n }\n session->stat = &check_stat;\n session->out.Reset();\n return kContinue;\n } else {\n session->stat = &close_stat;\n return kContinue;\n }\n return kDone;\n}\n\nconst std::string& ProcessSessionStat::GetName() const {\n static std::string name = \"process\";\n return name;\n}\n\nint ProcessSessionStat::Handler(int16_t event, ConnSession* session) {\n if (strncmp(session->in.GetBuffer(), \"keep_alive\", 10) == 0) {\n session->keep_alive = true;\n session->in.Reset();\n event_del(&session->timeout_event);\n session->stat = &check_stat;\n return kContinue;\n }\n\n string req(session->in.GetBuffer(), session->in.GetBufferLen());\n string resp;\n session->iothread->thread().dispatch(req, resp);\n session->out.Attach(resp.data(), resp.size());\n\n event_del(&session->event);\n event_set(&session->event, session->fd,\n EV_WRITE | EV_PERSIST, ConnSession::Handler, session);\n event_base_set(session->iothread->thread().base, &session->event);\n if (event_add(&session->event, 0) != 0) {\n session->stat = &close_stat;\n return kContinue;\n }\n session->stat = &write_stat;\n return kDone;\n}\n\n} // namespace libwxfreq\n" }, { "path": "net/iothread.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef LIBFREQ_NET_IOTHREAD_H_\n#define LIBFREQ_NET_IOTHREAD_H_\n#include \n#include \n#include \"server_inner.h\"\n\n#define LIBFREQ_BUFFER_SIZE 128\nnamespace libwxfreq {\nclass IOThread {\n public:\n explicit IOThread(Thread * t);\n void run();\n Thread& thread();\n private:\n static void* ThreadEntry(void *);\n Thread *thread_;\n};\n\nclass ConnSession;\nclass SessionStat {\n public:\n virtual const std::string& GetName() const = 0;\n virtual int Handler(int16_t event, ConnSession* session) = 0;\n protected:\n enum {\n kDone = 0,\n kContinue = 1,\n };\n};\n\nclass NotifySessionStat:public SessionStat {\n public:\n const std::string& GetName() const;\n int Handler(int16_t event, ConnSession* session);\n};\n\nclass CloseSessionStat:public SessionStat {\n public:\n const std::string& GetName() const;\n int Handler(int16_t event, ConnSession* session);\n};\n\nclass ReadSessionStat:public SessionStat {\n public:\n const std::string& GetName() const;\n int Handler(int16_t event, ConnSession* session);\n};\n\nclass CheckSessionStat:public SessionStat {\n public:\n const std::string& GetName() const;\n int Handler(int16_t event, ConnSession* session);\n};\n\nclass WriteSessionStat:public SessionStat {\n public:\n const std::string& GetName() const;\n int Handler(int16_t event, ConnSession* session);\n};\n\nclass ProcessSessionStat:public SessionStat {\n public:\n const std::string& GetName() const;\n int Handler(int16_t event, ConnSession* session);\n};\n\nclass SKBuff {\n public:\n SKBuff();\n ~SKBuff();\n char* GetWritePos();\n char* GetReadPos();\n size_t GetUseableSize();\n void Attach(SKBuff& buff);\n void Attach(const char* buff, size_t len);\n char* GetBuffer();\n size_t GetBufferLen();\n size_t GetUnReadSize();\n void Reset();\n void Shrink();\n size_t AddWritePos(size_t size);\n size_t AddReadPos(size_t size);\n private:\n void ExpandSize(size_t len);\n private:\n char array_[LIBFREQ_BUFFER_SIZE];\n char* buffer_;\n size_t write_pos_;\n size_t read_pos_;\n size_t capacity_;\n};\n\nstruct ConnSession {\n int fd;\n bool keep_alive;\n SessionStat *stat;\n SKBuff recv;\n SKBuff in;\n SKBuff out;\n IOThread* iothread;\n struct event event;\n struct event timeout_event;\n static void Handler(int, int16_t, void *);\n ~ConnSession();\n};\n} // namespace libwxfreq\n\n#endif // LIBFREQ_NET_IOTHREAD_H_\n" }, { "path": "net/server.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"server.h\"\n#include \"util.h\"\n\nnamespace libwxfreq {\nServer::Server(const char *ip, const int port, Dispatch func,\n const int worker_num)\n :server_ip_(ip), listent_port_(port), pool_(func, worker_num),\n acceptor_(this) {\n}\n\nServer::Server(Dispatch func)\n :pool_(func), acceptor_(this) {\n const ServerConf& conf = ServerConf::GetInstance();\n server_ip_ = conf.listen_ip();\n listent_port_ = conf.listen_port();\n pool_.SetWorkerNum(conf.worker_num());\n}\n\nvoid Server::Run() {\n if (Daemonize() != 0) exit(-1);\n if (!pool_.Run()) exit(-1);\n acceptor_.LoopAccept(server_ip_.c_str(), listent_port_);\n}\n\nWorkerPool& Server::pool() {\n return pool_;\n}\n\n} // namespace libwxfreq\n" }, { "path": "net/server.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef LIBFREQ_NET_SERVER_H_\n#define LIBFREQ_NET_SERVER_H_\n\n#include \n#include \n#include \"acceptor.h\"\n#include \"workerpool.h\"\n#include \"server_conf.h\"\n\nnamespace libwxfreq {\n\nclass Server {\n public:\n Server(const char *ip, const int port, Dispatch func,\n const int worker_num = 4);\n explicit Server(Dispatch func);\n void Run();\n WorkerPool& pool();\n private:\n int worker_num_;\n std::string server_ip_;\n int listent_port_;\n WorkerPool pool_;\n Acceptor acceptor_;\n};\n\n} // namespace libwxfreq\n\n#endif // LIBFREQ_NET_SERVER_H_\n" }, { "path": "net/server_conf.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"server_conf.h\"\n#include \"config.h\"\n\nnamespace libwxfreq {\nusing std::string;\nusing std::to_string;\n\nStatType::StatType(const StatType& left):typename_(left.typename_),\n init_(left.init_), shm_key_(left.shm_key_), item_count_(left.item_count_) {\n}\n\nStatType::StatType(const std::string& tyname, uint32_t init,\n uint32_t shm_key, uint32_t item_count)\n :typename_(tyname), init_(init), shm_key_(shm_key),\n item_count_(item_count) {\n}\n\nconst std::string& StatType::GetTypeName() const {\n return typename_;\n}\nuint32_t StatType::init() const {\n return init_;\n}\nuint32_t StatType::shm_key() const {\n return shm_key_;\n}\n\nuint32_t StatType::item_count() const {\n return item_count_;\n}\n\nServerConf& ServerConf::GetInstance() {\n static ServerConf conf_;\n return conf_;\n}\n\nServerConf::ServerConf() {\n has_init_ = false;\n}\n\nbool ServerConf::LoadFile(const std::string& file) {\n if (has_init_) return true;\n\n Config config(file);\n if (config.Init() != 0) return false;\n\n int32_t stat_num = 0;\n config.ReadItem(\"General\", \"libwxfreq_conf_path\", \"\", libwxfreq_conf_path_);\n config.ReadItem(\"General\", \"stat_type_num\", 0, stat_num);\n\n config.ReadItem(\"Server\", \"listen_ip\", \"\", listen_ip_);\n config.ReadItem(\"Server\", \"listen_port\", 0, listen_port_);\n config.ReadItem(\"Server\", \"worke_num\", 0, worke_num_);\n config.ReadItem(\"Server\", \"timeout\", 1, timeout_);\n stat_vec_.clear();\n for (int i = 0; i < stat_num; i++) {\n string typename_, section_name = \"Stat_\";\n uint32_t init = 0, shm_key = 0, item_count;\n section_name += to_string(i);\n\n config.ReadItem(section_name, \"typename\", \"\", typename_);\n config.ReadItem(section_name, \"init\", 0, init);\n config.ReadItem(section_name, \"shm_key\", 0, shm_key);\n config.ReadItem(section_name, \"item_count\", 0, item_count);\n\n StatType type(typename_, init, shm_key, item_count);\n stat_vec_.push_back(type);\n }\n\n if (stat_num > 0) {\n has_init_ = true;\n }\n return has_init_;\n}\n\nconst std::string ServerConf::listen_ip() const {\n return listen_ip_;\n}\nconst std::string ServerConf::libwxfreq_conf_path() const {\n return libwxfreq_conf_path_;\n}\nuint32_t ServerConf::listen_port() const {\n return listen_port_;\n}\nuint32_t ServerConf::worker_num() const {\n return worke_num_;\n}\nconst std::vector& ServerConf::stat_vec() const {\n return stat_vec_;\n}\nuint32_t ServerConf::timeout() const {\n return timeout_;\n}\n\n} // namespace libwxfreq\n\n" }, { "path": "net/server_conf.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef LIBFREQ_NET_SERVER_CONF_H_\n#define LIBFREQ_NET_SERVER_CONF_H_\n#include \n#include \n#include \n\n\nnamespace libwxfreq {\nclass StatType {\n public:\n StatType(const StatType& left);\n StatType(const std::string& tyname, uint32_t init,\n uint32_t shm_key, uint32_t item_count);\n const std::string& GetTypeName() const;\n uint32_t init() const;\n uint32_t shm_key() const;\n uint32_t item_count() const;\n\n private:\n std::string typename_;\n uint32_t init_;\n uint32_t shm_key_;\n uint32_t item_count_;\n};\n\nclass ServerConf {\n public:\n static ServerConf& GetInstance();\n ServerConf(const ServerConf& a) = delete;\n bool LoadFile(const std::string& file);\n const std::string listen_ip() const;\n const std::string libwxfreq_conf_path() const;\n uint32_t listen_port() const;\n uint32_t worker_num() const;\n const std::vector& stat_vec() const;\n uint32_t timeout() const;\n\n private:\n ServerConf();\n\n private:\n std::string libwxfreq_conf_path_;\n uint32_t stat_type_num_;\n std::string listen_ip_;\n uint32_t listen_port_;\n uint32_t worke_num_;\n uint32_t timeout_;\n std::vector stat_vec_;\n bool has_init_;\n};\n\n} // namespace libwxfreq\n#endif // LIBFREQ_NET_SERVER_CONF_H_\n" }, { "path": "net/server_inner.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"server_inner.h\"\n\nnamespace libwxfreq {\nvoid ConnQueue::EnQueue(int fd) {\n if ((head_ + 1) % 1024 != tail_) {\n array_[head_++].fd = fd;\n head_ %= 1024;\n }\n}\n\nConnItem* ConnQueue::DeQueue() {\n ConnItem* item = NULL;\n if (tail_ != head_) {\n item = &array_[tail_++];\n tail_ %= 1024;\n }\n return item;\n}\n\n} // namespace libwxfreq\n" }, { "path": "net/server_inner.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_NET_SERVER_INNER_H_\n#define FREQLIB_NET_SERVER_INNER_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace libwxfreq {\ntypedef std::function< void(const std::string&, std::string& resp) > Dispatch;\n\nstruct ConnItem {\n int fd;\n};\n\nclass ConnQueue {\n public:\n ConnQueue() :head_(0), tail_(0) {\n }\n void EnQueue(int fd);\n ConnItem* DeQueue();\n private:\n struct ConnItem array_[1024];\n int head_;\n int tail_;\n};\n\nclass IOThread;\n\nstruct Thread {\n pthread_t tid;\n IOThread *iothread;\n struct event_base *base;\n int receive_fd; // worker end\n int send_fd; // control end\n struct ConnQueue queue;\n Dispatch dispatch;\n};\n\n} // namespace libwxfreq\n#endif // FREQLIB_NET_SERVER_INNER_H_\n" }, { "path": "net/util.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \n#include \n#include \n#include \n#include \n\nnamespace libwxfreq {\nint Daemonize() {\n int fd = 0;\n if (fork() != 0) exit(0);\n // child process, create a new session\n if (setsid() == -1) return -1;\n\n umask(0);\n chdir(\"/\");\n if ((fd = open(\"/dev/null\", O_RDWR, 0)) != -1) {\n if (dup2(fd, STDIN_FILENO) < 0) return -1;\n if (dup2(fd, STDOUT_FILENO) < 0) return -1;\n if (dup2(fd, STDERR_FILENO) < 0) return -1;\n }\n\n signal(SIGPIPE, SIG_IGN);\n return 0;\n}\n\n\nvoid SetSockNonBlock(int sock) {\n int flags;\n flags = fcntl(sock, F_GETFL, 0);\n if (flags >= 0) {\n fcntl(sock, F_SETFL, flags | O_NONBLOCK);\n }\n}\n\n} // namespace libwxfreq\n" }, { "path": "net/util.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef LIBFREQ_NET_UTIL_H_\n#define LIBFREQ_NET_UTIL_H_\nnamespace libwxfreq {\nint Daemonize();\nvoid SetSockNonBlock(int sock);\n} // namespace libwxfreq\n\n#endif // LIBFREQ_NET_UTIL_H_\n" }, { "path": "net/worker.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"libwxfreq.h\"\nnamespace libwxfreq {\nusing std::string;\nusing std::vector;\nusing std::ostringstream;\n\nvoid Echo(const string& req, string& resp) {\n resp = req;\n}\n\nvoid SplitReq(const char* req, vector& cmd) {\n const char* end = req;\n const char* start = NULL;\n\n while(*end != '\\n' && *end != '\\r' && *end != '\\0') {\n if (*end == ' ' || *end == '\\t') {\n if (start != NULL) {\n cmd.push_back(string(start, end - start));\n start = NULL;\n }\n } else {\n if (start == NULL) {\n start = end;\n }\n }\n end++;\n }\n if (start != NULL) {\n cmd.push_back(string(start, end - start));\n }\n}\n\n\nvoid TxtFreqReq(const string& req, string& resp) {\n vector parsed_req;\n SplitReq(req.c_str(), parsed_req);\n\n if (parsed_req.size() == 0) return;\n string cmd = parsed_req[0];\n ostringstream ostr;\n\n if (strcasecmp(cmd.c_str(), \"ReportAndCheck\") == 0) {\n if (parsed_req.size() != 5) {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"wrong cmd, [ReportAndCheck type user key appid cnt]\\\"}\\n\\n\";\n } else {\n uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);\n uint32_t cnt = strtoul(parsed_req[4].c_str(), NULL, 10);\n BlockResult bs = ReportAndCheck(parsed_req[1].c_str(),\n parsed_req[2].c_str(), appid, cnt);\n ostr << \"{\\\"code\\\": 0, \\\"block_level\\\": \" << bs.block_level << \", \\\"match_rule\\\": \\\"\" << bs.match_rule <<\"\\\"}\\n\\n\";\n }\n } else if (strcasecmp(cmd.c_str(), \"OnlyCheck\") == 0) {\n if (parsed_req.size() != 4) {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"wrong cmd, [OnlyCheck type user key appid]\\\"}\\n\\n\";\n } else {\n uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);\n BlockResult bs = OnlyCheck(parsed_req[1].c_str(),\n parsed_req[2].c_str(), appid);\n ostr << \"{\\\"code\\\": 0, \\\"block_level\\\": \" << bs.block_level << \", \\\"match_rule\\\": \\\"\" << bs.match_rule << \"\\\"}\\n\\n\";\n }\n } else if (strcasecmp(cmd.c_str(), \"OnlyReport\") == 0) {\n if (parsed_req.size() != 5) {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"wrong cmd, [OnlyReport type user key appid cnt]\\\"}\\n\\n\";\n } else {\n uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);\n uint32_t cnt = strtoul(parsed_req[4].c_str(), NULL, 10);\n int ret = OnlyReport(parsed_req[1].c_str(),\n parsed_req[2].c_str(), appid, cnt);\n ostr << \"{\\\"code\\\": \" << ret << \"}\\n\\n\";\n }\n } else if (strcasecmp(cmd.c_str(), \"GetCache\") == 0) {\n if (parsed_req.size() != 4) {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"wrong cmd, [GetCache type user key appid]\\\"}\\n\\n\";\n } else {\n uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);\n FreqCache cache = GetCache(parsed_req[1].c_str(),\n parsed_req[2].c_str(), appid);\n ostr << \"{\\\"code\\\": 0, \\\"cnt1\\\": \" << cache.level1_cnt <<\", \\\"cnt2\\\": \"\n << cache.level2_cnt <<\", \\\"cnt3\\\": \" << cache.level3_cnt << \"}\\n\\n\";\n }\n } else if (strcasecmp(cmd.c_str(), \"AddWhite\") == 0) {\n if (parsed_req.size() != 5) {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"wrong cmd, [AddWhite type user key appid linger_time]\\\"}\\n\\n\";\n } else {\n uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);\n uint32_t linger_time = strtoul(parsed_req[4].c_str(), NULL, 10);\n int ret = AddWhite(parsed_req[1].c_str(), parsed_req[2].c_str(), appid, linger_time);\n ostr << \"{\\\"code\\\": \" << ret << \"}\\n\\n\";\n }\n } else if (strcasecmp(cmd.c_str(), \"DeleteWhite\") == 0) {\n if (parsed_req.size() != 4) {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"wrong cmd, [DeleteWhite type user key appid]\\\"}\\n\\n\";\n } else {\n uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);\n int ret = DeleteWhite(parsed_req[1].c_str(), parsed_req[2].c_str(), appid);\n ostr << \"{\\\"code\\\": \" << ret << \"}\\n\\n\";\n }\n } else if (strcasecmp(cmd.c_str(), \"AddBlock\") == 0) {\n if (parsed_req.size() != 6) {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"wrong cmd, [AddBlock type user key appid linger_time block_level]\\\"}\\n\\n\";\n } else {\n uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);\n uint32_t linger_time = strtoul(parsed_req[4].c_str(), NULL, 10);\n uint32_t block_level = strtoul(parsed_req[5].c_str(), NULL, 10);\n int ret = AddBlock(parsed_req[1].c_str(), parsed_req[2].c_str(), appid, linger_time, block_level);\n ostr << \"{\\\"code\\\": \" << ret << \"}\\n\\n\";\n }\n } else if (strcasecmp(cmd.c_str(), \"DeleteBlock\") == 0) {\n if (parsed_req.size() != 4) {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"wrong cmd, [DeleteBlock type user key appid]\\\"}\\n\\n\";\n } else {\n uint32_t appid = strtoul(parsed_req[3].c_str(), NULL, 10);\n int ret = DeleteBlock(parsed_req[1].c_str(), parsed_req[2].c_str(), appid);\n ostr << \"{\\\"code\\\": \" << ret << \"}\\n\\n\";\n }\n } else if (strcasecmp(cmd.c_str(), \"Help\") == 0) {\n ostr << \"cmdlist:\\n1: ReportAndCheck\\n2: OnlyCheck\\n3: OnlyReport\\n\"\n \"4: GetCache\\n5: AddWhite\\n6: DeleteWhite\\n7: AddBlock\\n\"\n \"8: DeleteBlock\\n\\n\";\n } else {\n ostr << \"{\\\"code\\\": -1, \\\"msg\\\":\\\"unkown cmd \" << cmd << \", try [help]\\\"}\\n\\n\";\n }\n\n resp = ostr.str();\n}\n\n} // namespace libwxfreq\n" }, { "path": "net/worker.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef LIBFREQ_NET_WORKER_H_\n#define LIBFREQ_NET_WORKER_H_\n#include \nnamespace libwxfreq {\nvoid Echo(const std::string& req, std::string& resp);\nvoid TxtFreqReq(const std::string& req, std::string& resp);\n} // namespace libwxfreq\n\n#endif // LIBFREQ_NET_WORKER_H_\n\n\n" }, { "path": "net/workerpool.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"workerpool.h\"\n#include \n#include \n#include \n#include \"iothread.h\"\n\nnamespace libwxfreq {\nWorkerPool::WorkerPool(Dispatch func, int num)\n :pool_(NULL), pool_size_(num),\n last_accept_worker_index_(0), func_(func) {\n}\n\nvoid WorkerPool::SetWorkerNum(int num) {\n pool_size_ = num;\n}\n\nbool WorkerPool::Run() {\n pool_ = static_cast(malloc(pool_size_ * sizeof(Thread)));\n memset(reinterpret_cast(pool_), 0, pool_size_ * sizeof(Thread));\n for (int i = 0; i < pool_size_; i++) {\n int fds[2] = {0};\n if (pipe(fds) != 0) {\n return false;\n }\n pool_[i].receive_fd = fds[0];\n pool_[i].send_fd = fds[1];\n pool_[i].dispatch = func_;\n\n pool_[i].iothread = new IOThread(&pool_[i]);\n pool_[i].iothread->run();\n }\n return true;\n}\n\nWorkerPool::~WorkerPool() {\n delete[] pool_;\n}\n\nvoid WorkerPool::AddNewConn(int fd) {\n last_accept_worker_index_ = (last_accept_worker_index_ + 1) % pool_size_;\n Thread &cur = pool_[last_accept_worker_index_];\n cur.queue.EnQueue(fd);\n Notify(cur);\n}\n\nvoid WorkerPool::Notify(Thread& thread) {\n int fd = thread.send_fd;\n char cmd[1] = {'c'};\n write(fd, cmd, 1);\n}\n\n} // namespace libwxfreq\n" }, { "path": "net/workerpool.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef LIBFREQ_NET_WORKERPOOL_H_\n#define LIBFREQ_NET_WORKERPOOL_H_\n#include \"server_inner.h\"\nnamespace libwxfreq {\nclass WorkerPool {\n public:\n explicit WorkerPool(Dispatch func, int num = 0);\n void SetWorkerNum(int num);\n ~WorkerPool();\n void AddNewConn(int fd);\n bool Run();\n private:\n void Notify(Thread& thread);\n private:\n Thread* pool_;\n int pool_size_;\n size_t last_accept_worker_index_;\n Dispatch func_;\n};\n\n} // namespace libwxfreq\n#endif // LIBFREQ_NET_WORKERPOOL_H_\n" }, { "path": "server.conf", "content": "[General]\nlibwxfreq_conf_path = ./test.conf\nstat_type_num = 2\n\n[Server]\nlisten_ip = 10.0.0.1\nlisten_port = 5678 \nworke_num = 5\ntimeout = 10\n\n[Stat_0]\ntypename = user\ninit = 0\nshm_key = 20180313\nitem_count = 100000000\n\n[Stat_1]\ntypename = ip \ninit = 0\nshm_key = 20180314\nitem_count = 100000000\n\n" }, { "path": "test/demo.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"libwxfreq/libwxfreq.h\"\n#include \n#include \n#include \n#include \n\nint main(int argc, char* argv[]) {\n if (argc != 7){\n printf(\"%s conf appid key keytype shm_key sleep_time(ms)\\n\", argv[0]);\n return 0;\n }\n SetRuleConfFile(argv[1]); \n int appid = strtoul(argv[2], NULL, 10);\n const char* key = argv[3];\n const char* keytype = argv[4];\n int shm_key = strtoul(argv[5], NULL, 10);\n int sleep_time = strtoul(argv[6], NULL, 10);\n sleep_time *= 1000;\n \n int ret = RegisterNewShmStat(keytype, false, shm_key, 1000000);\n if (ret != 0) {\n printf(\"RegisterNewShmStat failed, ret = %d\\n\", ret);\n return -1;\n }\n\n if (InitFreq() == false) {\n printf(\"InitFreq failed\\n\");\n return -2;\n }\n\n while (true) {\n struct BlockResult res;\n res = ReportAndCheck(keytype, key, appid, 1);\n printf(\"time %lu blocklevel %u matchrule %s\\n\", time(NULL),\n res.block_level, res.match_rule);\n usleep(sleep_time);\n }\n}\n" }, { "path": "test/performance_test.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"libwxfreq/libwxfreq.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nstatic inline int Log(const char* format, ...) { }\nint main(int argc, char* argv[]) {\n if (argc != 3){\n printf(\"%s conf item_count\\n\", argv[0]);\n return 0;\n }\n SetRuleConfFile(argv[1]); \n SetLogFunc(Log);\n int item_count = strtoul(argv[2], NULL, 10);\n \n int ret = RegisterNewShmStat(\"user\", false, rand(), item_count);\n if (ret != 0) {\n printf(\"RegisterNewShmStat failed, ret = %d\\n\", ret);\n return -1;\n }\n\n if (InitFreq() == false) {\n printf(\"InitFreq failed\\n\");\n return -2;\n }\n\n int oom_num = 0, first_time = 0;\n struct timeval start, end;\n gettimeofday(&start, NULL);\n printf(\"start time %u %u\\n\", start.tv_sec, start.tv_usec);\n for (int i = 0; i < item_count; i++) {\n struct BlockResult res;\n res = ReportAndCheck(\"user\", \"test\", rand(), 1);\n if (res.block_level == kErrorOutOfMemory) {\n oom_num ++;\n if (first_time == 0) first_time = i + 1;\n }\n }\n gettimeofday(&end, NULL);\n printf(\"end time %u %u\\n\", end.tv_sec, end.tv_usec);\n printf(\"average time %u per second\\n\", (unsigned int)(item_count/(end.tv_sec-start.tv_sec)));\n printf(\"oom count %u first oom %u memory usage %.3f\\n\", oom_num, first_time, 100 - 100.0 * oom_num/item_count);\n return 0;\n}\n\n" }, { "path": "test/test.conf", "content": "[appid]\n100 = 30, 60, 90\n\n[rule1]\nmatch_appid = 100\nitem = min_interval >= 10\nblock_level = 1\nrule_type = user\n\n[rule2]\nmatch_appid = 100\nitem = mid_interval >= 15 \nblock_level = 1\nrule_type = user\n\n[rule3]\nmatch_appid = 100\nitem = max_interval >= 20\nblock_level = 1\nrule_type = user\n\n[rule4]\nmatch_appid = 100\nitem = min_interval >= 100\nblock_level = 1\nrule_type = ip \n\n[rule5]\nmatch_appid = 100\nitem = mid_interval >= 150\nblock_level = 1\nrule_type = ip \n\n[rule6]\nmatch_appid = 100\nitem = max_interval >= 200 \nblock_level = 1\nrule_type = ip \n" }, { "path": "test/tools.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"libwxfreq/libwxfreq.h\"\n#include \n#include \n#include \n#include \n#include \n\nint main(int argc, char* argv[]) {\n if (argc != 9){\n printf(\"%s conf appid key keytype shm_key func level time\\n\", argv[0]);\n return 0;\n }\n SetRuleConfFile(argv[1]); \n int appid = strtoul(argv[2], NULL, 10);\n const char* key = argv[3];\n const char* keytype = argv[4];\n int shm_key = strtoul(argv[5], NULL, 10);\n const char* func = argv[6];\n int block_level = strtoul(argv[7], NULL, 10);\n int linger_time = strtoul(argv[8], NULL, 10);\n \n int ret = RegisterNewShmStat(keytype, false, shm_key, 1000000);\n if (ret != 0) {\n printf(\"RegisterNewShmStat failed, ret = %d\\n\", ret);\n return -1;\n }\n\n if (InitFreq() == false) {\n printf(\"InitFreq failed\\n\");\n return -2;\n }\n\n if (strcmp(func, \"addblock\") == 0) {\n ret = AddBlock(keytype, key, appid, linger_time, block_level);\n } else if (strcmp(func, \"deleteblock\") == 0) { \n ret = DeleteBlock(keytype, key, appid);\n } else if (strcmp(func, \"addwhite\") == 0) { \n ret = AddWhite(keytype, key, appid, linger_time);\n } else if (strcmp(func, \"deletewhite\") == 0) { \n ret = DeleteWhite(keytype, key, appid);\n } else {\n printf(\"unknown func\\n\");\n return -3;\n }\n printf(\"%s return ret %d\\n\", func, ret);\n return ret;\n}\n" }, { "path": "util/config.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"config.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"log.h\"\n#include \"slice.h\"\n\nnamespace libwxfreq {\n\n\nConfig::Config(const std::string& conf_filename) {\n conf_filename_ = conf_filename;\n}\n\nconst std::string & Config::conf_filename() {\n return conf_filename_;\n}\n\nvoid Config::set_conf_filename(const std::string &conf_filename) {\n conf_filename_ = conf_filename;\n}\n\nint Config::Init(void) {\n int ret = LoadFile();\n if (0 != ret) {\n gLog(\"[%s][%d]: init config %s error return %d\\n\", __FILE__, __LINE__,\n conf_filename_.c_str(), ret);\n Reset();\n }\n return ret;\n}\n\nint Config::Reset(void) {\n conf_filename_buf_.clear();\n section_list_.clear();\n table_.Clear();\n}\n\n\nint Config::ParserConfig() {\n Slice section, key, value;\n const char *eol = NULL, *tmp = NULL;\n const char *ptr = conf_filename_buf_.c_str();\n const char *endptr = conf_filename_buf_.c_str() + conf_filename_buf_.size();\n\n while (ptr < endptr) {\n eol = strchr(ptr, '\\n'); // getline\n if (eol == NULL) eol = endptr; // last line\n while (*ptr == ' ' || *ptr == '\\t') ptr++; // ltrim\n\n if (*ptr == '[') { // section\n ptr += 1;\n tmp = ptr;\n while (*ptr != ']' && ptr < eol) ptr++;\n section = Slice(tmp, ptr);\n section.StrTrim(\"\\t \");\n section_list_.push_back(section);\n } else if (NULL == strchr(\"#;\\n\", *ptr)) { // item\n tmp = ptr;\n while (*ptr != '=' && ptr < eol) ptr++;\n key = Slice(tmp, ptr);\n key.StrTrim(\"\\t \");\n\n if (*ptr == '=' && section.start() != NULL) {\n ptr += 1;\n tmp = ptr;\n while (ptr < eol) ptr++;\n value = Slice(tmp, ptr);\n value.StrTrim(\"\\t \");\n table_.Add(section, key, value);\n }\n }\n ptr = eol + 1; // next line\n }\n return section_list_.size() > 0 ? 0 : -1;\n}\n\nint Config::LoadFile(void) {\n FILE * fp = fopen(conf_filename_.c_str(), \"r\");\n if (NULL != fp) {\n struct stat fileStat;\n if (0 == fstat(fileno(fp), &fileStat)) {\n Reset();\n if (fileStat.st_size == 0) return 0;\n char *tmp = reinterpret_cast(malloc(fileStat.st_size + 64));\n fread(tmp, fileStat.st_size, 1, fp);\n tmp[fileStat.st_size] = '\\0'; // append '\\0' make strchr happy\n conf_filename_buf_ = tmp;\n free(tmp);\n ParserConfig();\n } else {\n gLog(\"[%s][%d]: open confile %s error\\n\",\n __FILE__, __LINE__, conf_filename_.c_str());\n }\n fclose(fp);\n fp = NULL;\n }\n return section_list_.size() == 0 ? -1 : 0;\n}\n\n\nvoid Config::section_list(std::vector& sectionlist) {\n for (size_t i = 0; i < section_list_.size(); i++) {\n sectionlist.push_back(std::string(section_list_[i].start(),\n section_list_[i].end() - section_list_[i].start()));\n }\n}\n\nvoid Config::GetKeysBySection(const std::string §ion,\n std::vector& keys) const {\n std::vector slice_keys;\n table_.GetKeysBySection(section, slice_keys);\n for (std::vector::iterator it = slice_keys.begin();\n it != slice_keys.end(); ++it) {\n keys.push_back(it->ToStr());\n }\n}\n\n\nint Config::ReadItem(const std::string& section, const std::string& key,\n const char* defaultvalue, std::string& itemvalue) const {\n if (section.size() <= 0 || key.size() <= 0 ) return -1;\n std::string value;\n Slice idxsec = section;\n Slice idxkey = key;\n Slice val = table_.Get(idxsec, idxkey);\n\n if (val.start() != NULL) {\n value.assign(val.start(), val.end() - val.start());\n std::stringstream ss(value, std::ios_base::in);\n itemvalue = ss.str();\n return 0;\n } else {\n std::stringstream ss;\n ss << defaultvalue;\n itemvalue = ss.str();\n return 1;\n }\n}\n\n} // namespace libwxfreq\n\n" }, { "path": "util/config.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_CONFIG_H_\n#define FREQLIB_UTIL_CONFIG_H_\n\n#include \n#include \n#include \n#include \"config_hash.h\"\n\nnamespace libwxfreq {\n\nclass Config {\n public:\n Config() { }\n ~Config() { }\n explicit Config(const std::string &configfile);\n\n int Init();\n const std::string & conf_filename();\n void set_conf_filename(const std::string &conf_filename);\n\n void section_list(std::vector& sectionlist);\n void GetKeysBySection(const std::string §ion,\n std::vector& keys) const;\n\n template \n int ReadItem(const std::string& section, const std::string& key,\n const T1& defaultvalue, T2& itemvalue) const;\n\n int ReadItem(const std::string& section, const std::string& key,\n const char* defaultvalue, std::string& itemvalue) const;\n\n private:\n int Reset();\n int ParserConfig();\n int LoadFile(void);\n\n std::string conf_filename_;\n std::string conf_filename_buf_;\n std::vector section_list_;\n ConfigHashTable table_;\n};\n\ntemplate \nint Config::ReadItem(const std::string& section, const std::string& key,\n const T1& defaultvalue, T2& itemvalue) const {\n if (section.size() <= 0 || key.size() <= 0 ) return -1;\n\n std::string value;\n Slice idxsec = section;\n Slice idxkey = key;\n Slice val = table_.Get(idxsec, idxkey);\n\n if (val.start() != NULL) {\n value.assign(val.start(), val.end() - val.start());\n std::stringstream ss(value, std::ios_base::in);\n ss >> itemvalue;\n return 0;\n } else {\n std::stringstream ss;\n ss << defaultvalue;\n ss >> itemvalue;\n return 1;\n }\n}\n} // namespace libwxfreq\n\n#endif // FREQLIB_UTIL_CONFIG_H_\n" }, { "path": "util/config_hash.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \n#include \"config_hash.h\"\n\nnamespace libwxfreq {\n\n// ConfigHashNode\nConfigHashNode::ConfigHashNode():hash_num_(0) { }\n\nConfigHashNode::ConfigHashNode(const unsigned int hash_num, const Slice &sec,\n const Slice &key, const Slice &val) {\n hash_num_ = hash_num;\n section_ = sec;\n key_ = key;\n value_ = val;\n}\n\ninline unsigned ConfigHashNode::hash_num() const {\n return hash_num_;\n}\n\ninline const Slice& ConfigHashNode::section() const {\n return section_;\n}\n\ninline const Slice& ConfigHashNode::key() const {\n return key_;\n}\n\ninline const Slice & ConfigHashNode::value() const {\n return value_;\n}\n\n// ConfigHashTable\nConfigHashTable::ConfigHashTable() {\n}\n\nConfigHashTable::ConfigHashTable(const ConfigHashTable &other) {\n table_ = other.table_;\n}\n\nConfigHashTable::~ConfigHashTable() {\n}\n\nvoid ConfigHashTable::Clear() {\n table_.clear();\n}\n\n// using FNV1 hash function\ninline unsigned int ConfigHashTable::HashFun(const Slice &sec,\n const Slice &key) const {\n static const unsigned int FNV1_BASIS = 2166136261lu;\n static const unsigned int FNV1_PRIME = 16777619;\n unsigned int ret = FNV1_BASIS;\n const char *ptr, *endptr;\n\n endptr = sec.end();\n for (ptr = sec.start(); ptr < endptr; ptr++) {\n ret *= FNV1_PRIME;\n ret ^= tolower(*ptr);\n }\n\n endptr = key.end();\n for (ptr = key.start(); ptr < endptr; ptr++) {\n ret *= FNV1_PRIME;\n ret ^= tolower(*ptr);\n }\n return ret;\n}\n\nvoid ConfigHashTable::Add(const Slice &sec, const Slice &key,\n const Slice &val) {\n unsigned int hashnum = HashFun(sec, key);\n table_[hashnum].push_back(ConfigHashNode(hashnum, sec, key, val));\n\n std::vector::iterator it = table_[hashnum].end() - 1;\n unsigned int sec_hashnum = HashFun(sec, Slice());\n keys_table_[sec_hashnum].push_back(it);\n}\n\nSlice ConfigHashTable::Get(const Slice &sec, const Slice &key) const {\n unsigned int hashnum = HashFun(sec, key);\n\n HashTable::const_iterator tit = table_.find(hashnum);\n\n if (tit == table_.end()) return Slice();\n\n for (std::vector::const_iterator it = tit->second.begin();\n it != tit->second.end(); ++it) {\n if (it->hash_num() == hashnum && it->section() == sec && it->key() == key) {\n return it->value();\n }\n }\n\n return Slice();\n}\n\nvoid ConfigHashTable::GetKeysBySection(const Slice &sec,\n std::vector &keys) const {\n unsigned int hashnum = HashFun(sec, Slice());\n HashIteratorTable::const_iterator tit = keys_table_.find(hashnum);\n\n if (tit == keys_table_.end()) return;\n\n for (std::vector::iterator>::const_iterator\n it = tit->second.begin(); it != tit->second.end(); ++it) {\n if ((*it)->section() == sec) {\n keys.push_back((*it)->key());\n }\n }\n}\n\nbool ConfigHashTable::Empty() {\n return table_.empty();\n}\n\n} // namespace libwxfreq\n\n" }, { "path": "util/config_hash.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_CONFIG_HASH_H_\n#define FREQLIB_UTIL_CONFIG_HASH_H_\n\n#include\n#include\n#include\n#include \"slice.h\"\n\nnamespace libwxfreq {\n\nclass ConfigHashNode {\n public:\n ConfigHashNode();\n ConfigHashNode(const unsigned int hash_num, const Slice &sec,\n const Slice &key, const Slice &val);\n unsigned hash_num() const;\n const Slice & section() const;\n const Slice & key() const;\n const Slice & value() const;\n\n private:\n unsigned int hash_num_;\n Slice section_, key_, value_;\n};\n\n\n\nclass ConfigHashTable {\n public:\n ConfigHashTable();\n ConfigHashTable(const ConfigHashTable &other);\n ~ConfigHashTable();\n\n\n void Clear();\n void Add(const Slice &sec, const Slice &key, const Slice &val);\n Slice Get(const Slice &sec, const Slice &key) const;\n void GetKeysBySection(const Slice &sec, std::vector &keys) const;\n bool Empty();\n\n private:\n typedef std::map > HashTable;\n typedef std::map::iterator> > HashIteratorTable;\n // using FNV1 hash function\n inline unsigned int HashFun(const Slice &sec, const Slice &key) const;\n HashTable table_;\n HashIteratorTable keys_table_;\n};\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_UTIL_CONFIG_HASH_H_\n" }, { "path": "util/freq_stat.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_FREQ_STAT_H_\n#define FREQLIB_UTIL_FREQ_STAT_H_\n#include \n#include \n\nnamespace libwxfreq {\nclass FreqItem;\n\nclass FreqStat {\n public:\n virtual uint32_t& GetItem(const std::string& key, const uint32_t appid,\n const uint32_t cnt, FreqItem* &freq_item) = 0;\n virtual bool TryReload() = 0;\n void set_type_name(const std::string& type_name) {\n type_name_ = type_name;\n }\n \n const std::string& type_name() const {\n return type_name_;\n }\n\n virtual ~FreqStat() { }\n protected:\n std::string type_name_;\n};\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_UTIL_FREQ_STAT_H_\n" }, { "path": "util/log.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"log.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace libwxfreq {\n\nstatic const char* kDefaultLogFileName = \"/tmp/libwxfreq.log\";\n\nstatic inline pid_t gettid() {\n return syscall(SYS_gettid);\n}\n\nstatic void Logv(FILE *file, const char* format, va_list ap) {\n const uint64_t thread_id = gettid();\n\n char buffer[500];\n for (int iter = 0; iter < 2; iter++) {\n char* base;\n int bufsize;\n if (iter == 0) {\n bufsize = sizeof(buffer);\n base = buffer;\n } else {\n bufsize = 30000;\n base = new char[bufsize];\n }\n char* p = base;\n char* limit = base + bufsize;\n\n struct timeval now_tv;\n gettimeofday(&now_tv, NULL);\n const time_t seconds = now_tv.tv_sec;\n struct tm t;\n localtime_r(&seconds, &t);\n p += snprintf(p, limit - p,\n \"%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx \",\n t.tm_year + 1900,\n t.tm_mon + 1,\n t.tm_mday,\n t.tm_hour,\n t.tm_min,\n t.tm_sec,\n static_cast(now_tv.tv_usec),\n static_cast(thread_id));\n\n // Print the message\n if (p < limit) {\n va_list backup_ap;\n va_copy(backup_ap, ap);\n p += vsnprintf(p, limit - p, format, backup_ap);\n va_end(backup_ap);\n }\n\n // Truncate to available space if necessary\n if (p >= limit) {\n if (iter == 0) {\n continue; // Try again with larger buffer\n } else {\n p = limit - 1;\n }\n }\n\n // Add newline if necessary\n if (p == base || p[-1] != '\\n') {\n *p++ = '\\n';\n }\n\n assert(p <= limit);\n fwrite(base, 1, p - base, file);\n fflush(file);\n if (base != buffer) {\n delete[] base;\n }\n break;\n }\n}\n\nstatic int Log(const char* format, ...) {\n static FILE* file = fopen(kDefaultLogFileName, \"a\");\n if (file == NULL) return -1;\n va_list ap;\n va_start(ap, format);\n Logv(file, format, ap);\n va_end(ap);\n return 0;\n}\n\nLogFunc gLog = Log;\n\n} // namespace libwxfreq\n\n" }, { "path": "util/log.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_LOG_H_\n#define FREQLIB_UTIL_LOG_H_\n\nnamespace libwxfreq {\n\ntypedef int (*LogFunc)(const char* format, ...);\n\nextern LogFunc gLog;\n\n} // namespace libwxfreq\n#endif // FREQLIB_UTIL_LOG_H_\n" }, { "path": "util/map_freq_stat.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"map_freq_stat.h\"\n#include \n#include \"reload.h\"\n\nnamespace libwxfreq {\nstatic uint32_t dummy_timestamp = 0;\n\nbool FreqKeyCmp(const FreqKey&a, const FreqKey&b) {\n if (a.appid < b.appid) {\n return true;\n } else if (a.appid == b.appid) {\n return strncmp(a.key, b.key, sizeof(a.key)) < 0;\n }\n return false;\n}\n\nuint32_t& MapFreqStat::GetItem(const std::string& key, const uint32_t appid,\n const uint32_t cnt, FreqItem* &freq_item) {\n FreqKey freqkey;\n freqkey.appid = appid;\n strncpy(freqkey.key, key.c_str(), sizeof(freqkey.key));\n if (cnt != 0) {\n FreqItemForMap& freqitemformap = item_map[freqkey];\n freq_item = reinterpret_cast(&freqitemformap);\n return freqitemformap.timestamp;\n } else {\n ItemMap::iterator it = item_map.find(freqkey);\n if (it == item_map.end()) {\n freq_item = NULL;\n return dummy_timestamp;\n } else {\n freq_item = reinterpret_cast(&(it->second));\n return it->second.timestamp;\n }\n }\n}\n\nbool MapFreqStat::TryReload() {\n return gLoadFunc(type_name_.c_str()) == 0;\n}\n\n} // namespace libwxfreq\n" }, { "path": "util/map_freq_stat.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_MAP_FREQ_STAT_H_\n#define FREQLIB_UTIL_MAP_FREQ_STAT_H_\n#include \n#include \n#include \"freq_item.h\"\n#include \"freq_stat.h\"\n\nnamespace libwxfreq {\n#pragma pack(1)\nstruct FreqItemForMap : public FreqItem {\n uint32_t timestamp;\n};\n#pragma pack()\nbool FreqKeyCmp(const FreqKey&a, const FreqKey&b);\n\nclass MapFreqStat : public FreqStat {\n public:\n MapFreqStat() : item_map(FreqKeyCmp) { }\n virtual uint32_t& GetItem(const std::string& key, const uint32_t appid,\n const uint32_t cnt, FreqItem* &freq_item);\n virtual bool TryReload();\n\n private:\n typedef bool (*cmp) (const FreqKey&a, const FreqKey&b);\n typedef std::map ItemMap;\n ItemMap item_map;\n};\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_UTIL_MAP_FREQ_STAT_H_\n" }, { "path": "util/multi_hash_base.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_MULTI_HASH_BASE_H_\n#define FREQLIB_UTIL_MULTI_HASH_BASE_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"log.h\"\n\nnamespace libwxfreq {\n\nstatic const unsigned int MAX_LEVEL_CNT = 30;\nstatic const unsigned int MAIN_LEVEL_CNT = 5;\nstatic const float MAIN_LEVEL_ITEM_RATE = 0.9;\n\n\ntemplate \nclass MultiHashBase {\n\n enum MultiHashRet {\n enMultiHashKeyNotFound = -3,\n enMultiHashBucketFull = -2,\n enMultiHashNotInit = -1,\n enMultiHashOK = 0,\n enMultiHashFoundEmptyItem = 1,\n enMultiHashFoundExpiredItem = 2,\n};\n\n public:\n MultiHashBase();\n virtual ~MultiHashBase();\n typedef Key KeyType;\n typedef Value ValueType;\n\n public:\n int MultiHashInit(char *base, size_t len,\n unsigned int max_level = MAX_LEVEL_CNT,\n unsigned int main_level = MAIN_LEVEL_CNT);\n\n int GetValueForRead(const KeyType &input_key, ValueType *&value);\n int GetValueForWrite(const KeyType &input_key, ValueType *&value);\n\n protected:\n virtual int InitLevelHash();\n\n unsigned int item_size() { return item_size_ ;}\n virtual size_t HashKey(const KeyType &input_key) = 0;\n virtual bool KeyCmp(const KeyType &input_key, const KeyType &key_in_mem) = 0;\n virtual bool IsExpired(const KeyType &key, const ValueType &value,\n void *old_value) = 0;\n virtual bool ExpiredOccupy(const KeyType &input_key, KeyType &key_in_mem,\n ValueType &value_in_mem, void *old_value) = 0;\n virtual bool IsEmpty(const KeyType &key, const ValueType &value) = 0;\n virtual bool EmptyOccupy(const KeyType &input_key, KeyType &key_in_mem) = 0;\n virtual void AfterFindReadItem(ValueType &value) = 0;\n virtual void AfterFindWriteItem(ValueType &value) = 0;\n\n private:\n char *shm_mem_base_;\n unsigned int max_level_;\n unsigned int main_level_;\n unsigned int item_size_;\n size_t shm_mem_len_;\n size_t max_item_cnt_;\n std::vector size_meta_;\n\n private:\n static size_t GetBiggestPrimer(size_t ulluppervalue);\n static bool IsPrimer(size_t ullvalue);\n};\n\n\ntemplate \nMultiHashBase::MultiHashBase()\n : shm_mem_base_(NULL),\n shm_mem_len_(0),\n max_level_(MAX_LEVEL_CNT),\n main_level_(MAIN_LEVEL_CNT) {\n item_size_ = sizeof(KeyType) + sizeof(ValueType);\n}\n\ntemplate \nMultiHashBase::~MultiHashBase() { }\n\n\ntemplate \nint MultiHashBase::MultiHashInit(char *base, size_t len,\n unsigned int max_level,\n unsigned int main_level) {\n shm_mem_base_ = base;\n shm_mem_len_ = len;\n max_level_ = max_level >= 2 * MAX_LEVEL_CNT ? 2 * MAX_LEVEL_CNT : max_level;\n max_level_ = max_level_ <= MAX_LEVEL_CNT / 2 ? MAX_LEVEL_CNT/ 2 : max_level_;\n main_level_ = main_level >= MAIN_LEVEL_CNT ? MAIN_LEVEL_CNT : main_level;\n main_level_ = main_level_<= MAIN_LEVEL_CNT / 2 ? MAIN_LEVEL_CNT / 2 : main_level_;\n main_level_ = main_level_ <= max_level_ ? main_level_ : max_level_ - 1;\n\n max_item_cnt_ = len / item_size_;\n size_meta_.reserve(max_level_);\n int ret = InitLevelHash();\n if (0 != ret) {\n return -__LINE__;\n }\n return enMultiHashOK;\n}\n\ntemplate \nint MultiHashBase::InitLevelHash() {\n unsigned int conflict_level = max_level_ - main_level_;\n size_t level_size =\n (size_t)(MAIN_LEVEL_ITEM_RATE * max_item_cnt_ / main_level_);\n gLog(\"[%s][%d]: level_size %llu rate %f cnt %llu\\n\", __FILE__, __LINE__, \n level_size, MAIN_LEVEL_ITEM_RATE, max_item_cnt_);\n \n size_t current_level_size = 0;\n size_t had_push_item_cnt = 0;\n\n for (int i = 0; i < max_level_; i++) {\n if (i == max_level_ -1) {\n size_meta_[i] = max_item_cnt_ - had_push_item_cnt;\n break;\n }\n\n if (level_size <= 2) {\n max_level_ = i + 1;\n size_meta_[i] = max_item_cnt_ - had_push_item_cnt;\n return enMultiHashOK;\n }\n\n current_level_size = GetBiggestPrimer(level_size);\n size_meta_[i] = current_level_size;\n had_push_item_cnt += current_level_size;\n if (i + 1 == main_level_) {\n level_size = (max_item_cnt_ - had_push_item_cnt) / conflict_level;\n } else {\n level_size = current_level_size - 1;\n }\n }\n\n for (int i = 0; i < max_level_; i++) {\n gLog(\"[%s][%d]: level %d size %llu\\n\", __FILE__, __LINE__, \n i, size_meta_[i]);\n }\n return enMultiHashOK;\n}\n\ntemplate \nint MultiHashBase::GetValueForRead(const KeyType &input_key,\n ValueType *&value) {\n int ret = enMultiHashKeyNotFound;\n size_t offset = 0;\n unsigned int find_level = 0;\n size_t hash_key = HashKey(input_key);\n for (find_level = 0; find_level < max_level_; find_level++) {\n size_t index = hash_key % size_meta_[find_level];\n\n KeyType* key_in_mem =\n reinterpret_cast(shm_mem_base_ + offset + (item_size_ * index));\n ValueType* value_in_mem = reinterpret_cast(\n reinterpret_cast(key_in_mem) + sizeof(KeyType));\n\n if (KeyCmp(input_key, *key_in_mem) == true) {\n uint32_t timestamp = 0;\n if (IsExpired(*key_in_mem, *value_in_mem, ×tamp)) {\n ret = enMultiHashKeyNotFound;\n break;\n }\n value = value_in_mem;\n ret = enMultiHashOK;\n AfterFindReadItem(*value_in_mem);\n break;\n } else if (IsEmpty(*key_in_mem, *value_in_mem) == true) {\n ret = enMultiHashKeyNotFound;\n break;\n }\n offset += size_meta_[find_level] * item_size_;\n }\n\n return ret;\n}\n\ntemplate \nint MultiHashBase::GetValueForWrite(const KeyType &input_key,\n ValueType *&value) {\n int ret = enMultiHashBucketFull;\n ValueType * read_value_ptr;\n ret = GetValueForRead(input_key, read_value_ptr);\n if (ret == enMultiHashOK) {\n value = read_value_ptr;\n AfterFindWriteItem(*value);\n return ret;\n }\n\n size_t offset = 0;\n unsigned int find_level = 0;\n ret = enMultiHashBucketFull;\n size_t hash_key = HashKey(input_key);\n for (find_level = 0; find_level < max_level_; find_level++) {\n size_t index = hash_key % size_meta_[find_level];\n KeyType* key_in_mem =\n reinterpret_cast(shm_mem_base_ + offset + (item_size_ * index));\n ValueType* value_in_mem = reinterpret_cast(\n reinterpret_cast(key_in_mem) + sizeof(KeyType));\n\n uint32_t timestamp = 0;\n if (IsEmpty(*key_in_mem, *value_in_mem) == true) {\n if (EmptyOccupy(input_key, *key_in_mem) == true) {\n memset(reinterpret_cast(value_in_mem), 0 , sizeof(ValueType));\n ret = enMultiHashOK;\n value = value_in_mem;\n break;\n }\n } else if (IsExpired(*key_in_mem, *value_in_mem, ×tamp) == true) {\n if (ExpiredOccupy(input_key, *key_in_mem, *value_in_mem, ×tamp) == true) {\n memset(reinterpret_cast(value_in_mem), 0 , sizeof(ValueType));\n ret = enMultiHashOK;\n value = value_in_mem;\n break;\n }\n }\n offset += size_meta_[find_level] * item_size_;\n }\n if (ret == enMultiHashOK) {\n gLog(\"[%s][%d]: level %d hash %llu\\n\", __FILE__, __LINE__, \n find_level, hash_key);\n } else {\n gLog(\"[%s][%d]: not found hash %llu\\n\", __FILE__, __LINE__, hash_key);\n }\n return ret;\n}\n\ntemplate \nbool MultiHashBase::IsPrimer(size_t ullValue) {\n if (0 == (ullValue % 2)) {\n return false;\n }\n\n size_t ullEnd = (size_t)sqrt(ullValue) + 1;\n if (ullEnd > (ullValue / 2)) {\n ullEnd = ullValue / 2;\n }\n\n for (size_t i = 3; i <= ullEnd; i++) {\n if (0 == (ullValue % i)) {\n return false;\n }\n }\n\n return true;\n}\n\ntemplate \nsize_t MultiHashBase::GetBiggestPrimer(\n size_t ullUpperValue) {\n for (size_t i = ullUpperValue; i > 1; i--) {\n if (IsPrimer(i)) {\n return i;\n }\n }\n return 1;\n}\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_UTIL_MULTI_HASH_BASE_H_\n\n" }, { "path": "util/multi_hash_table.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"multi_hash_table.h\"\n#include \n#include \n#include \n#include \n#include \nnamespace libwxfreq {\n\nstatic size_t BKDRHash(const char *str) {\n unsigned int seed = 131; // 31 131 1313 13131 131313 etc..\n unsigned int hash = 0;\n while (*str) {\n hash = hash * seed + (*str++);\n }\n return hash;\n}\n\nstatic size_t IntHash(uint32_t appid) {\n register size_t key = appid;\n return key * 2654435761;\n}\n\nMultiHashTable::MultiHashTable():init_flag_(NULL) { }\nMultiHashTable::~MultiHashTable() { }\n\nint MultiHashTable::MultiHashTableInit(bool zero_init, key_t shmkey,\n int shmflag, uint64_t item_cnt) {\n size_t shm_size = (size_t)(item_size() * item_cnt + 1);\n\n int iShmID = 0;\n iShmID = ::shmget(shmkey, shm_size, shmflag | IPC_CREAT);\n if (iShmID < 0 && errno == EINVAL) {\n // 共享内存大小发生变化\n iShmID = ::shmget(shmkey, 0, 0);\n if (iShmID == -1) return -__LINE__;\n if (::shmctl(iShmID, IPC_RMID, NULL) == -1) return -__LINE__;\n iShmID = ::shmget(shmkey, shm_size, shmflag | IPC_CREAT);\n }\n if (iShmID <0) return -__LINE__;\n\n char *base = NULL;\n base = reinterpret_cast(::shmat(iShmID, NULL, 0));\n if (reinterpret_cast(-1) == base) {\n return -__LINE__;\n }\n\n init_flag_ = base + shm_size - 1;\n if (zero_init) {\n memset(base, 0, shm_size);\n }\n\n return MultiHashInit(base, shm_size);\n}\n\nsize_t MultiHashTable::HashKey(const KeyType &input_key) {\n return BKDRHash(input_key.key) * IntHash(input_key.appid);\n}\n\nbool MultiHashTable::KeyCmp(const KeyType &input_key,\n const KeyType &key_in_mem) {\n size_t size = sizeof(input_key.key);\n uint32_t input_hash = *reinterpret_cast(\n input_key.key + size - sizeof(uint32_t));\n uint32_t mem_hash = *reinterpret_cast(\n key_in_mem.key + size - sizeof(uint32_t));\n return input_key.appid == key_in_mem.appid &&\n input_hash == mem_hash &&\n strncmp(key_in_mem.key, input_key.key, size) == 0;\n}\n\nbool MultiHashTable::IsExpired(const KeyType &input_key,\n const ValueType &value, void *old_value) {\n *reinterpret_cast(old_value) = value.timestamp;\n return value.timestamp != 0 && time(NULL) - value.timestamp >= 90000;\n}\n\nbool MultiHashTable::IsEmpty(const KeyType &input_key,\n const ValueType &/*value*/) {\n return input_key.appid == 0;\n}\nvoid MultiHashTable::AfterFindReadItem(ValueType &value) { }\nvoid MultiHashTable::AfterFindWriteItem(ValueType &value) { }\n\nbool MultiHashTable::ExpiredOccupy(const KeyType &input_key,\n KeyType &key_in_mem, ValueType &value_in_mem,\n void *old_value) {\n if (__sync_bool_compare_and_swap(&value_in_mem.timestamp,\n *reinterpret_cast(old_value), 0)) {\n memcpy(key_in_mem.key, input_key.key, sizeof(input_key.key));\n key_in_mem.appid = input_key.appid;\n return true;\n }\n return false;\n}\n\nbool MultiHashTable::EmptyOccupy(const KeyType &input_key,\n KeyType &key_in_mem) {\n if (__sync_bool_compare_and_swap(&key_in_mem.appid, 0, input_key.appid)) {\n memcpy(key_in_mem.key, input_key.key, sizeof(input_key.key));\n return true;\n }\n return false;\n}\n\nbool MultiHashTable::IsValid() {\n return init_flag_!= NULL && *init_flag_ == 17;\n}\n\nvoid MultiHashTable::MarkAsValid() {\n if (init_flag_ != NULL) *init_flag_ = 17;\n}\n\nvoid MultiHashTable::MarkAsInValid() {\n if (init_flag_ != NULL) *init_flag_ = 0;\n}\n\n} // namespace libwxfreq\n\n" }, { "path": "util/multi_hash_table.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_MULTI_HASH_TABLE_H_\n#define FREQLIB_UTIL_MULTI_HASH_TABLE_H_\n\n#include \n#include \n#include \"multi_hash_base.h\"\n#include \"freq_item.h\"\n\nnamespace libwxfreq {\n\n#pragma pack(1)\nstruct FreqItemForShm:public libwxfreq::FreqItem {\n uint32_t timestamp;\n};\n#pragma pack()\n\nclass MultiHashTable : public MultiHashBase {\n public:\n MultiHashTable();\n virtual ~MultiHashTable();\n\n public:\n int MultiHashTableInit(bool zero_init, key_t shmkey, int shmflag,\n uint64_t item_cnt);\n bool IsValid();\n void MarkAsValid();\n void MarkAsInValid();\n\n protected:\n size_t HashKey(const KeyType &input_key);\n bool KeyCmp(const KeyType &input_key, const KeyType &key_in_mem);\n bool IsExpired(const KeyType &input_key, const ValueType &value,\n void *old_value);\n bool IsEmpty(const KeyType &input_key, const ValueType &value);\n void AfterFindReadItem(ValueType &value);\n void AfterFindWriteItem(ValueType &value);\n bool ExpiredOccupy(const KeyType &input_key, KeyType &key_in_mem,\n ValueType &value_in_mem, void *old_value);\n bool EmptyOccupy(const KeyType &input_key, KeyType &key_in_mem);\n\n private:\n char * init_flag_;\n};\n\n} // namespace libwxfreq\n#endif // FREQLIB_UTIL_MULTI_HASH_TABLE_H_\n" }, { "path": "util/options.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"options.h\"\n\nnamespace libwxfreq {\n\nextern LogFunc gLog;\nextern LoadFunc gLoadFunc;\nextern DumpFunc gDumpFunc;\n\nvoid SetLog(LogFunc log) {\n gLog = log;\n}\n\nvoid SetLoadFunc(LoadFunc func) {\n gLoadFunc = func;\n}\n\nvoid SetDumpFunc(DumpFunc func) {\n gDumpFunc = func;\n}\n\n} // namespace libwxfreq\n" }, { "path": "util/options.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_OPTIONS_H_\n#define FREQLIB_UTIL_OPTIONS_H_\n#include \"log.h\"\n#include \"reload.h\"\n\nnamespace libwxfreq {\n\nvoid SetLog(LogFunc log);\n\nvoid SetLoadFunc(LoadFunc func);\nvoid SetDumpFunc(DumpFunc func);\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_UTIL_OPTIONS_H_\n" }, { "path": "util/reload.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"reload.h\"\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"freq_manager.h\"\n#include \"appid_manager.h\"\n#include \"log.h\"\n\nnamespace libwxfreq {\n\nstatic const char* kDefaultWhiteFileNamePrefix = \"/tmp/libwxfreq_db.\";\n\nstatic int LoadFromLocalFile(const char* type_name) {\n char filename[64];\n snprintf(filename, sizeof(filename), \"%s%s\",\n kDefaultWhiteFileNamePrefix, type_name);\n std::ifstream ifs;\n ifs.open(filename, std::fstream::in);\n\n if (!ifs.is_open()) {\n gLog(\"[%s][%d]: can not open %s\\n\", __FILE__, __LINE__, filename);\n return -1;\n }\n\n char line[128];\n std::string opname, key;\n uint32_t appid = 0;\n unsigned int expire_time = 0, block_level = 0;\n unsigned int now = time(NULL);\n while (ifs.getline(line, sizeof(line))) {\n std::stringstream ss(line);\n ss >> opname >> key >> appid >> expire_time >> block_level;\n\n if (opname == \"AddWhite\") {\n if (expire_time > now) {\n if (FreqManager::AddWhite(type_name, key, appid,\n expire_time - now) == false) {\n gLog(\"[%s][%d]: reload whitelist failed, typename %s key %s\"\n \" appid %u expiretime %u\\n\", __FILE__, __LINE__, type_name,\n key.c_str(), appid, expire_time);\n ifs.close();\n return -1;\n }\n }\n } else if (opname == \"DeleteWhite\") {\n if (FreqManager::DeleteWhite(type_name, key, appid) == false) {\n gLog(\"[%s][%d]: reload deletewhite failed, typename %s key %s\"\n \" appid %u\\n\", __FILE__, __LINE__, type_name, key.c_str(), appid);\n ifs.close();\n return -1;\n }\n } else if (opname == \"AddBlock\") {\n if (expire_time > now) {\n if (FreqManager::AddBlock(type_name, key, appid,\n expire_time - now, block_level) == false) {\n gLog(\"[%s][%d]: reload block failed, typename %s key %s appid %u\"\n \" expiretime %u blocklevel %u\\n\", __FILE__, __LINE__, type_name,\n key.c_str(), appid, expire_time, block_level);\n ifs.close();\n return -1;\n }\n }\n } else if (opname == \"DeleteBlock\") {\n if (FreqManager::DeleteBlock(type_name, key, appid) == false) {\n gLog(\"[%s][%d]: reload deleteblock failed, typename %s key %s\"\n \" appid %u\\n\", __FILE__, __LINE__, type_name, key.c_str(), appid);\n ifs.close();\n return -1;\n }\n } else {\n gLog(\"[%s][%d]: reload unknow typename %s\\n\", type_name);\n }\n }\n\n ifs.close();\n return 0;\n}\n\nstatic int DumpToLocalFile(const char* opname, const char* type_name,\n const char *key, const uint32_t appid,\n const uint32_t linger_time,\n const uint32_t block_level) {\n char filename[64];\n snprintf(filename, sizeof(filename), \"%s%s\",\n kDefaultWhiteFileNamePrefix, type_name);\n static std::ofstream ofs(filename, std::fstream::out | std::fstream::app);\n if (!ofs.is_open()) {\n gLog(\"[%s][%d]: can not open %s\\n\", __FILE__, __LINE__, filename);\n return -1;\n }\n if (linger_time <= 600) return 0;\n ofs << opname << \"\\t\" << key << \"\\t\" << appid << \"\\t\" <<\n time(NULL) + linger_time << \"\\t\" << block_level << std::endl;\n return 0;\n}\n\nLoadFunc gLoadFunc = LoadFromLocalFile;\nDumpFunc gDumpFunc = DumpToLocalFile;\n\n} // namespace libwxfreq\n\n" }, { "path": "util/reload.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_RELOAD_H_\n#define FREQLIB_UTIL_RELOAD_H_\n#include \nnamespace libwxfreq {\n\ntypedef int (*LoadFunc)(const char* type_name);\ntypedef int (*DumpFunc)(const char* opname, const char* type_name,\n const char *key, const uint32_t appid,\n const uint32_t linger_time,\n const uint32_t block_level);\n\nextern LoadFunc gLoadFunc;\nextern DumpFunc gDumpFunc;\n\n} // namespace libwxfreq\n#endif // FREQLIB_UTIL_RELOAD_H_\n" }, { "path": "util/shm_freq_stat.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"shm_freq_stat.h\"\n#include \n#include \n#include \n#include \n#include \"freq_item.h\"\n#include \"log.h\"\n#include \"reload.h\"\n\nnamespace libwxfreq {\nstatic uint32_t dummy_timestamp = 0;\n\nbool ShmKeyLessCmp(const FreqKey& left, const FreqKey& right) {\n size_t size = sizeof(left.key);\n uint32_t lef_hash = *reinterpret_cast(\n left.key + size - sizeof(uint32_t));\n uint32_t right_hash = *reinterpret_cast(\n right.key + size - sizeof(uint32_t));\n if (lef_hash < right_hash) return true;\n if (lef_hash > right_hash) return false;\n if (left.appid < right.appid) return true;\n if (left.appid > right.appid) return false;\n return strncmp(right.key, left.key, size) < 0;\n}\n\nuint32_t APHash(const char *str) {\n uint32_t hash = 0;\n int i;\n\n for (i=0; *str; i++) {\n if ((i & 1) == 0) {\n hash ^= ((hash << 7) ^ (*str++) ^ (hash >> 3));\n } else {\n hash ^= (~((hash << 11) ^ (*str++) ^ (hash >> 5)));\n }\n }\n return (hash & 0x7FFFFFFF);\n}\n\nShmFreqStat::ShmFreqStat(bool zero_init, key_t key, unsigned int item_cnt) :\n shrink_(false), last_shrink_time_(0), item_map_(ShmKeyLessCmp) {\n int ret = table_.MultiHashTableInit(zero_init, key,\n S_IRUSR | S_IWUSR, item_cnt);\n if (ret != 0) {\n gLog(\"[%s][%d]: ShmFreqStat constuct failed, key %u ret %d exit %d\\n\",\n __FILE__, __LINE__, key, ret, ret);\n exit(ret);\n }\n count_= 0;\n pthread_rwlock_init(&lock_, NULL);\n}\n\nvoid ShmFreqStat::ShrinkMap(time_t now) {\n if (now - last_shrink_time_ > 3600 &&\n __sync_bool_compare_and_swap(&shrink_, false, true)) {\n for (ItemMap::iterator it = item_map_.begin();\n it != item_map_.end(); ++it) {\n ItemMap::iterator dit = it;\n if (static_cast(now - it->second.timestamp) < 90000) {\n FreqItemForShm* freqitemforshm = NULL;\n table_.GetValueForWrite(it->first, freqitemforshm);\n if (freqitemforshm != NULL) {\n memcpy(freqitemforshm, &it->second, sizeof(FreqItemForShm));\n pthread_rwlock_wrlock(&lock_);\n item_map_.erase(dit);\n pthread_rwlock_unlock(&lock_);\n } else { // TODO(arthurzou)\n }\n } else {\n pthread_rwlock_wrlock(&lock_);\n item_map_.erase(dit);\n pthread_rwlock_unlock(&lock_);\n }\n }\n last_shrink_time_ = now;\n shrink_ = false;\n gLog(\"[%s][%d]: ShmFreqStat shrinkmap type_name %s size = %u\\n\",\n __FILE__, __LINE__, type_name().c_str(), item_map_.size());\n }\n}\n\nuint32_t& ShmFreqStat::GetItem(const std::string& key, const uint32_t appid,\n const uint32_t cnt, FreqItem* &freq_item) {\n FreqKey freqkey;\n freqkey.appid = appid;\n uint32_t* key_hash_ptr = reinterpret_cast(\n freqkey.key + sizeof(freqkey.key) - sizeof(uint32_t));\n *key_hash_ptr = APHash(key.c_str());\n strncpy(freqkey.key, key.c_str(), sizeof(freqkey.key));\n\n time_t now = time(NULL);\n\n FreqItemForShm* freqitemforshm = NULL;\n if (cnt != 0) {\n table_.GetValueForWrite(freqkey, freqitemforshm);\n } else {\n table_.GetValueForRead(freqkey, freqitemforshm);\n }\n#ifdef USING_MAP\n ShrinkMap(now);\n if (freqitemforshm == NULL) { // sad face, find in map\n if (cnt != 0) {\n pthread_rwlock_wrlock(&lock_);\n freqitemforshm = &item_map_[freqkey];\n table_.MarkAsInValid();\n gLog(\"[%s][%d]: item_map_ type_name %s size %u\\n\", __FILE__, __LINE__, \n type_name().c_str(), item_map_.size());\n pthread_rwlock_unlock(&lock_);\n freq_item = reinterpret_cast(freqitemforshm);\n return freqitemforshm->timestamp;\n } else {\n pthread_rwlock_rdlock(&lock_);\n ItemMap::iterator it = item_map_.find(freqkey);\n gLog(\"[%s][%d]: item_map_ type_name %s size %u\\n\", __FILE__, __LINE__, \n type_name().c_str(), item_map_.size());\n if (item_map_.size() > 0) {\n table_.MarkAsInValid();\n } else {\n table_.MarkAsValid();\n }\n if (it == item_map_.end() ||\n static_cast(now - it->second.timestamp) >= 90000) {\n freq_item = NULL;\n pthread_rwlock_unlock(&lock_);\n return dummy_timestamp;\n } else {\n freq_item = reinterpret_cast(&(it->second));\n pthread_rwlock_unlock(&lock_);\n return it->second.timestamp;\n }\n }\n } else {\n freq_item = reinterpret_cast(freqitemforshm);\n if (freq_item->level3_cnt == 0) {\n count_++;\n gLog(\"[%s][%d]: using shm type_name %s count %u\\n\", __FILE__, __LINE__, \n type_name().c_str(), count_);\n }\n return freqitemforshm->timestamp;\n }\n#else\n if (freqitemforshm == NULL) {\n freq_item = NULL;\n return dummy_timestamp;\n } else {\n freq_item = reinterpret_cast(freqitemforshm);\n return freqitemforshm->timestamp;\n }\n#endif\n}\n\nShmFreqStat::~ShmFreqStat() {\n}\n\nbool ShmFreqStat::TryReload() {\n if (!table_.IsValid()) {\n if (gLoadFunc(type_name_.c_str()) == 0) {\n table_.MarkAsValid();\n return true;\n } else {\n return false;\n }\n }\n return true;\n}\n\n} // namespace libwxfreq\n" }, { "path": "util/shm_freq_stat.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_SHM_FREQ_STAT_H_\n#define FREQLIB_UTIL_SHM_FREQ_STAT_H_\n#include \n#include \n#include \n#include \"freq_stat.h\"\n#include \"multi_hash_table.h\"\n\nnamespace libwxfreq {\nclass FreqItem;\n\nclass ShmFreqStat : public FreqStat {\n public:\n ShmFreqStat(bool zero_init, key_t key, unsigned int item_cnt);\n virtual uint32_t& GetItem(const std::string& key, const uint32_t appid,\n const uint32_t cnt, FreqItem* &freq_item);\n virtual bool TryReload();\n virtual ~ShmFreqStat();\n\n private:\n typedef bool (*cmp) (const FreqKey&a, const FreqKey&b);\n typedef std::map ItemMap;\n void ShrinkMap(time_t now);\n ItemMap item_map_;\n pthread_rwlock_t lock_;\n unsigned int shrink_;\n unsigned int last_shrink_time_;\n unsigned int count_;\n MultiHashTable table_;\n};\n\n} // namespace libwxfreq\n\n#endif // FREQLIB_UTIL_SHM_FREQ_STAT_H_\n" }, { "path": "util/slice.cpp", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#include \"slice.h\"\n#include \n#include \n\n\nusing namespace std;\n\nnamespace libwxfreq {\n\nSlice::Slice() {\n start_ = end_ = NULL;\n}\n\nSlice::Slice(const char *start, const char *end) {\n start_ = start;\n end_ = end;\n}\n\nSlice::Slice(const std::string &other_slice) {\n start_ = other_slice.c_str();\n end_ = other_slice.c_str() + other_slice.size();\n}\n\nstd::string Slice::ToStr() const {\n std::string ret(start_, end_- start_);\n return ret;\n}\n\nvoid Slice::StrTrim(const char *delimiter) {\n while (start_ < end_ && strchr(delimiter, *start_) != 0) start_++;\n while (start_ < end_ && strchr(delimiter, *(end_-1)) != 0) end_--;\n}\n\nbool Slice::operator!=(const Slice &other_slice) const {\n return !(*this == other_slice);\n}\n\nbool Slice::operator==(const Slice &other_slice) const {\n if (end_ - start_ != other_slice.end_ - other_slice.start_) return false;\n return strncasecmp(start_, other_slice.start_, end_ - start_) == 0;\n}\n\nbool Slice::operator==(const std::string &other_slice) const {\n if (end_ - start_ != static_cast(other_slice.size())) return false;\n return strncasecmp(start_, other_slice.c_str(), end_ - start_) == 0;\n}\n\nbool Slice::operator<(const Slice &other_slice) const {\n if (start_ == other_slice.start_) return false;\n\n int minlen = end_ - start_, res = 0;\n if (other_slice.end_ - other_slice.start_ < minlen )\n minlen = other_slice.end_ - other_slice.start_;\n if ((res = strncasecmp( start_, other_slice.start_, minlen )) < 0)\n return true;\n else if (res > 0)\n return false;\n else\n return end_ - start_ < other_slice.end_ - other_slice.start_;\n}\n\n} // namespace libwxfreq\n" }, { "path": "util/slice.h", "content": "/*\n* Tencent is pleased to support the open source community by making libwxfreq available.\n*\n* Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.\n*\n* Licensed under the BSD 3-Clause License (the \"License\"); you may not use this file\n* except in compliance with the License. You may obtain a copy of the License at\n*\n* https://opensource.org/licenses/BSD-3-Clause\n*\n* Unless required by applicable law or agreed to in writing, software distributed\n* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific\n* language governing permissions and limitations under the License.\n*/\n\n#ifndef FREQLIB_UTIL_SLICE_H_\n#define FREQLIB_UTIL_SLICE_H_\n\n#include\n\nnamespace libwxfreq {\n\nclass Slice {\n public:\n Slice();\n Slice(const char *start, const char *end);\n Slice(const std::string &other_str);\n\n std::string ToStr() const;\n void StrTrim(const char *delimiter);\n\n inline const char *start() const { return start_; }\n inline const char *end() const { return end_; }\n\n bool operator!=(const Slice &other_slice) const;\n bool operator==(const Slice &other_slice) const;\n bool operator==(const std::string &other_str) const;\n bool operator<(const Slice &other_slice) const;\n\n private:\n const char *start_, *end_;\n};\n\n\n} // namespace libwxfreq\n#endif // FREQLIB_UTIL_SLICE_H_\n" } ]