[ { "path": ".editorconfig", "content": "# top-most EditorConfig file\nroot = true\n \n# all files\n[*]\nindent_style = tab\nindent_size = 4\n[src/kernel/list.h]\nindent_style = tab\nindent_size = 8\n[src/kernel/rbtree.*]\nindent_style = tab\nindent_size = 8\n" }, { "path": ".github/workflows/ci.yml", "content": "name: ci build\n\non:\n push:\n branches: [ master ]\n pull_request:\n branches: [ master ]\n\njobs:\n\n ubuntu-cmake:\n name: ubuntu\n runs-on: ubuntu-latest\n\n steps:\n - name: setup\n run: |\n sudo apt-get update\n sudo apt-get install -y cmake g++ libgtest-dev make libssl-dev\n sudo apt-get install -y valgrind\n sudo apt-get install -y libsnappy-dev libzstd-dev liblz4-dev\n - uses: actions/checkout@v2\n - name: make\n run: make KAFKA=y\n - name: make check\n run: make check KAFKA=y\n - name: make tutorial\n run: make tutorial KAFKA=y\n \n fedora-cmake:\n name: fedora\n runs-on: ubuntu-latest\n container:\n image: fedora:latest\n\n steps:\n - uses: actions/checkout@v3\n - run: cat /etc/os-release\n - name: install dependencies\n run: |\n sudo dnf -y update\n sudo dnf install -y cmake gcc-c++ gtest-devel make\n sudo dnf install -y openssl-devel valgrind\n sudo dnf install -y snappy-devel libzstd-devel lz4-devel zlib-devel\n - name: make\n run: make KAFKA=y\n - name: make check\n run: make check KAFKA=y\n - name: make tutorial\n run: make tutorial KAFKA=y\n\n freebsd-cmake:\n name: freebsd\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - name: Build and test on FreeBSD\n uses: vmactions/freebsd-vm@v1\n with:\n usesh: true\n mem: 2048\n copyback: false\n prepare: |\n pkg update -f\n pkg install -y cmake gmake gcc pkgconf openssl devel/googletest\n pkg install -y valgrind\n pkg install -y snappy zstd liblz4\n run: |\n freebsd-version\n gmake KAFKA=y\n gmake check KAFKA=y\n gmake tutorial KAFKA=y\n" }, { "path": ".github/workflows/xmake.yml", "content": "name: xmake build\n\non:\n workflow_dispatch:\n\njobs:\n build:\n name: build-linux\n runs-on: ubuntu-latest\n\n steps:\n - name: install dependencies\n run: |\n sudo apt-get update\n sudo apt-get install -y g++ libssl-dev libgtest-dev\n\n - name: setup xmake\n uses: xmake-io/github-action-setup-xmake@v1\n with:\n xmake-version: latest\n\n - name: pull code\n uses: actions/checkout@v2\n\n - name: xmake\n run: |\n xmake -r\n xmake -g test\n xmake -g tutorial\n xmake -g benchmark\n\n - name : run shared\n run: |\n xmake f -k shared\n xmake -r\n xmake -g test\n xmake -g tutorial\n xmake -g benchmark\n\n\n" }, { "path": ".gitignore", "content": "# Prerequisites\n*.d\n\n# Compiled Object files\n*.slo\n*.lo\n*.o\n*.obj\n\n# Precompiled Headers\n*.gch\n*.pch\n\n# Compiled Dynamic libraries\n*.so\n*.so.*\n*.dylib\n*.dll\n\n# Fortran module files\n*.mod\n*.smod\n\n# Compiled Static libraries\n*.lai\n*.la\n*.a\n*.lib\n\n# Executables\n*.exe\n*.out\n*.app\n\n# bazel env\nbazel-*\n\n# vscode configs\n.vscode\n\n# idea configs\n.idea\ncmake-build-debug/\nworkflow-config.cmake\n\n# xmake configs\n.xmake\nbuild.xmake\n_include\n" }, { "path": "BUILD", "content": "config_setting(\n\tname = 'linux',\n\tconstraint_values = [\n\t\t\"@platforms//os:linux\",\n\t],\n\tvisibility = ['//visibility:public'],\n)\n\ncc_library(\n\tname = 'workflow_hdrs',\n\thdrs = glob(['src/include/workflow/*']),\n\tincludes = ['src/include'],\n\tvisibility = [\"//visibility:public\"],\n\tlinkopts = [\n\t\t'-lpthread',\n\t\t'-lssl',\n\t\t'-lcrypto',\n\t],\n)\ncc_library(\n\tname = 'common_c',\n\tsrcs = [\n\t\t'src/kernel/mpoller.c',\n\t\t'src/kernel/msgqueue.c',\n\t\t'src/kernel/poller.c',\n\t\t'src/kernel/rbtree.c',\n\t\t'src/kernel/thrdpool.c',\n\t\t'src/util/crc32c.c',\n\t\t'src/util/json_parser.c',\n\t],\n\thdrs = glob(['src/*/*.h']) + glob(['src/*/*.inl']),\n\tincludes = [\n\t\t'src/kernel',\n\t\t'src/util',\n\t],\n\tcopts = ['-std=gnu90'],\n\tvisibility = [\"//visibility:public\"],\n)\ncc_library(\n\tname = 'common',\n\tsrcs = [\n\t\t'src/client/WFDnsClient.cc',\n\t\t'src/factory/DnsTaskImpl.cc',\n\t\t'src/factory/FileTaskImpl.cc',\n\t\t'src/factory/WFGraphTask.cc',\n\t\t'src/factory/WFResourcePool.cc',\n\t\t'src/factory/WFMessageQueue.cc',\n\t\t'src/factory/WFTaskFactory.cc',\n\t\t'src/factory/Workflow.cc',\n\t\t'src/manager/DnsCache.cc',\n\t\t'src/manager/RouteManager.cc',\n\t\t'src/manager/WFGlobal.cc',\n\t\t'src/nameservice/WFDnsResolver.cc',\n\t\t'src/nameservice/WFNameService.cc',\n\t\t'src/protocol/TLVMessage.cc',\n\t\t'src/protocol/DnsMessage.cc',\n\t\t'src/protocol/DnsUtil.cc',\n\t\t'src/protocol/SSLWrapper.cc',\n\t\t'src/protocol/PackageWrapper.cc',\n\t\t'src/protocol/dns_parser.c',\n\t\t'src/server/WFServer.cc',\n\t\t'src/kernel/CommRequest.cc',\n\t\t'src/kernel/CommScheduler.cc',\n\t\t'src/kernel/Communicator.cc',\n\t\t'src/kernel/Executor.cc',\n\t\t'src/kernel/SubTask.cc',\n\t] + select({\n\t\t':linux': [\n\t\t\t'src/kernel/IOService_linux.cc',\n\t\t],\n\t\t'//conditions:default': [\n\t\t\t'src/kernel/IOService_thread.cc',\n\t\t],\n\t}) + glob(['src/util/*.cc']),\n\thdrs = glob(['src/*/*.h']) + glob(['src/*/*.inl']),\n\tincludes = [\n\t\t'src/algorithm',\n\t\t'src/client',\n\t\t'src/factory',\n\t\t'src/kernel',\n\t\t'src/manager',\n\t\t'src/nameservice',\n\t\t'src/protocol',\n\t\t'src/server',\n\t\t'src/util',\n\t],\n\tdeps = ['workflow_hdrs', 'common_c'],\n\tvisibility = [\"//visibility:public\"],\n)\ncc_library(\n\tname = 'http',\n\thdrs = [\n\t\t'src/factory/HttpTaskImpl.inl',\n\t\t'src/protocol/HttpMessage.h',\n\t\t'src/protocol/HttpUtil.h',\n\t\t'src/protocol/http_parser.h',\n\t\t'src/server/WFHttpServer.h',\n\t\t'src/client/WFHttpChunkedClient.h',\n\t],\n\tincludes = [\n\t\t'src/protocol',\n\t\t'src/server',\n\t],\n\tsrcs = [\n\t\t'src/factory/HttpTaskImpl.cc',\n\t\t'src/protocol/HttpMessage.cc',\n\t\t'src/protocol/HttpUtil.cc',\n\t\t'src/protocol/http_parser.c',\n\t\t'src/client/WFHttpChunkedClient.cc',\n\t],\n\tdeps = [\n\t\t':common',\n\t],\n\tvisibility = [\"//visibility:public\"],\n)\ncc_library(\n\tname = 'redis',\n\thdrs = [\n\t\t'src/factory/RedisTaskImpl.inl',\n\t\t'src/protocol/RedisMessage.h',\n\t\t'src/protocol/redis_parser.h',\n\t\t'src/server/WFRedisServer.h',\n\t\t'src/client/WFRedisSubscriber.h',\n\t],\n\tincludes = [\n\t\t'src/protocol',\n\t\t'src/server',\n\t],\n\tsrcs = [\n\t\t'src/factory/RedisTaskImpl.cc',\n\t\t'src/protocol/RedisMessage.cc',\n\t\t'src/protocol/redis_parser.c',\n\t\t'src/client/WFRedisSubscriber.cc',\n\t],\n\tdeps = [\n\t\t':common',\n\t],\n\tvisibility = [\"//visibility:public\"],\n)\ncc_library(\n\tname = 'mysql',\n\thdrs = [\n\t\t'src/protocol/MySQLMessage.h',\n\t\t'src/protocol/MySQLMessage.inl',\n\t\t'src/protocol/MySQLResult.h',\n\t\t'src/protocol/MySQLResult.inl',\n\t\t'src/protocol/MySQLUtil.h',\n\t\t'src/protocol/mysql_byteorder.h',\n\t\t'src/protocol/mysql_parser.h',\n\t\t'src/protocol/mysql_stream.h',\n\t\t'src/protocol/mysql_types.h',\n\t\t'src/server/WFMySQLServer.h',\n\t\t'src/client/WFMySQLConnection.h',\n\t],\n\tincludes = [\n\t\t'src/protocol',\n\t\t'src/client',\n\t\t'src/server',\n\t],\n\tsrcs = [\n\t\t'src/factory/MySQLTaskImpl.cc',\n\t\t'src/protocol/MySQLMessage.cc',\n\t\t'src/protocol/MySQLResult.cc',\n\t\t'src/protocol/MySQLUtil.cc',\n\t\t'src/protocol/mysql_byteorder.c',\n\t\t'src/protocol/mysql_parser.c',\n\t\t'src/protocol/mysql_stream.c',\n\t\t'src/server/WFMySQLServer.cc',\n\t\t'src/client/WFMySQLConnection.cc',\n\t],\n\tdeps = [\n\t\t':common',\n\t],\n\tvisibility = [\"//visibility:public\"],\n)\n\ncc_library(\n\tname = 'upstream',\n\thdrs = [\n\t\t'src/manager/UpstreamManager.h',\n\t\t'src/nameservice/UpstreamPolicies.h',\n\t\t'src/nameservice/WFServiceGovernance.h',\n\t],\n\tincludes = [\n\t\t'src/manager',\n\t\t'src/nameservice',\n\t],\n\tsrcs = [\n\t\t'src/manager/UpstreamManager.cc',\n\t\t'src/nameservice/UpstreamPolicies.cc',\n\t\t'src/nameservice/WFServiceGovernance.cc',\n\t],\n\tdeps = [\n\t\t':common',\n\t],\n\tvisibility = [\"//visibility:public\"],\n)\n\ncc_library(\n\tname = 'kafka',\n\thdrs = [\n\t\t'src/client/WFKafkaClient.h',\n\t\t'src/factory/KafkaTaskImpl.inl',\n\t\t'src/protocol/KafkaDataTypes.h',\n\t\t'src/protocol/KafkaMessage.h',\n\t\t'src/protocol/KafkaResult.h',\n\t\t'src/protocol/kafka_parser.h',\n\t],\n\tincludes = [\n\t\t'src/client',\n\t\t'src/factory',\n\t\t'src/protocol',\n\t],\n\tsrcs = [\n\t\t'src/client/WFKafkaClient.cc',\n\t\t'src/protocol/KafkaDataTypes.cc',\n\t\t'src/protocol/kafka_parser.c',\n\t\t'src/factory/KafkaTaskImpl.cc',\n\t\t'src/protocol/KafkaMessage.cc',\n\t\t'src/protocol/KafkaResult.cc',\n\t],\n\tdeps = [\n\t\t':common',\n\t],\n\tvisibility = [\"//visibility:public\"],\n\tlinkopts = [\n\t\t'-lsnappy',\n\t\t'-llz4',\n\t\t'-lz',\n\t\t'-lzstd',\n\t],\n)\n\ncc_library(\n\tname = 'consul',\n\thdrs = [\n\t\t'src/client/WFConsulClient.h',\n\t\t'src/protocol/ConsulDataTypes.h',\n\t],\n\tincludes = [ \n\t\t'src/client',\n\t\t'src/factory',\n\t\t'src/protocol',\n\t\t'src/util',\n\t],\n\tsrcs = [ \n\t\t'src/client/WFConsulClient.cc',\n\t],\n\tdeps = [\n\t\t':common',\n\t\t':http',\n\t],\n\tvisibility = [\"//visibility:public\"],\n)\n\ncc_binary(\n\tname = 'helloworld',\n\tsrcs = ['tutorial/tutorial-00-helloworld.cc'],\n\tdeps = [':http'],\n)\ncc_binary(\n\tname = 'wget',\n\tsrcs = ['tutorial/tutorial-01-wget.cc'],\n\tdeps = [':http'],\n)\ncc_binary(\n\tname = 'redis_cli',\n\tsrcs = ['tutorial/tutorial-02-redis_cli.cc'],\n\tdeps = [':redis'],\n)\n\ncc_binary(\n\tname = 'wget_to_redis',\n\tsrcs = ['tutorial/tutorial-03-wget_to_redis.cc'],\n\tdeps = [':http', 'redis'],\n)\n\ncc_binary(\n\tname = 'http_echo_server',\n\tsrcs = ['tutorial/tutorial-04-http_echo_server.cc'],\n\tdeps = [':http'],\n)\n\ncc_binary(\n\tname = 'http_proxy',\n\tsrcs = ['tutorial/tutorial-05-http_proxy.cc'],\n\tdeps = [':http'],\n)\n\ncc_binary(\n\t name = 'parallel_wget',\n\t srcs = ['tutorial/tutorial-06-parallel_wget.cc'],\n\t deps = [':http'],\n)\n\ncc_binary(\n\tname = 'sort_task',\n\tsrcs = ['tutorial/tutorial-07-sort_task.cc'],\n\tdeps = [':common'],\n)\n\ncc_binary(\n\tname = 'matrix_multiply',\n\tsrcs = ['tutorial/tutorial-08-matrix_multiply.cc'],\n\tdeps = [':common'],\n)\n\ncc_binary(\n\tname = 'http_file_server',\n\tsrcs = ['tutorial/tutorial-09-http_file_server.cc'],\n\tdeps = [':http'],\n)\n\ncc_library(\n\tname = 'user_hdrs',\n\thdrs = ['tutorial/tutorial-10-user_defined_protocol/message.h'],\n\tincludes = ['tutorial/tutorial-10-user_defined_protocol'],\n)\n\ncc_binary(\n\tname = 'server',\n\tsrcs = [\n\t\t'tutorial/tutorial-10-user_defined_protocol/server.cc',\n\t\t'tutorial/tutorial-10-user_defined_protocol/message.cc',\n\t],\n\tdeps = [':common', ':user_hdrs'],\n)\n\ncc_binary(\n\tname = 'client',\n\tsrcs = [\n\t\t'tutorial/tutorial-10-user_defined_protocol/client.cc',\n\t\t'tutorial/tutorial-10-user_defined_protocol/message.cc',\n\t],\n\tdeps = [':common', ':user_hdrs'],\n)\n\ncc_binary(\n\tname = 'graph_task',\n\tsrcs = ['tutorial/tutorial-11-graph_task.cc'],\n\tdeps = [':http'],\n)\n\ncc_binary(\n\tname = 'mysql_cli',\n\tsrcs = ['tutorial/tutorial-12-mysql_cli.cc'],\n\tdeps = [':mysql'],\n)\n\ncc_binary(\n\tname = 'kafka_cli',\n\tsrcs = ['tutorial/tutorial-13-kafka_cli.cc'],\n\tdeps = [':kafka', ':workflow_hdrs'],\n)\n\ncc_binary(\n\tname = 'consul_cli',\n\tsrcs = ['tutorial/tutorial-14-consul_cli.cc'],\n\tdeps = [':consul'],\n)\n\ncc_binary(\n\tname = 'name_service',\n\tsrcs = ['tutorial/tutorial-15-name_service.cc'],\n\tdeps = [':http'],\n)\n\ncc_binary(\n\tname = 'graceful_restart_bootstrap',\n\tsrcs = [\n\t\t'tutorial/tutorial-16-graceful_restart/bootstrap.c',\n\t],\n)\n\ncc_binary(\n\tname = 'graceful_restart_server',\n\tsrcs = [\n\t\t'tutorial/tutorial-16-graceful_restart/server.cc',\n\t],\n\tdeps = [':http'],\n)\n\ncc_binary(\n\tname = 'dns_cli',\n\tsrcs = ['tutorial/tutorial-17-dns_cli.cc'],\n\tdeps = [':common'],\n)\n\ncc_binary(\n\tname = 'redis_subscriber',\n\tsrcs = ['tutorial/tutorial-18-redis_subscriber.cc'],\n\tdeps = [':redis'],\n)\n\ncc_binary(\n\tname = 'dns_server',\n\tsrcs = ['tutorial/tutorial-19-dns_server.cc'],\n\tdeps = [':common'],\n)\n" }, { "path": "CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\n\nset(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING \"build type\")\nset(CMAKE_SKIP_RPATH TRUE)\n\nproject(\n\tworkflow\n\tVERSION 1.0.0\n\tLANGUAGES C CXX\n)\n\nif (CYGWIN)\n\tmessage(FATAL_ERROR \"Sorry, DO NOT support Cygwin\")\nendif ()\n\nif (MINGW)\n\tmessage(FATAL_ERROR \"Sorry, DO NOT support MinGW\")\nendif ()\n\ninclude(GNUInstallDirs)\n\nset(CMAKE_CONFIG_INSTALL_FILE ${PROJECT_BINARY_DIR}/config.toinstall.cmake)\nset(CMAKE_CONFIG_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})\nset(INC_DIR ${PROJECT_SOURCE_DIR}/_include CACHE PATH \"workflow inc\")\nset(LIB_DIR ${PROJECT_SOURCE_DIR}/_lib CACHE PATH \"workflow lib\")\n\nset(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${LIB_DIR})\nset(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LIB_DIR})\n\nadd_custom_target(\n\tLINK_HEADERS ALL\n\tCOMMENT \"link headers...\"\n)\n\nINCLUDE(CMakeLists_Headers.txt)\n\nmacro(makeLink src dest target)\n\tadd_custom_command(\n\t\tTARGET ${target} PRE_BUILD\n\t\tCOMMAND ${CMAKE_COMMAND} -E copy_if_different ${src} ${dest}\n\t)\nendmacro()\n\nadd_custom_command(\n\tTARGET LINK_HEADERS PRE_BUILD\n\tCOMMAND ${CMAKE_COMMAND} -E make_directory ${INC_DIR}/${PROJECT_NAME}\n)\n\nforeach(header_file ${INCLUDE_HEADERS} ${INCLUDE_KERNEL_HEADERS})\n\tstring(REPLACE \"/\" \";\" arr ${header_file})\n\tlist(GET arr -1 file_name)\n\tmakeLink(${PROJECT_SOURCE_DIR}/${header_file} ${INC_DIR}/${PROJECT_NAME}/${file_name} LINK_HEADERS)\nendforeach()\n\nmessage(\"CMAKE_C_FLAGS_DEBUG is ${CMAKE_C_FLAGS_DEBUG}\")\nmessage(\"CMAKE_C_FLAGS_RELEASE is ${CMAKE_C_FLAGS_RELEASE}\")\nmessage(\"CMAKE_C_FLAGS_RELWITHDEBINFO is ${CMAKE_C_FLAGS_RELWITHDEBINFO}\")\nmessage(\"CMAKE_C_FLAGS_MINSIZEREL is ${CMAKE_C_FLAGS_MINSIZEREL}\")\n\nmessage(\"CMAKE_CXX_FLAGS_DEBUG is ${CMAKE_CXX_FLAGS_DEBUG}\")\nmessage(\"CMAKE_CXX_FLAGS_RELEASE is ${CMAKE_CXX_FLAGS_RELEASE}\")\nmessage(\"CMAKE_CXX_FLAGS_RELWITHDEBINFO is ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}\")\nmessage(\"CMAKE_CXX_FLAGS_MINSIZEREL is ${CMAKE_CXX_FLAGS_MINSIZEREL}\")\n\nset(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -Wall -fPIC -pipe -std=gnu90\")\nset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wall -fPIC -pipe -std=c++11 -fno-exceptions -Wno-invalid-offsetof\")\nif (APPLE)\n\tset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations\")\nendif()\n\nadd_subdirectory(src)\n\n####CONFIG\n\ninclude(CMakePackageConfigHelpers)\nset(CONFIG_INC_DIR ${INC_DIR})\nset(CONFIG_LIB_DIR ${LIB_DIR})\nconfigure_package_config_file(\n\t${PROJECT_NAME}-config.cmake.in\n\t${PROJECT_SOURCE_DIR}/${PROJECT_NAME}-config.cmake\n\tINSTALL_DESTINATION ${CMAKE_CONFIG_INSTALL_DIR}\n\tPATH_VARS CONFIG_INC_DIR CONFIG_LIB_DIR\n)\n\nset(CONFIG_INC_DIR ${CMAKE_INSTALL_INCLUDEDIR})\nset(CONFIG_LIB_DIR ${CMAKE_INSTALL_LIBDIR})\nconfigure_package_config_file(\n\t${PROJECT_NAME}-config.cmake.in\n\t${CMAKE_CONFIG_INSTALL_FILE}\n\tINSTALL_DESTINATION ${CMAKE_CONFIG_INSTALL_DIR}\n\tPATH_VARS CONFIG_INC_DIR CONFIG_LIB_DIR\n)\n\nwrite_basic_package_version_file(\n\t${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake\n\tVERSION ${WORKFLOW_VERSION}\n\tCOMPATIBILITY AnyNewerVersion \n)\n\ninstall(\n\tFILES ${CMAKE_CONFIG_INSTALL_FILE}\n\tDESTINATION ${CMAKE_CONFIG_INSTALL_DIR}\n\tCOMPONENT devel\n\tRENAME ${PROJECT_NAME}-config.cmake\n)\n\ninstall(\n\tFILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake\n\tDESTINATION ${CMAKE_CONFIG_INSTALL_DIR}\n\tCOMPONENT devel\n)\n\ninstall(\n\tFILES ${INCLUDE_HEADERS} ${INCLUDE_KERNEL_HEADERS}\n\tDESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}\n\tCOMPONENT devel\n)\n\ninstall(\n\tFILES README.md\n\tDESTINATION \"${CMAKE_INSTALL_DOCDIR}-${PROJECT_VERSION}\"\n\tCOMPONENT devel\n)\n" }, { "path": "CMakeLists_Headers.txt", "content": "cmake_minimum_required(VERSION 3.10)\n\nset(COMMON_KERNEL_HEADERS\n\tsrc/kernel/CommRequest.h\n\tsrc/kernel/CommScheduler.h\n\tsrc/kernel/Communicator.h\n\tsrc/kernel/SleepRequest.h\n\tsrc/kernel/ExecRequest.h\n\tsrc/kernel/IORequest.h\n\tsrc/kernel/Executor.h\n\tsrc/kernel/list.h\n\tsrc/kernel/mpoller.h\n\tsrc/kernel/poller.h\n\tsrc/kernel/msgqueue.h\n\tsrc/kernel/rbtree.h\n\tsrc/kernel/SubTask.h\n\tsrc/kernel/thrdpool.h\n)\n\nif (CMAKE_SYSTEM_NAME STREQUAL \"Linux\" OR CMAKE_SYSTEM_NAME STREQUAL \"Android\")\n\tset(INCLUDE_KERNEL_HEADERS\n\t\t${COMMON_KERNEL_HEADERS}\n\t\tsrc/kernel/IOService_linux.h\n\t)\nelseif (UNIX)\n\tset(INCLUDE_KERNEL_HEADERS\n\t\t${COMMON_KERNEL_HEADERS}\n\t\tsrc/kernel/IOService_thread.h\n\t)\nelse ()\n\tmessage(FATAL_ERROR \"IOService unsupported.\")\nendif ()\n\nset(INCLUDE_HEADERS\n\tsrc/protocol/ProtocolMessage.h\n\tsrc/protocol/http_parser.h\n\tsrc/protocol/HttpMessage.h\n\tsrc/protocol/HttpUtil.h\n\tsrc/protocol/redis_parser.h\n\tsrc/protocol/RedisMessage.h\n\tsrc/protocol/mysql_stream.h\n\tsrc/protocol/MySQLMessage.h\n\tsrc/protocol/MySQLMessage.inl\n\tsrc/protocol/MySQLResult.h\n\tsrc/protocol/MySQLResult.inl\n\tsrc/protocol/MySQLUtil.h\n\tsrc/protocol/mysql_parser.h\n\tsrc/protocol/mysql_types.h\n\tsrc/protocol/mysql_byteorder.h\n\tsrc/protocol/PackageWrapper.h\n\tsrc/protocol/SSLWrapper.h\n\tsrc/protocol/dns_types.h\n\tsrc/protocol/dns_parser.h\n\tsrc/protocol/DnsMessage.h\n\tsrc/protocol/DnsUtil.h\n\tsrc/protocol/TLVMessage.h\n\tsrc/protocol/ConsulDataTypes.h\n\tsrc/server/WFServer.h\n\tsrc/server/WFDnsServer.h\n\tsrc/server/WFHttpServer.h\n\tsrc/server/WFRedisServer.h\n\tsrc/server/WFMySQLServer.h\n\tsrc/client/WFHttpChunkedClient.h\n\tsrc/client/WFMySQLConnection.h\n\tsrc/client/WFRedisSubscriber.h\n\tsrc/client/WFConsulClient.h\n\tsrc/client/WFDnsClient.h\n\tsrc/manager/DnsCache.h\n\tsrc/manager/WFGlobal.h\n\tsrc/manager/UpstreamManager.h\n\tsrc/manager/RouteManager.h\n\tsrc/manager/EndpointParams.h\n\tsrc/manager/WFFuture.h\n\tsrc/manager/WFFacilities.h\n\tsrc/manager/WFFacilities.inl\n\tsrc/util/json_parser.h\n\tsrc/util/EncodeStream.h\n\tsrc/util/LRUCache.h\n\tsrc/util/StringUtil.h\n\tsrc/util/URIParser.h\n\tsrc/factory/WFConnection.h\n\tsrc/factory/WFTask.h\n\tsrc/factory/WFTask.inl\n\tsrc/factory/WFGraphTask.h\n\tsrc/factory/WFTaskError.h\n\tsrc/factory/WFTaskFactory.h\n\tsrc/factory/WFTaskFactory.inl\n\tsrc/factory/WFAlgoTaskFactory.h\n\tsrc/factory/WFAlgoTaskFactory.inl\n\tsrc/factory/Workflow.h\n\tsrc/factory/WFOperator.h\n\tsrc/factory/WFResourcePool.h\n\tsrc/factory/WFMessageQueue.h\n\tsrc/factory/HttpTaskImpl.inl\n\tsrc/factory/RedisTaskImpl.inl\n\tsrc/nameservice/WFNameService.h\n\tsrc/nameservice/WFDnsResolver.h\n\tsrc/nameservice/WFServiceGovernance.h\n\tsrc/nameservice/UpstreamPolicies.h\n)\n\nif(KAFKA STREQUAL \"y\")\n\tset(INCLUDE_HEADERS\n\t\t${INCLUDE_HEADERS}\n\t\tsrc/util/crc32c.h\n\t\tsrc/protocol/KafkaMessage.h\n\t\tsrc/protocol/KafkaDataTypes.h\n\t\tsrc/protocol/KafkaResult.h\n\t\tsrc/protocol/kafka_parser.h\n\t\tsrc/client/WFKafkaClient.h\n\t\tsrc/factory/KafkaTaskImpl.inl\n\t)\nendif()\n\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment include:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others’ private information, such as a physical or electronic address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at xiehan@sogou-inc.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]\n\n[homepage]: http://contributor-covenant.org\n[version]: http://contributor-covenant.org/version/1/4/\n" }, { "path": "GNUmakefile", "content": "ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))\nALL_TARGETS := all base check install preinstall clean tutorial\nMAKE_FILE := Makefile\n\nDEFAULT_BUILD_DIR := build.cmake\nBUILD_DIR := $(shell if [ -f $(MAKE_FILE) ]; then echo \".\"; else echo $(DEFAULT_BUILD_DIR); fi)\nCMAKE3 := $(shell if which cmake3>/dev/null ; then echo cmake3; else echo cmake; fi;)\n\n.PHONY: $(ALL_TARGETS)\n\nall: base\n\t$(MAKE) -C $(BUILD_DIR) -f Makefile\n\nbase:\n\tmkdir -p $(BUILD_DIR)\n\nifeq ($(DEBUG),y)\n\tcd $(BUILD_DIR) && $(CMAKE3) -D CMAKE_BUILD_TYPE=Debug -D CONSUL=$(CONSUL) -D KAFKA=$(KAFKA) -D MYSQL=$(MYSQL) -D REDIS=$(REDIS) -D UPSTREAM=$(UPSTREAM) $(ROOT_DIR)\nelse ifneq (\"${INSTALL_PREFIX}install_prefix\", \"install_prefix\")\n\tcd $(BUILD_DIR) && $(CMAKE3) -DCMAKE_INSTALL_PREFIX:STRING=${INSTALL_PREFIX} -D CONSUL=$(CONSUL) -D KAFKA=$(KAFKA) -D MYSQL=$(MYSQL) -D REDIS=$(REDIS) -D UPSTREAM=$(UPSTREAM) $(ROOT_DIR)\nelse\n\tcd $(BUILD_DIR) && $(CMAKE3) -D CONSUL=$(CONSUL) -D KAFKA=$(KAFKA) -D MYSQL=$(MYSQL) -D REDIS=$(REDIS) -D UPSTREAM=$(UPSTREAM) $(ROOT_DIR)\nendif\n\ntutorial: all\n\t$(MAKE) -C tutorial\n\ncheck: all\n\t$(MAKE) -C test check\n\ninstall preinstall: base\n\tmkdir -p $(BUILD_DIR)\n\tcd $(BUILD_DIR) && $(CMAKE3) $(ROOT_DIR)\n\t$(MAKE) -C $(BUILD_DIR) -f Makefile $@\n\nclean:\n\t-$(MAKE) -C test clean\n\t-$(MAKE) -C tutorial clean\n\trm -rf $(DEFAULT_BUILD_DIR)\n\trm -rf _include\n\trm -rf _lib\n\tfind . -name CMakeCache.txt | xargs rm -f\n\tfind . -name Makefile | xargs rm -f\n\tfind . -name \"*.cmake\" | xargs rm -f\n\tfind . -name CMakeFiles | xargs rm -rf\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2020 Sogou Inc. All rights reserved.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "LICENSE_GPLV2", "content": " GNU GENERAL PUBLIC LICENSE\n Version 2, June 1991\n\n Copyright (C) 1989, 1991 Free Software Foundation, Inc.,\n 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA\n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n Preamble\n\n The licenses for most software are designed to take away your\nfreedom to share and change it. By contrast, the GNU General Public\nLicense is intended to guarantee your freedom to share and change free\nsoftware--to make sure the software is free for all its users. This\nGeneral Public License applies to most of the Free Software\nFoundation's software and to any other program whose authors commit to\nusing it. (Some other Free Software Foundation software is covered by\nthe GNU Lesser General Public License instead.) You can apply it to\nyour programs, too.\n\n When we speak of free software, we are referring to freedom, not\nprice. Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthis service if you wish), that you receive source code or can get it\nif you want it, that you can change the software or use pieces of it\nin new free programs; and that you know you can do these things.\n\n To protect your rights, we need to make restrictions that forbid\nanyone to deny you these rights or to ask you to surrender the rights.\nThese restrictions translate to certain responsibilities for you if you\ndistribute copies of the software, or if you modify it.\n\n For example, if you distribute copies of such a program, whether\ngratis or for a fee, you must give the recipients all the rights that\nyou have. You must make sure that they, too, receive or can get the\nsource code. And you must show them these terms so they know their\nrights.\n\n We protect your rights with two steps: (1) copyright the software, and\n(2) offer you this license which gives you legal permission to copy,\ndistribute and/or modify the software.\n\n Also, for each author's protection and ours, we want to make certain\nthat everyone understands that there is no warranty for this free\nsoftware. If the software is modified by someone else and passed on, we\nwant its recipients to know that what they have is not the original, so\nthat any problems introduced by others will not reflect on the original\nauthors' reputations.\n\n Finally, any free program is threatened constantly by software\npatents. We wish to avoid the danger that redistributors of a free\nprogram will individually obtain patent licenses, in effect making the\nprogram proprietary. To prevent this, we have made it clear that any\npatent must be licensed for everyone's free use or not licensed at all.\n\n The precise terms and conditions for copying, distribution and\nmodification follow.\n\n GNU GENERAL PUBLIC LICENSE\n TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\n\n 0. This License applies to any program or other work which contains\na notice placed by the copyright holder saying it may be distributed\nunder the terms of this General Public License. The \"Program\", below,\nrefers to any such program or work, and a \"work based on the Program\"\nmeans either the Program or any derivative work under copyright law:\nthat is to say, a work containing the Program or a portion of it,\neither verbatim or with modifications and/or translated into another\nlanguage. (Hereinafter, translation is included without limitation in\nthe term \"modification\".) Each licensee is addressed as \"you\".\n\nActivities other than copying, distribution and modification are not\ncovered by this License; they are outside its scope. The act of\nrunning the Program is not restricted, and the output from the Program\nis covered only if its contents constitute a work based on the\nProgram (independent of having been made by running the Program).\nWhether that is true depends on what the Program does.\n\n 1. You may copy and distribute verbatim copies of the Program's\nsource code as you receive it, in any medium, provided that you\nconspicuously and appropriately publish on each copy an appropriate\ncopyright notice and disclaimer of warranty; keep intact all the\nnotices that refer to this License and to the absence of any warranty;\nand give any other recipients of the Program a copy of this License\nalong with the Program.\n\nYou may charge a fee for the physical act of transferring a copy, and\nyou may at your option offer warranty protection in exchange for a fee.\n\n 2. You may modify your copy or copies of the Program or any portion\nof it, thus forming a work based on the Program, and copy and\ndistribute such modifications or work under the terms of Section 1\nabove, provided that you also meet all of these conditions:\n\n a) You must cause the modified files to carry prominent notices\n stating that you changed the files and the date of any change.\n\n b) You must cause any work that you distribute or publish, that in\n whole or in part contains or is derived from the Program or any\n part thereof, to be licensed as a whole at no charge to all third\n parties under the terms of this License.\n\n c) If the modified program normally reads commands interactively\n when run, you must cause it, when started running for such\n interactive use in the most ordinary way, to print or display an\n announcement including an appropriate copyright notice and a\n notice that there is no warranty (or else, saying that you provide\n a warranty) and that users may redistribute the program under\n these conditions, and telling the user how to view a copy of this\n License. (Exception: if the Program itself is interactive but\n does not normally print such an announcement, your work based on\n the Program is not required to print an announcement.)\n\nThese requirements apply to the modified work as a whole. If\nidentifiable sections of that work are not derived from the Program,\nand can be reasonably considered independent and separate works in\nthemselves, then this License, and its terms, do not apply to those\nsections when you distribute them as separate works. But when you\ndistribute the same sections as part of a whole which is a work based\non the Program, the distribution of the whole must be on the terms of\nthis License, whose permissions for other licensees extend to the\nentire whole, and thus to each and every part regardless of who wrote it.\n\nThus, it is not the intent of this section to claim rights or contest\nyour rights to work written entirely by you; rather, the intent is to\nexercise the right to control the distribution of derivative or\ncollective works based on the Program.\n\nIn addition, mere aggregation of another work not based on the Program\nwith the Program (or with a work based on the Program) on a volume of\na storage or distribution medium does not bring the other work under\nthe scope of this License.\n\n 3. You may copy and distribute the Program (or a work based on it,\nunder Section 2) in object code or executable form under the terms of\nSections 1 and 2 above provided that you also do one of the following:\n\n a) Accompany it with the complete corresponding machine-readable\n source code, which must be distributed under the terms of Sections\n 1 and 2 above on a medium customarily used for software interchange; or,\n\n b) Accompany it with a written offer, valid for at least three\n years, to give any third party, for a charge no more than your\n cost of physically performing source distribution, a complete\n machine-readable copy of the corresponding source code, to be\n distributed under the terms of Sections 1 and 2 above on a medium\n customarily used for software interchange; or,\n\n c) Accompany it with the information you received as to the offer\n to distribute corresponding source code. (This alternative is\n allowed only for noncommercial distribution and only if you\n received the program in object code or executable form with such\n an offer, in accord with Subsection b above.)\n\nThe source code for a work means the preferred form of the work for\nmaking modifications to it. For an executable work, complete source\ncode means all the source code for all modules it contains, plus any\nassociated interface definition files, plus the scripts used to\ncontrol compilation and installation of the executable. However, as a\nspecial exception, the source code distributed need not include\nanything that is normally distributed (in either source or binary\nform) with the major components (compiler, kernel, and so on) of the\noperating system on which the executable runs, unless that component\nitself accompanies the executable.\n\nIf distribution of executable or object code is made by offering\naccess to copy from a designated place, then offering equivalent\naccess to copy the source code from the same place counts as\ndistribution of the source code, even though third parties are not\ncompelled to copy the source along with the object code.\n\n 4. You may not copy, modify, sublicense, or distribute the Program\nexcept as expressly provided under this License. Any attempt\notherwise to copy, modify, sublicense or distribute the Program is\nvoid, and will automatically terminate your rights under this License.\nHowever, parties who have received copies, or rights, from you under\nthis License will not have their licenses terminated so long as such\nparties remain in full compliance.\n\n 5. You are not required to accept this License, since you have not\nsigned it. However, nothing else grants you permission to modify or\ndistribute the Program or its derivative works. These actions are\nprohibited by law if you do not accept this License. Therefore, by\nmodifying or distributing the Program (or any work based on the\nProgram), you indicate your acceptance of this License to do so, and\nall its terms and conditions for copying, distributing or modifying\nthe Program or works based on it.\n\n 6. Each time you redistribute the Program (or any work based on the\nProgram), the recipient automatically receives a license from the\noriginal licensor to copy, distribute or modify the Program subject to\nthese terms and conditions. You may not impose any further\nrestrictions on the recipients' exercise of the rights granted herein.\nYou are not responsible for enforcing compliance by third parties to\nthis License.\n\n 7. If, as a consequence of a court judgment or allegation of patent\ninfringement or for any other reason (not limited to patent issues),\nconditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License. If you cannot\ndistribute so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you\nmay not distribute the Program at all. For example, if a patent\nlicense would not permit royalty-free redistribution of the Program by\nall those who receive copies directly or indirectly through you, then\nthe only way you could satisfy both it and this License would be to\nrefrain entirely from distribution of the Program.\n\nIf any portion of this section is held invalid or unenforceable under\nany particular circumstance, the balance of the section is intended to\napply and the section as a whole is intended to apply in other\ncircumstances.\n\nIt is not the purpose of this section to induce you to infringe any\npatents or other property right claims or to contest validity of any\nsuch claims; this section has the sole purpose of protecting the\nintegrity of the free software distribution system, which is\nimplemented by public license practices. Many people have made\ngenerous contributions to the wide range of software distributed\nthrough that system in reliance on consistent application of that\nsystem; it is up to the author/donor to decide if he or she is willing\nto distribute software through any other system and a licensee cannot\nimpose that choice.\n\nThis section is intended to make thoroughly clear what is believed to\nbe a consequence of the rest of this License.\n\n 8. If the distribution and/or use of the Program is restricted in\ncertain countries either by patents or by copyrighted interfaces, the\noriginal copyright holder who places the Program under this License\nmay add an explicit geographical distribution limitation excluding\nthose countries, so that distribution is permitted only in or among\ncountries not thus excluded. In such case, this License incorporates\nthe limitation as if written in the body of this License.\n\n 9. The Free Software Foundation may publish revised and/or new versions\nof the General Public License from time to time. Such new versions will\nbe similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\nEach version is given a distinguishing version number. If the Program\nspecifies a version number of this License which applies to it and \"any\nlater version\", you have the option of following the terms and conditions\neither of that version or of any later version published by the Free\nSoftware Foundation. If the Program does not specify a version number of\nthis License, you may choose any version ever published by the Free Software\nFoundation.\n\n 10. If you wish to incorporate parts of the Program into other free\nprograms whose distribution conditions are different, write to the author\nto ask for permission. For software which is copyrighted by the Free\nSoftware Foundation, write to the Free Software Foundation; we sometimes\nmake exceptions for this. Our decision will be guided by the two goals\nof preserving the free status of all derivatives of our free software and\nof promoting the sharing and reuse of software generally.\n\n NO WARRANTY\n\n 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\nFOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN\nOTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\nPROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\nOR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\nMERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS\nTO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE\nPROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\nREPAIR OR CORRECTION.\n\n 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\nREDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\nINCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\nOUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\nTO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\nYOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\nPROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\nPOSSIBILITY OF SUCH DAMAGES.\n\n END OF TERMS AND CONDITIONS\n\n How to Apply These Terms to Your New Programs\n\n If you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these terms.\n\n To do so, attach the following notices to the program. It is safest\nto attach them to the start of each source file to most effectively\nconvey the exclusion of warranty; and each file should have at least\nthe \"copyright\" line and a pointer to where the full notice is found.\n\n \n Copyright (C) \n\n This program is free software; you can redistribute it and/or modify\n it under the terms of the GNU General Public License as published by\n the Free Software Foundation; either version 2 of the License, or\n (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU General Public License for more details.\n\n You should have received a copy of the GNU General Public License along\n with this program; if not, write to the Free Software Foundation, Inc.,\n 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\n\nAlso add information on how to contact you by electronic and paper mail.\n\nIf the program is interactive, make it output a short notice like this\nwhen it starts in an interactive mode:\n\n Gnomovision version 69, Copyright (C) year name of author\n Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.\n This is free software, and you are welcome to redistribute it\n under certain conditions; type `show c' for details.\n\nThe hypothetical commands `show w' and `show c' should show the appropriate\nparts of the General Public License. Of course, the commands you use may\nbe called something other than `show w' and `show c'; they could even be\nmouse-clicks or menu items--whatever suits your program.\n\nYou should also get your employer (if you work as a programmer) or your\nschool, if any, to sign a \"copyright disclaimer\" for the program, if\nnecessary. Here is a sample; alter the names:\n\n Yoyodyne, Inc., hereby disclaims all copyright interest in the program\n `Gnomovision' (which makes passes at compilers) written by James Hacker.\n\n , 1 April 1989\n Ty Coon, President of Vice\n\nThis General Public License does not permit incorporating your program into\nproprietary programs. If your program is a subroutine library, you may\nconsider it more useful to permit linking proprietary applications with the\nlibrary. If this is what you want to do, use the GNU Lesser General\nPublic License instead of this License.\n" }, { "path": "README.md", "content": "[简体中文版（推荐）](README_cn.md)\n\n## Sogou C++ Workflow\n\n[![License](https://img.shields.io/badge/License-Apache%202.0-green.svg)](https://github.com/sogou/workflow/blob/master/LICENSE)\n[![Language](https://img.shields.io/badge/language-c++-red.svg)](https://en.cppreference.com/) \n[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey.svg)](https://img.shields.io/badge/platform-linux%20%7C%20macos20%7C%20windows-lightgrey.svg)\n[![Build Status](https://img.shields.io/github/actions/workflow/status/sogou/workflow/ci.yml?branch=master)](https://github.com/sogou/workflow/actions?query=workflow%3A%22ci+build%22++)\n\nAs **Sogou\\`s C++ server engine**, Sogou C++ Workflow supports almost all **back-end C++ online services** of Sogou, including all search services, cloud input method, online advertisements, etc., handling more than **10 billion** requests every day. This is an **enterprise-level programming engine** in light and elegant design which can satisfy most C++ back-end development requirements.\n\n#### You can use it:\n\n* To quickly build an **HTTP server**:\n\n~~~cpp\n#include \n#include \"workflow/WFHttpServer.h\"\n\nint main()\n{\n WFHttpServer server([](WFHttpTask *task) {\n task->get_resp()->append_output_body(\"Hello World!\");\n });\n\n if (server.start(8888) == 0) { // start server on port 8888\n getchar(); // press \"Enter\" to end.\n server.stop();\n }\n\n return 0;\n}\n~~~\n\n* As a **multifunctional asynchronous client**, it currently supports `HTTP`, `Redis`, `MySQL` and `Kafka` protocols.\n * ``MySQL`` protocol supports ``MariaDB``, ``TiDB`` as well.\n* To implement **client/server on user-defined protocol** and build your own **RPC system**.\n * [srpc](https://github.com/sogou/srpc) is based on it and it is an independent open source project, which supports srpc, brpc, trpc and thrift protocols.\n* To build **asynchronous workflow**; support common **series** and **parallel** structures, and also support any **DAG** structures.\n* As a **parallel computing tool**. In addition to **networking tasks**, Sogou C++ Workflow also includes **the scheduling of computing tasks**. All types of tasks can be put into **the same** flow.\n* As an **asynchronous file IO tool** in `Linux` system, with high performance exceeding any system call. Disk file IO is also a task.\n* To realize any **high-performance** and **high-concurrency** back-end service with a very complex relationship between computing and networking.\n* To build a **micro service** system.\n * This project has built-in **service governance** and **load balancing** features.\n* Wiki link : [PaaS Architecture](https://github.com/sogou/workflow/wiki)\n\n#### Compiling and Running Environment\n\n* This project supports `Linux`, `macOS`, `Windows`, `Android` and other operating systems.\n * `Windows` version is currently released as an independent [branch](https://github.com/sogou/workflow/tree/windows), using `iocp` to implement asynchronous networking. All user interfaces are consistent with the `Linux` version.\n* Supports all CPU platforms, including 32 or 64-bit `x86` processors, big-endian or little-endian `arm` processors, `loongson` processors.\n* Master branch requires `OpenSSL 1.1` or above, and BoringSSL is fully compatible. If you don't like SSL, you may checkout the [nossl](https://github.com/sogou/workflow/tree/nossl) branch.\n* Uses the `C++11` standard and therefore, it should be compiled with a compiler which supports `C++11`. Does not rely on `boost` or `asio`.\n* No other dependencies. However, if you need `Kafka` protocol, some compression libraries should be installed, including `lz4`, `zstd` and `snappy`.\n\n### Get Started (Linux, macOS):\n~~~sh\ngit clone https://github.com/sogou/workflow\ncd workflow\nmake\ncd tutorial\nmake\n~~~~\n\n#### With SRPC Tool (NEW!)：\nhttps://github.com/sogou/srpc/blob/master/tools/README.md\n\n#### With [apt-get](https://launchpad.net/ubuntu/+source/workflow) on Debian Linux, ubuntu:\nSogou C++ Workflow has been packaged for Debian Linux and ubuntu 22.04. \nTo install the Workflow library for development purposes:\n~~~~sh\nsudo apt-get install libworkflow-dev\n~~~~\n\nTo install the Workflow library for deployment:\n~~~~sh\nsudo apt-get install libworkflow1\n~~~~\n\n#### With [dnf](https://packages.fedoraproject.org/pkgs/workflow) on Fedora Linux:\nSogou C++ Workflow has been packaged for Fedora Linux. \nTo install the Workflow library for development purposes:\n~~~~sh\nsudo dnf install workflow-devel\n~~~~\n\nTo install the Workflow library for deployment:\n~~~~sh\nsudo dnf install workflow\n~~~~\n\n#### With xmake\n\nIf you want to use xmake to build workflow, you can see [xmake build document](docs/en/xmake.md)\n\n# Tutorials\n\n* Client\n * [Creating your first task：wget](docs/en/tutorial-01-wget.md)\n * [Implementing Redis set and get：redis\\_cli](docs/en/tutorial-02-redis_cli.md)\n * [More features about series：wget\\_to\\_redis](docs/en/tutorial-03-wget_to_redis.md)\n* Server\n * [First server：http\\_echo\\_server](docs/en/tutorial-04-http_echo_server.md)\n * [Asynchronous server：http\\_proxy](docs/en/tutorial-05-http_proxy.md)\n* Parallel tasks and Series　\n * [A simple parallel wget：parallel\\_wget](docs/en/tutorial-06-parallel_wget.md)\n* Important topics\n * [About error](docs/en/about-error.md)\n * [About timeout](docs/en/about-timeout.md)\n * [About global configuration](docs/en/about-config.md)\n * [About DNS](docs/en/about-dns.md)\n * [About exit](docs/en/about-exit.md)\n* Computing tasks\n * [Using the build-in algorithm factory：sort\\_task](docs/en/tutorial-07-sort_task.md)\n * [User-defined computing task：matrix\\_multiply](docs/en/tutorial-08-matrix_multiply.md)\n * [Use computing task in a simple way: go task](docs/en/about-go-task.md)\n* Asynchronous File IO tasks\n * [Http server with file IO：http\\_file\\_server](docs/en/tutorial-09-http_file_server.md)\n* User-defined protocol\n * [A simple user-defined protocol: client/server](docs/en/tutorial-10-user_defined_protocol.md)\n * [Use TLV message](docs/en/about-tlv-message.md)\n* Other important tasks/components\n * [About timer](docs/en/about-timer.md)\n * [About counter](docs/en/about-counter.md)\n * [About resource pool](docs/en/about-resource-pool.md)\n * [About module](docs/en/about-module.md)\n * [About DAG](docs/en/tutorial-11-graph_task.md)\n* Service governance\n * [About service governance](docs/en/about-service-governance.md)\n * [More documents about upstream](docs/en/about-upstream.md)\n* Connection context\n * [About connection context](docs/en/about-connection-context.md)\n* Built-in clients\n * [Asynchronous MySQL client：mysql\\_cli](docs/en/tutorial-12-mysql_cli.md)\n * [Asynchronous Kafka client: kafka\\_cli](docs/en/tutorial-13-kafka_cli.md)\n\n#### Programming Paradigm\n\nProgram = Protocol + Algorithm + Workflow\n\n* Protocol\n * In most cases, users use built-in common network protocols, such as HTTP, Redis or various rpc.\n * Users can also easily customize user-defined network protocol. In the customization, they only need to provide serialization and deserialization functions to define their own client/server.\n* Algorithm\n * In our design, the algorithm is a concept symmetrical to the protocol.\n * If protocol call is rpc, then algorithm call is an apc (Async Procedure Call).\n * We have provided some general algorithms, such as sort, merge, psort, reduce, which can be used directly.\n * Compared with a user-defined protocol, a user-defined algorithm is much more common. Any complicated computation with clear boundaries should be packaged into an algorithm.\n* Workflow\n * Workflow is the actual business logic, which is to put the protocols and algorithms into the flow graph for use.\n * The typical workflow is a closed series-parallel graph. Complex business logic may be a non-closed DAG.\n * The workflow graph can be constructed directly or dynamically generated based on the results of each step. All tasks are executed asynchronously.\n\nStructured Concurrency and Task Abstraction\n\n* Our system contains five basic tasks: communication, computation, file IO, timer, and counter.\n* All tasks are generated by the task factory, and users organize the concurrency structure by calling interfaces, such as series, parallel, DAG, etc.\n* In most cases, the tasks generated by the user through the task factory is a complex task which encapsulates multiple asynchronous processes, but it is transparent to the user.\n * For example, an HTTP request may include many asynchronous processes (DNS, redirection), but for user, it is just a networking task.\n * File sorting seems to be an algorithm, but it actually includes many complex interaction processes between file IO and CPU computation.\n * If you think of business logic as building circuits with well-designed electronic components, then each electronic component may be a complex circuit.\n * The task abstraction mechanism greatly reduces the number of tasks users need to create and the depth of callbacks.\n* Any task runs in a **SeriesWork** and the tasks in the same SeriesWork shares the series context, which simplifies data transfer between asynchronous tasks.\n\nCallback and Memory Reclamation Mechanism\n\n* All calls are executed asynchronously, and there is almost no operation that occupies a thread.\n* Explicit callback mechanism. Users are aware that they are writing asynchronous programs.\n* **A set of object lifecycle mechanisms greatly simplifies memory management for asynchronous programs.**\n * The lifecycle of any task created by the framework is from creation until the callback function finishes running. There is no risk of leakage.\n * If a task is created but the user does not want to run it, the user needs to release it through the `dismiss()` interface.\n * Any data in the task, such as the response of the network request, will also be recycled with the task. At this time, the user can use `std::move()` to move the required data.\n * The project doesn’t use `std::shared_ptr` to manage memory.\n\n* We try to avoid user's derivations, and encapsulate user behavior with `std::function` instead, including:\n * The callback of any task.\n * Any server's process. This conforms to the `FaaS` (Function as a Service) idea.\n * The realization of an algorithm is simply a `std::function`. But the algorithm can also be implemented by derivation.\n * If used deeply, one will find that everything can be derived.\n\n#### Any Other Questions?\n\nYou may check the [FAQ](https://github.com/sogou/workflow/issues/406) and [issues](https://github.com/sogou/workflow/issues) list first to see if you can find the answer.\n\nYou are very welcome to send the problems you encounter in use to [issues](https://github.com/sogou/workflow/issues), and we will answer them as soon as possible. At the same time, more issues will also help new users.\n\n" }, { "path": "README_cn.md", "content": "[English version](README.md)\n\n## Sogou C++ Workflow\n[![License](https://img.shields.io/badge/License-Apache%202.0-green.svg)](https://github.com/sogou/workflow/blob/master/LICENSE)\n[![Language](https://img.shields.io/badge/language-c++-red.svg)](https://en.cppreference.com/)\n[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey.svg)](https://img.shields.io/badge/platform-linux%20%7C%20macos20%7C%20windows-lightgrey.svg)\n[![Build Status](https://img.shields.io/github/actions/workflow/status/sogou/workflow/ci.yml?branch=master)](https://github.com/sogou/workflow/actions?query=workflow%3A%22ci+build%22++)\n\n搜狗公司C++服务器引擎，编程范式。支撑搜狗几乎所有后端C++在线服务，包括所有搜索服务，云输入法，在线广告等，每日处理数百亿请求。这是一个设计轻盈优雅的企业级程序引擎，可以满足大多数后端与嵌入式开发需求。 \n#### 你可以用来：\n* 快速搭建http服务器：\n~~~cpp\n#include \n#include \"workflow/WFHttpServer.h\"\n\nint main()\n{\n WFHttpServer server([](WFHttpTask *task) {\n task->get_resp()->append_output_body(\"Hello World!\");\n });\n\n if (server.start(8888) == 0) { // start server on port 8888\n getchar(); // press \"Enter\" to end.\n server.stop();\n }\n\n return 0;\n}\n~~~\n* 作为万能异步客户端。目前支持``http``，``redis``，``mysql``和``kafka``协议。\n * 轻松构建效率极高的spider。\n * ``mysql``协议同时也支持``MariaDB``和``TiDB``等数据库。\n* 实现自定义协议client/server，构建自己的RPC系统。\n * [srpc](https://github.com/sogou/srpc)就是以它为基础，作为独立项目开源。支持``srpc``，``brpc``，``trpc``和``thrift``等协议。\n* 构建异步任务流，支持常用的串并联，也支持更加复杂的DAG结构。\n* 作为并行计算工具使用。除了网络任务，我们也包含计算任务的调度。所有类型的任务都可以放入同一个流中。\n* 在``Linux``系统下作为文件异步IO工具使用，性能超过任何标准调用。磁盘IO也是一种任务。\n* 实现任何计算与通讯关系非常复杂的高性能高并发的后端服务。\n* 构建微服务系统。\n * 项目内置服务治理与负载均衡等功能。\n* Wiki链接 : [PaaS 架构图](https://github.com/sogou/workflow/wiki)\n\n#### 编译和运行环境\n* 项目支持``Linux``，``macOS``，``Windows``，``Android``等操作系统。\n * ``Windows``版以[windows](https://github.com/sogou/workflow/tree/windows)分支发布，使用``iocp``实现异步网络。用户接口与``Linux``版一致。\n* 支持所有CPU平台，包括32或64位``x86``处理器，大端或小端``arm``处理器，国产``loongson``龙芯处理器实测支持。\n* 需要依赖于``OpenSSL 1.1``或以上版本，也兼容BoringSSL。\n * 不喜欢SSL的用户可以使用[nossl](https://github.com/sogou/workflow/tree/nossl)分支，代码更简洁。\n* 项目使用了``C++11``标准，需要用支持``C++11``的编译器编译。但不依赖``boost``或``asio``。\n* 项目无其它依赖。如需使用``kafka``协议，需自行安装``lz4``，``zstd``和``snappy``几个压缩库。\n\n#### 快速开始（Linux, macOS）：\n~~~sh\ngit clone https://github.com/sogou/workflow # From gitee: git clone https://gitee.com/sogou/workflow\ncd workflow\nmake\ncd tutorial\nmake\n~~~\n#### 使用SRPC工具（NEW!）\nSRPC工具可以生成完整的workflow工程，根据用户命令生成对应的server，client或proxy框架，以及CMake工程文件和JSON格式的配置文件。 \n并且，工具会下载最小的必要的依赖。例如在用户指定产生RPC项目时，自动下载并配置好protobuf等依赖。 \nSRPC工具的使用方法可以参考：https://github.com/sogou/srpc/blob/master/tools/README_cn.md\n\n#### Debian Linux或ubuntu上使用[apt-get](https://launchpad.net/ubuntu/+source/workflow)安装：\n作为是Debian Linux与Ubuntu Linux 22.04版自带软件，可以通过``apt-get``命令直接安装开发包：\n~~~sh\nsudo apt-get install libworkflow-dev\n~~~\n或部署运行环境：\n~~~sh\nsudo apt-get install workflow1\n~~~\n注意ubuntu只有最新22.04版或以上自带workflow。更推荐用git直接下载最新源代码编译。\n#### Fedora Linux上使用[dnf](https://packages.fedoraproject.org/pkgs/workflow)安装：\nWorkflow也是Fedora Linux的自带软件，可以使用最新的rpm包管理工具``dnf``直接安装开发包：\n~~~~sh\nsudo dnf install workflow-devel\n~~~~\n或部署运行环境：\n~~~~sh\nsudo dnf install workflow\n~~~~\n#### 使用xmake\n如果你想用xmake去构建 workflow, 你可以看 [xmake build document](docs/xmake.md)\n\n# 示例教程\n * Client基础\n * [创建第一个任务：wget](docs/tutorial-01-wget.md)\n * [实现一次redis写入与读出：redis_cli](docs/tutorial-02-redis_cli.md)\n * [任务序列的更多功能：wget_to_redis](docs/tutorial-03-wget_to_redis.md)\n * Server基础\n * [第一个server：http_echo_server](docs/tutorial-04-http_echo_server.md)\n * [异步server的示例：http_proxy](docs/tutorial-05-http_proxy.md)\n * 并行任务与工作流　\n * [一个简单的并行抓取：parallel_wget](docs/tutorial-06-parallel_wget.md)\n * 几个重要的话题\n * [关于错误处理](docs/about-error.md)\n * [关于超时](docs/about-timeout.md)\n\t* [关于全局配置](docs/about-config.md)\n * [关于DNS](docs/about-dns.md)\n * [关于程序退出](docs/about-exit.md)\n * 计算任务\n * [使用内置算法工厂：sort_task](docs/tutorial-07-sort_task.md)\n * [自定义计算任务：matrix_multiply](docs/tutorial-08-matrix_multiply.md)\n * [更加简单的使用计算任务：go_task](docs/about-go-task.md)【推荐】\n * 文件异步IO任务\n * [异步IO的http server：http_file_server](docs/tutorial-09-http_file_server.md)\n * 用户定义协议基础\n * [简单的用户自定义协议client/server](docs/tutorial-10-user_defined_protocol.md)\n * [使用TLV格式消息](docs/about-tlv-message.md)\n * 其它一些重要任务与组件\n * [关于定时器](docs/about-timer.md)\n * [关于计数器](docs/about-counter.md)\n * [模块任务](docs/about-module.md)\n * [DAG图任务](docs/tutorial-11-graph_task.md)\n * [Selector任务](docs/about-selector.md)\n * 任务间通信\n * [条件任务与观察者模式](docs/about-conditional.md)\n * [资源池与消息队列](docs/about-resource-pool.md)\n * 服务治理\n * [关于服务治理](docs/about-service-governance.md)\n * [Upstream更多文档](docs/about-upstream.md)\n * [自定义名称服务策略](docs/tutorial-15-name_service.md)\n * 连接上下文的使用\n * [关于连接上下文](docs/about-connection-context.md)\n * 内置客户端\n * [异步MySQL客户端：mysql_cli](docs/tutorial-12-mysql_cli.md)\n * [异步kafka客户端：kafka_cli](docs/tutorial-13-kafka_cli.md)\n * [异步DNS客户端：dns_cli](docs/tutorial-17-dns_cli.md)\n * [Redis订阅客户端：redis_subscriber](docs/tutorial-18-redis_subscriber.md)\n\n#### 编程范式\n\n程序 = 协议 + 算法 + 任务流\n* 协议\n * 大多数情况下，用户使用的是内置的通用网络协议，例如http，redis或各种rpc。\n * 用户可以方便的自定义网络协议，只需提供序列化和反序列化函数，就可以定义出自己的client/server。\n* 算法\n * 在我们的设计里，算法是与协议对称的概念。\n * 如果说协议的调用是rpc，算法的调用就是一次apc（Async Procedure Call）。\n * 我们提供了一些通用算法，例如sort，merge，psort，reduce，可以直接使用。\n * 与自定义协议相比，自定义算法的使用要常见得多。任何一次边界清晰的复杂计算，都应该包装成算法。\n* 任务流\n * 任务流就是实际的业务逻辑，就是把开发好的协议与算法放在流程图里使用起来。\n * 典型的任务流是一个闭合的串并联图。复杂的业务逻辑，可能是一个非闭合的DAG。\n * 任务流图可以直接构建，也可以根据每一步的结果动态生成。所有任务都是异步执行的。\n\n结构化并发与任务隐藏\n* 我们系统中包含五种基础任务：通讯，计算，文件IO，定时器，计数器。\n* 一切任务都由任务工厂产生，用户通过调用接口组织并发结构。例如串联并联，DAG等。\n* 大多数情况下，用户通过任务工厂产生的任务，都隐藏了多个异步过程，但用户并不感知。\n * 例如，一次http请求，可能包含许多次异步过程（DNS，重定向），但对用户来讲，就是一次通信任务。\n * 文件排序，看起来就是一个算法，但其实包括复杂的文件IO与CPU计算的交互过程。\n * 如果把业务逻辑想象成用设计好的电子元件搭建电路，那么每个电子元件内部可能又是一个复杂电路。\n * 任务隐藏机制大幅减少了用户需要创建的任务数量和回调深度。\n* 任何任务都运行在某个串行流（series）里，共享series上下文，让异步任务之间数据传递变得简单。\n\n回调与内存回收机制\n* 一切调用都是异步执行，几乎不存在占着线程等待的操作。\n* 显式的回调机制。用户清楚自己在写异步程序。\n* **通过一套对象生命周期机制，大幅简化异步程序的内存管理**\n * 任何框架创建的任务，生命周期都是从创建到callback函数运行结束为止。没有泄漏风险。\n * 如果创建了任务之后不想运行，则需要通过dismiss()接口删除。\n * 任务中的数据，例如网络请求的resp，也会随着任务被回收。此时用户可通过``std::move()``把需要的数据移走。\n * 项目中不使用任何智能指针来管理内存。代码观感清新。\n* 尽量避免用户级别派生，以``std::function``封装用户行为，包括：\n * 任何任务的callback。\n * 任何server的process。符合``FaaS``（Function as a Service）思想。\n * 一个算法的实现，简单来讲也是一个``std::function``。\n * 如果深入使用，又会发现一切皆可派生。\n\n# 使用中有疑问？\n可以先查看[FAQ](https://github.com/sogou/workflow/issues/170)和[issues](https://github.com/sogou/workflow/issues)列表，看看是否能找到答案。 \n非常欢迎将您使用中遇到的问题发送到[issues](https://github.com/sogou/workflow/issues)，我们将第一时间进行解答。同时更多的issue对新用户也会带来帮助。 \n也可以通过QQ群：**618773193** 联系我们。\n\n\"qq_qrcode\"\n\n#### Gitee仓库\n用户可以在访问GitHub遇到困难时，使用我们的Gitee官方仓库：https://gitee.com/sogou/workflow \n**另外也麻烦在Gitee上star了项目的用户，尽量同步star一下[GitHub仓库](https://github.com/sogou/workflow)。谢谢！**\n" }, { "path": "WORKSPACE", "content": "" }, { "path": "benchmark/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\n\nset(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING \"build type\")\n\nproject(benchmark\n\t\tLANGUAGES C CXX\n)\n\nset(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR})\n\nfind_library(LIBRT rt)\nfind_package(OpenSSL REQUIRED)\nfind_package(workflow REQUIRED CONFIG HINTS ..)\ninclude_directories(${OPENSSL_INCLUDE_DIR} ${WORKFLOW_INCLUDE_DIR})\nlink_directories(${WORKFLOW_LIB_DIR})\nfind_library(WORKFLOW_LIB NAMES libworkflow.a workflow HINTS ${WORKFLOW_LIB_DIR})\n\nset(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -Wall -fPIC -pipe -std=gnu90\")\nset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wall -fPIC -pipe -std=c++11 -fno-exceptions -Wno-invalid-offsetof\")\nif (APPLE)\n\tset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations\")\nendif()\n\nset(BENCHMARK_LIST\n\tbenchmark-01-http_server\n\tbenchmark-02-http_server_long_req\n)\n\nif (APPLE)\n\tset(LIB ${WORKFLOW_LIB} pthread OpenSSL::SSL OpenSSL::Crypto)\nelse ()\n\tset(LIB ${WORKFLOW_LIB} pthread OpenSSL::SSL OpenSSL::Crypto ${LIBRT})\nendif ()\n\nforeach(src ${BENCHMARK_LIST})\n\tstring(REPLACE \"-\" \";\" arr ${src})\n\tlist(GET arr -1 bin_name)\n\tadd_executable(${bin_name} ${src}.cc)\n\ttarget_link_libraries(${bin_name} ${LIB})\nendforeach()\n" }, { "path": "benchmark/GNUmakefile", "content": "ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))\nALL_TARGETS := all clean\nMAKE_FILE := Makefile\n\nDEFAULT_BUILD_DIR := build\nBUILD_DIR := $(shell if [ -f $(MAKE_FILE) ]; then echo \".\"; else echo $(DEFAULT_BUILD_DIR); fi)\nCMAKE3 := $(shell if which cmake3>/dev/null ; then echo cmake3; else echo cmake; fi;)\n\n.PHONY: $(ALL_TARGETS)\n\nall:\n\tmkdir -p $(BUILD_DIR)\nifeq ($(DEBUG),y)\n\tcd $(BUILD_DIR) && $(CMAKE3) -D CMAKE_BUILD_TYPE=Debug $(ROOT_DIR)\nelse\n\tcd $(BUILD_DIR) && $(CMAKE3) $(ROOT_DIR)\nendif\n\t$(MAKE) -C $(BUILD_DIR) -f Makefile\n\nclean:\nifeq ($(MAKE_FILE), $(wildcard $(MAKE_FILE)))\n\t-$(MAKE) -f Makefile clean\nelse ifeq (build, $(wildcard build))\n\t-$(MAKE) -C build clean\nendif\n\trm -rf build\n\n" }, { "path": "benchmark/README.md", "content": "# 性能测试\n\nSogou C++ Workflow是一款性能优异的网络框架，本文介绍我们进行的性能测试，\n包括方案、代码、结果，以及与其他同类产品的对比。\n\n更多场景下的实验正在进行中，本文将持续更新。\n\n## HTTP Server\n\nHTTP Client/Server是Sogou C++ Workflow常见的应用场景，\n我们首先对Server端进行实验。\n\n### 环境\n\n我们部署了两台相同机器作为Server和Client，软硬件配置如下：\n\n| 软硬件 | 配置 |\n|:---:|:---|\n| CPU | 40 Cores, x86_64, Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz |\n| Memory | 192GB |\n| NIC | 25000Mbps |\n| OS | CentOS 7.8.2003 |\n| Kernel | Linux version 3.10.0-1127.el7.x86_64 |\n| GCC | 4.8.5 |\n\n两者间`ping`测得的RTT为0.1ms左右。\n\n### 对照组\n\n我们选择nginx和brpc作为对照组。\n选择前者是因为它在生产中部署十分广泛，性能不俗；\n对于后者，我们在本次实验中只关注HTTP Server方面的能力，\n其他的特性已有[单独的实验][Sogou RPC Benchmark]进行更为详尽的测试。\n\n事实上，我们也对此二者之外的其他某些框架同时进行了实验，\n但结果其性能表现相差较远，因此未在本文中体现。\n\n后续我们将选取更多合适的框架加入对比测试中。\n\n### Client工具\n\n本次实验我们使用的压测工具为[wrk][wrk]和[wrk2][wrk2]。\n前者适合测试特定并发下的QPS极限和延时，\n后者适合在特定QPS下测试延时分布。\n\n我们也尝试过使用其他测试工具，例如[ab][ab]等，但无法打出足够的压力。\n有鉴于此，我们也在着手开发基于Sogou C++ Workflow的benchmark工具。\n\n### 变量和指标\n\n一般而言，对网络框架的性能测试，切入的角度可谓纷繁多样。\n通过控制不同的变量、观测不同的指标，可以探究程序在不同场景下的适应能力。\n\n本次实验，我们选择其中最普遍常见的变量和指标：\n通过控制Client并发度和承载数据的大小，来测试QPS和延时的变化情况。\n另外，我们还测试了在掺杂慢请求的正常请求的延时分布。\n\n下面依次介绍两个测试场景。\n\n### 测试方法\n\n#### 启动http server\n1. 编译benchmark\n2. 进入到benchmark目录，执行 \n\n```\n./http_server 12 9000 11\n```\n\n说明: 启动参数分别为线程数、端口和响应的随机字符串长度。\n\n### wrk测试\n\n```\nwrk --latency -d10 -c200 --timeout 8 -t 6 http://127.0.0.1:9000\n```\n**命令行解释**\n\n-c200: 启动200个连接\n\n-t6: 开启6个线程做压力测试\n\n-d10: 压测持续10s\n\n--timeout 8: 连接超时时间8s\n\n### 不同并发度和数据长度下的QPS和延时\n\n#### 代码和配置\n\n我们搭建了一个极其简约的HTTP服务器，\n忽略掉所有的业务逻辑，\n将测试点聚焦在纯粹的网络框架性能上。\n\n代码片段如下，\n完整代码移步[这里][benchmark-01 Code]。\n\n```cpp\n// ...\n\nauto * resp = task->get_resp();\nresp->add_header_pair(\"Date\", timestamp);\nresp->add_header_pair(\"Content-Type\", \"text/plain; charset=UTF-8\");\nresp->append_output_body_nocopy(content.data(), content.size());\n\n// ...\n```\n\n可以从上述代码中看到，\n对于到来的任何HTTP请求，\n我们都会返回一段固定的内容作为Body，\n并设置必要的Header，\n包括代码中指明的`content-type`、`date`，\n以及自动填充的`connection`和`content-length`。\n\nHTTP Body的固定内容是在Server启动时随机生成的ASCII字符串，\n其长度可以通过启动参数配置。\n同时可以配置的还有使用的poller线程数和监听的端口号。\n前者我们在本次测试中固定为16，\n因此Sogou C++ Workflow将使用16个poller线程和20个handler线程（默认配置）。\n\n对于nginx和brpc，\n我们也构建了相同的返回内容，\n并为nginx配置了40个进程、\nbrpc配置了40个线程。\n\n\n#### 变量\n\n我们控制并发度在`[1, 2K]`之间翻倍增长，\n数据长度在`[16B, 64KB]`之间翻倍增长，\n两者正交。\n\n#### 指标\n\n鉴于并发度和数据长度组合之后数量较多，\n我们选择其中部分数据绘制为曲线。\n\n##### 固定数据长度下QPS与并发度关系\n\n![Concurrency and QPS][Con-QPS]\n\n上图可以看出，当数据长度保持不变，\nQPS随着并发度提高而增大，后趋于平稳。\n此过程中Sogou C++ Workflow一直有明显优势，\n高于brpc和nginx。\n特别是数据长度为64和512的两条曲线，\n并发度足够的时候，可以保持500K的QPS。\n\n注意上图中nginx-64与nginx-512的曲线重叠度很高，\n不易辨识。\n\n##### 固定并发度下QPS与数据长度关系\n\n![Body Length and QPS][Len-QPS]\n\n上图可以看出，当并发度保持不变，\n随着数据长度的增长，\nQPS保持平稳至4K时下降。\n此过程中，Sogou C++ Workflow也一直保持优势。\n\n##### 固定数据长度下延时与并发度关系\n\n![Concurrency and Latency][Con-Lat]\n\n上图可以看出，保持数据长度不变，\n延时随并发度提高而有所上升。\n此过程中，Sogou C++ Workflow略好于brpc，\n大好于nginx。\n\n##### 固定并发度下延时与数据长度关系\n\n![Body Length and Latency][Len-Lat]\n\n上图可以看出，并发度保持不变时，\n增大数据长度，造成延时上升。\n此过程中，Sogou C++ Workflow好于nginx，\n好于brpc。\n\n### 掺杂慢请求的延时分布\n\n#### 代码\n\n我们在上一个测试的基础上，简单添加了一个慢请求的逻辑，\n模拟业务场景中可能出现的特殊情况。\n\n代码片段如下，\n完整代码请移步[这里][benchmark-02 Code]。\n\n```cpp\n// ...\n\nif (std::strcmp(uri, \"/long_req/\") == 0)\n{\n auto timer_task = WFTaskFactory::create_timer_task(microseconds, nullptr);\n series_of(task)->push_back(timer_task);\n}\n// ...\n```\n\n我们在Server的process里进行判断，\n如果访问的是特定的路径，\n则添加一个`WFTimerTask`到Series的末尾，\n能够模拟一个异步耗时处理过程。\n类似地，对brpc使用`bthread_usleep()`函数进行异步睡眠。\n\n#### 配置\n\n在本次实验中，我们固定并发度为1024，数据长度为1024字节，\n分别以QPS为20K、100K和200K进行正常请求测试，\n测绘延时；\n与此同时，有另一路压力，进行慢请求，\nQPS是上述QPS的1%，\n数据不计入统计。\n慢请求的时长固定为5ms。\n\n#### 延时CDF图\n\n![Latency CDF][Lat CDF]\n\n从上图可以看出，当QPS为20K时，\nSogou C++ Workflow略次于brpc；\n当QPS为100K时，两者几乎相当；\n当QPS为200K时，Sogou C++ Workflow略好于brpc。\n总之，可以认为两者在这方面旗鼓相当。\n\n\n[Sogou RPC Benchmark]: https://github.com/holmes1412/sogou-rpc-benchmark\n[wrk]: https://github.com/wg/wrk\n[wrk2]: https://github.com/giltene/wrk2\n[ab]: https://httpd.apache.org/docs/2.4/programs/ab.html\n[benchmark-01 Code]: benchmark-01-http_server.cc\n[benchmark-02 Code]: benchmark-02-http_server_long_req.cc\n[Con-QPS]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-01.png\n[Len-QPS]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-02.png\n[Con-Lat]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-03.png\n[Len-Lat]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-04.png\n[Lat CDF]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-05.png\n" }, { "path": "benchmark/benchmark-01-http_server.cc", "content": "#include \n\n#include \n#include \n#include \n\n#include \"util/args.h\"\n#include \"util/content.h\"\n#include \"util/date.h\"\n\nstatic WFFacilities::WaitGroup wait_group{1};\n\nvoid signal_handler(int)\n{\n\twait_group.done();\n}\n\nint main(int argc, char ** argv)\n{\n\tsize_t pollers;\n\tunsigned short port;\n\tsize_t length;\n\n\tif (parse_args(argc, argv, pollers, port, length) != 3)\n\t{\n\t\treturn -1;\n\t}\n\n\tstd::signal(SIGINT, signal_handler);\n\tstd::signal(SIGTERM, signal_handler);\n\n\tWFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;\n\tsettings.poller_threads = pollers;\n\tWORKFLOW_library_init(&settings);\n\n\tconst std::string content = make_content(length);\n\tWFHttpServer server([&content](WFHttpTask * task)\n\t{\n\t\tauto * resp = task->get_resp();\n\n\t\tchar timestamp[32];\n\t\tdate(timestamp, sizeof(timestamp));\n\t\tresp->add_header_pair(\"Date\", timestamp);\n\n\t\tresp->add_header_pair(\"Content-Type\", \"text/plain; charset=UTF-8\");\n\n\t\tresp->append_output_body_nocopy(content.data(), content.size());\n\t});\n\n\tif (server.start(port) == 0)\n\t{\n\t\twait_group.wait();\n\t\tserver.stop();\n\t}\n\n\treturn 0;\n}\n\n" }, { "path": "benchmark/benchmark-02-http_server_long_req.cc", "content": "#include \n#include \n\n#include \n#include \n#include \n\n#include \"util/args.h\"\n#include \"util/content.h\"\n#include \"util/date.h\"\n\nstatic WFFacilities::WaitGroup wait_group{1};\n\nvoid signal_handler(int)\n{\n\twait_group.done();\n}\n\nint main(int argc, char ** argv)\n{\n\tsize_t pollers;\n\tunsigned short port;\n\tsize_t length;\n\tsize_t microseconds;\n\n\tif (parse_args(argc, argv, pollers, port, length, microseconds) != 4)\n\t{\n\t\treturn -1;\n\t}\n\n\tstd::signal(SIGINT, signal_handler);\n\tstd::signal(SIGTERM, signal_handler);\n\n\tWFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;\n\tsettings.poller_threads = pollers;\n\tWORKFLOW_library_init(&settings);\n\n\tconst std::string content = make_content(length);\n\tWFHttpServer server([&content, µseconds](WFHttpTask * task)\n\t{\n\t\tauto resp = task->get_resp();\n\n\t\tchar timestamp[32];\n\t\tdate(timestamp, sizeof(timestamp));\n\t\tresp->add_header_pair(\"Date\", timestamp);\n\n\t\tresp->add_header_pair(\"Content-Type\", \"text/plain; charset=UTF-8\");\n\n\t\tresp->append_output_body_nocopy(content.data(), content.size());\n\n\t\tauto req = task->get_req();\n\t\tauto uri = req->get_request_uri();\n\t\tif (std::strcmp(uri, \"/long_req/\") == 0)\n\t\t{\n\t\t\tauto timer_task = WFTaskFactory::create_timer_task(microseconds, nullptr);\n\t\t\tseries_of(task)->push_back(timer_task);\n\t\t}\n\t});\n\n\tif (server.start(port) == 0)\n\t{\n\t\twait_group.wait();\n\t\tserver.stop();\n\t}\n\n\treturn 0;\n}\n\n" }, { "path": "benchmark/util/args.h", "content": "#ifndef _BENCHMARK_ARGS_H_\n#define _BENCHMARK_ARGS_H_\n\n#include \n#include \n#include \n\nnamespace details\n{\n\tinline bool extract(const char * p, size_t & t)\n\t{\n\t\tchar * e;\n\t\tlong long ll = std::strtoll(p, &e, 0);\n\t\tif (*e || ll < 0)\n\t\t{\n\t\t\treturn false;\n\t\t}\n\t\tt = static_cast(ll);\n\t\treturn true;\n\t}\n\n\tinline bool extract(const char * p, unsigned short & t)\n\t{\n\t\tchar * e;\n\t\tlong long ll = std::strtoll(p, &e, 0);\n\t\tif (*e\n\t\t || ll < static_cast(std::numeric_limits::min())\n\t\t || ll > static_cast(std::numeric_limits::max())\n\t\t\t)\n\t\t{\n\t\t\treturn false;\n\t\t}\n\t\tt = static_cast(ll);\n\t\treturn true;\n\t}\n\n\tinline bool extract(const char * p, std::string & t)\n\t{\n\t\tt = p;\n\t\treturn true;\n\t}\n\n\tinline bool extract(const char * p, const char *& t)\n\t{\n\t\tt = p;\n\t\treturn true;\n\t}\n\n\ttemplate \n\tinline int parse_one(bool & flag, char **& p, char ** end, ARG & arg)\n\t{\n\t\tif (flag && (flag = p < end) && (flag = extract(*p, arg)))\n\t\t{\n\t\t\tp++;\n\t\t}\n\t\treturn 0;\n\t}\n\n\ttemplate \n\tinline size_t parse_all(char ** begin, char ** end, ARGS & ... args)\n\t{\n\t\tbool flag = true;\n\t\tchar ** p = begin;\n\t\tstatic_cast(std::initializer_list{parse_one(flag, p, end, args) ...});\n\t\treturn p - begin;\n\t}\n\n\ttemplate \n\tinline size_t parse_args(int & argc, char ** argv, ARGS & ... args)\n\t{\n\t\tif (argc <= 1)\n\t\t{\n\t\t\treturn 0;\n\t\t}\n\n\t\tsize_t length = argc - 1;\n\t\tchar ** begin = argv + 1;\n\t\tchar ** end = begin + length;\n\n\t\tsize_t done = parse_all(begin, end, args ...);\n\t\tstd::rotate(begin, begin + done, end);\n\t\tstd::reverse(end - done, end);\n\n\t\targc -= done;\n\t\treturn done;\n\t}\n}\n\ntemplate \ninline static size_t parse_args(int & argc, char ** argv, ARGS & ... args)\n{\n\treturn details::parse_args(argc, argv, args ...);\n}\n\n#endif //_BENCHMARK_ARGS_H_\n" }, { "path": "benchmark/util/content.h", "content": "#ifndef _BENCHMARK_CONTENT_H_\n#define _BENCHMARK_CONTENT_H_\n\n#include \n#include \n\nstatic inline std::string make_content(size_t length)\n{\n\tstd::mt19937_64 gen{42};\n\tstd::uniform_int_distribution dis{32, 126};\n\n\tstd::string s;\n\ts.reserve(length);\n\tfor (size_t i = 0; i < length; i++)\n\t{\n\t\ts.push_back(static_cast(dis(gen)));\n\t}\n\treturn s;\n}\n\n#endif //_BENCHMARK_CONTENT_H_\n" }, { "path": "benchmark/util/date.h", "content": "#ifndef _BENCHMARK_DATE_H_\n#define _BENCHMARK_DATE_H_\n\n#include \n\nstatic inline void date(char * buf, size_t n)\n{\n\tauto tt = std::time(nullptr);\n\tstd::tm cur{};\n\t// gmtime_r(&tt, &cur);\n\tlocaltime_r(&tt, &cur);\n\tstrftime(buf, n, \"%a, %d %b %Y %H:%M:%S %Z\", &cur);\n}\n\n#endif //_BENCHMARK_DATE_H_\n" }, { "path": "benchmark/xmake.lua", "content": "set_group(\"benchmark\")\nset_default(false)\n\nadd_deps(\"workflow\")\n\nif not is_plat(\"macosx\") then\n add_ldflags(\"-lrt\")\nend\n\nfunction all_benchs()\n local res = {}\n for _, x in ipairs(os.files(\"**.cc\")) do\n local item = {}\n local s = path.filename(x)\n table.insert(item, s:sub(1, #s - 3)) -- target\n table.insert(item, path.relative(x, \".\")) -- source\n table.insert(res, item)\n end\n return res\nend\n\nfor _, bench in ipairs(all_benchs()) do\ntarget(bench[1])\n set_kind(\"binary\")\n add_files(bench[2])\nend\n" }, { "path": "docs/about-conditional.md", "content": "# 条件任务与观察者模式\n\n有的时候，我们需要让任务在某个条件下才被执行。条件任务（WFConditional）就是用于解决这种问题。 \n条件任务是一种任务包装器，可以包装任何的任务并取代原任务。通过对条件任务发送信号来触发被包装任务的执行。 \n\n# 条件任务的创建\n在[WFTaskFactory.h](/src/factory/WFTaskFactory.h)里，可以看到条件任务的创建接口。\n~~~cpp\nclass WFTaskFactory\n{\npublic:\n static WFConditional *create_conditional(SubTask *task);\n static WFConditional *create_conditional(SubTask *task, void **msgbuf);\n};\n~~~\n可以看到，我们通过工厂的create_conditional接口创建条件任务。 \n其中，task为被包装的任务。msgbuf是用于接收消息的缓冲区，如果无需关注消息的具体内容，msgbuf可以缺省。 \nWFConditional的主要接口：\n~~~cpp\nclass WFConditional : public WFGenericTask\n{\npublic:\n virtual void signal(void *msg);\n ...\n};\n~~~\nWFConditional是一种任务，所以，它满足普通workflow任务的一切属性。特别的接口只有signal，用于发送信号。 \n\n# 示例\n\n以下示例，通过timer和conditional，实现一个延迟1秒执行的计算任务。\n~~~cpp\nint main()\n{\n WFGoTask *task = WFTaskFactory::create_go_task(\"test\", [](){ printf(\"Done\\n\"); });\n WFConditional *cond = WFTaskFactory::create_conditional(task);\n WFTimerTask *timer = WFTaskFactory::create_timer_task(1, 0, [cond](void *){\n cond->signal(NULL);\n });\n timer->start();\n cond->start();\n getchar();\n}\n~~~\n这个示例里，在定时器的回调里向cond发送信号，让被包装的go task可以被执行。 \n注意，无论cond->signal()与cond->start()哪一个先被调用，程序都完全正确。 \n\n# 观察者模式\n\n我们看到，如果直接对cond发送信息，需要发送者直接持有cond的指针，这在一些情况下并不是很方便。 \n于是，我们引入了观察者模式，也就是命名的条件任务。通过向某个名称发送信号，同时唤醒所有在这个名称下的条件任务。 \n命名条件任务的创建与唤醒：\n~~~cpp\nclass WFTaskFactory\n{\npublic:\n static WFConditional *create_conditional(const std::string& cond_name, SubTask *task);\n static WFConditional *create_conditional(const std::string& cond_name, SubTask *task, void **msgbuf);\n static int signal_by_name(const std::string& cond_name, void *msg);\n static int signal_by_name(const std::string& cond_name, void *msg, size_t max);\n template\n static int signal_by_name(const std::string& cond_name, T *const msg[], size_t max);\n};\n~~~\n我们看到，与普通条件任务唯一区别是，命名条件任务创建时，需要传入一个cond_name。 \n而signal_by_name()接口，默认将msg发送到所有在这个名称上等待的条件任务，将它们全部唤醒。 \n也可以通过max参数指定唤醒的最大任务数。此时，msg还可以是一个指针数组，可给不同的条件任务发送不同的消息。 \n任何一个signal_by_name的重载函数，其返回值都是表示实际唤醒的条件任务个数。 \n这就相当于实现了观察者模式。 \n\n# 示例\n还是上面的延迟计算示例，我们增加到两个计算任务并用观察者模式来实现。用\"slot1\"作为条件任务名。\n~~~cpp\nint main()\n{\n WFGoTask *task1 = WFTaskFactory::create_go_task(\"test\", [](){ printf(\"test1 done\\n\"); });\n WFGoTask *task2 = WFTaskFactory::create_go_task(\"test\", [](){ printf(\"test2 done\\n\"); });\n WFConditional *cond1 = WFTaskFactory::create_conditional(\"slot1\", task1);\n WFConditional *cond2 = WFTaskFactory::create_conditional(\"slot1\", task2);\n WFTimerTask *timer = WFTaskFactory::create_timer_task(1, 0, [](void *){\n WFTaskFactory::signal_by_name(\"slot1\", NULL);\n });\n timer->start();\n cond1->start();\n cond2->start();\n getchar();\n}\n~~~\n我们看到，在这个示例里，timer在回调中通过signal_by_name方法，同时唤醒了slot1下两个计算任务。 \n\n# 使用条件任务注意事项\n\nWorkflow里的任何任务，如果创建之后不想运行，都可以通过dismiss接口直接释放。 \n对于条件任务，如果要被dismiss（或者在某个被cancel的series里），必须保证这个条件任务没有被signal过。\n以下代码的行为无定义：\n~~~cpp\nint main()\n{\n WFEmptyTask *task = WFTaskFactory::create_empty_task();\n WFConditional *cond = WFTaskFactory::create_conditional(\"slot1\", task);\n WFTimerTask *timer = WFTaskFactory::create_timer_task(0, 0, [](void *) {\n WFTaskFactory::signal_by_name(\"slot1\");\n });\n timer->start();\n cond->dismiss(); // 取消任务\n getchar();\n}\n~~~\n显然，如果timer的callback里已经执行或正在执行了signal_by_name，cond被signal，再dismiss()是一种错误行为。 \n这种情况一般也只会出现在命名条件任务里。所以，dismiss一个命名条件任务，需要特别的小心。 \n" }, { "path": "docs/about-config.md", "content": "# 关于全局配置\n\n全局配置用于配置全局默认参数，以适应的实际业务需求，提升程序性能。\n全局配置的修改必须在使用框架任何调用之前，否则修改可能无法生效。\n另外，一些全局配置选项，可以被upstream配置覆盖。这部分请参考upstream相关文档。\n\n# 修改默认配置\n\n在[WFGlobal.h](../src/manager/WFGlobal.h)里，包含了全局配置的结构体与默认值：\n~~~cpp\nstruct WFGlobalSettings\n{\n struct EndpointParams endpoint_params;\n struct EndpointParams dns_server_params;\n unsigned int dns_ttl_default; ///< in seconds, DNS TTL when network request success\n unsigned int dns_ttl_min; ///< in seconds, DNS TTL when network request fail\n int dns_threads;\n int poller_threads;\n int handler_threads;\n int compute_threads; ///< auto-set by system CPU number if value<=0\n int fio_max_events;\n const char *resolv_conf_path;\n const char *hosts_path;\n};\n\n\nstatic constexpr struct WFGlobalSettings GLOBAL_SETTINGS_DEFAULT =\n{\n .endpoint_params = ENDPOINT_PARAMS_DEFAULT,\n .dns_server_params = ENDPOINT_PARAMS_DEFAULT,\n .dns_ttl_default = 12 * 3600,\n .dns_ttl_min = 180,\n .dns_threads = 4,\n .poller_threads = 4,\n .handler_threads = 20,\n .compute_threads = -1,\n .fio_max_events = 4096,\n .resolv_conf_path = \"/etc/resolv.conf\",\n .hosts_path = \"/etc/hosts\",\n};\n~~~\n\n其中EndpointParams结构体和默认值在[EndpointParams.h](../src/manager/EndpointParams.h)文件里：\n\n~~~cpp\n\nstruct EndpointParams\n{\n size_t max_connections;\n int connect_timeout;\n int response_timeout;\n int ssl_connect_timeout;\n bool use_tls_sni;\n};\n\nstatic constexpr struct EndpointParams ENDPOINT_PARAMS_DEFAULT =\n{\n .max_connections = 200,\n .connect_timeout = 10 * 1000,\n .response_timeout = 10 * 1000,\n .ssl_connect_timeout = 10 * 1000,\n .use_tls_sni = false,\n};\n~~~\n\n举个例子，把默认的连接超时改为5秒，dns默认ttl改为1小时，用于消息反序列化的poller线程增加到10个：\n\n~~~cpp\n#include \"workflow/WFGlobal.h\"\n\nint main()\n{\n struct WFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;\n\n settings.endpoint_params.connect_timeout = 5 * 1000;\n settings.dns_ttl_default = 3600;\n settings.poller_threads = 10;\n WORKFLOW_library_init(&settings);\n\n ...\n}\n~~~\n\n大多数参数的意义都比较清晰。注意dns ttl相关参数，单位是**秒**。endpoint相关超时参数单位是**毫秒**，并且可以用-1表示无限。 \ndns_threads表示并行访问dns的线程数。但目前我们默认使用我们自己的异步DNS解析，所以并不会创建DNS线程（Window平台除外）。 \ndns_server_params表示是我们访问DNS server的参数，包括最大并发连接，以及连接与响应超时。 \ncompute_threads表示用于计算的线程数，默认-1代表与当前节点CPU核数相同。 \nfio_max_events是异步文件IO的最大并发事件数。\nresolv_conf_path是dns配置文件的路径，unix平台下默认为\"/etc/resolv.conf\"。Windows下默认为NULL，将使用多线程dns解析。 \nhosts_path是hosts文件路径。unix平台下默认为\"/etc/hosts“。只有配置了resolv_conf_path，这个配置才起作用。 \n\n与网络性能相关的两个参数为poller_threads和handler_threads：\n* poller线程主要负责epoll（kqueue）和消息反序列化。\n* handler线程是网络任务callback和process所在线程。\n\n所有框架需要的资源，都是在第一次被使用时才申请的。例如用户没有用到dns解析，那么异步dns解析器或dns线程不会被启动。 \n" }, { "path": "docs/about-connection-context.md", "content": "# 关于连接上下文\n\n连接上下文是使用本框架编程的一个高级课题。使用上会有一些复杂性。 \n从之前的示例里可以看出，无论是client还是server任务，我们并没有手段指定使用的具体连接。 \n但是有一些业务场景，特别是server端，可能是需要维护连接状态的。也就是说我们需要把一段上下文和连接绑定。 \n我们的框架里，是提供了连接上下文机制给用户使用的。 \n\n# 连接上下文的应用场景\n\nhttp协议可以说是一种完全无连接状态的协议，http会话，是通过cookie来实现的。这种协议对于我们的框架最友好。类似的还有kafka。 \n而redis和mysql的连接则是明显带状态，redis通过SELECT命令，指定当前连接上的数据库ID。mysql则是一个彻彻底底的有状态连接。 \n使用框架的redis或非事务mysql client任务时，由于URL里已经包含了所有和连接选择有关的信息，包括：\n* 用户名密码\n* 数据库名或数据库号\n* mysql的字符集\n\n框架会根据这些信息自动登录和选择可复用的连接，用户无需关心连接上下文的问题。 \n这也是为什么，框架里redis的SELECT命令和mysql的USE命令是禁止用户使用的，切换数据库需要用新的URL创建任务。 \n事务型mysql，可以固定连接，这部分内容请参考mysql相关文档。 \n但是，如果我们实现一个redis协议的server，那我们需要知道当前连接上的状态了。 \n\n此外，我们还可以通过连接上下文件被释放的事件来感知连接被远端关闭。\n\n# 使用连接上下文的方法\n\n我们需要强调的是，一般情况下只有server任务需要使用连接上下文，并且只需要在process函数内部使用，这也是最安全最简单的用法。 \n但是，任务在callback里也可以使用或修改连接上下文，只是使用的时候需要考虑并发的问题。我们会详细地讨论相关问题。 \n任何网络任务都可以调用接口获得连接对象，进而获得或修改连接上下文。在[WFTask.h](../src/factory/WFTask.h)里，调用如下：\n~~~cpp\ntemplate\nclass WFNetworkTask : public CommRequest\n{\npublic:\n virtual WFConnection *get_connection() const = 0;\n ...\n};\n~~~\n文件[WFConneciton.h](../src/factory/WFConnection.h)里，包含了对连接对象的操作接口：\n~~~cpp\nclass WFConnection : public CommConnection\n{\npublic:\n void *get_context() const;\n void set_context(void *context, std::function deleter);\n void set_context(void *context);\n void *test_set_context(void *test_context, void *new_context,\n std::function deleter);\n void *test_set_context(void *test_context, void *new_context);\n};\n~~~\nget_connection()只可在process或callback里调用，而且如果callback里调用，需要检查返回值是否为NULL。 \n如果成功取得WFConnection对象，就可以操作连接上下文了。连接上下文是一个void *指针。 \n设置连接上下文可以同时传入deleter函数，在连接被关闭时，deleter被自动调用。 \n如果调用无deleter参数的接口，可以只设置新的上下文，保持原有的deleter不变。 \n\n# 访问连接上下文的时机和并发问题\n\nclient task被创建的时候，连接对象没有确定，因此所有client task对连接上下文的使用只有在callback里。 \nserver task可能在两个地方使用连接上下文，process和callback。 \n在callback里使用连接上下文时，需要考虑并发问题，因为同一个连接，会被多个task复用，并且同时运行到callback。 \n所以，我们推荐只process函数里访问或修改连接上下文，process过程中连接不会被复用或释放，是最简单安全的方法。 \n注意，我们指的process只包括process函数内部，在process函数结束后，callback之前，get_connection调用一律返回NULL。 \nWFConnection的test_set_context()，就是为了解决callback里使用连接上下文是的并发问题，但我们不推荐使用。 \n总之，如果你不是对系统实现非常了解，请只在server task的process函数里使用连接上下文。 \n\n# 示例：减少Http/1.1的请求header传输\n\nhttp协议可以说是一个连接无状态的协议，同一个连接上，每一次请求都必须发送完整的header。 \n假设请求里的cookie非常大，那么这显然就增加了很大的数据传输量。我们可以通过server端连接上下文来解决这个问题。 \n我们约定http request里的cookie，对本连接上所有后续请求有效，后续请求header里可以不再发送cookie。 \n以下是server端代码：\n~~~cpp\nvoid process(WFHttpTask *server_task)\n{\n protocol::HttpRequest *req = server_task->get_req();\n protocol::HttpHeaderCursor cursor(req);\n WFConnection *conn = server_task->get_connection();\n void *context = conn->get_context();\n std::string cookie;\n\n if (cursor.find(\"Cookie\", cookie))\n {\n if (context)\n delete (std::string *)context;\n context = new std::string(cookie);\n conn->set_context(context, [](void *p) { delete (std::string *)p; });\n }\n else if (context)\n cookie = *(std::string *)context;\n\n ...\n}\n~~~\n通过这种方式，与client端约定好每次只在连接的第一个请求传输cookie，就可以实现流量的节省。 \nclient端的实现需要用到一个新的回调函数，用法如下： \n~~~cpp\n\nusing namespace protocol;\n\nvoid prepare_func(WFHttpTask *task)\n{\n if (task->get_task_seq() == 0)\n task->get_req()->add_header_pair(\"Cookie\", my_cookie);\n}\n\nint some_function()\n{\n WFHttpTask *task = WFTaskFactory::create_http_task(...);\n task->set_prepare(prepare_func);\n ...\n}\n~~~\n在这个示例中，当http task是连接上的首个请求时，我们设置了cookie。如果不是首个请求，根据约定，不再设置cookie。 \n另外，prepare函数里，可以安全的使用连接上下文。同一个连接上，prepare不会并发。\n" }, { "path": "docs/about-counter.md", "content": "# 关于计数器\n\n计数器是我们框架中一种非常重要的基础任务，计数器本质上是一个不占线程的信号量。 \n计数器主要用于工作流的控制，包括匿名计数器和命名计数器两种，可以实现非常复杂的业务逻辑。 \n\n# 计数器的创建\n\n由于计数器也是一种任务，它的创建同样通过WFTaskFactory来完成，包括两种创建方法：\n~~~cpp\nusing counter_callback_t = std::function;\n\nclass WFTaskFactory\n{\n ...\n static WFCounterTask *create_counter_task(unsigned int target_value,\n counter_callback_t callback);\n \n static WFCounterTask *create_counter_task(const std::string& counter_name,\n unsigned int target_value,\n counter_callback_t callback);\n\n ...\n};\n~~~\n每个计数器都包含一个target_value，当计数器的计数到达target_value，callback被调用。 \n以上两个接口分别产生匿名计数器和命名计数器，匿名计数器直接通过WFCounterTask的count方法来增加计数： \n~~~cpp\nclass WFCounterTask\n{\npublic:\n virtual void count()\n {\n ...\n }\n ...\n}\n~~~\n如果创建计数器时，传入一个counter_name，则产生一个命名计数器，可以通过count_by_name函数来增加计数。 \n\n# 用匿名计数器实现任务并行\n\n在[并行抓取](./tutorial-06-parallel_wget.md)的示例中，我们通过创建一个ParallelWork来实现多个series并行。 \n通过ParallelWork和SeriesWork的组合，可以构建任意的串并连图，已经可以满足大多数应用场景需求。 \n而计数器的存在，可以让我们构建更复杂的任务依赖关系，比如实现一个全连接的神经网络。 \n以下简单的代码，可代替ParallelWork，实现一个并行的http抓取。 \n~~~cpp\nvoid http_callback(WFHttpTask *task)\n{\n /* Save http page. */\n ...\n\n WFCounterTask *counter = (WFCounterTask *)task->user_data;\n counter->count();\n}\n\nstd::mutex mutex;\nstd::condition_variable cond;\nbool finished = false;\n\nvoid counter_callback(WFCounterTask *counter)\n{\n mutex.lock();\n finished = true;\n cond.notify_one();\n mutex.unlock();\n}\n\nint main(int argc, char *argv[])\n{\n WFCounterTask *counter = create_counter_task(url_count, counter_callback);\n WFHttpTask *task;\n std::string url[url_count];\n\n /* init urls */\n ...\n\n for (int i = 0; i < url_count; i++)\n {\n task = create_http_task(url[i], http_callback);\n task->user_data = counter;\n task->start();\n }\n\n counter->start();\n std::unique_lock lock(mutex);\n while (!finished) \n cond.wait(lock);\n lock.unlock();\n return 0;\n}\n~~~\n以上创建一个目标值为url_count的计数器，每个http任务完成之后，调用一次count。 \n注意，匿名计数器的count次数不可以超过目标值，否则counter可能已经callback销毁了，程序行为无定义。 \ncounter->start()调用可以放在for循环之前。counter只要被创建，就可以调用其count接口，无论counter是否已经启动。 \n匿名计数器的count接口调用，也可以写成counter->WFCounterTask::count(); 在非常注重性能的应用下可以这么用。 \n\n# Server与其它异步引擎结合使用\n\n某些情况下，我们的server可能需要调用非本框架的异步客户端等待结果。简单的方法我们可以在process里同步等待，通过条件变量来唤醒。 \n这么做的缺点是我们占用了一个处理线程，把其它框架的异步客户端变为同步客户端。但通过counter，我们可以不占线程地等待。 \n方法很简单：\n~~~cpp\n\nvoid some_callback(void *context)\n{\n protocol::HttpResponse *resp = get_resp_from_context(context);\n WFCounterTask *counter = get_counter_from_context(context);\n /* write data to resp. */\n ...\n counter->count();\n}\n\nvoid process(WFHttpTask *task)\n{\n WFCounterTask *counter = WFTaskFactory::create_counter_task(1, nullptr);\n\n SomeOtherAsyncClient client(some_callback, context);\n\n *series_of(task) << counter;\n}\n~~~\n在这里，我们可以把server任务所在的series理解为一个协程，而目标值为1的counter，可以理解为一个条件变量。 \nCounter的缺点是count操作不传递数据。如果业务有数据传达的需求，可以使用[Mailbox任务](https://github.com/sogou/workflow/blob/master/src/factory/WFTaskFactory.h#L268)。 \n\n# 命名计数器\n\n对匿名计数器进行count操作时，直接访问了counter对象指针。这就必然要求在操作时，调用count的次数不超过目标值。 \n但想象这样一个应用场景，我们同时启动4个任务，只要其中有任意3个任务完成，工作流就可以继续进行。 \n我们可以用一个目标值为3的计数器，每个任务完成之后，count一次，这样只要任务3个任务完成，计数器就被callback。 \n但这样的问题是，当第4个任务完成，再调用counter->count()的时候，计数器已经是一个野指针了，程序崩溃。 \n这时候我们可以用命名计数器来解决这个问题。通过给计数器命名，并通过名字来计数，例如以下实现：\n~~~cpp\nvoid counter_callback(WFCounterTask *counter)\n{\n WFRedisTask *next = WFTaskFactory::create_redis_task(...);\n series_of(counter)->push_back(next);\n}\n\nint main(void)\n{\n WFHttpTask *tasks[4];\n WFCounterTask *counter;\n\n counter = WFTaskFactory::create_counter_task(\"c1\", 3, counter_callback);\n counter->start();\n\n for (int i = 0; i < 4; i++)\n {\n tasks[i] = WFTaskFactory::create_http_task(..., [](WFHttpTask *task){\n WFTaskFactory::count_by_name(\"c1\"); });\n tasks[i]->start();\n }\n\n ...\n\n}\n~~~\n这个示例中，调起4个并发的http任务，其中3个完成了，立刻启动一个redis任务。实际应用中，可能还需要加入数据传递的代码。 \n示例中创建命名为\"c1\"的计数器，在http回调里，使用WFTaskFactory::count_by_name()调用来进行计数。\n~~~cpp\nclass WFTaskFactory\n{\n ...\n static int count_by_name(const std::string& counter_name);\n\n static int count_by_name(const std::string& counter_name, unsigned int n);\n ...\n};\n~~~\nWFTaskFactory::count_by_name方法还可以传入一个整数n，表示这一次操作要增加的计数值，显然： \ncount_by_name(\"c1\")等价于count_by_name(\"c1\", 1)。 \n如果\"c1\"计数器不存在（未创建或已经完成），那么对\"c1\"的操作不产生任何效果，因此不会有匿名计数器野指针的问题。 \n函数的返回值表示被唤醒的计数器个数。当n大于1时，count_by_name操作可能让多个计数器达到目标值。 \n\n# 命名计数器详细行为定义\n\n调用WFTaskFactory::count_by_name(name, n)的时候：\n* 如果name不存在（未创建或已经完成），无任何行为。\n* 如果只有一个名字为name的计数器：\n * 如果该计数器剩余的值小于或等于n，计数完成，callback被调用，该计数器被销毁。结束。\n * 如果计数器剩余值大于n，则计数值加n。结束。\n* 如果存在多个同名为name的计数器：\n * 按照创建顺序，取第一个计数器，假设其剩余值为m：\n * 如果m值大于n，则计数加n。结束（剩余值为m-n）。\n * 如果m小于或等于n，计数完成，callback被调用，第一个计数器被销毁。置n=n-m。\n * 如果n为0，结束。\n * 如果n大于0，再取出下一个同名计数器，重复整个的操作。\n\n虽然描述很复杂，但总结起来就一句话，按照创建顺序，依次访问所有名字为name的计数器，直到n为0。 \n也就是说，一次count_by_name(name, n)可以唤醒多个计数器。 \n用好计数器，可以实现非常复杂的业务逻辑。计数器在我们框架里，往往用于实现异步锁，或者用于任务之间的通道。形态上更像一种控制任务。 \n" }, { "path": "docs/about-dns.md", "content": "# 关于DNS\n当使用域名请求网络时，首先需要通过域名解析获取服务器地址，再使用网络地址进行后续的请求。Workflow已经实现了完备的域名解析和缓存系统，通常来说用户无需知晓内部机制即可流畅地发起网络任务。\n\n## DNS相关配置\nWorkflow中的全局配置包括\n\n~~~cpp\nstruct WFGlobalSettings\n{\n struct EndpointParams endpoint_params;\n struct EndpointParams dns_server_params;\n unsigned int dns_ttl_default;\n unsigned int dns_ttl_min;\n int dns_threads;\n int poller_threads;\n int handler_threads;\n int compute_threads;\n int fio_max_events;\n const char *resolv_conf_path;\n const char *hosts_path;\n};\n~~~\n\n其中与域名解析相关的配置项有\n\n* dns_server_params\n * address_family: 该项会在后续展开说明\n * max_connections: 向DNS服务器发送请求的最大并发数，默认为200\n * connect_timeout/response_timeout/ssl_connect_timeout: 参考[超时](about-timeout.md)相关说明\n* dns_threads: 当使用同步方式实现域名解析时，解析操作会在独立的线程池中执行，该项指定线程池的线程数，默认为4\n* dns_ttl_default: 域名解析成功的结果会被放到域名缓存中，该项指定其存活时间，单位为秒，默认值1小时，当解析结果过期后会重新解析以获取最新内容\n* dns_ttl_min: 当通信失败时，有可能出现缓存的结果已经失效的情况，该项指定一个较短的存活时间，当通信失败时以更频繁的速率更新缓存，单位为秒，默认值1分钟\n* resolv_conf_path: 该文件保存了访问DNS相关的配置，在常见的Linux发行版上通常位于`/etc/resolv.conf`，若该项配置为`NULL`则表示使用多线程同步解析的模式\n* hosts_path: 该文件是一个本地的域名查找表，若被解析的域名命中该表则不会向DNS发起请求，在常见的Linux发行版上通常位于`/etc/hosts`，若该项配置为`NULL`则表示不使用查找表\n\n### resolv.conf扩展功能\nWorkflow对`resolv.conf`配置文件进行了扩展，用户可以通过修改配置以支持`DNS over TLS(DoT)`功能，**注意**直接修改`/etc/resolv.conf`会影响其他进程，可以将该文件复制一份用于修改，并将Workflow的`resolv_conf_path`配置修改为新文件的路径。例如使用`dnss`协议的`nameserver`会通过SSL进行连接\n\n~~~bash\nnameserver dnss://8.8.8.8/\nnameserver dnss://[2001:4860:4860::8888]/\n~~~\n\n### Address Family\n在某些网络环境下，虽然本机支持IPv6，但因未被分配公网IPv6地址而无法与外部通信（例如本地IPv6地址以`fe80`开始）。此时可以将`endpoint_params.address_family`设置为`AF_INET`来强制域名解析时仅解析IPv4地址。同样的，`resolv.conf`文件中可能同时指定了`nameserver`的IPv4地址和IPv6地址，此时可以将`dns_server_params.address_family`设置为`AF_INET`或`AF_INET6`来强制仅使用IPv4或IPv6地址来访问DNS。\n\n### 使用Upstream配置\n全局配置默认对每个域名生效，若需要对某些域名单独指定不同的配置，则可使用[Upstream](./about-upstream.md#Address属性)功能。使用Upstream可以单独指定`dns_ttl_default`、`dns_ttl_min`配置项，以及通过`endpoint_params.address_family`单独指定该域名使用的IP地址类别。\n\n\n## 域名解析与缓存策略\n网络任务通常需要通过域名解析获取到需要访问的IP地址，Workflow中域名解析相关策略如下\n\n1. 检查域名缓存是否有该域名对应的IP地址，若有缓存且未过期，则使用该组IP地址\n2. 检查域名是否为IPv4、IPv6地址或`Unix Domain Socket`，若是则直接使用该地址，无需发起域名解析\n3. 检查`hosts_path`文件中是否包含该域名对应的IP地址，若有则直接使用该地址\n4. 获取异步锁，保证同一域名的解析请求在同一时刻仅发起一次，并向DNS发起解析请求\n5. 解析成功后会将解析结果保存到当前进程的域名缓存中，以供下次使用，并释放异步锁\n6. 解析失败后会释放异步锁且将失败原因通知给等在同一个异步锁上的所有任务，通知结束后再发起的新的任务则会再次请求DNS\n\n许多需要大量发起网络请求的场景都会配备域名缓存组件，如果每次发起网络任务时都向DNS发起解析请求，则DNS必然会不堪重负。Workflow设置了缓存存活时长（dns_ttl_default和dns_ttl_min）来保证缓存会在合理的时间后过期，以及时更新域名的解析结果。当某个域名的缓存项过期后，首先发现过期的任务会将其存活时间延长5秒并向DNS发起解析请求，5秒内同一域名上的请求会直接使用缓存的DNS解析结果，而无需等待本次解析结束。\n\n异步锁机制可以保证**同一域名**的解析请求在同一时刻仅发起一次，在没有锁保护的情况下，若短时间内对同一域名发起大量网络任务，每个任务都会因无法从缓存中获取结果而向DNS发起解析请求，这会对DNS带来很大且不必要的负担。这里的同一域名表示的是`(host, port, family)`三元组，若通过Upstream的方式对某域名分别要求只使用IPv4和IPv6，则他们会被不同的异步锁保护，也就有可能同时发起DNS请求。\n\n\n### 异步域名解析\nWorkflow实现了完备的DNS任务（参考[dns_cli](./tutorial-17-dns_cli.md)），若指定了`resolv_conf_path`配置项，则向DNS发起域名解析时会使用异步请求的方式进行，在类Unix系统下，Workflow默认使用`/etc/resolv.conf`作为该配置的值。异步域名解析不会阻塞任何线程，也不会独占线程池，可以更高效地完成域名解析的任务。\n\n### 同步域名解析\n若指定`resolv_conf_path`为`NULL`，则会通过调用`getaddrinfo`函数来实现同步域名解析，该方式会使用独立的线程池，其线程数通过`dns_threads`参数配置。若短时间内需要发起较多的域名解析请求，则同步的方式会带来较大的延迟。\n" }, { "path": "docs/about-error.md", "content": "# 关于错误处理\n\n任何软件系统里，错误处理都是一个重要而复杂的问题。在我们框架内部，错误处理可以说是无处不在并且极其繁琐的。 \n而在我们暴露给用户的接口里，我们尽可能地让事情变简单，但用户还是不可避免地需要了解一些错误信息。\n\n### 禁用C++异常\n\n我们框架内不使用C++异常，用户编译自己代码的时候，最好也加上-fno-exceptions标志，以减少代码大小。 \n参考业界通用做法，我们会忽略new操作失败的可能，并且内部也避免用new去分配大块内存。而C语言风格的内存分配则是有查错的。 \n\n### 关于工厂函数\n\n从之前的实例中我们看到，所有的task，series都是从WFTaskFactory或Workflow这两个工厂类产生的。 \n这些工厂类，以及我们以后可能遇到的更多的工厂类接口，都是确保成功的。也就是说，一定不会返回NULL。用户无需对返回值做检查。 \n为了达到这个目的，当URL不合法时，工厂也能正常产生task。并且在任务的callback里再得到错误。\n\n### 任务的状态和错误码\n\n在之前的示例里，我们经常在callback里看到这样的代码：\n~~~cpp\nvoid callback(WFXxxTask *task)\n{\n int state = task->get_state();\n int error = task->get_error();\n ...\n}\n~~~\n其中，state代表任务的结束状态，在[WFTask.h](../src/factory/WFTask.h)文件中，可以看到所有可能的状态值：\n~~~cpp\nenum\n{\n WFT_STATE_UNDEFINED = -1,\n WFT_STATE_SUCCESS = CS_STATE_SUCCESS,\n WFT_STATE_TOREPLY = CS_STATE_TOREPLY, /* for server task only */\n WFT_STATE_NOREPLY = CS_STATE_TOREPLY + 1, /* for server task only */\n WFT_STATE_SYS_ERROR = CS_STATE_ERROR,\n WFT_STATE_SSL_ERROR = 65,\n WFT_STATE_DNS_ERROR = 66, /* for client task only */\n WFT_STATE_TASK_ERROR = 67,\n WFT_STATE_ABORTED = CS_STATE_STOPPED /* main process terminated */\n};\n~~~\n##### 需要关注的几个状态：\n * SUCCESS：任务成功。client接收到完整的回复，或server把回复完全写进入发送缓冲（但不能确保对方一定能收到）。\n * SYS_ERROR: 系统错误。这种情况，task->get_error()得到的是系统错误码errno。\n * 当get_error()得到ETIMEDOUT，可以调用task->get_timeout_reason()进一步得到超时原因。\n * DNS_ERROR: DNS解析错误。get_error()得到的是getaddrinfo()调用的返回码。关于DNS，有一篇文档专门说明[about-dns.md](./about-dns.md)。\n * server任务永远不会有DNS_ERROR。\n * SSL_ERROR: SSL错误。get_error()得到的是SSL_get_error()的返回值。\n * 目前SSL错误信息没有做得很全，得不到ERR_get_error()的值。所以，基本上get_error()返回值也就三个可能：\n * SSL_ERROR_ZERO_RETURN, SSL_ERROR_X509_LOOKUP, SSL_ERROR_SSL。\n * 更加详细的SSL错误信息，我们在后续版本会考虑加入。\n * TASK_ERROR: 任务错误。常见的例如URL不合法，登录失败等。get_error()的返回值可以在[WFTaskError.h](../src/factory/WFTaskError.h)中查看。\n\n##### 用户一般无需关注的几个状态：\n * UNDEFINED: 刚创建完，还没有运行的client任务，状态是UNDEFINED。\n * TOREPLY: server任务回复之前，没有被调用过task->noreply()，都是TOREPLY状态。\n * NOREPLY: server任务被调用了task->noreply()之后，一直是NOREPLY状态。callback里也是这个状态。连接会被关闭。\n\n### 其它错误处理需求\n除了任务本身的错误处理，各种具体协议的消息接口上，也会有判断错误的需要。一般这些接口都通过返回false来表示错误，并且通过errno传递错误原因。 \n此外，一些更复杂的用法，可能需要接触到更复杂一点的错误信息。我们在具体的文档里再做介绍。\n" }, { "path": "docs/about-exit.md", "content": "# 关于程序退出\n\n由于我们的大多数调用都是非阻塞的，所以在之前的示例里我们都需要用一些机制来防止main函数提前退出。 \n例如wget示例中等待用户的Ctrl-C，或者像parallel_wget在所有抓取结束之后唤醒主线程。 \n而在几个server的示例中，stop()操作是阻塞的，可以确保所有server task的正常结束，主线程可安全退出。\n\n# 程序安全退出的原则\n\n一般情况下，用户只要正常写程序，模仿示例中的方法，不太会有什么关于退出的疑惑。但这里还是需要把程序正常退出的条件定义好。 \n* 用户不可以在callback或process等任何回调函数里调用系统的exit()函数，否则行为无定义。\n* 主线程可以安全结束（main函数调用exit()或return）的条件是所有任务已经运行到callback，并且没有新的任务被调起。\n * 我们所有的示例都符合这个假设，在callback里唤醒main函数。这是安全的，不用担心main返回的时候，callback还没结束的情况。\n * ParallelWork是一种task，也需要运行到callback。\n * 这一条规则某下情况下可以违反，我们将在下一节解释。\n* 所有server必须stop完成，否则行为无定义。因为stop操作用户都会调，所以一般server程序不会有什么退出方面的问题。\n * server的stop会等待所有server任务所在series结束。但如果用户在process直接start一个新任务，则需要考虑任务结束的问题。\n\n# 为什么需要等待运行中的任务callback？能不能提前结束程序？\n\n首先解释一下需要等待任务callback再结束程序的原因。在大多数情况下，我们通过任务工厂产生的任务，都是一个复合任务。 \nhttp抓取任务为例，一个http任务可能需要先解析dns，再发起http抓取。如遇到302重定向，可能需要再次dns。任务失败可能还会重试。 \n也就是说，我们一个异步任务可能包含多个异步过程，但对用户完全无感。而内部每个异步过程之间，并不会检查程序是否已经退出。 \n如果用户明确知道一个任务是原子任务，例如以IP地址（或肯定能dns cache命中）创建http任务，并且无重定向或重试。 \n那么，这个任务可以被程序退出打断并提前来到callback，callback里任务的状态是WFT_STATE_ABORTED。 \n例如以下程序是绝对安全的：\n~~~cpp\nvoid callback(WFHttpTask *task)\n{\n // 这里打印的结果大概率是2，WFT_STATE_ABORTED。\n printf(\"state = %d\\n\", task->get_state());\n}\n\nint main()\n{\n WFHttpTask *task = WFTaskFactory::create_http_task(\"https://127.0.0.1/\", 0, 0, callback);\n task->start();\n // 这里直接结束程序\n return 1;\n}\n~~~\n如果dns cache命中，也是安全的。因为内部无需再发起一个dns异步任务了。例如：\n~~~cpp\nWFFacilities::WaitGroup wg(1);\n\nvoid callback_normal(WFHttpTask *task)\n{\n wg.done();\n}\n\nvoid callback_abort(WFHttpTask *task)\n{\n // 这里打印的结果大概率是2，WFT_STATE_ABORTED。\n printf(\"state = %d\\n\", task->get_state());\n}\n\nint main()\n{\n WFHttpTask *task = WFTaskFactory::create_http_task(\"https://www.sogou.com/\", 3, 2, callback_normal);\n task->start();\n // 等待第一个访问www.sogou.com的任务结束。\n wg.wait();\n // 第二次访问www.sogou.com, dns信息已经被cache。\n WFHttpTask *task = WFTaskFactory::create_http_task(\"https://www.sogou.com/\", 0, 0, callback_abort);\n task->start();\n // 这里直接结束程序\n return 1;\n}\n~~~\n所以，对于网络任务而言，只要能确定是一个原子任务，都可以被程序结束打断。这个原则可以扩展到任何类型的任务。 \n例如，定时器任务是一个就原子任务，以下程序也是绝对安全的：\n~~~cpp\nvoid callback(WFTimerTask *task)\n{\n // 这里打印的结果肯定是2，WFT_STATE_ABORTED。\n printf(\"state = %d\\n\", task->get_state());\n}\nint main()\n{\n WFTimerTask *task = WFTaskFactory::create_timer_task(1000000, callback);\n task->start();\n // 这里直接结束程序\n return 1;\n}\n~~~\n在[关于定时器](https://github.com/sogou/workflow/blob/master/docs/about-timer.md)的文档里，我们将会详细展开描述。 \n此外，单线程的计算任务，文件IO任务，也可以在callback之前直接结束程序。 \n其中，已经在执行计算的计算任务，程序会等待计算结束，最终以SUCCESS状态callback。还未被调起的，则以ABORTED状态退出。 \n文件IO任务，只要已经start，肯定会等待IO完成。因此直接退出程序完全安全。\n\n# 关于OpenSSL 1.1版本在退出时的内存泄露\n\n我们发现某些openssl1.1版本，存在退出时内存释放不完全的问题，通过valgrind内存检查工具可以看出内存泄露。 \n这个问题只有在用户使用了SSL，例如抓取了https网页时才会发生，而且一般情况下用户可以忽略这个泄露。\n如果一定要解决，方法如下：\n~~~cpp\n#include \n\nint main()\n{\n#if OPENSSL_VERSION_NUMBER >= 0x10100000L\n OPENSSL_init_ssl(0, NULL);\n#endif\n ...\n}\n~~~\n也就是说在使用我们的库之前，先初始化openssl。如果你有需要也可以同时配置openssl的参数。 \n注意这个函数只在openssl1.1以上版本才有提供，所以调用之前需要先判断openssl版本。 \n这个内存泄露与openssl1.1的内存释放原理有关。我们提供的这个方案可以解决这个问题（但我们还是建议用户忽略）。\n" }, { "path": "docs/about-go-task.md", "content": "# 关于go task\n\n我们提供了另一种更简单的使用计算任务的方法，模仿go语言实现的go task。 \n使用go task来实计算任务无需定义输入与输出，所有数据通过函数参数传递。\n\n# 创建go task\n~~~cpp\nclass WFTaskFactory\n{\n ...\npublic:\n template\n static WFGoTask *create_go_task(const std::string& queue_name,\n FUNC&& func, ARGS&&... args);\n};\n~~~\n函数参数的queue_name为计算队列名，其作用在之前示例文档中有过介绍。 \nfunc可以是函数指针，函数对象，仿函数，lambda函数，类的成员函数等任意可调用对象。 \nargs为func的参数列表。注意当func是一个类的非静态成员函数时，args的第一个成员必须是对象地址。\n\n# 示例\n我们想异步的运行一个加法函数：void add(int a, int b, int& res); \n并且我们还想在函数运行结束的时候打印出结果。于是可以这样实现：\n~~~cpp\n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\nvoid add(int a, int b, int& res)\n{\n res = a + b;\n}\n\nint main(void)\n{\n WFFacilities::WaitGroup wait_group(1);\n int a = 1;\n int b = 1;\n int res;\n\n WFGoTask *task = WFTaskFactory::create_go_task(\"test\", add, a, b, std::ref(res));\n task->set_callback([&](WFGoTask *task) {\n printf(\"%d + %d = %d\\n\", a, b, res);\n wait_group.done();\n });\n \n task->start();\n wait_group.wait();\n return 0;\n}\n~~~\n以上的示例异步运行一个加法，打印结果并退出程序。go task的使用与其它的任务没有多少区别，也有user_data域可以使用。 \n唯一一点不同，是go task创建时不传callback，但和其它任务一样可以set_callback。 \n如果go task函数的某个参数是引用，需要使用std::ref，否则会变成值传递，这是c++11的特征。\n\n# 把workflow当成线程池\n\n用户可以只使用go task，这样可以将workflow退化成一个线程池，而且线程数量默认等于机器cpu数。 \n但是这个线程池比一般的线程池又有更多的功能，比如每个任务有queue name，任务之间还可以组成各种串并联或更复杂的依赖关系。\n\n# 带执行时间限制的go task\n通过create_timedgo_task接口（这里无法重载create_go_task接口），可以创建带时间限制的go task：\n~~~cpp\nclass WFTaskFactory\n{\n /* Create 'Go' task with running time limit in seconds plus nanoseconds.\n * If time exceeded, state WFT_STATE_SYS_ERROR and error ETIMEDOUT will be got in callback. */\n template\n static WFGoTask *create_timedgo_task(time_t seconds, long nanoseconds,\n const std::string& queue_name,\n FUNC&& func, ARGS&&... args);\n};\n~~~\n相比创建普通的go task，create_timedgo_task函数需要多传两个参数，seconds和nanoseconds。 \n如果func的运行时间到达seconds+nanosconds时限，task直接callback，且state为WFT_STATE_SYS_ERROR，error为ETIMEDOUT。 \n注意，框架无法中断用户执行中的任务。func依然会继续执行到结束，但不会再次callback。另外，nanoseconds取值区间在\\[0,10亿）。 \n另外，当我们给go task加上了运行时间限制，callback的时机可能会先于func函数的结束，任务所在series可能也会先于func结束。 \n如果我们在func里访问series，可能就是一个错误了。例如：\n~~~cpp\nvoid f(SeriesWork *series)\n{\n series->set_context(...); // 错误。当f是一个带超时的go task，此时series可能已经失效了。\n}\n\nint http_callback(WFHttpTask *task)\n{\n SeriesWork *series = series_of(task);\n WFGoTask *go = WFTaskFactory::create_timedgo_task(1, 0, \"test\", f, series); // 1秒超时的go task\n series_of(task)->push_back(go);\n}\n~~~\n这也是为什么，我们不推荐在计算任务的执行函数里，对任务所在的series进行操作。对series的操作，应该在callback里进行，例如：\n~~~cpp\nint main()\n{\n WFGoTask *task = WFTaskFactory::create_timedgo_task(1, 0, \"test\", f);\n task->set_callback([](WFGoTask *task) {\n SeriesWork *series = series_of(task):\n void *context = series->get_context();\n if (task->get_state() == WFT_STATE_SUCCESS) // 成功执行完\n {\n ...\n }\n else // state == WFT_STATE_SYS_ERROR && error == ETIMEDOUT // 超过运行时间限制\n {\n ...\n }\n });\n}\n~~~\n但是，在计算函数里使用task，是安全的。所以，可以使用task->user_data，在计算函数和callback之间传递数据。例如：\n~~~cpp\nint main()\n{\n WFGoTask *task = WFTaskFactory::create_timedgo_task(1, 0, \"test\", [&task]() {\n task->user_data = (void *)123;\n });\n task->set_callback([](WFGoTask *task) {\n SeriesWork *series = series_of(task):\n void *context = series->get_context();\n if (task->get_state() == WFT_STATE_SUCCESS) // 成功执行完\n {\n\t\t int result = (int)task->user_data;\n }\n else // state == WFT_STATE_SYS_ERROR && error == ETIMEDOUT // 超过运行时间限制\n {\n\t\t ...\n }\n });\n task->start();\n ...\n}\n~~~~\n# 重置go task的执行函数\n在某些时候，我们想在go task的执行函数里访问task，如上面的例子，将计算结果写入task的user_data域。 \n上例中，我们使用了引用捕获。但明显引用捕获会有一些问题。比如task本身的生命周期。我们更希望在执行函数里直接捕获go task指针。 \n直接进行值捕获明显是错误的，例如：\n~~~cpp\nWFGoTask *task = WFTaskFactory::create_timedgo_task(1, 0, \"test\", [task]() {\n task->user_data = (void *)123;\n });\n~~~\n这段代码并不能在lambda函数里得到task指针，因为捕获执行时，task还没有赋值。但我们可以通过以下的代码，实现这个需求：\n~~~cpp\nWFGoTask *task = WFTaskFactory::create_timedgo_task(1, 0, \"test\", nullptr); // 执行函数可以初始化为nullptr\nWFTaskFactory::reset_go_task(task, [task]() {\n task->user_data = (void *)123;\n });\n~~~\nWFTaskFactory::reset_get_task()函数，用于重置go task的执行函数。 \n因为task已经创建完毕，这时候在lambda函数里捕获task，就是一个正确的行为了。\n\n" }, { "path": "docs/about-module.md", "content": "# 关于模块任务\n\n我们的任务流是以task为元素。但很多情况下，用户需要模块级的封装，比如几个task完成一个特定的功能。 \n用原有的方法，就不得不让最后一个task的callback来衔接下一个任务，或者填写server任务的resp。这样不太合理。 \n因此，我们引入了WFModuleTask，方便用户封装模块，降低不同功能模块之间task的耦合。\n\n# 模块任务的创建\n\n我们把模块定义成一种特殊的任务，WFModuleTask。模块的内部包括一个sub_series用于运行模块内的任务。 \n对任务来讲，它无需关心自己是否运行在模块内。因为模块内的sub_series和普通series没有任何区别。 \n在[WFTaskFactory.h](/src/factory/WFTaskFactory.h)里，包括了包括了模块任务的创建接口：\n~~~cpp\nusing module_callback_t = std::function;\n\nclass WFTaskFactory\n{\n static WFModuleTask *create_module_task(SubTask *first, module_callback_t callback);\n};\n~~~\ncreate_module_task()的第一个参数first代表模块首任务，这与创建series类似。 \nmodule callback参数要求是const指针。这主要是防止用户在callback里，继续向module中添加任务。 \n\n# WFModuleTask的主要接口\n\n因为我们把模块也定义成这一种任务，所以，可以像使用其它任务一样使用模块。但模块没有state和error域。 \n在[WFTask.h](/src/factory/WFTask.h)里，定义了WFModuleTask类。\n~~~cpp\nclass ModuleTask : public ParallelTask, protected SeriesWork // 不必关注这个派生关系\n{\npublic:\n void start() { .. }\n void dismiss() { ... }\n\npublic:\n SeriesWork *sub_series() { return this; }\n const SeriesWork *sub_series() const { return this; }\n\npublic:\n void *user_data;\n};\n~~~\nmodule特有的sub_series接口返回module内任务运行的series。module本质上是一个子任务流。 \nsub_series也是一个普通的series，用户可以调用它的set_context()，get_context()，push_back()等函数。 \n但我们不太建议给sub_series设置callback，因为没有什么必要，使用module的callback就可以了。 \n注意，在module的callback参数表，是const WFModuleTask \\*，也就只能得到一个const的sub_series。 \n因此，在模块任务的callback里，只能调用sub_series的get_context()得到series上下文。 \n\n# 示例\n\n在一个http server的处理逻辑中，我们把所有处理逻辑设计成一个模块。\n~~~cpp\nstruct ModuleCtx\n{\n std::string body;\n};\n\nvoid http_callback(WFHttpTask *http_task)\n{\n SeriesWork *series = series_of(http_task); // 这个series就是module的sub_series。\n struct ModuleCtx *ctx = (struct ModuleCtx *)series->get_context();\n const void *body;\n size_t size;\n\n if (http_task->get_resp()->get_parsed_body(&body, &size))\n {\n ctx->body.assign(body, size);\n }\n\n ParallelWork *pwork = Workflow::create_parallel_work(…)；// 做一些别的操作\n series->push_back(pwork);\n}\n\nvoid process(WFHttpTask *server_task)\n{\n WFHttpTask *http_task = WFTaskFactory::create_http_task(…, http_callback);\n WFModuleTask *module = WFTaskFactory::create_module_task(http_task, [server_task](const WFModuleTask *mod) {\n struct ModuleCxt *ctx = (struct ModuleCtx *)mod->sub_series()->get_context();\n server_task->get_resp()->append_output_body(ctx->body);\n delete ctx;\n });\n module->sub_series()->set_context(new ModuleCtx);\n series_of(server_task)->push_back(module);\n}\n~~~\n通过这个方法，module里的任务只需操作series context，最终由module的callback汇总填写resp。任务耦合性大幅降低。\n" }, { "path": "docs/about-resource-pool.md", "content": "# 资源池\n\n在我们用workflow写异步程序时经常会遇到这样一些场景：\n* 任务运行时需要先从某个池子里获得一个资源。任务运行结束，则会把资源放回池子，让下一个需要资源的任务运行。\n* 网络通信时需要对某一个或一些通信目标做总的并发度限制，但又不希望占用线程等待。\n* 我们有许多随机到达的任务，处在不同的series里。但这些任务必须**串行**的运行。\n\n所有这些需求，都可以用资源池模块来解决。我们的[WFDnsResolver](https://github.com/sogou/workflow/blob/master/src/nameservice/WFDnsResolver.cc)就是通过这个方法来实现对dns server的并发度控制的。\n\n# 资源池的接口\n在[WFResourcePool.h](https://github.com/sogou/workflow/blob/master/src/factory/WFResourcePool.h)里，定义了资源池模块的接口：\n~~~cpp\nclass WFResourcePool\n{\npublic:\n WFConditional *get(SubTask *task, void **resbuf);\n WFConditional *get(SubTask *task);\n void post(void *res);\n ...\n\nprotected:\n virtual void *pop()\n {\n return this->data.res[this->data.index++];\n }\n\n virtual void push(void *res)\n {\n this->data.res[--this->data.index] = res;\n }\n ...\n\npublic:\n WFResourcePool(void *const *res, size_t n);\n WFResourcePool(size_t n);\n ...\n};\n~~~\n#### 构造函数\n第一个构造函数接受一个资源数组，长度为n。数组每个元素为一个void \\*。内部会再分配一份相同大小的内存，把数组复制走。 \n如果你的初始资源都是nullptr，那么你可以使用第二个构造函数，只需要传n，而无需先建立一个全部为nullptr的指针数组。 \n大概看看内部实现就明白了：\n~~~cpp\nvoid WFResourcePool::create(size_t n)\n{\n this->data.res = new void *[n];\n this->data.value = n;\n ...\n}\n\nWFResourcePool::WFResourcePool(void *const *res, size_t n)\n{\n this->create(n);\n memcpy(this->data.res, res, n * sizeof (void *));\n}\n\nWFResourcePool::WFResourcePool(size_t n)\n{\n this->create(n);\n memset(this->data.res, 0, n * sizeof (void *));\n}\n~~~\n\n#### 使用接口\n用户使用get()接口，把任务打包成一个conditional。conditional是一个条件任务，条件满足时运行其包装的任务。 \nget()接口可包含第二个参数是一个void \\*\\*resbuf，用于保存所获得的资源。 \n接下来，用户只需要用这个conditional取代原来的任务使用就好了，可以start或串进任务流。 \n注意conditional是在它被执行时去尝试获得资源的，而不是在它被创建的时候。要不然的话，以下代码就会被卡死：\n~~~cpp\nWFResourcePool pool(1);\n\nint f()\n{\n WFHttpTask *t1 = WFTaskFactory::create_http_task(..., [](void *){pool.post(nullptr);});\n WFHttpTask *t2 = WFTaskFactory::create_http_task(..., [](void *){pool.post(nullptr);});\n\n WFConditional *c1 = pool.get(t1, &t1->user_data); // 用user_data来保存res是一种实用方法。\n WFConditional *c2 = pool.get(t2, &t2->user_data);\n\n c2->start();\n // wait for t2 finish here.\n ...\n c1->start();\n ...\n}\n~~~\n以上代码c1先创建，等待t2结束后才运行。这里并不会出现c2卡死，因为conditional是在执行时才获得资源的。 \n当用户对资源使用完毕（一般在任务callback里），需要通过post()接口把资源放回池子。 \npost()时的res参数，**无需**与get()得到res的一致。 \n\n#### 派生\n从上面的pop()和push()函数我们可以看到，我们对资源的使用默认是FILO，即先进后出的。 \n使用FILO的原因是，大多数场景下，刚刚被释放的资源应该优先被复用。 \n但是，用户可以通过派生的方式，非常简单的实现一个FIFO资源池。只需要重写pop()和push()两个virtual函数即可。 \n如果需要，你还可以实现可动态扩展和收缩的资源池。\n\n# 示例\n我们准备抓取一份URL列表，但要求总的并发度不超过max_p。我们当然可以用parallel来实现，但使用资源池可以更简单：\n~~~cpp\nint fetch_with_max(std::vector& url_list, size_t max_p)\n{\n WFResourcePool pool(max_p);\n\n for (std::string& url : url_list)\n {\n WFHttpTask *task = WFTaskFactory::create_http_task(url, [&pool](WFHttpTask *task) {\n pool.post(nullptr);\n });\n WFConditional *cond = pool.get(task); // 无需保存res，可以不传resbuf参数。\n cond->start();\n }\n\n // wait_here...\n}\n~~~\n\n# 消息队列\n\n消息队列是一种比资源使用方法类似的组件。它们的区别在于：\n* 资源池的总资源数量是固定的，在创建时就已经确定。而消息队列的长度则不受限制。\n* 资源池的存取方式是先进后出，刚刚释放的资源会先被复用。而消息队列则是先进先出。\n* 资源池使用方式是先获取，后归还。没有获取就直接归还资源，可能导致缓冲区溢出。消息队列没有这样的约束。\n* 实现上，资源池使用的是数组，消息队列使用链表。总体来讲，在实现和使用上，消息队列都比资源池简单一些。\n\n# 消息队列接口\n\n在[WFMessageQueue.h](https://github.com/sogou/workflow/blob/master/src/factory/WFMessageQueue.h)里，定义了消息队列模块的接口：\n~~~cpp\nclass WFMessageQueue\n{\npublic:\n WFConditional *get(SubTask *task, void **msgbuf);\n WFConditional *get(SubTask *task);\n void post(void *msg);\n ...\n\npublic:\n WFMessageQueue();\n ...\n};\n~~~\n由于了解过资源池的用法，消息队列的使用方式我们也就无需再详细展开。模式和资源池一样，都是在获得消息（或资源）时，任务被拉起。 \n消息队列的get和post接口，无需像资源池一样遵循先获取再放回的原则，任何任务都可以随时从队列中存取消息。 \n如果有需要，用户同样可以派生WFMessageQueue类，实现先进后出的消息读取模式。 \n" }, { "path": "docs/about-selector.md", "content": "# 关于Selector任务\n\n我们业务中经常有一些需求，从几个异步分支中选择第一个成功完成的结果进行处理，丢弃其它结果。 \nSelector任务就是为了上述这种多选一场景而设计的。 \n\n# Selector解决的问题\n常见的多选一场景例如： \n* 向多个下游发送网络请求，只要任意一个下游返回正确结果，工作流程就可以继续。\n* 执行一组复杂的操作，操作执行完成或整体超时，流程都会继续。\n* 并行计算中，任何一个线程计算出预期的结果即完成，例如MD5碰撞计算。\n* 网络应用中的‘backup request’，也可以用selector配合timer来实现。\n\n在selector任务被引入之前，这些场景很难被很好解决，涉及到任务生命周期以及丢弃结果的资源回收等问题。 \n\n# 创建Selector任务\nSelector也是一种任务，所以一般由WFTaskFactory里的工厂函数产生：\n~~~cpp\nusing selector_callback_t = std::function;\n\nclass WFTaskFactory\n{\npublic:\n static WFSelectorTask *create_selector_task(size_t candidates,\n selector_callback_t callback);\n};\n~~~\n其中，candidates参数代表从多少个候选路径中选择。Selector任务创建后，必须有candidates次被提交才会被销毁。 \n因此，用户可以放心的（也是必须的）向selector提交candidates次，无需要担心selector的生命周期问题。 \n\n# Selector类的接口\nWFSelectorTask类包括两个主要接口。其中，对提交者来讲，只需要关注submit函数。对于等待者，只需使用到get_message。 \n~~~cpp\nclass WFSelectorTask : public WFGenericTask\n{\npublic:\n virtual int submit(void *msg);\n\n void *get_message() const;\n};\n~~~\n当第一个非空指针的msg被提交，submit函数返回1表示接受。随后的submit调用都返回0代表消息被拒绝。 \nSelector运行后接收到一个有效消息就进入callback了，但在收到所有submit之前，不会被销毁。 \n注意空指针永远不会被接受，所以submit一个NULL将返回0。一般来讲，submit(NULL)用于表示这个分支失败了。 \n如果所有候选都提交了NULL，selector运行到callback时，state=WFT_STATE_SYS_ERROR, error=ENOMSG。 \n作为等待者，在selector的callback里调用另外一个接口get_message()就可以得到被成功接受的消息了。 \n\n# 示例\n我们同时抓取两个http网页，并设置一个超时。当任意一个先抓取成功或超时，打印出抓取成功的URL或出错信息。 \n示例中使用wait group来保证两个抓取任务已经结束才退出程序。而timer可以被程序退出打断，无需等待。 \n~~~cpp\n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\nWFSelectorTask *selector;\nWFFacilities::WaitGroup wait_group(2);\n\nvoid http_callback(WFHttpTask *t)\n{\n if (t->get_state() == WFT_STATE_SUCCESS)\n selector->submit(t->user_data);\n else\n selector->submit(NULL);\n\n wait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n if (argc != 4)\n {\n fprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n exit(1); \n }\n\n selector = WFTaskFactory::create_selector_task(3, [](WFSelectorTask *selector) {\n void *msg = selector->get_message();\n if (msg)\n printf(\"%s\\n\", (char *)msg);\n else\n printf(\"failed\\n\");\n });\n\n auto *t = WFTaskFactory::create_http_task(argv[1], 0, 0, http_callback);\n t->user_data = argv[1];\n t->start();\n\n t = WFTaskFactory::create_http_task(argv[2], 0, 0, http_callback);\n t->user_data = argv[2];\n t->start();\n\n auto *timer = WFTaskFactory::create_timer_task(atoi(argv[3]), 0, [](WFTimerTask *timer){\n if (timer->get_state() == WFT_STATE_SUCCESS)\n selector->submit((void *)\"timeout\");\n else\n selector->submit(NULL);\n });\n timer->start();\n\n selector->start();\n\n wait_group.wait();\n return 0;\n}\n~~~\n\n\n\n" }, { "path": "docs/about-service-governance.md", "content": "# 关于服务治理\n\n我们拥有一套完整的机制，来管理我们所依赖的服务。这套机制包括以下的几个功能：\n* 用户级DNS。\n* 服务地址的选取\n * 包括多种选取机制，如权重随机，一致性哈希，用户指定选取方式等。\n* 服务的熔断与恢复。\n* 负载均衡。\n* 单个服务的独立参数配置。\n* 服务的主备关系等。\n\n所有这些功能都依赖于我们的upstream子系统。利用好这个系统，我们可以轻易地实现更复杂的服务网格功能。\n\n# upstream名\n\nupstream名相当于程序内部的域名，但相比一般的域名，upstream拥有更多的功能，包括：\n* 域名通常只能指向一组ip地址，upstream名可以指向一组ip地址或域名。\n* upstream指向的对象（域名或ip），可以包括端口信息。\n* upstream有管理和选择目标的强大功能，每个目标可以包含大量属性。\n* upstream的更新，是实时而且完全线程安全的，而域名的DNS信息，并不能实时更新。\n\n实现上，如果无需访问外网，用upstream可以完全代替域名和DNS。\n\n# upstream的创建与删除\n\n在[UpstreamManager.h](../src/manager/UpstreamManager.h)里，包括几个upstream创建接口：\n~~~cpp\nusing upstream_route_t = std::function;\n\nclass UpstreamManager\n{\npublic:\n static int upstream_create_consistent_hash(const std::string& name,\n upstream_route_t consitent_hash);\n\n static int upstream_create_weighted_random(const std::string& name,\n bool try_another);\n\n static int upstream_create_manual(const std::string& name,\n upstream_route_t select,\n bool try_another,\n upstream_route_t consitent_hash);\n\n static int upstream_delete(const std::string& name);\n ...\n};\n~~~\n三个函数创建分别为3种类型的upstream：一致性hash，权重随机和用户手动选取。 \n参数name为upstream名，创建之后，就和域名一样的使用了。 \nconsistent_hash和select参数，都是一个类型为upstream_route_t的std::function，用于指定路由方式。 \n而try_another表示，如果选取到的目标不可用（熔断），是否继续尝试找到一个可用目标。consistent_hash模式没有这个属性。 \nupstream_route_t参数接收的3个参数分别是url里的path, query和fragment部分。例如URL为：http://abc.com/home/index.html?a=1#bottom \n则这三个参数分别为\"/home/index.html\", \"a=1\"和\"bottom\"。用户可以根据这三个部分，选择目标服务器，或者进行一致性hash。 \n注意，以上接口中，consistent_hash参数都可以传nullptr，我们将使用默认的一致性哈希算法。 \n\n# 示例1：权重分配\n\n我们想把50%访问www.sogou.com的请求，打到127.0.0.1:8000和127.0.0.1:8080两个地址，并且让他们的负载为1:4。 \n我们无需要关心域名www.sogou.com之下，有多少个ip地址。总之实际域名会接收50%的请求。\n~~~cpp\n#include \"workflow/UpstreamManager.h\"\n#include \"workflow/WFTaskFactory.h\"\n\nint main()\n{\n UpstreamManager::upstream_create_weighted_random(\"www.sogou.com\", false);\n struct AddressParams params = ADDRESS_PARAMS_DEFAULT;\n\n params.weight = 5;\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"www.sogou.com\", ¶ms);\n params.weight = 1;\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"127.0.0.1:8000\", ¶ms);\n params.weight = 4;\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"127.0.0.1:8080\", ¶ms);\n\n WFHttpTask *task = WFTaskFactory::create_http_task(\"http://www.sogou.com/index.html\", ...);\n ...\n}\n~~~\n请注意，以上这些函数可以在任何场景下调用，完全线程安全，并实时生效。 \n另外，由于我们一切协议，包括用户自定义协议都有URL，所以upstream功能可作用于一切协议。\n\n# 示例2：手动选择\n\n同样是上面的例子，我们想让url里，query为\"123\"的请求，打到127.0.0.1:8000，如果是\"abc\"，打到8080端口，其它打正常域名。 \n~~~cpp\n#include \"workflow/UpstreamManager.h\"\n#include \"workflow/WFTaskFactory.h\"\n\nint my_select(const char *path, const char *query, const char *fragment)\n{\n if (strcmp(query, \"123\") == 0)\n return 1;\n else if (strcmp(query, \"abc\") == 0)\n return 2;\n else\n return 0;\n}\n\nint main()\n{\n UpstreamManager::upstream_create_manual(\"www.sogou.com\", my_select, false, nullptr);\n\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"www.sogou.com\");\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"127.0.0.1:8000\");\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"127.0.0.1:8080\");\n\n /* This URL will route to 127.0.0.1:8080 */\n WFHttpTask *task = WFTaskFactory::create_http_task(\"http://www.sogou.com/index.html?abc\", ...);\n ...\n}\n~~~\n由于我们原生提供了redis和mysql协议，用这个方法，可以极其方便的实现数据库的读写分离功能（注：非事务的操作）。 \n以上两个例子，upstream名用的是www.sogou.com，这本身也是一个域名。当然用户可以更简单的用字符串sogou，这样创建任务时：\n~~~cpp\n WFHttpTask *task = WFTaskFactory::create_http_task(\"http://sogou/home/1.html?abc\", ...);\n~~~\n总之url的host部分，如果是一个已经创建的upstream，则会被当作upstream使用。 \n\n# 示例3：一致性hash\n\n这个场景里，我们要从10个redis实例中，随机选择一台机器通信。但保证同一个url肯定访问一个确定的目标。方法很简单：\n~~~cpp\nint main()\n{\n UpstreamManager::upstream_create_consistent_hash(\"redis.name\", nullptr);\n\n UpstreamManager::upstream_add_server(\"redis.name\", \"10.135.35.53\");\n UpstreamManager::upstream_add_server(\"redis.name\", \"10.135.35.54\");\n UpstreamManager::upstream_add_server(\"redis.name\", \"10.135.35.55\");\n ...\n UpstreamManager::upstream_add_server(\"redis.name\", \"10.135.35.62\");\n\n auto *task = WFTaskFactory::create_redis_task(\"redis://:mypassword@redis.name/2?a=hello#111\", ...);\n ...\n}\n~~~\n我们的redis任务并不识别query部分，用户可以随意填写。path部分的2为redis库号。 \n这个时候，consistent_hash函数将得到\"/2\"，\"a=hello\"和\"111\"三个参数，但因为我们用nullptr，默认一致性hash将被调用。 \nupstream里的服务器没有指定端口号，于是将使用url里的端口。redis默认为6379。 \nconsitent_hash并没有try_another选项，如果目标熔断，将自动选取另一个。相同url还将得到相同选择（cache友好）。 \n\n# upstream server的参数\n\n示例１中，我们通过params参数设置了server的权重。当然server参数远不止权重一项。这个结构定义如下：\n~~~cpp\n// In EndpointParams.h\nstruct EndpointParams\n{\n size_t max_connections;\n int connect_timeout;\n int response_timeout;\n int ssl_connect_timeout;\n bool use_tls_sni;\n};\n\n// In ServiceGovernance.h\nstruct AddressParams\n{\n struct EndpointParams endpoint_params; ///< Connection config\n unsigned int dns_ttl_default; ///< in seconds, DNS TTL when network request success\n unsigned int dns_ttl_min; ///< in seconds, DNS TTL when network request fail\n/**\n * - The max_fails directive sets the number of consecutive unsuccessful attempts to communicate with the server.\n * - After 30s following the server failure, upstream probe the server with some live client’s requests.\n * - If the probes have been successful, the server is marked as a live one.\n * - If max_fails is set to 1, it means server would out of upstream selection in 30 seconds when failed only once\n */\n unsigned int max_fails; ///< [1, INT32_MAX] max_fails = 0 means max_fails = 1\n unsigned short weight; ///< [1, 65535] weight = 0 means weight = 1. only for main server\n int server_type; ///< 0 for main and 1 for backup\n int group_id; ///< -1 means no group. Backup without group will backup for any main node\n};\n~~~\n大多数参数的作用一眼了然。其中endpoint_params和dns相关参数，可以覆盖全局的配置。 \n例如，全局对每个目标ip最大连接数为200，但我想为10.135.35.53设置最多1000连接数，可以这么做：\n~~~cpp\n UpstreamManager::upstream_create_weighted_random(\"10.135.35.53\", false);\n struct AddressParams params = ADDRESS_PARAMS_DEFAULT;\n params.endpoint_params.max_connections = 1000;\n UpstreamManager::upstream_add_server(\"10.135.35.53\", \"10.135.35.53\", ¶ms);\n~~~\nmax_fails参数为最大出错次数，如果选取目标连续出错达到max_fails则熔断，如果upstream的try_another属性为false，则任务失败， \n在任务callback里，get_state()=WFT_STATE_TASK_ERROR，get_error()=WFT_ERR_UPSTREAM_UNAVAILABLE。 \n如果try_another为true，并且所有server都熔断的话，会得到同样错误。熔断时间为30秒。 \nserver_type和group_id用于主备功能。所有upstream必需有type为0(主节点)的server，否则upstream不可用。 \n类型为1（备份节点）的server，会在同group_id的主节点熔断情况下被使用。 \n\n更多upstream功能查询：[about-upstream.md](./about-upstream.md)。\n" }, { "path": "docs/about-timeout.md", "content": "# 关于超时\n\n为了让所有通信任务可以在用户的预期下精确运行，我们提供了大量的超时配置功能，并且确保这些超时的准确性。 \n这些超时配置里，有些是全局的，比如连接超时，但你又可以通过upstream功能，给某个域名配置自己的连接超时。 \n有一些超时是任务级的，比如完整发送一条消息的超时。因为用户需要根据消息大小，动态配置这个值。 \n当然对server来讲，又有自己的超时整体配置。总之，超时是一件很复杂的事，我们会做得很精确。 \n所有超时都采用poll风格，也就是int型，毫秒级，-1表示无限。 \n另外，正如我们在项目介绍里说的，所有的配置你都可以忽略，可以等遇到实际需求了再进行调整。\n\n### 基础通信超时配置\n\n在[EndpointParams.h](../src/manager/EndpointParams.h)文件里，可以看到：\n~~~cpp\nstruct EndpointParams\n{\n size_t max_connections;\n int connect_timeout;\n int response_timeout;\n int ssl_connect_timeout;\n};\n\nstatic constexpr struct EndpointParams ENDPOINT_PARAMS_DEFAULT =\n{\n .max_connections = 200,\n .connect_timeout = 10 * 1000,\n .response_timeout = 10 * 1000,\n .ssl_connect_timeout = 10 * 1000,\n};\n~~~\n其中，与超时相关的配置包括以下3项。\n * connect_timeout: 与目标建立连接的超时。默认为10秒。\n * response_timeout: 等待目标响应的超时，默认为10秒。代表成功发送到目标、或从目标读取到一块数据的超时。\n * ssl_connect_timeout: 与目标完成SSL握手的超时。默认为10秒。\n\n这个结构体是通信连接的最基础的配置，后续几乎所有的通信配置都会含有这个结构体。\n\n### 全局超时配置\n\n在[WFGlobal.h](../src/manager/WFGlobal.h)文件里，可以看到我们一个全局配置信息：\n~~~cpp\nstruct WFGlobalSettings\n{\n EndpointParams endpoint_params;\n unsigned int dns_ttl_default;\n unsigned int dns_ttl_min;\n int dns_threads;\n int poller_threads;\n int handler_threads;\n int compute_threads;\n};\n\nstatic constexpr struct WFGlobalSettings GLOBAL_SETTINGS_DEFAULT =\n{\n .endpoint_params = ENDPOINT_PARAMS_DEFAULT,\n .dns_ttl_default = 12 * 3600, /* in seconds */\n .dns_ttl_min = 180, /* reacquire when communication error */\n .dns_threads = 8,\n .poller_threads = 2,\n .handler_threads = 20,\n .compute_threads = -1\n};\n//compute_threads<=0 means auto-set by system cpu number\n~~~\n其中，与超时相关的配置就是EndpointParams endpoint_params这一项\n\n修改全局配置的方法是，调用我们任何工厂函数之前，执行类似下面的操作：\n~~~cpp\nint main()\n{\n struct WFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;\n settings.endpoint_params.connect_timeout = 2 * 1000;\n settings.endpoint_params.response_timeout = -1;\n WORKFLOW_library_init(&settings);\n}\n~~~\n上例把连接超时修改为2秒，远程服务器响应超时为无限。这种配置下，每次任务里都必须配置接收完整消息的超时，否则可能陷入无限的等待。 \n全局的超时配置，可以通过upstream功能，被单独的地址配置覆盖，比如你可以指定某个域名的连接超时。 \nUpstream每一个AddressParams也有一个EndpointParams endpoint_params项，使用方式与Global相仿。 \n具体结构详见[upstream文档](tutorial-10-upstream.md#Address属性)\n\n### Server超时配置\n\n在[http_proxy](./tutorial-05-http_proxy.md)示例的里，我们介绍过server启动配置。其中超时相关的配置包括：\n * peer_response_timeout: 这个的定义和全局的response_timeout一样，指的是远程client的响应超时，默认为10秒。\n * receive_timeout: 接收一条完整请求的超时，默认为-1。\n * keep_alive_timeout: 连接保持时间。默认1分钟。redis server为5分钟。\n * ssl_accept_timeout: 完成ssl握手的超时，默认为10秒。\n\n在这个默认配置下，client可以每9秒发送一个字节，让server一直接收而不引起超时。所以，如果服务用于公网，需要配置receive_timeout。 \n\n### 任务级别的超时配置\n\n任务级别的超时配置通过网络任务的几个接口调用来完成：\n~~~cpp\ntemplate \nclass WFNetworkTask : public CommRequest\n{\n...\npublic:\n /* All in milliseconds. timeout == -1 for unlimited. */\n void set_send_timeout(int timeout) { this->send_timeo = timeout; }\n void set_receive_timeout(int timeout) { this->receive_timeo = timeout; }\n void set_keep_alive(int timeout) { this->keep_alive_timeo = timeout; }\n void set_watch_timeout(int timeout) { this->watch_timeo = timeout; }\n...\n}\n~~~\n其中，set_send_timeout()设置发送完整消息的超时，默认值为-1。 \nset_receive_timeout()只对client任务有效，指接收完整server回复的超时，默认值为-1。 \n * server任务的receive_timeout在server启动配置里。对server任务设置receive_timeout没有意义，因为消息已经接收完成。\n\nset_keep_alive()接口设置连接保持超时。一般来讲，框架能很好的处理连接保持的问题，用户不需要调用。 \n如果是http协议，client或server想要使用短连接，可通过添加HTTP header来完成，尽量不要用这个接口去修改。 \n如果一个redis client想要在请求之后关闭连接，则需要用这个接口。显然，在callback里set_keep_alive()是无效的（连接已经被复用）。 \n\nset_watch_timeout()接口为client任务专有，代表一个client任务的请求发出之后，接收到第一个返回包的最大等待时间。 \n利用watch timeout，可以避免一些需要等待数据推送的client任务受到response timeout和receive timeout的约束而超时。 \n设置了watch timeout之后，从接收到第一个数据包再开始计算receive timeout。\n\n### 任务的同步等待超时\n\n有一个非常特殊的超时配置，是全局唯一一个同步等待超时。我们并不鼓励使用，但在某些应用场景下能得到很好的效果。 \n目前框架里，目标服务器是有连接上限的（全局和upstream都可以配置）。如果连接已经达到上限，默认的情况下，client任务失败返回。 \ncallback里task->get_state()得到WFT_STATE_SYS_ERROR, task->get_error()得到EAGAIN。如果任务配置了retry，会自动发起重试。 \n在这里，我们允许通过task->set_wait_timeout()接口，配置一个同步等待超时，如果在这段时间内，有连接被释放，则任务可以占用这个连接。 \n如果用户配置了wait_timeout，并且在超时之前没有拿到连接，则callback得到WFT_STATE_SYS_ERROR状态和ETIMEDOUT错误。\n~~~cpp\nclass CommRequest : public SubTask, public CommSession\n{\npublic:\n ...\n void set_wait_timeout(int wait_timeout) { this->wait_timeout = wait_timeout; }\n}\n~~~\n\n### 超时的原因查看\n\n通信task包含一个get_timeout_reason()接口，用于返回超时原因，但不是很细致，包括以下几个返回值：\n * TOR_NOT_TIMEOUT: 不是超时。\n * TOR_WAIT_TIMEOUT: 同步等待超时。\n * TOR_CONNECT_TIMEOUT: 连接超时。包括TCP，SCTP等协议的连接和SSL连接超时，都是这个值。\n * TOR_TRANSMIT_TIMEOUT: 一切传输超时。不能进一步区分是发送阶段还是接收阶段。以后可能会细化。\n * server任务，超时原因一定是TRANSMIT_TIMEOUT，并且一定是发送回复的阶段。\n\n### 超时功能的实现\n\n框架内部，需要处理的超时种类比我们在这里展现的还要更多。除了wait_timeout，全都是依赖于Linux的timerfd或kqueue的timer事件。 \n每个poller线程包含一个timerfd，默认配置下，poller线程数为4，可以满足大多数应用的需要了。 \n目前的超时算法利用了链表+红黑树的数据结构，时间复杂度在O(1)和O(logn)之间，其中n为poller线程的fd数量。 \n超时处理目前看不是瓶颈所在，因为Linux内核epoll相关调用也是O(logn)时间复杂度，我们把超时都做到O(1)也区别不大。\n" }, { "path": "docs/about-timer.md", "content": "# 关于定时器\n\n定时器的作用是不占线程的等待一个确定时间，同样通过callback来通知定时器到期。\n\n# 定时器的创建\n\nWFTaskFactory类里包括四个定时相关的接口：\n~~~cpp\nusing timer_callback_t = std::function;\n\nclass WFTaskFactory\n{\n ...\npublic:\n static WFTimerTask *create_timer_task(time_t seconds, long nanoseconds,\n timer_callback_t callback);\n\n static WFTimerTask *create_timer_task(const std::string& timer_name,\n time_t seconds, long nanoseconds,\n timer_callback_t callback);\n\n static int cancel_by_name(const std::string& timer_name)\n {\n cancel_by_name(const std::string& timer_name, (size_t)-1);\n }\n\n static int cancel_by_name(const std::string& timer_name, size_t max);\n};\n~~~\n我们通过seconds和nanoseconds两个参数来指定一个定时器的定时时间。其中，seconds指定秒数而nanoseconds为纳秒数。 \n* seconds参数可以传递-1，产生一个无限时长的定时器，一般用于命名定时器，为了将来调用cancel取消定时。 \n* nanoseconds的取值范围在[0,1000000000)，否则timer运行之后会立刻错误返回，错误码为EINVAL。\n\n在创建定时器任务时，可以传入一个timer_name作为定时器名，用于cancel_by_name接口取消定时。 \n定时器也是一种任务，因此使用方式与其它类型任务无异，同样有user_data域可以利用。 \n\n# 取消定时\n\n如果在创建定时器任务时传入一个名称，那么这个定时器就可以在被提前中断。 \n中断一个定时任务的方法是通过WFTaskFactory::cancel_by_name这个接口，这个接口默认情况下，会取消这个名称下的所有定时器。 \n因此，我们也支持传入一个max参数，让操作最多取消max个定时器。无论哪个接口，返回值都是代表实际被取消的定时器个数。 \n如果没有这个名称下的定时器，cancel操作不会产生任何效果，并返回0。 \n定时器在被创建之后就可取消，并非一定要等它被启动之后。以这个代码为例：\n~~~cpp\n#include \n#include \"workflow/WFTaskFactory.h\"\n\nint main()\n{\n WFTimerTask *timer = WFTaskFactory::create_timer_task(\"test\", 10000, 0, [](WFTimerTask *){\n printf(\"timer callback, state = %d, error = %d.\\n\", task->get_state(), task->get_error());\n });\n\n WFTaskFactory::cancel_by_name(\"test\");\n\n timer->start();\n\n getchar();\n return 0;\n}\n~~~\n程序会在立即打印出'timer callback, state = 1, error = 125.\"，因为定时器在运行之前就已经被取消了。所以，定时任务启动后立即callback，状态码为WFT_STATE_SYS_ERROR，错误码为ECANCELED。 \n使用中需要注意的是，命名定时器比匿名定时器是会多出一些开销的，原因是我们需要维护查找表，会有加锁解锁等操作。如果你的定时器没有提前中断的需要，就不要在创建时传入timer_name了。 \n\n# 程序退出打断定时器\n\n在[关于程序退出](./about-exit.md)里讲到，main函数结束或exit()被调用的时候，所有任务必须里运行到callback，并且没有新的任务被调起。 \n这时就可能出现一个问题，定时器的定时周期可以非常长，如果是不可中断的定时器，那么等待定时器到期，程序退出需要很长时间。 \n而实现上，程序退出是可以打断定时器，让定时器回到callback的。如果定时器被程序退出打断，get_state()会得到一个WFT_STATE_ABORTED状态。 \n当然如果定时器被程序退出打断，则不能再调起新的任务。 \n以下这个程序，每间隔一秒抓取一个一个http页面。当所有url抓完毕，程序直接退出，不用等待timer回到callback，退出不会有延迟。 \n~~~cpp\nbool program_terminate = false;\n\nvoid timer_callback(WFTimerTask *timer)\n{\n mutex.lock();\n if (!program_terminate)\n {\n WFHttpTask *task;\n if (urls_to_fetch > 0)\n {\n task = WFTaskFactory::create_http_task(...);\n series_of(timer)->push_back(task);\n }\n\n series_of(timer)->push_back(WFTaskFactory::create_timer_task(1, 0, timer_callback));\n }\n mutex.unlock();\n}\n\n...\nint main()\n{\n ....\n /* all urls done */\n mutex.lock();\n program_terminate = true;\n mutex.unlock();\n return 0;\n}\n~~~\n以上程序，timer_callback必须在锁里判断program_terminate条件，否则可能在程序已经结束的情况下又调起新任务。\n" }, { "path": "docs/about-tlv-message.md", "content": "# 关于TLV(Type-Length-Value)格式的消息\nTLV消息是一种由类型，长度，内容组成的消息。由于其结构简单通用，而且方便嵌套和扩展，特别适用于定义通信消息。 \n为方便用户实现自定义协议，我们内置了TLV消息的支持。 \n\n# TLV消息的结构\nTLV消息并没有具体规定Type和Length这两个字段占的字节数据。在我们的协议里，它们分别占4字节（网络序）。 \n也就是说，我们的消息有8字节的消息头，以及不超过32GB的Value内容。Type和Value域的含义我们不做规定。 \n\n# TLVMessage类\n由于TLV的定义内容很少，所以[TLVMessage](/src/protocol/TLVMessage.h)需要用到的接口很少。\n~~~cpp\nnamespace protocol\n{\nclass TLVMessage : public ProtocolMessage\n{\npublic:\n int get_type() const { return this->type; }\n void set_type(int type) { this->type = type; }\n\n std::string *get_value() { return &this->value; }\n void set_value(std::string value) { this->value = std::move(value); }\n\nprotected:\n int type;\n std::string value;\n\n ...\n};\n\nusing TLVRequest = TLVMessage;\nusing TLVResposne = TLVMessage;\n}\n~~~\n用户直接使用TLV消息来做数据传输的话，只需要用到上面的几个接口。分别为设置和获取Type与Value。 \nValue直接以std::string返回，方便用户必要的时候直接通过std::move移动数据。 \n\n# 基于TLV消息的echo server/client\n以下代码，直接启动一个基于TLV消息的server，并通过命令行产生client task进行交互。建议运行一下：\n~~~cpp\n#include \n#include \n#include \n#include \"workflow/WFGlobal.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"workflow/TLVMessage.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFServer.h\"\n\nusing namespace protocol;\n\nusing WFTLVServer = WFServer;\nusing WFTLVTask = WFNetworkTask;\nusing tlv_callback_t = std::function;\n\nWFTLVTask *create_tlv_task(const char *host, unsigned short port, tlv_callback_t callback)\n{\n auto *task = WFNetworkTaskFactory::create_client_task(\n TT_TCP, host, port, 0, std::move(callback));\n task->set_keep_alive(60 * 1000);\n return task;\n}\n\nint main()\n{\n WFTLVServer server([](WFTLVTask *task) {\n *task->get_resp() = std::move(*task->get_req());\n });\n\n if (server.start(8888) != 0) {\n perror(\"server.start\");\n exit(1);\n }\n\n auto&& create = [](WFRepeaterTask *)->SubTask * {\n std::string string;\n printf(\"Input string (Ctrl-D to exit): \");\n std::cin >> string;\n if (string.empty())\n return NULL;\n\n auto *task = create_tlv_task(\"127.0.0.1\", 8888, [](WFTLVTask *task) {\n if (task->get_state() == WFT_STATE_SUCCESS)\n printf(\"Server Response: %s\\n\", task->get_resp()->get_value()->c_str());\n else {\n const char *str = WFGlobal::get_error_string(task->get_state(), task->get_error());\n fprintf(stderr, \"Error: %s\\n\", str);\n }\n });\n\n task->get_req()->set_value(std::move(string));\n return task;\n };\n\n WFFacilities::WaitGroup wait_group(1);\n WFRepeaterTask *repeater = WFTaskFactory::create_repeater_task(std::move(create), nullptr);\n Workflow::start_series_work(repeater, [&wait_group](const SeriesWork *) {\n wait_group.done();\n });\n\n wait_group.wait();\n server.stop();\n return 0;\n}\n\n~~~\n\n# 派生TLVMessage\n上面的echo server实例，我们直接使用了原始的TLVMessage。但建议在具体的应用中，用户可以对消息进行派生。 \n在派生类里，提供更加丰富的接口来设置和提取消息内容，避免直接操作原始Value域，并形成自己的二级协议。 \n例如，我们实现一个JSON的协议，可以：\n~~~cpp\n#include \"workflow/json-parser.h\" // 内置的json解析器\n\nclass JsonMessage : public TLVMessage\n{\npublic:\n void set_json_value(const json_value_t *val)\n {\n this->type = JSON_TYPE;\n this->json_to_string(val, &this->value); // 需要实现一下\n }\n\n json_value_t *get_json_value() const\n {\n if (this->type == JSON_TYPE)\n return json_parser_parse(this->value.c_str()); // json-parser的函数\n else\n return NULL;\n }\n};\n\nusing JsonRequest = JsonMessage;\nusing JsonResponse = JsonMessage;\n\nusing JsonServer = WFServer;\n~~~\n这个例子只是为了说明派生的重要性，实际应用中，派生类可能要远远比这个复杂。 \n" }, { "path": "docs/about-upstream.md", "content": "# 关于Upstream\n\n在nginx里，Upstream代表了反向代理的负载均衡配置。在这里，我们扩充Upstream的含义，让其具备以下几个特点：\n1. 每一个Upstream都是一个独立的反向代理\n2. 访问一个Upstream等价于，在一组服务/目标/上下游，使用合适的策略选择其中一个进行访问\n3. Upstream具备负载均衡、出错处理、熔断和其他服务治理能力\n4. 对于同一个请求的多次重试，Upstream可以避开已试过的目标\n5. 通过Upstream可以对不同下游配置不同的连接参数\n6. 动态增删目标地址实时生效，方便对接任意的服务发现系统\n\n### Upstream相对于域名DNS解析的优势\n\nUpstream和域名DNS解析都可以将一组ip配置到一个Host，但是\n1. DNS域名解析是不针对于端口号的，相同ip不同端口的服务DNS域名是不能配置到一起的；但Upstream可以\n2. DNS域名解析对应的一组address，必定是ip；Upstream对应的一组address，可以是ip、域名或unix-domain-socket\n3. 通常情况下，DNS域名解析会被操作系统或网络上DNS服务器所缓存，更新时间受到ttl的限制；Upstream可以做到实时更新实时生效\n4. DNS域名解析消耗比Upstream解析和选取大很多\n\n### Workflow的Upstream\n\n这是一个本地反向代理模块，代理配置对server和client都生效。 \n\n支持动态配置，可用于服务发现系统，目前[workflow-k8s](https://github.com/sogou/workflow-k8s)可以对接Kubernetes的API Server。 \n\nUpstream名不包括端口，但Upstream请求支持指定端口（如果使用非内置协议，Upstream名暂时需要加上端口号以保证构造时的解析成功）。 \n\n每一个Upstream配置自己的独立名称UpstreamName，并添加设定着一组Address，这些Address可以是：\n1. ip4\n2. ip6\n2. 域名\n3. unix-domain-socket\n\n### 为什么要替代nginx的Upstream\n\n#### nginx的Upstream工作方式\n1. 只支持http/https协议\n2. 需要搭建一个nginx服务，启动进程占用socket等其他资源\n3. 请求先打到nginx上，nginx再向远端转发请求，这会多一次通信开销\n\n#### workflow本地Upstream工作方式\n1. 协议无关，你甚至可以通过upstream访问mysql、redis、mongodb等等\n2. 无需额外启动其他进程或端口，直接在进程内模拟反向代理的功能\n3. 选取过程是基本的计算和查表，不会有额外的通信开销\n\n# 使用Upstream\n\n### 常用接口\n~~~cpp\nclass UpstreamManager\n{\npublic:\n static int upstream_create_consistent_hash(const std::string& name,\n upstream_route_t consitent_hash);\n static int upstream_create_weighted_random(const std::string& name,\n bool try_another);\n static int upstream_create_manual(const std::string& name,\n upstream_route_t select,\n bool try_another,\n upstream_route_t consitent_hash);\n static int upstream_create_vnswrr(const std::string& name);\n static int upstream_delete(const std::string& name);\n\npublic:\n static int upstream_add_server(const std::string& name,\n const std::string& address);\n static int upstream_add_server(const std::string& name,\n const std::string& address,\n const struct AddressParams *address_params);\n static int upstream_remove_server(const std::string& name,\n const std::string& address);\n ...\n}\n~~~\n\n### 例1 在多个目标中随机访问\n配置一个本地反向代理，将本地发出的my_proxy.name所有请求均匀的打到6个目标server上\n~~~cpp\nUpstreamManager::upstream_create_weighted_random(\n \"my_proxy.name\",\n true);//如果遇到熔断机器，再次尝试直至找到可用或全部熔断\n\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.2.100:8081\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.2.100:8082\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.10.10\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"test.sogou.com:8080\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"abc.sogou.com\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"abc.sogou.com\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"/dev/unix_domain_scoket_sample\");\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://my_proxy.name/somepath?a=10\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n基本原理\n1. 随机选择一个目标\n2. 如果try_another配置为true，那么将在所有存活的目标中随机选择一个\n3. 仅在main中选择，选中目标所在group的主备和无group的备都视为有效的可选对象\n\n### 例2 在多个目标中按照权重大小随机访问\n配置一个本地反向代理，将本地发出的weighted.random所有请求按照5/20/1的权重分配打到3个目标server上\n~~~cpp\nUpstreamManager::upstream_create_weighted_random(\n \"weighted.random\",\n false);//如果遇到熔断机器，不再尝试，这种情况下此次请求必定失败\n\nAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\naddress_params.weight = 5;//权重为5\nUpstreamManager::upstream_add_server(\"weighted.random\", \"192.168.2.100:8081\", &address_params);//权重5\naddress_params.weight = 20;//权重为20\nUpstreamManager::upstream_add_server(\"weighted.random\", \"192.168.2.100:8082\", &address_params);//权重20\nUpstreamManager::upstream_add_server(\"weighted.random\", \"abc.sogou.com\");//权重1\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://weighted.random:9090\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n基本原理\n1. 按照权重分配，随机选择一个目标，权重越大概率越大\n2. 如果try_another配置为true，那么将在所有存活的目标中按照权重分配随机选择一个\n3. 仅在main中选择，选中目标所在group的主备和无group的备都视为有效的可选对象\n\n### 例3 在多个目标中按照框架默认的一致性哈希访问\n~~~cpp\nUpstreamManager::upstream_create_consistent_hash(\n \"abc.local\",\n nullptr);//nullptr代表使用框架默认的一致性哈希函数\n\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8081\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8082\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.10.10\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"test.sogou.com:8080\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"abc.sogou.com\");\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://abc.local/service/method\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n基本原理\n1. 每1个main视为16个虚拟节点\n2. 框架会使用std::hash对所有节点的address+虚拟index+此address加到此upstream的次数进行运算，作为一致性哈希的node值\n3. 框架会使用std::hash对path+query+fragment进行运算，作为一致性哈希data值\n4. 每次都选择存活node最近的值作为目标\n5. 对于每一个main、只要有存活group内main/有存活group内backup/有存活no group backup，即视为存活\n6. 如果upstream_add_server()时加上AddressParams，并配上权重weight，则每1个main视为16 * weight个虚拟节点，适用于带权一致性哈希或者希望一致性哈希标准差更小的场景\n\n### 例4 自定义一致性哈希函数\n~~~cpp\nUpstreamManager::upstream_create_consistent_hash(\n \"abc.local\",\n [](const char *path, const char *query, const char *fragment) -> unsigned int {\n unsigned int hash = 0;\n\n while (*path)\n hash = (hash * 131) + (*path++);\n\n while (*query)\n hash = (hash * 131) + (*query++);\n\n while (*fragment)\n hash = (hash * 131) + (*fragment++);\n\n return hash;\n });\n\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8081\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8082\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.10.10\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"test.sogou.com:8080\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"abc.sogou.com\");\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://abc.local/sompath?a=1#flag100\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n基本原理\n1. 框架会使用用户自定义的一致性哈希函数作为data值\n2. 其余与上例原理一致\n\n### 例5 自定义选取策略\n~~~cpp\nUpstreamManager::upstream_create_manual(\n \"xyz.cdn\",\n [](const char *path, const char *query, const char *fragment) -> unsigned int {\n return atoi(fragment);\n },\n true,//如果选择到已经熔断的目标，将进行二次选取\n nullptr);//nullptr代表二次选取时使用框架默认的一致性哈希函数\n\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"192.168.2.100:8081\");\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"192.168.2.100:8082\");\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"192.168.10.10\");\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"test.sogou.com:8080\");\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"abc.sogou.com\");\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://xyz.cdn/sompath?key=somename#3\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n基本原理\n1. 框架首先依据用户提供的普通选取函数、按照取模，在main列表中确定选取\n2. 对于每一个main、只要有存活group内main/有存活group内backup/有存活no group backup，即视为存活\n3. 如果选中目标不再存活且try_another设为true，将再使用一致性哈希函数进行二次选取\n4. 如果触发二次选取，一致性哈希将保证一定会选择一个存活目标、除非全部机器都被熔断掉\n\n### 例6 简单的主备模式\n~~~cpp\nUpstreamManager::upstream_create_weighted_random(\n \"simple.name\",\n true);//一主一备这项设什么没区别\n\nAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\naddress_params.server_type = 0;\nUpstreamManager::upstream_add_server(\"simple.name\", \"main01.test.ted.bj.sogou\", &address_params);//主\naddress_params.server_type = 1;\nUpstreamManager::upstream_add_server(\"simple.name\", \"backup01.test.ted.gd.sogou\", &address_params);//备\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://simple.name/request\", 0, 0, nullptr);\nauto *redis_task = WFTaskFactory::create_redis_task(\"redis://simple.name/2\", 0, nullptr);\nredis_task->get_req()->set_query(\"MGET\", {\"key1\", \"key2\", \"key3\", \"key4\"});\n(*http_task * redis_task).start();\n~~~\n基本原理\n1. 主备模式与前面所展示的任何模式都不冲突，可以同时生效\n2. 主备数量各自独立，没有限制。主和主之间平等，备与备之间平等，主备之间不平等。\n3. 只要有主活着，请求一直会使用某一个主\n4. 如果主都被熔断，备将作为替代目标接管请求直至有主恢复正常\n5. 在每一个策略中，存活的备都可以作为主的存活依据\n\n### 例7 主备+一致性哈希+分组\n~~~cpp\nUpstreamManager::upstream_create_consistent_hash(\n \"abc.local\",\n nullptr);//nullptr代表使用框架默认的一致性哈希函数\n\nAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\naddress_params.server_type = 0;\naddress_params.group_id = 1001;\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8081\", &address_params);//main in group 1001\naddress_params.server_type = 1;\naddress_params.group_id = 1001;\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8082\", &address_params);//backup for group 1001\naddress_params.server_type = 0;\naddress_params.group_id = 1002;\nUpstreamManager::upstream_add_server(\"abc.local\", \"main01.test.ted.bj.sogou\", &address_params);//main in group 1002\naddress_params.server_type = 1;\naddress_params.group_id = 1002;\nUpstreamManager::upstream_add_server(\"abc.local\", \"backup01.test.ted.gd.sogou\", &address_params);//backup for group 1002\naddress_params.server_type = 1;\naddress_params.group_id = -1;\nUpstreamManager::upstream_add_server(\"abc.local\", \"test.sogou.com:8080\", &address_params);//backup for no group mean backup for all group and no group\nUpstreamManager::upstream_add_server(\"abc.local\", \"abc.sogou.com\");//main, no group\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://abc.local/service/method\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n基本原理\n1. 组号-1代表无组，这种目标不属于任何组\n2. 无组的main之间是平等的，甚至可以视为同一个组。但与有组的main之间是隔离的\n3. 无组的backup可以为全局任何组目标/任何无组目标作为备\n4. 组号可以区分哪些主备是在一起工作的\n5. 不同组之间的备是相互隔离的，只为本组的main服务\n6. 添加目标的默认组号-1，type为0，表示主节点。\n\n### 例8 NVSWRR平滑按权重选取策略\n~~~cpp\nUpstreamManager::upstream_create_vnswrr(\"nvswrr.random\");\n\nAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\naddress_params.weight = 3;//权重为3\nUpstreamManager::upstream_add_server(\"nvswrr.random\", \"192.168.2.100:8081\", &address_params);//权重3\naddress_params.weight = 2;//权重为2\nUpstreamManager::upstream_add_server(\"nvswrr.random\", \"192.168.2.100:8082\", &address_params);//权重2\nUpstreamManager::upstream_add_server(\"nvswrr.random\", \"abc.sogou.com\");//权重1\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://nvswrr.random:9090\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n基本原理\n1. 虚拟节点初始化顺序按照[SWRR算法](https://github.com/nginx/nginx/commit/52327e0627f49dbda1e8db695e63a4b0af4448b1)选取\n2. 虚拟节点运行时分批初始化，避免密集型计算集中，每批次虚拟节点使用完后再进行下一批次虚拟节点列表初始化\n3. 兼具[SWRR算法](https://github.com/nginx/nginx/commit/52327e0627f49dbda1e8db695e63a4b0af4448b1)的平滑、分散特点，又能具备O(1)的时间复杂度\n4. 算法具体细节参见[tengine](https://github.com/alibaba/tengine/pull/1306)\n\n# Upstream选择策略\n\n当发起请求的url的URIHost填UpstreamName时，视做对与名字对应的Upstream发起请求，接下来将会在Upstream记录的这组Address中进行选择：\n1. 权重随机策略：按照权重随机选择\n2. 一致性哈希策略：框架使用标准的一致性哈希算法，用户可以自定义对请求uri的一致性哈希函数consistent_hash\n3. 手动策略：根据用户提供的对请求uri的select函数进行确定的选择，如果选中了已经熔断的目标：\n a. 如果try_another为false，这次请求将返回失败\n b. 如果try_another为true，框架使用标准的一致性哈希算法重新选取，用户可以自定义对请求uri的一致性哈希函数consistent_hash\n4. 主备策略：按照先主后备的优先级，只要主可以用就选择主。此策略可以与[1]、[2]、[3]中的任何一个同时生效，相互影响。\n\nround-robin/weighted-round-robin：视为与[1]等价，暂不提供 \n框架建议普通用户使用策略[2]，可以保证集群具有良好的容错性和可扩展性 \n对于复杂需求场景，高级用户可以使用策略[3]，订制复杂的选择逻辑\n\n# Address属性\n\n~~~cpp\nstruct EndpointParams\n{\n size_t max_connections;\n int connect_timeout;\n int response_timeout;\n int ssl_connect_timeout;\n bool use_tls_sni;\n};\n\nstatic constexpr struct EndpointParams ENDPOINT_PARAMS_DEFAULT =\n{\n .max_connections = 200,\n .connect_timeout = 10 * 1000,\n .response_timeout = 10 * 1000,\n .ssl_connect_timeout = 10 * 1000,\n .use_tls_sni = false,\n};\n\nstruct AddressParams\n{\n struct EndpointParams endpoint_params;\n unsigned int dns_ttl_default;\n unsigned int dns_ttl_min;\n unsigned int max_fails;\n unsigned short weight;\n int server_type;\n int group_id;\n};\n\nstatic constexpr struct AddressParams ADDRESS_PARAMS_DEFAULT =\n{\n .endpoint_params = ENDPOINT_PARAMS_DEFAULT,\n .dns_ttl_default = 12 * 3600,\n .dns_ttl_min = 180,\n .max_fails = 200,\n .weight = 1, //only for main of UPSTREAM_WEIGHTED_RANDOM\n .server_type = 0,\n .group_id = -1,\n};\n~~~\n每个Addreess都可以配置自己的自定义参数：\n * EndpointParams的max_connections, connect_timeout, response_timeout, ssl_connect_timeout：连接相关的参数\n * dns_ttl_default：dns cache中默认的ttl，单位秒，默认12小时，dns cache是针对当前进程的，即进程退出就会消失，配置也仅对当前进程有效\n * dns_ttl_min：dns最短生效时间，单位秒，默认3分钟，用于在通信失败重试时是否进行重新dns的决策\n * max_fails：触发熔断的【连续】失败次数（注：每次通信成功，计数会清零）\n * weight：权重，默认1，仅对main有效，用于Upstream随机策略选取和一致性哈希选取，权重大越容易被选中\n * server_type：主备配置，默认主。无论什么时刻，同组的主优先级永远高于其他的备\n * group_id：分组依据，默认-1。-1代表无分组(游离)，游离的备可视为任何主的备，有组的备优先级永远高于游离的备。\n\n# 关于熔断\n\n## MTTR\n\n平均修复时间（Mean time to repair，MTTR），是描述产品由故障状态转为工作状态时修理时间的平均值。\n\n## 服务雪崩效应\n\n服务雪崩效应是一种因“服务提供者的故障”（原因），导致“服务调用者故障”（结果），并将不可用逐渐/逐级放大的现象 \n若不加以有效控制，效应不会收敛，而且会以几何级放大，犹如雪崩，雪崩效应因此得名 \n日常表现通常为：起初只是一个很小的服务or模块异常/超时，引起下游其他依赖的服务随之异常/超时，产生连锁反应，最终导致绝大多数甚至全部的服务陷入瘫痪 \n随着故障的修复，效应随之消失，所以效应持续时间通常等于MTTR\n\n## 熔断机制\n\n当某一个目标的错误or异常触达到预先设定的阈值条件时，暂时认为这个目标不可用，剔除目标，即熔断开启进入熔断期 \n在熔断持续时间达到MTTR时长后，会进入半熔断状态，(尝试)恢复目标\n如果恢复的时候发现其他所有目标都被熔断，会同一时间把所有目标恢复\n熔断机制策略可以有效阻止雪崩效应\n\n## Upstream熔断保护机制\n\nMTTR=30秒，暂时不可配置，后续会考虑开放给用户自行配置 \n当某一个Addrees连续失败次数达到设定上限（默认200次），这个Address会被熔断MTTR=30秒 \nAddress在熔断期间，一旦被策略选中，Upstream会根据具体配置决定是否尝试其他Address、如何尝试 \n\n请注意满足下面1-4的某个情景，通信任务将得到一个WFT_ERR_UPSTREAM_UNAVAILABLE = 1004的错误：\n1. 权重随机策略，全部目标都处于熔断期\n2. 一致性哈希策略，全部目标都处于熔断期\n3. 手动策略 && try_another==true，全部目标都处于熔断期 \n4. 手动策略 && try_another==false，且同时满足下面三个条件： \n 1). select函数选中的main处于熔断期，，且游离的备都处于熔断期 \n 2). 这个main是游离的主，或者这个main所在的group其他目标都处于熔断期 \n 3). 所有游离的备都处于熔断期 \n\n# Upstream端口优先级\n\n1. 优先选择显式配置在Upstream Address上的端口号\n2. 若没有，再选择显式配置在请求url中的端口号\n3. 若都没有，使用协议默认端口号\n\n~~~text\n配置 UpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.2.100:8081\");\n请求 http://my_proxy.name:456/test.html => http://192.168.2.100:8081/test.html\n请求 http://my_proxy.name/test.html => http://192.168.2.100:8081/test.html\n~~~\n\n~~~text\n配置 UpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.10.10\");\n请求 http://my_proxy.name:456/test.html => http://192.168.10.10:456/test.html\n请求 http://my_proxy.name/test.html => http://192.168.10.10:80/test.html\n~~~\n\n" }, { "path": "docs/benchmark.md", "content": "# 性能测试\n\nSogou C++ Workflow是一款性能优异的网络框架，本文介绍我们进行的性能测试，\n包括方案、代码、结果，以及与其他同类产品的对比。\n\n更多场景下的实验正在进行中，本文将持续更新。\n\n## HTTP Server\n\nHTTP Client/Server是Sogou C++ Workflow常见的应用场景，\n我们首先对Server端进行实验。\n\n### 环境\n\n我们部署了两台相同机器作为Server和Client，软硬件配置如下：\n\n| 软硬件 | 配置 |\n|:---:|:---|\n| CPU | 40 Cores, x86_64, Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz |\n| Memory | 192GB |\n| NIC | 25000Mbps |\n| OS | CentOS 7.8.2003 |\n| Kernel | Linux version 3.10.0-1127.el7.x86_64 |\n| GCC | 4.8.5 |\n\n两者间`ping`测得的RTT为0.1ms左右。\n\n### 对照组\n\n我们选择nginx和brpc作为对照组。\n选择前者是因为它在生产中部署十分广泛，性能不俗；\n对于后者，我们在本次实验中只关注HTTP Server方面的能力，\n其他的特性已有[单独的实验][Sogou RPC Benchmark]进行更为详尽的测试。\n\n事实上，我们也对此二者之外的其他某些框架同时进行了实验，\n但结果其性能表现相差较远，因此未在本文中体现。\n\n后续我们将选取更多合适的框架加入对比测试中。\n\n### Client工具\n\n本次实验我们使用的压测工具为[wrk][wrk]和[wrk2][wrk2]。\n前者适合测试特定并发下的QPS极限和延时，\n后者适合在特定QPS下测试延时分布。\n\n我们也尝试过使用其他测试工具，例如[ab][ab]等，但无法打出足够的压力。\n有鉴于此，我们也在着手开发基于Sogou C++ Workflow的benchmark工具。\n\n### 变量和指标\n\n一般而言，对网络框架的性能测试，切入的角度可谓纷繁多样。\n通过控制不同的变量、观测不同的指标，可以探究程序在不同场景下的适应能力。\n\n本次实验，我们选择其中最普遍常见的变量和指标：\n通过控制Client并发度和承载数据的大小，来测试QPS和延时的变化情况。\n另外，我们还测试了在掺杂慢请求的正常请求的延时分布。\n\n下面依次介绍两个测试场景。\n\n### 不同并发度和数据长度下的QPS和延时\n\n#### 代码和配置\n\n我们搭建了一个极其简约的HTTP服务器，\n忽略掉所有的业务逻辑，\n将测试点聚焦在纯粹的网络框架性能上。\n\n代码片段如下，\n完整代码移步[这里][benchmark-01 Code]。\n\n```cpp\n// ...\n\nauto * resp = task->get_resp();\nresp->add_header_pair(\"Date\", timestamp);\nresp->add_header_pair(\"Content-Type\", \"text/plain; charset=UTF-8\");\nresp->append_output_body_nocopy(content.data(), content.size());\n\n// ...\n```\n\n可以从上述代码中看到，\n对于到来的任何HTTP请求，\n我们都会返回一段固定的内容作为Body，\n并设置必要的Header，\n包括代码中指明的`content-type`、`date`，\n以及自动填充的`connection`和`content-length`。\n\nHTTP Body的固定内容是在Server启动时随机生成的ASCII字符串，\n其长度可以通过启动参数配置。\n同时可以配置的还有使用的poller线程数和监听的端口号。\n前者我们在本次测试中固定为16，\n因此Sogou C++ Workflow将使用16个poller线程和20个handler线程（默认配置）。\n\n对于nginx和brpc，\n我们也构建了相同的返回内容，\n并为nginx配置了40个进程、\nbrpc配置了40个线程。\n\n\n#### 变量\n\n我们控制并发度在`[1, 2K]`之间翻倍增长，\n数据长度在`[16B, 64KB]`之间翻倍增长，\n两者正交。\n\n#### 指标\n\n鉴于并发度和数据长度组合之后数量较多，\n我们选择其中部分数据绘制为曲线。\n\n##### 固定数据长度下QPS与并发度关系\n\n![Concurrency and QPS][Con-QPS]\n\n上图可以看出，当数据长度保持不变，\nQPS随着并发度提高而增大，后趋于平稳。\n此过程中Sogou C++ Workflow一直有明显优势，\n高于brpc和nginx。\n特别是数据长度为64和512的两条曲线，\n并发度足够的时候，可以保持500K的QPS。\n\n注意上图中nginx-64与nginx-512的曲线重叠度很高，\n不易辨识。\n\n##### 固定并发度下QPS与数据长度关系\n\n![Body Length and QPS][Len-QPS]\n\n上图可以看出，当并发度保持不变，\n随着数据长度的增长，\nQPS保持平稳至4K时下降。\n此过程中，Sogou C++ Workflow也一直保持优势。\n\n##### 固定数据长度下延时与并发度关系\n\n![Concurrency and Latency][Con-Lat]\n\n上图可以看出，保持数据长度不变，\n延时随并发度提高而有所上升。\n此过程中，Sogou C++ Workflow略好于brpc，\n大好于nginx。\n\n##### 固定并发度下延时与数据长度关系\n\n![Body Length and Latency][Len-Lat]\n\n上图可以看出，并发度保持不变时，\n增大数据长度，造成延时上升。\n此过程中，Sogou C++ Workflow好于nginx，\n好于brpc。\n\n### 掺杂慢请求的延时分布\n\n#### 代码\n\n我们在上一个测试的基础上，简单添加了一个慢请求的逻辑，\n模拟业务场景中可能出现的特殊情况。\n\n代码片段如下，\n完整代码请移步[这里][benchmark-02 Code]。\n\n```cpp\n// ...\n\nif (std::strcmp(uri, \"/long_req/\") == 0)\n{\n auto timer_task = WFTaskFactory::create_timer_task(microseconds, nullptr);\n series_of(task)->push_back(timer_task);\n}\n// ...\n```\n\n我们在Server的process里进行判断，\n如果访问的是特定的路径，\n则添加一个`WFTimerTask`到Series的末尾，\n能够模拟一个异步耗时处理过程。\n类似地，对brpc使用`bthread_usleep()`函数进行异步睡眠。\n\n#### 配置\n\n在本次实验中，我们固定并发度为1024，数据长度为1024字节，\n分别以QPS为20K、100K和200K进行正常请求测试，\n测绘延时；\n与此同时，有另一路压力，进行慢请求，\nQPS是上述QPS的1%，\n数据不计入统计。\n慢请求的时长固定为5ms。\n\n#### 延时CDF图\n\n![Latency CDF][Lat CDF]\n\n从上图可以看出，当QPS为20K时，\nSogou C++ Workflow略次于brpc；\n当QPS为100K时，两者几乎相当；\n当QPS为200K时，Sogou C++ Workflow略好于brpc。\n总之，可以认为两者在这方面旗鼓相当。\n\n\n[Sogou RPC Benchmark]: https://github.com/holmes1412/sogou-rpc-benchmark\n[wrk]: https://github.com/wg/wrk\n[wrk2]: https://github.com/giltene/wrk2\n[ab]: https://httpd.apache.org/docs/2.4/programs/ab.html\n[benchmark-01 Code]: ../benchmark/benchmark-01-http_server.cc\n[benchmark-02 Code]: ../benchmark/benchmark-02-http_server_long_req.cc\n[Con-QPS]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-01.png\n[Len-QPS]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-02.png\n[Con-Lat]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-03.png\n[Len-Lat]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-04.png\n[Lat CDF]: https://raw.githubusercontent.com/wiki/sogou/workflow/img/benchmark-05.png\n" }, { "path": "docs/bugs.md", "content": "# 已知BUG列表\n\n### OpenSSL 1.1.1及以下，出现网络任务状态为WFT_STATE_SYS_ERROR，错误为0。\n这是OpenSSL 1.1.1及以下的bug，在SSL_get_error()为SSL_ERROR_SYSCALL时，errno被置为0。由于框架会把SSL_ERROR_SYSCALL转为系统错误，这会导致我们得到一个错误码0的系统错误：\n~~~cpp\nvoid callback(WFHttpTask *task)\n{\n int state = task->get_state();\n int error = task->get_error();\n printf(\"%d, %d\\n\", state, error); // 此处得到1，0，其中1是WFT_STATE_SYS_ERROR。\n}\n~~~\n显然只有在SSL通信下可能出现在这个问题。这个bug在OpenSSL 3.0里被修复，建议升级到OpenSSL 3.0或以上。 \n相关issue：https://github.com/openssl/openssl/issues/12416\n### 访问HTTPS网页，当打开TLS SNI并使用upstream时出现SSL error。\n当我们创建Http任务，http header里的Host域填写的是原始URL里的host部分。例如：\n~~~cpp\nvoid f()\n{\n auto *task = WFTaskFactory::create_http_task(\"https://sogou/index.html\", 0, 0, nullptr);\n}\n~~~\n这时候http request里的Host必然填写的是\"sogou\"。此时如果sogou是一个upstream名，指向域名www.sogou.com。并且我们开启了TLS SNI，那么SNI server name信息就是www.sogou.com，与http header里的Host是不一致的，会导致SSL错误。 \n要解决这个问题，用户可以在通过设置prepare函数，在发送请求前修改Host，让它与最终URL里的一致：\n~~~cpp\nvoid f();\n{\n auto *task = WFTaskFactory::create_http_task(\"https://sogou/index.html\", 0, 0, nullptr);\n task->set_prepare([](WFHttpTask *task){\n auto *t = static_cast *>(task);\n task->get_req()->set_header_pair(\"Host\", t->get_current_uri()->host); // 这里得到实际uri里的host。\n });\n}\n~~~\n只有打开了TLS SNI功能并使用upstream会出这个不一致问题。当然，很多时候我们配置upstream来访问http网站，也需要做这个修改，否则对方可能不会接受你的Host信息。\n" }, { "path": "docs/en/CONTRIBUTING.md", "content": "# Contribution Guide\n\nSogou C++ Workflow is community-driven and welcomes any contributor. \n\nThis document outlines some conventions about development steps, commit message formatting and contact points to make it easier to get your contribution accepted. \n\n- [Code of Conduct](#code-of-conduct)\n- [Getting started](#getting-started)\n- [First Contribution](#first-contribution)\n - [Find a good first topic](#find-a-good-first-topic)\n - [Work on an existed issue](#work-on-an-existed-issue)\n - [File a new issue](#file-a-new-issue)\n- [Contributor workflow](#contributor-workflow)\n - [Creating Pull Requests](#creating-pull-requests)\n - [Code Review](#code-review)\n - [Testing and building](#testing-and-building)\n\n# Code of Conduct\n\nPlease make sure to read and observe our [Code of Conduct](/CODE_OF_CONDUCT.md).\n\n# Getting started\n\n- Fork the repository on GitHub.\n- Make your changes on your fork repository.\n- Submit a PR.\n\n# First Contribution\n\nWe will help you to contribute in different areas like filing issues, developing features, fixing critical bugs and getting your work reviewed and merged.\n\nIf you have questions about the development process, feel free to [file an issue](https://github.com/sogou/workflow/issues/new/choose).\n\nWe are always in need of help, be it fixing documentation, reporting bugs or writing some code.\nLook at places where you feel best coding practices aren't followed, code refactoring is needed or tests are missing.\nHere is how you get started.\n\n### Find a good first topic\n\nYou can start by finding an existing issue with the \n[help-wanted](https://github.com/sogou/workflow/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) and\n[good first issue](https://github.com/sogou/workflow/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)\n label in this repository. These issues are well suited for new contributors as a beginner-friendly issues.\n\nWe can help new contributors who wish to work on such issues.\n\nAnother good way to contribute is to find a documentation improvement, such as a missing/broken link.\n\n#### Work on an existed issue\n\nWhen you are willing to take on an issue, just reply on the issue. The maintainer will assign it to you.\n\n### File a new issue\n\nWhile we encourage everyone to contribute code, it is also appreciated when someone reports an issue.\n\nPlease follow the prompted submission guidelines while opening an issue.\n\n# Contributor workflow\n\nTo contribute to the code base, please follow the workflow as defined in this section.\n\n1. Create a topic branch from where you want to base your work. This is usually master.\n2. Make commits of logical units and add test case if the change fixes a bug or adds new functionality.\n3. Run tests and make sure all the tests are passed.\n4. Make sure your commit messages are in the proper format.\n5. Push your changes to a topic branch in your fork of the repository.\n6. Submit a pull request.\n\nThis is a rough outline of what a contributor's workflow looks like. For more details, you are encouraged to communicate with the reviewers before sending a pull request.\n\nThanks for your contributions!\n\n## Creating Pull Requests\n\nOur project generally follows the standard [github pull request](https://help.github.com/articles/about-pull-requests/) process.\nTo submit a proposed change, please develop the code/fix and add new test cases.\nAfter that, run these local verifications before submitting pull request to predict the pass or fail of continuous integration.\n\n## Code Review\n\nTo make it easier for your Pull Request to receive reviews, break large changes into a logical series of smaller patches which individually make easily understandable changes, and in aggregate solve a broader issue.\n\nIf this is an independent modification, then it is recommended that you provide a tutorial and corresponding documents, and communicate with us. \n\n## Testing and building\n\nMake sure the the [travis-ci](https://travis-ci.com/github/sogou/workflow/pull_requests) passed.\n\nOnce Your PR has been merged, you become a contributor. Thank you for your contribution!\n" }, { "path": "docs/en/about-config.md", "content": "# About global configuration\n\nGlobal configuration is used to configure default global parameters to meet the actual business requirements and improve performance. The change of the global configuration must be made before you call any intefaces in the framework. Otherwise the change may not take effect. In addition, some global configuration items can be overridden in the upstream configuration. Please see upstream documents for reference.\n\n# Changing default configuration\n\n[WFGlobal.h](/src/manager/WFGlobal.h) defines the struts and the default values of the global configuration.\n\n~~~cpp\nstruct WFGlobalSettings\n{\n struct EndpointParams endpoint_params;\n struct EndpointParams dns_server_params;\n unsigned int dns_ttl_default; ///< in seconds, DNS TTL when network request success\n unsigned int dns_ttl_min; ///< in seconds, DNS TTL when network request fail\n int dns_threads;\n int poller_threads;\n int handler_threads;\n int compute_threads; ///< auto-set by system CPU number if value<=0\n int fio_max_events;\n const char *resolv_conf_path;\n const char *hosts_path;\n};\n\n\nstatic constexpr struct WFGlobalSettings GLOBAL_SETTINGS_DEFAULT =\n{\n .endpoint_params = ENDPOINT_PARAMS_DEFAULT,\n .dns_server_params = ENDPOINT_PARAMS_DEFAULT,\n .dns_ttl_default = 12 * 3600,\n .dns_ttl_min = 180,\n .dns_threads = 4,\n .poller_threads = 4,\n .handler_threads = 20,\n .compute_threads = -1,\n\t.fio_max_events = 4096,\n .resolv_conf_path = \"/etc/resolv.conf\",\n .hosts_path = \"/etc/hosts\",\n};\n~~~\n\n[EndpointParams.h](/src/manager/EndpointParams.h) defines the struture of EndpointParams and the default values.\n\n~~~cpp\n\nstruct EndpointParams\n{\n size_t max_connections;\n int connect_timeout;\n int response_timeout;\n int ssl_connect_timeout;\n bool use_tls_sni;\n};\n\nstatic constexpr struct EndpointParams ENDPOINT_PARAMS_DEFAULT =\n{\n .max_connections = 200,\n .connect_timeout = 10 * 1000,\n .response_timeout = 10 * 1000,\n .ssl_connect_timeout = 10 * 1000,\n .use_tls_sni = false,\n};\n~~~\n\nIf you want to change the default connecting timeout to 5 seconds, the default TTL for DNS to 1 hour and increase the number of poller threads for message deserialization to 10, you can follow the example below:\n\n~~~cpp\n#include \"workflow/WFGlobal.h\"\n\nint main()\n{\n struct WFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;\n\n settings.endpoint_params.connect_timeout = 5 * 1000;\n settings.dns_ttl_default = 3600;\n settings.poller_threads = 10;\n WORKFLOW_library_init(&settings);\n\n ...\n}\n\n~~~\n\nMost of the parameters are self-explanatory. Note: the ttl and related parameters in DNS configuration are in **seconds**. The timeout for endpoint is in **milliseconds**, and -1 indicates an infinite timeout. \ndns\\_threads indicates the total number of threads accessing DNS in parallel, but by default, we use asynchronous DNS resolving and don't create any dns threads (Except windows platform). \ndns\\_server\\_params indicates parameters that we access DNS server, including the maximum cocurrent connections, and the DNS server's connecting and response timeout. \ncompute\\_threads indicates the number of threads used for computation. The default value is -1, meaning the number of threads is the same as the number of CPU cores in the current node. \nfio\\_max\\_events indicates the maximum number of concurrent asynchronous file IO events. \nresolv\\_conf\\_path indicates the path of dns resolving configuration file. The default value is \"/etc/resolv.conf\" on unix platforms and NULL on windows. On the windows platform, we still use multi-threaded dns resolving by default. \nhosts_path indicates the path of the **hosts** file. The default value is \"/etc/hosts\" on unix platforms. If resolv_conf_path is NULL, this configuration will be ignored. \npoller\\_threads and handler\\_threads are the two parameters for tuning network performance:\n\n* poller\\_threads is mainly used for epoll (kqueue) and message deserialization.\n* handler\\_threads is the number of threads for the callback and the process of a network task.\n\nAll resources required by the framework are applied for when they are used for the first time. For example, if a user task does not involve DNS resolution, the asynchronous DNS resolver or DNS threads will not be created.\n" }, { "path": "docs/en/about-connection-context.md", "content": "# About connection context\n\nConnection context is an advanced programming topic in this framework. \nFrom the previous examples, we can see that we cannot assign one specific connection for a client task or a server task. \nHowever, in some business scenarios, especially for the server, we may need to maintain the connection status. In other words, we need to bind a context to a connection. \nIn the framework, we provide a connection context for users.\n\n# Application senarios for connection context\n\nHTTP is a completely stateless protocol, and HTTP session is realized with cookies. HTTP, Kafka and other stateless protocols are most friendly with our framework. \nThe connection used by Redis and MySQL is obviously stateful. Redis specifies the database ID on the current connection with the SELECT command. MySQL uses a completely stateful connection. \nWhen you use Redis or non-transactional MySQL client tasks in the framework, the URL already contains all the information related to the connection, including:\n\n* username and password\n* database name or database ID\n* the character set for MySQL\n\nThe framework will automatically log in or select a reusable connection based on the above information, and you do not need to care about the connection context. \nDue to this limitation, in the framework, you cannot use the SELECT command of Redis and the USE command of MySQL. If you want to switch databases, you should use a new URL to create the task. \nTransactional MySQL tasks can use fixed connections. Please see MySQL documentations for relevant details. \nHowever, if you implement a server based on Redis protocol, you need to know the current connection status.\n\nBy using the deleter function of connection context, users can also get notified when the connection was closed by the peer.\n\n# How to use connection context\n\nNote: generally, only the server tasks need to use the connection context, and the connection context is used only inside the process function, which is also the safest and simplest. \nYou can also use or modify the connection context in the callback, but you should consider the concurrency problem. We’ll discuss the related issues in details. \nYou can obtain the connection object in any network task through interfaces, and then obtain or modify the connection context. [WFTask.h](../src/factory/WFTask.h) contains a sample call:\n\n~~~cpp\ntemplate\nclass WFNetworkTask : public CommRequest\n{\npublic:\n virtual WFConnection *get_connection() const = 0;\n ...\n};\n~~~\n\n[WFConneciton.h ](../src/factory/WFConnection.h)contains the interfaces for performing operations on the connection objects:\n\n~~~cpp\nclass WFConnection : public CommConnection\n{\npublic:\n void *get_context() const;\n void set_context(void *context, std::function deleter);\n void set_context(void *context);\n void *test_set_context(void *test_context, void *new_context,\n std::function deleter);\n void *test_set_context(void *test_context, void *new_context);\n};\n~~~\n\n**get\\_connection()** can only be called in a process or a callback. If you call it in the callback, please check whether the return value is NULL. \nIf you get the WFConnection object successfully, you can perform operations on the connection context. A connection context is a void \\* pointer. When the connection is closed, the deleter is automatically called. When using the setting context functions without ``deleter`` argument, the original deleter will be kept unchanged.\n\n# Timing and concurrency for accessing connection context\n\nWhen a client task is created, the connection object is not determined. Thus, for all client tasks, you can only use the connection context in the callback. \nFor server tasks, you may use connection context in the process or the callback. \nWhen you use connection context in a callback, you need to consider concurrency, because the same connection may be reused by multiple tasks and reach the callbacks at the same time. \nTherefore, we recommend that the connection context should be accessed or modified only in the process function, because the connection will not be reused or released in the process, which is the simplest and safest. \nNote: the process in the above paragraphs means only the places inside the process function. In the places after the process function and before the callback, get\\_connection() always returns NULL. \n**test\\_set\\_context()** in the WFConnection is used to solve the concurrency issues for using connection context in the callback, but it is not recommended. \nIn a word, if you are not very familiar with the system implementation, please use the connection context only in the process function of the server tasks.\n\n# Example: how to reduce the request header fields in HTTP/1.1\n\nHTTP protocol is a stateless connection protocol, and a complete header must be sent for every request on the same connection. \nIf the cookie in the request is very large, it will obviously increase the data transmission overload. You can use the server-side connection context to solve this issue. \nYou can specify that the cookie in the HTTP request is valid for all subsequent requests on the same connection, and omit the cookie in the subsequent request headers. \nPlease see the following codes on the server side:\n\n~~~cpp\nvoid process(WFHttpTask *server_task)\n{\n protocol::HttpRequest *req = server_task->get_req();\n protocol::HttpHeaderCursor cursor(req);\n WFConnection *conn = server_task->get_connection();\n void *context = conn->get_context();\n std::string cookie;\n\n if (cursor.find(\"Cookie\", cookie))\n {\n if (context)\n delete (std::string *)context;\n context = new std::string(cookie);\n conn->set_context(context, [](void *p) { delete (std::string *)p; });\n }\n else if (context)\n cookie = *(std::string *)context;\n\n ...\n}\n~~~\n\nIn this way, if you arrange with the client that the cookie is transmitted only at the first request of the connection, the traffic can be reduced. \nThe implementation in the client side needs to use a new **prepare** function. Please see the codes below:\n\n~~~cpp\n\nusing namespace protocol;\n\nvoid prepare_func(WFHttpTask *task)\n{\n if (task->get_task_seq() == 0)\n task->get_req()->add_header_pair(\"Cookie\", my_cookie);\n}\n\nint some_function()\n{\n WFHttpTask *task = WFTaskFactory::create_http_task(...);\n task->set_prepare(prepare_func);\n ...\n}\n~~~\n\nIn the example, when the HTTP task is the first request on the connection, the cookie is set. If it is not the first request, according to our arrangement, we do not set the cookie. \nIn addition, you may use the connection context safely in the **prepare** function. **prepare** will not be concurrent on the same connection. \n" }, { "path": "docs/en/about-counter.md", "content": "# About counter\n\nCounters are very important basic tasks in our framework. A counter is essentially a semaphore that does not occupy thread. \nCounters are mainly used for workflow control. It includes anonymous counters and named counters, and can realize very complex business logic.\n\n# Creating a counter\n\nAs a counter is also a task, it is created through WFTaskFactory. You can create a counter with one of the following two methods:\n\n~~~cpp\nusing counter_callback_t = std::function;\n\nclass WFTaskFactory\n{\n ...\n static WFCounterTask *create_counter_task(unsigned int target_value,\n counter_callback_t callback);\n \n static WFCounterTask *create_counter_task(const std::string& counter_name,\n unsigned int target_value,\n counter_callback_t callback);\n\n ...\n};\n~~~\n\nEach counter contains a target\\_value. When the count in the counter reaches the target\\_value, its callback is called. \nThe above two interfaces generate a anonymous counter and a named counter respectively. The anonymous counter directly increases the count through the count method in the WFCounterTask:\n\n~~~cpp\nclass WFCounterTask\n{\npublic:\n virtual void count()\n {\n ...\n }\n ...\n}\n~~~\n\nIf a counter\\_name is passed when you create a counter, a named counter is generated, and the count can be increased with the count\\_by\\_name function.\n\n# Creating parallel tasks with anonymous counters\n\nIn the example of [parallel wget](/docs/en/tutorial-06-parallel_wget.md), we created a ParallelWork to achieve the parallel execution of several series. \nWith the combination of ParallelWork and SeriesWork, you can build series-parallel graphs in any form, which can meet the requirements in most scenarios. \nCounters allow us to build more complex dependencies between the tasks, such as a fully connected neural network. \nThe following simple code can replace ParallelWork to realize parallel HTTP crawling.\n\n~~~cpp\nvoid http_callback(WFHttpTask *task)\n{\n /* Save http page. */\n ...\n\n WFCounterTask *counter = (WFCounterTask *)task->user_data;\n counter->count();\n}\n\nstd::mutex mutex;\nstd::condition_variable cond;\nbool finished = false;\n\nvoid counter_callback(WFCounterTask *counter)\n{\n mutex.lock();\n finished = true;\n cond.notify_one();\n mutex.unlock();\n}\n\nint main(int argc, char *argv[])\n{\n WFCounterTask *counter = create_counter_task(url_count, counter_callback);\n WFHttpTask *task;\n std::string url[url_count];\n\n /* init urls */\n ...\n\n for (int i = 0; i < url_count; i++)\n {\n task = create_http_task(url[i], http_callback);\n task->user_data = counter;\n task->start();\n }\n\n counter->start();\n std::unique_lock lock(mutex);\n while (!finished) \n cond.wait(lock);\n lock.unlock();\n return 0;\n}\n~~~\n\nThe above code creates a counter with the target value as url\\_count, and calls the count once after each HTTP task is completed. \nNote that the times **count()** a anonymous counter cannot exceed it's target value. Otherwise the counter may have been destroyed after the callback, and the program behavior is undefined. \nThe call of **counter->start()** can be placed before the for loop. After a counter is created, you can call its count interface, no matter whether the counter has been started or not. \nYou can also use **counter->WFCounterTask::count()** to call the count interface of an anonymous counter; this can be used in performance-sensitive applications.\n\n# Using a server together with other asynchronous engines\n\nIn some cases, our server may need to call asynchronous clients in other frameworks and wait for the results. A simple method is that we wait synchronously in the process and then are waken up through conditional variables. \nIts disadvantage is that we occupy a processing thread and turn asynchronous clients in other frameworks into synchronous clients. But with the counter method, we can wait without occupying threads. The method is very simple:\n\n~~~cpp\n\nvoid some_callback(void *context)\n{\n protocol::HttpResponse *resp = get_resp_from_context(context);\n WFCounterTask *counter = get_counter_from_context(context);\n /* write data to resp. */\n ...\n counter->count();\n}\n\nvoid process(WFHttpTask *task)\n{\n WFCounterTask *counter = WFTaskFactory::create_counter_task(1, nullptr);\n\n SomeOtherAsyncClient client(some_callback, context);\n\n *series_of(task) << counter;\n}\n~~~\n\nHere, we can consider the series of a server task as a coroutine, and the counter whose target value is 1 can be considered as a conditional variable.\n\n# Named counters\n\nWhen the count operation is executed on the anonymous counter, the counter object pointer is directly accessed. This inevitably requires that the number of calls to count should not exceed the target value during operation. \nBut imagine an application scenario where we start four tasks at the same time, and as long as any three tasks are completed, the workflow can continue. \nWe can use a counter with a target value of 3, and count once after each task is completed. As long as three tasks are completed, the callback of the counter will be executed. \nBut the problem is that when the fourth task is finished and **counter->count()** is called again, the counter is already a wild pointer and the program crashes. \nIn this case, we can use named counters to solve this problem. By naming the counter and counting by name, we can have the following implementation:\n\n~~~cpp\nvoid counter_callback(WFCounterTask *counter)\n{\n WFRedisTask *next = WFTaskFactory::create_redis_task(...);\n series_of(counter)->push_back(next);\n}\n\nint main(void)\n{\n WFHttpTask *tasks[4];\n WFCounterTask *counter;\n\n counter = WFTaskFactory::create_counter_task(\"c1\", 3, counter_callback);\n counter->start();\n\n for (int i = 0; i < 4; i++)\n {\n tasks[i] = WFTaskFactory::create_http_task(..., [](WFHttpTask *task){\n WFTaskFactory::count_by_name(\"c1\"); });\n tasks[i]->start();\n }\n\n ...\n\n}\n~~~\n\nIn this example, four concurrent HTTP tasks are started, three of which are completed, and a Redis task is started immediately. In the practical application, you may need to add the code of data transmission. \nIn the example, a counter named \"c1\" is created, and in the HTTP callback, call **WFTaskFactory::count\\_by\\_name()** to increase the count.\n\n~~~cpp\nclass WFTaskFactory\n{\n ...\n static int count_by_name(const std::string& counter_name);\n\n static int count_by_name(const std::string& counter_name, unsigned int n);\n ...\n};\n~~~\n\nYou can pass an integer n to **WFTaskFactory::count\\_by\\_name**, indicating the count value to be increased in this operation. Obviously: \n**count\\_by\\_name(\"c1\")** is equivalent to **count\\_by\\_name(\"c1\", 1)**. \nIf the \"c1\" counter does not exist (not created or already completed), the operation on \"c1\" will have no effect, so the wild pointer problem in an anonymous counter will not happen here. \nThe **count\\_by\\_name()** function returns the number of counters that was waked up by the operation. When **n** is greater that 1, more than one counter may reach target value.\n\n# Definition of the detailed behaviors of named counters\n\nWhen you **call WFTaskFactory::count\\_by\\_name(name, n)**:\n\n* if the name does not exist (not created or already completed), there is no action.\n* if there is only one counter with that name:\n * if the remaining value of the counter is less than or equal to n, the counting is completed, the callback is called, and the counter is destroyed. end.\n * if the remaining value of the counter is greater than n, the count value is increased by n. end.\n* if there are multiple counters with that name:\n * according to the order of creation, take the first counter and assume that its remaining value is m:\n * if m is greater than n, the count value is increased by n. end (the remaining value is m-n).\n * if m is less than or equal to n, the counting is completed, the callback is called, and the counter is destroyed. set n = n-m.\n * If n is 0, the procedure ends.\n * If n is greater than 0, take out the next counter with the same name and repeat the whole operation.\n\nAlthough the description is very complicated, it can be summed up in one sentence. Access all counters with that name according to the order of creation one by one until n is 0. \nIn other words, one **count\\_by\\_name(name, n)** may wake up multiple counters. \nThe counters can be used to realize very complex business logic if you can use them well. In our framework, counters are often used to implement asynchronous locks or to build channels between tasks. It is more like a control task in form.\n" }, { "path": "docs/en/about-dns.md", "content": "# About DNS\nWhen using a domain name to request the network, you first need to obtain the server address through domain name resolution, and then use the network address to make subsequent requests. Workflow has implemented a complete domain name resolution and caching system. Generally speaking, users can initiate network tasks smoothly without knowing the internal mechanism.\n\n## DNS related configuration\nGlobal configuration in Workflow includes\n\n~~~cpp\nstruct WFGlobalSettings\n{\n struct EndpointParams endpoint_params;\n struct EndpointParams dns_server_params;\n unsigned int dns_ttl_default;\n unsigned int dns_ttl_min;\n int dns_threads;\n int poller_threads;\n int handler_threads;\n int compute_threads;\n int fio_max_events;\n const char *resolv_conf_path;\n const char *hosts_path;\n};\n~~~\n\nAmong them, the configuration items related to domain name resolution include\n\n* dns_server_params\n * address_family: This item will be explained later\n * max_connections: The maximum number of concurrent requests sent to the DNS server, the default is 200\n * connect_timeout/response_timeout/ssl_connect_timeout: refer to [timeout](about-timeout.md) for related instructions\n* dns_threads: When using synchronous mode to implement domain name resolution, the resolution operation will be executed in an independent thread pool. This item specifies the number of threads in the thread pool. The default is 4.\n* dns_ttl_default: The result of successful domain name resolution will be placed in the domain name cache. This item specifies its survival time in seconds. The default value is 1 hour. When the resolution result expires, it will be re-parsed to obtain the latest content.\n* dns_ttl_min: When communication fails, the cached result may have expired. This item specifies a shorter survival time. When communication fails, the cache is updated at a more frequent rate. The unit is seconds. The default value is 1 minute.\n* resolv_conf_path: This file saves the configuration related to accessing DNS. It is usually located in `/etc/resolv.conf` on common Linux distributions. If this item is configured as `NULL`, it means using multi-threaded synchronous resolution mode.\n* hosts_path: This file is a local domain name lookup table. If the resolved domain name hits this table, it will not initiate a request to DNS. It is usually located in `/etc/hosts` on common Linux distributions. If this item is configured as `NULL` means not to use the lookup table\n\n### resolv.conf extensions\nWorkflow has extended the `resolv.conf` configuration file. Users can modify the configuration to support the `DNS over TLS(DoT)`. **Note** directly modifying `/etc/resolv.conf` will affect other processes. You can make a copy of the file for modification, and modify the `resolv_conf_path` configuration of Workflow to the path of the new file. For example, a `nameserver` using the `dnss` protocol will connect via SSL\n\n~~~bash\nnameserver dnss://8.8.8.8/\nnameserver dnss://[2001:4860:4860::8888]/\n~~~\n\n### Address Family\nIn some network environments, although the machine supports IPv6, it cannot communicate with the outside because it has not been assigned a public IPv6 address (for example, the local IPv6 address starts with `fe80`). At this time, you can set `endpoint_params.address_family` to `AF_INET` to force only IPv4 addresses to be resolved during domain name resolution. Similarly, the `resolv.conf` file may specify both the IPv4 address and the IPv6 address of the `nameserver`. In this case, you can set `dns_server_params.address_family` to `AF_INET` or `AF_INET6` to force the use of only IPv4 or IPv6 addresses to access DNS.\n\n### Use Upstream configuration\nThe global configuration takes effect for each domain name by default. If you need to specify different configurations for certain domain names, you can use the [Upstream](./about-upstream.md#Address attribute) function. Using Upstream, you can individually specify the `dns_ttl_default` and `dns_ttl_min` configuration items, and individually specify the IP address family used by the domain name through `endpoint_params.address_family`.\n\n\n## Domain name resolution and caching strategy\nNetwork tasks usually require domain name resolution to obtain the IP address that needs to be accessed. The relevant strategies for domain name resolution in Workflow are as follows:\n\n1. Check whether the domain name cache has the IP address corresponding to the domain name. If there is a cache and it has not expired, use this set of IP addresses.\n2. Check whether the domain name is an IPv4, IPv6 address or `Unix Domain Socket`. If so, use the address directly without initiating domain name resolution.\n3. Check whether the `hosts_path` file contains the IP address corresponding to the domain name. If so, use the address directly.\n4. Obtain an asynchronous lock to ensure that a resolution request for the same domain name is only initiated once at the same time, and initiate a resolution request to DNS\n5. After successful parsing, the parsing result will be saved to the domain name cache of the current process for next use, and the asynchronous lock will be released.\n6. After the parsing fails, the asynchronous lock will be released and the failure reason will be notified to all tasks waiting on the same asynchronous lock. New tasks initiated after the notification is completed will request DNS again.\n\nMany scenarios that require a large number of network requests will be equipped with a domain name caching component. If a resolution request is sent to the DNS every time a network task is initiated, the DNS will inevitably be overwhelmed. Workflow sets the cache survival time (dns_ttl_default and dns_ttl_min) to ensure that the cache will expire after a reasonable period of time and the domain name resolution results can be updated in a timely manner. When a cache item of a domain name expires, the first task found to be expired will extend its survival time by 5 seconds and initiate a resolution request to DNS. Requests on the same domain name within 5 seconds will directly use the cached DNS resolution results without waiting.\n\nThe asynchronous lock mechanism can ensure that the resolution request for the **same domain name** is only initiated once at the same time. Without lock protection, if a large number of network tasks are initiated for the same domain name in a short period of time, each task will be unable to be retrieved from the cache. Too many resolution request to DNS will place a large and unnecessary burden on DNS. The same domain name here represents the `(host, port, family)` triplet. If a domain name is required to only use IPv4 and IPv6 through Upstream, they will be protected by different asynchronous locks, and it is possible to request DNS at the same time.\n\n\n### Asynchronous domain name resolution\nWorkflow implements a complete DNS task. If the `resolv_conf_path` configuration item is specified, an asynchronous request will be used when initiating domain name resolution to DNS. Under Unix-like systems, Workflow uses `/etc/resolv.conf` as the value of this configuration by default. Asynchronous domain name resolution does not block any threads or monopolize the thread pool, and can complete the task of domain name resolution more efficiently.\n\n### Synchronous domain name resolution\nIf `resolv_conf_path` is specified as `NULL`, synchronous domain name resolution will be achieved by calling the `getaddrinfo` function. This method will use an independent thread pool, and the number of threads is configured through the `dns_threads` parameter. If a large number of domain name resolution requests need to be initiated in a short period of time, the synchronization method will cause a large delay.\n" }, { "path": "docs/en/about-error.md", "content": "# About error handling\n\nError handling is an important and complex problem in any software system. Within our framework, error handling is ubiquitous and extremely cumbersome. \nIn the interfaces we exposed to users, we try to make things as simple as possible, but users still inevitably need to know some error messages.\n\n### Disabling C++ exceptions\n\nC++ exceptions are not used in our framework. When you compile your own code, it is best to add **-fno-exceptions** flag to reduce the code size. \nAccording to the common practice in the industry, we ignore the possibility of the failure of **new** operation, and avoid using new to allocate large blocks of memory internally. And there are error checks in memory allocation in C style.\n\n### About factory functions\n\nFrom the previous examples, you can see that all task and series are generated from two factory classes, WFTaskFactory or Workflow. \nThese factory classes, as well as more factory class interfaces that we may encounter in the future, ensure success. In other words, they never return NULL. And you do not need to check the return value. \nTo achieve this goal, even when the URL is illegal, the factory still generates the task normally. And you will get the error in the callback of the task.\n\n### States and error codes of a task\n\nIn the previous examples, you often see such codes in the callback:\n\n~~~cpp\nvoid callback(WFXxxTask *task)\n{\n int state = task->get_state();\n int error = task->get_error();\n ...\n}\n~~~\n\nin which, the state indicates the end state of a task. [WFTask.h](/src/factory/WFTask.h) contains all possible states:\n\n~~~cpp\nenum\n{\n WFT_STATE_UNDEFINED = -1,\n WFT_STATE_SUCCESS = CS_STATE_SUCCESS,\n WFT_STATE_TOREPLY = CS_STATE_TOREPLY, /* for server task only */\n WFT_STATE_NOREPLY = CS_STATE_TOREPLY + 1, /* for server task only */\n WFT_STATE_SYS_ERROR = CS_STATE_ERROR,\n WFT_STATE_SSL_ERROR = 65,\n WFT_STATE_DNS_ERROR = 66, /* for client task only */\n WFT_STATE_TASK_ERROR = 67,\n WFT_STATE_ABORTED = CS_STATE_STOPPED /* main process terminated */\n};\n~~~\n\n##### Please note the following states:\n\n* SUCCESS: the task is successfully completed. The client receives the complete reply, or the server writes the reply completely into the send buffer (but there is no guarantee that the peer will receive it).\n* SYS\\_ERROR: system error. In this case, use **task->get\\_error()** to get the system error code **errno**.\n * When **get\\_error()** gets ETIMEDOUT, you can call **task->get\\_timeout\\_reason()** to get the timeout reasons.\n* DNS\\_ERROR: DNS resolution error. Use **get\\_error()** to get the return code of **getaddrinfo()**. For DNS, please see the article for details [about-dns.md](/docs/en/about-dns.md). \n * The server task never has a DNS\\_ERROR.\n* SSL\\_ERROR: SSL error. Use **get\\_error()** to get the return value of **SSL\\_get\\_error()**.\n * Currently SSL error information is not complete, and you can not get the value of **ERR\\_get\\_error()**. Therefore, basically there are three possible return value of **get\\_error()**:\n * SSL\\_ERROR\\_ZERO\\_RETURN, SSL\\_ERROR\\_X509\\_LOOKUP, SSL\\_ERROR\\_SSL.\n * We will consider adding more detailed SSL error information in the future versions.\n* TASK\\_ERROR: task errors. Common errors include illegal URL, login failure, etc. [WFTaskError.h](/src/factory/WFTaskError.h) lists the return values of **get\\_error()**.\n\n##### You do not need to pay attention to the following states:\n\n* UNDEFINED: Client tasks that have just been created and have not yet been run are in UNDEFINED state.\n* TOREPLY: Server tasks that have not sent replies or called **task->noreply()** are in TOREPLY state.\n* NOREPLY: Server tasks that have called **task->noreply()** are always in NOREPLY state. The callback of these tasks are also in NOREPLY state. And the connection will be closed.\n\n### Other error handling requirements\n\nIn addition to the error handling of the task itself, you also need to check the errors of the message interfaces of various protocols. Generally, these interfaces indicate errors by returning false, and show the error reasons in the errno. \nIn addition, you may encounter more complicated error messages when you use some complex operations. You will learn them in detailed documents.\n" }, { "path": "docs/en/about-exit.md", "content": "# About exit\n\nAs most of our calls are non-blocking, we need some mechanisms to prevent the main function from exiting early in the previous examples. \nFor example, in the wget example, we wait for the user's Ctrl-C, or in the parallel\\_wget example, we wake up the main thread after all crawling tasks are finished. \nIn several server examples, the **stop()** operation is blocking, which can ensure the normal end of all server tasks and the safe exit of the main thread.\n\n# Principles on the safe exit\n\nGenerally, as long as you writes the program normally and follows the methods in the examples, there is little doubt about exit. However, it is still necessary to define the conditions for normal program exit.\n\n* You can't call **exit()** of the system in any callback functions such as the callback or the process, otherwise the behavior is undefined.\n* The condition that a main thread can safely end (call **exit()** or return in the main function) is that all tasks have been run to callbacks and no new tasks is started.\n * All our examples are consistent with this assumption, waking up the main function in the callback. This is safe, and there is no need to worry about the situation where the callback is not finished when the main function returns.\n * ParallelWork is a kind of tasks, which also needs to run to its callback.\n * This rule can be violated under certain circumstances. We will talk about it in the following section.\n* All server must stop, otherwise the behavior is undefined. Because all users know how to call the stop operation, generally a server program will not have any exit problems.\n * Server's stop() method will block until all server tasks' series end. But if you start a task directly in process function, you have to take care of the end this task.\n\n# Why do I need to wait for the callback of a running task? Can the program be ended early?\n\nFirst, explain why you need to wait till the callback of tasks before ending the program. \nIn most cases, the tasks generated through the task factory are composite tasks. For example, an http client task may need to resolve the dns, and then initiate the http crawl. And if a 302 redirect is encountered, dns resolving may be needed again. If the task fails, retrying may be involved. \nIn other words, any asynchronous tasks may contain multiple asynchronous processes, but it is completely insensitive to users. But between each internal asynchronous process, it does not check whether the program has exited. \nIf the user clearly knows that a task is an atomic task, for example, an http task created with an IP address (or a dns cache can definitely be hit), and there is no redirection or retry. Then, this task can be interrupted by the program's exit and come to the callback early, and the state of the task in the callback is WFT_STATE_ABORTED. \nFor example, the following program is always safe:\n~~~cpp\nvoid callback(WFHttpTask *task)\n{\n // most probably print 2，WFT_STATE_ABORTED。\n printf(\"state = %d\\n\", task->get_state());\n}\n\nint main()\n{\n WFHttpTask *task = WFTaskFactory::create_http_task(\"https://127.0.0.1/\", 0, 0, callback);\n task->start();\n // end the main process directly\n return 1;\n}\n~~~\nIf the dns cache hits, it is safe. Because there is no need to initiate a dns asynchronous task internally. E.g:\n~~~cpp\nWFFacilities::WaitGroup wg(1)\n\nvoid callback_normal(WFHttpTask *task)\n{\n wg.done();\n}\n\nvoid callback_abort(WFHttpTask *task)\n{\n // most probably print 2，WFT_STATE_ABORTED。\n printf(\"state = %d\\n\", task->get_state());\n}\n\nint main()\n{\n WFHttpTask *task = WFTaskFactory::create_http_task(\"https://www.sogou.com/\", 3, 2, callback_normal);\n task->start();\n // wait for the end of the first task\n wg.wait();\n // Access wwww.sogou.com again. Hit the dns cache definitely.\n WFHttpTask *task = WFTaskFactory::create_http_task(\"https://www.sogou.com/\", 0, 0, callback_abort);\n task->start();\n // end the main process directly\n return 1;\n}\n~~~\nTherefore, for a network task, as long as it can be determined to be an atomic task, it can be interrupted by the end of the program. This principle can be extended to any type of task. \nFor example, the timer task is an atomic task, and the following program is also safe:\n~~~cpp\nvoid callback(WFTimerTask *task)\n{\n // definitely print 2，WFT_STATE_ABORTED。\n printf(\"state = %d\\n\", task->get_state());\n}\nint main()\n{\n WFTimerTask *task = WFTaskFactory::create_timer_task(1000000, callback);\n task->start();\n // end the main process directly\n return 1;\n}\n~~~\nIn the documentation (About Timer)(https://github.com/sogou/workflow/blob/master/docs/en/about-timer.md), we will describe them in detail. \nIn addition, you can also end the program before the callback of single-threaded computing tasks and file IO tasks. Among them, the computing task that is already running, the program will wait for the task to end, and finally callback in the SUCCESS state. If it has not begun running, it will canceled and you will get an ABORTED state in callback. \nAs long as the file IO task has been started, it will always wait for the IO to complete. Therefore, it is always safe to exit the program directly.\n\n# About memory leakage of OpenSSL 1.1 in exiting\n\nWe found that some OpenSSL 1.1 versions have the problem of incomplete memory release in exiting. The memory leak can be seen by Valgrind memcheck tool. \nThis problem only happens when you use SSL, such as crawling HTTPS web pages, and usually you can ignore this leak. If it must be solved, you can use the following method:\n\n~~~cpp\n#include \n\nint main()\n{\n#if OPENSSL_VERSION_NUMBER >= 0x10100000L\n OPENSSL_init_ssl(0, NULL);\n#endif\n ...\n}\n~~~\n\nIn other words, before using our library, you should initialize OpenSSL. You can also configure OpenSSL parameters at the same time if necessary. \nPlease note that this function is only available in OpenSSL version 1.1 or above, so you need to check the openSSL version before calling it. \nThis memory leak is related to the memory release mechanism of OpenSSL 1.1. The solution provided by us can solve this problem (but we still recommend you to ignore it).\n" }, { "path": "docs/en/about-go-task.md", "content": "# About go task\n\nWe provide a simpler way to use computing task, which is inspired by the golang, and we name it 'go task'. \nWhen using go task, no input nor output type has to be defined. All data are passed through function's arguments.\n\n# Creating a go task\n~~~cpp\nclass WFTaskFactory\n{\n ...\npublic:\n template\n static WFGoTask *create_go_task(const std::string& queue_name,\n FUNC&& func, ARGS&&... args);\n};\n~~~\n\n# Example\nWe want to run an 'add' function asychronously: void add(int a, int b, int& res); \nStill, we want the result printed after the 'add' function is finished. We may create a go task:\n~~~cpp\n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\nvoid add(int a, int b, int& res)\n{\n res = a + b;\n}\n\nint main(void)\n{\n WFFacilities::WaitGroup wait_group(1);\n int a = 1;\n int b = 1;\n int res;\n\n WFGoTask *task = WFTaskFactory::create_go_task(\"test\", add, a, b, std::ref(res));\n task->set_callback([&](WFGoTask *task) {\n printf(\"%d + %d = %d\\n\", a, b, res);\n wait_group.done();\n });\n \n task->start();\n wait_group.wait();\n return 0;\n}\n~~~\nThe above example runs an add function asynchronously, prints the result and exits normally. The creating and running of go task have little difference from other kinds of tasks, and the user_data field is also available. \nNote that when creating a go task, we donot pass a callback function. But you may set_callback later like other kinds of tasks. \nIf an argument of the go task's function is a reference, you should use `std::ref` when passing it to the task, otherwise it will be passed as a value. \n\n# Go task with running time limit\nYou may create a go task with running time limit by calling WFTaskFactory::create_timedgo_task():\n~~~cpp\nclass WFTaskFactory\n{\n /* Create 'Go' task with running time limit in seconds plus nanoseconds.\n * If time exceeded, state WFT_STATE_SYS_ERROR and error ETIMEDOUT will be got in callback. */\n template\n static WFGoTask *create_timedgo_task(time_t seconds, long nanoseconds,\n const std::string& queue_name,\n FUNC&& func, ARGS&&... args);\n};\n~~~\nCompared with creating a normal go task, the ``create_timedgo_task`` function needs to pass two more parameters, seconds and nanoseconds. If the running time of ``func`` reaches the seconds+nanosconds time limit, the task callback directly, and the state is WFT_STATE_SYS_ERROR and the error is ETIMEDOUT. \nNote that the framework cannot interrupt the user's ongoing task. ``func`` will still continue to execute to the end, but will not callback again. In addition, the value range of nanoseconds is [0,1 billion).\n\n\n# Use the whole library as a thread pool\n\nYou may use go task only. In this way the workflow library becomes a thread pool，and the default thread number is equal to the cpu number of the host. \nBut this thread pool has some special features. Every thread task is associated with a queue name that will indicate scheduling, and you may set up the dependency of all tasks too.\n\n" }, { "path": "docs/en/about-module.md", "content": "# About Module Task\n\nOur **series** has tasks as elements. But in many cases, users need module-level encapsulation, such as several tasks to complete a specific function. With the original method, you have to let the callback of the last task connect to the next task, or fill in the response of the server task. Therefore, we introduced WFModuleTask, which is convenient for users to encapsulate modules and reduce the coupling of tasks between different functional modules.\n\n# Create a Module Task\n\nWe define a **module** as a kind of task, WFModuleTask. Inside the module includes a sub_series for running tasks within the module. Any task doesn't need to care if it runs inside a module. Because the sub_series inside the module is no different from the normal series. \nIn [WFTaskFactory.h](/src/factory/WFTaskFactory.h), the creation interface of module task:\n~~~cpp\nusing module_callback_t = std::function;\n\nclass WFTaskFactory\n{\n static WFModuleTask *create_module_task(SubTask *first, module_callback_t callback);\n};\n~~~\nThe first create_module_task() is the first task of the module. Similar to creating a series. \nThe module task’s callback request a **const** pointer argument in order to prevent user pushing more tasks to module in callback.\n\n# WFModuleTask Interfaces\n\nBecause we define modules as this kind of task, we can use modules like any other task. But modules do not have **state** and **error** fields. \nIn [WFTask.h](/src/factory/WFTask.h), we define the class of WFModuleTask:\n~~~cpp\nclass ModuleTask : public ParallelTask, protected SeriesWork\n{\npublic:\n void start() { .. }\n void dismiss() { ... }\n\npublic:\n SeriesWork *sub_series() { return this; }\n const SeriesWork *sub_series() const { return this; }\n\npublic:\n void *user_data;\n};\n~~~\nThe **sub_series** interface returns the series of tasks running in the module. A module is essentially a sub-flow. \nsub_series is also an ordinary series, and users can call its set_context(), get_context(), push_back() and other functions. \nBut we don't recommend setting a callback for sub_series, use module task’s callback instead. \n\n# Example\n\nIn the processing logic of an http server, we design all processing logic as a module.\n~~~cpp\nstruct ModuleCtx {\n std::string body;\n};\n\nvoid http_callback(WFHttpTask *http_task)\n{\n SeriesWork *series = series_of(http_task); // This series is module’s sub_series\n struct ModuleCtx *ctx = (struct ModuleCtx *)series->get_context();\n const void *body;\n size_t size;\n If (http_task->get_resp()->get_parsed_body(&body, &size))\n {\n ctx->body.assign(body, size);\n }\n\n ParallelWork *pwork = Workflow::create_parallel_work(…)；// Do some other things\n series->push_back(pwork);\n}\n\nvoid process(WFHttpTask *server_task)\n{\n WFHttpTask *http_task = WFTaskFactory::create_http_task(…, http_callback);\n WFModuleTask *module = WFTaskFactory::create_module_task(http_task, [server_task](const WFModuleTask *mod) {\n struct ModuleCxt *ctx = (ModuleCtx *)mod->sub_series()->get_context();\n server_task->get_resp()->append_output_body(ctx->body);\n delete ctx;\n });\n module->sub_series()->set_context(new ModuleCtx);\n series_of(server_task)->push_back(module);\n}\n~~~\nThrough this method, the tasks in the module only need to operate the series context, and finally the **resp** is filled in by the callback of the module. Task coupling is greatly reduced.\n" }, { "path": "docs/en/about-resource-pool.md", "content": "# Conditional task and resource pool\nWhen we use workflow to write asynchronous programs, we often encounter such scenarios:\n* A task needs to obtain a resource from a certain pool before running, and put it back to the pool after it finishs.\n* We may need to limit the max concurrency of accessing one or more communication targets. But don't want to occupy a thread when waiting.\n* We have many tasks that arrive randomly, in different series. But these tasks must be run serially.\n\nAll these needs can be solved with the resource pool module. Our [WFDnsResolver](https://github.com/sogou/workflow/blob/master/src/nameservice/WFDnsResolver.cc) uses this method to control the concurrency of querying the dns server.\n\n# Interfaces of resource pool\n\nIn [WFResourcePool.h](https://github.com/sogou/workflow/blob/master/src/factory/WFResourcePool.h) we define the interfaces of resource pool:\n~~~cpp\nclass WFResourcePool\n{\npublic:\n WFConditional *get(SubTask *task, void **resbuf);\n WFConditional *get(SubTask *task);\n void post(void *res);\n ...\n\nprotected:\n virtual void *pop()\n {\n return this->data.res[this->data.index++];\n }\n\n virtual void push(void *res)\n {\n this->data.res[--this->data.index] = res;\n }\n ...\n\npublic:\n WFResourcePool(void *const *res, size_t n);\n WFResourcePool(size_t n);\n ...\n};\n~~~\n### Constructors\nThe first constructor accept a resource array, with the lenght n. Each element of the array is a **void \\*** representing a resource. The whole array will be copied by the constructor. \nIf all the initial resources are **nullptr**, you may use the second constructor which has only one argument n, representing the number of resources. \nYou may take a look of the implementation codes:\n~~~cpp\nvoid WFResourcePool::create(size_t n)\n{\n this->data.res = new void *[n];\n this->data.value = n;\n ...\n}\n\nWFResourcePool::WFResourcePool(void *const *res, size_t n)\n{\n this->create(n);\n memcpy(this->data.res, res, n * sizeof (void *));\n}\n\nWFResourcePool::WFResourcePool(size_t n)\n{\n this->create(n);\n memset(this->data.res, 0, n * sizeof (void *));\n}\n~~~\n\n### Application interfaces\nUsers use **get()** method of resource pool to wrap a task. **get()** returns a conditional, which is also a task. Conditional will runs the task it wrap when it obtain a resource from the pool. **get()** may accept a second argument **void \\*\\* resbuf**, which is the buffer that will store the resource abtained. After the **get()** operation, users can use the returned conditional to substain the original task. It can be started or put to any series just like an ordinary task. \nAfter the user task is finished, **post()** need to be called to return a resource to the pool. Typically, **post()** is called in user task's callback.\n\n### Derivation\nThe using of resource pool is FILO. It means the last released resource will be the next one to be obtained. You may subclass WFResourcePool to implement a FIFO pool.\n\n### Example\nWe have a URL list to be crawled. But we limit the max concurreny of crawling task to be **max_p**. We may use ParallelWork to implement this function of course. But with resource pool, everything is much simpler:\n~~~cpp\nint fetch_with_max(std::vector& url_list, size_t max_p)\n{\n WFResourcePool pool(max_p);\n\n for (std::string& url : url_list)\n {\n WFHttpTask *task = WFTaskFactory::create_http_task(url, [&pool](WFHttpTask *task) {\n pool.post(nullptr);\n });\n WFConditional *cond = pool.get(task);\n cond->start();\n }\n\n // wait_here...\n}\n~~~\n" }, { "path": "docs/en/about-service-governance.md", "content": "# About service governance\n\nWe have a complete mechanism to manage the services we depend on. This mechanism includes the following functions:\n\n* User level DNS.\n* Selection of service addresses.\n * Including a variety of selection mechanisms, such as random weight, consistent hash, manual selection methods, etc.\n* Service circuit breaker and recovery.\n* Load balancing.\n* Configuring independent parameters for a single service.\n* Main/backup relations for a service, etc.\n\nAll these functions depend on our upstream subsystem. By making good use of this system, we can easily implement more complex service mesh functions.\n\n# upstream name\n\nupstream name is equivalent to the domain name inside the program. However, compared with the general domain name, upstream has more functions, including:\n\n* Generally, a domain name can only point to a set of IP addresses; an upstream name can point to a set of IP addresses or domain names.\n* The objects (domain names or IPs) pointed by the upstream may include port information.\n* upstream has powerful functions for managing and selecting targets, and each target can contain a large number of attributes.\n* upstream update is real-time and completely thread-safe, while the DNS of domain names cannot be updated in real time.\n\nIn practice, if you don't need to access the external network, the domain names and DNS can be completely replaced by upstream.\n\n# Creating and deleting upstream\n\n[UpstreamMananer.h](/src/manager/UpstreamManager.h) contains several interfaces for creating upstream:\n\n~~~cpp\nusing upstream_route_t = std::function;\n\nclass UpstreamManager\n{\npublic:\n static int upstream_create_consistent_hash(const std::string& name,\n upstream_route_t consitent_hash);\n\n static int upstream_create_weighted_random(const std::string& name,\n bool try_another);\n\n static int upstream_create_manual(const std::string& name,\n upstream_route_t select,\n bool try_another,\n upstream_route_t consitent_hash);\n\n static int upstream_delete(const std::string& name);\n ...\n};\n~~~\n\nThe three functions create three types of upstream: consistent hash, weighted random and manual selection. \nThe parameter **name** means upstream name, which is used in the same way as a domain name after creation. \n**consistent\\_hash** and **select** parameters are both **std::function** of **upstream\\_route\\_t**, which are used to specify the routing method. \nAnd try\\_another indicates whether to continue trying to find an available target if the selected target is unavailable (blown). consistent\\_hash mode does not have this attribute. \nThe upstream\\_route\\_t parameter receives three parameters: path, query and fragment in a URL. For example, if the URL is http://abc.com/home/index.html?a=1#bottom, the three parameters are \"/home/index.html\", \"a=1” and \"bottom” respectively. Based on these three parts, the system can select the target server or perform consistent hashing. \nPlease note that you call pass nullptr to all consistent\\_hash parameters in the above interfaces, and the framework will use the default consistent hash algorithm.\n\n# Example 1: weight allocation\n\nWe want to allocate 50% of the requests to www.sogou.com to 127.0.0.1:8000 and 127.0.0.1:8080, and make their load be 1:4. \nWe don't need to care about the number of IP addresses behind the domain name www.sogou.com. In short, the actual domain name will receive 50% of the requests.\n\n~~~cpp\n#include \"workflow/UpstreamManager.h\"\n#include \"workflow/WFTaskFactory.h\"\n\nint main()\n{\n UpstreamManager::upstream_create_weighted_random(\"www.sogou.com\", false);\n struct AddressParams params = ADDRESS_PARAMS_DEFAULT;\n\n params.weight = 5;\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"www.sogou.com\", ¶ms);\n params.weight = 1;\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"127.0.0.1:8000\", ¶ms);\n params.weight = 4;\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"127.0.0.1:8080\", ¶ms);\n\n WFHttpTask *task = WFTaskFactory::create_http_task(\"http://www.sogou.com/index.html\", ...);\n ...\n}\n~~~\n\nPlease note that these functions can be called in any scenario. They are completely thread-safe and takes effect instantly. \nIn addition, because all our protocols, including user-defined protocols, have URLs, the upstream function can be applied to all protocols.\n\n# Example 2: manual selection\n\nIn the same example as above, we want to allocate 127.0.0.1:8000 if the query in the request URLs is \"123\", port 8080 if the query is \"abc\", and normal domain names for other requests.\n\n~~~cpp\n#include \"workflow/UpstreamManager.h\"\n#include \"workflow/WFTaskFactory.h\"\n\nint my_select(const char *path, const char *query, const char *fragment)\n{\n if (strcmp(query, \"123\") == 0)\n return 1;\n else if (strcmp(query, \"abc\") == 0)\n return 2;\n else\n return 0;\n}\n\nint main()\n{\n UpstreamManager::upstream_create_manual(\"www.sogou.com\", my_select, false, nullptr);\n\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"www.sogou.com\");\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"127.0.0.1:8000\");\n UpstreamManager::upstream_add_server(\"www.sogou.com\", \"127.0.0.1:8080\");\n\n /* This URL will route to 127.0.0.1:8080 */\n WFHttpTask *task = WFTaskFactory::create_http_task(\"http://www.sogou.com/index.html?abc\", ...);\n ...\n}\n~~~\n\nBecause Redis and MySQL protocols are provided natively, it is very convenient to realize the read-write separation function of the database with this method (Note: non-transactional operation). \nIn the above two examples, the upstream name is www.sogou.com, which is also a domain name. Of course, you can use a simpler string sogou as upstream name. Thus:\n\n~~~cpp\n WFHttpTask *task = WFTaskFactory::create_http_task(\"http://sogou/home/1.html?abc\", ...);\n~~~\n\nIn a word, if the host part of the URL is a created upstream, it will be used as an upstream.\n\n# Example 3: consistent hash\n\nIn this scenario, we will randomly select one machine from 10 Redis instances and communicate with it. But we must ensure that the same URL always accesses the same specific target. The method is very simple:\n\n~~~cpp\nint main()\n{\n UpstreamManager::upstream_create_consistent_hash(\"redis.name\", nullptr);\n\n UpstreamManager::upstream_add_server(\"redis.name\", \"10.135.35.53\");\n UpstreamManager::upstream_add_server(\"redis.name\", \"10.135.35.54\");\n UpstreamManager::upstream_add_server(\"redis.name\", \"10.135.35.55\");\n ...\n UpstreamManager::upstream_add_server(\"redis.name\", \"10.135.35.62\");\n\n auto *task = WFTaskFactory::create_redis_task(\"redis://:mypassword@redis.name/2?a=hello#111\", ...);\n ...\n}\n~~~\n\nOur Redis task does not recognize the query part, so you can fill it out at will. 2 in the path indicates the Redis database ID. \nAt this time, the consistent\\_hash function will get three parameters: \"/2\", \"a=hello\" and \"111\". Because we use nullptr, the default consistent hash will be called. \nAs we does not specify the port number for the server in upstream, it will use the port in the URL. The default port of Redis is 6379. \nThere is no try\\_another option for consitent\\_hash. If the target is blown, another one will be automatically selected. The same URL will always get the same server (cache friendly).\n\n# Parameters of upstream server\n\nIn Example 1, we set the weight of a server through params. But the server parameters is far more than just a weight. Its struct is defined as follows:\n\n~~~cpp\n// In EndpointParams.h\nstruct EndpointParams\n{\n size_t max_connections;\n int connect_timeout;\n int response_timeout;\n int ssl_connect_timeout;\n bool use_tls_sni;\n};\n\n// In ServiceGovernance.h\nstruct AddressParams\n{\n struct EndpointParams endpoint_params; ///< Connection config\n unsigned int dns_ttl_default; ///< in seconds, DNS TTL when network request success\n unsigned int dns_ttl_min; ///< in seconds, DNS TTL when network request fail\n/**\n * - The max_fails directive sets the number of consecutive unsuccessful attempts to communicate with the server.\n * - After 30s following the server failure, upstream probe the server with some alive client’s requests.\n * - If the probes have been successful, the server is marked as an alive one.\n * - If max_fails is set to 1, it means server would out of upstream selection in 30 seconds when failed only once\n */\n unsigned int max_fails; ///< [1, INT32_MAX] max_fails = 0 means max_fails = 1\n unsigned short weight; ///< [1, 65535] weight = 0 means weight = 1. only for main\n int server_type; ///< 0 for main and 1 for backup\n int group_id; ///< -1 means no group. Backup without group will backup for any main\n};\n~~~\n\nMost of the parameters are self-explanatory. Among these parameters, endpoint\\_params, dns and other parameters will override the global configuration. \nFor example, if the global maximum number of connections to each target IP is 200, but you want to set a maximum of 1000 connections for 10.135.35.53, please follow the instructions below:\n\n~~~cpp\n UpstreamManager::upstream_create_weighted_random(\"10.135.35.53\", false);\n struct AddressParams params = ADDRESS_PARAMS_DEFAULT;\n params.endpoint_params.max_connections = 1000;\n UpstreamManager::upstream_add_server(\"10.135.35.53\", \"10.135.35.53\", ¶ms);\n~~~\n\nmax\\_fails parameter indicates the maximum number of failure. If the selected target continuously fails, and the number of failure reaches max\\_failures, it will enter the fusing state. If the try\\_another attribute of upstream is false, the task will fail. \nIn the callback of the task, get\\_state()=WFT\\_STATE\\_TASK\\_ERROR，get\\_error()=WFT\\_ERR\\_UPSTREAM\\_UNAVAILABLE. \nIf try\\_another is true and all server are blown, you will get the same error. The fusing time is 30 seconds. \nServer\\_type and group\\_id are used for main/backup features. All upstream must have a server whose type is 0, representing main, otherwise the upstream is unavailable. \nBackup servers (server_type 1) will be used when the main servers of the same group\\_id is blown.\n\nFor more information on the features of upstream, please see [about-upstream.md](/docs/en/about-upstream.md).\n" }, { "path": "docs/en/about-timeout.md", "content": "# About timeout\n\nIn order to make all communication tasks run as accurately as expected by users, the framework provides a large number of timeout configuration functions and ensure the accuracy of these timeouts. \nSome of these timeout configurations are global, such as connection timeout, but you may configure your own connection timeout for a perticular domain name through the upstream.\nSome timeouts are task-level, such as sending a message completely, because users needs to dynamically configure this value according to the message size. \nOf course, a server may have its own overall timeout configuration. In a word, timeout is a complicated matter, and the framework will do it accurately. \nAll timeouts are in **poll** style. It is an **int** in milliseconds and -1 means infinite. \nIn addition, as said in the project introduction, you can ignore all the configurations, and adjust them when you meet the actual requirements.\n\n### Timeout configuration for basic communication\n\n[EndpointParams.h](/src/manager/EndpointParams.h) contains the following items:\n\n~~~cpp\nstruct EndpointParams\n{\n size_t max_connections;\n int connect_timeout;\n int response_timeout;\n int ssl_connect_timeout;\n};\n\nstatic constexpr struct EndpointParams ENDPOINT_PARAMS_DEFAULT =\n{\n .max_connections = 200,\n .connect_timeout = 10 * 1000,\n .response_timeout = 10 * 1000,\n .ssl_connect_timeout = 10 * 1000,\n};\n~~~\n\nin which there are three DNS-related configuration items. Please ignore them right now. Items related to timeout: \n\n* connect\\_timeout: timeout for establishing a connection with the target. The default value is 10 seconds.\n* response\\_timeout: timeout for waiting for the target response; the default value is 10 seconds. It is the timeout for sending a block of data to the target or reading a block of data from the target.\n* ssl\\_connect\\_timeout: timeout for completing SSL handshakes with the target. The default value is 10 seconds.\n\nThis struct is the most basic configuration for the communication connection, and almost all subsequent communication configurations contain this struct.\n\n### Global timeout configuration\n\nYou can see the global settings in [WFGlobal.h](/src/manager/WFGlobal.h).\n\n~~~cpp\nstruct WFGlobalSettings\n{\n EndpointParams endpoint_params;\n unsigned int dns_ttl_default;\n unsigned int dns_ttl_min;\n int dns_threads;\n int poller_threads;\n int handler_threads;\n int compute_threads;\n};\n\nstatic constexpr struct WFGlobalSettings GLOBAL_SETTINGS_DEFAULT =\n{\n .endpoint_params = ENDPOINT_PARAMS_DEFAULT,\n .dns_ttl_default = 12 * 3600, /* in seconds */\n .dns_ttl_min = 180, /* reacquire when communication error */\n .dns_threads = 8,\n .poller_threads = 2,\n .handler_threads = 20,\n .compute_threads = -1\n};\n//compute_threads<=0 means auto-set by system cpu number\n~~~\n\nin which there is one timeout related configuration item: EndpointParams endpoint\\_params\n\nYou can perform operations like the following to change the global configuration before calling any of our factory functions:\n\n~~~cpp\nint main()\n{\n struct WFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;\n settings.endpoint_params.connect_timeout = 2 * 1000;\n settings.endpoint_params.response_timeout = -1;\n WORKFLOW_library_init(&settings);\n}\n~~~\n\nThe above example changes the connection timeout to 2 seconds, and the server response timeout is infinite. In this configuration, the timeout for receiving complete messages must be configured in each task, otherwise it may fall into infinite waiting. \nThe global configuration can be overridden by the configuration for an individual address in the upstream feature. For example, you can specify a connection timeout for a specific domain name. \nIn Upstream, each AddressParams also has the EndpointParams endpoint\\_params item, and you can configure it in the same way as you configure the Global item. \nFor the detailed structures, please see [upstream documents.](/docs/en/tutorial-10-upstream.md#Address)\n\n### Configuring server timeout\n\nThe [http\\_proxy](/docs/en/tutorial-05-http_proxy.md) example demonstrates the server startup configuration. In which the timeout-related configuration items include:\n\n* peer\\_response\\_timeout: its definition is the same as the global peer\\_response\\_timeout, which indicates the response timeout of the remote client, and the default value is 10 seconds.\n* receive\\_timeout: timeout for receiving a complete request. The default value is -1.\n* keep\\_alive\\_timeout: timeout for keeping a connection. The default value is 1 minute. For a Redis server, the default value is 5 minutes.\n* ssl\\_accept\\_timeout: timeout for completing SSL handshakes. The default value is 10 seconds.\n\nUnder this default configuration, the client can send one byte every 9 seconds, so that the server can always receive it and no timeout occurs. Therefore, if the service is used for public network, you need to configure receive\\_timeout.\n\n### Configuring task-level timeout\n\nTask-level timeout configuration is accomplished through calling several interfaces in a network task:\n\n~~~cpp\ntemplate \nclass WFNetworkTask : public CommRequest\n{\n...\npublic:\n /* All in milliseconds. timeout == -1 for unlimited. */\n void set_send_timeout(int timeout) { this->send_timeo = timeout; }\n void set_receive_timeout(int timeout) { this->receive_timeo = timeout; }\n void set_keep_alive(int timeout) { this->keep_alive_timeo = timeout; }\n void set_watch_timeout(int timeout) { this->watch_timeo = timeout; }\n...\n}\n~~~\n\nIn the above code, **set\\_send\\_timeout()** sets the timeout for sending a complete message, and the default value is -1. \n**set\\_receive\\_timeout()** is only valid for the client task, and it indicates the timeout for receiving a complete server reply. The default value is -1.\n\n * The receive\\_timeout of a server task is in the server startup configuration. All server tasks handled by users have successfully received complete requests.\n\n**set\\_keep\\_alive()** interface sets the timeout for keeping a connection. Generally, the framework can handle the connection maintenance well, and you do not need to call it. \nWhen an HTTP protocol is used, if a client or a server wants to use short connection, you can add an HTTP header to support it. Please do not modify it with this interface if you have other options. \nIf a Redis client wants to close the connection after a request, you need to use this interface. Obviously, **set\\_keep\\_alive()** is invalid in the callback (the connection has been reused).\n\n**set\\_watch\\_timeout()** is specific for client task only. It indicate the maximum time of waiting the first response package. This may prevent the client task from being timed out by the limit of **response\\_timeout** and **receive\\_timeout**. The framework will caculate **receive\\_timeoout** after receiving the first package if **watch\\_timeout** is set.\n\n### Timeout for synchronous task waiting \n\nThere is a very special timeout configuration, and it is the only global synchronous waiting timeout. It is not recommended, but you can get good results with it in some application scenarios. \nIn the current framework, the target server has a connection limit (you can set it in both global and upstream configurations). If the number of connections have reached the upper limit, the client task fails and returns an error by default. \nIn the callback, **task->get\\_state ()** gets WFT\\_STATE\\_SYS\\_ERROR, and **task->get\\_error()** gets EAGAIN. If the task is configured with retry, a retry will be automatically initiated. \nHere, it is allowed to configure a synchronous waiting timeout through the **task->set\\_wait\\_timeout()** interface. If a connection is released during this time period, the task can occupy this connection. \nIf you sets wait\\_timeout and does not get the connection before the timeout, the callback will get WFT\\_STATE\\_SYS\\_ERROR status and ETIMEDOUT error.\n\n~~~cpp\nclass CommRequest : public SubTask, public CommSession\n{\npublic:\n ...\n void set_wait_timeout(int wait_timeout) { this->wait_timeout = wait_timeout; }\n}\n~~~\n\n### Viewing the reasons for timeout\n\nCommunication tasks contain a **get\\_timeout\\_reason()** interface, which is used to return the timeout reason, but the reason is not very detailed. It includes the following return values:\n\n* TOR\\_NOT\\_TIMEOUT: not a timeout.\n* TOR\\_WAIT\\_TIMEOUT: timed out for synchronous waiting\n* TOR\\_CONNECT\\_TIMEOUT: connection timed out. The connections on TCP, SCTP, SSL and other protocols all use this timeout.\n* TOR\\_TRANSMIT\\_TIMEOUT: timed out for all transmissions. It is impossible to further distinguish whether it is in the sending stage or in the receiving stage. It may be refined later.\n * For a server task, if the timeout reason is TRANSMIT\\_TIMEOUT, it must be in the stage of sending replies.\n\n### Implementation of timeout functions\n\nWithin the framework, there are more types of timeouts than those we show here. Except for wait\\_timeout, all of them depend on the timer\\_fd on Linux or kqueue timer on BSD system, one for each poller thread. \nBy default, the number of poller threads is 4, which can meet the requirements of most applications. \nThe current timeout algorithm uses the data structure of linked list and red-black tree. Its time complexity is between O(1) and O(logn), where n is the fd number of the a poller thread. \nCurrently timeout processing is not the bottleneck, because the time complexity of related calls of epoll in Linux kernel is also O(logn). If the time complexity of all timeouts in our framework reaches O(1), there is no much difference.\n" }, { "path": "docs/en/about-timer.md", "content": "# About timer\n\n Timers are used to specify a certain waiting time without occupying a thread. The expiration of a timer is notified also by a callback.\n\n# Creating a timer\n\nTimer interfaces in WFTaskFactory：\n~~~cpp\nusing timer_callback_t = std::function;\n\nclass WFTaskFactory\n{\n ...\npublic:\n static WFTimerTask *create_timer_task(time_t seconds, long nanoseconds,\n timer_callback_t callback);\n\n static WFTimerTask *create_timer_task(const std::string& timer_name,\n time_t seconds, long nanoseconds,\n timer_callback_t callback);\n\n static int cancel_by_name(const std::string& timer_name)\n {\n cancel_by_name(const std::string& timer_name, (size_t)-1);\n }\n\n static int cancel_by_name(const std::string& timer_name, size_t max);\n};\n~~~\nWe specify the timing time of a timer through the seconds and nanoseconds parameters. Among them, the value range of nanoseconds is [0,1000000000). When creating a timer, a timer_name can be specified. And we may interrupt a timer by calling **cancel_by_name** with this name later. \nAs a standard workflow task, there is also a user\\_data field in the timer task that can be used to transfer some user data. Its starting method is the same as other tasks, and the procedure for adding it into the workflow is also the same.\n\n# Canceling a timer\n\nA named timer can be interrupted throught WFTaskFacotry::cancel_by_name interface, which will cancel all timers under the name by default. So we provide another cancel interface with the second argument **max** for user to cancel at most **max** timers. Each interface returns the number of timers that was actually canceled. And of course, if no timer under the name, nothing performed and returns 0.\nYou can cancel a timer right after it's created, for example:\n~~~cpp\n#include \n#include \"workflow/WFTaskFactory.h\"\n\nint main()\n{\n WFTimerTask *timer = WFTaskFactory::create_timer_task(\"test\", 10000, 0, [](WFTimerTask *){\n printf(\"timer callback, state = %d, error = %d.\\n\", task->get_state(), task->get_error());\n });\n\n WFTaskFactory::cancel_by_name(\"test\");\n\n timer->start();\n\n getchar();\n return 0;\n}\n~~~\nThis program prints 'timer callback, state = 1, error = 125.\"，immediately because the timer has be canceled before started, and it will run to callback soon after it's started. And the state code would be WFT_STATE_SYS_ERROR and the error code would be ECANCELED. \nBy the way, create named timer when and only when you may need to cancel it, because it costs more. In other scenarios just use anonymous timer. \n\n# Interrupting timer by program exit\n\nIn [About exit](/docs/en/about-exit.md), you learn that the condition that a main thread can safely end (calls **exit()** or return in the main function) is that all tasks have been run to the callback and no new task is started. \nThen, there may be a problem, if you wait for the timer to expire, it will take a long time for the program to exit. \nBut in practice, exiting the program can interrupt the timer safely and make it return to the callback. If the timer is interrupted by exiting the program, **get\\_state()** will return a WFT\\_STATE\\_ABORTED state. \nOf course, if the timer is interrupted by exiting the program, no new tasks can be started. \nThe following program demonstrates crawling one HTTP page at every one second. When all URLs are crawled, the program exits directly without waiting for the timer to return to the callback, and there will be no delay in exiting.\n\n~~~cpp\nbool program_terminate = false;\n\nvoid timer_callback(WFTimerTask *timer)\n{\n mutex.lock();\n if (!program_terminate)\n {\n WFHttpTask *task;\n if (urls_to_fetch > 0)\n {\n task = WFTaskFactory::create_http_task(...);\n series_of(timer)->push_back(task);\n }\n\n series_of(timer)->push_back(WFTaskFactory::create_timer_task(1, 0, timer_callback));\n }\n mutex.unlock();\n}\n\n...\nint main()\n{\n ....\n /* all urls done */\n mutex.lock();\n program_terminate = true;\n mutex.unlock();\n return 0;\n}\n~~~\n\nIn the above program, the timer\\_callback must check the program\\_terminate condition in the lock, otherwise a new task may be started when the program has terminated. \n" }, { "path": "docs/en/about-tlv-message.md", "content": "# About TLV (Type-Length-Value) format message\nA TLV message is a message consisting of type, length, and value. Because its format is simple and universal, and it is convenient for nesting and expansion, it is especially suitable for defining communication messages. To facilitate users to implement custom protocols, we have built-in support for TLV messages.\n# TLV message structure\nThe general TLV structure does not specify the bytes of the Type or Length field. In our protocol, they occupy 4 bytes each (network order). In other words, our message has an 8-byte message header and a Value content of no more than 32GB. We do not specify the meaning of the Type and Value fields.\n# TLVMessage class\nBecause the definition of TLV format is simple. The interfaces of this TLVMessage are very simple too.\n~~~cpp\nnamespace protocol\n{\nclass TLVMessage : public ProtocolMessage\n{\npublic:\n int get_type() const { return this->type; }\n void set_type(int type) { this->type = type; }\n\n std::string *get_value() { return &this->value; }\n void set_value(std::string value) { this->value = std::move(value); }\n\nprotected:\n int type;\n std::string value;\n\n ...\n};\n\nusing TLVRequest = TLVMessage;\nusing TLVResposne = TLVMessage;\n}\n~~~\nIf users directly use TLV messages for data transmission, they only need to use the above interfaces. Set and get Type and Value respectively. Value is directly returned as ``std::string``, which is convenient for users to move data directly through ``std::move`` when necessary.\n# An echo server/client example based on TLV message\nThe following code directly starts a server based on TLV messages, and generates a client task through the command line for interaction.\n~~~cpp\n#include \n#include \n#include \n#include \"workflow/WFGlobal.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"workflow/TLVMessage.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFServer.h\"\n\nusing namespace protocol;\n\nusing WFTLVServer = WFServer;\nusing WFTLVTask = WFNetworkTask;\nusing tlv_callback_t = std::function;\n\nWFTLVTask *create_tlv_task(const char *host, unsigned short port, tlv_callback_t callback)\n{\n auto *task = WFNetworkTaskFactory::create_client_task(\n TT_TCP, host, port, 0, std::move(callback));\n task->set_keep_alive(60 * 1000);\n return task;\n}\n\nint main()\n{\n WFTLVServer server([](WFTLVTask *task) {\n *task->get_resp() = std::move(*task->get_req());\n });\n\n if (server.start(8888) != 0) {\n perror(\"server.start\");\n exit(1);\n }\n\n auto&& create = [](WFRepeaterTask *)->SubTask * {\n std::string string;\n printf(\"Input string (Ctrl-D to exit): \");\n std::cin >> string;\n if (string.empty())\n return NULL;\n\n auto *task = create_tlv_task(\"127.0.0.1\", 8888, [](WFTLVTask *task) {\n if (task->get_state() == WFT_STATE_SUCCESS)\n printf(\"Server Response: %s\\n\", task->get_resp()->get_value()->c_str());\n else {\n const char *str = WFGlobal::get_error_string(task->get_state(), task->get_error());\n fprintf(stderr, \"Error: %s\\n\", str);\n }\n });\n\n task->get_req()->set_value(std::move(string));\n return task;\n };\n\n WFFacilities::WaitGroup wait_group(1);\n WFRepeaterTask *repeater = WFTaskFactory::create_repeater_task(std::move(create), nullptr);\n Workflow::start_series_work(repeater, [&wait_group](const SeriesWork *) {\n wait_group.done();\n });\n\n wait_group.wait();\n server.stop();\n return 0;\n}\n~~~\n# To extend TLVMessage\nIn the echo server example above, we directly use the original TLVMessage. However, it is suggested that in specific applications, users can derive TLVMessage. In the derived class, provide a richer interface to set and extract message content, avoid direct manipulation of the original Value field, and form its own secondary protocol.\nFor example, if we implement a JSON protocol, we can:\n~~~cpp\n#include \"workflow/json-parser.h\" // built-in JSON parser\n\nclass JsonMessage : public TLVMessage\n{\npublic:\n void set_json_value(const json_value_t *val)\n {\n this->type = JSON_TYPE;\n this->json_to_string(val, &this->value); // you have to implement this function\n }\n\n json_value_t *get_json_value() const\n {\n if (this->type == JSON_TYPE)\n return json_parser_parse(this->value.c_str()); // json-parser's interface\n else\n return NULL;\n }\n};\n\nusing JsonRequest = JsonMessage;\nusing JsonResponse = JsonMessage;\n\nusing JsonServer = WFServer;\n~~~\nThis example is just to illustrate the importance of derivation. In actual applications, derived classes may be far more complicated than this.\n" }, { "path": "docs/en/about-upstream.md", "content": "# About Upstream\n\nIn nginx, Upstream represents the load balancing configuration of the reverse proxy. Here, we expand the meaning of Upstream so that it has the following characteristics:\n\n1. Each Upstream is an independent reverse proxy\n2. Accessing an Upstream is equivalent to using an appropriate strategy to select one in a group of services/targets/upstream and downstream for access\n3. Upstream has load balancing, error handling, circuit breaker and other service governance capabilities\n4. For multiple retries of the same request, Upstream can avoid addresses that already tried\n5. Different connection parameters can be configured for different addresses through Upstream\n6. Dynamically adding/removing target will take effect in real time, which is convenient for any service discovery system\n\n### Advantages of Upstream over domain name DNS resolution\n\nBoth Upstream and domain name DNS resolution can configure a group of ip to a host, but\n\n1. DNS domain name resolution doesn’t address port number. The service DNS domain names with the same IP and different ports cannot be configured together; but it is possible for Upstream.\n2. The set of addresses corresponding to DNS domain name resolution must be ip; while the set of addresses corresponding to Upstream can be ip, domain name or unix-domain-socket\n3. Normally, DNS domain name resolution will be cached by operating system or DNS server on the network, and the update time is limited by ttl; while Upstream can be updated in real time and take effect in real time\n4. The consumption of DNS domain name is much greater than that of Upstream resolution and selection\n\n### Upstream of Workflow\n\nThis is a local reverse proxy module, and the proxy configuration is effective for both server and client.\n\nSupport dynamic configuration and available for any service discovery system. Currently, [workflow-k8s](https://github.com/sogou/workflow-k8s) can be used to acquire Pods information from the API server of Kubernetes.\n\nUpstream name does not include port, but upstream request supports specified port. (However, for non-built-in protocols, Upstream name temporarily needs to be added with the port to ensure parsing during construction).\n\nEach Upstream is configured with its own independent name UpstreamName, and a set of Addresses is added and set. These Addresses can be:\n\n1. ip4\n2. ip6\n3. Domain name\n4. unix-domain-socket\n\n### Why to replace nginx's Upstream\n\n#### Upstream working mode of nginx\n\n1. Supports http/https protocol only\n\n2. Needs to build a nginx service, start the start process occupies socket and other resources\n\n3. The request is sent to nginx first, and nginx forwards the request to remote end, which will increase one more network communication overhead\n\n#### Local Upstream working method of workflow\n\n1. Protocol irrelevant, you can even access mysql, redis, mongodb, etc. through upstream\n\n2. You can directly simulate the function of reverse proxy in the process, no need to start other processes or ports \n\n3. The selection process is basic calculation and table lookup, no additional network communication overhead\n\n# Use Upstream\n\n### Common interfaces\n\n~~~cpp\nclass UpstreamManager\n{\npublic:\n static int upstream_create_consistent_hash(const std::string& name,\n upstream_route_t consitent_hash);\n static int upstream_create_weighted_random(const std::string& name,\n bool try_another);\n static int upstream_create_manual(const std::string& name,\n upstream_route_t select,\n bool try_another,\n upstream_route_t consitent_hash);\n static int upstream_delete(const std::string& name);\n\npublic:\n static int upstream_add_server(const std::string& name,\n const std::string& address);\n static int upstream_add_server(const std::string& name,\n const std::string& address,\n const struct AddressParams *address_params);\n static int upstream_remove_server(const std::string& name,\n const std::string& address);\n ...\n}\n~~~\n\n### Example 1 Random access in multiple targets\n\nConfigure a local reverse proxy to evenly send all the local requests for **my_proxy.name** to 6 target servers\n\n~~~cpp\nUpstreamManager::upstream_create_weighted_random(\n \"my_proxy.name\",\n true); // In case of fusing, retry till the available is found or all fuses are blown\n\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.2.100:8081\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.2.100:8082\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.10.10\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"test.sogou.com:8080\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"abc.sogou.com\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"abc.sogou.com\");\nUpstreamManager::upstream_add_server(\"my_proxy.name\", \"/dev/unix_domain_scoket_sample\");\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://my_proxy.name/somepath?a=10\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n\nBasic principles\n\n1. Select a target randomly\n2. If try_another is configured as true, one of all surviving targets will be selected randomly\n3. Select in the main servers only, the mains and backups of the group where the selected target is located and the backup without group are regarded as valid optional objects\n\n### Example 2 Random access among multiple targets based on weights\n\nConfigure a local reverse proxy, send all **weighted.random** requests to the 3 target servers based on the weight distribution of 5/20/1\n\n~~~cpp\nUpstreamManager::upstream_create_weighted_random(\n \"weighted.random\",\n false); // If you don’t retry in case of fusing, the request will surely fail\n\nAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\naddress_params.weight = 5; //weight is 5\nUpstreamManager::upstream_add_server(\"weighted.random\", \"192.168.2.100:8081\", &address_params); // weight is 5\naddress_params.weight = 20; // weight is 20\nUpstreamManager::upstream_add_server(\"weighted.random\", \"192.168.2.100:8082\", &address_params); // weight is 20\nUpstreamManager::upstream_add_server(\"weighted.random\", \"abc.sogou.com\"); // weight is 1\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://weighted.random:9090\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n\nBasic principles\n\n1. According to the weight distribution, randomly select a target, the greater the weight is, the greater the probability is\n2. If try_another is configured as true, one of all surviving targets will be selected randomly as per weights.\n3. Select in the main servers only, the main and backup of the group where the selected target is located and the backup without group are regarded as valid optional objects\n\n### Example 3 Access among multiple targets based on the framework's default consistent hash\n\n~~~cpp\nUpstreamManager::upstream_create_consistent_hash(\n \"abc.local\",\n nullptr); // nullptr represents using the default consistent hash function of the framework\n\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8081\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8082\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.10.10\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"test.sogou.com:8080\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"abc.sogou.com\");\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://abc.local/service/method\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n\nBasic principles\n\n1. Each main server is regarded as 16 virtual nodes\n2. The framework will use std::hash to calculate \"the address + virtual index of all nodes + the number of times for this address to add into this Upstream\" as the node value of the consistent hash\n3. The framework will use std::hash to calculate path + query + fragment as a consistent hash data value\n4. Choose the value nearest to the surviving node as the target each time\n5. For each main, as long as there is a main in surviving group, or there is a backup in surviving group, or there is a surviving no group backup, it is regarded as surviving\n6. If weight on AddressParams is set with upstream_add_server(), each main server is regarded as 16 * weight virtual nodes. This is suitable for weighted consistent hash or shrinking the standard deviation of consistent hash\n\n### Example 4 User-defined consistent hash function\n\n~~~cpp\nUpstreamManager::upstream_create_consistent_hash(\n \"abc.local\",\n [](const char *path, const char *query, const char *fragment) -> unsigned int {\n unsigned int hash = 0;\n\n while (*path)\n hash = (hash * 131) + (*path++);\n\n while (*query)\n hash = (hash * 131) + (*query++);\n\n while (*fragment)\n hash = (hash * 131) + (*fragment++);\n\n return hash;\n });\n\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8081\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8082\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.10.10\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"test.sogou.com:8080\");\nUpstreamManager::upstream_add_server(\"abc.local\", \"abc.sogou.com\");\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://abc.local/sompath?a=1#flag100\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n\nBasic principles\n\n1. The framework will use a user-defined consistent hash function as the data value\n2. The rest is the same as the above principles\n\n### Example 5 User-defined selection strategy\n\n~~~cpp\nUpstreamManager::upstream_create_manual(\n \"xyz.cdn\",\n [](const char *path, const char *query, const char *fragment) -> unsigned int {\n return atoi(fragment);\n },\n true, // If a blown target is selected, a second selection will be made\n nullptr); // nullptr represents using the default consistent hash function of the framework in the second selection\n\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"192.168.2.100:8081\");\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"192.168.2.100:8082\");\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"192.168.10.10\");\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"test.sogou.com:8080\");\nUpstreamManager::upstream_add_server(\"xyz.cdn\", \"abc.sogou.com\");\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://xyz.cdn/sompath?key=somename#3\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n\nBasic principles\n\n1. The framework first determines the selection in the main server list according to the normal selection function provided by the user and then get the modulo\n2. For each main server, as long as there is a main server in surviving group, or there is a backup in surviving group, or there is a surviving no group backup, it is regarded as surviving\n3. If the selected target no longer survives and try_another is set as true, a second selection will be made using consistent hash function\n4. If the second selection is triggered, the consistent hash will ensure that a survival target will be selected, unless all machines are blown\n\n### Example 6 Simple main-backup mode\n~~~cpp\nUpstreamManager::upstream_create_weighted_random(\n \"simple.name\",\n true);//One main, one backup, nothing is different in this item\n\nAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\naddress_params.server_type = 0; /* 1 for main server */\nUpstreamManager::upstream_add_server(\"simple.name\", \"main01.test.ted.bj.sogou\", &address_params); // main\naddress_params.server_type = 1; /* 0 for backup server */\nUpstreamManager::upstream_add_server(\"simple.name\", \"backup01.test.ted.gd.sogou\", &address_params); //backup\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://simple.name/request\", 0, 0, nullptr);\nauto *redis_task = WFTaskFactory::create_redis_task(\"redis://simple.name/2\", 0, nullptr);\nredis_task->get_req()->set_query(\"MGET\", {\"key1\", \"key2\", \"key3\", \"key4\"});\n(*http_task * redis_task).start();\n~~~\n\nBasic principles\n1. The main-backup mode does not conflict with any of the modes shown above, and it can take effect at the same time\n2. The number of main/backup is independent of each other and there is no limit. All main servers are coequal to each other, and all backup servers are coequal to each others, but main and backup are not coequal to each other.\n3. As long as a main server is alive, the request will always use a main server.\n4. If all main servers are blown, backup server will take over the request as a substitute target until any main server works well again\n5. In every strategy, surviving backup can be used as the basis for the survival of main\n\n### Example 7 Main-backup + consistent hash + grouping\n~~~cpp\nUpstreamManager::upstream_create_consistent_hash(\n \"abc.local\",\n nullptr);//nullptr represents using the default consistent hash function of the framework \n\nAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\naddress_params.server_type = 0;\naddress_params.group_id = 1001;\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8081\", &address_params);//main in group 1001\naddress_params.server_type = 1;\naddress_params.group_id = 1001;\nUpstreamManager::upstream_add_server(\"abc.local\", \"192.168.2.100:8082\", &address_params);//backup for group 1001\naddress_params.server_type = 0;\naddress_params.group_id = 1002;\nUpstreamManager::upstream_add_server(\"abc.local\", \"backup01.test.ted.bj.sogou\", &address_params);//main in group 1002\naddress_params.server_type = 1;\naddress_params.group_id = 1002;\nUpstreamManager::upstream_add_server(\"abc.local\", \"backup01.test.ted.gd.sogou\", &address_params);//backup for group 1002\naddress_params.server_type = 1;\naddress_params.group_id = -1;\nUpstreamManager::upstream_add_server(\"abc.local\", \"test.sogou.com:8080\", &address_params);//backup with no group mean backup for all groups and no group\nUpstreamManager::upstream_add_server(\"abc.local\", \"abc.sogou.com\");//main, no group\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://abc.local/service/method\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n\nBasic principles\n\n1. Group number -1 means no group, this kind of target does not belong to any group\n2. The main servers without a group are coequal to each other, and they can even be regarded as one group. But they are isolated from the other main servers with a group\n3. A backup without a group can serve as a backup for any group target of Global/any target without a group\n4. The group number can identify which main and backup are working together\n5. The backups of different groups are isolated from each other, and they serve the main servers of their own group only\n6. Add the default group number -1 of the target, and the type is main\n\n### Example 8 NVSWRR selection weighting strategy\n~~~cpp\nUpstreamManager::upstream_create_vnswrr(\"nvswrr.random\");\n\nAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\naddress_params.weight = 3;//weight is 3\nUpstreamManager::upstream_add_server(\"nvswrr.random\", \"192.168.2.100:8081\", &address_params);//weight is 3\naddress_params.weight = 2;//weight is 2\nUpstreamManager::upstream_add_server(\"nvswrr.random\", \"192.168.2.100:8082\", &address_params);//weight is 2\nUpstreamManager::upstream_add_server(\"nvswrr.random\", \"abc.sogou.com\");//weight is 1\n\nauto *http_task = WFTaskFactory::create_http_task(\"http://nvswrr.random:9090\", 0, 0, nullptr);\nhttp_task->start();\n~~~\n1. The virtual node initialization sequence is selected according to the [SWRR algorithm](https://github.com/nginx/nginx/commit/52327e0627f49dbda1e8db695e63a4b0af4448b1)\n2. The virtual nodes are initialized in batches during operation to avoid intensive computing concentration. After each batch of virtual nodes is used up, the next batch of virtual node lists can be initialized.\n3. It has both the smooth and scattered characteristics of [SWRR algorithm](https://github.com/nginx/nginx/commit/52327e0627f49dbda1e8db695e63a4b0af4448b1) and the time complexity of O(1)\n4. For specific details of the algorithm, see tengine(https://github.com/alibaba/tengine/pull/1306)\n\n# Upstream selection strategy\n\nWhen the URIHost of the url that initiates the request is filled with UpstreamName, it is regarded as a request to the Upstream corresponding to the name, and then it will be selected from the set of Addresses recorded by the Upstream:\n\n1. Weight random strategy: selection randomly according to weight\n2. Consistent hash strategy: The framework uses a standard consistent hashing algorithm, and users can define the consistent hash function consistent_hash for the requested uri\n3. Manual strategy: make definite selection according to the select function that user provided for the requested uri, if the blown target is selected: **a.** If try_another is false, this request will return to failure **b.** If try_another is true, the framework uses standard consistent hash algorithm to make a second selection, and the user can define the consistent hash function consistent_hash for the requested uri\n4. Main-backup strategy: According to the priority of main first, backup next, select a main server as long as it can be used. This strategy can take effect concurrently with any of [1], [2], and [3], and they influence each other.\n\nRound-robin/weighted-round-robin: regarded as equivalent to [1], not available for now\n\nThe framework recommends common users to use strategy [2], which can ensure that the cluster has good fault tolerance and scalability\n\nFor complex scenarios, advanced users can use strategy [3] to customize complex selection logic\n\n# Address attribute\n~~~cpp\nstruct EndpointParams\n{\n size_t max_connections;\n int connect_timeout;\n int response_timeout;\n int ssl_connect_timeout;\n bool use_tls_sni;\n};\n\nstatic constexpr struct EndpointParams ENDPOINT_PARAMS_DEFAULT =\n{\n .max_connections = 200,\n .connect_timeout = 10 * 1000,\n .response_timeout = 10 * 1000,\n .ssl_connect_timeout = 10 * 1000,\n .use_tls_sni = false,\n};\n\nstruct AddressParams\n{\n struct EndpointParams endpoint_params;\n unsigned int dns_ttl_default;\n unsigned int dns_ttl_min;\n unsigned int max_fails;\n unsigned short weight;\n int server_type; /* 0 for main and 1 for backup. */\n int group_id;\n};\n\nstatic constexpr struct AddressParams ADDRESS_PARAMS_DEFAULT =\n{\n .endpoint_params = ENDPOINT_PARAMS_DEFAULT,\n .dns_ttl_default = 12 * 3600,\n .dns_ttl_min = 180,\n .max_fails = 200,\n .weight = 1, // only for main of UPSTREAM_WEIGHTED_RANDOM\n .server_type = 0,\n .group_id = -1,\n};\n~~~\n\nEach address can be configured with custom parameters:\n\n * Max_connections, connect_timeout, response_timeout, ssl_connect_timeout of EndpointParams: connection-related parameters\n * dns_ttl_default: The default ttl in the dns cache in seconds, and the default value is 12 hours. The dns cache is for the current process, that is, the process will disappear after exiting, and the configuration is only valid for the current process\n * dns_ttl_min: The shortest effective time of dns in seconds, and the default value is 3 minutes. It is used to decide whether to perform dns again when communication fails and retry.\n * max_fails: the number of [continuous] failures that triggered fusing (Note: each time the communication is successful, the count will be cleared)\n * Weight: weight, the default value is 1, which is only valid for main. It is used for Upstream weighted random strategy selection and consistent hash strategy selection, the larger the weight is, the easier it is to be selected.\n * server_type: main/backup configuration, main by default (server_type=0). At any time, the main servers in the same group are always at higher priority than backups\n * group_id: basis for grouping, the default value is -1. -1 means no grouping (free). A free backup can be regarded as backup to any main server. Any backup with group is always at higher priority than any free backup.\n\n# About fuse\n\n## MTTR\nMean time to repair (MTTR) is the average value of the repair time when the product changes from a fault state to a working state.\n\n## Service avalanche effect\n\nService avalanche effect is a phenomenon in which \"service caller failure\" (result) is caused by \"service provider's failure\" (cause), and the unavailability is amplified gradually/level by level\n\nIf it is not controlled effectively, the effect will not converge, but will be amplified geometrically, just like an avalanche, that’s why it is called avalanche effect\n\nDescription of the phenomenon: at first it is just a small service or module abnormality/timeout, causing abnormality/timeout of other downstream dependent services, then causing a chain reaction, eventually leading to paralysis of most or all services\n\nAs the fault is repaired, the effect will disappear, so the duration of the effect is usually equal to MTTR\n\n## Fuse mechanism\n\nWhen the error or abnormal touch of a certain target meets the preset threshold condition, the target is temporarily considered unavailable, and the target is removed, namely fuse is started and enters the fuse period\n\nAfter the fuse duration reaches MTTR duration, turn into half-open status, (attempt to) restore the target\n\nIf all targets are found fused whenever recovering one target, all targets will be restored at the same time\n\nFuse mechanism strategy can effectively prevent avalanche effect\n\n## Upstream fuse protection mechanism\n\nMTTR=30 seconds, which is temporarily not configurable, but we will consider opening it to be configured by users in the future.\n\nWhen the number of consecutive failures of a certain Address reaches the set upper limit (200 times by default), this Address will be blown, MTTR=30 seconds.\n\nDuring the fusing period, once the Address is selected by the strategy, Upstream will decide whether to try other Addresses and how to try according to the specific configuration\n\nPlease note that if one of the following 1-4 scenarios is met, the communication task will get an error of WFT_ERR_UPSTREAM_UNAVAILABLE = 1004:\n\n 1. Weight random strategy, all targets are in the fusing period\n 2. Consistent hash strategy, all targets are in the fusing period\n 3. Manual strategy and try_another==true, all targets are in the fusing period\n 4. Manual strategy and try_another==false, and all the following three conditions shall meet at the same time:\n\n 1). The main selected by the select function is in the fusing period, and all free devices are in the fusing period\n\n 2). The main is a free main, or other targets in the group where the main is located are all in the fusing period\n\n 3). All free devices are in the fusing period\n \n# Upstream port priority\n\n1. Priority is given to the port number explicitly configured on the Upstream Address\n\n2. If not, select the port number explicitly configured in the request url\n\n3. If none, use the default port number of the protocol\n\n~~~cpp\nConfigure UpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.2.100:8081\");\nRequest http://my_proxy.name:456/test.html => http://192.168.2.100:8081/test.html\nRequest http://my_proxy.name/test.html => http://192.168.2.100:8081/test.html\n~~~\n\n~~~cpp\nConfigure UpstreamManager::upstream_add_server(\"my_proxy.name\", \"192.168.10.10\");\nRequest http://my_proxy.name:456/test.html => http://192.168.10.10:456/test.html\nRequest http://my_proxy.name/test.html => http://192.168.10.10:80/test.html\n~~~\n" }, { "path": "docs/en/tutorial-01-wget.md", "content": "# Creating your first task: wget\n\n# Sample code\n\n[tutorial-01-wget.cc](/tutorial/tutorial-01-wget.cc)\n\n# About wget\n\nwget reads HTTP/HTTPS URLs from stdin, crawls the webpages and then print the content to stdout. It also outputs the HTTP headers of the request and the response to stderr. \nFor convenience, wget exits with Ctrl-C, but it will ensure that all resources are completely released first.\n\n# Creating and starting an HTTP task\n\n~~~cpp\nWFHttpTask *task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX, wget_callback);\nprotocol::HttpRequest *req = task->get_req();\nreq->add_header_pair(\"Accept\", \"*/*\");\nreq->add_header_pair(\"User-Agent\", \"Wget/1.14 (gnu-linux)\");\nreq->add_header_pair(\"Connection\", \"close\");\ntask->start();\npause();\n~~~\n\n**WFTaskFactory::create\\_http\\_task()** generates an HTTP task. In [WFTaskFactory.h](/src/factory/WFTaskFactory.h), the prototype is defined as follows:\n\n~~~cpp\nWFHttpTask *create_http_task(const std::string& url,\n int redirect_max, int retry_max,\n http_callback_t callback);\n~~~\n\nThe first few parameters are self-explanatory. **http\\_callback\\_t** is the callback of an HTTP task, which is defined below:\n\n~~~cpp\nusing http_callback_t = std::function;\n~~~\n\nTo put it simply, it’s the funtion that has **Task** as one parameter and does not return any value. You can pass NULL to this callback, indicating that there is no callback. The callback in all tasks follows the same rule. \nPlease note that all factory functions do not return failure, so even if the URL is illegal, don't worry that the task is a null pointer. All errors are handled in the callback. \nYou can use **task->get\\_req()** to get the request of the task. The default method is GET via HTTP/1.1 on long connections. The framework automatically adds request\\_uri, Host and other parameters. The framework will add other HTTP header fields automatically according to the actual requirements, including Content-Length or Connection before sending the request. You may also use **add\\_header\\_pair()** to add your own header. For more interfaces on HTTP messages, please see [HttpMessage.h](/src/protocol/HttpMessage.h). \n**task->start()** starts the task. It’s non-blocking and will not fail. Then the callback of the task will be called. As it’s an asynchronous task, obviously you cannot use the task pointer after **start()**. \nTo make the example as simple as possible, call **pause()** after **start()** to prevent the program from exiting. You can press Ctrl-C to exit the program.\n\n# Handling crawled HTTP results\n\nThis example demonstrates how to handle the results with a general function. Of course, **std::function** supports more features.\n\n~~~cpp\nvoid wget_callback(WFHttpTask *task)\n{\n protocol::HttpRequest *req = task->get_req();\n protocol::HttpResponse *resp = task->get_resp();\n int state = task->get_state();\n int error = task->get_error();\n\n // handle error states\n ...\n\n std::string name;\n std::string value;\n // print request to stderr\n fprintf(stderr, \"%s %s %s\\r\\n\", req->get_method(), req->get_http_version(), req->get_request_uri());\n protocol::HttpHeaderCursor req_cursor(req);\n while (req_cursor.next(name, value))\n fprintf(stderr, \"%s: %s\\r\\n\", name.c_str(), value.c_str());\n fprintf(stderr, \"\\r\\n\");\n \n // print response header to stderr\n ...\n\n // print response body to stdin\n void *body;\n size_t body_len;\n resp->get_parsed_body(&body, &body_len); // always success.\n fwrite(body, 1, body_len, stdout);\n fflush(stdout);\n}\n~~~\n\nIn this callback, the task is generated by the factory. \nYou can use **task->get\\_state()** and **task->get\\_error()** to obtain the running status and the error code of the task respectively. Let's skip the error handling first. \nUse **task->get\\_resp()** to get the response of the task, which is slightly different from the request, as they are both derived from HttpMessage. \nThen, use the HttpHeaderCursor to scan the headers of the request and the response. [HttpUtil.h](/src/protocol/HttpUtil.h) contains the definition of the Cursor.\n\n~~~cpp\nclass HttpHeaderCursor\n{\npublic:\n HttpHeaderCursor(const HttpMessage *message);\n ...\n void rewind();\n ...\n bool next(std::string& name, std::string& value);\n bool find(const std::string& name, std::string& value);\n ...\n};\n~~~\n\nThere should be no doubt about the use of this cursor. \nThe next line **resp->get\\_parsed\\_body()** obtains the HTTP body of the response. This call always returns true when the task is successful, and the body points to the data area. \nThe call gets the raw HTTP body, and does not decode the chunk. If you want to decode the chunk, you can use the HttpChunkCursor in [HttpUtil.h](/src/protocol/HttpUtil.h). \nIn addition, **find()** will change the pointer inside the cursor. If you want to iterate over the header after you use **find()**, please use **rewind()** to return to the cursor header.\n" }, { "path": "docs/en/tutorial-02-redis_cli.md", "content": "# Implementing Redis set and get: redis\\_cli\n\n# Sample code\n\n[tutorial-02-redis\\_cli.cc](/tutorial/tutorial-02-redis_cli.cc)\n\n# About redis\\_cli\n\nThe program reads the Redis server address and a key/value pair from the command line. Then execute SET to write this KV pair and then read them to verify that the writing is sucessful. \nCommand: ./redis_cli \\ \\ \\ \nFor the sake of simplicity, press Ctrl-C to exit the program.\n\n# Format of Redis URL\n\nredis://:password@host:port/dbnum?query#fragment \nIf SSL is used, use: \nrediss://:password@host:port/dbnum?query#fragment \npassword is optional. The default port is 6379; the default dbnum is 0, and its range is from 0 to 15. \nquery and fragment are not used in the factory and you can define them by yourself. For example, if you want to use upstream selection , you can define your own query and fragment. For relevant details, please see upstream documents. \nSample Redis URL: \nredis://127.0.0.1/ \nredis://:12345678@redis.some-host.com/1\n\n# Creating and starting a Redis task\n\nCreating a Redis task is almost the same as creating an HTTP task. The only difference is the omission of redirect\\_max.\n\n~~~cpp\nusing redis_callback_t = std::function;\n\nWFRedisTask *create_redis_task(const std::string& url,\n int retry_max,\n redis_callback_t callback);\n~~~\n\nIn this example, we want to store some user data in the Redis task, including URL and key, and use them in the callback. \nWe can use **std::function** to bind the parameters. Here we use **void \\*user\\_data** pointer in the task. The pointer is a public member of the task.\n\n~~~cpp\nstruct tutorial_task_data\n{\n std::sring url;\n std::string key;\n};\n...\nstruct tutorial_task_data data;\ndata.url = argv[1];\ndata.key = argv[2];\n\nWFRedisTask *task = WFTaskFactory::create_redis_task(data.url, RETRY_MAX, redis_callback);\n\nprotocol::RedisRequest *req = task->get_req();\nreq->set_request(\"SET\", { data.key, argv[3] });\n\ntask->user_data = &data;\ntask->start();\npause();\n~~~\n\nSimilar to **get\\_req()** in an HTTP task, **get\\_req()** in an Redis task returns the Redis request for that task.\nYou can see the functions of RedisRequest in [RedisMessage.h](/src/protocol/RedisMessage.h), where **set\\_request** is used to set Redis command.\n\n~~~cpp\nvoid set_request(const std::string& command, const std::vector& params);\n~~~\n\nThere is little doubt about this interface for people who frequently use Redis. However, please note that you cannot use SELECT and AUTH commands in the request. \nThe reason is that as you can't specify the connection every time you send a request and the next request after SELECT may not be initiated on the same connection, this command is meaningless. \nPlease specify the database name and password in the Redis URL. And the URL of every request must contain these data. \nIn addition, this redis client fully supports redis cluster mode. The client will process MOVED and ASK response, and redirect correctly. \n\n# Handling results\n\nAfter you successfully run the SET command, send the GET command to verify the writing. GET also uses the same callback. Therefore, the function will determine the source command of the results. \nLet's skip the error handling first again.\n\n~~~cpp\nvoid redis_callback(WFRedisTask *task)\n{\n protocol::RedisRequest *req = task->get_req();\n protocol::RedisResponse *resp = task->get_resp();\n int state = task->get_state();\n int error = task->get_error();\n protocol::RedisValue val;\n\n ...\n resp->get_result(val);\n std::string cmd;\n req->get_command(cmd);\n if (cmd == \"SET\")\n {\n tutorial_task_data *data = (tutorial_task_data *)task->user_data;\n WFRedisTask *next = WFTaskFactory::create_redis_task(data->url, RETRY_MAX, redis_callback);\n\n next->get_req()->set_request(\"GET\", { data->key });\n series_of(task)->push_back(next);\n fprintf(stderr, \"Redis SET request success. Trying to GET...\\n\");\n }\n else /* if (cmd == 'GET') */\n {\n // print the GET result\n ...\n fprintf(stderr, \"Finished. Press Ctrl-C to exit.\\n\");\n }\n}\n~~~\n\nRedisValue is the results of one Redis request. You can also see the interface in [RedisMessage.h](/src/protocol/RedisMessage.h). \nYou need to pay special attention to the callback in the line **series\\_of(task)->push\\_back(next)**. It`s the firt time we use the functions of Workflow. \nHere **next** means the Redis task we are about to start: run GET operation. We do not use **next->start()** to start the task. We use **push\\_back** to append the next task to the end of the current task queue instead. \nThe difference between the two methods is:\n\n* When a task is initiated by **start**, the task is started immediately; when a task is **push\\_back** to the queue, the **next** task is initiated after the callback.\n * The obvious advantage is that the **push\\_back** method can ensure that the log printing is not chaotic. Otherwise, if you use the **next->start()**, the \\\"Finished.\\\" in the sample may be printed out first.\n* If you use **start** to initiate the next task, the current task series ends and the next task will initiate a new series.\n * You can set a callback for a series. For the sake of simplicity, the sample omit it.\n * In the parallel tasks, a series is a branch of the parallel task. If the series ends, it is considered that the brand also ends. The following tutorials demonstrates how to use parallel tasks.\n\nIn a word, if you want to start the next task after one task, you usually use **push\\_back** operation (in some cases, **push\\_front** may be used). \n**series\\_of()** is a very important call and it is a global function that does not belong to any class. [Workflow.h](/src/factory/Workflow.h#L140) contains its definition and implementation.\n\n~~~cpp\nstatic inline SeriesWork *series_of(const SubTask *task)\n{\n return (SeriesWork *)task->get_pointer();\n}\n~~~\n\nAll tasks are derived from SubTask. And any running task must belong to one series. You can call **series\\_of()** to get the series of a task. \n**push\\_back** is a function in the SeriesWork class, which is used to append a task to the end of the series. **push\\_front** is a similar function. In the sample, you can use either function.\n\n~~~cpp\nclass SeriesWork\n{\n ...\npublic:\n void push_back(SubTask *task);\n void push_front(SubTask *task);\n ...\n}\n~~~\n\nSeriesWork class plays an important role in our system. In the next tutorial, you will learn more functions in SeriesWork.\n" }, { "path": "docs/en/tutorial-03-wget_to_redis.md", "content": "# More features about series: wget\\_to\\_redis\n\n# Sample code\n\n[tutorial-03-wget\\_to\\_redis.cc](/tutorial/tutorial-03-wget_to_redis.cc)\n\n# About wget\\_to\\_redis\n\nThe program reads one HTTP URL and one redis URL from the command line, crawls the HTTP web page and saves the content to Redis, with the key as the HTTP URL. \nDiffering from the other two examples, we add a wake-up mechanism. The program can automatically exit and users are not required to press Ctrl-C.\n\n# Creating and configuring an HTTP task\n\nSimilar to the previous example, in this example, we also executes two requests in series. The biggest difference is that we inform the main thread that the execution of the task has finished and quit normally. \nIn addition, we add two more calls to limit the size of the crawled HTTP response content and the maximum time to receive the reply.\n\n~~~cpp\nWFHttpTask *http_task = WFTaskFactory::create_http_task(...);\n...\nhttp_task->get_resp()->set_size_limit(20 * 1024 * 1024);\nhttp_task->set_receive_timeout(30 * 1000);\n~~~\n\n**set\\_size\\_limit()** is a function in HttpMessage.It is used to limit the packet size of incoming HTTP message. Actually this interface is required in all protocol messages. \n**set\\_receive\\_timeout()** sets the timeout for receiving data, in milliseconds. \nThe above code limits the size of the HTTP message to no more than 20M and the time for receiving the complete message to no more than 30 seconds. You can learn more about timeout configuration in the following documents.\n\n# Creating and starting a SeriesWork\n\nIn the previous two examples, we call **task->start()** directly to start the first task. The actual procedure in **task->start()** is: \ncreate a SeriesWork with the task as the head and then start the series. In [WFTask.h](/src/factory/WFTask.h), you can see the implemetation of **start**.\n\n~~~cpp\ntemplate\nclass WFNetWorkTask : public CommRequest\n{\npublic:\n void start()\n {\n assert(!series_of(this));\n Workflow::start_series_work(this, nullptr);\n }\n ...\n};\n~~~\n\nWe want to set a callback for that series and add some context. Therefore, instead of using the **start** interface of the task, we create our own series. \nYou cannot new, delete or inherit a SeriesWork. It can only be generated through **Workflow::create\\_series\\_work()** interface. In [Workflow.h](/src/factory/Workflow.h), \ngenerally we use the following call:\n\n~~~cpp\nusing series_callback_t = std::function;\n\nclass Workflow\n{\npublic:\n static SeriesWork *create_series_work(SubTask *first, series_callback_t callback);\n};\n~~~\n\nIn the sample code, our usage is as follows:\n\n~~~cpp\nstruct tutorial_series_context\n{\n std::string http_url;\n std::string redis_url;\n size_t body_len;\n bool success;\n};\n...\nstruct tutorial_series_context context;\n...\nSeriesWork *series = Workflow::create_series_work(http_task, series_callback);\nseries->set_context(&context);\nseries->start();\n~~~\n\nIn the previous example, we use the pointer **void \\*user\\_data** in the task to save the context. However, in this example, we put the context in the series, which is more reasonable. The series is a complete task chain, and all tasks can obtain and modify the context. \nThe callback function of the series is called after all the tasks in that series are finished. Here, we simply use a lamda function to print the running results and wake up the main thread.\n\n# Other work\n\nThere's nothing special left. After the HTTP crawling is successful, a Redis task is started to write the data into the database. If the crawling fails or the length of the HTTP body is 0, the Redis task will not be started. \nIn any case, the program can exit normally after all tasks are finished, because all tasks are in the same series.\n" }, { "path": "docs/en/tutorial-04-http_echo_server.md", "content": "# First server: http\\_echo\\_server\n\n# Sample code\n\n[tutorial-04-http\\_echo\\_server.cc](/tutorial/tutorial-04-http_echo_server.cc)\n\n# About http\\_echo\\_server\n\nIt is an HTTP server that returns an HTML page, which displays the header data in the HTTP request sent by the browser. \nThe log of the program contains the client address and the sequence of the request (the number of requests on the current connection). When 10 requests are completed on the same connection, the server actively closes the connection. \nThe program exits normally after users press Ctrl-C, and all resources are completely reclaimed.\n\n# Creating and starting an HTTP server\n\nIn this example, we use the default parameters of an HTTP server. It is very simple to create and start an HTTP server.\n\n~~~cpp\nWFHttpServer server(process);\nport = atoi(argv[1]);\nif (server.start(port) == 0)\n{\n pause();\n server.stop();\n}\n...\n~~~\n\nThe procedure is too simple to explain. Please note that the start process is non-blocking, so please pause the program. Obviously you can start several server objects and then pause. \nAfter a server is started, you can use **stop()** interface to shut down the server at any time. Stopping a server is non-violent and will be done until all the processing requests in the server are completed. \nTherefore, **stop** is a blocking operation. If non-blocking shutdown is required, please use **shutdown+wait\\_finish** interface. \nThere are several overloaded functions with **start()**. [WFServer.h](/src/server/WFServer.h) contains the following interfaces:\n\n~~~cpp\nclass WFServerBase\n{\npublic:\n /* To start TCP server. */\n int start(unsigned short port);\n int start(int family, unsigned short port);\n int start(const char *host, unsigned short port);\n int start(int family, const char *host, unsigned short port);\n int start(const struct sockaddr *bind_addr, socklen_t addrlen);\n\n /* To start an SSL server */\n int start(unsigned short port, const char *cert_file, const char *key_file);\n int start(int family, unsigned short port,\n const char *cert_file, const char *key_file);\n int start(const char *host, unsigned short port,\n const char *cert_file, const char *key_file);\n int start(int family, const char *host, unsigned short port,\n const char *cert_file, const char *key_file);\n int start(const struct sockaddr *bind_addr, socklen_t addrlen,\n const char *cert_file, const char *key_file); \n \n /* For graceful restart or multi-process server. */\n int serve(int listen_fd);\n int serve(int listen_fd, const char *cert_file, const char *key_file);\n\n /* Get the listening address. Used when started a server on a random port. */\n int get_listen_addr(struct sockaddr *addr, socklen_t *addrlen) const;\n};\n~~~\nThere interfaces are easy to understand. If the **port** number is zero, the server will be started on a random port, and you may need to call **get_listen_addr** to abtain the actual listening address (mainly for the actual port) after the server is started. \nWhen you start an SSL server, the cert\\_file and key\\_file should be in PEM format. \nThe last two **serve()** interfaces have the parameter **listen\\_fd**, which is used for graceful restart or for building a simple non-TCP (such as SCTP) server. \nPlease note that one server object corresponds to one **listen\\_fd**. If the server is running on both IPv4 and IPv6 protocols, you should:\n\n~~~cpp\n{\n WFHttpServer server_v4(process);\n WFHttpServer server_v6(process);\n server_v4.start(AF_INET, port);\n server_v6.start(AF_INET6, port);\n ...\n // now stop...\n server_v4.shutdown(); /* shutdown() is nonblocking */\n server_v6.shutdown();\n server_v4.wait_finish();\n server_v6.wait_finish();\n}\n~~~\n\nIn the above code, the two servers cannot share the connection counter. Therefore, it is recommended to start the IPv6 server only, because the IPv6 server can accept IPv4 connection.\n\n# Business logic of an HTTP echo server\n\nWhen you build an HTTP server, you pass a process parameter, which is also an **std::function**, as defined below:\n\n~~~cpp\nusing http_process_t = std::function;\nusing WFHttpServer = WFServer;\n\ntemplate<>\nWFHttpServer::WFServer(http_process_t proc) :\n WFServerBase(&HTTP_SERVER_PARAMS_DEFAULT),\n process(std::move(proc))\n{\n}\n~~~\n\nActually, the type of **http\\_proccess\\_t** and the type of **http\\_callback\\_t** are exactly the same. Both are used to handle WFHttpTask. \nThe job of the server is to populate the response based on the request. \nSimilarly, we use an ordinary function to implement the process. The process iterates over the HTTP header of the request line by line and then writes them into an HTML page.\n\n~~~cpp\nvoid process(WFHttpTask *server_task)\n{\n protocol::HttpRequest *req = server_task->get_req();\n protocol::HttpResponse *resp = server_task->get_resp();\n long seq = server_task->get_task_seq();\n protocol::HttpHeaderCursor cursor(req);\n std::string name;\n std::string value;\n char buf[8192];\n int len;\n\n /* Set response message body. */\n resp->append_output_body_nocopy(\"\", 6);\n len = snprintf(buf, 8192, \"

%s %s %s

\", req->get_method(),\n req->get_request_uri(), req->get_http_version());\n resp->append_output_body(buf, len);\n\n while (cursor.next(name, value))\n {\n len = snprintf(buf, 8192, \"

%s: %s

\", name.c_str(), value.c_str());\n resp->append_output_body(buf, len);\n }\n\n resp->append_output_body_nocopy(\"\", 7);\n\n /* Set status line if you like. */\n resp->set_http_version(\"HTTP/1.1\");\n resp->set_status_code(\"200\");\n resp->set_reason_phrase(\"OK\");\n\n resp->add_header_pair(\"Content-Type\", \"text/html\");\n resp->add_header_pair(\"Server\", \"Sogou WFHttpServer\");\n if (seq == 9) /* no more than 10 requests on the same connection. */\n resp->add_header_pair(\"Connection\", \"close\");\n\n // print log\n ...\n}\n~~~\n\nYou have learned most of the HttpMessage related operations. The only new operation here is **append\\_output\\_body()**. \nObviously, it is not very efficient for the users to generate a complete HTTP body and pass it to the framework. The user only needs to call the **append** interface to append the discrete data to the message block by block. \n**append\\_output\\_body()** operation will move the data, and another interface with the suffix **\\_nocopy** will directly use the reference to the pointer. Please do not make it point to the local variables when you use it. \n[HttpMessage.h](../src/protocol/HttpMessage.h) contains the declaration of relevant calls. \n\n~~~cpp\nclass HttpMessage\n{\npublic:\n bool append_output_body(const void *buf, size_t size);\n bool append_output_body_nocopy(const void *buf, size_t size);\n ...\n bool append_output_body(const std::string& buf);\n};\n~~~\n\nOnce again, please note that when you use **append\\_output\\_body\\_nocopy()**, the lifecycle of the data referenced by the buf must at least be extended to the callback of the task. \nAnother variable seq in the function is obtained by **server\\_task->get\\_task\\_seq()**, which indicates the number of requests on the current connection, starting from 0. \nIn the program, the connection is forcibly closed after 10 requests are completed, thus:\n\n~~~cpp\n if (seq == 9) /* no more than 10 requests on the same connection. */\n resp->add_header_pair(\"Connection\", \"close\");\n~~~\n\nYou can also use **task->set\\_keep\\_alive()** to close the connection. However, for the connection using HTTP protocol, it is recommended to set the “close” option in HTTP header. \nIn this example, because the response page is very small, we didn't pay attention to the reply status. In the next tutorial **http\\_proxy**, you will learn how to get the reply status.\n" }, { "path": "docs/en/tutorial-05-http_proxy.md", "content": "# Asynchronous server: http\\_proxy\n\n# Sample code\n\n[tutorial-05-http\\_proxy.cc](/tutorial/tutorial-05-http_proxy.cc)\n\n# About http\\_proxy\n\nIt is an HTTP proxy server. You can use it in a browser after proper configuration. It supports all HTTP methods. \nAs HTTPS proxy follows different principles, this example does not support HTTPS proxy. You can only browse HTTP websites. \nIn the implementation, this proxy must crawl the entire HTTP page and then forward it. Therefore, there will be noticeable latency when you upload/download a large file.\n\n# Changing server configuration\n\nIn the previous example, we use the default parameters of an HTTP server. In this tutorial, we will made some changes and limit the size of the request so as to prevent malicious attack.\n\n~~~cpp\nint main(int argc, char *argv[])\n{\n ...\n struct WFServerParams params = HTTP_SERVER_PARAMS_DEFAULT;\n params.request_size_limit = 8 * 1024 * 1024;\n\n WFHttpServer server(¶ms, process);\n if (server.start(port) == 0)\n {\n pause();\n server.stop();\n }\n else\n {\n perror(\"cannot start server\");\n exit(1);\n }\n\n return 0; \n}\n~~~\nUnlike the previous example, we pass an additional parameter to the server struct. Let’s see the configuration items in the HTTP server. \nIn [WFHttpServer.h](/src/server/WFHttpServer.h), the default parameters for an HTTP server include:\n~~~cpp\nstatic constexpr struct WFServerParams HTTP_SERVER_PARAMS_DEFAULT =\n{\n .transport_type = TT_TCP,\n .max_connections = 2000,\n .peer_response_timeout = 10 * 1000,\n .receive_timeout = -1,\n .keep_alive_timeout = 60 * 1000,\n .request_size_limit = (size_t)-1,\n .ssl_accept_timeout = 10 * 1000,\n};\n~~~\n**transport\\_type**: the transport layer protocol. Besides the default type TT_TCP, you may specify TT_UDP, or TT_SCTP on Linux platform. \n**max\\_connections**: the maximum number of connections is 2000. When it is exceeded, the least recently used keep-alive connection will be closed. If there is no keep-alive connection, the server will refuse new connections. \n**peer\\_response\\_timeout**: set the maximum duration for reading or sending out a block of data. The default setting is 10 seconds. \n**receive\\_timeout**: set the maximum duration for receiving a complete request; -1 means unlimited time. \n**keep\\_alive\\_timeout**: set the maximum duration for maintaining a connection. The default setting is 1 minute. \n**request\\_size\\_limit**: set the maximum size of a request packet. The default setting is unlimited packet size. \n**ssl\\_accept\\_timeout**: set the maximum duration for an SSL handshake. The default setting is 10 seconds. \nThere is no **send\\_timeout** in the parameters. **send\\_timeout** sets the timeout for sending a complete response. This parameter should be determined according to the size of the response packet.\n\n# Business logic of a proxy server\n\nEssentially, this proxy server forwards a user's request intactly to the corresponding web server, and then forwards the reply from the web server intactly to the user. \nIn the request sent by a browser to the proxy, the Request URL contains scheme, host and port, which should be removed before forwarding. \nFor example, when the browser visits `http://www.sogou.com/`, the first line of the request sent by the browser to the proxy is:\n `GET` `http://www.sogou.com/` `HTTP/1.1` \nwhich should be rewritten as: \n`GET` `/` `HTTP/1.1`\n\n~~~cpp\nvoid process(WFHttpTask *proxy_task)\n{\n auto *req = proxy_task->get_req();\n SeriesWork *series = series_of(proxy_task);\n WFHttpTask *http_task; /* for requesting remote webserver. */\n\n tutorial_series_context *context = new tutorial_series_context;\n context->url = req->get_request_uri();\n context->proxy_task = proxy_task;\n\n series->set_context(context);\n series->set_callback([](const SeriesWork *series) {\n delete (tutorial_series_context *)series->get_context();\n });\n\n http_task = WFTaskFactory::create_http_task(req->get_request_uri(), 0, 0,\n http_callback);\n\n const void *body;\n size_t len;\n\n /* Copy user's request to the new task's reuqest using std::move() */\n req->set_request_uri(http_task->get_req()->get_request_uri());\n req->get_parsed_body(&body, &len);\n req->append_output_body_nocopy(body, len);\n *http_task->get_req() = std::move(*req);\n\n /* also, limit the remote webserver response size. */\n http_task->get_resp()->set_size_limit(200 * 1024 * 1024);\n\n *series << http_task;\n}\n~~~\n\nThe above contains the entire content of the process. It first parses the struct of an HTTP request sent by a web server. \n**req->get\\_request\\_uri()** is used to get the complete URL of the request sent by a browser. And then build a HTTP task to the server based on this URL. \nBoth the retry times and the redirection times of this HTTP task is 0, because the redirection is handled by the browser and the browser will be resend the request when it meets 302, etc.\n\n~~~cpp\n req->set_request_uri(http_task->get_req()->get_request_uri());\n req->get_parsed_body(&body, &len);\n req->append_output_body_nocopy(body, len);\n *http_task->get_req() = std::move(*req);\n~~~\n\nIn fact, the above four lines generates a HTTP request to the web server. req is the received HTTP request, and it will be moved directly to the new request via **std::move()**. \nThe first line removes the `http://host:port` in the request\\_uri and keeps the part after the path. \nThe second line and the third line specify the parsed HTTP body as the HTTP body for output. The reason for this operation is that in the HttpMessage implementation, the http body obtained by parsing and the http body to send out are two fields, so we need to simply set it here, without copying the memory. \nThe fourth line transfers the request content to the request sent to the web server at one time. After the HTTP request is constructed, the request is placed at the end of the current series, and the process function ends.\n\n# Principles behind an asynchronous server\n\nObviously, the process function is only part of the proxy logic. We also need to handle the HTTP response returned from the web server and generates the response for the browser. \nIn the example of echo server, we populate the response page directly without network communication. However, in the proxy server, we have to wait for the response from the web server. \nOf course, we can occupy the thread of this process function and wait for the returned result, but this synchronous waiting mode is obviously not desirable. \nThus, it is better that we reply to the user's request asynchronously after receiving the results for the request, and no thread is occupied while we are waiting for the result. \nTherefore, we set a context for the current series in the head of the process, which contains the proxy\\_task itself. In this way, we can populate the results asynchronously.\n\n~~~cpp\nstruct tutorial_series_context\n{\n std::string url;\n WFHttpTask *proxy_task;\n bool is_keep_alive;\n};\n\nvoid process(WFHttpTask *proxy_task)\n{\n SeriesWork *series = series_of(proxy_task);\n ...\n tutorial_series_context *context = new tutorial_series_context;\n context->url = req->get_request_uri();\n context->proxy_task = proxy_task;\n\n series->set_context(context);\n series->set_callback([](const SeriesWork *series) {\n delete (tutorial_series_context *)series->get_context();\n });\n ...\n}\n~~~\n\nIn the previous client example, we said that any running task is in a series, and the server task is no exception. \nThus, we can get the current series and set the context. In which the URL is mainly used for the subsequent logs, and the proxy\\_task is the main content, which is used for resp later. \nNext, Let’s see how to handle the responses from the web server.\n\n~~~cpp\nvoid http_callback(WFHttpTask *task)\n{\n int state = task->get_state();\n auto *resp = task->get_resp();\n SeriesWork *series = series_of(task);\n tutorial_series_context *context =\n (tutorial_series_context *)series->get_context();\n auto *proxy_resp = context->proxy_task->get_resp();\n\n ...\n if (state == WFT_STATE_SUCCESS)\n {\n const void *body;\n size_t len;\n\n /* set a callback for getting reply status. */\n context->proxy_task->set_callback(reply_callback);\n\n /* Copy the remote webserver's response, to proxy response. */\n resp->get_parsed_body(&body, &len);\n resp->append_output_body_nocopy(body, len);\n *proxy_resp = std::move(*resp);\n ...\n }\n else\n {\n // return a \"404 Not found\" page\n ...\n }\n}\n~~~\n\nHere we focus on the successful cases only. If the proxy gets a complete HTTP page from the web server, no matter what the return code is, it is considered a success. All failure will simply return a 404 page. \nBecause the data returned to the user may be very large, the maximum size is set to 200MB in this example. Therefore, unlike the previous examples, we need to check the success/failure status of the reply. \nThe type of an HTTP server task is identical to the type of an HTTP client task created by ourselves. Both are WFHttpTask. The difference is that a server task is created by the framework, and its callback is initially empty. \nThe callback of a server task is the same as that of a client. Both are called after an HTTP interaction is completed. Therefore, for all server tasks, the callback is called after the reply is completed. \nThe following three lines of code are explained before. They transfer the response packets from the web server to the proxy response packets without copying. \nAfter the **http\\_callback** function is ended, the reply to the browser is sent out. Everything is done asynchronously. \nThe remaining function **reply\\_callback()** is used just to print some logs here. The proxy task will be automatically deleted after this callback is finished. \nFinally, the context is destroyed in the callback of the series.\n\n# Timing of a server reply\n\nPlease note that the reply message is sent automatically after all other tasks in the series are finished, so there is no **task->reply()** interface. \nHowever, there is a **task->noreply()**. If this interface is called for the server task, the connection will be closed directly at the original reply time. But the callback will still be called (its state is NOREPLY). \nIn the callback of a server task, you can also call **series\\_of()** to get the series of that server task. Then, you can still add new tasks to this series, although the reply has finished. \n" }, { "path": "docs/en/tutorial-06-parallel_wget.md", "content": "# A simple parallel wget: parallel\\_wget\n\n# Sample code\n\n[tutorial-06-parallel\\_wget.cc](/tutorial/tutorial-06-parallel_wget.cc)\n\n# About parallel\\_wget\n\nIt is our first example on parallel tasks. \nThe program reads multiple HTTP URLs (separated by spaces) from the command line, crawls these URLs in parallel, and prints the crawled results to the standard output according to the input order.\n\n# Creating a parallel task\n\nIn the previous example, you have already learned the SeriesWork class.\n\n* SeriesWork consists of a series of tasks that are executed sequentially. The series finishes when all its tasks finish.\n* ParallelWork class, corresponding to the SeriesWork, consists of multiple series that are executed in parallel. The parallel work finishes when all its series finish.\n* ParallelWork is a task.\n\nAccording to the above definition, you can generate any complex workflow dynamically or statically. \nThe Workflow class has two interfaces for generating parallel tasks:\n\n~~~cpp\nclass Workflow\n{\n ...\npublic:\n static ParallelWork *\n create_parallel_work(parallel_callback_t callback);\n\n static ParallelWork *\n create_parallel_work(SeriesWork *const all_series[], size_t n,\n parallel_callback_t callback);\n\n ...\n};\n~~~\n\nThe first interface creates an empty parallel task, and the second interface creates parallel tasks with a series array. \nBefore you start the parallel work, you can use **add\\_series()** interface of the ParallelWork to add series to the parallel tasks generated by either interface. \nIn the sample code, we create an empty parallel task and then add the series one by one.\n\n~~~cpp\nint main(int argc, char *argv[])\n{\n ParallelWork *pwork = Workflow::create_parallel_work(callback);\n SeriesWork *series;\n WFHttpTask *task;\n HttpRequest *req;\n tutorial_series_context *ctx;\n int i;\n\n for (i = 1; i < argc; i++)\n {\n std::string url(argv[i]);\n ...\n task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n [](WFHttpTask *task)\n {\n // store resp to ctx.\n });\n\n req = task->get_req();\n // add some headers.\n ...\n\n ctx = new tutorial_series_context;\n ctx->url = std::move(url);\n series = Workflow::create_series_work(task, nullptr);\n series->set_context(ctx);\n pwork->add_series(series);\n }\n ...\n}\n~~~\n\nYou can see that we first create an HTTP task in the code, but the HTTP task cannot be directly added to the parallel task, so we need to use it to create a series first. \nEach series has its own context, which is used to save the URL and the crawled results. You can learn related methods in our previous examples.\n\n# Saving and using the crawled results\n\nThe callback of an HTTP task is a simple lambda function, which saves the crawled result in its own series context, so that it can be retrieved by the parallel task.\n\n~~~cpp\n task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n [](WFHttpTask *task)\n {\n tutorial_series_context *ctx =\n (tutorial_series_context *)series_of(task)->get_context();\n ctx->state = task->get_state();\n ctx->error = task->get_error();\n ctx->resp = std::move(*task->get_resp());\n });\n~~~\n\nThis is necessary, because HTTP tasks will be recycled after the callback, so we have to use **std::move()** to move the resp. \nIn the callback of parallel tasks, we can easily get the results:\n\n~~~cpp\nvoid callback(const ParallelWork *pwork)\n{\n tutorial_series_context *ctx;\n const void *body;\n size_t size;\n size_t i;\n\n for (i = 0; i < pwork->size(); i++)\n {\n ctx = (tutorial_series_context *)pwork->series_at(i)->get_context();\n printf(\"%s\\n\", ctx->url.c_str());\n if (ctx->state == WFT_STATE_SUCCESS)\n {\n ctx->resp.get_parsed_body(&body, &size);\n printf(\"%zu%s\\n\", size, ctx->resp.is_chunked() ? \" chunked\" : \"\");\n fwrite(body, 1, size, stdout);\n printf(\"\\n\");\n }\n else\n printf(\"ERROR! state = %d, error = %d\\n\", ctx->state, ctx->error);\n\n delete ctx;\n }\n}\n~~~\n\nHere, you can see the two new interfaces of ParallelWork, **size()** and **series\\_at(i)**, which are used to obtain the number of the series in parallel and the ith parallel series respectively. \nYou can use **series->get\\_context()** to get the context of the series and print out the results.The printing order must be the same as with the order you add the series into the work. \nIn this example, there is no other work after the parallel tasks finish. \nAs we said above, ParallelWork is a kind of tasks, so you can use **series\\_of()** to get its series and add a new task. \nHowever, if the crawled results are used in the new task, you need to use **std::move()** to move the data to the context of the series of that parallel task.\n\n# Starting a parallel task\n\nAs a parallel task is a kind of tasks, so there is nothing special in starting a parallel task. You can call **start()** directly, or you can use it to build or start a series. \nIn this example, we start a series, wake up the main process in the callback of this series, and exit the program normally. \nWe can also wake up the main process in the callback of parallel tasks, and there is little difference in the program behaviors. However, it is more formal to wake up the main process in the callback of the series.\n" }, { "path": "docs/en/tutorial-07-sort_task.md", "content": "# Using the built-in algorithm factory: sort\\_task\n\n# Sample code\n\n[tutorial-07-sort\\_task.cc](/tutorial/tutorial-07-sort_task.cc)\n\n# About sort\\_task\n\nThe program reads a number n from the command line, sorts the random n positive integers in ascending order, and then sorts the results in descending order. You can add the second parameter \"p” to the program, and then it can be sorted in parallel. For example: \n ./sort\\_task 100000000 p \nThe above command will sort 100 million integers in ascending order and then in descending order. The two sortings are done in parallel respectively.\n\n# About computing tasks\n\nComputing tasks (or thread tasks) is a very important function in the framework. When you use the task flow, it is not recommended to directly perform very complicated computation in the callback. \nAll the computations that consume a lot of CPU time can be encapsulated into computing tasks and handed over to the system for scheduling. There is no difference in the usage between computing tasks and networking tasks. \nThe algorithm factory of the system provides some common computing tasks, such as sorting, merging and so on. You can also easily define your own computing tasks.\n\n# Creating sorting tasks in ascending order \n\n~~~cpp\nint main(int argc, char *argv[])\n{\n ...\n WFSortTask *task;\n if (use_parallel_sort)\n task = WFAlgoTaskFactory::create_psort_task(\"sort\", array, end, callback);\n else\n task = WFAlgoTaskFactory::create_sort_task(\"sort\", array, end, callback);\n ...\n task->start();\n ...\n}\n~~~\n\nUnlike WFHttpTask or WFRedisTask, the sorting task has one more template parameter to represent the type of array data to be sorted. \n**create\\_sort\\_task** and **create\\_psort\\_task** produce a common sorting task and a parallel sorting task respectively. \nTheir ****parameters and return values are the same.**** \nThe only thing that needs special explanation is the first parameter \"sort\", which is the name of the computation queue. It is used to instruct the internal task scheduling. The latter part in this article explains the usage of the queue name. \nThere is no difference in the starting methods and usage between computing tasks and networking tasks.\n\n# Handling results\n\nLike a networking task, the results are handled in the callback. In this example, the ascending sorting is followed by one descending sorting.\n\n~~~cpp\nusing namespace algorithm;\n\nvoid callback(void SortTask *task)\n{\n SortInput *input = task->get_input();\n int *first = input->first;\n int *last = input->last;\n\n // print result\n ...\n \n if (task->user_data == NULL)\n {\n auto cmp = [](int a1, int a2){ return a2 < a1; };\n WFSortTask *reverse;\n\n if (use_parallel_sort)\n reverse = WFAlgoTaskFactory::create_psort_task(\"sort\", first, last, cmp, callback);\n else\n reverse = WFAlgoTaskFactory::create_sort_task(\"sort\", first, last, cmp, callback);\n \n reverse->user_data = (void *)1; /* as a flag */\n series_of(task)->push_back(reverse);\n }\n else\n {\n // all done. Signal main thread to exit.\n ... \n }\n}\n~~~\n\nYou can use **get\\_input ()** interface of a computing task to get the input data, and use **get\\_output ()** to get the output data. For sorting tasks, the input and output are of the same type, and the content are exactly the same. \n[WFAlgoTaskFactory.h](/src/factory/WFAlgoTaskFactory.h) contains the definitions of the input and output of sorting tasks.\n\n~~~cpp\nnamespace algorithm\n{\n\ntemplate \nstruct SortInput\n{\n T *first;\n T *last;\n};\n\ntemplate \nusing SortOutput = SortInput;\n\n}\n\ntemplate \nusing WFSortTask = WFThreadTask,\n algorithm::SortOutput>;\n\ntemplate \nusing sort_callback_t = std::function *)>;\n\n~~~\n\nObviously, the first and last in the input or output mean the head pointer and the tail pointer of the array to be sorted. \nNext, we will create a descending sorting task. In this case, we need to pass in a comparison function.\n\n~~~cpp\n auto cmp = [](int a1, int a2)->bool{ return a2 < a1; };\n reverse = WFAlgoTaskFactory::create_sort_task(\"sort\", first, last, cmp, callback);\n~~~\n\nOur usage differs slightly from **std::sort()**. Our first and last are pointers, not iterators. \nSimilarly, you can use **create\\_psort\\_task()** to create a parallel sorting task. And the use of series in the sorting task is no different from that in the networking task.\n\n# About the configuration of the computing threads\n\nIf you don't make any configuration, the calculation scheduler will set the number of threads as the number of the CPU cores in the machine. You can change the value with the following method:\n\n~~~cpp\n#include \"workflow/WFGlobal.h\"\n\nint main()\n{\n struct WFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;\n settings.compute_threads = 16;\n WORKFLOW_library_init(&settings);\n ...\n}\n~~~\n\nWith the above configuration, the system will create 16 threads for computations.\n\n# About the parallel sorting algorithm\n\nThe built-in parallel sorting algorithm use block+two-way merge. Its space complexity is O(1). \nThe algorithm uses globally configured computing threads for computation, but at most 128 threads can be used. Because no extra space is used, the speedup ratio will be smaller than the number of threads, and the average CPU usage will be smaller. \nFor the detailed implementation, please see [WFAlgoTaskFactory.inl](/src/factory/WFAlgoTaskFactory.inl).\n\n# About the name of a calculation task queue\n\nThe computing task does not have priority levels. The only thing that can affect the scheduling order is the queue name of a computing task. In this example, the queue name is a string \"sort\". \nTo name a queue is very simple. Please note the following items:\n\n* The queue name is a static string, and new queue names cannot be generated infinitely. For example, you cannot generate the queue name according to the request id, because each queue is allocated a small block of resources internally.\n* If the computing threads are not 100% occupied, all tasks are started in real time, and the queue names have no effect.\n* If there are multiple computing steps in a service flow and they are interspersed among multiple network communications, you can simply give each calculation step a name, which is better than using one name as a whole.\n * If all computing tasks use the same name, the scheduling order of all tasks is consistent with the order of submission, which will affect the average response time in some scenarios.\n * If each kind of computing task has an independent name, it means that they are scheduled fairly. And the same kind of tasks are scheduled sequentially, the practical effect is better.\n* In a word, unless the computing load of the machine is already very heavy, you do not need to pay special attention to the queue name and you can just give each kind of task a name.\n" }, { "path": "docs/en/tutorial-08-matrix_multiply.md", "content": "# User-defined computing tasks: matrix\\_multiply\n\n# Sample code\n\n[tutorial-08-matrix\\_multiply.cc](/tutorial/tutorial-08-matrix_multiply.cc)\n\n# About matrix\\_multiply\n\nThe program multiplies two matrices and prints the results on the screen. \nThe main purpose of the example is to show how to implement a user-defined CPU computing task.\n\n# About computing tasks\n\nYou need to provide three types of basic information when you define a computer task: INPUT, OUTPUT, and routine. \nINPUT and OUTPUT are two template parameters, which can be of any type. routine means the process from INPUT to OUTPUT, which is defined as follows:\n\n~~~cpp\ntemplate \nclass __WFThreadTask\n{\n ...\n std::function routine;\n ...\n};\n~~~\n\nIt can be seen that routine is a simple computing process from INPUT to OUTPUT. The INPUT pointer is not necessarily be const, but you can also pass the function of const INPUT \\*. \nFor example, to implement an adding task, you can:\n\n~~~cpp\nstruct add_input\n{\n int x;\n int y;\n};\n\nstruct add_ouput\n{\n int res;\n};\n\nvoid add_routine(const add_input *input, add_output *output)\n{\n output->res = input->x + input->y;\n}\n\ntypedef WFThreadTask add_task;\n~~~\n\nIn the example of matrix multiplication, the input is two matrices and the output is one matrix. They are defined as follows:\n\n~~~cpp\nnamespace algorithm\n{\n\nusing Matrix = std::vector>;\n\nstruct MMInput\n{\n Matrix a;\n Matrix b;\n};\n\nstruct MMOutput\n{\n int error;\n size_t m, n, k;\n Matrix c;\n};\n\nvoid matrix_multiply(const MMInput *in, MMOutput *out)\n{\n ...\n}\n\n}\n~~~\n\nAs the input matrices may be illegal in matrix multiplication, so there is an error field in the output to indicate errors.\n\n# Generating computing tasks\n\nAfter you define the types of input and output and the algorithm process, you can use WFThreadTaskFactory to generate a computing task. \nIn [WFTaskFactory.h](/src/factory/WFTaskFactory.h), the computing task factory is defined as follows:\n\n~~~cpp\ntemplate \nclass WFThreadTaskFactory\n{\nprivate:\n using T = WFThreadTask;\n\npublic:\n static T *create_thread_task(const std::string& queue_name,\n std::function routine,\n std::function callback);\n\n static T *create_thread_task(time_t seconds, long nanoseconds,\n const std::string& queue_name,\n std::function routine,\n std::function callback);\n ...\n};\n~~~\nThere are two interfaces for creating tasks here. The second interface supports the user to pass in the task running time limit, we will introduce this function in the next section. Slightly different from the previous network factory class or the algorithm factory class, this factory requires two template parameters: INPUT and OUTPUT. \nqueue\\_name is explained in the previous example. routine is the computation process, and callback means the callback. \nIn our example, we see this call:\n\n~~~cpp\nusing MMTask = WFThreadTask;\n\nusing namespace algorithm;\n\nint main()\n{\n typedef WFThreadTaskFactory MMFactory;\n MMTask *task = MMFactory::create_thread_task(\"matrix_multiply_task\",\n matrix_multiply,\n callback);\n\n MMInput *input = task->get_input();\n\n input->a = {{1, 2, 3}, {4, 5, 6}};\n input->b = {{7, 8}, {9, 10}, {11, 12}};\n ...\n}\n~~~\n\nAfter the task is generated, use **get\\_input()** interface to get the pointer of the input data. This is similar to the **get\\_req()** in a network task. \nThe start and the end of a task is the same as those of a network task. Similarly, the callback is very simple:\n\n~~~cpp\nvoid callback(MMTask *task) // MMtask = WFThreadTask\n{\n MMInput *input = task->get_input();\n MMOutput *output = task->get_output();\n\n assert(task->get_state() == WFT_STATE_SUCCESS);\n\n if (output->error)\n printf(\"Error: %d %s\\n\", output->error, strerror(output->error));\n else\n {\n printf(\"Matrix A\\n\");\n print_matrix(input->a, output->m, output->k);\n printf(\"Matrix B\\n\");\n print_matrix(input->b, output->k, output->n);\n printf(\"Matrix A * Matrix B =>\\n\");\n print_matrix(output->c, output->m, output->n);\n }\n}\n~~~\n\nYou can ignore the the possibility of failure in the ordinary computing tasks, and the end state is always SUCCESS. \nThe callback simply prints out the input and the output. If the input data are illegal, the error will be printed out.\n\n# Computing task with running time limit\nObviously, our framework can not interrupt a computing task because it's a user function, and the users have to make sure the function will terminate normally. But we support users to create a computing task with a running time limit, and if the task doesn't finish within this time, the task will callback directly:\n~~~cpp\n template \n class WFThreadTaskFactory\n {\n private:\n using T = WFThreadTask;\n\n public:\n static T *create_thread_task(time_t seconds, long nanoseconds,\n const std::string& queue_name,\n std::function routine,\n std::function callback);\n ...\n };\n ~~~\nThis create_thread_task function needs to pass two more parameters, seconds and nanoseconds. If the running time of func reaches the seconds+nanosconds time limit, the task callback directly, and the state is WFT_STATE_SYS_ERROR and the error is ETIMEDOUT. But the task routine will continue to run till the end.\n\n# Symmetry of the algorithm and the protocol\n\nIn our system, algorithms and protocols are highly symmetrical on a very abstract level. \nThere are thread tasks with user-defined algorithms, obviously there are network tasks with user-defined protocols. \nA user-defined algorithm requires the user to provide the algorithm procedure, and a user-defined protocol requires the user to provide the procedure of serialization and deserialization. You can see an introduction in [Simple client/server based on user-defined protocols](/tutorial-10-user_defined_protocol.md) \nFor the user-defined algorithms and the user-defined protocols, both must be very pure . \nFor example, an algorithm is just a conversion procedure from INPUT to OUPUT, and the algorithm does not know the existence of task, series, etc. \nThe implementation of an HTTP protocol only cares about serialization and deserialization, and does not need to care about the task definition. Instead, the HTTP protocol is referred to in an http task.\n\n# Composite features of thread tasks and network tasks\n\nIn this example, we use WFThreadTaskFactory to build a thread task. This is the simplest way to get a computing task, and it is sufficient in most cases. \nSimilarly, you can simply define a server and a client with a user-defined protocol. \nHowever, in the previous example, we can use the algorithm factory to generate a parallel sorting task, which is obviously not possible with a routine. \nFor a network task, such as a Kafka task, interactions with several machines may be required to get results, but it is completely transparent to users. \nTherefore, our tasks are composite. If you use our framework skillfully, you can design many composite components.\n" }, { "path": "docs/en/tutorial-09-http_file_server.md", "content": "# Http server with file IO: http\\_file\\_server\n\n# Sample code\n\n[tutorial-09-http\\_file\\_server.cc](/tutorial/tutorial-09-http_file_server.cc)\n\n# About http\\_file\\_server\n\nhttp\\_file\\_server is a web server. You can start a web server after specifying the startup port and the root path (the default setting is the current path). \nYou can also specify a certificate file and a key file in PEM format to start an HTTPS web server. User may access the server through command line, the request will be sent to IP address 127.0.0.1. \nThe program mainly demonstrates how to use disk IO tasks. In the Linux system, we use the aio interface in the kernel of Linux, and the file reading is completely asynchronous.\n\n# Starting a server\n\nFor starting a server, the steps are almost the same as those when starting an echo server or an HTTP proxy. There is one more way to start an SSL server here:\n\n~~~cpp\nclass WFServerBase\n{\n ...\n int start(unsigned short port, const char *cert_file, const char *key_file);\n ...\n};\n~~~\n\nIn other words, you can specify a cert file and a key file in PEM format to start an SSL server. \nIn addition, when you define a server, you can use **std::bind()** to bind a root parameter to the process. The root parameter means the root path of the service.\n\n~~~cpp\nvoid process(WFHttpTask *server_task, const char *root)\n{\n ...\n}\n\nint main(int argc, char *argv[])\n{\n ...\n const char *root = (argc >= 3 ? argv[2] : \".\");\n auto&& proc = std::bind(process, std::placeholders::_1, root);\n WFHttpServer server(proc);\n\n // start server\n ...\n}\n~~~\n\n# Handling requests\n\nSimilar to http\\_proxy, no threads are occupied in file reading. Instead, an asynchronous task is generated to read files, and a reply to the request is generated after the reading is completed. \nPlease note again that the complete reply data should be read into the memory before the reply message is sent. Therefore, it is not suitable for transferring very large files.\n\n~~~cpp\nvoid process(WFHttpTask *server_task, const char *root)\n{\n // generate abs path.\n ...\n\n int fd = open(abs_path.c_str(), O_RDONLY);\n if (fd >= 0)\n {\n size_t size = lseek(fd, 0, SEEK_END);\n void *buf = malloc(size); /* As an example, assert(buf != NULL); */\n WFFileIOTask *pread_task;\n\n pread_task = WFTaskFactory::create_pread_task(fd, buf, size, 0,\n pread_callback);\n /* To implement a more complicated server, please use series' context\n * instead of tasks' user_data to pass/store internal data. */\n pread_task->user_data = resp; /* pass resp pointer to pread task. */\n server_task->user_data = buf; /* to free() in callback() */\n server_task->set_callback([](WFHttpTask *t){ free(t->user_data); });\n series_of(server_task)->push_back(pread_task);\n }\n else\n {\n resp->set_status_code(\"404\");\n resp->append_output_body(\"404 Not Found.\");\n }\n}\n~~~\n\nUnlike http\\_proxy that generates a new HTTP client task, here a pread task is generated by the factory. \n[WFAlgoTaskFactory.h](/src/factory/WFTaskFactory.h) contains the definitions of relevant interfaces.\n\n~~~cpp\nstruct FileIOArgs\n{\n int fd;\n void *buf;\n size_t count;\n off_t offset;\n};\n\n...\nusing WFFileIOTask = WFFileTask;\nusing fio_callback_t = std::function;\n...\n\nclass WFTaskFactory\n{\npublic:\n ...\n static WFFileIOTask *create_pread_task(int fd, void *buf, size_t count, off_t offset,\n fio_callback_t callback);\n\n static WFFileIOTask *create_pwrite_task(int fd, void *buf, size_t count, off_t offset,\n fio_callback_t callback);\n\n /* Interface with file path name */\n\tstatic WFFileIOTask *create_pread_task(const std::string& path, void *buf, size_t count, off_t offset,\n fio_callback_t callback);\n\n static WFFileIOTask *create_pwrite_task(const std::string& path, void *buf, size_t count, off_t offset,\n fio_callback_t callback); \n};\n~~~\n\nBoth pread and pwrite return WFFileIOTask. We do not distinguish between sort and psort, and we do not distinguish between client and server task. They all follow the same principle. \nIn addition to these two interfaces, preadv and pwritev return WFFileVIOTask; fsync and fdatasync return WFFileSyncTask. You can see the details in the header file. \nThe example uses the user\\_data field of the task to save the global data of the service. For larger services, we recommend to use series context. You can see the [proxy examples](/tutorial/tutorial-05-http_proxy.cc) for details.\n\n# Handling file reading results\n\n~~~cpp\nusing namespace protocol;\n\nvoid pread_callback(WFFileIOTask *task)\n{\n FileIOArgs *args = task->get_args();\n long ret = task->get_retval();\n HttpResponse *resp = (HttpResponse *)task->user_data;\n\n /* close fd only when you created File IO task with **fd** interface. */\n close(args->fd);\n if (ret < 0)\n {\n resp->set_status_code(\"503\");\n resp->append_output_body(\"503 Internal Server Error.\");\n }\n else /* Use '_nocopy' carefully. */\n resp->append_output_body_nocopy(args->buf, ret);\n}\n~~~\n\nUse **get\\_args()** of the file task to get the input parameters. Here it is a FileIOArgs struct, and it's **fd** field will be -1 if the task was created with **pathname**. \nUse **get\\_retval()** to get the return value of the operation. If ret < 0, the task fails. Otherwise, the ret is the size of the read data. \nIn the file task, ret < 0 and task->get\\_state()! = WFT\\_STATE\\_SUCCESS are completely equivalent. \nThe memory of the buf domain is managed by ourselves. You can use **append\\_output\\_body\\_nocopy()** to pass that memory to resp. \nAfter the reply is completed, we will **free()** this block of memory with this line in the process: \nserver\\_task->set\\_callback(\\[](WFHttpTask \\*t){ free(t->user\\_data); });\n\n# Interact with the server through command line\n\nAfter the server is started, users may access it through command line. Simply input the file name that you want to get, or input Ctrl-D to end the program. The repeating process is implemnted by using WFRepeaterTask, which can be created by this factory function:\n~~~cpp\nusing repeated_create_t = std::function;\nusing repeater_callback_t = std::function;\n\nclass WFTaskFactory\n{\n WFRpeaterTask *create_repeater_task(repeated_create_t create, repeater_callback_t callback);\n};\n~~~\nAs above, a repeater task is created with a task creator function. The repeater calls the task creator repeatedly and run the task until the creator return a NULL pointer. When the using's input is not empty, our creator will create an HTTP task on IP 127.0.0.1 to access the server. \n~~~cpp\n{\n\tauto&& create = [&scheme, port](WFRepeaterTask *)->SubTask *{\n\t\t...\n\t\tscanf(\"%1023s\", buf);\n\t\tif (*buf == '\\0')\n\t\t\treturn NULL;\n\n\t\tstd::string url = scheme + \"127.0.0.1:\" + std::to_string(port) + \"/\" + buf;\n\t\tWFHttpTask *task = WFTaskFactory::create_http_task(url, 0, 0,\n\t\t\t\t\t\t\t\t\t[](WFHttpTask *task) {\n\t\t\t...\n\t\t});\n\n\t\treturn task;\n\t};\n\t\n\tWFFacilities::WaitGroup wg(1);\n\tWFRepeaterTask *repeater;\n\trepeater = WFTaskFactory::create_repeater_task(create, [&wg](WFRepeaterTask *) {\n\t\twg.done();\n\t});\n\n\trepeater->start();\n\twg.wait();\n\n\tserver.stop();\n}\n~~~\nFinally, when the creator returned NULL, the repeater's callback is called and the program will be ended.\n\n# About the implementation of the file IO\n\nLinux operating system supports a set of asynchronous IO system calls with high efficiency and very little CPU occupation. If you use our framework in a Linux system, this set of interfaces are used by default. \nWe have implemented a set of posix aio interfaces to support other UNIX systems, and used the sigevent notification method of threads, but it is no longer in use because of its low efficiency. \nCurrently, for non-Linux systems, asynchronous IO is always simulated by multi-threading. When an IO task arrives, a thread is created in real time to execute IO tasks, and then a callback is used to return to the handler thread pool. \nMulti-threaded IO is also the only choice in macOS, because macOS does not have good sigevent support and posix aio will not work in macOS. \nSome UNIX systems do not support fdatasync. In this case, an fdatasync task is equivalent to an fsync task.\n" }, { "path": "docs/en/tutorial-10-user_defined_protocol.md", "content": "# A simple user-defined protocol: client/server \n\n# Sample codes\n\n[message.h](/tutorial/tutorial-10-user_defined_protocol/message.h) \n[message.cc](/tutorial/tutorial-10-user_defined_protocol/message.cc) \n[server.cc](/tutorial/tutorial-10-user_defined_protocol/server.cc) \n[client.cc](/tutorial/tutorial-10-user_defined_protocol/client.cc)\n\n# About user\\_defined\\_protocol\n\nThis example designs a simple communication protocol, and builds a server and a client on that protocol. The server converts the message sent by client into uppercase and returns it to the client.\n\n# Protocol format\n\nThe protocol message contains one 4-byte head and one message body. Head is an integer in network byte order, indicating the length of body. \nThe formats of the request messages and the response messages are identical.\n\n# Protocol implementation\n\nA user-defined protocol should provide its own serialization and deserialization methods, which are virtual functions in ProtocolMeessage class. \nIn addition, for the convenience of use, we strongly recommend users to implement the **move constructor** and **move assignment** for messages (for std::move ()). [ProtocolMessage.h](/src/protocol/ProtocolMessage.h) contains the following serialization and deserialization interfaces:\n\n~~~cpp\nnamespace protocol\n{\n\nclass ProtocolMessage : public CommMessageOut, public CommMessageIn\n{\nprivate:\n virtual int encode(struct iovec vectors[], int max);\n\n /* You have to implement one of the 'append' functions, and the first one\n * with arguement 'size_t *size' is recommmended. */\n virtual int append(const void *buf, size_t *size);\n virtual int append(const void *buf, size_t size);\n\n ...\n};\n\n}\n~~~\n\n### Serialization function: encode\n\n* The encode function is called before the message is sent, and it is called only once for each message.\n* In the encode function, you need to serialize the message into a vector array, and the number of array elements must not exceed max. Current the value of max is 2048.\n* For the definition of **struct iovec**, please see the system calls **readv** or **writev**.\n* Normally the return value of the encode function is between 0 and max, indicating how many vector are used in the message.\n * In case of UDP protocol, please note that the total length must not be more than 64k, and no more than 1024 vectors are used (in Linux, writev writes only 1024 vectors at one time).\n* The encode -1 indicates errors. To return -1, you need to set errno. If the return value is > max, you will get an EOVERFLOW error. All errors are obtained in the callback.\n* For performance reasons, the content pointed to by the iov\\_base pointer in the vector will not be copied. So it generally points to the member of the message class.\n\n### Deserialization function: append\n\n* The append function is called every time a data block is received. Therefore, for each message, it may be called multiple times.\n* buf and size are the content and the length of received data block respectively. You need to move the data content.\n * If the interface **append(const void \\*buf, size\\_t \\*size)** is implemented, you can tell the framework how much length is consumed at this time by modifying \\* size. remaining size = received size - consumed size, and the remaining part of the buf will be received again when the append is called next time. This function is more convenient for protocol parsing. Of course, you can also move the whole content and manage it by yourself. In this case, you do not need to modify \\*size.\n* If the **append** function returns 0, it indicates that the message is incomplete and the transmission continues. The return value of 1 indicates the end of the message. -1 indicates errors, and you need to set errno.\n* In a word, the append function is used to tell the framework whether the message transmission is completed or not. Please don't perform complicated and unnecessary protocol parsing in the append.\n\n### Setting the errno\n\n* If encode or append returns -1 or other negative numbers, it should be interpreted as failure, and you should set the errno to pass the error reason. You can obtain this error in the callback.\n* If the system calls or the library functions such as libc fail (for example, malloc), libc will definitely set errno, and you do not need to set it again.\n* Some errors of illegal messages are quite common. For example, EBADMSG or EMSGSIZE can be used to indicate that the message content is wrong and the message is too large respectively.\n* You can use a value that exceeds the errno range defined in the system to indicate a user-defined error. Generally, you can use a value greater than 256.\n* Please do not use a negative errno. Because negative numbers are used inside the framework to indicate SSL errors.\n\nIn our example, the serialization and deserialization of messages are very simple. \nThe header file [message.h](/tutorial/tutorial-10-user_defined_protocol/message.h) declares the request class and the response class.\n\n~~~cpp\nnamespace protocol\n{\n\nclass TutorialMessage : public ProtocolMessage\n{\nprivate:\n virtual int encode(struct iovec vectors[], int max);\n virtual int append(const void *buf, size_t size);\n ...\n};\n\nusing TutorialRequest = TutorialMessage;\nusing TutorialResponse = TutorialMessage;\n\n}\n~~~\n\nBoth the request class and the response class belong to the same type of messages. You can directly introduce them with using. \nNote that both the request and the response can be constructed without parameters. In other words, you must provide a constructor without parameters or no constructor. In addition, the response object may be destroyed and reconstruct during communication if retrial occurs, therefore it should be a RAII class, otherwise things will be complicated). \n[message.cc](/tutorial/tutorial-10-user_defined_protocol/message.cc) contains the implementation of encode and append:\n\n~~~cpp\nnamespace protocol\n{\n\nint TutorialMessage::encode(struct iovec vectors[], int max/*max==8192*/)\n{\n uint32_t n = htonl(this->body_size);\n\n memcpy(this->head, &n, 4);\n vectors[0].iov_base = this->head;\n vectors[0].iov_len = 4;\n vectors[1].iov_base = this->body;\n vectors[1].iov_len = this->body_size;\n\n return 2; /* return the number of vectors used, no more then max. */\n}\n\nint TutorialMessage::append(const void *buf, size_t size)\n{\n if (this->head_received < 4)\n {\n size_t head_left;\n void *p;\n\n p = &this->head[this->head_received];\n head_left = 4 - this->head_received;\n if (size < 4 - this->head_received)\n {\n memcpy(p, buf, size);\n this->head_received += size;\n return 0;\n }\n\n memcpy(p, buf, head_left);\n size -= head_left;\n buf = (const char *)buf + head_left;\n\n p = this->head;\n this->body_size = ntohl(*(uint32_t *)p);\n if (this->body_size > this->size_limit)\n {\n errno = EMSGSIZE;\n return -1;\n }\n\n this->body = (char *)malloc(this->body_size);\n if (!this->body)\n return -1;\n\n this->body_received = 0;\n }\n\n size_t body_left = this->body_size - this->body_received;\n\n if (size > body_left)\n {\n errno = EBADMSG;\n return -1;\n }\n\n memcpy(this->body, buf, size);\n if (size < body_left)\n return 0;\n\n return 1;\n}\n\n}\n~~~\n\nThe implementation of encode is very simple, in which two vectors are always, pointing to the head and the body respectively. Note that the iov\\_base pointer must point to a member of the message class. \nWhen you use append, you should ensure that the 4-byte head is received completely before reading the message body. Moreover, we can't guarantee that the first append must contain a complete head, so the process is a little cumbersome. \nThe append implements the size\\_limit function, and an EMSGSIZE error will be returned if the size\\_limit is exceeded. You can ignore the size_limit field if you don't need to limit the message size. \nBecause we require the communication protocol is two way with a request and a response, users do not need to consider the so-called \"TCP packet sticking\" problem. The problem should be treated as an error message directly. \nNow, with the definition and implementation of messages, we can build a server and a client.\n\n# Server and client definitions\n\nWith the request and response classes, we can build a server and a client based on this protocol. The previous example explains the type definitions related to an HTTP protocol:\n\n~~~cpp\nusing WFHttpTask = WFNetworkTask;\nusing http_callback_t = std::function;\n\nusing WFHttpServer = WFServer;\nusing http_process_t = std::function;\n~~~\n\nSimilarly, for the protocol in this tutorial, there is no difference in the definitions of data types:\n\n~~~cpp\nusing WFTutorialTask = WFNetworkTask;\nusing tutorial_callback_t = std::function;\n\nusing WFTutorialServer = WFServer;\nusing tutorial_process_t = std::function;\n~~~\n\n# server\n\nThere is no difference between this server and an ordinary HTTP server. We give priority to IPv6 startup, which does not affect the client requests in IPv4. In addition, the maximum request size is limited to 4KB. \nPlease see [server.cc](/tutorial/tutorial-10-user_defined_protocol/server.cc) for the complete code.\n\n# client\n\nThe logic of the client is to receive the user input from standard IO, construct a request, send it to the server and get the results. Here we use WFRepeaterTask to implement the repeating process, terminates if the user's input is empty. For the sake of security, we limit the packet size of the server reply to 4KB. \nThe only thing that a client needs to know is how to generate a client task on a user-defined protocol. There are three interface options in [WFTaskFactory.h](/src/factory/WFTaskFactory.h):\n\n~~~cpp\ntemplate\nclass WFNetworkTaskFactory\n{\nprivate:\n\tusing T = WFNetworkTask;\n\npublic:\n\tstatic T *create_client_task(TransportType type,\n\t\t\t\t\t\t\t\t const std::string& host,\n\t\t\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(TransportType type,\n\t\t\t\t\t\t\t\t const std::string& url,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(TransportType type,\n\t\t\t\t\t\t\t\t const ParsedURI& uri,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(TransportType type,\n\t\t\t\t\t\t\t\t const struct sockaddr *addr,\n\t\t\t\t\t\t\t\t socklen_t addrlen,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n ...\n};\n~~~\n\nAmong them, TransportType specifies the transport layer protocol, and the current options include TT\\_TCP, TT\\_UDP, TT\\_SCTP, TT\\_TCP\\_SSL and TT\\_SCTP\\_SSL. \nThere is little difference between the interfaces. In our example, the URL is not needed for the time being. We use a domain name and a port to create a task. \nThe actual code is shown as follows. We inherited the WFTaskFactory class, but this derivation is not required.\n\n~~~cpp\nusing namespace protocol;\n\nclass MyFactory : public WFTaskFactory\n{\npublic:\n static WFTutorialTask *create_tutorial_task(const std::string& host,\n unsigned short port,\n int retry_max,\n tutorial_callback_t callback)\n {\n using NTF = WFNetworkTaskFactory;\n WFTutorialTask *task = NTF::create_client_task(TT_TCP, host, port,\n retry_max,\n std::move(callback));\n task->set_keep_alive(30 * 1000);\n return task;\n }\n};\n~~~\n\nYou can see that we used the WFNetworkTaskFactory\\ class to create a client task. \nNext, by calling the **set\\_keep\\_alive()** interface of the task, the connection is kept for 30 seconds after the communication is completed. Otherwise, the short connection will be used by default. \nThe previous examples have explained the knowledge in other codes of the above client. Please see [client.cc](/tutorial/tutorial-10-user_defined_protocol/client.cc).\n\n# How is the request on an built-in protocol generated\n\nCurrently, there are five built-in protocols in the framework: HTTP, Redis, MySQL, Kafka and DNS. Can we generate an HTTP or Redis task in the same way? For example:\n\n~~~cpp\nWFHttpTask *task = WFNetworkTaskFactory::create_client_task(...);\n~~~\n\nPlease note that an HTTP task generated in this way will lose a lot of functions. For example, it is impossible to identify whether to use persistent connection according to the header, and it is impossible to identify redirection, etc. \nSimilarly, if a MySQL task is generated in this way, it may not run at all, because there is no login authentication process. \nA Kafka request may need to have complicated interactions with multiple brokers, so the request created in this way obviously cannot complete this process. \nThis shows that the generation of one message in each built-in protocol is far more complicated than that in this example. Similarly, if you need to implement a communication protocol with more functions, there are still many codes to write.\n" }, { "path": "docs/en/tutorial-11-graph_task.md", "content": "# Direct Acyclic Graph (DAG)：graph_task\n# Sample code\n\n[tutorial-11-graph_task.cc](/tutorial/tutorial-11-graph_task.cc)\n\n# About graph_task\n\nThe graph_task example demonstrates how to implement more complex inter-task dependencies by building a DAG.\n\n# Create tasks in the DAG\n\nIn this tutorial, we create a timer task, two http fetching task, and a 'go' task. Timer task executes a delay of 1 second before \nfetching, http tasks fetch the home page of 'sogou' and 'baidu' in parallel, and after all of that, go task will print the fetching result. \nThe dependencies of the tasks are:\n~~~\n +-------+ \n +---->| Http1 |-----+ \n | +-------+ |\n +-------+ +-v--+ \n | Timer | | Go | \n +-------+ +-^--+ \n | +-------+ | \n +---->| Http2 |-----+ \n +-------+ \n~~~\n\n# Create the graph task\n\nGraph is a kind of task as well. We can create a graph task by this function：\n~~~cpp\nclass WFTaskFactory\n{\npublic:\n static WFGraphTask *create_graph_task(graph_callback_t callback);\n ...\n};\n~~~\nThe graph is a empty graph after it's creation. Of course you may run an empty graph and will get to it callback immediately.\n\n# Create graph nodes\n\nWe'v got 4 orindary tasks, which can not been added to the graph directly but need to be turned into graph nodes:\n~~~cpp\n{\n /* Create graph nodes */\n WFGraphNode& a = graph->create_graph_node(timer);\n WFGraphNode& b = graph->create_graph_node(http_task1);\n WFGraphNode& c = graph->create_graph_node(http_task2);\n WFGraphNode& d = graph->create_graph_node(go_task);\n}\n~~~\nThe ``create_graph_node`` interface of WFGraphTask creates a graph node that refers to a task. And we can use the references of \ngraph nodes to specify the dependencies of them. Otherwise, they are all standalone nodes, and will run in parallel when the \ngraph task is started.\n\n# Build the graph\nBy using the '-->' or '<--' operators, we can specify the dependencies:\n~~~cpp\n{\n /* Build the graph */\n a-->b;\n a-->c;\n b-->d;\n c-->d;\n}\n~~~\nAnd now we'v built the graph that we described. And we can use it like an orindary task. \nAlso, any of the following codes is legal and equivalent:\n~~~cpp\n{\n a-->b-->d;\n a-->c-->d;\n}\n~~~\n~~~cpp\n{\n d<--b<--a;\n d<--c<--a;\n}\n~~~\n~~~cpp\n{\n d<--b<--a-->c-->d;\n}\n~~~\n\n# Canceling successors\n\nIn graph tasks, we extend SeriesWork's **cancel** operation. When the series of a graph node is canceled, the operation will apply on all it's successive nodes recursively. The **cancel** operation is usually used in a task's callback:\n~~~cpp\nint main()\n{\n WFGraphTask *graph = WFTaskFactory::create_graph_task(graph_callback);\n WFHttpTask *task = WFTaskFactory::create_http_task(url, 0, 0, [](WFHttpTask *t){\n if (t->get_state() != WFT_STATE_SUCCESS)\n series_of(t)->cancel();\n });\n WFGraphNode& a = graph->create_graph_node(task);\n WFGraphNode& b = ...;\n WFGraphNode& c = ...;\n WFGraphNode& d = ...;\n a-->b-->c;\n b-->d;\n graph->start();\n ...\n}\n~~~\nIn this case, when http task failed, nodes b, c, d will all be canceled, because the operation is recursive.\n\n# Data passing\n\nBecause the tasks in a graph don't share a same series, there is no general method for passing data between graph nodes.\n\n# Acknowledgement\n\nSome designs are inspired by [taskflow](https://github.com/taskflow/taskflow).\n\n" }, { "path": "docs/en/tutorial-12-mysql_cli.md", "content": "# Asynchronous MySQL client: mysql\\_cli\n\n# Sample code\n\n[tutorial-12-mysql\\_cli.cc](/tutorial/tutorial-12-mysql_cli.cc)\n\n# About mysql\\_cli\n\nThe usage of mysql\\_cli in the tutorial is similar to that of the official client. It is an asynchronous MySQL client with an interactive command line interface.\n\nTo start the program, run the command: ./mysql_cli \\\n\nAfter startup, you can directly enter MySQL command in the terminal to interact with db, or enter `quit` or `Ctrl-C` to exit.\n\n# Format of MySQL URL\n\nmysql://username:password@host:port/dbname?character\\_set=charset&character\\_set\\_results=charset\n\n- set scheme to be **mysqls://** for accessing MySQL with SSL connnection (MySQL server 5.7 or above is required).\n\n- fill in the username and the password for the MySQL database; Special characters in password need to be escaped.\n~~~cpp\n// Password: @@@@####\nstd::string url = \"mysql://root:\" + StringUtil::url_encode_component(\"@@@@####\") + \"@127.0.0.1\";\n~~~\n\n- the default port number is 3306;\n\n- **dbname** is the name of the database to be used. It is recommended to provide a dbname if SQL statements only operates on one database;\n\n- If you have upstream selection requirements for MySQL, please see [upstream documents](/docs/en/about-upstream.md).\n\n- **character_set** indicates a character set used for the client, with the same meaning of --default-character-set in official client. The default value is utf8. For details, please see official MySQL documents [character-set.html](https://dev.mysql.com/doc/internals/en/character-set.html).\n\n- **character_set_results** indicates a character set for client, connection and results. If you wants to use `SET NAMES ` in SQL statements, please set it here.\n\nSample MySQL URL:\n\nmysql://root:password@127.0.0.1\n\nmysql://@test.mysql.com:3306/db1?character\\_set=utf8&character_set_results=utf8\n\nmysqls://localhost/db1?character\\_set=big5\n\n# Creating and starting a MySQL task\n\nYou can use WFTaskFactory to create a MySQL task. The usage of creating interface and callback functions are similar to other tasks in workflow:\n\n~~~cpp\nusing mysql_callback_t = std::function;\n\nWFMySQLTask *create_mysql_task(const std::string& url, int retry_max, mysql_callback_t callback);\n\nvoid set_query(const std::string& query);\n~~~\n\nYou can call **set\\_query()** on the request to write SQL statements after creating a WFMySQLTask.\n\nIf **set_query()** had **NOT** been called before the task started, the user might get **WFT_ERR_MYSQL_QUERY_NOT_SET** in callback.\n\nOther functions, including callback, series and user\\_data are used in a way similar to other tasks in workflow.\n\nThe following codes show some general usage:\n~~~cpp\nint main(int argc, char *argv[])\n{\n ...\n WFMySQLTask *task = WFTaskFactory::create_mysql_task(url, RETRY_MAX, mysql_callback);\n task->get_req()->set_query(\"SHOW TABLES;\");\n ...\n task->start();\n ...\n}\n~~~\n\n# Supported commands\n\nCurrently the supported command is **COM\\_QUERY**, which can cover the basic requirements for adding, deleting, modifying and querying data, creating and deleting databases, creating and deleting tables, prepare, using stored procedures and using transactions.\n\nBecause the program doesn't support the selection of databases (**USE** command) in our interactive commands, if there are **cross-database** operations in SQL statements, you can specify the database and table with **db\\_name.table\\_name**.\n\nAny other command can be **spliced together** and then passed to WFMySQLTask with `set_query()`. (including INSERT/UPDATE/SELECT/PREPARE/CALL)\n\nThe spliced commands will be executed sequentially until an error occurs, and the previous commands will be executed successfully.\n\nFor example:\n\n~~~cpp\nreq->set_query(\"SELECT * FROM table1; CALL procedure1(); INSERT INTO table3 (id) VALUES (1);\");\n~~~\n\n# Parsing results\n\nSimilar to other tasks in workflow, you can use **task->get\\_resp()** to get **MySQLResponse**. For details on the interfaces, please see [MySQLResult.h](/src/protocol/MySQLResult.h).\n\nOne request will get one response, which is a 3-dimensional structure.\n- one response consists of one or more result sets;\n- the type of each result set may be **MYSQL_STATUS_GET_RESULT** or **MYSQL_STATUS_OK**;\n- one result set of type **MYSQL_STATUS_GET_RESULT** consists of one ore more rows;\n- one row consists of one or more fields, or data cells;\n\nThe two types of result sets can be judged by ``cursor->get_cursor_status()``.\n\n| |MYSQL_STATUS_GET_RESULT|MYSQL_STATUS_OK|\n|------|-----------------------|---------------|\n|SQL command|SELECT(including each SELECT in PROCEDURE)|INSERT / UPDATE / DELETE / ...|\n|Semantics|Read. One result set consists of a 2-dimensional structure
reprecenting the response of one read operation.|Write. One result set reprecents the results of
one write operation.|\n|main APIs|fetch_fields();
fetch_row(&row_arr);
...|get_insert_id();
get_affected_rows();
...|\n\nWhen errors occur in spliced commands, you may first get the multiple result sets through **MySQLResultCursor**, the commands which have been executed successfully. Then determine whether ``resp->get_packet_type()`` equals to **MYSQL_PACKET_ERROR** and get the specific error information through ``resp->get_error_code()`` and ``resp->get_error_msg()``.\n\nA **PROCEDURE** command containing N **SELECT** statements will return N result sets of **MYSQL_STATUS_GET_RESULT** and 1 result set of **MYSQL_STATUS_OK**. The user ignores this **MYSQL_STATUS_OK** result set is fine.\n\nTo get all the data, the specific steps should be:\n\n1. checking the task state (state at communication): you can check whether the task is successfully executed by checking whether ``task->get_state()`` is equal to **WFT_STATE_SUCCESS**;\n\n2. determining the type of the response packet (state at parsing the return packet): call ``resp->get_packet_type()`` to check the type of the last SQL query return packet. The common types include:\n\n- MYSQL\\_PACKET\\_OK: parsed successfully, should use cursor to get all the result sets.\n- MYSQL\\_PACKET\\_EOF: parsed successfully, should use cursor to get all the result sets.\n- MYSQL\\_PACKET\\_ERROR: requests: failed or partial failed, may use cursor to get the result sets of those successful commands.\n\n3. traverse the result sets: you can use **MySQLResultCursor** to read the content in the result set. Because the data returned by a MySQL server contains multiple result sets, the cursor will **automatically point to the reading position of the first result set** at first.\n\n4. checking the result set state (state at reading the result sets): **cursor->get_cursor_status()** returns the following states:\n\n- MYSQL\\_STATUS\\_GET\\_RESULT: current result set is a READ result set;\n- MYSQL\\_STATUS\\_END: the last record of the current READ result set has been read;\n- MYSQL\\_STATUS\\_OK: current result set is a WRITE result set;\n- MYSQL\\_STATUS\\_ERROR: parsing error;\n\n5. reading the basic content of **MYSQL_STATUS_OK** result set:\n - ``unsigned long long get_affected_rows() const;``\n - ``unsigned long long get_insert_id() const;``\n - ``int get_warnings() const;``\n - ``std::string get_info() const;``\n \n6. reading each field and each columns of **MYSQL_STATUS_GET_RESULT** result set:\n - `int get_field_count() const;`\n - `const MySQLField *fetch_field();`\n - `const MySQLField *const *fetch_fields() const;`\n\n7. reading each line of **MYSQL_STATUS_GET_RESULT** result set: you can use ``cursor->fetch_row()`` to read by row until the return value is false, in which the offset within the cursor that points to the row in the current result set will be moved:\n- `int get_rows_count() const;`\n- `bool fetch_row(std::vector& row_arr);`\n- `bool fetch_row(std::map& row_map);`\n- `bool fetch_row(std::unordered_map& row_map);`\n- `bool fetch_row_nocopy(const void **data, size_t *len, int *data_type);`\n\n8. taking out all the rows in the current **MYSQL_STATUS_GET_RESULT** result set directly: you can use ``cursor->fetch_all()`` to read all rows, and the cursor that is used to record the rows internally will be moved directly to the end; The cursor state changes to **MYSQL_STATUS_END**:\n- `bool fetch_all(std::vector>& rows);`\n\n9. returning to the head of the current **MYSQL_STATUS_GET_RESULT** result set: if it is necessary to read this result set again, you can use ``cursor->rewind()`` to return to the head of the current result set, and then read it via the operations in Step 7 or Step 8;\n\n10. getting the next result set: because the data packet returned by MySQL server may contains multiple result sets (for example, each SELECT/INSERT/... statement gets one result set; or the multiple result sets returned by calling a PROCEDURE). Therefore, you can use ``cursor->next_result_set()`` to jump to the next result set. If the return value is false, it means that all result sets have been taken.\n\n11. returning to the first result set: use **cursor->first\\_result\\_set()** to return to the heads of all result sets, and then you can repeat the operations from Step 4.\n\n12. getting the data of each column (MySQLCell): the row read in Step 5 is composed of multiple columns, and the result of each column is one MySQLCell. It mainly uses the following interfaces:\n\n- `int get_data_type();` returns MYSQL\\_TYPE\\_LONG, MYSQL\\_TYPE\\_STRING, and etc. For the details, please see [mysql\\_types.h](/src/protocol/mysql_types.h).\n- `bool is_TYPE() const;` the TYPE is int, string or ulonglong. It is used to check the data type.\n- `TYPE as_TYPE() const;` same as the above. It reads the data from MySQLCell in a certain type.\n- `void get_cell_nocopy(const void **data, size_t *len, int *data_type) const;` nocopy interface.\n\nThe whole example is shown below:\n\n~~~cpp\nvoid task_callback(WFMySQLTask *task)\n{\n // step-1. Check the status of the task\n if (task->get_state() != WFT_STATE_SUCCESS)\n {\n fprintf(stderr, \"task error = %d\\n\", task->get_error());\n return;\n }\n\n MySQLResultCursor cursor(task->get_resp());\n bool test_first_result_set_flag = false;\n bool test_rewind_flag = false;\n\n // step-2. Check other status of repsponse packet\n if (resp->get_packet_type() == MYSQL_PACKET_ERROR)\n {\n fprintf(stderr, \"ERROR. error_code=%d %s\\n\",\n task->get_resp()->get_error_code(),\n task->get_resp()->get_error_msg().c_str());\n }\n\nbegin:\n // step-3. Traverse the result sets\n do {\n // step-4. Check the status of the result set\n if (cursor.get_cursor_status() == MYSQL_STATUS_OK)\n {\n // step-5. Read the basic content of MYSQL_STATUS_OK result set\n fprintf(stderr, \"OK. %llu rows affected. %d warnings. insert_id=%llu.\\n\",\n cursor.get_affected_rows(), cursor.get_warnings(), cursor.get_insert_id());\n }\n else if (cursor.get_cursor_status() == MYSQL_STATUS_GET_RESULT)\n {\n fprintf(stderr, \"field_count=%u rows_count=%u \",\n cursor.get_field_count(), cursor.get_rows_count());\n\n // step-6. Read each fields. This is a nocopy api\n const MySQLField *const *fields = cursor.fetch_fields();\n for (int i = 0; i < cursor.get_field_count(); i++)\n {\n fprintf(stderr, \"db=%s table=%s name[%s] type[%s]\\n\",\n fields[i]->get_db().c_str(), fields[i]->get_table().c_str(),\n fields[i]->get_name().c_str(), datatype2str(fields[i]->get_data_type()));\n }\n\n // step-8. Read all the rows. You may use while (cursor.fetch_row(map/vector)) to get each rows accoding to step-7\n std::vector> rows;\n\n cursor.fetch_all(rows);\n for (unsigned int j = 0; j < rows.size(); j++)\n {\n // step-12. Read each cell\n for (unsigned int i = 0; i < rows[j].size(); i++)\n {\n fprintf(stderr, \"[%s][%s]\", fields[i]->get_name().c_str(),\n datatype2str(rows[j][i].get_data_type()));\n // step-12. Check the type wih is_string()and transform the type with as_string()\n if (rows[j][i].is_string())\n {\n std::string res = rows[j][i].as_string();\n fprintf(stderr, \"[%s]\\n\", res.c_str());\n }\n else if (rows[j][i].is_int())\n {\n fprintf(stderr, \"[%d]\\n\", rows[j][i].as_int());\n } // else if ...\n }\n }\n }\n // step-10. Get the next result set\n } while (cursor.next_result_set());\n\n if (test_first_result_set_flag == false)\n {\n test_first_result_set_flag = true;\n // step-11. Go back to the first result set\n cursor.first_result_set();\n goto begin;\n }\n\n if (test_rewind_flag == false)\n {\n test_rewind_flag = true;\n // step-9. Go back to the first position of the current result set\n cursor.rewind();\n goto begin;\n }\n\n return;\n}\n~~~\n\n# WFMySQLConnection\n\nSince it is a highly concurrent asynchronous client, this means that the client may have more than one connection to the server. As both MySQL transactions and preparation are stateful, in order to ensure that one transaction or preparation ocupies one connection exclusively, you can use our encapsulated secondary factory WFMySQLConnection to create a task. Each WFMySQLConnection guarantees that one connection is occupied exclusively. For the details, please see [WFMySQLConnection.h](/src/client/WFMySQLConnection.h).\n\n### 1\\. Creating and initializing WFMySQLConnection\n\nWhen creating a WFMySQLConnection, you need to pass in **id**, and the subsequent calls on this WFMySQLConnection will use this id and url to find the corresponding unique connection.\n\nWhen initializing a WFMySQLConnection, you need to pass a URL, and then you do not need to set the URL for the task created on this connection.\n\n~~~cpp\nclass WFMySQLConnection\n{\npublic:\n WFMySQLConnection(int id);\n int init(const std::string& url);\n ...\n};\n~~~\n\n### 2\\. Creating a task and closing a connection\n\nWith **create\\_query\\_task()**, you can create a task by writing an SQL request and a callback function. The task is garuanteed to be sent on this connection.\n\nSometimes you need to close this connection manually. Because when you stop using it, this connection will be kept until MySQL server time out. During this period, if you use the same id and url to create a WFMySQLConnection, you may reuse the connection.\n\nTherefore, we suggest that if you do not want to reuse the connection, you should use **create\\_disconnect\\_task()** to create a task and manually close the connection.\n\n~~~cpp\nclass WFMySQLConnection\n{\npublic:\n ...\n WFMySQLTask *create_query_task(const std::string& query,\n mysql_callback_t callback);\n WFMySQLTask *create_disconnect_task(mysql_callback_t callback);\n}\n~~~\n\nWFMySQLConnection is equivalent to a secondary factory. In the framework, we arrange that the life cycle of any factory object does not need to be maintained until the task ends. The following code is completely legal:\n\n~~~cpp\n WFMySQLConnection *conn = new WFMySQLConnection(1234);\n conn->init(url);\n auto *task = conn->create_query_task(\"SELECT * from table\", my_callback);\n conn->deinit();\n delete conn;\n task->start();\n~~~\n\n### 3\\. Cautions\n\nDo not generate new connection id infinitely, becauase easy id will occupy a little memory. The connection will be put to an internal connection pool if user didn't run a disconnect task, and will been reused by another WFMySQLConnection object initialized with the same id and url.\n\nRunning tasks on the same connection parallelly will fail with error **EAGAIN**.\n\nIf you have started `BEGIN` but have not `COMMIT` or `ROLLBACK` during the transaction and the connection has been interrupted during the transaction, the connection will be automatically reconnected internally by the framework, and you will get **ECONNRESET** error in the next task request. In this case, the transaction statements those have not been `COMMIT` would be expired and you may need to send them again.\n\n### 4\\. Preparation\n\nYou can also use the WFMySQLConnection for **PREPARE**. And you can easily use it to **defend against SQL injection**. If the connection is reconnected, you also get an **ECONNRESET** error.\n\n### 5\\. Complete example\n\n~~~cpp\nWFMySQLConnection conn(1);\nconn.init(\"mysql://root@127.0.0.1/test\");\n\n// test transaction\nconst char *query = \"BEGIN;\";\nWFMySQLTask *t1 = conn.create_query_task(query, task_callback);\nquery = \"SELECT * FROM check_tiny FOR UPDATE;\";\nWFMySQLTask *t2 = conn.create_query_task(query, task_callback);\nquery = \"INSERT INTO check_tiny VALUES (8);\";\nWFMySQLTask *t3 = conn.create_query_task(query, task_callback);\nquery = \"COMMIT;\";\nWFMySQLTask *t4 = conn.create_query_task(query, task_callback);\nWFMySQLTask *t5 = conn.create_disconnect_task(task_callback);\nSeriesWork *series = create_series_work(t1, nullptr);\n*series << t2 << t3 << t4 << t5;\nseries->start();\n~~~\n" }, { "path": "docs/en/tutorial-13-kafka_cli.md", "content": "# Asynchronous Kafka Client: kafka_cli\n\n# Sample Codes\n\n[tutorial-13-kafka_cli.cc](/tutorial/tutorial-13-kafka_cli.cc)\n\n# About Compiler\n\nBecause of supporting multiple compression methods of Kafka, [zlib](https://github.com/madler/zlib.git), [snappy](https://github.com/google/snappy.git), [lz4(>=1.7.5)](https://github.com/lz4/lz4.git), [zstd](https://github.com/facebook/zstd.git) and other third-party libraries are used in the compression algorithms in the Kafka protocol, and they must be installed before the compilation.\n\nIt supports both CMake and Bazel for compiling.\n\nCMake: You can use **make KAFKA=y** to compile a separate library for Kafka protocol(libwfkafka.a和libwfkafka.so) and use **cd tutorial; make KAFKA=y** to compile kafka_cli.\n\nBazel: You can use **bazel build kafka** to compile a separate library for Kafka protocol and use **bazel build kafka_cli** to compile kafka_cli.\n\n\n# About kafka_cli\n\nKafka_cli is a kafka client for producing and fetching messages in Kafka. \n\nWhen you compile the source codes, type the command **make KAFKA=y** in the **tutorial** directory or type the command **make KAFKA=y tutorial** in the root directory of the project.\n\nThe program then reads kafka broker server addresses and the current task type (produce/fetch) from the command line:\n\n./kafka_cli \\ [p/c]\n\nThe program exists automatically after all the tasks are completed, and all the resources will be completedly freed.\n\nIn the command, the broker_url may contain several urls seperated by comma(,).\n\n- For instance, kafka://host:port,kafka://host1:port... or: **kafkas**://host:port,**kafkas**://host1:port for kafka over SSL;\n- The default port is 9092 for TCP and 9093 for SSL;\n- Do not mix 'kafkas://' with \"kafka://\", otherwise the init function will fail with errno EINVAL;\n- If you want to use upstream policy at this layer, please refer to [upstream documents](/docs/en/about-upstream.md).\n\nThe following are several Kafka broker_url samples:\n\nkafka://127.0.0.1/\n\nkafka://kafka.host:9090/\n\nkafka://10.160.23.23:9000,10.123.23.23,kafka://kafka.sogou\n\nkafkas://broker1.kafka.sogou,kafkas://broker2.kafka.sogou\n\nIllegal broker_url sample (The first one is SSL, and the second one is not):\n\nkafkas://broker1.kafka.sogou,broker2.kafka.sogou\n\n# Principles and Features\n\nKafka client has no third-party dependencies internally except for the libraries used in the compression. With the high performance of Workflow, When properly configured and in fair environments, tens of thousands of Kafka requests can be processed in one second.\n\nInternally, a Kafka client divides each request into parallel tasks according to the brokers used. In parallel tasks, there is one sub-task for each broker address.\n\nIn this way, the efficiency is maximized. Besides, the connection reuse mechanism in the Workflow ensures that the total number of connections is kept within a reasonable range.\n\nIf there are multiple topic partitions under one broker address, you may create multiple clients and then create and start separate tasks for each topic partition to increase the throughput.\n\n# Creating and Starting Kafka Tasks\n\nTo create and start a Kafka task, create a **WFKafkaClient** first and then call **init** to initialize that **WFKafkaClient**.\n\n~~~cpp\nint init(const std::string& broker_url);\n\nint init(const std::string& broker_url, const std::string& group);\n~~~\n\nIn the above code snippet, **broker_url** means the address of the kafka broker cluster. Its format is the same as the broker_url in the above section.\n\n**group** means the group_name of a consumer group, which is used for the consumer group in a fetch task. In the case of produce tasks or fetch tasks without any consumer groups, do not use this interface.\n\nFor a consumer group, you can specify the heartbeat interval in milliseconds to keep the heartbeats.\n\n~~~cpp\nvoid set_heartbeat_interval(size_t interval_ms);\n~~~\n\nThen you can create a Kafka task with that **WFKafkaClient**.\n\n~~~cpp\nusing kafka_callback_t = std::function;\n\nWFKafkaTask *create_kafka_task(const std::string& query, int retry_max, kafka_callback_t cb);\n\nWFKafkaTask *create_kafka_task(int retry_max, kafka_callback_t cb);\n~~~\n\nIn the above code snippet, **query** includes the type of the task, the topic and other properties. **retry_max** means the maximum number of retries. **cb** is the user-defined callback function, which will be called after the task is completed. \n\nYou can also change the default settings of the task to meet the requirements. For details, refer to [KafkaDataTypes.h](/src/protocol/KafkaDataTypes.h).\n\n~~~cpp\nKafkaConfig config;\nconfig.set_client_id(\"workflow\");\ntask->set_config(std::move(config));\n~~~\n\nThe supported configuration items are described below: \n\nItem name | Type | Default value | Description \n------ | ---- | -------| ------- \nproduce_timeout | int | 100ms | Maximum time for produce. \nproduce_msg_max_bytes | int | 1000000 bytes | Maximum length for one message. \nproduce_msgset_cnt | int | 10000 | Maximun numbers of messges in one communication set \nproduce_msgset_max_bytes | int | 1000000 bytes | Maximum length of messages in one communication. \nfetch_timeout | int | 100ms | Maximum timeout for fetch. \nfetch_min_bytes | int | 1 byte | Minimum length of messages in one fetch communication. \nfetch_max_bytes | int | 50M bytes | Maximum length of messages in one fetch communication. \nfetch_msg_max_bytes | int | 1M bytes | Maximum length of one single message in a fetch communication. \noffset_timestamp | long long int | -1 | Initialized offfset in the consumer group mode when there is no offset history. -2 means the oldest offset; -1 means the latest offset. \nsession_timeout | int | 10s | Maximum initialization timeout for joining a consumer group. \nrebalance_timeout | int | 10s | Maximum timeout for synchronizing a consumer group information after a client joins the consumer group. \nproduce_acks | int | -1 | Number of brokers to ensure the successful replication of a message before the return of a produce task. -1 indicates all replica brokers. \nallow_auto_topic_creation | bool | true | Flag for controlling whether a topic is created automatically for the produce task if it does not exist. \nbroker_version | char * | NULL | Version number for brokers, which should be manually specified when the version number is smaller than 0.10. \ncompress_type | int | NoCompress | Compression type for produce messages. \nclient_id | char * | NULL | Identifier of a client. \ncheck_crcs | bool | false | Flag for controlling whether to check crc32 in the messages for a fetch task. \noffset_store | int | 0 | When joining the consumer group, whether to use the last submission offset, 1 means to use the specified offset, and 0 means to use the last submission preferentially.\nsasl_mechanisms | char * | NULL | Sasl certification type, currently only supports plain, and is on the ongoing development of sasl support.\nsasl_username | char * | NULL | Username required for sasl authentication.\nsasl_password | char * | NULL | Password required for sasl authentication.\n\n\nAfter configuring all the parameters, you can call **start** interface to start the Kafka task.\n\n# About Produce Tasks\n\n1\\. After you create and initialize a **WFKafkaClient**, you can specify the topic or other information in the **query** to create **WFKafkaTask** tasks.\n\nFor example:\n\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tclient = new WFKafkaClient();\n\tclient->init(url);\n\ttask = client->create_kafka_task(\"api=fetch&topic=xxx&topic=yyy\", 3, kafka_callback);\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n\n2\\. After the **WFKafkaTask** is created, call **set_key**, **set_value**, **add_header_pair** and other methods to build a **KafkaRecord**. \n\nFor information about more interfaces on **KafkaRecord**, refer to [KafkaDataTypes.h](/src/protocol/KafkaDataTypes.h).\n\nThen you can call **add_produce_record** to add a **KafkaRecord**. For the detailed definitions of the interfaces, refer to [WFKafkaClient.h](/src/client/WFKafkaClient.h).\n\nThe second parameter **partition** in **add_produce_record**, >=0 means the specified **partition**; -1 means that the **partition** is chosen randomly or the user-defined **kafka_partitioner_t** is used. \n\nFor **kafka_partitioner_t**, you can call the **set_partitioner** interface to specify the user-defined rules.\n\nFor example:\n\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tWFKafkaClient *client_fetch = new WFKafkaClient();\n\tclient_fetch->init(url);\n\ttask = client_fetch->create_kafka_task(\"api=produce&topic=xxx&topic=yyy\", 3, kafka_callback);\n\ttask->set_partitioner(partitioner);\n\n\tKafkaRecord record;\n\trecord.set_key(\"key1\", strlen(\"key1\"));\n\trecord.set_value(buf, sizeof(buf));\n\trecord.add_header_pair(\"hk1\", 3, \"hv1\", 3);\n\ttask->add_produce_record(\"workflow_test1\", -1, std::move(record));\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n\n3\\. You can use one of the four compressions supported by Kafka in the produce task by configuration. \n\nFor example:\n\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tWFKafkaClient *client_fetch = new WFKafkaClient();\n\tclient_fetch->init(url);\n\ttask = client_fetch->create_kafka_task(\"api=produce&topic=xxx&topic=yyy\", 3, kafka_callback);\n\n\tKafkaConfig config;\n\tconfig.set_compress_type(Kafka_Zstd);\n\ttask->set_config(std::move(config));\n\n\tKafkaRecord record;\n\trecord.set_key(\"key1\", strlen(\"key1\"));\n\trecord.set_value(buf, sizeof(buf));\n\trecord.add_header_pair(\"hk1\", 3, \"hv1\", 3);\n\ttask->add_produce_record(\"workflow_test1\", -1, std::move(record));\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n\n# About Fetch Tasks\n\nYou may use consumer group mode or manual mode for fetch tasks.\n\n1\\. Manual mode\n\nIn this mode, you do not need to specify consumer groups, but you must specify topic, partition and offset.\n\nFor example:\n\n~~~cpp\n\tclient = new WFKafkaClient();\n\tclient->init(url);\n\ttask = client->create_kafka_task(\"api=fetch\", 3, kafka_callback);\n\n\tKafkaToppar toppar;\n\ttoppar.set_topic_partition(\"workflow_test1\", 0);\n\ttoppar.set_offset(0);\n\ttask->add_toppar(toppar);\n~~~\n\n2\\. Consumer group mode\n\nIn this mode, you must specify the name of the consumer group when initializing a client.\n\nFor example:\n\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tWFKafkaClient *client_fetch = new WFKafkaClient();\n\tclient_fetch->init(url, cgroup_name);\n\ttask = client_fetch->create_kafka_task(\"api=fetch&topic=xxx&topic=yyy\", 3, kafka_callback);\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n\n3\\. Committing offset\n\nIn the consumer group mode, after a message is consumed, you can create a commit task in the callback to automatically submit the consumption record. \n\nFor example:\n\n~~~cpp\nvoid kafka_callback(WFKafkaTask *task)\n{\n\t...\n\tcommit_task = client.create_kafka_task(\"api=commit\", 3, kafka_callback);\n\n\t...\n\tcommit_task->start();\n\t...\n}\n~~~\n\n# Closing the Client\n\nIn the consumer group mode, before you close a client, you must call **create_leavegroup_task** to create a **leavegroup_task**.\n\nThis task will send a **leavegroup** packet. If no **leavegroup_task** is started, the group does not know that the client is leaving and will trigger rebalance.\n\n# Processing Kafka Results\n\nThe data structure of the message result set is KafkaResult, and you can call **get_result()** in the **WFKafkaTask** to retrieve the results.\n\nThen you can call the **fetch_record** in the **KafkaResult** to retrieve all records of the task. The record is a two-dimensional vector.\n\nThe first dimension is topic partition, and the second dimension is the **KafkaRecord** under that topic partition.\n\n[KafkaResult.h](/src/protocol/KafkaResult.h) contains the definition of **KafkaResult**.\n\n~~~cpp\nvoid kafka_callback(WFKafkaTask *task)\n{\n\tint state = task->get_state();\n\tint error = task->get_error();\n\n\t// handle error states\n\t...\n\n\tprotocol::KafkaResult *result = task->get_result();\n\tresult->fetch_records(records);\n\n\tfor (auto &v : records)\n\t{\n\t\tfor (auto &w: v)\n\t\t{\n\t\t\tconst void *value;\n\t\t\tsize_t value_len;\n\t\t\tw->get_value(&value, &value_len);\n\t\t\tprintf(\"produce\\ttopic: %s, partition: %d, status: %d, offset: %lld, val_len: %zu\\n\",\n\t\t\t\t w->get_topic(), w->get_partition(), w->get_status(), w->get_offset(), value_len);\n\t\t}\n\t}\n\t...\n\n\tprotocol::KafkaResult new_result = std::move(*task->get_result());\n\tif (new_result.fetch_records(records))\n\t{\n\t\tfor (auto &v : records)\n\t\t{\n\t\t\tif (v.empty())\n\t\t\t\tcontinue;\n\n\t\t\tfor (auto &w: v)\n\t\t\t{\n\t\t\t\tif (fp)\n\t\t\t\t{\n \tconst void *value;\n\t\t\t\t\tsize_t value_len;\n\t\t\t\t\tw->get_value(&value, &value_len);\n\t\t\t\t\tfwrite(w->get_value(), w->get_value_len(), 1, fp);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t...\n}\n~~~\n" }, { "path": "docs/en/xmake.md", "content": "# xmake compiling\n\n```\n// compile workflow library\nxmake\n\n// compile test\nxmake -g test\n// run test\nxmake run -g test\n\n// compile tutorial\nxmake -g tutorial\n\n// compile benchmark\nxmake -g benchmark\n```\n\n## running\n\n`xmake run -h` can see which targets you can run\n\nSelect a target to run, for instance:\n\n```\nxmake run tutorial-06-parallel_wget\n```\n\n## xmake install\n\n```\nsudo xmake install\n```\n\n## Compile static / shared library\n\n```\n// compile static lib\nxmake f -k static\nxmake -r\n```\n\n```\n// compile shard lib\nxmake f -k shared\nxmake -r\n```\n\n`tips : -r means -rebuild`\n\n## build options\n\n`xmake f --help` can see our defined options.\n\n```\nCommand options (Project Configuration):\n\n --workflow_inc=WORKFLOW_INC workflow inc (default: /media/psf/pro/workflow/_include)\n --upstream=[y|n] build upstream component (default: y)\n --consul=[y|n] build consul component\n --workflow_lib=WORKFLOW_LIB workflow lib (default: /media/psf/pro/workflow/_lib)\n --redis=[y|n] build redis component (default: y)\n --kafka=[y|n] build kafka component\n --mysql=[y|n] build mysql component (default: y)\n```\n\nYou can cut or integrate each components with the following commands\n\n```\nxmake f --redis=n --kafka=y --mysql=n\nxmake -r\n```\n" }, { "path": "docs/tutorial-01-wget.md", "content": "# 创建第一个任务：wget\n# 示例代码\n\n[tutorial-01-wget.cc](/tutorial/tutorial-01-wget.cc)\n\n# 关于wget\n程序从stdin读取http/https URL，抓取网页并把内容打印到stdout，并将请求和响应的http header打印在stderr。 \n为了简单起见，程序用Ctrl-C退出，但会保证所有资源先被完全释放。\n\n# 创建并启动http任务\n~~~cpp\nWFHttpTask *task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX, wget_callback);\nprotocol::HttpRequest *req = task->get_req();\nreq->add_header_pair(\"Accept\", \"*/*\");\nreq->add_header_pair(\"User-Agent\", \"Wget/1.14 (gnu-linux)\");\nreq->add_header_pair(\"Connection\", \"close\");\ntask->start();\npause();\n~~~\nWFTaskFactory::create_http_task()产生一个http任务，在[WFTaskFactory.h](../src/factory/WFTaskFactory.h)文件里，原型定义如下：\n~~~cpp\nWFHttpTask *create_http_task(const std::string& url,\n int redirect_max, int retry_max,\n http_callback_t callback);\n~~~\n前几个参数不用过多解释，http_callback_t是http任务的callback，定义如下：\n~~~cpp\nusing http_callback_t = std::function;\n~~~\n说白了，就是一个参数为Task本身，没有返回值的函数。这个callback可以传NULL，表示无需callback。我们一切任务的callback都是这个风格。 \n需要说明的是，所有工厂函数不会返回失败，所以不用担心task为空指针，哪怕是url不合法。一切错误都在callback再处理。 \ntask->get_req()函数得到任务的request，默认是GET方法，HTTP/1.1，长连接。框架会自动加上request_uri，Host等。 \n框架会在发送前根据需要自动加上Content-Length或Connection这些http header。用户也可以通过add_header_pair()方法添加自己的header。 \n关于http消息的更多接口，可以在[HttpMessage.h](../src/protocol/HttpMessage.h)中查看。 \ntask->start()启动任务，非阻塞，并且不会失败。之后callback必然会在被调用。因为异步的原因，start()以后显然不能再用task指针了。 \n为了让示例尽量简单，start()之后调用pause()防止程序退出，用户需要Ctrl-C结束程序。\n\n# 处理http抓取结果\n在这个示例中，我们使用一个普遍的函数处理结果。当然，std::function支持更多的功能。\n~~~cpp\nvoid wget_callback(WFHttpTask *task)\n{\n protocol::HttpRequest *req = task->get_req();\n protocol::HttpResponse *resp = task->get_resp();\n int state = task->get_state();\n int error = task->get_error();\n\n // handle error states\n ...\n\n std::string name;\n std::string value;\n // print request to stderr\n fprintf(stderr, \"%s %s %s\\r\\n\", req->get_method(), req->get_http_version(), req->get_request_uri());\n protocol::HttpHeaderCursor req_cursor(req);\n while (req_cursor.next(name, value))\n fprintf(stderr, \"%s: %s\\r\\n\", name.c_str(), value.c_str());\n fprintf(stderr, \"\\r\\n\");\n \n // print response header to stderr\n ...\n\n // print response body to stdin\n const void *body;\n size_t body_len;\n resp->get_parsed_body(&body, &body_len); // always success.\n fwrite(body, 1, body_len, stdout);\n fflush(stdout);\n}\n~~~\n在这个callback里，task就是我们通过工厂产生的task。 \ntask->get_state()与task->get_error()分别获得任务的运行状态和错误码。我们先略过错误处理的部分。 \ntask->get_resp()得到任务的response，这个和request区别不大，都是HttpMessage的派生。 \n之后通过HttpHeaderCursor对象，对request和response的header进行扫描。在[HttpUtil.h](../src/protocol/HttpUtil.h)可以看到Cursor的定义。\n\n~~~cpp\nclass HttpHeaderCursor\n{\npublic:\n HttpHeaderCursor(const HttpMessage *message);\n ...\n void rewind();\n ...\n bool next(std::string& name, std::string& value);\n bool find(const std::string& name, std::string& value);\n ...\n};\n~~~\n相信这个cursor在使用上应该不会有什么疑惑。 \n之后一行resp->get_parsed_body()获得response的http body。这个调用在任务成功的状态下，必然返回true，body指向数据区。 \n这个调用得到的是原始的http body，不解码chunk编码。如需解码chunk编码，可使用[HttpUtil.h](../src/protocol/HttpUtil.h)里的HttpChunkCursor。\n另外需要说明的是，find()接口会修改cursor内部的指针，即使用过find()过后如果仍然想对header进行遍历，需要通过rewind()接口回到cursor头部。\n\n" }, { "path": "docs/tutorial-02-redis_cli.md", "content": "# 实现一次redis写入与读出：redis_cli\n# 示例代码\n\n[tutorial-02-redis_cli.cc](/tutorial/tutorial-02-redis_cli.cc)\n\n# 关于redis_cli\n\n程序从命令行读入一个redis服务器地址，以及以一对key，value。执行SET命令写入这对KV，之后再读出验证写入是否成功。 \n程序运行方法：./redis_cli \\ \\ \\ \n为简单起见，程序需要用Ctrl-C结束。\n\n# Redis URL的格式\n\nredis://:password@host:port/dbnum?query#fragment \n如果是SSL，则为： \nrediss://:password@host:port/dbnum?query#fragment \npassword是可选项。port的缺省值是6379，dbnum缺省值0，范围0-15。 \nquery和fragment部分工厂里不作解释，用户可自行定义。比如，用户有upstream选取需求，可以自定义query和fragment。相关内容参考upstream文档。 \nredis URL示例： \nredis://127.0.0.1/ \nredis://:12345678@redis.some-host.com/1\n\n# 创建并启动Redis任务\n\n创建Redis任务与创建http任务并没有什么区别，少了redirect_max参数。\n~~~cpp\nusing redis_callback_t = std::function;\n\nWFRedisTask *create_redis_task(const std::string& url,\n int retry_max,\n redis_callback_t callback);\n~~~\n在这个示例里，我们想在redis task里存一些用户信息，包括url和key，以便在callback里使用。 \n当然，我们可利用std::function来绑定参数，但在这里我们利用了task里的void *user_data指针。这是task的一个public成员。\n~~~cpp\nstruct tutorial_task_data\n{\n std::sring url;\n std::string key;\n};\n...\nstruct tutorial_task_data data;\ndata.url = argv[1];\ndata.key = argv[2];\n\nWFRedisTask *task = WFTaskFactory::create_redis_task(data.url, RETRY_MAX, redis_callback);\n\nprotocol::RedisRequest *req = task->get_req();\nreq->set_request(\"SET\", { data.key, argv[3] });\n\ntask->user_data = &data;\ntask->start();\npause();\n~~~\n与http task的get_req()类似，redis task的get_req()返回任务对应的redis request。 \nRedisRequest提供的功能可以在[RedisMessage.h](../src/protocol/RedisMessage.h)查看。\n其中，set_request接口用于设置redis命令。 \n~~~cpp\nvoid set_request(const std::string& command, const std::vector& params);\n~~~\n相信经常使用redis的人，对这个接口不会有什么疑问。但必须注意，我们的请求是禁止SELECT命令和AUTH命令的。 \n因为用户每次请求并不能指定具体连接，SELECT之后下一次请求并不能保证在同一个连接上发起，那么这个命令对用户来讲没有任何意义。 \n对数据库选择和密码的指定，请在redis URL里完成。并且，必须是每次请求的URL都带着这些信息。 \n另外，我们的redis client是支持cluster模式的，可以自动处理MOVED和ASK回复并重定向。用户不能自己发送ASKING命令。 \n\n# 处理请求结果\n\n程序在SET命令成功之后，再发起一次GET命令，验证写入的结果。GET命令也用同一个callback。所以，函数里会判断这是哪个命令的结果。 \n同样，我们先忽略错误处理部分。\n~~~cpp\nvoid redis_callback(WFRedisTask *task)\n{\n protocol::RedisRequest *req = task->get_req();\n protocol::RedisResponse *resp = task->get_resp();\n int state = task->get_state();\n int error = task->get_error();\n protocol::RedisValue val;\n\n ...\n resp->get_result(val);\n std::string cmd;\n req->get_command(cmd);\n if (cmd == \"SET\")\n {\n tutorial_task_data *data = (tutorial_task_data *)task->user_data;\n WFRedisTask *next = WFTaskFactory::create_redis_task(data->url, RETRY_MAX, redis_callback);\n\n next->get_req()->set_request(\"GET\", { data->key });\n series_of(task)->push_back(next);\n fprintf(stderr, \"Redis SET request success. Trying to GET...\\n\");\n }\n else /* if (cmd == 'GET') */\n {\n // print the GET result\n ...\n fprintf(stderr, \"Finished. Press Ctrl-C to exit.\\n\");\n }\n}\n~~~\nRedisValue是一次redis request得到的结果，同样在[RedisMessage.h](../src/protocol/RedisMessage.h)里可以看到其接口。 \ncallback需要特别解释的，是series_of(task)->push_back(next)这个语句。因为这是我们第一次使用到Workflow的功能。 \n在这里next是我们下一个要发起的redis task，执行GET操作。我们并不是执行next->start()来启动任务，而是把next任务push_back到当前任务序列的末尾。 \n这两种方法的区别在于：\n * 用start来启动任务，任务是被立刻启动的，而push_back的方法，next任务是在callback结束之后被启动。\n * 最起码的好处是，push_back方法可以保证log打印不会乱。否则，用next->start()的话，示例中\"Finished.\"这个log可能会被先打印。\n * 用start来启动下一个任务的话，当前任务序列（series）就结束了，next任务会新启动一个新的series。\n * series是可以设置callback的，虽然在示例中没有用到。\n * 在并行任务里，series是并行任务的一个分枝，series结束就会认为分枝结束。并行相关内容在后续教程中讲解。\n\n总之，如果你想在一个任务之后启动下一个任务，一般是使用push_back操作来完成（还有些情况可能要用到push_front）。 \n而series_of()则是一个非常重要的调用，是一个不属于任何类的全局函数。其定义和实现在[Workflow.h](../src/factory/Workflow.h#L140)里：\n~~~cpp\nstatic inline SeriesWork *series_of(const SubTask *task)\n{\n return (SeriesWork *)task->get_pointer();\n}\n~~~\n任何task都是SubTask类型的派生。而任何运行中的task，一定属于某个series。通过series_of调用，得到了任务所在的series。 \n而push_back是SeriesWork类的一个调用，其功能是将一个task放到series的末尾。类似调用还有push_front。本示例中，用哪个调用并没有区别。\n~~~cpp\nclass SeriesWork\n{\n ...\npublic:\n void push_back(SubTask *task);\n void push_front(SubTask *task);\n ...\n}\n~~~\nSeriesWork类在我们整个体系中，扮演重要的角色。在下一个示例中，我们将展现SeriesWork更多的功能。\n" }, { "path": "docs/tutorial-03-wget_to_redis.md", "content": "# 任务序列的更多功能：wget_to_redis\n# 示例代码\n\n[tutorial-03-wget_to_redis.cc](/tutorial/tutorial-03-wget_to_redis.cc)\n\n# 关于wget_to_redis\n\n程序从命令行读入一条http URL和一条redis URL，把抓取下来的Http页面（以http URL为key）存入redis。 \n与之前两个示例不同，我们加入唤醒机制，让程序可以自动退出，无需Ctrl-C。\n\n# 创建Http任务并设置参数\n\n和上一个示例类似，本示例也是串行执行两个请求。最大的区别是，我们要通知主线程任务已经执行结束，并正常退出。 \n另外，我们多加入两个调用，限制一下http抓取返回内容的大小，以及接收回复的最大时间。\n~~~cpp\nWFHttpTask *http_task = WFTaskFactory::create_http_task(...);\n...\nhttp_task->get_resp()->set_size_limit(20 * 1024 * 1024);\nhttp_task->set_receive_timeout(30 * 1000);\n~~~\nset_size_limit()是HttpMessage的调用，用于限制接收http消息时包的大小。事实上所有的协议消息都要求提供这个接口。 \nset_receive_timeout()是接收数据的超时，单位为ms。 \n上述代码限制http消息不超过20M，完整接收时间不超过30秒。我们还有更多更丰富的超时配置，后述文档中再介绍。 \n\n# 创建并启动SeriesWork\n\n之前两组示例中，我们直接调用task->start()启动第一个任务。task->start()操作实际的工作方法是， \n先创建一个以task为首任务的SeriesWork，再启动这个series。在[WFTask.h](../src/factory/WFTask.h)里，可以看到start的实现：\n~~~cpp\ntemplate\nclass WFNetWorkTask : public CommRequest\n{\npublic:\n void start()\n {\n assert(!series_of(this));\n Workflow::start_series_work(this, nullptr);\n }\n ...\n};\n~~~\n我们想给series设置一个callback，并加入一些上下文。所以我们不使用任务的start接口，而是自己创建一个series。 \nSeriesWork不能new，delete，也不能派生。只能通过Workflow::create_series_work()接口产生。在[Workflow.h](../src/factory/Workflow.h)中， \n通常是用这个调用：\n~~~cpp\nusing series_callback_t = std::function;\n\nclass Workflow\n{\npublic:\n static SeriesWork *create_series_work(SubTask *first, series_callback_t callback);\n};\n~~~\n在示例代码中，我们的用法是：\n~~~cpp\nstruct tutorial_series_context\n{\n std::string http_url;\n std::string redis_url;\n size_t body_len;\n bool success;\n};\n...\nstruct tutorial_series_context context;\n...\nSeriesWork *series = Workflow::create_series_work(http_task, series_callback);\nseries->set_context(&context);\nseries->start();\n~~~\n之前的示例，我们用task里的void *user_data指针保存上下文信息。但这个示例中，我们把上文信息放在series里， \n这么做显然更合理一些，series是完整的任务链，所有任务都能得到并修改上下文。 \nseries的callback函数在series所有任务被执行完之后调用，在这里，我们简单的用一个lamda函数，打印运行结果并唤醒主线程。 \n\n# 其余的工作\n\n剩下的事情就没有什么特别的了，http抓取成功之后启动一个redis任务写库。如果抓取失败或http body长度为0，则不再启动redis任务。 \n无论是什么情况，程序都能在所有任务结束之后正常退出，因为任务都在同一个series里。\n" }, { "path": "docs/tutorial-04-http_echo_server.md", "content": "# 第一个server：http_echo_server\n# 示例代码\n\n[tutorial-04-http_echo_server.cc](/tutorial/tutorial-04-http_echo_server.cc)\n\n# 关于http_echo_server\n\n这是一个http server，返回一个html页面，显示浏览器发送的http请求的header信息。 \n程序log里会打印出请求的client地址，请求序号（当前连接上的第几次请求）。当同一连接上完成10次请求，server主动关闭连接。 \n程序通过Ctrl-C正常结束，一切资源完全回收。\n\n# 创建与启动http server\n\n本示例里，我们采用http server的默认参数。创建和启动过程非常简单。\n~~~cpp\nWFHttpServer server(process);\nport = atoi(argv[1]);\nif (server.start(port) == 0)\n{\n pause();\n server.stop();\n}\n...\n~~~\n这个过程实在太简单，没有什么好讲。要注意start是非阻塞的，所以要pause住程序。显然你也可以启动多个server对象再pause。 \nserver启动之后，任何时刻都可以通过stop()接口关停server。关停是非暴力式的，会等待正在服务的请求执行完。 \n所以，stop是一个阻塞操作。如果需要非阻塞的关闭，可使用shutdown+wait_finish接口。 \nstart()接口有好几个重载函数，在[WFServer.h](../src/server/WFServer.h)里，可以看到如下一些接口：\n~~~cpp\nclass WFServerBase\n{\npublic:\n /* To start TCP server. */\n int start(unsigned short port);\n int start(int family, unsigned short port);\n int start(const char *host, unsigned short port);\n int start(int family, const char *host, unsigned short port);\n int start(const struct sockaddr *bind_addr, socklen_t addrlen);\n\n /* To start an SSL server */\n int start(unsigned short port, const char *cert_file, const char *key_file);\n int start(int family, unsigned short port,\n const char *cert_file, const char *key_file);\n int start(const char *host, unsigned short port,\n const char *cert_file, const char *key_file);\n int start(int family, const char *host, unsigned short port,\n const char *cert_file, const char *key_file);\n int start(const struct sockaddr *bind_addr, socklen_t addrlen,\n const char *cert_file, const char *key_file);\n\n /* For graceful restart or multi-process server. */\n int serve(int listen_fd);\n int serve(int listen_fd, const char *cert_file, const char *key_file);\n\n /* Get the listening address. Used when started a server on a random port. */\n int get_listen_addr(struct sockaddr *addr, socklen_t *addrlen) const;\n};\n~~~\n这些接口都比较好理解。任何一个start函数，当端口号为0时，将使用随机端口。此时用户可能需要在server启动完成之后通过get_listen_addr获得实际监听地址。 \n启动SSL server时，cert_file和key_file为PEM格式。 \n最后两个带listen_fd的serve()接口，主要用于优雅重启。或者简单建立一个非TCP协议（如SCTP）的server。 \n需要特别提醒一下，我们一个server对象对应一个listen_fd，如果在IPv4和IPv6两个协议上都运行server，需要：\n~~~cpp\n{\n WFHttpServer server_v4(process);\n WFHttpServer server_v6(process);\n server_v4.start(AF_INET, port);\n server_v6.start(AF_INET6, port);\n ...\n // now stop...\n server_v4.shutdown(); /* shutdown() is nonblocking */\n server_v6.shutdown();\n server_v4.wait_finish();\n server_v6.wait_finish();\n}\n~~~\n这种方式我们没有办法让两个server共享连接记数。所以推荐只启动IPv6 server，因为IPv6 server可以接受IPv4的连接。\n\n# http echo server的业务逻辑\n\n我们看到在构造http server的时候，传入了一个process参数，这也是一个std::function，定义如下： \n~~~cpp\nusing http_process_t = std::function;\nusing WFHttpServer = WFServer;\n\ntemplate<>\nWFHttpServer::WFServer(http_process_t proc) :\n WFServerBase(&HTTP_SERVER_PARAMS_DEFAULT),\n process(std::move(proc))\n{\n}\n~~~\n其实这个http_proccess_t和的http_callback_t类型是完全一样的。都是处理一个WFHttpTask。 \n对server来讲，我们的目标就是根据request，填写好response。 \n同样我们用一个普通函数实现process。逐条读出request的http header写入html页面。\n~~~cpp\nvoid process(WFHttpTask *server_task)\n{\n protocol::HttpRequest *req = server_task->get_req();\n protocol::HttpResponse *resp = server_task->get_resp();\n long seq = server_task->get_task_seq();\n protocol::HttpHeaderCursor cursor(req);\n std::string name;\n std::string value;\n char buf[8192];\n int len;\n\n /* Set response message body. */\n resp->append_output_body_nocopy(\"\", 6);\n len = snprintf(buf, 8192, \"

%s %s %s

\", req->get_method(),\n req->get_request_uri(), req->get_http_version());\n resp->append_output_body(buf, len);\n\n while (cursor.next(name, value))\n {\n len = snprintf(buf, 8192, \"

%s: %s

\", name.c_str(), value.c_str());\n resp->append_output_body(buf, len);\n }\n\n resp->append_output_body_nocopy(\"\", 7);\n\n /* Set status line if you like. */\n resp->set_http_version(\"HTTP/1.1\");\n resp->set_status_code(\"200\");\n resp->set_reason_phrase(\"OK\");\n\n resp->add_header_pair(\"Content-Type\", \"text/html\");\n resp->add_header_pair(\"Server\", \"Sogou WFHttpServer\");\n if (seq == 9) /* no more than 10 requests on the same connection. */\n resp->add_header_pair(\"Connection\", \"close\");\n\n // print log\n ...\n}\n~~~\n大多数HttpMessage相关操作之前已经介绍过了，在这里唯一的一个新操作是append_output_body()。 \n显然让用户生成完整的http body再传给我们并不太高效。用户只需要调用append接口，把离散的数据一块块扩展到message里就可以了。 \nappend_output_body()操作会把数据复制走，另一个带_nocopy后缀的接口会直接引用指针，使用时需要注意不可以指向局部变量。 \n相关几个调用在[HttpMessage.h](../src/protocol/HttpMessage.h)可以看到其声明：\n~~~cpp\nclass HttpMessage\n{\npublic:\n bool append_output_body(const void *buf, size_t size);\n bool append_output_body_nocopy(const void *buf, size_t size);\n ...\n bool append_output_body(const std::string& buf);\n};\n~~~\n再次强调，使用append_output_body_nocopy()接口时，buf指向的数据的生命周期至少需要延续到task的callback。 \n函数中另外一个变量seq，通过server_task->get_task_seq()得到，表示该请求是当前连接上的第几次请求，从0开始计。 \n程序中，完成10次请求之后就强行关闭连接，于是：\n~~~cpp\n if (seq == 9) /* no more than 10 requests on the same connection. */\n resp->add_header_pair(\"Connection\", \"close\");\n~~~\n关闭连接还可以通过task->set_keep_alive()接口来完成，但对于http协议，还是推荐使用设置header的方式。 \n这个示例中，因为返回的页面很小，我们没有关注回复成功与否。下一个示例http_proxy我们将看到如果获得回复的状态。\n\n" }, { "path": "docs/tutorial-05-http_proxy.md", "content": "# 异步server的示例：http_proxy\n# 示例代码\n\n[tutorial-05-http_proxy.cc](/tutorial/tutorial-05-http_proxy.cc)\n\n# 关于http_proxy\n\n这是一个http代理服务器，可以配置在浏览器里使用。支持所有的http method。 \n因为https代理的原理不同，这个示例并不支持https代理，你只能浏览http网站。 \n这个proxy在实现上需要抓取下来完整的http页面再转发，下载/上传大文件会有延迟。\n\n# 修改server配置\n\n之前的示例我们使用了默认的http server参数。但这个例子里，我们做一点修改，限制请求的大小，防止被恶意攻击。\n~~~cpp\nint main(int argc, char *argv[])\n{\n ...\n struct WFServerParams params = HTTP_SERVER_PARAMS_DEFAULT;\n params.request_size_limit = 8 * 1024 * 1024;\n\n WFHttpServer server(¶ms, process);\n if (server.start(port) == 0)\n {\n pause();\n server.stop();\n }\n else\n {\n perror(\"cannot start server\");\n exit(1);\n }\n\n return 0; \n}\n~~~\n与上一个示例不同，我们在server构造，多传入一个参数结构。我们可以看看http server有哪些配置。 \n在[WFHttpServer.h](../src/server/WFHttpServer.h)里，http server的默认参数如下：\n~~~cpp\nstatic constexpr struct WFServerParams HTTP_SERVER_PARAMS_DEFAULT =\n{\n .transport_type = TT_TCP,\n .max_connections = 2000,\n .peer_response_timeout = 10 * 1000,\n .receive_timeout = -1,\n .keep_alive_timeout = 60 * 1000,\n .request_size_limit = (size_t)-1,\n .ssl_accept_timeout = 10 * 1000,\n};\n~~~\ntransport_type：传输层协议，默认为TCP。除了TT_TCP外，可选择的还有TT_UDP和Linux下支持的TT_SCTP。 \nmax_connections：最大连接数2000，达到上限之后会关闭最久未使用的keep-alive连接。没找到keep-alive连接，则拒绝新连接。 \npeer_response_timeout：每读取到一块数据或发送出一块数据的超时时间为10秒。 \nreceive_timeout：接收一条完整的请求超时时间为-1，无限。 \nkeep_alive_timeout：连接保持1分钟。 \nrequest_size_limit：请求包最大大小，无限制。 \nssl_accept_timeout：完成ssl握手超时，10秒。 \n参数里没有send_timeout，即完整的回复超时。这个参数需要每次请求根据自己回复包的大小来确定。 \n\n# 代理服务器业务逻辑\n\n这个代理服务器本质上是将用户请求原封不动转发到对应的web server，再将web server的回复原封不动转发给用户。\n浏览器发给proxy的请求里，request uri包含了scheme和host，port，转发时需要去除。 \n例如，访问`http://www.sogou.com/`， 浏览器发送给proxy请求首行是： \n`GET` `http://www.sogou.com/` `HTTP/1.1`\n需要改写为： \n`GET` `/` `HTTP/1.1` \n~~~cpp\nvoid process(WFHttpTask *proxy_task)\n{\n auto *req = proxy_task->get_req();\n SeriesWork *series = series_of(proxy_task);\n WFHttpTask *http_task; /* for requesting remote webserver. */\n\n tutorial_series_context *context = new tutorial_series_context;\n context->url = req->get_request_uri();\n context->proxy_task = proxy_task;\n\n series->set_context(context);\n series->set_callback([](const SeriesWork *series) {\n delete (tutorial_series_context *)series->get_context();\n });\n\n http_task = WFTaskFactory::create_http_task(req->get_request_uri(), 0, 0,\n http_callback);\n\n const void *body;\n size_t len;\n\n /* Copy user's request to the new task's reuqest using std::move() */\n req->set_request_uri(http_task->get_req()->get_request_uri());\n req->get_parsed_body(&body, &len);\n req->append_output_body_nocopy(body, len);\n *http_task->get_req() = std::move(*req);\n\n /* also, limit the remote webserver response size. */\n http_task->get_resp()->set_size_limit(200 * 1024 * 1024);\n\n *series << http_task;\n}\n~~~\n以上是process的全部内容。先解析向web server发送的http请求的构造。 \nreq->get_request_uri()调用得到浏览器请求的完整URL，通过这个URL构建发往server的http任务。 \n这个http任务重试与重定向次数都是0，因为重定向是由浏览器处理，遇到302等会重新发请求。 \n~~~cpp\n req->set_request_uri(http_task->get_req()->get_request_uri());\n req->get_parsed_body(&body, &len);\n req->append_output_body_nocopy(body, len);\n *http_task->get_req() = std::move(*req);\n~~~\n上面4个语句，其实是在生成发往web server的http请求。req是我们收到的http请求，我们最终要通过std::move()把它直接移动到新请求上。 \n第一行实际上就是将request_uri里的`http://host:port`部分去掉，只保留path之后的部分。 \n第二第三行把解析下来的http body指定为向外输出的http body。需要做这个操作的原因是，我们的HttpMessage实现里，\n解析得到的body和发送请求的body是两个域，所以这里需要简单的设置一下，无需复制内存。 \n第四行，一次性把请求内容转移给向web server发送的请求。\n构造好http请求后，将这个请求放到当前series末尾，process函数结束。\n\n# 异步server的工作原理\n\n显然process函数并不是proxy逻辑的全部，我们还需要处理从web server返回的http response，填写返回给浏览器的response。 \n在echo server的示例里，我们并不需要进行网络通信，直接填写返回页面就好。但proxy我们需要等待web server的结果。 \n我们当然可以占用这个process函数的线程，等待结果返回，但这种同步等待的方式明显不是我们想要的。 \n那么，我们就需要在异步得到请求结果之后，再去回复用户请求，在等待结果期间，不能占用任何的线程。 \n所以，在process的头部，我们给当前series设置了一个context，context里包含了proxy_task本身，以便我们异步填写结果。\n~~~cpp\nstruct tutorial_series_context\n{\n std::string url;\n WFHttpTask *proxy_task;\n bool is_keep_alive;\n};\n\nvoid process(WFHttpTask *proxy_task)\n{\n SeriesWork *series = series_of(proxy_task);\n ...\n tutorial_series_context *context = new tutorial_series_context;\n context->url = req->get_request_uri();\n context->proxy_task = proxy_task;\n\n series->set_context(context);\n series->set_callback([](const SeriesWork *series) {\n delete (tutorial_series_context *)series->get_context();\n });\n ...\n}\n~~~\n之前client的示例中我们说过，任何一个运行中的任务，都处在一个series里，server任务也不例外。 \n所以，我们可以得到当前series，并设置context。其中url主要是后续打日志之用，proxy_task是主要内容，后续需要填写resp。 \n接下来我们就可以看看处理web server响应的部分了。\n~~~cpp\nvoid http_callback(WFHttpTask *task)\n{\n int state = task->get_state();\n auto *resp = task->get_resp();\n SeriesWork *series = series_of(task);\n tutorial_series_context *context =\n (tutorial_series_context *)series->get_context();\n auto *proxy_resp = context->proxy_task->get_resp();\n\n ...\n if (state == WFT_STATE_SUCCESS)\n {\n const void *body;\n size_t len;\n\n /* set a callback for getting reply status. */\n context->proxy_task->set_callback(reply_callback);\n\n /* Copy the remote webserver's response, to proxy response. */\n resp->get_parsed_body(&body, &len);\n resp->append_output_body_nocopy(body, len);\n *proxy_resp = std::move(*resp);\n ...\n }\n else\n {\n // return a \"404 Not found\" page\n ...\n }\n}\n~~~\n我们只关注成功的情况。一切可以从web server得到一个完整http页面，不管什么返回码，都是成功。所有失败的情况，简单返回一个404页面。\n因为返回给用户的数据可能很大，在我们这个示例里，设置为200MB上限。所以，和之前的示例不同，我们需要查看reply成功/失败状态。 \nhttp server任务和我们自行创建的http client任务的类型是完全相同的，都是WFHttpTask。不同的是server任务是框架创建的，它的callback初始为空。 \nserver任务的callback和client一样，是在http交互完成之后被调用。所以，对server任务来讲，就是reply完成之后被调用。 \n后面三行代码我们应该很熟悉了，无拷贝地将web server响应包转移到proxy响应包。 \n在这个http_callback函数结束之后，对浏览器的回复被发送出，一切都是在异步的过程中进行。 \n剩下的一个函数是reply_callback()，在这里只为了打印一些log。在这个callback执行结束后，proxy task会被自动delete。 \n最后，series的callback里销毁context。\n\n# Server回复的时机\n\n这里需要说明一下，回复消息的时机是在series里所有其它任务被执行完后，自动回复，所以并没有task->reply()接口。 \n但是，有task->noreply()调用，如果对server任务执行了这个调用，在原本回复的时刻，直接关闭连接。但callback依然会被调用（状态为NOREPLY）。 \n在server任务的callback里，同样可以通过series_of()操作获得任务的series。那么，我们依然可以往这个series里追加新任务，虽然回复已经完成。 \n\n" }, { "path": "docs/tutorial-06-parallel_wget.md", "content": "# 一个简单的并行抓取：parallel_wget\n# 示例代码\n\n[tutorial-06-parallel_wget.cc](/tutorial/tutorial-06-parallel_wget.cc)\n\n# 关于parallel_wget\n\n这是我们第一个并行任务的示例。 \n程序从命令行读入多个http URL（以空格分割），并行抓取这些URL，并按照输入顺序将抓取结果打印到标准输出。\n\n# 创建并行任务\n\n之前的示例里，我们已经接触过了SeriesWork类。\n * SeriesWork由任务构成，代表一系列任务的串行执行。所有任务结束，则这个series结束。 \n * 与SeriesWork对应的ParallelWork类，parallel由series构成，代表若干个series的并行执行。所有series结束，则这个parallel结束。 \n * ParallelWork是一种任务。 \n\n根据上述的定义，我们就可以动态或静态的生成任意复杂的工作流了。 \nWorkflow类里，有两个接口用于产生并行任务：\n~~~cpp\nclass Workflow\n{\n ...\npublic:\n static ParallelWork *\n create_parallel_work(parallel_callback_t callback);\n\n static ParallelWork *\n create_parallel_work(SeriesWork *const all_series[], size_t n,\n parallel_callback_t callback);\n\n ...\n};\n~~~\n第一个接口创建一个空的并行任务，第二个接口用一个series数组创建并行任务。 \n无论用哪个接口产生的并行任务，在启动之前都可以用ParallelWork的add_series()接口添加series。 \n在示例代码里，我们创建一个空的并行任务，并逐个添加series。\n~~~cpp\nint main(int argc, char *argv[])\n{\n ParallelWork *pwork = Workflow::create_parallel_work(callback);\n SeriesWork *series;\n WFHttpTask *task;\n HttpRequest *req;\n tutorial_series_context *ctx;\n int i;\n\n for (i = 1; i < argc; i++)\n {\n std::string url(argv[i]);\n ...\n task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n [](WFHttpTask *task)\n {\n // store resp to ctx.\n });\n\n req = task->get_req();\n // add some headers.\n ...\n\n ctx = new tutorial_series_context;\n ctx->url = std::move(url);\n series = Workflow::create_series_work(task, nullptr);\n series->set_context(ctx);\n pwork->add_series(series);\n }\n ...\n}\n~~~\n从代码中看到，我们先创建http任务，但http任务并不能直接加入到并行任务里，需要先用它创建一个series。 \n每个series都带有context，用于保存url和抓取结果。相关的方法我们在之前的示例里都介绍过。\n\n# 保存和使用抓取结果\n\nhttp任务的callback是一个简单的lambda函数，把抓取结果保存在自己的series context里，以便并行任务获取。\n~~~cpp\n task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n [](WFHttpTask *task)\n {\n tutorial_series_context *ctx =\n (tutorial_series_context *)series_of(task)->get_context();\n ctx->state = task->get_state();\n ctx->error = task->get_error();\n ctx->resp = std::move(*task->get_resp());\n });\n~~~\n这个做法是必须的，因为http任务在callback之后就会被回收，我们只能把resp通过std::move()操作移走。 \n而在并行任务的callback里，我们可以很方便的获得结果：\n~~~cpp\nvoid callback(const ParallelWork *pwork)\n{\n tutorial_series_context *ctx;\n const void *body;\n size_t size;\n size_t i;\n\n for (i = 0; i < pwork->size(); i++)\n {\n ctx = (tutorial_series_context *)pwork->series_at(i)->get_context();\n printf(\"%s\\n\", ctx->url.c_str());\n if (ctx->state == WFT_STATE_SUCCESS)\n {\n ctx->resp.get_parsed_body(&body, &size);\n printf(\"%zu%s\\n\", size, ctx->resp.is_chunked() ? \" chunked\" : \"\");\n fwrite(body, 1, size, stdout);\n printf(\"\\n\");\n }\n else\n printf(\"ERROR! state = %d, error = %d\\n\", ctx->state, ctx->error);\n\n delete ctx;\n }\n}\n~~~\n在这里，我们看到ParallelWork的两个新接口，size()和series_at(i)，分别获得它的并行series个数，和第i个并行series。 \n通过series->get_context()取到对应series的上下文，打印结果。打印顺序必然和我们放入顺序一致。 \n在这个示例中，并行任务执行完就没有其它工作了。 \n我们上面说过，ParallelWork是一种任务，所以同样我们可以用series_of()获得它所在的series并添加新任务。 \n但是，如果新任务还要使用到抓取结果，我们需要再次用std::move()把数据移到并行任务所在series的上下文里。 \n\n# 并行任务启动\n\n并行任务是一种任务，所以并行任务的启动并没有什么特别，可以直接调用start()，也可以用它建立或启动一个series。 \n在这个示例里，我们启动一个series，在这个series的callback里唤醒主进程，正常退出程序。 \n我们也可以在并行任务的callback里唤醒主进程，程序行为上区别不大。但在series callback里唤醒更加规范一点。\n\n" }, { "path": "docs/tutorial-07-sort_task.md", "content": "# 使用内置算法工厂：sort_task\n# 示例代码\n\n[tutorial-07-sort_task.cc](/tutorial/tutorial-07-sort_task.cc)\n\n# 关于sort_task\n\n程序从命令行读入数字n，将随机的n个正整数先升序排列，再把结果再降序排列。 \n程序可加入第二个参数\"p\"，则可以进行并行排序。例如： \n$ ./sort_task 100000000 p \n上面的命令将先升序排列1亿个整数，再降序排列。两次排序都采用并行。 \n\n# 关于计算任务\n\n计算任务（或称线程任务），是我们非常重要的一个功能。在使用我们任务流的时候，并不建议在callback里直接进行非常复杂的计算。 \n所有需要消耗大量CPU时间的计算，都可以封装成计算任务交给系统去调度。计算任务和通信任务在使用方法上并没有什么区别。 \n系统的算法工厂提供了一些常用的计算任务，比如排序，归并等。用户也可以很方便定义自己的计算任务。\n\n# 创建升序排序任务\n~~~cpp\nint main(int argc, char *argv[])\n{\n ...\n WFSortTask *task;\n if (use_parallel_sort)\n task = WFAlgoTaskFactory::create_psort_task(\"sort\", array, end, callback);\n else\n task = WFAlgoTaskFactory::create_sort_task(\"sort\", array, end, callback);\n ...\n task->start();\n ...\n}\n~~~\n和WFHttpTask或WFRedisTask不同，排序任务多了一个模板参数代表要排序的数组数据类型。 \ncreate_sort_task和create_psort_task分别产生一个普通排序任务和一个并行排序任务。 \n这两个调用的参数和返回值并没有区别。 \n唯一需要特别说明的是第一个参数\"sort\"，这个是计算队列名，用于影响内部的任务调度。本篇文档后面会介绍队列名的用法。 \n计算任务的启动方法与使用方法和网络通信任务并没有什么区别。 \n\n# 处理结果\n\n和通信任务一样，我们在callback里处理结果。这个示例里，升序排序之后会再发起一次降序排序。\n~~~cpp\nusing namespace algorithm;\n\nvoid callback(void SortTask *task)\n{\n SortInput *input = task->get_input();\n int *first = input->first;\n int *last = input->last;\n\n // print result\n ...\n \n if (task->user_data == NULL)\n {\n auto cmp = [](int a1, int a2){ return a2 < a1; };\n WFSortTask *reverse;\n\n if (use_parallel_sort)\n reverse = WFAlgoTaskFactory::create_psort_task(\"sort\", first, last, cmp, callback);\n else\n reverse = WFAlgoTaskFactory::create_sort_task(\"sort\", first, last, cmp, callback);\n \n reverse->user_data = (void *)1; /* as a flag */\n series_of(task)->push_back(reverse);\n }\n else\n {\n // all done. Signal main thread to exit.\n ... \n }\n}\n~~~\n计算任务的get_input()接口得到输入数据，get_output()得到输出数据。对于排序任务，输入和输出是相同类型，内容也完全相同。 \n在[WFAlgoTaskFactory.h](../src/factory/WFAlgoTaskFactory.h)里，可以看到排序任务输入输出的定义：\n~~~cpp\nnamespace algorithm\n{\n\ntemplate \nstruct SortInput\n{\n T *first;\n T *last;\n};\n\ntemplate \nusing SortOutput = SortInput;\n\n}\n\ntemplate \nusing WFSortTask = WFThreadTask,\n algorithm::SortOutput>;\n\ntemplate \nusing sort_callback_t = std::function *)>;\n\n~~~\n显然，input或output里的first, last分别为排序数组的首尾指针。 \n接下来我们会创建一个降序排序的任务，这时候，我们就需要传进去一个比较函数了。 \n~~~cpp\n auto cmp = [](int a1, int a2)->bool{ return a2 < a1; };\n reverse = WFAlgoTaskFactory::create_sort_task(\"sort\", first, last, cmp, callback);\n~~~\n可以说我们的用法和std::sort()区别不是很大。但我们的first和last是指针，而不是用iterator。 \n同样，用create_psort_task()可以创建一个并行排序任务。而对series的使用，和通信任务没有区别。 \n\n# 关于计算线程数的配置\n\n如果你不做任何配置，计算调度器将使用当前机器CPU个数的线程数。你也可以通过以下的方式，修改这个值：\n~~~cpp\n#include \"workflow/WFGlobal.h\"\n\nint main()\n{\n struct WFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;\n settings.compute_threads = 16;\n WORKFLOW_library_init(&settings);\n ...\n}\n~~~\n通过上面的配置，我们将创建16个线程用于计算。\n\n# 关于并行排序算法\n\n内置的并行排序算法，使用分块+二路归并。空间复杂度为O(1)。 \n算法使用全局配置的计算线程进行计算，但最多使用128个线程。因为不使用额外空间，加速比会小于线程数量，平均CPU占用也比较小。 \n具体实现可参考[WFAlgoTaskFactory.inl](../src/factory/WFAlgoTaskFactory.inl)\n\n# 关于计算队列名\n\n我们的计算任务并没有优化级的概念，唯一可以影响调度顺序的是计算任务的队列名，本示例中队列名为字符串\"sort\"。 \n队列名的指定非常简单，需要说明以下几点： \n * 队列名是一个静态字符串，不可以无限产生新的队列名。例如不可以根据请求id来产生队列名，因为内部会为每个队列分配一小块资源。 \n * 当计算线程没有被100%占满，所有任务都是实时调起，队列名没有任何影响。\n * 如果一个服务流程里有多个计算步骤，穿插在多个网络通信之间，可以简单的给每种计算步骤起一个名字，这个会比整体用一个名字要好。\n * 如果所有计算任务用同一个名字，那么所有任务的被调度的顺序与提交顺序一致，在某些场景下会影响平均响应时间。\n * 每种计算任务有一个独立名字，那么相当于每种任务之间是公平调度的，而同一种任务内部是顺序调度的，实践效果更好。\n * 总之，除非机器的计算负载已经非常繁重，否则没有必要特别关心队列名，只要每种任务起一个名字就可以了。\n" }, { "path": "docs/tutorial-08-matrix_multiply.md", "content": "# 自定义计算任务：matrix_multiply\n# 示例代码\n\n[tutorial-08-matrix_multiply.cc](/tutorial/tutorial-08-matrix_multiply.cc)\n\n# 关于matrix_multiply\n\n程序执行代码里两个矩阵的乘法，并将相乘结果打印在屏幕上。 \n示例的主要目的是展现怎么实现一个自定义CPU计算任务。\n\n# 定义计算任务\n\n定义计算任务需要提供3个基本信息，分别为INPUT，OUTPUT，和routine。 \nINPUT和OUTPUT是两个模板参数，可以是任何类型。routine表示从INPUT到OUTPUT的过程，定义如下： \n~~~cpp\ntemplate \nclass __WFThreadTask\n{\n ...\n std::function routine;\n ...\n};\n~~~\n可以看出routine是一个简单的从INPUT到OUTPUT的计算过程。INPUT指针不要求是const，但用户也可以传const INPUT *的函数。 \n比如一个加法任务，就可这么做：\n~~~cpp\nstruct add_input\n{\n int x;\n int y;\n};\n\nstruct add_ouput\n{\n int res;\n};\n\nvoid add_routine(const add_input *input, add_output *output)\n{\n output->res = input->x + input->y;\n}\n\ntypedef WFThreadTask add_task;\n~~~\n在我们的矩阵乘法的示例里，输入是两个矩阵，输出为一个矩阵。其定义如下：\n~~~cpp\nnamespace algorithm\n{\n\nusing Matrix = std::vector>;\n\nstruct MMInput\n{\n Matrix a;\n Matrix b;\n};\n\nstruct MMOutput\n{\n int error;\n size_t m, n, k;\n Matrix c;\n};\n\nvoid matrix_multiply(const MMInput *in, MMOutput *out)\n{\n ...\n}\n\n}\n~~~\n矩阵乘法存在有输入矩阵不合法的问题，所以output里多了一个error域，用来表示错误。\n\n# 生成计算任务\n\n定义好输入输出的类型，以及算法的过程之后，就可以通过WFThreadTaskFactory工厂来产生计算任务了。 \n在[WFTaskFactory.h](../src/factory/WFTaskFactory.h)里，计算工厂类的定义如下：\n~~~cpp\ntemplate \nclass WFThreadTaskFactory\n{\nprivate:\n using T = WFThreadTask;\n\npublic:\n static T *create_thread_task(const std::string& queue_name,\n std::function routine,\n std::function callback);\n\n static T *create_thread_task(time_t seconds, long nanoseconds,\n const std::string& queue_name,\n std::function routine,\n std::function callback);\n ...\n};\n~~~\n这里包含两个创建任务的接口。第二个接口支持用户传入一下任务运行时间限制，我们在一下节介绍这个功能。 \n与之前的网络工厂类或算法工厂类略有不同，这个类需要INPUT和OUTPUT两个模板参数。 \nqueue_name相关的知识在上一个示例里已经有介绍。routine就是你的计算过程，callback是回调。 \n在我们的示例里，我们看到了这个调用的使用：\n~~~cpp\nusing MMTask = WFThreadTask;\n\nusing namespace algorithm;\n\nint main()\n{\n typedef WFThreadTaskFactory MMFactory;\n MMTask *task = MMFactory::create_thread_task(\"matrix_multiply_task\",\n matrix_multiply,\n callback);\n\n MMInput *input = task->get_input();\n\n input->a = {{1, 2, 3}, {4, 5, 6}};\n input->b = {{7, 8}, {9, 10}, {11, 12}};\n ...\n}\n~~~\n产生了task之后，通过get_input()接口得到输入数据的指针。这个可以类比网络任务的get_req()。 \n任务的发起和结束什么，与网络任务并没有什么区别。同样，回调也很简单：\n~~~cpp\nvoid callback(MMTask *task) // MMtask = WFThreadTask\n{\n MMInput *input = task->get_input();\n MMOutput *output = task->get_output();\n\n assert(task->get_state() == WFT_STATE_SUCCESS);\n\n if (output->error)\n printf(\"Error: %d %s\\n\", output->error, strerror(output->error));\n else\n {\n printf(\"Matrix A\\n\");\n print_matrix(input->a, output->m, output->k);\n printf(\"Matrix B\\n\");\n print_matrix(input->b, output->k, output->n);\n printf(\"Matrix A * Matrix B =>\\n\");\n print_matrix(output->c, output->m, output->n);\n }\n}\n~~~\n普通的计算任务可以忽略失败的可能性，结束状态肯定是SUCCESS。 \ncallback里简单打印了输入输出。如果输入数据不合法，则打印错误。\n\n# 带运行时间限制的计算任务\n\n显然，我们的框架无法打断用户的计算任务，因为用户的计算任务是一个函数，用户需要自行确保函数可以正常结束。 \n但我们支持用户指定一个时间限制，当计算无法在指定时间内完成，任务可以提前回到callback。带运行时间限制的接口定义如下：\n~~~cpp\ntemplate \nclass WFThreadTaskFactory\n{\nprivate:\n using T = WFThreadTask;\n\npublic:\n static T *create_thread_task(time_t seconds, long nanoseconds,\n const std::string& queue_name,\n std::function routine,\n std::function callback);\n ...\n};\n~~~\n参数seconds和nanoseconds构成了运行时限。在这里，nanoseconds的取值范围在\\[0,1000000000)。 \n当任务无法在运行时限内结束，会直接回到callback，并且任务的状态为WFT_STATE_SYS_ERROR且错误码为ETIMEDOUT。 \n还是用matrix_multiply的例子，我们可以这样写：\n~~~cpp\nvoid callback(MMTask *task) // MMtask = WFThreadTask\n{\n MMInput *input = task->get_input();\n MMOutput *output = task->get_output();\n\n if (task->get_state() == WFT_STATE_SYS_ERROR && task->get_error() == ETIMEDOUT)\n {\n printf(\"Run out of time.\\n\");\n return;\n }\n\n assert(task->get_state() == WFT_STATE_SUCCESS)\n\n if (output->error)\n printf(\"Error: %d %s\\n\", output->error, strerror(output->error));\n else\n {\n printf(\"Matrix A\\n\");\n print_matrix(input->a, output->m, output->k);\n printf(\"Matrix B\\n\");\n print_matrix(input->b, output->k, output->n);\n printf(\"Matrix A * Matrix B =>\\n\");\n print_matrix(output->c, output->m, output->n);\n }\n}\n\nusing namespace algorithm;\n\nint main()\n{\n typedef WFThreadTaskFactory MMFactory;\n MMTask *task = MMFactory::create_thread_task(0, 1000000,\n \"matrix_multiply_task\",\n matrix_multiply,\n callback);\n\n MMInput *input = task->get_input();\n\n input->a = {{1, 2, 3}, {4, 5, 6}};\n input->b = {{7, 8}, {9, 10}, {11, 12}};\n ...\n}\n~~~\n上面的示例，限制了任务运行时间不超过1毫秒，否则，以WFT_STATE_SYS_ERROR的状态返回。 \n再次提醒，我们并不会中断用户的实际运行函数。当任务超时并callback，计算函数还会一直运行直到结束。 \n如果用户希望函数不再继续执行，需要在代码中自行加入检查点来实现这样的功能。可以在INPUT里加入flag，例如：\n~~~cpp\nvoid callback(MMTask *task) // MMtask = WFThreadTask\n{\n if (task->get_state() == WFT_STATE_SYS_ERROR && task->get_error() == ETIMEDOUT)\n {\n task->get_input()->flag = true;\n printf(\"Run out of time.\\n\");\n return;\n }\n ...\n}\n\nvoid matrix_multiply(const MMInput *in, MMOutput *out)\n{\n while (!in->flag)\n {\n ....\n }\n}\n~~~\n\n# 算法与协议的对称性\n\n在我们的体系里，算法与协议在一个非常抽象的层面上是具有高度对称性的。 \n有自定义算法的线程任务，那显然也存在自定义协议的网络任务。 \n自定义算法要求提供算法的过程，而自定义协议则需要用户提供序列化和反序列化的过程，[简单的用户自定义协议client/server](./tutorial-10-user_defined_protocol.md)有介绍。 \n无论是自定义算法还是自定义协议，我们都必须强调算法和协议都是非常纯粹的。 \n例如算法就是一个从INPUT到OUPUT的转换过程，算法并不知道task，series等的存在。 \nHTTP协议的实现上，也只关心序列化反序列化，无需要关心什么是task。而是在http task里去引用HTTP协议。 \n\n# 线程任务与网络任务的复合性\n\n在这个示例里，我们通过WFThreadTaskFactory构建了一个线程任务。可以说这是一种最简单的计算任务构建，大多数情况下也够用了。 \n同样，用户可以非常简单的定义一个自有协议的server和client。 \n但在上一个示例里我们看到，我们可以通过算法工厂产生一个并行排序任务，这显然不是通过一个routine就能做到的。 \n对于网络任务，比如一个kafka任务，可能要经过与多台机器的交互才能得到结果，但对用户来讲是完全透明的。 \n所以，我们的任务都是具有复合性的，如果你熟练使用我们的框架，可以设计出很多复杂的组件出来。\n" }, { "path": "docs/tutorial-09-http_file_server.md", "content": "# 异步IO的http server：http_file_server\n# 示例代码\n\n[tutorial-09-http_file_server.cc](/tutorial/tutorial-09-http_file_server.cc)\n\n# 关于http_file_server\n\nhttp_file_server是一个web服务器，用户指定启动端口，根路径（默认为程序当路程），就可以启动一个web server。 \n用户还可以指定一个PEM格式的certificate file和key file，启动一个https web server。 \n程序在启动server之后，可以从命令行接受用户输入，并通过127.0.0.1地址来访问这个server。 \n程序主要展示了磁盘IO任务的用法。在Linux系统下，我们利用了Linux底层的aio接口，文件读取完全异步。\n\n# 启动server\n\n启动server这块，和之前的echo server或http proxy没有什么大区别。在这里只是多了一种SSL server的启动方式：\n~~~cpp\nclass WFServerBase\n{\n ...\n int start(unsigned short port, const char *cert_file, const char *key_file);\n ...\n};\n~~~\n也就是说，start操作可以指定一个PEM格式的cert文件和key文件，启动一个SSL server。 \n此外，我们在定义server时，用std::bind()给process绑定了一个root参数，代表服务的根路径。\n~~~cpp\nvoid process(WFHttpTask *server_task, const char *root)\n{\n ...\n}\n\nint main(int argc, char *argv[])\n{\n ...\n const char *root = (argc >= 3 ? argv[2] : \".\");\n auto&& proc = std::bind(process, std::placeholders::_1, root);\n WFHttpServer server(proc);\n\n // start server\n ...\n}\n~~~\n\n# 处理请求\n\n与http_proxy类似，我们不占用任何线程读取文件，而是产生一个异步的读文件任务，在读取完成之后回复请求。 \n再次说明一下，我们需要把完整回复数据读取到内存，才开始回复消息。所以不适合用来传输太大的文件。\n~~~cpp\nvoid process(WFHttpTask *server_task, const char *root)\n{\n // generate abs path.\n ...\n\n int fd = open(abs_path.c_str(), O_RDONLY);\n if (fd >= 0)\n {\n size_t size = lseek(fd, 0, SEEK_END);\n void *buf = malloc(size); /* As an example, assert(buf != NULL); */\n WFFileIOTask *pread_task;\n\n pread_task = WFTaskFactory::create_pread_task(fd, buf, size, 0,\n pread_callback);\n /* To implement a more complicated server, please use series' context\n * instead of tasks' user_data to pass/store internal data. */\n pread_task->user_data = resp; /* pass resp pointer to pread task. */\n server_task->user_data = buf; /* to free() in callback() */\n server_task->set_callback([](WFHttpTask *t){ free(t->user_data); });\n series_of(server_task)->push_back(pread_task);\n }\n else\n {\n resp->set_status_code(\"404\");\n resp->append_output_body(\"404 Not Found.\");\n }\n}\n~~~\n与http_proxy产生一个新的http client任务不同，这里我们通过factory产生了一个pread任务。 \n在[WFTaskFactory.h](../src/factory/WFTaskFactory.h)里，我们可以看到相关的接口。\n~~~cpp\nstruct FileIOArgs\n{\n int fd;\n void *buf;\n size_t count;\n off_t offset;\n};\n\n...\nusing WFFileIOTask = WFFileTask;\nusing fio_callback_t = std::function;\n...\n\nclass WFTaskFactory\n{\npublic:\n ...\n static WFFileIOTask *create_pread_task(int fd, void *buf, size_t count, off_t offset,\n fio_callback_t callback);\n\n static WFFileIOTask *create_pwrite_task(int fd, void *buf, size_t count, off_t offset,\n fio_callback_t callback);\n ...\n\n /* Interface with file path name */\n\tstatic WFFileIOTask *create_pread_task(const std::string& path, void *buf, size_t count, off_t offset,\n fio_callback_t callback);\n\n static WFFileIOTask *create_pwrite_task(const std::string& path, void *buf, size_t count, off_t offset,\n fio_callback_t callback); \n};\n~~~\n无论是pread还是pwrite，返回的都是WFFileIOTask。这与不区分sort或psort，不区分client或server task是一个道理。 \n除这两个接口还有preadv和pwritev，返回WFFileVIOTask，以及fsync，fdatasync，返回WFFileSyncTask。可以在头文件里查看。 \n示例用了task的user_data域保存服务的全局数据。但对于大服务，我们推荐使用series context。可以参考前面的[proxy示例](../tutorial/tutorial-05-http_proxy.cc)。\n\n# 处理读文件结果\n\n~~~cpp\nusing namespace protocol;\n\nvoid pread_callback(WFFileIOTask *task)\n{\n FileIOArgs *args = task->get_args();\n long ret = task->get_retval();\n HttpResponse *resp = (HttpResponse *)task->user_data;\n\n /* close fd only when you created File IO task with **fd** interface. */\n close(args->fd);\n if (ret < 0)\n {\n resp->set_status_code(\"503\");\n resp->append_output_body(\"503 Internal Server Error.\");\n }\n else /* Use '_nocopy' carefully. */\n resp->append_output_body_nocopy(args->buf, ret);\n}\n~~~\n文件任务的get_args()得到输入参数，这里是FileIOArgs结构，如果是用文件路径名创建的文件任务，其中的fd域等于-1。 \nget_retval()是操作的返回值。当ret < 0, 任务错误。否则ret为读取到数据的大小。 \n在文件任务里，ret < 0与task->get_state() != WFT_STATE_SUCCESS完全等价。 \nbuf域的内存我们是自己管理的，可以通过append_output_body_nocopy()传给resp。 \n在回复完成后，我们会free()这块内存，这个语句在process里： \nserver_task->set_callback([](WFHttpTask *t){ free(t->user_data); });\n\n# 命令行交互\n\n启动server后，用户可以在控制台输入文件名来访问server。当输入文件名为空（Ctrl-D），关闭server并结束程序。 \n这里，我们使用了WFRepeaterTask来实现这个循环接受输入的过程。WFRepeaterTask是一种循环任务，产生的接口如下：\n~~~cpp\nusing repeated_create_t = std::function;\nusing repeater_callback_t = std::function;\n\nclass WFTaskFactory\n{\n WFRpeaterTask *create_repeater_task(repeated_create_t create, repeater_callback_t callback);\n};\n~~~\n通过create函数，可以创建一个repeater任务。repeater内部会反复调用create，产生一个任务并运行，直到create返回空指针。 \n在我们的这个示例里，create函数内部调用scanf。当用户输入为空时，create返回NULL，整个循环过程结束。 \n当用户输入不为空（文件名），产生一个访问127.0.0.1地址的http任务来访问我们开启的server。\n~~~cpp\n{\n\tauto&& create = [&scheme, port](WFRepeaterTask *)->SubTask *{\n\t\t...\n\t\tscanf(\"%1023s\", buf);\n\t\tif (*buf == '\\0')\n\t\t\treturn NULL;\n\n\t\tstd::string url = scheme + \"127.0.0.1:\" + std::to_string(port) + \"/\" + buf;\n\t\tWFHttpTask *task = WFTaskFactory::create_http_task(url, 0, 0,\n\t\t\t\t\t\t\t\t\t[](WFHttpTask *task) {\n\t\t\t...\n\t\t});\n\n\t\treturn task;\n\t};\n\t\n\tWFFacilities::WaitGroup wg(1);\n\tWFRepeaterTask *repeater;\n\trepeater = WFTaskFactory::create_repeater_task(create, [&wg](WFRepeaterTask *) {\n\t\twg.done();\n\t});\n\n\trepeater->start();\n\twg.wait();\n\n\tserver.stop();\n}\n~~~\n最后，当create返回NULL，repeater被callback。我们关闭server并结束程序。 \n\n# 关于文件异步IO的实现\n\nLinux操作系统支持一套效率很高，CPU占用非常少的异步IO系统调用。在Linux系统下使用我们的框架将默认使用这套接口。 \n我们曾经实现过一套posix aio接口用于支持其它UNIX系统，并使用线程的sigevent通知方式，但由于其效率太低，已经不再使用了。 \n目前，对于非Linux系统，异步IO一律是用多线程实现，在IO任务到达时，实时创建线程执行IO任务，callback回到handler线程池。 \n多线程IO也是macOS下的唯一选择，因为macOS没有良好的sigevent支持，posix aio行不通。 \n某些UNIX系统不支持fdatasync调用，这种情况下，fdatasync任务将等价于fsync任务。\n\n" }, { "path": "docs/tutorial-10-user_defined_protocol.md", "content": "# 简单的用户自定义协议client/server\n# 示例代码\n\n[message.h](/tutorial/tutorial-10-user_defined_protocol/message.h) \n[message.cc](/tutorial/tutorial-10-user_defined_protocol/message.cc) \n[server.cc](/tutorial/tutorial-10-user_defined_protocol/server.cc) \n[client.cc](/tutorial/tutorial-10-user_defined_protocol/client.cc) \n\n# 关于user_defined_protocol\n\n本示例设计一个简单的通信协议，并在协议上构建server和client。server将client发送的消息转换成大写并返回。\n\n# 协议的格式\n\n协议消息包含一个4字节的head和一个message body。head是一个网络序的整数，指明body的长度。 \n请求和响应消息的格式一致。\n\n# 协议的实现\n\n用户自定义协议，需要提供协议的序列化和反序列化方法，这两个方法都是ProtocolMessage类的虚函数。 \n另外，为了使用方便，我们强烈建议用户实现消息的移动构造和移动赋值（用于std::move()）。\n在[ProtocolMessage.h](../src/protocol/ProtocolMessage.h)里，序列化反序列化接口如下：\n~~~cpp\nnamespace protocol\n{\n\nclass ProtocolMessage : public CommMessageOut, public CommMessageIn\n{\nprivate:\n virtual int encode(struct iovec vectors[], int max);\n\n /* You have to implement one of the 'append' functions, and the first one\n * with arguement 'size_t *size' is recommmended. */\n virtual int append(const void *buf, size_t *size);\n virtual int append(const void *buf, size_t size);\n\n ...\n};\n\n}\n~~~\n### 序列化函数encode\n * encode函数在消息被发送之前调用，每条消息只调用一次。\n * encode函数里，用户需要将消息序列化到一个vector数组，数组元素个数不超过max。目前max的值为2048。\n * 结构体struct iovec定义在请参考系统调用readv和writev。\n * encode函数正确情况下的返回值在0到max之间，表示消息使用了多少个vector。\n * 如果是UDP协议，请注意总长度不超过64k，并且使用不超过1024个vector（Linux一次writev只能1024个vector）。\n * encode返回-1表示错误。返回-1时，需要置errno。如果返回值>max，将得到一个EOVERFLOW错误。错误都在callback里得到。\n * 为了性能考虑vector里的iov_base指针指向的内容不会被复制。所以一般指向消息类的成员。\n\n### 反序列化函数append\n * append函数在每次收到一个数据块时被调用。因此，每条消息可能会调用多次。\n * buf和size分别是收到的数据块内容和长度。用户需要把数据内容复制走。\n * 如果实现了append(const void \\*buf, size_t \\*size)接口，可以通过修改\\*size来告诉框架本次消费了多少长度。收到的size - 消费的size = 剩余的size，剩余的那部分buf会由下一次append被调起时再次收到。此功能更方便协议解析，当然用户也可以全部复制走自行管理，则无需修改\\*size。\n * append函数返回0表示消息还不完整，传输继续。返回1表示消息结束。-1表示错误，需要置errno。\n * 总之append的作用就是用于告诉框架消息是否已经传输结束。不要在append里做复杂的非必要的协议解析。\n\n### errno的设置\n * encode或append返回-1或其它负数都会被理解为失败，需要通过errno来传递错误原因。用户会在callback里得到这个错误。\n * 如果是系统调用或libc等库函数失败（比如malloc）,libc肯定会设置好errno，用户无需再设置。\n * 一些消息不合法的错误是比较常见的，比如可以用EBADMSG，EMSGSIZE分别表示消息内容错误，和消息太大。\n * 用户可以选择超过系统定义errno范围的值来表示一些自定义错误。一般大于256的值是可以用的。\n * 请不要使用负数errno。因为框架内部用了负数来代表SSL错误。\n\n在我们的示例里，消息的序列化反序列化都非常的简单。 \n头文件[message.h](../tutorial/tutorial-10-user_defined_protocol/message.h)里，声明了request和response类：\n~~~cpp\nnamespace protocol\n{\n\nclass TutorialMessage : public ProtocolMessage\n{\nprivate:\n virtual int encode(struct iovec vectors[], int max);\n virtual int append(const void *buf, size_t size);\n ...\n};\n\nusing TutorialRequest = TutorialMessage;\nusing TutorialResponse = TutorialMessage;\n\n}\n~~~\nrequest和response类，都是同一种类型的消息。直接using就可以。 \n注意request和response必须可以无参数的被构造，也就是说需要有无参数的构造函数，或完全没有构造函数。 \n此外，通讯过程中，如果发生重试，response对象会被销毁并重新构造。因此，它最好是一个RAII类。否则处理起来会比较复杂。 \n[message.cc](../tutorial/tutorial-10-user_defined_protocol/message.cc)里包含了encode和append的实现：\n~~~cpp\nnamespace protocol\n{\n\nint TutorialMessage::encode(struct iovec vectors[], int max/*max==8192*/)\n{\n uint32_t n = htonl(this->body_size);\n\n memcpy(this->head, &n, 4);\n vectors[0].iov_base = this->head;\n vectors[0].iov_len = 4;\n vectors[1].iov_base = this->body;\n vectors[1].iov_len = this->body_size;\n\n return 2; /* return the number of vectors used, no more then max. */\n}\n\nint TutorialMessage::append(const void *buf, size_t size)\n{\n if (this->head_received < 4)\n {\n size_t head_left;\n void *p;\n\n p = &this->head[this->head_received];\n head_left = 4 - this->head_received;\n if (size < 4 - this->head_received)\n {\n memcpy(p, buf, size);\n this->head_received += size;\n return 0;\n }\n\n memcpy(p, buf, head_left);\n size -= head_left;\n buf = (const char *)buf + head_left;\n\n p = this->head;\n this->body_size = ntohl(*(uint32_t *)p);\n if (this->body_size > this->size_limit)\n {\n errno = EMSGSIZE;\n return -1;\n }\n\n this->body = (char *)malloc(this->body_size);\n if (!this->body)\n return -1;\n\n this->body_received = 0;\n }\n\n size_t body_left = this->body_size - this->body_received;\n\n if (size > body_left)\n {\n errno = EBADMSG;\n return -1;\n }\n\n memcpy(this->body, buf, size);\n if (size < body_left)\n return 0;\n\n return 1;\n}\n\n}\n~~~\nencode的实现非常简单，固定使用了两个vector，分别指向head和body。需要注意iov_base指针必须指向消息类的成员。 \nappend需要保证4字节的head接收完整，再读取message body。而且我们并不能保证第一次append一定包含完整的head，所以过程略为繁琐。 \nappend实现了size_limit功能，超过size_limit的会返回EMSGSIZE错误。用户如果不需要限制消息大小，可以忽略size_limit这个域。 \n由于我们要求通信协议是一来一回的，所谓的“TCP黏包”问题不需要考虑，直接当错误消息处理。 \n现在，有了消息的定义和实现，我们就可以建立server和client了。　\n\n# server和client的定义\n\n有了request和response类，我们就可以建立基于这个协议的server和client。前面的示例里我们介绍过Http协议相关的类型定义：\n~~~cpp\nusing WFHttpTask = WFNetworkTask;\nusing http_callback_t = std::function;\n\nusing WFHttpServer = WFServer;\nusing http_process_t = std::function;\n~~~\n同样的，对这个Tutorial协议，数据类型的定义并没有什么区别：\n~~~cpp\nusing WFTutorialTask = WFNetworkTask;\nusing tutorial_callback_t = std::function;\n\nusing WFTutorialServer = WFServer;\nusing tutorial_process_t = std::function;\n~~~\n\n# server端\n\nserver与普通的http server没有什么区别。我们优先IPv6启动，这不影响IPv4的client请求。另外限制请求最多不超过4KB。 \n代码请自行参考[server.cc](../tutorial/tutorial-10-user_defined_protocol/server.cc) \n\n# client端\n\nclient端的逻辑是从标准IO接收用户输入，构造出请求发往server并得到结果。这里我们使用了WFRepeaterTask来实现这个重复过程，直到用户的输入为空。\n此外，为了安全我们限制server回复包不超4KB。 \nclient端唯一需要了解的就是怎么产生一个自定义协议的client任务，在[WFTaskFactory.h](../src/factory/WFTaskFactory.h)有四个接口可以选择：\n~~~cpp\ntemplate\nclass WFNetworkTaskFactory\n{\nprivate:\n\tusing T = WFNetworkTask;\n\npublic:\n\tstatic T *create_client_task(TransportType type,\n\t\t\t\t\t\t\t\t const std::string& host,\n\t\t\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(TransportType type,\n\t\t\t\t\t\t\t\t const std::string& url,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(TransportType type,\n\t\t\t\t\t\t\t\t const ParsedURI& uri,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(TransportType type,\n\t\t\t\t\t\t\t\t const struct sockaddr *addr,\n\t\t\t\t\t\t\t\t socklen_t addrlen,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n ...\n};\n~~~\n其中，TransportType指定传输层协议，目前可选的值包括TT_TCP，TT_UDP，TT_SCTP和TT_TCP_SSL。 \n四个接口的区别不大，在我们这个示例里暂时不需要URL，我们用域名和端口来创建任务。 \n如果用户需要使用Unix Domain Protocol访问server，则需要用最后一个接口，直接传入sockaddr。 \n实际的调用代码如下。我们派生了WFTaskFactory类，但这个派生并非必须的。\n~~~cpp\nusing namespace protocol;\n\nclass MyFactory : public WFTaskFactory\n{\npublic:\n static WFTutorialTask *create_tutorial_task(const std::string& host,\n unsigned short port,\n int retry_max,\n tutorial_callback_t callback)\n {\n using NTF = WFNetworkTaskFactory;\n WFTutorialTask *task = NTF::create_client_task(TT_TCP, host, port,\n retry_max,\n std::move(callback));\n task->set_keep_alive(30 * 1000);\n return task;\n }\n};\n~~~\n可以看到我们用了WFNetworkTaskFactory类来创建client任务。 \n接下来通过任务的set_keep_alive()接口，让连接在通信完成之后保持30秒，否则，将默认采用短连接。 \nclient的其它代码涉及的知识点在之前的示例里都包含了。请参考[client.cc](../tutorial/tutorial-10-user_defined_protocol/client.cc)\n\n# 内置协议的请求是怎么产生的\n\n现在系统中内置了http, redis，mysql，kafka，dns等协议。我们可以通过相同的方法产生一个http或redis任务吗？比如： \n~~~cpp\nWFHttpTask *task = WFNetworkTaskFactory::create_client_task(...);\n~~~\n需要说明的是，这样产生的http任务，会损失很多的功能，比如，无法根据header来识别是否用持久连接，无法识别重定向等。 \n同样，如果这样产生一个MySQL任务，可能根本就无法运行起来。因为缺乏登录认证过程。 \n一个kafka请求可能需要和多台broker有复杂的交互过程，这样创建的请求显然也无法完成这一过程。 \n可见每一种内置协议消息的产生过程都远远比这个示例复杂。同样，如果用户需要实现一个更多功能的通信协议，还有许多代码要写。\n" }, { "path": "docs/tutorial-11-graph_task.md", "content": "# 有向无环图（DAG）的使用：graph_task\n# 示例代码\n\n[tutorial-11-graph_task.cc](/tutorial/tutorial-11-graph_task.cc)\n\n# 关于graph_task\n\ngraph_task示例通过建立一个有向无环图，演示如何用workflow框架实现更加复杂的任务间依赖关系。\n\n# 创建DAG中的任务\n\nDAG中的任务，可以是workflow框架的任何一种任务。在本示例中，我们创建了一个timer任务，两个http任务，以及一个go任务。 \nTimer执行一秒的等待，http1和http2分别抓取sogou和baidu的首页，go任务打印结果。它们之间的依赖关系如下：\n~~~\n +-------+ \n +---->| Http1 |-----+ \n | +-------+ |\n +-------+ +-v--+ \n | Timer | | Go | \n +-------+ +-^--+ \n | +-------+ | \n +---->| Http2 |-----+ \n +-------+ \n~~~\n创建DAG中任务的方法与创建普通任务的方法没有区别，这里不再展开。\n\n# 创建图任务\n\nDAG图在我们的框架里也是一种任务，通过以下代码，我们可以创建一个图任务：\n~~~cpp\n{\n WFGraphTask *graph = WFTaskFactory::create_graph_task([](WFGraphTask *) {\n printf(\"Graph task complete. Wakeup main process\\n\");\n wait_group.done();\n });\n}\n~~~\n可以看到，图任务的类型为WFGraphTask，创建函数只有一个参数，即任务的回调。显然一个新建的图任务，是一张空图。\n\n# 创建图节点\n\n接下来，我们需要通过之前创建的4个普通任务（timer，http_task1，http_task2，go_task），产生4个图节点：\n~~~cpp\n{\n /* Create graph nodes */\n WFGraphNode& a = graph->create_graph_node(timer);\n WFGraphNode& b = graph->create_graph_node(http_task1);\n WFGraphNode& c = graph->create_graph_node(http_task2);\n WFGraphNode& d = graph->create_graph_node(go_task);\n}\n~~~\nWFGraphTask的create_graph_node接口，产生一个图节点并返回节点的引用，用户通过这个节点引用来建立节点之间的依赖。 \n如果我们不为节点建立依赖直接运行图任务，那么显然所有节点都是孤立节点，将全部并发执行。\n\n# 建立依赖\n通过非常形象的'-->'运算符，我们可以建立节点的依赖关系：\n~~~cpp\n{\n /* Build the graph */\n a-->b;\n a-->c;\n b-->d;\n c-->d;\n}\n~~~\n这样我们就建立起了上述结构的DAG图啦。 \n除’—>’运算符，我们同样支持’<—‘。并且它们都可以连着写。所以，以下程序都是合法且等价的：\n~~~cpp\n{\n a-->b-->d;\n a-->c-->d;\n}\n~~~\n~~~cpp\n{\n d<--b<--a;\n d<--c<--a;\n}\n~~~\n~~~cpp\n{\n d<--b<--a-->c-->d;\n}\n~~~\n接下来直接运行graph，或者把graph放入任务流中就可以运行啦，和一般的任务没有区别。 \n当然，把一个图任务变成另一个图的节点，也是完全正确的行为。\n\n# 取消后继节点\n\n在图任务里，我们扩展了series的cancel操作，这个操作会取消该节点的所有后继结点。 \n取消操作一般在节点任务的callback里执行，例如：\n~~~cpp\nint main()\n{\n WFGraphTask *graph = WFTaskFactory::create_graph_task(graph_callback);\n WFHttpTask *task = WFTaskFactory::create_http_task(url, 0, 0, [](WFHttpTask *t){\n if (t->get_state() != WFT_STATE_SUCCESS)\n series_of(t)->cancel();\n });\n WFGraphNode& a = graph->create_graph_node(task);\n WFGraphNode& b = ...;\n WFGraphNode& c = ...;\n WFGraphNode& d = ...;\n a-->b-->c;\n b-->d;\n graph->start();\n ...\n}\n~~~\n注意取消后继节点的操作是递归的，这个例子里，如果http任务失败，b,c,d三个节点的任务都会被取消。\n\n# 数据传递\n\n图节点之间目前没有统一的数据传递方法，它们并不共享某一个series。因此，节点间数据传递需要用户解决。\n\n# 致谢\n\n部分思路来自于[taskflow](https://github.com/taskflow/taskflow)项目。\n\n" }, { "path": "docs/tutorial-12-mysql_cli.md", "content": "# 异步MySQL客户端：mysql_cli\n# 示例代码\n\n[tutorial-12-mysql_cli.cc](/tutorial/tutorial-12-mysql_cli.cc)\n\n# 关于mysql_cli\n\n教程中的mysql_cli使用方式与官方客户端相似，是一个命令行交互式的异步MySQL客户端。\n\n程序运行方式：./mysql_cli \\\n\n启动之后可以直接在终端输入mysql命令与db进行交互，输入quit或Ctrl-C退出。\n\n# MySQL URL的格式\n\nmysql://username:password@host:port/dbname?character_set=charset&character_set_results=charset\n\n- 如果以SSL连接访问MySQL，则scheme设为**mysqls://**。MySQL server 5.7及以上支持；\n\n- username和password按需填写，如果密码里包含特殊字符，需要转义后再拼接URL；\n~~~cpp\n// 密码为：@@@@####\nstd::string url = \"mysql://root:\" + StringUtil::url_encode_component(\"@@@@####\") + \"@127.0.0.1\";\n~~~\n- port默认为3306；\n\n- dbname为要用的数据库名，一般如果SQL语句只操作一个db的话建议填写；\n\n- 如果用户在这一层有upstream选取需求，可以参考[upstream文档](/docs/about-upstream.md)；\n\n- character_set为client的字符集，等价于使用官方客户端启动时的参数``--default-character-set``的配置，默认utf8，具体可以参考MySQL官方文档[character-set.html](https://dev.mysql.com/doc/internals/en/character-set.html)。\n\n- character_set_results为client、connection和results的字符集，如果想要在SQL语句里使用``SET NAME``来指定这些字符集的话，请把它配置到url的这个位置。\n\nMySQL URL示例：\n\nmysql://root:password@127.0.0.1\n\nmysql://@test.mysql.com:3306/db1?character_set=utf8&character_set_results=utf8\n\nmysqls://localhost/db1?character\\_set=big5\n\n# 创建并启动MySQL任务\n\n用户可以使用WFTaskFactory创建MySQL任务，创建接口与回调函数的用法都与workflow其他任务类似:\n~~~cpp\nusing mysql_callback_t = std::function;\n\nWFMySQLTask *create_mysql_task(const std::string& url, int retry_max, mysql_callback_t callback);\n\nvoid set_query(const std::string& query);\n~~~\n用户创建完WFMySQLTask之后，可以对req调用 **set_query()** 写入SQL语句。\n\n如果没调用过 **set_query()** ，task就被start起来的话，则用户会在callback里得到**WFT_ERR_MYSQL_QUERY_NOT_SET**。\n\n其他包括callback、series、user_data等与workflow其他task用法类似。\n\n大致使用示例如下：\n~~~cpp\nint main(int argc, char *argv[])\n{\n ...\n WFMySQLTask *task = WFTaskFactory::create_mysql_task(url, RETRY_MAX, mysql_callback);\n task->get_req()->set_query(\"SHOW TABLES;\");\n ...\n task->start();\n ...\n}\n~~~\n\n# 支持的命令\n\n目前支持的命令为**COM_QUERY**，已经能涵盖用户基本的增删改查、建库删库、建表删表、预处理、使用存储过程和使用事务的需求。\n\n因为我们的交互命令中不支持选库（**USE**命令），所以，如果SQL语句中有涉及到**跨库**的操作，则可以通过**db_name.table_name**的方式指定具体哪个库的哪张表。\n\n其他所有命令都可以**拼接**到一起通过 ``set_query()`` 传给WFMySQLTask（包括INSERT/UPDATE/SELECT/DELETE/PREPARE/CALL）。\n\n拼接的命令会被按序执行直到命令发生错误，前面的命令会执行成功。\n\n举个例子：\n~~~cpp\nreq->set_query(\"SELECT * FROM table1; CALL procedure1(); INSERT INTO table3 (id) VALUES (1);\");\n~~~\n\n# 结果解析\n\n与workflow其他任务类似，可以用``task->get_resp()``拿到**MySQLResponse**，我们可以通过**MySQLResultCursor**遍历结果集。具体接口可以查看：[MySQLResult.h](/src/protocol/MySQLResult.h)\n\n一次请求所对应的回复中，其数据是一个三维结构：\n- 一个回复中包含了一个或多个结果集（result set）；\n- 一个结果集的类型可能是**MYSQL_STATUS_GET_RESULT**或者**MYSQL_STATUS_OK**；\n- **MYSQL_STATUS_GET_RESULT**类型的结果集包含了一行或多行（row）；\n- 一行包含了一列或多个列，或者说一到多个阈（Field/Cell），具体数据结构为**MySQLField**和**MySQLCell**；\n\n结果集的两种类型，可以通过``cursor->get_cursor_status()``进行判断：\n\n| |MYSQL_STATUS_GET_RESULT|MYSQL_STATUS_OK|\n|------|-----------------------|---------------|\n|SQL命令|SELECT（包括存储过程中的每一个SELECT）|INSERT / UPDATE / DELETE / ...|\n|对应语义|读操作，一个结果集表示一份读操作返回的二维表|写操作，一个结果集表示一个写操作是否成功|\n|主要接口|fetch_fields();
fetch_row(&row_arr);
...|get_insert_id();
get_affected_rows();
...|\n\n由于拼接语句可能存在错误，因此这种情况，可以通过**MySQLResultCursor**拿到前面正确执行过的语句多个结果集，以及最后判断``resp->get_packet_type()``为**MYSQL_PACKET_ERROR**时，通过``resp->get_error_code()``和``resp->get_error_msg()``拿到具体错误信息。\n\n一个包含n条**SELECT**语句的**存储过程**，会返回n个**MYSQL_STATUS_GET_RESULT**的结果集和1个**MYSQL_STATUS_OK**的结果集，用户自行忽略此**MYSQL_STATUS_OK**结果集即可。\n\n具体使用从外到内的步骤应该是：\n\n1. 判断任务状态（代表通信层面状态）：用户通过判断 ``task->get_state()`` 等于**WFT_STATE_SUCCESS**来查看任务执行是否成功；\n\n2. 判断回复包类型（代表返回包解析状态）：调用 **resp->get_packet_type()** 查看最后一条MySQL语句的返回包类型，常见的几个类型为：\n - MYSQL_PACKET_OK：成功，可以用cursor遍历结果；\n - MYSQL_PACKET_EOF：成功，可以用cursor遍历结果；\n - MYSQL_PACKET_ERROR：失败或部分失败，成功的部分可以用cursor遍历结果；\n\n3. 遍历结果集。用户可以使用**MySQLResultCursor**读取结果集中的内容，因为MySQL server返回的数据是多结果集的，因此一开始cursor会**自动指向第一个结果集**的读取位置。\n\n4. 判断结果集状态（代表结果集读取状态）：通过 ``cursor->get_cursor_status()`` 可以拿到的几种状态：\n - MYSQL_STATUS_GET_RESULT：此结果集为读请求类型；\n - MYSQL_STATUS_END：读结果集已读完最后一行；\n - MYSQL_STATUS_OK：此结果集为写请求类型；\n - MYSQL_STATUS_ERROR：解析错误；\n\n5. 读取**MYSQL_STATUS_OK**结果集中的基本内容：\n - ``unsigned long long get_affected_rows() const;``\n - ``unsigned long long get_insert_id() const;``\n - ``int get_warnings() const;``\n - ``std::string get_info() const;``\n\n6. 读取**MYSQL_STATUS_GET_RESULT**结果集中的columns中每个field：\n - ``int get_field_count() const;``\n - ``const MySQLField *fetch_field();``\n - ``const MySQLField *const *fetch_fields() const;``\n\n7. 读取**MYSQL_STATUS_GET_RESULT**结果集中的每一行：按行读取可以使用 ``cursor->fetch_row()`` 直到返回值为false。其中会移动cursor内部对当前结果集的指向每行的offset：\n - ``int get_rows_count() const;``\n - ``bool fetch_row(std::vector& row_arr);``\n - ``bool fetch_row(std::map& row_map);``\n - ``bool fetch_row(std::unordered_map& row_map);``\n - ``bool fetch_row_nocopy(const void **data, size_t *len, int *data_type);``\n\n8. 直接把当前**MYSQL_STATUS_GET_RESULT**结果集的所有行拿出：所有行的读取可以使用 **cursor->fetch_all()** ，内部用来记录行的cursor会直接移动到最后；当前cursor状态会变成**MYSQL_STATUS_END**：\n - ``bool fetch_all(std::vector>& rows);``\n\n9. 返回当前**MYSQL_STATUS_GET_RESULT**结果集的头部：如果有必要重读这个结果集，可以使用 **cursor->rewind()** 回到当前结果集头部，再通过第7步或第8步进行读取；\n\n10. 拿到下一个结果集：因为MySQL server返回的数据包可能是包含多结果集的（比如每个select/insert/...语句为一个结果集；或者call procedure返回的多结果集数据），因此用户可以通过 **cursor->next_result_set()** 跳到下一个结果集，返回值为false表示所有结果集已取完。\n\n11. 返回第一个结果集：**cursor->first_result_set()** 可以让我们返回到所有结果集的头部，然后可以从第4步开始重新拿数据；\n\n12. **MYSQL_STATUS_GET_RESULT**结果集每列具体数据MySQLCell：第7步中读取到的一行，由多列组成，每列结果为MySQLCell，基本使用接口有：\n - ``int get_data_type();`` 返回MYSQL_TYPE_LONG、MYSQL_TYPE_STRING...具体参考[mysql_types.h](/src/protocol/mysql_types.h)\n - ``bool is_TYPE() const;`` TYPE为int、string、ulonglong，判断是否是某种类型\n - ``TYPE as_TYPE() const;`` 同上，以某种类型读出MySQLCell的数据\n - ``void get_cell_nocopy(const void **data, size_t *len, int *data_type) const;`` nocopy接口\n\n整体示例如下：\n~~~cpp\nvoid task_callback(WFMySQLTask *task)\n{\n // step-1. 判断任务状态 \n if (task->get_state() != WFT_STATE_SUCCESS)\n {\n fprintf(stderr, \"task error = %d\\n\", task->get_error());\n return;\n }\n\n MySQLResultCursor cursor(task->get_resp());\n bool test_first_result_set_flag = false;\n bool test_rewind_flag = false;\n\n // step-2. 判断回复包其他状态\n if (resp->get_packet_type() == MYSQL_PACKET_ERROR)\n {\n fprintf(stderr, \"ERROR. error_code=%d %s\\n\",\n task->get_resp()->get_error_code(),\n task->get_resp()->get_error_msg().c_str());\n }\n\nbegin:\n // step-3. 遍历结果集\n do {\n // step-4. 判断结果集状态\n if (cursor.get_cursor_status() == MYSQL_STATUS_OK)\n {\n // step-5. MYSQL_STATUS_OK结果集的基本内容\n fprintf(stderr, \"OK. %llu rows affected. %d warnings. insert_id=%llu.\\n\",\n cursor.get_affected_rows(), cursor.get_warnings(), cursor.get_insert_id());\n }\n else if (cursor.get_cursor_status() == MYSQL_STATUS_GET_RESULT)\n {\n fprintf(stderr, \"field_count=%u rows_count=%u \",\n cursor.get_field_count(), cursor.get_rows_count());\n\n // step-6. 读取每个fields。这是个nocopy api\n const MySQLField *const *fields = cursor.fetch_fields();\n for (int i = 0; i < cursor.get_field_count(); i++)\n {\n fprintf(stderr, \"db=%s table=%s name[%s] type[%s]\\n\",\n fields[i]->get_db().c_str(), fields[i]->get_table().c_str(),\n fields[i]->get_name().c_str(), datatype2str(fields[i]->get_data_type()));\n }\n\n // step-8. 把所有行读出，也可以while (cursor.fetch_row(map/vector)) 按step-7拿每一行\n std::vector> rows;\n\n cursor.fetch_all(rows);\n for (unsigned int j = 0; j < rows.size(); j++)\n {\n // step-12. 具体每个cell的读取\n for (unsigned int i = 0; i < rows[j].size(); i++)\n {\n fprintf(stderr, \"[%s][%s]\", fields[i]->get_name().c_str(),\n datatype2str(rows[j][i].get_data_type()));\n // step-12. 判断具体类型is_string()和转换具体类型as_string()\n if (rows[j][i].is_string())\n {\n std::string res = rows[j][i].as_string();\n fprintf(stderr, \"[%s]\\n\", res.c_str());\n }\n else if (rows[j][i].is_int())\n {\n fprintf(stderr, \"[%d]\\n\", rows[j][i].as_int());\n } // else if ...\n }\n }\n }\n // step-10. 拿下一个结果集\n } while (cursor.next_result_set());\n\n if (test_first_result_set_flag == false)\n {\n test_first_result_set_flag = true;\n // step-11. 返回第一个结果集\n cursor.first_result_set();\n goto begin;\n }\n\n if (test_rewind_flag == false)\n {\n test_rewind_flag = true;\n // step-9. 返回当前结果集头部\n cursor.rewind();\n goto begin;\n }\n\n return;\n}\n~~~\n\n# WFMySQLConnection\n\n由于我们是高并发异步客户端，这意味着我们对一个server的连接可能会不止一个。而MySQL的事务和预处理都是带状态的，为了保证一次事务或预处理独占一个连接，用户可以使用我们封装的二级工厂WFMySQLConnection来创建任务，每个WFMySQLConnection保证独占一个连接，具体参考[WFMySQLConnection.h](/src/client/WFMySQLConnection.h)。\n\n### 1. WFMySQLConnection的创建与初始化\n\n创建一个WFMySQLConnection的时候需要传入一个**id**，之后的调用内部都会由这个id和url去找到对应的那个连接。\n\n初始化需要传入**url**，之后在这个connection上创建的任务就不需要再设置url了。\n\n~~~cpp\nclass WFMySQLConnection\n{\npublic:\n WFMySQLConnection(int id);\n int init(const std::string& url);\n ...\n};\n~~~\n\n### 2. 创建任务与关闭连接\n\n通过 **create_query_task()** ，写入SQL请求和回调函数即可创建任务，该任务一定从这一个connection发出。\n\n有时候我们需要手动关闭这个连接。因为当我们不再使用它的时候，这个连接会一直保持到MySQL server超时。期间如果使用同一个id和url去创建WFMySQLConnection的话就可以复用到这个连接。\n\n因此我们建议如果不准备复用连接，应使用 **create_disconnect_task()** 创建一个任务，手动关闭这个连接。\n~~~cpp\nclass WFMySQLConnection\n{\npublic:\n ...\n WFMySQLTask *create_query_task(const std::string& query,\n mysql_callback_t callback);\n WFMySQLTask *create_disconnect_task(mysql_callback_t callback);\n}\n~~~\nWFMySQLConnection相当于一个二级工厂，我们约定任何工厂对象的生命周期无需保持到任务结束，以下代码完全合法：\n~~~cpp\n WFMySQLConnection *conn = new WFMySQLConnection(1234);\n conn->init(url);\n auto *task = conn->create_query_task(\"SELECT * from table\", my_callback);\n conn->deinit();\n delete conn;\n task->start();\n~~~\n\n### 3. 注意事项\n\n不可以无限制的产生id来生成连接对象，因为每个id会占用一小块内存，无限产生id会使内存不断增加。当一个连接使用完毕，可以不创建和运行disconnect task，而是让这个连接进入内部连接池。下一个connection通过相同的id和url初始化，会自动复用这个连接。\n\n同一个连接上的多个任务并行启动，会得到EAGAIN错误。\n\n如果在使用事务期间已经开始BEGIN但还没有COMMIT或ROLLBACK，且期间连接发生过中断，则连接会被框架内部自动重连，用户会在下一个task请求中拿到**ECONNRESET**错误。此时还没COMMIT的事务语句已经失效，需要重新再发一遍。\n\n### 4. 预处理\n\n用户也可以通过WFMySQLConnection来做预处理**PREPARE**，因此用户可以很方便地用作**防SQL注入**。如果连接发生了重连，也会得到一个**ECONNRESET**错误。\n\n### 5. 完整示例\n\n~~~cpp\nWFMySQLConnection conn(1);\nconn.init(\"mysql://root@127.0.0.1/test\");\n\n// test transaction\nconst char *query = \"BEGIN;\";\nWFMySQLTask *t1 = conn.create_query_task(query, task_callback);\nquery = \"SELECT * FROM check_tiny FOR UPDATE;\";\nWFMySQLTask *t2 = conn.create_query_task(query, task_callback);\nquery = \"INSERT INTO check_tiny VALUES (8);\";\nWFMySQLTask *t3 = conn.create_query_task(query, task_callback);\nquery = \"COMMIT;\";\nWFMySQLTask *t4 = conn.create_query_task(query, task_callback);\nWFMySQLTask *t5 = conn.create_disconnect_task(task_callback);\n((*t1) > t2 > t3 > t4 > t5).start();\n~~~\n\n" }, { "path": "docs/tutorial-13-kafka_cli.md", "content": "# 异步Kafka客户端：kafka_cli\n# 示例代码\n\n[tutorial-13-kafka_cli.cc](/tutorial/tutorial-13-kafka_cli.cc)\n\n# 编译\n\n由于支持Kafka的多种压缩方式，因此系统需要预先安装[zlib](https://github.com/madler/zlib.git),[snappy](https://github.com/google/snappy.git),[lz4(>=1.7.5)](https://github.com/lz4/lz4.git),[zstd](https://github.com/facebook/zstd.git)等第三方库。\n\n支持CMake和Bazel两种编译方式。\n\nCMake：执行命令make KAFKA=y 编译独立的类库（libwfkafka.a和libwfkafka.so）支持kafka协议；cd tutorial; make KAFKA=y 可以编译kafka_cli\n\nBazel：执行bazel build kafka 编译支持kafka协议的类库；执行bazel build kafka_cli 编译kafka_cli\n\n\n# 关于kafka_cli\n\n这是一个kafka client，可以完成kafka的消息生产(produce)和消息消费(fetch)。\n\n编译时需要在tutorial目录中执行编译命令make KAFKA=y或者在项目根目录执行make KAFKA=y tutorial。\n\n该程序从命令行读取一个kafka broker服务器地址和本次任务的类型(produce/fetch)：\n\n./kafka_cli \\ [p/c]\n\n程序会在执行完任务后自动退出，一切资源完全回收。\n\n其中broker_url可以有多个url组成，多个url之间以,分割\n\n- 形式如：kafka://host:port,kafka://host1:port... 或：**kafkas**://host:port,**kafkas**://host1:port代表使用SSL通信。\n- port的默认值在普通TCP连接下是9092，SSL下为9093。\n- \"kafka://\"前缀可以缺省。这时候使用默认使用TCL通信。\n- 多个url，必须都采用TCP或都采用SSL。否则init函数返回-1，错误码为EINVAL。\n- 如果用户在这一层有upstream选取需求，可以参考[upstream文档](../docs/about-upstream.md)。\n\nKafka broker_url示例：\n\nkafka://127.0.0.1/\n\nkafka://kafka.host:9090/\n\nkafka://10.160.23.23:9000,10.123.23.23,kafka://kafka.sogou\n\nkafkas://broker1.kafka.sogou,kafkas://broker2.kafka.sogou\n\n错误的url示例（第一个broker为SSL，第二个broker非SSL）：\n\nkafkas://broker1.kafka.sogou,broker2.kafka.sogou\n\n# 实现原理和特性\n\nkafka client内部实现上除了压缩功能外没有依赖第三方库，同时利用了workflow的高性能，在合理的配置和环境下，每秒钟可以处理几万次Kafka请求。\n\n在内部实现上，kafka client会把一次请求按照内部使用到的broker分拆成并行parallel任务，每个broker地址对应parallel任务中的一个子任务，\n\n这样可以最大限度的提升效率，同时利用workflow内部对连接的复用机制使得整体的连接数控制在一个合理的范围。\n\n如果一个broker地址下有多个topic partition，为了提高吞吐，应该创建多个client，然后按照topic partition分别创建任务独立启动。\n\n\n# 创建并启动Kafka任务\n\n首先需要创建一个WFKafkaClient对象，然后调用init函数初始化WFKafkaClient对象，\n~~~cpp\nint init(const std::string& broker_url);\n\nint init(const std::string& broker_url, const std::string& group);\n~~~\n其中broker_url是kafka broker集群的地址，格式可以参考上面的broker_url，\n\ngroup是消费者组的group_name，用在基于消费者组的fetch任务中，如果是produce任务或者没有使用消费者组的fetch任务，则不需要使用此接口；\n\n用消费者组的时候，可以设置heartbeat的间隔时间，时间单位是毫秒，用于维持心跳：\n~~~cpp\nvoid set_heartbeat_interval(size_t interval_ms);\n~~~\n\n后面再通过WFKafkaClient对象创建kafka任务\n~~~cpp\nusing kafka_callback_t = std::function;\n\nWFKafkaTask *create_kafka_task(const std::string& query, int retry_max, kafka_callback_t cb);\n\nWFKafkaTask *create_kafka_task(int retry_max, kafka_callback_t cb);\n~~~\n其中query中包含此次任务的类型以及topic等属性，retry_max表示最大重试次数，cb为用户自定义的callback函数，当task执行完毕后会被调用，\n\n接着还可以修改task的默认配置以满足实际需要，详细接口可以在[KafkaDataTypes.h](../src/protocol/KafkaDataTypes.h)中查看\n~~~cpp\nKafkaConfig config;\nconfig.set_client_id(\"workflow\");\ntask->set_config(std::move(config));\n~~~\n支持的配置选项描述如下：\n配置名 | 类型 | 默认值 | 含义\n------ | ---- | -------| -------\nproduce_timeout | int | 100ms | produce的超时时间\nproduce_msg_max_bytes | int | 1000000 bytes | 单个消息的最大长度限制\nproduce_msgset_cnt | int | int | 10000 | 一次通信消息集合的最大条数\nproduce_msgset_max_bytes | int | 1000000 bytes | 一次通信消息集合的最大长度限制\nfetch_timeout | int | 100ms | fetch的超时时间\nfetch_min_bytes | int | 1 byte | 一次fetch通信最小消息的长度\nfetch_max_bytes | int | 50M bytes | 一次fetch通信最大消息的长度\nfetch_msg_max_bytes | int | 1M bytes | 一次fetch通信单个消息的最大长度\noffset_timestamp | long long int | -1 | 消费者组模式下，没有找到历史offset时，初始化的offset，-2表示最久，-1表示最新\nsession_timeout | int | 10s | 加入消费者组初始化时的超时时间\nrebalance_timeout | int | 10s | 加入消费者组同步信息阶段的超时时间\nproduce_acks | int | -1 | produce任务在返回之前应确保消息成功复制的broker节点数，-1表示所有的复制broker节点\nallow_auto_topic_creation | bool | true | produce时topic不存在时，是否自动创建topic\nbroker_version | char * | NULL | 指定broker的版本号，<0.10时需要手动指定\ncompress_type | int | NoCompress | produce消息的压缩类型\nclient_id | char * | NULL | 表示client的id\ncheck_crcs | bool | false | fetch任务中是否校验消息的crc32\noffset_store | int | 0 | 加入消费者组时，是否使用上次提交offset，1表示使用指定的offset，0表示优先使用上次提交\nsasl_mechanisms | char * | NULL | sasl认证类型，目前支持plain和scram\nsasl_username | char * | NULL | sasl认证所需的username\nsasl_password | char * | NULL | sasl认证所需的password\n\n\n最后就可以调用start接口启动kafka任务。\n\n# produce任务\n\n1、在创建并初始化WFKafkaClient之后，可以在query中直接指定topic等信息创建WFKafkaTask任务\n\n使用示例如下：\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tclient = new WFKafkaClient();\n\tclient->init(url);\n\ttask = client->create_kafka_task(\"api=fetch&topic=xxx&topic=yyy\", 3, kafka_callback);\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n\n2、在创建完WFKafkaTask之后，先通过调用set_key, set_value, add_header_pair等方法构建KafkaRecord，\n\n关于KafkaRecord的更多接口，可以在[KafkaDataTypes.h](../src/protocol/KafkaDataTypes.h)中查看\n\n然后应该通过调用add_produce_record添加KafkaRecord，关于更多接口的详细定义，可以在[WFKafkaClient.h](../src/client/WFKafkaClient.h)中查看\n\n需要注意的是，add_produce_record的第二个参数partition，当>=0是表示指定的partition，-1表示随机指定partition或者调用自定义的kafka_partitioner_t\n\nkafka_partitioner_t可以通过set_partitioner接口设置自定义规则。\n\n使用示例如下：\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tWFKafkaClient *client_fetch = new WFKafkaClient();\n\tclient_fetch->init(url);\n\ttask = client_fetch->create_kafka_task(\"api=produce&topic=xxx&topic=yyy\", 3, kafka_callback);\n\ttask->set_partitioner(partitioner);\n\n\tKafkaRecord record;\n\trecord.set_key(\"key1\", strlen(\"key1\"));\n\trecord.set_value(buf, sizeof(buf));\n\trecord.add_header_pair(\"hk1\", 3, \"hv1\", 3);\n\ttask->add_produce_record(\"workflow_test1\", -1, std::move(record));\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n\n3、produce还可以使用kafka支持的4种压缩协议，通过设置配置项来实现\n\n使用示例如下：\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tWFKafkaClient *client_fetch = new WFKafkaClient();\n\tclient_fetch->init(url);\n\ttask = client_fetch->create_kafka_task(\"api=produce&topic=xxx&topic=yyy\", 3, kafka_callback);\n\n\tKafkaConfig config;\n\tconfig.set_compress_type(Kafka_Zstd);\n\ttask->set_config(std::move(config));\n\n\tKafkaRecord record;\n\trecord.set_key(\"key1\", strlen(\"key1\"));\n\trecord.set_value(buf, sizeof(buf));\n\trecord.add_header_pair(\"hk1\", 3, \"hv1\", 3);\n\ttask->add_produce_record(\"workflow_test1\", -1, std::move(record));\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n\n\n# fetch任务\n\nfetch任务支持消费者组模式和手动模式\n\n1、手动模式\n\n无需指定消费者组，同时需要用户指定topic、partition和offset\n\n使用示例如下：\n~~~cpp\n\tclient = new WFKafkaClient();\n\tclient->init(url);\n\ttask = client->create_kafka_task(\"api=fetch\", 3, kafka_callback);\n\n\tKafkaToppar toppar;\n\ttoppar.set_topic_partition(\"workflow_test1\", 0);\n\ttoppar.set_offset(0);\n\ttask->add_toppar(toppar);\n~~~\n\n2、消费者组模式\n\n在初始化client的时候需要指定消费者组的名称\n\n使用示例如下：\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tWFKafkaClient *client_fetch = new WFKafkaClient();\n\tclient_fetch->init(url, cgroup_name);\n\ttask = client_fetch->create_kafka_task(\"api=fetch&topic=xxx&topic=yyy\", 3, kafka_callback);\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n\n3、offset的提交\n\n在消费者组模式下，用户消费消息后，可以在callback函数中，通过创建commit任务来自动提交消费的记录，使用示例如下：\n~~~cpp\nvoid kafka_callback(WFKafkaTask *task)\n{\n\t...\n\tcommit_task = client.create_kafka_task(\"api=commit\", 3, kafka_callback);\n\n\t...\n\tcommit_task->start();\n\t...\n}\n~~~\n\n\n# 关于client的关闭\n\n在消费者组模式下，client在关闭之前需要调用create_leavegroup_task创建leavegroup_task，\n\n它会发送leavegroup协议包，如果没有启动leavegroup_task，会导致消费者组没有正确退出，触发这个组的rebalance。\n\n\n# 处理kafka结果\n\n消息的结果集的数据结构是KafkaResult，可以通过调用WFKafkaTask的get_result()接口获得，\n\n然后调用KafkaResult的fetch_record接口可以将本次task相关的record取出来，它是一个KafkaRecord的二维vector，\n\n第一维是topic partition，第二维是某个topic partition下对应的KafkaRecord，\n\n在[KafkaResult.h](../src/protocol/KafkaResult.h)中可以看到KafkaResult的定义\n~~~cpp\nvoid kafka_callback(WFKafkaTask *task)\n{\n\tint state = task->get_state();\n\tint error = task->get_error();\n\n\t// handle error states\n\t...\n\n\tprotocol::KafkaResult *result = task->get_result();\n\tresult->fetch_records(records);\n\n\tfor (auto &v : records)\n\t{\n\t\tfor (auto &w: v)\n\t\t{\n\t\t\tconst void *value;\n\t\t\tsize_t value_len;\n\t\t\tw->get_value(&value, &value_len);\n\t\t\tprintf(\"produce\\ttopic: %s, partition: %d, status: %d, offset: %lld, val_len: %zu\\n\",\n\t\t\t\t w->get_topic(), w->get_partition(), w->get_status(), w->get_offset(), value_len);\n\t\t}\n\t}\n\t...\n\n\tprotocol::KafkaResult new_result = std::move(*task->get_result());\n\tif (new_result.fetch_records(records))\n\t{\n\t\tfor (auto &v : records)\n\t\t{\n\t\t\tif (v.empty())\n\t\t\t\tcontinue;\n\n\t\t\tfor (auto &w: v)\n\t\t\t{\n\t\t\t\tif (fp)\n\t\t\t\t{\n \tconst void *value;\n\t\t\t\t\tsize_t value_len;\n\t\t\t\t\tw->get_value(&value, &value_len);\n\t\t\t\t\tfwrite(w->get_value(), w->get_value_len(), 1, fp);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\t...\n}\n~~~\n\n# 认证\n认证信息需要在配置中设置，以sasl为例:\n~~~cpp\nint main(int argc, char *argv[])\n{\n\t...\n\tclient = new WFKafkaClient();\n\tclient->init(url);\n\n\ttask = client->create_kafka_task(\"api=fetch&topic=xxx&topic=yyy\", 3, kafka_callback);\n\tconfig.set_sasl_username(\"fetch\");\n\tconfig.set_sasl_password(\"fetch-secret\");\n\tconfig.set_sasl_mech(\"SCRAM-SHA-256\");\n\ttask->set_config(std::move(config));\n\n\t...\n\ttask->start();\n\t...\n}\n~~~\n" }, { "path": "docs/tutorial-15-name_service.md", "content": "# 自定义命名服务策略：name_service\n# 示例代码\n[tutorial-15-name_service.cc](/tutorial/tutorial-15-name_service.cc)\n\n# 关于name_service\n本示例通过一个用户定义文本文件来指定名称服务策略。文件格式定义与系统hosts文件兼容，目前也支持指向域名。例如：\n~~~\n127.0.0.1 www.myhost.com\n192.168.10.10 host1\nwwww.sogou.com sogou # 扩展功能，'sogou'指向'www.sogou.com'\n~~~\n用户在命令行输入抓取的URL和名称服务文件来抓取目标网页。如果输入URL的域名在文件中不存在，则正常使用DNS。\n\n# 自定义名称服务策略\n所有名称服务策略，从WFNSPolic派生。其唯一需要实现的是create_router_task函数。\n~~~cpp\nclass MyNSPolicy : public WFNSPolicy\n{\npublic:\n WFRouterTask *create_router_task(const struct WFNSParams *params, router_callback_t callback) override;\n ..\n};\n~~~\n在这个示例里，我们并不需要引入很复杂的选取策略，只需要从一个文本文件把域名做转化。 \n所以，我们可以把转化结果交给全局的dns resolver，让dns resolve产生真正的路由任务。\n~~~cpp\nWFRouterTask *MyNSPolicy::create_router_task(const struct WFNSParams *params, router_callback_t callback)\n{\n WFDnsResolver *dns_resolver = WFGlobal::get_dns_resolver();\n\n if (params->uri.host)\n {\n FILE *fp = fopen(this->path.c_str(), \"r\");\n if (fp)\n {\n std::string dest = this->read_from_fp(fp, params->uri.host);\n if (dest.size() > 0)\n {\n /* Update the uri structure's 'host' field directly.\n\t * You can also update the 'port' field if needed. */\n\t free(params->uri.host);\n params->uri.host = strdup(dest.c_str());\n }\n\n fclose(fp);\n }\n }\n\n /* Simply, use the global dns resolver to create a router task. */\n return dns_resolver->create_router_task(params, std::move(callback));\n}\n~~~\n其中read_from_fp函数从文本文件中读取信息并做转换，这个函数的实现大家可以直接看源代码。 \n得到转换结果之后，用新的host覆盖原params里uri的host即可。最后，调用dns resover产生路由任务。\n\n# 注册名称服务\nWorkflow里，可以给每个单独的域名指定一个名称服务策略。如果一个域名找不到指定策略，则使用默认。 \n一般情况下，默认名称服务策略即是dns resolver。下面，我们把我定义好的策略注册到输入URL的域名上：\n~~~cpp\nint main()\n{\n ...\n /* Create an naming policy. */\n MyNSPolicy *policy = new MyNSPolicy(filename);\n\n /* Get the global name service object.*/\n WFNameService *ns = WFGlobal::get_name_service();\n\n /* Add the our name with policy to global name service.\n * You can add mutilply names with one policy object. */\n ns->add_policy(name, policy);\n ...\n}\n~~~\n其中，name为URL里的域名。这样的话，这个域名下的所有URL，都将使用我们自定义的名称服务策略了。 \n在程序退出之前，我们也需要把这个策略从全局名称服务中删除，防止内存泄漏：\n~~~cpp\nint main()\n{\n ...\n /* clean up */\n ns->del_policy(name);\n delete policy;\n return 0;\n}\n~~~\n# 设置默认名称服务策略\n在这个例子中，其实我们并没有修改默认名称服务策略。有些情况下，我们可能想让所有的host都使用这个名称服务策略。 \n这种情况，我们也可以修改默认的策略，让这个策略对所有的host都生效。只需要调用全局名称服务的set_default_policy函数：\n~~~cpp\nint main()\n{\n MyNSPolicy *policy = new MyNSPolicy(filename);\n WFNameService *ns = WFGlobal::get_name_service();\n ns->set_default_policy(policy);\n ...\n\n /* Reset default policy to dns resolver and clean up */\n ns->set_default_policy(WFGlobal::get_dns_resolver());\n delete policy;\n return 0;\n}\n~~~\n" }, { "path": "docs/tutorial-17-dns_cli.md", "content": "# 使用workflow请求DNS\n作为一款优秀的异步编程框架，workflow帮助用户处理了大量的细节，其中就包括域名解析，因此在大部分情况下，用户无需关心如何请求DNS服务。正如workflow中的其他模块一样，DNS解析模块设计的同样完备而优雅，若恰好需要实现一些域名解析任务，workflow中的WFDnsClient和WFDnsTask无疑是一个绝佳的选择。\n\n[about-dns](about-dns.md)中介绍了如何配置DNS相关参数，而本篇文档的重点在于介绍如何创建DNS任务以及获取解析结果。\n\n[tutorial-17-dns_cli.cc](/tutorial/tutorial-17-dns_cli.cc)\n\n## 使用WFDnsClient创建任务\nWFDnsClient是经过封装的高级接口，其行为类似于系统提供的`resolv.conf`配置文件，帮助用户代理了重试、search列表拼接、server轮换等功能，使用起来非常简单。WFDnsClient的初始化方式有以下几种情况，当函数返回0时表示初始化成功\n\n- 使用一个DNS IPv4地址初始化，下述两种写法等价\n```cpp\nclient.init(\"8.8.8.8\");\n// or\nclient.init(\"dns://8.8.8.8/\");\n```\n- 使用一个DNS IPv6地址初始化\n```cpp\nclient.init(\"[2402:4e00::]:53\");\n```\n- 使用DNS over TLS(DoT)地址初始化，默认端口号为853\n```cpp\nclient.init(\"dnss://120.53.53.53/\");\n```\n- 使用多个由逗号分隔的DNS地址初始化\n```cpp\nclient.init(\"dns://8.8.8.8/,119.29.29.29\");\n```\n- 显式指定重试策略的初始化，示例代码等价于下述`resolv.conf`描述的策略\n```\nnameserver 8.8.8.8\nsearch sogou.com tencent.com\noptions nodts:1 attempts:2 rotate\n```\n```cpp\nclient.init(\"8.8.8.8\", \"sogou.com,tencent.com\", 1, 2, true);\n```\n\n使用WFDnsClient创建的任务默认为`DNS_TYPE_A`、`DNS_CLASS_IN`类型的解析请求，且已经设置了递归解析的选项，即`task->get_req()->set_rd(1)`。了解了`WFDnsClient`的初始化的方式，仅需八行即可发起一个DNS解析任务\n\n```cpp\nint main()\n{\n WFDnsClient client;\n client.init(\"8.8.8.8\");\n\n WFDnsTask *task = client.create_dns_task(\"www.sogou.com\", dns_callback);\n task->start();\n\n pause();\n\n client.deinit();\n return 0;\n}\n```\n\n## 使用工厂函数创建任务\n若不需要WFDnsClient提供的额外功能，或想自行组织重试策略，可使用工厂函数创建任务。\n\n使用工厂函数创建任务时，可以在`url path`中指定要被解析的域名，工厂函数创建的任务默认为`DNS_TYPE_A`、`DNS_CLASS_IN`类型的解析请求，创建后可以通过`set_question_type`和`set_question_class`修改，例如\n\n```cpp\nstd::string url = \"dns://8.8.8.8/www.sogou.com\";\nWFDnsTask *task = WFTaskFactory::create_dns_task(url, 0, dns_callback);\nprotocol::DnsRequest *req = task->get_req();\nreq->set_rd(1);\nreq->set_question_type(DNS_TYPE_AAAA);\nreq->set_question_class(DNS_CLASS_IN);\n```\n\n若不在创建任务时指定要被解析的域名(此时默认的任务是对根域名`.`进行解析)，在创建任务后可以使用`set_question`函数设置域名等参数，例如\n\n```cpp\nstd::string url = \"dns://8.8.8.8/\";\nWFDnsTask *task = WFTaskFactory::create_dns_task(url, 0, dns_callback);\nprotocol::DnsRequest *req = task->get_req();\nreq->set_rd(1);\nreq->set_question(\"www.zhihu.com\", DNS_TYPE_AAAA, DNS_CLASS_IN);\n```\n\n## 借助工具获取结果\n一次成功的DNS请求会获得完整的DNS请求结果，有两种简便的接口可以从结果中获取信息\n\n### DnsUtil::getaddrinfo\n该函数类似于系统的`getaddrinfo`函数，调用成功时返回零并成功获得一组`struct addrinfo`，调用失败时返回`EAI_*`类型的错误码。对该函数的成功调用最终**都应该**使用`DnsUtil::freeaddrinfo`释放资源\n\n```cpp\nvoid dns_callback(WFDnsTask *task)\n{\n // ignore handle error states\n\n struct addrinfo *res;\n protocol::DnsResponse *resp = task->get_resp();\n int ret = protocol::DnsUtil::getaddrinfo(resp, 80, &res);\n // ignore check ret == 0\n\n char ip_str[INET6_ADDRSTRLEN + 1] = { 0 };\n for (struct addrinfo *p = res; p; p = p->ai_next)\n {\n void *addr = nullptr;\n if (p->ai_family == AF_INET)\n addr = &((struct sockaddr_in *)p->ai_addr)->sin_addr;\n else if (p->ai_family == AF_INET6)\n addr = &((struct sockaddr_in6 *)p->ai_addr)->sin6_addr;\n\n if (addr)\n {\n inet_ntop(p->ai_family, addr, ip_str, p->ai_addrlen);\n printf(\"ip:%s\\n\", ip_str);\n }\n }\n\n protocol::DnsUtil::freeaddrinfo(res);\n}\n```\n\n### DnsResultCursor\n`DnsUtil::getaddrinfo`一般用于获取`IPv4`、`IPv6`地址，而使用DnsResultCursor可以完整地遍历DNS结果。DNS解析结果分为answer、authority、additional三个区域，一般情况下主要内容位于answer区域，此处分别判断每个区域是否有内容，并调用`show_result`以逐一展示结果\n\n```cpp\nvoid dns_callback(WFDnsTask *task)\n{\n // ignore handle error states\n\n protocol::DnsResponse *resp = task->get_resp();\n protocol::DnsResultCursor cursor(resp);\n\n if(resp->get_ancount() > 0)\n {\n cursor.reset_answer_cursor();\n printf(\";; ANSWER SECTION:\\n\");\n show_result(cursor);\n }\n if(resp->get_nscount() > 0)\n {\n cursor.reset_authority_cursor();\n printf(\";; AUTHORITY SECTION\\n\");\n show_result(cursor);\n }\n if(resp->get_arcount() > 0)\n {\n cursor.reset_additional_cursor();\n printf(\";; ADDITIONAL SECTION\\n\");\n show_result(cursor);\n }\n}\n```\n\n根据请求类型不同，结果中包含的数据可以多种多样，常见的有\n\n- DNS_TYPE_A: IPv4类型的地址\n- DNS_TYPE_AAAA: IPv6类型的地址\n- DNS_TYPE_NS: 该域名的权威DNS服务器\n- DNS_TYPE_CNAME: 该域名的权威名称\n\n```cpp\nvoid show_result(protocol::DnsResultCursor &cursor)\n{\n char information[1024];\n const char *info;\n struct dns_record *record;\n struct dns_record_soa *soa;\n struct dns_record_srv *srv;\n struct dns_record_mx *mx;\n\n while(cursor.next(&record))\n {\n switch (record->type)\n {\n case DNS_TYPE_A:\n info = inet_ntop(AF_INET, record->rdata, information, 64);\n break;\n case DNS_TYPE_AAAA:\n info = inet_ntop(AF_INET6, record->rdata, information, 64);\n break;\n case DNS_TYPE_NS:\n case DNS_TYPE_CNAME:\n case DNS_TYPE_PTR:\n info = (const char *)(record->rdata);\n break;\n case DNS_TYPE_SOA:\n soa = (struct dns_record_soa *)(record->rdata);\n sprintf(information, \"%s %s %u %d %d %d %u\",\n soa->mname, soa->rname, soa->serial, soa->refresh,\n soa->retry, soa->expire, soa->minimum\n );\n info = information;\n break;\n case DNS_TYPE_SRV:\n srv = (struct dns_record_srv *)(record->rdata);\n sprintf(information, \"%u %u %u %s\",\n srv->priority, srv->weight, srv->port, srv->target\n );\n info = information;\n break;\n case DNS_TYPE_MX:\n mx = (struct dns_record_mx *)(record->rdata);\n sprintf(information, \"%d %s\", mx->preference, mx->exchange);\n info = information;\n break;\n default:\n info = \"Unknown\";\n }\n\n printf(\"%s\\t%d\\t%s\\t%s\\t%s\\n\",\n record->name, record->ttl,\n dns_class2str(record->rclass),\n dns_type2str(record->type),\n info\n );\n }\n printf(\"\\n\");\n}\n```\n" }, { "path": "docs/tutorial-18-redis_subscriber.md", "content": "# Redis订阅模式\n\n## 示例代码\n[tutorial-18-redis_subscriber.cc](/tutorial/tutorial-18-redis_subscriber.cc)\n\n## 创建订阅客户端和任务\n在Workflow中，一个客户端网络任务通常是向服务端发出一个请求并接收一个回复，而Redis订阅任务不同，它会先发出一个订阅请求，然后源源不断地接收服务端推送过来的消息，在这个过程中，客户端还可以新增或取消channels、patterns。\n\n用于实现Redis订阅功能的任务是`WFRedisSubscribeTask`，与普通的Redis任务不同，它不从任务工厂产生，而是需要使用`WFRedisSubscriber`来创建。例如\n\n```cpp\nWFRedisSubscriber suber;\n\nif (suber.init(url) != 0)\n{\n std::cerr << \"Subscriber init failed \" << strerror(errno) << std::endl;\n exit(1);\n}\n\n// ...\n\nWFRedisSubscribeTask *task;\ntask = suber.create_subscribe_task(channels, extract, callback);\n\ntask->set_watch_timeout(1000000); // 1000秒\ntask->start();\n\n// 这里可以使用task的相关接口改变订阅内容\n// ...\n\ntask->release();\nsuber.deinit();\n```\n\n初始化`WFRedisSubscriber`需要使用`Redis URL`，这与普通Redis任务相同，不再赘述。创建订阅任务时，需要提供三个参数\n\n- channels/patterns: 一个或多个被订阅的channel(subscribe)或pattern(psubscribe)\n- extract: 收到服务端推送消息时的处理函数\n- callback: 任务结束后的回调函数\n\n这个例子中为`watch_timeout`设置了一个很长的时间，若这个时间较短，且服务端长时间未推送消息，则连接会因为超时而断开，订阅任务也会直接失败，请根据实际情况合理设置。\n\n当任务处理完成后，需要通过`task->release()`来释放这个任务，这也是与其他任务的一个不同之处。\n\n## 处理订阅消息\n服务端推送的消息由创建任务时指定的`extract`函数处理。后续描述中，subscribe对应channel，psubscribe对应pattern。\n\n1. 服务端推送的消息格式是具有三个元素的数组，第一个元素是字符串\"message\"或\"pmessage\"，第二个元素是该消息的channel或pattern的名称，第三个元素是消息的内容。\n2. subscribe或psubscribe请求的回复是具有三个元素的数组，第一个元素是字符串\"subscribe\"或\"psubscribe\"，第二个元素是channel或pattern的名称，第三个元素是当前通过subscribe或psubscribe命令已经订阅了多少个channel或pattern，是一个整数。如果一个请求订阅了多个channel或pattern，会有多个回复。\n3. unsubscribe或punsubscribe请求的回复是具有三个元素的数组，格式与订阅命令相似。当取消订阅但不指定channel或pattern时，表示取消所有该类型的订阅，对于所有已经订阅的channel或pattern，返回一个回复消息。若当前类型未订阅任何channel或pattern，则返回一个消息，其中名称部分为nil。\n\n更多详情可参阅redis文档。\n\n处理消息的一个示例如下，简单地将内容打印到标准输出\n\n```cpp\nvoid extract(WFRedisSubscribeTask *task)\n{\n\tauto *resp = task->get_resp();\n\tprotocol::RedisValue value;\n\n\tresp->get_result(value);\n\n\tif (value.is_array())\n\t{\n\t\tfor (size_t i = 0; i < value.arr_size(); i++)\n\t\t{\n\t\t\tif (value[i].is_string())\n\t\t\t\tstd::cout << value[i].string_value();\n\t\t\telse if (value[i].is_int())\n\t\t\t\tstd::cout << value[i].int_value();\n\t\t\telse if (value[i].is_nil())\n\t\t\t\tstd::cout << \"nil\";\n\t\t\telse\n\t\t\t\tstd::cout << \"Unexpected value in array!\";\n\n\t\t\tstd::cout << \"\\n\";\n\t\t}\n\t}\n\telse\n\t\tstd::cout << \"Unexpected value!\\n\";\n}\n```\n\n## 改变订阅内容\n在任务过程中，可以通过下述接口新增或取消订阅，注意在带有channels或patterns参数的接口中，请勿传入空数组。\n\n```cpp\n// ...\n\ntask->start();\n\n// 新增订阅一组channels\ntask->subscribe(channels);\n\n// 取消订阅一组channels\ntask->unsubscribe(channels);\n\n// 取消订阅所有channels\ntask->unsubscribe();\n\n// 新增订阅一组patterns\ntask->psubscribe(patterns);\n\n// 取消订阅一组patterns\ntask->punsubscribe(patterns);\n\n// 取消订阅所有patterns\ntask->punsubscribe();\n\ntask->release();\n```\n\n当所有channels和patterns都被取消订阅后，任务会直接结束，此后不能再新增订阅，请注意该细节。也可以直接通过`task->quit()`来主动结束任务。\n\n此外，订阅模式下可以通过`task->ping()`或`task->ping(message)`向Redis服务器发起`ping`请求。当任务设置了较小的`watch_timeout`，但服务端可能长时间没有消息推送时，通过定时发出`ping`请求可以令服务端推送`pong`响应，此时任务便不会因为超时而失败。\n" }, { "path": "docs/tutorial-19-dns_server.md", "content": "# 使用workflow实现DNS服务器\n前述文档已经讲解了使用workflow实现服务器的方法，workflow框架贴心地为用户处理了底层逻辑和各种细节，因此本文档主要介绍如何组装DNS消息。\n\n[tutorial-19-dns_server.cc](/tutorial/tutorial-19-dns_server.cc)\n\nDNS协议内容中包含三个section，有`DNS_ANSWER_SECTION`、`DNS_AUTHORITY_SECTION`、`DNS_ADDITIONAL_SECTION`，每个section中可包含零或多条资源记录`Resource record`。目前`protocol::DnsResponse`支持添加的资源记录类型有`DNS_TYPE_A`、`DNS_TYPE_AAAA`、`DNS_TYPE_CNAME`、`DNS_TYPE_PTR`、`DNS_TYPE_SOA`、`DNS_TYPE_SRV`、`DNS_TYPE_MX`，其接口如下所示。\n\n```cpp\nint add_a_record(int section, const char *name,\n\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t const void *data);\n\nint add_aaaa_record(int section, const char *name,\n\t\t\t\t\tuint16_t rclass, uint32_t ttl,\n\t\t\t\t\tconst void *data);\n\nint add_ns_record(int section, const char *name,\n\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t const char *data);\n\nint add_cname_record(int section, const char *name,\n\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t const char *data);\n\nint add_ptr_record(int section, const char *name,\n\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t const char *data);\n\nint add_soa_record(int section, const char *name,\n\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t const char *mname, const char *rname,\n\t\t\t\t uint32_t serial, int32_t refresh,\n\t\t\t\t int32_t retry, int32_t expire, uint32_t minimum);\n\nint add_srv_record(int section, const char *name,\n\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t uint16_t priority, uint16_t weight, uint16_t port,\n\t\t\t\t const char *target);\n\nint add_mx_record(int section, const char *name,\n\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t int16_t preference, const char *exchange);\n\nint add_raw_record(int section, const char *name, uint16_t type,\n\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t const void *data, uint16_t dlen);\n```\n\n例如要添加一条AAAA记录，可使用下述方式实现\n\n```cpp\nstruct in6_addr addr;\n\ninet_pton(AF_INET6, \"1234:5678:9abc:def0::\", (void *)&addr);\nresp->add_aaaa_record(DNS_ANSWER_SECTION,\n\t\t\t\t\t name.c_str(), DNS_CLASS_IN, 600, &addr);\n```\n\n对于未支持的资源记录类型，可通过`add_raw_record`接口添加，例如要添加一条TXT记录，可使用下述方式实现\n\n```cpp\nconst char *raw_txt_data = \"\\x0dmy dns server\\x0fyour dns server\";\nuint16_t data_len = 30;\n\nresp->add_raw_record(DNS_ANSWER_SECTION, name.c_str(), DNS_TYPE_TXT,\n\t\t\t\t\t DNS_CLASS_IN, 1200, raw_txt_data, data_len);\n```\n\n注意，默认情况下`WFDnsServer`会启动一个UDP服务，若需要启动TCP服务，可通过修改WFServerParams中的transport_type字段为`TT_TCP`来实现。DNS客户端通常会优先使用UDP协议发起请求，当要回复的消息过大时，可仅添加部分资源记录，并通过`resp->set_tc(1)`设置截断标记，指示客户端可使用TCP协议重新请求。\n" }, { "path": "docs/xmake.md", "content": "# 编译\n\n```\n// 编译workflow库\nxmake\n\n// 编译test\nxmake -g test\n// 运行test文件\nxmake run -g test\n\n// 编译tutorial\nxmake -g tutorial\n\n// 编译benchmark\nxmake -g benchmark\n```\n\n## 运行\n\n`xmake run -h` 可以查看运行哪些target\n\n选择一个target即可运行\n\n比如\n\n```\nxmake run tutorial-06-parallel_wget\n```\n\n## 安装\n\n```\nsudo xmake install\n```\n\n## 切换编译静态库/动态库\n\n```\n// 编译静态库\nxmake f -k static\nxmake -r\n```\n\n```\n// 编译动态库\nxmake f -k shared\nxmake -r\n```\n\n`tips : -r 代表 -rebuild`\n\n## 进行定制化裁剪\n\n`xmake f --help` 可查看我们定制的option\n\n```\nCommand options (Project Configuration):\n\n --workflow_inc=WORKFLOW_INC workflow inc (default: /media/psf/pro/workflow/_include)\n --upstream=[y|n] build upstream component (default: y)\n --consul=[y|n] build consul component\n --workflow_lib=WORKFLOW_LIB workflow lib (default: /media/psf/pro/workflow/_lib)\n --redis=[y|n] build redis component (default: y)\n --kafka=[y|n] build kafka component\n --mysql=[y|n] build mysql component (default: y)\n```\n\n你可以通过如下命令来进行各个模块的裁剪或集成\n\n```\nxmake f --redis=n --kafka=y --mysql=n\nxmake -r\n```\n" }, { "path": "src/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\n\nif(ANDROID)\n\tinclude_directories(${OPENSSL_INCLUDE_DIR})\n\tlink_directories(${OPENSSL_LINK_DIR})\nelse()\n\tfind_package(OpenSSL REQUIRED)\nendif ()\n\ninclude_directories(${OPENSSL_INCLUDE_DIR} ${INC_DIR}/workflow)\n\nif (KAFKA STREQUAL \"y\")\n\tfind_package(ZLIB REQUIRED)\n\tinclude_directories(${ZLIB_INCLUDE_DIRS})\n\n\tfind_path(SNAPPY_INCLUDE_PATH NAMES snappy.h)\n\tfind_library(SNAPPY_LIB NAMES snappy)\n\tif ((NOT SNAPPY_INCLUDE_PATH) OR (NOT SNAPPY_LIB))\n\t\tmessage(FATAL_ERROR \"Fail to find snappy with KAFKA=y\")\n\tendif ()\n\tinclude_directories(${SNAPPY_INCLUDE_PATH})\n\n\tfind_path(ZSTD_INCLUDE_PATH NAMES zstd.h)\n\tfind_library(ZSTD_LIB NAMES zstd)\n\tif ((NOT ZSTD_INCLUDE_PATH) OR (NOT ZSTD_LIB))\n\t\tmessage(FATAL_ERROR \"Fail to find zstd with KAFKA=y\")\n\tendif ()\n\tinclude_directories(${ZSTD_INCLUDE_PATH})\n\n\tfind_path(LZ4_INCLUDE_PATH NAMES lz4.h)\n\tfind_library(LZ4_LIB NAMES lz4)\n\tif ((NOT LZ4_INCLUDE_PATH) OR (NOT LZ4_LIB))\n\t\tmessage(FATAL_ERROR \"Fail to find lz4 with KAFKA=y\")\n\tendif ()\n\tinclude_directories(${LZ4_INCLUDE_PATH})\nendif ()\n\nadd_subdirectory(kernel)\nadd_subdirectory(util)\nadd_subdirectory(manager)\nadd_subdirectory(protocol)\nadd_subdirectory(factory)\nadd_subdirectory(nameservice)\nadd_subdirectory(server)\nadd_subdirectory(client)\n\nadd_dependencies(kernel LINK_HEADERS)\nadd_dependencies(util LINK_HEADERS)\nadd_dependencies(manager LINK_HEADERS)\nadd_dependencies(protocol LINK_HEADERS)\nadd_dependencies(factory LINK_HEADERS)\nadd_dependencies(nameservice LINK_HEADERS)\nadd_dependencies(server LINK_HEADERS)\nadd_dependencies(client LINK_HEADERS)\n\nset(STATIC_LIB_NAME ${PROJECT_NAME}-static)\nset(SHARED_LIB_NAME ${PROJECT_NAME}-shared)\n\nadd_library(\n\t${STATIC_LIB_NAME} STATIC\n\t$\n\t$\n\t$\n\t$\n\t$\n\t$\n\t$\n\t$\n)\n\nadd_library(\n\t${SHARED_LIB_NAME} SHARED\n\t$\n\t$\n\t$\n\t$\n\t$\n\t$\n\t$\n\t$\n)\n\nif(ANDROID)\n\ttarget_link_libraries(${SHARED_LIB_NAME} PUBLIC ssl crypto c)\n\ttarget_link_libraries(${STATIC_LIB_NAME} PUBLIC ssl crypto c)\nelse()\n\ttarget_link_libraries(${SHARED_LIB_NAME} PUBLIC OpenSSL::SSL OpenSSL::Crypto pthread)\n\ttarget_link_libraries(${STATIC_LIB_NAME} PUBLIC OpenSSL::SSL OpenSSL::Crypto pthread)\nendif ()\n\nset_target_properties(${STATIC_LIB_NAME} PROPERTIES OUTPUT_NAME ${PROJECT_NAME})\nset_target_properties(${SHARED_LIB_NAME} PROPERTIES OUTPUT_NAME ${PROJECT_NAME} VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})\n\nif (KAFKA STREQUAL \"y\")\n\tadd_dependencies(client_kafka LINK_HEADERS)\n\tadd_dependencies(util_kafka LINK_HEADERS)\n\tadd_dependencies(protocol_kafka LINK_HEADERS)\n\tadd_dependencies(factory_kafka LINK_HEADERS)\n\n\tset(KAFKA_STATIC_LIB_NAME \"wfkafka-static\")\n\tadd_library(\n\t\t${KAFKA_STATIC_LIB_NAME} STATIC\n\t\t$\n\t\t$\n\t\t$\n\t\t$\n\t)\n\ttarget_link_libraries(${KAFKA_STATIC_LIB_NAME} PUBLIC\n\t\t\t\t${STATIC_LIB_NAME} ZLIB::ZLIB ${LZ4_LIB} ${ZSTD_LIB} ${SNAPPY_LIB})\n\tset_target_properties(${KAFKA_STATIC_LIB_NAME} PROPERTIES OUTPUT_NAME \"wfkafka\")\n\n\tset(KAFKA_SHARED_LIB_NAME \"wfkafka-shared\")\n\tadd_library(\n\t\t${KAFKA_SHARED_LIB_NAME} SHARED\n\t\t$\n\t\t$\n\t\t$\n\t\t$\n\t)\n\ttarget_link_libraries(${KAFKA_SHARED_LIB_NAME} PUBLIC\n\t\t\t\t${SHARED_LIB_NAME} ZLIB::ZLIB ${LZ4_LIB} ${ZSTD_LIB} ${SNAPPY_LIB})\n\tset_target_properties(${KAFKA_SHARED_LIB_NAME} PROPERTIES OUTPUT_NAME \"wfkafka\" VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})\nendif ()\n\ninstall(\n\tTARGETS ${STATIC_LIB_NAME}\n\tARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}\n\tCOMPONENT devel\n)\n\ninstall(\n\tTARGETS ${SHARED_LIB_NAME}\n\tLIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}\n\tCOMPONENT devel\n)\n\nif (KAFKA STREQUAL \"y\")\n\tinstall(\n\t\tTARGETS ${KAFKA_STATIC_LIB_NAME}\n\t\tARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}\n\t\tCOMPONENT devel\n\t)\n\tinstall(\n\t\tTARGETS ${KAFKA_SHARED_LIB_NAME}\n\t\tLIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}\n\t\tCOMPONENT devel\n\t)\nendif ()\n\ntarget_include_directories(${STATIC_LIB_NAME} BEFORE PUBLIC\n\t\"$\"\n\t\"$\")\ntarget_include_directories(${SHARED_LIB_NAME} BEFORE PUBLIC\n\t\"$\"\n\t\"$\")\nif (KAFKA STREQUAL \"y\")\n\ttarget_include_directories(${KAFKA_STATIC_LIB_NAME} BEFORE PUBLIC\n\t\t\"$\"\n\t\t\"$\")\n\ttarget_include_directories(${KAFKA_SHARED_LIB_NAME} BEFORE PUBLIC\n\t\t\"$\"\n\t\t\"$\")\nendif ()\n" }, { "path": "src/client/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\nproject(client)\n\nset(SRC\n\tWFDnsClient.cc\n\tWFHttpChunkedClient.cc\n)\n\nif (NOT REDIS STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tWFRedisSubscriber.cc\n\t)\nendif ()\n\nif (NOT MYSQL STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tWFMySQLConnection.cc\n\t)\nendif ()\n\nif (NOT CONSUL STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tWFConsulClient.cc\n\t)\nendif ()\n\nadd_library(${PROJECT_NAME} OBJECT ${SRC})\n\nif (KAFKA STREQUAL \"y\")\n\tset(SRC\n\t\tWFKafkaClient.cc\n\t)\n\tadd_library(\"client_kafka\" OBJECT ${SRC})\nendif ()\n\n" }, { "path": "src/client/WFConsulClient.cc", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhenpeng (wangzhenpeng@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"json_parser.h\"\n#include \"StringUtil.h\"\n#include \"URIParser.h\"\n#include \"HttpUtil.h\"\n#include \"WFConsulClient.h\"\n\nusing namespace protocol;\n\nWFConsulTask::WFConsulTask(const std::string& proxy_url,\n\t\t\t\t\t\t const std::string& service_namespace,\n\t\t\t\t\t\t const std::string& service_name,\n\t\t\t\t\t\t const std::string& service_id,\n\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t consul_callback_t&& cb) :\n\tproxy_url(proxy_url),\n\tcallback(std::move(cb))\n{\n\tthis->service.service_name = service_name;\n\tthis->service.service_namespace = service_namespace;\n\tthis->service.service_id = service_id;\n\tthis->api_type = CONSUL_API_TYPE_UNKNOWN;\n\tthis->retry_max = retry_max;\n\tthis->finish = false;\n\tthis->consul_index = 0;\n\tthis->ssl_ctx = NULL;\n}\n\nvoid WFConsulTask::set_service(const struct protocol::ConsulService *service)\n{\n\tthis->service.tags = service->tags;\n\tthis->service.meta = service->meta;\n\tthis->service.tag_override = service->tag_override;\n\tthis->service.service_address = service->service_address;\n\tthis->service.lan = service->lan;\n\tthis->service.lan_ipv4 = service->lan_ipv4;\n\tthis->service.lan_ipv6 = service->lan_ipv6;\n\tthis->service.virtual_address = service->virtual_address;\n\tthis->service.wan = service->wan;\n\tthis->service.wan_ipv4 = service->wan_ipv4;\n\tthis->service.wan_ipv6 = service->wan_ipv6;\n}\n\nstatic bool parse_discover_result(const json_value_t *root,\n\t\t\t\t\t\t\tstd::vector& result);\nstatic bool parse_list_service_result(const json_value_t *root,\n\t\t\t\t\t\t\tstd::vector& result);\n\nbool WFConsulTask::get_discover_result(\n\tstd::vector& result) const\n{\n\tjson_value_t *root;\n\tint errno_bak;\n\tbool ret;\n\n\tif (this->api_type != CONSUL_API_TYPE_DISCOVER)\n\t{\n\t\terrno = EPERM;\n\t\treturn false;\n\t}\n\n\terrno_bak = errno;\n\terrno = EBADMSG;\n\tstd::string body = HttpUtil::decode_chunked_body(&this->http_resp);\n\troot = json_value_parse(body.c_str());\n\tif (!root)\n\t\treturn false;\n\n\tret = parse_discover_result(root, result);\n\tjson_value_destroy(root);\n\tif (ret)\n\t\terrno = errno_bak;\n\n\treturn ret;\n}\n\nbool WFConsulTask::get_list_service_result(\n\tstd::vector& result) const\n{\n\tjson_value_t *root;\n\tint errno_bak;\n\tbool ret;\n\n\tif (this->api_type != CONSUL_API_TYPE_LIST_SERVICE)\n\t{\n\t\terrno = EPERM;\n\t\treturn false;\n\t}\n\n\terrno_bak = errno;\n\terrno = EBADMSG;\n\tstd::string body = HttpUtil::decode_chunked_body(&this->http_resp);\n\troot = json_value_parse(body.c_str());\n\tif (!root)\n\t\treturn false;\n\n\tret = parse_list_service_result(root, result);\n\tjson_value_destroy(root);\n\tif (ret)\n\t\terrno = errno_bak;\n\n\treturn ret;\n}\n\nvoid WFConsulTask::dispatch()\n{\n\tWFHttpTask *task;\n\n\tif (this->finish)\n\t{\n\t\tthis->subtask_done();\n\t\treturn;\n\t}\n\n\tswitch(this->api_type)\n\t{\n\tcase CONSUL_API_TYPE_DISCOVER:\n\t\ttask = create_discover_task();\n\t\tbreak;\n\n\tcase CONSUL_API_TYPE_LIST_SERVICE:\n\t\ttask = create_list_service_task();\n\t\tbreak;\n\n\tcase CONSUL_API_TYPE_DEREGISTER:\n\t\ttask = create_deregister_task();\n\t\tbreak;\n\n\tcase CONSUL_API_TYPE_REGISTER:\n\t\ttask = create_register_task();\n\t\tif (task)\n\t\t\tbreak;\n\n\t\tif (1)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t}\n\t\telse\n\t\t{\n\tdefault:\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_CONSUL_API_UNKNOWN;\n\t\t}\n\n\t\tthis->finish = true;\n\t\tthis->subtask_done();\n\t\treturn;\n\t}\n\n\tauto *t = (WFComplexClientTask *)task;\n\tt->set_ssl_ctx(this->ssl_ctx);\n\n\tseries_of(this)->push_front(this);\n\tseries_of(this)->push_front(task);\n\tthis->subtask_done();\n}\n\nSubTask *WFConsulTask::done()\n{\n\tSeriesWork *series = series_of(this);\n\n\tif (finish)\n\t{\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t}\n\n\treturn series->pop();\n}\n\nstatic std::string convert_time_to_str(int milliseconds)\n{\n\tstd::string str_time;\n\tint seconds = milliseconds / 1000;\n\n\tif (seconds >= 180)\n\t\tstr_time = std::to_string(seconds / 60) + \"m\";\n\telse\n\t\tstr_time = std::to_string(seconds) + \"s\";\n\n\treturn str_time;\n}\n\nstd::string WFConsulTask::generate_discover_request()\n{\n\tstd::string url = this->proxy_url;\n\n\turl += \"/v1/health/service/\" + this->service.service_name;\n\turl += \"?dc=\" + this->config.get_datacenter();\n\turl += \"&ns=\" + this->service.service_namespace;\n\tstd::string passing = this->config.get_passing() ? \"true\" : \"false\";\n\turl += \"&passing=\" + passing;\n\turl += \"&token=\" + this->config.get_token();\n\turl += \"&filter=\" + this->config.get_filter_expr();\n\n\t//consul blocking query\n\tif (this->config.blocking_query())\n\t{\n\t\turl += \"&index=\" + std::to_string(this->get_consul_index());\n\t\turl += \"&wait=\" + convert_time_to_str(this->config.get_wait_ttl());\n\t}\n\n\treturn url;\n}\n\nWFHttpTask *WFConsulTask::create_discover_task()\n{\n\tstd::string url = generate_discover_request();\n\tWFHttpTask *task = WFTaskFactory::create_http_task(url, 0, this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t discover_callback);\n\tHttpRequest *req = task->get_req();\n\n\treq->add_header_pair(\"Content-Type\", \"application/json\");\n\ttask->user_data = this;\n\treturn task;\n}\n\nWFHttpTask *WFConsulTask::create_list_service_task()\n{\n\tstd::string url = this->proxy_url;\n\n\turl += \"/v1/catalog/services?token=\" + this->config.get_token();\n\turl += \"&dc=\" + this->config.get_datacenter();\n\turl += \"&ns=\" + this->service.service_namespace;\n\t\n\tWFHttpTask *task = WFTaskFactory::create_http_task(url, 0, this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t list_service_callback);\n\tHttpRequest *req = task->get_req();\n\n\treq->add_header_pair(\"Content-Type\", \"application/json\");\n\ttask->user_data = this;\n\treturn task;\n}\n\nstatic void print_json_value(const json_value_t *val, int depth,\n\t\t\t\t\t\t\t std::string& json_str);\n\nstatic bool create_register_request(const json_value_t *root,\n\t\t\t\t\t\t\t\t\tconst struct ConsulService *service,\n\t\t\t\t\t\t\t\t\tconst ConsulConfig& config);\n\nWFHttpTask *WFConsulTask::create_register_task()\n{\n\tstd::string payload;\n\n\tstd::string url = this->proxy_url;\n\turl += \"/v1/agent/service/register?replace-existing-checks=\";\n\turl += this->config.get_replace_checks() ? \"true\" : \"false\";\n\n\tWFHttpTask *task = WFTaskFactory::create_http_task(url, 0, this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t register_callback);\n\tHttpRequest *req = task->get_req();\n\n\treq->set_method(HttpMethodPut);\n\treq->add_header_pair(\"Content-Type\", \"application/json\");\n\n\tif (!this->config.get_token().empty())\n\t\treq->add_header_pair(\"X-Consul-Token\", this->config.get_token());\n\n\tjson_value_t *root = json_value_create(JSON_VALUE_OBJECT);\n\tif (root)\n\t{\n\t\tif (create_register_request(root, &this->service, this->config))\n\t\t\tprint_json_value(root, 0, payload);\n\n\t\tjson_value_destroy(root);\n\t\tif (!payload.empty() && req->append_output_body(payload))\n\t\t{\n\t\t\ttask->user_data = this;\n\t\t\treturn task;\n\t\t}\n\t}\n\n\ttask->dismiss();\n\treturn NULL;\n}\n\nWFHttpTask *WFConsulTask::create_deregister_task()\n{\n\tstd::string url = this->proxy_url;\n\n\turl += \"/v1/agent/service/deregister/\" + this->service.service_id;\n\turl += \"?ns=\" + this->service.service_namespace;\n\n\tWFHttpTask *task = WFTaskFactory::create_http_task(url, 0, this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t register_callback);\n\tHttpRequest *req = task->get_req();\n\n\treq->set_method(HttpMethodPut);\n\treq->add_header_pair(\"Content-Type\", \"application/json\");\n\n\tstd::string token = this->config.get_token();\n\tif (!token.empty())\n\t\treq->add_header_pair(\"X-Consul-Token\", token);\n\n\ttask->user_data = this;\n\treturn task;\n}\n\nbool WFConsulTask::check_task_result(WFHttpTask *task, WFConsulTask *consul_task)\n{\n\tif (task->get_state() != WFT_STATE_SUCCESS)\n\t{\n\t\tconsul_task->state = task->get_state();\n\t\tconsul_task->error = task->get_error();\n\t\treturn false;\n\t}\n\n\tprotocol::HttpResponse *resp = task->get_resp();\n\tif (strcmp(resp->get_status_code(), \"200\") != 0)\n\t{\n\t\tconsul_task->state = WFT_STATE_TASK_ERROR;\n\t\tconsul_task->error = WFT_ERR_CONSUL_CHECK_RESPONSE_FAILED;\n\t\treturn false;\n\t}\n\n\treturn true;\n}\n\nlong long WFConsulTask::get_consul_index(HttpResponse *resp)\n{\n\tlong long consul_index = 0;\n\n\t// get consul-index from http header\n\tprotocol::HttpHeaderCursor cursor(resp);\n\tstd::string consul_index_str;\n\tif (cursor.find(\"X-Consul-Index\", consul_index_str))\n\t{\n\t\tconsul_index = strtoll(consul_index_str.c_str(), NULL, 10);\n\t\tif (consul_index < 0)\n\t\t\tconsul_index = 0;\n\t}\n\n\treturn consul_index;\n}\n\nvoid WFConsulTask::discover_callback(WFHttpTask *task)\n{\n\tWFConsulTask *t = (WFConsulTask*)task->user_data;\n\n\tif (WFConsulTask::check_task_result(task, t))\n\t{\n\t\tprotocol::HttpResponse *resp = task->get_resp();\n\t\tlong long consul_index = t->get_consul_index(resp);\n\t\tlong long last_consul_index = t->get_consul_index();\n\t\tt->set_consul_index(consul_index < last_consul_index ? 0 : consul_index);\n\t\tt->state = task->get_state();\n\t}\n\n\tt->http_resp = std::move(*task->get_resp());\n\tt->finish = true;\n}\n\nvoid WFConsulTask::list_service_callback(WFHttpTask *task)\n{\n\tWFConsulTask *t = (WFConsulTask*)task->user_data;\n\n\tif (WFConsulTask::check_task_result(task, t))\n\t\tt->state = task->get_state();\n\n\tt->http_resp = std::move(*task->get_resp());\n\tt->finish = true;\n}\n\nvoid WFConsulTask::register_callback(WFHttpTask *task)\n{\n\tWFConsulTask *t = (WFConsulTask *)task->user_data;\n\n\tif (WFConsulTask::check_task_result(task, t))\n\t\tt->state = task->get_state();\n\n\tt->http_resp = std::move(*task->get_resp());\n\tt->finish = true;\n}\n\nint WFConsulClient::init(const std::string& proxy_url, ConsulConfig config,\n\t\t\t\t\t\t SSL_CTX *ssl_ctx)\n{\n\tParsedURI uri;\n\n\tif (URIParser::parse(proxy_url, uri) >= 0)\n\t{\n\t\tthis->proxy_url = uri.scheme;\n\t\tthis->proxy_url += \"://\";\n\t\tthis->proxy_url += uri.host;\n\t\tif (uri.port)\n\t\t{\n\t\t\tthis->proxy_url += \":\";\n\t\t\tthis->proxy_url += uri.port;\n\t\t}\n\n\t\tthis->config = std::move(config);\n\t\tthis->ssl_ctx = ssl_ctx;\n\t\treturn 0;\n\t}\n\telse if (uri.state == URI_STATE_INVALID)\n\t\terrno = EINVAL;\n\n\treturn -1;\n}\n\nWFConsulTask *WFConsulClient::create_discover_task(\n\t\t\t\t\t\t\t\t\t\tconst std::string& service_namespace,\n\t\t\t\t\t\t\t\t\t\tconst std::string& service_name,\n\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\tconsul_callback_t cb)\n{\n\tWFConsulTask *task = new WFConsulTask(this->proxy_url, service_namespace,\n\t\t\t\t\t\t\t\t\t\t service_name, \"\", retry_max,\n\t\t\t\t\t\t\t\t\t\t std::move(cb));\n\ttask->set_api_type(CONSUL_API_TYPE_DISCOVER);\n\ttask->set_config(this->config);\n\ttask->set_ssl_ctx(this->ssl_ctx);\n\treturn task;\n}\n\nWFConsulTask *WFConsulClient::create_list_service_task(\n\t\t\t\t\t\t\t\t\t\tconst std::string& service_namespace,\n\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\tconsul_callback_t cb)\n{\n\tWFConsulTask *task = new WFConsulTask(this->proxy_url, service_namespace,\n\t\t\t\t\t\t\t\t\t\t \"\", \"\", retry_max,\n\t\t\t\t\t\t\t\t\t\t std::move(cb));\n\ttask->set_api_type(CONSUL_API_TYPE_LIST_SERVICE);\n\ttask->set_config(this->config);\n\ttask->set_ssl_ctx(this->ssl_ctx);\n\treturn task;\n}\n\nWFConsulTask *WFConsulClient::create_register_task(\n\t\t\t\t\t\t\t\t\t\tconst std::string& service_namespace,\n\t\t\t\t\t\t\t\t\t\tconst std::string& service_name,\n\t\t\t\t\t\t\t\t\t\tconst std::string& service_id,\n\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\tconsul_callback_t cb)\n{\n\tWFConsulTask *task = new WFConsulTask(this->proxy_url, service_namespace,\n\t\t\t\t\t\t\t\t\t\t service_name, service_id, retry_max,\n\t\t\t\t\t\t\t\t\t\t std::move(cb));\n\ttask->set_api_type(CONSUL_API_TYPE_REGISTER);\n\ttask->set_config(this->config);\n\ttask->set_ssl_ctx(this->ssl_ctx);\n\treturn task;\n}\n\nWFConsulTask *WFConsulClient::create_deregister_task(\n\t\t\t\t\t\t\t\t\t\tconst std::string& service_namespace,\n\t\t\t\t\t\t\t\t\t\tconst std::string& service_id,\n\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\tconsul_callback_t cb)\n{\n\tWFConsulTask *task = new WFConsulTask(this->proxy_url, service_namespace,\n\t\t\t\t\t\t\t\t\t\t \"\", service_id, retry_max,\n\t\t\t\t\t\t\t\t\t\t std::move(cb));\n\ttask->set_api_type(CONSUL_API_TYPE_DEREGISTER);\n\ttask->set_config(this->config);\n\ttask->set_ssl_ctx(this->ssl_ctx);\n\treturn task;\n}\n\nstatic bool create_tagged_address(const ConsulAddress& consul_address,\n\t\t\t\t\t\t\t\t const std::string& name,\n\t\t\t\t\t\t\t\t json_object_t *tagged_obj)\n{\n\tif (consul_address.first.empty())\n\t\treturn true;\n\n\tconst json_value_t *val = json_object_append(tagged_obj, name.c_str(),\n\t\t\t\t\t\t\t\t\t\t\t\t JSON_VALUE_OBJECT);\n\tif (!val)\n\t\treturn false;\n\n\tjson_object_t *obj = json_value_object(val);\n\n\tif (!json_object_append(obj, \"Address\", JSON_VALUE_STRING,\n\t\t\t\t\t\t\tconsul_address.first.c_str()))\n\t\treturn false;\n\n\tif (!json_object_append(obj, \"Port\", JSON_VALUE_NUMBER,\n\t\t\t\t\t\t\t(double)consul_address.second))\n\t\treturn false;\n\n\treturn true;\n}\n\nstatic bool create_health_check(const ConsulConfig& config, json_object_t *obj)\n{\n\tconst json_value_t *val;\n\tstd::string str;\n\n\tif (!config.get_health_check())\n\t\treturn true;\n\n\tval = json_object_append(obj, \"Check\", JSON_VALUE_OBJECT);\n\tif (!val)\n\t\treturn false;\n\n\tobj = json_value_object(val);\n\n\tstr = config.get_check_name();\n\tif (!json_object_append(obj, \"Name\", JSON_VALUE_STRING, str.c_str()))\n\t\treturn false;\n\n\tstr = config.get_check_notes();\n\tif (!json_object_append(obj, \"Notes\", JSON_VALUE_STRING, str.c_str()))\n\t\treturn false;\n\n\tstr = config.get_check_http_url();\n\tif (!str.empty())\n\t{\n\t\tif (!json_object_append(obj, \"HTTP\", JSON_VALUE_STRING, str.c_str()))\n\t\t\treturn false;\n\n\t\tstr = config.get_check_http_method();\n\t\tif (!json_object_append(obj, \"Method\", JSON_VALUE_STRING, str.c_str()))\n\t\t\treturn false;\n\n\t\tstr = config.get_http_body();\n\t\tif (!json_object_append(obj, \"Body\", JSON_VALUE_STRING, str.c_str()))\n\t\t\treturn false;\n\n\t\tval = json_object_append(obj, \"Header\", JSON_VALUE_OBJECT);\n\t\tif (!val)\n\t\t\treturn false;\n\n\t\tjson_object_t *header_obj = json_value_object(val);\n\n\t\tfor (const auto& header : *config.get_http_headers())\n\t\t{\n\t\t\tval = json_object_append(header_obj, header.first.c_str(),\n\t\t\t\t\t\t\t\t\t JSON_VALUE_ARRAY);\n\t\t\tif (!val)\n\t\t\t\treturn false;\n\n\t\t\tjson_array_t *arr = json_value_array(val);\n\n\t\t\tfor (const auto& value : header.second)\n\t\t\t{\n\t\t\t\tif (!json_array_append(arr, JSON_VALUE_STRING, value.c_str()))\n\t\t\t\t\treturn false;\n\t\t\t}\n\t\t}\n\t}\n\n\tstr = config.get_check_tcp();\n\tif (!str.empty())\n\t{\n\t\tif (!json_object_append(obj, \"TCP\", JSON_VALUE_STRING, str.c_str()))\n\t\t\treturn false;\n\t}\n\n\tstr = config.get_initial_status();\n\tif (!json_object_append(obj, \"Status\", JSON_VALUE_STRING, str.c_str()))\n\t\treturn false;\n\n\tstr = convert_time_to_str(config.get_auto_deregister_time());\n\tif (!json_object_append(obj, \"DeregisterCriticalServiceAfter\",\n\t\t\t\t\t\t\tJSON_VALUE_STRING, str.c_str()))\n\t\treturn false;\n\n\tstr = convert_time_to_str(config.get_check_interval());\n\tif (!json_object_append(obj, \"Interval\", JSON_VALUE_STRING, str.c_str()))\n\t\treturn false;\n\n\tstr = convert_time_to_str(config.get_check_timeout());\n\tif (!json_object_append(obj, \"Timeout\", JSON_VALUE_STRING, str.c_str()))\n\t\treturn false;\n\n\tif (!json_object_append(obj, \"SuccessBeforePassing\", JSON_VALUE_NUMBER,\n\t\t\t\t\t\t\t(double)config.get_success_times()))\n\t\treturn false;\n\n\tif (!json_object_append(obj, \"FailuresBeforeCritical\", JSON_VALUE_NUMBER,\n\t\t\t\t\t\t\t(double)config.get_failure_times()))\n\t\treturn false;\n\n\treturn true;\n}\n\nstatic bool create_register_request(const json_value_t *root,\n\t\t\t\t\t\t\t\t\tconst struct ConsulService *service,\n\t\t\t\t\t\t\t\t\tconst ConsulConfig& config)\n{\n\tconst json_value_t *val;\n\tjson_object_t *obj;\n\n\tobj = json_value_object(root);\n\tif (!obj)\n\t\treturn false;\n\n\tif (!json_object_append(obj, \"ID\", JSON_VALUE_STRING,\n\t\t\t\t\t\t\tservice->service_id.c_str()))\n\t\treturn false;\n\n\tif (!json_object_append(obj, \"Name\", JSON_VALUE_STRING,\n\t\t\t\t\t\t\tservice->service_name.c_str()))\n\t\treturn false;\n\n\tif (!service->service_namespace.empty())\n\t{\n\t\tif (!json_object_append(obj, \"ns\", JSON_VALUE_STRING,\n\t\t\t\t\t\t\t\tservice->service_namespace.c_str()))\n\t\t\treturn false;\n\t}\n\n\tval = json_object_append(obj, \"Tags\", JSON_VALUE_ARRAY);\n\tif (!val)\n\t\treturn false;\n\n\tjson_array_t *arr = json_value_array(val);\n\n\tfor (const auto& tag : service->tags)\n\t{\n\t\tif (!json_array_append(arr, JSON_VALUE_STRING, tag.c_str()))\n\t\t\treturn false;\n\t}\n\n\tif (!json_object_append(obj, \"Address\", JSON_VALUE_STRING,\n\t\t\t\t\t\t\tservice->service_address.first.c_str()))\n\t\treturn false;\n\n\tif (!json_object_append(obj, \"Port\", JSON_VALUE_NUMBER,\n\t\t\t\t\t\t\t(double)service->service_address.second))\n\t\treturn false;\n\n\tval = json_object_append(obj, \"Meta\", JSON_VALUE_OBJECT);\n\tif (!val)\n\t\treturn false;\n\n\tjson_object_t *meta_obj = json_value_object(val);\n\n\tfor (const auto& meta_kv : service->meta)\n\t{\n\t\tif (!json_object_append(meta_obj, meta_kv.first.c_str(),\n\t\t\t\t\t\t\t\tJSON_VALUE_STRING, meta_kv.second.c_str()))\n\t\t\treturn false;\n\t}\n\n\tint type = service->tag_override ? JSON_VALUE_TRUE : JSON_VALUE_FALSE;\n\tif (!json_object_append(obj, \"EnableTagOverride\", type))\n\t\treturn false;\n\n\tval = json_object_append(obj, \"TaggedAddresses\", JSON_VALUE_OBJECT);\n\tif (!val)\n\t\treturn false;\n\n\tjson_object_t *tagged_obj = json_value_object(val);\n\tif (!tagged_obj)\n\t\treturn false;\n\n\tif (!create_tagged_address(service->lan, \"lan\", tagged_obj))\n\t\treturn false;\n\n\tif (!create_tagged_address(service->lan_ipv4, \"lan_ipv4\", tagged_obj))\n\t\treturn false;\n\n\tif (!create_tagged_address(service->lan_ipv6, \"lan_ipv6\", tagged_obj))\n\t\treturn false;\n\n\tif (!create_tagged_address(service->virtual_address, \"virtual\", tagged_obj))\n\t\treturn false;\n\n\tif (!create_tagged_address(service->wan, \"wan\", tagged_obj))\n\t\treturn false;\n\n\tif (!create_tagged_address(service->wan_ipv4, \"wan_ipv4\", tagged_obj))\n\t\treturn false;\n\n\tif (!create_tagged_address(service->wan_ipv6, \"wan_ipv6\", tagged_obj))\n\t\treturn false;\n\n\t// create health check\n\tif (!create_health_check(config, obj))\n\t\treturn false;\n\n\treturn true;\n}\n\nstatic bool parse_list_service_result(const json_value_t *root,\n\t\t\t\t\t\t\tstd::vector& result)\n{\n\tconst json_object_t *obj;\n\tconst json_value_t *val;\n\tconst json_array_t *arr;\n\tconst char *key;\n\tconst char *str;\n\n\tobj = json_value_object(root);\n\tif (!obj)\n\t\treturn false;\n\n\tjson_object_for_each(key, val, obj)\n\t{\n\t\tstruct ConsulServiceTags instance;\n\n\t\tinstance.service_name = key;\n\t\tarr = json_value_array(val);\n\t\tif (!arr)\n\t\t\treturn false;\n\n\t\tconst json_value_t *tag_val;\n\t\tjson_array_for_each(tag_val, arr)\n\t\t{\n\t\t\tstr = json_value_string(tag_val);\n\t\t\tif (!str)\n\t\t\t\treturn false;\n\n\t\t\tinstance.tags.emplace_back(str);\n\t\t}\n\n\t\tresult.emplace_back(std::move(instance));\n\t}\n\n\treturn true;\n}\n\nstatic bool parse_discover_node(const json_object_t *obj,\n\t\t\t\t\t\t\t\tstruct ConsulServiceInstance *instance)\n{\n\tconst json_value_t *val;\n\tconst char *str;\n\n\tval = json_object_find(\"Node\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tobj = json_value_object(val);\n\tif (!obj)\n\t\treturn false;\n\n\tval = json_object_find(\"ID\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tstr = json_value_string(val);\n\tif (!str)\n\t\treturn false;\n\n\tinstance->node_id = str;\n\tval = json_object_find(\"Node\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tstr = json_value_string(val);\n\tif (!str)\n\t\treturn false;\n\n\tinstance->node_name = str;\n\tval = json_object_find(\"Address\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tstr = json_value_string(val);\n\tif (!str)\n\t\treturn false;\n\n\tinstance->node_address = str;\n\tval = json_object_find(\"Datacenter\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tstr = json_value_string(val);\n\tif (!str)\n\t\treturn false;\n\n\tinstance->dc = str;\n\n\tval = json_object_find(\"Meta\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tconst json_object_t *meta_obj = json_value_object(val);\n\tif (!meta_obj)\n\t\treturn false;\n\n\tconst char *meta_k;\n\tconst json_value_t *meta_v;\n\n\tjson_object_for_each(meta_k, meta_v, meta_obj)\n\t{\n\t\tstr = json_value_string(meta_v);\n\t\tif (!str)\n\t\t\treturn false;\n\n\t\tinstance->node_meta[meta_k] = str;\n\t}\n\n\tval = json_object_find(\"CreateIndex\", obj);\n\tif (val && json_value_type(val) == JSON_VALUE_NUMBER)\n\t\tinstance->create_index = json_value_number(val);\n\n\tval = json_object_find(\"ModifyIndex\", obj);\n\tif (val && json_value_type(val) == JSON_VALUE_NUMBER)\n\t\tinstance->modify_index = json_value_number(val);\n\n\treturn true;\n}\n\nstatic bool parse_tagged_address(const char *name, \n\t\t\t\t\t\t\t\t const json_value_t *tagged_val,\n\t\t\t\t\t\t\t\t ConsulAddress& tagged_address)\n{\n\tconst json_value_t *val;\n\tconst json_object_t *obj;\n\tconst char *str;\n\n\tobj = json_value_object(tagged_val);\n\tif (!obj)\n\t\treturn false;\n\n\tval = json_object_find(name, obj);\n\tif (!val)\n\t\treturn false;\n\n\tobj = json_value_object(val);\n\tif (!obj)\n\t\treturn false;\n\n\tval = json_object_find(\"Address\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tstr = json_value_string(val);\n\tif (!str)\n\t\treturn false;\n\n\ttagged_address.first = str;\n\tval = json_object_find(\"Port\", obj);\n\tif (!val || json_value_type(val) != JSON_VALUE_NUMBER)\n\t\treturn false;\n\n\ttagged_address.second = json_value_number(val);\n\treturn true;\n}\n\nstatic bool parse_service(const json_object_t *obj,\n\t\t\t\t\t\t struct ConsulService *service)\n{\n\tconst json_value_t *val;\n\tconst char *str;\n\n\tval = json_object_find(\"Service\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tobj = json_value_object(val);\n\tif (!obj)\n\t\treturn false;\n\n\tval = json_object_find(\"ID\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tstr = json_value_string(val);\n\tif (!str)\n\t\treturn false;\n\n\tservice->service_id = str;\n\tval = json_object_find(\"Service\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tstr = json_value_string(val);\n\tif (!str)\n\t\treturn false;\n\n\tservice->service_name = str;\n\tval = json_object_find(\"Namespace\", obj);\n\tif (val)\n\t{\n\t\tstr = json_value_string(val);\n\t\tif (!str)\n\t\t\treturn false;\n\n\t\tservice->service_namespace = str;\n\t}\n\n\tval = json_object_find(\"Address\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tstr = json_value_string(val);\n\tif (!str)\n\t\treturn false;\n\n\tservice->service_address.first = str;\n\n\tval = json_object_find(\"Port\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tservice->service_address.second = json_value_number(val);\n\tval = json_object_find(\"TaggedAddresses\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tparse_tagged_address(\"lan\", val, service->lan);\n\tparse_tagged_address(\"lan_ipv4\", val, service->lan_ipv4);\n\tparse_tagged_address(\"lan_ipv6\", val, service->lan_ipv6);\n\tparse_tagged_address(\"virtual\", val, service->virtual_address);\n\tparse_tagged_address(\"wan\", val, service->wan);\n\tparse_tagged_address(\"wan_ipv4\", val, service->wan_ipv4);\n\tparse_tagged_address(\"wan_ipv6\", val, service->wan_ipv6);\n\n\tval = json_object_find(\"Tags\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tconst json_array_t *tags_arr = json_value_array(val);\n\tif (tags_arr)\n\t{\n\t\tconst json_value_t *tags_value;\n\t\tjson_array_for_each(tags_value, tags_arr)\n\t\t{\n\t\t\tstr = json_value_string(tags_value);\n\t\t\tif (!str)\n\t\t\t\treturn false;\n\n\t\t\tservice->tags.emplace_back(str);\n\t\t}\n\t}\n\n\tval = json_object_find(\"Meta\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tconst json_object_t *meta_obj = json_value_object(val);\n\tif (!meta_obj)\n\t\treturn false;\n\n\tconst char *meta_k;\n\tconst json_value_t *meta_v;\n\tjson_object_for_each(meta_k, meta_v, meta_obj)\n\t{\n\t\tstr = json_value_string(meta_v);\n\t\tif (!str)\n\t\t\treturn false;\n\n\t\tservice->meta[meta_k] = str; \n\t}\n\n\tval = json_object_find(\"EnableTagOverride\", obj);\n\tif (val)\n\t\tservice->tag_override = (json_value_type(val) == JSON_VALUE_TRUE);\n\n\treturn true;\n}\n\nstatic bool parse_health_check(const json_object_t *obj,\n\t\t\t\t\t\t\t struct ConsulServiceInstance *instance)\n{\n\tconst json_value_t *val;\n\tconst char *str;\n\n\tval = json_object_find(\"Checks\", obj);\n\tif (!val)\n\t\treturn false;\n\n\tconst json_array_t *check_arr = json_value_array(val);\n\tif (!check_arr)\n\t\treturn false;\n\n\tconst json_value_t *arr_val;\n\tjson_array_for_each(arr_val, check_arr)\n\t{\n\t\tobj = json_value_object(arr_val);\n\t\tif (!obj)\n\t\t\treturn false;\n\n\t\tval = json_object_find(\"ServiceName\", obj);\n\t\tif (!val)\n\t\t\treturn false;\n\n\t\tstr = json_value_string(val);\n\t\tif (!str)\n\t\t\treturn false;\n\n\t\tstd::string check_service_name = str;\n\t\tval = json_object_find(\"ServiceID\", obj);\n\t\tif (!val)\n\t\t\treturn false;\n\n\t\tstr = json_value_string(val);\n\t\tif (!str)\n\t\t\treturn false;\n\n\t\tstd::string check_service_id = str;\n\t\tif (check_service_id.empty() || check_service_name.empty())\n\t\t\tcontinue;\n\n\t\tval = json_object_find(\"CheckID\", obj);\n\t\tif (!val)\n\t\t\treturn false;\n\n\t\tstr = json_value_string(val);\n\t\tif (!str)\n\t\t\treturn false;\n\n\t\tinstance->check_id = str;\n\n\t\tval = json_object_find(\"Name\", obj);\n\t\tif (!val)\n\t\t\treturn false;\n\n\t\tstr = json_value_string(val);\n\t\tif (!str)\n\t\t\treturn false;\n\n\t\tinstance->check_name = str;\n\t\tval = json_object_find(\"Status\", obj);\n\t\tif (!val)\n\t\t\treturn false;\n\n\t\tstr = json_value_string(val);\n\t\tif (!str)\n\t\t\treturn false;\n\n\t\tinstance->check_status = str;\n\t\tval = json_object_find(\"Notes\", obj);\n\t\tif (val)\n\t\t{\n\t\t\tstr = json_value_string(val);\n\t\t\tif (!str)\n\t\t\t\treturn false;\n\n\t\t\tinstance->check_notes = str;\n\t\t}\n\n\t\tval = json_object_find(\"Output\", obj);\n\t\tif (val)\n\t\t{\n\t\t\tstr = json_value_string(val);\n\t\t\tif (!str)\n\t\t\t\treturn false;\n\n\t\t\tinstance->check_output = str;\n\t\t}\n\n\t\tval = json_object_find(\"Type\", obj);\n\t\tif (val)\n\t\t{\n\t\t\tstr = json_value_string(val);\n\t\t\tif (!str)\n\t\t\t\treturn false;\n\n\t\t\tinstance->check_type = str;\n\t\t}\n\n\t\tbreak; //only one effective service health check\n\t}\n\n\treturn true;\n}\n\nstatic bool parse_discover_result(const json_value_t *root,\n\t\t\t\t\tstd::vector& result)\n{\n\tconst json_array_t *arr = json_value_array(root);\n\tconst json_value_t *val;\n\tconst json_object_t *obj;\n\n\tif (!arr)\n\t\treturn false;\n\n\tjson_array_for_each(val, arr)\n\t{\n\t\tstruct ConsulServiceInstance instance;\n\n\t\tobj = json_value_object(val);\n\t\tif (!obj)\n\t\t\treturn false;\n\n\t\tif (!parse_discover_node(obj, &instance))\n\t\t\treturn false;\n\n\t\tif (!parse_service(obj, &instance.service))\n\t\t\treturn false;\n\n\t\tparse_health_check(obj, &instance);\n\t\tresult.emplace_back(std::move(instance));\n\t}\n\n\treturn true;\n}\n\nstatic void print_json_string(const char *str, std::string& json_str)\n{\n\tjson_str += \"\\\"\";\n\twhile (*str)\n\t{\n\t\tswitch (*str)\n\t\t{\n\t\tcase '\\r':\n\t\t\tjson_str += \"\\\\r\";\n\t\t\tbreak;\n\t\tcase '\\n':\n\t\t\tjson_str += \"\\\\n\";\n\t\t\tbreak;\n\t\tcase '\\f':\n\t\t\tjson_str += \"\\\\f\";\n\t\t\tbreak;\n\t\tcase '\\b':\n\t\t\tjson_str += \"\\\\b\";\n\t\t\tbreak;\n\t\tcase '\\\"':\n\t\t\tjson_str += \"\\\\\\\"\";\n\t\t\tbreak;\n\t\tcase '\\t':\n\t\t\tjson_str += \"\\\\t\";\n\t\t\tbreak;\n\t\tcase '\\\\':\n\t\t\tjson_str += \"\\\\\\\\\";\n\t\t\tbreak;\n\t\tdefault:\n\t\t\tif ((unsigned char)*str < 0x20)\n\t\t\t{\n\t\t\t\tchar buf[8];\n\t\t\t\tsprintf(buf, \"\\\\u00%02x\", *str);\n\t\t\t\tjson_str += buf;\n\t\t\t}\n\t\t\telse\n\t\t\t\tjson_str += *str;\n\t\t\tbreak;\n\t\t}\n\t\tstr++;\n\t}\n\tjson_str += \"\\\"\";\n}\n\nstatic void print_json_number(double number, std::string& json_str)\n{\n\tlong long integer = number;\n\n\tif (integer == number)\n\t\tjson_str += std::to_string(integer);\n\telse\n\t\tjson_str += std::to_string(number);\n}\n\nstatic void print_json_object(const json_object_t *obj, int depth,\n\t\t\t\t\t\t\t std::string& json_str)\n{\n\tconst char *name;\n\tconst json_value_t *val;\n\tint n = 0;\n\tint i;\n\n\tjson_str += \"{\\n\";\n\tjson_object_for_each(name, val, obj)\n\t{\n\t\tif (n != 0)\n\t\t\tjson_str += \",\\n\";\n\n\t\tn++;\n\t\tfor (i = 0; i < depth + 1; i++)\n\t\t\tjson_str += \" \";\n\n\t\tprint_json_string(name, json_str);\n\t\tjson_str += \": \";\n\t\tprint_json_value(val, depth + 1, json_str);\n\t}\n\n\tjson_str += \"\\n\";\n\tfor (i = 0; i < depth; i++)\n\t\tjson_str += \" \";\n\n\tjson_str += \"}\";\n}\n\nstatic void print_json_array(const json_array_t *arr, int depth,\n\t\t\t\t\t\t\t std::string& json_str)\n{\n\tconst json_value_t *val;\n\tint n = 0;\n\tint i;\n\n\tjson_str += \"[\\n\";\n\tjson_array_for_each(val, arr)\n\t{\n\t\tif (n != 0)\n\t\t\tjson_str += \",\\n\";\n\n\t\tn++;\n\t\tfor (i = 0; i < depth + 1; i++)\n\t\t\tjson_str += \" \";\n\n\t\tprint_json_value(val, depth + 1, json_str);\n\t}\n\n\tjson_str += \"\\n\";\n\tfor (i = 0; i < depth; i++)\n\t\tjson_str += \" \";\n\n\tjson_str += \"]\";\n}\n\nstatic void print_json_value(const json_value_t *val, int depth,\n\t\t\t\t\t\t\t std::string& json_str)\n{\n\tswitch (json_value_type(val))\n\t{\n\tcase JSON_VALUE_STRING:\n\t\tprint_json_string(json_value_string(val), json_str);\n\t\tbreak;\n\tcase JSON_VALUE_NUMBER:\n\t\tprint_json_number(json_value_number(val), json_str);\n\t\tbreak;\n\tcase JSON_VALUE_OBJECT:\n\t\tprint_json_object(json_value_object(val), depth, json_str);\n\t\tbreak;\n\tcase JSON_VALUE_ARRAY:\n\t\tprint_json_array(json_value_array(val), depth, json_str);\n\t\tbreak;\n\tcase JSON_VALUE_TRUE:\n\t\tjson_str += \"true\";\n\t\tbreak;\n\tcase JSON_VALUE_FALSE:\n\t\tjson_str += \"false\";\n\t\tbreak;\n\tcase JSON_VALUE_NULL:\n\t\tjson_str += \"null\";\n\t\tbreak;\n\t}\n}\n\n" }, { "path": "src/client/WFConsulClient.h", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhenpeng (wangzhenpeng@sogou-inc.com)\n*/\n\n#ifndef _WFCONSULCLIENT_H_\n#define _WFCONSULCLIENT_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \"HttpMessage.h\"\n#include \"WFTaskFactory.h\"\n#include \"ConsulDataTypes.h\"\n\nclass WFConsulTask;\nusing consul_callback_t = std::function;\n\nenum\n{\n\tCONSUL_API_TYPE_UNKNOWN = 0,\n\tCONSUL_API_TYPE_DISCOVER,\n\tCONSUL_API_TYPE_LIST_SERVICE,\n\tCONSUL_API_TYPE_REGISTER,\n\tCONSUL_API_TYPE_DEREGISTER,\n};\n\nclass WFConsulTask : public WFGenericTask\n{\npublic:\n\tbool get_discover_result(\n\t\t\tstd::vector& result) const;\n\n\tbool get_list_service_result(\n\t\t\tstd::vector& result) const;\n\npublic:\n\tvoid set_service(const struct protocol::ConsulService *service);\n\n\tvoid set_api_type(int api_type)\n\t{\n\t\tthis->api_type = api_type;\n\t}\n\n\tint get_api_type() const\n\t{\n\t\treturn this->api_type;\n\t}\n\n\tvoid set_callback(consul_callback_t cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\n\tvoid set_consul_index(long long consul_index)\n\t{\n\t\tthis->consul_index = consul_index;\n\t}\n\n\tlong long get_consul_index() const { return this->consul_index; }\n\n\tconst protocol::HttpResponse *get_http_resp() const\n\t{\n\t\treturn &this->http_resp;\n\t}\n\nprotected:\n\tvoid set_config(protocol::ConsulConfig conf)\n\t{\n\t\tthis->config = std::move(conf);\n\t}\n\n\tvoid set_ssl_ctx(SSL_CTX *ssl_ctx)\n\t{\n\t\tthis->ssl_ctx = ssl_ctx;\n\t}\n\nprotected:\n\tvirtual void dispatch();\n\tvirtual SubTask *done();\n\n\tWFHttpTask *create_discover_task();\n\tWFHttpTask *create_list_service_task();\n\tWFHttpTask *create_register_task();\n\tWFHttpTask *create_deregister_task();\n\n\tstd::string generate_discover_request();\n\tlong long get_consul_index(protocol::HttpResponse *resp);\n\n\tstatic bool check_task_result(WFHttpTask *task, WFConsulTask *consul_task);\n\tstatic void discover_callback(WFHttpTask *task);\n\tstatic void list_service_callback(WFHttpTask *task);\n\tstatic void register_callback(WFHttpTask *task);\n\nprotected:\n\tprotocol::ConsulConfig config;\n\tSSL_CTX *ssl_ctx;\n\tstruct protocol::ConsulService service;\n\tstd::string proxy_url;\n\tint retry_max;\n\tint api_type;\n\tbool finish;\n\tlong long consul_index;\n\tprotocol::HttpResponse http_resp;\n\tconsul_callback_t callback;\n\nprotected:\n\tWFConsulTask(const std::string& proxy_url,\n\t\t\t\t const std::string& service_namespace,\n\t\t\t\t const std::string& service_name,\n\t\t\t\t const std::string& service_id,\n\t\t\t\t int retry_max, consul_callback_t&& cb);\n\tvirtual ~WFConsulTask() { }\n\tfriend class WFConsulClient;\n};\n\nclass WFConsulClient\n{\npublic:\n\t// example: http://127.0.0.1:8500\n\tint init(const std::string& proxy_url)\n\t{\n\t\treturn this->init(proxy_url, NULL);\n\t}\n\n\tint init(const std::string& proxy_url, protocol::ConsulConfig config)\n\t{\n\t\treturn this->init(proxy_url, std::move(config), NULL);\n\t}\n\n\t// with specific SSL_CTX\n\tint init(const std::string& proxy_url, SSL_CTX *ctx_ctx)\n\t{\n\t\treturn this->init(proxy_url, protocol::ConsulConfig(), ssl_ctx);\n\t}\n\n\tint init(const std::string& proxy_url, protocol::ConsulConfig config,\n\t\t\t SSL_CTX *ctx);\n\n\tvoid deinit() { }\n\n\tWFConsulTask *create_discover_task(const std::string& service_namespace,\n\t\t\t\t\t\t\t\t\t const std::string& service_name,\n\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t consul_callback_t cb);\n\n\tWFConsulTask *create_list_service_task(const std::string& service_namespace,\n\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t consul_callback_t cb);\n\n\tWFConsulTask *create_register_task(const std::string& service_namespace,\n\t\t\t\t\t\t\t\t\t const std::string& service_name,\n\t\t\t\t\t\t\t\t\t const std::string& service_id,\n\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t consul_callback_t cb);\n\n\tWFConsulTask *create_deregister_task(const std::string& service_namespace,\n\t\t\t\t\t\t\t\t\t\t const std::string& service_id,\n\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t consul_callback_t cb);\n\nprotected:\n\tstd::string proxy_url;\n\tprotocol::ConsulConfig config;\n\tSSL_CTX *ssl_ctx;\n\npublic:\n\tvirtual ~WFConsulClient() { }\n};\n\n#endif\n\n" }, { "path": "src/client/WFDnsClient.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"URIParser.h\"\n#include \"StringUtil.h\"\n#include \"dns_types.h\"\n#include \"DnsMessage.h\"\n#include \"WFDnsClient.h\"\n\nusing namespace protocol;\n\nusing DnsCtx = std::function;\nusing ComplexTask = WFComplexClientTask;\n\nclass DnsParams\n{\npublic:\n\tstruct dns_params\n\t{\n\t\tstd::vector uris;\n\t\tstd::vector search_list;\n\t\tint ndots;\n\t\tint attempts;\n\t\tbool rotate;\n\t};\n\npublic:\n\tDnsParams()\n\t{\n\t\tthis->ref = new std::atomic(1);\n\t\tthis->params = new dns_params();\n\t}\n\n\tDnsParams(const DnsParams& p)\n\t{\n\t\tthis->ref = p.ref;\n\t\tthis->params = p.params;\n\t\tthis->incref();\n\t}\n\n\tDnsParams& operator=(const DnsParams& p)\n\t{\n\t\tif (this != &p)\n\t\t{\n\t\t\tthis->decref();\n\t\t\tthis->ref = p.ref;\n\t\t\tthis->params = p.params;\n\t\t\tthis->incref();\n\t\t}\n\t\treturn *this;\n\t}\n\n\t~DnsParams() { this->decref(); }\n\n\tconst dns_params *get_params() const { return this->params; }\n\tdns_params *get_params() { return this->params; }\n\nprivate:\n\tvoid incref() { (*this->ref)++; }\n\tvoid decref()\n\t{\n\t\tif (--*this->ref == 0)\n\t\t{\n\t\t\tdelete this->params;\n\t\t\tdelete this->ref;\n\t\t}\n\t}\n\nprivate:\n\tdns_params *params;\n\tstd::atomic *ref;\n};\n\nenum\n{\n\tDNS_STATUS_TRY_ORIGIN_DONE = 0,\n\tDNS_STATUS_TRY_ORIGIN_FIRST = 1,\n\tDNS_STATUS_TRY_ORIGIN_LAST = 2\n};\n\nstruct DnsStatus\n{\n\tstd::string origin_name;\n\tstd::string current_name;\n\tsize_t next_server;\t\t\t// next server to try\n\tsize_t last_server;\t\t\t// last server to try\n\tsize_t next_domain;\t\t\t// next search domain to try\n\tint attempts_left;\n\tint try_origin_state;\n};\n\nstatic int __get_ndots(const std::string& s)\n{\n\tint ndots = 0;\n\tfor (size_t i = 0; i < s.size(); i++)\n\t\tndots += s[i] == '.';\n\treturn ndots;\n}\n\nstatic bool __has_next_name(const DnsParams::dns_params *p,\n\t\t\t\t\t\t\tstruct DnsStatus *s)\n{\n\tif (s->try_origin_state == DNS_STATUS_TRY_ORIGIN_FIRST)\n\t{\n\t\ts->current_name = s->origin_name;\n\t\ts->try_origin_state = DNS_STATUS_TRY_ORIGIN_DONE;\n\t\treturn true;\n\t}\n\n\tif (s->next_domain < p->search_list.size())\n\t{\n\t\ts->current_name = s->origin_name;\n\t\ts->current_name.push_back('.');\n\t\ts->current_name.append(p->search_list[s->next_domain]);\n\n\t\ts->next_domain++;\n\t\treturn true;\n\t}\n\n\tif (s->try_origin_state == DNS_STATUS_TRY_ORIGIN_LAST)\n\t{\n\t\ts->current_name = s->origin_name;\n\t\ts->try_origin_state = DNS_STATUS_TRY_ORIGIN_DONE;\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nstatic void __callback_internal(WFDnsTask *task, const DnsParams& params,\n\t\t\t\t\t\t\t\tstruct DnsStatus& s)\n{\n\tComplexTask *ctask = static_cast(task);\n\tint state = task->get_state();\n\tDnsRequest *req = task->get_req();\n\tDnsResponse *resp = task->get_resp();\n\tconst auto *p = params.get_params();\n\tint rcode = resp->get_rcode();\n\n\tbool try_next_server = state != WFT_STATE_SUCCESS ||\n\t\t\t\t\t\t rcode == DNS_RCODE_SERVER_FAILURE ||\n\t\t\t\t\t\t rcode == DNS_RCODE_NOT_IMPLEMENTED ||\n\t\t\t\t\t\t rcode == DNS_RCODE_REFUSED;\n\tbool try_next_name = rcode == DNS_RCODE_FORMAT_ERROR ||\n\t\t\t\t\t\t rcode == DNS_RCODE_NAME_ERROR ||\n\t\t\t\t\t\t resp->get_ancount() == 0;\n\n\tif (try_next_server)\n\t{\n\t\tif (s.last_server == s.next_server)\n\t\t\ts.attempts_left--;\n\t\tif (s.attempts_left <= 0)\n\t\t\treturn;\n\n\t\ts.next_server = (s.next_server + 1) % p->uris.size();\n\t\tctask->set_redirect(p->uris[s.next_server]);\n\t\treturn;\n\t}\n\n\tif (try_next_name && __has_next_name(p, &s))\n\t{\n\t\treq->set_question_name(s.current_name.c_str());\n\t\tctask->set_redirect(p->uris[s.next_server]);\n\t\treturn;\n\t}\n}\n\nint WFDnsClient::init(const std::string& url)\n{\n\treturn this->init(url, \"\", 1, 2, false);\n}\n\nint WFDnsClient::init(const std::string& url, const std::string& search_list,\n\t\t\t\t\t int ndots, int attempts, bool rotate)\n{\n\tstd::vector hosts;\n\tstd::vector uris;\n\tstd::string host;\n\tParsedURI uri;\n\n\tthis->id = 0;\n\thosts = StringUtil::split_filter_empty(url, ',');\n\n\tfor (size_t i = 0; i < hosts.size(); i++)\n\t{\n\t\thost = hosts[i];\n\t\tif (strncasecmp(host.c_str(), \"dns://\", 6) != 0 &&\n\t\t\tstrncasecmp(host.c_str(), \"dnss://\", 7) != 0)\n\t\t{\n\t\t\thost = \"dns://\" + host;\n\t\t}\n\n\t\tif (URIParser::parse(host, uri) != 0)\n\t\t\treturn -1;\n\n\t\turis.emplace_back(std::move(uri));\n\t}\n\n\tif (uris.empty() || ndots < 0 || attempts < 1)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\tthis->params = new DnsParams;\n\tDnsParams::dns_params *q = ((DnsParams *)this->params)->get_params();\n\tq->uris = std::move(uris);\n\tq->search_list = StringUtil::split_filter_empty(search_list, ',');\n\tq->ndots = ndots > 15 ? 15 : ndots;\n\tq->attempts = attempts > 5 ? 5 : attempts;\n\tq->rotate = rotate;\n\n\treturn 0;\n}\n\nvoid WFDnsClient::deinit()\n{\n\tdelete (DnsParams *)this->params;\n\tthis->params = NULL;\n}\n\nWFDnsTask *WFDnsClient::create_dns_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\tdns_callback_t callback)\n{\n\tDnsParams::dns_params *p = ((DnsParams *)this->params)->get_params();\n\tstruct DnsStatus status;\n\tsize_t next_server;\n\tWFDnsTask *task;\n\tDnsRequest *req;\n\n\tnext_server = p->rotate ? this->id++ % p->uris.size() : 0;\n\n\tstatus.origin_name = name;\n\tstatus.next_domain = 0;\n\tstatus.attempts_left = p->attempts;\n\tstatus.try_origin_state = DNS_STATUS_TRY_ORIGIN_FIRST;\n\n\tif (!name.empty() && name.back() == '.')\n\t\tstatus.next_domain = p->search_list.size();\n\telse if (__get_ndots(name) < p->ndots)\n\t\tstatus.try_origin_state = DNS_STATUS_TRY_ORIGIN_LAST;\n\n\t__has_next_name(p, &status);\n\n\ttask = WFTaskFactory::create_dns_task(p->uris[next_server], 0,\n\t\t\t\t\t\t\t\t\t\t std::move(callback));\n\tstatus.next_server = next_server;\n\tstatus.last_server = (next_server + p->uris.size() - 1) % p->uris.size();\n\n\treq = task->get_req();\n\treq->set_question(status.current_name.c_str(), DNS_TYPE_A, DNS_CLASS_IN);\n\treq->set_rd(1);\n\n\tComplexTask *ctask = static_cast(task);\n\t*ctask->get_mutable_ctx() = std::bind(__callback_internal,\n\t\t\t\t\t\t\t\t\t\t std::placeholders::_1,\n\t\t\t\t\t\t\t\t\t\t *(DnsParams *)params, status);\n\n\treturn task;\n}\n\n" }, { "path": "src/client/WFDnsClient.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#ifndef _WFDNSCLIENT_H_\n#define _WFDNSCLIENT_H_\n\n#include \n#include \n#include \"WFTaskFactory.h\"\n#include \"dns_types.h\"\n#include \"DnsMessage.h\"\n\nclass WFDnsClient\n{\npublic:\n\tint init(const std::string& url);\n\tint init(const std::string& url, const std::string& search_list,\n\t\t\t int ndots, int attempts, bool rotate);\n\tvoid deinit();\n\n\tWFDnsTask *create_dns_task(const std::string& name,\n\t\t\t\t\t\t\t dns_callback_t callback);\n\nprivate:\n\tvoid *params;\n\tstd::atomic id;\n\npublic:\n\tvirtual ~WFDnsClient() { }\n};\n\n#endif\n\n" }, { "path": "src/client/WFHttpChunkedClient.cc", "content": "/*\n Copyright (c) 2025 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \"HttpTaskImpl.inl\"\n#include \"WFHttpChunkedClient.h\"\n\nvoid WFHttpChunkedTask::task_extract(protocol::HttpMessageChunk *chunk,\n\t\t\t\t\t\t\t\t\t WFHttpTask *task)\n{\n\tauto *t = (WFHttpChunkedTask *)task->user_data;\n\n\tt->chunk = chunk;\n\tif (t->extract)\n\t{\n\t\tif (chunk || t->extract_flag)\n\t\t\tt->extract(t);\n\t}\n}\n\nvoid WFHttpChunkedTask::task_callback(WFHttpTask *task)\n{\n\tauto *t = (WFHttpChunkedTask *)task->user_data;\n\n\tt->state = task->get_state();\n\tt->error = task->get_error();\n\tt->chunk = NULL;\n\tif (t->callback)\n\t\tt->callback(t);\n\n\tt->task = NULL;\n\tdelete t;\n}\n\nWFHttpChunkedTask *\nWFHttpChunkedClient::create_chunked_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t int redirect_max,\n\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t callback_t callback)\n{\n\tWFHttpTask *task = __WFHttpTaskFactory::create_chunked_task(url,\n\t\t\t\t\t\t\t\t\t\tredirect_max,\n\t\t\t\t\t\t\t\t\t\tWFHttpChunkedTask::task_extract,\n\t\t\t\t\t\t\t\t\t\tWFHttpChunkedTask::task_callback);\n\treturn new WFHttpChunkedTask(task, std::move(extract), std::move(callback));\n}\n\nWFHttpChunkedTask *\nWFHttpChunkedClient::create_chunked_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t int redirect_max,\n\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t callback_t callback)\n{\n\tWFHttpTask *task = __WFHttpTaskFactory::create_chunked_task(uri,\n\t\t\t\t\t\t\t\t\t\tredirect_max,\n\t\t\t\t\t\t\t\t\t\tWFHttpChunkedTask::task_extract,\n\t\t\t\t\t\t\t\t\t\tWFHttpChunkedTask::task_callback);\n\treturn new WFHttpChunkedTask(task, std::move(extract), std::move(callback));\n}\n\n" }, { "path": "src/client/WFHttpChunkedClient.h", "content": "/*\n Copyright (c) 2025 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFHTTPCHUNKEDCLIENT_H_\n#define _WFHTTPCHUNKEDCLIENT_H_\n\n#include \n#include \n#include \n#include \"HttpMessage.h\"\n#include \"WFTask.h\"\n#include \"WFTaskFactory.h\"\n\nclass WFHttpChunkedTask : public WFGenericTask\n{\npublic:\n\tprotocol::HttpMessageChunk *get_chunk()\n\t{\n\t\treturn this->chunk;\n\t}\n\n\tconst protocol::HttpMessageChunk *get_chunk() const\n\t{\n\t\treturn this->chunk;\n\t}\n\npublic:\n\tprotocol::HttpRequest *get_req()\n\t{\n\t\treturn this->task->get_req();\n\t}\n\n\tprotocol::HttpResponse *get_resp()\n\t{\n\t\treturn this->task->get_resp();\n\t}\n\n\tconst protocol::HttpRequest *get_req() const\n\t{\n\t\treturn this->task->get_req();\n\t}\n\n\tconst protocol::HttpResponse *get_resp() const\n\t{\n\t\treturn this->task->get_resp();\n\t}\n\npublic:\n\t/* Timeout of waiting for the first package of each chunk. If not set,\n\t the max waiting time will be the global 'response_timeout'. */\n\tvoid set_watch_timeout(int timeout)\n\t{\n\t\tthis->task->set_watch_timeout(timeout);\n\t}\n\n\t/* Timeout of receiving a complete chunk. */\n\tvoid set_receive_timeout(int timeout)\n\t{\n\t\tthis->task->set_receive_timeout(timeout);\n\t}\n\n\t/* Timeout of sending the HTTP request. */\n\tvoid set_send_timeout(int timeout)\n\t{\n\t\tthis->task->set_send_timeout(timeout);\n\t}\n\n\t/* Speicify HTTP keep alive timeout. */\n\tvoid set_keep_alive(int timeout)\n\t{\n\t\tthis->task->set_keep_alive(timeout);\n\t}\n\n\t/* Equal to 'set_receive_timeout()'. For compatibility purpose only. */\n\tvoid set_recv_timeout(int timeout)\n\t{\n\t\tthis->set_receive_timeout(timeout);\n\t}\n\npublic:\n\tvoid set_ssl_ctx(SSL_CTX *ctx)\n\t{\n\t\tusing HttpRequest = protocol::HttpRequest;\n\t\tusing HttpResponse = protocol::HttpResponse;\n\t\tauto *t = (WFComplexClientTask *)this->task;\n\t\tt->set_ssl_ctx(ctx);\n\t}\n\n\tvoid extract_on_header(bool on)\n\t{\n\t\tthis->extract_flag = on;\n\t}\n\npublic:\n\tvoid set_extract(std::function ex)\n\t{\n\t\tthis->extract = std::move(ex);\n\t}\n\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\npublic:\n\tconst WFHttpTask *get_http_task() const\n\t{\n\t\treturn this->task;\n\t}\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tseries_of(this)->push_front(this->task);\n\t\tthis->subtask_done();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\treturn series_of(this)->pop();\n\t}\n\nprotected:\n\tstatic void task_extract(protocol::HttpMessageChunk *chunk,\n\t\t\t\t\t\t\t WFHttpTask *task);\n\tstatic void task_callback(WFHttpTask *task);\n\nprotected:\n\tWFHttpTask *task;\n\tprotocol::HttpMessageChunk *chunk;\n\tbool extract_flag;\n\tstd::function extract;\n\tstd::function callback;\n\nprotected:\n\tWFHttpChunkedTask(WFHttpTask *task,\n\t\t\t\t\t std::function&& ex,\n\t\t\t\t\t std::function&& cb) :\n\t\textract(std::move(ex)),\n\t\tcallback(std::move(cb))\n\t{\n\t\ttask->user_data = this;\n\t\tthis->task = task;\n\t\tthis->extract_flag = false;\n\t}\n\n\tvirtual ~WFHttpChunkedTask()\n\t{\n\t\tif (this->task)\n\t\t\tthis->task->dismiss();\n\t}\n\n\tfriend class WFHttpChunkedClient;\n};\n\nclass WFHttpChunkedClient\n{\npublic:\n\tusing extract_t = std::function;\n\tusing callback_t = std::function;\n\npublic:\n\tstatic WFHttpChunkedTask *create_chunked_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\t\t int redirect_max,\n\t\t\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t\t\t callback_t callback);\n\n\tstatic WFHttpChunkedTask *create_chunked_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t int redirect_max,\n\t\t\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t\t\t callback_t callback);\n};\n\n#endif\n\n" }, { "path": "src/client/WFKafkaClient.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"WFTaskError.h\"\n#include \"StringUtil.h\"\n#include \"KafkaTaskImpl.inl\"\n#include \"WFKafkaClient.h\"\n\n#define KAFKA_HEARTBEAT_INTERVAL\t(3 * 1000 * 1000)\n\n#define KAFKA_CGROUP_UNINIT\t\t0\n#define KAFKA_CGROUP_DOING\t\t1\n#define KAFKA_CGROUP_DONE\t\t2\n#define KAFKA_CGROUP_NONE\t\t3\n\n#define KAFKA_HEARTBEAT_UNINIT\t0\n#define KAFKA_HEARTBEAT_DOING\t1\n#define KAFKA_HEARTBEAT_DONE\t2\n\n#define KAFKA_DEINIT\t\t\t(1<<30)\n\nusing namespace protocol;\n\nusing ComplexKafkaTask = WFComplexClientTask;\n\nclass KafkaMember\n{\npublic:\n\tKafkaMember() : scheme(\"kafka://\"), ref(1)\n\t{\n\t\tthis->transport_type = TT_TCP;\n\t\tthis->cgroup_status = KAFKA_CGROUP_NONE;\n\t\tthis->heartbeat_status = KAFKA_HEARTBEAT_UNINIT;\n\t\tthis->meta_doing = false;\n\t\tthis->cgroup_outdated = false;\n\t\tthis->client_deinit = false;\n\t\tthis->heartbeat_series = NULL;\n\t}\n\n\tvoid incref()\n\t{\n\t\t++this->ref;\n\t}\n\n\tvoid decref()\n\t{\n\t\tif (--this->ref == 0)\n\t\t\tdelete this;\n\t}\n\n\tenum TransportType transport_type;\n\tstd::string scheme;\n\tstd::vector broker_hosts;\n\tSSL_CTX *ssl_ctx;\n\tKafkaCgroup cgroup;\n\tKafkaMetaList meta_list;\n\tKafkaBrokerMap broker_map;\n\tKafkaConfig config;\n\tstd::map meta_status;\n\tstd::mutex mutex;\n\tchar cgroup_status;\n\tchar heartbeat_status;\n\tbool meta_doing;\n\tbool cgroup_outdated;\n\tbool client_deinit;\n\tvoid *heartbeat_series;\n\tsize_t cgroup_wait_cnt;\n\tsize_t meta_wait_cnt;\n\tstd::atomic ref;\n};\n\nclass KafkaClientTask : public WFKafkaTask\n{\npublic:\n\tKafkaClientTask(const std::string& query, int retry_max,\n\t\t\t\t\tkafka_callback_t&& callback,\n\t\t\t\t\tWFKafkaClient *client) :\n\t\tWFKafkaTask(retry_max, std::move(callback))\n\t{\n\t\tthis->api_type = Kafka_Unknown;\n\t\tthis->kafka_error = 0;\n\t\tthis->member = client->member;\n\t\tthis->query = query;\n\n\t\tthis->member->incref();\n\t\tthis->member->mutex.lock();\n\t\tthis->config = client->member->config;\n\t\tif (!this->member->broker_hosts.empty())\n\t\t{\n\t\t\tint rpos = rand() % this->member->broker_hosts.size();\n\t\t\tthis->url = this->member->broker_hosts.at(rpos);\n\t\t}\n\t\tthis->member->mutex.unlock();\n\n\t\tthis->info_generated = false;\n\t\tthis->msg = NULL;\n\t}\n\n\tvirtual ~KafkaClientTask()\n\t{\n\t\tthis->member->decref();\n\t}\n\n\tstd::string *get_url() { return &this->url; }\n\nprotected:\n\tvirtual bool add_topic(const std::string& topic);\n\n\tvirtual bool add_toppar(const KafkaToppar& toppar);\n\n\tvirtual bool add_produce_record(const std::string& topic, int partition,\n\t\t\t\t\t\t\t\t\tKafkaRecord record);\n\n\tvirtual bool add_offset_toppar(const KafkaToppar& toppar);\n\n\tvirtual void dispatch();\n\n\tvirtual void parse_query();\n\tvirtual void generate_info();\n\nprivate:\n\tstatic void kafka_meta_callback(__WFKafkaTask *task);\n\n\tstatic void kafka_merge_meta_list(KafkaMetaList *dst,\n\t\t\t\t\t\t\t\t\t KafkaMetaList *src);\n\n\tstatic void kafka_merge_broker_list(const std::string& scheme,\n\t\t\t\t\t\t\t\t\t\tstd::vector *hosts,\n\t\t\t\t\t\t\t\t\t\tKafkaBrokerMap *dst,\n\t\t\t\t\t\t\t\t\t\tKafkaBrokerList *src);\n\n\tstatic void kafka_cgroup_callback(__WFKafkaTask *task);\n\n\tstatic void kafka_offsetcommit_callback(__WFKafkaTask *task);\n\n\tstatic void kafka_parallel_callback(const ParallelWork *pwork);\n\n\tstatic void kafka_timer_callback(WFTimerTask *task);\n\n\tstatic void kafka_heartbeat_callback(__WFKafkaTask *task);\n\n\tstatic void kafka_leavegroup_callback(__WFKafkaTask *task);\n\n\tstatic void kafka_rebalance_proc(KafkaMember *member, SeriesWork *series);\n\n\tstatic void kafka_rebalance_callback(__WFKafkaTask *task);\n\n\tvoid kafka_move_task_callback(__WFKafkaTask *task);\n\n\tvoid kafka_process_toppar_offset(KafkaToppar *task_toppar);\n\n\tbool compare_topics(KafkaClientTask *task);\n\n\tbool check_cgroup();\n\n\tbool check_meta();\n\n\tint arrange_toppar(int api_type);\n\n\tint arrange_produce();\n\n\tint arrange_fetch();\n\n\tint arrange_commit();\n\n\tint arrange_offset();\n\n\tint dispatch_locked();\n\n\tKafkaBroker *get_broker(int node_id)\n\t{\n\t\treturn this->member->broker_map.find_item(node_id);\n\t}\n\n\tint get_node_id(const KafkaToppar *toppar);\n\n\tbool get_meta_status(KafkaMetaList **uninit_meta_list);\n\tvoid set_meta_status(bool status);\n\n\tstd::string get_userinfo() { return this->userinfo; }\n\nprivate:\n\tKafkaMember *member;\n\tKafkaBroker broker;\n\tstd::map toppar_list_map;\n\tstd::string url;\n\tstd::string query;\n\tstd::set topic_set;\n\tstd::string userinfo;\n\tbool info_generated;\n\tbool wait_cgroup;\n\tvoid *msg;\n\n\tfriend class WFKafkaClient;\n};\n\nint KafkaClientTask::get_node_id(const KafkaToppar *toppar)\n{\n\tint preferred_read_replica = toppar->get_preferred_read_replica();\n\tif (preferred_read_replica >= 0)\n\t\treturn preferred_read_replica;\n\n\tbool flag = false;\n\tthis->member->meta_list.rewind();\n\tKafkaMeta *meta;\n\twhile ((meta = this->member->meta_list.get_next()) != NULL)\n\t{\n\t\tif (strcmp(meta->get_topic(), toppar->get_topic()) == 0)\n\t\t{\n\t\t\tflag = true;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tconst kafka_broker_t *broker = NULL;\n\tif (flag)\n\t\tbroker = meta->get_broker(toppar->get_partition());\n\n\tif (!broker)\n\t\treturn -1;\n\n\treturn broker->node_id;\n}\n\nvoid KafkaClientTask::kafka_offsetcommit_callback(__WFKafkaTask *task)\n{\n\tKafkaClientTask *t = (KafkaClientTask *)task->user_data;\n\tif (task->get_state() == WFT_STATE_SUCCESS)\n\t\tt->result.set_resp(std::move(*task->get_resp()), 0);\n\n\tt->finish = true;\n\tt->state = task->get_state();\n\tt->error = task->get_error();\n\tt->kafka_error = *static_cast(task)->get_mutable_ctx();\n}\n\nvoid KafkaClientTask::kafka_leavegroup_callback(__WFKafkaTask *task)\n{\n\tKafkaClientTask *t = (KafkaClientTask *)task->user_data;\n\tt->finish = true;\n\tt->state = task->get_state();\n\tt->error = task->get_error();\n\tt->kafka_error = *static_cast(task)->get_mutable_ctx();\n}\n\nvoid KafkaClientTask::kafka_rebalance_callback(__WFKafkaTask *task)\n{\n\tKafkaMember *member = (KafkaMember *)task->user_data;\n\tSeriesWork *series = series_of(task);\n\tsize_t max;\n\n\tmember->mutex.lock();\n\tif (member->client_deinit)\n\t{\n\t\tmember->mutex.unlock();\n\t\tmember->decref();\n\t\treturn;\n\t}\n\n\tif (task->get_state() == WFT_STATE_SUCCESS)\n\t{\n\t\tmember->cgroup_status = KAFKA_CGROUP_DONE;\n\t\tmember->cgroup = std::move(*(task->get_resp()->get_cgroup()));\n\n\t\tif (member->heartbeat_status == KAFKA_HEARTBEAT_UNINIT)\n\t\t{\n\t\t\t__WFKafkaTask *kafka_task;\n\t\t\tKafkaBroker *coordinator = member->cgroup.get_coordinator();\n\t\t\tkafka_task = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx, \"\", 0,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t kafka_heartbeat_callback);\n\t\t\tkafka_task->user_data = member;\n\t\t\tkafka_task->get_req()->set_api_type(Kafka_Heartbeat);\n\t\t\tkafka_task->get_req()->set_cgroup(member->cgroup);\n\t\t\tkafka_task->get_req()->set_broker(*coordinator);\n\t\t\tseries->push_back(kafka_task);\n\n\t\t\tmember->heartbeat_status = KAFKA_HEARTBEAT_DOING;\n\t\t\tmember->heartbeat_series = series;\n\t\t}\n\n\t\tmax = member->cgroup_wait_cnt;\n\t\tchar name[64];\n\t\tsnprintf(name, 64, \"%p.cgroup\", member);\n\t\tmember->mutex.unlock();\n\n\t\tWFTaskFactory::signal_by_name(name, NULL, max);\n\t}\n\telse\n\t{\n\t\tkafka_rebalance_proc(member, series);\n\t\tmember->mutex.unlock();\n\t}\n}\n\nvoid KafkaClientTask::kafka_rebalance_proc(KafkaMember *member, SeriesWork *series)\n{\n\tKafkaBroker *coordinator = member->cgroup.get_coordinator();\n\t__WFKafkaTask *task;\n\ttask = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx, \"\", 0,\n\t\t\t\t\t\t\t\t\t\t\t\t kafka_rebalance_callback);\n\ttask->user_data = member;\n\ttask->get_req()->set_config(member->config);\n\ttask->get_req()->set_api_type(Kafka_FindCoordinator);\n\ttask->get_req()->set_cgroup(member->cgroup);\n\ttask->get_req()->set_meta_list(member->meta_list);\n\n\tmember->cgroup_status = KAFKA_CGROUP_DOING;\n\tmember->heartbeat_status = KAFKA_HEARTBEAT_UNINIT;\n\tmember->cgroup_outdated = false;\n\n\tseries->push_back(task);\n}\n\nvoid KafkaClientTask::kafka_heartbeat_callback(__WFKafkaTask *task)\n{\n\tKafkaMember *member = (KafkaMember *)task->user_data;\n\tSeriesWork *series = series_of(task);\n\tKafkaResponse *resp = task->get_resp();\n\n\tmember->mutex.lock();\n\n\tif (member->client_deinit || member->heartbeat_series != series)\n\t{\n\t\tmember->mutex.unlock();\n\t\tmember->decref();\n\t\treturn;\n\t}\n\n\tif (resp->get_cgroup()->get_error() == 0)\n\t{\n\t\tmember->heartbeat_status = KAFKA_HEARTBEAT_DONE;\n\t\tWFTimerTask *timer_task;\n\t\ttimer_task = WFTaskFactory::create_timer_task(KAFKA_HEARTBEAT_INTERVAL,\n\t\t\t\t\t\t\t\t\t\t\t\t\t kafka_timer_callback);\n\t\ttimer_task->user_data = member;\n\t\tseries->push_back(timer_task);\n\t}\n\telse\n\t\tkafka_rebalance_proc(member, series);\n\n\tmember->mutex.unlock();\n}\n\nvoid KafkaClientTask::kafka_timer_callback(WFTimerTask *task)\n{\n\tKafkaMember *member = (KafkaMember *)task->user_data;\n\tSeriesWork *series = series_of(task);\n\n\tmember->mutex.lock();\n\tif (member->client_deinit || member->heartbeat_series != series)\n\t{\n\t\tmember->mutex.unlock();\n\t\tmember->decref();\n\t\treturn;\n\t}\n\n\tmember->heartbeat_status = KAFKA_HEARTBEAT_DOING;\n\n\t__WFKafkaTask *kafka_task;\n\tKafkaBroker *coordinator = member->cgroup.get_coordinator();\n\tkafka_task = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx, \"\", 0,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t kafka_heartbeat_callback);\n\n\tkafka_task->user_data = member;\n\tkafka_task->get_req()->set_config(member->config);\n\tkafka_task->get_req()->set_api_type(Kafka_Heartbeat);\n\tkafka_task->get_req()->set_cgroup(member->cgroup);\n\tkafka_task->get_req()->set_broker(*coordinator);\n\tseries->push_back(kafka_task);\n\n\tmember->mutex.unlock();\n}\n\nvoid KafkaClientTask::kafka_merge_meta_list(KafkaMetaList *dst,\n\t\t\t\t\t\t\t\t\t\t\tKafkaMetaList *src)\n{\n\tsrc->rewind();\n\tKafkaMeta *src_meta;\n\twhile ((src_meta = src->get_next()) != NULL)\n\t{\n\t\tdst->rewind();\n\n\t\tKafkaMeta *dst_meta;\n\t\twhile ((dst_meta = dst->get_next()) != NULL)\n\t\t{\n\t\t\tif (strcmp(dst_meta->get_topic(), src_meta->get_topic()) == 0)\n\t\t\t{\n\t\t\t\tdst->del_cur();\n\t\t\t\tdelete dst_meta;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tdst->add_item(*src_meta);\n\t}\n}\n\nvoid KafkaClientTask::kafka_merge_broker_list(const std::string& scheme,\n\t\t\t\t\t\t\t\t\t\t\t std::vector *hosts,\n\t\t\t\t\t\t\t\t\t\t\t KafkaBrokerMap *dst,\n\t\t\t\t\t\t\t\t\t\t\t KafkaBrokerList *src)\n{\n\thosts->clear();\n\tsrc->rewind();\n\tKafkaBroker *src_broker;\n\twhile ((src_broker = src->get_next()) != NULL)\n\t{\n\t\tstd::string host = scheme + src_broker->get_host() + \":\" +\n\t\t\t\t\t\t std::to_string(src_broker->get_port());\n\t\thosts->emplace_back(std::move(host));\n\n\t\tif (!dst->find_item(src_broker->get_node_id()))\n\t\t\tdst->add_item(*src_broker, src_broker->get_node_id());\n\t}\n}\n\nvoid KafkaClientTask::kafka_meta_callback(__WFKafkaTask *task)\n{\n\tKafkaClientTask *t = (KafkaClientTask *)task->user_data;\n\tvoid *msg = NULL;\n\tsize_t max;\n\n\tt->member->mutex.lock();\n\tt->state = task->get_state();\n\tt->error = task->get_error();\n\tt->kafka_error = *static_cast(task)->get_mutable_ctx();\n\tif (t->state == WFT_STATE_SUCCESS)\n\t{\n\t\tkafka_merge_meta_list(&t->member->meta_list,\n\t\t\t\t\t\t\t task->get_resp()->get_meta_list());\n\n\t\tt->meta_list.rewind();\n\t\tKafkaMeta *meta;\n\t\twhile ((meta = t->meta_list.get_next()) != NULL)\n\t\t\t(t->member->meta_status)[meta->get_topic()] = true;\n\n\t\tkafka_merge_broker_list(t->member->scheme,\n\t\t\t\t\t\t\t\t&t->member->broker_hosts,\n\t\t\t\t\t\t\t\t&t->member->broker_map,\n\t\t\t\t\t\t\t\ttask->get_resp()->get_broker_list());\n\t}\n\telse\n\t{\n\t\tt->meta_list.rewind();\n\t\tKafkaMeta *meta;\n\t\twhile ((meta = t->meta_list.get_next()) != NULL)\n\t\t\t(t->member->meta_status)[meta->get_topic()] = false;\n\n\t\tt->finish = true;\n\t\tmsg = t;\n\t}\n\n\tt->member->meta_doing = false;\n\tmax = t->member->meta_wait_cnt;\n\tchar name[64];\n\tsnprintf(name, 64, \"%p.meta\", t->member);\n\tt->member->mutex.unlock();\n\n\tWFTaskFactory::signal_by_name(name, msg, max);\n}\n\nvoid KafkaClientTask::kafka_cgroup_callback(__WFKafkaTask *task)\n{\n\tKafkaClientTask *t = (KafkaClientTask *)task->user_data;\n\tKafkaMember *member = t->member;\n\tSeriesWork *heartbeat_series = NULL;\n\tvoid *msg = NULL;\n\tsize_t max;\n\n\tmember->mutex.lock();\n\tt->state = task->get_state();\n\tt->error = task->get_error();\n\tt->kafka_error = *static_cast(task)->get_mutable_ctx();\n\n\tif (t->state == WFT_STATE_SUCCESS)\n\t{\n\t\tmember->cgroup = std::move(*(task->get_resp()->get_cgroup()));\n\n\t\tkafka_merge_meta_list(&member->meta_list,\n\t\t\t\t\t\t\t task->get_resp()->get_meta_list());\n\n\t\tt->meta_list.rewind();\n\t\tKafkaMeta *meta;\n\t\twhile ((meta = t->meta_list.get_next()) != NULL)\n\t\t\t(member->meta_status)[meta->get_topic()] = true;\n\n\t\tkafka_merge_broker_list(member->scheme,\n\t\t\t\t\t\t\t\t&member->broker_hosts,\n\t\t\t\t\t\t\t\t&member->broker_map,\n\t\t\t\t\t\t\t\ttask->get_resp()->get_broker_list());\n\n\t\tmember->cgroup_status = KAFKA_CGROUP_DONE;\n\n\t\tif (member->heartbeat_status == KAFKA_HEARTBEAT_UNINIT)\n\t\t{\n\t\t\t__WFKafkaTask *kafka_task;\n\t\t\tKafkaBroker *coordinator = member->cgroup.get_coordinator();\n\t\t\tkafka_task = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx, \"\", 0,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t kafka_heartbeat_callback);\n\t\t\tkafka_task->user_data = member;\n\t\t\tmember->incref();\n\n\t\t\tkafka_task->get_req()->set_config(member->config);\n\t\t\tkafka_task->get_req()->set_api_type(Kafka_Heartbeat);\n\t\t\tkafka_task->get_req()->set_cgroup(member->cgroup);\n\t\t\tkafka_task->get_req()->set_broker(*coordinator);\n\n\t\t\theartbeat_series = Workflow::create_series_work(kafka_task, nullptr);\n\t\t\tmember->heartbeat_status = KAFKA_HEARTBEAT_DOING;\n\t\t\tmember->heartbeat_series = heartbeat_series;\n\t\t}\n\t}\n\telse\n\t{\n\t\tmember->cgroup_status = KAFKA_CGROUP_UNINIT;\n\t\tmember->heartbeat_status = KAFKA_HEARTBEAT_UNINIT;\n\t\tmember->heartbeat_series = NULL;\n\t\tt->finish = true;\n\t\tmsg = t;\n\t}\n\n\tmax = member->cgroup_wait_cnt;\n\tchar name[64];\n\tsnprintf(name, 64, \"%p.cgroup\", member);\n\tmember->mutex.unlock();\n\n\tWFTaskFactory::signal_by_name(name, msg, max);\n\n\tif (heartbeat_series)\n\t\theartbeat_series->start();\n}\n\nvoid KafkaClientTask::kafka_parallel_callback(const ParallelWork *pwork)\n{\n\tKafkaClientTask *t = (KafkaClientTask *)pwork->get_context();\n\tt->finish = true;\n\tt->state = WFT_STATE_TASK_ERROR;\n\tt->error = 0;\n\n\tstd::pair *state_error;\n\tbool flag = false;\n\tint16_t state = WFT_STATE_SUCCESS;\n\tint16_t error = 0;\n\tint kafka_error = 0;\n\tfor (size_t i = 0; i < pwork->size(); i++)\n\t{\n\t\tstate_error = (std::pair *)pwork->series_at(i)->get_context();\n\t\tif ((state_error->first >> 16) != WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tif (!flag)\n\t\t\t{\n\t\t\t\tflag = true;\n\t\t\t\tt->member->mutex.lock();\n\t\t\t\tt->set_meta_status(false);\n\t\t\t\tt->member->mutex.unlock();\n\t\t\t}\n\t\t\tstate = state_error->first >> 16;\n\t\t\terror = state_error->first & 0xffff;\n\t\t\tkafka_error = state_error->second;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tt->state = WFT_STATE_SUCCESS;\n\t\t}\n\n\t\tdelete state_error;\n\t}\n\n\tif (t->state != WFT_STATE_SUCCESS)\n\t{\n\t\tt->state = state;\n\t\tt->error = error;\n\t\tt->kafka_error = kafka_error;\n\t}\n}\n\nvoid KafkaClientTask::kafka_process_toppar_offset(KafkaToppar *task_toppar)\n{\n\tKafkaToppar *toppar;\n\n\tstruct list_head *pos;\n\tlist_for_each(pos, this->member->cgroup.get_assigned_toppar_list())\n\t{\n\t\ttoppar = this->member->cgroup.get_assigned_toppar_by_pos(pos);\n\t\tif (strcmp(toppar->get_topic(), task_toppar->get_topic()) == 0 &&\n\t\t\ttoppar->get_partition() == task_toppar->get_partition())\n\t\t{\n\t\t\tlong long offset = task_toppar->get_offset() - 1;\n\t\t\tKafkaRecord *last_record = task_toppar->get_tail_record();\n\t\t\tif (last_record)\n\t\t\t\toffset = last_record->get_offset();\n\t\t\ttoppar->set_offset(offset + 1);\n\t\t\ttoppar->set_low_watermark(task_toppar->get_low_watermark());\n\t\t\ttoppar->set_high_watermark(task_toppar->get_high_watermark());\n\t\t}\n\t}\n}\n\nvoid KafkaClientTask::kafka_move_task_callback(__WFKafkaTask *task)\n{\n\tauto *state_error = new std::pair;\n\tint16_t state = task->get_state();\n\tint16_t error = task->get_error();\n\n\t/* 'state' is always positive. */\n\tstate_error->first = (state << 16) | error;\n\tstate_error->second = *static_cast(task)->get_mutable_ctx();\n\tseries_of(task)->set_context(state_error);\n\n\tKafkaTopparList *toppar_list = task->get_resp()->get_toppar_list();\n\n\tif (task->get_state() == WFT_STATE_SUCCESS &&\n\t\ttask->get_resp()->get_api_type() == Kafka_Fetch)\n\t{\n\t\ttoppar_list->rewind();\n\t\tKafkaToppar *task_toppar;\n\n\t\twhile ((task_toppar = toppar_list->get_next()) != NULL)\n\t\t\tkafka_process_toppar_offset(task_toppar);\n\t}\n\n\tif (task->get_state() == WFT_STATE_SUCCESS)\n\t{\n\t\tlong idx = (long)(task->user_data);\n\t\tthis->result.set_resp(std::move(*task->get_resp()), idx);\n\t}\n}\n\nvoid KafkaClientTask::generate_info()\n{\n\tif (this->info_generated)\n\t\treturn;\n\n\tif (this->config.get_sasl_mech())\n\t{\n\t\tconst char *username = this->config.get_sasl_username();\n\t\tconst char *password = this->config.get_sasl_password();\n\n\t\tthis->userinfo.clear();\n\t\tif (username)\n\t\t\tthis->userinfo += StringUtil::url_encode_component(username);\n\t\tthis->userinfo += \":\";\n\t\tif (password)\n\t\t\tthis->userinfo += StringUtil::url_encode_component(password);\n\t\tthis->userinfo += \":\";\n\t\tthis->userinfo += this->config.get_sasl_mech();\n\t\tthis->userinfo += \":\";\n\t\tthis->userinfo += std::to_string((intptr_t)this->member);\n\t}\n\telse\n\t{\n\t\tchar buf[64];\n\t\tsnprintf(buf, 64, \"user:pass:sasl:%p\", this->member);\n\t\tthis->userinfo = buf;\n\t}\n\n\tconst char *hostport = this->url.c_str() + this->member->scheme.size();\n\tthis->url = this->member->scheme + this->userinfo + \"@\" + hostport;\n\tthis->info_generated = true;\n}\n\nvoid KafkaClientTask::parse_query()\n{\n\tauto query_kv = URIParser::split_query_strict(this->query);\n\tint api_type = this->api_type;\n\tfor (const auto &kv : query_kv)\n\t{\n\t\tif (strcasecmp(kv.first.c_str(), \"api\") == 0 &&\n\t\t\tapi_type == Kafka_Unknown)\n\t\t{\n\t\t\tfor (auto& v : kv.second)\n\t\t\t{\n\t\t\t\tif (strcasecmp(v.c_str(), \"fetch\") == 0)\n\t\t\t\t\tthis->api_type = Kafka_Fetch;\n\t\t\t\telse if (strcasecmp(v.c_str(), \"produce\") == 0)\n\t\t\t\t\tthis->api_type = Kafka_Produce;\n\t\t\t\telse if (strcasecmp(v.c_str(), \"commit\") == 0)\n\t\t\t\t\tthis->api_type = Kafka_OffsetCommit;\n\t\t\t\telse if (strcasecmp(v.c_str(), \"meta\") == 0)\n\t\t\t\t\tthis->api_type = Kafka_Metadata;\n\t\t\t\telse if (strcasecmp(v.c_str(), \"leavegroup\") == 0)\n\t\t\t\t\tthis->api_type = Kafka_LeaveGroup;\n\t\t\t\telse if (strcasecmp(v.c_str(), \"listoffsets\") == 0)\n\t\t\t\t\tthis->api_type = Kafka_ListOffsets;\n\t\t\t}\n\t\t}\n\t\telse if (strcasecmp(kv.first.c_str(), \"topic\") == 0)\n\t\t{\n\t\t\tfor (auto& v : kv.second)\n\t\t\t\tthis->add_topic(v);\n\t\t}\n\t}\n}\n\nbool KafkaClientTask::get_meta_status(KafkaMetaList **uninit_meta_list)\n{\n\tthis->meta_list.rewind();\n\tKafkaMeta *meta;\n\tstd::set unique;\n\tbool status = true;\n\n\twhile ((meta = this->meta_list.get_next()) != NULL)\n\t{\n\t\tif (!unique.insert(meta->get_topic()).second)\n\t\t\tcontinue;\n\n\t\tif (!this->member->meta_status[meta->get_topic()])\n\t\t{\n\t\t\tif (status)\n\t\t\t{\n\t\t\t\t*uninit_meta_list = new KafkaMetaList;\n\t\t\t\tstatus = false;\n\t\t\t}\n\n\t\t\t(*uninit_meta_list)->add_item(*meta);\n\t\t}\n\t}\n\n\treturn status;\n}\n\nvoid KafkaClientTask::set_meta_status(bool status)\n{\n\tthis->member->meta_list.rewind();\n\tKafkaMeta *meta;\n\twhile ((meta = this->member->meta_list.get_next()) != NULL)\n\t\tthis->member->meta_status[meta->get_topic()] = false;\n}\n\nbool KafkaClientTask::compare_topics(KafkaClientTask *task)\n{\n\tauto first1 = topic_set.cbegin(), last1 = topic_set.cend();\n\tauto first2 = task->topic_set.cbegin(), last2 = task->topic_set.cend();\n\tint cmp;\n\n\t// check whether task->topic_set is a subset of topic_set\n\twhile (first1 != last1 && first2 != last2)\n\t{\n\t\tcmp = first1->compare(*first2);\n\t\tif (cmp == 0)\n\t\t{\n\t\t\t++first1;\n\t\t\t++first2;\n\t\t}\n\t\telse if (cmp < 0)\n\t\t\t++first1;\n\t\telse\n\t\t\treturn false;\n\t}\n\n\treturn first2 == last2;\n}\n\nbool KafkaClientTask::check_cgroup()\n{\n\tKafkaMember *member = this->member;\n\n\tif (member->cgroup_outdated && member->cgroup_status != KAFKA_CGROUP_DOING)\n\t{\n\t\tmember->cgroup_outdated = false;\n\t\tmember->cgroup_status = KAFKA_CGROUP_UNINIT;\n\t\tmember->heartbeat_series = NULL;\n\t\tmember->heartbeat_status = KAFKA_HEARTBEAT_UNINIT;\n\t}\n\n\tif (member->cgroup_status == KAFKA_CGROUP_DOING)\n\t{\n\t\tWFConditional *cond;\n\t\tchar name[64];\n\t\tsnprintf(name, 64, \"%p.cgroup\", this->member);\n\t\tthis->wait_cgroup = true;\n\t\tcond = WFTaskFactory::create_conditional(name, this, &this->msg);\n\t\tseries_of(this)->push_front(cond);\n\t\tmember->cgroup_wait_cnt++;\n\t\treturn false;\n\t}\n\n\tif ((this->api_type == Kafka_Fetch || this->api_type == Kafka_OffsetCommit) &&\n\t\t(member->cgroup_status == KAFKA_CGROUP_UNINIT))\n\t{\n\t\t__WFKafkaTask *task;\n\n\t\ttask = __WFKafkaTaskFactory::create_kafka_task(this->url, member->ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t kafka_cgroup_callback);\n\t\ttask->user_data = this;\n\t\ttask->get_req()->set_config(this->config);\n\t\ttask->get_req()->set_api_type(Kafka_FindCoordinator);\n\t\ttask->get_req()->set_cgroup(member->cgroup);\n\t\ttask->get_req()->set_meta_list(member->meta_list);\n\t\tseries_of(this)->push_front(this);\n\t\tseries_of(this)->push_front(task);\n\t\tmember->cgroup_status = KAFKA_CGROUP_DOING;\n\t\tmember->cgroup_wait_cnt = 0;\n\t\treturn false;\n\t}\n\n\treturn true;\n}\n\nbool KafkaClientTask::check_meta()\n{\n\tKafkaMember *member = this->member;\n\tKafkaMetaList *uninit_meta_list;\n\n\tif (this->get_meta_status(&uninit_meta_list))\n\t\treturn true;\n\n\tif (member->meta_doing)\n\t{\n\t\tWFConditional *cond;\n\t\tchar name[64];\n\t\tsnprintf(name, 64, \"%p.meta\", this->member);\n\t\tthis->wait_cgroup = false;\n\t\tcond = WFTaskFactory::create_conditional(name, this, &this->msg);\n\t\tseries_of(this)->push_front(cond);\n\t\tmember->meta_wait_cnt++;\n\t}\n\telse\n\t{\n\t\t__WFKafkaTask *task;\n\n\t\ttask = __WFKafkaTaskFactory::create_kafka_task(this->url, member->ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t kafka_meta_callback);\n\t\ttask->user_data = this;\n\t\ttask->get_req()->set_config(this->config);\n\t\ttask->get_req()->set_api_type(Kafka_Metadata);\n\t\ttask->get_req()->set_meta_list(*uninit_meta_list);\n\t\tseries_of(this)->push_front(this);\n\t\tseries_of(this)->push_front(task);\n\t\tmember->meta_wait_cnt = 0;\n\t\tmember->meta_doing = true;\n\t}\n\n\tdelete uninit_meta_list;\n\treturn false;\n}\n\nint KafkaClientTask::dispatch_locked()\n{\n\tKafkaMember *member = this->member;\n\tKafkaBroker *coordinator;\n\t__WFKafkaTask *task;\n\tParallelWork *parallel;\n\tSeriesWork *series;\n\n\tif (this->check_cgroup() == false)\n\t\treturn member->cgroup_wait_cnt > 0;\n\n\tif (this->check_meta() == false)\n\t\treturn member->meta_wait_cnt > 0;\n\n\tif (arrange_toppar(this->api_type) < 0)\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_KAFKA_ARRANGE_FAILED;\n\t\tthis->finish = true;\n\t\treturn 0;\n\t}\n\n\tif (this->member->cgroup_outdated)\n\t{\n\t\tseries_of(this)->push_front(this);\n\t\treturn 0;\n\t}\n\n\tswitch(this->api_type)\n\t{\n\tcase Kafka_Produce:\n\t\tif (this->toppar_list_map.size() == 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_KAFKA_PRODUCE_FAILED;\n\t\t\tthis->finish = true;\n\t\t\tbreak;\n\t\t}\n\n\t\tparallel = Workflow::create_parallel_work(kafka_parallel_callback);\n\t\tthis->result.create(this->toppar_list_map.size());\n\t\tparallel->set_context(this);\n\t\tfor (auto &v : this->toppar_list_map)\n\t\t{\n\t\t\tauto cb = std::bind(&KafkaClientTask::kafka_move_task_callback, this,\n\t\t\t\t\t\t\t\tstd::placeholders::_1);\n\t\t\tKafkaBroker *broker = get_broker(v.first);\n\t\t\ttask = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t broker->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t broker->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t this->get_userinfo(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t std::move(cb));\n\t\t\ttask->get_req()->set_config(this->config);\n\t\t\ttask->get_req()->set_toppar_list(v.second);\n\t\t\ttask->get_req()->set_broker(*broker);\n\t\t\ttask->get_req()->set_api_type(Kafka_Produce);\n\t\t\ttask->user_data = (void *)parallel->size();\n\t\t\tseries = Workflow::create_series_work(task, nullptr);\n\t\t\tparallel->add_series(series);\n\t\t}\n\t\tseries_of(this)->push_front(this);\n\t\tseries_of(this)->push_front(parallel);\n\t\tbreak;\n\n\tcase Kafka_Fetch:\n\t\tif (this->toppar_list_map.size() == 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_KAFKA_FETCH_FAILED;\n\t\t\tthis->finish = true;\n\t\t\tbreak;\n\t\t}\n\n\t\tparallel = Workflow::create_parallel_work(kafka_parallel_callback);\n\t\tthis->result.create(this->toppar_list_map.size());\n\t\tparallel->set_context(this);\n\t\tfor (auto &v : this->toppar_list_map)\n\t\t{\n\t\t\tauto cb = std::bind(&KafkaClientTask::kafka_move_task_callback, this,\n\t\t\t\t\t\t\t\tstd::placeholders::_1);\n\t\t\tKafkaBroker *broker = get_broker(v.first);\n\t\t\ttask = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t broker->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t broker->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t this->get_userinfo(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t std::move(cb));\n\n\t\t\ttask->get_req()->set_config(this->config);\n\t\t\ttask->get_req()->set_toppar_list(v.second);\n\t\t\ttask->get_req()->set_broker(*broker);\n\t\t\ttask->get_req()->set_api_type(Kafka_Fetch);\n\t\t\ttask->user_data = (void *)parallel->size();\n\t\t\tseries = Workflow::create_series_work(task, nullptr);\n\t\t\tparallel->add_series(series);\n\t\t}\n\n\t\tseries_of(this)->push_front(this);\n\t\tseries_of(this)->push_front(parallel);\n\t\tbreak;\n\n\tcase Kafka_Metadata:\n\t\tthis->finish = true;\n\t\tbreak;\n\n\tcase Kafka_OffsetCommit:\n\t\tif (!member->cgroup.get_group())\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_KAFKA_COMMIT_FAILED;\n\t\t\tthis->finish = true;\n\t\t\tbreak;\n\t\t}\n\n\t\tthis->result.create(1);\n\t\tcoordinator = member->cgroup.get_coordinator();\n\t\ttask = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t this->get_userinfo(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t kafka_offsetcommit_callback);\n\t\ttask->user_data = this;\n\t\ttask->get_req()->set_config(this->config);\n\t\ttask->get_req()->set_cgroup(member->cgroup);\n\t\ttask->get_req()->set_broker(*coordinator);\n\t\ttask->get_req()->set_toppar_list(this->toppar_list);\n\t\ttask->get_req()->set_api_type(this->api_type);\n\t\tseries_of(this)->push_front(this);\n\t\tseries_of(this)->push_front(task);\n\t\tbreak;\n\n\tcase Kafka_LeaveGroup:\n\t\tif (!member->cgroup.get_group())\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_KAFKA_LEAVEGROUP_FAILED;\n\t\t\tthis->finish = true;\n\t\t\tbreak;\n\t\t}\n\n\t\tcoordinator = member->cgroup.get_coordinator();\n\t\tif (!coordinator->get_host())\n\t\t\tbreak;\n\n\t\ttask = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t coordinator->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t this->get_userinfo(), 0,\n\t\t\t\t\t\t\t\t\t\t\t\t\t kafka_leavegroup_callback);\n\t\ttask->user_data = this;\n\t\ttask->get_req()->set_config(this->config);\n\t\ttask->get_req()->set_api_type(Kafka_LeaveGroup);\n\t\ttask->get_req()->set_broker(*coordinator);\n\t\ttask->get_req()->set_cgroup(member->cgroup);\n\t\tseries_of(this)->push_front(this);\n\t\tseries_of(this)->push_front(task);\n\t\tbreak;\n\n\tcase Kafka_ListOffsets:\n\t\tif (this->toppar_list_map.size() == 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_KAFKA_LIST_OFFSETS_FAILED;\n\t\t\tthis->finish = true;\n\t\t\tbreak;\n\t\t}\n\n\t\tparallel = Workflow::create_parallel_work(kafka_parallel_callback);\n\t\tthis->result.create(this->toppar_list_map.size());\n\t\tparallel->set_context(this);\n\t\tfor (auto &v : this->toppar_list_map)\n\t\t{\n\t\t\tauto cb = std::bind(&KafkaClientTask::kafka_move_task_callback, this,\n\t\t\t\t\t\t\t\tstd::placeholders::_1);\n\t\t\tKafkaBroker *broker = get_broker(v.first);\n\t\t\ttask = __WFKafkaTaskFactory::create_kafka_task(member->transport_type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t broker->get_host(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t broker->get_port(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t member->ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t this->get_userinfo(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t this->retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t std::move(cb));\n\t\t\ttask->get_req()->set_config(this->config);\n\t\t\ttask->get_req()->set_toppar_list(v.second);\n\t\t\ttask->get_req()->set_broker(*broker);\n\t\t\ttask->get_req()->set_api_type(Kafka_ListOffsets);\n\t\t\ttask->user_data = (void *)parallel->size();\n\t\t\tseries = Workflow::create_series_work(task, nullptr);\n\t\t\tparallel->add_series(series);\n\t\t}\n\t\tseries_of(this)->push_front(this);\n\t\tseries_of(this)->push_front(parallel);\n\t\tbreak;\n\n\tdefault:\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_KAFKA_API_UNKNOWN;\n\t\tthis->finish = true;\n\t\tbreak;\n\t}\n\n\treturn 0;\n}\n\nvoid KafkaClientTask::dispatch()\n{\n\tif (this->finish)\n\t{\n\t\tthis->subtask_done();\n\t\treturn;\n\t}\n\n\tif (this->msg)\n\t{\n\t\tKafkaClientTask *task = static_cast(this->msg);\n\t\tif (this->wait_cgroup || this->compare_topics(task) == true)\n\t\t{\n\t\t\tthis->state = task->get_state();\n\t\t\tthis->error = task->get_error();\n\t\t\tthis->kafka_error = get_kafka_error();\n\t\t\tthis->finish = true;\n\t\t\tthis->subtask_done();\n\t\t\treturn;\n\t\t}\n\n\t\tthis->msg = NULL;\n\t}\n\n\tif (!this->query.empty())\n\t\tthis->parse_query();\n\n\tthis->generate_info();\n\n\tint flag;\n\tthis->member->mutex.lock();\n\tflag = this->dispatch_locked();\n\tif (flag)\n\t\tthis->subtask_done();\n\tthis->member->mutex.unlock();\n\n\tif (!flag)\n\t\tthis->subtask_done();\n}\n\nbool KafkaClientTask::add_topic(const std::string& topic)\n{\n\tbool flag = false;\n\tthis->member->mutex.lock();\n\n\tthis->topic_set.insert(topic);\n\tthis->member->meta_list.rewind();\n\tKafkaMeta *meta;\n\twhile ((meta = this->member->meta_list.get_next()) != NULL)\n\t{\n\t\tif (meta->get_topic() == topic)\n\t\t{\n\t\t\tflag = true;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (!flag)\n\t{\n\t\tthis->member->meta_status[topic] = false;\n\n\t\tKafkaMeta tmp;\n\t\tif (!tmp.set_topic(topic))\n\t\t{\n\t\t\tthis->member->mutex.unlock();\n\t\t\treturn false;\n\t\t}\n\n\t\tthis->meta_list.add_item(tmp);\n\t\tthis->member->meta_list.add_item(tmp);\n\n\t\tif (this->member->cgroup.get_group())\n\t\t\tthis->member->cgroup_outdated = true;\n\t}\n\telse\n\t{\n\t\tthis->meta_list.rewind();\n\t\tKafkaMeta *exist;\n\t\twhile ((exist = this->meta_list.get_next()) != NULL)\n\t\t{\n\t\t\tif (strcmp(exist->get_topic(), meta->get_topic()) == 0)\n\t\t\t{\n\t\t\t\tthis->member->mutex.unlock();\n\t\t\t\treturn true;\n\t\t\t}\n\t\t}\n\n\t\tthis->meta_list.add_item(*meta);\n\t}\n\n\tthis->member->mutex.unlock();\n\n\treturn true;\n}\n\nbool KafkaClientTask::add_toppar(const KafkaToppar& toppar)\n{\n\tif (this->member->cgroup.get_group())\n\t\treturn false;\n\n\tbool flag = false;\n\tthis->member->mutex.lock();\n\n\tthis->member->meta_list.rewind();\n\tKafkaMeta *meta;\n\twhile ((meta = this->member->meta_list.get_next()) != NULL)\n\t{\n\t\tif (strcmp(meta->get_topic(), toppar.get_topic()) == 0)\n\t\t{\n\t\t\tflag = true;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tthis->topic_set.insert(toppar.get_topic());\n\tif (!flag)\n\t{\n\t\tKafkaMeta tmp;\n\t\tif (!tmp.set_topic(toppar.get_topic()))\n\t\t{\n\t\t\tthis->member->mutex.unlock();\n\t\t\treturn false;\n\t\t}\n\n\t\tKafkaToppar new_toppar;\n\t\tif (!new_toppar.set_topic_partition(toppar.get_topic(), toppar.get_partition()))\n\t\t{\n\t\t\tthis->member->mutex.unlock();\n\t\t\treturn false;\n\t\t}\n\n\t\tnew_toppar.set_offset(toppar.get_offset());\n\t\tnew_toppar.set_offset_timestamp(toppar.get_offset_timestamp());\n\t\tnew_toppar.set_low_watermark(toppar.get_low_watermark());\n\t\tnew_toppar.set_high_watermark(toppar.get_high_watermark());\n\t\tthis->toppar_list.add_item(new_toppar);\n\n\t\tthis->meta_list.add_item(tmp);\n\t\tthis->member->meta_list.add_item(tmp);\n\n\t\tif (this->member->cgroup.get_group())\n\t\t\tthis->member->cgroup_outdated = true;\n\t}\n\telse\n\t{\n\t\tthis->toppar_list.rewind();\n\t\tKafkaToppar *exist;\n\t\twhile ((exist = this->toppar_list.get_next()) != NULL)\n\t\t{\n\t\t\tif (strcmp(exist->get_topic(), toppar.get_topic()) == 0 &&\n\t\t\t\texist->get_partition() == toppar.get_partition())\n\t\t\t{\n\t\t\t\tthis->member->mutex.unlock();\n\t\t\t\treturn true;\n\t\t\t}\n\t\t}\n\n\t\tKafkaToppar new_toppar;\n\t\tif (!new_toppar.set_topic_partition(toppar.get_topic(), toppar.get_partition()))\n\t\t{\n\t\t\tthis->member->mutex.unlock();\n\t\t\treturn true;\n\t\t}\n\n\t\tnew_toppar.set_offset(toppar.get_offset());\n\t\tnew_toppar.set_offset_timestamp(toppar.get_offset_timestamp());\n\t\tnew_toppar.set_low_watermark(toppar.get_low_watermark());\n\t\tnew_toppar.set_high_watermark(toppar.get_high_watermark());\n\t\tthis->toppar_list.add_item(new_toppar);\n\n\t\tthis->meta_list.add_item(*meta);\n\t}\n\n\tthis->member->mutex.unlock();\n\n\treturn true;\n}\n\nbool KafkaClientTask::add_produce_record(const std::string& topic,\n\t\t\t\t\t\t\t\t\t\t int partition,\n\t\t\t\t\t\t\t\t\t\t KafkaRecord record)\n{\n\tif (!add_topic(topic))\n\t\treturn false;\n\n\tbool flag = false;\n\tthis->toppar_list.rewind();\n\tKafkaToppar *toppar;\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tif (toppar->get_topic() == topic &&\n\t\t\ttoppar->get_partition() == partition)\n\t\t{\n\t\t\tflag = true;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (!flag)\n\t{\n\t\tKafkaToppar new_toppar;\n\t\tif (!new_toppar.set_topic_partition(topic, partition))\n\t\t\treturn false;\n\n\t\tnew_toppar.add_record(std::move(record));\n\t\tthis->toppar_list.add_item(std::move(new_toppar));\n\t}\n\telse\n\t\ttoppar->add_record(std::move(record));\n\n\treturn true;\n}\n\nstatic bool check_replace_toppar(KafkaTopparList *toppar_list, KafkaToppar *toppar)\n{\n\tbool flag = false;\n\ttoppar_list->rewind();\n\tKafkaToppar *exist;\n\twhile ((exist = toppar_list->get_next()) != NULL)\n\t{\n\t\tif (strcmp(exist->get_topic(), toppar->get_topic()) == 0 &&\n\t\t\texist->get_partition() == toppar->get_partition())\n\t\t{\n\t\t\tflag = true;\n\t\t\tif (toppar->get_offset() > exist->get_offset())\n\t\t\t{\n\t\t\t\ttoppar_list->add_item(std::move(*toppar));\n\t\t\t\ttoppar_list->del_cur();\n\t\t\t\tdelete exist;\n\t\t\t\treturn true;\n\t\t\t}\n\t\t}\n\t}\n\n\tif (!flag)\n\t{\n\t\ttoppar_list->add_item(std::move(*toppar));\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nint KafkaClientTask::arrange_toppar(int api_type)\n{\n\tswitch(api_type)\n\t{\n\tcase Kafka_Produce:\n\t\treturn this->arrange_produce();\n\n\tcase Kafka_Fetch:\n\t\treturn this->arrange_fetch();\n\n\tcase Kafka_ListOffsets:\n\t\treturn this->arrange_offset();\n\n\tcase Kafka_OffsetCommit:\n\t\treturn this->arrange_commit();\n\n\tdefault:\n\t\treturn 0;\n\t}\n}\n\nbool KafkaClientTask::add_offset_toppar(const protocol::KafkaToppar& toppar)\n{\n\tif (!add_topic(toppar.get_topic()))\n\t\treturn false;\n\n\tKafkaToppar *exist;\n\tbool found = false;\n\twhile ((exist = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tif (strcmp(exist->get_topic(), toppar.get_topic()) == 0 &&\n\t\t\t\texist->get_partition() == toppar.get_partition())\n\t\t{\n\t\t\tfound = true;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (!found)\n\t{\n\t\tKafkaToppar toppar_t;\n\t\ttoppar_t.set_topic_partition(toppar.get_topic(), toppar.get_partition());\n\t\tthis->toppar_list.add_item(std::move(toppar_t));\n\t}\n\n\treturn true;\n}\n\nint KafkaClientTask::arrange_offset()\n{\n\tthis->toppar_list.rewind();\n\tKafkaToppar *toppar;\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tint node_id = get_node_id(toppar);\n\t\tif (node_id < 0)\n\t\t\treturn -1;\n\n\t\tif (this->toppar_list_map.find(node_id) == this->toppar_list_map.end())\n\t\t\tthis->toppar_list_map[node_id] = (KafkaTopparList());\n\n\t\tKafkaToppar new_toppar;\n\t\tif (!new_toppar.set_topic_partition(toppar->get_topic(), toppar->get_partition()))\n\t\t\treturn -1;\n\n\t\tthis->toppar_list_map[node_id].add_item(std::move(new_toppar));\n\t}\n\n\treturn 0;\n}\n\nint KafkaClientTask::arrange_commit()\n{\n\tthis->toppar_list.rewind();\n\tKafkaTopparList new_toppar_list;\n\tKafkaToppar *toppar;\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tcheck_replace_toppar(&new_toppar_list, toppar);\n\t}\n\n\tthis->toppar_list = std::move(new_toppar_list);\n\treturn 0;\n}\n\nint KafkaClientTask::arrange_fetch()\n{\n\tthis->meta_list.rewind();\n\tfor (auto& topic : topic_set)\n\t{\n\t\tif (this->member->cgroup.get_group())\n\t\t{\n\t\t\tthis->member->cgroup.assigned_toppar_rewind();\n\t\t\tKafkaToppar *toppar;\n\t\t\twhile ((toppar = this->member->cgroup.get_assigned_toppar_next()) != NULL)\n\t\t\t{\n\t\t\t\tif (topic.compare(toppar->get_topic()) == 0)\n\t\t\t\t{\n\t\t\t\t\tint node_id = get_node_id(toppar);\n\t\t\t\t\tif (node_id < 0)\n\t\t\t\t\t\treturn -1;\n\n\t\t\t\t\tif (this->toppar_list_map.find(node_id) == this->toppar_list_map.end())\n\t\t\t\t\t\tthis->toppar_list_map[node_id] = (KafkaTopparList());\n\n\t\t\t\t\tKafkaToppar new_toppar;\n\t\t\t\t\tif (!new_toppar.set_topic_partition(toppar->get_topic(), toppar->get_partition()))\n\t\t\t\t\t\treturn -1;\n\n\t\t\t\t\tnew_toppar.set_offset(toppar->get_offset());\n\t\t\t\t\tnew_toppar.set_low_watermark(toppar->get_low_watermark());\n\t\t\t\t\tnew_toppar.set_high_watermark(toppar->get_high_watermark());\n\t\t\t\t\tthis->toppar_list_map[node_id].add_item(std::move(new_toppar));\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tthis->toppar_list.rewind();\n\t\t\tKafkaToppar *toppar;\n\t\t\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t\t\t{\n\t\t\t\tif (topic.compare(toppar->get_topic()) == 0)\n\t\t\t\t{\n\t\t\t\t\tint node_id = get_node_id(toppar);\n\t\t\t\t\tif (node_id < 0)\n\t\t\t\t\t\treturn -1;\n\n\t\t\t\t\tif (this->toppar_list_map.find(node_id) == this->toppar_list_map.end())\n\t\t\t\t\t\tthis->toppar_list_map[node_id] = KafkaTopparList();\n\n\t\t\t\t\tKafkaToppar new_toppar;\n\t\t\t\t\tif (!new_toppar.set_topic_partition(toppar->get_topic(), toppar->get_partition()))\n\t\t\t\t\t\treturn -1;\n\n\t\t\t\t\tnew_toppar.set_offset(toppar->get_offset());\n\t\t\t\t\tnew_toppar.set_offset_timestamp(toppar->get_offset_timestamp());\n\t\t\t\t\tnew_toppar.set_low_watermark(toppar->get_low_watermark());\n\t\t\t\t\tnew_toppar.set_high_watermark(toppar->get_high_watermark());\n\t\t\t\t\tthis->toppar_list_map[node_id].add_item(std::move(new_toppar));\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nint KafkaClientTask::arrange_produce()\n{\n\tthis->toppar_list.rewind();\n\tKafkaToppar *toppar;\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tif (toppar->get_partition() < 0)\n\t\t{\n\t\t\ttoppar->record_rewind();\n\t\t\tKafkaRecord *record;\n\t\t\twhile ((record = toppar->get_record_next()) != NULL)\n\t\t\t{\n\t\t\t\tint partition_num;\n\t\t\t\tconst KafkaMeta *meta;\n\t\t\t\tmeta = get_meta(toppar->get_topic(), &this->member->meta_list);\n\t\t\t\tif (!meta)\n\t\t\t\t\treturn -1;\n\n\t\t\t\tpartition_num = meta->get_partition_elements();\n\t\t\t\tif (partition_num <= 0)\n\t\t\t\t\treturn -1;\n\n\t\t\t\tint partition = -1;\n\t\t\t\tif (this->partitioner)\n\t\t\t\t{\n\t\t\t\t\tconst void *key;\n\t\t\t\t\tsize_t key_len;\n\t\t\t\t\trecord->get_key(&key, &key_len);\n\t\t\t\t\tpartition = this->partitioner(toppar->get_topic(), key,\n\t\t\t\t\t\t\t\t\t\t\t\t key_len, partition_num);\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t\tpartition = rand() % partition_num;\n\n\t\t\t\tKafkaToppar *new_toppar = get_toppar(toppar->get_topic(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t partition,\n\t\t\t\t\t\t\t\t\t\t\t\t\t &this->toppar_list);\n\t\t\t\tif (!new_toppar)\n\t\t\t\t{\n\t\t\t\t\tKafkaToppar tmp;\n\t\t\t\t\tif (!tmp.set_topic_partition(toppar->get_topic(), partition))\n\t\t\t\t\t\treturn -1;\n\n\t\t\t\t\tnew_toppar = this->toppar_list.add_item(std::move(tmp));\n\t\t\t\t}\n\n\t\t\t\trecord->get_raw_ptr()->toppar = new_toppar->get_raw_ptr();\n\t\t\t\tnew_toppar->add_record(std::move(*record));\n\t\t\t\ttoppar->del_record_cur();\n\t\t\t\tdelete record;\n\t\t\t}\n\t\t\tthis->toppar_list.del_cur();\n\t\t\tdelete toppar;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tKafkaRecord *record;\n\t\t\twhile ((record = toppar->get_record_next()) != NULL)\n\t\t\t\trecord->get_raw_ptr()->toppar = toppar->get_raw_ptr();\n\t\t}\n\t}\n\n\tthis->toppar_list.rewind();\n\tKafkaTopparList toppar_list;\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tint node_id = get_node_id(toppar);\n\t\tif (node_id < 0)\n\t\t\treturn -1;\n\n\t\tif (this->toppar_list_map.find(node_id) == this->toppar_list_map.end())\n\t\t\tthis->toppar_list_map[node_id] = KafkaTopparList();\n\n\t\tthis->toppar_list_map[node_id].add_item(std::move(*toppar));\n\t}\n\n\treturn 0;\n}\n\nSubTask *WFKafkaTask::done()\n{\n\tSeriesWork *series = series_of(this);\n\n\tauto cb = [] (WFTimerTask *task)\n\t{\n\t\tWFKafkaTask *kafka_task = (WFKafkaTask *)task->user_data;\n\t\tif (kafka_task->callback)\n\t\t\tkafka_task->callback(kafka_task);\n\n\t\tdelete kafka_task;\n\t};\n\n\tif (finish)\n\t{\n\t\tif (this->state == WFT_STATE_TASK_ERROR)\n\t\t{\n\t\t\tWFTimerTask *timer;\n\t\t\ttimer = WFTaskFactory::create_timer_task(0, 0, std::move(cb));\n\t\t\ttimer->user_data = this;\n\t\t\tseries->push_front(timer);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (this->callback)\n\t\t\t\tthis->callback(this);\n\n\t\t\tdelete this;\n\t\t}\n\t}\n\n\treturn series->pop();\n}\n\nint WFKafkaClient::init(const std::string& broker, SSL_CTX *ssl_ctx)\n{\n\tstd::vector broker_hosts;\n\tstd::string::size_type ppos = 0;\n\tstd::string::size_type pos;\n\tbool use_ssl;\n\n\tuse_ssl = (strncasecmp(broker.c_str(), \"kafkas://\", 9) == 0);\n\twhile (1)\n\t{\n\t\tpos = broker.find(',', ppos);\n\t\tstd::string host = broker.substr(ppos, pos - ppos);\n\t\tif (use_ssl)\n\t\t{\n\t\t\tif (strncasecmp(host.c_str(), \"kafkas://\", 9) != 0)\n\t\t\t{\n\t\t\t\terrno = EINVAL;\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t}\n\t\telse if (strncasecmp(host.c_str(), \"kafka://\", 8) != 0)\n\t\t{\n\t\t\tif (strncasecmp(host.c_str(), \"kafkas://\", 9) == 0)\n\t\t\t{\n\t\t\t\terrno = EINVAL;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\thost = \"kafka://\" + host;\n\t\t}\n\n\t\tbroker_hosts.emplace_back(host);\n\t\tif (pos == std::string::npos)\n\t\t\tbreak;\n\n\t\tppos = pos + 1;\n\t}\n\n\tthis->member = new KafkaMember;\n\tthis->member->broker_hosts = std::move(broker_hosts);\n\tthis->member->ssl_ctx = ssl_ctx;\n\tif (use_ssl)\n\t{\n\t\tthis->member->transport_type = TT_TCP_SSL;\n\t\tthis->member->scheme = \"kafkas://\";\n\t}\n\n\treturn 0;\n}\n\nint WFKafkaClient::init(const std::string& broker, const std::string& group,\n\t\t\t\t\t\tSSL_CTX *ssl_ctx)\n{\n\tif (this->init(broker, ssl_ctx) < 0)\n\t\treturn -1;\n\n\tthis->member->cgroup.set_group(group);\n\tthis->member->cgroup_status = KAFKA_CGROUP_UNINIT;\n\treturn 0;\n}\n\nint WFKafkaClient::deinit()\n{\n\tthis->member->mutex.lock();\n\tthis->member->client_deinit = true;\n\tthis->member->mutex.unlock();\n\tthis->member->decref();\n\treturn 0;\n}\n\nWFKafkaTask *WFKafkaClient::create_kafka_task(const std::string& query,\n\t\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t kafka_callback_t cb)\n{\n\tWFKafkaTask *task = new KafkaClientTask(query, retry_max, std::move(cb), this);\n\treturn task;\n}\n\nWFKafkaTask *WFKafkaClient::create_kafka_task(int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t kafka_callback_t cb)\n{\n\tWFKafkaTask *task = new KafkaClientTask(\"\", retry_max, std::move(cb), this);\n\treturn task;\n}\n\nWFKafkaTask *WFKafkaClient::create_leavegroup_task(int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t kafka_callback_t cb)\n{\n\tWFKafkaTask *task = new KafkaClientTask(\"api=leavegroup\", retry_max,\n\t\t\t\t\t\t\t\t\t\t\tstd::move(cb), this);\n\treturn task;\n}\n\nvoid WFKafkaClient::set_config(protocol::KafkaConfig conf)\n{\n\tthis->member->config = std::move(conf);\n}\n\nKafkaMetaList *WFKafkaClient::get_meta_list()\n{\n\treturn &this->member->meta_list;\n}\n\nconst KafkaMetaList *WFKafkaClient::get_meta_list() const\n{\n\treturn &this->member->meta_list;\n}\n\n" }, { "path": "src/client/WFKafkaClient.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#ifndef _WFKAFKACLIENT_H_\n#define _WFKAFKACLIENT_H_\n\n#include \n#include \n#include \n#include \n#include \"WFTask.h\"\n#include \"KafkaMessage.h\"\n#include \"KafkaResult.h\"\n\nclass WFKafkaTask;\nclass WFKafkaClient;\n\nusing kafka_callback_t = std::function;\nusing kafka_partitioner_t = std::function;\n\nclass WFKafkaTask : public WFGenericTask\n{\npublic:\n\tvirtual bool add_topic(const std::string& topic) = 0;\n\n\tvirtual bool add_toppar(const protocol::KafkaToppar& toppar) = 0;\n\n\tvirtual bool add_produce_record(const std::string& topic, int partition,\n\t\t\t\t\t\t\t\t\tprotocol::KafkaRecord record) = 0;\n\n\tvirtual bool add_offset_toppar(const protocol::KafkaToppar& toppar) = 0;\n\n\tvoid add_commit_record(const protocol::KafkaRecord& record)\n\t{\n\t\tprotocol::KafkaToppar toppar;\n\t\ttoppar.set_topic_partition(record.get_topic(), record.get_partition());\n\t\ttoppar.set_offset(record.get_offset());\n\t\ttoppar.set_error(0);\n\t\tthis->toppar_list.add_item(std::move(toppar));\n\t}\n\n\tvoid add_commit_toppar(const protocol::KafkaToppar& toppar)\n\t{\n\t\tprotocol::KafkaToppar toppar_t;\n\t\ttoppar_t.set_topic_partition(toppar.get_topic(), toppar.get_partition());\n\t\ttoppar_t.set_offset(toppar.get_offset());\n\t\ttoppar_t.set_error(0);\n\t\tthis->toppar_list.add_item(std::move(toppar_t));\n\t}\n\n\tvoid add_commit_item(const std::string& topic, int partition,\n\t\t\t\t\t\t long long offset)\n\t{\n\t\tprotocol::KafkaToppar toppar;\n\t\ttoppar.set_topic_partition(topic, partition);\n\t\ttoppar.set_offset(offset);\n\t\ttoppar.set_error(0);\n\t\tthis->toppar_list.add_item(std::move(toppar));\n\t}\n\n\tvoid set_api_type(int api_type)\n\t{\n\t\tthis->api_type = api_type;\n\t}\n\n\tint get_api_type() const\n\t{\n\t\treturn this->api_type;\n\t}\n\n\tvoid set_config(protocol::KafkaConfig conf)\n\t{\n\t\tthis->config = std::move(conf);\n\t}\n\n\tvoid set_partitioner(kafka_partitioner_t partitioner)\n\t{\n\t\tthis->partitioner = std::move(partitioner);\n\t}\n\n\tprotocol::KafkaResult *get_result()\n\t{\n\t\treturn &this->result;\n\t}\n\n\tconst protocol::KafkaResult *get_result() const\n\t{\n\t\treturn &this->result;\n\t}\n\n\tint get_kafka_error() const\n\t{\n\t\treturn this->kafka_error;\n\t}\n\n\tvoid set_callback(kafka_callback_t cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tWFKafkaTask(int retry_max, kafka_callback_t&& cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t\tthis->retry_max = retry_max;\n\t\tthis->finish = false;\n\t}\n\n\tvirtual ~WFKafkaTask() { }\n\n\tvirtual SubTask *done();\n\nprotected:\n\tprotocol::KafkaConfig config;\n\tprotocol::KafkaTopparList toppar_list;\n\tprotocol::KafkaMetaList meta_list;\n\tprotocol::KafkaResult result;\n\tkafka_callback_t callback;\n\tkafka_partitioner_t partitioner;\n\tint api_type;\n\tint kafka_error;\n\tint retry_max;\n\tbool finish;\n\nprivate:\n\tfriend class WFKafkaClient;\n};\n\nclass WFKafkaClient\n{\npublic:\n\t// example: kafka://10.160.23.23:9000\n\t// example: kafka://kafka.sogou\n\t// example: kafka.sogou:9090\n\t// example: kafka://10.160.23.23:9000,10.123.23.23,kafka://kafka.sogou\n\t// example: kafkas://kafka.sogou -> kafka over TLS\n\tint init(const std::string& broker_url)\n\t{\n\t\treturn this->init(broker_url, NULL);\n\t}\n\n\tint init(const std::string& broker_url, const std::string& group)\n\t{\n\t\treturn this->init(broker_url, group, NULL);\n\t}\n\n\t// With a specific SSL_CTX. Effective only on brokers over TLS.\n\tint init(const std::string& broker_url, SSL_CTX *ssl_ctx);\n\n\tint init(const std::string& broker_url, const std::string& group,\n\t\t\t SSL_CTX *ssl_ctx);\n\n\tint deinit();\n\n\t// example: topic=xxx&topic=yyy&api=fetch\n\t// example: api=commit\n\tWFKafkaTask *create_kafka_task(const std::string& query,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t kafka_callback_t cb);\n\n\tWFKafkaTask *create_kafka_task(int retry_max, kafka_callback_t cb);\n\n\tvoid set_config(protocol::KafkaConfig conf);\n\npublic:\n\t/* If you don't leavegroup manually, rebalance would be triggered */\n\tWFKafkaTask *create_leavegroup_task(int retry_max,\n\t\t\t\t\t\t\t\t\t\tkafka_callback_t callback);\n\npublic:\n\tprotocol::KafkaMetaList *get_meta_list();\n\n\tconst protocol::KafkaMetaList *get_meta_list() const;\n\nprivate:\n\tclass KafkaMember *member;\n\tfriend class KafkaClientTask;\n};\n\n#endif\n\n" }, { "path": "src/client/WFMySQLConnection.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"URIParser.h\"\n#include \"WFMySQLConnection.h\"\n\nint WFMySQLConnection::init(const std::string& url, SSL_CTX *ssl_ctx)\n{\n\tstd::string query;\n\tParsedURI uri;\n\n\tif (URIParser::parse(url, uri) >= 0)\n\t{\n\t\tif (uri.query)\n\t\t{\n\t\t\tquery = uri.query;\n\t\t\tquery += '&';\n\t\t}\n\n\t\tquery += \"transaction=INTERNAL_CONN_ID_\" + std::to_string(this->id);\n\t\tfree(uri.query);\n\t\turi.query = strdup(query.c_str());\n\t\tif (uri.query)\n\t\t{\n\t\t\tthis->uri = std::move(uri);\n\t\t\tthis->ssl_ctx = ssl_ctx;\n\t\t\treturn 0;\n\t\t}\n\t}\n\telse if (uri.state == URI_STATE_INVALID)\n\t\terrno = EINVAL;\n\n\treturn -1;\n}\n\n" }, { "path": "src/client/WFMySQLConnection.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFMYSQLCONNECTION_H_\n#define _WFMYSQLCONNECTION_H_\n\n#include \n#include \n#include \n#include \n#include \"URIParser.h\"\n#include \"WFTaskFactory.h\"\n\nclass WFMySQLConnection\n{\npublic:\n\t/* example: mysql://username:passwd@127.0.0.1/dbname?character_set=utf8\n\t * IP string is recommmended in url. When using a domain name, the first\n\t * address resovled will be used. Don't use upstream name as a host. */\n\tint init(const std::string& url)\n\t{\n\t\treturn this->init(url, NULL);\n\t}\n\n\tint init(const std::string& url, SSL_CTX *ssl_ctx);\n\n\tvoid deinit() { }\n\npublic:\n\tWFMySQLTask *create_query_task(const std::string& query,\n\t\t\t\t\t\t\t\t mysql_callback_t callback)\n\t{\n\t\tWFMySQLTask *task = WFTaskFactory::create_mysql_task(this->uri, 0,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::move(callback));\n\t\tthis->set_ssl_ctx(task);\n\t\ttask->get_req()->set_query(query);\n\t\treturn task;\n\t}\n\n\t/* If you don't disconnect manually, the TCP connection will be\n\t * kept alive after this object is deleted, and maybe reused by\n\t * another WFMySQLConnection object with same id and url. */\n\tWFMySQLTask *create_disconnect_task(mysql_callback_t callback)\n\t{\n\t\tWFMySQLTask *task = this->create_query_task(\"\", std::move(callback));\n\t\tthis->set_ssl_ctx(task);\n\t\ttask->set_keep_alive(0);\n\t\treturn task;\n\t}\n\nprotected:\n\tvoid set_ssl_ctx(WFMySQLTask *task) const\n\t{\n\t\tusing MySQLRequest = protocol::MySQLRequest;\n\t\tusing MySQLResponse = protocol::MySQLResponse;\n\t\tauto *t = (WFComplexClientTask *)task;\n\t\t/* 'ssl_ctx' can be NULL and will use default. */\n\t\tt->set_ssl_ctx(this->ssl_ctx);\n\t}\n\nprotected:\n\tParsedURI uri;\n\tSSL_CTX *ssl_ctx;\n\tint id;\n\npublic:\n\t/* Make sure that concurrent connections have different id.\n\t * When a connection object is deleted, id can be reused. */\n\tWFMySQLConnection(int id) { this->id = id; }\n\tvirtual ~WFMySQLConnection() { }\n};\n\n#endif\n\n" }, { "path": "src/client/WFRedisSubscriber.cc", "content": "/*\n Copyright (c) 2024 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"URIParser.h\"\n#include \"RedisTaskImpl.inl\"\n#include \"WFRedisSubscriber.h\"\n\nint WFRedisSubscribeTask::sync_send(const std::string& command,\n\t\t\t\t\t\t\t\t\tconst std::vector& params)\n{\n\tstd::string str(\"*\" + std::to_string(1 + params.size()) + \"\\r\\n\");\n\tint ret;\n\n\tstr += \"$\" + std::to_string(command.size()) + \"\\r\\n\" + command + \"\\r\\n\";\n\tfor (const std::string& p : params)\n\t\tstr += \"$\" + std::to_string(p.size()) + \"\\r\\n\" + p + \"\\r\\n\";\n\n\tthis->mutex.lock();\n\tif (this->task)\n\t{\n\t\tret = this->task->push(str.c_str(), str.size());\n\t\tif (ret == (int)str.size())\n\t\t\tret = 0;\n\t\telse\n\t\t{\n\t\t\tif (ret >= 0)\n\t\t\t\terrno = ENOBUFS;\n\t\t\tret = -1;\n\t\t}\n\t}\n\telse\n\t{\n\t\terrno = ENOENT;\n\t\tret = -1;\n\t}\n\n\tthis->mutex.unlock();\n\treturn ret;\n}\n\nvoid WFRedisSubscribeTask::task_extract(WFRedisTask *task)\n{\n\tauto *t = (WFRedisSubscribeTask *)task->user_data;\n\n\tif (t->extract)\n\t\tt->extract(t);\n}\n\nvoid WFRedisSubscribeTask::task_callback(WFRedisTask *task)\n{\n\tauto *t = (WFRedisSubscribeTask *)task->user_data;\n\n\tt->mutex.lock();\n\tt->task = NULL;\n\tt->mutex.unlock();\n\n\tt->state = task->get_state();\n\tt->error = task->get_error();\n\tif (t->callback)\n\t\tt->callback(t);\n\n\tt->release();\n}\n\nint WFRedisSubscriber::init(const std::string& url, SSL_CTX *ssl_ctx)\n{\n\tif (URIParser::parse(url, this->uri) >= 0)\n\t{\n\t\tthis->ssl_ctx = ssl_ctx;\n\t\treturn 0;\n\t}\n\n\tif (this->uri.state == URI_STATE_INVALID)\n\t\terrno = EINVAL;\n\n\treturn -1;\n}\n\nWFRedisTask *\nWFRedisSubscriber::create_redis_task(const std::string& command,\n\t\t\t\t\t\t\t\t\t const std::vector& params)\n{\n\tWFRedisTask *task = __WFRedisTaskFactory::create_subscribe_task(this->uri,\n\t\t\t\t\t\t\t\t\tWFRedisSubscribeTask::task_extract,\n\t\t\t\t\t\t\t\t\tWFRedisSubscribeTask::task_callback);\n\tthis->set_ssl_ctx(task);\n\ttask->get_req()->set_request(command, params);\n\treturn task;\n}\n\nWFRedisSubscribeTask *\nWFRedisSubscriber::create_subscribe_task(\n\t\t\t\t\t\tconst std::vector& channels,\n\t\t\t\t\t\textract_t extract, callback_t callback)\n{\n\tWFRedisTask *task = this->create_redis_task(\"SUBSCRIBE\", channels);\n\treturn new WFRedisSubscribeTask(task, std::move(extract),\n\t\t\t\t\t\t\t\t\tstd::move(callback));\n}\n\nWFRedisSubscribeTask *\nWFRedisSubscriber::create_psubscribe_task(\n\t\t\t\t\t\tconst std::vector& patterns,\n\t\t\t\t\t\textract_t extract, callback_t callback)\n{\n\tWFRedisTask *task = this->create_redis_task(\"PSUBSCRIBE\", patterns);\n\treturn new WFRedisSubscribeTask(task, std::move(extract),\n\t\t\t\t\t\t\t\t\tstd::move(callback));\n}\n\n" }, { "path": "src/client/WFRedisSubscriber.h", "content": "/*\n Copyright (c) 2024 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFREDISSUBSCRIBER_H_\n#define _WFREDISSUBSCRIBER_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"RedisMessage.h\"\n#include \"WFTask.h\"\n#include \"WFTaskFactory.h\"\n\nclass WFRedisSubscribeTask : public WFGenericTask\n{\npublic:\n\t/* Note: Call 'get_resp()' only in the 'extract' function or\n\t before the task is started to set response size limit. */\n\tprotocol::RedisResponse *get_resp()\n\t{\n\t\treturn this->task->get_resp();\n\t}\n\n\tconst protocol::RedisResponse *get_resp() const\n\t{\n\t\treturn this->task->get_resp();\n\t}\n\npublic:\n\t/* User needs to call 'release()' exactly once, anywhere. */\n\tvoid release()\n\t{\n\t\tif (this->flag.exchange(true))\n\t\t\tdelete this;\n\t}\n\npublic:\n\t/* Note: After 'release()' is called, all the requesting functions\n\t should not be called except in 'extract', because the task\n\t point may have been deleted because 'callback' finished. */\n\n\tint subscribe(const std::vector& channels)\n\t{\n\t\treturn this->sync_send(\"SUBSCRIBE\", channels);\n\t}\n\n\tint unsubscribe(const std::vector& channels)\n\t{\n\t\treturn this->sync_send(\"UNSUBSCRIBE\", channels);\n\t}\n\n\tint unsubscribe()\n\t{\n\t\treturn this->sync_send(\"UNSUBSCRIBE\", { });\n\t}\n\n\tint psubscribe(const std::vector& patterns)\n\t{\n\t\treturn this->sync_send(\"PSUBSCRIBE\", patterns);\n\t}\n\n\tint punsubscribe(const std::vector& patterns)\n\t{\n\t\treturn this->sync_send(\"PUNSUBSCRIBE\", patterns);\n\t}\n\n\tint punsubscribe()\n\t{\n\t\treturn this->sync_send(\"PUNSUBSCRIBE\", { });\n\t}\n\n\tint ping(const std::string& message)\n\t{\n\t\treturn this->sync_send(\"PING\", { message });\n\t}\n\n\tint ping()\n\t{\n\t\treturn this->sync_send(\"PING\", { });\n\t}\n\n\tint quit()\n\t{\n\t\treturn this->sync_send(\"QUIT\", { });\n\t}\n\npublic:\n\t/* All 'timeout' proxy functions can only be called only before\n\t the task is started or in 'extract'. */\n\n\t/* Timeout of waiting for each message. Very useful. If not set,\n\t the max waiting time will be the global 'response_timeout'. */\n\tvoid set_watch_timeout(int timeout)\n\t{\n\t\tthis->task->set_watch_timeout(timeout);\n\t}\n\n\t/* Timeout of receiving a complete message. */\n\tvoid set_receive_timeout(int timeout)\n\t{\n\t\tthis->task->set_receive_timeout(timeout);\n\t}\n\n\t/* Timeout of sending the first subscribe request. */\n\tvoid set_send_timeout(int timeout)\n\t{\n\t\tthis->task->set_send_timeout(timeout);\n\t}\n\n\t/* The default keep alive timeout is 0. If you want to keep\n\t the connection alive, make sure not to send any request\n\t after all channels/patterns were unsubscribed. */\n\tvoid set_keep_alive(int timeout)\n\t{\n\t\tthis->task->set_keep_alive(timeout);\n\t}\n\n\t/* For compatibility purpose only. */\n\tvoid set_recv_timeout(int timeout)\n\t{\n\t\tthis->set_receive_timeout(timeout);\n\t}\n\npublic:\n\t/* Call 'set_extract' or 'set_callback' only before the task\n\t is started, or in 'extract'. */\n\n\tvoid set_extract(std::function ex)\n\t{\n\t\tthis->extract = std::move(ex);\n\t}\n\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tseries_of(this)->push_front(this->task);\n\t\tthis->subtask_done();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\treturn series_of(this)->pop();\n\t}\n\nprotected:\n\tint sync_send(const std::string& command,\n\t\t\t\t const std::vector& params);\n\tstatic void task_extract(WFRedisTask *task);\n\tstatic void task_callback(WFRedisTask *task);\n\nprotected:\n\tWFRedisTask *task;\n\tstd::mutex mutex;\n\tstd::atomic flag;\n\tstd::function extract;\n\tstd::function callback;\n\nprotected:\n\tWFRedisSubscribeTask(WFRedisTask *task,\n\t\t\t\t\t\t std::function&& ex,\n\t\t\t\t\t\t std::function&& cb) :\n\t\tflag(false),\n\t\textract(std::move(ex)),\n\t\tcallback(std::move(cb))\n\t{\n\t\ttask->user_data = this;\n\t\tthis->task = task;\n\t}\n\n\tvirtual ~WFRedisSubscribeTask()\n\t{\n\t\tif (this->task)\n\t\t\tthis->task->dismiss();\n\t}\n\n\tfriend class WFRedisSubscriber;\n};\n\nclass WFRedisSubscriber\n{\npublic:\n\tint init(const std::string& url)\n\t{\n\t\treturn this->init(url, NULL);\n\t}\n\n\tint init(const std::string& url, SSL_CTX *ssl_ctx);\n\n\tvoid deinit() { }\n\npublic:\n\tusing extract_t = std::function;\n\tusing callback_t = std::function;\n\npublic:\n\tWFRedisSubscribeTask *\n\tcreate_subscribe_task(const std::vector& channels,\n\t\t\t\t\t\t extract_t extract, callback_t callback);\n\n\tWFRedisSubscribeTask *\n\tcreate_psubscribe_task(const std::vector& patterns,\n\t\t\t\t\t\t extract_t extract, callback_t callback);\n\nprotected:\n\tvoid set_ssl_ctx(WFRedisTask *task) const\n\t{\n\t\tusing RedisRequest = protocol::RedisRequest;\n\t\tusing RedisResponse = protocol::RedisResponse;\n\t\tauto *t = (WFComplexClientTask *)task;\n\t\t/* 'ssl_ctx' can be NULL and will use default. */\n\t\tt->set_ssl_ctx(this->ssl_ctx);\n\t}\n\nprotected:\n\tWFRedisTask *create_redis_task(const std::string& command,\n\t\t\t\t\t\t\t\t const std::vector& params);\n\nprotected:\n\tParsedURI uri;\n\tSSL_CTX *ssl_ctx;\n\npublic:\n\tvirtual ~WFRedisSubscriber() { }\n};\n\n#endif\n\n" }, { "path": "src/client/xmake.lua", "content": "target(\"client\")\n set_kind(\"object\")\n add_files(\"*.cc\")\n remove_files(\"WFKafkaClient.cc\")\n if not has_config(\"redis\") then\n remove_files(\"WFRedisSubscriber.cc\")\n end\n if not has_config(\"mysql\") then\n remove_files(\"WFMySQLConnection.cc\")\n end\n if not has_config(\"consul\") then\n remove_files(\"WFConsulClient.cc\")\n end\n\ntarget(\"kafka_client\")\n if has_config(\"kafka\") then\n add_files(\"WFKafkaClient.cc\")\n set_kind(\"object\")\n add_deps(\"client\")\n add_packages(\"zlib\", \"snappy\", \"zstd\", \"lz4\")\n else\n set_kind(\"phony\")\n end\n" }, { "path": "src/factory/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\nproject(factory)\n\nset(SRC\n\tWFGraphTask.cc\n\tDnsTaskImpl.cc\n\tWFTaskFactory.cc\n\tWorkflow.cc\n\tHttpTaskImpl.cc\n\tWFResourcePool.cc\n\tWFMessageQueue.cc\n\tFileTaskImpl.cc\n)\n\nif (NOT MYSQL STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tMySQLTaskImpl.cc\n\t)\nendif ()\n\nif (NOT REDIS STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tRedisTaskImpl.cc\n\t)\nendif ()\n\nadd_library(${PROJECT_NAME} OBJECT ${SRC})\n\nif (KAFKA STREQUAL \"y\")\n\tset(SRC\n\t\tKafkaTaskImpl.cc\n\t)\n\tadd_library(\"factory_kafka\" OBJECT ${SRC})\nendif ()\n" }, { "path": "src/factory/DnsTaskImpl.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"dns_types.h\"\n#include \"DnsMessage.h\"\n#include \"WFTaskError.h\"\n#include \"WFTaskFactory.h\"\n#include \"WFServer.h\"\n\nusing namespace protocol;\n\n#define DNS_KEEPALIVE_DEFAULT\t(60 * 1000)\n\n/**********Client**********/\n\nclass ComplexDnsTask : public WFComplexClientTask>\n{\n\tstatic struct addrinfo hints;\n\tstatic std::atomic seq;\n\npublic:\n\tComplexDnsTask(int retry_max, dns_callback_t&& cb):\n\t\tWFComplexClientTask(retry_max, std::move(cb))\n\t{\n\t\tthis->set_transport_type(TT_UDP);\n\t}\n\nprotected:\n\tvirtual CommMessageOut *message_out();\n\tvirtual bool init_success();\n\tvirtual bool finish_once();\n\nprivate:\n\tbool need_redirect();\n};\n\nstruct addrinfo ComplexDnsTask::hints =\n{\n\t.ai_flags = AI_NUMERICSERV | AI_NUMERICHOST,\n\t.ai_family = AF_UNSPEC,\n\t.ai_socktype = SOCK_STREAM\n};\n\nstd::atomic ComplexDnsTask::seq(0);\n\nCommMessageOut *ComplexDnsTask::message_out()\n{\n\tDnsRequest *req = this->get_req();\n\tDnsResponse *resp = this->get_resp();\n\tenum TransportType type = this->get_transport_type();\n\n\tif (req->get_id() == 0)\n\t\treq->set_id(++ComplexDnsTask::seq * 99991 % 65535 + 1);\n\tresp->set_request_id(req->get_id());\n\tresp->set_request_name(req->get_question_name());\n\treq->set_single_packet(type == TT_UDP);\n\tresp->set_single_packet(type == TT_UDP);\n\n\treturn this->WFClientTask::message_out();\n}\n\nbool ComplexDnsTask::init_success()\n{\n\tif (uri_.scheme && strcasecmp(uri_.scheme, \"dnss\") == 0)\n\t\tthis->WFComplexClientTask::set_transport_type(TT_TCP_SSL);\n\telse if (!uri_.scheme || strcasecmp(uri_.scheme, \"dns\") != 0)\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\t\treturn false;\n\t}\n\n\tif (!this->route_result_.request_object)\n\t{\n\t\tenum TransportType type = this->get_transport_type();\n\t\tstruct addrinfo *addr;\n\t\tint ret;\n\n\t\tret = getaddrinfo(uri_.host, uri_.port, &hints, &addr);\n\t\tif (ret != 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_DNS_ERROR;\n\t\t\tthis->error = ret;\n\t\t\treturn false;\n\t\t}\n\n\t\tauto *ep = &WFGlobal::get_global_settings()->dns_server_params;\n\t\tret = WFGlobal::get_route_manager()->get(type, addr, info_, ep,\n\t\t\t\t\t\t\t\t\t\t\t\t uri_.host, ssl_ctx_,\n\t\t\t\t\t\t\t\t\t\t\t\t route_result_);\n\t\tfreeaddrinfo(addr);\n\t\tif (ret < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t\treturn false;\n\t\t}\n\t}\n\n\treturn true;\n}\n\nbool ComplexDnsTask::finish_once()\n{\n\tif (this->state == WFT_STATE_SUCCESS)\n\t{\n\t\tif (need_redirect())\n\t\t\tthis->set_redirect(uri_);\n\t\telse if (this->state != WFT_STATE_SUCCESS)\n\t\t\tthis->disable_retry();\n\t}\n\n\t/* If retry times meet retry max and there is no redirect,\n\t * we ask the client for a retry or redirect.\n\t */\n\tif (retry_times_ == retry_max_ && !redirect_ && *this->get_mutable_ctx())\n\t{\n\t\t/* Reset type to UDP before a client redirect. */\n\t\tthis->set_transport_type(TT_UDP);\n\t\t(*this->get_mutable_ctx())(this);\n\t}\n\n\treturn true;\n}\n\nbool ComplexDnsTask::need_redirect()\n{\n\tDnsResponse *client_resp = this->get_resp();\n\tenum TransportType type = this->get_transport_type();\n\n\tif (type == TT_UDP && client_resp->get_tc() == 1)\n\t{\n\t\tthis->set_transport_type(TT_TCP);\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\n/**********Client Factory**********/\n\nWFDnsTask *WFTaskFactory::create_dns_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t dns_callback_t callback)\n{\n\tParsedURI uri;\n\n\tURIParser::parse(url, uri);\n\treturn WFTaskFactory::create_dns_task(uri, retry_max, std::move(callback));\n}\n\nWFDnsTask *WFTaskFactory::create_dns_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t dns_callback_t callback)\n{\n\tComplexDnsTask *task = new ComplexDnsTask(retry_max, std::move(callback));\n\tconst char *name;\n\n\tif (uri.path && uri.path[0] && uri.path[1])\n\t\tname = uri.path + 1;\n\telse\n\t\tname = \".\";\n\n\tDnsRequest *req = task->get_req();\n\treq->set_question(name, DNS_TYPE_A, DNS_CLASS_IN);\n\n\ttask->init(uri);\n\ttask->set_keep_alive(DNS_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\n\n/**********Server**********/\n\nclass WFDnsServerTask : public WFServerTask\n{\npublic:\n\tWFDnsServerTask(CommService *service,\n\t\t\t\t\tstd::function& proc) :\n\t\tWFServerTask(service, WFGlobal::get_scheduler(), proc)\n\t{\n\t\tthis->type = ((WFServerBase *)service)->get_params()->transport_type;\n\t}\n\nprotected:\n\tvirtual CommMessageIn *message_in()\n\t{\n\t\tthis->get_req()->set_single_packet(this->type == TT_UDP);\n\t\treturn this->WFServerTask::message_in();\n\t}\n\n\tvirtual CommMessageOut *message_out()\n\t{\n\t\tthis->get_resp()->set_single_packet(this->type == TT_UDP);\n\t\treturn this->WFServerTask::message_out();\n\t}\n\n\tvirtual void handle(int state, int error);\n\nprotected:\n\tenum TransportType type;\n};\n\nvoid WFDnsServerTask::handle(int state, int error)\n{\n\tif (state == WFT_STATE_TOREPLY)\n\t{\n\t\tDnsRequest *req = this->get_req();\n\t\tDnsResponse *resp = this->get_resp();\n\n\t\tresp->set_question_name(req->get_question_name());\n\t\tresp->set_question_type(req->get_question_type());\n\t\tresp->set_question_class(req->get_question_class());\n\t\tresp->set_opcode(req->get_opcode());\n\t\tresp->set_id(req->get_id());\n\t\tresp->set_rd(req->get_rd());\n\t\tresp->set_qr(1);\n\t}\n\n\treturn WFServerTask::handle(state, error);\n}\n\n/**********Server Factory**********/\n\nWFDnsTask *WFServerTaskFactory::create_dns_task(CommService *service,\n\t\t\t\t\t\tstd::function& proc)\n{\n\treturn new WFDnsServerTask(service, proc);\n}\n\n" }, { "path": "src/factory/FileTaskImpl.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Li Jinghao (lijinghao@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"WFGlobal.h\"\n#include \"WFTaskFactory.h\"\n\nclass WFFilepreadTask : public WFFileIOTask\n{\npublic:\n\tWFFilepreadTask(int fd, void *buf, size_t count, off_t offset,\n\t\t\t\t\tIOService *service, fio_callback_t&& cb) :\n\t\tWFFileIOTask(service, std::move(cb))\n\t{\n\t\tthis->args.fd = fd;\n\t\tthis->args.buf = buf;\n\t\tthis->args.count = count;\n\t\tthis->args.offset = offset;\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->prep_pread(this->args.fd, this->args.buf, this->args.count,\n\t\t\t\t\t\t this->args.offset);\n\t\treturn 0;\n\t}\n};\n\nclass WFFilepwriteTask : public WFFileIOTask\n{\npublic:\n\tWFFilepwriteTask(int fd, const void *buf, size_t count, off_t offset,\n\t\t\t\t\t IOService *service, fio_callback_t&& cb) :\n\t\tWFFileIOTask(service, std::move(cb))\n\t{\n\t\tthis->args.fd = fd;\n\t\tthis->args.buf = (void *)buf;\n\t\tthis->args.count = count;\n\t\tthis->args.offset = offset;\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->prep_pwrite(this->args.fd, this->args.buf, this->args.count,\n\t\t\t\t\t\t this->args.offset);\n\t\treturn 0;\n\t}\n};\n\nclass WFFilepreadvTask : public WFFileVIOTask\n{\npublic:\n\tWFFilepreadvTask(int fd, const struct iovec *iov, int iovcnt, off_t offset,\n\t\t\t\t\t IOService *service, fvio_callback_t&& cb) :\n\t\tWFFileVIOTask(service, std::move(cb))\n\t{\n\t\tthis->args.fd = fd;\n\t\tthis->args.iov = iov;\n\t\tthis->args.iovcnt = iovcnt;\n\t\tthis->args.offset = offset;\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->prep_preadv(this->args.fd, this->args.iov, this->args.iovcnt,\n\t\t\t\t\t\t this->args.offset);\n\t\treturn 0;\n\t}\n};\n\nclass WFFilepwritevTask : public WFFileVIOTask\n{\npublic:\n\tWFFilepwritevTask(int fd, const struct iovec *iov, int iovcnt, off_t offset,\n\t\t\t\t\t IOService *service, fvio_callback_t&& cb) :\n\t\tWFFileVIOTask(service, std::move(cb))\n\t{\n\t\tthis->args.fd = fd;\n\t\tthis->args.iov = iov;\n\t\tthis->args.iovcnt = iovcnt;\n\t\tthis->args.offset = offset;\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->prep_pwritev(this->args.fd, this->args.iov, this->args.iovcnt,\n\t\t\t\t\t\t this->args.offset);\n\t\treturn 0;\n\t}\n};\n\nclass WFFilefsyncTask : public WFFileSyncTask\n{\npublic:\n\tWFFilefsyncTask(int fd, IOService *service, fsync_callback_t&& cb) :\n\t\tWFFileSyncTask(service, std::move(cb))\n\t{\n\t\tthis->args.fd = fd;\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->prep_fsync(this->args.fd);\n\t\treturn 0;\n\t}\n};\n\nclass WFFilefdatasyncTask : public WFFileSyncTask\n{\npublic:\n\tWFFilefdatasyncTask(int fd, IOService *service, fsync_callback_t&& cb) :\n\t\tWFFileSyncTask(service, std::move(cb))\n\t{\n\t\tthis->args.fd = fd;\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->prep_fdatasync(this->args.fd);\n\t\treturn 0;\n\t}\n};\n\n/* File tasks created with path name. */\n\nclass __WFFilepreadTask : public WFFilepreadTask\n{\npublic:\n\t__WFFilepreadTask(const std::string& path, void *buf, size_t count,\n\t\t\t\t\t off_t offset, IOService *service, fio_callback_t&& cb):\n\t\tWFFilepreadTask(-1, buf, count, offset, service, std::move(cb)),\n\t\tpathname(path)\n\t{\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->args.fd = open(this->pathname.c_str(), O_RDONLY);\n\t\tif (this->args.fd < 0)\n\t\t\treturn -1;\n\n\t\treturn WFFilepreadTask::prepare();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tif (this->args.fd >= 0)\n\t\t{\n\t\t\tclose(this->args.fd);\n\t\t\tthis->args.fd = -1;\n\t\t}\n\n\t\treturn WFFilepreadTask::done();\n\t}\n\nprotected:\n\tstd::string pathname;\n};\n\nclass __WFFilepwriteTask : public WFFilepwriteTask\n{\npublic:\n\t__WFFilepwriteTask(const std::string& path, const void *buf, size_t count,\n\t\t\t\t\t off_t offset, IOService *service, fio_callback_t&& cb):\n\t\tWFFilepwriteTask(-1, buf, count, offset, service, std::move(cb)),\n\t\tpathname(path)\n\t{\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->args.fd = open(this->pathname.c_str(), O_WRONLY | O_CREAT, 0644);\n\t\tif (this->args.fd < 0)\n\t\t\treturn -1;\n\n\t\treturn WFFilepwriteTask::prepare();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tif (this->args.fd >= 0)\n\t\t{\n\t\t\tclose(this->args.fd);\n\t\t\tthis->args.fd = -1;\n\t\t}\n\n\t\treturn WFFilepwriteTask::done();\n\t}\n\nprotected:\n\tstd::string pathname;\n};\n\nclass __WFFilepreadvTask : public WFFilepreadvTask\n{\npublic:\n\t__WFFilepreadvTask(const std::string& path, const struct iovec *iov,\n\t\t\t\t\t int iovcnt, off_t offset, IOService *service,\n\t\t\t\t\t fvio_callback_t&& cb) :\n\t\tWFFilepreadvTask(-1, iov, iovcnt, offset, service, std::move(cb)),\n\t\tpathname(path)\n\t{\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->args.fd = open(this->pathname.c_str(), O_RDONLY);\n\t\tif (this->args.fd < 0)\n\t\t\treturn -1;\n\n\t\treturn WFFilepreadvTask::prepare();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tif (this->args.fd >= 0)\n\t\t{\n\t\t\tclose(this->args.fd);\n\t\t\tthis->args.fd = -1;\n\t\t}\n\n\t\treturn WFFilepreadvTask::done();\n\t}\n\nprotected:\n\tstd::string pathname;\n};\n\nclass __WFFilepwritevTask : public WFFilepwritevTask\n{\npublic:\n\t__WFFilepwritevTask(const std::string& path, const struct iovec *iov,\n\t\t\t\t\t\tint iovcnt, off_t offset, IOService *service,\n\t\t\t\t\t\tfvio_callback_t&& cb) :\n\t\tWFFilepwritevTask(-1, iov, iovcnt, offset, service, std::move(cb)),\n\t\tpathname(path)\n\t{\n\t}\n\nprotected:\n\tvirtual int prepare()\n\t{\n\t\tthis->args.fd = open(this->pathname.c_str(), O_WRONLY | O_CREAT, 0644);\n\t\tif (this->args.fd < 0)\n\t\t\treturn -1;\n\n\t\treturn WFFilepwritevTask::prepare();\n\t}\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tif (this->args.fd >= 0)\n\t\t{\n\t\t\tclose(this->args.fd);\n\t\t\tthis->args.fd = -1;\n\t\t}\n\n\t\treturn WFFilepwritevTask::done();\n\t}\n\nprotected:\n\tstd::string pathname;\n};\n\n/* Factory functions with fd. */\n\nWFFileIOTask *WFTaskFactory::create_pread_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t void *buf,\n\t\t\t\t\t\t\t\t\t\t\t size_t count,\n\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t fio_callback_t callback)\n{\n\treturn new WFFilepreadTask(fd, buf, count, offset,\n\t\t\t\t\t\t\t WFGlobal::get_io_service(),\n\t\t\t\t\t\t\t std::move(callback));\n}\n\nWFFileIOTask *WFTaskFactory::create_pwrite_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t\tconst void *buf,\n\t\t\t\t\t\t\t\t\t\t\t\tsize_t count,\n\t\t\t\t\t\t\t\t\t\t\t\toff_t offset,\n\t\t\t\t\t\t\t\t\t\t\t\tfio_callback_t callback)\n{\n\treturn new WFFilepwriteTask(fd, buf, count, offset,\n\t\t\t\t\t\t\t\tWFGlobal::get_io_service(),\n\t\t\t\t\t\t\t\tstd::move(callback));\n}\n\nWFFileVIOTask *WFTaskFactory::create_preadv_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t\t const struct iovec *iovec,\n\t\t\t\t\t\t\t\t\t\t\t\t int iovcnt,\n\t\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t\t fvio_callback_t callback)\n{\n\treturn new WFFilepreadvTask(fd, iovec, iovcnt, offset,\n\t\t\t\t\t\t\t\tWFGlobal::get_io_service(),\n\t\t\t\t\t\t\t\tstd::move(callback));\n}\n\nWFFileVIOTask *WFTaskFactory::create_pwritev_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t\t const struct iovec *iovec,\n\t\t\t\t\t\t\t\t\t\t\t\t int iovcnt,\n\t\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t\t fvio_callback_t callback)\n{\n\treturn new WFFilepwritevTask(fd, iovec, iovcnt, offset,\n\t\t\t\t\t\t\t\t WFGlobal::get_io_service(),\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\nWFFileSyncTask *WFTaskFactory::create_fsync_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t\t fsync_callback_t callback)\n{\n\treturn new WFFilefsyncTask(fd,\n\t\t\t\t\t\t\t WFGlobal::get_io_service(),\n\t\t\t\t\t\t\t std::move(callback));\n}\n\nWFFileSyncTask *WFTaskFactory::create_fdatasync_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t\t\t fsync_callback_t callback)\n{\n\treturn new WFFilefdatasyncTask(fd,\n\t\t\t\t\t\t\t\t WFGlobal::get_io_service(),\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n/* Factory functions with path name. */\n\nWFFileIOTask *WFTaskFactory::create_pread_task(const std::string& path,\n\t\t\t\t\t\t\t\t\t\t\t void *buf,\n\t\t\t\t\t\t\t\t\t\t\t size_t count,\n\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t fio_callback_t callback)\n{\n\treturn new __WFFilepreadTask(path, buf, count, offset,\n\t\t\t\t\t\t\t\t WFGlobal::get_io_service(),\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\nWFFileIOTask *WFTaskFactory::create_pwrite_task(const std::string& path,\n\t\t\t\t\t\t\t\t\t\t\t\tconst void *buf,\n\t\t\t\t\t\t\t\t\t\t\t\tsize_t count,\n\t\t\t\t\t\t\t\t\t\t\t\toff_t offset,\n\t\t\t\t\t\t\t\t\t\t\t\tfio_callback_t callback)\n{\n\treturn new __WFFilepwriteTask(path, buf, count, offset,\n\t\t\t\t\t\t\t\t WFGlobal::get_io_service(),\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\nWFFileVIOTask *WFTaskFactory::create_preadv_task(const std::string& path,\n\t\t\t\t\t\t\t\t\t\t\t\t const struct iovec *iovec,\n\t\t\t\t\t\t\t\t\t\t\t\t int iovcnt,\n\t\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t\t fvio_callback_t callback)\n{\n\treturn new __WFFilepreadvTask(path, iovec, iovcnt, offset,\n\t\t\t\t\t\t\t\t WFGlobal::get_io_service(),\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\nWFFileVIOTask *WFTaskFactory::create_pwritev_task(const std::string& path,\n\t\t\t\t\t\t\t\t\t\t\t\t const struct iovec *iovec,\n\t\t\t\t\t\t\t\t\t\t\t\t int iovcnt,\n\t\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t\t fvio_callback_t callback)\n{\n\treturn new __WFFilepwritevTask(path, iovec, iovcnt, offset,\n\t\t\t\t\t\t\t\t WFGlobal::get_io_service(),\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n" }, { "path": "src/factory/HttpTaskImpl.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Liu Kai (liukaidx@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"WFTaskError.h\"\n#include \"WFTaskFactory.h\"\n#include \"StringUtil.h\"\n#include \"WFGlobal.h\"\n#include \"HttpUtil.h\"\n#include \"SSLWrapper.h\"\n#include \"PackageWrapper.h\"\n#include \"HttpTaskImpl.inl\"\n\nusing namespace protocol;\n\n#define HTTP_KEEPALIVE_DEFAULT\t(60 * 1000)\n#define HTTP_KEEPALIVE_MAX\t\t(300 * 1000)\n\n/**********Client**********/\n\nstatic int __encode_auth(const char *p, std::string& auth)\n{\n\tsize_t len = strlen(p);\n\tsize_t base64_len = (len + 2) / 3 * 4;\n\tchar *base64 = (char *)malloc(base64_len + 1);\n\n\tif (!base64)\n\t\treturn -1;\n\n\tEVP_EncodeBlock((unsigned char *)base64, (const unsigned char *)p, len);\n\tauth.append(\"Basic \");\n\tauth.append(base64, base64_len);\n\n\tfree(base64);\n\treturn 0;\n}\n\nclass ComplexHttpTask : public WFComplexClientTask\n{\npublic:\n\tComplexHttpTask(int redirect_max,\n\t\t\t\t\tint retry_max,\n\t\t\t\t\thttp_callback_t&& callback):\n\t\tWFComplexClientTask(retry_max, std::move(callback)),\n\t\tredirect_max_(redirect_max),\n\t\tredirect_count_(0)\n\t{\n\t\tHttpRequest *client_req = this->get_req();\n\n\t\tclient_req->set_method(HttpMethodGet);\n\t\tclient_req->set_http_version(\"HTTP/1.1\");\n\t}\n\nprotected:\n\tvirtual CommMessageOut *message_out();\n\tvirtual CommMessageIn *message_in();\n\tvirtual int keep_alive_timeout();\n\tvirtual bool init_success();\n\tvirtual void init_failed();\n\tvirtual bool finish_once();\n\nprotected:\n\tbool need_redirect(const ParsedURI& uri, ParsedURI& new_uri);\n\tbool redirect_url(HttpResponse *client_resp,\n\t\t\t\t\t const ParsedURI& uri, ParsedURI& new_uri);\n\tvoid set_empty_request();\n\tvoid check_response();\n\nprivate:\n\tint redirect_max_;\n\tint redirect_count_;\n};\n\nCommMessageOut *ComplexHttpTask::message_out()\n{\n\tHttpRequest *req = this->get_req();\n\tstruct HttpMessageHeader header;\n\tbool is_alive;\n\n\tif (!req->is_chunked() && !req->has_content_length_header())\n\t{\n\t\tsize_t body_size = req->get_output_body_size();\n\t\tconst char *method = req->get_method();\n\n\t\tif (body_size != 0 || strcmp(method, \"POST\") == 0 ||\n\t\t\t\t\t\t\t strcmp(method, \"PUT\") == 0)\n\t\t{\n\t\t\tchar buf[32];\n\t\t\theader.name = \"Content-Length\";\n\t\t\theader.name_len = strlen(\"Content-Length\");\n\t\t\theader.value = buf;\n\t\t\theader.value_len = sprintf(buf, \"%zu\", body_size);\n\t\t\treq->add_header(&header);\n\t\t}\n\t}\n\n\tif (req->has_connection_header())\n\t\tis_alive = req->is_keep_alive();\n\telse\n\t{\n\t\theader.name = \"Connection\";\n\t\theader.name_len = strlen(\"Connection\");\n\t\tis_alive = (this->keep_alive_timeo != 0);\n\t\tif (is_alive)\n\t\t{\n\t\t\theader.value = \"Keep-Alive\";\n\t\t\theader.value_len = strlen(\"Keep-Alive\");\n\t\t}\n\t\telse\n\t\t{\n\t\t\theader.value = \"close\";\n\t\t\theader.value_len = strlen(\"close\");\n\t\t}\n\n\t\treq->add_header(&header);\n\t}\n\n\tif (!is_alive)\n\t\tthis->keep_alive_timeo = 0;\n\telse if (req->has_keep_alive_header())\n\t{\n\t\tHttpHeaderCursor cursor(req);\n\n\t\t//req---Connection: Keep-Alive\n\t\t//req---Keep-Alive: timeout=0,max=100\n\t\theader.name = \"Keep-Alive\";\n\t\theader.name_len = strlen(\"Keep-Alive\");\n\t\theader.value = NULL;\n\t\theader.value_len = 0;\n\t\tif (cursor.find(&header))\n\t\t{\n\t\t\tstd::string keep_alive((const char *)header.value, header.value_len);\n\t\t\tstd::vector params = StringUtil::split(keep_alive, ',');\n\n\t\t\tfor (const auto& kv : params)\n\t\t\t{\n\t\t\t\tstd::vector arr = StringUtil::split(kv, '=');\n\t\t\t\tif (arr.size() < 2)\n\t\t\t\t\tarr.emplace_back(\"0\");\n\n\t\t\t\tstd::string key = StringUtil::strip(arr[0]);\n\t\t\t\tstd::string val = StringUtil::strip(arr[1]);\n\t\t\t\tif (strcasecmp(key.c_str(), \"timeout\") == 0)\n\t\t\t\t{\n\t\t\t\t\tthis->keep_alive_timeo = 1000 * atoi(val.c_str());\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif ((unsigned int)this->keep_alive_timeo > HTTP_KEEPALIVE_MAX)\n\t\t\tthis->keep_alive_timeo = HTTP_KEEPALIVE_MAX;\n\t}\n\n\treturn this->WFComplexClientTask::message_out();\n}\n\nCommMessageIn *ComplexHttpTask::message_in()\n{\n\tHttpResponse *resp = this->get_resp();\n\n\tif (strcmp(this->get_req()->get_method(), HttpMethodHead) == 0)\n\t\tresp->parse_zero_body();\n\n\treturn this->WFComplexClientTask::message_in();\n}\n\nint ComplexHttpTask::keep_alive_timeout()\n{\n\treturn this->resp.is_keep_alive() ? this->keep_alive_timeo : 0;\n}\n\nvoid ComplexHttpTask::set_empty_request()\n{\n\tHttpRequest *client_req = this->get_req();\n\tHttpHeaderCursor cursor(client_req);\n\tstruct HttpMessageHeader header = {\n\t\t.name\t\t=\t\"Host\",\n\t\t.name_len\t=\tstrlen(\"Host\"),\n\t};\n\n\tclient_req->set_request_uri(\"/\");\n\tcursor.find_and_erase(&header);\n\n\theader.name = \"Authorization\";\n\theader.name_len = strlen(\"Authorization\");\n\tcursor.find_and_erase(&header);\n}\n\nvoid ComplexHttpTask::init_failed()\n{\n\tthis->set_empty_request();\n}\n\nbool ComplexHttpTask::init_success()\n{\n\tHttpRequest *client_req = this->get_req();\n\tstd::string request_uri;\n\tstd::string header_host;\n\tbool is_ssl;\n\n\tif (uri_.scheme && strcasecmp(uri_.scheme, \"http\") == 0)\n\t\tis_ssl = false;\n\telse if (uri_.scheme && strcasecmp(uri_.scheme, \"https\") == 0)\n\t\tis_ssl = true;\n\telse\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\t\treturn false;\n\t}\n\n\t//todo http+unix\n\t//https://stackoverflow.com/questions/26964595/whats-the-correct-way-to-use-a-unix-domain-socket-in-requests-framework\n\t//https://stackoverflow.com/questions/27037990/connecting-to-postgres-via-database-url-and-unix-socket-in-rails\n\n\tif (uri_.path && uri_.path[0])\n\t\trequest_uri = uri_.path;\n\telse\n\t\trequest_uri = \"/\";\n\n\tif (uri_.query && uri_.query[0])\n\t{\n\t\trequest_uri += \"?\";\n\t\trequest_uri += uri_.query;\n\t}\n\n\tif (uri_.host && uri_.host[0])\n\t\theader_host = uri_.host;\n\n\tif (uri_.port && uri_.port[0])\n\t{\n\t\tint port = atoi(uri_.port);\n\n\t\tif (is_ssl)\n\t\t{\n\t\t\tif (port != 443)\n\t\t\t{\n\t\t\t\theader_host += \":\";\n\t\t\t\theader_host += uri_.port;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (port != 80)\n\t\t\t{\n\t\t\t\theader_host += \":\";\n\t\t\t\theader_host += uri_.port;\n\t\t\t}\n\t\t}\n\t}\n\n\tthis->WFComplexClientTask::set_transport_type(is_ssl ? TT_TCP_SSL : TT_TCP);\n\tclient_req->set_request_uri(request_uri.c_str());\n\tclient_req->set_header_pair(\"Host\", header_host.c_str());\n\n\tif (uri_.userinfo && uri_.userinfo[0])\n\t{\n\t\tstd::string userinfo(uri_.userinfo);\n\t\tstd::string http_auth;\n\n\t\tStringUtil::url_decode(userinfo);\n\n\t\tif (__encode_auth(userinfo.c_str(), http_auth) < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t\treturn false;\n\t\t}\n\n\t\tclient_req->set_header_pair(\"Authorization\", http_auth.c_str());\n\t}\n\n\treturn true;\n}\n\nbool ComplexHttpTask::redirect_url(HttpResponse *client_resp,\n\t\t\t\t\t\t\t\t const ParsedURI& uri, ParsedURI& new_uri)\n{\n\tif (redirect_count_ < redirect_max_)\n\t{\n\t\tredirect_count_++;\n\t\tstd::string url;\n\t\tHttpHeaderCursor cursor(client_resp);\n\n\t\tif (!cursor.find(\"Location\", url) || url.empty())\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_HTTP_BAD_REDIRECT_HEADER;\n\t\t\treturn false;\n\t\t}\n\n\t\tif (url[0] == '/')\n\t\t{\n\t\t\tif (url[1] != '/')\n\t\t\t{\n\t\t\t\tif (uri.port)\n\t\t\t\t\turl = ':' + (uri.port + url);\n\n\t\t\t\turl = \"//\" + (uri.host + url);\n\t\t\t}\n\n\t\t\turl = uri.scheme + (':' + url);\n\t\t}\n\n\t\tURIParser::parse(url, new_uri);\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nbool ComplexHttpTask::need_redirect(const ParsedURI& uri, ParsedURI& new_uri)\n{\n\tHttpRequest *client_req = this->get_req();\n\tHttpResponse *client_resp = this->get_resp();\n\tconst char *status_code_str = client_resp->get_status_code();\n\tconst char *method = client_req->get_method();\n\n\tif (!status_code_str || !method)\n\t\treturn false;\n\n\tint status_code = atoi(status_code_str);\n\n\tswitch (status_code)\n\t{\n\tcase 301:\n\tcase 302:\n\tcase 303:\n\t\tif (redirect_url(client_resp, uri, new_uri))\n\t\t{\n\t\t\tif (strcasecmp(method, HttpMethodGet) != 0 &&\n\t\t\t\tstrcasecmp(method, HttpMethodHead) != 0)\n\t\t\t{\n\t\t\t\tclient_req->set_method(HttpMethodGet);\n\t\t\t}\n\n\t\t\treturn true;\n\t\t}\n\t\telse\n\t\t\tbreak;\n\n\tcase 307:\n\tcase 308:\n\t\tif (redirect_url(client_resp, uri, new_uri))\n\t\t\treturn true;\n\t\telse\n\t\t\tbreak;\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\treturn false;\n}\n\nvoid ComplexHttpTask::check_response()\n{\n\tHttpResponse *resp = this->get_resp();\n\n\tresp->end_parsing();\n\tif (this->state == WFT_STATE_SYS_ERROR && this->error == ECONNRESET)\n\t{\n\t\t/* Servers can end the message by closing the connection. */\n\t\tif (resp->is_header_complete() && !resp->is_chunked() &&\n\t\t\t!resp->has_content_length_header())\n\t\t{\n\t\t\tthis->state = WFT_STATE_SUCCESS;\n\t\t\tthis->error = 0;\n\t\t}\n\t}\n}\n\nbool ComplexHttpTask::finish_once()\n{\n\tif (this->state != WFT_STATE_SUCCESS)\n\t\tthis->check_response();\n\n\tif (this->state == WFT_STATE_SUCCESS)\n\t{\n\t\tParsedURI new_uri;\n\t\tif (this->need_redirect(uri_, new_uri))\n\t\t{\n\t\t\tif (uri_.userinfo && strcasecmp(uri_.host, new_uri.host) == 0)\n\t\t\t{\n\t\t\t\tif (!new_uri.userinfo)\n\t\t\t\t{\n\t\t\t\t\tnew_uri.userinfo = uri_.userinfo;\n\t\t\t\t\turi_.userinfo = NULL;\n\t\t\t\t}\n\t\t\t}\n\t\t\telse if (uri_.userinfo)\n\t\t\t{\n\t\t\t\tHttpRequest *client_req = this->get_req();\n\t\t\t\tHttpHeaderCursor cursor(client_req);\n\t\t\t\tstruct HttpMessageHeader header = {\n\t\t\t\t\t.name = \"Authorization\",\n\t\t\t\t\t.name_len = strlen(\"Authorization\")\n\t\t\t\t};\n\n\t\t\t\tcursor.find_and_erase(&header);\n\t\t\t}\n\n\t\t\tthis->set_redirect(new_uri);\n\t\t}\n\t\telse if (this->state != WFT_STATE_SUCCESS)\n\t\t\tthis->disable_retry();\n\t}\n\n\treturn true;\n}\n\n/*******Proxy Client*******/\n\nstatic SSL *__create_ssl(SSL_CTX *ssl_ctx)\n{\n\tBIO *wbio;\n\tBIO *rbio;\n\tSSL *ssl;\n\n\trbio = BIO_new(BIO_s_mem());\n\tif (rbio)\n\t{\n\t\twbio = BIO_new(BIO_s_mem());\n\t\tif (wbio)\n\t\t{\n\t\t\tssl = SSL_new(ssl_ctx);\n\t\t\tif (ssl)\n\t\t\t{\n\t\t\t\tSSL_set_bio(ssl, rbio, wbio);\n\t\t\t\treturn ssl;\n\t\t\t}\n\n\t\t\tBIO_free(wbio);\n\t\t}\n\n\t\tBIO_free(rbio);\n\t}\n\n\treturn NULL;\n}\n\nclass ComplexHttpProxyTask : public ComplexHttpTask\n{\npublic:\n\tComplexHttpProxyTask(int redirect_max,\n\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t http_callback_t&& callback):\n\t\tComplexHttpTask(redirect_max, retry_max, std::move(callback)),\n\t\tis_user_request_(true)\n\t{ }\n\n\tvoid set_user_uri(ParsedURI&& uri) { user_uri_ = std::move(uri); }\n\tvoid set_user_uri(const ParsedURI& uri) { user_uri_ = uri; }\n\n\tvirtual const ParsedURI *get_current_uri() const { return &user_uri_; }\n\nprotected:\n\tvirtual CommMessageOut *message_out();\n\tvirtual CommMessageIn *message_in();\n\tvirtual int keep_alive_timeout();\n\tvirtual int first_timeout();\n\tvirtual bool init_success();\n\tvirtual bool finish_once();\n\nprotected:\n\tvirtual WFConnection *get_connection() const\n\t{\n\t\tWFConnection *conn = this->ComplexHttpTask::get_connection();\n\n\t\tif (conn && is_ssl_)\n\t\t\treturn (SSLConnection *)conn->get_context();\n\n\t\treturn conn;\n\t}\n\nprivate:\n\tstruct SSLConnection : public WFConnection\n\t{\n\t\tSSL *ssl;\n\t\tSSLHandshaker handshaker;\n\t\tSSLWrapper wrapper;\n\t\tSSLConnection(SSL *ssl) : handshaker(ssl), wrapper(&wrapper, ssl)\n\t\t{\n\t\t\tthis->ssl = ssl;\n\t\t}\n\t};\n\n\tSSLHandshaker *get_ssl_handshaker() const\n\t{\n\t\treturn &((SSLConnection *)this->get_connection())->handshaker;\n\t}\n\n\tSSLWrapper *get_ssl_wrapper(ProtocolMessage *msg) const\n\t{\n\t\tSSLConnection *conn = (SSLConnection *)this->get_connection();\n\t\tconn->wrapper = SSLWrapper(msg, conn->ssl);\n\t\treturn &conn->wrapper;\n\t}\n\n\tint init_ssl_connection();\n\n\tstd::string proxy_auth_;\n\tParsedURI user_uri_;\n\tbool is_ssl_;\n\tbool is_user_request_;\n\tshort state_;\n\tint error_;\n};\n\nint ComplexHttpProxyTask::init_ssl_connection()\n{\n\tstatic SSL_CTX *ssl_ctx = WFGlobal::get_ssl_client_ctx();\n\tSSL *ssl = __create_ssl(ssl_ctx_ ? ssl_ctx_ : ssl_ctx);\n\tWFConnection *conn;\n\n\tif (!ssl)\n\t\treturn -1;\n\n\tSSL_set_tlsext_host_name(ssl, user_uri_.host);\n\tSSL_set_connect_state(ssl);\n\n\tconn = this->ComplexHttpTask::get_connection();\n\tSSLConnection *ssl_conn = new SSLConnection(ssl);\n\n\tauto&& deleter = [] (void *ctx)\n\t{\n\t\tSSLConnection *ssl_conn = (SSLConnection *)ctx;\n\t\tSSL_free(ssl_conn->ssl);\n\t\tdelete ssl_conn;\n\t};\n\tconn->set_context(ssl_conn, std::move(deleter));\n\treturn 0;\n}\n\nCommMessageOut *ComplexHttpProxyTask::message_out()\n{\n\tlong long seqid = this->get_seq();\n\n\tif (seqid == 0) // CONNECT\n\t{\n\t\tHttpRequest *conn_req = new HttpRequest;\n\t\tstd::string request_uri(user_uri_.host);\n\n\t\trequest_uri += \":\";\n\t\tif (user_uri_.port)\n\t\t\trequest_uri += user_uri_.port;\n\t\telse\n\t\t\trequest_uri += is_ssl_ ? \"443\" : \"80\";\n\n\t\tconn_req->set_method(\"CONNECT\");\n\t\tconn_req->set_request_uri(request_uri);\n\t\tconn_req->set_http_version(\"HTTP/1.1\");\n\t\tconn_req->add_header_pair(\"Host\", request_uri.c_str());\n\n\t\tif (!proxy_auth_.empty())\n\t\t\tconn_req->add_header_pair(\"Proxy-Authorization\", proxy_auth_);\n\n\t\tis_user_request_ = false;\n\t\treturn conn_req;\n\t}\n\telse if (seqid == 1 && is_ssl_) // HANDSHAKE\n\t{\n\t\tis_user_request_ = false;\n\t\treturn get_ssl_handshaker();\n\t}\n\n\tauto *msg = (ProtocolMessage *)this->ComplexHttpTask::message_out();\n\treturn is_ssl_ ? get_ssl_wrapper(msg) : msg;\n}\n\nCommMessageIn *ComplexHttpProxyTask::message_in()\n{\n\tlong long seqid = this->get_seq();\n\n\tif (seqid == 0)\n\t{\n\t\tHttpResponse *conn_resp = new HttpResponse;\n\t\tconn_resp->parse_zero_body();\n\t\treturn conn_resp;\n\t}\n\telse if (seqid == 1 && is_ssl_)\n\t\treturn get_ssl_handshaker();\n\n\tauto *msg = (ProtocolMessage *)this->ComplexHttpTask::message_in();\n\treturn is_ssl_ ? get_ssl_wrapper(msg) : msg;\n}\n\nint ComplexHttpProxyTask::keep_alive_timeout()\n{\n\tlong long seqid = this->get_seq();\n\n\tstate_ = WFT_STATE_SUCCESS;\n\terror_ = 0;\n\tif (seqid == 0)\n\t{\n\t\tHttpResponse *resp = this->get_resp();\n\t\tconst char *code_str;\n\t\tint status_code;\n\n\t\t*resp = std::move(*(HttpResponse *)this->get_message_in());\n\t\tcode_str = resp->get_status_code();\n\t\tstatus_code = code_str ? atoi(code_str) : 0;\n\n\t\tswitch (status_code)\n\t\t{\n\t\tcase 200:\n\t\t\tbreak;\n\t\tcase 407:\n\t\t\tthis->disable_retry();\n\t\tdefault:\n\t\t\tstate_ = WFT_STATE_TASK_ERROR;\n\t\t\terror_ = WFT_ERR_HTTP_PROXY_CONNECT_FAILED;\n\t\t\treturn 0;\n\t\t}\n\n\t\tthis->clear_resp();\n\n\t\tif (is_ssl_ && init_ssl_connection() < 0)\n\t\t{\n\t\t\tstate_ = WFT_STATE_SYS_ERROR;\n\t\t\terror_ = errno;\n\t\t\treturn 0;\n\t\t}\n\n\t\treturn HTTP_KEEPALIVE_DEFAULT;\n\t}\n\telse if (seqid == 1 && is_ssl_)\n\t\treturn HTTP_KEEPALIVE_DEFAULT;\n\n\treturn this->ComplexHttpTask::keep_alive_timeout();\n}\n\nint ComplexHttpProxyTask::first_timeout()\n{\n\treturn is_user_request_ ? this->watch_timeo : 0;\n}\n\nbool ComplexHttpProxyTask::init_success()\n{\n\tif (!uri_.scheme || strcasecmp(uri_.scheme, \"http\") != 0)\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\t\treturn false;\n\t}\n\n\tif (user_uri_.state == URI_STATE_ERROR)\n\t{\n\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\tthis->error = uri_.error;\n\t\treturn false;\n\t}\n\telse if (user_uri_.state != URI_STATE_SUCCESS)\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_PARSE_FAILED;\n\t\treturn false;\n\t}\n\n\tif (user_uri_.scheme && strcasecmp(user_uri_.scheme, \"http\") == 0)\n\t\tis_ssl_ = false;\n\telse if (user_uri_.scheme && strcasecmp(user_uri_.scheme, \"https\") == 0)\n\t\tis_ssl_ = true;\n\telse\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\t\treturn false;\n\t}\n\n\tint user_port;\n\tif (user_uri_.port)\n\t{\n\t\tuser_port = atoi(user_uri_.port);\n\t\tif (user_port <= 0 || user_port > 65535)\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_URI_PORT_INVALID;\n\t\t\treturn false;\n\t\t}\n\t}\n\telse\n\t\tuser_port = is_ssl_ ? 443 : 80;\n\n\tstd::string info(\"http-proxy|remote:\");\n\tinfo += is_ssl_ ? \"https://\" : \"http://\";\n\tinfo += user_uri_.host;\n\tinfo += \":\";\n\tif (user_uri_.port)\n\t\tinfo += user_uri_.port;\n\telse\n\t\tinfo += is_ssl_ ? \"443\" : \"80\";\n\n\tif (uri_.userinfo && uri_.userinfo[0])\n\t{\n\t\tstd::string userinfo(uri_.userinfo);\n\n\t\tStringUtil::url_decode(userinfo);\n\t\tproxy_auth_.clear();\n\n\t\tif (__encode_auth(userinfo.c_str(), proxy_auth_) < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t\treturn false;\n\t\t}\n\n\t\tinfo += \"|auth:\";\n\t\tinfo += proxy_auth_;\n\t}\n\n\tthis->WFComplexClientTask::set_info(info);\n\n\tstd::string request_uri;\n\tstd::string header_host;\n\n\tif (user_uri_.path && user_uri_.path[0])\n\t\trequest_uri = user_uri_.path;\n\telse\n\t\trequest_uri = \"/\";\n\n\tif (user_uri_.query && user_uri_.query[0])\n\t{\n\t\trequest_uri += \"?\";\n\t\trequest_uri += user_uri_.query;\n\t}\n\n\tif (user_uri_.host && user_uri_.host[0])\n\t\theader_host = user_uri_.host;\n\n\tif ((is_ssl_ && user_port != 443) || (!is_ssl_ && user_port != 80))\n\t{\n\t\theader_host += \":\";\n\t\theader_host += uri_.port;\n\t}\n\n\tHttpRequest *client_req = this->get_req();\n\tclient_req->set_request_uri(request_uri.c_str());\n\tclient_req->set_header_pair(\"Host\", header_host.c_str());\n\tthis->WFComplexClientTask::set_transport_type(TT_TCP);\n\n\tif (user_uri_.userinfo && user_uri_.userinfo[0])\n\t{\n\t\tstd::string userinfo(user_uri_.userinfo);\n\t\tstd::string http_auth;\n\n\t\tStringUtil::url_decode(userinfo);\n\n\t\tif (__encode_auth(userinfo.c_str(), http_auth) < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t\treturn false;\n\t\t}\n\n\t\tclient_req->set_header_pair(\"Authorization\", http_auth.c_str());\n\t}\n\n\treturn true;\n}\n\nbool ComplexHttpProxyTask::finish_once()\n{\n\tif (!is_user_request_)\n\t{\n\t\tif (this->state == WFT_STATE_SUCCESS && state_ != WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tthis->state = state_;\n\t\t\tthis->error = error_;\n\t\t}\n\n\t\tif (this->get_seq() == 0)\n\t\t{\n\t\t\tdelete this->get_message_in();\n\t\t\tdelete this->get_message_out();\n\t\t}\n\n\t\tis_user_request_ = true;\n\t\treturn false;\n\t}\n\n\tif (this->state != WFT_STATE_SUCCESS)\n\t\tthis->check_response();\n\n\tif (this->state == WFT_STATE_SUCCESS)\n\t{\n\t\tParsedURI new_uri;\n\t\tif (this->need_redirect(user_uri_, new_uri))\n\t\t{\n\t\t\tif (user_uri_.userinfo &&\n\t\t\t\tstrcasecmp(user_uri_.host, new_uri.host) == 0)\n\t\t\t{\n\t\t\t\tif (!new_uri.userinfo)\n\t\t\t\t{\n\t\t\t\t\tnew_uri.userinfo = user_uri_.userinfo;\n\t\t\t\t\tuser_uri_.userinfo = NULL;\n\t\t\t\t}\n\t\t\t}\n\t\t\telse if (user_uri_.userinfo)\n\t\t\t{\n\t\t\t\tHttpRequest *client_req = this->get_req();\n\t\t\t\tHttpHeaderCursor cursor(client_req);\n\t\t\t\tstruct HttpMessageHeader header = {\n\t\t\t\t\t.name = \"Authorization\",\n\t\t\t\t\t.name_len = strlen(\"Authorization\")\n\t\t\t\t};\n\n\t\t\t\tcursor.find_and_erase(&header);\n\t\t\t}\n\n\t\t\tuser_uri_ = std::move(new_uri);\n\t\t\tthis->set_redirect(uri_);\n\t\t}\n\t\telse if (this->state != WFT_STATE_SUCCESS)\n\t\t\tthis->disable_retry();\n\t}\n\n\treturn true;\n}\n\n/*******Chunked Client******/\n\nclass ComplexHttpChunkedTask : public ComplexHttpTask\n{\nprotected:\n\tvirtual CommMessageIn *message_in();\n\n\tvirtual int keep_alive_timeout()\n\t{\n\t\treturn resp_is_keep_alive_ ? this->keep_alive_timeo : 0;\n\t}\n\n\tvirtual bool finish_once()\n\t{\n\t\treturn chunking_ ? true : ComplexHttpTask::finish_once();\n\t}\n\nprotected:\n\tclass ChunkWrapper : public PackageWrapper\n\t{\n\tprotected:\n\t\tvirtual ProtocolMessage *next_in(ProtocolMessage *msg);\n\n\tprotected:\n\t\tComplexHttpChunkedTask *task_;\n\n\tpublic:\n\t\tChunkWrapper(ComplexHttpChunkedTask *task) :\n\t\t\tPackageWrapper(NULL)\n\t\t{\n\t\t\ttask_ = task;\n\t\t}\n\n\t\tfriend class ComplexHttpChunkedTask;\n\t};\n\nprotected:\n\tbool chunking_;\n\tbool resp_is_keep_alive_;\n\tHttpMessageChunk chunk_;\n\tChunkWrapper wrapper_;\n\tstd::function extract_;\n\npublic:\n\tComplexHttpChunkedTask(int redirect_max,\n\t\t\t\t\t\t std::function&& extract,\n\t\t\t\t\t\t http_callback_t&& callback) :\n\t\tComplexHttpTask(redirect_max, 0, std::move(callback)),\n\t\twrapper_(this),\n\t\textract_(std::move(extract))\n\t{\n\t\tchunking_ = false;\n\t}\n};\n\nCommMessageIn *ComplexHttpChunkedTask::message_in()\n{\n\tHttpResponse *resp = this->get_resp();\n\n\tif (strcmp(this->get_req()->get_method(), HttpMethodHead) == 0)\n\t\treturn ComplexHttpTask::message_in();\n\n\tresp->parse_zero_body();\n\twrapper_.set_message(resp);\n\treturn &wrapper_;\n}\n\nProtocolMessage *\nComplexHttpChunkedTask::ChunkWrapper::next_in(ProtocolMessage *msg)\n{\n\tHttpResponse *resp = task_->get_resp();\n\tconst void *chunk_data;\n\tsize_t size;\n\n\tif (msg == resp)\n\t{\n\t\tint status_code = atoi(resp->get_status_code());\n\n\t\ttask_->resp_is_keep_alive_ = resp->is_keep_alive();\n\t\tif (status_code / 100 == 1 || status_code == 204 || status_code == 304)\n\t\t\treturn NULL;\n\n\t\tif (status_code / 100 == 2)\n\t\t{\n\t\t\ttask_->extract_(NULL, task_);\n\t\t\tif (resp->is_chunked())\n\t\t\t{\n\t\t\t\tsize = resp->get_size_limit();\n\t\t\t\ttask_->chunk_.set_size_limit(size);\n\t\t\t\ttask_->chunking_ = true;\n\t\t\t\treturn &task_->chunk_;\n\t\t\t}\n\t\t}\n\n\t\thttp_parser_t *parser = (http_parser_t *)resp->get_parser();\n\t\tif (parser->transfer_length != 0)\n\t\t\treturn NULL;\n\n\t\tif (resp->is_chunked())\n\t\t\tparser->transfer_length = (size_t)-1;\n\t\telse if (parser->content_length != 0)\n\t\t\tparser->transfer_length = parser->content_length;\n\t\telse\n\t\t\treturn NULL;\n\n\t\tparser->complete = 0;\n\t\treturn resp;\n\t}\n\n\ttask_->chunk_.get_chunk_data(&chunk_data, &size);\n\tif (size == 0)\n\t\treturn NULL;\n\n\tsize = task_->chunk_.get_size_limit() - size;\n\ttask_->extract_(&task_->chunk_, task_);\n\n\ttask_->chunk_.~HttpMessageChunk();\n\tnew(&task_->chunk_) HttpMessageChunk;\n\ttask_->chunk_.set_size_limit(size);\n\treturn &task_->chunk_;\n}\n\n/**********Client Factory**********/\n\nWFHttpTask *WFTaskFactory::create_http_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\tint redirect_max,\n\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\thttp_callback_t callback)\n{\n\tauto *task = new ComplexHttpTask(redirect_max,\n\t\t\t\t\t\t\t\t\t retry_max,\n\t\t\t\t\t\t\t\t\t std::move(callback));\n\tParsedURI uri;\n\n\tURIParser::parse(url, uri);\n\ttask->init(std::move(uri));\n\ttask->set_keep_alive(HTTP_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\nWFHttpTask *WFTaskFactory::create_http_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\tint redirect_max,\n\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\thttp_callback_t callback)\n{\n\tauto *task = new ComplexHttpTask(redirect_max,\n\t\t\t\t\t\t\t\t\t retry_max,\n\t\t\t\t\t\t\t\t\t std::move(callback));\n\n\ttask->init(uri);\n\ttask->set_keep_alive(HTTP_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\nWFHttpTask *WFTaskFactory::create_http_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\tconst std::string& proxy_url,\n\t\t\t\t\t\t\t\t\t\t\tint redirect_max,\n\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\thttp_callback_t callback)\n{\n\tauto *task = new ComplexHttpProxyTask(redirect_max,\n\t\t\t\t\t\t\t\t\t\t retry_max,\n\t\t\t\t\t\t\t\t\t\t std::move(callback));\n\n\tParsedURI uri, user_uri;\n\tURIParser::parse(url, user_uri);\n\tURIParser::parse(proxy_url, uri);\n\n\ttask->set_user_uri(std::move(user_uri));\n\ttask->set_keep_alive(HTTP_KEEPALIVE_DEFAULT);\n\ttask->init(std::move(uri));\n\treturn task;\n}\n\nWFHttpTask *WFTaskFactory::create_http_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\tconst ParsedURI& proxy_uri,\n\t\t\t\t\t\t\t\t\t\t\tint redirect_max,\n\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\thttp_callback_t callback)\n{\n\tauto *task = new ComplexHttpProxyTask(redirect_max,\n\t\t\t\t\t\t\t\t\t\t retry_max,\n\t\t\t\t\t\t\t\t\t\t std::move(callback));\n\n\ttask->set_user_uri(uri);\n\ttask->set_keep_alive(HTTP_KEEPALIVE_DEFAULT);\n\ttask->init(proxy_uri);\n\treturn task;\n}\n\n\nWFHttpTask *__WFHttpTaskFactory::create_chunked_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\t\t\t int redirect_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t\t\t\t http_callback_t callback)\n{\n\tauto *task = new ComplexHttpChunkedTask(redirect_max,\n\t\t\t\t\t\t\t\t\t\t\tstd::move(extract),\n\t\t\t\t\t\t\t\t\t\t\tstd::move(callback));\n\tParsedURI uri;\n\n\tURIParser::parse(url, uri);\n\ttask->init(std::move(uri));\n\ttask->set_keep_alive(HTTP_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\nWFHttpTask *__WFHttpTaskFactory::create_chunked_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t int redirect_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t\t\t\t http_callback_t callback)\n{\n\tauto *task = new ComplexHttpChunkedTask(redirect_max,\n\t\t\t\t\t\t\t\t\t\t\tstd::move(extract),\n\t\t\t\t\t\t\t\t\t\t\tstd::move(callback));\n\n\ttask->init(uri);\n\ttask->set_keep_alive(HTTP_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\n/**********Server**********/\n\nclass WFHttpServerTask : public WFServerTask\n{\nprivate:\n\tusing TASK = WFNetworkTask;\n\npublic:\n\tWFHttpServerTask(CommService *service, std::function& proc) :\n\t\tWFServerTask(service, WFGlobal::get_scheduler(), proc)\n\t{}\n\nprotected:\n\tvirtual void handle(int state, int error);\n\tvirtual CommMessageOut *message_out();\n\nprotected:\n\tbool req_is_keep_alive_;\n\tbool req_has_keep_alive_header_;\n\tstd::string req_keep_alive_;\n};\n\nvoid WFHttpServerTask::handle(int state, int error)\n{\n\tif (state == WFT_STATE_TOREPLY)\n\t{\n\t\treq_is_keep_alive_ = this->req.is_keep_alive();\n\t\tif (req_is_keep_alive_ && this->req.has_keep_alive_header())\n\t\t{\n\t\t\tHttpHeaderCursor cursor(&this->req);\n\t\t\tstruct HttpMessageHeader header = {\n\t\t\t\t.name\t\t=\t\"Keep-Alive\",\n\t\t\t\t.name_len\t=\tstrlen(\"Keep-Alive\"),\n\t\t\t};\n\n\t\t\treq_has_keep_alive_header_ = cursor.find(&header);\n\t\t\tif (req_has_keep_alive_header_)\n\t\t\t{\n\t\t\t\treq_keep_alive_.assign((const char *)header.value,\n\t\t\t\t\t\t\t\t\t\theader.value_len);\n\t\t\t}\n\t\t}\n\t\telse\n\t\t\treq_has_keep_alive_header_ = false;\n\t}\n\n\tthis->WFServerTask::handle(state, error);\n}\n\nCommMessageOut *WFHttpServerTask::message_out()\n{\n\tHttpResponse *resp = this->get_resp();\n\tstruct HttpMessageHeader header;\n\n\tif (!resp->get_http_version())\n\t\tresp->set_http_version(\"HTTP/1.1\");\n\n\tconst char *status_code_str = resp->get_status_code();\n\tif (!status_code_str || !resp->get_reason_phrase())\n\t{\n\t\tint status_code;\n\n\t\tif (status_code_str)\n\t\t\tstatus_code = atoi(status_code_str);\n\t\telse\n\t\t\tstatus_code = HttpStatusOK;\n\n\t\tHttpUtil::set_response_status(resp, status_code);\n\t}\n\n\tif (!resp->is_chunked() && !resp->has_content_length_header())\n\t{\n\t\tchar buf[32];\n\t\theader.name = \"Content-Length\";\n\t\theader.name_len = strlen(\"Content-Length\");\n\t\theader.value = buf;\n\t\theader.value_len = sprintf(buf, \"%zu\", resp->get_output_body_size());\n\t\tresp->add_header(&header);\n\t}\n\n\tbool is_alive;\n\n\tif (resp->has_connection_header())\n\t\tis_alive = resp->is_keep_alive();\n\telse\n\t\tis_alive = req_is_keep_alive_;\n\n\tif (!is_alive)\n\t\tthis->keep_alive_timeo = 0;\n\telse\n\t{\n\t\t//req---Connection: Keep-Alive\n\t\t//req---Keep-Alive: timeout=5,max=100\n\n\t\tif (req_has_keep_alive_header_)\n\t\t{\n\t\t\tint flag = 0;\n\t\t\tstd::vector params = StringUtil::split(req_keep_alive_, ',');\n\n\t\t\tfor (const auto& kv : params)\n\t\t\t{\n\t\t\t\tstd::vector arr = StringUtil::split(kv, '=');\n\t\t\t\tif (arr.size() < 2)\n\t\t\t\t\tarr.emplace_back(\"0\");\n\n\t\t\t\tstd::string key = StringUtil::strip(arr[0]);\n\t\t\t\tstd::string val = StringUtil::strip(arr[1]);\n\t\t\t\tif (!(flag & 1) && strcasecmp(key.c_str(), \"timeout\") == 0)\n\t\t\t\t{\n\t\t\t\t\tflag |= 1;\n\t\t\t\t\t// keep_alive_timeo = 5000ms when Keep-Alive: timeout=5\n\t\t\t\t\tthis->keep_alive_timeo = 1000 * atoi(val.c_str());\n\t\t\t\t\tif (flag == 3)\n\t\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\telse if (!(flag & 2) && strcasecmp(key.c_str(), \"max\") == 0)\n\t\t\t\t{\n\t\t\t\t\tflag |= 2;\n\t\t\t\t\tif (this->get_seq() >= atoi(val.c_str()))\n\t\t\t\t\t{\n\t\t\t\t\t\tthis->keep_alive_timeo = 0;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\n\t\t\t\t\tif (flag == 3)\n\t\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif ((unsigned int)this->keep_alive_timeo > HTTP_KEEPALIVE_MAX)\n\t\t\tthis->keep_alive_timeo = HTTP_KEEPALIVE_MAX;\n\t\t//if (this->keep_alive_timeo < 0 || this->keep_alive_timeo > HTTP_KEEPALIVE_MAX)\n\n\t}\n\n\tif (!resp->has_connection_header())\n\t{\n\t\theader.name = \"Connection\";\n\t\theader.name_len = 10;\n\t\tif (this->keep_alive_timeo == 0)\n\t\t{\n\t\t\theader.value = \"close\";\n\t\t\theader.value_len = 5;\n\t\t}\n\t\telse\n\t\t{\n\t\t\theader.value = \"Keep-Alive\";\n\t\t\theader.value_len = 10;\n\t\t}\n\n\t\tresp->add_header(&header);\n\t}\n\n\treturn this->WFServerTask::message_out();\n}\n\n/**********Server Factory**********/\n\nWFHttpTask *WFServerTaskFactory::create_http_task(CommService *service,\n\t\t\t\t\t\t\tstd::function& process)\n{\n\treturn new WFHttpServerTask(service, process);\n}\n\n" }, { "path": "src/factory/HttpTaskImpl.inl", "content": "/*\n Copyright (c) 2025 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \"HttpMessage.h\"\n#include \"WFTaskFactory.h\"\n\n// Internal, for WFHttpChunkedTask only.\n\nclass __WFHttpTaskFactory\n{\nprivate:\n\tusing extract_t = std::function;\n\npublic:\n\tstatic WFHttpTask *create_chunked_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t int redirect_max,\n\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t http_callback_t callback);\n\n\tstatic WFHttpTask *create_chunked_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t int redirect_max,\n\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t http_callback_t callback);\n};\n\n" }, { "path": "src/factory/KafkaTaskImpl.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n\t http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"StringUtil.h\"\n#include \"KafkaTaskImpl.inl\"\n\nusing namespace protocol;\n\n#define KAFKA_KEEPALIVE_DEFAULT\t(60 * 1000)\n#define KAFKA_ROUNDTRIP_TIMEOUT (5 * 1000)\n\nstatic KafkaCgroup __create_cgroup(const KafkaCgroup *c)\n{\n\tKafkaCgroup g;\n\tconst char *member_id = c->get_member_id();\n\n\tif (member_id)\n\t\tg.set_member_id(member_id);\n\n\tg.set_group(c->get_group());\n\n\treturn g;\n}\n\n/**********Client**********/\n\nclass __ComplexKafkaTask : public WFComplexClientTask\n{\npublic:\n\t__ComplexKafkaTask(int retry_max, __kafka_callback_t&& callback) :\n\t\tWFComplexClientTask(retry_max, std::move(callback))\n\t{\n\t\tis_user_request_ = true;\n\t\tis_redirect_ = false;\n\t\tctx_ = 0;\n\t}\n\nprotected:\n\tvirtual CommMessageOut *message_out();\n\tvirtual CommMessageIn *message_in();\n\tvirtual bool init_success();\n\tvirtual bool finish_once();\n\nprivate:\n\tstruct KafkaConnectionInfo\n\t{\n\t\tkafka_api_t api;\n\t\tkafka_sasl_t sasl;\n\t\tstd::string mechanisms;\n\n\t\tKafkaConnectionInfo()\n\t\t{\n\t\t\tkafka_api_init(&this->api);\n\t\t\tkafka_sasl_init(&this->sasl);\n\t\t}\n\n\t\t~KafkaConnectionInfo()\n\t\t{\n\t\t\tkafka_api_deinit(&this->api);\n\t\t\tkafka_sasl_deinit(&this->sasl);\n\t\t}\n\n\t\tbool init(const char *mechanisms)\n\t\t{\n\t\t\tthis->mechanisms = mechanisms;\n\n\t\t\tif (strncasecmp(mechanisms, \"SCRAM\", 5) == 0)\n\t\t\t{\n\t\t\t\tif (strcasecmp(mechanisms, \"SCRAM-SHA-1\") == 0)\n\t\t\t\t{\n\t\t\t\t\tthis->sasl.scram.evp = EVP_sha1();\n\t\t\t\t\tthis->sasl.scram.scram_h = SHA1;\n\t\t\t\t\tthis->sasl.scram.scram_h_size = SHA_DIGEST_LENGTH;\n\t\t\t\t}\n\t\t\t\telse if (strcasecmp(mechanisms, \"SCRAM-SHA-256\") == 0)\n\t\t\t\t{\n\t\t\t\t\tthis->sasl.scram.evp = EVP_sha256();\n\t\t\t\t\tthis->sasl.scram.scram_h = SHA256;\n\t\t\t\t\tthis->sasl.scram.scram_h_size = SHA256_DIGEST_LENGTH;\n\t\t\t\t}\n\t\t\t\telse if (strcasecmp(mechanisms, \"SCRAM-SHA-512\") == 0)\n\t\t\t\t{\n\t\t\t\t\tthis->sasl.scram.evp = EVP_sha512();\n\t\t\t\t\tthis->sasl.scram.scram_h = SHA512;\n\t\t\t\t\tthis->sasl.scram.scram_h_size = SHA512_DIGEST_LENGTH;\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t\treturn false;\n\t\t\t}\n\n\t\t\treturn true;\n\t\t}\n\t};\n\n\tvirtual int keep_alive_timeout();\n\tvirtual int first_timeout();\n\tbool has_next();\n\tbool process_produce();\n\tbool process_fetch();\n\tbool process_metadata();\n\tbool process_list_offsets();\n\tbool process_find_coordinator();\n\tbool process_join_group();\n\tbool process_sync_group();\n\tbool process_sasl_authenticate();\n\tbool process_sasl_handshake();\n\n\tbool is_user_request_;\n\tbool is_redirect_;\n\tstd::string user_info_;\n};\n\nCommMessageOut *__ComplexKafkaTask::message_out()\n{\n\tlong long seqid = this->get_seq();\n\tif (seqid == 0)\n\t{\n\t\tKafkaConnectionInfo *conn_info = new KafkaConnectionInfo;\n\n\t\tthis->get_req()->set_api(&conn_info->api);\n\t\tthis->get_connection()->set_context(conn_info, [](void *ctx) {\n\t\t\tdelete (KafkaConnectionInfo *)ctx;\n\t\t});\n\n\t\tif (!this->get_req()->get_config()->get_broker_version())\n\t\t{\n\t\t\tKafkaRequest *req = new KafkaRequest;\n\t\t\treq->duplicate(*this->get_req());\n\t\t\treq->set_api_type(Kafka_ApiVersions);\n\t\t\tis_user_request_ = false;\n\t\t\treturn req;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tkafka_api_version_t *api;\n\t\t\tsize_t api_cnt;\n\t\t\tconst char *v = this->get_req()->get_config()->get_broker_version();\n\t\t\tint ret = kafka_api_version_is_queryable(v, &api, &api_cnt);\n\t\t\tkafka_api_version_t *p = NULL;\n\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tp = (kafka_api_version_t *)malloc(api_cnt * sizeof(*p));\n\t\t\t\tif (p)\n\t\t\t\t{\n\t\t\t\t\tmemcpy(p, api, api_cnt * sizeof(kafka_api_version_t));\n\t\t\t\t\tconn_info->api.api = p;\n\t\t\t\t\tconn_info->api.elements = api_cnt;\n\t\t\t\t\tconn_info->api.features = kafka_get_features(p, api_cnt);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (!p)\n\t\t\t\treturn NULL;\n\n\t\t\tseqid++;\n\t\t}\n\t}\n\n\tif (seqid == 1)\n\t{\n\t\tconst char *sasl_mech = this->get_req()->get_config()->get_sasl_mech();\n\t\tKafkaConnectionInfo *conn_info =\n\t\t\t(KafkaConnectionInfo *)this->get_connection()->get_context();\n\t\tif (sasl_mech && conn_info->sasl.status == 0)\n\t\t{\n\t\t\tif (!conn_info->init(sasl_mech))\n\t\t\t\treturn NULL;\n\n\t\t\tthis->get_req()->set_api(&conn_info->api);\n\t\t\tthis->get_req()->set_sasl(&conn_info->sasl);\n\n\t\t\tKafkaRequest *req = new KafkaRequest;\n\t\t\treq->duplicate(*this->get_req());\n\t\t\tif (conn_info->api.features & KAFKA_FEATURE_SASL_HANDSHAKE)\n\t\t\t\treq->set_api_type(Kafka_SaslHandshake);\n\t\t\telse\n\t\t\t\treq->set_api_type(Kafka_SaslAuthenticate);\n\t\t\treq->set_correlation_id(1);\n\t\t\tis_user_request_ = false;\n\t\t\treturn req;\n\t\t}\n\t}\n\n\tKafkaConnectionInfo *conn_info =\n\t\t(KafkaConnectionInfo *)this->get_connection()->get_context();\n\tKafkaRequest *req = this->get_req();\n\treq->set_api(&conn_info->api);\n\n\tif (req->get_api_type() == Kafka_Fetch ||\n\t\treq->get_api_type() == Kafka_ListOffsets)\n\t{\n\t\tKafkaTopparList *req_toppar_lst = req->get_toppar_list();\n\t\tKafkaToppar *toppar;\n\t\tKafkaTopparList toppar_list;\n\t\tbool flag = false;\n\t\tlong long cfg_ts = req->get_config()->get_offset_timestamp();\n\t\tlong long tp_ts;\n\n\t\treq_toppar_lst->rewind();\n\t\twhile ((toppar = req_toppar_lst->get_next()) != NULL)\n\t\t{\n\t\t\ttp_ts = toppar->get_offset_timestamp();\n\t\t\tif (tp_ts == KAFKA_TIMESTAMP_UNINIT)\n\t\t\t\ttp_ts = cfg_ts;\n\n\t\t\tif (toppar->get_offset() == KAFKA_OFFSET_UNINIT)\n\t\t\t{\n\t\t\t\tif (tp_ts == KAFKA_TIMESTAMP_EARLIEST)\n\t\t\t\t\ttoppar->set_offset(toppar->get_low_watermark());\n\t\t\t\telse if (tp_ts < 0)\n\t\t\t\t{\n\t\t\t\t\ttoppar->set_offset(toppar->get_high_watermark());\n\t\t\t\t\ttp_ts = KAFKA_TIMESTAMP_LATEST;\n\t\t\t\t}\n\t\t\t}\n\t\t\telse if (toppar->get_offset() == KAFKA_OFFSET_OVERFLOW)\n\t\t\t{\n\t\t\t\tif (tp_ts == KAFKA_TIMESTAMP_EARLIEST)\n\t\t\t\t\ttoppar->set_offset(toppar->get_low_watermark());\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\ttoppar->set_offset(toppar->get_high_watermark());\n\t\t\t\t\ttp_ts = KAFKA_TIMESTAMP_LATEST;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (toppar->get_offset() < 0)\n\t\t\t{\n\t\t\t\ttoppar->set_offset_timestamp(tp_ts);\n\t\t\t\ttoppar_list.add_item(*toppar);\n\t\t\t\tflag = true;\n\t\t\t}\n\t\t}\n\n\t\tif (flag)\n\t\t{\n\t\t\tKafkaRequest *new_req = new KafkaRequest;\n\t\t\tnew_req->set_api(&conn_info->api);\n\t\t\tnew_req->set_broker(*req->get_broker());\n\t\t\tnew_req->set_toppar_list(toppar_list);\n\t\t\tnew_req->set_config(*req->get_config());\n\t\t\tnew_req->set_api_type(Kafka_ListOffsets);\n\t\t\tnew_req->set_correlation_id(seqid);\n\t\t\tis_user_request_ = false;\n\t\t\treturn new_req;\n\t\t}\n\t}\n\n\tthis->get_req()->set_correlation_id(seqid);\n\treturn this->WFComplexClientTask::message_out();\n}\n\nCommMessageIn *__ComplexKafkaTask::message_in()\n{\n\tKafkaRequest *req = static_cast(this->get_message_out());\n\tKafkaResponse *resp = this->get_resp();\n\tKafkaCgroup *cgroup;\n\n\tresp->set_api_type(req->get_api_type());\n\tresp->set_api_version(req->get_api_version());\n\tresp->duplicate(*req);\n\n\tswitch (req->get_api_type())\n\t{\n\tcase Kafka_FindCoordinator:\n\tcase Kafka_Heartbeat:\n\t\tcgroup = req->get_cgroup();\n\t\tif (cgroup->get_group())\n\t\t\tresp->set_cgroup(__create_cgroup(cgroup));\n\t\tbreak;\n\tdefault:\n\t\tbreak;\n\t}\n\n\treturn this->WFComplexClientTask::message_in();\n}\n\nbool __ComplexKafkaTask::init_success()\n{\n\tenum TransportType type;\n\n\tif (uri_.scheme && strcasecmp(uri_.scheme, \"kafka\") == 0)\n\t\ttype = TT_TCP;\n\telse if (uri_.scheme && strcasecmp(uri_.scheme, \"kafkas\") == 0)\n\t\ttype = TT_TCP_SSL;\n\telse\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\t\treturn false;\n\t}\n\n\tstd::string username, password, sasl, client;\n\tif (uri_.userinfo)\n\t{\n\t\tconst char *pos = strchr(uri_.userinfo, ':');\n\t\tif (pos)\n\t\t{\n\t\t\tusername = std::string(uri_.userinfo, pos - uri_.userinfo);\n\t\t\tStringUtil::url_decode(username);\n\t\t\tconst char *pos1 = strchr(pos + 1, ':');\n\t\t\tif (pos1)\n\t\t\t{\n\t\t\t\tpassword = std::string(pos + 1, pos1 - pos - 1);\n\t\t\t\tStringUtil::url_decode(password);\n\t\t\t\tconst char *pos2 = strchr(pos1 + 1, ':');\n\t\t\t\tif (pos2)\n\t\t\t\t{\n\t\t\t\t\tsasl = std::string(pos1 + 1, pos2 - pos1 - 1);\n\t\t\t\t\tclient = std::string(pos1 + 1);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif (username.empty() || password.empty() || sasl.empty() || client.empty())\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\t\t\treturn false;\n\t\t}\n\n\t\tuser_info_ = uri_.userinfo;\n\t\tsize_t info_len = username.size() + password.size() + sasl.size() +\n\t\t\tclient.size() + 50;\n\t\tchar *info = new char[info_len];\n\n\t\tsnprintf(info, info_len, \"%s|user:%s|pass:%s|sasl:%s|client:%s|\", \"kafka\",\n\t\t\t\tusername.c_str(), password.c_str(), sasl.c_str(), client.c_str());\n\n\t\tthis->WFComplexClientTask::set_info(info);\n\t\tdelete []info;\n\t}\n\n\tthis->WFComplexClientTask::set_transport_type(type);\n\treturn true;\n}\n\nint __ComplexKafkaTask::keep_alive_timeout()\n{\n\tif (this->get_resp()->get_broker()->get_error())\n\t\treturn 0;\n\n\treturn this->WFComplexClientTask::keep_alive_timeout();\n}\n\nint __ComplexKafkaTask::first_timeout()\n{\n\tKafkaRequest *client_req = this->get_req();\n\tint ret = 0;\n\n\tswitch(client_req->get_api_type())\n\t{\n\tcase Kafka_Fetch:\n\t\tret = client_req->get_config()->get_fetch_timeout();\n\t\tbreak;\n\n\tcase Kafka_JoinGroup:\n\t\tret = client_req->get_config()->get_session_timeout();\n\t\tbreak;\n\n\tcase Kafka_SyncGroup:\n\t\tret = client_req->get_config()->get_rebalance_timeout();\n\t\tbreak;\n\n\tcase Kafka_Produce:\n\t\tret = client_req->get_config()->get_produce_timeout();\n\t\tbreak;\n\n\tdefault:\n\t\treturn 0;\n\t}\n\n\treturn ret + KAFKA_ROUNDTRIP_TIMEOUT;\n}\n\nbool __ComplexKafkaTask::process_find_coordinator()\n{\n\tKafkaCgroup *cgroup = this->get_resp()->get_cgroup();\n\tctx_ = cgroup->get_error();\n\tif (ctx_)\n\t{\n\t\tthis->error = WFT_ERR_KAFKA_CGROUP_FAILED;\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\treturn false;\n\t}\n\telse\n\t{\n\t\tthis->get_req()->set_cgroup(*cgroup);\n\t\tKafkaBroker *coordinator = cgroup->get_coordinator();\n\t\tstd::string url(uri_.scheme);\n\t\turl += \"://\";\n\t\turl += user_info_ + \"@\";\n\t\turl += coordinator->get_host();\n\t\turl += \":\" + std::to_string(coordinator->get_port());\n\n\t\tParsedURI uri;\n\t\tURIParser::parse(url, uri);\n\t\tset_redirect(std::move(uri));\n\t\tthis->get_req()->set_api_type(Kafka_JoinGroup);\n\t\tis_redirect_ = true;\n\t\treturn true;\n\t}\n}\n\nbool __ComplexKafkaTask::process_join_group()\n{\n\tKafkaResponse *msg = this->get_resp();\n\tswitch(msg->get_cgroup()->get_error())\n\t{\n\tcase KAFKA_MEMBER_ID_REQUIRED:\n\t\tthis->get_req()->set_api_type(Kafka_JoinGroup);\n\t\tbreak;\n\n\tcase KAFKA_UNKNOWN_MEMBER_ID:\n\t\tmsg->get_cgroup()->set_member_id(\"\");\n\t\tthis->get_req()->set_api_type(Kafka_JoinGroup);\n\t\tbreak;\n\n\tcase 0:\n\t\tthis->get_req()->set_api_type(Kafka_Metadata);\n\t\tbreak;\n\n\tdefault:\n\t\tctx_ = msg->get_cgroup()->get_error();\n\t\tthis->error = WFT_ERR_KAFKA_CGROUP_FAILED;\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\treturn false;\n\t}\n\n\treturn true;\n}\n\nbool __ComplexKafkaTask::process_sync_group()\n{\n\tctx_ = this->get_resp()->get_cgroup()->get_error();\n\tif (ctx_)\n\t{\n\t\tthis->error = WFT_ERR_KAFKA_CGROUP_FAILED;\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\treturn false;\n\t}\n\telse\n\t{\n\t\tthis->get_req()->set_api_type(Kafka_OffsetFetch);\n\t\treturn true;\n\t}\n}\n\nbool __ComplexKafkaTask::process_metadata()\n{\n\tKafkaResponse *msg = this->get_resp();\n\tmsg->get_meta_list()->rewind();\n\tKafkaMeta *meta;\n\twhile ((meta = msg->get_meta_list()->get_next()) != NULL)\n\t{\n\t\tswitch (meta->get_error())\n\t\t{\n\t\tcase KAFKA_LEADER_NOT_AVAILABLE:\n\t\t\tthis->get_req()->set_api_type(Kafka_Metadata);\n\t\t\treturn true;\n\t\tcase 0:\n\t\t\tbreak;\n\t\tdefault:\n\t\t\tctx_ = meta->get_error();\n\t\t\tthis->error = WFT_ERR_KAFKA_META_FAILED;\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\treturn false;\n\t\t}\n\t}\n\n\tthis->get_req()->set_meta_list(*msg->get_meta_list());\n\tif (msg->get_cgroup()->get_group())\n\t{\n\t\tif (msg->get_cgroup()->is_leader())\n\t\t{\n\t\t\tKafkaCgroup *cgroup = msg->get_cgroup();\n\t\t\tif (cgroup->run_assignor(msg->get_meta_list(),\n\t\t\t\t\t\t\t\t\t cgroup->get_protocol_name()) < 0)\n\t\t\t{\n\t\t\t\tthis->error = WFT_ERR_KAFKA_CGROUP_ASSIGN_FAILED;\n\t\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\t\treturn false;\n\t\t\t}\n\t\t}\n\n\t\tthis->get_req()->set_api_type(Kafka_SyncGroup);\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nbool __ComplexKafkaTask::process_fetch()\n{\n\tbool ret = false;\n\tKafkaToppar *toppar;\n\tthis->get_resp()->get_toppar_list()->rewind();\n\twhile ((toppar = this->get_resp()->get_toppar_list()->get_next()) != NULL)\n\t{\n\t\tint toppar_error = toppar->get_error();\n\n\t\tif (toppar_error == KAFKA_OFFSET_OUT_OF_RANGE)\n\t\t{\n\t\t\ttoppar->set_offset(KAFKA_OFFSET_OVERFLOW);\n\t\t\ttoppar->set_low_watermark(KAFKA_OFFSET_UNINIT);\n\t\t\ttoppar->set_high_watermark(KAFKA_OFFSET_UNINIT);\n\t\t\tret = true;\n\t\t}\n\t\telse if (toppar_error)\n\t\t{\n\t\t\tctx_ = toppar_error;\n\t\t\tthis->error = WFT_ERR_KAFKA_FETCH_FAILED;\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\treturn false;\n\t\t}\n\t}\n\treturn ret;\n}\n\nbool __ComplexKafkaTask::process_list_offsets()\n{\n\tKafkaToppar *toppar;\n\tthis->get_resp()->get_toppar_list()->rewind();\n\twhile ((toppar = this->get_resp()->get_toppar_list()->get_next()) != NULL)\n\t{\n\t\tif (toppar->get_error())\n\t\t{\n\t\t\tthis->error = toppar->get_error();\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t}\n\t}\n\treturn false;\n}\n\nbool __ComplexKafkaTask::process_produce()\n{\n\tKafkaToppar *toppar;\n\tthis->get_resp()->get_toppar_list()->rewind();\n\twhile ((toppar = this->get_resp()->get_toppar_list()->get_next()) != NULL)\n\t{\n\t\tif (!toppar->record_reach_end())\n\t\t{\n\t\t\tthis->get_req()->set_api_type(Kafka_Produce);\n\t\t\treturn true;\n\t\t}\n\n\t\tif (toppar->get_error())\n\t\t{\n\t\t\tctx_ = toppar->get_error();\n\t\t\tthis->error = WFT_ERR_KAFKA_PRODUCE_FAILED;\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\treturn false;\n\t\t}\n\t}\n\treturn false;\n}\n\nbool __ComplexKafkaTask::process_sasl_handshake()\n{\n\tctx_ = this->get_resp()->get_broker()->get_error();\n\tif (ctx_)\n\t{\n\t\tthis->error = WFT_ERR_KAFKA_SASL_DISALLOWED;\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\treturn false;\n\t}\n\treturn true;\n}\n\nbool __ComplexKafkaTask::process_sasl_authenticate()\n{\n\tctx_ = this->get_resp()->get_broker()->get_error();\n\tif (ctx_)\n\t{\n\t\tthis->error = WFT_ERR_KAFKA_SASL_DISALLOWED;\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t}\n\treturn false;\n}\n\nbool __ComplexKafkaTask::has_next()\n{\n\tswitch (this->get_resp()->get_api_type())\n\t{\n\tcase Kafka_Produce:\n\t\treturn this->process_produce();\n\tcase Kafka_Fetch:\n\t\treturn this->process_fetch();\n\tcase Kafka_Metadata:\n\t\treturn this->process_metadata();\n\tcase Kafka_FindCoordinator:\n\t\treturn this->process_find_coordinator();\n\tcase Kafka_JoinGroup:\n\t\treturn this->process_join_group();\n\tcase Kafka_SyncGroup:\n\t\treturn this->process_sync_group();\n\tcase Kafka_SaslHandshake:\n\t\treturn this->process_sasl_handshake();\n\tcase Kafka_SaslAuthenticate:\n\t\treturn this->process_sasl_authenticate();\n\tcase Kafka_ListOffsets:\n\t\treturn this->process_list_offsets();\n\tcase Kafka_OffsetCommit:\n\tcase Kafka_OffsetFetch:\n\tcase Kafka_LeaveGroup:\n\tcase Kafka_DescribeGroups:\n\tcase Kafka_Heartbeat:\n\t\tctx_ = this->get_resp()->get_cgroup()->get_error();\n\t\tif (ctx_)\n\t\t{\n\t\t\tthis->error = WFT_ERR_KAFKA_CGROUP_FAILED;\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t}\n\n\t\tbreak;\n\tcase Kafka_ApiVersions:\n\t\tbreak;\n\tdefault:\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_KAFKA_API_UNKNOWN;\n\t\tbreak;\n\t}\n\treturn false;\n}\n\nbool __ComplexKafkaTask::finish_once()\n{\n\tbool finish = true;\n\tif (this->state == WFT_STATE_SUCCESS)\n\t\tfinish = !has_next();\n\n\tif (!is_user_request_)\n\t{\n\t\tdelete this->get_message_out();\n\t\tthis->get_resp()->clear_buf();\n\t}\n\n\tif (is_redirect_ && this->state == WFT_STATE_UNDEFINED)\n\t{\n\t\tthis->get_req()->clear_buf();\n\t\tis_redirect_ = false;\n\t}\n\telse if (this->state == WFT_STATE_SUCCESS)\n\t{\n\t\tif (!is_user_request_)\n\t\t{\n\t\t\tis_user_request_ = true;\n\t\t\treturn false;\n\t\t}\n\n\t\tif (!finish)\n\t\t{\n\t\t\tthis->get_req()->clear_buf();\n\t\t\tthis->get_resp()->clear_buf();\n\t\t\treturn false;\n\t\t}\n\t}\n\telse\n\t{\n\t\tthis->get_resp()->set_api_type(this->get_req()->get_api_type());\n\t\tthis->get_resp()->set_api_version(this->get_req()->get_api_version());\n\t}\n\n\tis_user_request_ = true;\n\treturn true;\n}\n\n/**********Factory**********/\n// kafka://user:password:sasl@host:port/api=type&topic=name\n__WFKafkaTask *__WFKafkaTaskFactory::create_kafka_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\t\t\t SSL_CTX *ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t __kafka_callback_t callback)\n{\n\tauto *task = new __ComplexKafkaTask(retry_max, std::move(callback));\n\ttask->set_ssl_ctx(ssl_ctx);\n\n\tParsedURI uri;\n\tURIParser::parse(url, uri);\n\ttask->init(std::move(uri));\n\ttask->set_keep_alive(KAFKA_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\n__WFKafkaTask *__WFKafkaTaskFactory::create_kafka_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t SSL_CTX *ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t __kafka_callback_t callback)\n{\n\tauto *task = new __ComplexKafkaTask(retry_max, std::move(callback));\n\ttask->set_ssl_ctx(ssl_ctx);\n\n\ttask->init(uri);\n\ttask->set_keep_alive(KAFKA_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\n__WFKafkaTask *__WFKafkaTaskFactory::create_kafka_task(enum TransportType type,\n\t\t\t\t\t\t\t\t\t\t\t\t\t const char *host,\n\t\t\t\t\t\t\t\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t\t\t\t\t\t\t\t SSL_CTX *ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\t const std::string& info,\n\t\t\t\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t __kafka_callback_t callback)\n{\n\tauto *task = new __ComplexKafkaTask(retry_max, std::move(callback));\n\ttask->set_ssl_ctx(ssl_ctx);\n\n\tParsedURI uri;\n\tchar buf[32];\n\n\tif (type == TT_TCP_SSL)\n\t\turi.scheme = strdup(\"kafkas\");\n\telse\n\t\turi.scheme = strdup(\"kafka\");\n\n\tif (!info.empty())\n\t\turi.userinfo = strdup(info.c_str());\n\n\turi.host = strdup(host);\n\tsprintf(buf, \"%u\", port);\n\turi.port = strdup(buf);\n\n\tif (!uri.scheme || !uri.host || !uri.port ||\n\t\t(!info.empty() && !uri.userinfo))\n\t{\n\t\turi.state = URI_STATE_ERROR;\n\t\turi.error = errno;\n\t}\n\telse\n\t\turi.state = URI_STATE_SUCCESS;\n\n\ttask->init(std::move(uri));\n\ttask->set_keep_alive(KAFKA_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\n" }, { "path": "src/factory/KafkaTaskImpl.inl", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#include \n#include \"WFTaskFactory.h\"\n#include \"KafkaMessage.h\"\n\n// Kafka internal task. For __ComplexKafkaTask usage only\nusing __WFKafkaTask = WFNetworkTask;\nusing __kafka_callback_t = std::function;\n\nclass __WFKafkaTaskFactory\n{\npublic:\n\t/* __WFKafkaTask is create by __ComplexKafkaTask. This is an internal\n\t * interface for create internal task. It should not be created directly by common\n\t * user task.\n\t */\n\tstatic __WFKafkaTask *create_kafka_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\tSSL_CTX *ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t__kafka_callback_t callback);\n\n\tstatic __WFKafkaTask *create_kafka_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\tSSL_CTX *ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t__kafka_callback_t callback);\n\n\tstatic __WFKafkaTask *create_kafka_task(enum TransportType type,\n\t\t\t\t\t\t\t\t\t\t\tconst char *host,\n\t\t\t\t\t\t\t\t\t\t\tunsigned short port,\n\t\t\t\t\t\t\t\t\t\t\tSSL_CTX *ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\tconst std::string& info,\n\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t__kafka_callback_t callback);\n};\n\n" }, { "path": "src/factory/MySQLTaskImpl.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"WFTaskError.h\"\n#include \"WFTaskFactory.h\"\n#include \"StringUtil.h\"\n#include \"WFGlobal.h\"\n#include \"mysql_types.h\"\n\nusing namespace protocol;\n\n#define MYSQL_KEEPALIVE_DEFAULT\t\t(60 * 1000)\n#define MYSQL_KEEPALIVE_TRANSACTION\t(3600 * 1000)\n\n/**********Client**********/\n\nclass ComplexMySQLTask : public WFComplexClientTask\n{\nprotected:\n\tvirtual bool check_request();\n\tvirtual CommMessageOut *message_out();\n\tvirtual CommMessageIn *message_in();\n\tvirtual int keep_alive_timeout();\n\tvirtual int first_timeout();\n\tvirtual bool init_success();\n\tvirtual bool finish_once();\n\nprotected:\n\tvirtual WFConnection *get_connection() const\n\t{\n\t\tWFConnection *conn = this->WFComplexClientTask::get_connection();\n\n\t\tif (conn)\n\t\t{\n\t\t\tvoid *ctx = conn->get_context();\n\t\t\tif (ctx)\n\t\t\t\tconn = (WFConnection *)ctx;\n\t\t}\n\n\t\treturn conn;\n\t}\n\nprivate:\n\tenum ConnState\n\t{\n\t\tST_SSL_REQUEST,\n\t\tST_AUTH_REQUEST,\n\t\tST_AUTH_SWITCH_REQUEST,\n\t\tST_CLEAR_PASSWORD_REQUEST,\n\t\tST_SHA256_PUBLIC_KEY_REQUEST,\n\t\tST_CSHA2_PUBLIC_KEY_REQUEST,\n\t\tST_RSA_AUTH_REQUEST,\n\t\tST_CHARSET_REQUEST,\n\t\tST_FIRST_USER_REQUEST,\n\t\tST_USER_REQUEST\n\t};\n\n\tstruct MyConnection : public WFConnection\n\t{\n\t\tstd::string str;\t// shared by auth, auth_swich and rsa_auth requests\n\t\tunsigned char seed[20];\n\t\tenum ConnState state;\n\t\tunsigned char mysql_seqid;\n\t\tSSL *ssl;\n\t\tSSLWrapper wrapper;\n\t\tMyConnection(SSL *ssl) : wrapper(&wrapper, ssl)\n\t\t{\n\t\t\tthis->ssl = ssl;\n\t\t}\n\t};\n\n\tint check_handshake(MySQLHandshakeResponse *resp);\n\tint auth_switch(MySQLAuthResponse *resp, MyConnection *conn);\n\n\tstruct MySSLWrapper : public SSLWrapper\n\t{\n\t\tMySSLWrapper(ProtocolMessage *msg, SSL *ssl) :\n\t\t\tSSLWrapper(msg, ssl)\n\t\t{ }\n\t\tProtocolMessage *get_msg() const { return this->message; }\n\t\tvirtual ~MySSLWrapper() { delete this->message; }\n\t};\n\nprivate:\n\tstd::string username_;\n\tstd::string password_;\n\tstd::string db_;\n\tstd::string res_charset_;\n\tshort character_set_;\n\tshort state_;\n\tint error_;\n\tbool is_ssl_;\n\tbool is_user_request_;\n\npublic:\n\tComplexMySQLTask(int retry_max, mysql_callback_t&& callback):\n\t\tWFComplexClientTask(retry_max, std::move(callback)),\n\t\tcharacter_set_(33),\n\t\tis_user_request_(true)\n\t{}\n};\n\nbool ComplexMySQLTask::check_request()\n{\n\tif (this->req.query_is_unset() == false)\n\t{\n\t\tif (this->req.get_command() == MYSQL_COM_QUERY)\n\t\t{\n\t\t\tstd::string query = this->req.get_query();\n\n\t\t\tif (strncasecmp(query.c_str(), \"USE \", 4) &&\n\t\t\t\tstrncasecmp(query.c_str(), \"SET NAMES \", 10) &&\n\t\t\t\tstrncasecmp(query.c_str(), \"SET CHARSET \", 12) &&\n\t\t\t\tstrncasecmp(query.c_str(), \"SET CHARACTER SET \", 18))\n\t\t\t{\n\t\t\t\treturn true;\n\t\t\t}\n\t\t}\n\n\t\tthis->error = WFT_ERR_MYSQL_COMMAND_DISALLOWED;\n\t}\n\telse\n\t\tthis->error = WFT_ERR_MYSQL_QUERY_NOT_SET;\n\n\tthis->state = WFT_STATE_TASK_ERROR;\n\treturn false;\n}\n\nstatic SSL *__create_ssl(SSL_CTX *ssl_ctx)\n{\n\tBIO *wbio;\n\tBIO *rbio;\n\tSSL *ssl;\n\n\trbio = BIO_new(BIO_s_mem());\n\tif (rbio)\n\t{\n\t\twbio = BIO_new(BIO_s_mem());\n\t\tif (wbio)\n\t\t{\n\t\t\tssl = SSL_new(ssl_ctx);\n\t\t\tif (ssl)\n\t\t\t{\n\t\t\t\tSSL_set_bio(ssl, rbio, wbio);\n\t\t\t\treturn ssl;\n\t\t\t}\n\n\t\t\tBIO_free(wbio);\n\t\t}\n\n\t\tBIO_free(rbio);\n\t}\n\n\treturn NULL;\n}\n\nCommMessageOut *ComplexMySQLTask::message_out()\n{\n\tMySQLAuthSwitchRequest *auth_switch_req;\n\tMySQLRSAAuthRequest *rsa_auth_req;\n\tMySQLAuthRequest *auth_req;\n\tMySQLRequest *req;\n\n\tis_user_request_ = false;\n\tif (this->get_seq() == 0)\n\t\treturn new MySQLHandshakeRequest;\n\n\tauto *conn = (MyConnection *)this->get_connection();\n\tswitch (conn->state)\n\t{\n\tcase ST_SSL_REQUEST:\n\t\treq = new MySQLSSLRequest(character_set_, conn->ssl);\n\t\treq->set_seqid(conn->mysql_seqid);\n\t\treturn req;\n\n\tcase ST_AUTH_REQUEST:\n\t\treq = new MySQLAuthRequest;\n\t\tauth_req = (MySQLAuthRequest *)req;\n\t\tauth_req->set_auth(username_, password_, db_, character_set_);\n\t\tauth_req->set_auth_plugin_name(std::move(conn->str));\n\t\tauth_req->set_seed(conn->seed);\n\t\tbreak;\n\n\tcase ST_CLEAR_PASSWORD_REQUEST:\n\t\tconn->str = \"mysql_clear_password\";\n\tcase ST_AUTH_SWITCH_REQUEST:\n\t\treq = new MySQLAuthSwitchRequest;\n\t\tauth_switch_req = (MySQLAuthSwitchRequest *)req;\n\t\tauth_switch_req->set_password(password_);\n\t\tauth_switch_req->set_auth_plugin_name(std::move(conn->str));\n\t\tauth_switch_req->set_seed(conn->seed);\n\t\tbreak;\n\n\tcase ST_SHA256_PUBLIC_KEY_REQUEST:\n\t\treq = new MySQLPublicKeyRequest;\n\t\t((MySQLPublicKeyRequest *)req)->set_sha256();\n\t\tbreak;\n\n\tcase ST_CSHA2_PUBLIC_KEY_REQUEST:\n\t\treq = new MySQLPublicKeyRequest;\n\t\t((MySQLPublicKeyRequest *)req)->set_caching_sha2();\n\t\tbreak;\n\n\tcase ST_RSA_AUTH_REQUEST:\n\t\treq = new MySQLRSAAuthRequest;\n\t\trsa_auth_req = (MySQLRSAAuthRequest *)req;\n\t\trsa_auth_req->set_password(password_);\n\t\trsa_auth_req->set_public_key(std::move(conn->str));\n\t\trsa_auth_req->set_seed(conn->seed);\n\t\tbreak;\n\n\tcase ST_CHARSET_REQUEST:\n\t\treq = new MySQLRequest;\n\t\treq->set_query(\"SET NAMES \" + res_charset_);\n\t\tbreak;\n\n\tcase ST_FIRST_USER_REQUEST:\n\t\tif (this->is_fixed_conn())\n\t\t{\n\t\t\tauto *target = (RouteManager::RouteTarget *)this->target;\n\n\t\t\t/* If it's a transaction task, generate a ECONNRESET error when\n\t\t\t * the target was reconnected. */\n\t\t\tif (target->state)\n\t\t\t{\n\t\t\t\tis_user_request_ = true;\n\t\t\t\terrno = ECONNRESET;\n\t\t\t\treturn NULL;\n\t\t\t}\n\n\t\t\ttarget->state = 1;\n\t\t}\n\n\tcase ST_USER_REQUEST:\n\t\tis_user_request_ = true;\n\t\treq = (MySQLRequest *)this->WFComplexClientTask::message_out();\n\t\tbreak;\n\n\tdefault:\n\t\tassert(0);\n\t\treturn NULL;\n\t}\n\n\tif (!is_user_request_ && conn->state != ST_CHARSET_REQUEST)\n\t\treq->set_seqid(conn->mysql_seqid);\n\n\tif (!is_ssl_)\n\t\treturn req;\n\n\tif (is_user_request_)\n\t{\n\t\tconn->wrapper = SSLWrapper(req, conn->ssl);\n\t\treturn &conn->wrapper;\n\t}\n\telse\n\t\treturn new MySSLWrapper(req, conn->ssl);\n}\n\nCommMessageIn *ComplexMySQLTask::message_in()\n{\n\tMySQLResponse *resp;\n\n\tif (this->get_seq() == 0)\n\t\treturn new MySQLHandshakeResponse;\n\n\tauto *conn = (MyConnection *)this->get_connection();\n\tswitch (conn->state)\n\t{\n\tcase ST_SSL_REQUEST:\n\t\treturn new SSLHandshaker(conn->ssl);\n\n\tcase ST_AUTH_REQUEST:\n\tcase ST_AUTH_SWITCH_REQUEST:\n\t\tresp = new MySQLAuthResponse;\n\t\tbreak;\n\n\tcase ST_CLEAR_PASSWORD_REQUEST:\n\tcase ST_RSA_AUTH_REQUEST:\n\t\tresp = new MySQLResponse;\n\t\tbreak;\n\n\tcase ST_SHA256_PUBLIC_KEY_REQUEST:\n\tcase ST_CSHA2_PUBLIC_KEY_REQUEST:\n\t\tresp = new MySQLPublicKeyResponse;\n\t\tbreak;\n\n\tcase ST_CHARSET_REQUEST:\n\t\tresp = new MySQLResponse;\n\t\tbreak;\n\n\tcase ST_FIRST_USER_REQUEST:\n\tcase ST_USER_REQUEST:\n\t\tresp = (MySQLResponse *)this->WFComplexClientTask::message_in();\n\t\tbreak;\n\n\tdefault:\n\t\tassert(0);\n\t\treturn NULL;\n\t}\n\n\tif (!is_ssl_)\n\t\treturn resp;\n\n\tif (is_user_request_)\n\t{\n\t\tconn->wrapper = SSLWrapper(resp, conn->ssl);\n\t\treturn &conn->wrapper;\n\t}\n\telse\n\t\treturn new MySSLWrapper(resp, conn->ssl);\n}\n\nint ComplexMySQLTask::check_handshake(MySQLHandshakeResponse *resp)\n{\n\tSSL *ssl = NULL;\n\n\tif (resp->host_disallowed())\n\t{\n\t\tthis->resp = std::move(*(MySQLResponse *)resp);\n\t\tstate_ = WFT_STATE_TASK_ERROR;\n\t\terror_ = WFT_ERR_MYSQL_HOST_NOT_ALLOWED;\n\t\treturn 0;\n\t}\n\n\tif (is_ssl_)\n\t{\n\t\tif (resp->get_capability_flags() & 0x800)\n\t\t{\n\t\t\tstatic SSL_CTX *ssl_ctx = WFGlobal::get_ssl_client_ctx();\n\n\t\t\tssl = __create_ssl(ssl_ctx_ ? ssl_ctx_ : ssl_ctx);\n\t\t\tif (!ssl)\n\t\t\t{\n\t\t\t\tstate_ = WFT_STATE_SYS_ERROR;\n\t\t\t\terror_ = errno;\n\t\t\t\treturn 0;\n\t\t\t}\n\n\t\t\tSSL_set_connect_state(ssl);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tthis->resp = std::move(*(MySQLResponse *)resp);\n\t\t\tstate_ = WFT_STATE_TASK_ERROR;\n\t\t\terror_ = WFT_ERR_MYSQL_SSL_NOT_SUPPORTED;\n\t\t\treturn 0;\n\t\t}\n\n\t}\n\n\tauto *conn = this->get_connection();\n\tauto *my_conn = new MyConnection(ssl);\n\n\tmy_conn->str = resp->get_auth_plugin_name();\n\tif (!password_.empty() && my_conn->str == \"sha256_password\")\n\t\tmy_conn->str = \"caching_sha2_password\";\n\n\tresp->get_seed(my_conn->seed);\n\tmy_conn->state = is_ssl_ ? ST_SSL_REQUEST : ST_AUTH_REQUEST;\n\tmy_conn->mysql_seqid = resp->get_seqid() + 1;\n\tconn->set_context(my_conn, [](void *ctx) {\n\t\tauto *my_conn = (MyConnection *)ctx;\n\t\tif (my_conn->ssl)\n\t\t\tSSL_free(my_conn->ssl);\n\t\tdelete my_conn;\n\t});\n\n\treturn MYSQL_KEEPALIVE_DEFAULT;\n}\n\nint ComplexMySQLTask::auth_switch(MySQLAuthResponse *resp, MyConnection *conn)\n{\n\tstd::string name = resp->get_auth_plugin_name();\n\n\tif (conn->state != ST_AUTH_REQUEST ||\n\t\t(name == \"mysql_clear_password\" && !is_ssl_))\n\t{\n\t\tstate_ = WFT_STATE_SYS_ERROR;\n\t\terror_ = EBADMSG;\n\t\treturn 0;\n\t}\n\n\tif (password_.empty())\n\t{\n\t\tconn->state = ST_CLEAR_PASSWORD_REQUEST;\n\t}\n\telse if (name == \"sha256_password\")\n\t{\n\t\tif (is_ssl_)\n\t\t\tconn->state = ST_CLEAR_PASSWORD_REQUEST;\n\t\telse\n\t\t\tconn->state = ST_SHA256_PUBLIC_KEY_REQUEST;\n\t}\n\telse\n\t{\n\t\tconn->str = std::move(name);\n\t\tconn->state = ST_AUTH_SWITCH_REQUEST;\n\t}\n\n\tresp->get_seed(conn->seed);\n\tconn->mysql_seqid = resp->get_seqid() + 1;\n\treturn MYSQL_KEEPALIVE_DEFAULT;\n}\n\nint ComplexMySQLTask::keep_alive_timeout()\n{\n\tauto *msg = (ProtocolMessage *)this->get_message_in();\n\tMySQLAuthResponse *auth_resp;\n\tMySQLResponse *resp;\n\n\tstate_ = WFT_STATE_SUCCESS;\n\terror_ = 0;\n\tif (this->get_seq() == 0)\n\t\treturn check_handshake((MySQLHandshakeResponse *)msg);\n\n\tauto *conn = (MyConnection *)this->get_connection();\n\tif (conn->state == ST_SSL_REQUEST)\n\t{\n\t\tconn->state = ST_AUTH_REQUEST;\n\t\tconn->mysql_seqid++;\n\t\treturn MYSQL_KEEPALIVE_DEFAULT;\n\t}\n\n\tif (is_ssl_)\n\t\tresp = (MySQLResponse *)((MySSLWrapper *)msg)->get_msg();\n\telse\n\t\tresp = (MySQLResponse *)msg;\n\n\tswitch (conn->state)\n\t{\n\tcase ST_AUTH_REQUEST:\n\tcase ST_AUTH_SWITCH_REQUEST:\n\tcase ST_CLEAR_PASSWORD_REQUEST:\n\tcase ST_RSA_AUTH_REQUEST:\n\t\tif (resp->is_ok_packet())\n\t\t{\n\t\t\tif (!res_charset_.empty())\n\t\t\t\tconn->state = ST_CHARSET_REQUEST;\n\t\t\telse\n\t\t\t\tconn->state = ST_FIRST_USER_REQUEST;\n\n\t\t\tbreak;\n\t\t}\n\n\t\tif (resp->is_error_packet() ||\n\t\t\tconn->state == ST_CLEAR_PASSWORD_REQUEST ||\n\t\t\tconn->state == ST_RSA_AUTH_REQUEST)\n\t\t{\n\t\t\tthis->resp = std::move(*resp);\n\t\t\tstate_ = WFT_STATE_TASK_ERROR;\n\t\t\terror_ = WFT_ERR_MYSQL_ACCESS_DENIED;\n\t\t\treturn 0;\n\t\t}\n\n\t\tauth_resp = (MySQLAuthResponse *)resp;\n\t\tif (auth_resp->is_continue())\n\t\t{\n\t\t\tif (is_ssl_)\n\t\t\t\tconn->state = ST_CLEAR_PASSWORD_REQUEST;\n\t\t\telse\n\t\t\t\tconn->state = ST_CSHA2_PUBLIC_KEY_REQUEST;\n\n\t\t\tbreak;\n\t\t}\n\n\t\treturn auth_switch(auth_resp, conn);\n\n\tcase ST_SHA256_PUBLIC_KEY_REQUEST:\n\tcase ST_CSHA2_PUBLIC_KEY_REQUEST:\n\t\tconn->str = ((MySQLPublicKeyResponse *)resp)->get_public_key();\n\t\tconn->state = ST_RSA_AUTH_REQUEST;\n\t\tbreak;\n\n\tcase ST_CHARSET_REQUEST:\n\t\tif (!resp->is_ok_packet())\n\t\t{\n\t\t\tthis->resp = std::move(*resp);\n\t\t\tstate_ = WFT_STATE_TASK_ERROR;\n\t\t\terror_ = WFT_ERR_MYSQL_INVALID_CHARACTER_SET;\n\t\t\treturn 0;\n\t\t}\n\n\t\tconn->state = ST_FIRST_USER_REQUEST;\n\t\treturn MYSQL_KEEPALIVE_DEFAULT;\n\n\tcase ST_FIRST_USER_REQUEST:\n\t\tconn->state = ST_USER_REQUEST;\n\tcase ST_USER_REQUEST:\n\t\treturn this->keep_alive_timeo;\n\n\tdefault:\n\t\tassert(0);\n\t\treturn 0;\n\t}\n\n\tconn->mysql_seqid = resp->get_seqid() + 1;\n\treturn MYSQL_KEEPALIVE_DEFAULT;\n}\n\nint ComplexMySQLTask::first_timeout()\n{\n\treturn is_user_request_ ? this->watch_timeo : 0;\n}\n\n/*\n+--------------------+---------------------+-----+\n| CHARACTER_SET_NAME | COLLATION_NAME | ID |\n+--------------------+---------------------+-----+\n| big5 | big5_chinese_ci | 1 |\n| dec8 | dec8_swedish_ci | 3 |\n| cp850 | cp850_general_ci | 4 |\n| hp8 | hp8_english_ci | 6 |\n| koi8r | koi8r_general_ci | 7 |\n| latin1 | latin1_swedish_ci | 8 |\n| latin2 | latin2_general_ci | 9 |\n| swe7 | swe7_swedish_ci | 10 |\n| ascii | ascii_general_ci | 11 |\n| ujis | ujis_japanese_ci | 12 |\n| sjis | sjis_japanese_ci | 13 |\n| hebrew | hebrew_general_ci | 16 |\n| tis620 | tis620_thai_ci | 18 |\n| euckr | euckr_korean_ci | 19 |\n| koi8u | koi8u_general_ci | 22 |\n| gb2312 | gb2312_chinese_ci | 24 |\n| greek | greek_general_ci | 25 |\n| cp1250 | cp1250_general_ci | 26 |\n| gbk | gbk_chinese_ci | 28 |\n| latin5 | latin5_turkish_ci | 30 |\n| armscii8 | armscii8_general_ci | 32 |\n| utf8 | utf8_general_ci | 33 |\n| ucs2 | ucs2_general_ci | 35 |\n| cp866 | cp866_general_ci | 36 |\n| keybcs2 | keybcs2_general_ci | 37 |\n| macce | macce_general_ci | 38 |\n| macroman | macroman_general_ci | 39 |\n| cp852 | cp852_general_ci | 40 |\n| latin7 | latin7_general_ci | 41 |\n| cp1251 | cp1251_general_ci | 51 |\n| utf16 | utf16_general_ci | 54 |\n| utf16le | utf16le_general_ci | 56 |\n| cp1256 | cp1256_general_ci | 57 |\n| cp1257 | cp1257_general_ci | 59 |\n| utf32 | utf32_general_ci | 60 |\n| binary | binary | 63 |\n| geostd8 | geostd8_general_ci | 92 |\n| cp932 | cp932_japanese_ci | 95 |\n| eucjpms | eucjpms_japanese_ci | 97 |\n| gb18030 | gb18030_chinese_ci | 248 |\n| utf8mb4 | utf8mb4_0900_ai_ci | 255 |\n+--------------------+---------------------+-----+\n*/\n\nstatic int __mysql_get_character_set(const std::string& charset)\n{\n\tstatic std::unordered_map charset_map = {\n\t\t{\"big5\",\t1},\n\t\t{\"dec8\",\t3},\n\t\t{\"cp850\",\t4},\n\t\t{\"hp8\",\t\t5},\n\t\t{\"koi8r\",\t6},\n\t\t{\"latin1\",\t7},\n\t\t{\"latin2\",\t8},\n\t\t{\"swe7\",\t10},\n\t\t{\"ascii\",\t11},\n\t\t{\"ujis\",\t12},\n\t\t{\"sjis\",\t13},\n\t\t{\"hebrew\",\t16},\n\t\t{\"tis620\",\t18},\n\t\t{\"euckr\",\t19},\n\t\t{\"koi8u\",\t22},\n\t\t{\"gb2312\",\t24},\n\t\t{\"greek\",\t25},\n\t\t{\"cp1250\",\t26},\n\t\t{\"gbk\",\t\t28},\n\t\t{\"latin5\",\t30},\n\t\t{\"armscii8\",32},\n\t\t{\"utf8\",\t33},\n\t\t{\"ucs2\",\t35},\n\t\t{\"cp866\",\t36},\n\t\t{\"keybcs2\",\t37},\n\t\t{\"macce\",\t38},\n\t\t{\"macroman\",39},\n\t\t{\"cp852\",\t40},\n\t\t{\"latin7\",\t41},\n\t\t{\"cp1251\",\t51},\n\t\t{\"utf16\",\t54},\n\t\t{\"utf16le\",\t56},\n\t\t{\"cp1256\",\t57},\n\t\t{\"cp1257\",\t59},\n\t\t{\"utf32\",\t60},\n\t\t{\"binary\",\t63},\n\t\t{\"geostd8\",\t92},\n\t\t{\"cp932\",\t95},\n\t\t{\"eucjpms\",\t97},\n\t\t{\"gb18030\",\t248},\n\t\t{\"utf8mb4\",\t255},\n\t};\n\n\tconst auto it = charset_map.find(charset);\n\n\tif (it != charset_map.cend())\n\t\treturn it->second;\n\n\treturn -1;\n}\n\nbool ComplexMySQLTask::init_success()\n{\n\tif (uri_.scheme && strcasecmp(uri_.scheme, \"mysql\") == 0)\n\t\tis_ssl_ = false;\n\telse if (uri_.scheme && strcasecmp(uri_.scheme, \"mysqls\") == 0)\n\t\tis_ssl_ = true;\n\telse\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\t\treturn false;\n\t}\n\n\t//todo mysql+unix\n\tusername_.clear();\n\tpassword_.clear();\n\tdb_.clear();\n\tif (uri_.userinfo)\n\t{\n\t\tconst char *colon = NULL;\n\t\tconst char *pos = uri_.userinfo;\n\n\t\twhile (*pos && *pos != ':')\n\t\t\tpos++;\n\n\t\tif (*pos == ':')\n\t\t\tcolon = pos++;\n\n\t\tif (colon)\n\t\t{\n\t\t\tif (colon > uri_.userinfo)\n\t\t\t{\n\t\t\t\tusername_.assign(uri_.userinfo, colon - uri_.userinfo);\n\t\t\t\tStringUtil::url_decode(username_);\n\t\t\t}\n\n\t\t\tif (*pos)\n\t\t\t{\n\t\t\t\tpassword_.assign(pos);\n\t\t\t\tStringUtil::url_decode(password_);\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tusername_.assign(uri_.userinfo);\n\t\t\tStringUtil::url_decode(username_);\n\t\t}\n\t}\n\n\tif (uri_.path && uri_.path[0] == '/' && uri_.path[1])\n\t{\n\t\tdb_.assign(uri_.path + 1);\n\t\tStringUtil::url_decode(db_);\n\t}\n\n\tstd::string transaction;\n\n\tif (uri_.query)\n\t{\n\t\tauto query_kv = URIParser::split_query(uri_.query);\n\n\t\tfor (auto& kv : query_kv)\n\t\t{\n\t\t\tif (strcasecmp(kv.first.c_str(), \"transaction\") == 0)\n\t\t\t\ttransaction = std::move(kv.second);\n\t\t\telse if (strcasecmp(kv.first.c_str(), \"character_set\") == 0)\n\t\t\t{\n\t\t\t\tcharacter_set_ = __mysql_get_character_set(kv.second);\n\t\t\t\tif (character_set_ < 0)\n\t\t\t\t{\n\t\t\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\t\t\tthis->error = WFT_ERR_MYSQL_INVALID_CHARACTER_SET;\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t}\n\t\t\telse if (strcasecmp(kv.first.c_str(), \"character_set_results\") == 0)\n\t\t\t\tres_charset_ = std::move(kv.second);\n\t\t}\n\t}\n\n\tsize_t info_len = username_.size() + password_.size() + db_.size() +\n\t\t\t\t\t res_charset_.size() + 50;\n\tchar *info = new char[info_len];\n\n\tsnprintf(info, info_len, \"%s|user:%s|pass:%s|db:%s|\"\n\t\t\t\t\t\t\t \"charset:%d|rcharset:%s\",\n\t\t\t is_ssl_ ? \"mysqls\" : \"mysql\", username_.c_str(), password_.c_str(),\n\t\t\t db_.c_str(), character_set_, res_charset_.c_str());\n\tthis->WFComplexClientTask::set_transport_type(TT_TCP);\n\n\tif (!transaction.empty())\n\t{\n\t\tthis->set_fixed_addr(true);\n\t\tthis->set_fixed_conn(true);\n\t\tthis->WFComplexClientTask::set_info(info + (\"|txn:\" + transaction));\n\t}\n\telse\n\t\tthis->WFComplexClientTask::set_info(info);\n\n\tdelete []info;\n\treturn true;\n}\n\nbool ComplexMySQLTask::finish_once()\n{\n\tif (!is_user_request_)\n\t{\n\t\tdelete this->get_message_out();\n\t\tdelete this->get_message_in();\n\n\t\tif (this->state == WFT_STATE_SUCCESS && state_ != WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tthis->state = state_;\n\t\t\tthis->error = error_;\n\t\t\tthis->disable_retry();\n\t\t}\n\n\t\tis_user_request_ = true;\n\t\treturn false;\n\t}\n\n\tif (this->is_fixed_conn())\n\t{\n\t\tif (this->state != WFT_STATE_SUCCESS || this->keep_alive_timeo == 0)\n\t\t{\n\t\t\tif (this->target)\n\t\t\t\t((RouteManager::RouteTarget *)this->target)->state = 0;\n\t\t}\n\t}\n\n\treturn true;\n}\n\n/**********Client Factory**********/\n\n// mysql://user:password@host:port/db_name\n// url = \"mysql://admin:123456@192.168.1.101:3301/test\"\n// url = \"mysql://127.0.0.1:3306\"\nWFMySQLTask *WFTaskFactory::create_mysql_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t mysql_callback_t callback)\n{\n\tauto *task = new ComplexMySQLTask(retry_max, std::move(callback));\n\tParsedURI uri;\n\n\tURIParser::parse(url, uri);\n\ttask->init(std::move(uri));\n\tif (task->is_fixed_conn())\n\t\ttask->set_keep_alive(MYSQL_KEEPALIVE_TRANSACTION);\n\telse\n\t\ttask->set_keep_alive(MYSQL_KEEPALIVE_DEFAULT);\n\n\treturn task;\n}\n\nWFMySQLTask *WFTaskFactory::create_mysql_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t mysql_callback_t callback)\n{\n\tauto *task = new ComplexMySQLTask(retry_max, std::move(callback));\n\n\ttask->init(uri);\n\tif (task->is_fixed_conn())\n\t\ttask->set_keep_alive(MYSQL_KEEPALIVE_TRANSACTION);\n\telse\n\t\ttask->set_keep_alive(MYSQL_KEEPALIVE_DEFAULT);\n\n\treturn task;\n}\n\n/**********Server**********/\n\nclass WFMySQLServerTask : public WFServerTask\n{\npublic:\n\tWFMySQLServerTask(CommService *service,\n\t\t\t\t\t std::function& proc):\n\t\tWFServerTask(service, WFGlobal::get_scheduler(), proc)\n\t{}\n\nprotected:\n\tvirtual SubTask *done();\n\tvirtual CommMessageOut *message_out();\n\tvirtual CommMessageIn *message_in();\n};\n\nSubTask *WFMySQLServerTask::done()\n{\n\tif (this->get_seq() == 0)\n\t\tdelete this->get_message_in();\n\n\treturn this->WFServerTask::done();\n}\n\nCommMessageOut *WFMySQLServerTask::message_out()\n{\n\tlong long seqid = this->get_seq();\n\n\tif (seqid == 0)\n\t\tthis->resp.set_ok_packet();\t// always success\n\n\treturn this->WFServerTask::message_out();\n}\n\nCommMessageIn *WFMySQLServerTask::message_in()\n{\n\tlong long seqid = this->get_seq();\n\n\tif (seqid == 0)\n\t\treturn new MySQLAuthRequest;\n\n\treturn this->WFServerTask::message_in();\n}\n\n/**********Server Factory**********/\n\nWFMySQLTask *WFServerTaskFactory::create_mysql_task(CommService *service,\n\t\t\t\t\t\t\tstd::function& process)\n{\n\treturn new WFMySQLServerTask(service, process);\n}\n\n" }, { "path": "src/factory/RedisTaskImpl.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Li Yingxin (liyingxin@sogou-inc.com)\n Liu Kai (liukaidx@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"PackageWrapper.h\"\n#include \"WFTaskError.h\"\n#include \"WFTaskFactory.h\"\n#include \"StringUtil.h\"\n#include \"RedisTaskImpl.inl\"\n\nusing namespace protocol;\n\n#define REDIS_KEEPALIVE_DEFAULT\t\t(60 * 1000)\n#define REDIS_REDIRECT_MAX\t\t\t3\n\n/**********Client**********/\n\nclass ComplexRedisTask : public WFComplexClientTask\n{\npublic:\n\tComplexRedisTask(int retry_max, redis_callback_t&& callback):\n\t\tWFComplexClientTask(retry_max, std::move(callback)),\n\t\tdb_num_(0),\n\t\tis_user_request_(true),\n\t\tredirect_count_(0)\n\t{}\n\nprotected:\n\tvirtual bool check_request();\n\tvirtual CommMessageOut *message_out();\n\tvirtual CommMessageIn *message_in();\n\tvirtual int keep_alive_timeout();\n\tvirtual int first_timeout();\n\tvirtual bool init_success();\n\tvirtual bool finish_once();\n\nprotected:\n\tbool need_redirect();\n\n\tstd::string username_;\n\tstd::string password_;\n\tint db_num_;\n\tbool succ_;\n\tbool is_user_request_;\n\tint redirect_count_;\n};\n\nbool ComplexRedisTask::check_request()\n{\n\tstd::string command;\n\n\tif (this->req.get_command(command) &&\n\t\t(strcasecmp(command.c_str(), \"AUTH\") == 0 ||\n\t\t strcasecmp(command.c_str(), \"SELECT\") == 0 ||\n\t\t strcasecmp(command.c_str(), \"RESET\") == 0 ||\n\t\t strcasecmp(command.c_str(), \"ASKING\") == 0))\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_REDIS_COMMAND_DISALLOWED;\n\t\treturn false;\n\t}\n\n\treturn true;\n}\n\nCommMessageOut *ComplexRedisTask::message_out()\n{\n\tlong long seqid = this->get_seq();\n\n\tif (seqid <= 1)\n\t{\n\t\tif (seqid == 0 && (!password_.empty() || !username_.empty()))\n\t\t{\n\t\t\tauto *auth_req = new RedisRequest;\n\n\t\t\tif (!username_.empty())\n\t\t\t\tauth_req->set_request(\"AUTH\", {username_, password_});\n\t\t\telse\n\t\t\t\tauth_req->set_request(\"AUTH\", {password_});\n\n\t\t\tsucc_ = false;\n\t\t\tis_user_request_ = false;\n\t\t\treturn auth_req;\n\t\t}\n\n\t\tif (db_num_ > 0 &&\n\t\t\t(seqid == 0 || !password_.empty() || !username_.empty()))\n\t\t{\n\t\t\tauto *select_req = new RedisRequest;\n\t\t\tchar buf[32];\n\n\t\t\tsprintf(buf, \"%d\", db_num_);\n\t\t\tselect_req->set_request(\"SELECT\", {buf});\n\n\t\t\tsucc_ = false;\n\t\t\tis_user_request_ = false;\n\t\t\treturn select_req;\n\t\t}\n\t}\n\n\treturn this->WFComplexClientTask::message_out();\n}\n\nCommMessageIn *ComplexRedisTask::message_in()\n{\n\tRedisRequest *req = this->get_req();\n\tRedisResponse *resp = this->get_resp();\n\n\tif (is_user_request_)\n\t\tresp->set_asking(req->is_asking());\n\telse\n\t\tresp->set_asking(false);\n\n\treturn this->WFComplexClientTask::message_in();\n}\n\nint ComplexRedisTask::keep_alive_timeout()\n{\n\tif (this->is_user_request_)\n\t\treturn this->keep_alive_timeo;\n\n\tRedisResponse *resp = this->get_resp();\n\n\tsucc_ = (resp->parse_success() &&\n\t\t\t resp->result_ptr()->type != REDIS_REPLY_TYPE_ERROR);\n\n\treturn succ_ ? REDIS_KEEPALIVE_DEFAULT : 0;\n}\n\nint ComplexRedisTask::first_timeout()\n{\n\treturn is_user_request_ ? this->watch_timeo : 0;\n}\n\nbool ComplexRedisTask::init_success()\n{\n\tenum TransportType type;\n\n\tif (uri_.scheme && strcasecmp(uri_.scheme, \"redis\") == 0)\n\t\ttype = TT_TCP;\n\telse if (uri_.scheme && strcasecmp(uri_.scheme, \"rediss\") == 0)\n\t\ttype = TT_TCP_SSL;\n\telse\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\t\treturn false;\n\t}\n\n\t//todo redis+unix\n\t//https://stackoverflow.com/questions/26964595/whats-the-correct-way-to-use-a-unix-domain-socket-in-requests-framework\n\t//https://stackoverflow.com/questions/27037990/connecting-to-postgres-via-database-url-and-unix-socket-in-rails\n\n\tif (uri_.userinfo)\n\t{\n\t\tchar *p = strchr(uri_.userinfo, ':');\n\t\tif (p)\n\t\t{\n\t\t\tusername_.assign(uri_.userinfo, p);\n\t\t\tpassword_.assign(p + 1);\n\t\t\tStringUtil::url_decode(username_);\n\t\t\tStringUtil::url_decode(password_);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tusername_.assign(uri_.userinfo);\n\t\t\tStringUtil::url_decode(username_);\n\t\t}\n\t}\n\n\tif (uri_.path && uri_.path[0] == '/' && uri_.path[1])\n\t\tdb_num_ = atoi(uri_.path + 1);\n\n\tsize_t info_len = username_.size() + password_.size() + 32 + 32;\n\tchar *info = new char[info_len];\n\n\tsprintf(info, \"redis|user:%s|pass:%s|db:%d\", username_.c_str(),\n\t\t\tpassword_.c_str(), db_num_);\n\tthis->WFComplexClientTask::set_transport_type(type);\n\tthis->WFComplexClientTask::set_info(info);\n\n\tdelete []info;\n\treturn true;\n}\n\nbool ComplexRedisTask::need_redirect()\n{\n\tRedisRequest *client_req = this->get_req();\n\tRedisResponse *client_resp = this->get_resp();\n\tredis_reply_t *reply = client_resp->result_ptr();\n\n\tif (reply->type == REDIS_REPLY_TYPE_ERROR)\n\t{\n\t\tif (reply->str == NULL)\n\t\t\treturn false;\n\n\t\tif (strncasecmp(reply->str, \"NOAUTH \", 7) == 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_REDIS_ACCESS_DENIED;\n\t\t\treturn false;\n\t\t}\n\n\t\tbool asking = false;\n\t\tif (strncasecmp(reply->str, \"ASK \", 4) == 0)\n\t\t\tasking = true;\n\t\telse if (strncasecmp(reply->str, \"MOVED \", 6) != 0)\n\t\t\treturn false;\n\n\t\tif (redirect_count_ >= REDIS_REDIRECT_MAX)\n\t\t\treturn false;\n\n\t\tstd::string err_str(reply->str, reply->len);\n\t\tauto split_result = StringUtil::split_filter_empty(err_str, ' ');\n\t\tif (split_result.size() == 3)\n\t\t{\n\t\t\tclient_req->set_asking(asking);\n\n\t\t\t// format: COMMAND SLOT HOSTPORT\n\t\t\t// example: MOVED/ASK 123 127.0.0.1:6379\n\t\t\tstd::string& hostport = split_result[2];\n\t\t\tredirect_count_++;\n\n\t\t\tParsedURI uri;\n\t\t\tstd::string url;\n\t\t\turl.append(uri_.scheme);\n\t\t\turl.append(\"://\");\n\t\t\turl.append(hostport);\n\n\t\t\tURIParser::parse(url, uri);\n\t\t\tstd::swap(uri.host, uri_.host);\n\t\t\tstd::swap(uri.port, uri_.port);\n\t\t\tstd::swap(uri.state, uri_.state);\n\t\t\tstd::swap(uri.error, uri_.error);\n\n\t\t\treturn true;\n\t\t}\n\t}\n\n\treturn false;\n}\n\nbool ComplexRedisTask::finish_once()\n{\n\tif (!is_user_request_)\n\t{\n\t\tis_user_request_ = true;\n\t\tdelete this->get_message_out();\n\n\t\tif (this->state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tif (succ_)\n\t\t\t\tthis->clear_resp();\n\t\t\telse\n\t\t\t{\n\t\t\t\tthis->disable_retry();\n\t\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\t\tthis->error = WFT_ERR_REDIS_ACCESS_DENIED;\n\t\t\t}\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tif (this->state == WFT_STATE_SUCCESS)\n\t{\n\t\tif (need_redirect())\n\t\t\tthis->set_redirect(uri_);\n\t\telse if (this->state != WFT_STATE_SUCCESS)\n\t\t\tthis->disable_retry();\n\t}\n\n\treturn true;\n}\n\n/****** Redis Subscribe ******/\n\nclass ComplexRedisSubscribeTask : public ComplexRedisTask\n{\npublic:\n\tvirtual int push(const void *buf, size_t size)\n\t{\n\t\tif (finished_)\n\t\t{\n\t\t\terrno = ENOENT;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (!watching_)\n\t\t{\n\t\t\terrno = EAGAIN;\n\t\t\treturn -1;\n\t\t}\n\n\t\treturn this->scheduler->push(buf, size, this);\n\t}\n\nprotected:\n\tvirtual CommMessageIn *message_in()\n\t{\n\t\tif (!is_user_request_)\n\t\t\treturn this->ComplexRedisTask::message_in();\n\n\t\treturn &wrapper_;\n\t}\n\n\tvirtual int first_timeout()\n\t{\n\t\treturn watching_ ? this->watch_timeo : 0;\n\t}\n\nprotected:\n\tclass SubscribeWrapper : public PackageWrapper\n\t{\n\tprotected:\n\t\tvirtual ProtocolMessage *next_in(ProtocolMessage *message);\n\n\tprotected:\n\t\tComplexRedisSubscribeTask *task_;\n\n\tpublic:\n\t\tSubscribeWrapper(ComplexRedisSubscribeTask *task) :\n\t\t\tPackageWrapper(task->get_resp())\n\t\t{\n\t\t\ttask_ = task;\n\t\t}\n\t};\n\nprotected:\n\tSubscribeWrapper wrapper_;\n\tbool watching_;\n\tbool finished_;\n\tstd::function extract_;\n\npublic:\n\tComplexRedisSubscribeTask(std::function&& extract,\n\t\t\t\t\t\t\t redis_callback_t&& callback) :\n\t\tComplexRedisTask(0, std::move(callback)),\n\t\twrapper_(this),\n\t\textract_(std::move(extract))\n\t{\n\t\twatching_ = false;\n\t\tfinished_ = false;\n\t}\n};\n\nProtocolMessage *\nComplexRedisSubscribeTask::SubscribeWrapper::next_in(ProtocolMessage *message)\n{\n\tredis_reply_t *reply = task_->resp.result_ptr();\n\n\tif (reply->type != REDIS_REPLY_TYPE_ARRAY)\n\t{\n\t\ttask_->finished_ = true;\n\t\treturn NULL;\n\t}\n\n\tif (reply->elements == 3 &&\n\t\treply->element[2]->type == REDIS_REPLY_TYPE_INTEGER &&\n\t\treply->element[2]->integer == 0)\n\t{\n\t\ttask_->finished_ = true;\n\t}\n\n\ttask_->watching_ = true;\n\ttask_->extract_(task_);\n\n\tRedisResponse resp;\n\t*(protocol::ProtocolMessage *)&resp = std::move(task_->resp);\n\ttask_->resp = std::move(resp);\n\treturn task_->finished_ ? NULL : &task_->resp;\n}\n\n/**********Factory**********/\n\n// redis://:password@host:port/db_num\n// url = \"redis://:admin@192.168.1.101:6001/3\"\n// url = \"redis://127.0.0.1:6379\"\nWFRedisTask *WFTaskFactory::create_redis_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t redis_callback_t callback)\n{\n\tauto *task = new ComplexRedisTask(retry_max, std::move(callback));\n\tParsedURI uri;\n\n\tURIParser::parse(url, uri);\n\ttask->init(std::move(uri));\n\ttask->set_keep_alive(REDIS_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\nWFRedisTask *WFTaskFactory::create_redis_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t\t redis_callback_t callback)\n{\n\tauto *task = new ComplexRedisTask(retry_max, std::move(callback));\n\n\ttask->init(uri);\n\ttask->set_keep_alive(REDIS_KEEPALIVE_DEFAULT);\n\treturn task;\n}\n\nWFRedisTask *\n__WFRedisTaskFactory::create_subscribe_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\textract_t extract,\n\t\t\t\t\t\t\t\t\t\t\tredis_callback_t callback)\n{\n\tauto *task = new ComplexRedisSubscribeTask(std::move(extract),\n\t\t\t\t\t\t\t\t\t\t\t std::move(callback));\n\tParsedURI uri;\n\n\tURIParser::parse(url, uri);\n\ttask->init(std::move(uri));\n\treturn task;\n}\n\nWFRedisTask *\n__WFRedisTaskFactory::create_subscribe_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\textract_t extract,\n\t\t\t\t\t\t\t\t\t\t\tredis_callback_t callback)\n{\n\tauto *task = new ComplexRedisSubscribeTask(std::move(extract),\n\t\t\t\t\t\t\t\t\t\t\t std::move(callback));\n\n\ttask->init(uri);\n\treturn task;\n}\n\n" }, { "path": "src/factory/RedisTaskImpl.inl", "content": "/*\n Copyright (c) 2024 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \"WFTaskFactory.h\"\n\n// Internal, for WFRedisSubscribeTask only.\n\nclass __WFRedisTaskFactory\n{\nprivate:\n\tusing extract_t = std::function;\n\npublic:\n\tstatic WFRedisTask *create_subscribe_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t\t redis_callback_t callback);\n\n\tstatic WFRedisTask *create_subscribe_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t extract_t extract,\n\t\t\t\t\t\t\t\t\t\t\t redis_callback_t callback);\n};\n\n" }, { "path": "src/factory/WFAlgoTaskFactory.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFALGOTASKFACTORY_H_\n#define _WFALGOTASKFACTORY_H_\n\n#include \n#include \n#include \n#include \"WFTask.h\"\n\nnamespace algorithm\n{\n\ntemplate\nstruct SortInput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct SortOutput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct MergeInput\n{\n\tT *first1;\n\tT *last1;\n\tT *first2;\n\tT *last2;\n\tT *d_first;\n};\n\ntemplate\nstruct MergeOutput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct ShuffleInput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct ShuffleOutput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct RemoveInput\n{\n\tT *first;\n\tT *last;\n\tT value;\n};\n\ntemplate\nstruct RemoveOutput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct UniqueInput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct UniqueOutput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct ReverseInput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct ReverseOutput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nstruct RotateInput\n{\n\tT *first;\n\tT *middle;\n\tT *last;\n};\n\ntemplate\nstruct RotateOutput\n{\n\tT *first;\n\tT *last;\n};\n\ntemplate\nusing ReduceInput = std::vector>;\n\ntemplate\nusing ReduceOutput = std::vector>;\n\n} /* namespace algorithm */\n\ntemplate\nusing WFSortTask = WFThreadTask,\n\t\t\t\t\t\t\t\talgorithm::SortOutput>;\ntemplate\nusing sort_callback_t = std::function *)>;\n\ntemplate\nusing WFMergeTask = WFThreadTask,\n\t\t\t\t\t\t\t\t algorithm::MergeOutput>;\ntemplate\nusing merge_callback_t = std::function *)>;\n\ntemplate\nusing WFShuffleTask = WFThreadTask,\n\t\t\t\t\t\t\t\t algorithm::ShuffleOutput>;\ntemplate\nusing shuffle_callback_t = std::function *)>;\n\ntemplate\nusing WFRemoveTask = WFThreadTask,\n\t\t\t\t\t\t\t\t algorithm::RemoveOutput>;\ntemplate\nusing remove_callback_t = std::function *)>;\n\ntemplate\nusing WFUniqueTask = WFThreadTask,\n\t\t\t\t\t\t\t\t algorithm::UniqueOutput>;\ntemplate\nusing unique_callback_t = std::function *)>;\n\ntemplate\nusing WFReverseTask = WFThreadTask,\n\t\t\t\t\t\t\t\t algorithm::ReverseOutput>;\ntemplate\nusing reverse_callback_t = std::function *)>;\n\ntemplate\nusing WFRotateTask = WFThreadTask,\n\t\t\t\t\t\t\t\t algorithm::RotateOutput>;\ntemplate\nusing rotate_callback_t = std::function *)>;\n\nclass WFAlgoTaskFactory\n{\npublic:\n\ttemplate>\n\tstatic WFSortTask *create_sort_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFSortTask *create_sort_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t CMP compare,\n\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFSortTask *create_psort_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\tT *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\tCB callback);\n\n\ttemplate>\n\tstatic WFSortTask *create_psort_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\tT *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\tCMP compare,\n\t\t\t\t\t\t\t\t\t\t\tCB callback);\n\n\ttemplate>\n\tstatic WFMergeTask *create_merge_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t T *first1, T *last1,\n\t\t\t\t\t\t\t\t\t\t\t T *first2, T *last2,\n\t\t\t\t\t\t\t\t\t\t\t T *d_first,\n\t\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFMergeTask *create_merge_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t T *first1, T *last1,\n\t\t\t\t\t\t\t\t\t\t\t T *first2, T *last2,\n\t\t\t\t\t\t\t\t\t\t\t T *d_first,\n\t\t\t\t\t\t\t\t\t\t\t CMP compare,\n\t\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFShuffleTask *create_shuffle_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFShuffleTask *create_shuffle_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t URBG generator,\n\t\t\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFRemoveTask *create_remove_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t T value,\n\t\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFUniqueTask *create_unique_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFReverseTask *create_reverse_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t CB callback);\n\n\ttemplate>\n\tstatic WFRotateTask *create_rotate_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t T *first, T *middle, T *last,\n\t\t\t\t\t\t\t\t\t\t\t CB callback);\n};\n\n#include \"WFAlgoTaskFactory.inl\"\n\n#endif\n\n" }, { "path": "src/factory/WFAlgoTaskFactory.inl", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \"Workflow.h\"\n#include \"WFGlobal.h\"\n\n/********** Classes without CMP **********/\n\ntemplate\nclass __WFSortTask : public WFSortTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tstd::sort(this->input.first, this->input.last);\n\t\tthis->output.first = this->input.first;\n\t\tthis->output.last = this->input.last;\n\t}\n\npublic:\n\t__WFSortTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t T *first, T *last,\n\t\t\t\t sort_callback_t&& cb) :\n\t\tWFSortTask(queue, executor, std::move(cb))\n\t{\n\t\tthis->input.first = first;\n\t\tthis->input.last = last;\n\t\tthis->output.first = NULL;\n\t\tthis->output.last = NULL;\n\t}\n};\n\ntemplate\nclass __WFMergeTask : public WFMergeTask\n{\nprotected:\n\tvirtual void execute();\n\npublic:\n\t__WFMergeTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t T *first1, T *last1, T *first2, T *last2, T *d_first,\n\t\t\t\t merge_callback_t&& cb) :\n\t\tWFMergeTask(queue, executor, std::move(cb))\n\t{\n\t\tthis->input.first1 = first1;\n\t\tthis->input.last1 = last1;\n\t\tthis->input.first2 = first2;\n\t\tthis->input.last2 = last2;\n\t\tthis->input.d_first = d_first;\n\t\tthis->output.first = NULL;\n\t\tthis->output.last = NULL;\n\t}\n};\n\ntemplate\nvoid __WFMergeTask::execute()\n{\n\tauto *input = &this->input;\n\tauto *output = &this->output;\n\n\tif (input->first1 == input->d_first && input->last1 == input->first2)\n\t{\n\t\tstd::inplace_merge(input->first1, input->first2, input->last2);\n\t\toutput->last = input->last2;\n\t}\n\telse if (input->first2 == input->d_first && input->last2 == input->first1)\n\t{\n\t\tstd::inplace_merge(input->first2, input->first1, input->last1);\n\t\toutput->last = input->last1;\n\t}\n\telse\n\t{\n\t\toutput->last = std::merge(input->first1, input->last1,\n\t\t\t\t\t\t\t\t input->first2, input->last2,\n\t\t\t\t\t\t\t\t input->d_first);\n\t}\n\n\toutput->first = input->d_first;\n}\n\ntemplate\nclass __WFParSortTask : public __WFSortTask\n{\npublic:\n\tvirtual void dispatch();\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tif (this->flag)\n\t\t\treturn series_of(this)->pop();\n\n\t\treturn this->WFSortTask::done();\n\t}\n\n\tvirtual void execute();\n\nprotected:\n\tint depth;\n\tint flag;\n\npublic:\n\t__WFParSortTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t\tT *first, T *last, int depth,\n\t\t\t\t\tsort_callback_t&& cb) :\n\t\t__WFSortTask(queue, executor, first, last, std::move(cb))\n\t{\n\t\tthis->depth = depth;\n\t\tthis->flag = 0;\n\t}\n};\n\ntemplate\nvoid __WFParSortTask::dispatch()\n{\n\tsize_t n = this->input.last - this->input.first;\n\n\tif (!this->flag && this->depth < 7 && n >= 32)\n\t{\n\t\tSeriesWork *series = series_of(this);\n\t\tT *middle = this->input.first + n / 2;\n\t\tauto *task1 =\n\t\t\tnew __WFParSortTask(this->queue, this->executor,\n\t\t\t\t\t\t\t\t this->input.first, middle,\n\t\t\t\t\t\t\t\t this->depth + 1,\n\t\t\t\t\t\t\t\t nullptr);\n\t\tauto *task2 =\n\t\t\tnew __WFParSortTask(this->queue, this->executor,\n\t\t\t\t\t\t\t\t middle, this->input.last,\n\t\t\t\t\t\t\t\t this->depth + 1,\n\t\t\t\t\t\t\t\t nullptr);\n\t\tSeriesWork *sub_series[2] = {\n\t\t\tWorkflow::create_series_work(task1, nullptr),\n\t\t\tWorkflow::create_series_work(task2, nullptr)\n\t\t};\n\t\tParallelWork *parallel =\n\t\t\tWorkflow::create_parallel_work(sub_series, 2, nullptr);\n\n\t\tseries->push_front(this);\n\t\tseries->push_front(parallel);\n\t\tthis->flag = 1;\n\t\tthis->subtask_done();\n\t}\n\telse\n\t\tthis->__WFSortTask::dispatch();\n}\n\ntemplate\nvoid __WFParSortTask::execute()\n{\n\tif (this->flag)\n\t{\n\t\tsize_t n = this->input.last - this->input.first;\n\t\tT *middle = this->input.first + n / 2;\n\n\t\tstd::inplace_merge(this->input.first, middle, this->input.last);\n\t\tthis->output.first = this->input.first;\n\t\tthis->output.last = this->input.last;\n\t\tthis->flag = 0;\n\t}\n\telse\n\t\tthis->__WFSortTask::execute();\n}\n\n/********** Classes with CMP **********/\n\ntemplate\nclass __WFSortTaskCmp : public __WFSortTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tstd::sort(this->input.first, this->input.last,\n\t\t\t\t std::move(this->compare));\n\t\tthis->output.first = this->input.first;\n\t\tthis->output.last = this->input.last;\n\t}\n\nprotected:\n\tCMP compare;\n\npublic:\n\t__WFSortTaskCmp(ExecQueue *queue, Executor *executor,\n\t\t\t\t\tT *first, T *last, CMP&& cmp,\n\t\t\t\t\tsort_callback_t&& cb) :\n\t\t__WFSortTask(queue, executor, first, last, std::move(cb)),\n\t\tcompare(std::move(cmp))\n\t{\n\t}\n};\n\ntemplate\nclass __WFMergeTaskCmp : public __WFMergeTask\n{\nprotected:\n\tvirtual void execute();\n\nprotected:\n\tCMP compare;\n\npublic:\n\t__WFMergeTaskCmp(ExecQueue *queue, Executor *executor,\n\t\t\t\t\t T *first1, T *last1, T *first2, T *last2,\n\t\t\t\t\t T *d_first, CMP&& cmp,\n\t\t\t\t\t merge_callback_t&& cb) :\n\t\t__WFMergeTask(queue, executor, first1, last1, first2, last2, d_first,\n\t\t\t\t\t\t std::move(cb)),\n\t\tcompare(std::move(cmp))\n\t{\n\t}\n};\n\ntemplate\nvoid __WFMergeTaskCmp::execute()\n{\n\tauto *input = &this->input;\n\tauto *output = &this->output;\n\n\tif (input->first1 == input->d_first && input->last1 == input->first2)\n\t{\n\t\tstd::inplace_merge(input->first1, input->first2, input->last2,\n\t\t\t\t\t\t std::move(this->compare));\n\t\toutput->last = input->last2;\n\t}\n\telse if (input->first2 == input->d_first && input->last2 == input->first1)\n\t{\n\t\tstd::inplace_merge(input->first2, input->first1, input->last1,\n\t\t\t\t\t\t std::move(this->compare));\n\t\toutput->last = input->last1;\n\t}\n\telse\n\t{\n\t\toutput->last = std::merge(input->first1, input->last1,\n\t\t\t\t\t\t\t\t input->first2, input->last2,\n\t\t\t\t\t\t\t\t input->d_first,\n\t\t\t\t\t\t\t\t std::move(this->compare));\n\t}\n\n\toutput->first = input->d_first;\n}\n\ntemplate\nclass __WFParSortTaskCmp : public __WFSortTaskCmp\n{\npublic:\n\tvirtual void dispatch();\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tif (this->flag)\n\t\t\treturn series_of(this)->pop();\n\n\t\treturn this->WFSortTask::done();\n\t}\n\n\tvirtual void execute();\n\nprotected:\n\tint depth;\n\tint flag;\n\npublic:\n\t__WFParSortTaskCmp(ExecQueue *queue, Executor *executor,\n\t\t\t\t\t T *first, T *last, CMP cmp, int depth,\n\t\t\t\t\t sort_callback_t&& cb) :\n\t\t__WFSortTaskCmp(queue, executor, first, last, std::move(cmp),\n\t\t\t\t\t\t\t\tstd::move(cb))\n\t{\n\t\tthis->depth = depth;\n\t\tthis->flag = 0;\n\t}\n};\n\ntemplate\nvoid __WFParSortTaskCmp::dispatch()\n{\n\tsize_t n = this->input.last - this->input.first;\n\n\tif (!this->flag && this->depth < 7 && n >= 32)\n\t{\n\t\tSeriesWork *series = series_of(this);\n\t\tT *middle = this->input.first + n / 2;\n\t\tauto *task1 =\n\t\t\tnew __WFParSortTaskCmp(this->queue, this->executor,\n\t\t\t\t\t\t\t\t\t\t this->input.first, middle,\n\t\t\t\t\t\t\t\t\t\t this->compare, this->depth + 1,\n\t\t\t\t\t\t\t\t\t\t nullptr);\n\t\tauto *task2 =\n\t\t\tnew __WFParSortTaskCmp(this->queue, this->executor,\n\t\t\t\t\t\t\t\t\t\t middle, this->input.last,\n\t\t\t\t\t\t\t\t\t\t this->compare, this->depth + 1,\n\t\t\t\t\t\t\t\t\t\t nullptr);\n\t\tSeriesWork *sub_series[2] = {\n\t\t\tWorkflow::create_series_work(task1, nullptr),\n\t\t\tWorkflow::create_series_work(task2, nullptr)\n\t\t};\n\t\tParallelWork *parallel =\n\t\t\tWorkflow::create_parallel_work(sub_series, 2, nullptr);\n\n\t\tseries->push_front(this);\n\t\tseries->push_front(parallel);\n\t\tthis->flag = 1;\n\t\tthis->subtask_done();\n\t}\n\telse\n\t\tthis->__WFSortTaskCmp::dispatch();\n}\n\ntemplate\nvoid __WFParSortTaskCmp::execute()\n{\n\tif (this->flag)\n\t{\n\t\tsize_t n = this->input.last - this->input.first;\n\t\tT *middle = this->input.first + n / 2;\n\n\t\tstd::inplace_merge(this->input.first, middle, this->input.last,\n\t\t\t\t\t\t std::move(this->compare));\n\t\tthis->output.first = this->input.first;\n\t\tthis->output.last = this->input.last;\n\t\tthis->flag = 0;\n\t}\n\telse\n\t\tthis->__WFSortTaskCmp::execute();\n}\n\n/********** Factory functions without CMP **********/\n\ntemplate\nWFSortTask *WFAlgoTaskFactory::create_sort_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFSortTask(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t first, last,\n\t\t\t\t\t\t\t std::move(callback));\n}\n\ntemplate\nWFMergeTask *WFAlgoTaskFactory::create_merge_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *first1, T *last1,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *first2, T *last2,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *d_first,\n\t\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFMergeTask(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\tfirst1, last1, first2, last2, d_first,\n\t\t\t\t\t\t\t\tstd::move(callback));\n}\n\ntemplate\nWFSortTask *WFAlgoTaskFactory::create_psort_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\tT *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t\tCB callback)\n{\n\treturn new __WFParSortTask(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t first, last, 0,\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n/********** Factory functions with CMP **********/\n\ntemplate\nWFSortTask *WFAlgoTaskFactory::create_sort_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t CMP compare,\n\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFSortTaskCmp(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t\t first, last, std::move(compare),\n\t\t\t\t\t\t\t\t\t std::move(callback));\n}\n\ntemplate\nWFMergeTask *WFAlgoTaskFactory::create_merge_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *first1, T *last1,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *first2, T *last2,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *d_first,\n\t\t\t\t\t\t\t\t\t\t\t\t\t CMP compare,\n\t\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFMergeTaskCmp(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t\t\tWFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t\t\tfirst1, last1, first2, last2,\n\t\t\t\t\t\t\t\t\t\td_first, std::move(compare),\n\t\t\t\t\t\t\t\t\t\tstd::move(callback));\n}\n\ntemplate\nWFSortTask *WFAlgoTaskFactory::create_psort_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\tT *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t\tCMP compare,\n\t\t\t\t\t\t\t\t\t\t\t\t\tCB callback)\n{\n\treturn new __WFParSortTaskCmp(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t\t\t first, last, std::move(compare), 0,\n\t\t\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n/****************** Shuffle ******************/\n\ntemplate\nclass __WFShuffleTask : public WFShuffleTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tstd::shuffle(this->input.first, this->input.last,\n\t\t\t\t\t std::mt19937_64(rand()));\n\t\tthis->output.first = this->input.first;\n\t\tthis->output.last = this->input.last;\n\t}\n\npublic:\n\t__WFShuffleTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t\tT *first, T *last,\n\t\t\t\t\tshuffle_callback_t&& cb) :\n\t\tWFShuffleTask(queue, executor, std::move(cb))\n\t{\n\t\tthis->input.first = first;\n\t\tthis->input.last = last;\n\t\tthis->output.first = NULL;\n\t\tthis->output.last = NULL;\n\t}\n};\n\ntemplate\nclass __WFShuffleTaskGen : public __WFShuffleTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tstd::shuffle(this->input.first, this->input.last,\n\t\t\t\t\t std::move(this->generator));\n\t\tthis->output.first = this->input.first;\n\t\tthis->output.last = this->input.last;\n\t}\n\nprotected:\n\tURBG generator;\n\npublic:\n\t__WFShuffleTaskGen(ExecQueue *queue, Executor *executor,\n\t\t\t\t\t T *first, T *last, URBG&& gen,\n\t\t\t\t\t shuffle_callback_t&& cb) :\n\t\t__WFShuffleTask(queue, executor, std::move(cb)),\n\t\tgenerator(std::move(gen))\n\t{\n\t}\n};\n\ntemplate\nWFShuffleTask *WFAlgoTaskFactory::create_shuffle_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFShuffleTask(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t first, last,\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\ntemplate\nWFShuffleTask *WFAlgoTaskFactory::create_shuffle_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t URBG generator,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFShuffleTaskGen(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t\t\t first, last, std::move(generator),\n\t\t\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n/****************** Remove ******************/\n\ntemplate\nclass __WFRemoveTask : public WFRemoveTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tthis->output.last = std::remove(this->input.first, this->input.last,\n\t\t\t\t\t\t\t\t\t\tthis->input.value);\n\t\tthis->output.first = this->input.first;\n\t}\n\npublic:\n\t__WFRemoveTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t T *first, T *last, T&& value,\n\t\t\t\t remove_callback_t&& cb) :\n\t\tWFRemoveTask(queue, executor, std::move(cb))\n\t{\n\t\tthis->input.first = first;\n\t\tthis->input.last = last;\n\t\tthis->input.value = std::move(value);\n\t\tthis->output.first = NULL;\n\t\tthis->output.last = NULL;\n\t}\n};\n\ntemplate\nWFRemoveTask *WFAlgoTaskFactory::create_remove_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T value,\n\t\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFRemoveTask(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t first, last, std::move(value),\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n/****************** Unique ******************/\n\ntemplate\nclass __WFUniqueTask : public WFUniqueTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tthis->output.last = std::unique(this->input.first, this->input.last);\n\t\tthis->output.first = this->input.first;\n\t}\n\npublic:\n\t__WFUniqueTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t T *first, T *last,\n\t\t\t\t unique_callback_t&& cb) :\n\t\tWFUniqueTask(queue, executor, std::move(cb))\n\t{\n\t\tthis->input.first = first;\n\t\tthis->input.last = last;\n\t\tthis->output.first = NULL;\n\t\tthis->output.last = NULL;\n\t}\n};\n\ntemplate\nWFUniqueTask *WFAlgoTaskFactory::create_unique_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFUniqueTask(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t first, last,\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n/****************** Reverse ******************/\n\ntemplate\nclass __WFReverseTask : public WFReverseTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tstd::reverse(this->input.first, this->input.last);\n\t\tthis->output.first = this->input.first;\n\t\tthis->output.last = this->input.last;\n\t}\n\npublic:\n\t__WFReverseTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t\tT *first, T *last,\n\t\t\t\t\treverse_callback_t&& cb) :\n\t\tWFReverseTask(queue, executor, std::move(cb))\n\t{\n\t\tthis->input.first = first;\n\t\tthis->input.last = last;\n\t\tthis->output.first = NULL;\n\t\tthis->output.last = NULL;\n\t}\n};\n\ntemplate\nWFReverseTask *WFAlgoTaskFactory::create_reverse_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFReverseTask(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t first, last,\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n/****************** Rotate ******************/\n\ntemplate\nclass __WFRotateTask : public WFRotateTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tstd::rotate(this->input.first, this->input.middle, this->input.last);\n\t\tthis->output.first = this->input.first;\n\t\tthis->output.last = this->input.last;\n\t}\n\npublic:\n\t__WFRotateTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t\tT *first, T* middle, T *last,\n\t\t\t\t\trotate_callback_t&& cb) :\n\t\tWFRotateTask(queue, executor, std::move(cb))\n\t{\n\t\tthis->input.first = first;\n\t\tthis->input.middle = middle;\n\t\tthis->input.last = last;\n\t\tthis->output.first = NULL;\n\t\tthis->output.last = NULL;\n\t}\n};\n\ntemplate\nWFRotateTask *WFAlgoTaskFactory::create_rotate_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t T *first, T *middle, T *last,\n\t\t\t\t\t\t\t\t\t\t\t\t\t CB callback)\n{\n\treturn new __WFRotateTask(WFGlobal::get_exec_queue(name),\n\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t first, middle, last,\n\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n" }, { "path": "src/factory/WFConnection.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFCONNECTION_H_\n#define _WFCONNECTION_H_\n\n#include \n#include \n#include \n#include \"Communicator.h\"\n\nclass WFConnection : public CommConnection\n{\npublic:\n\tvoid *get_context() const\n\t{\n\t\treturn this->context;\n\t}\n\n\tvoid set_context(void *context, std::function deleter)\n\t{\n\t\tthis->context = context;\n\t\tthis->deleter = std::move(deleter);\n\t}\n\n\tvoid set_context(void *context)\n\t{\n\t\tthis->context = context;\n\t}\n\n\tvoid *test_set_context(void *test_context, void *new_context,\n\t\t\t\t\t\t std::function deleter)\n\t{\n\t\tif (this->context.compare_exchange_strong(test_context, new_context))\n\t\t{\n\t\t\tthis->deleter = std::move(deleter);\n\t\t\treturn new_context;\n\t\t}\n\n\t\treturn test_context;\n\t}\n\n\tvoid *test_set_context(void *test_context, void *new_context)\n\t{\n\t\tif (this->context.compare_exchange_strong(test_context, new_context))\n\t\t\treturn new_context;\n\n\t\treturn test_context;\n\t}\n\nprivate:\n\tstd::atomic context;\n\tstd::function deleter;\n\npublic:\n\tWFConnection() : context(NULL) { }\n\nprotected:\n\tvirtual ~WFConnection()\n\t{\n\t\tif (this->deleter)\n\t\t\tthis->deleter(this->context);\n\t}\n};\n\n#endif\n\n" }, { "path": "src/factory/WFGraphTask.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \"Workflow.h\"\n#include \"WFGraphTask.h\"\n\nSubTask *WFGraphNode::done()\n{\n\tSeriesWork *series = series_of(this);\n\n\tif (!this->user_data)\n\t{\n\t\tthis->value = 1;\n\t\tthis->user_data = (void *)1;\n\t}\n\telse\n\t\tdelete this;\n\n\treturn series->pop();\n}\n\nWFGraphNode::~WFGraphNode()\n{\n\tif (this->user_data)\n\t{\n\t\tif (series_of(this)->is_canceled())\n\t\t{\n\t\t\tfor (WFGraphNode *node : this->successors)\n\t\t\t\tseries_of(node)->SeriesWork::cancel();\n\t\t}\n\n\t\tfor (WFGraphNode *node : this->successors)\n\t\t\tnode->WFCounterTask::count();\n\t}\n}\n\nWFGraphNode& WFGraphTask::create_graph_node(SubTask *task)\n{\n\tWFGraphNode *node = new WFGraphNode;\n\tSeriesWork *series = Workflow::create_series_work(node, node, nullptr);\n\n\tseries->push_back(task);\n\tthis->parallel->add_series(series);\n\treturn *node;\n}\n\nvoid WFGraphTask::dispatch()\n{\n\tSeriesWork *series = series_of(this);\n\n\tif (this->parallel)\n\t{\n\t\tseries->push_front(this);\n\t\tseries->push_front(this->parallel);\n\t\tthis->parallel = NULL;\n\t}\n\telse\n\t\tthis->state = WFT_STATE_SUCCESS;\n\n\tthis->subtask_done();\n}\n\nSubTask *WFGraphTask::done()\n{\n\tSeriesWork *series = series_of(this);\n\n\tif (this->state == WFT_STATE_SUCCESS)\n\t{\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t}\n\n\treturn series->pop();\n}\n\nWFGraphTask::~WFGraphTask()\n{\n\tif (this->parallel)\n\t{\n\t\tfor (SeriesWork *series : *this->parallel)\n\t\t\tseries->unset_last_task();\n\n\t\tthis->parallel->dismiss();\n\t}\n}\n" }, { "path": "src/factory/WFGraphTask.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFGRAPHTASK_H_\n#define _WFGRAPHTASK_H_\n\n#include \n#include \n#include \n#include \"Workflow.h\"\n#include \"WFTask.h\"\n\nclass WFGraphNode : protected WFCounterTask\n{\npublic:\n\tvoid precede(WFGraphNode& node)\n\t{\n\t\tnode.value++;\n\t\tthis->successors.push_back(&node);\n\t}\n\n\tvoid succeed(WFGraphNode& node)\n\t{\n\t\tnode.precede(*this);\n\t}\n\nprotected:\n\tvirtual SubTask *done();\n\nprotected:\n\tstd::vector successors;\n\nprotected:\n\tWFGraphNode() : WFCounterTask(0, nullptr) { }\n\tvirtual ~WFGraphNode();\n\tfriend class WFGraphTask;\n};\n\nstatic inline WFGraphNode& operator --(WFGraphNode& node, int)\n{\n\treturn node;\n}\n\nstatic inline WFGraphNode& operator > (WFGraphNode& prec, WFGraphNode& succ)\n{\n\tprec.precede(succ);\n\treturn succ;\n}\n\nstatic inline WFGraphNode& operator < (WFGraphNode& succ, WFGraphNode& prec)\n{\n\tsucc.succeed(prec);\n\treturn prec;\n}\n\nstatic inline WFGraphNode& operator --(WFGraphNode& node)\n{\n\treturn node;\n}\n\nclass WFGraphTask : public WFGenericTask\n{\npublic:\n\tWFGraphNode& create_graph_node(SubTask *task);\n\npublic:\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual void dispatch();\n\tvirtual SubTask *done();\n\nprotected:\n\tParallelWork *parallel;\n\tstd::function callback;\n\npublic:\n\tWFGraphTask(std::function&& cb) :\n\t\tcallback(std::move(cb))\n\t{\n\t\tthis->parallel = Workflow::create_parallel_work(nullptr);\n\t}\n\nprotected:\n\tvirtual ~WFGraphTask();\n};\n\n#endif\n\n" }, { "path": "src/factory/WFMessageQueue.cc", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \"list.h\"\n#include \"WFTask.h\"\n#include \"WFMessageQueue.h\"\n\nclass __MQConditional : public WFConditional\n{\npublic:\n\tstruct list_head list;\n\tstruct WFMessageQueue::Data *data;\n\npublic:\n\tvirtual void dispatch();\n\tvirtual void signal(void *msg) { }\n\npublic:\n\t__MQConditional(SubTask *task, void **msgbuf,\n\t\t\t\t\tstruct WFMessageQueue::Data *data) :\n\t\tWFConditional(task, msgbuf)\n\t{\n\t\tthis->data = data;\n\t}\n\n\t__MQConditional(SubTask *task,\n\t\t\t\t\tstruct WFMessageQueue::Data *data) :\n\t\tWFConditional(task)\n\t{\n\t\tthis->data = data;\n\t}\n};\n\nvoid __MQConditional::dispatch()\n{\n\tstruct WFMessageQueue::Data *data = this->data;\n\n\tdata->mutex.lock();\n\tif (!list_empty(&data->msg_list))\n\t\tthis->WFConditional::signal(data->pop());\n\telse\n\t\tlist_add_tail(&this->list, &data->wait_list);\n\n\tdata->mutex.unlock();\n\tthis->WFConditional::dispatch();\n}\n\nWFConditional *WFMessageQueue::get(SubTask *task, void **msgbuf)\n{\n\treturn new __MQConditional(task, msgbuf, &this->data);\n}\n\nWFConditional *WFMessageQueue::get(SubTask *task)\n{\n\treturn new __MQConditional(task, &this->data);\n}\n\nvoid WFMessageQueue::post(void *msg)\n{\n\tstruct WFMessageQueue::Data *data = &this->data;\n\tWFConditional *cond;\n\n\tdata->mutex.lock();\n\tif (!list_empty(&data->wait_list))\n\t{\n\t\tcond = list_entry(data->wait_list.next, __MQConditional, list);\n\t\tlist_del(data->wait_list.next);\n\t}\n\telse\n\t{\n\t\tcond = NULL;\n\t\tthis->push(msg);\n\t}\n\n\tdata->mutex.unlock();\n\tif (cond)\n\t\tcond->WFConditional::signal(msg);\n}\n\n" }, { "path": "src/factory/WFMessageQueue.h", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFMESSAGEQUEUE_H_\n#define _WFMESSAGEQUEUE_H_\n\n#include \n#include \"list.h\"\n#include \"WFTask.h\"\n\nclass WFMessageQueue\n{\npublic:\n\tWFConditional *get(SubTask *task, void **msgbuf);\n\tWFConditional *get(SubTask *task);\n\tvoid post(void *msg);\n\npublic:\n\tstruct Data\n\t{\n\t\tvoid *pop() { return this->queue->pop(); }\n\t\tvoid push(void *msg) { this->queue->push(msg); }\n\n\t\tstruct list_head msg_list;\n\t\tstruct list_head wait_list;\n\t\tstd::mutex mutex;\n\t\tWFMessageQueue *queue;\n\t};\n\nprotected:\n\tstruct MessageEntry\n\t{\n\t\tstruct list_head list;\n\t\tvoid *msg;\n\t};\n\nprotected:\n\tvirtual void *pop()\n\t{\n\t\tstruct MessageEntry *entry;\n\t\tvoid *msg;\n\n\t\tentry = list_entry(this->data.msg_list.next, struct MessageEntry, list);\n\t\tlist_del(&entry->list);\n\t\tmsg = entry->msg;\n\t\tdelete entry;\n\n\t\treturn msg;\n\t}\n\n\tvirtual void push(void *msg)\n\t{\n\t\tstruct MessageEntry *entry = new struct MessageEntry;\n\t\tentry->msg = msg;\n\t\tlist_add_tail(&entry->list, &this->data.msg_list);\n\t}\n\nprotected:\n\tstruct Data data;\n\npublic:\n\tWFMessageQueue()\n\t{\n\t\tINIT_LIST_HEAD(&this->data.msg_list);\n\t\tINIT_LIST_HEAD(&this->data.wait_list);\n\t\tthis->data.queue = this;\n\t}\n\n\tvirtual ~WFMessageQueue() { }\n};\n\n#endif\n\n" }, { "path": "src/factory/WFOperator.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _WFOPERATOR_H_\n#define _WFOPERATOR_H_\n\n#include \"Workflow.h\"\n\n/**\n * @file WFOperator.h\n * @brief Workflow Series/Parallel/Task operator\n */\n\n/**\n * @brief S=S>P\n * @note Equivalent to x.push_back(&y)\n*/\nstatic inline SeriesWork& operator>(SeriesWork& x, ParallelWork& y);\n\n/**\n * @brief S=P>S\n * @note Equivalent to y.push_front(&x)\n*/\nstatic inline SeriesWork& operator>(ParallelWork& x, SeriesWork& y);\n\n/**\n * @brief S=S>t\n * @note Equivalent to x.push_back(&y)\n*/\nstatic inline SeriesWork& operator>(SeriesWork& x, SubTask& y);\n\n/**\n * @brief S=S>t\n * @note Equivalent to x.push_back(y)\n*/\nstatic inline SeriesWork& operator>(SeriesWork& x, SubTask *y);\n\n/**\n * @brief S=t>S\n * @note Equivalent to y.push_front(&x)\n*/\nstatic inline SeriesWork& operator>(SubTask& x, SeriesWork& y);\n\n/**\n * @brief S=t>S\n * @note Equivalent to y.push_front(x)\n*/\nstatic inline SeriesWork& operator>(SubTask *x, SeriesWork& y);\n\n/**\n * @brief S=P>P\n * @note Equivalent to Workflow::create_series_work(&x, nullptr)->push_back(&y)\n*/\nstatic inline SeriesWork& operator>(ParallelWork& x, ParallelWork& y);\n\n/**\n * @brief S=P>t\n * @note Equivalent to Workflow::create_series_work(&x, nullptr)->push_back(&y)\n*/\nstatic inline SeriesWork& operator>(ParallelWork& x, SubTask& y);\n\n/**\n * @brief S=P>t\n * @note Equivalent to Workflow::create_series_work(&x, nullptr)->push_back(y)\n*/\nstatic inline SeriesWork& operator>(ParallelWork& x, SubTask *y);\n\n/**\n * @brief S=t>P\n * @note Equivalent to Workflow::create_series_work(&x, nullptr)->push_back(&y)\n*/\nstatic inline SeriesWork& operator>(SubTask& x, ParallelWork& y);\n\n/**\n * @brief S=t>P\n * @note Equivalent to Workflow::create_series_work(x, nullptr)->push_back(&y)\n*/\nstatic inline SeriesWork& operator>(SubTask *x, ParallelWork& y);\n\n/**\n * @brief S=t>t\n * @note Equivalent to Workflow::create_series_work(&x, nullptr)->push_back(&y)\n*/\nstatic inline SeriesWork& operator>(SubTask& x, SubTask& y);\n\n/**\n * @brief S=t>t\n * @note Equivalent to Workflow::create_series_work(&x, nullptr)->push_back(y)\n*/\nstatic inline SeriesWork& operator>(SubTask& x, SubTask *y);\n\n/**\n * @brief S=t>t\n * @note Equivalent to Workflow::create_series_work(x, nullptr)->push_back(&y)\n*/\nstatic inline SeriesWork& operator>(SubTask *x, SubTask& y);\n//static inline SeriesWork& operator>(SubTask *x, SubTask *y);//compile error!\n\n/**\n * @brief P=P*S\n * @note Equivalent to x.add_series(&y)\n*/\nstatic inline ParallelWork& operator*(ParallelWork& x, SeriesWork& y);\n\n/**\n * @brief P=S*P\n * @note Equivalent to y.push_back(&x)\n*/\nstatic inline ParallelWork& operator*(SeriesWork& x, ParallelWork& y);\n\n/**\n * @brief P=P*t\n * @note Equivalent to x.add_series(Workflow::create_series_work(&y, nullptr))\n*/\nstatic inline ParallelWork& operator*(ParallelWork& x, SubTask& y);\n\n/**\n * @brief P=P*t\n * @note Equivalent to x.add_series(Workflow::create_series_work(y, nullptr))\n*/\nstatic inline ParallelWork& operator*(ParallelWork& x, SubTask *y);\n\n/**\n * @brief P=t*P\n * @note Equivalent to y.add_series(Workflow::create_series_work(&x, nullptr))\n*/\nstatic inline ParallelWork& operator*(SubTask& x, ParallelWork& y);\n\n/**\n * @brief P=t*P\n * @note Equivalent to y.add_series(Workflow::create_series_work(x, nullptr))\n*/\nstatic inline ParallelWork& operator*(SubTask *x, ParallelWork& y);\n\n/**\n * @brief P=S*S\n * @note Equivalent to Workflow::create_parallel_work({&x, &y}, 2, nullptr)\n*/\nstatic inline ParallelWork& operator*(SeriesWork& x, SeriesWork& y);\n\n/**\n * @brief P=S*t\n * @note Equivalent to Workflow::create_parallel_work({&y}, 1, nullptr)->add_series(&x)\n*/\nstatic inline ParallelWork& operator*(SeriesWork& x, SubTask& y);\n\n/**\n * @brief P=S*t\n * @note Equivalent to Workflow::create_parallel_work({y}, 1, nullptr)->add_series(&x)\n*/\nstatic inline ParallelWork& operator*(SeriesWork& x, SubTask *y);\n\n/**\n * @brief P=t*S\n * @note Equivalent to Workflow::create_parallel_work({&x}, 1, nullptr)->add_series(&y)\n*/\nstatic inline ParallelWork& operator*(SubTask& x, SeriesWork& y);\n\n/**\n * @brief P=t*S\n * @note Equivalent to Workflow::create_parallel_work({x}, 1, nullptr)->add_series(&y)\n*/\nstatic inline ParallelWork& operator*(SubTask *x, SeriesWork& y);\n\n/**\n * @brief P=t*t\n * @note Equivalent to Workflow::create_parallel_work({Workflow::create_series_work(&x, nullptr), Workflow::create_series_work(&y, nullptr)}, 2, nullptr)\n*/\nstatic inline ParallelWork& operator*(SubTask& x, SubTask& y);\n\n/**\n * @brief P=t*t\n * @note Equivalent to Workflow::create_parallel_work({Workflow::create_series_work(&x, nullptr), Workflow::create_series_work(y, nullptr)}, 2, nullptr)\n*/\nstatic inline ParallelWork& operator*(SubTask& x, SubTask *y);\n\n/**\n * @brief P=t*t\n * @note Equivalent to Workflow::create_parallel_work({Workflow::create_series_work(x, nullptr), Workflow::create_series_work(&y, nullptr)}, 2, nullptr)\n*/\nstatic inline ParallelWork& operator*(SubTask *x, SubTask& y);\n//static inline ParallelWork& operator*(SubTask *x, SubTask *y);//compile error!\n\n\n//S=S>t\nstatic inline SeriesWork& operator>(SeriesWork& x, SubTask& y)\n{\n\tx.push_back(&y);\n\treturn x;\n}\nstatic inline SeriesWork& operator>(SeriesWork& x, SubTask *y)\n{\n\treturn x > *y;\n}\n//S=t>S\nstatic inline SeriesWork& operator>(SubTask& x, SeriesWork& y)\n{\n\ty.push_front(&x);\n\treturn y;\n}\nstatic inline SeriesWork& operator>(SubTask *x, SeriesWork& y)\n{\n\treturn *x > y;\n}\n//S=t>t\nstatic inline SeriesWork& operator>(SubTask& x, SubTask& y)\n{\n\tSeriesWork *series = Workflow::create_series_work(&x, nullptr);\n\tseries->push_back(&y);\n\treturn *series;\n}\nstatic inline SeriesWork& operator>(SubTask& x, SubTask *y)\n{\n\treturn x > *y;\n}\nstatic inline SeriesWork& operator>(SubTask *x, SubTask& y)\n{\n\treturn *x > y;\n}\n//S=S>P\nstatic inline SeriesWork& operator>(SeriesWork& x, ParallelWork& y)\n{\n\treturn x > (SubTask&)y;\n}\n//S=P>S\nstatic inline SeriesWork& operator>(ParallelWork& x, SeriesWork& y)\n{\n\treturn y > (SubTask&)x;\n}\n//S=P>P\nstatic inline SeriesWork& operator>(ParallelWork& x, ParallelWork& y)\n{\n\treturn (SubTask&)x > (SubTask&)y;\n}\n//S=P>t\nstatic inline SeriesWork& operator>(ParallelWork& x, SubTask& y)\n{\n\treturn (SubTask&)x > y;\n}\nstatic inline SeriesWork& operator>(ParallelWork& x, SubTask *y)\n{\n\treturn x > *y;\n}\n//S=t>P\nstatic inline SeriesWork& operator>(SubTask& x, ParallelWork& y)\n{\n\treturn x > (SubTask&)y;\n}\nstatic inline SeriesWork& operator>(SubTask *x, ParallelWork& y)\n{\n\treturn *x > y;\n}\n\n//P=P*S\nstatic inline ParallelWork& operator*(ParallelWork& x, SeriesWork& y)\n{\n\tx.add_series(&y);\n\treturn x;\n}\n//P=S*P\nstatic inline ParallelWork& operator*(SeriesWork& x, ParallelWork& y)\n{\n\treturn y * x;\n}\n//P=P*t\nstatic inline ParallelWork& operator*(ParallelWork& x, SubTask& y)\n{\n\tx.add_series(Workflow::create_series_work(&y, nullptr));\n\treturn x;\n}\nstatic inline ParallelWork& operator*(ParallelWork& x, SubTask *y)\n{\n\treturn x * (*y);\n}\n//P=t*P\nstatic inline ParallelWork& operator*(SubTask& x, ParallelWork& y)\n{\n\treturn y * x;\n}\nstatic inline ParallelWork& operator*(SubTask *x, ParallelWork& y)\n{\n\treturn (*x) * y;\n}\n//P=S*S\nstatic inline ParallelWork& operator*(SeriesWork& x, SeriesWork& y)\n{\n\tSeriesWork *arr[2] = {&x, &y};\n\treturn *Workflow::create_parallel_work(arr, 2, nullptr);\n}\n//P=S*t\nstatic inline ParallelWork& operator*(SeriesWork& x, SubTask& y)\n{\n\treturn x * (*Workflow::create_series_work(&y, nullptr));\n}\nstatic inline ParallelWork& operator*(SeriesWork& x, SubTask *y)\n{\n\treturn x * (*y);\n}\n//P=t*S\nstatic inline ParallelWork& operator*(SubTask& x, SeriesWork& y)\n{\n\treturn y * x;\n}\nstatic inline ParallelWork& operator*(SubTask *x, SeriesWork& y)\n{\n\treturn (*x) * y;\n}\n//P=t*t\nstatic inline ParallelWork& operator*(SubTask& x, SubTask& y)\n{\n\tSeriesWork *arr[2] = {Workflow::create_series_work(&x, nullptr),\n\t\t\t\t\t\t Workflow::create_series_work(&y, nullptr)};\n\treturn *Workflow::create_parallel_work(arr, 2, nullptr);\n}\nstatic inline ParallelWork& operator*(SubTask& x, SubTask *y)\n{\n\treturn x * (*y);\n}\nstatic inline ParallelWork& operator*(SubTask *x, SubTask& y)\n{\n\treturn (*x) * y;\n}\n\n\n#endif\n\n" }, { "path": "src/factory/WFResourcePool.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \"list.h\"\n#include \"WFTask.h\"\n#include \"WFResourcePool.h\"\n\nclass __RPConditional : public WFConditional\n{\npublic:\n\tstruct list_head list;\n\tstruct WFResourcePool::Data *data;\n\npublic:\n\tvirtual void dispatch();\n\tvirtual void signal(void *res) { }\n\npublic:\n\t__RPConditional(SubTask *task, void **resbuf,\n\t\t\t\t\tstruct WFResourcePool::Data *data) :\n\t\tWFConditional(task, resbuf)\n\t{\n\t\tthis->data = data;\n\t}\n\n\t__RPConditional(SubTask *task,\n\t\t\t\t\tstruct WFResourcePool::Data *data) :\n\t\tWFConditional(task)\n\t{\n\t\tthis->data = data;\n\t}\n};\n\nvoid __RPConditional::dispatch()\n{\n\tstruct WFResourcePool::Data *data = this->data;\n\n\tdata->mutex.lock();\n\tif (--data->value >= 0)\n\t\tthis->WFConditional::signal(data->pop());\n\telse\n\t\tlist_add_tail(&this->list, &data->wait_list);\n\n\tdata->mutex.unlock();\n\tthis->WFConditional::dispatch();\n}\n\nWFConditional *WFResourcePool::get(SubTask *task, void **resbuf)\n{\n\treturn new __RPConditional(task, resbuf, &this->data);\n}\n\nWFConditional *WFResourcePool::get(SubTask *task)\n{\n\treturn new __RPConditional(task, &this->data);\n}\n\nvoid WFResourcePool::create(size_t n)\n{\n\tthis->data.res = new void *[n];\n\tthis->data.value = n;\n\tthis->data.index = 0;\n\tINIT_LIST_HEAD(&this->data.wait_list);\n\tthis->data.pool = this;\n}\n\nWFResourcePool::WFResourcePool(void *const *res, size_t n)\n{\n\tthis->create(n);\n\tmemcpy(this->data.res, res, n * sizeof (void *));\n}\n\nWFResourcePool::WFResourcePool(size_t n)\n{\n\tthis->create(n);\n\tmemset(this->data.res, 0, n * sizeof (void *));\n}\n\nvoid WFResourcePool::post(void *res)\n{\n\tstruct WFResourcePool::Data *data = &this->data;\n\tWFConditional *cond;\n\n\tdata->mutex.lock();\n\tif (++data->value <= 0)\n\t{\n\t\tcond = list_entry(data->wait_list.next, __RPConditional, list);\n\t\tlist_del(data->wait_list.next);\n\t}\n\telse\n\t{\n\t\tcond = NULL;\n\t\tthis->push(res);\n\t}\n\n\tdata->mutex.unlock();\n\tif (cond)\n\t\tcond->WFConditional::signal(res);\n}\n\n" }, { "path": "src/factory/WFResourcePool.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFRESOURCEPOOL_H_\n#define _WFRESOURCEPOOL_H_\n\n#include \n#include \"list.h\"\n#include \"WFTask.h\"\n\nclass WFResourcePool\n{\npublic:\n\tWFConditional *get(SubTask *task, void **resbuf);\n\tWFConditional *get(SubTask *task);\n\tvoid post(void *res);\n\npublic:\n\tstruct Data\n\t{\n\t\tvoid *pop() { return this->pool->pop(); }\n\t\tvoid push(void *res) { this->pool->push(res); }\n\n\t\tvoid **res;\n\t\tlong value;\n\t\tsize_t index;\n\t\tstruct list_head wait_list;\n\t\tstd::mutex mutex;\n\t\tWFResourcePool *pool;\n\t};\n\nprotected:\n\tvirtual void *pop()\n\t{\n\t\treturn this->data.res[this->data.index++];\n\t}\n\n\tvirtual void push(void *res)\n\t{\n\t\tthis->data.res[--this->data.index] = res;\n\t}\n\nprotected:\n\tstruct Data data;\n\nprivate:\n\tvoid create(size_t n);\n\npublic:\n\tWFResourcePool(void *const *res, size_t n);\n\tWFResourcePool(size_t n);\n\tvirtual ~WFResourcePool() { delete []this->data.res; }\n};\n\n#endif\n\n" }, { "path": "src/factory/WFTask.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFTASK_H_\n#define _WFTASK_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \"Executor.h\"\n#include \"ExecRequest.h\"\n#include \"Communicator.h\"\n#include \"CommScheduler.h\"\n#include \"CommRequest.h\"\n#include \"SleepRequest.h\"\n#include \"IORequest.h\"\n#include \"Workflow.h\"\n#include \"WFConnection.h\"\n\nenum\n{\n\tWFT_STATE_UNDEFINED = -1,\n\tWFT_STATE_SUCCESS = CS_STATE_SUCCESS,\n\tWFT_STATE_TOREPLY = CS_STATE_TOREPLY,\t\t/* for server task only */\n\tWFT_STATE_NOREPLY = CS_STATE_TOREPLY + 1,\t/* for server task only */\n\tWFT_STATE_SYS_ERROR = CS_STATE_ERROR,\n\tWFT_STATE_SSL_ERROR = 65,\n\tWFT_STATE_DNS_ERROR = 66,\t\t\t\t\t/* for client task only */\n\tWFT_STATE_TASK_ERROR = 67,\n\tWFT_STATE_ABORTED = CS_STATE_STOPPED\n};\n\ntemplate\nclass WFThreadTask : public ExecRequest\n{\npublic:\n\tvoid start()\n\t{\n\t\tassert(!series_of(this));\n\t\tWorkflow::start_series_work(this, nullptr);\n\t}\n\n\tvoid dismiss()\n\t{\n\t\tassert(!series_of(this));\n\t\tdelete this;\n\t}\n\npublic:\n\tINPUT *get_input() { return &this->input; }\n\tOUTPUT *get_output() { return &this->output; }\n\n\tconst INPUT *get_input() const { return &this->input; }\n\tconst OUTPUT *get_output() const { return &this->output; }\n\npublic:\n\tvoid *user_data;\n\npublic:\n\tint get_state() const { return this->state; }\n\tint get_error() const { return this->error; }\n\npublic:\n\tvoid set_callback(std::function *)> cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tINPUT input;\n\tOUTPUT output;\n\tstd::function *)> callback;\n\npublic:\n\tWFThreadTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t std::function *)>&& cb) :\n\t\tExecRequest(queue, executor),\n\t\tcallback(std::move(cb))\n\t{\n\t\tthis->user_data = NULL;\n\t\tthis->state = WFT_STATE_UNDEFINED;\n\t\tthis->error = 0;\n\t}\n\nprotected:\n\tvirtual ~WFThreadTask() { }\n};\n\ntemplate\nclass WFNetworkTask : public CommRequest\n{\npublic:\n\t/* start(), dismiss() are for client tasks only. */\n\tvoid start()\n\t{\n\t\tassert(!series_of(this));\n\t\tWorkflow::start_series_work(this, nullptr);\n\t}\n\n\tvoid dismiss()\n\t{\n\t\tassert(!series_of(this));\n\t\tdelete this;\n\t}\n\npublic:\n\tREQ *get_req() { return &this->req; }\n\tRESP *get_resp() { return &this->resp; }\n\n\tconst REQ *get_req() const { return &this->req; }\n\tconst RESP *get_resp() const { return &this->resp; }\n\npublic:\n\tvoid *user_data;\n\npublic:\n\tint get_state() const { return this->state; }\n\tint get_error() const { return this->error; }\n\n\t/* Call when error is ETIMEDOUT, return values:\n\t * TOR_NOT_TIMEOUT, TOR_WAIT_TIMEOUT, TOR_CONNECT_TIMEOUT,\n\t * TOR_TRANSMIT_TIMEOUT (send or receive).\n\t * SSL connect timeout also returns TOR_CONNECT_TIMEOUT. */\n\tint get_timeout_reason() const { return this->timeout_reason; }\n\n\t/* Call only in callback or server's process. */\n\tlong long get_task_seq() const\n\t{\n\t\tif (!this->target)\n\t\t{\n\t\t\terrno = ENOTCONN;\n\t\t\treturn -1;\n\t\t}\n\n\t\treturn this->get_seq();\n\t}\n\n\tint get_peer_addr(struct sockaddr *addr, socklen_t *addrlen) const;\n\n\tvirtual WFConnection *get_connection() const = 0;\n\npublic:\n\t/* All in milliseconds. timeout == -1 for unlimited. */\n\tvoid set_send_timeout(int timeout) { this->send_timeo = timeout; }\n\tvoid set_receive_timeout(int timeout) { this->receive_timeo = timeout; }\n\tvoid set_keep_alive(int timeout) { this->keep_alive_timeo = timeout; }\n\tvoid set_watch_timeout(int timeout) { this->watch_timeo = timeout; }\n\npublic:\n\t/* Do not reply this request. */\n\tvoid noreply()\n\t{\n\t\tif (this->state == WFT_STATE_TOREPLY)\n\t\t\tthis->state = WFT_STATE_NOREPLY;\n\t}\n\n\t/* Push reply data synchronously. */\n\tvirtual int push(const void *buf, size_t size)\n\t{\n\t\tif (this->state != WFT_STATE_TOREPLY &&\n\t\t\tthis->state != WFT_STATE_NOREPLY)\n\t\t{\n\t\t\terrno = ENOENT;\n\t\t\treturn -1;\n\t\t}\n\n\t\treturn this->scheduler->push(buf, size, this);\n\t}\n\n\t/* To check if the connection was closed before replying.\n\t Always returns 'true' in callback. */\n\tbool closed() const\n\t{\n\t\tswitch (this->state)\n\t\t{\n\t\tcase WFT_STATE_UNDEFINED:\n\t\t\treturn false;\n\t\tcase WFT_STATE_TOREPLY:\n\t\tcase WFT_STATE_NOREPLY:\n\t\t\treturn !this->target->has_idle_conn();\n\t\tdefault:\n\t\t\treturn true;\n\t\t}\n\t}\n\npublic:\n\tvoid set_prepare(std::function *)> prep)\n\t{\n\t\tthis->prepare = std::move(prep);\n\t}\n\npublic:\n\tvoid set_callback(std::function *)> cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual int send_timeout() { return this->send_timeo; }\n\tvirtual int receive_timeout() { return this->receive_timeo; }\n\tvirtual int keep_alive_timeout() { return this->keep_alive_timeo; }\n\tvirtual int first_timeout() { return this->watch_timeo; }\n\nprotected:\n\tint send_timeo;\n\tint receive_timeo;\n\tint keep_alive_timeo;\n\tint watch_timeo;\n\tREQ req;\n\tRESP resp;\n\tstd::function *)> prepare;\n\tstd::function *)> callback;\n\nprotected:\n\tWFNetworkTask(CommSchedObject *object, CommScheduler *scheduler,\n\t\t\t\t std::function *)>&& cb) :\n\t\tCommRequest(object, scheduler),\n\t\tcallback(std::move(cb))\n\t{\n\t\tthis->send_timeo = -1;\n\t\tthis->receive_timeo = -1;\n\t\tthis->keep_alive_timeo = 0;\n\t\tthis->watch_timeo = 0;\n\t\tthis->target = NULL;\n\t\tthis->timeout_reason = TOR_NOT_TIMEOUT;\n\t\tthis->user_data = NULL;\n\t\tthis->state = WFT_STATE_UNDEFINED;\n\t\tthis->error = 0;\n\t}\n\n\tvirtual ~WFNetworkTask() { }\n};\n\nclass WFTimerTask : public SleepRequest\n{\npublic:\n\tvoid start()\n\t{\n\t\tassert(!series_of(this));\n\t\tWorkflow::start_series_work(this, nullptr);\n\t}\n\n\tvoid dismiss()\n\t{\n\t\tassert(!series_of(this));\n\t\tdelete this;\n\t}\n\npublic:\n\tvoid *user_data;\n\npublic:\n\tint get_state() const { return this->state; }\n\tint get_error() const { return this->error; }\n\npublic:\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tstd::function callback;\n\npublic:\n\tWFTimerTask(CommScheduler *scheduler,\n\t\t\t\tstd::function cb) :\n\t\tSleepRequest(scheduler),\n\t\tcallback(std::move(cb))\n\t{\n\t\tthis->user_data = NULL;\n\t\tthis->state = WFT_STATE_UNDEFINED;\n\t\tthis->error = 0;\n\t}\n\nprotected:\n\tvirtual ~WFTimerTask() { }\n};\n\ntemplate\nclass WFFileTask : public IORequest\n{\npublic:\n\tvoid start()\n\t{\n\t\tassert(!series_of(this));\n\t\tWorkflow::start_series_work(this, nullptr);\n\t}\n\n\tvoid dismiss()\n\t{\n\t\tassert(!series_of(this));\n\t\tdelete this;\n\t}\n\npublic:\n\tARGS *get_args() { return &this->args; }\n\n\tconst ARGS *get_args() const { return &this->args; }\n\n\tlong get_retval() const\n\t{\n\t\tif (this->state == WFT_STATE_SUCCESS)\n\t\t\treturn this->get_res();\n\t\telse\n\t\t\treturn -1;\n\t}\n\npublic:\n\tvoid *user_data;\n\npublic:\n\tint get_state() const { return this->state; }\n\tint get_error() const { return this->error; }\n\npublic:\n\tvoid set_callback(std::function *)> cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tARGS args;\n\tstd::function *)> callback;\n\npublic:\n\tWFFileTask(IOService *service,\n\t\t\t std::function *)>&& cb) :\n\t\tIORequest(service),\n\t\tcallback(std::move(cb))\n\t{\n\t\tthis->user_data = NULL;\n\t\tthis->state = WFT_STATE_UNDEFINED;\n\t\tthis->error = 0;\n\t}\n\nprotected:\n\tvirtual ~WFFileTask() { }\n};\n\nclass WFGenericTask : public SubTask\n{\npublic:\n\tvoid start()\n\t{\n\t\tassert(!series_of(this));\n\t\tWorkflow::start_series_work(this, nullptr);\n\t}\n\n\tvoid dismiss()\n\t{\n\t\tassert(!series_of(this));\n\t\tdelete this;\n\t}\n\npublic:\n\tvoid *user_data;\n\npublic:\n\tint get_state() const { return this->state; }\n\tint get_error() const { return this->error; }\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tthis->subtask_done();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tint state;\n\tint error;\n\npublic:\n\tWFGenericTask()\n\t{\n\t\tthis->user_data = NULL;\n\t\tthis->state = WFT_STATE_UNDEFINED;\n\t\tthis->error = 0;\n\t}\n\nprotected:\n\tvirtual ~WFGenericTask() { }\n};\n\nclass WFCounterTask : public WFGenericTask\n{\npublic:\n\tvirtual void count()\n\t{\n\t\tif (--this->value == 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SUCCESS;\n\t\t\tthis->subtask_done();\n\t\t}\n\t}\n\npublic:\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tthis->WFCounterTask::count();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tstd::atomic value;\n\tstd::function callback;\n\npublic:\n\tWFCounterTask(unsigned int target_value,\n\t\t\t\t std::function&& cb) :\n\t\tvalue(target_value + 1),\n\t\tcallback(std::move(cb))\n\t{\n\t}\n\nprotected:\n\tvirtual ~WFCounterTask() { }\n};\n\nclass WFMailboxTask : public WFGenericTask\n{\npublic:\n\tvirtual void send(void *msg)\n\t{\n\t\t*this->mailbox = msg;\n\t\tif (this->flag.exchange(true))\n\t\t{\n\t\t\tthis->state = WFT_STATE_SUCCESS;\n\t\t\tthis->subtask_done();\n\t\t}\n\t}\n\n\tvoid **get_mailbox() const { return this->mailbox; }\n\npublic:\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tif (this->flag.exchange(true))\n\t\t{\n\t\t\tthis->state = WFT_STATE_SUCCESS;\n\t\t\tthis->subtask_done();\n\t\t}\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tvoid **mailbox;\n\tstd::atomic flag;\n\tstd::function callback;\n\npublic:\n\tWFMailboxTask(void **mailbox,\n\t\t\t\t std::function&& cb) :\n\t\tflag(false),\n\t\tcallback(std::move(cb))\n\t{\n\t\tthis->mailbox = mailbox;\n\t}\n\n\tWFMailboxTask(std::function&& cb) :\n\t\tflag(false),\n\t\tcallback(std::move(cb))\n\t{\n\t\tthis->mailbox = &this->user_data;\n\t}\n\nprotected:\n\tvirtual ~WFMailboxTask() { }\n};\n\nclass WFSelectorTask : public WFGenericTask\n{\npublic:\n\tvirtual int submit(void *msg)\n\t{\n\t\tvoid *tmp = NULL;\n\t\tint ret = 0;\n\n\t\tif (this->message.compare_exchange_strong(tmp, msg) && msg)\n\t\t{\n\t\t\tret = 1;\n\t\t\tif (this->flag.exchange(true))\n\t\t\t{\n\t\t\t\tthis->state = WFT_STATE_SUCCESS;\n\t\t\t\tthis->subtask_done();\n\t\t\t}\n\t\t}\n\n\t\tif (--this->nleft == 0)\n\t\t{\n\t\t\tif (!this->message)\n\t\t\t{\n\t\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\t\tthis->error = ENOMSG;\n\t\t\t\tthis->subtask_done();\n\t\t\t}\n\n\t\t\tdelete this;\n\t\t}\n\n\t\treturn ret;\n\t}\n\n\tvoid *get_message() const { return this->message; }\n\npublic:\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tif (this->flag.exchange(true))\n\t\t{\n\t\t\tthis->state = WFT_STATE_SUCCESS;\n\t\t\tthis->subtask_done();\n\t\t}\n\n\t\tif (--this->nleft == 0)\n\t\t{\n\t\t\tif (!this->message)\n\t\t\t{\n\t\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\t\tthis->error = ENOMSG;\n\t\t\t\tthis->subtask_done();\n\t\t\t}\n\n\t\t\tdelete this;\n\t\t}\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tstd::atomic message;\n\tstd::atomic flag;\n\tstd::atomic nleft;\n\tstd::function callback;\n\npublic:\n\tWFSelectorTask(size_t candidates,\n\t\t\t\t std::function&& cb) :\n\t\tmessage(NULL),\n\t\tflag(false),\n\t\tnleft(candidates + 1),\n\t\tcallback(std::move(cb))\n\t{\n\t}\n\nprotected:\n\tvirtual ~WFSelectorTask() { }\n};\n\nclass WFConditional : public WFGenericTask\n{\npublic:\n\tvirtual void signal(void *msg)\n\t{\n\t\t*this->msgbuf = msg;\n\t\tif (this->flag.exchange(true))\n\t\t\tthis->subtask_done();\n\t}\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tseries_of(this)->push_front(this->task);\n\t\tthis->task = NULL;\n\t\tif (this->flag.exchange(true))\n\t\t\tthis->subtask_done();\n\t}\n\nprotected:\n\tstd::atomic flag;\n\tSubTask *task;\n\tvoid **msgbuf;\n\npublic:\n\tWFConditional(SubTask *task, void **msgbuf) :\n\t\tflag(false)\n\t{\n\t\tthis->task = task;\n\t\tthis->msgbuf = msgbuf;\n\t}\n\n\tWFConditional(SubTask *task) :\n\t\tflag(false)\n\t{\n\t\tthis->task = task;\n\t\tthis->msgbuf = &this->user_data;\n\t}\n\nprotected:\n\tvirtual ~WFConditional()\n\t{\n\t\tdelete this->task;\n\t}\n};\n\nclass WFGoTask : public ExecRequest\n{\npublic:\n\tvoid start()\n\t{\n\t\tassert(!series_of(this));\n\t\tWorkflow::start_series_work(this, nullptr);\n\t}\n\n\tvoid dismiss()\n\t{\n\t\tassert(!series_of(this));\n\t\tdelete this;\n\t}\n\npublic:\n\tvoid *user_data;\n\npublic:\n\tint get_state() const { return this->state; }\n\tint get_error() const { return this->error; }\n\npublic:\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tstd::function callback;\n\npublic:\n\tWFGoTask(ExecQueue *queue, Executor *executor) :\n\t\tExecRequest(queue, executor)\n\t{\n\t\tthis->user_data = NULL;\n\t\tthis->state = WFT_STATE_UNDEFINED;\n\t\tthis->error = 0;\n\t}\n\nprotected:\n\tvirtual ~WFGoTask() { }\n};\n\nclass WFRepeaterTask : public WFGenericTask\n{\npublic:\n\tvoid set_create(std::function create)\n\t{\n\t\tthis->create = std::move(create);\n\t}\n\npublic:\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tSubTask *task = this->create(this);\n\n\t\tif (task)\n\t\t{\n\t\t\tseries_of(this)->push_front(this);\n\t\t\tseries_of(this)->push_front(task);\n\t\t}\n\t\telse\n\t\t\tthis->state = WFT_STATE_SUCCESS;\n\n\t\tthis->subtask_done();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->state != WFT_STATE_UNDEFINED)\n\t\t{\n\t\t\tif (this->callback)\n\t\t\t\tthis->callback(this);\n\n\t\t\tdelete this;\n\t\t}\n\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tstd::function create;\n\tstd::function callback;\n\npublic:\n\tWFRepeaterTask(std::function&& create,\n\t\t\t\t std::function&& cb) :\n\t\tcreate(std::move(create)),\n\t\tcallback(std::move(cb))\n\t{\n\t}\n\nprotected:\n\tvirtual ~WFRepeaterTask() { }\n};\n\nclass WFModuleTask : public ParallelTask, protected SeriesWork\n{\npublic:\n\tvoid start()\n\t{\n\t\tassert(!series_of(this));\n\t\tWorkflow::start_series_work(this, nullptr);\n\t}\n\n\tvoid dismiss()\n\t{\n\t\tassert(!series_of(this));\n\t\tdelete this;\n\t}\n\npublic:\n\tSeriesWork *sub_series() { return this; }\n\n\tconst SeriesWork *sub_series() const { return this; }\n\npublic:\n\tvoid *user_data;\n\npublic:\n\tvoid set_callback(std::function cb)\n\t{\n\t\tthis->callback = std::move(cb);\n\t}\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tSubTask *first;\n\tstd::function callback;\n\npublic:\n\tWFModuleTask(SubTask *first,\n\t\t\t\t std::function&& cb) :\n\t\tParallelTask(&this->first, 1),\n\t\tSeriesWork(first, nullptr),\n\t\tcallback(std::move(cb))\n\t{\n\t\tthis->first = first;\n\t\tthis->set_in_parallel(this);\n\t\tthis->user_data = NULL;\n\t}\n\nprotected:\n\tvirtual ~WFModuleTask()\n\t{\n\t\tif (!this->is_finished())\n\t\t\tthis->dismiss_recursive();\n\t}\n};\n\n#include \"WFTask.inl\"\n\n#endif\n\n" }, { "path": "src/factory/WFTask.inl", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\ntemplate\nint WFNetworkTask::get_peer_addr(struct sockaddr *addr,\n\t\t\t\t\t\t\t\t\t\t\tsocklen_t *addrlen) const\n{\n\tconst struct sockaddr *p;\n\tsocklen_t len;\n\n\tif (this->target)\n\t{\n\t\tthis->target->get_addr(&p, &len);\n\t\tif (*addrlen >= len)\n\t\t{\n\t\t\tmemcpy(addr, p, len);\n\t\t\t*addrlen = len;\n\t\t\treturn 0;\n\t\t}\n\n\t\terrno = ENOBUFS;\n\t}\n\telse\n\t\terrno = ENOTCONN;\n\n\treturn -1;\n}\n\ntemplate\nclass WFClientTask : public WFNetworkTask\n{\nprotected:\n\tvirtual CommMessageOut *message_out()\n\t{\n\t\t/* By setting 'prepare' function, users can modify the request after\n\t\t * the connection is established. */\n\t\tif (this->prepare)\n\t\t\tthis->prepare(this);\n\n\t\treturn &this->req;\n\t}\n\n\tvirtual CommMessageIn *message_in() { return &this->resp; }\n\nprotected:\n\tvirtual WFConnection *get_connection() const\n\t{\n\t\tCommConnection *conn;\n\n\t\tif (this->target)\n\t\t{\n\t\t\tconn = this->CommSession::get_connection();\n\t\t\tif (conn)\n\t\t\t\treturn (WFConnection *)conn;\n\t\t}\n\n\t\terrno = ENOTCONN;\n\t\treturn NULL;\n\t}\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->state == WFT_STATE_SYS_ERROR && this->error < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SSL_ERROR;\n\t\t\tthis->error = -this->error;\n\t\t}\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\npublic:\n\tWFClientTask(CommSchedObject *object, CommScheduler *scheduler,\n\t\t\t\t std::function *)>&& cb) :\n\t\tWFNetworkTask(object, scheduler, std::move(cb))\n\t{\n\t}\n\nprotected:\n\tvirtual ~WFClientTask() { }\n};\n\ntemplate\nclass WFServerTask : public WFNetworkTask\n{\nprotected:\n\tvirtual CommMessageOut *message_out()\n\t{\n\t\t/* By setting 'prepare' function, users can modify the response before\n\t\t * replying to the client. */\n\t\tif (this->prepare)\n\t\t\tthis->prepare(this);\n\n\t\treturn &this->resp;\n\t}\n\n\tvirtual CommMessageIn *message_in() { return &this->req; }\n\tvirtual void handle(int state, int error);\n\nprotected:\n\tvirtual WFConnection *get_connection() const\n\t{\n\t\tif (this->processor.task == this)\n\t\t\treturn (WFConnection *)this->CommSession::get_connection();\n\n\t\terrno = this->processor.task ? ENOTCONN : EPERM;\n\t\treturn NULL;\n\t}\n\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tif (this->state == WFT_STATE_TOREPLY)\n\t\t{\n\t\t\tthis->processor.task = this;\n\t\t\tif (this->scheduler->reply(this) >= 0)\n\t\t\t\treturn;\n\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t}\n\t\telse\n\t\t\tthis->scheduler->shutdown(this);\n\n\t\tthis->processor.task = (WFServerTask *)-1;\n\t\tthis->subtask_done();\n\t}\n\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->state == WFT_STATE_SYS_ERROR && this->error < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SSL_ERROR;\n\t\t\tthis->error = -this->error;\n\t\t}\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tthis->processor.task = NULL;\n\t\treturn series->pop();\n\t}\n\nprotected:\n\tclass Processor : public SubTask\n\t{\n\tpublic:\n\t\tProcessor(WFServerTask *task,\n\t\t\t\t std::function *)>& proc) :\n\t\t\tprocess(proc)\n\t\t{\n\t\t\tthis->task = task;\n\t\t}\n\n\t\tvirtual void dispatch()\n\t\t{\n\t\t\tthis->process(this->task);\n\t\t\tthis->task = NULL;\t/* As a flag, disable getting connection. */\n\t\t\tthis->subtask_done();\n\t\t}\n\n\t\tvirtual SubTask *done()\n\t\t{\n\t\t\treturn series_of(this)->pop();\n\t\t}\n\n\t\tstd::function *)>& process;\n\t\tWFServerTask *task;\n\t} processor;\n\n\tclass Series : public SeriesWork\n\t{\n\tpublic:\n\t\tSeries(WFServerTask *task) :\n\t\t\tSeriesWork(&task->processor, nullptr)\n\t\t{\n\t\t\tthis->set_last_task(task);\n\t\t\tthis->task = task;\n\t\t}\n\n\t\tvirtual ~Series()\n\t\t{\n\t\t\tdelete this->task;\n\t\t}\n\n\t\tWFServerTask *task;\n\t};\n\npublic:\n\tWFServerTask(CommService *service, CommScheduler *scheduler,\n\t\t\t\t std::function *)>& proc) :\n\t\tWFNetworkTask(NULL, scheduler, nullptr),\n\t\tprocessor(this, proc)\n\t{\n\t}\n\nprotected:\n\tvirtual ~WFServerTask()\n\t{\n\t\tif (this->target)\n\t\t\t((Series *)series_of(&this->processor))->task = NULL;\n\t}\n};\n\ntemplate\nvoid WFServerTask::handle(int state, int error)\n{\n\tif (state == WFT_STATE_TOREPLY)\n\t{\n\t\tthis->state = WFT_STATE_TOREPLY;\n\t\tthis->target = this->get_target();\n\t\tnew Series(this);\n\t\tthis->processor.dispatch();\n\t}\n\telse if (this->state == WFT_STATE_TOREPLY)\n\t{\n\t\tthis->state = state;\n\t\tthis->error = error;\n\t\tif (error == ETIMEDOUT)\n\t\t\tthis->timeout_reason = TOR_TRANSMIT_TIMEOUT;\n\n\t\tthis->subtask_done();\n\t}\n\telse\n\t\tdelete this;\n}\n\n" }, { "path": "src/factory/WFTaskError.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _WFTASKERROR_H_\n#define _WFTASKERROR_H_\n\n/**\n * @file WFTaskError.h\n * @brief Workflow Task Error Code List\n */\n\n/**\n * @brief Defination of task state code\n * @note Only for WFNetworkTask and only when get_state()==WFT_STATE_TASK_ERROR\n */\nenum\n{\n\t//COMMON\n\tWFT_ERR_URI_PARSE_FAILED = 1001, ///< URI, parse failed\n\tWFT_ERR_URI_SCHEME_INVALID = 1002, ///< URI, invalid scheme\n\tWFT_ERR_URI_PORT_INVALID = 1003, ///< URI, invalid port\n\tWFT_ERR_UPSTREAM_UNAVAILABLE = 1004, ///< Upstream, all target server down\n\n\t//HTTP\n\tWFT_ERR_HTTP_BAD_REDIRECT_HEADER = 2001, ///< Http, 301/302/303/307/308 Location header value is NULL\n\tWFT_ERR_HTTP_PROXY_CONNECT_FAILED = 2002, ///< Http, proxy CONNECT return non 200\n\n\t//REDIS\n\tWFT_ERR_REDIS_ACCESS_DENIED = 3001, ///< Redis, invalid password\n\tWFT_ERR_REDIS_COMMAND_DISALLOWED = 3002, ///< Redis, command disabled, cannot be \"AUTH\"/\"SELECT\"\n\n\t//MYSQL\n\tWFT_ERR_MYSQL_HOST_NOT_ALLOWED = 4001, ///< MySQL\n\tWFT_ERR_MYSQL_ACCESS_DENIED = 4002, ///< MySQL, authentication failed\n\tWFT_ERR_MYSQL_INVALID_CHARACTER_SET = 4003, ///< MySQL, invalid charset, not found in MySQL-Documentation\n\tWFT_ERR_MYSQL_COMMAND_DISALLOWED = 4004, ///< MySQL, sql command disabled, cannot be \"USE\"/\"SET NAMES\"/\"SET CHARSET\"/\"SET CHARACTER SET\"\n\tWFT_ERR_MYSQL_QUERY_NOT_SET = 4005, ///< MySQL, query not set sql, maybe forget please check\n\tWFT_ERR_MYSQL_SSL_NOT_SUPPORTED = 4006,\t\t///< MySQL, SSL not supported by the server\n\n\t//KAFKA\n\tWFT_ERR_KAFKA_PARSE_RESPONSE_FAILED = 5001, ///< Kafka parse response failed\n\tWFT_ERR_KAFKA_PRODUCE_FAILED = 5002,\n\tWFT_ERR_KAFKA_FETCH_FAILED = 5003,\n\tWFT_ERR_KAFKA_CGROUP_FAILED = 5004,\n\tWFT_ERR_KAFKA_COMMIT_FAILED = 5005,\n\tWFT_ERR_KAFKA_META_FAILED = 5006,\n\tWFT_ERR_KAFKA_LEAVEGROUP_FAILED = 5007,\n\tWFT_ERR_KAFKA_API_UNKNOWN = 5008,\t\t\t///< api type not supported\n\tWFT_ERR_KAFKA_VERSION_DISALLOWED = 5009,\t///< broker version not supported\n\tWFT_ERR_KAFKA_SASL_DISALLOWED = 5010,\t\t///< sasl not supported\n\tWFT_ERR_KAFKA_ARRANGE_FAILED = 5011,\t\t///< arrange toppar failed\n\tWFT_ERR_KAFKA_LIST_OFFSETS_FAILED = 5012,\n\tWFT_ERR_KAFKA_CGROUP_ASSIGN_FAILED = 5013,\n\n\t//CONSUL\n\tWFT_ERR_CONSUL_API_UNKNOWN = 6001,\t\t///< api type not supported\n\tWFT_ERR_CONSUL_CHECK_RESPONSE_FAILED = 6002,\t///< Consul http code failed\n};\n\n#endif\n" }, { "path": "src/factory/WFTaskFactory.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"rbtree.h\"\n#include \"WFGlobal.h\"\n#include \"WFTaskFactory.h\"\n\nclass __WFTimerTask : public WFTimerTask\n{\nprotected:\n\tvirtual int duration(struct timespec *value)\n\t{\n\t\tvalue->tv_sec = this->seconds;\n\t\tvalue->tv_nsec = this->nanoseconds;\n\t\treturn 0;\n\t}\n\nprotected:\n\ttime_t seconds;\n\tlong nanoseconds;\n\npublic:\n\t__WFTimerTask(time_t seconds, long nanoseconds, CommScheduler *scheduler,\n\t\t\t\t timer_callback_t&& cb) :\n\t\tWFTimerTask(scheduler, std::move(cb))\n\t{\n\t\tthis->seconds = seconds;\n\t\tthis->nanoseconds = nanoseconds;\n\t}\n};\n\nclass __WFCanceledTimerTask : public __WFTimerTask\n{\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tif (this->scheduler->sleep(this) >= 0)\n\t\t\tthis->cancel();\n\t\telse\n\t\t\tthis->handle(WFT_STATE_SYS_ERROR, errno);\n\t}\n\npublic:\n\t__WFCanceledTimerTask(CommScheduler *scheduler, timer_callback_t&& cb) :\n\t\t__WFTimerTask(-1, 0, scheduler, std::move(cb))\n\t{\n\t}\n};\n\nWFTimerTask *WFTaskFactory::create_timer_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t timer_callback_t callback)\n{\n\treturn new __WFTimerTask(seconds, nanoseconds, WFGlobal::get_scheduler(),\n\t\t\t\t\t\t\t std::move(callback));\n}\n\nWFTimerTask *WFTaskFactory::create_timer_task(timer_callback_t callback)\n{\n\treturn new __WFCanceledTimerTask(WFGlobal::get_scheduler(),\n\t\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n/* Deprecated. */\nWFTimerTask *WFTaskFactory::create_timer_task(unsigned int microseconds,\n\t\t\t\t\t\t\t\t\t\t\t timer_callback_t callback)\n{\n\treturn WFTaskFactory::create_timer_task(microseconds / 1000000,\n\t\t\t\t\t\t\t\t\t\t\tmicroseconds % 1000000 * 1000,\n\t\t\t\t\t\t\t\t\t\t\tstd::move(callback));\n}\n\n/****************** Named Tasks ******************/\n\ntemplate\nstruct __NamedObjectList\n{\n\t__NamedObjectList(const std::string& str):\n\t\tname(str)\n\t{\n\t\tINIT_LIST_HEAD(&this->head);\n\t}\n\n\tvoid push_back(T *node)\n\t{\n\t\tlist_add_tail(&node->list, &this->head);\n\t}\n\n\tbool empty() const\n\t{\n\t\treturn list_empty(&this->head);\n\t}\n\n\tbool del(T *node, rb_root *root)\n\t{\n\t\tlist_del(&node->list);\n\t\tif (this->empty())\n\t\t{\n\t\t\trb_erase(&this->rb, root);\n\t\t\treturn true;\n\t\t}\n\t\telse\n\t\t\treturn false;\n\t}\n\n\tstruct rb_node rb;\n\tstruct list_head head;\n\tstd::string name;\n};\n\ntemplate\nstatic T *__get_object_list(const std::string& name, struct rb_root *root,\n\t\t\t\t\t\t\tbool insert)\n{\n\tstruct rb_node **p = &root->rb_node;\n\tstruct rb_node *parent = NULL;\n\tT *objs;\n\tint n;\n\n\twhile (*p)\n\t{\n\t\tparent = *p;\n\t\tobjs = rb_entry(*p, T, rb);\n\t\tn = name.compare(objs->name);\n\t\tif (n < 0)\n\t\t\tp = &(*p)->rb_left;\n\t\telse if (n > 0)\n\t\t\tp = &(*p)->rb_right;\n\t\telse\n\t\t\treturn objs;\n\t}\n\n\tif (insert)\n\t{\n\t\tobjs = new T(name);\n\t\trb_link_node(&objs->rb, parent, p);\n\t\trb_insert_color(&objs->rb, root);\n\t\treturn objs;\n\t}\n\n\treturn NULL;\n}\n\n/****************** Named Timer ******************/\n\nclass __WFNamedTimerTask;\n\nstruct __timer_node\n{\n\tstruct list_head list;\n\t__WFNamedTimerTask *task;\n};\n\nstatic class __NamedTimerMap\n{\npublic:\n\tusing TimerList = __NamedObjectList;\n\npublic:\n\tWFTimerTask *create(const std::string& name,\n\t\t\t\t\t\ttime_t seconds, long nanoseconds,\n\t\t\t\t\t\tCommScheduler *scheduler,\n\t\t\t\t\t\ttimer_callback_t&& cb);\n\npublic:\n\tint cancel(const std::string& name, size_t max);\n\nprivate:\n\tstruct rb_root root_;\n\tstd::mutex mutex_;\n\npublic:\n\t__NamedTimerMap()\n\t{\n\t\troot_.rb_node = NULL;\n\t}\n\n\tfriend class __WFNamedTimerTask;\n} __timer_map;\n\nclass __WFNamedTimerTask : public __WFTimerTask\n{\npublic:\n\t__WFNamedTimerTask(time_t seconds, long nanoseconds,\n\t\t\t\t\t CommScheduler *scheduler,\n\t\t\t\t\t timer_callback_t&& cb) :\n\t\t__WFTimerTask(seconds, nanoseconds, scheduler, std::move(cb)),\n\t\tflag_(false)\n\t{\n\t\tnode_.task = this;\n\t}\n\n\tvoid push_to(__NamedTimerMap::TimerList *timers)\n\t{\n\t\ttimers->push_back(&node_);\n\t\ttimers_ = timers;\n\t}\n\n\tvirtual ~__WFNamedTimerTask()\n\t{\n\t\tif (node_.task)\n\t\t{\n\t\t\tbool erased = false;\n\n\t\t\t__timer_map.mutex_.lock();\n\t\t\tif (node_.task)\n\t\t\t\terased = timers_->del(&node_, &__timer_map.root_);\n\n\t\t\t__timer_map.mutex_.unlock();\n\t\t\tif (erased)\n\t\t\t\tdelete timers_;\n\t\t}\n\t}\n\nprotected:\n\tvirtual void dispatch();\n\tvirtual void handle(int state, int error);\n\nprivate:\n\tstruct __timer_node node_;\n\t__NamedTimerMap::TimerList *timers_;\n\tstd::atomic flag_;\n\tstd::mutex mutex_;\n\tfriend class __NamedTimerMap;\n};\n\nvoid __WFNamedTimerTask::dispatch()\n{\n\tint ret;\n\n\tmutex_.lock();\n\tret = this->scheduler->sleep(this);\n\tif (ret >= 0 && flag_.exchange(true))\n\t\tthis->cancel();\n\n\tmutex_.unlock();\n\tif (ret < 0)\n\t\tthis->handle(WFT_STATE_SYS_ERROR, errno);\n}\n\nvoid __WFNamedTimerTask::handle(int state, int error)\n{\n\tbool canceled = true;\n\n\tif (node_.task)\n\t{\n\t\tbool erased = false;\n\n\t\t__timer_map.mutex_.lock();\n\t\tif (node_.task)\n\t\t{\n\t\t\tcanceled = false;\n\t\t\terased = timers_->del(&node_, &__timer_map.root_);\n\t\t\tnode_.task = NULL;\n\t\t}\n\n\t\t__timer_map.mutex_.unlock();\n\t\tif (erased)\n\t\t\tdelete timers_;\n\t}\n\n\tif (canceled)\n\t{\n\t\tstate = WFT_STATE_SYS_ERROR;\n\t\terror = ECANCELED;\n\t}\n\n\tmutex_.lock();\n\tmutex_.unlock();\n\tthis->__WFTimerTask::handle(state, error);\n}\n\nWFTimerTask *__NamedTimerMap::create(const std::string& name,\n\t\t\t\t\t\t\t\t\t time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t CommScheduler *scheduler,\n\t\t\t\t\t\t\t\t\t timer_callback_t&& cb)\n{\n\tauto *task = new __WFNamedTimerTask(seconds, nanoseconds, scheduler,\n\t\t\t\t\t\t\t\t\t\tstd::move(cb));\n\tmutex_.lock();\n\ttask->push_to(__get_object_list(name, &root_, true));\n\tmutex_.unlock();\n\treturn task;\n}\n\nint __NamedTimerMap::cancel(const std::string& name, size_t max)\n{\n\tstruct __timer_node *node;\n\tTimerList *timers;\n\tint ret = 0;\n\n\tmutex_.lock();\n\ttimers = __get_object_list(name, &root_, false);\n\tif (timers)\n\t{\n\t\twhile (1)\n\t\t{\n\t\t\tif (max == 0)\n\t\t\t{\n\t\t\t\ttimers = NULL;\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tnode = list_entry(timers->head.next, struct __timer_node, list);\n\t\t\tlist_del(&node->list);\n\t\t\tif (node->task->flag_.exchange(true))\n\t\t\t\tnode->task->cancel();\n\n\t\t\tnode->task = NULL;\n\t\t\tmax--;\n\t\t\tret++;\n\t\t\tif (timers->empty())\n\t\t\t{\n\t\t\t\trb_erase(&timers->rb, &root_);\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\n\tmutex_.unlock();\n\tdelete timers;\n\treturn ret;\n}\n\nWFTimerTask *WFTaskFactory::create_timer_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t timer_callback_t callback)\n{\n\treturn __timer_map.create(name, seconds, nanoseconds,\n\t\t\t\t\t\t\t WFGlobal::get_scheduler(),\n\t\t\t\t\t\t\t std::move(callback));\n}\n\nint WFTaskFactory::cancel_by_name(const std::string& name, size_t max)\n{\n\treturn __timer_map.cancel(name, max);\n}\n\n/****************** Named Counter ******************/\n\nclass __WFNamedCounterTask;\n\nstruct __counter_node\n{\n\tstruct list_head list;\n\tunsigned int target_value;\n\t__WFNamedCounterTask *task;\n};\n\nstatic class __NamedCounterMap\n{\npublic:\n\tusing CounterList = __NamedObjectList;\n\npublic:\n\tWFCounterTask *create(const std::string& name, unsigned int target_value,\n\t\t\t\t\t\t counter_callback_t&& cb);\n\n\tint count_n(const std::string& name, unsigned int n);\n\tvoid count(CounterList *counters, struct __counter_node *node);\n\n\tvoid remove(CounterList *counters, struct __counter_node *node)\n\t{\n\t\tbool erased;\n\n\t\tmutex_.lock();\n\t\terased = counters->del(node, &root_);\n\t\tmutex_.unlock();\n\t\tif (erased)\n\t\t\tdelete counters;\n\t}\n\nprivate:\n\tbool count_n_locked(CounterList *counters, unsigned int n,\n\t\t\t\t\t\tstruct list_head *task_list);\n\tstruct rb_root root_;\n\tstd::mutex mutex_;\n\npublic:\n\t__NamedCounterMap()\n\t{\n\t\troot_.rb_node = NULL;\n\t}\n} __counter_map;\n\nclass __WFNamedCounterTask : public WFCounterTask\n{\npublic:\n\t__WFNamedCounterTask(unsigned int target_value, counter_callback_t&& cb) :\n\t\tWFCounterTask(1, std::move(cb))\n\t{\n\t\tnode_.target_value = target_value;\n\t\tnode_.task = this;\n\t}\n\n\tvoid push_to(__NamedCounterMap::CounterList *counters)\n\t{\n\t\tcounters->push_back(&node_);\n\t\tcounters_ = counters;\n\t}\n\n\tvirtual void count()\n\t{\n\t\t__counter_map.count(counters_, &node_);\n\t}\n\n\tvirtual ~__WFNamedCounterTask()\n\t{\n\t\tif (this->value != 0)\n\t\t\t__counter_map.remove(counters_, &node_);\n\t}\n\nprivate:\n\tstruct __counter_node node_;\n\t__NamedCounterMap::CounterList *counters_;\n};\n\nWFCounterTask *__NamedCounterMap::create(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t unsigned int target_value,\n\t\t\t\t\t\t\t\t\t\t counter_callback_t&& cb)\n{\n\tif (target_value == 0)\n\t\treturn new WFCounterTask(0, std::move(cb));\n\n\tauto *task = new __WFNamedCounterTask(target_value, std::move(cb));\n\tmutex_.lock();\n\ttask->push_to(__get_object_list(name, &root_, true));\n\tmutex_.unlock();\n\treturn task;\n}\n\nbool __NamedCounterMap::count_n_locked(CounterList *counters, unsigned int n,\n\t\t\t\t\t\t\t\t\t struct list_head *task_list)\n{\n\tstruct __counter_node *node;\n\n\twhile (n > 0)\n\t{\n\t\tnode = list_entry(counters->head.next, struct __counter_node, list);\n\t\tif (n >= node->target_value)\n\t\t{\n\t\t\tn -= node->target_value;\n\t\t\tnode->target_value = 0;\n\t\t\tlist_move_tail(&node->list, task_list);\n\t\t\tif (counters->empty())\n\t\t\t{\n\t\t\t\trb_erase(&counters->rb, &root_);\n\t\t\t\treturn true;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tnode->target_value -= n;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\treturn false;\n}\n\nint __NamedCounterMap::count_n(const std::string& name, unsigned int n)\n{\n\tLIST_HEAD(task_list);\n\tstruct __counter_node *node;\n\tCounterList *counters;\n\tbool erased = false;\n\tint ret = 0;\n\n\tmutex_.lock();\n\tcounters = __get_object_list(name, &root_, false);\n\tif (counters)\n\t\terased = count_n_locked(counters, n, &task_list);\n\n\tmutex_.unlock();\n\tif (erased)\n\t\tdelete counters;\n\n\twhile (!list_empty(&task_list))\n\t{\n\t\tnode = list_entry(task_list.next, struct __counter_node, list);\n\t\tlist_del(&node->list);\n\t\tnode->task->WFCounterTask::count();\n\t\tret++;\n\t}\n\n\treturn ret;\n}\n\nvoid __NamedCounterMap::count(CounterList *counters,\n\t\t\t\t\t\t\t struct __counter_node *node)\n{\n\t__WFNamedCounterTask *task = NULL;\n\tbool erased = false;\n\n\tmutex_.lock();\n\tif (--node->target_value == 0)\n\t{\n\t\ttask = node->task;\n\t\terased = counters->del(node, &root_);\n\t}\n\n\tmutex_.unlock();\n\tif (erased)\n\t\tdelete counters;\n\n\tif (task)\n\t\ttask->WFCounterTask::count();\n}\n\nWFCounterTask *WFTaskFactory::create_counter_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t unsigned int target_value,\n\t\t\t\t\t\t\t\t\t\t\t\t counter_callback_t callback)\n{\n\treturn __counter_map.create(name, target_value, std::move(callback));\n}\n\nint WFTaskFactory::count_by_name(const std::string& name, unsigned int n)\n{\n\treturn __counter_map.count_n(name, n);\n}\n\n/****************** Named Mailbox ******************/\n\nclass __WFNamedMailboxTask;\n\nstruct __mailbox_node\n{\n\tstruct list_head list;\n\t__WFNamedMailboxTask *task;\n};\n\nstatic class __NamedMailboxMap\n{\npublic:\n\tusing MailboxList = __NamedObjectList;\n\npublic:\n\tWFMailboxTask *create(const std::string& name, void **mailbox,\n\t\t\t\t\t\t mailbox_callback_t&& cb);\n\tWFMailboxTask *create(const std::string& name, mailbox_callback_t&& cb);\n\n\tint send(const std::string& name, void *const msg[], size_t max, int inc);\n\tvoid send(MailboxList *mailboxes, struct __mailbox_node *node, void *msg);\n\n\tvoid remove(MailboxList *mailboxes, struct __mailbox_node *node)\n\t{\n\t\tbool erased;\n\n\t\tmutex_.lock();\n\t\terased = mailboxes->del(node, &root_);\n\t\tmutex_.unlock();\n\t\tif (erased)\n\t\t\tdelete mailboxes;\n\t}\n\nprivate:\n\tbool send_max_locked(MailboxList *mailboxes, size_t max,\n\t\t\t\t\t\t struct list_head *task_list);\n\tstruct rb_root root_;\n\tstd::mutex mutex_;\n\npublic:\n\t__NamedMailboxMap()\n\t{\n\t\troot_.rb_node = NULL;\n\t}\n} __mailbox_map;\n\nclass __WFNamedMailboxTask : public WFMailboxTask\n{\npublic:\n\t__WFNamedMailboxTask(void **mailbox, mailbox_callback_t&& cb) :\n\t\tWFMailboxTask(mailbox, std::move(cb))\n\t{\n\t\tnode_.task = this;\n\t}\n\n\t__WFNamedMailboxTask(mailbox_callback_t&& cb) :\n\t\tWFMailboxTask(std::move(cb))\n\t{\n\t\tnode_.task = this;\n\t}\n\n\tvoid push_to(__NamedMailboxMap::MailboxList *mailboxes)\n\t{\n\t\tmailboxes->push_back(&node_);\n\t\tmailboxes_ = mailboxes;\n\t}\n\n\tvirtual void send(void *msg)\n\t{\n\t\t__mailbox_map.send(mailboxes_, &node_, msg);\n\t}\n\n\tvirtual ~__WFNamedMailboxTask()\n\t{\n\t\tif (!this->flag)\n\t\t\t__mailbox_map.remove(mailboxes_, &node_);\n\t}\n\nprivate:\n\tstruct __mailbox_node node_;\n\t__NamedMailboxMap::MailboxList *mailboxes_;\n};\n\nWFMailboxTask *__NamedMailboxMap::create(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t void **mailbox,\n\t\t\t\t\t\t\t\t\t\t mailbox_callback_t&& cb)\n{\n\tauto *task = new __WFNamedMailboxTask(mailbox, std::move(cb));\n\tmutex_.lock();\n\ttask->push_to(__get_object_list(name, &root_, true));\n\tmutex_.unlock();\n\treturn task;\n}\n\nWFMailboxTask *__NamedMailboxMap::create(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t mailbox_callback_t&& cb)\n{\n\tauto *task = new __WFNamedMailboxTask(std::move(cb));\n\tmutex_.lock();\n\ttask->push_to(__get_object_list(name, &root_, true));\n\tmutex_.unlock();\n\treturn task;\n}\n\nbool __NamedMailboxMap::send_max_locked(MailboxList *mailboxes, size_t max,\n\t\t\t\t\t\t\t\t\t\tstruct list_head *task_list)\n{\n\tif (max == (size_t)-1)\n\t\tlist_splice(&mailboxes->head, task_list);\n\telse\n\t{\n\t\tdo\n\t\t{\n\t\t\tif (max == 0)\n\t\t\t\treturn false;\n\n\t\t\tlist_move_tail(mailboxes->head.next, task_list);\n\t\t\tmax--;\n\t\t} while (!mailboxes->empty());\n\t}\n\n\trb_erase(&mailboxes->rb, &root_);\n\treturn true;\n}\n\nint __NamedMailboxMap::send(const std::string& name, void *const msg[],\n\t\t\t\t\t\t\tsize_t max, int inc)\n{\n\tLIST_HEAD(task_list);\n\tstruct __mailbox_node *node;\n\tMailboxList *mailboxes;\n\tbool erased = false;\n\tint ret = 0;\n\n\tmutex_.lock();\n\tmailboxes = __get_object_list(name, &root_, false);\n\tif (mailboxes)\n\t\terased = send_max_locked(mailboxes, max, &task_list);\n\n\tmutex_.unlock();\n\tif (erased)\n\t\tdelete mailboxes;\n\n\twhile (!list_empty(&task_list))\n\t{\n\t\tnode = list_entry(task_list.next, struct __mailbox_node, list);\n\t\tlist_del(&node->list);\n\t\tnode->task->WFMailboxTask::send(*msg);\n\t\tmsg += inc;\n\t\tret++;\n\t}\n\n\treturn ret;\n}\n\nvoid __NamedMailboxMap::send(MailboxList *mailboxes,\n\t\t\t\t\t\t\t struct __mailbox_node *node,\n\t\t\t\t\t\t\t void *msg)\n{\n\tbool erased;\n\n\tmutex_.lock();\n\terased = mailboxes->del(node, &root_);\n\tmutex_.unlock();\n\tif (erased)\n\t\tdelete mailboxes;\n\n\tnode->task->WFMailboxTask::send(msg);\n}\n\nWFMailboxTask *WFTaskFactory::create_mailbox_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t void **mailbox,\n\t\t\t\t\t\t\t\t\t\t\t\t mailbox_callback_t callback)\n{\n\treturn __mailbox_map.create(name, mailbox, std::move(callback));\n}\n\nWFMailboxTask *WFTaskFactory::create_mailbox_task(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t mailbox_callback_t callback)\n{\n\treturn __mailbox_map.create(name, std::move(callback));\n}\n\nint WFTaskFactory::send_by_name(const std::string& name, void *msg,\n\t\t\t\t\t\t\t\tsize_t max)\n{\n\treturn __mailbox_map.send(name, &msg, max, 0);\n}\n\ntemplate<>\nint WFTaskFactory::send_by_name(const std::string& name, void *const msg[],\n\t\t\t\t\t\t\t\tsize_t max)\n{\n\treturn __mailbox_map.send(name, msg, max, 1);\n}\n\n/****************** Named Conditional ******************/\n\nclass __WFNamedConditional;\n\nstruct __conditional_node\n{\n\tstruct list_head list;\n\t__WFNamedConditional *cond;\n};\n\nstatic class __NamedConditionalMap\n{\npublic:\n\tusing ConditionalList = __NamedObjectList;\n\npublic:\n\tWFConditional *create(const std::string& name, SubTask *task,\n\t\t\t\t\t\t void **msgbuf);\n\tWFConditional *create(const std::string& name, SubTask *task);\n\n\tint signal(const std::string& name, void *const msg[], size_t max, int inc);\n\tvoid signal(ConditionalList *conds, struct __conditional_node *node,\n\t\t\t\tvoid *msg);\n\n\tvoid remove(ConditionalList *conds, struct __conditional_node *node)\n\t{\n\t\tbool erased;\n\n\t\tmutex_.lock();\n\t\terased = conds->del(node, &root_);\n\t\tmutex_.unlock();\n\t\tif (erased)\n\t\t\tdelete conds;\n\t}\n\nprivate:\n\tbool signal_max_locked(ConditionalList *conds, size_t max,\n\t\t\t\t\t\t struct list_head *cond_list);\n\tstruct rb_root root_;\n\tstd::mutex mutex_;\n\npublic:\n\t__NamedConditionalMap()\n\t{\n\t\troot_.rb_node = NULL;\n\t}\n} __conditional_map;\n\nclass __WFNamedConditional : public WFConditional\n{\npublic:\n\t__WFNamedConditional(SubTask *task, void **msgbuf) :\n\t\tWFConditional(task, msgbuf)\n\t{\n\t\tnode_.cond = this;\n\t}\n\n\t__WFNamedConditional(SubTask *task) :\n\t\tWFConditional(task)\n\t{\n\t\tnode_.cond = this;\n\t}\n\n\tvoid push_to(__NamedConditionalMap::ConditionalList *conds)\n\t{\n\t\tconds->push_back(&node_);\n\t\tconds_ = conds;\n\t}\n\n\tvirtual void signal(void *msg)\n\t{\n\t\t__conditional_map.signal(conds_, &node_, msg);\n\t}\n\n\tvirtual ~__WFNamedConditional()\n\t{\n\t\tif (!this->flag)\n\t\t\t__conditional_map.remove(conds_, &node_);\n\t}\n\nprivate:\n\tstruct __conditional_node node_;\n\t__NamedConditionalMap::ConditionalList *conds_;\n};\n\nWFConditional *__NamedConditionalMap::create(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t SubTask *task, void **msgbuf)\n{\n\tauto *cond = new __WFNamedConditional(task, msgbuf);\n\tmutex_.lock();\n\tcond->push_to(__get_object_list(name, &root_, true));\n\tmutex_.unlock();\n\treturn cond;\n}\n\nWFConditional *__NamedConditionalMap::create(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t SubTask *task)\n{\n\tauto *cond = new __WFNamedConditional(task);\n\tmutex_.lock();\n\tcond->push_to(__get_object_list(name, &root_, true));\n\tmutex_.unlock();\n\treturn cond;\n}\n\nbool __NamedConditionalMap::signal_max_locked(ConditionalList *conds,\n\t\t\t\t\t\t\t\t\t\t\t size_t max,\n\t\t\t\t\t\t\t\t\t\t\t struct list_head *cond_list)\n{\n\tif (max == (size_t)-1)\n\t\tlist_splice(&conds->head, cond_list);\n\telse\n\t{\n\t\tdo\n\t\t{\n\t\t\tif (max == 0)\n\t\t\t\treturn false;\n\n\t\t\tlist_move_tail(conds->head.next, cond_list);\n\t\t\tmax--;\n\t\t} while (!conds->empty());\n\t}\n\n\trb_erase(&conds->rb, &root_);\n\treturn true;\n}\n\nint __NamedConditionalMap::signal(const std::string& name, void *const msg[],\n\t\t\t\t\t\t\t\t size_t max, int inc)\n{\n\tLIST_HEAD(cond_list);\n\tstruct __conditional_node *node;\n\tConditionalList *conds;\n\tbool erased = false;\n\tint ret = 0;\n\n\tmutex_.lock();\n\tconds = __get_object_list(name, &root_, false);\n\tif (conds)\n\t\terased = signal_max_locked(conds, max, &cond_list);\n\n\tmutex_.unlock();\n\tif (erased)\n\t\tdelete conds;\n\n\twhile (!list_empty(&cond_list))\n\t{\n\t\tnode = list_entry(cond_list.next, struct __conditional_node, list);\n\t\tlist_del(&node->list);\n\t\tnode->cond->WFConditional::signal(*msg);\n\t\tmsg += inc;\n\t\tret++;\n\t}\n\n\treturn ret;\n}\n\nvoid __NamedConditionalMap::signal(ConditionalList *conds,\n\t\t\t\t\t\t\t\t struct __conditional_node *node,\n\t\t\t\t\t\t\t\t void *msg)\n{\n\tbool erased;\n\n\tmutex_.lock();\n\terased = conds->del(node, &root_);\n\tmutex_.unlock();\n\tif (erased)\n\t\tdelete conds;\n\n\tnode->cond->WFConditional::signal(msg);\n}\n\nWFConditional *WFTaskFactory::create_conditional(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t SubTask *task, void **msgbuf)\n{\n\treturn __conditional_map.create(name, task, msgbuf);\n}\n\nWFConditional *WFTaskFactory::create_conditional(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t SubTask *task)\n{\n\treturn __conditional_map.create(name, task);\n}\n\nint WFTaskFactory::signal_by_name(const std::string& name, void *msg,\n\t\t\t\t\t\t\t\t size_t max)\n{\n\treturn __conditional_map.signal(name, &msg, max, 0);\n}\n\ntemplate<>\nint WFTaskFactory::signal_by_name(const std::string& name, void *const msg[],\n\t\t\t\t\t\t\t\t size_t max)\n{\n\treturn __conditional_map.signal(name, msg, max, 1);\n}\n\n/****************** Named Guard ******************/\n\nclass __WFNamedGuard;\n\nstruct __guard_node\n{\n\tstruct list_head list;\n\t__WFNamedGuard *guard;\n};\n\nstatic class __NamedGuardMap\n{\npublic:\n\tstruct GuardList : public __NamedObjectList\n\t{\n\t\tGuardList(const std::string& name) : __NamedObjectList(name)\n\t\t{\n\t\t\tacquired = false;\n\t\t\trefcnt = 0;\n\t\t}\n\n\t\tbool acquired;\n\t\tsize_t refcnt;\n\t\tstd::mutex mutex;\n\t};\n\npublic:\n\tWFConditional *create(const std::string& name, SubTask *task);\n\tWFConditional *create(const std::string& name, SubTask *task,\n\t\t\t\t\t\t void **msgbuf);\n\n\tstruct __guard_node *release(const std::string& name);\n\n\tvoid unref(GuardList *guards)\n\t{\n\t\tmutex_.lock();\n\t\tif (--guards->refcnt == 0)\n\t\t\trb_erase(&guards->rb, &root_);\n\t\telse\n\t\t\tguards = NULL;\n\n\t\tmutex_.unlock();\n\t\tdelete guards;\n\t}\n\nprivate:\n\tstruct rb_root root_;\n\tstd::mutex mutex_;\n\npublic:\n\t__NamedGuardMap()\n\t{\n\t\troot_.rb_node = NULL;\n\t}\n} __guard_map;\n\nclass __WFNamedGuard : public WFConditional\n{\npublic:\n\t__WFNamedGuard(SubTask *task) : WFConditional(task)\n\t{\n\t\tnode_.guard = this;\n\t}\n\n\t__WFNamedGuard(SubTask *task, void **msgbuf) : WFConditional(task, msgbuf)\n\t{\n\t\tnode_.guard = this;\n\t}\n\n\tvirtual ~__WFNamedGuard()\n\t{\n\t\tif (!this->flag)\n\t\t\t__guard_map.unref(guards_);\n\t}\n\nprotected:\n\tvirtual void dispatch();\n\tvirtual void signal(void *msg) { }\n\nprivate:\n\tstruct __guard_node node_;\n\t__NamedGuardMap::GuardList *guards_;\n\tfriend __NamedGuardMap;\n};\n\nvoid __WFNamedGuard::dispatch()\n{\n\tguards_->mutex.lock();\n\tif (guards_->acquired)\n\t\tguards_->push_back(&node_);\n\telse\n\t{\n\t\tguards_->acquired = true;\n\t\tthis->WFConditional::signal(NULL);\n\t}\n\n\tguards_->mutex.unlock();\n\tthis->WFConditional::dispatch();\n}\n\nWFConditional *__NamedGuardMap::create(const std::string& name,\n\t\t\t\t\t\t\t\t\t SubTask *task)\n{\n\tauto *guard = new __WFNamedGuard(task);\n\tmutex_.lock();\n\tguard->guards_ = __get_object_list(name, &root_, true);\n\tguard->guards_->refcnt++;\n\tmutex_.unlock();\n\treturn guard;\n}\n\nWFConditional *__NamedGuardMap::create(const std::string& name,\n\t\t\t\t\t\t\t\t\t SubTask *task, void **msgbuf)\n{\n\tauto *guard = new __WFNamedGuard(task, msgbuf);\n\tmutex_.lock();\n\tguard->guards_ = __get_object_list(name, &root_, true);\n\tguard->guards_->refcnt++;\n\tmutex_.unlock();\n\treturn guard;\n}\n\nstruct __guard_node *__NamedGuardMap::release(const std::string& name)\n{\n\tstruct __guard_node *node = NULL;\n\tGuardList *guards;\n\n\tmutex_.lock();\n\tguards = __get_object_list(name, &root_, false);\n\tif (guards)\n\t{\n\t\tif (--guards->refcnt == 0)\n\t\t\trb_erase(&guards->rb, &root_);\n\t\telse\n\t\t{\n\t\t\tguards->mutex.lock();\n\t\t\tif (!guards->empty())\n\t\t\t{\n\t\t\t\tnode = list_entry(guards->head.next, struct __guard_node, list);\n\t\t\t\tlist_del(&node->list);\n\t\t\t}\n\t\t\telse\n\t\t\t\tguards->acquired = false;\n\n\t\t\tguards->mutex.unlock();\n\t\t\tguards = NULL;\n\t\t}\n\t}\n\n\tmutex_.unlock();\n\tdelete guards;\n\treturn node;\n}\n\nWFConditional *WFTaskFactory::create_guard(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t SubTask *task)\n{\n\treturn __guard_map.create(name, task);\n}\n\nWFConditional *WFTaskFactory::create_guard(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t SubTask *task, void **msgbuf)\n{\n\treturn __guard_map.create(name, task, msgbuf);\n}\n\nint WFTaskFactory::release_guard(const std::string& name, void *msg)\n{\n\tstruct __guard_node *node = __guard_map.release(name);\n\n\tif (!node)\n\t\treturn 0;\n\n\tnode->guard->WFConditional::signal(msg);\n\treturn 1;\n}\n\nint WFTaskFactory::release_guard_safe(const std::string& name, void *msg)\n{\n\tstruct __guard_node *node = __guard_map.release(name);\n\tWFTimerTask *timer;\n\n\tif (!node)\n\t\treturn 0;\n\n\ttimer = WFTaskFactory::create_timer_task([node](WFTimerTask *timer) {\n\t\tnode->guard->WFConditional::signal(timer->user_data);\n\t});\n\ttimer->user_data = msg;\n\ttimer->start();\n\treturn 1;\n}\n\n/**************** Timed Go Task *****************/\n\nvoid __WFTimedGoTask::dispatch()\n{\n\tWFTimerTask *timer;\n\n\ttimer = WFTaskFactory::create_timer_task(this->seconds, this->nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t __WFTimedGoTask::timer_callback);\n\ttimer->user_data = this;\n\n\tthis->__WFGoTask::dispatch();\n\ttimer->start();\n}\n\nSubTask *__WFTimedGoTask::done()\n{\n\tif (this->callback)\n\t\tthis->callback(this);\n\n\treturn series_of(this)->pop();\n}\n\nvoid __WFTimedGoTask::handle(int state, int error)\n{\n\tif (--this->ref == 3)\n\t{\n\t\tthis->state = state;\n\t\tthis->error = error;\n\t\tthis->subtask_done();\n\t}\n\n\tif (--this->ref == 0)\n\t\tdelete this;\n}\n\nvoid __WFTimedGoTask::timer_callback(WFTimerTask *timer)\n{\n\t__WFTimedGoTask *task = (__WFTimedGoTask *)timer->user_data;\n\n\tif (--task->ref == 3)\n\t{\n\t\tif (timer->get_state() == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\ttask->state = WFT_STATE_SYS_ERROR;\n\t\t\ttask->error = ETIMEDOUT;\n\t\t}\n\t\telse\n\t\t{\n\t\t\ttask->state = timer->get_state();\n\t\t\ttask->error = timer->get_error();\n\t\t}\n\n\t\ttask->subtask_done();\n\t}\n\n\tif (--task->ref == 0)\n\t\tdelete task;\n}\n\n" }, { "path": "src/factory/WFTaskFactory.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _WFTASKFACTORY_H_\n#define _WFTASKFACTORY_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \"URIParser.h\"\n#include \"RedisMessage.h\"\n#include \"HttpMessage.h\"\n#include \"MySQLMessage.h\"\n#include \"DnsMessage.h\"\n#include \"Workflow.h\"\n#include \"WFTask.h\"\n#include \"WFGraphTask.h\"\n#include \"EndpointParams.h\"\n\n// Network Client/Server tasks\n\nusing WFHttpTask = WFNetworkTask;\nusing http_callback_t = std::function;\n\nusing WFRedisTask = WFNetworkTask;\nusing redis_callback_t = std::function;\n\nusing WFMySQLTask = WFNetworkTask;\nusing mysql_callback_t = std::function;\n\nusing WFDnsTask = WFNetworkTask;\nusing dns_callback_t = std::function;\n\n// File IO tasks\n\nstruct FileIOArgs\n{\n\tint fd;\n\tvoid *buf;\n\tsize_t count;\n\toff_t offset;\n};\n\nstruct FileVIOArgs\n{\n\tint fd;\n\tconst struct iovec *iov;\n\tint iovcnt;\n\toff_t offset;\n};\n\nstruct FileSyncArgs\n{\n\tint fd;\n};\n\nusing WFFileIOTask = WFFileTask;\nusing fio_callback_t = std::function;\n\nusing WFFileVIOTask = WFFileTask;\nusing fvio_callback_t = std::function;\n\nusing WFFileSyncTask = WFFileTask;\nusing fsync_callback_t = std::function;\n\n// Timer and counter\nusing timer_callback_t = std::function;\nusing counter_callback_t = std::function;\n\nusing mailbox_callback_t = std::function;\n\nusing selector_callback_t = std::function;\n\n// Graph (DAG) task.\nusing graph_callback_t = std::function;\n\nusing WFEmptyTask = WFGenericTask;\n\nusing WFDynamicTask = WFGenericTask;\nusing dynamic_create_t = std::function;\n\nusing repeated_create_t = std::function;\nusing repeater_callback_t = std::function;\n\nusing module_callback_t = std::function;\n\nclass WFTaskFactory\n{\npublic:\n\tstatic WFHttpTask *create_http_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\tint redirect_max,\n\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\thttp_callback_t callback);\n\n\tstatic WFHttpTask *create_http_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\tint redirect_max,\n\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\thttp_callback_t callback);\n\n\tstatic WFHttpTask *create_http_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\tconst std::string& proxy_url,\n\t\t\t\t\t\t\t\t\t\tint redirect_max,\n\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\thttp_callback_t callback);\n\n\tstatic WFHttpTask *create_http_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\tconst ParsedURI& proxy_uri,\n\t\t\t\t\t\t\t\t\t\tint redirect_max,\n\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\thttp_callback_t callback);\n\n\tstatic WFRedisTask *create_redis_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t redis_callback_t callback);\n\n\tstatic WFRedisTask *create_redis_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t redis_callback_t callback);\n\n\tstatic WFMySQLTask *create_mysql_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t mysql_callback_t callback);\n\n\tstatic WFMySQLTask *create_mysql_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t\t mysql_callback_t callback);\n\n\tstatic WFDnsTask *create_dns_task(const std::string& url,\n\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t dns_callback_t callback);\n\n\tstatic WFDnsTask *create_dns_task(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t\t dns_callback_t callback);\n\npublic:\n\tstatic WFFileIOTask *create_pread_task(int fd,\n\t\t\t\t\t\t\t\t\t\t void *buf,\n\t\t\t\t\t\t\t\t\t\t size_t count,\n\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t fio_callback_t callback);\n\n\tstatic WFFileIOTask *create_pwrite_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\tconst void *buf,\n\t\t\t\t\t\t\t\t\t\t\tsize_t count,\n\t\t\t\t\t\t\t\t\t\t\toff_t offset,\n\t\t\t\t\t\t\t\t\t\t\tfio_callback_t callback);\n\n\tstatic WFFileVIOTask *create_preadv_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t const struct iovec *iov,\n\t\t\t\t\t\t\t\t\t\t\t int iovcnt,\n\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t fvio_callback_t callback);\n\n\tstatic WFFileVIOTask *create_pwritev_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t const struct iovec *iov,\n\t\t\t\t\t\t\t\t\t\t\t int iovcnt,\n\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t fvio_callback_t callback);\n\n\tstatic WFFileSyncTask *create_fsync_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t fsync_callback_t callback);\n\n\tstatic WFFileSyncTask *create_fdatasync_task(int fd,\n\t\t\t\t\t\t\t\t\t\t\t\t fsync_callback_t callback);\n\n\t/* File tasks with path name. */\npublic:\n\tstatic WFFileIOTask *create_pread_task(const std::string& path,\n\t\t\t\t\t\t\t\t\t\t void *buf,\n\t\t\t\t\t\t\t\t\t\t size_t count,\n\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t fio_callback_t callback);\n\n\tstatic WFFileIOTask *create_pwrite_task(const std::string& path,\n\t\t\t\t\t\t\t\t\t\t\tconst void *buf,\n\t\t\t\t\t\t\t\t\t\t\tsize_t count,\n\t\t\t\t\t\t\t\t\t\t\toff_t offset,\n\t\t\t\t\t\t\t\t\t\t\tfio_callback_t callback);\n\n\tstatic WFFileVIOTask *create_preadv_task(const std::string& path,\n\t\t\t\t\t\t\t\t\t\t\t const struct iovec *iov,\n\t\t\t\t\t\t\t\t\t\t\t int iovcnt,\n\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t fvio_callback_t callback);\n\n\tstatic WFFileVIOTask *create_pwritev_task(const std::string& path,\n\t\t\t\t\t\t\t\t\t\t\t const struct iovec *iov,\n\t\t\t\t\t\t\t\t\t\t\t int iovcnt,\n\t\t\t\t\t\t\t\t\t\t\t off_t offset,\n\t\t\t\t\t\t\t\t\t\t\t fvio_callback_t callback);\n\npublic:\n\tstatic WFTimerTask *create_timer_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t timer_callback_t callback);\n\n\t/* Create a named timer. */\n\tstatic WFTimerTask *create_timer_task(const std::string& timer_name,\n\t\t\t\t\t\t\t\t\t\t time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t timer_callback_t callback);\n\n\t/* Cancel all timers under the name. */\n\tstatic int cancel_by_name(const std::string& timer_name)\n\t{\n\t\treturn WFTaskFactory::cancel_by_name(timer_name, (size_t)-1);\n\t}\n\n\t/* Cancel at most 'max' timers under the name. */\n\tstatic int cancel_by_name(const std::string& timer_name, size_t max);\n\n\t/* Timer to be canceled immediately after started. */\n\tstatic WFTimerTask *create_timer_task(timer_callback_t callback);\n\n\t/* Timer in microseconds. (deprecated) */\n\tstatic WFTimerTask *create_timer_task(unsigned int microseconds,\n\t\t\t\t\t\t\t\t\t\t timer_callback_t callback);\n\npublic:\n\t/* Create an unnamed counter. Call counter->count() directly.\n\t * NOTE: never call count() exceeding target_value. */\n\tstatic WFCounterTask *create_counter_task(unsigned int target_value,\n\t\t\t\t\t\t\t\t\t\t\t counter_callback_t callback)\n\t{\n\t\treturn new WFCounterTask(target_value, std::move(callback));\n\t}\n\n\t/* Create a named counter. */\n\tstatic WFCounterTask *create_counter_task(const std::string& counter_name,\n\t\t\t\t\t\t\t\t\t\t\t unsigned int target_value,\n\t\t\t\t\t\t\t\t\t\t\t counter_callback_t callback);\n\n\t/* Count by a counter's name. When count_by_name(), it's safe to count\n\t * exceeding target_value. When multiple counters share a same name,\n\t * this operation will be performed on the first created. */\n\tstatic int count_by_name(const std::string& counter_name)\n\t{\n\t\treturn WFTaskFactory::count_by_name(counter_name, 1);\n\t}\n\n\t/* Count by name with a value n. When multiple counters share this name,\n\t * the operation is performed on the counters in the sequence of its\n\t * creation, and more than one counter may reach target value. */\n\tstatic int count_by_name(const std::string& counter_name, unsigned int n);\n\npublic:\n\tstatic WFMailboxTask *create_mailbox_task(void **mailbox,\n\t\t\t\t\t\t\t\t\t\t\t mailbox_callback_t callback)\n\t{\n\t\treturn new WFMailboxTask(mailbox, std::move(callback));\n\t}\n\n\t/* Use 'user_data' as mailbox. */\n\tstatic WFMailboxTask *create_mailbox_task(mailbox_callback_t callback)\n\t{\n\t\treturn new WFMailboxTask(std::move(callback));\n\t}\n\n\tstatic WFMailboxTask *create_mailbox_task(const std::string& mailbox_name,\n\t\t\t\t\t\t\t\t\t\t\t void **mailbox,\n\t\t\t\t\t\t\t\t\t\t\t mailbox_callback_t callback);\n\n\tstatic WFMailboxTask *create_mailbox_task(const std::string& mailbox_name,\n\t\t\t\t\t\t\t\t\t\t\t mailbox_callback_t callback);\n\n\t/* The 'msg' will be sent to the all mailbox tasks under the name, and\n\t * would be lost if no task matched. */\n\tstatic int send_by_name(const std::string& mailbox_name, void *msg)\n\t{\n\t\treturn WFTaskFactory::send_by_name(mailbox_name, msg, (size_t)-1);\n\t}\n\n\tstatic int send_by_name(const std::string& mailbox_name, void *msg,\n\t\t\t\t\t\t\tsize_t max);\n\n\ttemplate\n\tstatic int send_by_name(const std::string& mailbox_name, T *const msg[],\n\t\t\t\t\t\t\tsize_t max);\n\npublic:\n\tstatic WFSelectorTask *create_selector_task(size_t candidates,\n\t\t\t\t\t\t\t\t\t\t\t\tselector_callback_t callback)\n\t{\n\t\treturn new WFSelectorTask(candidates, std::move(callback));\n\t}\n\npublic:\n\tstatic WFConditional *create_conditional(SubTask *task, void **msgbuf)\n\t{\n\t\treturn new WFConditional(task, msgbuf);\n\t}\n\n\tstatic WFConditional *create_conditional(SubTask *task)\n\t{\n\t\treturn new WFConditional(task);\n\t}\n\n\tstatic WFConditional *create_conditional(const std::string& cond_name,\n\t\t\t\t\t\t\t\t\t\t\t SubTask *task, void **msgbuf);\n\n\tstatic WFConditional *create_conditional(const std::string& cond_name,\n\t\t\t\t\t\t\t\t\t\t\t SubTask *task);\n\n\tstatic int signal_by_name(const std::string& cond_name, void *msg)\n\t{\n\t\treturn WFTaskFactory::signal_by_name(cond_name, msg, (size_t)-1);\n\t}\n\n\tstatic int signal_by_name(const std::string& cond_name, void *msg,\n\t\t\t\t\t\t\t size_t max);\n\n\ttemplate\n\tstatic int signal_by_name(const std::string& cond_name, T *const msg[],\n\t\t\t\t\t\t\t size_t max);\n\npublic:\n\tstatic WFConditional *create_guard(const std::string& resource_name,\n\t\t\t\t\t\t\t\t\t SubTask *task);\n\n\tstatic WFConditional *create_guard(const std::string& resource_name,\n\t\t\t\t\t\t\t\t\t SubTask *task, void **msgbuf);\n\n\t/* The 'guard' is acquired after started, so call 'release_guard' after\n\t and only after the task is finished, typically in its callback.\n\t The function returns 1 if another is signaled, otherwise returns 0. */\n\tstatic int release_guard(const std::string& resource_name, void *msg);\n\n\tstatic int release_guard_safe(const std::string& resource_name, void *msg);\n\npublic:\n\ttemplate\n\tstatic WFGoTask *create_go_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\tFUNC&& func, ARGS&&... args);\n\n\t/* Create 'Go' task with running time limit in seconds plus nanoseconds.\n\t * If time exceeded, state WFT_STATE_SYS_ERROR and error ETIMEDOUT\n\t * will be got in callback. */\n\ttemplate\n\tstatic WFGoTask *create_timedgo_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t FUNC&& func, ARGS&&... args);\n\n\t/* Create 'Go' task on user's executor and execution queue. */\n\ttemplate\n\tstatic WFGoTask *create_go_task(ExecQueue *queue, Executor *executor,\n\t\t\t\t\t\t\t\t\tFUNC&& func, ARGS&&... args);\n\n\ttemplate\n\tstatic WFGoTask *create_timedgo_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t ExecQueue *queue, Executor *executor,\n\t\t\t\t\t\t\t\t\t\t FUNC&& func, ARGS&&... args);\n\n\t/* For capturing 'task' itself in go task's running function. */\n\ttemplate\n\tstatic void reset_go_task(WFGoTask *task, FUNC&& func, ARGS&&... args);\n\npublic:\n\tstatic WFGraphTask *create_graph_task(graph_callback_t callback)\n\t{\n\t\treturn new WFGraphTask(std::move(callback));\n\t}\n\npublic:\n\tstatic WFEmptyTask *create_empty_task()\n\t{\n\t\treturn new WFEmptyTask;\n\t}\n\n\tstatic WFDynamicTask *create_dynamic_task(dynamic_create_t create);\n\n\tstatic WFRepeaterTask *create_repeater_task(repeated_create_t create,\n\t\t\t\t\t\t\t\t\t\t\t\trepeater_callback_t callback)\n\t{\n\t\treturn new WFRepeaterTask(std::move(create), std::move(callback));\n\t}\n\npublic:\n\tstatic WFModuleTask *create_module_task(SubTask *first,\n\t\t\t\t\t\t\t\t\t\t\tmodule_callback_t callback)\n\t{\n\t\treturn new WFModuleTask(first, std::move(callback));\n\t}\n\n\tstatic WFModuleTask *create_module_task(SubTask *first, SubTask *last,\n\t\t\t\t\t\t\t\t\t\t\tmodule_callback_t callback)\n\t{\n\t\tWFModuleTask *task = new WFModuleTask(first, std::move(callback));\n\t\ttask->sub_series()->set_last_task(last);\n\t\treturn task;\n\t}\n};\n\ntemplate\nclass WFNetworkTaskFactory\n{\nprivate:\n\tusing T = WFNetworkTask;\n\npublic:\n\tstatic T *create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t const std::string& host,\n\t\t\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t const std::string& url,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t const ParsedURI& uri,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t const struct sockaddr *addr,\n\t\t\t\t\t\t\t\t socklen_t addrlen,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\n\n\tstatic T *create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t const struct sockaddr *addr,\n\t\t\t\t\t\t\t\t socklen_t addrlen,\n\t\t\t\t\t\t\t\t SSL_CTX *ssl_ctx,\n\t\t\t\t\t\t\t\t int retry_max,\n\t\t\t\t\t\t\t\t std::function callback);\npublic:\n\tstatic T *create_server_task(CommService *service,\n\t\t\t\t\t\t\t\t std::function& process);\n};\n\ntemplate\nclass WFThreadTaskFactory\n{\nprivate:\n\tusing T = WFThreadTask;\n\npublic:\n\tstatic T *create_thread_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\tstd::function routine,\n\t\t\t\t\t\t\t\tstd::function callback);\n\n\t/* Create thread task with running time limit. */\n\tstatic T *create_thread_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\tconst std::string& queue_name,\n\t\t\t\t\t\t\t\tstd::function routine,\n\t\t\t\t\t\t\t\tstd::function callback);\n\npublic:\n\t/* Create thread task on user's executor and execution queue. */\n\tstatic T *create_thread_task(ExecQueue *queue, Executor *executor,\n\t\t\t\t\t\t\t\tstd::function routine,\n\t\t\t\t\t\t\t\tstd::function callback);\n\n\t/* With running time limit. */\n\tstatic T *create_thread_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\tExecQueue *queue, Executor *executor,\n\t\t\t\t\t\t\t\tstd::function routine,\n\t\t\t\t\t\t\t\tstd::function callback);\n};\n\n#include \"WFTaskFactory.inl\"\n\n#endif\n\n" }, { "path": "src/factory/WFTaskFactory.inl", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"WFGlobal.h\"\n#include \"Workflow.h\"\n#include \"WFTask.h\"\n#include \"RouteManager.h\"\n#include \"URIParser.h\"\n#include \"WFTaskError.h\"\n#include \"EndpointParams.h\"\n#include \"WFNameService.h\"\n\nclass __WFDynamicTask : public WFDynamicTask\n{\nprotected:\n\tvirtual void dispatch()\n\t{\n\t\tseries_of(this)->push_front(this->create(this));\n\t\tthis->WFDynamicTask::dispatch();\n\t}\n\nprotected:\n\tstd::function create;\n\npublic:\n\t__WFDynamicTask(std::function&& create) :\n\t\tcreate(std::move(create))\n\t{\n\t}\n};\n\ninline WFDynamicTask *\nWFTaskFactory::create_dynamic_task(dynamic_create_t create)\n{\n\treturn new __WFDynamicTask(std::move(create));\n}\n\ntemplate<>\nint WFTaskFactory::send_by_name(const std::string&, void *const *, size_t);\n\ntemplate\nint WFTaskFactory::send_by_name(const std::string& mailbox_name, T *const msg[],\n\t\t\t\t\t\t\t\tsize_t max)\n{\n\treturn WFTaskFactory::send_by_name(mailbox_name, (void *const *)msg, max);\n}\n\ntemplate<>\nint WFTaskFactory::signal_by_name(const std::string&, void *const *, size_t);\n\ntemplate\nint WFTaskFactory::signal_by_name(const std::string& cond_name, T *const msg[],\n\t\t\t\t\t\t\t\t size_t max)\n{\n\treturn WFTaskFactory::signal_by_name(cond_name, (void *const *)msg, max);\n}\n\ntemplate\nclass WFComplexClientTask : public WFClientTask\n{\nprotected:\n\tusing task_callback_t = std::function *)>;\n\npublic:\n\tWFComplexClientTask(int retry_max, task_callback_t&& cb):\n\t\tWFClientTask(NULL, WFGlobal::get_scheduler(), std::move(cb))\n\t{\n\t\ttype_ = TT_TCP;\n\t\tssl_ctx_ = NULL;\n\t\tfixed_addr_ = false;\n\t\tfixed_conn_ = false;\n\t\tretry_max_ = retry_max;\n\t\tretry_times_ = 0;\n\t\tredirect_ = false;\n\t\tns_policy_ = NULL;\n\t\trouter_task_ = NULL;\n\t}\n\nprotected:\n\t// new api for children\n\tvirtual bool init_success() { return true; }\n\tvirtual void init_failed() {}\n\tvirtual bool check_request() { return true; }\n\tvirtual WFRouterTask *route();\n\tvirtual bool finish_once() { return true; }\n\npublic:\n\tvoid init(const ParsedURI& uri)\n\t{\n\t\turi_ = uri;\n\t\tinit_with_uri();\n\t}\n\n\tvoid init(ParsedURI&& uri)\n\t{\n\t\turi_ = std::move(uri);\n\t\tinit_with_uri();\n\t}\n\n\tvoid init(enum TransportType type,\n\t\t\t const struct sockaddr *addr,\n\t\t\t socklen_t addrlen,\n\t\t\t const std::string& info);\n\n\tvoid set_transport_type(enum TransportType type)\n\t{\n\t\ttype_ = type;\n\t}\n\n\tenum TransportType get_transport_type() const { return type_; }\n\n\tvoid set_ssl_ctx(SSL_CTX *ssl_ctx) { ssl_ctx_ = ssl_ctx; }\n\n\tvirtual const ParsedURI *get_current_uri() const { return &uri_; }\n\n\tvoid set_redirect(const ParsedURI& uri)\n\t{\n\t\tredirect_ = true;\n\t\tinit(uri);\n\t}\n\n\tvoid set_redirect(enum TransportType type, const struct sockaddr *addr,\n\t\t\t\t\t socklen_t addrlen, const std::string& info)\n\t{\n\t\tredirect_ = true;\n\t\tinit(type, addr, addrlen, info);\n\t}\n\n\tbool is_fixed_addr() const { return this->fixed_addr_; }\n\n\tbool is_fixed_conn() const { return this->fixed_conn_; }\n\nprotected:\n\tvoid set_fixed_addr(int fixed) { this->fixed_addr_ = fixed; }\n\n\tvoid set_fixed_conn(int fixed) { this->fixed_conn_ = fixed; }\n\n\tvoid set_info(const std::string& info)\n\t{\n\t\tinfo_.assign(info);\n\t}\n\n\tvoid set_info(const char *info)\n\t{\n\t\tinfo_.assign(info);\n\t}\n\nprotected:\n\tvirtual void dispatch();\n\tvirtual SubTask *done();\n\n\tvoid clear_resp()\n\t{\n\t\tprotocol::ProtocolMessage head(std::move(this->resp));\n\t\tthis->resp.~RESP();\n\t\tnew(&this->resp) RESP;\n\t\t*(protocol::ProtocolMessage *)&this->resp = std::move(head);\n\t}\n\n\tvoid disable_retry()\n\t{\n\t\tretry_times_ = retry_max_;\n\t}\n\nprotected:\n\tenum TransportType type_;\n\tParsedURI uri_;\n\tstd::string info_;\n\tSSL_CTX *ssl_ctx_;\n\tbool fixed_addr_;\n\tbool fixed_conn_;\n\tbool redirect_;\n\tCTX ctx_;\n\tint retry_max_;\n\tint retry_times_;\n\tWFNSPolicy *ns_policy_;\n\tWFRouterTask *router_task_;\n\tRouteManager::RouteResult route_result_;\n\tWFNSTracing tracing_;\n\npublic:\n\tCTX *get_mutable_ctx() { return &ctx_; }\n\nprivate:\n\tvoid clear_prev_state();\n\tvoid init_with_uri();\n\tbool set_port();\n\tvoid router_callback(void *t);\n\tvoid switch_callback(void *t);\n};\n\ntemplate\nvoid WFComplexClientTask::clear_prev_state()\n{\n\tns_policy_ = NULL;\n\troute_result_.clear();\n\tif (tracing_.deleter)\n\t{\n\t\ttracing_.deleter(tracing_.data);\n\t\ttracing_.deleter = NULL;\n\t}\n\ttracing_.data = NULL;\n\tretry_times_ = 0;\n\tthis->state = WFT_STATE_UNDEFINED;\n\tthis->error = 0;\n\tthis->timeout_reason = TOR_NOT_TIMEOUT;\n}\n\ntemplate\nvoid WFComplexClientTask::init(enum TransportType type,\n\t\t\t\t\t\t\t\t\t\t\t const struct sockaddr *addr,\n\t\t\t\t\t\t\t\t\t\t\t socklen_t addrlen,\n\t\t\t\t\t\t\t\t\t\t\t const std::string& info)\n{\n\tif (redirect_)\n\t\tclear_prev_state();\n\n\tauto params = WFGlobal::get_global_settings()->endpoint_params;\n\tstruct addrinfo addrinfo = { };\n\taddrinfo.ai_family = addr->sa_family;\n\taddrinfo.ai_addr = (struct sockaddr *)addr;\n\taddrinfo.ai_addrlen = addrlen;\n\n\ttype_ = type;\n\tinfo_.assign(info);\n\tparams.use_tls_sni = false;\n\tif (WFGlobal::get_route_manager()->get(type, &addrinfo, info_, ¶ms,\n\t\t\t\t\t\t\t\t\t\t \"\", ssl_ctx_, route_result_) < 0)\n\t{\n\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\tthis->error = errno;\n\t}\n\telse if (this->init_success())\n\t\treturn;\n\n\tthis->init_failed();\n}\n\ntemplate\nbool WFComplexClientTask::set_port()\n{\n\tif (uri_.port)\n\t{\n\t\tint port = atoi(uri_.port);\n\n\t\tif (port <= 0 || port > 65535)\n\t\t{\n\t\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\t\tthis->error = WFT_ERR_URI_PORT_INVALID;\n\t\t\treturn false;\n\t\t}\n\n\t\treturn true;\n\t}\n\n\tif (uri_.scheme)\n\t{\n\t\tconst char *port_str = WFGlobal::get_default_port(uri_.scheme);\n\n\t\tif (port_str)\n\t\t{\n\t\t\turi_.port = strdup(port_str);\n\t\t\tif (uri_.port)\n\t\t\t\treturn true;\n\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t\treturn false;\n\t\t}\n\t}\n\n\tthis->state = WFT_STATE_TASK_ERROR;\n\tthis->error = WFT_ERR_URI_SCHEME_INVALID;\n\treturn false;\n}\n\ntemplate\nvoid WFComplexClientTask::init_with_uri()\n{\n\tif (redirect_)\n\t{\n\t\tclear_prev_state();\n\t\tns_policy_ = WFGlobal::get_dns_resolver();\n\t}\n\n\tif (uri_.state == URI_STATE_SUCCESS)\n\t{\n\t\tif (this->set_port())\n\t\t{\n\t\t\tif (this->init_success())\n\t\t\t\treturn;\n\t\t}\n\t}\n\telse if (uri_.state == URI_STATE_ERROR)\n\t{\n\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\tthis->error = uri_.error;\n\t}\n\telse\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_URI_PARSE_FAILED;\n\t}\n\n\tthis->init_failed();\n}\n\ntemplate\nWFRouterTask *WFComplexClientTask::route()\n{\n\tauto&& cb = std::bind(&WFComplexClientTask::router_callback,\n\t\t\t\t\t\t this,\n\t\t\t\t\t\t std::placeholders::_1);\n\tstruct WFNSParams params = {\n\t\t.type\t\t\t=\ttype_,\n\t\t.uri\t\t\t=\turi_,\n\t\t.info\t\t\t=\tinfo_.c_str(),\n\t\t.ssl_ctx\t\t=\tssl_ctx_,\n\t\t.fixed_addr\t\t=\tfixed_addr_,\n\t\t.fixed_conn\t\t=\tfixed_conn_,\n\t\t.retry_times\t=\tretry_times_,\n\t\t.tracing\t\t=\t&tracing_,\n\t};\n\n\tif (!ns_policy_)\n\t{\n\t\tWFNameService *ns = WFGlobal::get_name_service();\n\t\tns_policy_ = ns->get_policy(uri_.host ? uri_.host : \"\");\n\t}\n\n\treturn ns_policy_->create_router_task(¶ms, std::move(cb));\n}\n\ntemplate\nvoid WFComplexClientTask::router_callback(void *t)\n{\n\tWFRouterTask *task = (WFRouterTask *)t;\n\n\tthis->state = task->get_state();\n\tif (this->state == WFT_STATE_SUCCESS)\n\t\troute_result_ = std::move(*task->get_result());\n\telse if (this->state == WFT_STATE_UNDEFINED)\n\t{\n\t\t/* should not happend */\n\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\tthis->error = ENOSYS;\n\t}\n\telse\n\t\tthis->error = task->get_error();\n}\n\ntemplate\nvoid WFComplexClientTask::dispatch()\n{\n\tswitch (this->state)\n\t{\n\tcase WFT_STATE_UNDEFINED:\n\t\tif (this->check_request())\n\t\t{\n\t\t\tif (this->route_result_.request_object)\n\t\t\t{\n\tcase WFT_STATE_SUCCESS:\n\t\t\t\tthis->set_request_object(route_result_.request_object);\n\t\t\t\tthis->WFClientTask::dispatch();\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\trouter_task_ = this->route();\n\t\t\tseries_of(this)->push_front(this);\n\t\t\tseries_of(this)->push_front(router_task_);\n\t\t}\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\tthis->subtask_done();\n}\n\ntemplate\nvoid WFComplexClientTask::switch_callback(void *t)\n{\n\tif (!redirect_)\n\t{\n\t\tif (this->state == WFT_STATE_SYS_ERROR && this->error < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SSL_ERROR;\n\t\t\tthis->error = -this->error;\n\t\t}\n\n\t\tif (tracing_.deleter)\n\t\t{\n\t\t\ttracing_.deleter(tracing_.data);\n\t\t\ttracing_.deleter = NULL;\n\t\t}\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\t}\n\n\tif (redirect_)\n\t{\n\t\tredirect_ = false;\n\t\tclear_resp();\n\t\tthis->target = NULL;\n\t\tseries_of(this)->push_front(this);\n\t}\n\telse\n\t\tdelete this;\n}\n\ntemplate\nSubTask *WFComplexClientTask::done()\n{\n\tSeriesWork *series = series_of(this);\n\n\tif (router_task_)\n\t{\n\t\trouter_task_ = NULL;\n\t\treturn series->pop();\n\t}\n\n\tbool is_user_request = this->finish_once();\n\n\tif (ns_policy_)\n\t{\n\t\tif (this->state == WFT_STATE_SYS_ERROR ||\n\t\t\tthis->state == WFT_STATE_DNS_ERROR)\n\t\t{\n\t\t\tns_policy_->failed(&route_result_, &tracing_, this->target);\n\t\t}\n\t\telse if (route_result_.request_object)\n\t\t{\n\t\t\tns_policy_->success(&route_result_, &tracing_, this->target);\n\t\t}\n\t}\n\n\tif (this->state == WFT_STATE_SUCCESS)\n\t{\n\t\tif (!is_user_request)\n\t\t\treturn this;\n\t}\n\telse if (this->state == WFT_STATE_SYS_ERROR)\n\t{\n\t\tif (retry_times_ < retry_max_)\n\t\t{\n\t\t\tredirect_ = true;\n\t\t\tif (ns_policy_)\n\t\t\t\troute_result_.clear();\n\n\t\t\tthis->state = WFT_STATE_UNDEFINED;\n\t\t\tthis->error = 0;\n\t\t\tthis->timeout_reason = 0;\n\t\t\tretry_times_++;\n\t\t}\n\t}\n\n\t/* When the target or the connection is NULL, it's very likely that we are\n\t * in the caller's thread. Running a timer will switch callback function to\n\t * a handler thread, and this can prevent stack overflow. */\n\tif (!this->target || !this->CommSession::get_connection())\n\t{\n\t\tauto&& cb = std::bind(&WFComplexClientTask::switch_callback,\n\t\t\t\t\t\t\t this,\n\t\t\t\t\t\t\t std::placeholders::_1);\n\t\tWFTimerTask *timer = WFTaskFactory::create_timer_task(std::move(cb));\n\t\tseries->push_front(timer);\n\t}\n\telse\n\t\tthis->switch_callback(NULL);\n\n\treturn series->pop();\n}\n\n/**********Template Network Factory**********/\n\ntemplate\nWFNetworkTask *\nWFNetworkTaskFactory::create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t\t\t\t\t\tconst std::string& host,\n\t\t\t\t\t\t\t\t\t\t\t\t\tunsigned short port,\n\t\t\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::function *)> callback)\n{\n\tauto *task = new WFComplexClientTask(retry_max, std::move(callback));\n\tParsedURI uri;\n\tchar buf[32];\n\n\tsprintf(buf, \"%u\", port);\n\turi.scheme = strdup(\"scheme\");\n\turi.host = strdup(host.c_str());\n\turi.port = strdup(buf);\n\tif (!uri.scheme || !uri.host || !uri.port)\n\t{\n\t\turi.state = URI_STATE_ERROR;\n\t\turi.error = errno;\n\t}\n\telse\n\t\turi.state = URI_STATE_SUCCESS;\n\n\ttask->init(std::move(uri));\n\ttask->set_transport_type(type);\n\treturn task;\n}\n\ntemplate\nWFNetworkTask *\nWFNetworkTaskFactory::create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t\t\t\t\t\tconst std::string& url,\n\t\t\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::function *)> callback)\n{\n\tauto *task = new WFComplexClientTask(retry_max, std::move(callback));\n\tParsedURI uri;\n\n\tURIParser::parse(url, uri);\n\ttask->init(std::move(uri));\n\ttask->set_transport_type(type);\n\treturn task;\n}\n\ntemplate\nWFNetworkTask *\nWFNetworkTaskFactory::create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t\t\t\t\t\tconst ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::function *)> callback)\n{\n\tauto *task = new WFComplexClientTask(retry_max, std::move(callback));\n\n\ttask->init(uri);\n\ttask->set_transport_type(type);\n\treturn task;\n}\n\ntemplate\nWFNetworkTask *\nWFNetworkTaskFactory::create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t\t\t\t\t\tconst struct sockaddr *addr,\n\t\t\t\t\t\t\t\t\t\t\t\t\tsocklen_t addrlen,\n\t\t\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::function *)> callback)\n{\n\tauto *task = new WFComplexClientTask(retry_max, std::move(callback));\n\n\ttask->init(type, addr, addrlen, \"\");\n\treturn task;\n}\n\ntemplate\nWFNetworkTask *\nWFNetworkTaskFactory::create_client_task(enum TransportType type,\n\t\t\t\t\t\t\t\t\t\t\t\t\tconst struct sockaddr *addr,\n\t\t\t\t\t\t\t\t\t\t\t\t\tsocklen_t addrlen,\n\t\t\t\t\t\t\t\t\t\t\t\t\tSSL_CTX *ssl_ctx,\n\t\t\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::function *)> callback)\n{\n\tauto *task = new WFComplexClientTask(retry_max, std::move(callback));\n\n\ttask->set_ssl_ctx(ssl_ctx);\n\ttask->init(type, addr, addrlen, \"\");\n\treturn task;\n}\n\ntemplate\nWFNetworkTask *\nWFNetworkTaskFactory::create_server_task(CommService *service,\n\t\t\t\tstd::function *)>& process)\n{\n\treturn new WFServerTask(service, WFGlobal::get_scheduler(), process);\n}\n\n/**********Server Factory**********/\n\nclass WFServerTaskFactory\n{\npublic:\n\tstatic WFDnsTask *create_dns_task(CommService *service,\n\t\t\t\t\tstd::function& process);\n\n\tstatic WFHttpTask *create_http_task(CommService *service,\n\t\t\t\t\tstd::function& process);\n\n\tstatic WFMySQLTask *create_mysql_task(CommService *service,\n\t\t\t\t\tstd::function& process);\n};\n\n/************Go Task Factory************/\n\nclass __WFGoTask : public WFGoTask\n{\npublic:\n\tvoid set_go_func(std::function func)\n\t{\n\t\tthis->go = std::move(func);\n\t}\n\nprotected:\n\tvirtual void execute()\n\t{\n\t\tthis->go();\n\t}\n\nprotected:\n\tstd::function go;\n\npublic:\n\t__WFGoTask(ExecQueue *queue, Executor *executor,\n\t\t\t std::function&& func) :\n\t\tWFGoTask(queue, executor),\n\t\tgo(std::move(func))\n\t{\n\t}\n};\n\nclass __WFTimedGoTask : public __WFGoTask\n{\nprotected:\n\tvirtual void dispatch();\n\tvirtual SubTask *done();\n\nprotected:\n\tvirtual void handle(int state, int error);\n\nprotected:\n\tstatic void timer_callback(WFTimerTask *timer);\n\nprotected:\n\ttime_t seconds;\n\tlong nanoseconds;\n\tstd::atomic ref;\n\npublic:\n\t__WFTimedGoTask(time_t seconds, long nanoseconds,\n\t\t\t\t\tExecQueue *queue, Executor *executor,\n\t\t\t\t\tstd::function&& func) :\n\t\t__WFGoTask(queue, executor, std::move(func)),\n\t\tref(4)\n\t{\n\t\tthis->seconds = seconds;\n\t\tthis->nanoseconds = nanoseconds;\n\t}\n};\n\ntemplate\nWFGoTask *WFTaskFactory::create_go_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\tFUNC&& func, ARGS&&... args)\n{\n\tauto&& tmp = std::bind(std::forward(func),\n\t\t\t\t\t\t std::forward(args)...);\n\treturn new __WFGoTask(WFGlobal::get_exec_queue(queue_name),\n\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t std::move(tmp));\n}\n\ntemplate\nWFGoTask *WFTaskFactory::create_timedgo_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t FUNC&& func, ARGS&&... args)\n{\n\tauto&& tmp = std::bind(std::forward(func),\n\t\t\t\t\t\t std::forward(args)...);\n\treturn new __WFTimedGoTask(seconds, nanoseconds,\n\t\t\t\t\t\t\t WFGlobal::get_exec_queue(queue_name),\n\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t std::move(tmp));\n}\n\ntemplate\nWFGoTask *WFTaskFactory::create_go_task(ExecQueue *queue, Executor *executor,\n\t\t\t\t\t\t\t\t\t\tFUNC&& func, ARGS&&... args)\n{\n\tauto&& tmp = std::bind(std::forward(func),\n\t\t\t\t\t\t std::forward(args)...);\n\treturn new __WFGoTask(queue, executor, std::move(tmp));\n}\n\ntemplate\nWFGoTask *WFTaskFactory::create_timedgo_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t ExecQueue *queue, Executor *executor,\n\t\t\t\t\t\t\t\t\t\t\t FUNC&& func, ARGS&&... args)\n{\n\tauto&& tmp = std::bind(std::forward(func),\n\t\t\t\t\t\t std::forward(args)...);\n\treturn new __WFTimedGoTask(seconds, nanoseconds,\n\t\t\t\t\t\t\t queue, executor,\n\t\t\t\t\t\t\t std::move(tmp));\n}\n\ntemplate\nvoid WFTaskFactory::reset_go_task(WFGoTask *task, FUNC&& func, ARGS&&... args)\n{\n\tauto&& tmp = std::bind(std::forward(func),\n\t\t\t\t\t\t std::forward(args)...);\n\t((__WFGoTask *)task)->set_go_func(std::move(tmp));\n}\n\n/**********Create go task with nullptr func**********/\n\ntemplate<> inline\nWFGoTask *WFTaskFactory::create_go_task(const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\tstd::nullptr_t&&)\n{\n\treturn new __WFGoTask(WFGlobal::get_exec_queue(queue_name),\n\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t nullptr);\n}\n\ntemplate<> inline\nWFGoTask *WFTaskFactory::create_timedgo_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t const std::string& queue_name,\n\t\t\t\t\t\t\t\t\t\t\t std::nullptr_t&&)\n{\n\treturn new __WFTimedGoTask(seconds, nanoseconds,\n\t\t\t\t\t\t\t WFGlobal::get_exec_queue(queue_name),\n\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t nullptr);\n}\n\ntemplate<> inline\nWFGoTask *WFTaskFactory::create_go_task(ExecQueue *queue, Executor *executor,\n\t\t\t\t\t\t\t\t\t\tstd::nullptr_t&&)\n{\n\treturn new __WFGoTask(queue, executor, nullptr);\n}\n\ntemplate<> inline\nWFGoTask *WFTaskFactory::create_timedgo_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t ExecQueue *queue, Executor *executor,\n\t\t\t\t\t\t\t\t\t\t\t std::nullptr_t&&)\n{\n\treturn new __WFTimedGoTask(seconds, nanoseconds, queue, executor, nullptr);\n}\n\ntemplate<> inline\nvoid WFTaskFactory::reset_go_task(WFGoTask *task, std::nullptr_t&&)\n{\n\t((__WFGoTask *)task)->set_go_func(nullptr);\n}\n\n/**********Template Thread Task Factory**********/\n\ntemplate\nclass __WFThreadTask : public WFThreadTask\n{\nprotected:\n\tvirtual void execute()\n\t{\n\t\tthis->routine(&this->input, &this->output);\n\t}\n\nprotected:\n\tstd::function routine;\n\npublic:\n\t__WFThreadTask(ExecQueue *queue, Executor *executor,\n\t\t\t\t std::function&& rt,\n\t\t\t\t std::function *)>&& cb) :\n\t\tWFThreadTask(queue, executor, std::move(cb)),\n\t\troutine(std::move(rt))\n\t{\n\t}\n};\n\ntemplate\nclass __WFTimedThreadTask : public __WFThreadTask\n{\nprotected:\n\tvirtual void dispatch();\n\tvirtual SubTask *done();\n\nprotected:\n\tvirtual void handle(int state, int error);\n\nprotected:\n\tstatic void timer_callback(WFTimerTask *timer);\n\nprotected:\n\ttime_t seconds;\n\tlong nanoseconds;\n\tstd::atomic ref;\n\npublic:\n\t__WFTimedThreadTask(time_t seconds, long nanoseconds,\n\t\t\t\t\t\tExecQueue *queue, Executor *executor,\n\t\t\t\t\t\tstd::function&& rt,\n\t\t\t\t\t\tstd::function *)>&& cb) :\n\t\t__WFThreadTask(queue, executor, std::move(rt), std::move(cb)),\n\t\tref(4)\n\t{\n\t\tthis->seconds = seconds;\n\t\tthis->nanoseconds = nanoseconds;\n\t}\n};\n\ntemplate\nvoid __WFTimedThreadTask::dispatch()\n{\n\tWFTimerTask *timer;\n\n\ttimer = WFTaskFactory::create_timer_task(this->seconds, this->nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t __WFTimedThreadTask::timer_callback);\n\ttimer->user_data = this;\n\n\tthis->__WFThreadTask::dispatch();\n\ttimer->start();\n}\n\ntemplate\nSubTask *__WFTimedThreadTask::done()\n{\n\tif (this->callback)\n\t\tthis->callback(this);\n\n\treturn series_of(this)->pop();\n}\n\ntemplate\nvoid __WFTimedThreadTask::handle(int state, int error)\n{\n\tif (--this->ref == 3)\n\t{\n\t\tthis->state = state;\n\t\tthis->error = error;\n\t\tthis->subtask_done();\n\t}\n\n\tif (--this->ref == 0)\n\t\tdelete this;\n}\n\ntemplate\nvoid __WFTimedThreadTask::timer_callback(WFTimerTask *timer)\n{\n\tauto *task = (__WFTimedThreadTask *)timer->user_data;\n\n\tif (--task->ref == 3)\n\t{\n\t\tif (timer->get_state() == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\ttask->state = WFT_STATE_SYS_ERROR;\n\t\t\ttask->error = ETIMEDOUT;\n\t\t}\n\t\telse\n\t\t{\n\t\t\ttask->state = timer->get_state();\n\t\t\ttask->error = timer->get_error();\n\t\t}\n\n\t\ttask->subtask_done();\n\t}\n\n\tif (--task->ref == 0)\n\t\tdelete task;\n}\n\ntemplate\nWFThreadTask *\nWFThreadTaskFactory::create_thread_task(const std::string& queue_name,\n\t\t\t\t\t\tstd::function routine,\n\t\t\t\t\t\tstd::function *)> callback)\n{\n\treturn new __WFThreadTask(WFGlobal::get_exec_queue(queue_name),\n\t\t\t\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t\t\t\t std::move(routine),\n\t\t\t\t\t\t\t\t\t\t\t std::move(callback));\n}\n\ntemplate\nWFThreadTask *\nWFThreadTaskFactory::create_thread_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\tconst std::string& queue_name,\n\t\t\t\t\t\tstd::function routine,\n\t\t\t\t\t\tstd::function *)> callback)\n{\n\treturn new __WFTimedThreadTask(seconds, nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t\t WFGlobal::get_exec_queue(queue_name),\n\t\t\t\t\t\t\t\t\t\t\t\t WFGlobal::get_compute_executor(),\n\t\t\t\t\t\t\t\t\t\t\t\t std::move(routine),\n\t\t\t\t\t\t\t\t\t\t\t\t std::move(callback));\n}\n\ntemplate\nWFThreadTask *\nWFThreadTaskFactory::create_thread_task(ExecQueue *queue, Executor *executor,\n\t\t\t\t\t\tstd::function routine,\n\t\t\t\t\t\tstd::function *)> callback)\n{\n\treturn new __WFThreadTask(queue, executor,\n\t\t\t\t\t\t\t\t\t\t\t std::move(routine),\n\t\t\t\t\t\t\t\t\t\t\t std::move(callback));\n}\n\ntemplate\nWFThreadTask *\nWFThreadTaskFactory::create_thread_task(time_t seconds, long nanoseconds,\n\t\t\t\t\t\tExecQueue *queue, Executor *executor,\n\t\t\t\t\t\tstd::function routine,\n\t\t\t\t\t\tstd::function *)> callback)\n{\n\treturn new __WFTimedThreadTask(seconds, nanoseconds,\n\t\t\t\t\t\t\t\t\t\t\t\t queue, executor,\n\t\t\t\t\t\t\t\t\t\t\t\t std::move(routine),\n\t\t\t\t\t\t\t\t\t\t\t\t std::move(callback));\n}\n\n" }, { "path": "src/factory/Workflow.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \"Workflow.h\"\n\nSeriesWork::SeriesWork(SubTask *first, series_callback_t&& cb) :\n\tcallback(std::move(cb))\n{\n\tthis->queue = this->buf;\n\tthis->queue_size = sizeof this->buf / sizeof *this->buf;\n\tthis->front = 0;\n\tthis->back = 0;\n\tthis->canceled = false;\n\tthis->finished = false;\n\tassert(!series_of(first));\n\tfirst->set_pointer(this);\n\tthis->first = first;\n\tthis->last = NULL;\n\tthis->context = NULL;\n\tthis->in_parallel = NULL;\n}\n\nSeriesWork::~SeriesWork()\n{\n\tif (this->queue != this->buf)\n\t\tdelete []this->queue;\n}\n\nvoid SeriesWork::dismiss_recursive()\n{\n\tSubTask *task = first;\n\n\tthis->callback = nullptr;\n\tdo\n\t{\n\t\tdelete task;\n\t\ttask = this->pop_task();\n\t} while (task);\n}\n\nvoid SeriesWork::expand_queue()\n{\n\tint size = 2 * this->queue_size;\n\tSubTask **queue = new SubTask *[size];\n\tint i, j;\n\n\ti = 0;\n\tj = this->front;\n\tdo\n\t{\n\t\tqueue[i++] = this->queue[j++];\n\t\tif (j == this->queue_size)\n\t\t\tj = 0;\n\t} while (j != this->back);\n\n\tif (this->queue != this->buf)\n\t\tdelete []this->queue;\n\n\tthis->queue = queue;\n\tthis->queue_size = size;\n\tthis->front = 0;\n\tthis->back = i;\n}\n\nvoid SeriesWork::push_front(SubTask *task)\n{\n\tthis->mutex.lock();\n\tif (--this->front == -1)\n\t\tthis->front = this->queue_size - 1;\n\n\ttask->set_pointer(this);\n\tthis->queue[this->front] = task;\n\tif (this->front == this->back)\n\t\tthis->expand_queue();\n\n\tthis->mutex.unlock();\n}\n\nvoid SeriesWork::push_back(SubTask *task)\n{\n\tthis->mutex.lock();\n\ttask->set_pointer(this);\n\tthis->queue[this->back] = task;\n\tif (++this->back == this->queue_size)\n\t\tthis->back = 0;\n\n\tif (this->front == this->back)\n\t\tthis->expand_queue();\n\n\tthis->mutex.unlock();\n}\n\nSubTask *SeriesWork::pop()\n{\n\tbool canceled = this->canceled;\n\tSubTask *task = this->pop_task();\n\n\tif (!canceled)\n\t\treturn task;\n\n\twhile (task)\n\t{\n\t\tdelete task;\n\t\ttask = this->pop_task();\n\t}\n\n\treturn NULL;\n}\n\nSubTask *SeriesWork::pop_task()\n{\n\tSubTask *task;\n\n\tthis->mutex.lock();\n\tif (this->front != this->back)\n\t{\n\t\ttask = this->queue[this->front];\n\t\tif (++this->front == this->queue_size)\n\t\t\tthis->front = 0;\n\t}\n\telse\n\t{\n\t\ttask = this->last;\n\t\tthis->last = NULL;\n\t}\n\n\tthis->mutex.unlock();\n\tif (!task)\n\t{\n\t\tthis->finished = true;\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tif (!this->in_parallel)\n\t\t\tdelete this;\n\t}\n\n\treturn task;\n}\n\nParallelWork::ParallelWork(parallel_callback_t&& cb) :\n\tParallelTask(new SubTask *[2 * 4], 0),\n\tcallback(std::move(cb))\n{\n\tthis->buf_size = 4;\n\tthis->all_series = (SeriesWork **)&this->subtasks[this->buf_size];\n\tthis->context = NULL;\n}\n\nParallelWork::ParallelWork(SeriesWork *const all_series[], size_t n,\n\t\t\t\t\t\t parallel_callback_t&& cb) :\n\tParallelTask(new SubTask *[2 * (n > 4 ? n : 4)], n),\n\tcallback(std::move(cb))\n{\n\tsize_t i;\n\n\tthis->buf_size = (n > 4 ? n : 4);\n\tthis->all_series = (SeriesWork **)&this->subtasks[this->buf_size];\n\tfor (i = 0; i < n; i++)\n\t{\n\t\tassert(!all_series[i]->in_parallel);\n\t\tall_series[i]->in_parallel = this;\n\t\tthis->all_series[i] = all_series[i];\n\t\tthis->subtasks[i] = all_series[i]->first;\n\t}\n\n\tthis->context = NULL;\n}\n\nvoid ParallelWork::expand_buf()\n{\n\tSubTask **buf;\n\tsize_t size;\n\n\tthis->buf_size *= 2;\n\tbuf = new SubTask *[2 * this->buf_size];\n\tsize = this->subtasks_nr * sizeof (void *);\n\tmemcpy(buf, this->subtasks, size);\n\tmemcpy(buf + this->buf_size, this->all_series, size);\n\n\tdelete []this->subtasks;\n\tthis->subtasks = buf;\n\tthis->all_series = (SeriesWork **)&buf[this->buf_size];\n}\n\nvoid ParallelWork::add_series(SeriesWork *series)\n{\n\tif (this->subtasks_nr == this->buf_size)\n\t\tthis->expand_buf();\n\n\tassert(!series->in_parallel);\n\tseries->in_parallel = this;\n\tthis->all_series[this->subtasks_nr] = series;\n\tthis->subtasks[this->subtasks_nr] = series->first;\n\tthis->subtasks_nr++;\n}\n\nSubTask *ParallelWork::done()\n{\n\tSeriesWork *series = series_of(this);\n\tsize_t i;\n\n\tif (this->callback)\n\t\tthis->callback(this);\n\n\tfor (i = 0; i < this->subtasks_nr; i++)\n\t\tdelete this->all_series[i];\n\n\tthis->subtasks_nr = 0;\n\tdelete this;\n\treturn series->pop();\n}\n\nParallelWork::~ParallelWork()\n{\n\tsize_t i;\n\n\tfor (i = 0; i < this->subtasks_nr; i++)\n\t{\n\t\tthis->all_series[i]->in_parallel = NULL;\n\t\tthis->all_series[i]->dismiss_recursive();\n\t}\n\n\tdelete []this->subtasks;\n}\n\n" }, { "path": "src/factory/Workflow.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WORKFLOW_H_\n#define _WORKFLOW_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \"SubTask.h\"\n\nclass SeriesWork;\nclass ParallelWork;\n\nusing series_callback_t = std::function;\nusing parallel_callback_t = std::function;\n\nclass Workflow\n{\npublic:\n\tstatic SeriesWork *\n\tcreate_series_work(SubTask *first, series_callback_t callback);\n\n\tstatic void\n\tstart_series_work(SubTask *first, series_callback_t callback);\n\n\tstatic ParallelWork *\n\tcreate_parallel_work(parallel_callback_t callback);\n\n\tstatic ParallelWork *\n\tcreate_parallel_work(SeriesWork *const all_series[], size_t n,\n\t\t\t\t\t\t parallel_callback_t callback);\n\n\tstatic void\n\tstart_parallel_work(SeriesWork *const all_series[], size_t n,\n\t\t\t\t\t\tparallel_callback_t callback);\n\npublic:\n\tstatic SeriesWork *\n\tcreate_series_work(SubTask *first, SubTask *last,\n\t\t\t\t\t series_callback_t callback);\n\n\tstatic void\n\tstart_series_work(SubTask *first, SubTask *last,\n\t\t\t\t\t series_callback_t callback);\n};\n\nclass SeriesWork\n{\npublic:\n\tvoid start()\n\t{\n\t\tassert(!this->in_parallel);\n\t\tthis->first->dispatch();\n\t}\n\n\t/* Call dismiss() only when you don't want to start a created series.\n\t * This operation is recursive, so only call on the \"root\". */\n\tvoid dismiss()\n\t{\n\t\tassert(!this->in_parallel);\n\t\tthis->dismiss_recursive();\n\t}\n\npublic:\n\tvoid push_back(SubTask *task);\n\tvoid push_front(SubTask *task);\n\npublic:\n\tvoid *get_context() const { return this->context; }\n\tvoid set_context(void *context) { this->context = context; }\n\npublic:\n\t/* Cancel a running series. Typically, called in the callback of a task\n\t * that belongs to the series. All subsequent tasks in the series will be\n\t * destroyed immediately and recursively (ParallelWork), without callback.\n\t * But the callback of this canceled series will still be called. */\n\tvirtual void cancel() { this->canceled = true; }\n\n\t/* Parallel work's callback may check the cancellation state of each\n\t * sub-series, and cancel it's super-series recursively. */\n\tbool is_canceled() const { return this->canceled; }\n\n\t/* 'false' until the time of callback. Mainly for sub-class. */\n\tbool is_finished() const { return this->finished; }\n\npublic:\n\tvoid set_callback(series_callback_t callback)\n\t{\n\t\tthis->callback = std::move(callback);\n\t}\n\npublic:\n\tvirtual void *get_specific(const char *key) { return NULL; }\n\npublic:\n\t/* The following functions are intended for task implementations only. */\n\tSubTask *pop();\n\n\tSubTask *get_last_task() const { return this->last; }\n\n\tvoid set_last_task(SubTask *last)\n\t{\n\t\tlast->set_pointer(this);\n\t\tthis->last = last;\n\t}\n\n\tvoid unset_last_task() { this->last = NULL; }\n\n\tconst ParallelTask *get_in_parallel() const { return this->in_parallel; }\n\nprotected:\n\tvoid set_in_parallel(const ParallelTask *task) { this->in_parallel = task; }\n\n\tvoid dismiss_recursive();\n\nprotected:\n\tvoid *context;\n\tseries_callback_t callback;\n\nprivate:\n\tSubTask *pop_task();\n\tvoid expand_queue();\n\nprivate:\n\tSubTask *buf[4];\n\tSubTask *first;\n\tSubTask *last;\n\tSubTask **queue;\n\tint queue_size;\n\tint front;\n\tint back;\n\tbool canceled;\n\tbool finished;\n\tconst ParallelTask *in_parallel;\n\tstd::mutex mutex;\n\nprotected:\n\tSeriesWork(SubTask *first, series_callback_t&& callback);\n\tvirtual ~SeriesWork();\n\tfriend class ParallelWork;\n\tfriend class Workflow;\n};\n\nstatic inline SeriesWork *series_of(const SubTask *task)\n{\n\treturn (SeriesWork *)task->get_pointer();\n}\n\nstatic inline SeriesWork& operator *(const SubTask& task)\n{\n\treturn *series_of(&task);\n}\n\nstatic inline SeriesWork& operator << (SeriesWork& series, SubTask *task)\n{\n\tseries.push_back(task);\n\treturn series;\n}\n\ninline SeriesWork *\nWorkflow::create_series_work(SubTask *first, series_callback_t callback)\n{\n\treturn new SeriesWork(first, std::move(callback));\n}\n\ninline void\nWorkflow::start_series_work(SubTask *first, series_callback_t callback)\n{\n\tnew SeriesWork(first, std::move(callback));\n\tfirst->dispatch();\n}\n\ninline SeriesWork *\nWorkflow::create_series_work(SubTask *first, SubTask *last,\n\t\t\t\t\t\t\t series_callback_t callback)\n{\n\tSeriesWork *series = new SeriesWork(first, std::move(callback));\n\tseries->set_last_task(last);\n\treturn series;\n}\n\ninline void\nWorkflow::start_series_work(SubTask *first, SubTask *last,\n\t\t\t\t\t\t\tseries_callback_t callback)\n{\n\tSeriesWork *series = new SeriesWork(first, std::move(callback));\n\tseries->set_last_task(last);\n\tfirst->dispatch();\n}\n\nclass ParallelWork : public ParallelTask\n{\npublic:\n\tvoid start()\n\t{\n\t\tassert(!series_of(this));\n\t\tWorkflow::start_series_work(this, nullptr);\n\t}\n\n\tvoid dismiss()\n\t{\n\t\tassert(!series_of(this));\n\t\tdelete this;\n\t}\n\npublic:\n\tvoid add_series(SeriesWork *series);\n\npublic:\n\tvoid *get_context() const { return this->context; }\n\tvoid set_context(void *context) { this->context = context; }\n\npublic:\n\tSeriesWork *series_at(size_t index)\n\t{\n\t\tif (index < this->subtasks_nr)\n\t\t\treturn this->all_series[index];\n\t\telse\n\t\t\treturn NULL;\n\t}\n\n\tconst SeriesWork *series_at(size_t index) const\n\t{\n\t\tif (index < this->subtasks_nr)\n\t\t\treturn this->all_series[index];\n\t\telse\n\t\t\treturn NULL;\n\t}\n\n\tSeriesWork& operator[] (size_t index) { return *this->all_series[index]; }\n\n\tconst SeriesWork& operator[] (size_t index) const\n\t{\n\t\treturn *this->all_series[index];\n\t}\n\n\tsize_t size() const { return this->subtasks_nr; }\n\npublic:\n\tSeriesWork *const *begin() { return this->all_series; }\n\tSeriesWork *const *end() { return this->all_series + this->subtasks_nr; }\n\n\tconst SeriesWork *const *begin() const\n\t{\n\t\treturn (const SeriesWork **)this->all_series;\n\t}\n\n\tconst SeriesWork *const *end() const\n\t{\n\t\treturn (const SeriesWork **)this->all_series + this->subtasks_nr;\n\t}\n\npublic:\n\tvoid set_callback(parallel_callback_t callback)\n\t{\n\t\tthis->callback = std::move(callback);\n\t}\n\nprotected:\n\tvirtual SubTask *done();\n\nprotected:\n\tvoid *context;\n\tparallel_callback_t callback;\n\nprivate:\n\tvoid expand_buf();\n\nprivate:\n\tsize_t buf_size;\n\tSeriesWork **all_series;\n\nprotected:\n\tParallelWork(parallel_callback_t&& callback);\n\tParallelWork(SeriesWork *const all_series[], size_t n,\n\t\t\t\t parallel_callback_t&& callback);\n\tvirtual ~ParallelWork();\n\tfriend class Workflow;\n};\n\ninline ParallelWork *\nWorkflow::create_parallel_work(parallel_callback_t callback)\n{\n\treturn new ParallelWork(std::move(callback));\n}\n\ninline ParallelWork *\nWorkflow::create_parallel_work(SeriesWork *const all_series[], size_t n,\n\t\t\t\t\t\t\t parallel_callback_t callback)\n{\n\treturn new ParallelWork(all_series, n, std::move(callback));\n}\n\ninline void\nWorkflow::start_parallel_work(SeriesWork *const all_series[], size_t n,\n\t\t\t\t\t\t\t parallel_callback_t callback)\n{\n\tParallelWork *p = new ParallelWork(all_series, n, std::move(callback));\n\tWorkflow::start_series_work(p, nullptr);\n}\n\n#endif\n\n" }, { "path": "src/factory/xmake.lua", "content": "target(\"factory\")\n add_files(\"*.cc\")\n set_kind(\"object\")\n if not has_config(\"mysql\") then\n remove_files(\"MySQLTaskImpl.cc\")\n end\n if not has_config(\"redis\") then\n remove_files(\"RedisTaskImpl.cc\")\n end\n remove_files(\"KafkaTaskImpl.cc\")\n\ntarget(\"kafka_factory\")\n if has_config(\"kafka\") then\n add_files(\"KafkaTaskImpl.cc\")\n set_kind(\"object\")\n add_deps(\"factory\")\n add_packages(\"zlib\", \"snappy\", \"zstd\", \"lz4\")\n else\n set_kind(\"phony\")\n end\n" }, { "path": "src/kernel/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\nproject(kernel)\n\nif (CMAKE_SYSTEM_NAME STREQUAL \"Linux\" OR CMAKE_SYSTEM_NAME STREQUAL \"Android\")\n\tset(IOSERVICE_FILE IOService_linux.cc)\nelseif (UNIX)\n\tset(IOSERVICE_FILE IOService_thread.cc)\nelse ()\n\tmessage(FATAL_ERROR \"IOService unsupported.\")\nendif ()\n\nset(SRC\n\t${IOSERVICE_FILE}\n\tmpoller.c\n\tpoller.c\n\trbtree.c\n\tmsgqueue.c\n\tthrdpool.c\n\tCommRequest.cc\n\tCommScheduler.cc\n\tCommunicator.cc\n\tExecutor.cc\n\tSubTask.cc\n)\n\nadd_library(${PROJECT_NAME} OBJECT ${SRC})\n\n" }, { "path": "src/kernel/CommRequest.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \"CommScheduler.h\"\n#include \"CommRequest.h\"\n\nvoid CommRequest::handle(int state, int error)\n{\n\tthis->state = state;\n\tthis->error = error;\n\tif (error != ETIMEDOUT)\n\t\tthis->timeout_reason = TOR_NOT_TIMEOUT;\n\telse if (!this->target)\n\t\tthis->timeout_reason = TOR_WAIT_TIMEOUT;\n\telse if (!this->get_message_out())\n\t\tthis->timeout_reason = TOR_CONNECT_TIMEOUT;\n\telse\n\t\tthis->timeout_reason = TOR_TRANSMIT_TIMEOUT;\n\n\tthis->subtask_done();\n}\n\n" }, { "path": "src/kernel/CommRequest.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _COMMREQUEST_H_\n#define _COMMREQUEST_H_\n\n#include \n#include \n#include \"SubTask.h\"\n#include \"Communicator.h\"\n#include \"CommScheduler.h\"\n\nclass CommRequest : public SubTask, public CommSession\n{\npublic:\n\tCommRequest(CommSchedObject *object, CommScheduler *scheduler)\n\t{\n\t\tthis->scheduler = scheduler;\n\t\tthis->object = object;\n\t\tthis->wait_timeout = 0;\n\t}\n\n\tCommSchedObject *get_request_object() const { return this->object; }\n\tvoid set_request_object(CommSchedObject *object) { this->object = object; }\n\tint get_wait_timeout() const { return this->wait_timeout; }\n\tvoid set_wait_timeout(int timeout) { this->wait_timeout = timeout; }\n\npublic:\n\tvirtual void dispatch()\n\t{\n\t\tif (this->scheduler->request(this, this->object, this->wait_timeout,\n\t\t\t\t\t\t\t\t\t &this->target) < 0)\n\t\t{\n\t\t\tthis->handle(CS_STATE_ERROR, errno);\n\t\t}\n\t}\n\nprotected:\n\tint state;\n\tint error;\n\nprotected:\n\tCommTarget *target;\n#define TOR_NOT_TIMEOUT\t\t\t0\n#define TOR_WAIT_TIMEOUT\t\t1\n#define TOR_CONNECT_TIMEOUT\t\t2\n#define TOR_TRANSMIT_TIMEOUT\t3\n\tint timeout_reason;\n\nprotected:\n\tint wait_timeout;\n\tCommSchedObject *object;\n\tCommScheduler *scheduler;\n\nprotected:\n\tvirtual void handle(int state, int error);\n};\n\n#endif\n\n" }, { "path": "src/kernel/CommScheduler.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"CommScheduler.h\"\n\n#define PTHREAD_COND_TIMEDWAIT(cond, mutex, abstime) \\\n\t((abstime) ? pthread_cond_timedwait(cond, mutex, abstime) : \\\n\t\t\t\t pthread_cond_wait(cond, mutex))\n\nstatic struct timespec *__get_abstime(int timeout, struct timespec *ts)\n{\n\tif (timeout < 0)\n\t\treturn NULL;\n\n\tclock_gettime(CLOCK_REALTIME, ts);\n\tts->tv_sec += timeout / 1000;\n\tts->tv_nsec += timeout % 1000 * 1000000;\n\tif (ts->tv_nsec >= 1000000000)\n\t{\n\t\tts->tv_nsec -= 1000000000;\n\t\tts->tv_sec++;\n\t}\n\n\treturn ts;\n}\n\nint CommSchedTarget::init(const struct sockaddr *addr, socklen_t addrlen,\n\t\t\t\t\t\t int connect_timeout, int response_timeout,\n\t\t\t\t\t\t size_t max_connections)\n{\n\tint ret;\n\n\tif (max_connections == 0)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\tif (this->CommTarget::init(addr, addrlen, connect_timeout,\n\t\t\t\t\t\t\t response_timeout) >= 0)\n\t{\n\t\tret = pthread_mutex_init(&this->mutex, NULL);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tret = pthread_cond_init(&this->cond, NULL);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tthis->max_load = max_connections;\n\t\t\t\tthis->cur_load = 0;\n\t\t\t\tthis->wait_cnt = 0;\n\t\t\t\tthis->group = NULL;\n\t\t\t\treturn 0;\n\t\t\t}\n\n\t\t\tpthread_mutex_destroy(&this->mutex);\n\t\t}\n\n\t\terrno = ret;\n\t\tthis->CommTarget::deinit();\n\t}\n\n\treturn -1;\n}\n\nvoid CommSchedTarget::deinit()\n{\n\tpthread_cond_destroy(&this->cond);\n\tpthread_mutex_destroy(&this->mutex);\n\tthis->CommTarget::deinit();\n}\n\nCommTarget *CommSchedTarget::acquire(int wait_timeout)\n{\n\tpthread_mutex_t *mutex = &this->mutex;\n\tint ret;\n\n\tpthread_mutex_lock(mutex);\n\tif (this->group)\n\t{\n\t\tmutex = &this->group->mutex;\n\t\tpthread_mutex_lock(mutex);\n\t\tpthread_mutex_unlock(&this->mutex);\n\t}\n\n\tif (this->cur_load >= this->max_load)\n\t{\n\t\tif (wait_timeout != 0)\n\t\t{\n\t\t\tstruct timespec ts;\n\t\t\tstruct timespec *abstime = __get_abstime(wait_timeout, &ts);\n\n\t\t\tdo\n\t\t\t{\n\t\t\t\tthis->wait_cnt++;\n\t\t\t\tret = PTHREAD_COND_TIMEDWAIT(&this->cond, mutex, abstime);\n\t\t\t\tthis->wait_cnt--;\n\t\t\t} while (this->cur_load >= this->max_load && ret == 0);\n\t\t}\n\t\telse\n\t\t\tret = EAGAIN;\n\t}\n\n\tif (this->cur_load < this->max_load)\n\t{\n\t\tthis->cur_load++;\n\t\tif (this->group)\n\t\t{\n\t\t\tthis->group->cur_load++;\n\t\t\tthis->group->heapify(this->index);\n\t\t}\n\n\t\tret = 0;\n\t}\n\n\tpthread_mutex_unlock(mutex);\n\tif (ret)\n\t{\n\t\terrno = ret;\n\t\treturn NULL;\n\t}\n\n\treturn this;\n}\n\nvoid CommSchedTarget::release()\n{\n\tpthread_mutex_lock(&this->mutex);\n\tif (this->group)\n\t\tpthread_mutex_lock(&this->group->mutex);\n\n\tthis->cur_load--;\n\tif (this->wait_cnt > 0)\n\t\tpthread_cond_signal(&this->cond);\n\n\tif (this->group)\n\t{\n\t\tthis->group->cur_load--;\n\t\tif (this->wait_cnt == 0 && this->group->wait_cnt > 0)\n\t\t\tpthread_cond_signal(&this->group->cond);\n\n\t\tthis->group->heap_adjust(this->index, this->has_idle_conn());\n\t\tpthread_mutex_unlock(&this->group->mutex);\n\t}\n\n\tpthread_mutex_unlock(&this->mutex);\n}\n\nint CommSchedGroup::target_cmp(CommSchedTarget *target1,\n\t\t\t\t\t\t\t CommSchedTarget *target2)\n{\n\tsize_t load1 = target1->cur_load * target2->max_load;\n\tsize_t load2 = target2->cur_load * target1->max_load;\n\n\tif (load1 < load2)\n\t\treturn -1;\n\telse if (load1 > load2)\n\t\treturn 1;\n\telse\n\t\treturn 0;\n}\n\nvoid CommSchedGroup::heap_adjust(int index, int swap_on_equal)\n{\n\tCommSchedTarget *target = this->tg_heap[index];\n\tCommSchedTarget *parent;\n\n\twhile (index > 0)\n\t{\n\t\tparent = this->tg_heap[(index - 1) / 2];\n\t\tif (CommSchedGroup::target_cmp(target, parent) < swap_on_equal)\n\t\t{\n\t\t\tthis->tg_heap[index] = parent;\n\t\t\tparent->index = index;\n\t\t\tindex = (index - 1) / 2;\n\t\t}\n\t\telse\n\t\t\tbreak;\n\t}\n\n\tthis->tg_heap[index] = target;\n\ttarget->index = index;\n}\n\n/* Fastest heapify ever. */\nvoid CommSchedGroup::heapify(int top)\n{\n\tCommSchedTarget *target = this->tg_heap[top];\n\tint last = this->heap_size - 1;\n\tCommSchedTarget **child;\n\tint i;\n\n\twhile (i = 2 * top + 1, i < last)\n\t{\n\t\tchild = &this->tg_heap[i];\n\t\tif (CommSchedGroup::target_cmp(child[0], target) < 0)\n\t\t{\n\t\t\tif (CommSchedGroup::target_cmp(child[1], child[0]) < 0)\n\t\t\t{\n\t\t\t\tthis->tg_heap[top] = child[1];\n\t\t\t\tchild[1]->index = top;\n\t\t\t\ttop = i + 1;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tthis->tg_heap[top] = child[0];\n\t\t\t\tchild[0]->index = top;\n\t\t\t\ttop = i;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (CommSchedGroup::target_cmp(child[1], target) < 0)\n\t\t\t{\n\t\t\t\tthis->tg_heap[top] = child[1];\n\t\t\t\tchild[1]->index = top;\n\t\t\t\ttop = i + 1;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tthis->tg_heap[top] = target;\n\t\t\t\ttarget->index = top;\n\t\t\t\treturn;\n\t\t\t}\n\t\t}\n\t}\n\n\tif (i == last)\n\t{\n\t\tchild = &this->tg_heap[i];\n\t\tif (CommSchedGroup::target_cmp(child[0], target) < 0)\n\t\t{\n\t\t\tthis->tg_heap[top] = child[0];\n\t\t\tchild[0]->index = top;\n\t\t\ttop = i;\n\t\t}\n\t}\n\n\tthis->tg_heap[top] = target;\n\ttarget->index = top;\n}\n\nint CommSchedGroup::heap_insert(CommSchedTarget *target)\n{\n\tif (this->heap_size == this->heap_buf_size)\n\t{\n\t\tint new_size = 2 * this->heap_buf_size;\n\t\tvoid *new_base = realloc(this->tg_heap, new_size * sizeof (void *));\n\n\t\tif (new_base)\n\t\t{\n\t\t\tthis->tg_heap = (CommSchedTarget **)new_base;\n\t\t\tthis->heap_buf_size = new_size;\n\t\t}\n\t\telse\n\t\t\treturn -1;\n\t}\n\n\tthis->tg_heap[this->heap_size] = target;\n\ttarget->index = this->heap_size;\n\tthis->heap_adjust(this->heap_size, 0);\n\tthis->heap_size++;\n\treturn 0;\n}\n\nvoid CommSchedGroup::heap_remove(int index)\n{\n\tCommSchedTarget *target;\n\n\tthis->heap_size--;\n\tif (index != this->heap_size)\n\t{\n\t\ttarget = this->tg_heap[this->heap_size];\n\t\tthis->tg_heap[index] = target;\n\t\ttarget->index = index;\n\t\tthis->heap_adjust(index, 0);\n\t\tthis->heapify(target->index);\n\t}\n}\n\n#define COMMGROUP_INIT_SIZE\t\t4\n\nint CommSchedGroup::init()\n{\n\tsize_t size = COMMGROUP_INIT_SIZE * sizeof (void *);\n\tint ret;\n\n\tthis->tg_heap = (CommSchedTarget **)malloc(size);\n\tif (this->tg_heap)\n\t{\n\t\tret = pthread_mutex_init(&this->mutex, NULL);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tret = pthread_cond_init(&this->cond, NULL);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tthis->heap_buf_size = COMMGROUP_INIT_SIZE;\n\t\t\t\tthis->heap_size = 0;\n\t\t\t\tthis->max_load = 0;\n\t\t\t\tthis->cur_load = 0;\n\t\t\t\tthis->wait_cnt = 0;\n\t\t\t\treturn 0;\n\t\t\t}\n\n\t\t\tpthread_mutex_destroy(&this->mutex);\n\t\t}\n\n\t\terrno = ret;\n\t\tfree(this->tg_heap);\n\t}\n\n\treturn -1;\n}\n\nvoid CommSchedGroup::deinit()\n{\n\tpthread_cond_destroy(&this->cond);\n\tpthread_mutex_destroy(&this->mutex);\n\tfree(this->tg_heap);\n}\n\nint CommSchedGroup::add(CommSchedTarget *target)\n{\n\tint ret = -1;\n\n\tpthread_mutex_lock(&target->mutex);\n\tpthread_mutex_lock(&this->mutex);\n\tif (target->group == NULL && target->wait_cnt == 0)\n\t{\n\t\tif (this->heap_insert(target) >= 0)\n\t\t{\n\t\t\ttarget->group = this;\n\t\t\tthis->max_load += target->max_load;\n\t\t\tthis->cur_load += target->cur_load;\n\t\t\tif (this->wait_cnt > 0 && this->cur_load < this->max_load)\n\t\t\t\tpthread_cond_signal(&this->cond);\n\n\t\t\tret = 0;\n\t\t}\n\t}\n\telse if (target->group == this)\n\t\terrno = EEXIST;\n\telse if (target->group)\n\t\terrno = EINVAL;\n\telse\n\t\terrno = EBUSY;\n\n\tpthread_mutex_unlock(&this->mutex);\n\tpthread_mutex_unlock(&target->mutex);\n\treturn ret;\n}\n\nint CommSchedGroup::remove(CommSchedTarget *target)\n{\n\tint ret = -1;\n\n\tpthread_mutex_lock(&target->mutex);\n\tpthread_mutex_lock(&this->mutex);\n\tif (target->group == this && target->wait_cnt == 0)\n\t{\n\t\tthis->heap_remove(target->index);\n\t\tthis->max_load -= target->max_load;\n\t\tthis->cur_load -= target->cur_load;\n\t\ttarget->group = NULL;\n\t\tret = 0;\n\t}\n\telse if (target->group != this)\n\t\terrno = ENOENT;\n\telse\n\t\terrno = EBUSY;\n\n\tpthread_mutex_unlock(&this->mutex);\n\tpthread_mutex_unlock(&target->mutex);\n\treturn ret;\n}\n\nCommTarget *CommSchedGroup::acquire(int wait_timeout)\n{\n\tpthread_mutex_t *mutex = &this->mutex;\n\tCommSchedTarget *target;\n\tint ret;\n\n\tpthread_mutex_lock(mutex);\n\tif (this->cur_load >= this->max_load)\n\t{\n\t\tif (wait_timeout != 0)\n\t\t{\n\t\t\tstruct timespec ts;\n\t\t\tstruct timespec *abstime = __get_abstime(wait_timeout, &ts);\n\n\t\t\tdo\n\t\t\t{\n\t\t\t\tthis->wait_cnt++;\n\t\t\t\tret = PTHREAD_COND_TIMEDWAIT(&this->cond, mutex, abstime);\n\t\t\t\tthis->wait_cnt--;\n\t\t\t} while (this->cur_load >= this->max_load && ret == 0);\n\t\t}\n\t\telse\n\t\t\tret = EAGAIN;\n\t}\n\n\tif (this->cur_load < this->max_load)\n\t{\n\t\ttarget = this->tg_heap[0];\n\t\ttarget->cur_load++;\n\t\tthis->cur_load++;\n\t\tthis->heapify(0);\n\t\tret = 0;\n\t}\n\n\tpthread_mutex_unlock(mutex);\n\tif (ret)\n\t{\n\t\terrno = ret;\n\t\treturn NULL;\n\t}\n\n\treturn target;\n}\n\n" }, { "path": "src/kernel/CommScheduler.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _COMMSCHEDULER_H_\n#define _COMMSCHEDULER_H_\n\n#include \n#include \n#include \n#include \n#include \"Communicator.h\"\n\nclass CommSchedObject\n{\npublic:\n\tsize_t get_max_load() const { return this->max_load; }\n\tsize_t get_cur_load() const { return this->cur_load; }\n\nprivate:\n\tvirtual CommTarget *acquire(int wait_timeout) = 0;\n\nprotected:\n\tsize_t max_load;\n\tsize_t cur_load;\n\npublic:\n\tvirtual ~CommSchedObject() { }\n\tfriend class CommScheduler;\n};\n\nclass CommSchedGroup;\n\nclass CommSchedTarget : public CommSchedObject, public CommTarget\n{\npublic:\n\tint init(const struct sockaddr *addr, socklen_t addrlen,\n\t\t\t int connect_timeout, int response_timeout,\n\t\t\t size_t max_connections);\n\tvoid deinit();\n\npublic:\n\tint init(const struct sockaddr *addr, socklen_t addrlen, SSL_CTX *ssl_ctx,\n\t\t\t int connect_timeout, int ssl_connect_timeout, int response_timeout,\n\t\t\t size_t max_connections)\n\t{\n\t\tint ret = this->init(addr, addrlen, connect_timeout, response_timeout,\n\t\t\t\t\t\t\t max_connections);\n\n\t\tif (ret >= 0)\n\t\t\tthis->set_ssl(ssl_ctx, ssl_connect_timeout);\n\n\t\treturn ret;\n\t}\n\nprivate:\n\tvirtual CommTarget *acquire(int wait_timeout); /* final */\n\tvirtual void release(); /* final */\n\nprivate:\n\tCommSchedGroup *group;\n\tint index;\n\tint wait_cnt;\n\tpthread_mutex_t mutex;\n\tpthread_cond_t cond;\n\tfriend class CommSchedGroup;\n};\n\nclass CommSchedGroup : public CommSchedObject\n{\npublic:\n\tint init();\n\tvoid deinit();\n\tint add(CommSchedTarget *target);\n\tint remove(CommSchedTarget *target);\n\nprivate:\n\tvirtual CommTarget *acquire(int wait_timeout); /* final */\n\nprivate:\n\tCommSchedTarget **tg_heap;\n\tint heap_size;\n\tint heap_buf_size;\n\tint wait_cnt;\n\tpthread_mutex_t mutex;\n\tpthread_cond_t cond;\n\nprivate:\n\tstatic int target_cmp(CommSchedTarget *target1, CommSchedTarget *target2);\n\tvoid heapify(int top);\n\tvoid heap_adjust(int index, int swap_on_equal);\n\tint heap_insert(CommSchedTarget *target);\n\tvoid heap_remove(int index);\n\tfriend class CommSchedTarget;\n};\n\nclass CommScheduler\n{\npublic:\n\tint init(size_t poller_threads, size_t handler_threads)\n\t{\n\t\treturn this->comm.init(poller_threads, handler_threads);\n\t}\n\n\tvoid deinit()\n\t{\n\t\tthis->comm.deinit();\n\t}\n\n\t/* wait_timeout in milliseconds, -1 for no timeout. */\n\tint request(CommSession *session, CommSchedObject *object,\n\t\t\t\tint wait_timeout, CommTarget **target)\n\t{\n\t\tint ret = -1;\n\n\t\t*target = object->acquire(wait_timeout);\n\t\tif (*target)\n\t\t{\n\t\t\tret = this->comm.request(session, *target);\n\t\t\tif (ret < 0)\n\t\t\t\t(*target)->release();\n\t\t}\n\n\t\treturn ret;\n\t}\n\n\t/* for services. */\n\tint reply(CommSession *session)\n\t{\n\t\treturn this->comm.reply(session);\n\t}\n\n\tint shutdown(CommSession *session)\n\t{\n\t\treturn this->comm.shutdown(session);\n\t}\n\n\tint push(const void *buf, size_t size, CommSession *session)\n\t{\n\t\treturn this->comm.push(buf, size, session);\n\t}\n\n\tint bind(CommService *service)\n\t{\n\t\treturn this->comm.bind(service);\n\t}\n\n\tvoid unbind(CommService *service)\n\t{\n\t\tthis->comm.unbind(service);\n\t}\n\n\t/* for sleepers. */\n\tint sleep(SleepSession *session)\n\t{\n\t\treturn this->comm.sleep(session);\n\t}\n\n\t/* Call 'unsleep' only before 'handle()' returns. */\n\tint unsleep(SleepSession *session)\n\t{\n\t\treturn this->comm.unsleep(session);\n\t}\n\n\t/* for file aio services. */\n\tint io_bind(IOService *service)\n\t{\n\t\treturn this->comm.io_bind(service);\n\t}\n\n\tvoid io_unbind(IOService *service)\n\t{\n\t\tthis->comm.io_unbind(service);\n\t}\n\npublic:\n\tint is_handler_thread() const\n\t{\n\t\treturn this->comm.is_handler_thread();\n\t}\n\n\tint increase_handler_thread()\n\t{\n\t\treturn this->comm.increase_handler_thread();\n\t}\n\n\tint decrease_handler_thread()\n\t{\n\t\treturn this->comm.decrease_handler_thread();\n\t}\n\npublic:\n\tvoid customize_event_handler(CommEventHandler *handler)\n\t{\n\t\tthis->comm.customize_event_handler(handler);\n\t}\n\nprivate:\n\tCommunicator comm;\n\npublic:\n\tvirtual ~CommScheduler() { }\n};\n\n#endif\n\n" }, { "path": "src/kernel/Communicator.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"msgqueue.h\"\n#include \"thrdpool.h\"\n#include \"poller.h\"\n#include \"mpoller.h\"\n#include \"Communicator.h\"\n\nstruct CommConnEntry\n{\n\tstruct list_head list;\n\tCommConnection *conn;\n\tlong long seq;\n\tint sockfd;\n#define CONN_STATE_CONNECTING\t0\n#define CONN_STATE_CONNECTED\t1\n#define CONN_STATE_RECEIVING\t2\n#define CONN_STATE_SUCCESS\t\t3\n#define CONN_STATE_IDLE\t\t\t4\n#define CONN_STATE_KEEPALIVE\t5\n#define CONN_STATE_CLOSING\t\t6\n#define CONN_STATE_ERROR\t\t7\n\tint state;\n\tint error;\n\tint ref;\n\tstruct iovec *write_iov;\n\tSSL *ssl;\n\tCommSession *session;\n\tCommTarget *target;\n\tCommService *service;\n\tmpoller_t *mpoller;\n\t/* Connection entry's mutex is for client session only. */\n\tpthread_mutex_t mutex;\n};\n\nstatic inline int __set_fd_nonblock(int fd)\n{\n\tint flags = fcntl(fd, F_GETFL);\n\n\tif (flags >= 0)\n\t\tflags = fcntl(fd, F_SETFL, flags | O_NONBLOCK);\n\n\treturn flags;\n}\n\nstatic int __bind_sockaddr(int sockfd, const struct sockaddr *addr,\n\t\t\t\t\t\t socklen_t addrlen)\n{\n\tstruct sockaddr_storage ss;\n\tsocklen_t len;\n\n\tlen = sizeof (struct sockaddr_storage);\n\tif (getsockname(sockfd, (struct sockaddr *)&ss, &len) < 0)\n\t\treturn -1;\n\n\tss.ss_family = 0;\n\twhile (len != 0)\n\t{\n\t\tif (((char *)&ss)[--len] != 0)\n\t\t\tbreak;\n\t}\n\n\tif (len == 0)\n\t{\n\t\tif (bind(sockfd, addr, addrlen) < 0)\n\t\t\treturn -1;\n\t}\n\n\treturn 0;\n}\n\nstatic int __create_ssl(SSL_CTX *ssl_ctx, struct CommConnEntry *entry)\n{\n\tBIO *bio = BIO_new_socket(entry->sockfd, BIO_NOCLOSE);\n\n\tif (bio)\n\t{\n\t\tentry->ssl = SSL_new(ssl_ctx);\n\t\tif (entry->ssl)\n\t\t{\n\t\t\tSSL_set_bio(entry->ssl, bio, bio);\n\t\t\treturn 0;\n\t\t}\n\n\t\tBIO_free(bio);\n\t}\n\n\treturn -1;\n}\n\n#define SSL_WRITEV_BUFSIZE\t2048\n\nstatic int __ssl_writev(SSL *ssl, struct iovec vectors[], int cnt)\n{\n\tif (vectors[0].iov_len < SSL_WRITEV_BUFSIZE && cnt > 1)\n\t{\n\t\tchar *p = (char *)SSL_get_app_data(ssl);\n\t\tsize_t nleft = SSL_WRITEV_BUFSIZE;\n\t\tsize_t n;\n\t\tint i;\n\n\t\tif (!p)\n\t\t{\n\t\t\tp = (char *)malloc(SSL_WRITEV_BUFSIZE);\n\t\t\tif (!p)\n\t\t\t\treturn -1;\n\n\t\t\tif (SSL_set_app_data(ssl, p) <= 0)\n\t\t\t{\n\t\t\t\tfree(p);\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t}\n\n\t\tn = vectors[0].iov_len;\n\t\tmemcpy(p, vectors[0].iov_base, n);\n\t\tvectors[0].iov_base = p;\n\n\t\tp += n;\n\t\tnleft -= n;\n\t\tfor (i = 1; i < cnt; i++)\n\t\t{\n\t\t\tif (vectors[i].iov_len < nleft)\n\t\t\t\tn = vectors[i].iov_len;\n\t\t\telse\n\t\t\t\tn = nleft;\n\n\t\t\tmemcpy(p, vectors[i].iov_base, n);\n\t\t\tvectors[i].iov_base = (char *)vectors[i].iov_base + n;\n\t\t\tvectors[i].iov_len -= n;\n\n\t\t\tp += n;\n\t\t\tnleft -= n;\n\t\t\tif (nleft == 0)\n\t\t\t\tbreak;\n\t\t}\n\n\t\tvectors[0].iov_len = SSL_WRITEV_BUFSIZE - nleft;\n\t}\n\n\treturn SSL_write(ssl, vectors[0].iov_base, vectors[0].iov_len);\n}\n\nstatic void __release_conn(struct CommConnEntry *entry)\n{\n\tdelete entry->conn;\n\tif (!entry->service)\n\t\tpthread_mutex_destroy(&entry->mutex);\n\n\tif (entry->ssl)\n\t{\n\t\tfree(SSL_get_app_data(entry->ssl));\n\t\tSSL_free(entry->ssl);\n\t}\n\n\tclose(entry->sockfd);\n\tfree(entry);\n}\n\nint CommTarget::init(const struct sockaddr *addr, socklen_t addrlen,\n\t\t\t\t\t int connect_timeout, int response_timeout)\n{\n\tint ret;\n\n\tthis->addr = (struct sockaddr *)malloc(addrlen);\n\tif (this->addr)\n\t{\n\t\tret = pthread_mutex_init(&this->mutex, NULL);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tmemcpy(this->addr, addr, addrlen);\n\t\t\tthis->addrlen = addrlen;\n\t\t\tthis->connect_timeout = connect_timeout;\n\t\t\tthis->response_timeout = response_timeout;\n\t\t\tINIT_LIST_HEAD(&this->idle_list);\n\n\t\t\tthis->ssl_ctx = NULL;\n\t\t\tthis->ssl_connect_timeout = 0;\n\t\t\treturn 0;\n\t\t}\n\n\t\terrno = ret;\n\t\tfree(this->addr);\n\t}\n\n\treturn -1;\n}\n\nvoid CommTarget::deinit()\n{\n\tpthread_mutex_destroy(&this->mutex);\n\tfree(this->addr);\n}\n\nint CommMessageIn::feedback(const void *buf, size_t size)\n{\n\tstruct CommConnEntry *entry = this->entry;\n\tconst struct sockaddr *addr = NULL;\n\tsocklen_t addrlen = 0;\n\tint ret;\n\n\tif (!entry->ssl)\n\t{\n\t\tif (entry->service)\n\t\t\tentry->target->get_addr(&addr, &addrlen);\n\n\t\treturn sendto(entry->sockfd, buf, size, 0, addr, addrlen);\n\t}\n\n\tif (size == 0)\n\t\treturn 0;\n\n\tret = SSL_write(entry->ssl, buf, size);\n\tif (ret <= 0)\n\t{\n\t\tret = SSL_get_error(entry->ssl, ret);\n\t\tif (ret == SSL_ERROR_WANT_READ || ret == SSL_ERROR_WANT_WRITE)\n\t\t\terrno = EAGAIN;\n\t\telse if (ret != SSL_ERROR_SYSCALL)\n\t\t\terrno = -ret;\n\n\t\tret = -1;\n\t}\n\n\treturn ret;\n}\n\nvoid CommMessageIn::renew()\n{\n\tCommSession *session = this->entry->session;\n\tsession->timeout = -1;\n\tsession->begin_time.tv_sec = -1;\n\tsession->begin_time.tv_nsec = -1;\n}\n\nint CommService::init(const struct sockaddr *bind_addr, socklen_t addrlen,\n\t\t\t\t\t int listen_timeout, int response_timeout)\n{\n\tint ret;\n\n\tthis->bind_addr = (struct sockaddr *)malloc(addrlen);\n\tif (this->bind_addr)\n\t{\n\t\tret = pthread_mutex_init(&this->mutex, NULL);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tmemcpy(this->bind_addr, bind_addr, addrlen);\n\t\t\tthis->addrlen = addrlen;\n\t\t\tthis->listen_timeout = listen_timeout;\n\t\t\tthis->response_timeout = response_timeout;\n\t\t\tINIT_LIST_HEAD(&this->keep_alive_list);\n\n\t\t\tthis->ssl_ctx = NULL;\n\t\t\tthis->ssl_accept_timeout = 0;\n\t\t\treturn 0;\n\t\t}\n\n\t\terrno = ret;\n\t\tfree(this->bind_addr);\n\t}\n\n\treturn -1;\n}\n\nvoid CommService::deinit()\n{\n\tpthread_mutex_destroy(&this->mutex);\n\tfree(this->bind_addr);\n}\n\nint CommService::drain(int max)\n{\n\tstruct CommConnEntry *entry;\n\tstruct list_head *pos;\n\tint errno_bak;\n\tint cnt = 0;\n\n\terrno_bak = errno;\n\tpthread_mutex_lock(&this->mutex);\n\twhile (cnt != max && !list_empty(&this->keep_alive_list))\n\t{\n\t\tpos = this->keep_alive_list.prev;\n\t\tentry = list_entry(pos, struct CommConnEntry, list);\n\t\tlist_del(pos);\n\t\tcnt++;\n\n\t\t/* Cannot change the sequence of next two lines. */\n\t\tmpoller_del(entry->sockfd, entry->mpoller);\n\t\tentry->state = CONN_STATE_CLOSING;\n\t}\n\n\tpthread_mutex_unlock(&this->mutex);\n\terrno = errno_bak;\n\treturn cnt;\n}\n\ninline void CommService::incref()\n{\n\t__sync_add_and_fetch(&this->ref, 1);\n}\n\ninline void CommService::decref()\n{\n\tif (__sync_sub_and_fetch(&this->ref, 1) == 0)\n\t\tthis->handle_unbound();\n}\n\nclass CommServiceTarget : public CommTarget\n{\npublic:\n\tvoid incref()\n\t{\n\t\t__sync_add_and_fetch(&this->ref, 1);\n\t}\n\n\tvoid decref()\n\t{\n\t\tif (__sync_sub_and_fetch(&this->ref, 1) == 0)\n\t\t{\n\t\t\tthis->service->decref();\n\t\t\tthis->deinit();\n\t\t\tdelete this;\n\t\t}\n\t}\n\npublic:\n\tint shutdown();\n\nprivate:\n\tint sockfd;\n\tint ref;\n\nprivate:\n\tCommService *service;\n\nprivate:\n\tvirtual ~CommServiceTarget() { }\n\tfriend class Communicator;\n};\n\nint CommServiceTarget::shutdown()\n{\n\tstruct CommConnEntry *entry;\n\tint errno_bak;\n\tint ret = 0;\n\n\tpthread_mutex_lock(&this->mutex);\n\tif (!list_empty(&this->idle_list))\n\t{\n\t\tentry = list_entry(this->idle_list.next, struct CommConnEntry, list);\n\t\tlist_del(&entry->list);\n\n\t\tif (this->service->reliable)\n\t\t{\n\t\t\terrno_bak = errno;\n\t\t\tmpoller_del(entry->sockfd, entry->mpoller);\n\t\t\tentry->state = CONN_STATE_CLOSING;\n\t\t\terrno = errno_bak;\n\t\t}\n\t\telse\n\t\t{\n\t\t\t__release_conn(entry);\n\t\t\t__sync_sub_and_fetch(&this->ref, 1);\n\t\t}\n\n\t\tret = 1;\n\t}\n\n\tpthread_mutex_unlock(&this->mutex);\n\treturn ret;\n}\n\nCommSession::~CommSession()\n{\n\tCommServiceTarget *target;\n\n\tif (!this->passive)\n\t\treturn;\n\n\ttarget = (CommServiceTarget *)this->target;\n\tif (!this->out && target->has_idle_conn())\n\t\ttarget->shutdown();\n\n\ttarget->decref();\n}\n\ninline int Communicator::first_timeout(CommSession *session)\n{\n\tint timeout = session->target->response_timeout;\n\n\tif (timeout < 0 || (unsigned int)session->timeout <= (unsigned int)timeout)\n\t{\n\t\ttimeout = session->timeout;\n\t\tsession->timeout = 0;\n\t}\n\telse\n\t\tclock_gettime(CLOCK_MONOTONIC, &session->begin_time);\n\n\treturn timeout;\n}\n\nint Communicator::next_timeout(CommSession *session)\n{\n\tint timeout = session->target->response_timeout;\n\tstruct timespec cur_time;\n\tint time_used, time_left;\n\n\tif (session->timeout > 0)\n\t{\n\t\tclock_gettime(CLOCK_MONOTONIC, &cur_time);\n\t\ttime_used = 1000 * (cur_time.tv_sec - session->begin_time.tv_sec) +\n\t\t\t\t\t(cur_time.tv_nsec - session->begin_time.tv_nsec) / 1000000;\n\t\ttime_left = session->timeout - time_used;\n\t\tif (time_left <= timeout) /* here timeout >= 0 */\n\t\t{\n\t\t\ttimeout = time_left < 0 ? 0 : time_left;\n\t\t\tsession->timeout = 0;\n\t\t}\n\t}\n\n\treturn timeout;\n}\n\nint Communicator::first_timeout_send(CommSession *session)\n{\n\tsession->timeout = session->send_timeout();\n\treturn Communicator::first_timeout(session);\n}\n\nint Communicator::first_timeout_recv(CommSession *session)\n{\n\tsession->timeout = session->receive_timeout();\n\treturn Communicator::first_timeout(session);\n}\n\nvoid Communicator::shutdown_service(CommService *service)\n{\n\tclose(service->listen_fd);\n\tservice->listen_fd = -1;\n\tservice->drain(-1);\n\tservice->decref();\n}\n\n#ifndef IOV_MAX\n# define IOV_MAX\t16\n#endif\n\nint Communicator::send_message_sync(struct iovec vectors[], int cnt,\n\t\t\t\t\t\t\t\t\tstruct CommConnEntry *entry)\n{\n\tCommSession *session = entry->session;\n\tCommService *service;\n\tint timeout;\n\tssize_t n;\n\tint i;\n\n\twhile (cnt > 0)\n\t{\n\t\tif (!entry->ssl)\n\t\t{\n\t\t\tn = writev(entry->sockfd, vectors, cnt <= IOV_MAX ? cnt : IOV_MAX);\n\t\t\tif (n < 0)\n\t\t\t\treturn errno == EAGAIN ? cnt : -1;\n\t\t}\n\t\telse if (vectors->iov_len > 0)\n\t\t{\n\t\t\tn = __ssl_writev(entry->ssl, vectors, cnt);\n\t\t\tif (n <= 0)\n\t\t\t\treturn cnt;\n\t\t}\n\t\telse\n\t\t\tn = 0;\n\n\t\tfor (i = 0; i < cnt; i++)\n\t\t{\n\t\t\tif ((size_t)n >= vectors[i].iov_len)\n\t\t\t\tn -= vectors[i].iov_len;\n\t\t\telse if (n > 0)\n\t\t\t{\n\t\t\t\tvectors[i].iov_base = (char *)vectors[i].iov_base + n;\n\t\t\t\tvectors[i].iov_len -= n;\n\t\t\t\treturn cnt - i;\n\t\t\t}\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\tvectors += i;\n\t\tcnt -= i;\n\t}\n\n\tservice = entry->service;\n\tif (service)\n\t{\n\t\t__sync_add_and_fetch(&entry->ref, 1);\n\t\ttimeout = session->keep_alive_timeout();\n\t\tswitch (timeout)\n\t\t{\n\t\tdefault:\n\t\t\tmpoller_set_timeout(entry->sockfd, timeout, this->mpoller);\n\t\t\tpthread_mutex_lock(&service->mutex);\n\t\t\tif (service->listen_fd >= 0)\n\t\t\t{\n\t\t\t\tentry->state = CONN_STATE_KEEPALIVE;\n\t\t\t\tlist_add(&entry->list, &service->keep_alive_list);\n\t\t\t\tentry = NULL;\n\t\t\t}\n\n\t\t\tpthread_mutex_unlock(&service->mutex);\n\t\t\tif (entry)\n\t\t\t{\n\t\tcase 0:\n\t\t\t\tmpoller_del(entry->sockfd, this->mpoller);\n\t\t\t\tentry->state = CONN_STATE_CLOSING;\n\t\t\t}\n\t\t}\n\t}\n\telse\n\t{\n\t\tif (entry->state == CONN_STATE_IDLE)\n\t\t{\n\t\t\ttimeout = session->first_timeout();\n\t\t\tif (timeout == 0)\n\t\t\t\ttimeout = Communicator::first_timeout_recv(session);\n\t\t\telse\n\t\t\t{\n\t\t\t\tsession->timeout = -1;\n\t\t\t\tsession->begin_time.tv_sec = -1;\n\t\t\t\tsession->begin_time.tv_nsec = 0;\n\t\t\t}\n\n\t\t\tmpoller_set_timeout(entry->sockfd, timeout, this->mpoller);\n\t\t}\n\n\t\tentry->state = CONN_STATE_RECEIVING;\n\t}\n\n\treturn 0;\n}\n\nint Communicator::send_message_async(struct iovec vectors[], int cnt,\n\t\t\t\t\t\t\t\t\t struct CommConnEntry *entry)\n{\n\tstruct poller_data data;\n\tint timeout;\n\tint ret;\n\tint i;\n\n\tentry->write_iov = (struct iovec *)malloc(cnt * sizeof (struct iovec));\n\tif (entry->write_iov)\n\t{\n\t\tfor (i = 0; i < cnt; i++)\n\t\t\tentry->write_iov[i] = vectors[i];\n\t}\n\telse\n\t\treturn -1;\n\n\tdata.operation = PD_OP_WRITE;\n\tdata.fd = entry->sockfd;\n\tdata.ssl = entry->ssl;\n\tdata.partial_written = Communicator::partial_written;\n\tdata.context = entry;\n\tdata.write_iov = entry->write_iov;\n\tdata.iovcnt = cnt;\n\ttimeout = Communicator::first_timeout_send(entry->session);\n\tif (entry->state == CONN_STATE_IDLE)\n\t{\n\t\tret = mpoller_mod(&data, timeout, this->mpoller);\n\t\tif (ret < 0 && errno == ENOENT)\n\t\t\tentry->state = CONN_STATE_RECEIVING;\n\t}\n\telse\n\t{\n\t\tret = mpoller_add(&data, timeout, this->mpoller);\n\t\tif (ret >= 0)\n\t\t{\n\t\t\tif (this->stop_flag)\n\t\t\t\tmpoller_del(data.fd, this->mpoller);\n\t\t}\n\t}\n\n\tif (ret < 0)\n\t{\n\t\tfree(entry->write_iov);\n\t\tif (entry->state != CONN_STATE_RECEIVING)\n\t\t\treturn -1;\n\t}\n\n\treturn 1;\n}\n\n#define ENCODE_IOV_MAX\t\t2048\n\nint Communicator::send_message(struct CommConnEntry *entry)\n{\n\tstruct iovec vectors[ENCODE_IOV_MAX];\n\tstruct iovec *end;\n\tint cnt;\n\n\tcnt = entry->session->out->encode(vectors, ENCODE_IOV_MAX);\n\tif ((unsigned int)cnt > ENCODE_IOV_MAX)\n\t{\n\t\tif (cnt > ENCODE_IOV_MAX)\n\t\t\terrno = EOVERFLOW;\n\t\treturn -1;\n\t}\n\n\tend = vectors + cnt;\n\tcnt = this->send_message_sync(vectors, cnt, entry);\n\tif (cnt <= 0)\n\t\treturn cnt;\n\n\treturn this->send_message_async(end - cnt, cnt, entry);\n}\n\nvoid Communicator::handle_incoming_request(struct poller_result *res)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)res->data.context;\n\tCommTarget *target = entry->target;\n\tCommSession *session = NULL;\n\tint state;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_SUCCESS:\n\t\tsession = entry->session;\n\t\tstate = CS_STATE_TOREPLY;\n\t\tpthread_mutex_lock(&target->mutex);\n\t\tif (entry->state == CONN_STATE_SUCCESS)\n\t\t{\n\t\t\t__sync_add_and_fetch(&entry->ref, 1);\n\t\t\tentry->state = CONN_STATE_IDLE;\n\t\t\tlist_add(&entry->list, &target->idle_list);\n\t\t}\n\n\t\tpthread_mutex_unlock(&target->mutex);\n\t\tbreak;\n\n\tcase PR_ST_FINISHED:\n\t\tres->error = ECONNRESET;\n\t\tif (1)\n\tcase PR_ST_ERROR:\n\t\t\tstate = CS_STATE_ERROR;\n\t\telse\n\tcase PR_ST_DELETED:\n\tcase PR_ST_STOPPED:\n\t\t\tstate = CS_STATE_STOPPED;\n\n\t\tpthread_mutex_lock(&target->mutex);\n\t\tswitch (entry->state)\n\t\t{\n\t\tcase CONN_STATE_KEEPALIVE:\n\t\t\tpthread_mutex_lock(&entry->service->mutex);\n\t\t\tif (entry->state == CONN_STATE_KEEPALIVE)\n\t\t\t\tlist_del(&entry->list);\n\t\t\tpthread_mutex_unlock(&entry->service->mutex);\n\t\t\tbreak;\n\n\t\tcase CONN_STATE_IDLE:\n\t\t\tlist_del(&entry->list);\n\t\t\tbreak;\n\n\t\tcase CONN_STATE_ERROR:\n\t\t\tres->error = entry->error;\n\t\t\tstate = CS_STATE_ERROR;\n\t\tcase CONN_STATE_RECEIVING:\n\t\t\tsession = entry->session;\n\t\t\tbreak;\n\n\t\tcase CONN_STATE_SUCCESS:\n\t\t\t/* This may happen only if handler_threads > 1. */\n\t\t\tentry->state = CONN_STATE_CLOSING;\n\t\t\tentry = NULL;\n\t\t\tbreak;\n\t\t}\n\n\t\tpthread_mutex_unlock(&target->mutex);\n\t\tbreak;\n\t}\n\n\tif (entry)\n\t{\n\t\tif (session)\n\t\t\tsession->handle(state, res->error);\n\n\t\tif (__sync_sub_and_fetch(&entry->ref, 1) == 0)\n\t\t{\n\t\t\t__release_conn(entry);\n\t\t\t((CommServiceTarget *)target)->decref();\n\t\t}\n\t}\n}\n\nvoid Communicator::handle_incoming_reply(struct poller_result *res)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)res->data.context;\n\tCommTarget *target = entry->target;\n\tCommSession *session = NULL;\n\tpthread_mutex_t *mutex;\n\tint state;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_SUCCESS:\n\t\tsession = entry->session;\n\t\tstate = CS_STATE_SUCCESS;\n\t\tpthread_mutex_lock(&target->mutex);\n\t\tif (entry->state == CONN_STATE_SUCCESS)\n\t\t{\n\t\t\t__sync_add_and_fetch(&entry->ref, 1);\n\t\t\tif (session->timeout != 0) /* This is keep-alive timeout. */\n\t\t\t{\n\t\t\t\tentry->state = CONN_STATE_IDLE;\n\t\t\t\tlist_add(&entry->list, &target->idle_list);\n\t\t\t}\n\t\t\telse\n\t\t\t\tentry->state = CONN_STATE_CLOSING;\n\t\t}\n\n\t\tpthread_mutex_unlock(&target->mutex);\n\t\tbreak;\n\n\tcase PR_ST_FINISHED:\n\t\tres->error = ECONNRESET;\n\t\tif (1)\n\tcase PR_ST_ERROR:\n\t\t\tstate = CS_STATE_ERROR;\n\t\telse\n\tcase PR_ST_DELETED:\n\tcase PR_ST_STOPPED:\n\t\t\tstate = CS_STATE_STOPPED;\n\n\t\tmutex = &entry->mutex;\n\t\tpthread_mutex_lock(&target->mutex);\n\t\tpthread_mutex_lock(mutex);\n\t\tswitch (entry->state)\n\t\t{\n\t\tcase CONN_STATE_IDLE:\n\t\t\tlist_del(&entry->list);\n\t\t\tbreak;\n\n\t\tcase CONN_STATE_ERROR:\n\t\t\tres->error = entry->error;\n\t\t\tstate = CS_STATE_ERROR;\n\t\tcase CONN_STATE_RECEIVING:\n\t\t\tsession = entry->session;\n\t\t\tbreak;\n\n\t\tcase CONN_STATE_SUCCESS:\n\t\t\t/* This may happen only if handler_threads > 1. */\n\t\t\tentry->state = CONN_STATE_CLOSING;\n\t\t\tentry = NULL;\n\t\t\tbreak;\n\t\t}\n\n\t\tpthread_mutex_unlock(&target->mutex);\n\t\tpthread_mutex_unlock(mutex);\n\t\tbreak;\n\t}\n\n\tif (entry)\n\t{\n\t\tif (session)\n\t\t{\n\t\t\ttarget->release();\n\t\t\tsession->handle(state, res->error);\n\t\t}\n\n\t\tif (__sync_sub_and_fetch(&entry->ref, 1) == 0)\n\t\t\t__release_conn(entry);\n\t}\n}\n\nvoid Communicator::handle_read_result(struct poller_result *res)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)res->data.context;\n\n\tif (res->state != PR_ST_MODIFIED)\n\t{\n\t\tif (entry->service)\n\t\t\tthis->handle_incoming_request(res);\n\t\telse\n\t\t\tthis->handle_incoming_reply(res);\n\t}\n}\n\nvoid Communicator::handle_reply_result(struct poller_result *res)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)res->data.context;\n\tCommService *service = entry->service;\n\tCommSession *session = entry->session;\n\tCommTarget *target = entry->target;\n\tint timeout;\n\tint state;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_FINISHED:\n\t\ttimeout = session->keep_alive_timeout();\n\t\tif (timeout != 0)\n\t\t{\n\t\t\t__sync_add_and_fetch(&entry->ref, 1);\n\t\t\tres->data.operation = PD_OP_READ;\n\t\t\tres->data.create_message = Communicator::create_request;\n\t\t\tres->data.message = NULL;\n\t\t\tpthread_mutex_lock(&target->mutex);\n\t\t\tif (mpoller_add(&res->data, timeout, this->mpoller) >= 0)\n\t\t\t{\n\t\t\t\tpthread_mutex_lock(&service->mutex);\n\t\t\t\tif (!this->stop_flag && service->listen_fd >= 0)\n\t\t\t\t{\n\t\t\t\t\tentry->state = CONN_STATE_KEEPALIVE;\n\t\t\t\t\tlist_add(&entry->list, &service->keep_alive_list);\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\tmpoller_del(res->data.fd, this->mpoller);\n\t\t\t\t\tentry->state = CONN_STATE_CLOSING;\n\t\t\t\t}\n\n\t\t\t\tpthread_mutex_unlock(&service->mutex);\n\t\t\t}\n\t\t\telse\n\t\t\t\t__sync_sub_and_fetch(&entry->ref, 1);\n\n\t\t\tpthread_mutex_unlock(&target->mutex);\n\t\t}\n\n\t\tif (1)\n\t\t\tstate = CS_STATE_SUCCESS;\n\t\telse if (1)\n\tcase PR_ST_ERROR:\n\t\t\tstate = CS_STATE_ERROR;\n\t\telse\n\tcase PR_ST_DELETED:\t\t/* DELETED seems not possible. */\n\tcase PR_ST_STOPPED:\n\t\t\tstate = CS_STATE_STOPPED;\n\n\t\tsession->handle(state, res->error);\n\t\tif (__sync_sub_and_fetch(&entry->ref, 1) == 0)\n\t\t{\n\t\t\t__release_conn(entry);\n\t\t\t((CommServiceTarget *)target)->decref();\n\t\t}\n\n\t\tbreak;\n\t}\n}\n\nvoid Communicator::handle_request_result(struct poller_result *res)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)res->data.context;\n\tCommSession *session = entry->session;\n\tint timeout;\n\tint state;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_FINISHED:\n\t\tentry->state = CONN_STATE_RECEIVING;\n\t\tres->data.operation = PD_OP_READ;\n\t\tres->data.create_message = Communicator::create_reply;\n\t\tres->data.message = NULL;\n\t\ttimeout = session->first_timeout();\n\t\tif (timeout == 0)\n\t\t\ttimeout = Communicator::first_timeout_recv(session);\n\t\telse\n\t\t{\n\t\t\tsession->timeout = -1;\n\t\t\tsession->begin_time.tv_sec = -1;\n\t\t\tsession->begin_time.tv_nsec = 0;\n\t\t}\n\n\t\tif (mpoller_add(&res->data, timeout, this->mpoller) >= 0)\n\t\t{\n\t\t\tif (this->stop_flag)\n\t\t\t\tmpoller_del(res->data.fd, this->mpoller);\n\t\t\tbreak;\n\t\t}\n\n\t\tres->error = errno;\n\t\tif (1)\n\tcase PR_ST_ERROR:\n\t\t\tstate = CS_STATE_ERROR;\n\t\telse\n\tcase PR_ST_DELETED:\n\tcase PR_ST_STOPPED:\n\t\t\tstate = CS_STATE_STOPPED;\n\n\t\tentry->target->release();\n\t\tsession->handle(state, res->error);\n\t\tpthread_mutex_lock(&entry->mutex);\n\t\t/* do nothing */\n\t\tpthread_mutex_unlock(&entry->mutex);\n\t\tif (__sync_sub_and_fetch(&entry->ref, 1) == 0)\n\t\t\t__release_conn(entry);\n\n\t\tbreak;\n\t}\n}\n\nvoid Communicator::handle_write_result(struct poller_result *res)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)res->data.context;\n\n\tfree(entry->write_iov);\n\tif (entry->service)\n\t\tthis->handle_reply_result(res);\n\telse\n\t\tthis->handle_request_result(res);\n}\n\nstruct CommConnEntry *Communicator::accept_conn(CommServiceTarget *target,\n\t\t\t\t\t\t\t\t\t\t\t\tCommService *service)\n{\n\tstruct CommConnEntry *entry;\n\tsize_t size;\n\n\tif (__set_fd_nonblock(target->sockfd) >= 0)\n\t{\n\t\tsize = offsetof(struct CommConnEntry, mutex);\n\t\tentry = (struct CommConnEntry *)malloc(size);\n\t\tif (entry)\n\t\t{\n\t\t\tentry->conn = service->new_connection(target->sockfd);\n\t\t\tif (entry->conn)\n\t\t\t{\n\t\t\t\tentry->seq = 0;\n\t\t\t\tentry->mpoller = NULL;\n\t\t\t\tentry->service = service;\n\t\t\t\tentry->target = target;\n\t\t\t\tentry->ssl = NULL;\n\t\t\t\tentry->sockfd = target->sockfd;\n\t\t\t\tentry->state = CONN_STATE_CONNECTED;\n\t\t\t\tentry->ref = 1;\n\t\t\t\treturn entry;\n\t\t\t}\n\n\t\t\tfree(entry);\n\t\t}\n\t}\n\n\treturn NULL;\n}\n\nvoid Communicator::handle_connect_result(struct poller_result *res)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)res->data.context;\n\tCommSession *session = entry->session;\n\tCommTarget *target = entry->target;\n\tint timeout;\n\tint state;\n\tint ret;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_FINISHED:\n\t\tif (target->ssl_ctx && !entry->ssl)\n\t\t{\n\t\t\tif (__create_ssl(target->ssl_ctx, entry) >= 0 &&\n\t\t\t\ttarget->init_ssl(entry->ssl) >= 0)\n\t\t\t{\n\t\t\t\tret = 0;\n\t\t\t\tres->data.operation = PD_OP_SSL_CONNECT;\n\t\t\t\tres->data.ssl = entry->ssl;\n\t\t\t\ttimeout = target->ssl_connect_timeout;\n\t\t\t}\n\t\t\telse\n\t\t\t\tret = -1;\n\t\t}\n\t\telse if ((session->out = session->message_out()) != NULL)\n\t\t{\n\t\t\tret = this->send_message(entry);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tres->data.operation = PD_OP_READ;\n\t\t\t\tres->data.create_message = Communicator::create_reply;\n\t\t\t\tres->data.message = NULL;\n\t\t\t\ttimeout = session->first_timeout();\n\t\t\t\tif (timeout == 0)\n\t\t\t\t\ttimeout = Communicator::first_timeout_recv(session);\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\tsession->timeout = -1;\n\t\t\t\t\tsession->begin_time.tv_sec = -1;\n\t\t\t\t\tsession->begin_time.tv_nsec = 0;\n\t\t\t\t}\n\t\t\t}\n\t\t\telse if (ret > 0)\n\t\t\t\tbreak;\n\t\t}\n\t\telse\n\t\t\tret = -1;\n\n\t\tif (ret >= 0)\n\t\t{\n\t\t\tif (mpoller_add(&res->data, timeout, this->mpoller) >= 0)\n\t\t\t{\n\t\t\t\tif (this->stop_flag)\n\t\t\t\t\tmpoller_del(res->data.fd, this->mpoller);\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tres->error = errno;\n\t\tif (1)\n\tcase PR_ST_ERROR:\n\t\t\tstate = CS_STATE_ERROR;\n\t\telse\n\tcase PR_ST_DELETED:\n\tcase PR_ST_STOPPED:\n\t\t\tstate = CS_STATE_STOPPED;\n\n\t\ttarget->release();\n\t\tsession->handle(state, res->error);\n\t\t__release_conn(entry);\n\t\tbreak;\n\t}\n}\n\nvoid Communicator::handle_listen_result(struct poller_result *res)\n{\n\tCommService *service = (CommService *)res->data.context;\n\tstruct CommConnEntry *entry;\n\tCommServiceTarget *target;\n\tint timeout;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_SUCCESS:\n\t\ttarget = (CommServiceTarget *)res->data.result;\n\t\tentry = Communicator::accept_conn(target, service);\n\t\tif (entry)\n\t\t{\n\t\t\tentry->mpoller = this->mpoller;\n\t\t\tif (service->ssl_ctx)\n\t\t\t{\n\t\t\t\tif (__create_ssl(service->ssl_ctx, entry) >= 0 &&\n\t\t\t\t\tservice->init_ssl(entry->ssl) >= 0)\n\t\t\t\t{\n\t\t\t\t\tres->data.operation = PD_OP_SSL_ACCEPT;\n\t\t\t\t\ttimeout = service->ssl_accept_timeout;\n\t\t\t\t}\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tres->data.operation = PD_OP_READ;\n\t\t\t\tres->data.create_message = Communicator::create_request;\n\t\t\t\tres->data.message = NULL;\n\t\t\t\ttimeout = target->response_timeout;\n\t\t\t}\n\n\t\t\tif (res->data.operation != PD_OP_LISTEN)\n\t\t\t{\n\t\t\t\tres->data.fd = entry->sockfd;\n\t\t\t\tres->data.ssl = entry->ssl;\n\t\t\t\tres->data.context = entry;\n\t\t\t\tif (mpoller_add(&res->data, timeout, this->mpoller) >= 0)\n\t\t\t\t{\n\t\t\t\t\tif (this->stop_flag)\n\t\t\t\t\t\tmpoller_del(res->data.fd, this->mpoller);\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t__release_conn(entry);\n\t\t}\n\t\telse\n\t\t\tclose(target->sockfd);\n\n\t\ttarget->decref();\n\t\tbreak;\n\n\tcase PR_ST_DELETED:\n\t\tthis->shutdown_service(service);\n\t\tbreak;\n\n\tcase PR_ST_ERROR:\n\tcase PR_ST_STOPPED:\n\t\tservice->handle_stop(res->error);\n\t\tbreak;\n\t}\n}\n\nvoid Communicator::handle_recvfrom_result(struct poller_result *res)\n{\n\tCommService *service = (CommService *)res->data.context;\n\tstruct CommConnEntry *entry;\n\tCommSession *session;\n\tCommTarget *target;\n\tint state, error;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_SUCCESS:\n\t\tentry = (struct CommConnEntry *)res->data.result;\n\t\tsession = entry->session;\n\t\ttarget = entry->target;\n\t\tif (entry->state == CONN_STATE_SUCCESS)\n\t\t{\n\t\t\tstate = CS_STATE_TOREPLY;\n\t\t\terror = 0;\n\t\t\tentry->state = CONN_STATE_IDLE;\n\t\t\tlist_add(&entry->list, &target->idle_list);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tstate = CS_STATE_ERROR;\n\t\t\tif (entry->state == CONN_STATE_ERROR)\n\t\t\t\terror = entry->error;\n\t\t\telse\n\t\t\t\terror = EBADMSG;\n\t\t}\n\n\t\tsession->handle(state, error);\n\t\tif (state == CS_STATE_ERROR)\n\t\t{\n\t\t\t__release_conn(entry);\n\t\t\t((CommServiceTarget *)target)->decref();\n\t\t}\n\n\t\tbreak;\n\n\tcase PR_ST_DELETED:\n\t\tthis->shutdown_service(service);\n\t\tbreak;\n\n\tcase PR_ST_ERROR:\n\tcase PR_ST_STOPPED:\n\t\tservice->handle_stop(res->error);\n\t\tbreak;\n\t}\n}\n\nvoid Communicator::handle_ssl_accept_result(struct poller_result *res)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)res->data.context;\n\tCommTarget *target = entry->target;\n\tint timeout;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_FINISHED:\n\t\tres->data.operation = PD_OP_READ;\n\t\tres->data.create_message = Communicator::create_request;\n\t\tres->data.message = NULL;\n\t\ttimeout = target->response_timeout;\n\t\tif (mpoller_add(&res->data, timeout, this->mpoller) >= 0)\n\t\t{\n\t\t\tif (this->stop_flag)\n\t\t\t\tmpoller_del(res->data.fd, this->mpoller);\n\t\t\tbreak;\n\t\t}\n\n\tcase PR_ST_DELETED:\n\tcase PR_ST_ERROR:\n\tcase PR_ST_STOPPED:\n\t\t__release_conn(entry);\n\t\t((CommServiceTarget *)target)->decref();\n\t\tbreak;\n\t}\n}\n\nvoid Communicator::handle_sleep_result(struct poller_result *res)\n{\n\tSleepSession *session = (SleepSession *)res->data.context;\n\tint state;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_FINISHED:\n\t\tstate = SS_STATE_COMPLETE;\n\t\tbreak;\n\tcase PR_ST_DELETED:\n\t\tres->error = ECANCELED;\n\tcase PR_ST_ERROR:\n\t\tstate = SS_STATE_ERROR;\n\t\tbreak;\n\tcase PR_ST_STOPPED:\n\t\tstate = SS_STATE_DISRUPTED;\n\t\tbreak;\n\t}\n\n\tsession->handle(state, res->error);\n}\n\nvoid Communicator::handle_aio_result(struct poller_result *res)\n{\n\tIOService *service = (IOService *)res->data.context;\n\tIOSession *session;\n\tint state, error;\n\n\tswitch (res->state)\n\t{\n\tcase PR_ST_SUCCESS:\n\t\tsession = (IOSession *)res->data.result;\n\t\tpthread_mutex_lock(&service->mutex);\n\t\tlist_del(&session->list);\n\t\tpthread_mutex_unlock(&service->mutex);\n\t\tif (session->res >= 0)\n\t\t{\n\t\t\tstate = IOS_STATE_SUCCESS;\n\t\t\terror = 0;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tstate = IOS_STATE_ERROR;\n\t\t\terror = -session->res;\n\t\t}\n\n\t\tsession->handle(state, error);\n\t\tservice->decref();\n\t\tbreak;\n\n\tcase PR_ST_DELETED:\n\t\tthis->shutdown_io_service(service);\n\t\tbreak;\n\n\tcase PR_ST_ERROR:\n\tcase PR_ST_STOPPED:\n\t\tservice->handle_stop(res->error);\n\t\tbreak;\n\t}\n}\n\nvoid Communicator::handle_poller_result(struct poller_result *res)\n{\n\tswitch (res->data.operation)\n\t{\n\tcase PD_OP_TIMER:\n\t\tthis->handle_sleep_result(res);\n\t\tbreak;\n\tcase PD_OP_READ:\n\t\tthis->handle_read_result(res);\n\t\tbreak;\n\tcase PD_OP_WRITE:\n\t\tthis->handle_write_result(res);\n\t\tbreak;\n\tcase PD_OP_CONNECT:\n\tcase PD_OP_SSL_CONNECT:\n\t\tthis->handle_connect_result(res);\n\t\tbreak;\n\tcase PD_OP_LISTEN:\n\t\tthis->handle_listen_result(res);\n\t\tbreak;\n\tcase PD_OP_RECVFROM:\n\t\tthis->handle_recvfrom_result(res);\n\t\tbreak;\n\tcase PD_OP_SSL_ACCEPT:\n\t\tthis->handle_ssl_accept_result(res);\n\t\tbreak;\n\tcase PD_OP_EVENT:\n\tcase PD_OP_NOTIFY:\n\t\tthis->handle_aio_result(res);\n\t\tbreak;\n\tdefault:\n\t\tfree(res);\n\t\tthrdpool_exit(this->thrdpool);\n\t\treturn;\n\t}\n\n\tfree(res);\n}\n\nvoid Communicator::handler_thread_routine(void *context)\n{\n\tCommunicator *comm = (Communicator *)context;\n\tvoid *msg;\n\n\twhile ((msg = msgqueue_get(comm->msgqueue)) != NULL)\n\t\tcomm->handle_poller_result((struct poller_result *)msg);\n}\n\nint Communicator::append_message(const void *buf, size_t *size,\n\t\t\t\t\t\t\t\t poller_message_t *msg)\n{\n\tCommMessageIn *in = (CommMessageIn *)msg;\n\tstruct CommConnEntry *entry = in->entry;\n\tCommSession *session = entry->session;\n\tint timeout;\n\tint ret;\n\n\tret = in->append(buf, size);\n\tif (ret > 0)\n\t{\n\t\tentry->state = CONN_STATE_SUCCESS;\n\t\tif (!entry->service)\n\t\t{\n\t\t\ttimeout = session->keep_alive_timeout();\n\t\t\tsession->timeout = timeout; /* Reuse session's timeout field. */\n\t\t\tif (timeout == 0)\n\t\t\t{\n\t\t\t\tmpoller_del(entry->sockfd, entry->mpoller);\n\t\t\t\treturn ret;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t\ttimeout = -1;\n\t}\n\telse if (ret == 0 && session->timeout != 0)\n\t{\n\t\tif (session->begin_time.tv_sec < 0)\n\t\t{\n\t\t\tif (session->begin_time.tv_nsec < 0)\n\t\t\t\ttimeout = session->first_timeout();\n\t\t\telse\n\t\t\t\ttimeout = 0;\n\n\t\t\tif (timeout == 0)\n\t\t\t\ttimeout = Communicator::first_timeout_recv(session);\n\t\t\telse\n\t\t\t\tsession->begin_time.tv_nsec = 0;\n\t\t}\n\t\telse\n\t\t\ttimeout = Communicator::next_timeout(session);\n\t}\n\telse\n\t\treturn ret;\n\n\t/* This set_timeout() never fails, which is very important. */\n\tmpoller_set_timeout(entry->sockfd, timeout, entry->mpoller);\n\treturn ret;\n}\n\npoller_message_t *Communicator::create_request(void *context)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)context;\n\tCommService *service = entry->service;\n\tCommTarget *target = entry->target;\n\tCommSession *session;\n\tCommMessageIn *in;\n\tint timeout;\n\n\tif (entry->state == CONN_STATE_IDLE)\n\t{\n\t\tpthread_mutex_lock(&target->mutex);\n\t\t/* do nothing */\n\t\tpthread_mutex_unlock(&target->mutex);\n\t}\n\n\tpthread_mutex_lock(&service->mutex);\n\tif (entry->state == CONN_STATE_KEEPALIVE)\n\t\tlist_del(&entry->list);\n\telse if (entry->state != CONN_STATE_CONNECTED)\n\t\tentry = NULL;\n\n\tpthread_mutex_unlock(&service->mutex);\n\tif (!entry)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn NULL;\n\t}\n\n\tsession = service->new_session(entry->seq, entry->conn);\n\tif (!session)\n\t\treturn NULL;\n\n\tsession->passive = 1;\n\tentry->session = session;\n\tsession->target = target;\n\tsession->conn = entry->conn;\n\tsession->seq = entry->seq++;\n\tsession->out = NULL;\n\tsession->in = NULL;\n\n\ttimeout = Communicator::first_timeout_recv(session);\n\tmpoller_set_timeout(entry->sockfd, timeout, entry->mpoller);\n\tentry->state = CONN_STATE_RECEIVING;\n\n\t((CommServiceTarget *)target)->incref();\n\n\tin = session->message_in();\n\tif (in)\n\t{\n\t\tin->poller_message_t::append = Communicator::append_message;\n\t\tin->entry = entry;\n\t\tsession->in = in;\n\t}\n\n\treturn in;\n}\n\npoller_message_t *Communicator::create_reply(void *context)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)context;\n\tCommSession *session;\n\tCommMessageIn *in;\n\n\tif (entry->state == CONN_STATE_IDLE)\n\t{\n\t\tpthread_mutex_lock(&entry->mutex);\n\t\t/* do nothing */\n\t\tpthread_mutex_unlock(&entry->mutex);\n\t}\n\n\tif (entry->state != CONN_STATE_RECEIVING)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn NULL;\n\t}\n\n\tsession = entry->session;\n\tin = session->message_in();\n\tif (in)\n\t{\n\t\tin->poller_message_t::append = Communicator::append_message;\n\t\tin->entry = entry;\n\t\tsession->in = in;\n\t}\n\n\treturn in;\n}\n\nint Communicator::recv_request(const void *buf, size_t size,\n\t\t\t\t\t\t\t struct CommConnEntry *entry)\n{\n\tCommService *service = entry->service;\n\tCommTarget *target = entry->target;\n\tCommSession *session;\n\tCommMessageIn *in;\n\tsize_t n;\n\tint ret;\n\n\tsession = service->new_session(entry->seq, entry->conn);\n\tif (!session)\n\t\treturn -1;\n\n\tsession->passive = 1;\n\tentry->session = session;\n\tsession->target = target;\n\tsession->conn = entry->conn;\n\tsession->seq = entry->seq++;\n\tsession->out = NULL;\n\tsession->in = NULL;\n\n\tentry->state = CONN_STATE_RECEIVING;\n\n\t((CommServiceTarget *)target)->incref();\n\n\tin = session->message_in();\n\tif (in)\n\t{\n\t\tin->entry = entry;\n\t\tsession->in = in;\n\t\tdo\n\t\t{\n\t\t\tn = size;\n\t\t\tret = in->append(buf, &n);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tsize -= n;\n\t\t\t\tbuf = (const char *)buf + n;\n\t\t\t}\n\t\t\telse if (ret < 0)\n\t\t\t{\n\t\t\t\tentry->error = errno;\n\t\t\t\tentry->state = CONN_STATE_ERROR;\n\t\t\t}\n\t\t\telse\n\t\t\t\tentry->state = CONN_STATE_SUCCESS;\n\n\t\t} while (ret == 0 && size > 0);\n\t}\n\n\treturn 0;\n}\n\nint Communicator::partial_written(size_t n, void *context)\n{\n\tstruct CommConnEntry *entry = (struct CommConnEntry *)context;\n\tCommSession *session = entry->session;\n\tint timeout;\n\n\ttimeout = Communicator::next_timeout(session);\n\tmpoller_set_timeout(entry->sockfd, timeout, entry->mpoller);\n\treturn 0;\n}\n\nvoid *Communicator::accept(const struct sockaddr *addr, socklen_t addrlen,\n\t\t\t\t\t\t int sockfd, void *context)\n{\n\tCommService *service = (CommService *)context;\n\tCommServiceTarget *target = new CommServiceTarget;\n\n\tif (target)\n\t{\n\t\tif (target->init(addr, addrlen, 0, service->response_timeout) >= 0)\n\t\t{\n\t\t\tservice->incref();\n\t\t\ttarget->service = service;\n\t\t\ttarget->sockfd = sockfd;\n\t\t\ttarget->ref = 1;\n\t\t\treturn target;\n\t\t}\n\n\t\tdelete target;\n\t}\n\n\tclose(sockfd);\n\treturn NULL;\n}\n\nvoid *Communicator::recvfrom(const struct sockaddr *addr, socklen_t addrlen,\n\t\t\t\t\t\t\t const void *buf, size_t size, void *context)\n{\n\tCommService *service = (CommService *)context;\n\tstruct CommConnEntry *entry;\n\tCommServiceTarget *target;\n\tvoid *result;\n\tint sockfd;\n\n\tsockfd = dup(service->listen_fd);\n\tif (sockfd >= 0)\n\t{\n\t\tresult = Communicator::accept(addr, addrlen, sockfd, context);\n\t\tif (result)\n\t\t{\n\t\t\ttarget = (CommServiceTarget *)result;\n\t\t\tentry = Communicator::accept_conn(target, service);\n\t\t\tif (entry)\n\t\t\t{\n\t\t\t\tif (Communicator::recv_request(buf, size, entry) >= 0)\n\t\t\t\t\treturn entry;\n\n\t\t\t\t__release_conn(entry);\n\t\t\t}\n\t\t\telse\n\t\t\t\tclose(sockfd);\n\n\t\t\ttarget->decref();\n\t\t}\n\t}\n\n\treturn NULL;\n}\n\nvoid Communicator::callback(struct poller_result *res, void *context)\n{\n\tCommunicator *comm = (Communicator *)context;\n\tmsgqueue_put(res, comm->msgqueue);\n}\n\nint Communicator::create_handler_threads(size_t handler_threads)\n{\n\tstruct thrdpool_task task = {\n\t\t.routine\t=\tCommunicator::handler_thread_routine,\n\t\t.context\t=\tthis\n\t};\n\tsize_t i;\n\n\tthis->thrdpool = thrdpool_create(handler_threads, 0);\n\tif (this->thrdpool)\n\t{\n\t\tfor (i = 0; i < handler_threads; i++)\n\t\t{\n\t\t\tif (thrdpool_schedule(&task, this->thrdpool) < 0)\n\t\t\t\tbreak;\n\t\t}\n\n\t\tif (i == handler_threads)\n\t\t\treturn 0;\n\n\t\tmsgqueue_set_nonblock(this->msgqueue);\n\t\tthrdpool_destroy(NULL, this->thrdpool);\n\t}\n\n\treturn -1;\n}\n\nint Communicator::create_poller(size_t poller_threads)\n{\n\tstruct poller_params params = {\n\t\t.max_open_files\t\t=\t(size_t)sysconf(_SC_OPEN_MAX),\n\t\t.callback\t\t\t=\tCommunicator::callback,\n\t\t.context\t\t\t=\tthis\n\t};\n\n\tif ((ssize_t)params.max_open_files < 0)\n\t\treturn -1;\n\n\tthis->msgqueue = msgqueue_create(16 * 1024, sizeof (struct poller_result));\n\tif (this->msgqueue)\n\t{\n\t\tthis->mpoller = mpoller_create(¶ms, poller_threads);\n\t\tif (this->mpoller)\n\t\t{\n\t\t\tif (mpoller_start(this->mpoller) >= 0)\n\t\t\t\treturn 0;\n\n\t\t\tmpoller_destroy(this->mpoller);\n\t\t}\n\n\t\tmsgqueue_destroy(this->msgqueue);\n\t}\n\n\treturn -1;\n}\n\nint Communicator::init(size_t poller_threads, size_t handler_threads)\n{\n\tif (poller_threads == 0)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\tif (this->create_poller(poller_threads) >= 0)\n\t{\n\t\tif (this->create_handler_threads(handler_threads) >= 0)\n\t\t{\n\t\t\tthis->event_handler = NULL;\n\t\t\tthis->stop_flag = 0;\n\t\t\treturn 0;\n\t\t}\n\n\t\tmpoller_stop(this->mpoller);\n\t\tmpoller_destroy(this->mpoller);\n\t\tmsgqueue_destroy(this->msgqueue);\n\t}\n\n\treturn -1;\n}\n\nvoid Communicator::deinit()\n{\n\tthis->stop_flag = 1;\n\tmpoller_stop(this->mpoller);\n\tif (this->event_handler)\n\t\tthis->event_handler->wait();\n\n\tmsgqueue_set_nonblock(this->msgqueue);\n\tthrdpool_destroy(NULL, this->thrdpool);\n\tmpoller_destroy(this->mpoller);\n\tmsgqueue_destroy(this->msgqueue);\n}\n\nint Communicator::nonblock_connect(CommTarget *target)\n{\n\tint sockfd = target->create_connect_fd();\n\n\tif (sockfd >= 0)\n\t{\n\t\tif (__set_fd_nonblock(sockfd) >= 0)\n\t\t{\n\t\t\tif (connect(sockfd, target->addr, target->addrlen) >= 0 ||\n\t\t\t\terrno == EINPROGRESS)\n\t\t\t{\n\t\t\t\treturn sockfd;\n\t\t\t}\n\t\t}\n\n\t\tclose(sockfd);\n\t}\n\n\treturn -1;\n}\n\nstruct CommConnEntry *Communicator::launch_conn(CommSession *session,\n\t\t\t\t\t\t\t\t\t\t\t\tCommTarget *target)\n{\n\tstruct CommConnEntry *entry;\n\tint sockfd;\n\tint ret;\n\n\tsockfd = Communicator::nonblock_connect(target);\n\tif (sockfd >= 0)\n\t{\n\t\tentry = (struct CommConnEntry *)malloc(sizeof (struct CommConnEntry));\n\t\tif (entry)\n\t\t{\n\t\t\tret = pthread_mutex_init(&entry->mutex, NULL);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tentry->conn = target->new_connection(sockfd);\n\t\t\t\tif (entry->conn)\n\t\t\t\t{\n\t\t\t\t\tentry->seq = 0;\n\t\t\t\t\tentry->mpoller = NULL;\n\t\t\t\t\tentry->service = NULL;\n\t\t\t\t\tentry->target = target;\n\t\t\t\t\tentry->session = session;\n\t\t\t\t\tentry->ssl = NULL;\n\t\t\t\t\tentry->sockfd = sockfd;\n\t\t\t\t\tentry->state = CONN_STATE_CONNECTING;\n\t\t\t\t\tentry->ref = 1;\n\t\t\t\t\treturn entry;\n\t\t\t\t}\n\n\t\t\t\tpthread_mutex_destroy(&entry->mutex);\n\t\t\t}\n\t\t\telse\n\t\t\t\terrno = ret;\n\n\t\t\tfree(entry);\n\t\t}\n\n\t\tclose(sockfd);\n\t}\n\n\treturn NULL;\n}\n\nint Communicator::request_idle_conn(CommSession *session, CommTarget *target)\n{\n\tstruct CommConnEntry *entry;\n\tstruct list_head *pos;\n\tint ret = -1;\n\n\twhile (1)\n\t{\n\t\tpthread_mutex_lock(&target->mutex);\n\t\tif (!list_empty(&target->idle_list))\n\t\t{\n\t\t\tpos = target->idle_list.next;\n\t\t\tentry = list_entry(pos, struct CommConnEntry, list);\n\t\t\tlist_del(pos);\n\t\t\tpthread_mutex_lock(&entry->mutex);\n\t\t}\n\t\telse\n\t\t\tentry = NULL;\n\n\t\tpthread_mutex_unlock(&target->mutex);\n\t\tif (!entry)\n\t\t{\n\t\t\terrno = ENOENT;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (mpoller_set_timeout(entry->sockfd, -1, this->mpoller) >= 0)\n\t\t\tbreak;\n\n\t\tentry->state = CONN_STATE_CLOSING;\n\t\tpthread_mutex_unlock(&entry->mutex);\n\t}\n\n\tentry->session = session;\n\tsession->conn = entry->conn;\n\tsession->seq = entry->seq++;\n\tsession->out = session->message_out();\n\tif (session->out)\n\t\tret = this->send_message(entry);\n\n\tif (ret < 0)\n\t{\n\t\tentry->error = errno;\n\t\tmpoller_del(entry->sockfd, this->mpoller);\n\t\tentry->state = CONN_STATE_ERROR;\n\t\tret = 1;\n\t}\n\n\tpthread_mutex_unlock(&entry->mutex);\n\treturn ret;\n}\n\nint Communicator::request_new_conn(CommSession *session, CommTarget *target)\n{\n\tstruct CommConnEntry *entry;\n\tstruct poller_data data;\n\tint timeout;\n\n\tentry = Communicator::launch_conn(session, target);\n\tif (entry)\n\t{\n\t\tentry->mpoller = this->mpoller;\n\t\tsession->conn = entry->conn;\n\t\tsession->seq = entry->seq++;\n\t\tdata.operation = PD_OP_CONNECT;\n\t\tdata.fd = entry->sockfd;\n\t\tdata.ssl = NULL;\n\t\tdata.context = entry;\n\t\ttimeout = session->target->connect_timeout;\n\t\tif (mpoller_add(&data, timeout, this->mpoller) >= 0)\n\t\t\treturn 0;\n\n\t\t__release_conn(entry);\n\t}\n\n\treturn -1;\n}\n\nint Communicator::request(CommSession *session, CommTarget *target)\n{\n\tint errno_bak;\n\n\tif (session->passive)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\terrno_bak = errno;\n\tsession->target = target;\n\tsession->out = NULL;\n\tsession->in = NULL;\n\tif (this->request_idle_conn(session, target) < 0)\n\t{\n\t\tif (this->request_new_conn(session, target) < 0)\n\t\t{\n\t\t\tsession->conn = NULL;\n\t\t\tsession->seq = 0;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\terrno = errno_bak;\n\treturn 0;\n}\n\nint Communicator::nonblock_listen(CommService *service)\n{\n\tint sockfd = service->create_listen_fd();\n\tint ret;\n\n\tif (sockfd >= 0)\n\t{\n\t\tif (__set_fd_nonblock(sockfd) >= 0)\n\t\t{\n\t\t\tif (__bind_sockaddr(sockfd, service->bind_addr,\n\t\t\t\t\t\t\t\tservice->addrlen) >= 0)\n\t\t\t{\n\t\t\t\tret = listen(sockfd, SOMAXCONN);\n\t\t\t\tif (ret >= 0 || errno == EOPNOTSUPP)\n\t\t\t\t{\n\t\t\t\t\tservice->reliable = (ret >= 0);\n\t\t\t\t\treturn sockfd;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tclose(sockfd);\n\t}\n\n\treturn -1;\n}\n\nint Communicator::bind(CommService *service)\n{\n\tstruct poller_data data;\n\tint errno_bak = errno;\n\tint sockfd;\n\n\tsockfd = this->nonblock_listen(service);\n\tif (sockfd >= 0)\n\t{\n\t\tservice->listen_fd = sockfd;\n\t\tservice->ref = 1;\n\t\tdata.fd = sockfd;\n\t\tdata.context = service;\n\t\tdata.result = NULL;\n\t\tif (service->reliable)\n\t\t{\n\t\t\tdata.operation = PD_OP_LISTEN;\n\t\t\tdata.accept = Communicator::accept;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdata.operation = PD_OP_RECVFROM;\n\t\t\tdata.recvfrom = Communicator::recvfrom;\n\t\t}\n\n\t\tif (mpoller_add(&data, service->listen_timeout, this->mpoller) >= 0)\n\t\t{\n\t\t\terrno = errno_bak;\n\t\t\treturn 0;\n\t\t}\n\n\t\tclose(sockfd);\n\t}\n\n\treturn -1;\n}\n\nvoid Communicator::unbind(CommService *service)\n{\n\tint errno_bak = errno;\n\n\tif (mpoller_del(service->listen_fd, this->mpoller) < 0)\n\t{\n\t\t/* Error occurred on listen_fd or Communicator::deinit() called. */\n\t\tthis->shutdown_service(service);\n\t\terrno = errno_bak;\n\t}\n}\n\nint Communicator::reply_reliable(CommSession *session, CommTarget *target)\n{\n\tstruct CommConnEntry *entry;\n\tstruct list_head *pos;\n\tint ret = -1;\n\n\tpthread_mutex_lock(&target->mutex);\n\tif (!list_empty(&target->idle_list))\n\t{\n\t\tpos = target->idle_list.next;\n\t\tentry = list_entry(pos, struct CommConnEntry, list);\n\t\tlist_del(pos);\n\n\t\tsession->out = session->message_out();\n\t\tif (session->out)\n\t\t\tret = this->send_message(entry);\n\n\t\tif (ret < 0)\n\t\t{\n\t\t\tentry->error = errno;\n\t\t\tmpoller_del(entry->sockfd, this->mpoller);\n\t\t\tentry->state = CONN_STATE_ERROR;\n\t\t\tret = 1;\n\t\t}\n\t}\n\telse\n\t\terrno = ENOENT;\n\n\tpthread_mutex_unlock(&target->mutex);\n\treturn ret;\n}\n\nint Communicator::reply_message_unreliable(struct CommConnEntry *entry)\n{\n\tstruct iovec vectors[ENCODE_IOV_MAX];\n\tint cnt;\n\n\tcnt = entry->session->out->encode(vectors, ENCODE_IOV_MAX);\n\tif ((unsigned int)cnt > ENCODE_IOV_MAX)\n\t{\n\t\tif (cnt > ENCODE_IOV_MAX)\n\t\t\terrno = EOVERFLOW;\n\t\treturn -1;\n\t}\n\n\tif (cnt > 0)\n\t{\n\t\tstruct msghdr message = {\n\t\t\t.msg_name\t\t=\tentry->target->addr,\n\t\t\t.msg_namelen\t=\tentry->target->addrlen,\n\t\t\t.msg_iov\t\t=\tvectors,\n#ifdef __linux__\n\t\t\t.msg_iovlen\t\t=\t(size_t)cnt,\n#else\n\t\t\t.msg_iovlen\t\t=\tcnt,\n#endif\n\t\t};\n\t\tif (sendmsg(entry->sockfd, &message, 0) < 0)\n\t\t\treturn -1;\n\t}\n\n\treturn 0;\n}\n\nint Communicator::reply_unreliable(CommSession *session, CommTarget *target)\n{\n\tstruct CommConnEntry *entry;\n\tstruct list_head *pos;\n\n\tif (!list_empty(&target->idle_list))\n\t{\n\t\tpos = target->idle_list.next;\n\t\tentry = list_entry(pos, struct CommConnEntry, list);\n\t\tlist_del(pos);\n\n\t\tsession->out = session->message_out();\n\t\tif (session->out)\n\t\t{\n\t\t\tif (this->reply_message_unreliable(entry) >= 0)\n\t\t\t\treturn 0;\n\t\t}\n\n\t\t__release_conn(entry);\n\t\t((CommServiceTarget *)target)->decref();\n\t}\n\telse\n\t\terrno = ENOENT;\n\n\treturn -1;\n}\n\nint Communicator::reply(CommSession *session)\n{\n\tstruct CommConnEntry *entry;\n\tCommServiceTarget *target;\n\tint errno_bak;\n\tint ret;\n\n\tif (!session->passive)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\tif (session->out)\n\t{\n\t\terrno = ENOENT;\n\t\treturn -1;\n\t}\n\n\terrno_bak = errno;\n\ttarget = (CommServiceTarget *)session->target;\n\tif (target->service->reliable)\n\t\tret = this->reply_reliable(session, target);\n\telse\n\t\tret = this->reply_unreliable(session, target);\n\n\tif (ret == 0)\n\t{\n\t\tentry = session->in->entry;\n\t\tsession->handle(CS_STATE_SUCCESS, 0);\n\t\tif (__sync_sub_and_fetch(&entry->ref, 1) == 0)\n\t\t{\n\t\t\t__release_conn(entry);\n\t\t\ttarget->decref();\n\t\t}\n\t}\n\telse if (ret < 0)\n\t\treturn -1;\n\n\terrno = errno_bak;\n\treturn 0;\n}\n\nint Communicator::push(const void *buf, size_t size, CommSession *session)\n{\n\tCommMessageIn *in = session->in;\n\tpthread_mutex_t *mutex;\n\tint ret;\n\n\tif (!in)\n\t{\n\t\terrno = ENOENT;\n\t\treturn -1;\n\t}\n\n\tif (session->passive)\n\t\tmutex = &session->target->mutex;\n\telse\n\t\tmutex = &in->entry->mutex;\n\n\tpthread_mutex_lock(mutex);\n\tif ((!session->passive || session->target->has_idle_conn()) &&\n\t\tin->entry->session == session)\n\t{\n\t\tret = in->inner()->feedback(buf, size);\n\t}\n\telse\n\t{\n\t\terrno = ENOENT;\n\t\tret = -1;\n\t}\n\n\tpthread_mutex_unlock(mutex);\n\treturn ret;\n}\n\nint Communicator::shutdown(CommSession *session)\n{\n\tCommServiceTarget *target;\n\n\tif (!session->passive)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\ttarget = (CommServiceTarget *)session->target;\n\tif (session->out || !target->shutdown())\n\t{\n\t\terrno = ENOENT;\n\t\treturn -1;\n\t}\n\n\treturn 0;\n}\n\nint Communicator::sleep(SleepSession *session)\n{\n\tstruct timespec value;\n\n\tif (session->duration(&value) >= 0)\n\t{\n\t\tif (mpoller_add_timer(&value, session, &session->timer, &session->index,\n\t\t\t\t\t\t\t this->mpoller) >= 0)\n\t\t{\n\t\t\treturn 0;\n\t\t}\n\t}\n\n\treturn -1;\n}\n\nint Communicator::unsleep(SleepSession *session)\n{\n\treturn mpoller_del_timer(session->timer, session->index, this->mpoller);\n}\n\n#ifdef __linux__\n\nvoid Communicator::shutdown_io_service(IOService *service)\n{\n\tpthread_mutex_lock(&service->mutex);\n\tclose(service->event_fd);\n\tservice->event_fd = -1;\n\tpthread_mutex_unlock(&service->mutex);\n\tservice->decref();\n}\n\nint Communicator::io_bind(IOService *service)\n{\n\tstruct poller_data data;\n\tint event_fd;\n\n\tevent_fd = service->create_event_fd();\n\tif (event_fd >= 0)\n\t{\n\t\tif (__set_fd_nonblock(event_fd) >= 0)\n\t\t{\n\t\t\tservice->ref = 1;\n\t\t\tdata.operation = PD_OP_EVENT;\n\t\t\tdata.fd = event_fd;\n\t\t\tdata.event = IOService::aio_finish;\n\t\t\tdata.context = service;\n\t\t\tdata.result = NULL;\n\t\t\tif (mpoller_add(&data, -1, this->mpoller) >= 0)\n\t\t\t{\n\t\t\t\tservice->event_fd = event_fd;\n\t\t\t\treturn 0;\n\t\t\t}\n\t\t}\n\n\t\tclose(event_fd);\n\t}\n\n\treturn -1;\n}\n\nvoid Communicator::io_unbind(IOService *service)\n{\n\tint errno_bak = errno;\n\n\tif (mpoller_del(service->event_fd, this->mpoller) < 0)\n\t{\n\t\t/* Error occurred on event_fd or Communicator::deinit() called. */\n\t\tthis->shutdown_io_service(service);\n\t\terrno = errno_bak;\n\t}\n}\n\n#else\n\nvoid Communicator::shutdown_io_service(IOService *service)\n{\n\tpthread_mutex_lock(&service->mutex);\n\tclose(service->pipe_fd[0]);\n\tclose(service->pipe_fd[1]);\n\tservice->pipe_fd[0] = -1;\n\tservice->pipe_fd[1] = -1;\n\tpthread_mutex_unlock(&service->mutex);\n\tservice->decref();\n}\n\nint Communicator::io_bind(IOService *service)\n{\n\tstruct poller_data data;\n\tint pipe_fd[2];\n\n\tif (service->create_pipe_fd(pipe_fd) >= 0)\n\t{\n\t\tif (__set_fd_nonblock(pipe_fd[0]) >= 0)\n\t\t{\n\t\t\tservice->ref = 1;\n\t\t\tdata.operation = PD_OP_NOTIFY;\n\t\t\tdata.fd = pipe_fd[0];\n\t\t\tdata.notify = IOService::aio_finish;\n\t\t\tdata.context = service;\n\t\t\tdata.result = NULL;\n\t\t\tif (mpoller_add(&data, -1, this->mpoller) >= 0)\n\t\t\t{\n\t\t\t\tservice->pipe_fd[0] = pipe_fd[0];\n\t\t\t\tservice->pipe_fd[1] = pipe_fd[1];\n\t\t\t\treturn 0;\n\t\t\t}\n\t\t}\n\n\t\tclose(pipe_fd[0]);\n\t\tclose(pipe_fd[1]);\n\t}\n\n\treturn -1;\n}\n\nvoid Communicator::io_unbind(IOService *service)\n{\n\tint errno_bak = errno;\n\n\tif (mpoller_del(service->pipe_fd[0], this->mpoller) < 0)\n\t{\n\t\t/* Error occurred on pipe_fd or Communicator::deinit() called. */\n\t\tthis->shutdown_io_service(service);\n\t\terrno = errno_bak;\n\t}\n}\n\n#endif\n\nint Communicator::is_handler_thread() const\n{\n\treturn thrdpool_in_pool(this->thrdpool);\n}\n\nextern \"C\" void __thrdpool_schedule(const struct thrdpool_task *, void *,\n\t\t\t\t\t\t\t\t\tthrdpool_t *);\n\nint Communicator::increase_handler_thread()\n{\n\tvoid *buf = malloc(4 * sizeof (void *));\n\n\tif (buf)\n\t{\n\t\tif (thrdpool_increase(this->thrdpool) >= 0)\n\t\t{\n\t\t\tstruct thrdpool_task task = {\n\t\t\t\t.routine\t=\tCommunicator::handler_thread_routine,\n\t\t\t\t.context\t=\tthis\n\t\t\t};\n\t\t\t__thrdpool_schedule(&task, buf, this->thrdpool);\n\t\t\treturn 0;\n\t\t}\n\n\t\tfree(buf);\n\t}\n\n\treturn -1;\n}\n\nint Communicator::decrease_handler_thread()\n{\n\tstruct poller_result *res;\n\tsize_t size;\n\n\tsize = sizeof (struct poller_result) + sizeof (void *);\n\tres = (struct poller_result *)malloc(size);\n\tif (res)\n\t{\n\t\tres->data.operation = -1;\n\t\tmsgqueue_put_head(res, this->msgqueue);\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nvoid Communicator::event_handler_routine(void *context)\n{\n\tstruct poller_result *res = (struct poller_result *)context;\n\tCommunicator *comm = *(Communicator **)(res + 1);\n\tcomm->handle_poller_result(res);\n}\n\nvoid Communicator::callback_custom(struct poller_result *res, void *context)\n{\n\tCommunicator *comm = (Communicator *)context;\n\tCommEventHandler *handler = comm->event_handler;\n\n\tif (handler)\n\t{\n\t\t*(Communicator **)(res + 1) = comm;\n\t\thandler->schedule(Communicator::event_handler_routine, res);\n\t}\n\telse\n\t\tCommunicator::callback(res, context);\n}\n\nvoid Communicator::customize_event_handler(CommEventHandler *handler)\n{\n\tthis->event_handler = handler;\n\tif (handler)\n\t\tmpoller_set_callback(Communicator::callback_custom, this->mpoller);\n\telse\n\t\tmpoller_set_callback(Communicator::callback, this->mpoller);\n}\n\n" }, { "path": "src/kernel/Communicator.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _COMMUNICATOR_H_\n#define _COMMUNICATOR_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"poller.h\"\n\nclass CommConnection\n{\npublic:\n\tvirtual ~CommConnection() { }\n};\n\nclass CommTarget\n{\npublic:\n\tint init(const struct sockaddr *addr, socklen_t addrlen,\n\t\t\t int connect_timeout, int response_timeout);\n\tvoid deinit();\n\npublic:\n\tvoid get_addr(const struct sockaddr **addr, socklen_t *addrlen) const\n\t{\n\t\t*addr = this->addr;\n\t\t*addrlen = this->addrlen;\n\t}\n\n\tint has_idle_conn() const { return !list_empty(&this->idle_list); }\n\nprotected:\n\tvoid set_ssl(SSL_CTX *ssl_ctx, int ssl_connect_timeout)\n\t{\n\t\tthis->ssl_ctx = ssl_ctx;\n\t\tthis->ssl_connect_timeout = ssl_connect_timeout;\n\t}\n\n\tSSL_CTX *get_ssl_ctx() const { return this->ssl_ctx; }\n\nprivate:\n\tvirtual int create_connect_fd()\n\t{\n\t\treturn socket(this->addr->sa_family, SOCK_STREAM, 0);\n\t}\n\n\tvirtual CommConnection *new_connection(int connect_fd)\n\t{\n\t\treturn new CommConnection;\n\t}\n\n\tvirtual int init_ssl(SSL *ssl) { return 0; }\n\npublic:\n\tvirtual void release() { }\n\nprivate:\n\tstruct sockaddr *addr;\n\tsocklen_t addrlen;\n\tint connect_timeout;\n\tint response_timeout;\n\tint ssl_connect_timeout;\n\tSSL_CTX *ssl_ctx;\n\nprivate:\n\tstruct list_head idle_list;\n\tpthread_mutex_t mutex;\n\npublic:\n\tvirtual ~CommTarget() { }\n\tfriend class CommServiceTarget;\n\tfriend class Communicator;\n};\n\nclass CommMessageOut\n{\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max) = 0;\n\npublic:\n\tvirtual ~CommMessageOut() { }\n\tfriend class Communicator;\n};\n\nclass CommMessageIn : private poller_message_t\n{\nprivate:\n\tvirtual int append(const void *buf, size_t *size) = 0;\n\nprotected:\n\t/* Send small packet while receiving. Call only in append(). */\n\tvirtual int feedback(const void *buf, size_t size);\n\n\t/* In append(), reset the begin time of receiving to current time. */\n\tvirtual void renew();\n\n\t/* Return the deepest wrapped message. */\n\tvirtual CommMessageIn *inner() { return this; }\n\nprivate:\n\tstruct CommConnEntry *entry;\n\npublic:\n\tvirtual ~CommMessageIn() { }\n\tfriend class Communicator;\n};\n\n#define CS_STATE_SUCCESS\t0\n#define CS_STATE_ERROR\t\t1\n#define CS_STATE_STOPPED\t2\n#define CS_STATE_TOREPLY\t3\t/* for service session only. */\n\nclass CommSession\n{\nprivate:\n\tvirtual CommMessageOut *message_out() = 0;\n\tvirtual CommMessageIn *message_in() = 0;\n\tvirtual int send_timeout() { return -1; }\n\tvirtual int receive_timeout() { return -1; }\n\tvirtual int keep_alive_timeout() { return 0; }\n\tvirtual int first_timeout() { return 0; }\n\tvirtual void handle(int state, int error) = 0;\n\nprotected:\n\tCommTarget *get_target() const { return this->target; }\n\tCommConnection *get_connection() const { return this->conn; }\n\tCommMessageOut *get_message_out() const { return this->out; }\n\tCommMessageIn *get_message_in() const { return this->in; }\n\tlong long get_seq() const { return this->seq; }\n\nprivate:\n\tCommTarget *target;\n\tCommConnection *conn;\n\tCommMessageOut *out;\n\tCommMessageIn *in;\n\tlong long seq;\n\nprivate:\n\tstruct timespec begin_time;\n\tint timeout;\n\tint passive;\n\npublic:\n\tCommSession() { this->passive = 0; }\n\tvirtual ~CommSession();\n\tfriend class CommMessageIn;\n\tfriend class Communicator;\n};\n\nclass CommService\n{\npublic:\n\tint init(const struct sockaddr *bind_addr, socklen_t addrlen,\n\t\t\t int listen_timeout, int response_timeout);\n\tvoid deinit();\n\n\tint drain(int max);\n\npublic:\n\tvoid get_addr(const struct sockaddr **addr, socklen_t *addrlen) const\n\t{\n\t\t*addr = this->bind_addr;\n\t\t*addrlen = this->addrlen;\n\t}\n\nprotected:\n\tvoid set_ssl(SSL_CTX *ssl_ctx, int ssl_accept_timeout)\n\t{\n\t\tthis->ssl_ctx = ssl_ctx;\n\t\tthis->ssl_accept_timeout = ssl_accept_timeout;\n\t}\n\n\tSSL_CTX *get_ssl_ctx() const { return this->ssl_ctx; }\n\nprivate:\n\tvirtual CommSession *new_session(long long seq, CommConnection *conn) = 0;\n\tvirtual void handle_stop(int error) { }\n\tvirtual void handle_unbound() = 0;\n\nprivate:\n\tvirtual int create_listen_fd()\n\t{\n\t\treturn socket(this->bind_addr->sa_family, SOCK_STREAM, 0);\n\t}\n\n\tvirtual CommConnection *new_connection(int accept_fd)\n\t{\n\t\treturn new CommConnection;\n\t}\n\n\tvirtual int init_ssl(SSL *ssl) { return 0; }\n\nprivate:\n\tstruct sockaddr *bind_addr;\n\tsocklen_t addrlen;\n\tint listen_timeout;\n\tint response_timeout;\n\tint ssl_accept_timeout;\n\tSSL_CTX *ssl_ctx;\n\nprivate:\n\tvoid incref();\n\tvoid decref();\n\nprivate:\n\tint reliable;\n\tint listen_fd;\n\tint ref;\n\nprivate:\n\tstruct list_head keep_alive_list;\n\tpthread_mutex_t mutex;\n\npublic:\n\tvirtual ~CommService() { }\n\tfriend class CommServiceTarget;\n\tfriend class Communicator;\n};\n\n#define SS_STATE_COMPLETE\t0\n#define SS_STATE_ERROR\t\t1\n#define SS_STATE_DISRUPTED\t2\n\nclass SleepSession\n{\nprivate:\n\tvirtual int duration(struct timespec *value) = 0;\n\tvirtual void handle(int state, int error) = 0;\n\nprivate:\n\tvoid *timer;\n\tint index;\n\npublic:\n\tvirtual ~SleepSession() { }\n\tfriend class Communicator;\n};\n\n#ifdef __linux__\n# include \"IOService_linux.h\"\n#else\n# include \"IOService_thread.h\"\n#endif\n\nclass CommEventHandler\n{\nprivate:\n\tvirtual void schedule(void (*routine)(void *), void *context) = 0;\n\tvirtual void wait() = 0;\n\npublic:\n\tvirtual ~CommEventHandler() { }\n\tfriend class Communicator;\n};\n\nclass Communicator\n{\npublic:\n\tint init(size_t poller_threads, size_t handler_threads);\n\tvoid deinit();\n\n\tint request(CommSession *session, CommTarget *target);\n\tint reply(CommSession *session);\n\n\tint push(const void *buf, size_t size, CommSession *session);\n\n\tint shutdown(CommSession *session);\n\n\tint bind(CommService *service);\n\tvoid unbind(CommService *service);\n\n\tint sleep(SleepSession *session);\n\tint unsleep(SleepSession *session);\n\n\tint io_bind(IOService *service);\n\tvoid io_unbind(IOService *service);\n\npublic:\n\tint is_handler_thread() const;\n\n\tint increase_handler_thread();\n\tint decrease_handler_thread();\n\npublic:\n\tvoid customize_event_handler(CommEventHandler *handler);\n\nprivate:\n\tstruct __mpoller *mpoller;\n\tstruct __msgqueue *msgqueue;\n\tstruct __thrdpool *thrdpool;\n\tint stop_flag;\n\nprivate:\n\tCommEventHandler *event_handler;\n\nprivate:\n\tint create_poller(size_t poller_threads);\n\n\tint create_handler_threads(size_t handler_threads);\n\n\tvoid shutdown_service(CommService *service);\n\n\tvoid shutdown_io_service(IOService *service);\n\n\tint send_message_sync(struct iovec vectors[], int cnt,\n\t\t\t\t\t\t struct CommConnEntry *entry);\n\tint send_message_async(struct iovec vectors[], int cnt,\n\t\t\t\t\t\t struct CommConnEntry *entry);\n\n\tint send_message(struct CommConnEntry *entry);\n\n\tint request_new_conn(CommSession *session, CommTarget *target);\n\tint request_idle_conn(CommSession *session, CommTarget *target);\n\n\tint reply_message_unreliable(struct CommConnEntry *entry);\n\n\tint reply_reliable(CommSession *session, CommTarget *target);\n\tint reply_unreliable(CommSession *session, CommTarget *target);\n\n\tvoid handle_poller_result(struct poller_result *res);\n\n\tvoid handle_incoming_request(struct poller_result *res);\n\tvoid handle_incoming_reply(struct poller_result *res);\n\n\tvoid handle_request_result(struct poller_result *res);\n\tvoid handle_reply_result(struct poller_result *res);\n\n\tvoid handle_write_result(struct poller_result *res);\n\tvoid handle_read_result(struct poller_result *res);\n\n\tvoid handle_connect_result(struct poller_result *res);\n\tvoid handle_listen_result(struct poller_result *res);\n\n\tvoid handle_recvfrom_result(struct poller_result *res);\n\n\tvoid handle_ssl_accept_result(struct poller_result *res);\n\n\tvoid handle_sleep_result(struct poller_result *res);\n\n\tvoid handle_aio_result(struct poller_result *res);\n\n\tstatic void handler_thread_routine(void *context);\n\n\tstatic int nonblock_connect(CommTarget *target);\n\tstatic int nonblock_listen(CommService *service);\n\n\tstatic struct CommConnEntry *launch_conn(CommSession *session,\n\t\t\t\t\t\t\t\t\t\t\t CommTarget *target);\n\tstatic struct CommConnEntry *accept_conn(class CommServiceTarget *target,\n\t\t\t\t\t\t\t\t\t\t\t CommService *service);\n\n\tstatic int first_timeout(CommSession *session);\n\tstatic int next_timeout(CommSession *session);\n\n\tstatic int first_timeout_send(CommSession *session);\n\tstatic int first_timeout_recv(CommSession *session);\n\n\tstatic int append_message(const void *buf, size_t *size,\n\t\t\t\t\t\t\t poller_message_t *msg);\n\n\tstatic poller_message_t *create_request(void *context);\n\tstatic poller_message_t *create_reply(void *context);\n\n\tstatic int recv_request(const void *buf, size_t size,\n\t\t\t\t\t\t\tstruct CommConnEntry *entry);\n\n\tstatic int partial_written(size_t n, void *context);\n\n\tstatic void *accept(const struct sockaddr *addr, socklen_t addrlen,\n\t\t\t\t\t\tint sockfd, void *context);\n\n\tstatic void *recvfrom(const struct sockaddr *addr, socklen_t addrlen,\n\t\t\t\t\t\t const void *buf, size_t size, void *context);\n\n\tstatic void callback(struct poller_result *res, void *context);\n\nprivate:\n\tstatic void event_handler_routine(void *context);\n\n\tstatic void callback_custom(struct poller_result *res, void *context);\n\npublic:\n\tvirtual ~Communicator() { }\n};\n\n#endif\n\n" }, { "path": "src/kernel/ExecRequest.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _EXECREQUEST_H_\n#define _EXECREQUEST_H_\n\n#include \n#include \"SubTask.h\"\n#include \"Executor.h\"\n\nclass ExecRequest : public SubTask, public ExecSession\n{\npublic:\n\tExecRequest(ExecQueue *queue, Executor *executor)\n\t{\n\t\tthis->executor = executor;\n\t\tthis->queue = queue;\n\t}\n\n\tExecQueue *get_request_queue() const { return this->queue; }\n\tvoid set_request_queue(ExecQueue *queue) { this->queue = queue; }\n\npublic:\n\tvirtual void dispatch()\n\t{\n\t\tif (this->executor->request(this, this->queue) < 0)\n\t\t\tthis->handle(ES_STATE_ERROR, errno);\n\t}\n\nprotected:\n\tint state;\n\tint error;\n\nprotected:\n\tExecQueue *queue;\n\tExecutor *executor;\n\nprotected:\n\tvirtual void handle(int state, int error)\n\t{\n\t\tthis->state = state;\n\t\tthis->error = error;\n\t\tthis->subtask_done();\n\t}\n};\n\n#endif\n\n" }, { "path": "src/kernel/Executor.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"list.h\"\n#include \"thrdpool.h\"\n#include \"Executor.h\"\n\nstruct ExecSessionEntry\n{\n\tstruct list_head list;\n\tExecSession *session;\n\tthrdpool_t *thrdpool;\n};\n\nint ExecQueue::init()\n{\n\tint ret;\n\n\tret = pthread_mutex_init(&this->mutex, NULL);\n\tif (ret == 0)\n\t{\n\t\tINIT_LIST_HEAD(&this->session_list);\n\t\treturn 0;\n\t}\n\n\terrno = ret;\n\treturn -1;\n}\n\nvoid ExecQueue::deinit()\n{\n\tpthread_mutex_destroy(&this->mutex);\n}\n\nint Executor::init(size_t nthreads)\n{\n\tthis->thrdpool = thrdpool_create(nthreads, 0);\n\tif (this->thrdpool)\n\t\treturn 0;\n\n\treturn -1;\n}\n\nvoid Executor::deinit()\n{\n\tthrdpool_destroy(Executor::executor_cancel, this->thrdpool);\n}\n\nextern \"C\" void __thrdpool_schedule(const struct thrdpool_task *, void *,\n\t\t\t\t\t\t\t\t\tthrdpool_t *);\n\nvoid Executor::executor_thread_routine(void *context)\n{\n\tExecQueue *queue = (ExecQueue *)context;\n\tstruct ExecSessionEntry *entry;\n\tExecSession *session;\n\tint empty;\n\n\tentry = list_entry(queue->session_list.next, struct ExecSessionEntry, list);\n\tpthread_mutex_lock(&queue->mutex);\n\tlist_del(&entry->list);\n\tempty = list_empty(&queue->session_list);\n\tpthread_mutex_unlock(&queue->mutex);\n\n\tsession = entry->session;\n\tif (!empty)\n\t{\n\t\tstruct thrdpool_task task = {\n\t\t\t.routine\t=\tExecutor::executor_thread_routine,\n\t\t\t.context\t=\tqueue\n\t\t};\n\t\t__thrdpool_schedule(&task, entry, entry->thrdpool);\n\t}\n\telse\n\t\tfree(entry);\n\n\tsession->execute();\n\tsession->handle(ES_STATE_FINISHED, 0);\n}\n\nvoid Executor::executor_cancel(const struct thrdpool_task *task)\n{\n\tExecQueue *queue = (ExecQueue *)task->context;\n\tstruct ExecSessionEntry *entry;\n\tstruct list_head *pos, *tmp;\n\tExecSession *session;\n\n\tlist_for_each_safe(pos, tmp, &queue->session_list)\n\t{\n\t\tentry = list_entry(pos, struct ExecSessionEntry, list);\n\t\tlist_del(pos);\n\t\tsession = entry->session;\n\t\tfree(entry);\n\n\t\tsession->handle(ES_STATE_CANCELED, 0);\n\t}\n}\n\nint Executor::request(ExecSession *session, ExecQueue *queue)\n{\n\tstruct ExecSessionEntry *entry;\n\n\tsession->queue = queue;\n\tentry = (struct ExecSessionEntry *)malloc(sizeof (struct ExecSessionEntry));\n\tif (entry)\n\t{\n\t\tentry->session = session;\n\t\tentry->thrdpool = this->thrdpool;\n\t\tpthread_mutex_lock(&queue->mutex);\n\t\tlist_add_tail(&entry->list, &queue->session_list);\n\t\tif (queue->session_list.next == &entry->list)\n\t\t{\n\t\t\tstruct thrdpool_task task = {\n\t\t\t\t.routine\t=\tExecutor::executor_thread_routine,\n\t\t\t\t.context\t=\tqueue\n\t\t\t};\n\t\t\tif (thrdpool_schedule(&task, this->thrdpool) < 0)\n\t\t\t{\n\t\t\t\tlist_del(&entry->list);\n\t\t\t\tfree(entry);\n\t\t\t\tentry = NULL;\n\t\t\t}\n\t\t}\n\n\t\tpthread_mutex_unlock(&queue->mutex);\n\t}\n\n\treturn -!entry;\n}\n\nint Executor::increase_thread()\n{\n\treturn thrdpool_increase(this->thrdpool);\n}\n\nint Executor::decrease_thread()\n{\n\treturn thrdpool_decrease(this->thrdpool);\n}\n\n" }, { "path": "src/kernel/Executor.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _EXECUTOR_H_\n#define _EXECUTOR_H_\n\n#include \n#include \n#include \"list.h\"\n\nclass ExecQueue\n{\npublic:\n\tint init();\n\tvoid deinit();\n\nprivate:\n\tstruct list_head session_list;\n\tpthread_mutex_t mutex;\n\npublic:\n\tvirtual ~ExecQueue() { }\n\tfriend class Executor;\n};\n\n#define ES_STATE_FINISHED\t0\n#define ES_STATE_ERROR\t\t1\n#define ES_STATE_CANCELED\t2\n\nclass ExecSession\n{\nprivate:\n\tvirtual void execute() = 0;\n\tvirtual void handle(int state, int error) = 0;\n\nprotected:\n\tExecQueue *get_queue() const { return this->queue; }\n\nprivate:\n\tExecQueue *queue;\n\npublic:\n\tvirtual ~ExecSession() { }\n\tfriend class Executor;\n};\n\nclass Executor\n{\npublic:\n\tint init(size_t nthreads);\n\tvoid deinit();\n\n\tint request(ExecSession *session, ExecQueue *queue);\n\npublic:\n\tint increase_thread();\n\tint decrease_thread();\n\nprivate:\n\tstruct __thrdpool *thrdpool;\n\nprivate:\n\tstatic void executor_thread_routine(void *context);\n\tstatic void executor_cancel(const struct thrdpool_task *task);\n\npublic:\n\tvirtual ~Executor() { }\n};\n\n#endif\n\n" }, { "path": "src/kernel/IORequest.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _IOREQUEST_H_\n#define _IOREQUEST_H_\n\n#include \n#include \"SubTask.h\"\n#include \"Communicator.h\"\n\nclass IORequest : public SubTask, public IOSession\n{\npublic:\n\tIORequest(IOService *service)\n\t{\n\t\tthis->service = service;\n\t}\n\npublic:\n\tvirtual void dispatch()\n\t{\n\t\tif (this->service->request(this) < 0)\n\t\t\tthis->handle(IOS_STATE_ERROR, errno);\n\t}\n\nprotected:\n\tint state;\n\tint error;\n\nprotected:\n\tIOService *service;\n\nprotected:\n\tvirtual void handle(int state, int error)\n\t{\n\t\tthis->state = state;\n\t\tthis->error = error;\n\t\tthis->subtask_done();\n\t}\n};\n\n#endif\n\n" }, { "path": "src/kernel/IOService_linux.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"IOService_linux.h\"\n\n/* Linux async I/O interface from libaio.h */\n\ntypedef struct io_context *io_context_t;\n\ntypedef enum io_iocb_cmd {\n\tIO_CMD_PREAD = 0,\n\tIO_CMD_PWRITE = 1,\n\n\tIO_CMD_FSYNC = 2,\n\tIO_CMD_FDSYNC = 3,\n\n\tIO_CMD_POLL = 5,\n\tIO_CMD_NOOP = 6,\n\tIO_CMD_PREADV = 7,\n\tIO_CMD_PWRITEV = 8,\n} io_iocb_cmd_t;\n\n/* little endian, 32 bits */\n#if defined(__i386__) || (defined(__arm__) && !defined(__ARMEB__)) || \\\n defined(__sh__) || defined(__bfin__) || defined(__MIPSEL__) || \\\n defined(__cris__) || (defined(__riscv) && __riscv_xlen == 32) || \\\n (defined(__GNUC__) && defined(__BYTE_ORDER__) && \\\n __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_LONG__ == 4)\n#define PADDED(x, y)\tx; unsigned y\n#define PADDEDptr(x, y)\tx; unsigned y\n#define PADDEDul(x, y)\tunsigned long x; unsigned y\n\n/* little endian, 64 bits */\n#elif defined(__ia64__) || defined(__x86_64__) || defined(__alpha__) || \\\n (defined(__aarch64__) && defined(__AARCH64EL__)) || \\\n (defined(__riscv) && __riscv_xlen == 64) || \\\n (defined(__GNUC__) && defined(__BYTE_ORDER__) && \\\n __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_LONG__ == 8)\n#define PADDED(x, y)\tx, y\n#define PADDEDptr(x, y)\tx\n#define PADDEDul(x, y)\tunsigned long x\n\n/* big endian, 64 bits */\n#elif defined(__powerpc64__) || defined(__s390x__) || \\\n (defined(__sparc__) && defined(__arch64__)) || \\\n (defined(__aarch64__) && defined(__AARCH64EB__)) || \\\n (defined(__GNUC__) && defined(__BYTE_ORDER__) && \\\n __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ && __SIZEOF_LONG__ == 8)\n#define PADDED(x, y)\tunsigned y; x\n#define PADDEDptr(x,y)\tx\n#define PADDEDul(x, y)\tunsigned long x\n\n/* big endian, 32 bits */\n#elif defined(__PPC__) || defined(__s390__) || \\\n (defined(__arm__) && defined(__ARMEB__)) || \\\n defined(__sparc__) || defined(__MIPSEB__) || defined(__m68k__) || \\\n defined(__hppa__) || defined(__frv__) || defined(__avr32__) || \\\n (defined(__GNUC__) && defined(__BYTE_ORDER__) && \\\n __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ && __SIZEOF_LONG__ == 4)\n#define PADDED(x, y)\tunsigned y; x\n#define PADDEDptr(x, y)\tunsigned y; x\n#define PADDEDul(x, y)\tunsigned y; unsigned long x\n\n#else\n#error\tendian?\n#endif\n\nstruct io_iocb_poll {\n\tPADDED(int events, __pad1);\n};\t/* result code is the set of result flags or -'ve errno */\n\nstruct io_iocb_sockaddr {\n\tstruct sockaddr *addr;\n\tint\t\tlen;\n};\t/* result code is the length of the sockaddr, or -'ve errno */\n\nstruct io_iocb_common {\n\tPADDEDptr(void\t*buf, __pad1);\n\tPADDEDul(nbytes, __pad2);\n\tlong long\toffset;\n\tlong long\t__pad3;\n\tunsigned\tflags;\n\tunsigned\tresfd;\n};\t/* result code is the amount read or -'ve errno */\n\nstruct io_iocb_vector {\n\tconst struct iovec\t*vec;\n\tint\t\t\tnr;\n\tlong long\t\toffset;\n};\t/* result code is the amount read or -'ve errno */\n\nstruct iocb {\n\tPADDEDptr(void *data, __pad1);\t/* Return in the io completion event */\n\t/* key: For use in identifying io requests */\n\t/* aio_rw_flags: RWF_* flags (such as RWF_NOWAIT) */\n\tPADDED(unsigned key, aio_rw_flags);\n\n\tshort\t\taio_lio_opcode;\t\n\tshort\t\taio_reqprio;\n\tint\t\taio_fildes;\n\n\tunion {\n\t\tstruct io_iocb_common\t\tc;\n\t\tstruct io_iocb_vector\t\tv;\n\t\tstruct io_iocb_poll\t\tpoll;\n\t\tstruct io_iocb_sockaddr\tsaddr;\n\t} u;\n};\n\nstruct io_event {\n\tPADDEDptr(void *data, __pad1);\n\tPADDEDptr(struct iocb *obj, __pad2);\n\tPADDEDul(res, __pad3);\n\tPADDEDul(res2, __pad4);\n};\n\n#undef PADDED\n#undef PADDEDptr\n#undef PADDEDul\n\n/* Actual syscalls */\nstatic inline int io_setup(int maxevents, io_context_t *ctxp)\n{\n\treturn syscall(__NR_io_setup, maxevents, ctxp);\n}\n\nstatic inline int io_destroy(io_context_t ctx)\n{\n\treturn syscall(__NR_io_destroy, ctx);\n}\n\nstatic inline int io_submit(io_context_t ctx, long nr, struct iocb *ios[])\n{\n\treturn syscall(__NR_io_submit, ctx, nr, ios);\n}\n\nstatic inline int io_cancel(io_context_t ctx, struct iocb *iocb,\n\t\t\t\t\t\t\tstruct io_event *evt)\n{\n\treturn syscall(__NR_io_cancel, ctx, iocb, evt);\n}\n\nstatic inline int io_getevents(io_context_t ctx_id, long min_nr, long nr,\n\t\t\t\t\t\t\t struct io_event *events,\n\t\t\t\t\t\t\t struct timespec *timeout)\n{\n\treturn syscall(__NR_io_getevents, ctx_id, min_nr, nr, events, timeout);\n}\n\nstatic inline void io_set_eventfd(struct iocb *iocb, int eventfd)\n{\n\tiocb->u.c.flags |= (1 << 0) /* IOCB_FLAG_RESFD */;\n\tiocb->u.c.resfd = eventfd;\n}\n\nvoid IOSession::prep_pread(int fd, void *buf, size_t count, long long offset)\n{\n\tstruct iocb *iocb = (struct iocb *)this->iocb_buf;\n\n\tmemset(iocb, 0, sizeof(*iocb));\n\tiocb->aio_fildes = fd;\n\tiocb->aio_lio_opcode = IO_CMD_PREAD;\n\tiocb->u.c.buf = buf;\n\tiocb->u.c.nbytes = count;\n\tiocb->u.c.offset = offset;\n}\n\nvoid IOSession::prep_pwrite(int fd, void *buf, size_t count, long long offset)\n{\n\tstruct iocb *iocb = (struct iocb *)this->iocb_buf;\n\n\tmemset(iocb, 0, sizeof(*iocb));\n\tiocb->aio_fildes = fd;\n\tiocb->aio_lio_opcode = IO_CMD_PWRITE;\n\tiocb->u.c.buf = buf;\n\tiocb->u.c.nbytes = count;\n\tiocb->u.c.offset = offset;\n}\n\nvoid IOSession::prep_preadv(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t\t\tlong long offset)\n{\n\tstruct iocb *iocb = (struct iocb *)this->iocb_buf;\n\n\tmemset(iocb, 0, sizeof(*iocb));\n\tiocb->aio_fildes = fd;\n\tiocb->aio_lio_opcode = IO_CMD_PREADV;\n\tiocb->u.c.buf = (void *)iov;\n\tiocb->u.c.nbytes = iovcnt;\n\tiocb->u.c.offset = offset;\n}\n\nvoid IOSession::prep_pwritev(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t\t\t long long offset)\n{\n\tstruct iocb *iocb = (struct iocb *)this->iocb_buf;\n\n\tmemset(iocb, 0, sizeof(*iocb));\n\tiocb->aio_fildes = fd;\n\tiocb->aio_lio_opcode = IO_CMD_PWRITEV;\n\tiocb->u.c.buf = (void *)iov;\n\tiocb->u.c.nbytes = iovcnt;\n\tiocb->u.c.offset = offset;\n}\n\nvoid IOSession::prep_fsync(int fd)\n{\n\tstruct iocb *iocb = (struct iocb *)this->iocb_buf;\n\n\tmemset(iocb, 0, sizeof(*iocb));\n\tiocb->aio_fildes = fd;\n\tiocb->aio_lio_opcode = IO_CMD_FSYNC;\n}\n\nvoid IOSession::prep_fdatasync(int fd)\n{\n\tstruct iocb *iocb = (struct iocb *)this->iocb_buf;\n\n\tmemset(iocb, 0, sizeof(*iocb));\n\tiocb->aio_fildes = fd;\n\tiocb->aio_lio_opcode = IO_CMD_FDSYNC;\n}\n\nint IOService::init(int maxevents)\n{\n\tint ret;\n\n\tif (maxevents < 0)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\tthis->io_ctx = NULL;\n\tif (io_setup(maxevents, &this->io_ctx) >= 0)\n\t{\n\t\tret = pthread_mutex_init(&this->mutex, NULL);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tINIT_LIST_HEAD(&this->session_list);\n\t\t\tthis->event_fd = -1;\n\t\t\treturn 0;\n\t\t}\n\n\t\terrno = ret;\n\t\tio_destroy(this->io_ctx);\n\t}\n\n\treturn -1;\n}\n\nvoid IOService::deinit()\n{\n\tpthread_mutex_destroy(&this->mutex);\n\tio_destroy(this->io_ctx);\n}\n\ninline void IOService::incref()\n{\n\t__sync_add_and_fetch(&this->ref, 1);\n}\n\nvoid IOService::decref()\n{\n\tIOSession *session;\n\tstruct io_event event;\n\tint state, error;\n\n\tif (__sync_sub_and_fetch(&this->ref, 1) == 0)\n\t{\n\t\twhile (!list_empty(&this->session_list))\n\t\t{\n\t\t\tif (io_getevents(this->io_ctx, 1, 1, &event, NULL) > 0)\n\t\t\t{\n\t\t\t\tsession = (IOSession *)event.data;\n\t\t\t\tlist_del(&session->list);\n\t\t\t\tsession->res = event.res;\n\t\t\t\tif (session->res >= 0)\n\t\t\t\t{\n\t\t\t\t\tstate = IOS_STATE_SUCCESS;\n\t\t\t\t\terror = 0;\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\tstate = IOS_STATE_ERROR;\n\t\t\t\t\terror = -session->res;\n\t\t\t\t}\n\n\t\t\t\tsession->handle(state, error);\n\t\t\t}\n\t\t}\n\n\t\tthis->handle_unbound();\n\t}\n}\n\nint IOService::request(IOSession *session)\n{\n\tstruct iocb *iocb = (struct iocb *)session->iocb_buf;\n\tint ret = -1;\n\n\tpthread_mutex_lock(&this->mutex);\n\tif (this->event_fd < 0)\n\t\terrno = ENOENT;\n\telse if (session->prepare() >= 0)\n\t{\n\t\tio_set_eventfd(iocb, this->event_fd);\n\t\tiocb->data = session;\n\t\tif (io_submit(this->io_ctx, 1, &iocb) > 0)\n\t\t{\n\t\t\tlist_add_tail(&session->list, &this->session_list);\n\t\t\tret = 0;\n\t\t}\n\t}\n\n\tpthread_mutex_unlock(&this->mutex);\n\tif (ret < 0)\n\t\tsession->res = -errno;\n\n\treturn ret;\n}\n\nvoid *IOService::aio_finish(void *context)\n{\n\tIOService *service = (IOService *)context;\n\tIOSession *session;\n\tstruct io_event event;\n\n\tif (io_getevents(service->io_ctx, 1, 1, &event, NULL) > 0)\n\t{\n\t\tservice->incref();\n\t\tsession = (IOSession *)event.data;\n\t\tsession->res = event.res;\n\t\treturn session;\n\t}\n\n\treturn NULL;\n}\n\n" }, { "path": "src/kernel/IOService_linux.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _IOSERVICE_LINUX_H_\n#define _IOSERVICE_LINUX_H_\n\n#include \n#include \n#include \n#include \n#include \"list.h\"\n\n#define IOS_STATE_SUCCESS\t0\n#define IOS_STATE_ERROR\t\t1\n\nclass IOSession\n{\nprivate:\n\tvirtual int prepare() = 0;\n\tvirtual void handle(int state, int error) = 0;\n\nprotected:\n\t/* prepare() has to call one of the the prep_ functions. */\n\tvoid prep_pread(int fd, void *buf, size_t count, long long offset);\n\tvoid prep_pwrite(int fd, void *buf, size_t count, long long offset);\n\tvoid prep_preadv(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t long long offset);\n\tvoid prep_pwritev(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t long long offset);\n\tvoid prep_fsync(int fd);\n\tvoid prep_fdatasync(int fd);\n\nprotected:\n\tlong get_res() const { return this->res; }\n\nprivate:\n\tchar iocb_buf[64];\n\tlong res;\n\nprivate:\n\tstruct list_head list;\n\npublic:\n\tvirtual ~IOSession() { }\n\tfriend class IOService;\n\tfriend class Communicator;\n};\n\nclass IOService\n{\npublic:\n\tint init(int maxevents);\n\tvoid deinit();\n\n\tint request(IOSession *session);\n\nprivate:\n\tvirtual void handle_stop(int error) { }\n\tvirtual void handle_unbound() = 0;\n\nprivate:\n\tvirtual int create_event_fd()\n\t{\n\t\treturn eventfd(0, 0);\n\t}\n\nprivate:\n\tstruct io_context *io_ctx;\n\nprivate:\n\tvoid incref();\n\tvoid decref();\n\nprivate:\n\tint event_fd;\n\tint ref;\n\nprivate:\n\tstruct list_head session_list;\n\tpthread_mutex_t mutex;\n\nprivate:\n\tstatic void *aio_finish(void *context);\n\npublic:\n\tvirtual ~IOService() { }\n\tfriend class Communicator;\n};\n\n#endif\n\n" }, { "path": "src/kernel/IOService_thread.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"IOService_thread.h\"\n\ntypedef enum io_iocb_cmd {\n\tIO_CMD_PREAD = 0,\n\tIO_CMD_PWRITE = 1,\n\n\tIO_CMD_FSYNC = 2,\n\tIO_CMD_FDSYNC = 3,\n\n\tIO_CMD_NOOP = 6,\n\tIO_CMD_PREADV = 7,\n\tIO_CMD_PWRITEV = 8,\n} io_iocb_cmd_t;\n\nvoid IOSession::prep_pread(int fd, void *buf, size_t count, long long offset)\n{\n\tthis->fd = fd;\n\tthis->op = IO_CMD_PREAD;\n\tthis->buf = buf;\n\tthis->count = count;\n\tthis->offset = offset;\n}\n\nvoid IOSession::prep_pwrite(int fd, void *buf, size_t count, long long offset)\n{\n\tthis->fd = fd;\n\tthis->op = IO_CMD_PWRITE;\n\tthis->buf = buf;\n\tthis->count = count;\n\tthis->offset = offset;\n}\n\nvoid IOSession::prep_preadv(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t\t\tlong long offset)\n{\n\tthis->fd = fd;\n\tthis->op = IO_CMD_PREADV;\n\tthis->buf = (void *)iov;\n\tthis->count = iovcnt;\n\tthis->offset = offset;\n}\n\nvoid IOSession::prep_pwritev(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t\t\t long long offset)\n{\n\tthis->fd = fd;\n\tthis->op = IO_CMD_PWRITEV;\n\tthis->buf = (void *)iov;\n\tthis->count = iovcnt;\n\tthis->offset = offset;\n}\n\nvoid IOSession::prep_fsync(int fd)\n{\n\tthis->fd = fd;\n\tthis->op = IO_CMD_FSYNC;\n}\n\nvoid IOSession::prep_fdatasync(int fd)\n{\n\tthis->fd = fd;\n\tthis->op = IO_CMD_FDSYNC;\n}\n\nint IOService::init(int maxevents)\n{\n\tvoid *p;\n\tint ret;\n\n\tif (maxevents <= 0)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\tret = pthread_mutex_init(&this->mutex, NULL);\n\tif (ret)\n\t{\n\t\terrno = ret;\n\t\treturn -1;\n\t}\n\n\tp = dlsym(RTLD_DEFAULT, \"fdatasync\");\n\tif (p)\n\t\tthis->fdatasync = (int (*)(int))p;\n\telse\n\t\tthis->fdatasync = fsync;\n\n\tp = dlsym(RTLD_DEFAULT, \"preadv\");\n\tif (p)\n\t\tthis->preadv = (ssize_t (*)(int, const struct iovec *, int, off_t))p;\n\telse\n\t\tthis->preadv = IOService::preadv_emul;\n\n\tp = dlsym(RTLD_DEFAULT, \"pwritev\");\n\tif (p)\n\t\tthis->pwritev = (ssize_t (*)(int, const struct iovec *, int, off_t))p;\n\telse\n\t\tthis->pwritev = IOService::pwritev_emul;\n\n\tthis->maxevents = maxevents;\n\tthis->nevents = 0;\n\tINIT_LIST_HEAD(&this->session_list);\n\tthis->pipe_fd[0] = -1;\n\tthis->pipe_fd[1] = -1;\n\treturn 0;\n}\n\nvoid IOService::deinit()\n{\n\tpthread_mutex_destroy(&this->mutex);\n}\n\ninline void IOService::incref()\n{\n\t__sync_add_and_fetch(&this->ref, 1);\n}\n\nvoid IOService::decref()\n{\n\tIOSession *session;\n\tint state, error;\n\n\tif (__sync_sub_and_fetch(&this->ref, 1) == 0)\n\t{\n\t\twhile (!list_empty(&this->session_list))\n\t\t{\n\t\t\tsession = list_entry(this->session_list.next, IOSession, list);\n\t\t\tpthread_join(session->tid, NULL);\n\t\t\tlist_del(&session->list);\n\t\t\tif (session->res >= 0)\n\t\t\t{\n\t\t\t\tstate = IOS_STATE_SUCCESS;\n\t\t\t\terror = 0;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tstate = IOS_STATE_ERROR;\n\t\t\t\terror = -session->res;\n\t\t\t}\n\n\t\t\tsession->handle(state, error);\n\t\t}\n\n\t\tpthread_mutex_lock(&this->mutex);\n\t\t/* Wait for detached threads. */\n\t\tpthread_mutex_unlock(&this->mutex);\n\t\tthis->handle_unbound();\n\t}\n}\n\nint IOService::request(IOSession *session)\n{\n\tpthread_t tid;\n\tint ret = -1;\n\n\tpthread_mutex_lock(&this->mutex);\n\tif (this->pipe_fd[0] < 0)\n\t\terrno = ENOENT;\n\telse if (this->nevents >= this->maxevents)\n\t\terrno = EAGAIN;\n\telse if (session->prepare() >= 0)\n\t{\n\t\tsession->service = this;\n\t\tret = pthread_create(&tid, NULL, IOService::io_routine, session);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tsession->tid = tid;\n\t\t\tlist_add_tail(&session->list, &this->session_list);\n\t\t\tthis->nevents++;\n\t\t}\n\t\telse\n\t\t{\n\t\t\terrno = ret;\n\t\t\tret = -1;\n\t\t}\n\t}\n\n\tpthread_mutex_unlock(&this->mutex);\n\tif (ret < 0)\n\t\tsession->res = -errno;\n\n\treturn ret;\n}\n\nvoid *IOService::io_routine(void *arg)\n{\n\tIOSession *session = (IOSession *)arg;\n\tIOService *service = session->service;\n\tint fd = session->fd;\n\tssize_t ret;\n\n\tswitch (session->op)\n\t{\n\tcase IO_CMD_PREAD:\n\t\tret = pread(fd, session->buf, session->count, session->offset);\n\t\tbreak;\n\tcase IO_CMD_PWRITE:\n\t\tret = pwrite(fd, session->buf, session->count, session->offset);\n\t\tbreak;\n\tcase IO_CMD_FSYNC:\n\t\tret = fsync(fd);\n\t\tbreak;\n\tcase IO_CMD_FDSYNC:\n\t\tret = service->fdatasync(fd);\n\t\tbreak;\n\tcase IO_CMD_PREADV:\n\t\tret = service->preadv(fd, (const struct iovec *)session->buf,\n\t\t\t\t\t\t\t session->count, session->offset);\n\t\tbreak;\n\tcase IO_CMD_PWRITEV:\n\t\tret = service->pwritev(fd, (const struct iovec *)session->buf,\n\t\t\t\t\t\t\t session->count, session->offset);\n\t\tbreak;\n\tdefault:\n\t\terrno = EINVAL;\n\t\tret = -1;\n\t\tbreak;\n\t}\n\n\tif (ret < 0)\n\t\tret = -errno;\n\n\tsession->res = ret;\n\tpthread_mutex_lock(&service->mutex);\n\tif (service->pipe_fd[1] >= 0)\n\t\twrite(service->pipe_fd[1], &session, sizeof (void *));\n\n\tservice->nevents--;\n\tpthread_mutex_unlock(&service->mutex);\n\treturn NULL;\n}\n\nvoid *IOService::aio_finish(void *ptr, void *context)\n{\n\tIOService *service = (IOService *)context;\n\tIOSession *session = (IOSession *)ptr;\n\n\tservice->incref();\n\tpthread_detach(session->tid);\n\treturn session;\n}\n\nssize_t IOService::preadv_emul(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t\t\t off_t offset)\n{\n\tsize_t total = 0;\n\tssize_t n;\n\tint i;\n\n\tfor (i = 0; i < iovcnt; i++)\n\t{\n\t\tn = pread(fd, iov[i].iov_base, iov[i].iov_len, offset);\n\t\tif (n < 0)\n\t\t\treturn total == 0 ? -1 : total;\n\n\t\ttotal += n;\n\t\tif ((size_t)n < iov[i].iov_len)\n\t\t\treturn total;\n\n\t\toffset += n;\n\t}\n\n\treturn total;\n}\n\nssize_t IOService::pwritev_emul(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t\t\t\toff_t offset)\n{\n\tsize_t total = 0;\n\tssize_t n;\n\tint i;\n\n\tfor (i = 0; i < iovcnt; i++)\n\t{\n\t\tn = pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset);\n\t\tif (n < 0)\n\t\t\treturn total == 0 ? -1 : total;\n\n\t\ttotal += n;\n\t\tif ((size_t)n < iov[i].iov_len)\n\t\t\treturn total;\n\n\t\toffset += n;\n\t}\n\n\treturn total;\n}\n\n" }, { "path": "src/kernel/IOService_thread.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _IOSERVICE_THREAD_H_\n#define _IOSERVICE_THREAD_H_\n\n#include \n#include \n#include \n#include \n#include \"list.h\"\n\n#define IOS_STATE_SUCCESS\t0\n#define IOS_STATE_ERROR\t\t1\n\nclass IOSession\n{\nprivate:\n\tvirtual int prepare() = 0;\n\tvirtual void handle(int state, int error) = 0;\n\nprotected:\n\t/* prepare() has to call one of the the prep_ functions. */\n\tvoid prep_pread(int fd, void *buf, size_t count, long long offset);\n\tvoid prep_pwrite(int fd, void *buf, size_t count, long long offset);\n\tvoid prep_preadv(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t long long offset);\n\tvoid prep_pwritev(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t long long offset);\n\tvoid prep_fsync(int fd);\n\tvoid prep_fdatasync(int fd);\n\nprotected:\n\tlong get_res() const { return this->res; }\n\nprivate:\n\tint fd;\n\tint op;\n\tvoid *buf;\n\tsize_t count;\n\tlong long offset;\n\tlong res;\n\nprivate:\n\tstruct list_head list;\n\tclass IOService *service;\n\tpthread_t tid;\n\npublic:\n\tvirtual ~IOSession() { }\n\tfriend class IOService;\n\tfriend class Communicator;\n};\n\nclass IOService\n{\npublic:\n\tint init(int maxevents);\n\tvoid deinit();\n\n\tint request(IOSession *session);\n\nprivate:\n\tvirtual void handle_stop(int error) { }\n\tvirtual void handle_unbound() = 0;\n\nprivate:\n\tvirtual int create_pipe_fd(int pipe_fd[2])\n\t{\n\t\treturn pipe(pipe_fd);\n\t}\n\nprivate:\n\tint maxevents;\n\tint nevents;\n\nprivate:\n\tvoid incref();\n\tvoid decref();\n\nprivate:\n\tint pipe_fd[2];\n\tint ref;\n\nprivate:\n\tstruct list_head session_list;\n\tpthread_mutex_t mutex;\n\nprivate:\n\tstatic void *io_routine(void *arg);\n\tstatic void *aio_finish(void *ptr, void *context);\n\nprivate:\n\tint (*fdatasync)(int);\n\tssize_t (*preadv)(int, const struct iovec *, int, off_t);\n\tssize_t (*pwritev)(int, const struct iovec *, int, off_t);\n\nprivate:\n\tstatic ssize_t preadv_emul(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t\t\t off_t offset);\n\tstatic ssize_t pwritev_emul(int fd, const struct iovec *iov, int iovcnt,\n\t\t\t\t\t\t\t\toff_t offset);\n\npublic:\n\tvirtual ~IOService() { }\n\tfriend class Communicator;\n};\n\n#endif\n\n" }, { "path": "src/kernel/SleepRequest.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _SLEEPREQUEST_H_\n#define _SLEEPREQUEST_H_\n\n#include \n#include \"SubTask.h\"\n#include \"Communicator.h\"\n#include \"CommScheduler.h\"\n\nclass SleepRequest : public SubTask, public SleepSession\n{\npublic:\n\tSleepRequest(CommScheduler *scheduler)\n\t{\n\t\tthis->scheduler = scheduler;\n\t}\n\npublic:\n\tvirtual void dispatch()\n\t{\n\t\tif (this->scheduler->sleep(this) < 0)\n\t\t\tthis->handle(SS_STATE_ERROR, errno);\n\t}\n\nprotected:\n\tint cancel()\n\t{\n\t\treturn this->scheduler->unsleep(this);\n\t}\n\nprotected:\n\tint state;\n\tint error;\n\nprotected:\n\tCommScheduler *scheduler;\n\nprotected:\n\tvirtual void handle(int state, int error)\n\t{\n\t\tthis->state = state;\n\t\tthis->error = error;\n\t\tthis->subtask_done();\n\t}\n};\n\n#endif\n\n" }, { "path": "src/kernel/SubTask.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \"SubTask.h\"\n\nvoid SubTask::subtask_done()\n{\n\tSubTask *cur = this;\n\tParallelTask *parent;\n\n\twhile (1)\n\t{\n\t\tparent = cur->parent;\n\t\tcur = cur->done();\n\t\tif (cur)\n\t\t{\n\t\t\tcur->parent = parent;\n\t\t\tcur->dispatch();\n\t\t}\n\t\telse if (parent)\n\t\t{\n\t\t\tif (__sync_sub_and_fetch(&parent->nleft, 1) == 0)\n\t\t\t{\n\t\t\t\tcur = parent;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t}\n\n\t\tbreak;\n\t}\n}\n\nvoid ParallelTask::dispatch()\n{\n\tSubTask **end = this->subtasks + this->subtasks_nr;\n\tSubTask **p = this->subtasks;\n\n\tthis->nleft = this->subtasks_nr;\n\tif (this->nleft != 0)\n\t{\n\t\tdo\n\t\t{\n\t\t\t(*p)->parent = this;\n\t\t\t(*p)->dispatch();\n\t\t} while (++p != end);\n\t}\n\telse\n\t\tthis->subtask_done();\n}\n\n" }, { "path": "src/kernel/SubTask.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _SUBTASK_H_\n#define _SUBTASK_H_\n\n#include \n\nclass SubTask\n{\npublic:\n\tvirtual void dispatch() = 0;\n\nprivate:\n\tvirtual SubTask *done() = 0;\n\nprotected:\n\tvoid subtask_done();\n\npublic:\n\tvoid *get_pointer() const { return this->pointer; }\n\tvoid set_pointer(void *pointer) { this->pointer = pointer; }\n\nprivate:\n\tclass ParallelTask *parent;\n\tvoid *pointer;\n\npublic:\n\tSubTask()\n\t{\n\t\tthis->parent = NULL;\n\t\tthis->pointer = NULL;\n\t}\n\n\tvirtual ~SubTask() { }\n\tfriend class ParallelTask;\n};\n\nclass ParallelTask : public SubTask\n{\npublic:\n\tvirtual void dispatch();\n\nprotected:\n\tSubTask **subtasks;\n\tsize_t subtasks_nr;\n\nprivate:\n\tsize_t nleft;\n\npublic:\n\tParallelTask(SubTask **subtasks, size_t n)\n\t{\n\t\tthis->subtasks = subtasks;\n\t\tthis->subtasks_nr = n;\n\t}\n\n\tvirtual ~ParallelTask() { }\n\tfriend class SubTask;\n};\n\n#endif\n\n" }, { "path": "src/kernel/list.h", "content": "#ifndef _LINUX_LIST_H\n#define _LINUX_LIST_H\n\n/*\n * Circular doubly linked list implementation.\n *\n * Some of the internal functions (\"__xxx\") are useful when\n * manipulating whole lists rather than single entries, as\n * sometimes we already know the next/prev entries and we can\n * generate better code by using them directly rather than\n * using the generic single-entry routines.\n */\n\nstruct list_head {\n\tstruct list_head *next, *prev;\n};\n\n#define LIST_HEAD_INIT(name) { &(name), &(name) }\n\n#define LIST_HEAD(name) \\\n\tstruct list_head name = LIST_HEAD_INIT(name)\n\n/**\n * INIT_LIST_HEAD - Initialize a list_head structure\n * @list: list_head structure to be initialized.\n *\n * Initializes the list_head to point to itself. If it is a list header,\n * the result is an empty list.\n */\nstatic inline void INIT_LIST_HEAD(struct list_head *list)\n{\n\tlist->next = list;\n\tlist->prev = list;\n}\n\n/*\n * Insert a new entry between two known consecutive entries. \n *\n * This is only for internal list manipulation where we know\n * the prev/next entries already!\n */\nstatic inline void __list_add(struct list_head *entry,\n\t\t\t struct list_head *prev,\n\t\t\t struct list_head *next)\n{\n\tnext->prev = entry;\n\tentry->next = next;\n\tentry->prev = prev;\n\tprev->next = entry;\n}\n\n/**\n * list_add - add a new entry\n * @entry: new entry to be added\n * @head: list head to add it after\n *\n * Insert a new entry after the specified head.\n * This is good for implementing stacks.\n */\nstatic inline void list_add(struct list_head *entry, struct list_head *head)\n{\n\t__list_add(entry, head, head->next);\n}\n\n/**\n * list_add_tail - add a new entry\n * @entry: new entry to be added\n * @head: list head to add it before\n *\n * Insert a new entry before the specified head.\n * This is useful for implementing queues.\n */\nstatic inline void list_add_tail(struct list_head *entry,\n\t\t\t\t struct list_head *head)\n{\n\t__list_add(entry, head->prev, head);\n}\n\n/*\n * Delete a list entry by making the prev/next entries\n * point to each other.\n *\n * This is only for internal list manipulation where we know\n * the prev/next entries already!\n */\nstatic inline void __list_del(struct list_head *prev, struct list_head *next)\n{\n\tnext->prev = prev;\n\tprev->next = next;\n}\n\n/**\n * list_del - deletes entry from list.\n * @entry: the element to delete from the list.\n * Note: list_empty() on entry does not return true after this, the entry is\n * in an undefined state.\n */\nstatic inline void list_del(struct list_head *entry)\n{\n\t__list_del(entry->prev, entry->next);\n}\n\n/**\n * list_move - delete from one list and add as another's head\n * @entry: the entry to move\n * @head: the head that will precede our entry\n */\nstatic inline void list_move(struct list_head *entry, struct list_head *head)\n{\n\t__list_del(entry->prev, entry->next);\n\tlist_add(entry, head);\n}\n\n/**\n * list_move_tail - delete from one list and add as another's tail\n * @entry: the entry to move\n * @head: the head that will follow our entry\n */\nstatic inline void list_move_tail(struct list_head *entry,\n\t\t\t\t struct list_head *head)\n{\n\t__list_del(entry->prev, entry->next);\n\tlist_add_tail(entry, head);\n}\n\n/**\n * list_empty - tests whether a list is empty\n * @head: the list to test.\n */\nstatic inline int list_empty(const struct list_head *head)\n{\n\treturn head->next == head;\n}\n\nstatic inline void __list_splice(const struct list_head *list,\n\t\t\t\t struct list_head *prev,\n\t\t\t\t struct list_head *next)\n{\n\tstruct list_head *first = list->next;\n\tstruct list_head *last = list->prev;\n\n\tfirst->prev = prev;\n\tprev->next = first;\n\n\tlast->next = next;\n\tnext->prev = last;\n}\n\n/**\n * list_splice - join two lists\n * @list: the new list to add.\n * @head: the place to add it in the first list.\n */\nstatic inline void list_splice(const struct list_head *list,\n\t\t\t struct list_head *head)\n{\n\tif (!list_empty(list))\n\t\t__list_splice(list, head, head->next);\n}\n\n/**\n * list_splice_init - join two lists and reinitialise the emptied list.\n * @list: the new list to add.\n * @head: the place to add it in the first list.\n *\n * The list at @list is reinitialised\n */\nstatic inline void list_splice_init(struct list_head *list,\n\t\t\t\t struct list_head *head)\n{\n\tif (!list_empty(list)) {\n\t\t__list_splice(list, head, head->next);\n\t\tINIT_LIST_HEAD(list);\n\t}\n}\n\n/**\n * list_entry - get the struct for this entry\n * @ptr:\tthe &struct list_head pointer.\n * @type:\tthe type of the struct this is embedded in.\n * @member:\tthe name of the list_struct within the struct.\n */\n#define list_entry(ptr, type, member) \\\n\t((type *)((char *)(ptr)-(unsigned long)(&((type *)0)->member)))\n\n/**\n * list_for_each - iterate over a list\n * @pos:\tthe &struct list_head to use as a loop counter.\n * @head:\tthe head for your list.\n */\n#define list_for_each(pos, head) \\\n\tfor (pos = (head)->next; pos != (head); pos = pos->next)\n\n/**\n * list_for_each_prev - iterate over a list backwards\n * @pos:\tthe &struct list_head to use as a loop counter.\n * @head:\tthe head for your list.\n */\n#define list_for_each_prev(pos, head) \\\n\tfor (pos = (head)->prev; pos != (head); pos = pos->prev)\n\n/**\n * list_for_each_safe - iterate over a list safe against removal of list entry\n * @pos:\tthe &struct list_head to use as a loop counter.\n * @n:\t\tanother &struct list_head to use as temporary storage\n * @head:\tthe head for your list.\n */\n#define list_for_each_safe(pos, n, head) \\\n\tfor (pos = (head)->next, n = pos->next; pos != (head); \\\n\t pos = n, n = pos->next)\n\n/**\n * list_for_each_entry - iterate over list of given type\n * @pos:\tthe type * to use as a loop counter.\n * @head:\tthe head for your list.\n * @member:\tthe name of the list_struct within the struct.\n */\n#define list_for_each_entry(pos, head, member) \\\n\tfor (pos = list_entry((head)->next, typeof (*pos), member); \\\n\t &pos->member != (head); \\\n\t pos = list_entry(pos->member.next, typeof (*pos), member))\n\n/*\n * Singly linked list implementation.\n */\n\nstruct slist_node {\n\tstruct slist_node *next;\n};\n\nstruct slist_head {\n\tstruct slist_node first, *last;\n};\n\n#define SLIST_HEAD_INIT(name) { { (struct slist_node *)0 }, &(name).first }\n\n#define SLIST_HEAD(name) \\\n\tstruct slist_head name = SLIST_HEAD_INIT(name)\n\nstatic inline void INIT_SLIST_HEAD(struct slist_head *list)\n{\n\tlist->first.next = (struct slist_node *)0;\n\tlist->last = &list->first;\n}\n\nstatic inline void slist_add_after(struct slist_node *entry,\n\t\t\t\t struct slist_node *prev,\n\t\t\t\t struct slist_head *list)\n{\n\tentry->next = prev->next;\n\tprev->next = entry;\n\tif (!entry->next)\n\t\tlist->last = entry;\n}\n\nstatic inline void slist_add_head(struct slist_node *entry,\n\t\t\t\t struct slist_head *list)\n{\n\tslist_add_after(entry, &list->first, list);\n}\n\nstatic inline void slist_add_tail(struct slist_node *entry,\n\t\t\t\t struct slist_head *list)\n{\n\tentry->next = (struct slist_node *)0;\n\tlist->last->next = entry;\n\tlist->last = entry;\n}\n\nstatic inline void slist_del_after(struct slist_node *prev,\n\t\t\t\t struct slist_head *list)\n{\n\tprev->next = prev->next->next;\n\tif (!prev->next)\n\t\tlist->last = prev;\n}\n\nstatic inline void slist_del_head(struct slist_head *list)\n{\n\tslist_del_after(&list->first, list);\n}\n\nstatic inline int slist_empty(const struct slist_head *list)\n{\n\treturn !list->first.next;\n}\n\nstatic inline void __slist_splice(const struct slist_head *list,\n\t\t\t\t struct slist_node *prev,\n\t\t\t\t struct slist_head *head)\n{\n\tlist->last->next = prev->next;\n\tprev->next = list->first.next;\n\tif (!list->last->next)\n\t\thead->last = list->last;\n}\n\nstatic inline void slist_splice(const struct slist_head *list,\n\t\t\t\tstruct slist_node *prev,\n\t\t\t\tstruct slist_head *head)\n{\n\tif (!slist_empty(list))\n\t\t__slist_splice(list, prev, head);\n}\n\nstatic inline void slist_splice_init(struct slist_head *list,\n\t\t\t\t struct slist_node *prev,\n\t\t\t\t struct slist_head *head)\n{\n\tif (!slist_empty(list)) {\n\t\t__slist_splice(list, prev, head);\n\t\tINIT_SLIST_HEAD(list);\n\t}\n}\n\n#define slist_entry(ptr, type, member) \\\n\t((type *)((char *)(ptr)-(unsigned long)(&((type *)0)->member)))\n\n#define slist_for_each(pos, head) \\\n\tfor (pos = (head)->first.next; pos; pos = pos->next)\n\n#define slist_for_each_safe(pos, prev, head) \\\n\tfor (prev = &(head)->first, pos = prev->next; pos; \\\n\t prev = prev->next == pos ? pos : prev, pos = prev->next)\n\n#define slist_for_each_entry(pos, head, member) \\\n\tfor (pos = slist_entry((head)->first.next, typeof (*pos), member); \\\n\t &pos->member != (struct slist_node *)0; \\\n\t pos = slist_entry(pos->member.next, typeof (*pos), member))\n\n#endif\n" }, { "path": "src/kernel/mpoller.c", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"poller.h\"\n#include \"mpoller.h\"\n\nextern poller_t *__poller_create(void **, const struct poller_params *);\nextern void __poller_destroy(poller_t *);\n\nstatic int __mpoller_create(const struct poller_params *params,\n\t\t\t\t\t\t\tmpoller_t *mpoller)\n{\n\tvoid **nodes_buf = (void **)calloc(params->max_open_files, sizeof (void *));\n\tunsigned int i;\n\n\tif (nodes_buf)\n\t{\n\t\tfor (i = 0; i < mpoller->nthreads; i++)\n\t\t{\n\t\t\tmpoller->poller[i] = __poller_create(nodes_buf, params);\n\t\t\tif (!mpoller->poller[i])\n\t\t\t\tbreak;\n\t\t}\n\n\t\tif (i == mpoller->nthreads)\n\t\t{\n\t\t\tmpoller->nodes_buf = nodes_buf;\n\t\t\treturn 0;\n\t\t}\n\n\t\twhile (i > 0)\n\t\t\t__poller_destroy(mpoller->poller[--i]);\n\n\t\tfree(nodes_buf);\n\t}\n\n\treturn -1;\n}\n\nmpoller_t *mpoller_create(const struct poller_params *params, size_t nthreads)\n{\n\tmpoller_t *mpoller;\n\tsize_t size;\n\n\tif (nthreads == 0)\n\t\tnthreads = 1;\n\n\tsize = offsetof(mpoller_t, poller) + nthreads * sizeof (void *);\n\tmpoller = (mpoller_t *)malloc(size);\n\tif (mpoller)\n\t{\n\t\tmpoller->nthreads = (unsigned int)nthreads;\n\t\tif (__mpoller_create(params, mpoller) >= 0)\n\t\t\treturn mpoller;\n\n\t\tfree(mpoller);\n\t}\n\n\treturn NULL;\n}\n\nint mpoller_start(mpoller_t *mpoller)\n{\n\tunsigned int i;\n\n\tfor (i = 0; i < mpoller->nthreads; i++)\n\t{\n\t\tif (poller_start(mpoller->poller[i]) < 0)\n\t\t\tbreak;\n\t}\n\n\tif (i == mpoller->nthreads)\n\t\treturn 0;\n\n\twhile (i > 0)\n\t\tpoller_stop(mpoller->poller[--i]);\n\n\treturn -1;\n}\n\nvoid mpoller_set_callback(void (*callback)(struct poller_result *, void *),\n\t\t\t\t\t\t mpoller_t *mpoller)\n{\n\tunsigned int i;\n\n\tfor (i = 0; i < mpoller->nthreads; i++)\n\t\tpoller_set_callback(callback, mpoller->poller[i]);\n}\n\nvoid mpoller_stop(mpoller_t *mpoller)\n{\n\tunsigned int i;\n\n\tfor (i = 0; i < mpoller->nthreads; i++)\n\t\tpoller_stop(mpoller->poller[i]);\n}\n\nvoid mpoller_destroy(mpoller_t *mpoller)\n{\n\tunsigned int i;\n\n\tfor (i = 0; i < mpoller->nthreads; i++)\n\t\t__poller_destroy(mpoller->poller[i]);\n\n\tfree(mpoller->nodes_buf);\n\tfree(mpoller);\n}\n\n" }, { "path": "src/kernel/mpoller.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _MPOLLER_H_\n#define _MPOLLER_H_\n\n#include \n#include \"poller.h\"\n\ntypedef struct __mpoller mpoller_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nmpoller_t *mpoller_create(const struct poller_params *params, size_t nthreads);\nint mpoller_start(mpoller_t *mpoller);\nvoid mpoller_set_callback(void (*callback)(struct poller_result *, void *),\n\t\t\t\t\t\t mpoller_t *mpoller);\nvoid mpoller_stop(mpoller_t *mpoller);\nvoid mpoller_destroy(mpoller_t *mpoller);\n\n#ifdef __cplusplus\n}\n#endif\n\nstruct __mpoller\n{\n\tvoid **nodes_buf;\n\tunsigned int nthreads;\n\tpoller_t *poller[1];\n};\n\nstatic inline int mpoller_add(const struct poller_data *data, int timeout,\n\t\t\t\t\t\t\t mpoller_t *mpoller)\n{\n\tint index = (unsigned int)data->fd % mpoller->nthreads;\n\treturn poller_add(data, timeout, mpoller->poller[index]);\n}\n\nstatic inline int mpoller_del(int fd, mpoller_t *mpoller)\n{\n\tint index = (unsigned int)fd % mpoller->nthreads;\n\treturn poller_del(fd, mpoller->poller[index]);\n}\n\nstatic inline int mpoller_mod(const struct poller_data *data, int timeout,\n\t\t\t\t\t\t\t mpoller_t *mpoller)\n{\n\tint index = (unsigned int)data->fd % mpoller->nthreads;\n\treturn poller_mod(data, timeout, mpoller->poller[index]);\n}\n\nstatic inline int mpoller_set_timeout(int fd, int timeout, mpoller_t *mpoller)\n{\n\tint index = (unsigned int)fd % mpoller->nthreads;\n\treturn poller_set_timeout(fd, timeout, mpoller->poller[index]);\n}\n\nstatic inline int mpoller_add_timer(const struct timespec *value, void *context,\n\t\t\t\t\t\t\t\t\tvoid **timer, int *index,\n\t\t\t\t\t\t\t\t\tmpoller_t *mpoller)\n{\n\tstatic unsigned int n = 0;\n\t*index = n++ % mpoller->nthreads;\n\treturn poller_add_timer(value, context, timer, mpoller->poller[*index]);\n}\n\nstatic inline int mpoller_del_timer(void *timer, int index, mpoller_t *mpoller)\n{\n\treturn poller_del_timer(timer, mpoller->poller[index]);\n}\n\n#endif\n\n" }, { "path": "src/kernel/msgqueue.c", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"msgqueue.h\"\n\nstruct __msgqueue\n{\n\tsize_t msg_max;\n\tsize_t msg_cnt;\n\tint linkoff;\n\tint nonblock;\n\tvoid *head1;\n\tvoid *head2;\n\tvoid **get_head;\n\tvoid **put_head;\n\tvoid **put_tail;\n\tpthread_mutex_t get_mutex;\n\tpthread_mutex_t put_mutex;\n\tpthread_cond_t get_cond;\n\tpthread_cond_t put_cond;\n};\n\nvoid msgqueue_set_nonblock(msgqueue_t *queue)\n{\n\tqueue->nonblock = 1;\n\tpthread_mutex_lock(&queue->put_mutex);\n\tpthread_cond_signal(&queue->get_cond);\n\tpthread_cond_broadcast(&queue->put_cond);\n\tpthread_mutex_unlock(&queue->put_mutex);\n}\n\nvoid msgqueue_set_block(msgqueue_t *queue)\n{\n\tqueue->nonblock = 0;\n}\n\nvoid msgqueue_put(void *msg, msgqueue_t *queue)\n{\n\tvoid **link = (void **)((char *)msg + queue->linkoff);\n\n\t*link = NULL;\n\tpthread_mutex_lock(&queue->put_mutex);\n\twhile (queue->msg_cnt > queue->msg_max - 1 && !queue->nonblock)\n\t\tpthread_cond_wait(&queue->put_cond, &queue->put_mutex);\n\n\t*queue->put_tail = link;\n\tqueue->put_tail = link;\n\tqueue->msg_cnt++;\n\tpthread_mutex_unlock(&queue->put_mutex);\n\tpthread_cond_signal(&queue->get_cond);\n}\n\nvoid msgqueue_put_head(void *msg, msgqueue_t *queue)\n{\n\tvoid **link = (void **)((char *)msg + queue->linkoff);\n\n\tpthread_mutex_lock(&queue->put_mutex);\n\twhile (*queue->get_head)\n\t{\n\t\tif (pthread_mutex_trylock(&queue->get_mutex) == 0)\n\t\t{\n\t\t\tpthread_mutex_unlock(&queue->put_mutex);\n\t\t\t*link = *queue->get_head;\n\t\t\t*queue->get_head = link;\n\t\t\tpthread_mutex_unlock(&queue->get_mutex);\n\t\t\treturn;\n\t\t}\n\t}\n\n\twhile (queue->msg_cnt > queue->msg_max - 1 && !queue->nonblock)\n\t\tpthread_cond_wait(&queue->put_cond, &queue->put_mutex);\n\n\t*link = *queue->put_head;\n\tif (*link == NULL)\n\t\tqueue->put_tail = link;\n\n\t*queue->put_head = link;\n\tqueue->msg_cnt++;\n\tpthread_mutex_unlock(&queue->put_mutex);\n\tpthread_cond_signal(&queue->get_cond);\n}\n\nstatic size_t __msgqueue_swap(msgqueue_t *queue)\n{\n\tvoid **get_head = queue->get_head;\n\tsize_t cnt;\n\n\tpthread_mutex_lock(&queue->put_mutex);\n\twhile (queue->msg_cnt == 0 && !queue->nonblock)\n\t\tpthread_cond_wait(&queue->get_cond, &queue->put_mutex);\n\n\tcnt = queue->msg_cnt;\n\tif (cnt > queue->msg_max - 1)\n\t\tpthread_cond_broadcast(&queue->put_cond);\n\n\tqueue->get_head = queue->put_head;\n\tqueue->put_head = get_head;\n\tqueue->put_tail = get_head;\n\tqueue->msg_cnt = 0;\n\tpthread_mutex_unlock(&queue->put_mutex);\n\treturn cnt;\n}\n\nvoid *msgqueue_get(msgqueue_t *queue)\n{\n\tvoid *msg;\n\n\tpthread_mutex_lock(&queue->get_mutex);\n\tif (*queue->get_head || __msgqueue_swap(queue) > 0)\n\t{\n\t\tmsg = (char *)*queue->get_head - queue->linkoff;\n\t\t*queue->get_head = *(void **)*queue->get_head;\n\t}\n\telse\n\t\tmsg = NULL;\n\n\tpthread_mutex_unlock(&queue->get_mutex);\n\treturn msg;\n}\n\nmsgqueue_t *msgqueue_create(size_t maxlen, int linkoff)\n{\n\tmsgqueue_t *queue = (msgqueue_t *)malloc(sizeof (msgqueue_t));\n\tint ret;\n\n\tif (!queue)\n\t\treturn NULL;\n\n\tret = pthread_mutex_init(&queue->get_mutex, NULL);\n\tif (ret == 0)\n\t{\n\t\tret = pthread_mutex_init(&queue->put_mutex, NULL);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tret = pthread_cond_init(&queue->get_cond, NULL);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tret = pthread_cond_init(&queue->put_cond, NULL);\n\t\t\t\tif (ret == 0)\n\t\t\t\t{\n\t\t\t\t\tqueue->msg_max = maxlen;\n\t\t\t\t\tqueue->linkoff = linkoff;\n\t\t\t\t\tqueue->head1 = NULL;\n\t\t\t\t\tqueue->head2 = NULL;\n\t\t\t\t\tqueue->get_head = &queue->head1;\n\t\t\t\t\tqueue->put_head = &queue->head2;\n\t\t\t\t\tqueue->put_tail = &queue->head2;\n\t\t\t\t\tqueue->msg_cnt = 0;\n\t\t\t\t\tqueue->nonblock = 0;\n\t\t\t\t\treturn queue;\n\t\t\t\t}\n\n\t\t\t\tpthread_cond_destroy(&queue->get_cond);\n\t\t\t}\n\n\t\t\tpthread_mutex_destroy(&queue->put_mutex);\n\t\t}\n\n\t\tpthread_mutex_destroy(&queue->get_mutex);\n\t}\n\n\terrno = ret;\n\tfree(queue);\n\treturn NULL;\n}\n\nvoid msgqueue_destroy(msgqueue_t *queue)\n{\n\tpthread_cond_destroy(&queue->put_cond);\n\tpthread_cond_destroy(&queue->get_cond);\n\tpthread_mutex_destroy(&queue->put_mutex);\n\tpthread_mutex_destroy(&queue->get_mutex);\n\tfree(queue);\n}\n\n" }, { "path": "src/kernel/msgqueue.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _MSGQUEUE_H_\n#define _MSGQUEUE_H_\n\n#include \n\ntypedef struct __msgqueue msgqueue_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\n/* A simple implementation of message queue. The max pending messages may\n * reach two times 'maxlen' when the queue is in blocking mode, and infinite\n * in nonblocking mode. 'linkoff' is the offset from the head of each message,\n * where spaces of one pointer size should be available for internal usage.\n * 'linkoff' can be positive or negative or zero. */\n\nmsgqueue_t *msgqueue_create(size_t maxlen, int linkoff);\nvoid *msgqueue_get(msgqueue_t *queue);\nvoid msgqueue_put(void *msg, msgqueue_t *queue);\nvoid msgqueue_put_head(void *msg, msgqueue_t *queue);\nvoid msgqueue_set_nonblock(msgqueue_t *queue);\nvoid msgqueue_set_block(msgqueue_t *queue);\nvoid msgqueue_destroy(msgqueue_t *queue);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n\n" }, { "path": "src/kernel/poller.c", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#ifdef __linux__\n# include \n# include \n#else\n# include \n# undef LIST_HEAD\n# undef SLIST_HEAD\n#endif\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"rbtree.h\"\n#include \"poller.h\"\n\n#define POLLER_BUFSIZE\t\t\t(256 * 1024)\n#define POLLER_EVENTS_MAX\t\t256\n\nstruct __poller_node\n{\n\tint state;\n\tint error;\n\tstruct poller_data data;\n#pragma pack(1)\n\tunion\n\t{\n\t\tstruct list_head list;\n\t\tstruct rb_node rb;\n\t};\n#pragma pack()\n\tchar in_rbtree;\n\tchar removed;\n\tint event;\n\tstruct timespec timeout;\n\tstruct __poller_node *res;\n};\n\nstruct __poller\n{\n\tsize_t max_open_files;\n\tvoid (*callback)(struct poller_result *, void *);\n\tvoid *context;\n\n\tpthread_t tid;\n\tint pfd;\n\tint timerfd;\n\tint pipe_rd;\n\tint pipe_wr;\n\tint stopped;\n\tstruct rb_root timeo_tree;\n\tstruct rb_node *tree_first;\n\tstruct rb_node *tree_last;\n\tstruct list_head timeo_list;\n\tstruct list_head no_timeo_list;\n\tstruct __poller_node **nodes;\n\tpthread_mutex_t mutex;\n\tchar buf[POLLER_BUFSIZE];\n};\n\n#ifdef __linux__\n\nstatic inline int __poller_create_pfd()\n{\n\treturn epoll_create(1);\n}\n\nstatic inline int __poller_close_pfd(int fd)\n{\n\treturn close(fd);\n}\n\nstatic inline int __poller_add_fd(int fd, int event, void *data,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct epoll_event ev = {\n\t\t.events\t\t=\tevent,\n\t\t.data\t\t=\t{\n\t\t\t.ptr\t=\tdata\n\t\t}\n\t};\n\treturn epoll_ctl(poller->pfd, EPOLL_CTL_ADD, fd, &ev);\n}\n\nstatic inline int __poller_del_fd(int fd, int event, poller_t *poller)\n{\n\treturn epoll_ctl(poller->pfd, EPOLL_CTL_DEL, fd, NULL);\n}\n\nstatic inline int __poller_mod_fd(int fd, int old_event,\n\t\t\t\t\t\t\t\t int new_event, void *data,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct epoll_event ev = {\n\t\t.events\t\t=\tnew_event,\n\t\t.data\t\t=\t{\n\t\t\t.ptr\t=\tdata\n\t\t}\n\t};\n\treturn epoll_ctl(poller->pfd, EPOLL_CTL_MOD, fd, &ev);\n}\n\nstatic inline int __poller_create_timerfd()\n{\n\treturn timerfd_create(CLOCK_MONOTONIC, 0);\n}\n\nstatic inline int __poller_close_timerfd(int fd)\n{\n\treturn close(fd);\n}\n\nstatic inline int __poller_add_timerfd(int fd, poller_t *poller)\n{\n\tstatic struct poller_result node = {\n\t\t.data\t\t=\t{\n\t\t\t.operation\t=\tPD_OP_TIMER\n\t\t}\n\t};\n\treturn __poller_add_fd(fd, EPOLLIN | EPOLLET, &node, poller);\n}\n\nstatic inline int __poller_set_timerfd(int fd, const struct timespec *abstime,\n\t\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct itimerspec timer = {\n\t\t.it_interval\t=\t{ },\n\t\t.it_value\t\t=\t*abstime\n\t};\n\treturn timerfd_settime(fd, TFD_TIMER_ABSTIME, &timer, NULL);\n}\n\ntypedef struct epoll_event __poller_event_t;\n\nstatic inline int __poller_wait(__poller_event_t *events, int maxevents,\n\t\t\t\t\t\t\t\tpoller_t *poller)\n{\n\treturn epoll_wait(poller->pfd, events, maxevents, -1);\n}\n\nstatic inline void *__poller_event_data(const __poller_event_t *event)\n{\n\treturn event->data.ptr;\n}\n\n#else /* BSD, macOS */\n\nstatic inline int __poller_create_pfd()\n{\n\treturn kqueue();\n}\n\nstatic inline int __poller_close_pfd(int fd)\n{\n\treturn close(fd);\n}\n\nstatic inline int __poller_add_fd(int fd, int event, void *data,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct kevent ev;\n\tEV_SET(&ev, fd, event, EV_ADD, 0, 0, data);\n\treturn kevent(poller->pfd, &ev, 1, NULL, 0, NULL);\n}\n\nstatic inline int __poller_del_fd(int fd, int event, poller_t *poller)\n{\n\tstruct kevent ev;\n\tEV_SET(&ev, fd, event, EV_DELETE, 0, 0, NULL);\n\treturn kevent(poller->pfd, &ev, 1, NULL, 0, NULL);\n}\n\nstatic inline int __poller_mod_fd(int fd, int old_event,\n\t\t\t\t\t\t\t\t int new_event, void *data,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct kevent ev[2];\n\tEV_SET(&ev[0], fd, old_event, EV_DELETE, 0, 0, NULL);\n\tEV_SET(&ev[1], fd, new_event, EV_ADD, 0, 0, data);\n\treturn kevent(poller->pfd, ev, 2, NULL, 0, NULL);\n}\n\nstatic inline int __poller_create_timerfd()\n{\n\treturn 0;\n}\n\nstatic inline int __poller_close_timerfd(int fd)\n{\n\treturn 0;\n}\n\nstatic inline int __poller_add_timerfd(int fd, poller_t *poller)\n{\n\treturn 0;\n}\n\nstatic int __poller_set_timerfd(int fd, const struct timespec *abstime,\n\t\t\t\t\t\t\t\tpoller_t *poller)\n{\n\tstatic struct poller_result node = {\n\t\t.data\t\t=\t{\n\t\t\t.operation\t=\tPD_OP_TIMER\n\t\t}\n\t};\n\tstruct timespec curtime;\n\tlong long nseconds;\n\tstruct kevent ev;\n\tint flags;\n\n\tif (abstime->tv_sec || abstime->tv_nsec)\n\t{\n\t\tflags = EV_ADD | EV_ONESHOT;\n\t\tclock_gettime(CLOCK_MONOTONIC, &curtime);\n\t\tnseconds = 1000000000LL * (abstime->tv_sec - curtime.tv_sec);\n\t\tnseconds += abstime->tv_nsec - curtime.tv_nsec;\n\t\tif (nseconds < 0)\n\t\t\tnseconds = 0;\n\t}\n\telse\n\t{\n\t\tflags = EV_DELETE;\n\t\tnseconds = 0;\n\t}\n\n\tEV_SET(&ev, fd, EVFILT_TIMER, flags, NOTE_NSECONDS, nseconds, &node);\n\treturn kevent(poller->pfd, &ev, 1, NULL, 0, NULL);\n}\n\ntypedef struct kevent __poller_event_t;\n\nstatic inline int __poller_wait(__poller_event_t *events, int maxevents,\n\t\t\t\t\t\t\t\tpoller_t *poller)\n{\n\treturn kevent(poller->pfd, NULL, 0, events, maxevents, NULL);\n}\n\nstatic inline void *__poller_event_data(const __poller_event_t *event)\n{\n\treturn event->udata;\n}\n\n#define EPOLLIN\t\tEVFILT_READ\n#define EPOLLOUT\tEVFILT_WRITE\n#define EPOLLET\t\t0\n\n#endif\n\nstatic inline long __timeout_cmp(const struct __poller_node *node1,\n\t\t\t\t\t\t\t\t const struct __poller_node *node2)\n{\n\tlong ret = node1->timeout.tv_sec - node2->timeout.tv_sec;\n\n\tif (ret == 0)\n\t\tret = node1->timeout.tv_nsec - node2->timeout.tv_nsec;\n\n\treturn ret;\n}\n\nstatic void __poller_tree_insert(struct __poller_node *node, poller_t *poller)\n{\n\tstruct rb_node **p = &poller->timeo_tree.rb_node;\n\tstruct rb_node *parent = NULL;\n\tstruct __poller_node *entry;\n\n\tentry = rb_entry(poller->tree_last, struct __poller_node, rb);\n\tif (!*p)\n\t{\n\t\tpoller->tree_first = &node->rb;\n\t\tpoller->tree_last = &node->rb;\n\t}\n\telse if (__timeout_cmp(node, entry) >= 0)\n\t{\n\t\tparent = poller->tree_last;\n\t\tp = &parent->rb_right;\n\t\tpoller->tree_last = &node->rb;\n\t}\n\telse\n\t{\n\t\tdo\n\t\t{\n\t\t\tparent = *p;\n\t\t\tentry = rb_entry(*p, struct __poller_node, rb);\n\t\t\tif (__timeout_cmp(node, entry) < 0)\n\t\t\t\tp = &(*p)->rb_left;\n\t\t\telse\n\t\t\t\tp = &(*p)->rb_right;\n\t\t} while (*p);\n\n\t\tif (p == &poller->tree_first->rb_left)\n\t\t\tpoller->tree_first = &node->rb;\n\t}\n\n\tnode->in_rbtree = 1;\n\trb_link_node(&node->rb, parent, p);\n\trb_insert_color(&node->rb, &poller->timeo_tree);\n}\n\nstatic inline void __poller_tree_erase(struct __poller_node *node,\n\t\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tif (&node->rb == poller->tree_first)\n\t\tpoller->tree_first = rb_next(&node->rb);\n\n\tif (&node->rb == poller->tree_last)\n\t\tpoller->tree_last = rb_prev(&node->rb);\n\n\trb_erase(&node->rb, &poller->timeo_tree);\n\tnode->in_rbtree = 0;\n}\n\nstatic int __poller_remove_node(struct __poller_node *node, poller_t *poller)\n{\n\tint removed;\n\n\tpthread_mutex_lock(&poller->mutex);\n\tremoved = node->removed;\n\tif (!removed)\n\t{\n\t\tpoller->nodes[node->data.fd] = NULL;\n\n\t\tif (node->in_rbtree)\n\t\t\t__poller_tree_erase(node, poller);\n\t\telse\n\t\t\tlist_del(&node->list);\n\n\t\t__poller_del_fd(node->data.fd, node->event, poller);\n\t}\n\n\tpthread_mutex_unlock(&poller->mutex);\n\treturn removed;\n}\n\nstatic int __poller_append_message(const void *buf, size_t *n,\n\t\t\t\t\t\t\t\t struct __poller_node *node,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tpoller_message_t *msg = node->data.message;\n\tstruct __poller_node *res;\n\tint ret;\n\n\tif (!msg)\n\t{\n\t\tres = (struct __poller_node *)malloc(sizeof (struct __poller_node));\n\t\tif (!res)\n\t\t\treturn -1;\n\n\t\tmsg = node->data.create_message(node->data.context);\n\t\tif (!msg)\n\t\t{\n\t\t\tfree(res);\n\t\t\treturn -1;\n\t\t}\n\n\t\tnode->data.message = msg;\n\t\tnode->res = res;\n\t}\n\telse\n\t\tres = node->res;\n\n\tret = msg->append(buf, n, msg);\n\tif (ret > 0)\n\t{\n\t\tres->data = node->data;\n\t\tres->error = 0;\n\t\tres->state = PR_ST_SUCCESS;\n\t\tpoller->callback((struct poller_result *)res, poller->context);\n\n\t\tnode->data.message = NULL;\n\t\tnode->res = NULL;\n\t}\n\n\treturn ret;\n}\n\nstatic int __poller_handle_ssl_error(struct __poller_node *node, int ret,\n\t\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tint error = SSL_get_error(node->data.ssl, ret);\n\tint event;\n\n\tswitch (error)\n\t{\n\tcase SSL_ERROR_WANT_READ:\n\t\tevent = EPOLLIN | EPOLLET;\n\t\tbreak;\n\tcase SSL_ERROR_WANT_WRITE:\n\t\tevent = EPOLLOUT | EPOLLET;\n\t\tbreak;\n\tdefault:\n\t\terrno = -error;\n\tcase SSL_ERROR_SYSCALL:\n\t\treturn -1;\n\t}\n\n\tif (event == node->event)\n\t\treturn 0;\n\n\tpthread_mutex_lock(&poller->mutex);\n\tif (!node->removed)\n\t{\n\t\tret = __poller_mod_fd(node->data.fd, node->event, event, node, poller);\n\t\tif (ret >= 0)\n\t\t\tnode->event = event;\n\t}\n\telse\n\t\tret = 0;\n\n\tpthread_mutex_unlock(&poller->mutex);\n\treturn ret;\n}\n\nstatic void __poller_handle_read(struct __poller_node *node,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tssize_t nleft;\n\tsize_t n;\n\tchar *p;\n\n\twhile (1)\n\t{\n\t\tp = poller->buf;\n\t\tif (!node->data.ssl)\n\t\t{\n\t\t\tnleft = read(node->data.fd, p, POLLER_BUFSIZE);\n\t\t\tif (nleft < 0)\n\t\t\t{\n\t\t\t\tif (errno == EAGAIN)\n\t\t\t\t\treturn;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tnleft = SSL_read(node->data.ssl, p, POLLER_BUFSIZE);\n\t\t\tif (nleft <= 0)\n\t\t\t{\n\t\t\t\tif (__poller_handle_ssl_error(node, nleft, poller) >= 0)\n\t\t\t\t\treturn;\n\n\t\t\t\tif (errno == -SSL_ERROR_ZERO_RETURN)\n\t\t\t\t\tnleft = 0;\n\t\t\t\telse\n\t\t\t\t\tnleft = -1;\n\t\t\t}\n\t\t}\n\n\t\tif (nleft <= 0)\n\t\t\tbreak;\n\n\t\tdo\n\t\t{\n\t\t\tn = nleft;\n\t\t\tif (__poller_append_message(p, &n, node, poller) >= 0)\n\t\t\t{\n\t\t\t\tnleft -= n;\n\t\t\t\tp += n;\n\t\t\t}\n\t\t\telse\n\t\t\t\tnleft = -1;\n\t\t} while (nleft > 0);\n\n\t\tif (nleft < 0)\n\t\t\tbreak;\n\n\t\tif (node->removed)\n\t\t\treturn;\n\t}\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tif (nleft == 0)\n\t{\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_FINISHED;\n\t}\n\telse\n\t{\n\t\tnode->error = errno;\n\t\tnode->state = PR_ST_ERROR;\n\t}\n\n\tfree(node->res);\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\n#ifndef IOV_MAX\n# define IOV_MAX\t16\n#endif\n\nstatic void __poller_handle_write(struct __poller_node *node,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct iovec *iov = node->data.write_iov;\n\tsize_t count = 0;\n\tssize_t nleft;\n\tint iovcnt;\n\tint ret;\n\n\twhile (node->data.iovcnt > 0)\n\t{\n\t\tif (!node->data.ssl)\n\t\t{\n\t\t\tiovcnt = node->data.iovcnt;\n\t\t\tif (iovcnt > IOV_MAX)\n\t\t\t\tiovcnt = IOV_MAX;\n\n\t\t\tnleft = writev(node->data.fd, iov, iovcnt);\n\t\t\tif (nleft < 0)\n\t\t\t{\n\t\t\t\tret = errno == EAGAIN ? 0 : -1;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\telse if (iov->iov_len > 0)\n\t\t{\n\t\t\tnleft = SSL_write(node->data.ssl, iov->iov_base, iov->iov_len);\n\t\t\tif (nleft <= 0)\n\t\t\t{\n\t\t\t\tret = __poller_handle_ssl_error(node, nleft, poller);\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t\tnleft = 0;\n\n\t\tcount += nleft;\n\t\tdo\n\t\t{\n\t\t\tif (nleft >= iov->iov_len)\n\t\t\t{\n\t\t\t\tnleft -= iov->iov_len;\n\t\t\t\tiov->iov_base = (char *)iov->iov_base + iov->iov_len;\n\t\t\t\tiov->iov_len = 0;\n\t\t\t\tiov++;\n\t\t\t\tnode->data.iovcnt--;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tiov->iov_base = (char *)iov->iov_base + nleft;\n\t\t\t\tiov->iov_len -= nleft;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t} while (node->data.iovcnt > 0);\n\t}\n\n\tnode->data.write_iov = iov;\n\tif (node->data.iovcnt > 0 && ret >= 0)\n\t{\n\t\tif (count == 0)\n\t\t\treturn;\n\n\t\tif (node->data.partial_written(count, node->data.context) >= 0)\n\t\t\treturn;\n\t}\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tif (node->data.iovcnt == 0)\n\t{\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_FINISHED;\n\t}\n\telse\n\t{\n\t\tnode->error = errno;\n\t\tnode->state = PR_ST_ERROR;\n\t}\n\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic void __poller_handle_listen(struct __poller_node *node,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct __poller_node *res = node->res;\n\tstruct sockaddr_storage ss;\n\tstruct sockaddr *addr = (struct sockaddr *)&ss;\n\tsocklen_t addrlen;\n\tvoid *result;\n\tint sockfd;\n\n\twhile (1)\n\t{\n\t\taddrlen = sizeof (struct sockaddr_storage);\n\t\tsockfd = accept(node->data.fd, addr, &addrlen);\n\t\tif (sockfd < 0)\n\t\t{\n\t\t\tif (errno == EAGAIN || errno == EMFILE || errno == ENFILE)\n\t\t\t\treturn;\n\t\t\telse if (errno == ECONNABORTED)\n\t\t\t\tcontinue;\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\tresult = node->data.accept(addr, addrlen, sockfd, node->data.context);\n\t\tif (!result)\n\t\t\tbreak;\n\n\t\tres->data = node->data;\n\t\tres->data.result = result;\n\t\tres->error = 0;\n\t\tres->state = PR_ST_SUCCESS;\n\t\tpoller->callback((struct poller_result *)res, poller->context);\n\n\t\tres = (struct __poller_node *)malloc(sizeof (struct __poller_node));\n\t\tnode->res = res;\n\t\tif (!res)\n\t\t\tbreak;\n\n\t\tif (node->removed)\n\t\t\treturn;\n\t}\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tnode->error = errno;\n\tnode->state = PR_ST_ERROR;\n\tfree(node->res);\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic void __poller_handle_connect(struct __poller_node *node,\n\t\t\t\t\t\t\t\t\tpoller_t *poller)\n{\n\tsocklen_t len = sizeof (int);\n\tint error;\n\n\tif (getsockopt(node->data.fd, SOL_SOCKET, SO_ERROR, &error, &len) < 0)\n\t\terror = errno;\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tif (error == 0)\n\t{\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_FINISHED;\n\t}\n\telse\n\t{\n\t\tnode->error = error;\n\t\tnode->state = PR_ST_ERROR;\n\t}\n\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic void __poller_handle_recvfrom(struct __poller_node *node,\n\t\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct __poller_node *res = node->res;\n\tstruct sockaddr_storage ss;\n\tstruct sockaddr *addr = (struct sockaddr *)&ss;\n\tsocklen_t addrlen;\n\tvoid *result;\n\tssize_t n;\n\n\twhile (1)\n\t{\n\t\taddrlen = sizeof (struct sockaddr_storage);\n\t\tn = recvfrom(node->data.fd, poller->buf, POLLER_BUFSIZE, 0,\n\t\t\t\t\t addr, &addrlen);\n\t\tif (n < 0)\n\t\t{\n\t\t\tif (errno == EAGAIN)\n\t\t\t\treturn;\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\tresult = node->data.recvfrom(addr, addrlen, poller->buf, n,\n\t\t\t\t\t\t\t\t\t node->data.context);\n\t\tif (!result)\n\t\t\tbreak;\n\n\t\tres->data = node->data;\n\t\tres->data.result = result;\n\t\tres->error = 0;\n\t\tres->state = PR_ST_SUCCESS;\n\t\tpoller->callback((struct poller_result *)res, poller->context);\n\n\t\tres = (struct __poller_node *)malloc(sizeof (struct __poller_node));\n\t\tnode->res = res;\n\t\tif (!res)\n\t\t\tbreak;\n\n\t\tif (node->removed)\n\t\t\treturn;\n\t}\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tnode->error = errno;\n\tnode->state = PR_ST_ERROR;\n\tfree(node->res);\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic void __poller_handle_ssl_accept(struct __poller_node *node,\n\t\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tint ret = SSL_accept(node->data.ssl);\n\n\tif (ret <= 0)\n\t{\n\t\tif (__poller_handle_ssl_error(node, ret, poller) >= 0)\n\t\t\treturn;\n\t}\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tif (ret > 0)\n\t{\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_FINISHED;\n\t}\n\telse\n\t{\n\t\tnode->error = errno;\n\t\tnode->state = PR_ST_ERROR;\n\t}\n\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic void __poller_handle_ssl_connect(struct __poller_node *node,\n\t\t\t\t\t\t\t\t\t\tpoller_t *poller)\n{\n\tint ret = SSL_connect(node->data.ssl);\n\n\tif (ret <= 0)\n\t{\n\t\tif (__poller_handle_ssl_error(node, ret, poller) >= 0)\n\t\t\treturn;\n\t}\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tif (ret > 0)\n\t{\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_FINISHED;\n\t}\n\telse\n\t{\n\t\tnode->error = errno;\n\t\tnode->state = PR_ST_ERROR;\n\t}\n\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic void __poller_handle_ssl_shutdown(struct __poller_node *node,\n\t\t\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tint ret = SSL_shutdown(node->data.ssl);\n\n\tif (ret <= 0)\n\t{\n\t\tif (__poller_handle_ssl_error(node, ret, poller) >= 0)\n\t\t\treturn;\n\t}\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tif (ret > 0)\n\t{\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_FINISHED;\n\t}\n\telse\n\t{\n\t\tnode->error = errno;\n\t\tnode->state = PR_ST_ERROR;\n\t}\n\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic void __poller_handle_event(struct __poller_node *node,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct __poller_node *res = node->res;\n\tunsigned long long cnt = 0;\n\tunsigned long long value;\n\tvoid *result;\n\tssize_t n;\n\n\twhile (1)\n\t{\n\t\tn = read(node->data.fd, &value, sizeof (unsigned long long));\n\t\tif (n == sizeof (unsigned long long))\n\t\t\tcnt += value;\n\t\telse\n\t\t{\n\t\t\tif (n >= 0)\n\t\t\t\terrno = EINVAL;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (errno == EAGAIN)\n\t{\n\t\twhile (1)\n\t\t{\n\t\t\tif (cnt == 0)\n\t\t\t\treturn;\n\n\t\t\tcnt--;\n\t\t\tresult = node->data.event(node->data.context);\n\t\t\tif (!result)\n\t\t\t\tbreak;\n\n\t\t\tres->data = node->data;\n\t\t\tres->data.result = result;\n\t\t\tres->error = 0;\n\t\t\tres->state = PR_ST_SUCCESS;\n\t\t\tpoller->callback((struct poller_result *)res, poller->context);\n\n\t\t\tres = (struct __poller_node *)malloc(sizeof (struct __poller_node));\n\t\t\tnode->res = res;\n\t\t\tif (!res)\n\t\t\t\tbreak;\n\n\t\t\tif (node->removed)\n\t\t\t\treturn;\n\t\t}\n\t}\n\n\tif (cnt != 0)\n\t\twrite(node->data.fd, &cnt, sizeof (unsigned long long));\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tnode->error = errno;\n\tnode->state = PR_ST_ERROR;\n\tfree(node->res);\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic void __poller_handle_notify(struct __poller_node *node,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct __poller_node *res = node->res;\n\tvoid *result;\n\tssize_t n;\n\n\twhile (1)\n\t{\n\t\tn = read(node->data.fd, &result, sizeof (void *));\n\t\tif (n == sizeof (void *))\n\t\t{\n\t\t\tresult = node->data.notify(result, node->data.context);\n\t\t\tif (!result)\n\t\t\t\tbreak;\n\n\t\t\tres->data = node->data;\n\t\t\tres->data.result = result;\n\t\t\tres->error = 0;\n\t\t\tres->state = PR_ST_SUCCESS;\n\t\t\tpoller->callback((struct poller_result *)res, poller->context);\n\n\t\t\tres = (struct __poller_node *)malloc(sizeof (struct __poller_node));\n\t\t\tnode->res = res;\n\t\t\tif (!res)\n\t\t\t\tbreak;\n\n\t\t\tif (node->removed)\n\t\t\t\treturn;\n\t\t}\n\t\telse if (n < 0 && errno == EAGAIN)\n\t\t\treturn;\n\t\telse\n\t\t{\n\t\t\tif (n > 0)\n\t\t\t\terrno = EINVAL;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (__poller_remove_node(node, poller))\n\t\treturn;\n\n\tif (n == 0)\n\t{\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_FINISHED;\n\t}\n\telse\n\t{\n\t\tnode->error = errno;\n\t\tnode->state = PR_ST_ERROR;\n\t}\n\n\tfree(node->res);\n\tpoller->callback((struct poller_result *)node, poller->context);\n}\n\nstatic int __poller_handle_pipe(poller_t *poller)\n{\n\tstruct __poller_node **node = (struct __poller_node **)poller->buf;\n\tint stop = 0;\n\tint n;\n\tint i;\n\n\tn = read(poller->pipe_rd, node, POLLER_BUFSIZE) / sizeof (void *);\n\tfor (i = 0; i < n; i++)\n\t{\n\t\tif (node[i])\n\t\t{\n\t\t\tfree(node[i]->res);\n\t\t\tpoller->callback((struct poller_result *)node[i], poller->context);\n\t\t}\n\t\telse\n\t\t\tstop = 1;\n\t}\n\n\treturn stop;\n}\n\nstatic void __poller_handle_timeout(const struct __poller_node *time_node,\n\t\t\t\t\t\t\t\t\tpoller_t *poller)\n{\n\tstruct __poller_node *node;\n\tstruct list_head *pos, *tmp;\n\tLIST_HEAD(timeo_list);\n\n\tpthread_mutex_lock(&poller->mutex);\n\tlist_for_each_safe(pos, tmp, &poller->timeo_list)\n\t{\n\t\tnode = list_entry(pos, struct __poller_node, list);\n\t\tif (__timeout_cmp(node, time_node) > 0)\n\t\t\tbreak;\n\n\t\tif (node->data.fd >= 0)\n\t\t{\n\t\t\tpoller->nodes[node->data.fd] = NULL;\n\t\t\t__poller_del_fd(node->data.fd, node->event, poller);\n\t\t}\n\t\telse\n\t\t\tnode->removed = 1;\n\n\t\tlist_move_tail(pos, &timeo_list);\n\t}\n\n\twhile (poller->tree_first)\n\t{\n\t\tnode = rb_entry(poller->tree_first, struct __poller_node, rb);\n\t\tif (__timeout_cmp(node, time_node) > 0)\n\t\t\tbreak;\n\n\t\tif (node->data.fd >= 0)\n\t\t{\n\t\t\tpoller->nodes[node->data.fd] = NULL;\n\t\t\t__poller_del_fd(node->data.fd, node->event, poller);\n\t\t}\n\t\telse\n\t\t\tnode->removed = 1;\n\n\t\tpoller->tree_first = rb_next(poller->tree_first);\n\t\trb_erase(&node->rb, &poller->timeo_tree);\n\t\tlist_add_tail(&node->list, &timeo_list);\n\t\tif (!poller->tree_first)\n\t\t\tpoller->tree_last = NULL;\n\t}\n\n\tpthread_mutex_unlock(&poller->mutex);\n\tlist_for_each_safe(pos, tmp, &timeo_list)\n\t{\n\t\tnode = list_entry(pos, struct __poller_node, list);\n\t\tif (node->data.fd >= 0)\n\t\t{\n\t\t\tnode->error = ETIMEDOUT;\n\t\t\tnode->state = PR_ST_ERROR;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tnode->error = 0;\n\t\t\tnode->state = PR_ST_FINISHED;\n\t\t}\n\n\t\tfree(node->res);\n\t\tpoller->callback((struct poller_result *)node, poller->context);\n\t}\n}\n\nstatic void __poller_set_timer(poller_t *poller)\n{\n\tstruct __poller_node *node = NULL;\n\tstruct __poller_node *first;\n\tstruct timespec abstime;\n\n\tpthread_mutex_lock(&poller->mutex);\n\tif (!list_empty(&poller->timeo_list))\n\t\tnode = list_entry(poller->timeo_list.next, struct __poller_node, list);\n\n\tif (poller->tree_first)\n\t{\n\t\tfirst = rb_entry(poller->tree_first, struct __poller_node, rb);\n\t\tif (!node || __timeout_cmp(first, node) < 0)\n\t\t\tnode = first;\n\t}\n\n\tif (node)\n\t\tabstime = node->timeout;\n\telse\n\t{\n\t\tabstime.tv_sec = 0;\n\t\tabstime.tv_nsec = 0;\n\t}\n\n\t__poller_set_timerfd(poller->timerfd, &abstime, poller);\n\tpthread_mutex_unlock(&poller->mutex);\n}\n\nstatic void *__poller_thread_routine(void *arg)\n{\n\tpoller_t *poller = (poller_t *)arg;\n\t__poller_event_t events[POLLER_EVENTS_MAX];\n\tstruct __poller_node time_node;\n\tstruct __poller_node *node;\n\tint has_pipe_event;\n\tint nevents;\n\tint i;\n\n\twhile (1)\n\t{\n\t\t__poller_set_timer(poller);\n\t\tnevents = __poller_wait(events, POLLER_EVENTS_MAX, poller);\n\t\tclock_gettime(CLOCK_MONOTONIC, &time_node.timeout);\n\t\thas_pipe_event = 0;\n\t\tfor (i = 0; i < nevents; i++)\n\t\t{\n\t\t\tnode = (struct __poller_node *)__poller_event_data(&events[i]);\n\t\t\tswitch (node->data.operation)\n\t\t\t{\n\t\t\tcase PD_OP_READ:\n\t\t\t\t__poller_handle_read(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_WRITE:\n\t\t\t\t__poller_handle_write(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_LISTEN:\n\t\t\t\t__poller_handle_listen(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_CONNECT:\n\t\t\t\t__poller_handle_connect(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_RECVFROM:\n\t\t\t\t__poller_handle_recvfrom(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_SSL_ACCEPT:\n\t\t\t\t__poller_handle_ssl_accept(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_SSL_CONNECT:\n\t\t\t\t__poller_handle_ssl_connect(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_SSL_SHUTDOWN:\n\t\t\t\t__poller_handle_ssl_shutdown(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_EVENT:\n\t\t\t\t__poller_handle_event(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase PD_OP_NOTIFY:\n\t\t\t\t__poller_handle_notify(node, poller);\n\t\t\t\tbreak;\n\t\t\tcase -1:\n\t\t\t\thas_pipe_event = 1;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tif (has_pipe_event)\n\t\t{\n\t\t\tif (__poller_handle_pipe(poller))\n\t\t\t\tbreak;\n\t\t}\n\n\t\t__poller_handle_timeout(&time_node, poller);\n\t}\n\n\treturn NULL;\n}\n\nstatic int __poller_open_pipe(poller_t *poller)\n{\n\tstatic struct poller_result node = {\n\t\t.data\t\t=\t{\n\t\t\t.operation\t=\t-1\n\t\t}\n\t};\n\tint pipefd[2];\n\n\tif (pipe(pipefd) >= 0)\n\t{\n\t\tif (__poller_add_fd(pipefd[0], EPOLLIN, &node, poller) >= 0)\n\t\t{\n\t\t\tpoller->pipe_rd = pipefd[0];\n\t\t\tpoller->pipe_wr = pipefd[1];\n\t\t\treturn 0;\n\t\t}\n\n\t\tclose(pipefd[0]);\n\t\tclose(pipefd[1]);\n\t}\n\n\treturn -1;\n}\n\nstatic int __poller_create_timer(poller_t *poller)\n{\n\tint timerfd = __poller_create_timerfd();\n\n\tif (timerfd >= 0)\n\t{\n\t\tif (__poller_add_timerfd(timerfd, poller) >= 0)\n\t\t{\n\t\t\tpoller->timerfd = timerfd;\n\t\t\treturn 0;\n\t\t}\n\n\t\t__poller_close_timerfd(timerfd);\n\t}\n\n\treturn -1;\n}\n\npoller_t *__poller_create(void **nodes_buf, const struct poller_params *params)\n{\n\tpoller_t *poller = (poller_t *)malloc(sizeof (poller_t));\n\tint ret;\n\n\tif (!poller)\n\t\treturn NULL;\n\n\tpoller->pfd = __poller_create_pfd();\n\tif (poller->pfd >= 0)\n\t{\n\t\tif (__poller_create_timer(poller) >= 0)\n\t\t{\n\t\t\tret = pthread_mutex_init(&poller->mutex, NULL);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tpoller->nodes = (struct __poller_node **)nodes_buf;\n\t\t\t\tpoller->max_open_files = params->max_open_files;\n\t\t\t\tpoller->callback = params->callback;\n\t\t\t\tpoller->context = params->context;\n\n\t\t\t\tpoller->timeo_tree.rb_node = NULL;\n\t\t\t\tpoller->tree_first = NULL;\n\t\t\t\tpoller->tree_last = NULL;\n\t\t\t\tINIT_LIST_HEAD(&poller->timeo_list);\n\t\t\t\tINIT_LIST_HEAD(&poller->no_timeo_list);\n\n\t\t\t\tpoller->stopped = 1;\n\t\t\t\treturn poller;\n\t\t\t}\n\n\t\t\terrno = ret;\n\t\t\t__poller_close_timerfd(poller->timerfd);\n\t\t}\n\n\t\t__poller_close_pfd(poller->pfd);\n\t}\n\n\tfree(poller);\n\treturn NULL;\n}\n\npoller_t *poller_create(const struct poller_params *params)\n{\n\tvoid **nodes_buf = (void **)calloc(params->max_open_files, sizeof (void *));\n\tpoller_t *poller;\n\n\tif (nodes_buf)\n\t{\n\t\tpoller = __poller_create(nodes_buf, params);\n\t\tif (poller)\n\t\t\treturn poller;\n\n\t\tfree(nodes_buf);\n\t}\n\n\treturn NULL;\n}\n\nvoid __poller_destroy(poller_t *poller)\n{\n\tpthread_mutex_destroy(&poller->mutex);\n\t__poller_close_timerfd(poller->timerfd);\n\t__poller_close_pfd(poller->pfd);\n\tfree(poller);\n}\n\nvoid poller_destroy(poller_t *poller)\n{\n\tfree(poller->nodes);\n\t__poller_destroy(poller);\n}\n\nint poller_start(poller_t *poller)\n{\n\tpthread_t tid;\n\tint ret;\n\n\tpthread_mutex_lock(&poller->mutex);\n\tif (__poller_open_pipe(poller) >= 0)\n\t{\n\t\tret = pthread_create(&tid, NULL, __poller_thread_routine, poller);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tpoller->tid = tid;\n\t\t\tpoller->stopped = 0;\n\t\t}\n\t\telse\n\t\t{\n\t\t\terrno = ret;\n\t\t\tclose(poller->pipe_wr);\n\t\t\tclose(poller->pipe_rd);\n\t\t}\n\t}\n\n\tpthread_mutex_unlock(&poller->mutex);\n\treturn -poller->stopped;\n}\n\nstatic void __poller_insert_node(struct __poller_node *node,\n\t\t\t\t\t\t\t\t poller_t *poller)\n{\n\tstruct __poller_node *end;\n\n\tend = list_entry(poller->timeo_list.prev, struct __poller_node, list);\n\tif (list_empty(&poller->timeo_list))\n\t{\n\t\tlist_add(&node->list, &poller->timeo_list);\n\t\tend = rb_entry(poller->tree_first, struct __poller_node, rb);\n\t}\n\telse if (__timeout_cmp(node, end) >= 0)\n\t{\n\t\tlist_add_tail(&node->list, &poller->timeo_list);\n\t\treturn;\n\t}\n\telse\n\t{\n\t\t__poller_tree_insert(node, poller);\n\t\tif (&node->rb != poller->tree_first)\n\t\t\treturn;\n\n\t\tend = list_entry(poller->timeo_list.next, struct __poller_node, list);\n\t}\n\n\tif (!poller->tree_first || __timeout_cmp(node, end) < 0)\n\t\t__poller_set_timerfd(poller->timerfd, &node->timeout, poller);\n}\n\nstatic void __poller_node_set_timeout(int timeout, struct __poller_node *node)\n{\n\tclock_gettime(CLOCK_MONOTONIC, &node->timeout);\n\tnode->timeout.tv_sec += timeout / 1000;\n\tnode->timeout.tv_nsec += timeout % 1000 * 1000000;\n\tif (node->timeout.tv_nsec >= 1000000000)\n\t{\n\t\tnode->timeout.tv_nsec -= 1000000000;\n\t\tnode->timeout.tv_sec++;\n\t}\n}\n\nstatic int __poller_data_get_event(int *event, const struct poller_data *data)\n{\n\tswitch (data->operation)\n\t{\n\tcase PD_OP_READ:\n\t\t*event = EPOLLIN | EPOLLET;\n\t\treturn !!data->message;\n\tcase PD_OP_WRITE:\n\t\t*event = EPOLLOUT | EPOLLET;\n\t\treturn 0;\n\tcase PD_OP_LISTEN:\n\t\t*event = EPOLLIN;\n\t\treturn 1;\n\tcase PD_OP_CONNECT:\n\t\t*event = EPOLLOUT | EPOLLET;\n\t\treturn 0;\n\tcase PD_OP_RECVFROM:\n\t\t*event = EPOLLIN | EPOLLET;\n\t\treturn 1;\n\tcase PD_OP_SSL_ACCEPT:\n\t\t*event = EPOLLIN | EPOLLET;\n\t\treturn 0;\n\tcase PD_OP_SSL_CONNECT:\n\t\t*event = EPOLLOUT | EPOLLET;\n\t\treturn 0;\n\tcase PD_OP_SSL_SHUTDOWN:\n\t\t*event = EPOLLOUT | EPOLLET;\n\t\treturn 0;\n\tcase PD_OP_EVENT:\n\t\t*event = EPOLLIN | EPOLLET;\n\t\treturn 1;\n\tcase PD_OP_NOTIFY:\n\t\t*event = EPOLLIN | EPOLLET;\n\t\treturn 1;\n\tdefault:\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n}\n\nstatic struct __poller_node *__poller_new_node(const struct poller_data *data,\n\t\t\t\t\t\t\t\t\t\t\t int timeout, poller_t *poller)\n{\n\tstruct __poller_node *res = NULL;\n\tstruct __poller_node *node;\n\tint need_res;\n\tint event;\n\n\tif ((size_t)data->fd >= poller->max_open_files)\n\t{\n\t\terrno = data->fd < 0 ? EBADF : EMFILE;\n\t\treturn NULL;\n\t}\n\n\tneed_res = __poller_data_get_event(&event, data);\n\tif (need_res < 0)\n\t\treturn NULL;\n\n\tif (need_res)\n\t{\n\t\tres = (struct __poller_node *)malloc(sizeof (struct __poller_node));\n\t\tif (!res)\n\t\t\treturn NULL;\n\t}\n\n\tnode = (struct __poller_node *)malloc(sizeof (struct __poller_node));\n\tif (!node)\n\t{\n\t\tfree(res);\n\t\treturn NULL;\n\t}\n\n\tnode->data = *data;\n\tnode->event = event;\n\tnode->in_rbtree = 0;\n\tnode->removed = 0;\n\tnode->res = res;\n\tif (timeout >= 0)\n\t\t__poller_node_set_timeout(timeout, node);\n\n\treturn node;\n}\n\nint poller_add(const struct poller_data *data, int timeout, poller_t *poller)\n{\n\tstruct __poller_node *node;\n\n\tnode = __poller_new_node(data, timeout, poller);\n\tif (!node)\n\t\treturn -1;\n\n\tpthread_mutex_lock(&poller->mutex);\n\tif (!poller->nodes[data->fd])\n\t{\n\t\tif (__poller_add_fd(data->fd, node->event, node, poller) >= 0)\n\t\t{\n\t\t\tif (timeout >= 0)\n\t\t\t\t__poller_insert_node(node, poller);\n\t\t\telse\n\t\t\t\tlist_add_tail(&node->list, &poller->no_timeo_list);\n\n\t\t\tpoller->nodes[data->fd] = node;\n\t\t\tnode = NULL;\n\t\t}\n\t}\n\telse\n\t\terrno = EEXIST;\n\n\tpthread_mutex_unlock(&poller->mutex);\n\tif (node == NULL)\n\t\treturn 0;\n\n\tfree(node->res);\n\tfree(node);\n\treturn -1;\n}\n\nint poller_del(int fd, poller_t *poller)\n{\n\tstruct __poller_node *node;\n\tint stopped = 0;\n\n\tif ((size_t)fd >= poller->max_open_files)\n\t{\n\t\terrno = fd < 0 ? EBADF : EMFILE;\n\t\treturn -1;\n\t}\n\n\tpthread_mutex_lock(&poller->mutex);\n\tnode = poller->nodes[fd];\n\tif (node)\n\t{\n\t\tpoller->nodes[fd] = NULL;\n\n\t\tif (node->in_rbtree)\n\t\t\t__poller_tree_erase(node, poller);\n\t\telse\n\t\t\tlist_del(&node->list);\n\n\t\t__poller_del_fd(fd, node->event, poller);\n\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_DELETED;\n\t\tstopped = poller->stopped;\n\t\tif (!stopped)\n\t\t{\n\t\t\tnode->removed = 1;\n\t\t\twrite(poller->pipe_wr, &node, sizeof (void *));\n\t\t}\n\t}\n\telse\n\t\terrno = ENOENT;\n\n\tpthread_mutex_unlock(&poller->mutex);\n\tif (stopped)\n\t{\n\t\tfree(node->res);\n\t\tpoller->callback((struct poller_result *)node, poller->context);\n\t}\n\n\treturn -!node;\n}\n\nint poller_mod(const struct poller_data *data, int timeout, poller_t *poller)\n{\n\tstruct __poller_node *node;\n\tstruct __poller_node *orig;\n\tint stopped = 0;\n\n\tnode = __poller_new_node(data, timeout, poller);\n\tif (!node)\n\t\treturn -1;\n\n\tpthread_mutex_lock(&poller->mutex);\n\torig = poller->nodes[data->fd];\n\tif (orig)\n\t{\n\t\tif (__poller_mod_fd(data->fd, orig->event, node->event, node, poller) >= 0)\n\t\t{\n\t\t\tif (orig->in_rbtree)\n\t\t\t\t__poller_tree_erase(orig, poller);\n\t\t\telse\n\t\t\t\tlist_del(&orig->list);\n\n\t\t\torig->error = 0;\n\t\t\torig->state = PR_ST_MODIFIED;\n\t\t\tstopped = poller->stopped;\n\t\t\tif (!stopped)\n\t\t\t{\n\t\t\t\torig->removed = 1;\n\t\t\t\twrite(poller->pipe_wr, &orig, sizeof (void *));\n\t\t\t}\n\n\t\t\tif (timeout >= 0)\n\t\t\t\t__poller_insert_node(node, poller);\n\t\t\telse\n\t\t\t\tlist_add_tail(&node->list, &poller->no_timeo_list);\n\n\t\t\tpoller->nodes[data->fd] = node;\n\t\t\tnode = NULL;\n\t\t}\n\t}\n\telse\n\t\terrno = ENOENT;\n\n\tpthread_mutex_unlock(&poller->mutex);\n\tif (stopped)\n\t{\n\t\tfree(orig->res);\n\t\tpoller->callback((struct poller_result *)orig, poller->context);\n\t}\n\n\tif (node == NULL)\n\t\treturn 0;\n\n\tfree(node->res);\n\tfree(node);\n\treturn -1;\n}\n\nint poller_set_timeout(int fd, int timeout, poller_t *poller)\n{\n\tstruct __poller_node time_node;\n\tstruct __poller_node *node;\n\n\tif ((size_t)fd >= poller->max_open_files)\n\t{\n\t\terrno = fd < 0 ? EBADF : EMFILE;\n\t\treturn -1;\n\t}\n\n\tif (timeout >= 0)\n\t\t__poller_node_set_timeout(timeout, &time_node);\n\n\tpthread_mutex_lock(&poller->mutex);\n\tnode = poller->nodes[fd];\n\tif (node)\n\t{\n\t\tif (node->in_rbtree)\n\t\t\t__poller_tree_erase(node, poller);\n\t\telse\n\t\t\tlist_del(&node->list);\n\n\t\tif (timeout >= 0)\n\t\t{\n\t\t\tnode->timeout = time_node.timeout;\n\t\t\t__poller_insert_node(node, poller);\n\t\t}\n\t\telse\n\t\t\tlist_add_tail(&node->list, &poller->no_timeo_list);\n\t}\n\telse\n\t\terrno = ENOENT;\n\n\tpthread_mutex_unlock(&poller->mutex);\n\treturn -!node;\n}\n\nint poller_add_timer(const struct timespec *value, void *context, void **timer,\n\t\t\t\t\t poller_t *poller)\n{\n\tstruct __poller_node *node;\n\n\tif (value->tv_nsec < 0 || value->tv_nsec >= 1000000000)\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\tnode = (struct __poller_node *)malloc(sizeof (struct __poller_node));\n\tif (node)\n\t{\n\t\tmemset(&node->data, 0, sizeof (struct poller_data));\n\t\tnode->data.operation = PD_OP_TIMER;\n\t\tnode->data.fd = -1;\n\t\tnode->data.context = context;\n\t\tnode->in_rbtree = 0;\n\t\tnode->removed = 0;\n\t\tnode->res = NULL;\n\n\t\tif (value->tv_sec >= 0)\n\t\t{\n\t\t\tclock_gettime(CLOCK_MONOTONIC, &node->timeout);\n\t\t\tnode->timeout.tv_sec += value->tv_sec;\n\t\t\tnode->timeout.tv_nsec += value->tv_nsec;\n\t\t\tif (node->timeout.tv_nsec >= 1000000000)\n\t\t\t{\n\t\t\t\tnode->timeout.tv_nsec -= 1000000000;\n\t\t\t\tnode->timeout.tv_sec++;\n\t\t\t}\n\t\t}\n\n\t\t*timer = node;\n\t\tpthread_mutex_lock(&poller->mutex);\n\t\tif (value->tv_sec >= 0)\n\t\t\t__poller_insert_node(node, poller);\n\t\telse\n\t\t\tlist_add_tail(&node->list, &poller->no_timeo_list);\n\n\t\tpthread_mutex_unlock(&poller->mutex);\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nint poller_del_timer(void *timer, poller_t *poller)\n{\n\tstruct __poller_node *node = (struct __poller_node *)timer;\n\n\tpthread_mutex_lock(&poller->mutex);\n\tif (!node->removed)\n\t{\n\t\tnode->removed = 1;\n\n\t\tif (node->in_rbtree)\n\t\t\t__poller_tree_erase(node, poller);\n\t\telse\n\t\t\tlist_del(&node->list);\n\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_DELETED;\n\t}\n\telse\n\t{\n\t\terrno = ENOENT;\n\t\tnode = NULL;\n\t}\n\n\tpthread_mutex_unlock(&poller->mutex);\n\tif (node)\n\t\tpoller->callback((struct poller_result *)node, poller->context);\n\n\treturn -!node;\n}\n\nvoid poller_set_callback(void (*callback)(struct poller_result *, void *),\n\t\t\t\t\t\t poller_t *poller)\n{\n\tpoller->callback = callback;\n}\n\nvoid poller_stop(poller_t *poller)\n{\n\tstruct __poller_node *node;\n\tstruct list_head *pos, *tmp;\n\tLIST_HEAD(node_list);\n\tvoid *p = NULL;\n\n\twrite(poller->pipe_wr, &p, sizeof (void *));\n\tpthread_join(poller->tid, NULL);\n\tpoller->stopped = 1;\n\n\tpthread_mutex_lock(&poller->mutex);\n\tclose(poller->pipe_wr);\n\t__poller_handle_pipe(poller);\n\tclose(poller->pipe_rd);\n\n\tpoller->tree_first = NULL;\n\tpoller->tree_last = NULL;\n\twhile (poller->timeo_tree.rb_node)\n\t{\n\t\tnode = rb_entry(poller->timeo_tree.rb_node, struct __poller_node, rb);\n\t\trb_erase(&node->rb, &poller->timeo_tree);\n\t\tlist_add(&node->list, &node_list);\n\t}\n\n\tlist_splice_init(&poller->timeo_list, &node_list);\n\tlist_splice_init(&poller->no_timeo_list, &node_list);\n\tlist_for_each(pos, &node_list)\n\t{\n\t\tnode = list_entry(pos, struct __poller_node, list);\n\t\tif (node->data.fd >= 0)\n\t\t{\n\t\t\tpoller->nodes[node->data.fd] = NULL;\n\t\t\t__poller_del_fd(node->data.fd, node->event, poller);\n\t\t}\n\t\telse\n\t\t\tnode->removed = 1;\n\t}\n\n\tpthread_mutex_unlock(&poller->mutex);\n\tlist_for_each_safe(pos, tmp, &node_list)\n\t{\n\t\tnode = list_entry(pos, struct __poller_node, list);\n\t\tnode->error = 0;\n\t\tnode->state = PR_ST_STOPPED;\n\t\tfree(node->res);\n\t\tpoller->callback((struct poller_result *)node, poller->context);\n\t}\n}\n\n" }, { "path": "src/kernel/poller.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _POLLER_H_\n#define _POLLER_H_\n\n#include \n#include \n#include \n#include \n\ntypedef struct __poller poller_t;\ntypedef struct __poller_message poller_message_t;\n\nstruct __poller_message\n{\n\tint (*append)(const void *, size_t *, poller_message_t *);\n\tchar data[0];\n};\n\nstruct poller_data\n{\n#define PD_OP_TIMER\t\t\t0\n#define PD_OP_READ\t\t\t1\n#define PD_OP_WRITE\t\t\t2\n#define PD_OP_LISTEN\t\t3\n#define PD_OP_CONNECT\t\t4\n#define PD_OP_RECVFROM\t\t5\n#define PD_OP_SSL_READ\t\tPD_OP_READ\n#define PD_OP_SSL_WRITE\t\tPD_OP_WRITE\n#define PD_OP_SSL_ACCEPT\t6\n#define PD_OP_SSL_CONNECT\t7\n#define PD_OP_SSL_SHUTDOWN\t8\n#define PD_OP_EVENT\t\t\t9\n#define PD_OP_NOTIFY\t\t10\n\tshort operation;\n\tunsigned short iovcnt;\n\tint fd;\n\tSSL *ssl;\n\tunion\n\t{\n\t\tpoller_message_t *(*create_message)(void *);\n\t\tint (*partial_written)(size_t, void *);\n\t\tvoid *(*accept)(const struct sockaddr *, socklen_t, int, void *);\n\t\tvoid *(*recvfrom)(const struct sockaddr *, socklen_t,\n\t\t\t\t\t\t const void *, size_t, void *);\n\t\tvoid *(*event)(void *);\n\t\tvoid *(*notify)(void *, void *);\n\t};\n\tvoid *context;\n\tunion\n\t{\n\t\tpoller_message_t *message;\n\t\tstruct iovec *write_iov;\n\t\tvoid *result;\n\t};\n};\n\nstruct poller_result\n{\n#define PR_ST_SUCCESS\t\t0\n#define PR_ST_FINISHED\t\t1\n#define PR_ST_ERROR\t\t\t2\n#define PR_ST_DELETED\t\t3\n#define PR_ST_MODIFIED\t\t4\n#define PR_ST_STOPPED\t\t5\n\tint state;\n\tint error;\n\tstruct poller_data data;\n\t/* In callback, spaces of six pointers are available from here. */\n};\n\nstruct poller_params\n{\n\tsize_t max_open_files;\n\tvoid (*callback)(struct poller_result *, void *);\n\tvoid *context;\n};\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\npoller_t *poller_create(const struct poller_params *params);\nint poller_start(poller_t *poller);\nint poller_add(const struct poller_data *data, int timeout, poller_t *poller);\nint poller_del(int fd, poller_t *poller);\nint poller_mod(const struct poller_data *data, int timeout, poller_t *poller);\nint poller_set_timeout(int fd, int timeout, poller_t *poller);\nint poller_add_timer(const struct timespec *value, void *context, void **timer,\n\t\t\t\t\t poller_t *poller);\nint poller_del_timer(void *timer, poller_t *poller);\nvoid poller_set_callback(void (*callback)(struct poller_result *, void *),\n\t\t\t\t\t\t poller_t *poller);\nvoid poller_stop(poller_t *poller);\nvoid poller_destroy(poller_t *poller);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n\n" }, { "path": "src/kernel/rbtree.c", "content": "/*\n Red Black Trees\n (C) 1999 Andrea Arcangeli \n (C) 2002 David Woodhouse \n \n This program is free software; you can redistribute it and/or modify\n it under the terms of the GNU General Public License as published by\n the Free Software Foundation; either version 2 of the License, or\n (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU General Public License for more details.\n\n You should have received a copy of the GNU General Public License\n along with this program; if not, write to the Free Software\n Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n\n linux/lib/rbtree.c\n*/\n\n#include \"rbtree.h\"\n\nstatic void __rb_rotate_left(struct rb_node *node, struct rb_root *root)\n{\n\tstruct rb_node *right = node->rb_right;\n\n\tif ((node->rb_right = right->rb_left))\n\t\tright->rb_left->rb_parent = node;\n\tright->rb_left = node;\n\n\tif ((right->rb_parent = node->rb_parent))\n\t{\n\t\tif (node == node->rb_parent->rb_left)\n\t\t\tnode->rb_parent->rb_left = right;\n\t\telse\n\t\t\tnode->rb_parent->rb_right = right;\n\t}\n\telse\n\t\troot->rb_node = right;\n\tnode->rb_parent = right;\n}\n\nstatic void __rb_rotate_right(struct rb_node *node, struct rb_root *root)\n{\n\tstruct rb_node *left = node->rb_left;\n\n\tif ((node->rb_left = left->rb_right))\n\t\tleft->rb_right->rb_parent = node;\n\tleft->rb_right = node;\n\n\tif ((left->rb_parent = node->rb_parent))\n\t{\n\t\tif (node == node->rb_parent->rb_right)\n\t\t\tnode->rb_parent->rb_right = left;\n\t\telse\n\t\t\tnode->rb_parent->rb_left = left;\n\t}\n\telse\n\t\troot->rb_node = left;\n\tnode->rb_parent = left;\n}\n\nvoid rb_insert_color(struct rb_node *node, struct rb_root *root)\n{\n\tstruct rb_node *parent, *gparent;\n\n\twhile ((parent = node->rb_parent) && parent->rb_color == RB_RED)\n\t{\n\t\tgparent = parent->rb_parent;\n\n\t\tif (parent == gparent->rb_left)\n\t\t{\n\t\t\t{\n\t\t\t\tregister struct rb_node *uncle = gparent->rb_right;\n\t\t\t\tif (uncle && uncle->rb_color == RB_RED)\n\t\t\t\t{\n\t\t\t\t\tuncle->rb_color = RB_BLACK;\n\t\t\t\t\tparent->rb_color = RB_BLACK;\n\t\t\t\t\tgparent->rb_color = RB_RED;\n\t\t\t\t\tnode = gparent;\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (parent->rb_right == node)\n\t\t\t{\n\t\t\t\tregister struct rb_node *tmp;\n\t\t\t\t__rb_rotate_left(parent, root);\n\t\t\t\ttmp = parent;\n\t\t\t\tparent = node;\n\t\t\t\tnode = tmp;\n\t\t\t}\n\n\t\t\tparent->rb_color = RB_BLACK;\n\t\t\tgparent->rb_color = RB_RED;\n\t\t\t__rb_rotate_right(gparent, root);\n\t\t} else {\n\t\t\t{\n\t\t\t\tregister struct rb_node *uncle = gparent->rb_left;\n\t\t\t\tif (uncle && uncle->rb_color == RB_RED)\n\t\t\t\t{\n\t\t\t\t\tuncle->rb_color = RB_BLACK;\n\t\t\t\t\tparent->rb_color = RB_BLACK;\n\t\t\t\t\tgparent->rb_color = RB_RED;\n\t\t\t\t\tnode = gparent;\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (parent->rb_left == node)\n\t\t\t{\n\t\t\t\tregister struct rb_node *tmp;\n\t\t\t\t__rb_rotate_right(parent, root);\n\t\t\t\ttmp = parent;\n\t\t\t\tparent = node;\n\t\t\t\tnode = tmp;\n\t\t\t}\n\n\t\t\tparent->rb_color = RB_BLACK;\n\t\t\tgparent->rb_color = RB_RED;\n\t\t\t__rb_rotate_left(gparent, root);\n\t\t}\n\t}\n\n\troot->rb_node->rb_color = RB_BLACK;\n}\n\nstatic void __rb_erase_color(struct rb_node *node, struct rb_node *parent,\n\t\t\t struct rb_root *root)\n{\n\tstruct rb_node *other;\n\n\twhile ((!node || node->rb_color == RB_BLACK) && node != root->rb_node)\n\t{\n\t\tif (parent->rb_left == node)\n\t\t{\n\t\t\tother = parent->rb_right;\n\t\t\tif (other->rb_color == RB_RED)\n\t\t\t{\n\t\t\t\tother->rb_color = RB_BLACK;\n\t\t\t\tparent->rb_color = RB_RED;\n\t\t\t\t__rb_rotate_left(parent, root);\n\t\t\t\tother = parent->rb_right;\n\t\t\t}\n\t\t\tif ((!other->rb_left ||\n\t\t\t other->rb_left->rb_color == RB_BLACK)\n\t\t\t && (!other->rb_right ||\n\t\t\t\tother->rb_right->rb_color == RB_BLACK))\n\t\t\t{\n\t\t\t\tother->rb_color = RB_RED;\n\t\t\t\tnode = parent;\n\t\t\t\tparent = node->rb_parent;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tif (!other->rb_right ||\n\t\t\t\t other->rb_right->rb_color == RB_BLACK)\n\t\t\t\t{\n\t\t\t\t\tregister struct rb_node *o_left;\n\t\t\t\t\tif ((o_left = other->rb_left))\n\t\t\t\t\t\to_left->rb_color = RB_BLACK;\n\t\t\t\t\tother->rb_color = RB_RED;\n\t\t\t\t\t__rb_rotate_right(other, root);\n\t\t\t\t\tother = parent->rb_right;\n\t\t\t\t}\n\t\t\t\tother->rb_color = parent->rb_color;\n\t\t\t\tparent->rb_color = RB_BLACK;\n\t\t\t\tif (other->rb_right)\n\t\t\t\t\tother->rb_right->rb_color = RB_BLACK;\n\t\t\t\t__rb_rotate_left(parent, root);\n\t\t\t\tnode = root->rb_node;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tother = parent->rb_left;\n\t\t\tif (other->rb_color == RB_RED)\n\t\t\t{\n\t\t\t\tother->rb_color = RB_BLACK;\n\t\t\t\tparent->rb_color = RB_RED;\n\t\t\t\t__rb_rotate_right(parent, root);\n\t\t\t\tother = parent->rb_left;\n\t\t\t}\n\t\t\tif ((!other->rb_left ||\n\t\t\t other->rb_left->rb_color == RB_BLACK)\n\t\t\t && (!other->rb_right ||\n\t\t\t\tother->rb_right->rb_color == RB_BLACK))\n\t\t\t{\n\t\t\t\tother->rb_color = RB_RED;\n\t\t\t\tnode = parent;\n\t\t\t\tparent = node->rb_parent;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tif (!other->rb_left ||\n\t\t\t\t other->rb_left->rb_color == RB_BLACK)\n\t\t\t\t{\n\t\t\t\t\tregister struct rb_node *o_right;\n\t\t\t\t\tif ((o_right = other->rb_right))\n\t\t\t\t\t\to_right->rb_color = RB_BLACK;\n\t\t\t\t\tother->rb_color = RB_RED;\n\t\t\t\t\t__rb_rotate_left(other, root);\n\t\t\t\t\tother = parent->rb_left;\n\t\t\t\t}\n\t\t\t\tother->rb_color = parent->rb_color;\n\t\t\t\tparent->rb_color = RB_BLACK;\n\t\t\t\tif (other->rb_left)\n\t\t\t\t\tother->rb_left->rb_color = RB_BLACK;\n\t\t\t\t__rb_rotate_right(parent, root);\n\t\t\t\tnode = root->rb_node;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\tif (node)\n\t\tnode->rb_color = RB_BLACK;\n}\n\nvoid rb_erase(struct rb_node *node, struct rb_root *root)\n{\n\tstruct rb_node *child, *parent;\n\tint color;\n\n\tif (!node->rb_left)\n\t\tchild = node->rb_right;\n\telse if (!node->rb_right)\n\t\tchild = node->rb_left;\n\telse\n\t{\n\t\tstruct rb_node *old = node, *left;\n\n\t\tnode = node->rb_right;\n\t\twhile ((left = node->rb_left))\n\t\t\tnode = left;\n\t\tchild = node->rb_right;\n\t\tparent = node->rb_parent;\n\t\tcolor = node->rb_color;\n\n\t\tif (child)\n\t\t\tchild->rb_parent = parent;\n\t\tif (parent)\n\t\t{\n\t\t\tif (parent->rb_left == node)\n\t\t\t\tparent->rb_left = child;\n\t\t\telse\n\t\t\t\tparent->rb_right = child;\n\t\t}\n\t\telse\n\t\t\troot->rb_node = child;\n\n\t\tif (node->rb_parent == old)\n\t\t\tparent = node;\n\t\tnode->rb_parent = old->rb_parent;\n\t\tnode->rb_color = old->rb_color;\n\t\tnode->rb_right = old->rb_right;\n\t\tnode->rb_left = old->rb_left;\n\n\t\tif (old->rb_parent)\n\t\t{\n\t\t\tif (old->rb_parent->rb_left == old)\n\t\t\t\told->rb_parent->rb_left = node;\n\t\t\telse\n\t\t\t\told->rb_parent->rb_right = node;\n\t\t} else\n\t\t\troot->rb_node = node;\n\n\t\told->rb_left->rb_parent = node;\n\t\tif (old->rb_right)\n\t\t\told->rb_right->rb_parent = node;\n\t\tgoto color;\n\t}\n\n\tparent = node->rb_parent;\n\tcolor = node->rb_color;\n\n\tif (child)\n\t\tchild->rb_parent = parent;\n\tif (parent)\n\t{\n\t\tif (parent->rb_left == node)\n\t\t\tparent->rb_left = child;\n\t\telse\n\t\t\tparent->rb_right = child;\n\t}\n\telse\n\t\troot->rb_node = child;\n\n color:\n\tif (color == RB_BLACK)\n\t\t__rb_erase_color(child, parent, root);\n}\n\n/*\n * This function returns the first node (in sort order) of the tree.\n */\nstruct rb_node *rb_first(struct rb_root *root)\n{\n\tstruct rb_node *n;\n\n\tn = root->rb_node;\n\tif (!n)\n\t\treturn (struct rb_node *)0;\n\twhile (n->rb_left)\n\t\tn = n->rb_left;\n\treturn n;\n}\n\nstruct rb_node *rb_last(struct rb_root *root)\n{\n\tstruct rb_node *n;\n\n\tn = root->rb_node;\n\tif (!n)\n\t\treturn (struct rb_node *)0;\n\twhile (n->rb_right)\n\t\tn = n->rb_right;\n\treturn n;\n}\n\nstruct rb_node *rb_next(struct rb_node *node)\n{\n\t/* If we have a right-hand child, go down and then left as far\n\t as we can. */\n\tif (node->rb_right) {\n\t\tnode = node->rb_right; \n\t\twhile (node->rb_left)\n\t\t\tnode = node->rb_left;\n\t\treturn node;\n\t}\n\n\t/* No right-hand children. Everything down and left is\n\t smaller than us, so any 'next' node must be in the general\n\t direction of our parent. Go up the tree; any time the\n\t ancestor is a right-hand child of its parent, keep going\n\t up. First time it's a left-hand child of its parent, said\n\t parent is our 'next' node. */\n\twhile (node->rb_parent && node == node->rb_parent->rb_right)\n\t\tnode = node->rb_parent;\n\n\treturn node->rb_parent;\n}\n\nstruct rb_node *rb_prev(struct rb_node *node)\n{\n\t/* If we have a left-hand child, go down and then right as far\n\t as we can. */\n\tif (node->rb_left) {\n\t\tnode = node->rb_left; \n\t\twhile (node->rb_right)\n\t\t\tnode = node->rb_right;\n\t\treturn node;\n\t}\n\n\t/* No left-hand children. Go up till we find an ancestor which\n\t is a right-hand child of its parent */\n\twhile (node->rb_parent && node == node->rb_parent->rb_left)\n\t\tnode = node->rb_parent;\n\n\treturn node->rb_parent;\n}\n\nvoid rb_replace_node(struct rb_node *victim, struct rb_node *newnode,\n\t\t struct rb_root *root)\n{\n\tstruct rb_node *parent = victim->rb_parent;\n\n\t/* Set the surrounding nodes to point to the replacement */\n\tif (parent) {\n\t\tif (victim == parent->rb_left)\n\t\t\tparent->rb_left = newnode;\n\t\telse\n\t\t\tparent->rb_right = newnode;\n\t} else {\n\t\troot->rb_node = newnode;\n\t}\n\tif (victim->rb_left)\n\t\tvictim->rb_left->rb_parent = newnode;\n\tif (victim->rb_right)\n\t\tvictim->rb_right->rb_parent = newnode;\n\n\t/* Copy the pointers/colour from the victim to the replacement */\n\t*newnode = *victim;\n}\n\n" }, { "path": "src/kernel/rbtree.h", "content": "/*\n Red Black Trees\n (C) 1999 Andrea Arcangeli \n \n This program is free software; you can redistribute it and/or modify\n it under the terms of the GNU General Public License as published by\n the Free Software Foundation; either version 2 of the License, or\n (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU General Public License for more details.\n\n You should have received a copy of the GNU General Public License\n along with this program; if not, write to the Free Software\n Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n\n linux/include/linux/rbtree.h\n\n To use rbtrees you'll have to implement your own insert and search cores.\n This will avoid us to use callbacks and to drop drammatically performances.\n I know it's not the cleaner way, but in C (not in C++) to get\n performances and genericity...\n\n Some example of insert and search follows here. The search is a plain\n normal search over an ordered tree. The insert instead must be implemented\n int two steps: as first thing the code must insert the element in\n order as a red leaf in the tree, then the support library function\n rb_insert_color() must be called. Such function will do the\n not trivial work to rebalance the rbtree if necessary.\n\n-----------------------------------------------------------------------\nstatic inline struct page * rb_search_page_cache(struct inode * inode,\n\t\t\t\t\t\t unsigned long offset)\n{\n\trb_node_t * n = inode->i_rb_page_cache.rb_node;\n\tstruct page * page;\n\n\twhile (n)\n\t{\n\t\tpage = rb_entry(n, struct page, rb_page_cache);\n\n\t\tif (offset < page->offset)\n\t\t\tn = n->rb_left;\n\t\telse if (offset > page->offset)\n\t\t\tn = n->rb_right;\n\t\telse\n\t\t\treturn page;\n\t}\n\treturn NULL;\n}\n\nstatic inline struct page * __rb_insert_page_cache(struct inode * inode,\n\t\t\t\t\t\t unsigned long offset,\n\t\t\t\t\t\t rb_node_t * node)\n{\n\trb_node_t ** p = &inode->i_rb_page_cache.rb_node;\n\trb_node_t * parent = NULL;\n\tstruct page * page;\n\n\twhile (*p)\n\t{\n\t\tparent = *p;\n\t\tpage = rb_entry(parent, struct page, rb_page_cache);\n\n\t\tif (offset < page->offset)\n\t\t\tp = &(*p)->rb_left;\n\t\telse if (offset > page->offset)\n\t\t\tp = &(*p)->rb_right;\n\t\telse\n\t\t\treturn page;\n\t}\n\n\trb_link_node(node, parent, p);\n\n\treturn NULL;\n}\n\nstatic inline struct page * rb_insert_page_cache(struct inode * inode,\n\t\t\t\t\t\t unsigned long offset,\n\t\t\t\t\t\t rb_node_t * node)\n{\n\tstruct page * ret;\n\tif ((ret = __rb_insert_page_cache(inode, offset, node)))\n\t\tgoto out;\n\trb_insert_color(node, &inode->i_rb_page_cache);\n out:\n\treturn ret;\n}\n-----------------------------------------------------------------------\n*/\n\n#ifndef\t_LINUX_RBTREE_H\n#define\t_LINUX_RBTREE_H\n\n#pragma pack(1)\nstruct rb_node\n{\n\tstruct rb_node *rb_parent;\n\tstruct rb_node *rb_right;\n\tstruct rb_node *rb_left;\n\tchar rb_color;\n#define\tRB_RED\t\t0\n#define\tRB_BLACK\t1\n};\n#pragma pack()\n\nstruct rb_root\n{\n\tstruct rb_node *rb_node;\n};\n\n#define RB_ROOT (struct rb_root){ (struct rb_node *)0, }\n#define\trb_entry(ptr, type, member) \\\n\t((type *)((char *)(ptr)-(unsigned long)(&((type *)0)->member)))\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nextern void rb_insert_color(struct rb_node *node, struct rb_root *root);\nextern void rb_erase(struct rb_node *node, struct rb_root *root);\n\n/* Find logical next and previous nodes in a tree */\nextern struct rb_node *rb_next(struct rb_node *);\nextern struct rb_node *rb_prev(struct rb_node *);\nextern struct rb_node *rb_first(struct rb_root *);\nextern struct rb_node *rb_last(struct rb_root *);\n\n/* Fast replacement of a single node without remove/rebalance/add/rebalance */\nextern void rb_replace_node(struct rb_node *victim, struct rb_node *newnode, \n\t\t\t struct rb_root *root);\n\n#ifdef __cplusplus\n}\n#endif\n\nstatic inline void rb_link_node(struct rb_node *node, struct rb_node *parent,\n\t\t\t\tstruct rb_node **link)\n{\n\tnode->rb_parent = parent;\n\tnode->rb_color = RB_RED;\n\tnode->rb_left = node->rb_right = (struct rb_node *)0;\n\n\t*link = node;\n}\n\n#endif\n" }, { "path": "src/kernel/thrdpool.c", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"msgqueue.h\"\n#include \"thrdpool.h\"\n\nstruct __thrdpool\n{\n\tmsgqueue_t *msgqueue;\n\tsize_t nthreads;\n\tsize_t stacksize;\n\tpthread_t tid;\n\tpthread_mutex_t mutex;\n\tpthread_key_t key;\n\tpthread_cond_t *terminate;\n};\n\nstruct __thrdpool_task_entry\n{\n\tvoid *link;\n\tstruct thrdpool_task task;\n};\n\nstatic pthread_t __zero_tid;\n\nstatic void __thrdpool_exit_routine(void *context)\n{\n\tthrdpool_t *pool = (thrdpool_t *)context;\n\tpthread_t tid;\n\n\t/* One thread joins another. Don't need to keep all thread IDs. */\n\tpthread_mutex_lock(&pool->mutex);\n\ttid = pool->tid;\n\tpool->tid = pthread_self();\n\tif (--pool->nthreads == 0 && pool->terminate)\n\t\tpthread_cond_signal(pool->terminate);\n\n\tpthread_mutex_unlock(&pool->mutex);\n\tif (!pthread_equal(tid, __zero_tid))\n\t\tpthread_join(tid, NULL);\n\n\tpthread_exit(NULL);\n}\n\nstatic void *__thrdpool_routine(void *arg)\n{\n\tthrdpool_t *pool = (thrdpool_t *)arg;\n\tstruct __thrdpool_task_entry *entry;\n\tvoid (*task_routine)(void *);\n\tvoid *task_context;\n\n\tpthread_setspecific(pool->key, pool);\n\twhile (!pool->terminate)\n\t{\n\t\tentry = (struct __thrdpool_task_entry *)msgqueue_get(pool->msgqueue);\n\t\tif (!entry)\n\t\t\tbreak;\n\n\t\ttask_routine = entry->task.routine;\n\t\ttask_context = entry->task.context;\n\t\tfree(entry);\n\t\ttask_routine(task_context);\n\n\t\tif (pool->nthreads == 0)\n\t\t{\n\t\t\t/* Thread pool was destroyed by the task. */\n\t\t\tfree(pool);\n\t\t\treturn NULL;\n\t\t}\n\t}\n\n\t__thrdpool_exit_routine(pool);\n\treturn NULL;\n}\n\nstatic void __thrdpool_terminate(int in_pool, thrdpool_t *pool)\n{\n\tpthread_cond_t term = PTHREAD_COND_INITIALIZER;\n\n\tpthread_mutex_lock(&pool->mutex);\n\tmsgqueue_set_nonblock(pool->msgqueue);\n\tpool->terminate = &term;\n\n\tif (in_pool)\n\t{\n\t\t/* Thread pool destroyed in a pool thread is legal. */\n\t\tpthread_detach(pthread_self());\n\t\tpool->nthreads--;\n\t}\n\n\twhile (pool->nthreads > 0)\n\t\tpthread_cond_wait(&term, &pool->mutex);\n\n\tpthread_mutex_unlock(&pool->mutex);\n\tif (!pthread_equal(pool->tid, __zero_tid))\n\t\tpthread_join(pool->tid, NULL);\n\n\tpthread_cond_destroy(&term);\n}\n\nstatic int __thrdpool_create_threads(size_t nthreads, thrdpool_t *pool)\n{\n\tpthread_attr_t attr;\n\tpthread_t tid;\n\tint ret;\n\n\tret = pthread_attr_init(&attr);\n\tif (ret == 0)\n\t{\n\t\tif (pool->stacksize)\n\t\t\tret = pthread_attr_setstacksize(&attr, pool->stacksize);\n\n\t\tif (ret == 0)\n\t\t{\n\t\t\tpthread_mutex_lock(&pool->mutex);\n\t\t\twhile (pool->nthreads < nthreads)\n\t\t\t{\n\t\t\t\tret = pthread_create(&tid, &attr, __thrdpool_routine, pool);\n\t\t\t\tif (ret == 0)\n\t\t\t\t\tpool->nthreads++;\n\t\t\t\telse\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tpthread_mutex_unlock(&pool->mutex);\n\t\t}\n\n\t\tpthread_attr_destroy(&attr);\n\t\tif (ret == 0)\n\t\t\treturn 0;\n\n\t\t__thrdpool_terminate(0, pool);\n\t}\n\n\terrno = ret;\n\treturn -1;\n}\n\nthrdpool_t *thrdpool_create(size_t nthreads, size_t stacksize)\n{\n\tthrdpool_t *pool;\n\tint ret;\n\n\tpool = (thrdpool_t *)malloc(sizeof (thrdpool_t));\n\tif (!pool)\n\t\treturn NULL;\n\n\tpool->msgqueue = msgqueue_create(0, 0);\n\tif (pool->msgqueue)\n\t{\n\t\tret = pthread_mutex_init(&pool->mutex, NULL);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tret = pthread_key_create(&pool->key, NULL);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tpool->stacksize = stacksize;\n\t\t\t\tpool->nthreads = 0;\n\t\t\t\tpool->tid = __zero_tid;\n\t\t\t\tpool->terminate = NULL;\n\t\t\t\tif (__thrdpool_create_threads(nthreads, pool) >= 0)\n\t\t\t\t\treturn pool;\n\n\t\t\t\tpthread_key_delete(pool->key);\n\t\t\t}\n\n\t\t\tpthread_mutex_destroy(&pool->mutex);\n\t\t}\n\n\t\terrno = ret;\n\t\tmsgqueue_destroy(pool->msgqueue);\n\t}\n\n\tfree(pool);\n\treturn NULL;\n}\n\nvoid __thrdpool_schedule(const struct thrdpool_task *task, void *buf,\n\t\t\t\t\t\t thrdpool_t *pool)\n{\n\t((struct __thrdpool_task_entry *)buf)->task = *task;\n\tmsgqueue_put(buf, pool->msgqueue);\n}\n\nint thrdpool_schedule(const struct thrdpool_task *task, thrdpool_t *pool)\n{\n\tvoid *buf = malloc(sizeof (struct __thrdpool_task_entry));\n\n\tif (buf)\n\t{\n\t\t__thrdpool_schedule(task, buf, pool);\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nint thrdpool_in_pool(thrdpool_t *pool)\n{\n\treturn pthread_getspecific(pool->key) == pool;\n}\n\nint thrdpool_increase(thrdpool_t *pool)\n{\n\tpthread_attr_t attr;\n\tpthread_t tid;\n\tint ret;\n\n\tret = pthread_attr_init(&attr);\n\tif (ret == 0)\n\t{\n\t\tif (pool->stacksize)\n\t\t\tret = pthread_attr_setstacksize(&attr, pool->stacksize);\n\n\t\tif (ret == 0)\n\t\t{\n\t\t\tpthread_mutex_lock(&pool->mutex);\n\t\t\tret = pthread_create(&tid, &attr, __thrdpool_routine, pool);\n\t\t\tif (ret == 0)\n\t\t\t\tpool->nthreads++;\n\n\t\t\tpthread_mutex_unlock(&pool->mutex);\n\t\t}\n\n\t\tpthread_attr_destroy(&attr);\n\t\tif (ret == 0)\n\t\t\treturn 0;\n\t}\n\n\terrno = ret;\n\treturn -1;\n}\n\nint thrdpool_decrease(thrdpool_t *pool)\n{\n\tvoid *buf = malloc(sizeof (struct __thrdpool_task_entry));\n\tstruct __thrdpool_task_entry *entry;\n\n\tif (buf)\n\t{\n\t\tentry = (struct __thrdpool_task_entry *)buf;\n\t\tentry->task.routine = __thrdpool_exit_routine;\n\t\tentry->task.context = pool;\n\t\tmsgqueue_put_head(entry, pool->msgqueue);\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nvoid thrdpool_exit(thrdpool_t *pool)\n{\n\tif (thrdpool_in_pool(pool))\n\t\t__thrdpool_exit_routine(pool);\n}\n\nvoid thrdpool_destroy(void (*pending)(const struct thrdpool_task *),\n\t\t\t\t\t thrdpool_t *pool)\n{\n\tint in_pool = thrdpool_in_pool(pool);\n\tstruct __thrdpool_task_entry *entry;\n\n\t__thrdpool_terminate(in_pool, pool);\n\twhile (1)\n\t{\n\t\tentry = (struct __thrdpool_task_entry *)msgqueue_get(pool->msgqueue);\n\t\tif (!entry)\n\t\t\tbreak;\n\n\t\tif (pending && entry->task.routine != __thrdpool_exit_routine)\n\t\t\tpending(&entry->task);\n\n\t\tfree(entry);\n\t}\n\n\tpthread_key_delete(pool->key);\n\tpthread_mutex_destroy(&pool->mutex);\n\tmsgqueue_destroy(pool->msgqueue);\n\tif (!in_pool)\n\t\tfree(pool);\n}\n\n" }, { "path": "src/kernel/thrdpool.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _THRDPOOL_H_\n#define _THRDPOOL_H_\n\n#include \n\ntypedef struct __thrdpool thrdpool_t;\n\nstruct thrdpool_task\n{\n\tvoid (*routine)(void *);\n\tvoid *context;\n};\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nthrdpool_t *thrdpool_create(size_t nthreads, size_t stacksize);\nint thrdpool_schedule(const struct thrdpool_task *task, thrdpool_t *pool);\nint thrdpool_in_pool(thrdpool_t *pool);\nint thrdpool_increase(thrdpool_t *pool);\nint thrdpool_decrease(thrdpool_t *pool);\nvoid thrdpool_exit(thrdpool_t *pool);\nvoid thrdpool_destroy(void (*pending)(const struct thrdpool_task *),\n\t\t\t\t\t thrdpool_t *pool);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n\n" }, { "path": "src/kernel/xmake.lua", "content": "target(\"kernel\")\n set_kind(\"object\")\n add_files(\"*.cc\")\n add_files(\"*.c\")\n if is_plat(\"linux\", \"android\") then\n remove_files(\"IOService_thread.cc\")\n else\n \tremove_files(\"IOService_linux.cc\")\n end\n" }, { "path": "src/manager/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\nproject(manager)\n\nset(SRC\n\tDnsCache.cc\n\tRouteManager.cc\n\tWFGlobal.cc\n)\n\nif (NOT UPSTREAM STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tUpstreamManager.cc\n\t)\nendif ()\n\nadd_library(${PROJECT_NAME} OBJECT ${SRC})\n" }, { "path": "src/manager/DnsCache.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"DnsCache.h\"\n\n#define GET_CURRENT_SECOND\tstd::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()\n\n#define\tTTL_INC\t\t\t\t5\n\nconst DnsCache::DnsHandle *DnsCache::get_inner(const HostPort& host_port,\n\t\t\t\t\t\t\t\t\t\t\t int type)\n{\n\tint64_t cur = GET_CURRENT_SECOND;\n\tstd::lock_guard lock(mutex_);\n\tconst DnsHandle *handle = cache_pool_.get(host_port);\n\n\tif (handle && ((type == GET_TYPE_TTL && cur > handle->value.expire_time) ||\n\t\t(type == GET_TYPE_CONFIDENT && cur > handle->value.confident_time)))\n\t{\n\t\tif (!handle->value.delayed())\n\t\t{\n\t\t\tDnsHandle *h = const_cast(handle);\n\t\t\tif (type == GET_TYPE_TTL)\n\t\t\t\th->value.expire_time += TTL_INC;\n\t\t\telse\n\t\t\t\th->value.confident_time += TTL_INC;\n\n\t\t\th->value.addrinfo->ai_flags |= 2;\n\t\t}\n\n\t\tcache_pool_.release(handle);\n\t\treturn NULL;\n\t}\n\n\treturn handle;\n}\n\nconst DnsCache::DnsHandle *DnsCache::put(const HostPort& host_port,\n\t\t\t\t\t\t\t\t\t\t struct addrinfo *addrinfo,\n\t\t\t\t\t\t\t\t\t\t unsigned int dns_ttl_default,\n\t\t\t\t\t\t\t\t\t\t unsigned int dns_ttl_min)\n{\n\tint64_t expire_time;\n\tint64_t confident_time;\n\tint64_t cur_time = GET_CURRENT_SECOND;\n\n\tif (dns_ttl_min > dns_ttl_default)\n\t\tdns_ttl_min = dns_ttl_default;\n\n\tif (dns_ttl_min == (unsigned int)-1)\n\t\tconfident_time = INT64_MAX;\n\telse\n\t\tconfident_time = cur_time + dns_ttl_min;\n\n\tif (dns_ttl_default == (unsigned int)-1)\n\t\texpire_time = INT64_MAX;\n\telse\n\t\texpire_time = cur_time + dns_ttl_default;\n\n\tstd::lock_guard lock(mutex_);\n\treturn cache_pool_.put(host_port, {addrinfo, confident_time, expire_time});\n}\n\nconst DnsCache::DnsHandle *DnsCache::get(const DnsCache::HostPort& host_port)\n{\n\tstd::lock_guard lock(mutex_);\n\treturn cache_pool_.get(host_port);\n}\n\nvoid DnsCache::release(const DnsCache::DnsHandle *handle)\n{\n\tstd::lock_guard lock(mutex_);\n\tcache_pool_.release(handle);\n}\n\nvoid DnsCache::del(const DnsCache::HostPort& key)\n{\n\tstd::lock_guard lock(mutex_);\n\tcache_pool_.del(key);\n}\n\nDnsCache::DnsCache()\n{\n}\n\nDnsCache::~DnsCache()\n{\n}\n\n" }, { "path": "src/manager/DnsCache.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _DNSCACHE_H_\n#define _DNSCACHE_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \"LRUCache.h\"\n#include \"DnsUtil.h\"\n\n#define GET_TYPE_TTL\t\t0\n#define GET_TYPE_CONFIDENT\t1\n\nstruct DnsCacheValue\n{\n\tstruct addrinfo *addrinfo;\n\tint64_t confident_time;\n\tint64_t expire_time;\n\n\tbool delayed() const\n\t{\n\t\treturn addrinfo->ai_flags & 2;\n\t}\n};\n\n// RAII: NO. Release handle by user\n// Thread safety: YES\n// MUST call release when handle no longer used\nclass DnsCache\n{\npublic:\n\tusing HostPort = std::pair;\n\tusing DnsHandle = LRUHandle;\n\npublic:\n\t// get handler\n\t// Need call release when handle no longer needed\n\t//Handle *get(const KEY &key);\n\tconst DnsHandle *get(const HostPort& host_port);\n\n\tconst DnsHandle *get(const std::string& host, unsigned short port)\n\t{\n\t\treturn get(HostPort(host, port));\n\t}\n\n\tconst DnsHandle *get(const char *host, unsigned short port)\n\t{\n\t\treturn get(std::string(host), port);\n\t}\n\n\tconst DnsHandle *get_ttl(const HostPort& host_port)\n\t{\n\t\treturn get_inner(host_port, GET_TYPE_TTL);\n\t}\n\n\tconst DnsHandle *get_ttl(const std::string& host, unsigned short port)\n\t{\n\t\treturn get_ttl(HostPort(host, port));\n\t}\n\n\tconst DnsHandle *get_ttl(const char *host, unsigned short port)\n\t{\n\t\treturn get_ttl(std::string(host), port);\n\t}\n\n\tconst DnsHandle *get_confident(const HostPort& host_port)\n\t{\n\t\treturn get_inner(host_port, GET_TYPE_CONFIDENT);\n\t}\n\n\tconst DnsHandle *get_confident(const std::string& host, unsigned short port)\n\t{\n\t\treturn get_confident(HostPort(host, port));\n\t}\n\n\tconst DnsHandle *get_confident(const char *host, unsigned short port)\n\t{\n\t\treturn get_confident(std::string(host), port);\n\t}\n\n\tconst DnsHandle *put(const HostPort& host_port,\n\t\t\t\t\t\t struct addrinfo *addrinfo,\n\t\t\t\t\t\t unsigned int dns_ttl_default,\n\t\t\t\t\t\t unsigned int dns_ttl_min);\n\n\tconst DnsHandle *put(const std::string& host,\n\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t struct addrinfo *addrinfo,\n\t\t\t\t\t\t unsigned int dns_ttl_default,\n\t\t\t\t\t\t unsigned int dns_ttl_min)\n\t{\n\t\treturn put(HostPort(host, port), addrinfo, dns_ttl_default, dns_ttl_min);\n\t}\n\n\tconst DnsHandle *put(const char *host,\n\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t struct addrinfo *addrinfo,\n\t\t\t\t\t\t unsigned int dns_ttl_default,\n\t\t\t\t\t\t unsigned int dns_ttl_min)\n\t{\n\t\treturn put(std::string(host), port, addrinfo, dns_ttl_default, dns_ttl_min);\n\t}\n\n\t// release handle by get/put\n\tvoid release(const DnsHandle *handle);\n\n\t// delete from cache, deleter delay called when all inuse-handle release.\n\tvoid del(const HostPort& key);\n\n\tvoid del(const std::string& host, unsigned short port)\n\t{\n\t\tdel(HostPort(host, port));\n\t}\n\n\tvoid del(const char *host, unsigned short port)\n\t{\n\t\tdel(std::string(host), port);\n\t}\n\nprivate:\n\tconst DnsHandle *get_inner(const HostPort& host_port, int type);\n\n\tstd::mutex mutex_;\n\n\tclass ValueDeleter\n\t{\n\tpublic:\n\t\tvoid operator() (const DnsCacheValue& value) const\n\t\t{\n\t\t\tstruct addrinfo *ai = value.addrinfo;\n\n\t\t\tif (ai)\n\t\t\t{\n\t\t\t\tif (ai->ai_flags & 1)\n\t\t\t\t\tfreeaddrinfo(ai);\n\t\t\t\telse\n\t\t\t\t\tprotocol::DnsUtil::freeaddrinfo(ai);\n\t\t\t}\n\t\t}\n\t};\n\n\tLRUCache cache_pool_;\n\npublic:\n\t// To prevent inline calling LRUCache's constructor and deconstructor.\n\tDnsCache();\n\t~DnsCache();\n};\n\n#endif\n\n" }, { "path": "src/manager/EndpointParams.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _ENDPOINTPARAMS_H_\n#define _ENDPOINTPARAMS_H_\n\n#include \n#include \n\n/**\n * @file EndpointParams.h\n * @brief Network config for client task\n */\n\nenum TransportType\n{\n\tTT_TCP,\n\tTT_UDP,\n\tTT_SCTP,\n\tTT_TCP_SSL,\n\tTT_SCTP_SSL,\n};\n\nstruct EndpointParams\n{\n\tint address_family;\n\tsize_t max_connections;\n\tint connect_timeout;\n\tint response_timeout;\n\tint ssl_connect_timeout;\n\tbool use_tls_sni;\n};\n\nstatic constexpr struct EndpointParams ENDPOINT_PARAMS_DEFAULT =\n{\n\t.address_family\t\t\t=\tAF_UNSPEC,\n\t.max_connections\t\t=\t200,\n\t.connect_timeout\t\t=\t10 * 1000,\n\t.response_timeout\t\t=\t10 * 1000,\n\t.ssl_connect_timeout\t=\t10 * 1000,\n\t.use_tls_sni\t\t\t=\tfalse,\n};\n\n#endif\n\n" }, { "path": "src/manager/RouteManager.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"rbtree.h\"\n#include \"WFGlobal.h\"\n#include \"CommScheduler.h\"\n#include \"EndpointParams.h\"\n#include \"RouteManager.h\"\n#include \"StringUtil.h\"\n\n#define GET_CURRENT_SECOND\tstd::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()\n#define MTTR_SECOND\t\t\t30\n\nusing RouteTargetTCP = RouteManager::RouteTarget;\n\nclass RouteTargetUDP : public RouteManager::RouteTarget\n{\nprivate:\n\tvirtual int create_connect_fd()\n\t{\n\t\tconst struct sockaddr *addr;\n\t\tsocklen_t addrlen;\n\n\t\tthis->get_addr(&addr, &addrlen);\n\t\treturn socket(addr->sa_family, SOCK_DGRAM, 0);\n\t}\n};\n\nclass RouteTargetSCTP : public RouteManager::RouteTarget\n{\nprivate:\n#ifdef IPPROTO_SCTP\n\tvirtual int create_connect_fd()\n\t{\n\t\tconst struct sockaddr *addr;\n\t\tsocklen_t addrlen;\n\n\t\tthis->get_addr(&addr, &addrlen);\n\t\treturn socket(addr->sa_family, SOCK_STREAM, IPPROTO_SCTP);\n\t}\n#else\n\tvirtual int create_connect_fd()\n\t{\n\t\terrno = EPROTONOSUPPORT;\n\t\treturn -1;\n\t}\n#endif\n};\n\nusing RouteTargetTCP_SSL = RouteTargetTCP;\n\n/* To support TLS SNI. */\nclass RouteTargetTCP_TLS_SNI : public RouteTargetTCP_SSL\n{\nprivate:\n\tvirtual int init_ssl(SSL *ssl)\n\t{\n\t\tif (SSL_set_tlsext_host_name(ssl, this->hostname.c_str()) > 0)\n\t\t\treturn 0;\n\t\telse\n\t\t\treturn -1;\n\t}\n\nprivate:\n\tstd::string hostname;\n\npublic:\n\tRouteTargetTCP_TLS_SNI(const std::string& name) : hostname(name)\n\t{\n\t}\n};\n\nusing RouteTargetSCTP_SSL = RouteTargetSCTP;\n\nclass RouteTargetSCTP_TLS_SNI : public RouteTargetSCTP_SSL\n{\nprivate:\n\tvirtual int init_ssl(SSL *ssl)\n\t{\n\t\tif (SSL_set_tlsext_host_name(ssl, this->hostname.c_str()) > 0)\n\t\t\treturn 0;\n\t\telse\n\t\t\treturn -1;\n\t}\n\nprivate:\n\tstd::string hostname;\n\npublic:\n\tRouteTargetSCTP_TLS_SNI(const std::string& name) : hostname(name)\n\t{\n\t}\n};\n\n// protocol_name\\n user\\n pass\\n dbname\\n ai_addr ai_addrlen \\n....\n//\n\nstruct RouteParams\n{\n\tenum TransportType transport_type;\n\tconst struct addrinfo *addrinfo;\n\tuint64_t key;\n\tSSL_CTX *ssl_ctx;\n\tsize_t max_connections;\n\tint connect_timeout;\n\tint response_timeout;\n\tint ssl_connect_timeout;\n\tbool use_tls_sni;\n\tconst std::string& hostname;\n};\n\nclass RouteResultEntry\n{\npublic:\n\tstruct rb_node rb;\n\tCommSchedObject *request_object;\n\tCommSchedGroup *group;\n\tstd::mutex mutex;\n\tstd::vector targets;\n\tstruct list_head breaker_list;\n\tuint64_t key;\n\tint nleft;\n\tint nbreak;\n\n\tRouteResultEntry():\n\t\trequest_object(NULL),\n\t\tgroup(NULL)\n\t{\n\t\tINIT_LIST_HEAD(&this->breaker_list);\n\t\tthis->nleft = 0;\n\t\tthis->nbreak = 0;\n\t}\n\npublic:\n\tint init(const struct RouteParams *params);\n\tvoid deinit();\n\n\tvoid notify_unavailable(RouteManager::RouteTarget *target);\n\tvoid notify_available(RouteManager::RouteTarget *target);\n\tvoid check_breaker();\n\nprivate:\n\tvoid free_list();\n\tRouteManager::RouteTarget *create_target(const struct RouteParams *params,\n\t\t\t\t\t\t\t\t\t\t\t const struct addrinfo *addrinfo);\n\tint add_group_targets(const struct RouteParams *params);\n};\n\nstruct __breaker_node\n{\n\tRouteManager::RouteTarget *target;\n\tint64_t timeout;\n\tstruct list_head breaker_list;\n};\n\nRouteManager::RouteTarget *\nRouteResultEntry::create_target(const struct RouteParams *params,\n\t\t\t\t\t\t\t\tconst struct addrinfo *addr)\n{\n\tRouteManager::RouteTarget *target;\n\n\tswitch (params->transport_type)\n\t{\n\tcase TT_TCP:\n\t\ttarget = new RouteTargetTCP();\n\t\tbreak;\n\tcase TT_UDP:\n\t\ttarget = new RouteTargetUDP();\n\t\tbreak;\n\tcase TT_SCTP:\n\t\ttarget = new RouteTargetSCTP();\n\t\tbreak;\n\tcase TT_TCP_SSL:\n\t\tif (params->use_tls_sni)\n\t\t\ttarget = new RouteTargetTCP_TLS_SNI(params->hostname);\n\t\telse\n\t\t\ttarget = new RouteTargetTCP_SSL;\n\t\tbreak;\n\tcase TT_SCTP_SSL:\n\t\tif (params->use_tls_sni)\n\t\t\ttarget = new RouteTargetSCTP_TLS_SNI(params->hostname);\n\t\telse\n\t\t\ttarget = new RouteTargetSCTP_SSL;\n\t\tbreak;\n\tdefault:\n\t\terrno = EINVAL;\n\t\treturn NULL;\n\t}\n\n\tif (target->init(addr->ai_addr, addr->ai_addrlen, params->ssl_ctx,\n\t\t\t\t\t params->connect_timeout, params->ssl_connect_timeout,\n\t\t\t\t\t params->response_timeout, params->max_connections) < 0)\n\t{\n\t\tdelete target;\n\t\ttarget = NULL;\n\t}\n\n\treturn target;\n}\n\nint RouteResultEntry::init(const struct RouteParams *params)\n{\n\tconst struct addrinfo *addr = params->addrinfo;\n\tRouteManager::RouteTarget *target;\n\n\tif (addr == NULL)//0\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\tif (addr->ai_next == NULL)//1\n\t{\n\t\ttarget = this->create_target(params, addr);\n\t\tif (target)\n\t\t{\n\t\t\tthis->targets.push_back(target);\n\t\t\tthis->request_object = target;\n\t\t\tthis->key = params->key;\n\t\t\treturn 0;\n\t\t}\n\n\t\treturn -1;\n\t}\n\n\tthis->group = new CommSchedGroup();\n\tif (this->group->init() >= 0)\n\t{\n\t\tif (this->add_group_targets(params) >= 0)\n\t\t{\n\t\t\tthis->request_object = this->group;\n\t\t\tthis->key = params->key;\n\t\t\treturn 0;\n\t\t}\n\n\t\tthis->group->deinit();\n\t}\n\n\tdelete this->group;\n\treturn -1;\n}\n\nint RouteResultEntry::add_group_targets(const struct RouteParams *params)\n{\n\tRouteManager::RouteTarget *target;\n\tconst struct addrinfo *addr;\n\n\tfor (addr = params->addrinfo; addr; addr = addr->ai_next)\n\t{\n\t\ttarget = this->create_target(params, addr);\n\t\tif (target)\n\t\t{\n\t\t\tif (this->group->add(target) >= 0)\n\t\t\t{\n\t\t\t\tthis->targets.push_back(target);\n\t\t\t\tthis->nleft++;\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\ttarget->deinit();\n\t\t\tdelete target;\n\t\t}\n\n\t\tfor (auto *target : this->targets)\n\t\t{\n\t\t\tthis->group->remove(target);\n\t\t\ttarget->deinit();\n\t\t\tdelete target;\n\t\t}\n\n\t\treturn -1;\n\t}\n\n\treturn 0;\n}\n\nvoid RouteResultEntry::deinit()\n{\n\tfor (auto *target : this->targets)\n\t{\n\t\tif (this->group)\n\t\t\tthis->group->remove(target);\n\n\t\ttarget->deinit();\n\t\tdelete target;\n\t}\n\n\tif (this->group)\n\t{\n\t\tthis->group->deinit();\n\t\tdelete this->group;\n\t}\n\n\tstruct list_head *pos, *tmp;\n\t__breaker_node *node;\n\n\tlist_for_each_safe(pos, tmp, &this->breaker_list)\n\t{\n\t\tnode = list_entry(pos, __breaker_node, breaker_list);\n\t\tlist_del(pos);\n\t\tdelete node;\n\t}\n}\n\nvoid RouteResultEntry::notify_unavailable(RouteManager::RouteTarget *target)\n{\n\tif (this->targets.size() <= 1)\n\t\treturn;\n\n\tint errno_bak = errno;\n\tstd::lock_guard lock(this->mutex);\n\n\tif (this->nleft <= 1)\n\t\treturn;\n\n\tif (this->group->remove(target) < 0)\n\t{\n\t\terrno = errno_bak;\n\t\treturn;\n\t}\n\n\tauto *node = new __breaker_node;\n\n\tnode->target = target;\n\tnode->timeout = GET_CURRENT_SECOND + MTTR_SECOND;\n\tlist_add_tail(&node->breaker_list, &this->breaker_list);\n\tthis->nbreak++;\n\tthis->nleft--;\n}\n\nvoid RouteResultEntry::notify_available(RouteManager::RouteTarget *target)\n{\n\tif (this->targets.size() <= 1 || this->nbreak == 0)\n\t\treturn;\n\n\tint errno_bak = errno;\n\tstd::lock_guard lock(this->mutex);\n\n\tif (this->group->add(target) == 0)\n\t\tthis->nleft++;\n\telse\n\t\terrno = errno_bak;\n}\n\nvoid RouteResultEntry::check_breaker()\n{\n\tif (this->targets.size() <= 1 || this->nbreak == 0)\n\t\treturn;\n\n\tstruct list_head *pos, *tmp;\n\t__breaker_node *node;\n\tint errno_bak = errno;\n\tint64_t cur_time = GET_CURRENT_SECOND;\n\tstd::lock_guard lock(this->mutex);\n\n\tlist_for_each_safe(pos, tmp, &this->breaker_list)\n\t{\n\t\tnode = list_entry(pos, __breaker_node, breaker_list);\n\t\tif (cur_time >= node->timeout)\n\t\t{\n\t\t\tif (this->group->add(node->target) == 0)\n\t\t\t\tthis->nleft++;\n\t\t\telse\n\t\t\t\terrno = errno_bak;\n\n\t\t\tlist_del(pos);\n\t\t\tdelete node;\n\t\t\tthis->nbreak--;\n\t\t}\n\t}\n}\n\nstatic inline int __addr_cmp(const struct addrinfo *x, const struct addrinfo *y)\n{\n\t//todo ai_protocol\n\tif (x->ai_addrlen == y->ai_addrlen)\n\t\treturn memcmp(x->ai_addr, y->ai_addr, x->ai_addrlen);\n\telse if (x->ai_addrlen < y->ai_addrlen)\n\t\treturn -1;\n\telse\n\t\treturn 1;\n}\n\nstatic inline bool __addr_less(const struct addrinfo *x, const struct addrinfo *y)\n{\n\treturn __addr_cmp(x, y) < 0;\n}\n\nstatic uint64_t __fnv_hash(const unsigned char *data, size_t size)\n{\n\tuint64_t hash = 14695981039346656037ULL;\n\n\twhile (size)\n\t{\n\t\thash ^= (const uint64_t)*data++;\n\t\thash *= 1099511628211ULL;\n\t\tsize--;\n\t}\n\n\treturn hash;\n}\n\nstatic uint64_t __generate_key(enum TransportType type,\n\t\t\t\t\t\t\t const struct addrinfo *addrinfo,\n\t\t\t\t\t\t\t const std::string& other_info,\n\t\t\t\t\t\t\t const struct EndpointParams *ep_params,\n\t\t\t\t\t\t\t const std::string& hostname,\n\t\t\t\t\t\t\t SSL_CTX *ssl_ctx)\n{\n\tconst int params[] = {\n\t\tep_params->address_family, (int)ep_params->max_connections,\n\t\tep_params->connect_timeout, ep_params->response_timeout\n\t};\n\tstd::string buf((const char *)&type, sizeof (enum TransportType));\n\n\tif (!other_info.empty())\n\t\tbuf += other_info;\n\n\tbuf.append((const char *)params, sizeof params);\n\tif (type == TT_TCP_SSL || type == TT_SCTP_SSL)\n\t{\n\t\tbuf.append((const char *)ssl_ctx, sizeof (void *));\n\t\tbuf.append((const char *)&ep_params->ssl_connect_timeout, sizeof (int));\n\t\tif (ep_params->use_tls_sni)\n\t\t{\n\t\t\tbuf += hostname;\n\t\t\tbuf += '\\n';\n\t\t}\n\t}\n\n\tif (addrinfo->ai_next)\n\t{\n\t\tstd::vector sorted_addr;\n\n\t\tsorted_addr.push_back(addrinfo);\n\t\taddrinfo = addrinfo->ai_next;\n\t\tdo\n\t\t{\n\t\t\tsorted_addr.push_back(addrinfo);\n\t\t\taddrinfo = addrinfo->ai_next;\n\t\t} while (addrinfo);\n\n\t\tstd::sort(sorted_addr.begin(), sorted_addr.end(), __addr_less);\n\t\tfor (const struct addrinfo *p : sorted_addr)\n\t\t{\n\t\t\tbuf.append((const char *)&p->ai_addrlen, sizeof (socklen_t));\n\t\t\tbuf.append((const char *)p->ai_addr, p->ai_addrlen);\n\t\t}\n\t}\n\telse\n\t{\n\t\tbuf.append((const char *)&addrinfo->ai_addrlen, sizeof (socklen_t));\n\t\tbuf.append((const char *)addrinfo->ai_addr, addrinfo->ai_addrlen);\n\t}\n\n\treturn __fnv_hash((const unsigned char *)buf.c_str(), buf.size());\n}\n\nRouteManager::~RouteManager()\n{\n\tRouteResultEntry *entry;\n\n\twhile (cache_.rb_node)\n\t{\n\t\tentry = rb_entry(cache_.rb_node, RouteResultEntry, rb);\n\t\trb_erase(cache_.rb_node, &cache_);\n\t\tentry->deinit();\n\t\tdelete entry;\n\t}\n}\n\nint RouteManager::get(enum TransportType type,\n\t\t\t\t\t const struct addrinfo *addrinfo,\n\t\t\t\t\t const std::string& other_info,\n\t\t\t\t\t const struct EndpointParams *ep_params,\n\t\t\t\t\t const std::string& hostname, SSL_CTX *ssl_ctx,\n\t\t\t\t\t RouteResult& result)\n{\n\tif (type == TT_TCP_SSL || type == TT_SCTP_SSL)\n\t{\n\t\tstatic SSL_CTX *global_client_ctx = WFGlobal::get_ssl_client_ctx();\n\t\tif (ssl_ctx == NULL)\n\t\t\tssl_ctx = global_client_ctx;\n\t}\n\telse\n\t\tssl_ctx = NULL;\n\n\tuint64_t key = __generate_key(type, addrinfo, other_info, ep_params,\n\t\t\t\t\t\t\t\t hostname, ssl_ctx);\n\tstruct rb_node **p = &cache_.rb_node;\n\tstruct rb_node *parent = NULL;\n\tRouteResultEntry *bound = NULL;\n\tRouteResultEntry *entry;\n\tstd::lock_guard lock(mutex_);\n\n\twhile (*p)\n\t{\n\t\tparent = *p;\n\t\tentry = rb_entry(*p, RouteResultEntry, rb);\n\t\tif (key <= entry->key)\n\t\t{\n\t\t\tbound = entry;\n\t\t\tp = &(*p)->rb_left;\n\t\t}\n\t\telse\n\t\t\tp = &(*p)->rb_right;\n\t}\n\n\tif (bound && bound->key == key)\n\t{\n\t\tentry = bound;\n\t\tentry->check_breaker();\n\t}\n\telse\n\t{\n\t\tstruct RouteParams params = {\n\t\t\t.transport_type\t\t\t=\ttype,\n\t\t\t.addrinfo \t\t\t\t=\taddrinfo,\n\t\t\t.key\t\t\t\t\t=\tkey,\n\t\t\t.ssl_ctx\t\t\t\t=\tssl_ctx,\n\t\t\t.max_connections\t\t=\tep_params->max_connections,\n\t\t\t.connect_timeout\t\t=\tep_params->connect_timeout,\n\t\t\t.response_timeout\t\t=\tep_params->response_timeout,\n\t\t\t.ssl_connect_timeout\t=\tep_params->ssl_connect_timeout,\n\t\t\t.use_tls_sni\t\t\t=\tep_params->use_tls_sni,\n\t\t\t.hostname\t\t\t\t=\thostname,\n\t\t};\n\n\t\tentry = new RouteResultEntry;\n\t\tif (entry->init(¶ms) >= 0)\n\t\t{\n\t\t\trb_link_node(&entry->rb, parent, p);\n\t\t\trb_insert_color(&entry->rb, &cache_);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdelete entry;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\tresult.cookie = entry;\n\tresult.request_object = entry->request_object;\n\treturn 0;\n}\n\nvoid RouteManager::notify_unavailable(void *cookie, CommTarget *target)\n{\n\tif (cookie && target)\n\t\t((RouteResultEntry *)cookie)->notify_unavailable((RouteTarget *)target);\n}\n\nvoid RouteManager::notify_available(void *cookie, CommTarget *target)\n{\n\tif (cookie && target)\n\t\t((RouteResultEntry *)cookie)->notify_available((RouteTarget *)target);\n}\n\n" }, { "path": "src/manager/RouteManager.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _ROUTEMANAGER_H_\n#define _ROUTEMANAGER_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \"rbtree.h\"\n#include \"WFConnection.h\"\n#include \"EndpointParams.h\"\n#include \"CommScheduler.h\"\n\nclass RouteManager\n{\npublic:\n\tclass RouteResult\n\t{\n\tpublic:\n\t\tvoid *cookie;\n\t\tCommSchedObject *request_object;\n\n\tpublic:\n\t\tRouteResult(): cookie(NULL), request_object(NULL) { }\n\t\tvoid clear() { cookie = NULL; request_object = NULL; }\n\t};\n\n\tclass RouteTarget : public CommSchedTarget\n\t{\n\tpublic:\n\t\tint init(const struct sockaddr *addr, socklen_t addrlen, SSL_CTX *ssl_ctx,\n\t\t\t\t int connect_timeout, int ssl_connect_timeout, int response_timeout,\n\t\t\t\t size_t max_connections)\n\t\t{\n\t\t\tint ret = this->CommSchedTarget::init(addr, addrlen, ssl_ctx,\n\t\t\t\t\t\t\t\tconnect_timeout, ssl_connect_timeout,\n\t\t\t\t\t\t\t\tresponse_timeout, max_connections);\n\n\t\t\tif (ret >= 0 && ssl_ctx)\n\t\t\t\tSSL_CTX_up_ref(ssl_ctx);\n\n\t\t\treturn ret;\n\t\t}\n\n\t\tvoid deinit()\n\t\t{\n\t\t\tSSL_CTX *ssl_ctx = this->get_ssl_ctx();\n\n\t\t\tthis->CommSchedTarget::deinit();\n\t\t\tif (ssl_ctx)\n\t\t\t\tSSL_CTX_free(ssl_ctx);\n\t\t}\n\n\tpublic:\n\t\tint state;\n\n\tprivate:\n\t\tvirtual WFConnection *new_connection(int connect_fd)\n\t\t{\n\t\t\treturn new WFConnection;\n\t\t}\n\n\tpublic:\n\t\tRouteTarget() : state(0) { }\n\t};\n\npublic:\n\tint get(enum TransportType type,\n\t\t\tconst struct addrinfo *addrinfo,\n\t\t\tconst std::string& other_info,\n\t\t\tconst struct EndpointParams *ep_params,\n\t\t\tconst std::string& hostname, SSL_CTX *ssl_ctx,\n\t\t\tRouteResult& result);\n\n\tRouteManager()\n\t{\n\t\tcache_.rb_node = NULL;\n\t}\n\n\t~RouteManager();\n\nprivate:\n\tstd::mutex mutex_;\n\tstruct rb_root cache_;\n\npublic:\n\tstatic void notify_unavailable(void *cookie, CommTarget *target);\n\tstatic void notify_available(void *cookie, CommTarget *target);\n};\n\n#endif\n\n" }, { "path": "src/manager/UpstreamManager.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"UpstreamManager.h\"\n#include \"WFNameService.h\"\n#include \"WFGlobal.h\"\n#include \"UpstreamPolicies.h\"\n\nclass __UpstreamManager\n{\npublic:\n\tstatic __UpstreamManager *get_instance()\n\t{\n\t\tstatic __UpstreamManager kInstance;\n\t\treturn &kInstance;\n\t}\n\n\tvoid add_upstream_policy(UPSGroupPolicy *policy)\n\t{\n\t\tpthread_mutex_lock(&this->mutex);\n\t\tthis->upstream_policies.push_back(policy);\n\t\tpthread_mutex_unlock(&this->mutex);\n\t}\n\nprivate:\n\t__UpstreamManager() :\n\t\tmutex(PTHREAD_MUTEX_INITIALIZER)\n\t{\n\t}\n\n\t~__UpstreamManager()\n\t{\n\t\tfor (UPSGroupPolicy *policy : this->upstream_policies)\n\t\t\tdelete policy;\n\n\t\tpthread_mutex_destroy(&mutex);\n\t}\n\n\tpthread_mutex_t mutex;\n\tstd::vector upstream_policies;\n};\n\nint UpstreamManager::upstream_create_round_robin(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t bool try_another)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tauto *policy = new UPSRoundRobinPolicy(try_another);\n\n\tif (ns->add_policy(name.c_str(), policy) >= 0)\n\t{\n\t\t__UpstreamManager::get_instance()->add_upstream_policy(policy);\n\t\treturn 0;\n\t}\n\n\tdelete policy;\n\treturn -1;\n}\n\nstatic unsigned int __default_consistent_hash(const char *path,\n\t\t\t\t\t\t\t\t\t\t\t const char *query,\n\t\t\t\t\t\t\t\t\t\t\t const char *fragment)\n{\n\tstatic std::hash std_hash;\n\tstd::string str(path);\n\n\tstr += query;\n\tstr += fragment;\n\treturn std_hash(str);\n}\n\nint UpstreamManager::upstream_create_consistent_hash(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t upstream_route_t consistent_hash)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSConsistentHashPolicy *policy;\n\n\tpolicy = new UPSConsistentHashPolicy(\n\t\t\t\t\t\tconsistent_hash ? std::move(consistent_hash) :\n\t\t\t\t\t\t\t\t\t\t __default_consistent_hash);\n\tif (ns->add_policy(name.c_str(), policy) >= 0)\n\t{\n\t\t__UpstreamManager::get_instance()->add_upstream_policy(policy);\n\t\treturn 0;\n\t}\n\n\tdelete policy;\n\treturn -1;\n}\n\nint UpstreamManager::upstream_create_weighted_random(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t\t\t bool try_another)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tauto *policy = new UPSWeightedRandomPolicy(try_another);\n\n\tif (ns->add_policy(name.c_str(), policy) >= 0)\n\t{\n\t\t__UpstreamManager::get_instance()->add_upstream_policy(policy);\n\t\treturn 0;\n\t}\n\n\tdelete policy;\n\treturn -1;\n}\n\nint UpstreamManager::upstream_create_vnswrr(const std::string& name)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tauto *policy = new UPSVNSWRRPolicy();\n\n\tif (ns->add_policy(name.c_str(), policy) >= 0)\n\t{\n\t\t__UpstreamManager::get_instance()->add_upstream_policy(policy);\n\t\treturn 0;\n\t}\n\n\tdelete policy;\n\treturn -1;\n}\n\nint UpstreamManager::upstream_create_manual(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\tupstream_route_t select,\n\t\t\t\t\t\t\t\t\t\t\tbool try_another,\n\t\t\t\t\t\t\t\t\t\t\tupstream_route_t consistent_hash)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSManualPolicy *policy;\n\n\tpolicy = new UPSManualPolicy(try_another, std::move(select),\n\t\t\t\t\t\tconsistent_hash ? std::move(consistent_hash) :\n\t\t\t\t\t\t\t\t\t\t __default_consistent_hash);\n\tif (ns->add_policy(name.c_str(), policy) >= 0)\n\t{\n\t\t__UpstreamManager::get_instance()->add_upstream_policy(policy);\n\t\treturn 0;\n\t}\n\n\tdelete policy;\n\treturn -1;\n}\n\nint UpstreamManager::upstream_add_server(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t const std::string& address)\n{\n\treturn UpstreamManager::upstream_add_server(name, address,\n\t\t\t\t\t\t\t\t\t\t\t\t&ADDRESS_PARAMS_DEFAULT);\n}\n\nint UpstreamManager::upstream_add_server(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t const std::string& address,\n\t\t\t\t\t\t\t\t\t\t const AddressParams *address_params)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSGroupPolicy *policy = dynamic_cast(ns->get_policy(name.c_str()));\n\n\tif (policy)\n\t{\n\t\tpolicy->add_server(address, address_params);\n\t\treturn 0;\n\t}\n\n\terrno = ENOENT;\n\treturn -1;\n}\n\nint UpstreamManager::upstream_remove_server(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\tconst std::string& address)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSGroupPolicy *policy = dynamic_cast(ns->get_policy(name.c_str()));\n\n\tif (policy)\n\t\treturn policy->remove_server(address);\n\n\terrno = ENOENT;\n\treturn -1;\n}\n\nint UpstreamManager::upstream_delete(const std::string& name)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSGroupPolicy *policy = dynamic_cast(ns->del_policy(name.c_str()));\n\n\tif (policy)\n\t\treturn 0;\n\n\terrno = ENOENT;\n\treturn -1;\n}\n\nstd::vector\nUpstreamManager::upstream_main_address_list(const std::string& name)\n{\n\tstd::vector address;\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSGroupPolicy *policy = dynamic_cast(ns->get_policy(name.c_str()));\n\n\tif (policy)\n\t\tpolicy->get_main_address(address);\n\n\treturn address;\n}\n\nint UpstreamManager::upstream_disable_server(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t const std::string& address)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSGroupPolicy *policy = dynamic_cast(ns->get_policy(name.c_str()));\n\n\tif (policy)\n\t{\n\t\tpolicy->disable_server(address);\n\t\treturn 0;\n\t}\n\n\terrno = ENOENT;\n\treturn -1;\n}\n\nint UpstreamManager::upstream_enable_server(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\tconst std::string& address)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSGroupPolicy *policy = dynamic_cast(ns->get_policy(name.c_str()));\n\n\tif (policy)\n\t{\n\t\tpolicy->enable_server(address);\n\t\treturn 0;\n\t}\n\n\terrno = ENOENT;\n\treturn -1;\n}\n\nint UpstreamManager::upstream_replace_server(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t const std::string& address,\n\t\t\t\t\t\t\t\t\t\t\t const struct AddressParams *address_params)\n{\n\tWFNameService *ns = WFGlobal::get_name_service();\n\tUPSGroupPolicy *policy = dynamic_cast(ns->get_policy(name.c_str()));\n\n\tif (policy)\n\t{\n\t\tpolicy->replace_server(address, address_params);\n\t\treturn 0;\n\t}\n\n\terrno = ENOENT;\n\treturn -1;\n}\n\n" }, { "path": "src/manager/UpstreamManager.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _UPSTREAM_MANAGER_H_\n#define _UPSTREAM_MANAGER_H_\n\n#include \n#include \n#include \"WFServiceGovernance.h\"\n#include \"UpstreamPolicies.h\"\n#include \"WFGlobal.h\"\n\n/**\n * @file UpstreamManager.h\n * @brief Local Reverse Proxy & Load Balance & Service Discovery\n * @details\n * - This is very similar with Nginx-Upstream.\n * - Do not cost any other network resource, We just simulate in local to choose one target properly.\n * - This is working only for the current process.\n */\n\n/**\n * @brief Upstream Management Class\n * @details\n * - We support three modes:\n * 1. Weighted-Random\n * 2. Consistent-Hash\n * 3. Manual-Select\n * - Additional, we support Main-backup & Group for server and working well in any mode.\n *\n * @code{.cc}\n\tupstream_create_weighted_random(\"abc.sogou\", true); //UPSTREAM_WEIGHTED_RANDOM\n\tupstream_add_server(\"abc.sogou\", \"192.168.2.100:8081\"); //weight=1, max_fails=200\n\tupstream_add_server(\"abc.sogou\", \"192.168.2.100:9090\"); //weight=1, max_fails=200\n\tAddressParams params = ADDRESS_PARAMS_DEFAULT;\n\tparams.weight = 2;\n\tparams.max_fails = 6;\n\tupstream_add_server(\"abc.sogou\", \"www.sogou.com\", ¶ms); //weight=2, max_fails=6\n\n\t//send request with url like http://abc.sogou/somepath/somerequest\n\n\tupstream_create_consistent_hash(\"def.sogou\",\n\t\t\t\t\t\t\t\t [](const char *path,\n\t\t\t\t\t\t\t\t\t const char *query,\n\t\t\t\t\t\t\t\t\t const char *fragment) -> int {\n\t\t\t\t\t\t\t\t\t\t\treturn somehash(...));\n\t\t\t\t\t\t\t\t\t}); //UPSTREAM_CONSISTENT_HASH\n\tupstream_create_manual(\"xyz.sogou\",\n\t\t\t\t\t\t [](const char *path,\n\t\t\t\t\t\t\t const char *query,\n\t\t\t\t\t\t\t const char *fragment) -> int {\n\t\t\t\t\t\t\t\t\treturn select_xxx(...));\n\t\t\t\t\t\t },\n\t\t\t\t\t\t true,\n\t\t\t\t\t\t [](const char *path,\n\t\t\t\t\t\t\t const char *query,\n\t\t\t\t\t\t\t const char *fragment) -> int {\n\t\t\t\t\t\t\t\t\treturn rehash(...));\n\t\t\t\t\t\t },); //UPSTREAM_MANUAL\n * @endcode\n */\nclass UpstreamManager\n{\npublic:\n\t/**\n\t * @brief MODE 0: round-robin select\n\t * @param[in] name upstream name\n\t * @param[in] try_another when first choice is failed, try another one or not\n\t * @return success/fail\n\t * @retval 0 success\n\t * @retval -1 fail, more info see errno\n\t * @note\n\t * when first choose server is already down:\n\t * - if try_another==false, request will be failed\n\t * - if try_another==true, upstream will choose the next\n\t */\n\tstatic int upstream_create_round_robin(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t bool try_another);\n\n\t/**\n\t * @brief MODE 1: consistent-hashing select\n\t * @param[in] name upstream name\n\t * @param[in] consitent_hash consistent-hash functional\n\t * @return success/fail\n\t * @retval 0 success\n\t * @retval -1 fail, more info see errno\n\t * @note consitent_hash need to return value in 0~(2^31-1) Balance/Monotonicity/Spread/Smoothness\n\t * @note if consitent_hash==nullptr, upstream will use std::hash with request uri\n\t */\n\tstatic int upstream_create_consistent_hash(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t upstream_route_t consitent_hash);\n\n\t/**\n\t * @brief MODE 2: weighted-random select\n\t * @param[in] name upstream name\n\t * @param[in] try_another when first choice is failed, try another one or not\n\t * @return success/fail\n\t * @retval 0 success\n\t * @retval -1 fail, more info see errno\n\t * @note\n\t * when first choose server is already down:\n\t * - if try_another==false, request will be failed\n\t * - if try_another==true, upstream will choose from alive-servers by weight-random strategy\n\t */\n\tstatic int upstream_create_weighted_random(const std::string& name,\n\t\t\t\t\t\t\t\t\t\t\t bool try_another);\n\n\t/**\n\t * @brief MODE 3: manual select\n\t * @param[in] name upstream name\n\t * @param[in] select manual select functional, just tell us main-index.\n\t * @param[in] try_another when first choice is failed, try another one or not\n\t * @param[in] consitent_hash consistent-hash functional\n\t * @return success/fail\n\t * @retval 0 success\n\t * @retval -1 fail, more info see errno\n\t * @note\n\t * when first choose server is already down:\n\t * - if try_another==false, request will be failed, consistent_hash value will be ignored\n\t * - if try_another==true, upstream will work with consistent hash mode，if consitent_hash==NULL, upstream will use std::hash with request uri\n\t * @warning select functional cannot be nullptr!\n\t */\n\tstatic int upstream_create_manual(const std::string& name,\n\t\t\t\t\t\t\t\t\t upstream_route_t select,\n\t\t\t\t\t\t\t\t\t bool try_another,\n\t\t\t\t\t\t\t\t\t upstream_route_t consitent_hash);\n\n\t/**\n\t * @brief MODE 4: VNSWRR select\n\t * @param[in] name upstream name\n\t * @return success/fail\n\t * @retval 0 success\n\t * @retval -1 fail, more info see errno\n\t * @note\n\t */\n\tstatic int upstream_create_vnswrr(const std::string& name);\n\n\t/**\n\t * @brief Delete one upstream\n\t * @param[in] name upstream name\n\t * @return success/fail\n\t * @retval 0 success\n\t * @retval -1 fail, not found\n\t */\n\tstatic int upstream_delete(const std::string& name);\n\npublic:\n\t/**\n\t * @brief Add server into one upstream, with default config\n\t * @param[in] name upstream name\n\t * @param[in] address ip OR host OR ip:port OR host:port OR /unix-domain-socket\n\t * @return success/fail\n\t * @retval 0 success\n\t * @retval -1 fail, more info see errno\n\t * @warning Same address add twice, means two different server\n\t */\n\tstatic int upstream_add_server(const std::string& name,\n\t\t\t\t\t\t\t\t const std::string& address);\n\n\t/**\n\t * @brief Add server into one upstream, with custom config\n\t * @param[in] name upstream name\n\t * @param[in] address ip OR host OR ip:port OR host:port OR /unix-domain-socket\n\t * @param[in] address_params custom config for this target server\n\t * @return success/fail\n\t * @retval 0 success\n\t * @retval -1 fail, more info see errno\n\t * @warning Same address with different params, means two different server\n\t * @warning Same address with exactly same params, still means two different server\n\t */\n\tstatic int upstream_add_server(const std::string& name,\n\t\t\t\t\t\t\t\t const std::string& address,\n\t\t\t\t\t\t\t\t const struct AddressParams *address_params);\n\n\t/**\n\t * @brief Remove server from one upstream\n\t * @param[in] name upstream name\n\t * @param[in] address same as address when add by upstream_add_server\n\t * @return success/fail\n\t * @retval >=0 success, the amount of being removed server\n\t * @retval -1 fail, upstream name not found\n\t * @warning If server servers has the same address in this upstream, we will remove them all\n\t */\n\tstatic int upstream_remove_server(const std::string& name,\n\t\t\t\t\t\t\t\t\t const std::string& address);\n\n\t/**\n\t * @brief get all main servers address list from one upstream\n\t * @param[in] name upstream name\n\t * @return all main servers' address list\n\t * @warning If server servers has the same address in this upstream, then will appear in the vector multiply times\n\t */\n\tstatic std::vector upstream_main_address_list(const std::string& name);\n\npublic:\n\t/// @breif for plugin\n\tstatic int upstream_disable_server(const std::string& name, const std::string& address);\n\tstatic int upstream_enable_server(const std::string& name, const std::string& address);\n\n\tstatic int upstream_replace_server(const std::string& name,\n\t\t\t\t\t\t\t\t\t const std::string& address,\n\t\t\t\t\t\t\t\t\t const struct AddressParams *address_params);\n\n};\n\n#endif\n\n" }, { "path": "src/manager/WFFacilities.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFFACILITIES_H_\n#define _WFFACILITIES_H_\n\n#include \n#include \"WFFuture.h\"\n#include \"WFTaskFactory.h\"\n\nclass WFFacilities\n{\npublic:\n\tstatic void usleep(unsigned int microseconds);\n\tstatic WFFuture async_usleep(unsigned int microseconds);\n\npublic:\n\ttemplate\n\tstatic void go(const std::string& queue_name, FUNC&& func, ARGS&&... args);\n\npublic:\n\ttemplate\n\tstruct WFNetworkResult\n\t{\n\t\tRESP resp;\n\t\tlong long seqid;\n\t\tint task_state;\n\t\tint task_error;\n\t};\n\n\ttemplate\n\tstatic WFNetworkResult request(enum TransportType type, const std::string& url, REQ&& req, int retry_max);\n\n\ttemplate\n\tstatic WFFuture> async_request(enum TransportType type, const std::string& url, REQ&& req, int retry_max);\n\npublic:// async fileIO\n\tstatic WFFuture async_pread(int fd, void *buf, size_t count, off_t offset);\n\tstatic WFFuture async_pwrite(int fd, const void *buf, size_t count, off_t offset);\n\tstatic WFFuture async_preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset);\n\tstatic WFFuture async_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset);\n\tstatic WFFuture async_fsync(int fd);\n\tstatic WFFuture async_fdatasync(int fd);\n\npublic:\n\tclass WaitGroup\n\t{\n\tpublic:\n\t\tWaitGroup(int n);\n\t\t~WaitGroup();\n\n\t\tvoid wait() const;\n\t\tstd::future_status wait(int timeout) const;\n\t\tvoid add(int n);\n\t\tvoid done();\n\n\tprivate:\n\t\tstatic void __wait_group_callback(WFCounterTask *task);\n\n\t\tstd::atomic nleft;\n\t\tWFCounterTask *task;\n\t\tWFFuture future;\n\t};\n\nprivate:\n\tstatic void __timer_future_callback(WFTimerTask *task);\n\tstatic void __fio_future_callback(WFFileIOTask *task);\n\tstatic void __fvio_future_callback(WFFileVIOTask *task);\n\tstatic void __fsync_future_callback(WFFileSyncTask *task);\n};\n\n#include \"WFFacilities.inl\"\n\n#endif\n\n" }, { "path": "src/manager/WFFacilities.inl", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\ninline void WFFacilities::usleep(unsigned int microseconds)\n{\n\tasync_usleep(microseconds).get();\n}\n\ninline WFFuture WFFacilities::async_usleep(unsigned int microseconds)\n{\n\tauto *pr = new WFPromise();\n\tauto fr = pr->get_future();\n\tauto *task = WFTaskFactory::create_timer_task(microseconds, __timer_future_callback);\n\n\ttask->user_data = pr;\n\ttask->start();\n\treturn fr;\n}\n\ntemplate\nvoid WFFacilities::go(const std::string& queue_name, FUNC&& func, ARGS&&... args)\n{\n\tWFTaskFactory::create_go_task(queue_name, std::forward(func), std::forward(args)...)->start();\n}\n\ntemplate\nWFFacilities::WFNetworkResult WFFacilities::request(enum TransportType type, const std::string& url, REQ&& req, int retry_max)\n{\n\treturn async_request(type, url, std::forward(req), retry_max).get();\n}\n\ntemplate\nWFFuture> WFFacilities::async_request(enum TransportType type, const std::string& url, REQ&& req, int retry_max)\n{\n\tParsedURI uri;\n\tauto *pr = new WFPromise>();\n\tauto fr = pr->get_future();\n\tauto *task = new WFComplexClientTask(retry_max, [pr](WFNetworkTask *task) {\n\t\tWFNetworkResult res;\n\n\t\tres.seqid = task->get_task_seq();\n\t\tres.task_state = task->get_state();\n\t\tres.task_error = task->get_error();\n\t\tif (res.task_state == WFT_STATE_SUCCESS)\n\t\t\tres.resp = std::move(*task->get_resp());\n\n\t\tpr->set_value(std::move(res));\n\t\tdelete pr;\n\t});\n\n\tURIParser::parse(url, uri);\n\ttask->init(std::move(uri));\n\ttask->set_transport_type(type);\n\t*task->get_req() = std::forward(req);\n\ttask->start();\n\treturn fr;\n}\n\ninline WFFuture WFFacilities::async_pread(int fd, void *buf, size_t count, off_t offset)\n{\n\tauto *pr = new WFPromise();\n\tauto fr = pr->get_future();\n\tauto *task = WFTaskFactory::create_pread_task(fd, buf, count, offset, __fio_future_callback);\n\n\ttask->user_data = pr;\n\ttask->start();\n\treturn fr;\n}\n\ninline WFFuture WFFacilities::async_pwrite(int fd, const void *buf, size_t count, off_t offset)\n{\n\tauto *pr = new WFPromise();\n\tauto fr = pr->get_future();\n\tauto *task = WFTaskFactory::create_pwrite_task(fd, buf, count, offset, __fio_future_callback);\n\n\ttask->user_data = pr;\n\ttask->start();\n\treturn fr;\n}\n\ninline WFFuture WFFacilities::async_preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset)\n{\n\tauto *pr = new WFPromise();\n\tauto fr = pr->get_future();\n\tauto *task = WFTaskFactory::create_preadv_task(fd, iov, iovcnt, offset, __fvio_future_callback);\n\n\ttask->user_data = pr;\n\ttask->start();\n\treturn fr;\n}\n\ninline WFFuture WFFacilities::async_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset)\n{\n\tauto *pr = new WFPromise();\n\tauto fr = pr->get_future();\n\tauto *task = WFTaskFactory::create_pwritev_task(fd, iov, iovcnt, offset, __fvio_future_callback);\n\n\ttask->user_data = pr;\n\ttask->start();\n\treturn fr;\n}\n\ninline WFFuture WFFacilities::async_fsync(int fd)\n{\n\tauto *pr = new WFPromise();\n\tauto fr = pr->get_future();\n\tauto *task = WFTaskFactory::create_fsync_task(fd, __fsync_future_callback);\n\n\ttask->user_data = pr;\n\ttask->start();\n\treturn fr;\n}\n\ninline WFFuture WFFacilities::async_fdatasync(int fd)\n{\n\tauto *pr = new WFPromise();\n\tauto fr = pr->get_future();\n\tauto *task = WFTaskFactory::create_fdatasync_task(fd, __fsync_future_callback);\n\n\ttask->user_data = pr;\n\ttask->start();\n\treturn fr;\n}\n\ninline void WFFacilities::__timer_future_callback(WFTimerTask *task)\n{\n\tauto *pr = static_cast *>(task->user_data);\n\n\tpr->set_value();\n\tdelete pr;\n}\n\ninline void WFFacilities::__fio_future_callback(WFFileIOTask *task)\n{\n\tauto *pr = static_cast *>(task->user_data);\n\n\tpr->set_value(task->get_retval());\n\tdelete pr;\n}\n\ninline void WFFacilities::__fvio_future_callback(WFFileVIOTask *task)\n{\n\tauto *pr = static_cast *>(task->user_data);\n\n\tpr->set_value(task->get_retval());\n\tdelete pr;\n}\n\ninline void WFFacilities::__fsync_future_callback(WFFileSyncTask *task)\n{\n\tauto *pr = static_cast *>(task->user_data);\n\n\tpr->set_value(task->get_retval());\n\tdelete pr;\n}\n\ninline WFFacilities::WaitGroup::WaitGroup(int n) : nleft(n)\n{\n\tif (n <= 0)\n\t{\n\t\tthis->nleft = -1;\n\t\treturn;\n\t}\n\n\tauto *pr = new WFPromise();\n\n\tthis->task = WFTaskFactory::create_counter_task(1, __wait_group_callback);\n\tthis->future = pr->get_future();\n\tthis->task->user_data = pr;\n\tthis->task->start();\n}\n\ninline WFFacilities::WaitGroup::~WaitGroup()\n{\n\tif (this->nleft > 0)\n\t\tthis->task->count();\n}\n\ninline void WFFacilities::WaitGroup::wait() const\n{\n\tif (this->nleft < 0)\n\t\treturn;\n\n\tthis->future.wait();\n}\n\ninline std::future_status WFFacilities::WaitGroup::wait(int timeout) const\n{\n\tif (this->nleft < 0)\n\t\treturn std::future_status::ready;\n\n\tif (timeout < 0)\n\t{\n\t\tthis->future.wait();\n\t\treturn std::future_status::ready;\n\t}\n\n\treturn this->future.wait_for(std::chrono::milliseconds(timeout));\n}\n\ninline void WFFacilities::WaitGroup::add(int n)\n{\n\tif ((this->nleft += n) == 0)\n\t{\n\t\tthis->task->count();\n\t}\n}\n\ninline void WFFacilities::WaitGroup::done()\n{\n\tthis->add(-1);\n}\n\ninline void WFFacilities::WaitGroup::__wait_group_callback(WFCounterTask *task)\n{\n\tauto *pr = static_cast *>(task->user_data);\n\n\tpr->set_value();\n\tdelete pr;\n}\n\n" }, { "path": "src/manager/WFFuture.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#ifndef _WFFUTURE_H_\n#define _WFFUTURE_H_\n\n#include \n#include \n#include \n#include \"CommScheduler.h\"\n#include \"WFGlobal.h\"\n\ntemplate\nclass WFFuture\n{\npublic:\n\tWFFuture(std::future&& fr) :\n\t\tfuture(std::move(fr))\n\t{\n\t}\n\n\tWFFuture() = default;\n\tWFFuture(const WFFuture&) = delete;\n\tWFFuture(WFFuture&& move) = default;\n\n\tWFFuture& operator=(const WFFuture&) = delete;\n\tWFFuture& operator=(WFFuture&& move) = default;\n\n\tvoid wait() const;\n\n\ttemplate\n\tstd::future_status wait_for(const std::chrono::duration& time_duration) const;\n\n\ttemplate\n\tstd::future_status wait_until(const std::chrono::time_point& timeout_time) const;\n\n\tRES get()\n\t{\n\t\tthis->wait();\n\t\treturn this->future.get();\n\t}\n\n\tbool valid() const { return this->future.valid(); }\n\nprivate:\n\tstd::future future;\n};\n\ntemplate\nclass WFPromise\n{\npublic:\n\tWFPromise() = default;\n\tWFPromise(const WFPromise& promise) = delete;\n\tWFPromise(WFPromise&& move) = default;\n\tWFPromise& operator=(const WFPromise& promise) = delete;\n\tWFPromise& operator=(WFPromise&& move) = default;\n\n\tWFFuture get_future()\n\t{\n\t\treturn WFFuture(this->promise.get_future());\n\t}\n\n\tvoid set_value(const RES& value) { this->promise.set_value(value); }\n\tvoid set_value(RES&& value) { this->promise.set_value(std::move(value)); }\n\nprivate:\n\tstd::promise promise;\n};\n\ntemplate\nvoid WFFuture::wait() const\n{\n\tif (this->future.wait_for(std::chrono::seconds(0)) != std::future_status::ready)\n\t{\n\t\tint cookie = WFGlobal::sync_operation_begin();\n\t\tthis->future.wait();\n\t\tWFGlobal::sync_operation_end(cookie);\n\t}\n}\n\ntemplate\ntemplate\nstd::future_status WFFuture::wait_for(const std::chrono::duration& time_duration) const\n{\n\tstd::future_status status = std::future_status::ready;\n\n\tif (this->future.wait_for(std::chrono::seconds(0)) != std::future_status::ready)\n\t{\n\t\tint cookie = WFGlobal::sync_operation_begin();\n\t\tstatus = this->future.wait_for(time_duration);\n\t\tWFGlobal::sync_operation_end(cookie);\n\t}\n\n\treturn status;\n}\n\ntemplate\ntemplate\nstd::future_status WFFuture::wait_until(const std::chrono::time_point& timeout_time) const\n{\n\tstd::future_status status = std::future_status::ready;\n\n\tif (this->future.wait_for(std::chrono::seconds(0)) != std::future_status::ready)\n\t{\n\t\tint cookie = WFGlobal::sync_operation_begin();\n\t\tstatus = this->future.wait_until(timeout_time);\n\t\tWFGlobal::sync_operation_end(cookie);\n\t}\n\n\treturn status;\n}\n\n///// WFFuture template specialization\ntemplate<>\ninline void WFFuture::get()\n{\n\tthis->wait();\n\tthis->future.get();\n}\n\ntemplate<>\nclass WFPromise\n{\npublic:\n\tWFPromise() = default;\n\tWFPromise(const WFPromise& promise) = delete;\n\tWFPromise(WFPromise&& move) = default;\n\tWFPromise& operator=(const WFPromise& promise) = delete;\n\tWFPromise& operator=(WFPromise&& move) = default;\n\n\tWFFuture get_future()\n\t{\n\t\treturn WFFuture(this->promise.get_future());\n\t}\n\n\tvoid set_value() { this->promise.set_value(); }\n//\tvoid set_value(const RES& value) { this->promise.set_value(value); }\n//\tvoid set_value(RES&& value) { this->promise.set_value(std::move(value)); }\n\nprivate:\n\tstd::promise promise;\n};\n\n#endif\n\n" }, { "path": "src/manager/WFGlobal.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"CommScheduler.h\"\n#include \"Executor.h\"\n#include \"WFResourcePool.h\"\n#include \"WFTaskError.h\"\n#include \"WFDnsClient.h\"\n#include \"WFGlobal.h\"\n#include \"URIParser.h\"\n\nclass __WFGlobal\n{\npublic:\n\tstatic __WFGlobal *get_instance()\n\t{\n\t\tstatic __WFGlobal kInstance;\n\t\treturn &kInstance;\n\t}\n\n\tconst char *get_default_port(const std::string& scheme)\n\t{\n\t\tconst auto it = static_scheme_port_.find(scheme);\n\n\t\tif (it != static_scheme_port_.end())\n\t\t\treturn it->second;\n\n\t\tconst char *port = NULL;\n\t\tuser_scheme_port_mutex_.lock();\n\t\tconst auto it2 = user_scheme_port_.find(scheme);\n\n\t\tif (it2 != user_scheme_port_.end())\n\t\t\tport = it2->second.c_str();\n\n\t\tuser_scheme_port_mutex_.unlock();\n\t\treturn port;\n\t}\n\n\tvoid register_scheme_port(const std::string& scheme, unsigned short port)\n\t{\n\t\tuser_scheme_port_mutex_.lock();\n\t\tuser_scheme_port_[scheme] = std::to_string(port);\n\t\tuser_scheme_port_mutex_.unlock();\n\t}\n\n\tvoid sync_operation_begin()\n\t{\n\t\tbool inc;\n\n\t\tsync_mutex_.lock();\n\t\tinc = ++sync_count_ > sync_max_;\n\t\tif (inc)\n\t\t\tsync_max_ = sync_count_;\n\n\t\tsync_mutex_.unlock();\n\t\tif (inc)\n\t\t\tWFGlobal::increase_handler_thread();\n\t}\n\n\tvoid sync_operation_end()\n\t{\n\t\tint dec = 0;\n\n\t\tsync_mutex_.lock();\n\t\tif (--sync_count_ < (sync_max_ + 1) / 2)\n\t\t{\n\t\t\tdec = sync_max_ - 2 * sync_count_;\n\t\t\tsync_max_ -= dec;\n\t\t}\n\n\t\tsync_mutex_.unlock();\n\t\twhile (dec > 0)\n\t\t{\n\t\t\tWFGlobal::decrease_handler_thread();\n\t\t\tdec--;\n\t\t}\n\t}\n\nprivate:\n\t__WFGlobal();\n\nprivate:\n\tstd::unordered_map static_scheme_port_;\n\tstd::unordered_map user_scheme_port_;\n\tstd::mutex user_scheme_port_mutex_;\n\tstd::mutex sync_mutex_;\n\tint sync_count_;\n\tint sync_max_;\n};\n\n__WFGlobal::__WFGlobal()\n{\n\tstatic_scheme_port_[\"dns\"] = \"53\";\n\tstatic_scheme_port_[\"Dns\"] = \"53\";\n\tstatic_scheme_port_[\"DNS\"] = \"53\";\n\n\tstatic_scheme_port_[\"dnss\"] = \"853\";\n\tstatic_scheme_port_[\"Dnss\"] = \"853\";\n\tstatic_scheme_port_[\"DNSs\"] = \"853\";\n\tstatic_scheme_port_[\"DNSS\"] = \"853\";\n\n\tstatic_scheme_port_[\"http\"] = \"80\";\n\tstatic_scheme_port_[\"Http\"] = \"80\";\n\tstatic_scheme_port_[\"HTTP\"] = \"80\";\n\n\tstatic_scheme_port_[\"https\"] = \"443\";\n\tstatic_scheme_port_[\"Https\"] = \"443\";\n\tstatic_scheme_port_[\"HTTPs\"] = \"443\";\n\tstatic_scheme_port_[\"HTTPS\"] = \"443\";\n\n\tstatic_scheme_port_[\"redis\"] = \"6379\";\n\tstatic_scheme_port_[\"Redis\"] = \"6379\";\n\tstatic_scheme_port_[\"REDIS\"] = \"6379\";\n\n\tstatic_scheme_port_[\"rediss\"] = \"6379\";\n\tstatic_scheme_port_[\"Rediss\"] = \"6379\";\n\tstatic_scheme_port_[\"REDISs\"] = \"6379\";\n\tstatic_scheme_port_[\"REDISS\"] = \"6379\";\n\n\tstatic_scheme_port_[\"mysql\"] = \"3306\";\n\tstatic_scheme_port_[\"Mysql\"] = \"3306\";\n\tstatic_scheme_port_[\"MySql\"] = \"3306\";\n\tstatic_scheme_port_[\"MySQL\"] = \"3306\";\n\tstatic_scheme_port_[\"MYSQL\"] = \"3306\";\n\n\tstatic_scheme_port_[\"mysqls\"] = \"3306\";\n\tstatic_scheme_port_[\"Mysqls\"] = \"3306\";\n\tstatic_scheme_port_[\"MySqls\"] = \"3306\";\n\tstatic_scheme_port_[\"MySQLs\"] = \"3306\";\n\tstatic_scheme_port_[\"MYSQLs\"] = \"3306\";\n\tstatic_scheme_port_[\"MYSQLS\"] = \"3306\";\n\n\tstatic_scheme_port_[\"kafka\"] = \"9092\";\n\tstatic_scheme_port_[\"Kafka\"] = \"9092\";\n\tstatic_scheme_port_[\"KAFKA\"] = \"9092\";\n\n\tstatic_scheme_port_[\"kafkas\"] = \"9093\";\n\tstatic_scheme_port_[\"Kafkas\"] = \"9093\";\n\tstatic_scheme_port_[\"KAFKAs\"] = \"9093\";\n\tstatic_scheme_port_[\"KAFKAS\"] = \"9093\";\n\n\tsync_count_ = 0;\n\tsync_max_ = 0;\n}\n\nclass __SSLManager\n{\npublic:\n\tstatic __SSLManager *get_instance()\n\t{\n\t\tstatic __SSLManager kInstance;\n\t\treturn &kInstance;\n\t}\n\n\tSSL_CTX *get_ssl_client_ctx() { return ssl_client_ctx_; }\n\tSSL_CTX *new_ssl_server_ctx() { return SSL_CTX_new(SSLv23_server_method()); }\n\nprivate:\n\t__SSLManager()\n\t{\n\t\tssl_client_ctx_ = SSL_CTX_new(SSLv23_client_method());\n\t\tif (ssl_client_ctx_ == NULL)\n\t\t\tabort();\n\t}\n\n\t~__SSLManager()\n\t{\n\t\tSSL_CTX_free(ssl_client_ctx_);\n\t}\n\nprivate:\n\tSSL_CTX *ssl_client_ctx_;\n};\n\nclass __FileIOService : public IOService\n{\npublic:\n\t__FileIOService(CommScheduler *scheduler):\n\t\tscheduler_(scheduler),\n\t\tflag_(true)\n\t{}\n\n\tint bind()\n\t{\n\t\tmutex_.lock();\n\t\tflag_ = false;\n\n\t\tint ret = scheduler_->io_bind(this);\n\n\t\tif (ret < 0)\n\t\t\tflag_ = true;\n\n\t\tmutex_.unlock();\n\t\treturn ret;\n\t}\n\n\tvoid deinit()\n\t{\n\t\tstd::unique_lock lock(mutex_);\n\t\twhile (!flag_)\n\t\t\tcond_.wait(lock);\n\n\t\tlock.unlock();\n\t\tIOService::deinit();\n\t}\n\nprivate:\n\tvirtual void handle_unbound()\n\t{\n\t\tmutex_.lock();\n\t\tflag_ = true;\n\t\tcond_.notify_one();\n\t\tmutex_.unlock();\n\t}\n\n\tvirtual void handle_stop(int error)\n\t{\n\t\tscheduler_->io_unbind(this);\n\t}\n\n\tCommScheduler *scheduler_;\n\tstd::mutex mutex_;\n\tstd::condition_variable cond_;\n\tbool flag_;\n};\n\nclass __ThreadDnsManager\n{\npublic:\n\tstatic __ThreadDnsManager *get_instance()\n\t{\n\t\tstatic __ThreadDnsManager kInstance;\n\t\treturn &kInstance;\n\t}\n\n\tExecQueue *get_dns_queue() { return &dns_queue_; }\n\tExecutor *get_dns_executor() { return &dns_executor_; }\n\n\t__ThreadDnsManager()\n\t{\n\t\tint ret;\n\n\t\tret = dns_queue_.init();\n\t\tif (ret < 0)\n\t\t\tabort();\n\n\t\tret = dns_executor_.init(WFGlobal::get_global_settings()->dns_threads);\n\t\tif (ret < 0)\n\t\t\tabort();\n\t}\n\n\t~__ThreadDnsManager()\n\t{\n\t\tdns_executor_.deinit();\n\t\tdns_queue_.deinit();\n\t}\n\nprivate:\n\tExecQueue dns_queue_;\n\tExecutor dns_executor_;\n};\n\nclass __CommManager\n{\npublic:\n\tstatic __CommManager *get_instance()\n\t{\n\t\tstatic __CommManager kInstance;\n\t\t__CommManager::created_ = true;\n\t\treturn &kInstance;\n\t}\n\n\tCommScheduler *get_scheduler() { return &scheduler_; }\n\tIOService *get_io_service();\n\tstatic bool is_created() { return created_; }\n\nprivate:\n\t__CommManager():\n\t\tfio_service_(NULL),\n\t\tfio_flag_(false)\n\t{\n\t\tconst auto *settings = WFGlobal::get_global_settings();\n\t\tif (scheduler_.init(settings->poller_threads,\n\t\t\t\t\t\t\tsettings->handler_threads) < 0)\n\t\t\tabort();\n\n\t\tsignal(SIGPIPE, SIG_IGN);\n\t}\n\n\t~__CommManager()\n\t{\n\t\t// scheduler_.deinit() will triger fio_service to stop\n\t\tscheduler_.deinit();\n\t\tif (fio_service_)\n\t\t{\n\t\t\tfio_service_->deinit();\n\t\t\tdelete fio_service_;\n\t\t}\n\t}\n\nprivate:\n\tCommScheduler scheduler_;\n\t__FileIOService *fio_service_;\n\tvolatile bool fio_flag_;\n\tstd::mutex fio_mutex_;\n\nprivate:\n\tstatic bool created_;\n};\n\nbool __CommManager::created_ = false;\n\ninline IOService *__CommManager::get_io_service()\n{\n\tif (!fio_flag_)\n\t{\n\t\tfio_mutex_.lock();\n\t\tif (!fio_flag_)\n\t\t{\n\t\t\tint maxevents = WFGlobal::get_global_settings()->fio_max_events;\n\t\t\tint n = 65536;\n\n\t\t\tfio_service_ = new __FileIOService(&scheduler_);\n\t\t\twhile (fio_service_->init(maxevents) < 0)\n\t\t\t{\n\t\t\t\tif ((errno != EAGAIN && errno != EINVAL) || maxevents <= 16)\n\t\t\t\t\tabort();\n\n\t\t\t\twhile (n >= maxevents)\n\t\t\t\t\tn /= 2;\n\n\t\t\t\tmaxevents = n;\n\t\t\t}\n\n\t\t\tif (fio_service_->bind() < 0)\n\t\t\t\tabort();\n\n\t\t\tfio_flag_ = true;\n\t\t}\n\n\t\tfio_mutex_.unlock();\n\t}\n\n\treturn fio_service_;\n}\n\nclass __ExecManager\n{\nprotected:\n\tusing ExecQueueMap = std::unordered_map;\n\npublic:\n\tstatic __ExecManager *get_instance()\n\t{\n\t\tstatic __ExecManager kInstance;\n\t\treturn &kInstance;\n\t}\n\n\tExecQueue *get_exec_queue(const std::string& queue_name);\n\tExecutor *get_compute_executor() { return &compute_executor_; }\n\nprivate:\n\t__ExecManager():\n\t\trwlock_(PTHREAD_RWLOCK_INITIALIZER)\n\t{\n\t\tint compute_threads = WFGlobal::get_global_settings()->compute_threads;\n\n\t\tif (compute_threads < 0)\n\t\t\tcompute_threads = sysconf(_SC_NPROCESSORS_ONLN);\n\n\t\tif (compute_executor_.init(compute_threads) < 0)\n\t\t\tabort();\n\t}\n\n\t~__ExecManager()\n\t{\n\t\tcompute_executor_.deinit();\n\n\t\tfor (auto& kv : queue_map_)\n\t\t{\n\t\t\tkv.second->deinit();\n\t\t\tdelete kv.second;\n\t\t}\n\n\t\tpthread_rwlock_destroy(&rwlock_);\n\t}\n\nprivate:\n\tpthread_rwlock_t rwlock_;\n\tExecQueueMap queue_map_;\n\tExecutor compute_executor_;\n};\n\ninline ExecQueue *__ExecManager::get_exec_queue(const std::string& queue_name)\n{\n\tExecQueue *queue = NULL;\n\tExecQueueMap::const_iterator iter;\n\n\tpthread_rwlock_rdlock(&rwlock_);\n\titer = queue_map_.find(queue_name);\n\tif (iter != queue_map_.cend())\n\t\tqueue = iter->second;\n\n\tpthread_rwlock_unlock(&rwlock_);\n\tif (queue)\n\t\treturn queue;\n\n\tpthread_rwlock_wrlock(&rwlock_);\n\titer = queue_map_.find(queue_name);\n\tif (iter == queue_map_.cend())\n\t{\n\t\tqueue = new ExecQueue();\n\t\tif (queue->init() >= 0)\n\t\t\tqueue_map_.emplace(queue_name, queue);\n\t\telse\n\t\t{\n\t\t\tdelete queue;\n\t\t\tqueue = NULL;\n\t\t}\n\t}\n\telse\n\t\tqueue = iter->second;\n\n\tpthread_rwlock_unlock(&rwlock_);\n\treturn queue;\n}\n\nstatic std::string __dns_server_url(const std::string& url,\n\t\t\t\t\t\t\t\t\tconst struct addrinfo *hints)\n{\n\tstd::string host;\n\tParsedURI uri;\n\tstruct addrinfo *res;\n\tstruct in6_addr buf;\n\n\tif (strncasecmp(url.c_str(), \"dns://\", 6) == 0 ||\n\t\tstrncasecmp(url.c_str(), \"dnss://\", 7) == 0)\n\t{\n\t\thost = url;\n\t}\n\telse if (inet_pton(AF_INET6, url.c_str(), &buf) > 0)\n\t\thost = \"dns://[\" + url + \"]\";\n\telse\n\t\thost = \"dns://\" + url;\n\n\tif (URIParser::parse(host, uri) == 0 && uri.host && uri.host[0])\n\t{\n\t\tif (getaddrinfo(uri.host, \"53\", hints, &res) == 0)\n\t\t{\n\t\t\tfreeaddrinfo(res);\n\t\t\treturn host;\n\t\t}\n\t}\n\n\treturn \"\";\n}\n\nstatic void __split_merge_str(const char *p, bool is_nameserver,\n\t\t\t\t\t\t\t const struct addrinfo *hints,\n\t\t\t\t\t\t\t std::string& result)\n{\n\tconst char *start;\n\n\tif (!isspace(*p))\n\t\treturn;\n\n\twhile (1)\n\t{\n\t\twhile (isspace(*p))\n\t\t\tp++;\n\n\t\tstart = p;\n\t\twhile (*p && *p != '#' && *p != ';' && !isspace(*p))\n\t\t\tp++;\n\n\t\tif (start == p)\n\t\t\tbreak;\n\n\t\tstd::string str(start, p);\n\t\tif (is_nameserver)\n\t\t\tstr = __dns_server_url(str, hints);\n\n\t\tif (!str.empty())\n\t\t{\n\t\t\tif (!result.empty())\n\t\t\t\tresult.push_back(',');\n\n\t\t\tresult.append(str);\n\t\t}\n\t}\n}\n\nstatic inline const char *__try_options(const char *p, const char *q,\n\t\t\t\t\t\t\t\t\t\tconst char *r)\n{\n\tsize_t len = strlen(r);\n\tif ((size_t)(q - p) >= len && strncmp(p, r, len) == 0)\n\t\treturn p + len;\n\treturn NULL;\n}\n\nstatic void __set_options(const char *p,\n\t\t\t\t\t\t int *ndots, int *attempts, bool *rotate)\n{\n\tconst char *start;\n\tconst char *opt;\n\n\tif (!isspace(*p))\n\t\treturn;\n\n\twhile (1)\n\t{\n\t\twhile (isspace(*p))\n\t\t\tp++;\n\n\t\tstart = p;\n\t\twhile (*p && *p != '#' && *p != ';' && !isspace(*p))\n\t\t\tp++;\n\n\t\tif (start == p)\n\t\t\tbreak;\n\n\t\tif ((opt = __try_options(start, p, \"ndots:\")) != NULL)\n\t\t\t*ndots = atoi(opt);\n\t\telse if ((opt = __try_options(start, p, \"attempts:\")) != NULL)\n\t\t\t*attempts = atoi(opt);\n\t\telse if ((opt = __try_options(start, p, \"rotate\")) != NULL)\n\t\t\t*rotate = true;\n\t}\n}\n\nstatic int __parse_resolv_conf(const char *path,\n\t\t\t\t\t\t\t std::string& url, std::string& search_list,\n\t\t\t\t\t\t\t int *ndots, int *attempts, bool *rotate)\n{\n\tsize_t bufsize = 0;\n\tchar *line = NULL;\n\tFILE *fp;\n\tint ret;\n\n\tfp = fopen(path, \"r\");\n\tif (!fp)\n\t\treturn -1;\n\n\tconst struct WFGlobalSettings *settings = WFGlobal::get_global_settings();\n\tstruct addrinfo hints = {\n\t\t.ai_flags\t\t=\tAI_ADDRCONFIG | AI_NUMERICHOST | AI_NUMERICSERV,\n\t\t.ai_family\t\t=\tsettings->dns_server_params.address_family,\n\t\t.ai_socktype\t=\tSOCK_STREAM,\n\t};\n\n\twhile ((ret = getline(&line, &bufsize, fp)) > 0)\n\t{\n\t\tif (strncmp(line, \"nameserver\", 10) == 0)\n\t\t\t__split_merge_str(line + 10, true, &hints, url);\n\t\telse if (strncmp(line, \"search\", 6) == 0)\n\t\t\t__split_merge_str(line + 6, false, &hints, search_list);\n\t\telse if (strncmp(line, \"options\", 7) == 0)\n\t\t\t__set_options(line + 7, ndots, attempts, rotate);\n\t}\n\n\tret = ferror(fp) ? -1 : 0;\n\tfree(line);\n\tfclose(fp);\n\treturn ret;\n}\n\nclass __DnsClientManager\n{\npublic:\n\tstatic __DnsClientManager *get_instance()\n\t{\n\t\tstatic __DnsClientManager kInstance;\n\t\treturn &kInstance;\n\t}\n\npublic:\n\tWFDnsClient *get_dns_client() { return client_; }\n\tWFResourcePool *get_dns_respool() { return &respool_; };\n\nprivate:\n\t__DnsClientManager() : respool_(WFGlobal::get_global_settings()->\n\t\t\t\t\t\t\t\t\tdns_server_params.max_connections)\n\t{\n\t\tconst char *path = WFGlobal::get_global_settings()->resolv_conf_path;\n\n\t\tclient_ = NULL;\n\t\tif (path && path[0])\n\t\t{\n\t\t\tint ndots = 1;\n\t\t\tint attempts = 2;\n\t\t\tbool rotate = false;\n\t\t\tstd::string url;\n\t\t\tstd::string search;\n\n\t\t\t__parse_resolv_conf(path, url, search, &ndots, &attempts, &rotate);\n\t\t\tif (url.size() == 0)\n\t\t\t\turl = \"8.8.8.8\";\n\n\t\t\tclient_ = new WFDnsClient;\n\t\t\tif (client_->init(url, search, ndots, attempts, rotate) >= 0)\n\t\t\t\treturn;\n\n\t\t\tdelete client_;\n\t\t\tclient_ = NULL;\n\t\t}\n\t}\n\n\t~__DnsClientManager()\n\t{\n\t\tif (client_)\n\t\t{\n\t\t\tclient_->deinit();\n\t\t\tdelete client_;\n\t\t}\n\t}\n\n\tWFDnsClient *client_;\n\tWFResourcePool respool_;\n};\n\nstruct WFGlobalSettings WFGlobal::settings_ = GLOBAL_SETTINGS_DEFAULT;\nRouteManager WFGlobal::route_manager_;\nDnsCache WFGlobal::dns_cache_;\nWFDnsResolver WFGlobal::dns_resolver_;\nWFNameService WFGlobal::name_service_(&WFGlobal::dns_resolver_);\n\nbool WFGlobal::is_scheduler_created()\n{\n\treturn __CommManager::is_created();\n}\n\nCommScheduler *WFGlobal::get_scheduler()\n{\n\treturn __CommManager::get_instance()->get_scheduler();\n}\n\nSSL_CTX *WFGlobal::get_ssl_client_ctx()\n{\n\treturn __SSLManager::get_instance()->get_ssl_client_ctx();\n}\n\nSSL_CTX *WFGlobal::new_ssl_server_ctx()\n{\n\treturn __SSLManager::get_instance()->new_ssl_server_ctx();\n}\n\nExecQueue *WFGlobal::get_exec_queue(const std::string& queue_name)\n{\n\treturn __ExecManager::get_instance()->get_exec_queue(queue_name);\n}\n\nExecutor *WFGlobal::get_compute_executor()\n{\n\treturn __ExecManager::get_instance()->get_compute_executor();\n}\n\nIOService *WFGlobal::get_io_service()\n{\n\treturn __CommManager::get_instance()->get_io_service();\n}\n\nExecQueue *WFGlobal::get_dns_queue()\n{\n\treturn __ThreadDnsManager::get_instance()->get_dns_queue();\n}\n\nExecutor *WFGlobal::get_dns_executor()\n{\n\treturn __ThreadDnsManager::get_instance()->get_dns_executor();\n}\n\nWFDnsClient *WFGlobal::get_dns_client()\n{\n\treturn __DnsClientManager::get_instance()->get_dns_client();\n}\n\nWFResourcePool *WFGlobal::get_dns_respool()\n{\n\treturn __DnsClientManager::get_instance()->get_dns_respool();\n}\n\nconst char *WFGlobal::get_default_port(const std::string& scheme)\n{\n\treturn __WFGlobal::get_instance()->get_default_port(scheme);\n}\n\nvoid WFGlobal::register_scheme_port(const std::string& scheme,\n\t\t\t\t\t\t\t\t\tunsigned short port)\n{\n\t__WFGlobal::get_instance()->register_scheme_port(scheme, port);\n}\n\nint WFGlobal::sync_operation_begin()\n{\n\tif (WFGlobal::is_scheduler_created() &&\n\t\tWFGlobal::get_scheduler()->is_handler_thread())\n\t{\n\t\t__WFGlobal::get_instance()->sync_operation_begin();\n\t\treturn 1;\n\t}\n\n\treturn 0;\n}\n\nvoid WFGlobal::sync_operation_end(int cookie)\n{\n\tif (cookie)\n\t\t__WFGlobal::get_instance()->sync_operation_end();\n}\n\nstatic inline const char *__get_ssl_error_string(int error)\n{\n\tswitch (error)\n\t{\n\tcase SSL_ERROR_NONE:\n\t\treturn \"SSL Error None\";\n\n\tcase SSL_ERROR_ZERO_RETURN:\n\t\treturn \"SSL Error Zero Return\";\n\n\tcase SSL_ERROR_WANT_READ:\n\t\treturn \"SSL Error Want Read\";\n\n\tcase SSL_ERROR_WANT_WRITE:\n\t\treturn \"SSL Error Want Write\";\n\n\tcase SSL_ERROR_WANT_CONNECT:\n\t\treturn \"SSL Error Want Connect\";\n\n\tcase SSL_ERROR_WANT_ACCEPT:\n\t\treturn \"SSL Error Want Accept\";\n\n\tcase SSL_ERROR_WANT_X509_LOOKUP:\n\t\treturn \"SSL Error Want X509 Lookup\";\n\n#ifdef SSL_ERROR_WANT_ASYNC\n\tcase SSL_ERROR_WANT_ASYNC:\n\t\treturn \"SSL Error Want Async\";\n#endif\n\n#ifdef SSL_ERROR_WANT_ASYNC_JOB\n\tcase SSL_ERROR_WANT_ASYNC_JOB:\n\t\treturn \"SSL Error Want Async Job\";\n#endif\n\n#ifdef SSL_ERROR_WANT_CLIENT_HELLO_CB\n\tcase SSL_ERROR_WANT_CLIENT_HELLO_CB:\n\t\treturn \"SSL Error Want Client Hello CB\";\n#endif\n\n\tcase SSL_ERROR_SYSCALL:\n\t\treturn \"SSL System Error\";\n\n\tcase SSL_ERROR_SSL:\n\t\treturn \"SSL Error SSL\";\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\treturn \"Unknown\";\n}\n\nstatic inline const char *__get_task_error_string(int error)\n{\n\tswitch (error)\n\t{\n\tcase WFT_ERR_URI_PARSE_FAILED:\n\t\treturn \"URI Parse Failed\";\n\n\tcase WFT_ERR_URI_SCHEME_INVALID:\n\t\treturn \"URI Scheme Invalid\";\n\n\tcase WFT_ERR_URI_PORT_INVALID:\n\t\treturn \"URI Port Invalid\";\n\n\tcase WFT_ERR_UPSTREAM_UNAVAILABLE:\n\t\treturn \"Upstream Unavailable\";\n\n\tcase WFT_ERR_HTTP_BAD_REDIRECT_HEADER:\n\t\treturn \"Http Bad Redirect Header\";\n\n\tcase WFT_ERR_HTTP_PROXY_CONNECT_FAILED:\n\t\treturn \"Http Proxy Connect Failed\";\n\n\tcase WFT_ERR_REDIS_ACCESS_DENIED:\n\t\treturn \"Redis Access Denied\";\n\n\tcase WFT_ERR_REDIS_COMMAND_DISALLOWED:\n\t\treturn \"Redis Command Disallowed\";\n\n\tcase WFT_ERR_MYSQL_HOST_NOT_ALLOWED:\n\t\treturn \"MySQL Host Not Allowed\";\n\n\tcase WFT_ERR_MYSQL_ACCESS_DENIED:\n\t\treturn \"MySQL Access Denied\";\n\n\tcase WFT_ERR_MYSQL_INVALID_CHARACTER_SET:\n\t\treturn \"MySQL Invalid Character Set\";\n\n\tcase WFT_ERR_MYSQL_COMMAND_DISALLOWED:\n\t\treturn \"MySQL Command Disallowed\";\n\n\tcase WFT_ERR_MYSQL_QUERY_NOT_SET:\n\t\treturn \"MySQL Query Not Set\";\n\n\tcase WFT_ERR_MYSQL_SSL_NOT_SUPPORTED:\n\t\treturn \"MySQL SSL Not Supported\";\n\n\tcase WFT_ERR_KAFKA_PARSE_RESPONSE_FAILED:\n\t\treturn \"Kafka parse response failed\";\n\n\tcase WFT_ERR_KAFKA_PRODUCE_FAILED:\n\t\treturn \"Kafka produce api failed\";\n\n\tcase WFT_ERR_KAFKA_FETCH_FAILED:\n\t\treturn \"Kafka fetch api failed\";\n\n\tcase WFT_ERR_KAFKA_CGROUP_FAILED:\n\t\treturn \"Kafka cgroup failed\";\n\n\tcase WFT_ERR_KAFKA_COMMIT_FAILED:\n\t\treturn \"Kafka commit api failed\";\n\n\tcase WFT_ERR_KAFKA_META_FAILED:\n\t\treturn \"Kafka meta api failed\";\n\n\tcase WFT_ERR_KAFKA_LEAVEGROUP_FAILED:\n\t\treturn \"Kafka leavegroup failed\";\n\n\tcase WFT_ERR_KAFKA_API_UNKNOWN:\n\t\treturn \"Kafka api type unknown\";\n\n\tcase WFT_ERR_KAFKA_VERSION_DISALLOWED:\n\t\treturn \"Kafka broker version not supported\";\n\n case WFT_ERR_KAFKA_SASL_DISALLOWED:\n return \"Kafka sasl disallowed\";\n\t\n case WFT_ERR_KAFKA_ARRANGE_FAILED:\n return \"Kafka arrange failed\";\n\t\n case WFT_ERR_KAFKA_LIST_OFFSETS_FAILED:\n return \"Kafka list offsets failed\";\n\n case WFT_ERR_KAFKA_CGROUP_ASSIGN_FAILED:\n return \"Kafka cgroup assign failed\";\n\t\t\t\n\tcase WFT_ERR_CONSUL_API_UNKNOWN:\n\t\treturn \"Consul api type unknown\";\n\n\tcase WFT_ERR_CONSUL_CHECK_RESPONSE_FAILED:\n\t\treturn \"Consul check response failed\";\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\treturn \"Unknown\";\n}\n\nconst char *WFGlobal::get_error_string(int state, int error)\n{\n\tswitch (state)\n\t{\n\tcase WFT_STATE_SUCCESS:\n\t\treturn \"Success\";\n\n\tcase WFT_STATE_TOREPLY:\n\t\treturn \"To Reply\";\n\n\tcase WFT_STATE_NOREPLY:\n\t\treturn \"No Reply\";\n\n\tcase WFT_STATE_SYS_ERROR:\n\t\treturn strerror(error);\n\n\tcase WFT_STATE_SSL_ERROR:\n\t\treturn __get_ssl_error_string(error);\n\n\tcase WFT_STATE_DNS_ERROR:\n\t\treturn gai_strerror(error);\n\n\tcase WFT_STATE_TASK_ERROR:\n\t\treturn __get_task_error_string(error);\n\n\tcase WFT_STATE_ABORTED:\n\t\treturn \"Aborted\";\n\n\tcase WFT_STATE_UNDEFINED:\n\t\treturn \"Undefined\";\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\treturn \"Unknown\";\n}\n\nvoid WORKFLOW_library_init(const struct WFGlobalSettings *settings)\n{\n\tWFGlobal::set_global_settings(settings);\n}\n\n" }, { "path": "src/manager/WFGlobal.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _WFGLOBAL_H_\n#define _WFGLOBAL_H_\n\n#if __cplusplus < 201100\n#error CPLUSPLUS VERSION required at least C++11. Please use \"-std=c++11\".\n#include \n#endif\n\n#include \n#include \n#include \"CommScheduler.h\"\n#include \"DnsCache.h\"\n#include \"RouteManager.h\"\n#include \"Executor.h\"\n#include \"EndpointParams.h\"\n#include \"WFResourcePool.h\"\n#include \"WFNameService.h\"\n#include \"WFDnsResolver.h\"\n\n/**\n * @file WFGlobal.h\n * @brief Workflow Global Settings & Workflow Global APIs\n */\n\n/**\n * @brief Workflow Library Global Setting\n * @details\n * If you want set different settings with default, please call WORKFLOW_library_init at the beginning of the process\n*/\nstruct WFGlobalSettings\n{\n\tstruct EndpointParams endpoint_params;\n\tstruct EndpointParams dns_server_params;\n\tunsigned int dns_ttl_default;\t///< in seconds, DNS TTL when network request success\n\tunsigned int dns_ttl_min;\t\t///< in seconds, DNS TTL when network request fail\n\tint dns_threads;\n\tint poller_threads;\n\tint handler_threads;\n\tint compute_threads;\t\t\t///< auto-set by system CPU number if value<0\n\tint fio_max_events;\n\tconst char *resolv_conf_path;\n\tconst char *hosts_path;\n};\n\n/**\n * @brief Default Workflow Library Global Settings\n */\nstatic constexpr struct WFGlobalSettings GLOBAL_SETTINGS_DEFAULT =\n{\n\t.endpoint_params\t=\tENDPOINT_PARAMS_DEFAULT,\n\t.dns_server_params\t=\tENDPOINT_PARAMS_DEFAULT,\n\t.dns_ttl_default\t=\t3600,\n\t.dns_ttl_min\t\t=\t60,\n\t.dns_threads\t\t=\t4,\n\t.poller_threads\t\t=\t4,\n\t.handler_threads\t=\t20,\n\t.compute_threads\t=\t-1,\n\t.fio_max_events\t\t=\t4096,\n\t.resolv_conf_path\t=\t\"/etc/resolv.conf\",\n\t.hosts_path\t\t\t=\t\"/etc/hosts\",\n};\n\n/**\n * @brief Reset Workflow Library Global Setting\n * @param[in] settings custom settings pointer\n*/\nextern void WORKFLOW_library_init(const struct WFGlobalSettings *settings);\n\n/**\n * @brief Workflow Global Management Class\n * @details Workflow Global APIs\n */\nclass WFGlobal\n{\npublic:\n\t/**\n\t * @brief register default port for one scheme string\n\t * @param[in] scheme scheme string\n\t * @param[in] port default port value\n\t * @warning No effect when scheme is \"http\"/\"https\"/\"redis\"/\"rediss\"/\"mysql\"/\"kafka\"\n\t */\n\tstatic void register_scheme_port(const std::string& scheme,\n\t\t\t\t\t\t\t\t\t unsigned short port);\n\t/**\n\t * @brief get default port string for one scheme string\n\t * @param[in] scheme scheme string\n\t * @return port string const pointer\n\t * @retval NULL fail, scheme not found\n\t * @retval not NULL success\n\t */\n\tstatic const char *get_default_port(const std::string& scheme);\n\t/**\n\t * @brief get current global settings\n\t * @return current global settings const pointer\n\t * @note returnval never NULL\n\t */\n\tstatic const struct WFGlobalSettings *get_global_settings()\n\t{\n\t\treturn &settings_;\n\t}\n\n\tstatic void set_global_settings(const struct WFGlobalSettings *settings)\n\t{\n\t\tsettings_ = *settings;\n\t}\n\n\tstatic const char *get_error_string(int state, int error);\n\n\tstatic bool increase_handler_thread()\n\t{\n\t\treturn WFGlobal::get_scheduler()->increase_handler_thread() == 0;\n\t}\n\n\tstatic bool decrease_handler_thread()\n\t{\n\t\treturn WFGlobal::get_scheduler()->decrease_handler_thread() == 0;\n\t}\n\n\tstatic bool increase_compute_thread()\n\t{\n\t\treturn WFGlobal::get_compute_executor()->increase_thread() == 0;\n\t}\n\n\tstatic bool decrease_compute_thread()\n\t{\n\t\treturn WFGlobal::get_compute_executor()->decrease_thread() == 0;\n\t}\n\n\t// Internal usage only\npublic:\n\tstatic bool is_scheduler_created();\n\tstatic class CommScheduler *get_scheduler();\n\tstatic SSL_CTX *get_ssl_client_ctx();\n\tstatic SSL_CTX *new_ssl_server_ctx();\n\tstatic class ExecQueue *get_exec_queue(const std::string& queue_name);\n\tstatic class Executor *get_compute_executor();\n\tstatic class IOService *get_io_service();\n\tstatic class ExecQueue *get_dns_queue();\n\tstatic class Executor *get_dns_executor();\n\tstatic class WFDnsClient *get_dns_client();\n\tstatic class WFResourcePool *get_dns_respool();\n\n\tstatic class RouteManager *get_route_manager()\n\t{\n\t\treturn &route_manager_;\n\t}\n\n\tstatic class DnsCache *get_dns_cache()\n\t{\n\t\treturn &dns_cache_;\n\t}\n\n\tstatic class WFDnsResolver *get_dns_resolver()\n\t{\n\t\treturn &dns_resolver_;\n\t}\n\n\tstatic class WFNameService *get_name_service()\n\t{\n\t\treturn &name_service_;\n\t}\n\npublic:\n\tstatic int sync_operation_begin();\n\tstatic void sync_operation_end(int cookie);\n\nprivate:\n\tstatic struct WFGlobalSettings settings_;\n\tstatic RouteManager route_manager_;\n\tstatic DnsCache dns_cache_;\n\tstatic WFDnsResolver dns_resolver_;\n\tstatic WFNameService name_service_;\n};\n\n#endif\n\n" }, { "path": "src/manager/xmake.lua", "content": "target(\"manager\")\n add_files(\"*.cc\")\n set_kind(\"object\")\n if not has_config(\"upstream\") then\n remove_files(\"UpstreamManager.cc\")\n end\n" }, { "path": "src/nameservice/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\nproject(nameservice)\n\nset(SRC\n\tWFNameService.cc\n\tWFDnsResolver.cc\n)\n\nif (NOT UPSTREAM STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tWFServiceGovernance.cc\n\t\tUpstreamPolicies.cc\n\t)\nendif ()\n\nadd_library(${PROJECT_NAME} OBJECT ${SRC})\n" }, { "path": "src/nameservice/UpstreamPolicies.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"rbtree.h\"\n#include \"URIParser.h\"\n#include \"UpstreamPolicies.h\"\n\nclass EndpointGroup\n{\npublic:\n\tEndpointGroup(int group_id, UPSGroupPolicy *policy) :\n\t\t\tmutex(PTHREAD_MUTEX_INITIALIZER),\n\t\t\tgen(rand())\n\t{\n\t\tthis->id = group_id;\n\t\tthis->policy = policy;\n\t\tthis->nalives = 0;\n\t\tthis->weight = 0;\n\t}\n\n\t~EndpointGroup()\n\t{\n\t\tpthread_mutex_destroy(&this->mutex);\n\t}\n\n\tEndpointAddress *get_one(WFNSTracing *tracing);\n\tEndpointAddress *get_one_backup(WFNSTracing *tracing);\n\npublic:\n\tint id;\n\tUPSGroupPolicy *policy;\n\tstruct rb_node rb;\n\tpthread_mutex_t mutex;\n\tstd::mt19937 gen;\n\tstd::vector mains;\n\tstd::vector backups;\n\tstd::atomic nalives;\n\tint weight;\n};\n\nUPSAddrParams::UPSAddrParams(const struct AddressParams *params,\n\t\t\t\t\t\t\t const std::string& address) :\n\tPolicyAddrParams(params)\n{\n\tthis->weight = params->weight;\n\tthis->server_type = params->server_type;\n\tthis->group_id = params->group_id;\n\n\tif (this->group_id < 0)\n\t\tthis->group_id = -1;\n\n\tif (this->weight == 0)\n\t\tthis->weight = 1;\n}\n\nvoid UPSGroupPolicy::get_main_address(std::vector& addr_list)\n{\n\tUPSAddrParams *params;\n\tpthread_rwlock_rdlock(&this->rwlock);\n\n\tfor (const EndpointAddress *server : this->servers)\n\t{\n\t\tparams = static_cast(server->params);\n\t\tif (params->server_type == 0)\n\t\t\taddr_list.push_back(server->address);\n\t}\n\n\tpthread_rwlock_unlock(&this->rwlock);\n}\n\nUPSGroupPolicy::UPSGroupPolicy()\n{\n\tthis->group_map.rb_node = NULL;\n\tthis->default_group = new EndpointGroup(-1, this);\n\trb_link_node(&this->default_group->rb, NULL, &this->group_map.rb_node);\n\trb_insert_color(&this->default_group->rb, &this->group_map);\n}\n\nUPSGroupPolicy::~UPSGroupPolicy()\n{\n\tEndpointGroup *group;\n\n\twhile (this->group_map.rb_node)\n\t{\n\t\tgroup = rb_entry(this->group_map.rb_node, EndpointGroup, rb);\n\t\trb_erase(this->group_map.rb_node, &this->group_map);\n\t\tdelete group;\n\t}\n}\n\ninline bool UPSGroupPolicy::is_alive(const EndpointAddress *addr) const\n{\n\tUPSAddrParams *params = static_cast(addr->params);\n\treturn ((params->group_id < 0 &&\n\t\t\t\taddr->fail_count < addr->params->max_fails) ||\n\t\t\t(params->group_id >= 0 &&\n\t\t\t\tparams->group->nalives > 0));\n}\n\nvoid UPSGroupPolicy::recover_one_server(const EndpointAddress *addr)\n{\n\tthis->nalives++;\n\tUPSAddrParams *params = static_cast(addr->params);\n\tparams->group->nalives++;\n}\n\nvoid UPSGroupPolicy::fuse_one_server(const EndpointAddress *addr)\n{\n\tthis->nalives--;\n\tUPSAddrParams *params = static_cast(addr->params);\n\tparams->group->nalives--;\n}\n\nvoid UPSGroupPolicy::add_server(const std::string& address,\n\t\t\t\t\t\t\t\tconst AddressParams *params)\n{\n\tEndpointAddress *addr = new EndpointAddress(address,\n\t\t\t\t\t\t\t\t\tnew UPSAddrParams(params, address));\n\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tthis->add_server_locked(addr);\n\tpthread_rwlock_unlock(&this->rwlock);\n}\n\nint UPSGroupPolicy::replace_server(const std::string& address,\n\t\t\t\t\t\t\t\t const AddressParams *params)\n{\n\tint ret;\n\tEndpointAddress *addr = new EndpointAddress(address,\n\t\t\t\t\t\t\t\t\tnew UPSAddrParams(params, address));\n\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tret = this->remove_server_locked(address);\n\tthis->add_server_locked(addr);\n\tpthread_rwlock_unlock(&this->rwlock);\n\treturn ret;\n}\n\nbool UPSGroupPolicy::select(const ParsedURI& uri, WFNSTracing *tracing,\n\t\t\t\t\t\t\tEndpointAddress **addr)\n{\n\tpthread_rwlock_rdlock(&this->rwlock);\n\tunsigned int n = (unsigned int)this->servers.size();\n\n\tif (n == 0)\n\t{\n\t\tpthread_rwlock_unlock(&this->rwlock);\n\t\treturn false;\n\t}\n\n\tthis->check_breaker();\n\n\t// select_addr == NULL will happen only in consistent_hash\n\tEndpointAddress *select_addr = this->first_strategy(uri, tracing);\n\n\tif (!select_addr || select_addr->fail_count >= select_addr->params->max_fails)\n\t{\n\t\tif (select_addr)\n\t\t\tselect_addr = this->check_and_get(select_addr, true, tracing);\n\n\t\tif (!select_addr && this->try_another)\n\t\t\tselect_addr = this->another_strategy(uri, tracing);\n\t}\n\n\tif (!select_addr)\n\t\tselect_addr = this->default_group->get_one_backup(tracing);\n\n\tif (select_addr)\n\t{\n\t\t*addr = select_addr;\n\t\t++select_addr->ref;\n\t}\n\n\tpthread_rwlock_unlock(&this->rwlock);\n\treturn !!select_addr;\n}\n\n/*\n * addr_failed true: return an available one. If not exists, return NULL.\n * \t\t\t false: means addr maybe group-alive.\n * \t\t\t\t\t If addr is not available, get one from addr->group.\n */\nEndpointAddress *UPSGroupPolicy::check_and_get(EndpointAddress *addr,\n\t\t\t\t\t\t\t\t\t\t\t bool addr_failed,\n\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\tUPSAddrParams *params = static_cast(addr->params);\n\n\tif (addr_failed) // means fail_count >= max_fails\n\t{\n\t\tif (params->group_id == -1)\n\t\t\treturn NULL;\n\n\t\treturn params->group->get_one(tracing);\n\t}\n\n\tif (addr && addr->fail_count >= addr->params->max_fails &&\n\t\tparams->group_id >= 0)\n\t{\n\t\tEndpointAddress *tmp = params->group->get_one(tracing);\n\t\tif (tmp)\n\t\t\taddr = tmp;\n\t}\n\n\treturn addr;\n}\n\nEndpointAddress *EndpointGroup::get_one(WFNSTracing *tracing)\n{\n\tif (this->nalives == 0)\n\t\treturn NULL;\n\n\tEndpointAddress *server;\n\tEndpointAddress *addr = NULL;\n\tpthread_mutex_lock(&this->mutex);\n\n\tstd::shuffle(this->mains.begin(), this->mains.end(), this->gen);\n\tfor (size_t i = 0; i < this->mains.size(); i++)\n\t{\n\t\tserver = this->mains[i];\n\t\tif (server->fail_count < server->params->max_fails &&\n\t\t\tWFServiceGovernance::in_select_history(tracing, server) == false)\n\t\t{\n\t\t\taddr = server;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (!addr)\n\t{\n\t\tstd::shuffle(this->backups.begin(), this->backups.end(), this->gen);\n\t\tfor (size_t i = 0; i < this->backups.size(); i++)\n\t\t{\n\t\t\tserver = this->backups[i];\n\t\t\tif (server->fail_count < server->params->max_fails &&\n\t\t\t\tWFServiceGovernance::in_select_history(tracing, server) == false)\n\t\t\t{\n\t\t\t\taddr = server;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\n\tpthread_mutex_unlock(&this->mutex);\n\treturn addr;\n}\n\nEndpointAddress *EndpointGroup::get_one_backup(WFNSTracing *tracing)\n{\n\tif (this->nalives == 0)\n\t\treturn NULL;\n\n\tEndpointAddress *server;\n\tEndpointAddress *addr = NULL;\n\n\tpthread_mutex_lock(&this->mutex);\n\n\tstd::shuffle(this->backups.begin(), this->backups.end(), this->gen);\n\tfor (size_t i = 0; i < this->backups.size(); i++)\n\t{\n\t\tserver = this->backups[i];\n\t\tif (server->fail_count < server->params->max_fails &&\n\t\t\tWFServiceGovernance::in_select_history(tracing, server) == false)\n\t\t{\n\t\t\taddr = server;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tpthread_mutex_unlock(&this->mutex);\n\treturn addr;\n}\n\nvoid UPSGroupPolicy::add_server_locked(EndpointAddress *addr)\n{\n\tUPSAddrParams *params = static_cast(addr->params);\n\tint group_id = params->group_id;\n\trb_node **p = &this->group_map.rb_node;\n\trb_node *parent = NULL;\n\tEndpointGroup *group;\n\n\tthis->server_map[addr->address].push_back(addr);\n\n\tif (params->server_type == 0)\n\t\tthis->servers.push_back(addr);\n\n\twhile (*p)\n\t{\n\t\tparent = *p;\n\t\tgroup = rb_entry(*p, EndpointGroup, rb);\n\n\t\tif (group_id < group->id)\n\t\t\tp = &(*p)->rb_left;\n\t\telse if (group_id > group->id)\n\t\t\tp = &(*p)->rb_right;\n\t\telse\n\t\t\tbreak;\n\t}\n\n\tif (*p == NULL)\n\t{\n\t\tgroup = new EndpointGroup(group_id, this);\n\t\trb_link_node(&group->rb, parent, p);\n\t\trb_insert_color(&group->rb, &this->group_map);\n\t}\n\n\tpthread_mutex_lock(&group->mutex);\n\tparams->group = group;\n\tthis->recover_one_server(addr);\n\tif (params->server_type == 0)\n\t{\n\t\tgroup->mains.push_back(addr);\n\t\tgroup->weight += params->weight;\n\t}\n\telse\n\t\tgroup->backups.push_back(addr);\n\tpthread_mutex_unlock(&group->mutex);\n}\n\nint UPSGroupPolicy::remove_server_locked(const std::string& address)\n{\n\tconst auto map_it = this->server_map.find(address);\n\tsize_t n = this->servers.size();\n\tsize_t new_n = 0;\n\tint ret = 0;\n\n\tfor (size_t i = 0; i < n; i++)\n\t{\n\t\tif (this->servers[i]->address != address)\n\t\t\tthis->servers[new_n++] = this->servers[i];\n\t}\n\n\tthis->servers.resize(new_n);\n\n\tif (map_it != this->server_map.cend())\n\t{\n\t\tfor (EndpointAddress *addr : map_it->second)\n\t\t{\n\t\t\tUPSAddrParams *params = static_cast(addr->params);\n\t\t\tEndpointGroup *group = params->group;\n\t\t\tstd::vector *vec;\n\n\t\t\tif (params->server_type == 0)\n\t\t\t\tvec = &group->mains;\n\t\t\telse\n\t\t\t\tvec = &group->backups;\n\n\t\t\tpthread_mutex_lock(&group->mutex);\n\t\t\tif (params->server_type == 0)\n\t\t\t\tgroup->weight -= params->weight;\n\n\t\t\tfor (auto it = vec->begin(); it != vec->end(); ++it)\n\t\t\t{\n\t\t\t\tif (*it == addr)\n\t\t\t\t{\n\t\t\t\t\tvec->erase(it);\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (--addr->ref == 0)\n\t\t\t{\n\t\t\t\tthis->pre_delete_server(addr);\n\t\t\t\tdelete addr;\n\t\t\t}\n\n\t\t\tpthread_mutex_unlock(&group->mutex);\n\t\t\tret++;\n\t\t}\n\n\t\tthis->server_map.erase(map_it);\n\t}\n\n\treturn ret;\n}\n\nEndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int hash,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tWFNSTracing *tracing)\n{\n\tif (this->nalives == 0)\n\t\treturn NULL;\n\n\tstd::map::iterator it;\n\tit = this->addr_hash.lower_bound(hash);\n\n\tif (it == this->addr_hash.end())\n\t\tit = this->addr_hash.begin();\n\n\twhile (!this->is_alive(it->second))\n\t{\n\t\tit++;\n\t\tif (it == this->addr_hash.end())\n\t\t\tit = this->addr_hash.begin();\n\t}\n\n\treturn this->check_and_get(it->second, false, tracing);\n}\n\n#define VIRTUAL_GROUP_SIZE\t16\n\nvoid UPSGroupPolicy::hash_map_add_addr(EndpointAddress *addr)\n{\n\tUPSAddrParams *params = static_cast(addr->params);\n\n\tif (params->server_type == 0)\n\t{\n\t\tstatic std::hash std_hash;\n\t\tunsigned int hash_value;\n\t\tsize_t ip_count = this->server_map[addr->address].size();\n\n\t\tfor (int i = 0; i < VIRTUAL_GROUP_SIZE * params->weight; i++)\n\t\t{\n\t\t\thash_value = std_hash(addr->address + \"|v\" + std::to_string(i) +\n\t\t\t\t\t\t\t\t \"|n\" + std::to_string(ip_count));\n\t\t\tthis->addr_hash.insert(std::make_pair(hash_value, addr));\n\t\t}\n\t}\n}\n\nvoid UPSGroupPolicy::hash_map_remove_addr(const std::string& address)\n{\n\tstd::map::iterator it;\n\n\tfor (it = this->addr_hash.begin(); it != this->addr_hash.end();)\n\t{\n\t\tif (it->second->address == address)\n\t\t\tthis->addr_hash.erase(it++);\n\t\telse\n\t\t\tit++;\n\t}\n}\n\nint UPSRoundRobinPolicy::remove_server_locked(const std::string& address)\n{\n\tif (servers.size() != 0)\n\t{\n\t\tsize_t cur_idx = this->cur_idx % servers.size();\n\n\t\tfor (size_t i = 0; i < cur_idx; i++)\n\t\t{\n\t\t\tif (this->servers[i]->address == address)\n\t\t\t\tthis->cur_idx--;\n\t\t}\n\t}\n\n\treturn UPSGroupPolicy::remove_server_locked(address);\n}\n\nEndpointAddress *UPSRoundRobinPolicy::first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\treturn this->servers[this->cur_idx++ % this->servers.size()];\n}\n\nEndpointAddress *UPSRoundRobinPolicy::another_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\tEndpointAddress *addr = this->servers[this->cur_idx++ % this->servers.size()];\n\treturn this->check_and_get(addr, false, tracing);\n}\n\nvoid UPSWeightedRandomPolicy::add_server_locked(EndpointAddress *addr)\n{\n\tUPSAddrParams *params = static_cast(addr->params);\n\n\tUPSGroupPolicy::add_server_locked(addr);\n\tif (params->server_type == 0)\n\t\tthis->total_weight += params->weight;\n}\n\nint UPSWeightedRandomPolicy::remove_server_locked(const std::string& address)\n{\n\tUPSAddrParams *params;\n\tconst auto map_it = this->server_map.find(address);\n\n\tif (map_it != this->server_map.cend())\n\t{\n\t\tfor (EndpointAddress *addr : map_it->second)\n\t\t{\n\t\t\tparams = static_cast(addr->params);\n\t\t\tif (params->server_type == 0)\n\t\t\t\tthis->total_weight -= params->weight;\n\t\t}\n\t}\n\n\treturn UPSGroupPolicy::remove_server_locked(address);\n}\n\nint UPSWeightedRandomPolicy::select_history_weight(WFNSTracing *tracing)\n{\n\tstruct TracingData *tracing_data = (struct TracingData *)tracing->data;\n\n\tif (!tracing_data)\n\t\treturn 0;\n\n\tint ret = 0;\n\n\tfor (EndpointAddress *server : tracing_data->history)\n\t\tret += ((UPSAddrParams *)server->params)->weight;\n\n\treturn ret;\n}\n\nEndpointAddress *UPSWeightedRandomPolicy::first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\tint x = 0;\n\tint s = 0;\n\tsize_t idx;\n\tUPSAddrParams *params;\n\tint temp_weight = this->total_weight;\n\ttemp_weight -= UPSWeightedRandomPolicy::select_history_weight(tracing);\n\n\tif (temp_weight > 0)\n\t\tx = rand() % temp_weight;\n\n\tfor (idx = 0; idx < this->servers.size(); idx++)\n\t{\n\t\tif (WFServiceGovernance::in_select_history(tracing, this->servers[idx]))\n\t\t\tcontinue;\n\n\t\tparams = static_cast(this->servers[idx]->params);\n\t\ts += params->weight;\n\t\tif (s > x)\n\t\t\tbreak;\n\t}\n\tif (idx == this->servers.size())\n\t\tidx--;\n\n\treturn this->servers[idx];\n}\n\nEndpointAddress *UPSWeightedRandomPolicy::another_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\t/* When all servers are down, recover all servers if any server\n\t * reaches fusing timeout. */\n\tif (this->available_weight == 0)\n\t\tthis->try_clear_breaker();\n\n\tint temp_weight = this->available_weight;\n\tif (temp_weight == 0)\n\t\treturn NULL;\n\n\tUPSAddrParams *params;\n\tEndpointAddress *addr = NULL;\n\tint x = rand() % temp_weight;\n\tint s = 0;\n\n\tfor (EndpointAddress *server : this->servers)\n\t{\n\t\tif (this->is_alive(server))\n\t\t{\n\t\t\taddr = server;\n\t\t\tparams = static_cast(server->params);\n\t\t\ts += params->weight;\n\t\t\tif (s > x)\n\t\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (!addr)\n\t\treturn NULL;\n\n\treturn this->check_and_get(addr, false, tracing);\n}\n\nvoid UPSWeightedRandomPolicy::recover_one_server(const EndpointAddress *addr)\n{\n\tUPSAddrParams *params = static_cast(addr->params);\n\n\tthis->nalives++;\n\tif (params->group->nalives++ == 0 && params->group->id > 0)\n\t\tthis->available_weight += params->group->weight;\n\n\tif (params->group_id < 0 && params->server_type == 0)\n\t\tthis->available_weight += params->weight;\n}\n\nvoid UPSWeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr)\n{\n\tUPSAddrParams *params = static_cast(addr->params);\n\n\tthis->nalives--;\n\tif (--params->group->nalives == 0 && params->group->id > 0)\n\t\tthis->available_weight -= params->group->weight;\n\n\tif (params->group_id < 0 && params->server_type == 0)\n\t\tthis->available_weight -= params->weight;\n}\n\nEndpointAddress *UPSVNSWRRPolicy::first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\tint idx = this->cur_idx.fetch_add(1);\n\tint pos = 0;\n\tfor (int i = 0; i < this->total_weight; i++, idx++)\n\t{\n\t\tpos = this->pre_generated_vec[idx % this->pre_generated_vec.size()];\n\t\tif (WFServiceGovernance::in_select_history(tracing, this->servers[pos]))\n\t\t\tcontinue;\n\n\t\tbreak;\n\t}\n\treturn this->servers[pos];\n}\n\nvoid UPSVNSWRRPolicy::init_virtual_nodes()\n{\n\tUPSAddrParams *params;\n\tsize_t start_pos = this->pre_generated_vec.size();\n\tsize_t end_pos = this->total_weight;\n\tthis->pre_generated_vec.resize(end_pos);\n\n\tfor (size_t i = start_pos; i < end_pos; i++)\n\t{\n\t\tfor (size_t j = 0; j < this->servers.size(); j++)\n\t\t{\n\t\t\tconst EndpointAddress *server = this->servers[j];\n\t\t\tparams = static_cast(server->params);\n\t\t\tthis->current_weight_vec[j] += params->weight;\n\t\t}\n\t\tstd::vector::iterator biggest = std::max_element(this->current_weight_vec.begin(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t this->current_weight_vec.end());\n\t\tthis->pre_generated_vec[i] = std::distance(this->current_weight_vec.begin(), biggest);\n\t\tthis->current_weight_vec[this->pre_generated_vec[i]] -= this->total_weight;\n\t}\n}\n\nvoid UPSVNSWRRPolicy::init()\n{\n\tif (this->total_weight <= 0)\n\t\treturn;\n\n\tthis->pre_generated_vec.clear();\n\tthis->cur_idx = rand() % this->total_weight;\n\tstd::vector t(this->servers.size(), 0);\n\tthis->current_weight_vec.swap(t);\n\tthis->init_virtual_nodes();\n}\n\nvoid UPSVNSWRRPolicy::add_server_locked(EndpointAddress *addr)\n{\n\tUPSWeightedRandomPolicy::add_server_locked(addr);\n\tinit();\n}\n\nint UPSVNSWRRPolicy::remove_server_locked(const std::string& address)\n{\n\tint ret = UPSWeightedRandomPolicy::remove_server_locked(address);\n\tinit();\n\treturn ret;\n}\n\nEndpointAddress *UPSConsistentHashPolicy::first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\tunsigned int hash_value = this->consistent_hash(\n\t\t\t\t\t\t\t\t\t\turi.path ? uri.path : \"\",\n\t\t\t\t\t\t\t\t\t\turi.query ? uri.query : \"\",\n\t\t\t\t\t\t\t\t\t\turi.fragment ? uri.fragment : \"\");\n\treturn this->consistent_hash_with_group(hash_value, tracing);\n}\n\nvoid UPSConsistentHashPolicy::add_server_locked(EndpointAddress *addr)\n{\n\tUPSGroupPolicy::add_server_locked(addr);\n\tthis->hash_map_add_addr(addr);\n}\n\nint UPSConsistentHashPolicy::remove_server_locked(const std::string& address)\n{\n\tthis->hash_map_remove_addr(address);\n\n\treturn UPSGroupPolicy::remove_server_locked(address);\n}\n\nEndpointAddress *UPSManualPolicy::first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\tunsigned int idx = this->manual_select(uri.path ? uri.path : \"\",\n\t\t\t\t\t\t\t\t\t\t uri.query ? uri.query : \"\",\n\t\t\t\t\t\t\t\t\t\t uri.fragment ? uri.fragment : \"\");\n\n\tif (idx >= this->servers.size())\n\t\tidx %= this->servers.size();\n\n\treturn this->servers[idx];\n}\n\nEndpointAddress *UPSManualPolicy::another_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\tunsigned int hash_value = this->another_select(\n\t\t\t\t\t\t\t\t\t\turi.path ? uri.path : \"\",\n\t\t\t\t\t\t\t\t\t\turi.query ? uri.query : \"\",\n\t\t\t\t\t\t\t\t\t\turi.fragment ? uri.fragment : \"\");\n\treturn this->consistent_hash_with_group(hash_value, tracing);\n}\n\nvoid UPSManualPolicy::add_server_locked(EndpointAddress *addr)\n{\n\tUPSGroupPolicy::add_server_locked(addr);\n\n\tif (this->try_another)\n\t\tthis->hash_map_add_addr(addr);\n}\n\nint UPSManualPolicy::remove_server_locked(const std::string& address)\n{\n\tif (this->try_another)\n\t\tthis->hash_map_remove_addr(address);\n\n\treturn UPSGroupPolicy::remove_server_locked(address);\n}\n\n" }, { "path": "src/nameservice/UpstreamPolicies.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#ifndef _UPSTREAMPOLICIES_H_\n#define _UPSTREAMPOLICIES_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \"URIParser.h\"\n#include \"EndpointParams.h\"\n#include \"WFNameService.h\"\n#include \"WFServiceGovernance.h\"\n\nusing upstream_route_t = std::function;\n\nclass EndpointGroup;\nclass UPSGroupPolicy;\n\nclass UPSAddrParams : public PolicyAddrParams\n{\npublic:\n\tunsigned short weight;\n\tshort server_type;\n\tint group_id;\n\tEndpointGroup *group;\n\n\tUPSAddrParams(const struct AddressParams *params,\n\t\t\t\t const std::string& address);\n};\n\nclass UPSGroupPolicy : public WFServiceGovernance\n{\npublic:\n\tUPSGroupPolicy();\n\tvirtual ~UPSGroupPolicy();\n\npublic:\n\tvirtual bool select(const ParsedURI& uri, WFNSTracing *tracing,\n\t\t\t\t\t\tEndpointAddress **addr);\n\tvirtual void add_server(const std::string& address,\n\t\t\t\t\t\t\tconst struct AddressParams *params);\n\tvirtual int replace_server(const std::string& address,\n\t\t\t\t\t\t\t const struct AddressParams *params);\n\tvoid get_main_address(std::vector& addr_list);\n\nprotected:\n\tstruct rb_root group_map;\n\tEndpointGroup *default_group;\n\nprivate:\n\tvirtual void recover_one_server(const EndpointAddress *addr);\n\tvirtual void fuse_one_server(const EndpointAddress *addr);\n\nprotected:\n\tvirtual void add_server_locked(EndpointAddress *addr);\n\tvirtual int remove_server_locked(const std::string& address);\n\n\tEndpointAddress *check_and_get(EndpointAddress *addr, bool addr_failed,\n\t\t\t\t\t\t\t\t WFNSTracing *tracing);\n\n\tbool is_alive(const EndpointAddress *addr) const;\n\nprotected:\n\tEndpointAddress *consistent_hash_with_group(unsigned int hash,\n\t\t\t\t\t\t\t\t\t\t\t\tWFNSTracing *tracing);\n\tvoid hash_map_add_addr(EndpointAddress *addr);\n\tvoid hash_map_remove_addr(const std::string& address);\n\n\tstd::map addr_hash;\n};\n\nclass UPSRoundRobinPolicy : public UPSGroupPolicy\n{\npublic:\n\tUPSRoundRobinPolicy(bool try_another) :\n\t\tcur_idx(0)\n\t{\n\t\tthis->try_another = try_another;\n\t}\n\nprotected:\n\tvirtual EndpointAddress *first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\tWFNSTracing *tracing);\n\tvirtual EndpointAddress *another_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing);\n\nprotected:\n\tvirtual int remove_server_locked(const std::string& address);\n\nprotected:\n\tstd::atomic cur_idx;\n};\n\nclass UPSWeightedRandomPolicy : public UPSGroupPolicy\n{\npublic:\n\tUPSWeightedRandomPolicy(bool try_another)\n\t{\n\t\tthis->total_weight = 0;\n\t\tthis->available_weight = 0;\n\t\tthis->try_another = try_another;\n\t}\n\nprotected:\n\tvirtual EndpointAddress *first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\tWFNSTracing *tracing);\n\tvirtual EndpointAddress *another_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing);\n\nprotected:\n\tvirtual void add_server_locked(EndpointAddress *addr);\n\tvirtual int remove_server_locked(const std::string& address);\n\tint total_weight;\n\tint available_weight;\n\nprivate:\n\tvirtual void recover_one_server(const EndpointAddress *addr);\n\tvirtual void fuse_one_server(const EndpointAddress *addr);\n\tstatic int select_history_weight(WFNSTracing *tracing);\n};\n\nclass UPSVNSWRRPolicy : public UPSWeightedRandomPolicy\n{\npublic:\n\tUPSVNSWRRPolicy() : UPSWeightedRandomPolicy(false)\n\t{\n\t\tthis->cur_idx = 0;\n\t\tthis->try_another = false;\n\t};\n\nprotected:\n\tvirtual EndpointAddress *first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\tWFNSTracing *tracing);\n\nprivate:\n\tvirtual void add_server_locked(EndpointAddress *addr);\n\tvirtual int remove_server_locked(const std::string& address);\n\tvoid init();\n\tvoid init_virtual_nodes();\n\tstd::vector pre_generated_vec;\n\tstd::vector current_weight_vec;\n\tstd::atomic cur_idx;\n};\n\nclass UPSConsistentHashPolicy : public UPSGroupPolicy\n{\npublic:\n\tUPSConsistentHashPolicy(upstream_route_t consistent_hash) :\n\t\tconsistent_hash(std::move(consistent_hash))\n\t{\n\t}\n\nprotected:\n\tvirtual EndpointAddress *first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\tWFNSTracing *tracing);\n\nprivate:\n\tvirtual void add_server_locked(EndpointAddress *addr);\n\tvirtual int remove_server_locked(const std::string& address);\n\tupstream_route_t consistent_hash;\n};\n\nclass UPSManualPolicy : public UPSGroupPolicy\n{\npublic:\n\tUPSManualPolicy(bool try_another, upstream_route_t select,\n\t\t\t\t\tupstream_route_t try_another_select) :\n\t\tmanual_select(std::move(select)),\n\t\tanother_select(std::move(try_another_select))\n\t{\n\t\tthis->try_another = try_another;\n\t}\n\n\tEndpointAddress *first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\tWFNSTracing *tracing);\n\tEndpointAddress *another_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t WFNSTracing *tracing);\n\nprivate:\n\tvirtual void add_server_locked(EndpointAddress *addr);\n\tvirtual int remove_server_locked(const std::string& address);\n\tupstream_route_t manual_select;\n\tupstream_route_t another_select;\n};\n\n#endif\n" }, { "path": "src/nameservice/WFDnsResolver.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Liu Kai (liukaidx@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"EndpointParams.h\"\n#include \"RouteManager.h\"\n#include \"WFGlobal.h\"\n#include \"WFTaskFactory.h\"\n#include \"WFResourcePool.h\"\n#include \"WFNameService.h\"\n#include \"DnsCache.h\"\n#include \"DnsUtil.h\"\n#include \"WFDnsClient.h\"\n#include \"WFDnsResolver.h\"\n\n#define HOSTS_LINEBUF_INIT_SIZE\t128\n#define PORT_STR_MAX\t\t\t5\n\nclass DnsInput\n{\npublic:\n\tDnsInput() :\n\t\tport_(0),\n\t\tnumeric_host_(false),\n\t\tfamily_(AF_UNSPEC)\n\t{}\n\n\tDnsInput(const std::string& host, unsigned short port,\n\t\t\t bool numeric_host, int family) :\n\t\thost_(host),\n\t\tport_(port),\n\t\tnumeric_host_(numeric_host),\n\t\tfamily_(family)\n\t{}\n\n\tvoid reset(const std::string& host, unsigned short port)\n\t{\n\t\thost_.assign(host);\n\t\tport_ = port;\n\t\tnumeric_host_ = false;\n\t\tfamily_ = AF_UNSPEC;\n\t}\n\n\tvoid reset(const std::string& host, unsigned short port,\n\t\t\t bool numeric_host, int family)\n\t{\n\t\thost_.assign(host);\n\t\tport_ = port;\n\t\tnumeric_host_ = numeric_host;\n\t\tfamily_ = family;\n\t}\n\n\tconst std::string& get_host() const { return host_; }\n\tunsigned short get_port() const { return port_; }\n\tbool is_numeric_host() const { return numeric_host_; }\n\nprotected:\n\tstd::string host_;\n\tunsigned short port_;\n\tbool numeric_host_;\n\tint family_;\n\n\tfriend class DnsRoutine;\n};\n\nclass DnsOutput\n{\npublic:\n\tDnsOutput():\n\t\terror_(0),\n\t\taddrinfo_(NULL)\n\t{}\n\n\t~DnsOutput()\n\t{\n\t\tif (addrinfo_)\n\t\t{\n\t\t\tif (addrinfo_->ai_flags)\n\t\t\t\tfreeaddrinfo(addrinfo_);\n\t\t\telse\n\t\t\t\tfree(addrinfo_);\n\t\t}\n\t}\n\n\tint get_error() const { return error_; }\n\tconst struct addrinfo *get_addrinfo() const { return addrinfo_; }\n\n\t//if DONOT want DnsOutput release addrinfo, use move_addrinfo in callback\n\tstruct addrinfo *move_addrinfo()\n\t{\n\t\tstruct addrinfo *p = addrinfo_;\n\t\taddrinfo_ = NULL;\n\t\treturn p;\n\t}\n\nprotected:\n\tint error_;\n\tstruct addrinfo *addrinfo_;\n\n\tfriend class DnsRoutine;\n};\n\nclass DnsRoutine\n{\npublic:\n\tstatic void run(const DnsInput *in, DnsOutput *out);\n\tstatic void create(DnsOutput *out, int error, struct addrinfo *ai)\n\t{\n\t\tif (out->addrinfo_)\n\t\t{\n\t\t\tif (out->addrinfo_->ai_flags)\n\t\t\t\tfreeaddrinfo(out->addrinfo_);\n\t\t\telse\n\t\t\t\tfree(out->addrinfo_);\n\t\t}\n\n\t\tout->error_ = error;\n\t\tout->addrinfo_ = ai;\n\t}\n\nprivate:\n\tstatic void run_local_path(const std::string& path, DnsOutput *out);\n};\n\nvoid DnsRoutine::run_local_path(const std::string& path, DnsOutput *out)\n{\n\tstruct sockaddr_un *sun = NULL;\n\n\tif (path.size() + 1 <= sizeof sun->sun_path)\n\t{\n\t\tsize_t size = sizeof (struct addrinfo) + sizeof (struct sockaddr_un);\n\n\t\tout->addrinfo_ = (struct addrinfo *)calloc(size, 1);\n\t\tif (out->addrinfo_)\n\t\t{\n\t\t\tsun = (struct sockaddr_un *)(out->addrinfo_ + 1);\n\t\t\tsun->sun_family = AF_UNIX;\n\t\t\tmemcpy(sun->sun_path, path.c_str(), path.size());\n\n\t\t\tout->addrinfo_->ai_family = AF_UNIX;\n\t\t\tout->addrinfo_->ai_socktype = SOCK_STREAM;\n\t\t\tout->addrinfo_->ai_addr = (struct sockaddr *)sun;\n\t\t\tsize = offsetof(struct sockaddr_un, sun_path) + path.size() + 1;\n\t\t\tout->addrinfo_->ai_addrlen = size;\n\t\t\tout->error_ = 0;\n\t\t\treturn;\n\t\t}\n\t}\n\telse\n\t\terrno = EINVAL;\n\n\tout->error_ = EAI_SYSTEM;\n}\n\nvoid DnsRoutine::run(const DnsInput *in, DnsOutput *out)\n{\n\tif (in->host_[0] == '/')\n\t{\n\t\trun_local_path(in->host_, out);\n\t\treturn;\n\t}\n\n\tstruct addrinfo hints = {\n\t\t.ai_flags\t\t=\tAI_ADDRCONFIG | AI_NUMERICSERV,\n\t\t.ai_family\t\t=\tin->family_,\n\t\t.ai_socktype\t=\tSOCK_STREAM,\n\t};\n\tchar port_str[PORT_STR_MAX + 1];\n\n\tif (in->is_numeric_host())\n\t\thints.ai_flags |= AI_NUMERICHOST;\n\n\tsnprintf(port_str, PORT_STR_MAX + 1, \"%u\", in->port_);\n\tout->error_ = getaddrinfo(in->host_.c_str(), port_str,\n\t\t\t\t\t\t\t &hints, &out->addrinfo_);\n\tif (out->error_ == 0)\n\t\tout->addrinfo_->ai_flags = 1;\n}\n\n// Dns Thread task. For internal usage only.\nusing ThreadDnsTask = WFThreadTask;\nusing thread_dns_callback_t = std::function;\n\nstruct DnsContext\n{\n\tunsigned short port;\n\tint eai_error;\n\tstruct addrinfo *ai;\n};\n\nstatic int __default_family()\n{\n\tstruct addrinfo hints = {\n\t\t.ai_flags\t\t=\tAI_ADDRCONFIG,\n\t\t.ai_family\t\t=\tAF_UNSPEC,\n\t\t.ai_socktype\t=\tSOCK_STREAM,\n\t};\n\tstruct addrinfo *res;\n\tstruct addrinfo *cur;\n\tint family = AF_UNSPEC;\n\tbool v4 = false;\n\tbool v6 = false;\n\n\tif (getaddrinfo(NULL, \"1\", &hints, &res) == 0)\n\t{\n\t\tfor (cur = res; cur; cur = cur->ai_next)\n\t\t{\n\t\t\tif (cur->ai_family == AF_INET)\n\t\t\t\tv4 = true;\n\t\t\telse if (cur->ai_family == AF_INET6)\n\t\t\t\tv6 = true;\n\t\t}\n\n\t\tfreeaddrinfo(res);\n\t\tif (v4 ^ v6)\n\t\t\tfamily = v4 ? AF_INET : AF_INET6;\n\t}\n\n\treturn family;\n}\n\n// hosts line format: IP canonical_name [aliases...] [# Comment]\nstatic int __readaddrinfo_line(char *p, const char *name, const char *port,\n\t\t\t\t\t\t\t const struct addrinfo *hints,\n\t\t\t\t\t\t\t struct addrinfo **res)\n{\n\tconst char *ip = NULL;\n\tchar *start;\n\n\tstart = p;\n\twhile (*start != '\\0' && *start != '#')\n\t\tstart++;\n\t*start = '\\0';\n\n\twhile (1)\n\t{\n\t\twhile (isspace(*p))\n\t\t\tp++;\n\n\t\tstart = p;\n\t\twhile (*p != '\\0' && !isspace(*p))\n\t\t\tp++;\n\n\t\tif (start == p)\n\t\t\tbreak;\n\n\t\tif (*p != '\\0')\n\t\t\t*p++ = '\\0';\n\n\t\tif (ip == NULL)\n\t\t{\n\t\t\tip = start;\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (strcasecmp(name, start) == 0)\n\t\t{\n\t\t\tif (getaddrinfo(ip, port, hints, res) == 0)\n\t\t\t\treturn 0;\n\t\t}\n\t}\n\n\treturn 1;\n}\n\nstatic int __readaddrinfo(const char *path,\n\t\t\t\t\t\t const char *name, unsigned short port,\n\t\t\t\t\t\t const struct addrinfo *hints,\n\t\t\t\t\t\t struct addrinfo **res)\n{\n\tchar port_str[PORT_STR_MAX + 1];\n\tsize_t bufsize = 0;\n\tchar *line = NULL;\n\tint count = 0;\n\tint errno_bak;\n\tFILE *fp;\n\tint ret;\n\n\tfp = fopen(path, \"r\");\n\tif (!fp)\n\t\treturn EAI_SYSTEM;\n\n\tsnprintf(port_str, PORT_STR_MAX + 1, \"%u\", port);\n\n\terrno_bak = errno;\n\twhile ((ret = getline(&line, &bufsize, fp)) > 0)\n\t{\n\t\tif (__readaddrinfo_line(line, name, port_str, hints, res) == 0)\n\t\t{\n\t\t\tcount++;\n\t\t\tres = &(*res)->ai_next;\n\t\t}\n\t}\n\n\tret = ferror(fp) ? EAI_SYSTEM : EAI_NONAME;\n\tfree(line);\n\tfclose(fp);\n\tif (count != 0)\n\t{\n\t\terrno = errno_bak;\n\t\treturn 0;\n\t}\n\n\treturn ret;\n}\n\nstatic ThreadDnsTask *__create_thread_dns_task(const std::string& host,\n\t\t\t\t\t\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t\t\t\t\t\t int family,\n\t\t\t\t\t\t\t\t\t\t\t thread_dns_callback_t callback)\n{\n\tauto *task = WFThreadTaskFactory::\n\t\t\t\t\t\tcreate_thread_task(WFGlobal::get_dns_queue(),\n\t\t\t\t\t\t\t\t\t\t WFGlobal::get_dns_executor(),\n\t\t\t\t\t\t\t\t\t\t DnsRoutine::run,\n\t\t\t\t\t\t\t\t\t\t std::move(callback));\n\n\ttask->get_input()->reset(host, port, false, family);\n\treturn task;\n}\n\nstatic std::string __get_cache_host(const std::string& hostname,\n\t\t\t\t\t\t\t\t\tint family)\n{\n\tchar c;\n\n\tif (family == AF_UNSPEC)\n\t\tc = '*';\n\telse if (family == AF_INET)\n\t\tc = '4';\n\telse if (family == AF_INET6)\n\t\tc = '6';\n\telse\n\t\tc = '?';\n\n\treturn hostname + c;\n}\n\nstatic std::string __get_guard_name(const std::string& cache_host,\n\t\t\t\t\t\t\t\t\tunsigned short port)\n{\n\tstd::string guard_name(\"INTERNAL-dns:\");\n\tguard_name.append(cache_host).append(\":\");\n\tguard_name.append(std::to_string(port));\n\treturn guard_name;\n}\n\nvoid WFResolverTask::dispatch()\n{\n\tif (this->msg_)\n\t{\n\t\tthis->state = WFT_STATE_DNS_ERROR;\n\t\tthis->error = (intptr_t)msg_;\n\t\tthis->subtask_done();\n\t\treturn;\n\t}\n\n\tconst ParsedURI& uri = ns_params_.uri;\n\thost_ = uri.host ? uri.host : \"\";\n\tport_ = uri.port ? atoi(uri.port) : 0;\n\n\tDnsCache *dns_cache = WFGlobal::get_dns_cache();\n\tconst DnsCache::DnsHandle *addr_handle;\n\tstd::string hostname = host_;\n\tint family = ep_params_.address_family;\n\tstd::string cache_host = __get_cache_host(hostname, family);\n\n\tif (ns_params_.retry_times == 0)\n\t\taddr_handle = dns_cache->get_ttl(cache_host, port_);\n\telse\n\t\taddr_handle = dns_cache->get_confident(cache_host, port_);\n\n\tif (in_guard_ && (addr_handle == NULL || addr_handle->value.delayed()))\n\t{\n\t\tif (addr_handle)\n\t\t\tdns_cache->release(addr_handle);\n\n\t\tthis->request_dns();\n\t\treturn;\n\t}\n\n\tif (addr_handle)\n\t{\n\t\tRouteManager *route_manager = WFGlobal::get_route_manager();\n\t\tstruct addrinfo *addrinfo = addr_handle->value.addrinfo;\n\t\tstruct addrinfo first;\n\n\t\tif (ns_params_.fixed_addr && addrinfo->ai_next)\n\t\t{\n\t\t\tfirst = *addrinfo;\n\t\t\tfirst.ai_next = NULL;\n\t\t\taddrinfo = &first;\n\t\t}\n\n\t\tif (route_manager->get(ns_params_.type, addrinfo, ns_params_.info,\n\t\t\t\t\t\t\t &ep_params_, hostname, ns_params_.ssl_ctx,\n\t\t\t\t\t\t\t this->result) < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t}\n\t\telse\n\t\t\tthis->state = WFT_STATE_SUCCESS;\n\n\t\tdns_cache->release(addr_handle);\n\t\tthis->subtask_done();\n\t\treturn;\n\t}\n\n\tif (*host_)\n\t{\n\t\tchar front = host_[0];\n\t\tchar back = host_[hostname.size() - 1];\n\t\tstruct in6_addr addr;\n\t\tint ret;\n\n\t\tif (strchr(host_, ':'))\n\t\t\tret = inet_pton(AF_INET6, host_, &addr);\n\t\telse if (isdigit(back) && isdigit(front))\n\t\t\tret = inet_pton(AF_INET, host_, &addr);\n\t\telse if (front == '/')\n\t\t\tret = 1;\n\t\telse\n\t\t\tret = 0;\n\n\t\tif (ret == 1)\n\t\t{\n\t\t\t// 'true' means numeric host\n\t\t\tDnsInput dns_in(hostname, port_, true, AF_UNSPEC);\n\t\t\tDnsOutput dns_out;\n\n\t\t\tDnsRoutine::run(&dns_in, &dns_out);\n\t\t\tdns_callback_internal(&dns_out, (unsigned int)-1, (unsigned int)-1);\n\t\t\tthis->subtask_done();\n\t\t\treturn;\n\t\t}\n\t}\n\n\tconst char *hosts = WFGlobal::get_global_settings()->hosts_path;\n\tif (hosts)\n\t{\n\t\tstruct addrinfo hints = {\n\t\t\t.ai_flags\t\t=\tAI_ADDRCONFIG | AI_NUMERICSERV | AI_NUMERICHOST,\n\t\t\t.ai_family\t\t=\tep_params_.address_family,\n\t\t\t.ai_socktype\t=\tSOCK_STREAM,\n\t\t};\n\t\tstruct addrinfo *ai;\n\t\tint ret;\n\n\t\tret = __readaddrinfo(hosts, host_, port_, &hints, &ai);\n\t\tif (ret == 0)\n\t\t{\n\t\t\tDnsOutput out;\n\t\t\tDnsRoutine::create(&out, ret, ai);\n\t\t\tdns_callback_internal(&out, dns_ttl_default_, dns_ttl_min_);\n\t\t\tthis->subtask_done();\n\t\t\treturn;\n\t\t}\n\t}\n\n\tstd::string guard_name = __get_guard_name(cache_host, port_);\n\tWFConditional *guard = WFTaskFactory::create_guard(guard_name, this, &msg_);\n\n\tin_guard_ = true;\n\thas_next_ = true;\n\n\tseries_of(this)->push_front(guard);\n\tthis->subtask_done();\n}\n\nvoid WFResolverTask::request_dns()\n{\n\tWFDnsClient *client = WFGlobal::get_dns_client();\n\tif (client)\n\t{\n\t\tstatic int default_family = __default_family();\n\t\tWFResourcePool *respool = WFGlobal::get_dns_respool();\n\n\t\tint family = ep_params_.address_family;\n\t\tif (family == AF_UNSPEC)\n\t\t\tfamily = default_family;\n\n\t\tif (family == AF_INET || family == AF_INET6)\n\t\t{\n\t\t\tauto&& cb = std::bind(&WFResolverTask::dns_single_callback,\n\t\t\t\t\t\t\t\t this,\n\t\t\t\t\t\t\t\t std::placeholders::_1);\n\t\t\tWFDnsTask *dns_task = client->create_dns_task(host_, std::move(cb));\n\n\t\t\tif (family == AF_INET6)\n\t\t\t\tdns_task->get_req()->set_question_type(DNS_TYPE_AAAA);\n\n\t\t\tWFConditional *cond = respool->get(dns_task);\n\t\t\tseries_of(this)->push_front(cond);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tstruct DnsContext *dctx = new struct DnsContext[2];\n\t\t\tWFDnsTask *task_v4;\n\t\t\tWFDnsTask *task_v6;\n\t\t\tParallelWork *pwork;\n\n\t\t\tdctx[0].ai = NULL;\n\t\t\tdctx[1].ai = NULL;\n\t\t\tdctx[0].port = port_;\n\t\t\tdctx[1].port = port_;\n\n\t\t\ttask_v4 = client->create_dns_task(host_, dns_partial_callback);\n\t\t\ttask_v4->user_data = dctx;\n\n\t\t\ttask_v6 = client->create_dns_task(host_, dns_partial_callback);\n\t\t\ttask_v6->get_req()->set_question_type(DNS_TYPE_AAAA);\n\t\t\ttask_v6->user_data = dctx + 1;\n\n\t\t\tauto&& cb = std::bind(&WFResolverTask::dns_parallel_callback,\n\t\t\t\t\t\t\t\t this,\n\t\t\t\t\t\t\t\t std::placeholders::_1);\n\n\t\t\tpwork = Workflow::create_parallel_work(std::move(cb));\n\t\t\tpwork->set_context(dctx);\n\n\t\t\tWFConditional *cond_v4 = respool->get(task_v4);\n\t\t\tWFConditional *cond_v6 = respool->get(task_v6);\n\t\t\tpwork->add_series(Workflow::create_series_work(cond_v4, nullptr));\n\t\t\tpwork->add_series(Workflow::create_series_work(cond_v6, nullptr));\n\n\t\t\tseries_of(this)->push_front(pwork);\n\t\t}\n\t}\n\telse\n\t{\n\t\tThreadDnsTask *dns_task;\n\t\tauto&& cb = std::bind(&WFResolverTask::thread_dns_callback,\n\t\t\t\t\t\t\t this,\n\t\t\t\t\t\t\t std::placeholders::_1);\n\t\tdns_task = __create_thread_dns_task(host_, port_,\n\t\t\t\t\t\t\t\t\t\t\tep_params_.address_family,\n\t\t\t\t\t\t\t\t\t\t\tstd::move(cb));\n\t\tseries_of(this)->push_front(dns_task);\n\t}\n\n\thas_next_ = true;\n\tthis->subtask_done();\n}\n\nSubTask *WFResolverTask::done()\n{\n\tSeriesWork *series = series_of(this);\n\n\tif (!has_next_)\n\t\ttask_callback();\n\telse\n\t\thas_next_ = false;\n\n\treturn series->pop();\n}\n\nvoid WFResolverTask::dns_callback_internal(void *thrd_dns_output,\n\t\t\t\t\t\t\t\t\t\t unsigned int ttl_default,\n\t\t\t\t\t\t\t\t\t\t unsigned int ttl_min)\n{\n\tDnsOutput *dns_out = (DnsOutput *)thrd_dns_output;\n\tint dns_error = dns_out->get_error();\n\n\tif (dns_error)\n\t{\n\t\tif (dns_error == EAI_SYSTEM)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tthis->state = WFT_STATE_DNS_ERROR;\n\t\t\tthis->error = dns_error;\n\t\t}\n\t}\n\telse\n\t{\n\t\tRouteManager *route_manager = WFGlobal::get_route_manager();\n\t\tDnsCache *dns_cache = WFGlobal::get_dns_cache();\n\t\tstruct addrinfo *addrinfo = dns_out->move_addrinfo();\n\t\tconst DnsCache::DnsHandle *addr_handle;\n\t\tstd::string hostname = host_;\n\t\tint family = ep_params_.address_family;\n\t\tstd::string cache_host = __get_cache_host(hostname, family);\n\n\t\taddr_handle = dns_cache->put(cache_host, port_, addrinfo,\n\t\t\t\t\t\t\t\t\t (unsigned int)ttl_default,\n\t\t\t\t\t\t\t\t\t (unsigned int)ttl_min);\n\t\tif (route_manager->get(ns_params_.type, addrinfo, ns_params_.info,\n\t\t\t\t\t\t\t &ep_params_, hostname, ns_params_.ssl_ctx,\n\t\t\t\t\t\t\t this->result) < 0)\n\t\t{\n\t\t\tthis->state = WFT_STATE_SYS_ERROR;\n\t\t\tthis->error = errno;\n\t\t}\n\t\telse\n\t\t\tthis->state = WFT_STATE_SUCCESS;\n\n\t\tdns_cache->release(addr_handle);\n\t}\n}\n\nvoid WFResolverTask::dns_single_callback(void *net_dns_task)\n{\n\tWFDnsTask *dns_task = (WFDnsTask *)net_dns_task;\n\tWFGlobal::get_dns_respool()->post(NULL);\n\n\tif (dns_task->get_state() == WFT_STATE_SUCCESS)\n\t{\n\t\tstruct addrinfo *ai = NULL;\n\t\tint ret;\n\n\t\tret = protocol::DnsUtil::getaddrinfo(dns_task->get_resp(), port_, &ai);\n\t\tDnsOutput out;\n\t\tDnsRoutine::create(&out, ret, ai);\n\t\tdns_callback_internal(&out, dns_ttl_default_, dns_ttl_min_);\n\t}\n\telse\n\t{\n\t\tthis->state = WFT_STATE_DNS_ERROR;\n\t\tthis->error = EAI_AGAIN;\n\t}\n\n\ttask_callback();\n}\n\nvoid WFResolverTask::dns_partial_callback(void *net_dns_task)\n{\n\tWFDnsTask *dns_task = (WFDnsTask *)net_dns_task;\n\tWFGlobal::get_dns_respool()->post(NULL);\n\n\tstruct DnsContext *ctx = (struct DnsContext *)dns_task->user_data;\n\n\tctx->ai = NULL;\n\tif (dns_task->get_state() == WFT_STATE_SUCCESS)\n\t{\n\t\tprotocol::DnsResponse *resp = dns_task->get_resp();\n\t\tctx->eai_error = protocol::DnsUtil::getaddrinfo(resp, ctx->port,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t&ctx->ai);\n\t}\n\telse\n\t\tctx->eai_error = EAI_AGAIN;\n}\n\nvoid WFResolverTask::dns_parallel_callback(const void *parallel)\n{\n\tconst ParallelWork *pwork = (const ParallelWork *)parallel;\n\tstruct DnsContext *c4 = (struct DnsContext *)pwork->get_context();\n\tstruct DnsContext *c6 = c4 + 1;\n\n\tif (c4->eai_error == 0 || c6->eai_error == 0)\n\t{\n\t\tstruct addrinfo *ai = NULL;\n\t\tstruct addrinfo **pai = &ai;\n\t\tDnsOutput out;\n\n\t\t*pai = c4->ai;\n\t\twhile (*pai)\n\t\t\tpai = &(*pai)->ai_next;\n\n\t\t*pai = c6->ai;\n\t\tDnsRoutine::create(&out, 0, ai);\n\t\tdns_callback_internal(&out, dns_ttl_default_, dns_ttl_min_);\n\t}\n\telse\n\t{\n\t\tint eai_error = c4->eai_error;\n\n\t\tif (c6->eai_error == EAI_AGAIN)\n\t\t\teai_error = EAI_AGAIN;\n\n\t\tthis->state = WFT_STATE_DNS_ERROR;\n\t\tthis->error = eai_error;\n\t}\n\n\tdelete []c4;\n\ttask_callback();\n}\n\nvoid WFResolverTask::thread_dns_callback(void *thrd_dns_task)\n{\n\tThreadDnsTask *dns_task = (ThreadDnsTask *)thrd_dns_task;\n\n\tif (dns_task->get_state() == WFT_STATE_SUCCESS)\n\t{\n\t\tDnsOutput *out = dns_task->get_output();\n\t\tdns_callback_internal(out, dns_ttl_default_, dns_ttl_min_);\n\t}\n\telse\n\t{\n\t\tthis->state = dns_task->get_state();\n\t\tthis->error = dns_task->get_error();\n\t}\n\n\ttask_callback();\n}\n\nvoid WFResolverTask::task_callback()\n{\n\tif (in_guard_)\n\t{\n\t\tint family = ep_params_.address_family;\n\t\tstd::string cache_host = __get_cache_host(host_, family);\n\t\tstd::string guard_name = __get_guard_name(cache_host, port_);\n\n\t\tif (this->state == WFT_STATE_DNS_ERROR)\n\t\t\tmsg_ = (void *)(intptr_t)this->error;\n\n\t\tWFTaskFactory::release_guard_safe(guard_name, msg_);\n\t}\n\n\tif (this->callback)\n\t\tthis->callback(this);\n\n\tdelete this;\n}\n\nWFRouterTask *WFDnsResolver::create_router_task(const struct WFNSParams *params,\n\t\t\t\t\t\t\t\t\t\t\t\trouter_callback_t callback)\n{\n\tconst struct WFGlobalSettings *settings = WFGlobal::get_global_settings();\n\tunsigned int dns_ttl_default = settings->dns_ttl_default;\n\tunsigned int dns_ttl_min = settings->dns_ttl_min;\n\tconst struct EndpointParams *ep_params = &settings->endpoint_params;\n\treturn new WFResolverTask(params, dns_ttl_default, dns_ttl_min, ep_params,\n\t\t\t\t\t\t\t std::move(callback));\n}\n\n" }, { "path": "src/nameservice/WFDnsResolver.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFDNSRESOLVER_H_\n#define _WFDNSRESOLVER_H_\n\n#include \n#include \n#include \"EndpointParams.h\"\n#include \"WFNameService.h\"\n\nclass WFResolverTask : public WFRouterTask\n{\npublic:\n\tWFResolverTask(const struct WFNSParams *ns_params,\n\t\t\t\t unsigned int dns_ttl_default, unsigned int dns_ttl_min,\n\t\t\t\t const struct EndpointParams *ep_params,\n\t\t\t\t router_callback_t&& cb) :\n\t\tWFRouterTask(std::move(cb)),\n\t\tns_params_(*ns_params),\n\t\tep_params_(*ep_params)\n\t{\n\t\tif (ns_params_.fixed_conn)\n\t\t\tep_params_.max_connections = 1;\n\n\t\tdns_ttl_default_ = dns_ttl_default;\n\t\tdns_ttl_min_ = dns_ttl_min;\n\t\thas_next_ = false;\n\t\tin_guard_ = false;\n\t\tmsg_ = NULL;\n\t}\n\n\tWFResolverTask(const struct WFNSParams *ns_params,\n\t\t\t\t router_callback_t&& cb) :\n\t\tWFRouterTask(std::move(cb)),\n\t\tns_params_(*ns_params)\n\t{\n\t\tif (ns_params_.fixed_conn)\n\t\t\tep_params_.max_connections = 1;\n\n\t\thas_next_ = false;\n\t\tin_guard_ = false;\n\t\tmsg_ = NULL;\n\t}\n\nprotected:\n\tvirtual void dispatch();\n\tvirtual SubTask *done();\n\nprivate:\n\tvoid thread_dns_callback(void *thrd_dns_task);\n\tvoid dns_single_callback(void *net_dns_task);\n\tstatic void dns_partial_callback(void *net_dns_task);\n\tvoid dns_parallel_callback(const void *parallel);\n\tvoid dns_callback_internal(void *thrd_dns_output,\n\t\t\t\t\t\t\t unsigned int ttl_default,\n\t\t\t\t\t\t\t unsigned int ttl_min);\n\n\tvoid request_dns();\n\tvoid task_callback();\n\nprotected:\n\tstruct WFNSParams ns_params_;\n\tunsigned int dns_ttl_default_;\n\tunsigned int dns_ttl_min_;\n\tstruct EndpointParams ep_params_;\n\nprivate:\n\tconst char *host_;\n\tunsigned short port_;\n\tbool has_next_;\n\tbool in_guard_;\n\tvoid *msg_;\n};\n\nclass WFDnsResolver : public WFNSPolicy\n{\npublic:\n\tvirtual WFRouterTask *create_router_task(const struct WFNSParams *params,\n\t\t\t\t\t\t\t\t\t\t\t router_callback_t callback);\n};\n\n#endif\n\n" }, { "path": "src/nameservice/WFNameService.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"rbtree.h\"\n#include \"WFNameService.h\"\n\nstruct WFNSPolicyEntry\n{\n\tstruct rb_node rb;\n\tWFNSPolicy *policy;\n\tchar name[1];\n};\n\nint WFNameService::add_policy(const char *name, WFNSPolicy *policy)\n{\n\tstruct rb_node **p = &this->root.rb_node;\n\tstruct rb_node *parent = NULL;\n\tstruct WFNSPolicyEntry *entry;\n\tint n, ret = -1;\n\n\tpthread_rwlock_wrlock(&this->rwlock);\n\twhile (*p)\n\t{\n\t\tparent = *p;\n\t\tentry = rb_entry(*p, struct WFNSPolicyEntry, rb);\n\t\tn = strcasecmp(name, entry->name);\n\t\tif (n < 0)\n\t\t\tp = &(*p)->rb_left;\n\t\telse if (n > 0)\n\t\t\tp = &(*p)->rb_right;\n\t\telse\n\t\t\tbreak;\n\t}\n\n\tif (!*p)\n\t{\n\t\tsize_t len = strlen(name);\n\t\tsize_t size = offsetof(struct WFNSPolicyEntry, name) + len + 1;\n\n\t\tentry = (struct WFNSPolicyEntry *)malloc(size);\n\t\tif (entry)\n\t\t{\n\t\t\tmemcpy(entry->name, name, len + 1);\n\t\t\tentry->policy = policy;\n\t\t\trb_link_node(&entry->rb, parent, p);\n\t\t\trb_insert_color(&entry->rb, &this->root);\n\t\t\tret = 0;\n\t\t}\n\t}\n\telse\n\t\terrno = EEXIST;\n\n\tpthread_rwlock_unlock(&this->rwlock);\n\treturn ret;\n}\n\ninline struct WFNSPolicyEntry *WFNameService::get_policy_entry(const char *name)\n{\n\tstruct rb_node *p = this->root.rb_node;\n\tstruct WFNSPolicyEntry *entry;\n\tint n;\n\n\twhile (p)\n\t{\n\t\tentry = rb_entry(p, struct WFNSPolicyEntry, rb);\n\t\tn = strcasecmp(name, entry->name);\n\t\tif (n < 0)\n\t\t\tp = p->rb_left;\n\t\telse if (n > 0)\n\t\t\tp = p->rb_right;\n\t\telse\n\t\t\treturn entry;\n\t}\n\n\treturn NULL;\n}\n\nWFNSPolicy *WFNameService::get_policy(const char *name)\n{\n\tWFNSPolicy *policy = this->default_policy;\n\tstruct WFNSPolicyEntry *entry;\n\n\tif (this->root.rb_node)\n\t{\n\t\tpthread_rwlock_rdlock(&this->rwlock);\n\t\tentry = this->get_policy_entry(name);\n\t\tif (entry)\n\t\t\tpolicy = entry->policy;\n\n\t\tpthread_rwlock_unlock(&this->rwlock);\n\t}\n\n\treturn policy;\n}\n\nWFNSPolicy *WFNameService::del_policy(const char *name)\n{\n\tWFNSPolicy *policy = NULL;\n\tstruct WFNSPolicyEntry *entry;\n\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tentry = this->get_policy_entry(name);\n\tif (entry)\n\t{\n\t\tpolicy = entry->policy;\n\t\trb_erase(&entry->rb, &this->root);\n\t}\n\n\tpthread_rwlock_unlock(&this->rwlock);\n\tfree(entry);\n\treturn policy;\n}\n\nWFNameService::~WFNameService()\n{\n\tstruct WFNSPolicyEntry *entry;\n\n\twhile (this->root.rb_node)\n\t{\n\t\tentry = rb_entry(this->root.rb_node, struct WFNSPolicyEntry, rb);\n\t\trb_erase(&entry->rb, &this->root);\n\t\tfree(entry);\n\t}\n\n\tpthread_rwlock_destroy(&this->rwlock);\n}\n\n" }, { "path": "src/nameservice/WFNameService.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFNAMESERVICE_H_\n#define _WFNAMESERVICE_H_\n\n#include \n#include \n#include \n#include \"rbtree.h\"\n#include \"Communicator.h\"\n#include \"Workflow.h\"\n#include \"WFTask.h\"\n#include \"RouteManager.h\"\n#include \"URIParser.h\"\n#include \"EndpointParams.h\"\n\nclass WFRouterTask : public WFGenericTask\n{\npublic:\n\tRouteManager::RouteResult *get_result() { return &this->result; }\n\npublic:\n\tvoid set_state(int state) { this->state = state; }\n\tvoid set_error(int error) { this->error = error; }\n\nprotected:\n\tRouteManager::RouteResult result;\n\tstd::function callback;\n\nprotected:\n\tvirtual SubTask *done()\n\t{\n\t\tSeriesWork *series = series_of(this);\n\n\t\tif (this->callback)\n\t\t\tthis->callback(this);\n\n\t\tdelete this;\n\t\treturn series->pop();\n\t}\n\npublic:\n\tWFRouterTask(std::function&& cb) :\n\t\tcallback(std::move(cb))\n\t{\n\t}\n};\n\nclass WFNSTracing\n{\npublic:\n\tvoid *data;\n\tvoid (*deleter)(void *);\n\npublic:\n\tWFNSTracing()\n\t{\n\t\tthis->data = NULL;\n\t\tthis->deleter = NULL;\n\t}\n};\n\nstruct WFNSParams\n{\n\tenum TransportType type;\n\tParsedURI& uri;\n\tconst char *info;\n\tSSL_CTX *ssl_ctx;\n\tbool fixed_addr;\n\tbool fixed_conn;\n\tint retry_times;\n\tWFNSTracing *tracing;\n};\n\nusing router_callback_t = std::function;\n\nclass WFNSPolicy\n{\npublic:\n\tvirtual WFRouterTask *create_router_task(const struct WFNSParams *params,\n\t\t\t\t\t\t\t\t\t\t\t router_callback_t callback) = 0;\n\n\tvirtual void success(RouteManager::RouteResult *result,\n\t\t\t\t\t\t WFNSTracing *tracing,\n\t\t\t\t\t\t CommTarget *target)\n\t{\n\t\tRouteManager::notify_available(result->cookie, target);\n\t}\n\n\tvirtual void failed(RouteManager::RouteResult *result,\n\t\t\t\t\t\tWFNSTracing *tracing,\n\t\t\t\t\t\tCommTarget *target)\n\t{\n\t\tif (target)\n\t\t\tRouteManager::notify_unavailable(result->cookie, target);\n\t}\n\npublic:\n\tvirtual ~WFNSPolicy() { }\n};\n\nclass WFNameService\n{\npublic:\n\tint add_policy(const char *name, WFNSPolicy *policy);\n\tWFNSPolicy *get_policy(const char *name);\n\tWFNSPolicy *del_policy(const char *name);\n\npublic:\n\tWFNSPolicy *get_default_policy() const\n\t{\n\t\treturn this->default_policy;\n\t}\n\n\tvoid set_default_policy(WFNSPolicy *policy)\n\t{\n\t\tthis->default_policy = policy;\n\t}\n\nprivate:\n\tWFNSPolicy *default_policy;\n\tstruct rb_root root;\n\tpthread_rwlock_t rwlock;\n\nprivate:\n\tstruct WFNSPolicyEntry *get_policy_entry(const char *name);\n\npublic:\n\tWFNameService(WFNSPolicy *default_policy) :\n\t\trwlock(PTHREAD_RWLOCK_INITIALIZER)\n\t{\n\t\tthis->root.rb_node = NULL;\n\t\tthis->default_policy = default_policy;\n\t}\n\n\tvirtual ~WFNameService();\n};\n\n#endif\n\n" }, { "path": "src/nameservice/WFServiceGovernance.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \"URIParser.h\"\n#include \"WFTaskError.h\"\n#include \"StringUtil.h\"\n#include \"WFGlobal.h\"\n#include \"WFNameService.h\"\n#include \"WFDnsResolver.h\"\n#include \"WFServiceGovernance.h\"\n\n#define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()\n\n#define DNS_CACHE_LEVEL_1\t\t1\n#define DNS_CACHE_LEVEL_2\t\t2\n\n#define MTTR_SECONDS_DEFAULT\t30\n\nWFServiceGovernance::WFServiceGovernance() :\n\t\tbreaker_lock(PTHREAD_MUTEX_INITIALIZER),\n\t\trwlock(PTHREAD_RWLOCK_INITIALIZER)\n{\n\tthis->nalives = 0;\n\tthis->try_another = false;\n\tthis->mttr_seconds = MTTR_SECONDS_DEFAULT;\n\tINIT_LIST_HEAD(&this->breaker_list);\n}\n\nWFServiceGovernance::~WFServiceGovernance()\n{\n\tfor (EndpointAddress *addr : this->servers)\n\t\tdelete addr;\n\n\tpthread_rwlock_destroy(&this->rwlock);\n\tpthread_mutex_destroy(&this->breaker_lock);\n}\n\nPolicyAddrParams::PolicyAddrParams()\n{\n\tconst struct AddressParams *params = &ADDRESS_PARAMS_DEFAULT;\n\tthis->endpoint_params = params->endpoint_params;\n\tthis->dns_ttl_default = params->dns_ttl_default;\n\tthis->dns_ttl_min = params->dns_ttl_min;\n\tthis->max_fails = params->max_fails;\n}\n\nPolicyAddrParams::PolicyAddrParams(const struct AddressParams *params) :\n\tendpoint_params(params->endpoint_params)\n{\n\tthis->dns_ttl_default = params->dns_ttl_default;\n\tthis->dns_ttl_min = params->dns_ttl_min;\n\tthis->max_fails = params->max_fails;\n}\n\nEndpointAddress::EndpointAddress(const std::string& address,\n\t\t\t\t\t\t\t\t PolicyAddrParams *address_params)\n{\n\tstd::vector arr = StringUtil::split(address, ':');\n\n\tthis->params = address_params;\n\tif (this->params->max_fails == 0)\n\t\tthis->params->max_fails = 1;\n\n\tthis->address = address;\n\tthis->fail_count = 0;\n\tthis->ref = 1;\n\tthis->entry.list.next = NULL;\n\tthis->entry.ptr = this;\n\n\tif (arr.size() == 0)\n\t\tthis->host = \"\";\n\telse\n\t\tthis->host = arr[0];\n\n\tif (arr.size() <= 1)\n\t\tthis->port = \"\";\n\telse\n\t\tthis->port = arr[1];\n}\n\nclass WFSGResolverTask : public WFResolverTask\n{\npublic:\n\tWFSGResolverTask(const struct WFNSParams *params,\n\t\t\t\t\t WFServiceGovernance *sg,\n\t\t\t\t\t router_callback_t&& cb) :\n\t\tWFResolverTask(params, std::move(cb))\n\t{\n\t\tsg_ = sg;\n\t}\n\nprotected:\n\tvirtual void dispatch();\n\nprotected:\n\tWFServiceGovernance *sg_;\n};\n\nstatic void copy_host_port(ParsedURI& uri, const EndpointAddress *addr)\n{\n\tif (!addr->host.empty())\n\t{\n\t\tfree(uri.host);\n\t\turi.host = strdup(addr->host.c_str());\n\t}\n\n\tif (!addr->port.empty())\n\t{\n\t\tfree(uri.port);\n\t\turi.port = strdup(addr->port.c_str());\n\t}\n}\n\nvoid WFSGResolverTask::dispatch()\n{\n\tWFNSTracing *tracing = ns_params_.tracing;\n\tEndpointAddress *addr;\n\n\tif (!sg_)\n\t{\n\t\tthis->WFResolverTask::dispatch();\n\t\treturn;\n\t}\n\n\tif (sg_->select(ns_params_.uri, tracing, &addr))\n\t{\n\t\tauto *tracing_data = (WFServiceGovernance::TracingData *)tracing->data;\n\t\tif (!tracing_data)\n\t\t{\n\t\t\ttracing_data = new WFServiceGovernance::TracingData;\n\t\t\ttracing_data->sg = sg_;\n\t\t\ttracing->data = tracing_data;\n\t\t\ttracing->deleter = WFServiceGovernance::tracing_deleter;\n\t\t}\n\n\t\ttracing_data->history.push_back(addr);\n\t\tsg_ = NULL;\n\n\t\tcopy_host_port(ns_params_.uri, addr);\n\t\tdns_ttl_default_ = addr->params->dns_ttl_default;\n\t\tdns_ttl_min_ = addr->params->dns_ttl_min;\n\t\tep_params_ = addr->params->endpoint_params;\n\t\tthis->WFResolverTask::dispatch();\n\t}\n\telse\n\t{\n\t\tthis->state = WFT_STATE_TASK_ERROR;\n\t\tthis->error = WFT_ERR_UPSTREAM_UNAVAILABLE;\n\t\tthis->subtask_done();\n\t}\n}\n\nWFRouterTask *WFServiceGovernance::create_router_task(const struct WFNSParams *params,\n\t\t\t\t\t\t\t\t\t\t\t\t\t router_callback_t callback)\n{\n\treturn new WFSGResolverTask(params, this, std::move(callback));\n}\n\nvoid WFServiceGovernance::tracing_deleter(void *data)\n{\n\tstruct TracingData *tracing_data = (struct TracingData *)data;\n\n\tfor (EndpointAddress *addr : tracing_data->history)\n\t{\n\t\tif (--addr->ref == 0)\n\t\t{\n\t\t\tpthread_rwlock_wrlock(&tracing_data->sg->rwlock);\n\t\t\ttracing_data->sg->pre_delete_server(addr);\n\t\t\tpthread_rwlock_unlock(&tracing_data->sg->rwlock);\n\t\t\tdelete addr;\n\t\t}\n\t}\n\n\tdelete tracing_data;\n}\n\nbool WFServiceGovernance::in_select_history(WFNSTracing *tracing,\n\t\t\t\t\t\t\t\t\t\t\tEndpointAddress *addr)\n{\n\tstruct TracingData *tracing_data = (struct TracingData *)tracing->data;\n\n\tif (!tracing_data)\n\t\treturn false;\n\n\tfor (EndpointAddress *server : tracing_data->history)\n\t{\n\t\tif (server == addr)\n\t\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nvoid WFServiceGovernance::recover_server_from_breaker(EndpointAddress *addr)\n{\n\taddr->fail_count = 0;\n\tpthread_mutex_lock(&this->breaker_lock);\n\tif (addr->entry.list.next)\n\t{\n\t\tlist_del(&addr->entry.list);\n\t\taddr->entry.list.next = NULL;\n\t\tthis->recover_one_server(addr);\n\t}\n\tpthread_mutex_unlock(&this->breaker_lock);\n}\n\nvoid WFServiceGovernance::fuse_server_to_breaker(EndpointAddress *addr)\n{\n\tpthread_mutex_lock(&this->breaker_lock);\n\tif (!addr->entry.list.next)\n\t{\n\t\taddr->broken_timeout = GET_CURRENT_SECOND + this->mttr_seconds;\n\t\tlist_add_tail(&addr->entry.list, &this->breaker_list);\n\t\tthis->fuse_one_server(addr);\n\t}\n\tpthread_mutex_unlock(&this->breaker_lock);\n}\n\nvoid WFServiceGovernance::pre_delete_server(EndpointAddress *addr)\n{\n\tpthread_mutex_lock(&this->breaker_lock);\n\tif (addr->entry.list.next)\n\t{\n\t\tlist_del(&addr->entry.list);\n\t\taddr->entry.list.next = NULL;\n\t}\n\telse\n\t\tthis->fuse_one_server(addr);\n\tpthread_mutex_unlock(&this->breaker_lock);\n}\n\nvoid WFServiceGovernance::success(RouteManager::RouteResult *result,\n\t\t\t\t\t\t\t\t WFNSTracing *tracing,\n\t\t\t\t\t\t\t\t CommTarget *target)\n{\n\tstruct TracingData *tracing_data = (struct TracingData *)tracing->data;\n\tauto *v = &tracing_data->history;\n\tEndpointAddress *server = (*v)[v->size() - 1];\n\n\tserver->fail_count = 0;\n\tif (server->entry.list.next)\n\t{\n\t\tpthread_rwlock_wrlock(&this->rwlock);\n\t\tthis->recover_server_from_breaker(server);\n\t\tpthread_rwlock_unlock(&this->rwlock);\n\t}\n\n\tthis->WFNSPolicy::success(result, tracing, target);\n}\n\nvoid WFServiceGovernance::failed(RouteManager::RouteResult *result,\n\t\t\t\t\t\t\t\t WFNSTracing *tracing,\n\t\t\t\t\t\t\t\t CommTarget *target)\n{\n\tstruct TracingData *tracing_data = (struct TracingData *)tracing->data;\n\tauto *v = &tracing_data->history;\n\tEndpointAddress *server = (*v)[v->size() - 1];\n\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tif (++server->fail_count == server->params->max_fails)\n\t\tthis->fuse_server_to_breaker(server);\n\tpthread_rwlock_unlock(&this->rwlock);\n\n\tthis->WFNSPolicy::failed(result, tracing, target);\n}\n\nvoid WFServiceGovernance::check_breaker_locked(int64_t cur_time)\n{\n\tstruct list_head *pos, *tmp;\n\tstruct EndpointAddress::address_entry *entry;\n\tEndpointAddress *addr;\n\n\tlist_for_each_safe(pos, tmp, &this->breaker_list)\n\t{\n\t\tentry = list_entry(pos, struct EndpointAddress::address_entry, list);\n\t\taddr = entry->ptr;\n\n\t\tif (cur_time >= addr->broken_timeout)\n\t\t{\n\t\t\taddr->fail_count = addr->params->max_fails - 1;\n\t\t\tthis->recover_one_server(addr);\n\t\t\tlist_del(pos);\n\t\t\tpos->next = NULL;\n\t\t}\n\t\telse\n\t\t\tbreak;\n\t}\n}\n\nvoid WFServiceGovernance::check_breaker()\n{\n\tif (!list_empty(&this->breaker_list))\n\t{\n\t\tpthread_mutex_lock(&this->breaker_lock);\n\t\tthis->check_breaker_locked(GET_CURRENT_SECOND);\n\t\tpthread_mutex_unlock(&this->breaker_lock);\n\t}\n}\n\nvoid WFServiceGovernance::try_clear_breaker()\n{\n\tpthread_mutex_lock(&this->breaker_lock);\n\tif (!list_empty(&this->breaker_list))\n\t{\n\t\tstruct list_head *pos = this->breaker_list.next;\n\t\tstruct EndpointAddress::address_entry *entry;\n\t\tentry = list_entry(pos, struct EndpointAddress::address_entry, list);\n\t\tif (GET_CURRENT_SECOND >= entry->ptr->broken_timeout)\n\t\t\tthis->check_breaker_locked(INT64_MAX);\n\t}\n\tpthread_mutex_unlock(&this->breaker_lock);\n}\n\nEndpointAddress *WFServiceGovernance::first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\tunsigned int idx = rand() % this->servers.size();\n\treturn this->servers[idx];\n}\n\nEndpointAddress *WFServiceGovernance::another_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing)\n{\n\treturn this->first_strategy(uri, tracing);\n}\n\nbool WFServiceGovernance::select(const ParsedURI& uri, WFNSTracing *tracing,\n\t\t\t\t\t\t\t\t EndpointAddress **addr)\n{\n\tpthread_rwlock_rdlock(&this->rwlock);\n\tunsigned int n = (unsigned int)this->servers.size();\n\n\tif (n == 0)\n\t{\n\t\tpthread_rwlock_unlock(&this->rwlock);\n\t\treturn false;\n\t}\n\n\tthis->check_breaker();\n\tif (this->nalives == 0)\n\t{\n\t\tpthread_rwlock_unlock(&this->rwlock);\n\t\treturn false;\n\t}\n\n\tEndpointAddress *select_addr = this->first_strategy(uri, tracing);\n\n\tif (!select_addr ||\n\t\tselect_addr->fail_count >= select_addr->params->max_fails)\n\t{\n\t\tif (this->try_another)\n\t\t\tselect_addr = this->another_strategy(uri, tracing);\n\t}\n\n\tif (select_addr)\n\t{\n\t\t*addr = select_addr;\n\t\t++select_addr->ref;\n\t}\n\n\tpthread_rwlock_unlock(&this->rwlock);\n\treturn !!select_addr;\n}\n\nvoid WFServiceGovernance::add_server_locked(EndpointAddress *addr)\n{\n\tthis->server_map[addr->address].push_back(addr);\n\tthis->servers.push_back(addr);\n\tthis->recover_one_server(addr);\n}\n\nint WFServiceGovernance::remove_server_locked(const std::string& address)\n{\n\tconst auto map_it = this->server_map.find(address);\n\tsize_t n = this->servers.size();\n\tsize_t new_n = 0;\n\tint ret = 0;\n\n\tfor (size_t i = 0; i < n; i++)\n\t{\n\t\tif (this->servers[i]->address != address)\n\t\t\tthis->servers[new_n++] = this->servers[i];\n\t}\n\n\tthis->servers.resize(new_n);\n\n\tif (map_it != this->server_map.cend())\n\t{\n\t\tfor (EndpointAddress *addr : map_it->second)\n\t\t{\n\t\t\tif (--addr->ref == 0)\n\t\t\t{\n\t\t\t\tthis->pre_delete_server(addr);\n\t\t\t\tdelete addr;\n\t\t\t}\n\n\t\t\tret++;\n\t\t}\n\n\t\tthis->server_map.erase(map_it);\n\t}\n\n\treturn ret;\n}\n\nvoid WFServiceGovernance::add_server(const std::string& address,\n\t\t\t\t\t\t\t\t\t const AddressParams *params)\n{\n\tEndpointAddress *addr = new EndpointAddress(address,\n\t\t\t\t\t\t\t\t\tnew PolicyAddrParams(params));\n\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tthis->add_server_locked(addr);\n\tpthread_rwlock_unlock(&this->rwlock);\n}\n\nint WFServiceGovernance::remove_server(const std::string& address)\n{\n\tint ret;\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tret = this->remove_server_locked(address);\n\tpthread_rwlock_unlock(&this->rwlock);\n\treturn ret;\n}\n\nint WFServiceGovernance::replace_server(const std::string& address,\n\t\t\t\t\t\t\t\t\t\tconst AddressParams *params)\n{\n\tint ret;\n\tEndpointAddress *addr = new EndpointAddress(address,\n\t\t\t\t\t\t\t\t\tnew PolicyAddrParams(params));\n\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tret = this->remove_server_locked(address);\n\tthis->add_server_locked(addr);\n\tpthread_rwlock_unlock(&this->rwlock);\n\treturn ret;\n}\n\nvoid WFServiceGovernance::enable_server(const std::string& address)\n{\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tconst auto map_it = this->server_map.find(address);\n\tif (map_it != this->server_map.cend())\n\t{\n\t\tfor (EndpointAddress *addr : map_it->second)\n\t\t\tthis->recover_server_from_breaker(addr);\n\t}\n\tpthread_rwlock_unlock(&this->rwlock);\n}\n\nvoid WFServiceGovernance::disable_server(const std::string& address)\n{\n\tpthread_rwlock_wrlock(&this->rwlock);\n\tconst auto map_it = this->server_map.find(address);\n\tif (map_it != this->server_map.cend())\n\t{\n\t\tfor (EndpointAddress *addr : map_it->second)\n\t\t{\n\t\t\taddr->fail_count = addr->params->max_fails;\n\t\t\tthis->fuse_server_to_breaker(addr);\n\t\t}\n\t}\n\tpthread_rwlock_unlock(&this->rwlock);\n}\n\nvoid WFServiceGovernance::get_current_address(std::vector& addr_list)\n{\n\tpthread_rwlock_rdlock(&this->rwlock);\n\n\tfor (const EndpointAddress *server : this->servers)\n\t\taddr_list.push_back(server->address);\n\n\tpthread_rwlock_unlock(&this->rwlock);\n}\n\n" }, { "path": "src/nameservice/WFServiceGovernance.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFSERVICEGOVERNANCE_H_\n#define _WFSERVICEGOVERNANCE_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \"URIParser.h\"\n#include \"EndpointParams.h\"\n#include \"WFNameService.h\"\n\nstruct AddressParams\n{\n\tstruct EndpointParams endpoint_params; ///< Connection config\n\tunsigned int dns_ttl_default; ///< in seconds, DNS TTL when network request success\n\tunsigned int dns_ttl_min; ///< in seconds, DNS TTL when network request fail\n/**\n * - The max_fails directive sets the number of consecutive unsuccessful attempts to communicate with the server.\n * - After 30s following the server failure, upstream probe the server with some live client`s requests.\n * - If the probes have been successful, the server is marked as a live one.\n * - If max_fails is set to 1, it means server would out of upstream selection in 30 seconds when failed only once\n */\n\tunsigned int max_fails; ///< [1, INT32_MAX] max_fails = 0 means max_fails = 1\n\tunsigned short weight; ///< [1, 65535] weight = 0 means weight = 1. only for main server\n\tint server_type; ///< 0 for main and 1 for backup\n\tint group_id; ///< -1 means no group. Backup without group will be backup for any main\n};\n\nstatic constexpr struct AddressParams ADDRESS_PARAMS_DEFAULT =\n{\n\t.endpoint_params\t=\tENDPOINT_PARAMS_DEFAULT,\n\t.dns_ttl_default\t=\t12 * 3600,\n\t.dns_ttl_min\t\t=\t180,\n\t.max_fails\t\t\t=\t200,\n\t.weight\t\t\t\t=\t1,\n\t.server_type\t\t=\t0,\t/* 0 for main and 1 for backup. */\n\t.group_id\t\t\t=\t-1,\n};\n\nclass PolicyAddrParams\n{\npublic:\n\tstruct EndpointParams endpoint_params;\n\tunsigned int dns_ttl_default;\n\tunsigned int dns_ttl_min;\n\tunsigned int max_fails;\n\npublic:\n\tPolicyAddrParams();\n\tPolicyAddrParams(const struct AddressParams *params);\n\tvirtual ~PolicyAddrParams() { }\n};\n\nclass EndpointAddress\n{\npublic:\n\tstd::string address;\n\tstd::string host;\n\tstd::string port;\n\tunsigned int fail_count;\n\tstd::atomic ref;\n\tlong long broken_timeout;\n\tPolicyAddrParams *params;\n\n\tstruct address_entry\n\t{\n\t\tstruct list_head list;\n\t\tEndpointAddress *ptr;\n\t} entry;\n\npublic:\n\tEndpointAddress(const std::string& address, PolicyAddrParams *params);\n\tvirtual ~EndpointAddress() { delete this->params; }\n};\n\nclass WFServiceGovernance : public WFNSPolicy\n{\npublic:\n\tvirtual WFRouterTask *create_router_task(const struct WFNSParams *params,\n\t\t\t\t\t\t\t\t\t\t\t router_callback_t callback);\n\tvirtual void success(RouteManager::RouteResult *result,\n\t\t\t\t\t\t WFNSTracing *tracing,\n\t\t\t\t\t \t CommTarget *target);\n\tvirtual void failed(RouteManager::RouteResult *result,\n\t\t\t\t\t\tWFNSTracing *tracing,\n\t\t\t\t\t\tCommTarget *target);\n\n\tvirtual void add_server(const std::string& address,\n\t\t\t\t\t\t\tconst struct AddressParams *params);\n\tint remove_server(const std::string& address);\n\tvirtual int replace_server(const std::string& address,\n\t\t\t\t\t\t\t const struct AddressParams *params);\n\n\tvoid enable_server(const std::string& address);\n\tvoid disable_server(const std::string& address);\n\tvirtual void get_current_address(std::vector& addr_list);\n\n\tvoid set_mttr_seconds(unsigned int seconds)\n\t{\n\t\tthis->mttr_seconds = seconds;\n\t}\n\n\tstatic bool in_select_history(WFNSTracing *tracing, EndpointAddress *addr);\n\nprivate:\n\tvirtual bool select(const ParsedURI& uri, WFNSTracing *tracing,\n\t\t\t\t\t\tEndpointAddress **addr);\n\n\tvirtual void recover_one_server(const EndpointAddress *addr)\n\t{\n\t\tthis->nalives++;\n\t}\n\n\tvirtual void fuse_one_server(const EndpointAddress *addr)\n\t{\n\t\tthis->nalives--;\n\t}\n\n\tvirtual void add_server_locked(EndpointAddress *addr);\n\tvirtual int remove_server_locked(const std::string& address);\n\n\tvoid recover_server_from_breaker(EndpointAddress *addr);\n\tvoid fuse_server_to_breaker(EndpointAddress *addr);\n\tvoid check_breaker_locked(int64_t cur_time);\n\nprivate:\n\tstruct list_head breaker_list;\n\tpthread_mutex_t breaker_lock;\n\tunsigned int mttr_seconds;\n\nprotected:\n\tvirtual EndpointAddress *first_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\tWFNSTracing *tracing);\n\tvirtual EndpointAddress *another_strategy(const ParsedURI& uri,\n\t\t\t\t\t\t\t\t\t\t\t WFNSTracing *tracing);\n\tvoid check_breaker();\n\tvoid try_clear_breaker();\n\tvoid pre_delete_server(EndpointAddress *addr);\n\n\tstruct TracingData\n\t{\n\t\tstd::vector history;\n\t\tWFServiceGovernance *sg;\n\t};\n\n\tstatic void tracing_deleter(void *data);\n\n\tstd::vector servers;\n\tstd::unordered_map> server_map;\n\tpthread_rwlock_t rwlock;\n\tstd::atomic nalives;\n\tbool try_another;\n\npublic:\n\tWFServiceGovernance();\n\tvirtual ~WFServiceGovernance();\n\tfriend class WFSGResolverTask;\n};\n\n#endif\n\n" }, { "path": "src/nameservice/xmake.lua", "content": "target(\"nameservice\")\n add_files(\"*.cc\")\n set_kind(\"object\")\n if not has_config(\"upstream\") then\n remove_files(\"WFServiceGovernance.cc\", \"UpstreamPolicies.cc\")\n end\n\n" }, { "path": "src/protocol/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\nproject(protocol)\n\nset(SRC\n\tPackageWrapper.cc\n\tSSLWrapper.cc\n\tdns_parser.c\n\tDnsMessage.cc\n\tDnsUtil.cc\n\thttp_parser.c\n\tHttpMessage.cc\n\tHttpUtil.cc\n\tTLVMessage.cc\n)\n\nif (NOT MYSQL STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tmysql_stream.c\n\t\tmysql_parser.c\n\t\tmysql_byteorder.c\n\t\tMySQLMessage.cc\n\t\tMySQLResult.cc\n\t\tMySQLUtil.cc\n\t)\nendif ()\n\nif (NOT REDIS STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tredis_parser.c\n\t\tRedisMessage.cc\n\t)\nendif ()\n\nadd_library(${PROJECT_NAME} OBJECT ${SRC})\n\nif (KAFKA STREQUAL \"y\")\n\tset(SRC\n\t\tkafka_parser.c\n\t\tKafkaMessage.cc\n\t\tKafkaDataTypes.cc\n\t\tKafkaResult.cc\n\t)\n\tadd_library(\"protocol_kafka\" OBJECT ${SRC})\nendif ()\n" }, { "path": "src/protocol/ConsulDataTypes.h", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhenpeng (wangzhenpeng@sogou-inc.com)\n*/\n\n#ifndef _CONSULDATATYPES_H_\n#define _CONSULDATATYPES_H_\n\n#include \n#include \n#include \n#include \n#include \n\nnamespace protocol\n{\n\nclass ConsulConfig\n{\npublic:\n\t// common config\n\tvoid set_token(const std::string& token) { this->ptr->token = token; }\n\tstd::string get_token() const { return this->ptr->token; }\n\n\t// discover config\n\n\tvoid set_datacenter(const std::string& data_center)\n\t{\n\t\tthis->ptr->dc = data_center;\n\t}\n\tstd::string get_datacenter() const { return this->ptr->dc; }\n\n\tvoid set_near_node(const std::string& near_node)\n\t{\n\t\tthis->ptr->near = near_node;\n\t}\n\tstd::string get_near_node() const { return this->ptr->near; }\n\n\tvoid set_filter_expr(const std::string& filter_expr)\n\t{\n\t\tthis->ptr->filter = filter_expr;\n\t}\n\tstd::string get_filter_expr() const { return this->ptr->filter; }\n\n\t// blocking query wait, limited to 10 minutes, default:5m, unit:ms\n\tvoid set_wait_ttl(int wait_ttl) { this->ptr->wait_ttl = wait_ttl; }\n\tint get_wait_ttl() const { return this->ptr->wait_ttl; }\n\n\t// enable blocking query\n\tvoid set_blocking_query(bool enable_flag)\n\t{\n\t\tthis->ptr->blocking_query = enable_flag;\n\t}\n\tbool blocking_query() const { return this->ptr->blocking_query; }\n\n\t// only get health passing status service instance\n\tvoid set_passing(bool passing) { this->ptr->passing = passing; }\n\tbool get_passing() const { return this->ptr->passing; }\n\n\n\t// register config\n\n\tvoid set_replace_checks(bool replace_checks)\n\t{\n\t\tthis->ptr->replace_checks = replace_checks;\n\t}\n\tbool get_replace_checks() const\n\t{\n\t\treturn this->ptr->replace_checks;\n\t}\n\t\n\tvoid set_check_name(const std::string& check_name)\n\t{\n\t\tthis->ptr->check_cfg.check_name = check_name;\n\t}\n\tstd::string get_check_name() const { return this->ptr->check_cfg.check_name; }\n\n\tvoid set_check_http_url(const std::string& http_url)\n\t{\n\t\tthis->ptr->check_cfg.http_url = http_url;\n\t}\n\tstd::string get_check_http_url() const\n\t{\n\t\treturn this->ptr->check_cfg.http_url;\n\t}\n\n\tvoid set_check_http_method(const std::string& method)\n\t{\n\t\tthis->ptr->check_cfg.http_method = method;\n\t}\n\tstd::string get_check_http_method() const\n\t{\n\t\treturn this->ptr->check_cfg.http_method;\n\t}\n\n\tvoid add_http_header(const std::string& key,\n\t\t\t\t\t\t const std::vector& values)\n\t{\n\t\tthis->ptr->check_cfg.headers.emplace(key, values);\n\t}\n\tconst std::map> *get_http_headers() const\n\t{\n\t\treturn &this->ptr->check_cfg.headers;\n\t}\n\n\tvoid set_http_body(const std::string& body)\n\t{\n\t\tthis->ptr->check_cfg.http_body = body;\n\t}\n\tstd::string get_http_body() const { return this->ptr->check_cfg.http_body; }\n\n\tvoid set_check_interval(int interval)\n\t{\n\t\tthis->ptr->check_cfg.interval = interval;\n\t}\n\tint get_check_interval() const { return this->ptr->check_cfg.interval; }\n\n\tvoid set_check_timeout(int timeout)\n\t{\n\t\tthis->ptr->check_cfg.timeout = timeout;\n\t}\n\tint get_check_timeout() const { return this->ptr->check_cfg.timeout; }\n\n\tvoid set_check_notes(const std::string& notes)\n\t{\n\t\tthis->ptr->check_cfg.notes = notes;\n\t}\n\tstd::string get_check_notes() const { return this->ptr->check_cfg.notes; }\n\n\tvoid set_check_tcp(const std::string& tcp_address)\n\t{\n\t\tthis->ptr->check_cfg.tcp_address = tcp_address;\n\t}\n\tstd::string get_check_tcp() const { return this->ptr->check_cfg.tcp_address; }\n\n\tvoid set_initial_status(const std::string& initial_status)\n\t{\n\t\tthis->ptr->check_cfg.initial_status = initial_status;\n\t}\n\tstd::string get_initial_status() const\n\t{\n\t\treturn this->ptr->check_cfg.initial_status;\n\t}\n\n\tvoid set_auto_deregister_time(int milliseconds)\n\t{\n\t\tthis->ptr->check_cfg.auto_deregister_time = milliseconds;\n\t}\n\tint get_auto_deregister_time() const\n\t{\n\t\treturn this->ptr->check_cfg.auto_deregister_time;\n\t}\n\n\t// set success times before passing, refer to success_before_passing, default:0\n\tvoid set_success_times(int times)\n\t{ \n\t\tthis->ptr->check_cfg.success_times = times;\n\t}\n\tint get_success_times() const { return this->ptr->check_cfg.success_times; }\n\n\t// set failure times before critical, refer to failures_before_critical, default:0\n\tvoid set_failure_times(int times) { this->ptr->check_cfg.failure_times = times; }\n\tint get_failure_times() const { return this->ptr->check_cfg.failure_times; }\n\n\tvoid set_health_check(bool enable_flag)\n\t{\n\t\tthis->ptr->check_cfg.health_check = enable_flag;\n\t}\n\tbool get_health_check() const\n\t{\n\t\treturn this->ptr->check_cfg.health_check;\n\t}\n\npublic:\n\tConsulConfig()\n\t{\n\t\tthis->ptr = new Config;\t\n\t\tthis->ptr->blocking_query = false;\n\t\tthis->ptr->passing = false;\n\t\tthis->ptr->replace_checks = false;\n\t\tthis->ptr->wait_ttl = 300 * 1000;\n\t\tthis->ptr->check_cfg.interval = 5000;\n\t\tthis->ptr->check_cfg.timeout = 10000;\n\t\tthis->ptr->check_cfg.http_method = \"GET\";\n\t\tthis->ptr->check_cfg.initial_status = \"critical\";\n\t\tthis->ptr->check_cfg.auto_deregister_time = 10 * 60 * 1000;\n\t\tthis->ptr->check_cfg.success_times = 0;\n\t\tthis->ptr->check_cfg.failure_times = 0;\n\t\tthis->ptr->check_cfg.health_check = false;\n\n\t\tthis->ref = new std::atomic(1);\n\t}\n\n\tvirtual ~ConsulConfig()\n\t{\n\t\tif (--*this->ref == 0)\n\t\t{\n\t\t\tdelete this->ptr;\n\t\t\tdelete this->ref;\n\t\t}\n\t}\n\n\tConsulConfig(ConsulConfig&& move)\n\t{\n\t\tthis->ptr = move.ptr;\n\t\tthis->ref = move.ref;\n\t\tmove.ptr = new Config;\n\t\tmove.ref = new std::atomic(1);\n\t}\n\n\tConsulConfig(const ConsulConfig& copy)\n\t{\n\t\tthis->ptr = copy.ptr;\n\t\tthis->ref = copy.ref;\n\t\t++(*this->ref);\n\t}\n\n\tConsulConfig& operator= (ConsulConfig&& move)\n\t{\n\t\tif (this != &move)\n\t\t{\n\t\t\tthis->~ConsulConfig();\n\t\t\tthis->ptr = move.ptr;\n\t\t\tthis->ref = move.ref;\n\t\t\tmove.ptr = new Config;\n\t\t\tmove.ref = new std::atomic(1);\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tConsulConfig& operator= (const ConsulConfig& copy)\n\t{\n\t\tif (this != ©)\n\t\t{\n\t\t\tthis->~ConsulConfig();\n\t\t\tthis->ptr = copy.ptr;\n\t\t\tthis->ref = copy.ref;\n\t\t\t++(*this->ref);\n\t\t}\n\n\t\treturn *this;\n\t}\n\nprivate:\n\t// register health check config\n\tstruct HealthCheckConfig\n\t{\n\t\tstd::string check_name;\n\t\tstd::string notes;\n\t\tstd::string http_url;\n\t\tstd::string http_method;\n\t\tstd::string http_body;\n\t\tstd::string tcp_address;\n\t\tstd::string initial_status; // passing or critical, default:critical\n\t\tstd::map> headers;\n\t\tint auto_deregister_time; // refer to deregister_critical_service_after\n\t\tint interval;\n\t\tint timeout; // default 10000\n\t\tint success_times; // default:0 success times before passing\n\t\tint failure_times; // default:0 failure_before_critical\n\t\tbool health_check;\n\t};\n\n\tstruct Config\n\t{\n\t\t// common config\n\t\tstd::string token;\n\n\t\t// discover config\n\t\tstd::string dc;\n\t\tstd::string near;\n\t\tstd::string filter;\n\t\tint wait_ttl;\n\t\tbool blocking_query;\n\t\tbool passing;\n\n\t\t// register config\n\t\tbool replace_checks; //refer to replace_existing_checks\n\t\tHealthCheckConfig check_cfg;\n\t};\n\nprivate:\n\tstruct Config *ptr;\n\tstd::atomic *ref;\n};\n\n// k:address, v:port\nusing ConsulAddress = std::pair;\n\nstruct ConsulService\n{\n\tstd::string service_name;\n\tstd::string service_namespace;\n\tstd::string service_id;\n\tstd::vector tags;\n\tConsulAddress service_address;\n\tConsulAddress lan;\n\tConsulAddress lan_ipv4;\n\tConsulAddress lan_ipv6;\n\tConsulAddress virtual_address; \n\tConsulAddress wan;\n\tConsulAddress wan_ipv4;\n\tConsulAddress wan_ipv6;\n\tstd::map meta;\n\tbool tag_override;\n};\n\nstruct ConsulServiceInstance\n{\n\t// node info\n\tstd::string node_id;\n\tstd::string node_name;\n\tstd::string node_address;\n\tstd::string dc;\n\tstd::map node_meta;\n\tlong long create_index;\n\tlong long modify_index;\n\n\t// service info\n\tstruct ConsulService service;\n\n\t// service health check\n\tstd::string check_name;\n\tstd::string check_id;\n\tstd::string check_notes;\n\tstd::string check_output;\n\tstd::string check_status;\n\tstd::string check_type;\n};\n\nstruct ConsulServiceTags\n{\n\tstd::string service_name;\n\tstd::vector tags;\n};\n\n}\n\n#endif\n" }, { "path": "src/protocol/DnsMessage.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"dns_types.h\"\n#include \"dns_parser.h\"\n#include \"DnsMessage.h\"\n\n#define DNS_LABELS_MAX\t\t\t\t63\n#define DNS_MESSAGE_MAX_UDP_SIZE\t512\n\nnamespace protocol\n{\n\nstatic inline void __append_uint8(std::string& s, uint8_t tmp)\n{\n\ts.append((const char *)&tmp, sizeof (uint8_t));\n}\n\nstatic inline void __append_uint16(std::string& s, uint16_t tmp)\n{\n\ttmp = htons(tmp);\n\ts.append((const char *)&tmp, sizeof (uint16_t));\n}\n\nstatic inline void __append_uint32(std::string& s, uint32_t tmp)\n{\n\ttmp = htonl(tmp);\n\ts.append((const char *)&tmp, sizeof (uint32_t));\n}\n\nstatic inline int __append_name(std::string& s, const char *p)\n{\n\tconst char *name;\n\tsize_t len;\n\n\twhile (*p)\n\t{\n\t\tname = p;\n\t\twhile (*p && *p != '.')\n\t\t\tp++;\n\n\t\tlen = p - name;\n\t\tif (len > DNS_LABELS_MAX || (len == 0 && *p && *(p + 1)))\n\t\t{\n\t\t\terrno = EINVAL;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (len > 0)\n\t\t{\n\t\t\t__append_uint8(s, len);\n\t\t\ts.append(name, len);\n\t\t}\n\n\t\tif (*p == '.')\n\t\t\tp++;\n\t}\n\n\tlen = 0;\n\t__append_uint8(s, len);\n\n\treturn 0;\n}\n\nstatic inline int __append_record_list(std::string& s, int *count,\n\t\t\t\t\t\t\t\t\t dns_record_cursor_t *cursor)\n{\n\tint cnt = 0;\n\tstruct dns_record *record;\n\tstd::string record_buf;\n\tstd::string rdata_buf;\n\tint ret;\n\n\twhile (dns_record_cursor_next(&record, cursor) == 0)\n\t{\n\t\trecord_buf.clear();\n\t\tret = __append_name(record_buf, record->name);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\t__append_uint16(record_buf, record->type);\n\t\t__append_uint16(record_buf, record->rclass);\n\t\t__append_uint32(record_buf, record->ttl);\n\n\t\tswitch (record->type)\n\t\t{\n\t\tdefault: // encode unknown types as raw record\n\t\tcase DNS_TYPE_A:\n\t\tcase DNS_TYPE_AAAA:\n\t\t\t__append_uint16(record_buf, record->rdlength);\n\t\t\trecord_buf.append((const char *)record->rdata, record->rdlength);\n\t\t\tbreak;\n\n\t\tcase DNS_TYPE_NS:\n\t\tcase DNS_TYPE_CNAME:\n\t\tcase DNS_TYPE_PTR:\n\t\t\trdata_buf.clear();\n\t\t\tret = __append_name(rdata_buf, (const char *)record->rdata);\n\t\t\tif (ret < 0)\n\t\t\t\treturn ret;\n\n\t\t\t__append_uint16(record_buf, rdata_buf.size());\n\t\t\trecord_buf.append(rdata_buf);\n\t\t\tbreak;\n\n\t\tcase DNS_TYPE_SOA:\n\t\t{\n\t\t\tauto *soa = (struct dns_record_soa *)record->rdata;\n\n\t\t\trdata_buf.clear();\n\t\t\tret = __append_name(rdata_buf, soa->mname);\n\t\t\tif (ret < 0)\n\t\t\t\treturn ret;\n\t\t\tret = __append_name(rdata_buf, soa->rname);\n\t\t\tif (ret < 0)\n\t\t\t\treturn ret;\n\n\t\t\t__append_uint32(rdata_buf, soa->serial);\n\t\t\t__append_uint32(rdata_buf, soa->refresh);\n\t\t\t__append_uint32(rdata_buf, soa->retry);\n\t\t\t__append_uint32(rdata_buf, soa->expire);\n\t\t\t__append_uint32(rdata_buf, soa->minimum);\n\n\t\t\t__append_uint16(record_buf, rdata_buf.size());\n\t\t\trecord_buf.append(rdata_buf);\n\t\t\tbreak;\n\t\t}\n\n\t\tcase DNS_TYPE_SRV:\n\t\t{\n\t\t\tauto *srv = (struct dns_record_srv *)record->rdata;\n\n\t\t\trdata_buf.clear();\n\t\t\t__append_uint16(rdata_buf, srv->priority);\n\t\t\t__append_uint16(rdata_buf, srv->weight);\n\t\t\t__append_uint16(rdata_buf, srv->port);\n\t\t\tret = __append_name(rdata_buf, srv->target);\n\t\t\tif (ret < 0)\n\t\t\t\treturn ret;\n\n\t\t\t__append_uint16(record_buf, rdata_buf.size());\n\t\t\trecord_buf.append(rdata_buf);\n\t\t\tbreak;\n\t\t}\n\n\t\tcase DNS_TYPE_MX:\n\t\t{\n\t\t\tauto *mx = (struct dns_record_mx *)record->rdata;\n\n\t\t\trdata_buf.clear();\n\t\t\t__append_uint16(rdata_buf, mx->preference);\n\t\t\tret = __append_name(rdata_buf, mx->exchange);\n\t\t\tif (ret < 0)\n\t\t\t\treturn ret;\n\n\t\t\t__append_uint16(record_buf, rdata_buf.size());\n\t\t\trecord_buf.append(rdata_buf);\n\t\t\tbreak;\n\t\t}\n\t\t}\n\n\t\tcnt++;\n\t\ts.append(record_buf);\n\t}\n\n\tif (count)\n\t\t*count = cnt;\n\n\treturn 0;\n}\n\nDnsMessage::DnsMessage(DnsMessage&& msg) :\n\tProtocolMessage(std::move(msg))\n{\n\tthis->parser = msg.parser;\n\tmsg.parser = NULL;\n\n\tthis->cur_size = msg.cur_size;\n\tmsg.cur_size = 0;\n}\n\nDnsMessage& DnsMessage::operator = (DnsMessage&& msg)\n{\n\tif (&msg != this)\n\t{\n\t\t*(ProtocolMessage *)this = std::move(msg);\n\n\t\tif (this->parser)\n\t\t{\n\t\t\tdns_parser_deinit(this->parser);\n\t\t\tdelete this->parser;\n\t\t}\n\n\t\tthis->parser = msg.parser;\n\t\tmsg.parser = NULL;\n\n\t\tthis->cur_size = msg.cur_size;\n\t\tmsg.cur_size = 0;\n\t}\n\treturn *this;\n}\n\ninline struct list_head *DnsMessage::get_section(int section)\n{\n\tswitch (section)\n\t{\n\tcase DNS_ANSWER_SECTION:\n\t\treturn &parser->answer_list;\n\tcase DNS_AUTHORITY_SECTION:\n\t\treturn &parser->authority_list;\n\tcase DNS_ADDITIONAL_SECTION:\n\t\treturn &parser->additional_list;\n\tdefault:\n\t\terrno = EINVAL;\n\t\treturn NULL;\n\t}\n}\n\nint DnsMessage::add_a_record(int section, const char *name,\n\t\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t const void *data)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_raw_record(name, DNS_TYPE_A, rclass, ttl, 4, data, list);\n}\n\nint DnsMessage::add_aaaa_record(int section, const char *name,\n\t\t\t\t\t\t\t\tuint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t\tconst void *data)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_raw_record(name, DNS_TYPE_AAAA, rclass, ttl, 16, data, list);\n}\n\nint DnsMessage::add_ns_record(int section, const char *name,\n\t\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t const char *data)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_str_record(name, DNS_TYPE_NS, rclass, ttl, data, list);\n}\n\nint DnsMessage::add_cname_record(int section, const char *name,\n\t\t\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t\t const char *data)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_str_record(name, DNS_TYPE_CNAME, rclass, ttl, data, list);\n}\n\nint DnsMessage::add_ptr_record(int section, const char *name,\n\t\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t const char *data)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_str_record(name, DNS_TYPE_PTR, rclass, ttl, data, list);\n}\n\nint DnsMessage::add_soa_record(int section, const char *name,\n\t\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t const char *mname, const char *rname,\n\t\t\t\t\t\t\t uint32_t serial, int32_t refresh,\n\t\t\t\t\t\t\t int32_t retry, int32_t expire, uint32_t minimum)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_soa_record(name, rclass, ttl, mname, rname, serial,\n\t\t\t\t\t\t\t refresh, retry, expire, minimum, list);\n}\n\nint DnsMessage::add_srv_record(int section, const char *name,\n\t\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t uint16_t priority, uint16_t weight,\n\t\t\t\t\t\t\t uint16_t port, const char *target)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_srv_record(name, rclass, ttl, priority, weight,\n\t\t\t\t\t\t\t port, target, list);\n}\n\nint DnsMessage::add_mx_record(int section, const char *name,\n\t\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t int16_t preference, const char *exchange)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_mx_record(name, rclass, ttl, preference, exchange, list);\n}\n\nint DnsMessage::add_raw_record(int section, const char *name, uint16_t type,\n\t\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t\t const void *data, uint16_t dlen)\n{\n\tstruct list_head *list = get_section(section);\n\n\tif (!list)\n\t\treturn -1;\n\n\treturn dns_add_raw_record(name, type, rclass, ttl, dlen, data, list);\n}\n\nint DnsMessage::encode_reply()\n{\n\tdns_record_cursor_t cursor;\n\tstruct dns_header h;\n\tstd::string tmpbuf;\n\tconst char *p;\n\tint ancount;\n\tint nscount;\n\tint arcount;\n\tint ret;\n\n\tmsgbuf.clear();\n\tmsgsize = 0;\n\n\t// TODO\n\t// this is an incomplete and inefficient way, compress not used,\n\t// pointers can only be used for occurances of a domain name where\n\t// the format is not class specific\n\tdns_answer_cursor_init(&cursor, this->parser);\n\tret = __append_record_list(tmpbuf, &ancount, &cursor);\n\tdns_record_cursor_deinit(&cursor);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tdns_authority_cursor_init(&cursor, this->parser);\n\tret = __append_record_list(tmpbuf, &nscount, &cursor);\n\tdns_record_cursor_deinit(&cursor);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tdns_additional_cursor_init(&cursor, this->parser);\n\tret = __append_record_list(tmpbuf, &arcount, &cursor);\n\tdns_record_cursor_deinit(&cursor);\n\tif (ret < 0)\n\t\treturn ret;\n\n\th = this->parser->header;\n\th.id = htons(h.id);\n\th.qdcount = htons(1);\n\th.ancount = htons(ancount);\n\th.nscount = htons(nscount);\n\th.arcount = htons(arcount);\n\n\tmsgbuf.append((const char *)&h, sizeof (struct dns_header));\n\tp = parser->question.qname ? parser->question.qname : \".\";\n\tret = __append_name(msgbuf, p);\n\tif (ret < 0)\n\t\treturn ret;\n\n\t__append_uint16(msgbuf, parser->question.qtype);\n\t__append_uint16(msgbuf, parser->question.qclass);\n\n\tmsgbuf.append(tmpbuf);\n\n\tif (msgbuf.size() >= (1 << 16))\n\t{\n\t\terrno = EOVERFLOW;\n\t\treturn -1;\n\t}\n\n\tmsgsize = htons(msgbuf.size());\n\n\treturn 0;\n}\n\nint DnsMessage::encode(struct iovec vectors[], int)\n{\n\tstruct iovec *p = vectors;\n\n\tif (this->encode_reply() < 0)\n\t\treturn -1;\n\n\t// TODO\n\t// if this is a request, it won't exceed the 512 bytes UDP limit\n\t// if this is a response and exceed 512 bytes, we need a TrunCation reply\n\n\tif (!this->is_single_packet())\n\t{\n\t\tp->iov_base = &this->msgsize;\n\t\tp->iov_len = sizeof (uint16_t);\n\t\tp++;\n\t}\n\n\tp->iov_base = (void *)this->msgbuf.data();\n\tp->iov_len = msgbuf.size();\n\treturn p - vectors + 1;\n}\n\nint DnsMessage::append(const void *buf, size_t *size)\n{\n\tint ret = dns_parser_append_message(buf, size, this->parser);\n\n\tif (ret >= 0)\n\t{\n\t\tthis->cur_size += *size;\n\t\tif (this->cur_size > this->size_limit)\n\t\t{\n\t\t\terrno = EMSGSIZE;\n\t\t\tret = -1;\n\t\t}\n\t}\n\telse if (ret == -2)\n\t{\n\t\terrno = EBADMSG;\n\t\tret = -1;\n\t}\n\n\treturn ret;\n}\n\nint DnsResponse::append(const void *buf, size_t *size)\n{\n\tint ret = this->DnsMessage::append(buf, size);\n\tconst char *qname = this->parser->question.qname;\n\n\tif (ret >= 1 && (this->request_id != this->get_id() ||\n\t\tstrcasecmp(this->request_name.c_str(), qname) != 0))\n\t{\n\t\tif (!this->is_single_packet())\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\tret = -1;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdns_parser_deinit(this->parser);\n\t\t\tdns_parser_init(this->parser);\n\t\t\tret = 0;\n\t\t}\n\t}\n\n\treturn ret;\n}\n\n}\n\n" }, { "path": "src/protocol/DnsMessage.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#ifndef _DNSMESSAGE_H_\n#define _DNSMESSAGE_H_\n\n/**\n * @file DnsMessage.h\n * @brief Dns Protocol Interface\n */\n\n#include \n#include \n#include \"ProtocolMessage.h\"\n#include \"dns_parser.h\"\n\nnamespace protocol\n{\n\nclass DnsMessage : public ProtocolMessage\n{\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\npublic:\n\tint get_id() const\n\t{\n\t\treturn parser->header.id;\n\t}\n\tint get_qr() const\n\t{\n\t\treturn parser->header.qr;\n\t}\n\tint get_opcode() const\n\t{\n\t\treturn parser->header.opcode;\n\t}\n\tint get_aa() const\n\t{\n\t\treturn parser->header.aa;\n\t}\n\tint get_tc() const\n\t{\n\t\treturn parser->header.tc;\n\t}\n\tint get_rd() const\n\t{\n\t\treturn parser->header.rd;\n\t}\n\tint get_ra() const\n\t{\n\t\treturn parser->header.ra;\n\t}\n\tint get_rcode() const\n\t{\n\t\treturn parser->header.rcode;\n\t}\n\tint get_qdcount() const\n\t{\n\t\treturn parser->header.qdcount;\n\t}\n\tint get_ancount() const\n\t{\n\t\treturn parser->header.ancount;\n\t}\n\tint get_nscount() const\n\t{\n\t\treturn parser->header.nscount;\n\t}\n\tint get_arcount() const\n\t{\n\t\treturn parser->header.arcount;\n\t}\n\n\tvoid set_id(int id)\n\t{\n\t\tparser->header.id = id;\n\t}\n\tvoid set_qr(int qr)\n\t{\n\t\tparser->header.qr = qr;\n\t}\n\tvoid set_opcode(int opcode)\n\t{\n\t\tparser->header.opcode = opcode;\n\t}\n\tvoid set_aa(int aa)\n\t{\n\t\tparser->header.aa = aa;\n\t}\n\tvoid set_tc(int tc)\n\t{\n\t\tparser->header.tc = tc;\n\t}\n\tvoid set_rd(int rd)\n\t{\n\t\tparser->header.rd = rd;\n\t}\n\tvoid set_ra(int ra)\n\t{\n\t\tparser->header.ra = ra;\n\t}\n\tvoid set_rcode(int rcode)\n\t{\n\t\tparser->header.rcode = rcode;\n\t}\n\n\tint get_question_type() const\n\t{\n\t\treturn parser->question.qtype;\n\t}\n\tint get_question_class() const\n\t{\n\t\treturn parser->question.qclass;\n\t}\n\tstd::string get_question_name() const\n\t{\n\t\tconst char *name = parser->question.qname;\n\t\tif (name == NULL)\n\t\t\treturn \"\";\n\t\treturn name;\n\t}\n\tvoid set_question_type(int qtype)\n\t{\n\t\tparser->question.qtype = qtype;\n\t}\n\tvoid set_question_class(int qclass)\n\t{\n\t\tparser->question.qclass = qclass;\n\t}\n\tvoid set_question_name(const std::string& name)\n\t{\n\t\tchar *pname = parser->question.qname;\n\t\tif (pname != NULL)\n\t\t\tfree(pname);\n\t\tparser->question.qname = strdup(name.c_str());\n\t}\n\n\tint add_a_record(int section, const char *name,\n\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t const void *data);\n\n\tint add_aaaa_record(int section, const char *name,\n\t\t\t\t\t\tuint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\tconst void *data);\n\n\tint add_ns_record(int section, const char *name,\n\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t const char *data);\n\n\tint add_cname_record(int section, const char *name,\n\t\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t\t const char *data);\n\n\tint add_ptr_record(int section, const char *name,\n\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t const char *data);\n\n\tint add_soa_record(int section, const char *name,\n\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t const char *mname, const char *rname,\n\t\t\t\t\t uint32_t serial, int32_t refresh,\n\t\t\t\t\t int32_t retry, int32_t expire, uint32_t minimum);\n\n\tint add_srv_record(int section, const char *name,\n\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t uint16_t priority, uint16_t weight,\n\t\t\t\t\t uint16_t port, const char *target);\n\n\tint add_mx_record(int section, const char *name,\n\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t int16_t preference, const char *exchange);\n\n\tint add_raw_record(int section, const char *name, uint16_t type,\n\t\t\t\t\t uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t const void *data, uint16_t dlen);\n\n\t// Inner use only\n\tbool is_single_packet() const\n\t{\n\t\treturn parser->single_packet;\n\t}\n\tvoid set_single_packet(bool single)\n\t{\n\t\tparser->single_packet = single;\n\t}\n\npublic:\n\tDnsMessage() :\n\t\tparser(new dns_parser_t)\n\t{\n\t\tdns_parser_init(parser);\n\t\tthis->cur_size = 0;\n\t}\n\n\tvirtual ~DnsMessage()\n\t{\n\t\tif (this->parser)\n\t\t{\n\t\t\tdns_parser_deinit(parser);\n\t\t\tdelete this->parser;\n\t\t}\n\t}\n\n\tDnsMessage(DnsMessage&& msg);\n\tDnsMessage& operator = (DnsMessage&& msg);\n\nprotected:\n\tdns_parser_t *parser;\n\tstd::string msgbuf;\n\tsize_t cur_size;\n\nprivate:\n\tint encode_reply();\n\tint encode_truncation_reply();\n\tstruct list_head *get_section(int section);\n\n\t// size of msgbuf, but in network byte order\n\tuint16_t msgsize;\n};\n\nclass DnsRequest : public DnsMessage\n{\npublic:\n\tDnsRequest() = default;\n\tDnsRequest(DnsRequest&& req) = default;\n\tDnsRequest& operator = (DnsRequest&& req) = default;\n\n\tvoid set_question(const char *host, uint16_t qtype, uint16_t qclass)\n\t{\n\t\tdns_parser_set_question(host, qtype, qclass, this->parser);\n\t}\n};\n\nclass DnsResponse : public DnsMessage\n{\npublic:\n\tDnsResponse()\n\t{\n\t\tthis->request_id = 0;\n\t}\n\n\tDnsResponse(DnsResponse&& req) = default;\n\tDnsResponse& operator = (DnsResponse&& req) = default;\n\n\tconst dns_parser_t *get_parser() const\n\t{\n\t\treturn this->parser;\n\t}\n\n\tvoid set_request_id(uint16_t id)\n\t{\n\t\tthis->request_id = id;\n\t}\n\n\tvoid set_request_name(const std::string& name)\n\t{\n\t\tstd::string& req_name = this->request_name;\n\n\t\treq_name = name;\n\t\twhile (req_name.size() > 1 && req_name.back() == '.')\n\t\t\treq_name.pop_back();\n\t}\n\nprotected:\n\tvirtual int append(const void *buf, size_t *size);\n\nprivate:\n\tuint16_t request_id;\n\tstd::string request_name;\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/DnsUtil.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"dns_types.h\"\n#include \"DnsUtil.h\"\n\nnamespace protocol\n{\n\nint DnsUtil::getaddrinfo(const DnsResponse *resp,\n\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t struct addrinfo **addrinfo)\n{\n\tint ancount = resp->get_ancount();\n\tint rcode = resp->get_rcode();\n\tint status = 0;\n\tstruct addrinfo *res = NULL;\n\tstruct addrinfo **pres = &res;\n\tstruct dns_record *record;\n\tstruct addrinfo *ai;\n\tstd::string qname;\n\tconst char *cname;\n\tint family;\n\tint addrlen;\n\n\tswitch (rcode)\n\t{\n\tcase DNS_RCODE_NAME_ERROR:\n\t\tstatus = EAI_NONAME;\n\t\tbreak;\n\tcase DNS_RCODE_SERVER_FAILURE:\n\t\tstatus = EAI_AGAIN;\n\t\tbreak;\n\tcase DNS_RCODE_FORMAT_ERROR:\n\tcase DNS_RCODE_NOT_IMPLEMENTED:\n\tcase DNS_RCODE_REFUSED:\n\t\tstatus = EAI_FAIL;\n\t\tbreak;\n\t}\n\n\tqname = resp->get_question_name();\n\tcname = qname.c_str();\n\n\tDnsResultCursor cursor(resp);\n\tcursor.reset_answer_cursor();\n\t/* Forbid loop in cname chain */\n\twhile (cursor.find_cname(cname, &cname) && ancount-- > 0) { }\n\n\tif (rcode == DNS_RCODE_NO_ERROR && ancount <= 0)\n\t\tstatus = EAI_NODATA;\n\tif (status != 0)\n\t\treturn status;\n\n\tcursor.reset_answer_cursor();\n\twhile (cursor.next(&record))\n\t{\n\t\tif (!(record->rclass == DNS_CLASS_IN &&\n\t\t\t(record->type == DNS_TYPE_A || record->type == DNS_TYPE_AAAA) &&\n\t\t\tstrcasecmp(record->name, cname) == 0))\n\t\t\tcontinue;\n\n\t\tif (record->type == DNS_TYPE_A)\n\t\t{\n\t\t\tfamily = AF_INET;\n\t\t\taddrlen = sizeof (struct sockaddr_in);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tfamily = AF_INET6;\n\t\t\taddrlen = sizeof (struct sockaddr_in6);\n\t\t}\n\n\t\tai = (struct addrinfo *)calloc(sizeof (struct addrinfo) + addrlen, 1);\n\t\tif (ai == NULL)\n\t\t{\n\t\t\tif (res)\n\t\t\t\tDnsUtil::freeaddrinfo(res);\n\t\t\treturn EAI_MEMORY;\n\t\t}\n\n\t\tai->ai_family = family;\n\t\tai->ai_addrlen = addrlen;\n\t\tai->ai_addr = (struct sockaddr *)(ai + 1);\n\t\tai->ai_addr->sa_family = family;\n\n\t\tif (family == AF_INET)\n\t\t{\n\t\t\tstruct sockaddr_in *in = (struct sockaddr_in *)(ai->ai_addr);\n\t\t\tin->sin_port = htons(port);\n\t\t\tmemcpy(&in->sin_addr, record->rdata, sizeof (struct in_addr));\n\t\t}\n\t\telse\n\t\t{\n\t\t\tstruct sockaddr_in6 *in = (struct sockaddr_in6 *)(ai->ai_addr);\n\t\t\tin->sin6_port = htons(port);\n\t\t\tmemcpy(&in->sin6_addr, record->rdata, sizeof (struct in6_addr));\n\t\t}\n\n\t\t*pres = ai;\n\t\tpres = &ai->ai_next;\n\t}\n\n\tif (res == NULL)\n\t\treturn EAI_NODATA;\n\n\tif (cname)\n\t\tres->ai_canonname = strdup(cname);\n\n\t*addrinfo = res;\n\n\treturn 0;\n}\n\nvoid DnsUtil::freeaddrinfo(struct addrinfo *ai)\n{\n\tstruct addrinfo *p;\n\n\twhile (ai != NULL)\n\t{\n\t\tp = ai;\n\t\tai = ai->ai_next;\n\t\tfree(p->ai_canonname);\n\t\tfree(p);\n\t}\n}\n\n}\n\n" }, { "path": "src/protocol/DnsUtil.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#ifndef _DNSUTIL_H_\n#define _DNSUTIL_H_\n\n#include \n#include \"DnsMessage.h\"\n\n/**\n * @file DnsUtil.h\n * @brief Dns toolbox\n */\n\nnamespace protocol\n{\n\nclass DnsUtil\n{\npublic:\n\tstatic int getaddrinfo(const DnsResponse *resp,\n\t\t\t\t\t\t unsigned short port,\n\t\t\t\t\t\t struct addrinfo **res);\n\tstatic void freeaddrinfo(struct addrinfo *ai);\n};\n\nclass DnsResultCursor\n{\npublic:\n\tDnsResultCursor(const DnsResponse *resp) :\n\t\tparser(resp->get_parser())\n\t{\n\t\tdns_answer_cursor_init(&cursor, parser);\n\t\trecord = NULL;\n\t}\n\n\tDnsResultCursor(DnsResultCursor&& move) = delete;\n\tDnsResultCursor& operator=(DnsResultCursor&& move) = delete;\n\n\tvirtual ~DnsResultCursor() { }\n\n\tvoid reset_answer_cursor()\n\t{\n\t\tdns_answer_cursor_init(&cursor, parser);\n\t}\n\n\tvoid reset_authority_cursor()\n\t{\n\t\tdns_authority_cursor_init(&cursor, parser);\n\t}\n\n\tvoid reset_additional_cursor()\n\t{\n\t\tdns_additional_cursor_init(&cursor, parser);\n\t}\n\n\tbool next(struct dns_record **next_record)\n\t{\n\t\tint ret = dns_record_cursor_next(&record, &cursor);\n\t\tif (ret != 0)\n\t\t\trecord = NULL;\n\t\telse\n\t\t\t*next_record = record;\n\n\t\treturn ret == 0;\n\t}\n\n\tbool find_cname(const char *name, const char **cname)\n\t{\n\t\treturn dns_record_cursor_find_cname(name, cname, &cursor) == 0;\n\t}\n\nprivate:\n\tconst dns_parser_t *parser;\n\tdns_record_cursor_t cursor;\n\tstruct dns_record *record;\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/HttpMessage.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"HttpMessage.h\"\n\nnamespace protocol\n{\n\nstruct HttpMessageBlock\n{\n\tstruct list_head list;\n\tconst void *ptr;\n\tsize_t size;\n};\n\nbool HttpMessage::append_output_body(const void *buf, size_t size)\n{\n\tsize_t n = sizeof (struct HttpMessageBlock) + size;\n\tstruct HttpMessageBlock *block = (struct HttpMessageBlock *)malloc(n);\n\n\tif (block)\n\t{\n\t\tmemcpy(block + 1, buf, size);\n\t\tblock->ptr = block + 1;\n\t\tblock->size = size;\n\t\tlist_add_tail(&block->list, &this->output_body);\n\t\tthis->output_body_size += size;\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nbool HttpMessage::append_output_body_nocopy(const void *buf, size_t size)\n{\n\tsize_t n = sizeof (struct HttpMessageBlock);\n\tstruct HttpMessageBlock *block = (struct HttpMessageBlock *)malloc(n);\n\n\tif (block)\n\t{\n\t\tblock->ptr = buf;\n\t\tblock->size = size;\n\t\tlist_add_tail(&block->list, &this->output_body);\n\t\tthis->output_body_size += size;\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nsize_t HttpMessage::get_output_body_blocks(const void *buf[], size_t size[],\n\t\t\t\t\t\t\t\t\t\t size_t max) const\n{\n\tstruct HttpMessageBlock *block;\n\tstruct list_head *pos;\n\tsize_t n = 0;\n\n\tlist_for_each(pos, &this->output_body)\n\t{\n\t\tif (n == max)\n\t\t\tbreak;\n\n\t\tblock = list_entry(pos, struct HttpMessageBlock, list);\n\t\tbuf[n] = block->ptr;\n\t\tsize[n] = block->size;\n\t\tn++;\n\t}\n\n\treturn n;\n}\n\nbool HttpMessage::get_output_body_merged(void *buf, size_t *size) const\n{\n\tstruct HttpMessageBlock *block;\n\tstruct list_head *pos;\n\n\tif (*size < this->output_body_size)\n\t{\n\t\terrno = ENOSPC;\n\t\treturn false;\n\t}\n\n\tlist_for_each(pos, &this->output_body)\n\t{\n\t\tblock = list_entry(pos, struct HttpMessageBlock, list);\n\t\tmemcpy(buf, block->ptr, block->size);\n\t\tbuf = (char *)buf + block->size;\n\t}\n\n\t*size = this->output_body_size;\n\treturn true;\n}\n\nvoid HttpMessage::clear_output_body()\n{\n\tstruct HttpMessageBlock *block;\n\tstruct list_head *pos, *tmp;\n\n\tlist_for_each_safe(pos, tmp, &this->output_body)\n\t{\n\t\tblock = list_entry(pos, struct HttpMessageBlock, list);\n\t\tlist_del(pos);\n\t\tfree(block);\n\t}\n\n\tthis->output_body_size = 0;\n}\n\nstruct list_head *HttpMessage::combine_from(struct list_head *pos, size_t size)\n{\n\tsize_t n = sizeof (struct HttpMessageBlock) + size;\n\tstruct HttpMessageBlock *block = (struct HttpMessageBlock *)malloc(n);\n\tstruct HttpMessageBlock *entry;\n\tchar *ptr;\n\n\tif (block)\n\t{\n\t\tblock->ptr = block + 1;\n\t\tblock->size = size;\n\t\tptr = (char *)block->ptr;\n\n\t\tdo\n\t\t{\n\t\t\tentry = list_entry(pos, struct HttpMessageBlock, list);\n\t\t\tpos = pos->next;\n\t\t\tlist_del(&entry->list);\n\t\t\tmemcpy(ptr, entry->ptr, entry->size);\n\t\t\tptr += entry->size;\n\t\t\tfree(entry);\n\t\t} while (pos != &this->output_body);\n\n\t\tlist_add_tail(&block->list, &this->output_body);\n\t\treturn &block->list;\n\t}\n\n\treturn NULL;\n}\n\nint HttpMessage::encode(struct iovec vectors[], int max)\n{\n\tconst char *start_line[3];\n\thttp_header_cursor_t cursor;\n\tstruct HttpMessageHeader header;\n\tstruct HttpMessageBlock *block;\n\tstruct list_head *pos;\n\tsize_t size;\n\tint i;\n\n\tstart_line[0] = http_parser_get_method(this->parser);\n\tif (start_line[0])\n\t{\n\t\tstart_line[1] = http_parser_get_uri(this->parser);\n\t\tstart_line[2] = http_parser_get_version(this->parser);\n\t}\n\telse\n\t{\n\t\tstart_line[0] = http_parser_get_version(this->parser);\n\t\tstart_line[1] = http_parser_get_code(this->parser);\n\t\tstart_line[2] = http_parser_get_phrase(this->parser);\n\t}\n\n\tif (!start_line[0] || !start_line[1] || !start_line[2])\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tvectors[0].iov_base = (void *)start_line[0];\n\tvectors[0].iov_len = strlen(start_line[0]);\n\tvectors[1].iov_base = (void *)\" \";\n\tvectors[1].iov_len = 1;\n\n\tvectors[2].iov_base = (void *)start_line[1];\n\tvectors[2].iov_len = strlen(start_line[1]);\n\tvectors[3].iov_base = (void *)\" \";\n\tvectors[3].iov_len = 1;\n\n\tvectors[4].iov_base = (void *)start_line[2];\n\tvectors[4].iov_len = strlen(start_line[2]);\n\tvectors[5].iov_base = (void *)\"\\r\\n\";\n\tvectors[5].iov_len = 2;\n\n\ti = 6;\n\thttp_header_cursor_init(&cursor, this->parser);\n\twhile (http_header_cursor_next(&header.name, &header.name_len,\n\t\t\t\t\t\t\t\t &header.value, &header.value_len,\n\t\t\t\t\t\t\t\t &cursor) == 0)\n\t{\n\t\tif (i == max)\n\t\t\tbreak;\n\n\t\tvectors[i].iov_base = (void *)header.name;\n\t\tvectors[i].iov_len = header.name_len + 2 + header.value_len + 2;\n\t\ti++;\n\t}\n\n\thttp_header_cursor_deinit(&cursor);\n\tif (i + 1 >= max)\n\t{\n\t\terrno = EOVERFLOW;\n\t\treturn -1;\n\t}\n\n\tvectors[i].iov_base = (void *)\"\\r\\n\";\n\tvectors[i].iov_len = 2;\n\ti++;\n\n\tsize = this->output_body_size;\n\tlist_for_each(pos, &this->output_body)\n\t{\n\t\tif (i + 1 == max && pos != this->output_body.prev)\n\t\t{\n\t\t\tpos = this->combine_from(pos, size);\n\t\t\tif (!pos)\n\t\t\t\treturn -1;\n\t\t}\n\n\t\tblock = list_entry(pos, struct HttpMessageBlock, list);\n\t\tvectors[i].iov_base = (void *)block->ptr;\n\t\tvectors[i].iov_len = block->size;\n\t\tsize -= block->size;\n\t\ti++;\n\t}\n\n\treturn i;\n}\n\ninline int HttpMessage::append(const void *buf, size_t *size)\n{\n\tint ret = http_parser_append_message(buf, size, this->parser);\n\n\tif (ret >= 0)\n\t{\n\t\tthis->cur_size += *size;\n\t\tif (this->cur_size > this->size_limit)\n\t\t{\n\t\t\terrno = EMSGSIZE;\n\t\t\tret = -1;\n\t\t}\n\t}\n\telse if (ret == -2)\n\t{\n\t\terrno = EBADMSG;\n\t\tret = -1;\n\t}\n\n\treturn ret;\n}\n\nHttpMessage::HttpMessage(HttpMessage&& msg) :\n\tProtocolMessage(std::move(msg))\n{\n\tthis->parser = msg.parser;\n\tmsg.parser = NULL;\n\n\tINIT_LIST_HEAD(&this->output_body);\n\tlist_splice_init(&msg.output_body, &this->output_body);\n\tthis->output_body_size = msg.output_body_size;\n\tmsg.output_body_size = 0;\n\n\tthis->cur_size = msg.cur_size;\n\tmsg.cur_size = 0;\n}\n\nHttpMessage& HttpMessage::operator = (HttpMessage&& msg)\n{\n\tif (&msg != this)\n\t{\n\t\t*(ProtocolMessage *)this = std::move(msg);\n\n\t\tif (this->parser)\n\t\t{\n\t\t\thttp_parser_deinit(this->parser);\n\t\t\tdelete this->parser;\n\t\t}\n\n\t\tthis->parser = msg.parser;\n\t\tmsg.parser = NULL;\n\n\t\tthis->clear_output_body();\n\t\tlist_splice_init(&msg.output_body, &this->output_body);\n\t\tthis->output_body_size = msg.output_body_size;\n\t\tmsg.output_body_size = 0;\n\n\t\tthis->cur_size = msg.cur_size;\n\t\tmsg.cur_size = 0;\n\t}\n\n\treturn *this;\n}\n\n#define HTTP_100_STATUS_LINE\t\"HTTP/1.1 100 Continue\"\n#define HTTP_400_STATUS_LINE\t\"HTTP/1.1 400 Bad Request\"\n#define HTTP_413_STATUS_LINE\t\"HTTP/1.1 413 Request Entity Too Large\"\n#define HTTP_417_STATUS_LINE\t\"HTTP/1.1 417 Expectation Failed\"\n#define CONTENT_LENGTH_ZERO\t\t\"Content-Length: 0\"\n#define CONNECTION_CLOSE\t\t\"Connection: close\"\n#define CRLF\t\t\t\t\t\"\\r\\n\"\n\n#define HTTP_100_RESP\t\t\tHTTP_100_STATUS_LINE CRLF \\\n\t\t\t\t\t\t\t\tCRLF\n#define HTTP_400_RESP\t\t\tHTTP_400_STATUS_LINE CRLF \\\n\t\t\t\t\t\t\t\tCONTENT_LENGTH_ZERO CRLF \\\n\t\t\t\t\t\t\t\tCONNECTION_CLOSE CRLF \\\n\t\t\t\t\t\t\t\tCRLF\n#define HTTP_413_RESP\t\t\tHTTP_413_STATUS_LINE CRLF \\\n\t\t\t\t\t\t\t\tCONTENT_LENGTH_ZERO CRLF \\\n\t\t\t\t\t\t\t\tCONNECTION_CLOSE CRLF \\\n\t\t\t\t\t\t\t\tCRLF\n#define HTTP_417_RESP\t\t\tHTTP_417_STATUS_LINE CRLF \\\n\t\t\t\t\t\t\t\tCONTENT_LENGTH_ZERO CRLF \\\n\t\t\t\t\t\t\t\tCONNECTION_CLOSE CRLF \\\n\t\t\t\t\t\t\t\tCRLF\n\nint HttpRequest::handle_expect_continue()\n{\n\tsize_t trans_len = this->parser->transfer_length;\n\tint ret;\n\n\tif (trans_len != (size_t)-1)\n\t{\n\t\tif (this->parser->header_offset + trans_len > this->size_limit)\n\t\t{\n\t\t\tthis->feedback(HTTP_417_RESP, strlen(HTTP_417_RESP));\n\t\t\terrno = EMSGSIZE;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\tret = this->feedback(HTTP_100_RESP, strlen(HTTP_100_RESP));\n\tif (ret != strlen(HTTP_100_RESP))\n\t{\n\t\tif (ret >= 0)\n\t\t\terrno = ENOBUFS;\n\t\treturn -1;\n\t}\n\n\treturn 0;\n}\n\nint HttpRequest::append(const void *buf, size_t *size)\n{\n\tint ret = HttpMessage::append(buf, size);\n\n\tif (ret == 0)\n\t{\n\t\tif (this->parser->expect_continue &&\n\t\t\thttp_parser_header_complete(this->parser))\n\t\t{\n\t\t\tthis->parser->expect_continue = 0;\n\t\t\tret = this->handle_expect_continue();\n\t\t}\n\t}\n\telse if (ret < 0)\n\t{\n\t\tif (errno == EBADMSG)\n\t\t\tthis->feedback(HTTP_400_RESP, strlen(HTTP_400_RESP));\n\t\telse if (errno == EMSGSIZE)\n\t\t\tthis->feedback(HTTP_413_RESP, strlen(HTTP_413_RESP));\n\t}\n\n\treturn ret;\n}\n\nint HttpResponse::append(const void *buf, size_t *size)\n{\n\tint ret = HttpMessage::append(buf, size);\n\n\tif (ret > 0)\n\t{\n\t\tif (strcmp(http_parser_get_code(this->parser), \"100\") == 0)\n\t\t{\n\t\t\thttp_parser_deinit(this->parser);\n\t\t\thttp_parser_init(1, this->parser);\n\t\t\tret = 0;\n\t\t}\n\t}\n\n\treturn ret;\n}\n\nbool HttpMessageChunk::get_chunk_data(const void **data, size_t *size) const\n{\n\tif (this->chunk_data && this->nreceived == this->chunk_size + 2)\n\t{\n\t\t*data = this->chunk_data;\n\t\t*size = this->chunk_size;\n\t\treturn true;\n\t}\n\telse\n\t\treturn false;\n}\n\nbool HttpMessageChunk::move_chunk_data(void **data, size_t *size)\n{\n\tif (this->chunk_data && this->nreceived == this->chunk_size + 2)\n\t{\n\t\t*data = this->chunk_data;\n\t\t*size = this->chunk_size;\n\t\tthis->chunk_data = NULL;\n\t\tthis->nreceived = 0;\n\t\treturn true;\n\t}\n\telse\n\t\treturn false;\n}\n\nbool HttpMessageChunk::set_chunk_data(const void *data, size_t size)\n{\n\tchar *p = (char *)malloc(size + 3);\n\n\tif (p)\n\t{\n\t\tmemcpy(p, data, size);\n\t\tp[size] = '\\r';\n\t\tp[size + 1] = '\\n';\n\t\tp[size + 2] = '\\0';\n\n\t\tfree(this->chunk_data);\n\t\tthis->chunk_data = p;\n\t\tthis->chunk_size = size;\n\t\tthis->nreceived = size + 2;\n\t\treturn true;\n\t}\n\telse\n\t\treturn false;\n}\n\nint HttpMessageChunk::encode(struct iovec vectors[], int max)\n{\n\tint len = sprintf(this->chunk_line, \"%zx\\r\\n\", this->chunk_size);\n\n\tvectors[0].iov_base = this->chunk_line;\n\tvectors[0].iov_len = len;\n\tvectors[1].iov_base = this->chunk_data;\n\tvectors[1].iov_len = this->chunk_size + 2;\n\n\treturn 2;\n}\n\n#define MIN(x, y)\t((x) <= (y) ? (x) : (y))\n\nint HttpMessageChunk::append_chunk_line(const void *buf, size_t size)\n{\n\tchar *end;\n\tsize_t i;\n\n\tsize = MIN(size, sizeof this->chunk_line - this->nreceived);\n\tmemcpy(this->chunk_line + this->nreceived, buf, size);\n\tfor (i = 0; i + 1 < this->nreceived + size; i++)\n\t{\n\t\tif (this->chunk_line[i] == '\\r')\n\t\t{\n\t\t\tif (this->chunk_line[i + 1] != '\\n')\n\t\t\t{\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tthis->chunk_line[i] = '\\0';\n\t\t\tthis->chunk_size = strtoul(this->chunk_line, &end, 16);\n\t\t\tif (end == this->chunk_line)\n\t\t\t{\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tif (this->chunk_size > 64 * 1024 * 1024 ||\n\t\t\t\tthis->chunk_size > this->size_limit)\n\t\t\t{\n\t\t\t\terrno = EMSGSIZE;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tthis->chunk_data = malloc(this->chunk_size + 3);\n\t\t\tif (!this->chunk_data)\n\t\t\t\treturn -1;\n\n\t\t\tthis->nreceived = i + 2;\n\t\t\treturn 1;\n\t\t}\n\t}\n\n\tif (i == sizeof this->chunk_line - 1)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tthis->nreceived += size;\n\treturn 0;\n}\n\nint HttpMessageChunk::append(const void *buf, size_t *size)\n{\n\tsize_t nleft;\n\tsize_t n;\n\tint ret;\n\n\tif (!this->chunk_data)\n\t{\n\t\tn = this->nreceived;\n\t\tret = this->append_chunk_line(buf, *size);\n\t\tif (ret <= 0)\n\t\t\treturn ret;\n\n\t\tn = this->nreceived - n;\n\t\tthis->nreceived = 0;\n\t}\n\telse\n\t\tn = 0;\n\n\tif (this->chunk_size != 0)\n\t{\n\t\tnleft = this->chunk_size + 2 - this->nreceived;\n\t\tif (*size - n > nleft)\n\t\t\t*size = n + nleft;\n\n\t\tbuf = (const char *)buf + n;\n\t\tn = *size - n;\n\t\tmemcpy((char *)this->chunk_data + this->nreceived, buf, n);\n\t\tthis->nreceived += n;\n\t\tif (this->nreceived == this->chunk_size + 2)\n\t\t{\n\t\t\t((char *)this->chunk_data)[this->nreceived] = '\\0';\n\t\t\treturn 1;\n\t\t}\n\t}\n\telse\n\t{\n\t\twhile (n < *size)\n\t\t{\n\t\t\tchar c = ((const char *)buf)[n];\n\n\t\t\tif (this->nreceived == 0)\n\t\t\t{\n\t\t\t\tif (c == '\\r')\n\t\t\t\t\tthis->nreceived = 1;\n\t\t\t\telse\n\t\t\t\t\tthis->nreceived = (size_t)-2;\n\t\t\t}\n\t\t\telse if (this->nreceived == 1)\n\t\t\t{\n\t\t\t\tif (c == '\\n')\n\t\t\t\t{\n\t\t\t\t\t*size = n + 1;\n\t\t\t\t\tthis->nreceived = 2;\n\t\t\t\t\t((char *)this->chunk_data)[0] = '\\r';\n\t\t\t\t\t((char *)this->chunk_data)[1] = '\\n';\n\t\t\t\t\t((char *)this->chunk_data)[2] = '\\0';\n\t\t\t\t\treturn 1;\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t\telse if (this->nreceived == (size_t)-2)\n\t\t\t{\n\t\t\t\tif (c == '\\r')\n\t\t\t\t\tthis->nreceived = (size_t)-1;\n\t\t\t}\n\t\t\telse /* if (this->nreceived == (size_t)-1) */\n\t\t\t{\n\t\t\t\tif (c == '\\n')\n\t\t\t\t\tthis->nreceived = 0;\n\t\t\t\telse\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tn++;\n\t\t}\n\n\t\tif (n < *size)\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nHttpMessageChunk::HttpMessageChunk(HttpMessageChunk&& msg) :\n\tProtocolMessage(std::move(msg))\n{\n\tmemcpy(this->chunk_line, msg.chunk_line, sizeof this->chunk_line);\n\tthis->chunk_data = msg.chunk_data;\n\tmsg.chunk_data = NULL;\n\tthis->chunk_size = msg.chunk_size;\n\tthis->nreceived = msg.nreceived;\n\tmsg.nreceived = 0;\n}\n\nHttpMessageChunk& HttpMessageChunk::operator = (HttpMessageChunk&& msg)\n{\n\tif (&msg != this)\n\t{\n\t\t*(ProtocolMessage *)this = std::move(msg);\n\n\t\tmemcpy(this->chunk_line, msg.chunk_line, sizeof this->chunk_line);\n\t\tfree(this->chunk_data);\n\t\tthis->chunk_data = msg.chunk_data;\n\t\tmsg.chunk_data = NULL;\n\t\tthis->chunk_size = msg.chunk_size;\n\t\tthis->nreceived = msg.nreceived;\n\t\tmsg.nreceived = 0;\n\t}\n\n\treturn *this;\n}\n\n}\n\n" }, { "path": "src/protocol/HttpMessage.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _HTTPMESSAGE_H_\n#define _HTTPMESSAGE_H_\n\n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"ProtocolMessage.h\"\n#include \"http_parser.h\"\n\n/**\n * @file HttpMessage.h\n * @brief Http Protocol Interface\n */\n\nnamespace protocol\n{\n\nstruct HttpMessageHeader\n{\n\tconst void *name;\n\tsize_t name_len;\n\tconst void *value;\n\tsize_t value_len;\n};\n\nclass HttpMessage : public ProtocolMessage\n{\npublic:\n\tconst char *get_http_version() const\n\t{\n\t\treturn http_parser_get_version(this->parser);\n\t}\n\n\tbool set_http_version(const char *version)\n\t{\n\t\treturn http_parser_set_version(version, this->parser) == 0;\n\t}\n\n\tbool is_chunked() const\n\t{\n\t\treturn http_parser_chunked(this->parser);\n\t}\n\n\tbool is_keep_alive() const\n\t{\n\t\treturn http_parser_keep_alive(this->parser);\n\t}\n\n\tbool add_header(const struct HttpMessageHeader *header)\n\t{\n\t\treturn http_parser_add_header(header->name, header->name_len,\n\t\t\t\t\t\t\t\t\t header->value, header->value_len,\n\t\t\t\t\t\t\t\t\t this->parser) == 0;\n\t}\n\n\tbool add_header_pair(const char *name, const char *value)\n\t{\n\t\treturn http_parser_add_header(name, strlen(name),\n\t\t\t\t\t\t\t\t\t value, strlen(value),\n\t\t\t\t\t\t\t\t\t this->parser) == 0;\n\t}\n\n\tbool set_header(const struct HttpMessageHeader *header)\n\t{\n\t\treturn http_parser_set_header(header->name, header->name_len,\n\t\t\t\t\t\t\t\t\t header->value, header->value_len,\n\t\t\t\t\t\t\t\t\t this->parser) == 0;\n\t}\n\n\tbool set_header_pair(const char *name, const char *value)\n\t{\n\t\treturn http_parser_set_header(name, strlen(name),\n\t\t\t\t\t\t\t\t\t value, strlen(value),\n\t\t\t\t\t\t\t\t\t this->parser) == 0;\n\t}\n\n\tbool get_parsed_body(const void **body, size_t *size) const\n\t{\n\t\treturn http_parser_get_body(body, size, this->parser) == 0;\n\t}\n\n\t/* Output body is for sending. Want to transfer a message received, maybe:\n\t * msg->get_parsed_body(&body, &size);\n\t * msg->append_output_body_nocopy(body, size); */\n\tbool append_output_body(const void *buf, size_t size);\n\n\tbool append_output_body(const char *buf)\n\t{\n\t\treturn this->append_output_body(buf, strlen(buf));\n\t}\n\n\tbool append_output_body_nocopy(const void *buf, size_t size);\n\n\tbool append_output_body_nocopy(const char *buf)\n\t{\n\t\treturn this->append_output_body_nocopy(buf, strlen(buf));\n\t}\n\n\tsize_t get_output_body_size() const\n\t{\n\t\treturn this->output_body_size;\n\t}\n\n\tsize_t get_output_body_blocks(const void *buf[], size_t size[],\n\t\t\t\t\t\t\t\t size_t max) const;\n\n\tbool get_output_body_merged(void *buf, size_t *size) const;\n\n\tvoid clear_output_body();\n\n\t/* std::string interfaces */\npublic:\n\tbool get_http_version(std::string& version) const\n\t{\n\t\tconst char *str = this->get_http_version();\n\n\t\tif (str)\n\t\t{\n\t\t\tversion.assign(str);\n\t\t\treturn true;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tbool set_http_version(const std::string& version)\n\t{\n\t\treturn this->set_http_version(version.c_str());\n\t}\n\n\tbool add_header_pair(const std::string& name, const std::string& value)\n\t{\n\t\treturn http_parser_add_header(name.c_str(), name.size(),\n\t\t\t\t\t\t\t\t\t value.c_str(), value.size(),\n\t\t\t\t\t\t\t\t\t this->parser) == 0;\n\t}\n\n\tbool set_header_pair(const std::string& name, const std::string& value)\n\t{\n\t\treturn http_parser_set_header(name.c_str(), name.size(),\n\t\t\t\t\t\t\t\t\t value.c_str(), value.size(),\n\t\t\t\t\t\t\t\t\t this->parser) == 0;\n\t}\n\n\tbool append_output_body(const std::string& buf)\n\t{\n\t\treturn this->append_output_body(buf.c_str(), buf.size());\n\t}\n\n\tbool append_output_body_nocopy(const std::string& buf)\n\t{\n\t\treturn this->append_output_body_nocopy(buf.c_str(), buf.size());\n\t}\n\n\tbool get_output_body_merged(std::string& body) const\n\t{\n\t\tsize_t size = this->output_body_size;\n\t\tbody.resize(size);\n\t\treturn this->get_output_body_merged((void *)body.data(), &size);\n\t}\n\n\t/* for http task implementations. */\npublic:\n\tbool is_header_complete() const\n\t{\n\t\treturn http_parser_header_complete(this->parser);\n\t}\n\n\tbool has_connection_header() const\n\t{\n\t\treturn http_parser_has_connection(this->parser);\n\t}\n\n\tbool has_content_length_header() const\n\t{\n\t\treturn http_parser_has_content_length(this->parser);\n\t}\n\n\tbool has_keep_alive_header() const\n\t{\n\t\treturn http_parser_has_keep_alive(this->parser);\n\t}\n\n\tvoid end_parsing()\n\t{\n\t\thttp_parser_close_message(this->parser);\n\t}\n\n\t/* for header cursor implementations. */\n\tconst http_parser_t *get_parser() const\n\t{\n\t\treturn this->parser;\n\t}\n\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\nprotected:\n\thttp_parser_t *parser;\n\tsize_t cur_size;\n\nprivate:\n\tstruct list_head *combine_from(struct list_head *pos, size_t size);\n\nprivate:\n\tstruct list_head output_body;\n\tsize_t output_body_size;\n\npublic:\n\tHttpMessage(bool is_resp) : parser(new http_parser_t)\n\t{\n\t\thttp_parser_init(is_resp, this->parser);\n\t\tINIT_LIST_HEAD(&this->output_body);\n\t\tthis->output_body_size = 0;\n\t\tthis->cur_size = 0;\n\t}\n\n\tvirtual ~HttpMessage()\n\t{\n\t\tthis->clear_output_body();\n\t\tif (this->parser)\n\t\t{\n\t\t\thttp_parser_deinit(this->parser);\n\t\t\tdelete this->parser;\n\t\t}\n\t}\n\npublic:\n\tHttpMessage(HttpMessage&& msg);\n\tHttpMessage& operator = (HttpMessage&& msg);\n};\n\nclass HttpRequest : public HttpMessage\n{\npublic:\n\tconst char *get_method() const\n\t{\n\t\treturn http_parser_get_method(this->parser);\n\t}\n\n\tconst char *get_request_uri() const\n\t{\n\t\treturn http_parser_get_uri(this->parser);\n\t}\n\n\tbool set_method(const char *method)\n\t{\n\t\treturn http_parser_set_method(method, this->parser) == 0;\n\t}\n\n\tbool set_request_uri(const char *uri)\n\t{\n\t\treturn http_parser_set_uri(uri, this->parser) == 0;\n\t}\n\n\t/* std::string interfaces */\npublic:\n\tbool get_method(std::string& method) const\n\t{\n\t\tconst char *str = this->get_method();\n\n\t\tif (str)\n\t\t{\n\t\t\tmethod.assign(str);\n\t\t\treturn true;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tbool get_request_uri(std::string& uri) const\n\t{\n\t\tconst char *str = this->get_request_uri();\n\n\t\tif (str)\n\t\t{\n\t\t\turi.assign(str);\n\t\t\treturn true;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tbool set_method(const std::string& method)\n\t{\n\t\treturn this->set_method(method.c_str());\n\t}\n\n\tbool set_request_uri(const std::string& uri)\n\t{\n\t\treturn this->set_request_uri(uri.c_str());\n\t}\n\nprotected:\n\tvirtual int append(const void *buf, size_t *size);\n\nprivate:\n\tint handle_expect_continue();\n\npublic:\n\tHttpRequest() : HttpMessage(false) { }\n\npublic:\n\tHttpRequest(HttpRequest&& req) = default;\n\tHttpRequest& operator = (HttpRequest&& req) = default;\n};\n\nclass HttpResponse : public HttpMessage\n{\npublic:\n\tconst char *get_status_code() const\n\t{\n\t\treturn http_parser_get_code(this->parser);\n\t}\n\n\tconst char *get_reason_phrase() const\n\t{\n\t\treturn http_parser_get_phrase(this->parser);\n\t}\n\n\tbool set_status_code(const char *code)\n\t{\n\t\treturn http_parser_set_code(code, this->parser) == 0;\n\t}\n\n\tbool set_reason_phrase(const char *phrase)\n\t{\n\t\treturn http_parser_set_phrase(phrase, this->parser) == 0;\n\t}\n\n\t/* std::string interfaces */\npublic:\n\tbool get_status_code(std::string& code) const\n\t{\n\t\tconst char *str = this->get_status_code();\n\n\t\tif (str)\n\t\t{\n\t\t\tcode.assign(str);\n\t\t\treturn true;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tbool get_reason_phrase(std::string& phrase) const\n\t{\n\t\tconst char *str = this->get_reason_phrase();\n\n\t\tif (str)\n\t\t{\n\t\t\tphrase.assign(str);\n\t\t\treturn true;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tbool set_status_code(const std::string& code)\n\t{\n\t\treturn this->set_status_code(code.c_str());\n\t}\n\n\tbool set_reason_phrase(const std::string& phrase)\n\t{\n\t\treturn this->set_reason_phrase(phrase.c_str());\n\t}\n\npublic:\n\t/* Tell the parser, it is a HEAD response. For implementations. */\n\tvoid parse_zero_body()\n\t{\n\t\tthis->parser->transfer_length = 0;\n\t}\n\nprotected:\n\tvirtual int append(const void *buf, size_t *size);\n\npublic:\n\tHttpResponse() : HttpMessage(true) { }\n\npublic:\n\tHttpResponse(HttpResponse&& resp) = default;\n\tHttpResponse& operator = (HttpResponse&& resp) = default;\n};\n\nclass HttpMessageChunk : public ProtocolMessage\n{\npublic:\n\tbool get_chunk_data(const void **chunk_data, size_t *size) const;\n\tbool move_chunk_data(void **chunk_data, size_t *size);\n\tbool set_chunk_data(const void *chunk_data, size_t size);\n\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\nprivate:\n\tint append_chunk_line(const void *buf, size_t size);\n\nprivate:\n\tchar chunk_line[32];\n\tvoid *chunk_data;\n\tsize_t chunk_size;\n\tsize_t nreceived;\n\npublic:\n\tHttpMessageChunk()\n\t{\n\t\tthis->chunk_data = NULL;\n\t\tthis->nreceived = 0;\n\t}\n\n\tvirtual ~HttpMessageChunk()\n\t{\n\t\tfree(this->chunk_data);\n\t}\n\npublic:\n\tHttpMessageChunk(HttpMessageChunk&& msg);\n\tHttpMessageChunk& operator = (HttpMessageChunk&& msg);\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/HttpUtil.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"http_parser.h\"\n#include \"HttpMessage.h\"\n#include \"HttpUtil.h\"\n\nnamespace protocol\n{\n\nHttpHeaderMap::HttpHeaderMap(const HttpMessage *message)\n{\n\thttp_header_cursor_t cursor;\n\tstruct HttpMessageHeader header;\n\n\thttp_header_cursor_init(&cursor, message->get_parser());\n\twhile (http_header_cursor_next(&header.name, &header.name_len,\n\t\t\t\t\t\t\t\t &header.value, &header.value_len,\n\t\t\t\t\t\t\t\t &cursor) == 0)\n\t{\n\t\tstd::string key((const char *)header.name, header.name_len);\n\n\t\tstd::transform(key.begin(), key.end(), key.begin(), ::tolower);\n\t\theader_map_[key].emplace_back((const char *)header.value, header.value_len);\n\t}\n\n\thttp_header_cursor_deinit(&cursor);\n}\n\nbool HttpHeaderMap::key_exists(std::string key)\n{\n\tstd::transform(key.begin(), key.end(), key.begin(), ::tolower);\n\treturn header_map_.count(key) > 0;\n}\n\nstd::string HttpHeaderMap::get(std::string key)\n{\n\tstd::transform(key.begin(), key.end(), key.begin(), ::tolower);\n\tconst auto it = header_map_.find(key);\n\n\tif (it == header_map_.end() || it->second.empty())\n\t\treturn std::string();\n\n\treturn it->second[0];\n}\n\nbool HttpHeaderMap::get(std::string key, std::string& value)\n{\n\tstd::transform(key.begin(), key.end(), key.begin(), ::tolower);\n\tconst auto it = header_map_.find(key);\n\n\tif (it == header_map_.end() || it->second.empty())\n\t\treturn false;\n\n\tvalue = it->second[0];\n\treturn true;\n}\n\nstd::vector HttpHeaderMap::get_strict(std::string key)\n{\n\tstd::transform(key.begin(), key.end(), key.begin(), ::tolower);\n\treturn header_map_[key];\n}\n\nbool HttpHeaderMap::get_strict(std::string key, std::vector& values)\n{\n\tstd::transform(key.begin(), key.end(), key.begin(), ::tolower);\n\tconst auto it = header_map_.find(key);\n\n\tif (it == header_map_.end() || it->second.empty())\n\t\treturn false;\n\n\tvalues = it->second;\n\treturn true;\n}\n\nstd::string HttpUtil::decode_chunked_body(const HttpMessage *msg)\n{\n\tconst void *body;\n\tsize_t body_len;\n\tconst void *chunk;\n\tsize_t chunk_size;\n\tstd::string decode_result;\n\tHttpChunkCursor cursor(msg);\n\n\tif (msg->get_parsed_body(&body, &body_len))\n\t{\n\t\tdecode_result.reserve(body_len);\n\t\twhile (cursor.next(&chunk, &chunk_size))\n\t\t\tdecode_result.append((const char *)chunk, chunk_size);\n\t}\n\n\treturn decode_result;\n}\n\nvoid HttpUtil::set_response_status(HttpResponse *resp, int status_code)\n{\n\tchar buf[32];\n\tsprintf(buf, \"%d\", status_code);\n\tresp->set_status_code(buf);\n\n\tswitch (status_code)\n\t{\n\tcase 100:\n\t\tresp->set_reason_phrase(\"Continue\");\n\t\tbreak;\n\n\tcase 101:\n\t\tresp->set_reason_phrase(\"Switching Protocols\");\n\t\tbreak;\n\n\tcase 102:\n\t\tresp->set_reason_phrase(\"Processing\");\n\t\tbreak;\n\n\tcase 200:\n\t\tresp->set_reason_phrase(\"OK\");\n\t\tbreak;\n\n\tcase 201:\n\t\tresp->set_reason_phrase(\"Created\");\n\t\tbreak;\n\n\tcase 202:\n\t\tresp->set_reason_phrase(\"Accepted\");\n\t\tbreak;\n\n\tcase 203:\n\t\tresp->set_reason_phrase(\"Non-Authoritative Information\");\n\t\tbreak;\n\n\tcase 204:\n\t\tresp->set_reason_phrase(\"No Content\");\n\t\tbreak;\n\n\tcase 205:\n\t\tresp->set_reason_phrase(\"Reset Content\");\n\t\tbreak;\n\n\tcase 206:\n\t\tresp->set_reason_phrase(\"Partial Content\");\n\t\tbreak;\n\n\tcase 207:\n\t\tresp->set_reason_phrase(\"Multi-Status\");\n\t\tbreak;\n\n\tcase 208:\n\t\tresp->set_reason_phrase(\"Already Reported\");\n\t\tbreak;\n\n\tcase 226:\n\t\tresp->set_reason_phrase(\"IM Used\");\n\t\tbreak;\n\n\tcase 300:\n\t\tresp->set_reason_phrase(\"Multiple Choices\");\n\t\tbreak;\n\n\tcase 301:\n\t\tresp->set_reason_phrase(\"Moved Permanently\");\n\t\tbreak;\n\n\tcase 302:\n\t\tresp->set_reason_phrase(\"Found\");\n\t\tbreak;\n\n\tcase 303:\n\t\tresp->set_reason_phrase(\"See Other\");\n\t\tbreak;\n\n\tcase 304:\n\t\tresp->set_reason_phrase(\"Not Modified\");\n\t\tbreak;\n\n\tcase 305:\n\t\tresp->set_reason_phrase(\"Use Proxy\");\n\t\tbreak;\n\n\tcase 306:\n\t\tresp->set_reason_phrase(\"Switch Proxy\");\n\t\tbreak;\n\n\tcase 307:\n\t\tresp->set_reason_phrase(\"Temporary Redirect\");\n\t\tbreak;\n\n\tcase 308:\n\t\tresp->set_reason_phrase(\"Permanent Redirect\");\n\t\tbreak;\n\n\tcase 400:\n\t\tresp->set_reason_phrase(\"Bad Request\");\n\t\tbreak;\n\n\tcase 401:\n\t\tresp->set_reason_phrase(\"Unauthorized\");\n\t\tbreak;\n\n\tcase 402:\n\t\tresp->set_reason_phrase(\"Payment Required\");\n\t\tbreak;\n\n\tcase 403:\n\t\tresp->set_reason_phrase(\"Forbidden\");\n\t\tbreak;\n\n\tcase 404:\n\t\tresp->set_reason_phrase(\"Not Found\");\n\t\tbreak;\n\n\tcase 405:\n\t\tresp->set_reason_phrase(\"Method Not Allowed\");\n\t\tbreak;\n\n\tcase 406:\n\t\tresp->set_reason_phrase(\"Not Acceptable\");\n\t\tbreak;\n\n\tcase 407:\n\t\tresp->set_reason_phrase(\"Proxy Authentication Required\");\n\t\tbreak;\n\n\tcase 408:\n\t\tresp->set_reason_phrase(\"Request Timeout\");\n\t\tbreak;\n\n\tcase 409:\n\t\tresp->set_reason_phrase(\"Conflict\");\n\t\tbreak;\n\n\tcase 410:\n\t\tresp->set_reason_phrase(\"Gone\");\n\t\tbreak;\n\n\tcase 411:\n\t\tresp->set_reason_phrase(\"Length Required\");\n\t\tbreak;\n\n\tcase 412:\n\t\tresp->set_reason_phrase(\"Precondition Failed\");\n\t\tbreak;\n\n\tcase 413:\n\t\tresp->set_reason_phrase(\"Request Entity Too Large\");\n\t\tbreak;\n\n\tcase 414:\n\t\tresp->set_reason_phrase(\"Request-URI Too Long\");\n\t\tbreak;\n\n\tcase 415:\n\t\tresp->set_reason_phrase(\"Unsupported Media Type\");\n\t\tbreak;\n\n\tcase 416:\n\t\tresp->set_reason_phrase(\"Requested Range Not Satisfiable\");\n\t\tbreak;\n\n\tcase 417:\n\t\tresp->set_reason_phrase(\"Expectation Failed\");\n\t\tbreak;\n\n\tcase 418:\n\t\tresp->set_reason_phrase(\"I'm a teapot\");\n\t\tbreak;\n\n\tcase 420:\n\t\tresp->set_reason_phrase(\"Enhance Your Caim\");\n\t\tbreak;\n\n\tcase 421:\n\t\tresp->set_reason_phrase(\"Misdirected Request\");\n\t\tbreak;\n\n\tcase 422:\n\t\tresp->set_reason_phrase(\"Unprocessable Entity\");\n\t\tbreak;\n\n\tcase 423:\n\t\tresp->set_reason_phrase(\"Locked\");\n\t\tbreak;\n\n\tcase 424:\n\t\tresp->set_reason_phrase(\"Failed Dependency\");\n\t\tbreak;\n\n\tcase 425:\n\t\tresp->set_reason_phrase(\"Too Early\");\n\t\tbreak;\n\n\tcase 426:\n\t\tresp->set_reason_phrase(\"Upgrade Required\");\n\t\tbreak;\n\n\tcase 428:\n\t\tresp->set_reason_phrase(\"Precondition Required\");\n\t\tbreak;\n\n\tcase 429:\n\t\tresp->set_reason_phrase(\"Too Many Requests\");\n\t\tbreak;\n\n\tcase 431:\n\t\tresp->set_reason_phrase(\"Request Header Fields Too Large\");\n\t\tbreak;\n\n\tcase 444:\n\t\tresp->set_reason_phrase(\"No Response\");\n\t\tbreak;\n\n\tcase 450:\n\t\tresp->set_reason_phrase(\"Blocked by Windows Parental Controls\");\n\t\tbreak;\n\n\tcase 451:\n\t\tresp->set_reason_phrase(\"Unavailable For Legal Reasons\");\n\t\tbreak;\n\n\tcase 494:\n\t\tresp->set_reason_phrase(\"Request Header Too Large\");\n\t\tbreak;\n\n\tcase 500:\n\t\tresp->set_reason_phrase(\"Internal Server Error\");\n\t\tbreak;\n\n\tcase 501:\n\t\tresp->set_reason_phrase(\"Not Implemented\");\n\t\tbreak;\n\n\tcase 502:\n\t\tresp->set_reason_phrase(\"Bad Gateway\");\n\t\tbreak;\n\n\tcase 503:\n\t\tresp->set_reason_phrase(\"Service Unavailable\");\n\t\tbreak;\n\n\tcase 504:\n\t\tresp->set_reason_phrase(\"Gateway Timeout\");\n\t\tbreak;\n\n\tcase 505:\n\t\tresp->set_reason_phrase(\"HTTP Version Not Supported\");\n\t\tbreak;\n\n\tcase 506:\n\t\tresp->set_reason_phrase(\"Variant Also Negotiates\");\n\t\tbreak;\n\n\tcase 507:\n\t\tresp->set_reason_phrase(\"Insufficient Storage\");\n\t\tbreak;\n\n\tcase 508:\n\t\tresp->set_reason_phrase(\"Loop Detected\");\n\t\tbreak;\n\n\tcase 510:\n\t\tresp->set_reason_phrase(\"Not Extended\");\n\t\tbreak;\n\n\tcase 511:\n\t\tresp->set_reason_phrase(\"Network Authentication Required\");\n\t\tbreak;\n\n\tdefault:\n\t\tresp->set_reason_phrase(\"Unknown\");\n\t\tbreak;\n\t}\n}\n\nbool HttpHeaderCursor::next(std::string& name, std::string& value)\n{\n\tstruct HttpMessageHeader header;\n\n\tif (this->next(&header))\n\t{\n\t\tname.assign((const char *)header.name, header.name_len);\n\t\tvalue.assign((const char *)header.value, header.value_len);\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nbool HttpHeaderCursor::find(const std::string& name, std::string& value)\n{\n\tstruct HttpMessageHeader header = {\n\t\t.name\t\t=\tname.c_str(),\n\t\t.name_len\t=\tname.size(),\n\t};\n\n\tif (this->find(&header))\n\t{\n\t\tvalue.assign((const char *)header.value, header.value_len);\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nbool HttpHeaderCursor::find_and_erase(const std::string& name)\n{\n\tstruct HttpMessageHeader header = {\n\t\t.name\t\t=\tname.c_str(),\n\t\t.name_len\t=\tname.size(),\n\t};\n\treturn this->find_and_erase(&header);\n}\n\nHttpChunkCursor::HttpChunkCursor(const HttpMessage *msg)\n{\n\tif (msg->get_parsed_body(&this->body, &this->body_len))\n\t{\n\t\tthis->pos = this->body;\n\t\tthis->chunked = msg->is_chunked();\n\t\tthis->end = false;\n\t}\n\telse\n\t{\n\t\tthis->body = NULL;\n\t\tthis->end = true;\n\t}\n}\n\nbool HttpChunkCursor::next(const void **chunk, size_t *size)\n{\n\tif (this->end)\n\t\treturn false;\n\n\tif (!this->chunked)\n\t{\n\t\t*chunk = this->body;\n\t\t*size = this->body_len;\n\t\tthis->end = true;\n\t\treturn true;\n\t}\n\n\tconst char *cur = (const char *)this->pos;\n\tchar *end;\n\n\t*size = strtoul(cur, &end, 16);\n\tif (*size == 0)\n\t{\n\t\tthis->end = true;\n\t\treturn false;\n\t}\n\n\tcur = strchr(end, '\\r');\n\t*chunk = cur + 2;\n\tcur += *size + 4;\n\tthis->pos = cur;\n\treturn true;\n}\n\nvoid HttpChunkCursor::rewind()\n{\n\tif (this->body)\n\t{\n\t\tthis->pos = this->body;\n\t\tthis->end = false;\n\t}\n}\n\n}\n\n" }, { "path": "src/protocol/HttpUtil.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _HTTPUTIL_H_\n#define _HTTPUTIL_H_\n\n#include \n#include \n#include \n#include \"http_parser.h\"\n#include \"HttpMessage.h\"\n\n/**\n * @file HttpUtil.h\n * @brief Http toolbox\n */\n\n#define HttpMethodGet\t\t\"GET\"\n#define HttpMethodHead\t\t\"HEAD\"\n#define HttpMethodPost\t\t\"POST\"\n#define HttpMethodPut\t\t\"PUT\"\n#define HttpMethodPatch\t\t\"PATCH\"\n#define HttpMethodDelete\t\"DELETE\"\n#define HttpMethodConnect\t\"CONNECT\"\n#define HttpMethodOptions\t\"OPTIONS\"\n#define HttpMethodTrace\t\t\"TRACE\"\n\nenum\n{\n\tHttpStatusContinue = 100, // RFC 7231, 6.2.1\n\tHttpStatusSwitchingProtocols = 101, // RFC 7231, 6.2.2\n\tHttpStatusProcessing = 102, // RFC 2518, 10.1\n\n\tHttpStatusOK = 200, // RFC 7231, 6.3.1\n\tHttpStatusCreated = 201, // RFC 7231, 6.3.2\n\tHttpStatusAccepted = 202, // RFC 7231, 6.3.3\n\tHttpStatusNonAuthoritativeInfo = 203, // RFC 7231, 6.3.4\n\tHttpStatusNoContent = 204, // RFC 7231, 6.3.5\n\tHttpStatusResetContent = 205, // RFC 7231, 6.3.6\n\tHttpStatusPartialContent = 206, // RFC 7233, 4.1\n\tHttpStatusMultiStatus = 207, // RFC 4918, 11.1\n\tHttpStatusAlreadyReported = 208, // RFC 5842, 7.1\n\tHttpStatusIMUsed = 226, // RFC 3229, 10.4.1\n\n\tHttpStatusMultipleChoices = 300, // RFC 7231, 6.4.1\n\tHttpStatusMovedPermanently = 301, // RFC 7231, 6.4.2\n\tHttpStatusFound = 302, // RFC 7231, 6.4.3\n\tHttpStatusSeeOther = 303, // RFC 7231, 6.4.4\n\tHttpStatusNotModified = 304, // RFC 7232, 4.1\n\tHttpStatusUseProxy = 305, // RFC 7231, 6.4.5\n\n\tHttpStatusTemporaryRedirect = 307, // RFC 7231, 6.4.7\n\tHttpStatusPermanentRedirect = 308, // RFC 7538, 3\n\n\tHttpStatusBadRequest = 400, // RFC 7231, 6.5.1\n\tHttpStatusUnauthorized = 401, // RFC 7235, 3.1\n\tHttpStatusPaymentRequired = 402, // RFC 7231, 6.5.2\n\tHttpStatusForbidden = 403, // RFC 7231, 6.5.3\n\tHttpStatusNotFound = 404, // RFC 7231, 6.5.4\n\tHttpStatusMethodNotAllowed = 405, // RFC 7231, 6.5.5\n\tHttpStatusNotAcceptable = 406, // RFC 7231, 6.5.6\n\tHttpStatusProxyAuthRequired = 407, // RFC 7235, 3.2\n\tHttpStatusRequestTimeout = 408, // RFC 7231, 6.5.7\n\tHttpStatusConflict = 409, // RFC 7231, 6.5.8\n\tHttpStatusGone = 410, // RFC 7231, 6.5.9\n\tHttpStatusLengthRequired = 411, // RFC 7231, 6.5.10\n\tHttpStatusPreconditionFailed = 412, // RFC 7232, 4.2\n\tHttpStatusRequestEntityTooLarge = 413, // RFC 7231, 6.5.11\n\tHttpStatusRequestURITooLong = 414, // RFC 7231, 6.5.12\n\tHttpStatusUnsupportedMediaType = 415, // RFC 7231, 6.5.13\n\tHttpStatusRequestedRangeNotSatisfiable = 416, // RFC 7233, 4.4\n\tHttpStatusExpectationFailed = 417, // RFC 7231, 6.5.14\n\tHttpStatusTeapot = 418, // RFC 7168, 2.3.3\n\tHttpStatusEnhanceYourCaim = 420, // Twitter Search\n\tHttpStatusMisdirectedRequest = 421, // RFC 7540, 9.1.2\n\tHttpStatusUnprocessableEntity = 422, // RFC 4918, 11.2\n\tHttpStatusLocked = 423, // RFC 4918, 11.3\n\tHttpStatusFailedDependency = 424, // RFC 4918, 11.4\n\tHttpStatusTooEarly = 425, // RFC 8470, 5.2.\n\tHttpStatusUpgradeRequired = 426, // RFC 7231, 6.5.15\n\tHttpStatusPreconditionRequired = 428, // RFC 6585, 3\n\tHttpStatusTooManyRequests = 429, // RFC 6585, 4\n\tHttpStatusRequestHeaderFieldsTooLarge = 431, // RFC 6585, 5\n\tHttpStatusNoResponse = 444, // Nginx\n\tHttpStatusBlocked = 450, // Windows\n\tHttpStatusUnavailableForLegalReasons = 451, // RFC 7725, 3\n\tHttpStatusTooLargeForNginx = 494, // Nginx\n\n\tHttpStatusInternalServerError = 500, // RFC 7231, 6.6.1\n\tHttpStatusNotImplemented = 501, // RFC 7231, 6.6.2\n\tHttpStatusBadGateway = 502, // RFC 7231, 6.6.3\n\tHttpStatusServiceUnavailable = 503, // RFC 7231, 6.6.4\n\tHttpStatusGatewayTimeout = 504, // RFC 7231, 6.6.5\n\tHttpStatusHTTPVersionNotSupported = 505, // RFC 7231, 6.6.6\n\tHttpStatusVariantAlsoNegotiates = 506, // RFC 2295, 8.1\n\tHttpStatusInsufficientStorage = 507, // RFC 4918, 11.5\n\tHttpStatusLoopDetected = 508, // RFC 5842, 7.2\n\tHttpStatusNotExtended = 510, // RFC 2774, 7\n\tHttpStatusNetworkAuthenticationRequired = 511, // RFC 6585, 6\n};\n\nnamespace protocol\n{\n\n// static class\nclass HttpUtil\n{\npublic:\n\tstatic void set_response_status(HttpResponse *resp, int status_code);\n\tstatic std::string decode_chunked_body(const HttpMessage *msg);\n};\n\nclass HttpHeaderMap\n{\npublic:\n\tHttpHeaderMap(const HttpMessage *message);\n\n\tbool key_exists(std::string key);\n\tstd::string get(std::string key);\n\tbool get(std::string key, std::string& value);\n\tstd::vector get_strict(std::string key);\n\tbool get_strict(std::string key, std::vector& values);\n\nprivate:\n\tstd::unordered_map> header_map_;\n};\n\nclass HttpHeaderCursor\n{\npublic:\n\tHttpHeaderCursor(const HttpMessage *message);\n\tvirtual ~HttpHeaderCursor();\n\npublic:\n\tbool next(struct HttpMessageHeader *header);\n\tbool find(struct HttpMessageHeader *header);\n\tbool erase();\n\tbool find_and_erase(struct HttpMessageHeader *header);\n\tvoid rewind();\n\n\t/* std::string interface */\npublic:\n\tbool next(std::string& name, std::string& value);\n\tbool find(const std::string& name, std::string& value);\n\tbool find_and_erase(const std::string& name);\n\nprotected:\n\thttp_header_cursor_t cursor;\n};\n\nclass HttpChunkCursor\n{\npublic:\n\tHttpChunkCursor(const HttpMessage *message);\n\tvirtual ~HttpChunkCursor() { }\n\npublic:\n\tbool next(const void **chunk, size_t *size);\n\tvoid rewind();\n\nprotected:\n\tconst void *body;\n\tsize_t body_len;\n\tconst void *pos;\n\tbool chunked;\n\tbool end;\n};\n\n////////////////////\n\ninline HttpHeaderCursor::HttpHeaderCursor(const HttpMessage *message)\n{\n\thttp_header_cursor_init(&this->cursor, message->get_parser());\n}\n\ninline HttpHeaderCursor::~HttpHeaderCursor()\n{\n\thttp_header_cursor_deinit(&this->cursor);\n}\n\ninline bool HttpHeaderCursor::next(struct HttpMessageHeader *header)\n{\n\treturn http_header_cursor_next(&header->name, &header->name_len,\n\t\t\t\t\t\t\t\t &header->value, &header->value_len,\n\t\t\t\t\t\t\t\t &this->cursor) == 0;\n}\n\ninline bool HttpHeaderCursor::find(struct HttpMessageHeader *header)\n{\n\treturn http_header_cursor_find(header->name, header->name_len,\n\t\t\t\t\t\t\t\t &header->value, &header->value_len,\n\t\t\t\t\t\t\t\t &this->cursor) == 0;\n}\n\ninline bool HttpHeaderCursor::erase()\n{\n\treturn http_header_cursor_erase(&this->cursor) == 0;\n}\n\ninline bool HttpHeaderCursor::find_and_erase(struct HttpMessageHeader *header)\n{\n\tif (this->find(header))\n\t\treturn this->erase();\n\n\treturn false;\n}\n\ninline void HttpHeaderCursor::rewind()\n{\n\thttp_header_cursor_rewind(&this->cursor);\n}\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/KafkaDataTypes.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n\t http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"KafkaDataTypes.h\"\n\n#define MIN(x, y)\t((x) <= (y) ? (x) : (y))\n\nnamespace protocol\n{\n\nstd::string KafkaConfig::get_sasl_info() const\n{\n\tstd::string info;\n\n\tif (strcasecmp(this->ptr->mechanisms, \"plain\") == 0)\n\t{\n\t\tinfo += this->ptr->mechanisms;\n\t\tinfo += \"|\";\n\t\tinfo += this->ptr->username;\n\t\tinfo += \"|\";\n\t\tinfo += this->ptr->password;\n\t\tinfo += \"|\";\n\t}\n\telse if (strncasecmp(this->ptr->mechanisms, \"SCRAM\", 5) == 0)\n\t{\n\t\tinfo += this->ptr->mechanisms;\n\t\tinfo += \"|\";\n\t\tinfo += this->ptr->username;\n\t\tinfo += \"|\";\n\t\tinfo += this->ptr->password;\n\t\tinfo += \"|\";\n\t}\n\n\treturn info;\n}\n\nstatic bool compare_member(const kafka_member_t *m1, const kafka_member_t *m2)\n{\n\treturn strcmp(m1->member_id, m2->member_id) < 0;\n}\n\ninline void KafkaMetaSubscriber::sort_by_member()\n{\n\tstd::sort(this->member_vec.begin(), this->member_vec.end(), compare_member);\n}\n\nstatic bool operator<(const KafkaMetaSubscriber& s1, const KafkaMetaSubscriber& s2)\n{\n\treturn strcmp(s1.get_meta()->get_topic(), s2.get_meta()->get_topic()) < 0;\n}\n\n/*\n * For example, suppose there are two consumers C0 and C1, two topics t0 and t1, and each topic has 3 partitions,\n * resulting in partitions t0p0, t0p1, t0p2, t1p0, t1p1, and t1p2.\n *\n * The assignment will be:\n * C0: [t0p0, t0p1, t1p0, t1p1]\n * C1: [t0p2, t1p2]\n */\nint KafkaCgroup::kafka_range_assignor(kafka_member_t **members,\n\t\t\t\t\t\t\t\t\t int member_elements,\n\t\t\t\t\t\t\t\t\t void *meta_topic)\n{\n\tstd::vector *subscribers =\n\t\tstatic_cast *>(meta_topic);\n\n\t/* The range assignor works on a per-topic basis. */\n\tfor (auto& subscriber : *subscribers)\n\t{\n\t\tsubscriber.sort_by_member();\n\n\t\tint num_partitions_per_consumer =\n\t\t\t\tsubscriber.get_meta()->get_partition_elements() /\n\t\t\t\tsubscriber.get_member()->size();\n\n\t\t/* If it does not evenly divide, then the first few consumers\n\t\t * will have one extra partition. */\n\t\tint consumers_with_extra_partition =\n\t\t\t\tsubscriber.get_meta()->get_partition_elements() %\n\t\t\t\tsubscriber.get_member()->size();\n\n\t\tfor (int i = 0 ; i < (int)subscriber.get_member()->size(); i++)\n\t\t{\n\t\t\tint start = num_partitions_per_consumer * i +\n\t\t\t\t\tMIN(i, consumers_with_extra_partition);\n\t\t\tint length = num_partitions_per_consumer +\n\t\t\t\t\t(i + 1 > consumers_with_extra_partition ? 0 : 1);\n\n\t\t\tif (length == 0)\n\t\t\t\tcontinue;\n\n\t\t\tfor (int j = start; j < length + start; ++j)\n\t\t\t{\n\t\t\t\tKafkaToppar *toppar = new KafkaToppar;\n\t\t\t\tif (!toppar->set_topic_partition(subscriber.get_meta()->get_topic(), j))\n\t\t\t\t{\n\t\t\t\t\tdelete toppar;\n\t\t\t\t\treturn -1;\n\t\t\t\t}\n\n\t\t\t\tlist_add_tail(&toppar->list, &subscriber.get_member()->at(i)->assigned_toppar_list);\n\t\t\t}\n\t\t}\n\t}\n\n\treturn 0;\n}\n\n/*\n * For example, suppose there are two consumers C0 and C1, two topics t0 and\n * t1, and each topic has 3 partitions, resulting in partitions t0p0, t0p1,\n * t0p2, t1p0, t1p1, and t1p2.\n *\n * The assignment will be:\n * C0: [t0p0, t0p2, t1p1]\n * C1: [t0p1, t1p0, t1p2]\n */\nint KafkaCgroup::kafka_roundrobin_assignor(kafka_member_t **members,\n\t\t\t\t\t\t\t\t\t\t int member_elements,\n\t\t\t\t\t\t\t\t\t\t void *meta_topic)\n{\n\tstd::vector *subscribers =\n\t\tstatic_cast *>(meta_topic);\n\n\tint next = -1;\n\n\tstd::sort(subscribers->begin(), subscribers->end());\n\tstd::sort(members, members + member_elements, compare_member);\n\n\tfor (const auto& subscriber : *subscribers)\n\t{\n\t\tint partition_elements = subscriber.get_meta()->get_partition_elements();\n\n\t\tfor (int partition = 0; partition < partition_elements; ++partition)\n\t\t{\n\t\t\tnext = (next + 1) % subscriber.get_member()->size();\n\t\t\tstruct list_head *pos;\n\t\t\tKafkaToppar *toppar;\n\n\t\t\tint i = 0;\n\t\t\tfor (; i < member_elements; i++)\n\t\t\t{\n\t\t\t\tbool flag = false;\n\t\t\t\tlist_for_each(pos, &members[next + i]->toppar_list)\n\t\t\t\t{\n\t\t\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\t\t\tif (strcmp(subscriber.get_meta()->get_topic(), toppar->get_topic()) == 0)\n\t\t\t\t\t{\n\t\t\t\t\t\tflag = true;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (flag)\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (i >= member_elements)\n\t\t\t\treturn -1;\n\n\t\t\ttoppar = new KafkaToppar;\n\t\t\tif (!toppar->set_topic_partition(subscriber.get_meta()->get_topic(),\n\t\t\t\t\t\t\t\t\t\t\t partition))\n\t\t\t{\n\t\t\t\tdelete toppar;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tlist_add_tail(toppar->get_list(), &members[next]->assigned_toppar_list);\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nbool KafkaMeta::create_partitions(int partition_cnt)\n{\n\tif (partition_cnt <= 0)\n\t\treturn true;\n\n\tkafka_partition_t **partitions;\n\tpartitions = (kafka_partition_t **)malloc(sizeof(void *) * partition_cnt);\n\tif (!partitions)\n\t\treturn false;\n\n\tint i;\n\n\tfor (i = 0; i < partition_cnt; ++i)\n\t{\n\t\tpartitions[i] = (kafka_partition_t *)malloc(sizeof(kafka_partition_t));\n\t\tif (!partitions[i])\n\t\t\tbreak;\n\n\t\tkafka_partition_init(partitions[i]);\n\t}\n\n\tif (i != partition_cnt)\n\t{\n\t\twhile (--i >= 0)\n\t\t{\n\t\t\tkafka_partition_deinit(partitions[i]);\n\t\t\tfree(partitions[i]);\n\t\t}\n\n\t\tfree(partitions);\n\t\treturn false;\n\t}\n\n\tfor (i = 0; i < this->ptr->partition_elements; ++i)\n\t{\n\t\tkafka_partition_deinit(this->ptr->partitions[i]);\n\t\tfree(this->ptr->partitions[i]);\n\t}\n\n\tfree(this->ptr->partitions);\n\n\tthis->ptr->partitions = partitions;\n\tthis->ptr->partition_elements = partition_cnt;\n\treturn true;\n}\n\nvoid KafkaCgroup::add_subscriber(KafkaMetaList *meta_list, \n\t\t\t\t\t\t\t\t std::vector *subscribers)\n{\n\tmeta_list->rewind();\n\tKafkaMeta *meta;\n\n\twhile ((meta = meta_list->get_next()) != NULL)\n\t{\n\t\tKafkaMetaSubscriber subscriber;\n\n\t\tsubscriber.set_meta(meta);\n\t\tfor (int i = 0; i < this->get_member_elements(); ++i)\n\t\t{\n\t\t\tstruct list_head *pos;\n\t\t\tKafkaToppar *toppar;\n\t\t\tbool flag = false;\n\n\t\t\tlist_for_each(pos, &this->get_members()[i]->toppar_list)\n\t\t\t{\n\t\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\t\tif (strcmp(meta->get_topic(), toppar->get_topic()) == 0)\n\t\t\t\t{\n\t\t\t\t\tflag = true;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (flag)\n\t\t\t\tsubscriber.add_member(this->get_members()[i]);\n\t\t}\n\n\t\tif (!subscriber.get_member()->empty())\n\t\t\tsubscribers->emplace_back(subscriber);\n\t}\n}\n\nint KafkaCgroup::run_assignor(KafkaMetaList *meta_list,\n\t\t\t\t\t\t\t const char *protocol_name)\n{\n\tstd::vector subscribers;\n\tthis->add_subscriber(meta_list, &subscribers);\n\n\tstruct list_head *pos;\n\tkafka_group_protocol_t *protocol;\n\tbool flag = false;\n\tlist_for_each(pos, this->get_group_protocol())\n\t{\n\t\tprotocol = list_entry(pos, kafka_group_protocol_t, list);\n\t\tif (strcmp(protocol_name, protocol->protocol_name) == 0)\n\t\t{\n\t\t\tflag = true;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (!flag)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\treturn protocol->assignor(this->get_members(), this->get_member_elements(),\n\t\t\t\t\t\t\t &subscribers);\n}\n\nKafkaCgroup::KafkaCgroup()\n{\n\tthis->ptr = new kafka_cgroup_t;\n\tkafka_cgroup_init(this->ptr);\n\tkafka_group_protocol_t *protocol = new kafka_group_protocol_t;\n\tprotocol->protocol_name = new char[strlen(\"range\") + 1];\n\tmemcpy(protocol->protocol_name, \"range\", strlen(\"range\") + 1);\n\tprotocol->assignor = kafka_range_assignor;\n\tlist_add_tail(&protocol->list, &this->ptr->group_protocol_list);\n\tprotocol = new kafka_group_protocol_t;\n\tprotocol->protocol_name = new char[strlen(\"roundrobin\") + 1];\n\tmemcpy(protocol->protocol_name, \"roundrobin\", strlen(\"roundrobin\") + 1);\n\tprotocol->assignor = kafka_roundrobin_assignor;\n\tlist_add_tail(&protocol->list, &this->ptr->group_protocol_list);\n\tthis->ref = new std::atomic(1);\n\tthis->coordinator = NULL;\n}\n\nKafkaCgroup::~KafkaCgroup()\n{\n\tif (--*this->ref == 0)\n\t{\n\t\tfor (int i = 0; i < this->ptr->member_elements; ++i)\n\t\t{\n\t\t\tkafka_member_t *member = this->ptr->members[i];\n\t\t\tKafkaToppar *toppar;\n\t\t\tstruct list_head *pos, *tmp;\n\n\t\t\tlist_for_each_safe(pos, tmp, &member->toppar_list)\n\t\t\t{\n\t\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\t\tlist_del(pos);\n\t\t\t\tdelete toppar;\n\t\t\t}\n\n\t\t\tlist_for_each_safe(pos, tmp, &member->assigned_toppar_list)\n\t\t\t{\n\t\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\t\tlist_del(pos);\n\t\t\t\tdelete toppar;\n\t\t\t}\n\t\t}\n\n\t\tkafka_cgroup_deinit(this->ptr);\n\n\t\tstruct list_head *tmp, *pos;\n\t\tKafkaToppar *toppar;\n\n\t\tlist_for_each_safe(pos, tmp, &this->ptr->assigned_toppar_list)\n\t\t{\n\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\tlist_del(pos);\n\t\t\tdelete toppar;\n\t\t}\n\n\t\tkafka_group_protocol_t *protocol;\n\t\tlist_for_each_safe(pos, tmp, &this->ptr->group_protocol_list)\n\t\t{\n\t\t\tprotocol = list_entry(pos, kafka_group_protocol_t, list);\n\t\t\tlist_del(pos);\n\t\t\tdelete []protocol->protocol_name;\n\t\t\tdelete protocol;\n\t\t}\n\n\t\tdelete []this->ptr->group_name;\n\n\t\tdelete this->ptr;\n\t\tdelete this->ref;\n\t}\n\n\tdelete this->coordinator;\n}\n\nKafkaCgroup::KafkaCgroup(KafkaCgroup&& move)\n{\n\tthis->ptr = move.ptr;\n\tthis->ref = move.ref;\n\tmove.ptr = new kafka_cgroup_t;\n\tkafka_cgroup_init(move.ptr);\n\tmove.ref = new std::atomic(1);\n\tthis->coordinator = move.coordinator;\n\tmove.coordinator = NULL;\n}\n\nKafkaCgroup& KafkaCgroup::operator= (KafkaCgroup&& move)\n{\n\tif (this != &move)\n\t{\n\t\tthis->~KafkaCgroup();\n\t\tthis->ptr = move.ptr;\n\t\tthis->ref = move.ref;\n\t\tmove.ptr = new kafka_cgroup_t;\n\t\tkafka_cgroup_init(move.ptr);\n\t\tmove.ref = new std::atomic(1);\n\t\tthis->coordinator = move.coordinator;\n\t\tmove.coordinator = NULL;\n\t}\n\n\treturn *this;\n}\n\nKafkaCgroup::KafkaCgroup(const KafkaCgroup& copy)\n{\n\tthis->ptr = copy.ptr;\n\tthis->ref = copy.ref;\n\t++*this->ref;\n\n\tif (copy.coordinator)\n\t\tthis->coordinator = new KafkaBroker(copy.coordinator->get_raw_ptr());\n\telse\n\t\tthis->coordinator = NULL;\n}\n\nKafkaCgroup& KafkaCgroup::operator= (const KafkaCgroup& copy)\n{\n\tthis->~KafkaCgroup();\n\tthis->ptr = copy.ptr;\n\tthis->ref = copy.ref;\n\t++*this->ref;\n\n\tif (copy.coordinator)\n\t\tthis->coordinator = new KafkaBroker(copy.coordinator->get_raw_ptr());\n\telse\n\t\tthis->coordinator = NULL;\n\n\treturn *this;\n}\n\nbool KafkaCgroup::create_members(int member_cnt)\n{\n\tif (member_cnt == 0)\n\t\treturn true;\n\n\tkafka_member_t **members;\n\tmembers = (kafka_member_t **)malloc(sizeof(void *) * member_cnt);\n\tif (!members)\n\t\treturn false;\n\n\tint i;\n\n\tfor (i = 0; i < member_cnt; ++i)\n\t{\n\t\tmembers[i] = (kafka_member_t *)malloc(sizeof(kafka_member_t));\n\t\tif (!members[i])\n\t\t\tbreak;\n\n\t\tkafka_member_init(members[i]);\n\t\tINIT_LIST_HEAD(&members[i]->toppar_list);\n\t\tINIT_LIST_HEAD(&members[i]->assigned_toppar_list);\n\t}\n\n\tif (i != member_cnt)\n\t{\n\t\twhile (--i >= 0)\n\t\t{\n\t\t\tKafkaToppar *toppar;\n\t\t\tstruct list_head *pos, *tmp;\n\t\t\tlist_for_each_safe(pos, tmp, &members[i]->toppar_list)\n\t\t\t{\n\t\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\t\tlist_del(pos);\n\t\t\t\tdelete toppar;\n\t\t\t}\n\n\t\t\tlist_for_each_safe(pos, tmp, &members[i]->assigned_toppar_list)\n\t\t\t{\n\t\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\t\tlist_del(pos);\n\t\t\t\tdelete toppar;\n\t\t\t}\n\n\t\t\tkafka_member_deinit(members[i]);\n\t\t\tfree(members[i]);\n\t\t}\n\n\t\tfree(members);\n\t\treturn false;\n\t}\n\n\tfor (i = 0; i < this->ptr->member_elements; ++i)\n\t{\n\t\tKafkaToppar *toppar;\n\t\tstruct list_head *pos, *tmp;\n\n\t\tlist_for_each_safe(pos, tmp, &this->ptr->members[i]->toppar_list)\n\t\t{\n\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\tlist_del(pos);\n\t\t\tdelete toppar;\n\t\t}\n\n\t\tlist_for_each_safe(pos, tmp, &this->ptr->members[i]->assigned_toppar_list)\n\t\t{\n\t\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\t\tlist_del(pos);\n\t\t\tdelete toppar;\n\t\t}\n\n\t\tkafka_member_deinit(this->ptr->members[i]);\n\t\tfree(this->ptr->members[i]);\n\t}\n\n\tfree(this->ptr->members);\n\n\tthis->ptr->members = members;\n\tthis->ptr->member_elements = member_cnt;\n\treturn true;\n}\n\nvoid KafkaCgroup::add_assigned_toppar(KafkaToppar *toppar)\n{\n\tlist_add_tail(toppar->get_list(), &this->ptr->assigned_toppar_list);\n}\n\nvoid KafkaCgroup::assigned_toppar_rewind()\n{\n\tthis->curpos = &this->ptr->assigned_toppar_list;\n}\n\nKafkaToppar *KafkaCgroup::get_assigned_toppar_next()\n{\n\tif (this->curpos->next == &this->ptr->assigned_toppar_list)\n\t\treturn NULL;\n\n\tthis->curpos = this->curpos->next;\n\treturn list_entry(this->curpos, KafkaToppar, list);\n}\n\nvoid KafkaCgroup::del_assigned_toppar_cur()\n{\n\tassert(this->curpos != &this->ptr->assigned_toppar_list);\n\tthis->curpos = this->curpos->prev;\n\tlist_del(this->curpos->next);\n}\n\nbool KafkaRecord::add_header_pair(const void *key, size_t key_len,\n\t\t\t\t\t\t\t\t const void *val, size_t val_len)\n{\n\tkafka_record_header_t *header;\n\n\theader = (kafka_record_header_t *)malloc(sizeof(kafka_record_header_t));\n\tif (!header)\n\t\treturn false;\n\n\tkafka_record_header_init(header);\n\tif (kafka_record_header_set_kv(key, key_len, val, val_len, header) < 0)\n\t{\n\t\tfree(header);\n\t\treturn false;\n\t}\n\n\tlist_add_tail(&header->list, &this->ptr->header_list);\n\treturn true;\n}\n\nbool KafkaRecord::add_header_pair(const std::string& key,\n\t\t\t\t\t\t\t\t const std::string& val)\n{\n\treturn add_header_pair(key.c_str(), key.size(), val.c_str(), val.size());\n}\n\nKafkaToppar::~KafkaToppar()\n{\n\tif (--*this->ref == 0)\n\t{\n\t\tkafka_topic_partition_deinit(this->ptr);\n\n\t\tstruct list_head *tmp, *pos;\n\t\tKafkaRecord *record;\n\t\tlist_for_each_safe(pos, tmp, &this->ptr->record_list)\n\t\t{\n\t\t\trecord = list_entry(pos, KafkaRecord, list);\n\t\t\tlist_del(pos);\n\t\t\tdelete record;\n\t\t}\n\n\t\tdelete this->ptr;\n\t\tdelete this->ref;\n\t}\n}\n\nvoid KafkaBuffer::list_splice(KafkaBuffer *buffer)\n{\n\tstruct list_head *pre_insert;\n\tstruct list_head *pre_tail;\n\n\tthis->buf_size -= this->insert_buf_size;\n\n\tpre_insert = this->insert_pos->next;\n\t__list_splice(buffer->get_head(), this->insert_pos, pre_insert);\n\n\tpre_tail = this->block_list.get_tail();\n\tbuffer->get_head()->prev->next = this->block_list.get_head();\n\tthis->block_list.get_head()->prev = buffer->get_head()->prev;\n\n\tbuffer->get_head()->next = pre_insert;\n\tbuffer->get_head()->prev = pre_tail;\n\tpre_tail->next = buffer->get_head();\n\n\tpre_insert->prev = buffer->get_head();\n\n\tthis->buf_size += buffer->get_size();\n}\n\nsize_t KafkaBuffer::peek(const char **buf)\n{\n\tif (!this->inited)\n\t{\n\t\tthis->inited = true;\n\t\tthis->cur_pos = std::make_pair(this->block_list.get_next(), 0);\n\t}\n\n\tif (this->cur_pos.first == this->block_list.get_tail_entry() &&\n\t\tthis->cur_pos.second == this->block_list.get_tail_entry()->get_len())\n\t{\n\t\t*buf = NULL;\n\t\treturn 0;\n\t}\n\n\tKafkaBlock *block = this->cur_pos.first;\n\n\tif (this->cur_pos.second >= block->get_len())\n\t{\n\t\tblock = this->block_list.get_next();\n\t\tthis->cur_pos = std::make_pair(block, 0);\n\t}\n\n\t*buf = (char *)block->get_block() + this->cur_pos.second;\n\n\treturn block->get_len() - this->cur_pos.second;\n}\n\nKafkaToppar *get_toppar(const char *topic, int partition,\n\t\t\t\t\t\tKafkaTopparList *toppar_list)\n{\n\tstruct list_head *pos;\n\tKafkaToppar *toppar;\n\tlist_for_each(pos, toppar_list->get_head())\n\t{\n\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\tif (strcmp(toppar->get_topic(), topic) == 0 &&\n\t\t\ttoppar->get_partition() == partition)\n\t\t\treturn toppar;\n\t}\n\n\treturn NULL;\n}\n\nconst KafkaMeta *get_meta(const char *topic, KafkaMetaList *meta_list)\n{\n\tstruct list_head *pos;\n\tconst KafkaMeta *meta;\n\tlist_for_each(pos, meta_list->get_head())\n\t{\n\t\tmeta = list_entry(pos, KafkaMeta, list);\n\t\tif (strcmp(meta->get_topic(), topic) == 0)\n\t\t\treturn meta;\n\t}\n\n\treturn NULL;\n}\n\nKafkaSnappySink::~KafkaSnappySink()\n{\n}\n\nKafkaSnappySource::~KafkaSnappySource()\n{\n}\n\n} /* namespace protocol */\n" }, { "path": "src/protocol/KafkaDataTypes.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n\t http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#ifndef _KAFKA_DATATYPES_H_\n#define _KAFKA_DATATYPES_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"rbtree.h\"\n#include \"kafka_parser.h\"\n\nnamespace protocol\n{\n\ntemplate\nclass KafkaList\n{\npublic:\n\tKafkaList()\n\t{\n\t\tthis->t_list = new struct list_head;\n\n\t\tINIT_LIST_HEAD(this->t_list);\n\t\tthis->ref = new std::atomic(1);\n\t\tthis->curpos = this->t_list;\n\t}\n\n\t~KafkaList()\n\t{\n\t\tif (--*this->ref == 0)\n\t\t{\n\t\t\tstruct list_head *pos, *tmp;\n\t\t\tT *t;\n\n\t\t\tlist_for_each_safe(pos, tmp, this->t_list)\n\t\t\t{\n\t\t\t\tt = list_entry(pos, T, list);\n\t\t\t\tlist_del(pos);\n\t\t\t\tdelete t;\n\t\t\t}\n\n\t\t\tdelete this->t_list;\n\t\t\tdelete this->ref;\n\t\t}\n\t}\n\n\tKafkaList(KafkaList&& move)\n\t{\n\t\tthis->t_list = move.t_list;\n\t\tmove.t_list = new struct list_head;\n\t\tINIT_LIST_HEAD(move.t_list);\n\t\tthis->ref = move.ref;\n\t\tthis->curpos = this->t_list;\n\t\tmove.ref = new std::atomic(1);\n\t}\n\n\tKafkaList& operator= (KafkaList&& move)\n\t{\n\t\tif (this != &move)\n\t\t{\n\t\t\tthis->~KafkaList();\n\t\t\tthis->t_list = move.t_list;\n\t\t\tmove.t_list = new struct list_head;\n\t\t\tINIT_LIST_HEAD(move.t_list);\n\t\t\tthis->ref = move.ref;\n\t\t\tthis->curpos = this->t_list;\n\t\t\tmove.ref = new std::atomic(1);\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tKafkaList(const KafkaList& copy)\n\t{\n\t\tthis->ref = copy.ref;\n\t\t++*this->ref;\n\t\tthis->t_list = copy.t_list;\n\t\tthis->curpos = copy.curpos;\n\t}\n\n\tKafkaList& operator= (const KafkaList& copy)\n\t{\n\t\tif (this != ©)\n\t\t{\n\t\t\tthis->~KafkaList();\n\t\t\tthis->ref = copy.ref;\n\t\t\t++*this->ref;\n\t\t\tthis->t_list = copy.t_list;\n\t\t\tthis->curpos = copy.curpos;\n\t\t}\n\t\treturn *this;\n\t}\n\n\tT *add_item(T&& move)\n\t{\n\t\tT *t = new T;\n\n\t\t*t = std::move(move);\n\t\tlist_add_tail(t->get_list(), this->t_list);\n\t\treturn t;\n\t}\n\n\tvoid add_item(T& obj)\n\t{\n\t\tT *t = new T;\n\n\t\t*t = obj;\n\t\tlist_add_tail(t->get_list(), this->t_list);\n\t}\n\n\tstruct list_head *get_head() { return this->t_list; }\n\n\tstruct list_head *get_tail() { return this->t_list->prev; }\n\n\tT *get_first_entry()\n\t{\n\t\tif (this->t_list == this->t_list->next)\n\t\t\treturn NULL;\n\n\t\treturn list_entry(this->t_list->next, T, list);\n\t}\n\n\tT *get_tail_entry()\n\t{\n\t\tif (this->t_list == this->get_tail())\n\t\t\treturn NULL;\n\n\t\treturn list_entry(this->get_tail(), T, list);\n\t}\n\n\tT *get_entry(struct list_head *pos)\n\t{\n\t\treturn list_entry(pos, T, list);\n\t}\n\n\tvoid rewind()\n\t{\n\t\tthis->curpos = this->t_list;\n\t}\n\n\tT *get_next()\n\t{\n\t\tif (this->curpos->next == this->t_list)\n\t\t\treturn NULL;\n\n\t\tthis->curpos = this->curpos->next;\n\t\treturn list_entry(this->curpos, T, list);\n\t}\n\n\tvoid insert_pos(struct list_head *list, struct list_head *pos)\n\t{\n\t\t__list_add(list, pos, pos->next);\n\t}\n\n\tvoid del_cur()\n\t{\n\t\tassert(this->curpos != this->t_list);\n\t\tthis->curpos = this->curpos->prev;\n\t\tlist_del(this->curpos->next);\n\t}\n\nprivate:\n\tstruct list_head *t_list;\n\tstd::atomic *ref;\n\tstruct list_head *curpos;\n};\n\ntemplate\nclass KafkaMap\n{\npublic:\n\tKafkaMap()\n\t{\n\t\tthis->t_map = new struct rb_root;\n\t\tthis->t_map->rb_node = NULL;\n\n\t\tthis->ref = new std::atomic(1);\n\t}\n\n\t~KafkaMap()\n\t{\n\t\tif (--*this->ref == 0)\n\t\t{\n\t\t\tT *t;\n\t\t\twhile (this->t_map->rb_node)\n\t\t\t{\n\t\t\t\tt = rb_entry(this->t_map->rb_node, T, rb);\n\t\t\t\trb_erase(this->t_map->rb_node, this->t_map);\n\t\t\t\tdelete t;\n\t\t\t}\n\n\t\t\tdelete this->t_map;\n\t\t\tdelete this->ref;\n\t\t}\n\t}\n\n\tKafkaMap(const KafkaMap& copy)\n\t{\n\t\tthis->ref = copy.ref;\n\t\t++*this->ref;\n\t\tthis->t_map = copy.t_map;\n\t}\n\n\tKafkaMap& operator= (const KafkaMap& copy)\n\t{\n\t\tif (this != ©)\n\t\t{\n\t\t\tthis->~KafkaMap();\n\t\t\tthis->ref = copy.ref;\n\t\t\t++*this->ref;\n\t\t\tthis->t_map = copy.t_map;\n\t\t}\n\t\treturn *this;\n\t}\n\n\tT *find_item(const T& v) const\n\t{\n\t\trb_node **p = &this->t_map->rb_node;\n\t\tT *t;\n\n\t\twhile (*p)\n\t\t{\n\t\t\tt = rb_entry(*p, T, rb);\n\n\t\t\tif (v < *t)\n\t\t\t\tp = &(*p)->rb_left;\n\t\t\telse if (v > *t)\n\t\t\t\tp = &(*p)->rb_right;\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\treturn *p ? t : NULL;\n\t}\n\n\tvoid add_item(T& obj)\n\t{\n\t\trb_node **p = &this->t_map->rb_node;\n\t\trb_node *parent = NULL;\n\t\tT *t;\n\n\t\twhile (*p)\n\t\t{\n\t\t\tparent = *p;\n\t\t\tt = rb_entry(*p, T, rb);\n\n\t\t\tif (obj < *t)\n\t\t\t\tp = &(*p)->rb_left;\n\t\t\telse if (obj > *t)\n\t\t\t\tp = &(*p)->rb_right;\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\tif (*p == NULL)\n\t\t{\n\t\t\tT *nt = new T;\n\n\t\t\t*nt = obj;\n\t\t\trb_link_node(nt->get_rb(), parent, p);\n\t\t\trb_insert_color(nt->get_rb(), this->t_map);\n\t\t}\n\t}\n\n\tT *find_item(int id) const\n\t{\n\t\trb_node **p = &this->t_map->rb_node;\n\t\tT *t;\n\n\t\twhile (*p)\n\t\t{\n\t\t\tt = rb_entry(*p, T, rb);\n\n\t\t\tif (id < t->get_id())\n\t\t\t\tp = &(*p)->rb_left;\n\t\t\telse if (id > t->get_id())\n\t\t\t\tp = &(*p)->rb_right;\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\treturn *p ? t : NULL;\n\t}\n\n\tvoid add_item(T& obj, int id)\n\t{\n\t\trb_node **p = &this->t_map->rb_node;\n\t\trb_node *parent = NULL;\n\t\tT *t;\n\n\t\twhile (*p)\n\t\t{\n\t\t\tparent = *p;\n\t\t\tt = rb_entry(*p, T, rb);\n\n\t\t\tif (id < t->get_id())\n\t\t\t\tp = &(*p)->rb_left;\n\t\t\telse if (id > t->get_id())\n\t\t\t\tp = &(*p)->rb_right;\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\tif (*p == NULL)\n\t\t{\n\t\t\tT *nt = new T;\n\n\t\t\t*nt = obj;\n\t\t\trb_link_node(nt->get_rb(), parent, p);\n\t\t\trb_insert_color(nt->get_rb(), this->t_map);\n\t\t}\n\t}\n\n\tT *get_first_entry()\n\t{\n\t\tstruct rb_node *p = rb_first(this->t_map);\n\t\treturn rb_entry(p, T, rb);\n\t}\n\n\tT *get_tail_entry()\n\t{\n\t\tstruct rb_node *p = rb_last(this->t_map);\n\t\treturn rb_entry(p, T, rb);\n\t}\n\nprivate:\n\tstruct rb_root *t_map;\n\tstd::atomic *ref;\n};\n\nclass KafkaConfig\n{\npublic:\n\tvoid set_produce_timeout(int ms) { this->ptr->produce_timeout = ms; }\n\tint get_produce_timeout() const { return this->ptr->produce_timeout; }\n\n\tvoid set_produce_msg_max_bytes(int bytes)\n\t{\n\t\tthis->ptr->produce_msg_max_bytes = bytes;\n\t}\n\tint get_produce_msg_max_bytes() const\n\t{\n\t\treturn this->ptr->produce_msg_max_bytes;\n\t}\n\n\tvoid set_produce_msgset_cnt(int cnt)\n\t{\n\t\tthis->ptr->produce_msgset_cnt = cnt;\n\t}\n\tint get_produce_msgset_cnt() const\n\t{\n\t\treturn this->ptr->produce_msgset_cnt;\n\t}\n\n\tvoid set_produce_msgset_max_bytes(int bytes)\n\t{\n\t\tthis->ptr->produce_msgset_max_bytes = bytes;\n\t}\n\tint get_produce_msgset_max_bytes() const\n\t{\n\t\treturn this->ptr->produce_msgset_max_bytes;\n\t}\n\n\tvoid set_fetch_timeout(int ms) { this->ptr->fetch_timeout = ms; }\n\tint get_fetch_timeout() const { return this->ptr->fetch_timeout; }\n\n\tvoid set_fetch_min_bytes(int bytes) { this->ptr->fetch_min_bytes = bytes; }\n\tint get_fetch_min_bytes() const { return this->ptr->fetch_min_bytes; }\n\n\tvoid set_fetch_max_bytes(int bytes) { this->ptr->fetch_max_bytes = bytes; }\n\tint get_fetch_max_bytes() const { return this->ptr->fetch_max_bytes; }\n\n\tvoid set_fetch_msg_max_bytes(int bytes) { this->ptr->fetch_msg_max_bytes = bytes; }\n\tint get_fetch_msg_max_bytes() const { return this->ptr->fetch_msg_max_bytes; }\n\n\tvoid set_offset_timestamp(long long tm)\n\t{\n\t\tthis->ptr->offset_timestamp = tm;\n\t}\n\tlong long get_offset_timestamp() const\n\t{\n\t\treturn this->ptr->offset_timestamp;\n\t}\n\n\tvoid set_commit_timestamp(long long commit_timestamp)\n\t{\n\t\tthis->ptr->commit_timestamp = commit_timestamp;\n\t}\n\tlong long get_commit_timestamp() const { return this->ptr->commit_timestamp; }\n\n\tvoid set_session_timeout(int ms) { this->ptr->session_timeout = ms; }\n\tint get_session_timeout() const { return this->ptr->session_timeout; }\n\n\tvoid set_rebalance_timeout(int ms) { this->ptr->rebalance_timeout = ms; }\n\tint get_rebalance_timeout() const { return this->ptr->rebalance_timeout; }\n\n\tvoid set_retention_time_period(long long ms)\n\t{\n\t\tthis->ptr->retention_time_period = ms;\n\t}\n\tlong long get_retention_time_period() const\n\t{\n\t\treturn this->ptr->retention_time_period;\n\t}\n\n\tvoid set_produce_acks(int acks) { this->ptr->produce_acks = acks; }\n\tint get_produce_acks() const { return this->ptr->produce_acks; }\n\n\tvoid set_allow_auto_topic_creation(bool allow_auto_topic_creation)\n\t{\n\t\tthis->ptr->allow_auto_topic_creation = allow_auto_topic_creation;\n\t}\n\tbool get_allow_auto_topic_creation() const\n\t{\n\t\treturn this->ptr->allow_auto_topic_creation;\n\t}\n\n\tvoid set_api_version_request(int api_ver)\n\t{\n\t\tthis->ptr->api_version_request = api_ver;\n\t}\n\tint get_api_version_request() const\n\t{\n\t\treturn this->ptr->api_version_request;\n\t}\n\n\tbool set_broker_version(const char *version)\n\t{\n\t\tchar *p = strdup(version);\n\n\t\tif (!p)\n\t\t\treturn false;\n\n\t\tfree(this->ptr->broker_version);\n\t\tthis->ptr->broker_version = p;\n\t\treturn true;\n\t}\n\tconst char *get_broker_version() const\n\t{\n\t\treturn this->ptr->broker_version;\n\t}\n\n\tvoid set_compress_type(int type) { this->ptr->compress_type = type; }\n\tint get_compress_type() const { return this->ptr->compress_type; }\n\n\tconst char *get_client_id() { return this->ptr->client_id; }\n\tbool set_client_id(const char *client_id)\n\t{\n\t\tchar *p = strdup(client_id);\n\n\t\tif (!p)\n\t\t\treturn false;\n\n\t\tfree(this->ptr->client_id);\n\t\tthis->ptr->client_id = p;\n\t\treturn true;\n\t}\n\n\tbool get_check_crcs() const\n\t{\n\t\treturn this->ptr->check_crcs != 0;\n\t}\n\tvoid set_check_crcs(bool check_crcs)\n\t{\n\t\tthis->ptr->check_crcs = check_crcs;\n\t}\n\n\tint get_offset_store() const\n\t{\n\t\treturn this->ptr->offset_store;\n\t}\n\tvoid set_offset_store(int offset_store)\n\t{\n\t\tthis->ptr->offset_store = offset_store;\n\t}\n\n\tconst char *get_rack_id() const\n\t{\n\t\treturn this->ptr->rack_id;\n\t}\n\tbool set_rack_id(const char *rack_id)\n\t{\n\t\tchar *p = strdup(rack_id);\n\t\tif (!p)\n\t\t\treturn false;\n\n\t\tfree(this->ptr->rack_id);\n\t\tthis->ptr->rack_id = p;\n\t\treturn true;\n\t}\n\n\tconst char *get_sasl_mech() const\n\t{\n\t\treturn this->ptr->mechanisms;\n\t}\n\tbool set_sasl_mech(const char *mechanisms)\n\t{\n\t\tchar *p = strdup(mechanisms);\n\n\t\tif (!p)\n\t\t\treturn false;\n\n\t\tfree(this->ptr->mechanisms);\n\t\tthis->ptr->mechanisms = p;\n\t\tif (kafka_sasl_set_mechanisms(this->ptr) != 0)\n\t\t\treturn false;\n\n\t\treturn true;\n\t}\n\n\tconst char *get_sasl_username() const\n\t{\n\t\treturn this->ptr->username;\n\t}\n\tbool set_sasl_username(const char *username)\n\t{\n\t\treturn kafka_sasl_set_username(username, this->ptr) == 0;\n\t}\n\n\tconst char *get_sasl_password() const\n\t{\n\t\treturn this->ptr->password;\n\t}\n\tbool set_sasl_password(const char *password)\n\t{\n\t\treturn kafka_sasl_set_password(password, this->ptr) == 0;\n\t}\n\n\tstd::string get_sasl_info() const;\n\n\tbool new_client(kafka_sasl_t *sasl)\n\t{\n\t\treturn this->ptr->client_new(this->ptr, sasl) == 0;\n\t}\n\npublic:\n\tKafkaConfig()\n\t{\n\t\tthis->ptr = new kafka_config_t;\n\t\tkafka_config_init(this->ptr);\n\t\tthis->ref = new std::atomic(1);\n\t}\n\n\tvirtual ~KafkaConfig()\n\t{\n\t\tif (--*this->ref == 0)\n\t\t{\n\t\t\tkafka_config_deinit(this->ptr);\n\t\t\tdelete this->ptr;\n\t\t\tdelete this->ref;\n\t\t}\n\t}\n\n\tKafkaConfig(KafkaConfig&& move)\n\t{\n\t\tthis->ptr = move.ptr;\n\t\tthis->ref = move.ref;\n\t\tmove.ptr = new kafka_config_t;\n\t\tkafka_config_init(move.ptr);\n\t\tmove.ref = new std::atomic(1);\n\t}\n\n\tKafkaConfig& operator= (KafkaConfig&& move)\n\t{\n\t\tif (this != &move)\n\t\t{\n\t\t\tthis->~KafkaConfig();\n\t\t\tthis->ptr = move.ptr;\n\t\t\tthis->ref = move.ref;\n\t\t\tmove.ptr = new kafka_config_t;\n\t\t\tkafka_config_init(move.ptr);\n\t\t\tmove.ref = new std::atomic(1);\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tKafkaConfig(const KafkaConfig& copy)\n\t{\n\t\tthis->ptr = copy.ptr;\n\t\tthis->ref = copy.ref;\n\t\t++*this->ref;\n\t}\n\n\tKafkaConfig& operator= (const KafkaConfig& copy)\n\t{\n\t\tif (this != ©)\n\t\t{\n\t\t\tthis->~KafkaConfig();\n\t\t\tthis->ptr = copy.ptr;\n\t\t\tthis->ref = copy.ref;\n\t\t\t++*this->ref;\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tkafka_config_t *get_raw_ptr() { return this->ptr; }\n\nprivate:\n\tkafka_config_t *ptr;\n\tstd::atomic *ref;\n};\n\nclass KafkaRecord\n{\npublic:\n\tbool set_key(const void *key, size_t key_len)\n\t{\n\t\treturn kafka_record_set_key(key, key_len, this->ptr) == 0;\n\t}\n\tvoid get_key(const void **key, size_t *key_len) const\n\t{\n\t\t*key = this->ptr->key;\n\t\t*key_len = this->ptr->key_len;\n\t}\n\tsize_t get_key_len() const { return this->ptr->key_len; }\n\n\tbool set_value(const void *value, size_t value_len)\n\t{\n\t\treturn kafka_record_set_value(value, value_len, this->ptr) == 0;\n\t}\n\tvoid get_value(const void **value, size_t *value_len) const\n\t{\n\t\t*value = this->ptr->value;\n\t\t*value_len = this->ptr->value_len;\n\t}\n\tsize_t get_value_len() const { return this->ptr->value_len; }\n\n\tbool add_header_pair(const void *key, size_t key_len,\n\t\t\t\t\t\t const void *val, size_t val_len);\n\n\tbool add_header_pair(const std::string& key, const std::string& val);\n\n\tstruct list_head *get_list() { return &this->list; }\n\n\tconst char *get_topic() const { return this->ptr->toppar->topic_name; }\n\n\tvoid set_status(short err) { this->ptr->status = err; }\n\tshort get_status() const { return this->ptr->status; }\n\n\tint get_partition() const { return this->ptr->toppar->partition; }\n\n\tlong long get_offset() const { return this->ptr->offset; }\n\tvoid set_offset(long long offset) { this->ptr->offset = offset; }\n\n\tlong long get_timestamp() const { return this->ptr->timestamp; }\n\tvoid set_timestamp(long long timestamp) { this->ptr->timestamp = timestamp; }\n\npublic:\n\tKafkaRecord()\n\t{\n\t\tthis->ptr = new kafka_record_t;\n\t\tkafka_record_init(this->ptr);\n\t\tthis->ref = new std::atomic(1);\n\t}\n\n\t~KafkaRecord()\n\t{\n\t\tif (--*this->ref == 0)\n\t\t{\n\t\t\tkafka_record_deinit(this->ptr);\n\t\t\tdelete this->ptr;\n\t\t\tdelete this->ref;\n\t\t}\n\t}\n\n\tKafkaRecord(KafkaRecord&& move)\n\t{\n\t\tthis->ptr = move.ptr;\n\t\tthis->ref = move.ref;\n\t\tmove.ptr = new kafka_record_t;\n\t\tkafka_record_init(move.ptr);\n\t\tmove.ref = new std::atomic(1);\n\t}\n\n\tKafkaRecord& operator= (KafkaRecord&& move)\n\t{\n\t\tif (this != &move)\n\t\t{\n\t\t\tthis->~KafkaRecord();\n\t\t\tthis->ptr = move.ptr;\n\t\t\tthis->ref = move.ref;\n\t\t\tmove.ptr = new kafka_record_t;\n\t\t\tkafka_record_init(move.ptr);\n\t\t\tmove.ref = new std::atomic(1);\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tKafkaRecord(const KafkaRecord& copy)\n\t{\n\t\tthis->ptr = copy.ptr;\n\t\tthis->ref = copy.ref;\n\t\t++*this->ref;\n\t}\n\n\tKafkaRecord& operator= (const KafkaRecord& copy)\n\t{\n\t\tif (this != ©)\n\t\t{\n\t\t\tthis->~KafkaRecord();\n\t\t\tthis->ptr = copy.ptr;\n\t\t\tthis->ref = copy.ref;\n\t\t\t++*this->ref;\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tkafka_record_t *get_raw_ptr() const { return this->ptr; }\n\n\tstruct list_head *get_header_list() const { return &this->ptr->header_list; }\n\nprivate:\n\tstruct list_head list;\n\tkafka_record_t *ptr;\n\tstd::atomic *ref;\n\n\tfriend class KafkaMessage;\n\tfriend class KafkaResponse;\n\tfriend class KafkaToppar;\n};\n\n\nclass KafkaMeta;\nclass KafkaBroker;\nclass KafkaToppar;\n\nusing KafkaMetaList = KafkaList;\nusing KafkaBrokerList = KafkaList;\nusing KafkaBrokerMap = KafkaMap;\nusing KafkaTopparList = KafkaList;\nusing KafkaRecordList = KafkaList;\n\nextern KafkaToppar *get_toppar(const char *topic, int partition,\n\t\t\t\t\t\t\t KafkaTopparList *toppar_list);\n\nextern const KafkaMeta *get_meta(const char *topic, KafkaMetaList *meta_list);\n\nclass KafkaToppar\n{\npublic:\n\tbool set_topic_partition(const std::string& topic, int partition)\n\t{\n\t\treturn kafka_topic_partition_set_tp(topic.c_str(), partition,\n\t\t\t\t\t\t\t\t\t\t\tthis->ptr) == 0;\n\t}\n\n\tint get_preferred_read_replica() const\n\t{\n\t\treturn this->ptr->preferred_read_replica;\n\t}\n\n\tbool set_topic(const char *topic)\n\t{\n\t\tthis->ptr->topic_name = strdup(topic);\n\t\treturn this->ptr->topic_name != NULL;\n\t}\n\n\tconst char *get_topic() const { return this->ptr->topic_name; }\n\n\tint get_partition() const { return this->ptr->partition; }\n\n\tlong long get_offset() const { return this->ptr->offset; }\n\tvoid set_offset(long long offset) { this->ptr->offset = offset; }\n\n\tlong long get_offset_timestamp() const { return this->ptr->offset_timestamp; }\n\tvoid set_offset_timestamp(long long tm) { this->ptr->offset_timestamp = tm; }\n\n\tlong long get_high_watermark() const { return this->ptr->high_watermark; }\n\tvoid set_high_watermark(long long offset) const { this->ptr->high_watermark = offset; }\n\n\tlong long get_low_watermark() const { return this->ptr->low_watermark; }\n\tvoid set_low_watermark(long long offset) { this->ptr->low_watermark = offset; }\n\n\tvoid clear_records()\n\t{\n\t\tINIT_LIST_HEAD(&this->ptr->record_list);\n\t\tthis->curpos = &this->ptr->record_list;\n\t\tthis->startpos = this->endpos = this->curpos;\n\t}\n\npublic:\n\tKafkaToppar()\n\t{\n\t\tthis->ptr = new kafka_topic_partition_t;\n\t\tkafka_topic_partition_init(this->ptr);\n\t\tthis->ref = new std::atomic(1);\n\t\tthis->curpos = &this->ptr->record_list;\n\t\tthis->startpos = this->endpos = this->curpos;\n\t}\n\n\t~KafkaToppar();\n\n\tKafkaToppar(KafkaToppar&& move)\n\t{\n\t\tthis->ptr = move.ptr;\n\t\tthis->ref = move.ref;\n\t\tmove.ptr = new kafka_topic_partition_t;\n\t\tkafka_topic_partition_init(move.ptr);\n\t\tmove.ref = new std::atomic(1);\n\t\tthis->curpos = &this->ptr->record_list;\n\t\tthis->startpos = this->endpos = this->curpos;\n\t}\n\n\tKafkaToppar& operator= (KafkaToppar&& move)\n\t{\n\t\tif (this != &move)\n\t\t{\n\t\t\tthis->~KafkaToppar();\n\t\t\tthis->ptr = move.ptr;\n\t\t\tthis->ref = move.ref;\n\t\t\tmove.ptr = new kafka_topic_partition_t;\n\t\t\tkafka_topic_partition_init(move.ptr);\n\t\t\tmove.ref = new std::atomic(1);\n\t\t\tthis->curpos = &this->ptr->record_list;\n\t\t\tthis->startpos = this->endpos = this->curpos;\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tKafkaToppar(const KafkaToppar& copy)\n\t{\n\t\tthis->ptr = copy.ptr;\n\t\tthis->ref = copy.ref;\n\t\t++*this->ref;\n\t\tthis->curpos = copy.curpos;\n\t\tthis->startpos = copy.startpos;\n\t\tthis->endpos = copy.endpos;\n\t}\n\n\tKafkaToppar& operator= (const KafkaToppar& copy)\n\t{\n\t\tif (this != ©)\n\t\t{\n\t\t\tthis->~KafkaToppar();\n\t\t\tthis->ptr = copy.ptr;\n\t\t\tthis->ref = copy.ref;\n\t\t\t++*this->ref;\n\t\t\tthis->curpos = copy.curpos;\n\t\t\tthis->startpos = copy.startpos;\n\t\t\tthis->endpos = copy.endpos;\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tkafka_topic_partition_t *get_raw_ptr() { return this->ptr; }\n\n\tstruct list_head *get_list() { return &this->list; }\n\n\tstruct list_head *get_record() { return &this->ptr->record_list; }\n\n\tvoid set_error(short error) { this->ptr->error = error; }\n\tint get_error() const { return this->ptr->error; }\n\n\tvoid add_record(KafkaRecord&& record)\n\t{\n\t\tKafkaRecord *tmp = new KafkaRecord;\n\n\t\t*tmp =std::move(record);\n\t\tlist_add_tail(tmp->get_list(), &this->ptr->record_list);\n\t}\n\n\tvoid record_rewind()\n\t{\n\t\tthis->curpos = &this->ptr->record_list;\n\t}\n\n\tKafkaRecord *get_record_next()\n\t{\n\t\tif (this->curpos->next == &this->ptr->record_list)\n\t\t\treturn NULL;\n\n\t\tthis->curpos = this->curpos->next;\n\t\treturn list_entry(this->curpos, KafkaRecord, list);\n\t}\n\n\tvoid del_record_cur()\n\t{\n\t\tassert(this->curpos != &this->ptr->record_list);\n\t\tthis->curpos = this->curpos->prev;\n\t\tlist_del(this->curpos->next);\n\t}\n\n\tstruct list_head *get_record_startpos()\n\t{\n\t\treturn this->startpos;\n\t}\n\n\tstruct list_head *get_record_endpos()\n\t{\n\t\treturn this->endpos;\n\t}\n\n\tvoid restore_record_curpos()\n\t{\n\t\tthis->curpos = this->startpos;\n\t\tthis->endpos = NULL;\n\t}\n\n\tvoid save_record_startpos()\n\t{\n\t\tthis->startpos = this->curpos;\n\t}\n\n\tvoid save_record_endpos()\n\t{\n\t\tthis->endpos = this->curpos->next;\n\t}\n\n\tbool record_reach_end()\n\t{\n\t\treturn this->endpos == &this->ptr->record_list;\n\t}\n\n\tvoid record_rollback()\n\t{\n\t\tthis->curpos = this->curpos->prev;\n\t}\n\n\tKafkaRecord *get_tail_record()\n\t{\n\t\tif (&this->ptr->record_list != this->ptr->record_list.prev)\n\t\t{\n\t\t\treturn (KafkaRecord *)list_entry(this->ptr->record_list.prev,\n\t\t\t\t\tKafkaRecord, list);\n\t\t}\n\t\telse\n\t\t{\n\t\t\treturn NULL;\n\t\t}\n\t}\n\nprivate:\n\tstruct list_head list;\n\tkafka_topic_partition_t *ptr;\n\tstd::atomic *ref;\n\tstruct list_head *curpos;\n\tstruct list_head *startpos;\n\tstruct list_head *endpos;\n\n\tfriend class KafkaMessage;\n\tfriend class KafkaRequest;\n\tfriend class KafkaResponse;\n\tfriend class KafkaList;\n\tfriend class KafkaCgroup;\n\n\tfriend KafkaToppar *get_toppar(const char *topic, int partition,\n\t\t\t\t\t\t\t\t KafkaTopparList *toppar_list);\n};\n\nclass KafkaBroker\n{\npublic:\n\tconst char *get_host() const\n\t{\n\t\treturn this->ptr->host;\n\t}\n\n\tint get_port() const\n\t{\n\t\treturn this->ptr->port;\n\t}\n\n\tstd::string get_host_port() const\n\t{\n\t\tstd::string host_port(this->ptr->host);\n\t\thost_port += \":\";\n\t\thost_port += std::to_string(this->ptr->port);\n\t\treturn host_port;\n\t}\n\n\tint get_error()\n\t{\n\t\treturn this->ptr->error;\n\t}\n\npublic:\n\tKafkaBroker()\n\t{\n\t\tthis->ptr = new kafka_broker_t;\n\t\tkafka_broker_init(this->ptr);\n\t\tthis->ref = new std::atomic(1);\n\t}\n\n\t~KafkaBroker()\n\t{\n\t\tif (this->ref && --*this->ref == 0)\n\t\t{\n\t\t\tkafka_broker_deinit(this->ptr);\n\t\t\tdelete this->ptr;\n\t\t\tdelete this->ref;\n\t\t}\n\t}\n\n\tKafkaBroker(kafka_broker_t *ptr)\n\t{\n\t\tthis->ptr = ptr;\n\t\tthis->ref = NULL;\n\t}\n\n\tKafkaBroker(KafkaBroker&& move)\n\t{\n\t\tthis->ptr = move.ptr;\n\t\tthis->ref = move.ref;\n\t\tmove.ptr = new kafka_broker_t;\n\t\tkafka_broker_init(move.ptr);\n\t\tmove.ref = new std::atomic(1);\n\t}\n\n\tKafkaBroker& operator= (KafkaBroker&& move)\n\t{\n\t\tif (this != &move)\n\t\t{\n\t\t\tthis->~KafkaBroker();\n\t\t\tthis->ptr = move.ptr;\n\t\t\tthis->ref = move.ref;\n\t\t\tmove.ptr = new kafka_broker_t;\n\t\t\tkafka_broker_init(move.ptr);\n\t\t\tmove.ref = new std::atomic(1);\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tKafkaBroker(const KafkaBroker& copy)\n\t{\n\t\tthis->ptr = copy.ptr;\n\t\tthis->ref = copy.ref;\n\t\tif (this->ref)\n\t\t\t++*this->ref;\n\t}\n\n\tKafkaBroker& operator= (const KafkaBroker& copy)\n\t{\n\t\tif (this != ©)\n\t\t{\n\t\t\tthis->~KafkaBroker();\n\t\t\tthis->ptr = copy.ptr;\n\t\t\tthis->ref = copy.ref;\n\t\t\tif (this->ref)\n\t\t\t\t++*this->ref;\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tbool operator< (const KafkaBroker& broker) const\n\t{\n\t\treturn this->get_host_port() < broker.get_host_port();\n\t}\n\n\tbool operator> (const KafkaBroker& broker) const\n\t{\n\t\treturn this->get_host_port() > broker.get_host_port();\n\t}\n\n\tkafka_broker_t *get_raw_ptr() const { return this->ptr; }\n\n\tstruct list_head *get_list() { return &this->list; }\n\n\tstruct rb_node *get_rb() { return &this->rb; }\n\n\tint get_node_id() const { return this->ptr->node_id; }\n\n\tint get_id () const { return this->ptr->node_id; }\n\nprivate:\n\tstruct list_head list;\n\tstruct rb_node rb;\n\tkafka_broker_t *ptr;\n\tstd::atomic *ref;\n\n\tfriend class KafkaList;\n\tfriend class KafkaMap;\n};\n\nclass KafkaMeta\n{\npublic:\n\tconst char *get_topic() const { return this->ptr->topic_name; }\n\n\tconst kafka_broker_t *get_broker(int partition) const\n\t{\n\t\tif (partition >= this->ptr->partition_elements)\n\t\t\treturn NULL;\n\n\t\tfor (int i = 0; i < this->ptr->partition_elements; ++i)\n\t\t{\n\t\t\tif (partition == this->ptr->partitions[i]->partition_index)\n\t\t\t\treturn &this->ptr->partitions[i]->leader;\n\t\t}\n\n\t\treturn NULL;\n\t}\n\n\tkafka_partition_t **get_partitions() const { return this->ptr->partitions; }\n\n\tint get_partition_elements() const { return this->ptr->partition_elements; }\n\npublic:\n\tKafkaMeta()\n\t{\n\t\tthis->ptr = new kafka_meta_t;\n\t\tkafka_meta_init(this->ptr);\n\t\tthis->ref = new std::atomic(1);\n\t}\n\n\t~KafkaMeta()\n\t{\n\t\tif (--*this->ref == 0)\n\t\t{\n\t\t\tkafka_meta_deinit(this->ptr);\n\t\t\tdelete this->ptr;\n\t\t\tdelete this->ref;\n\t\t}\n\t}\n\n\tKafkaMeta(KafkaMeta&& move)\n\t{\n\t\tthis->ptr = move.ptr;\n\t\tthis->ref = move.ref;\n\t\tmove.ptr = new kafka_meta_t;\n\t\tkafka_meta_init(move.ptr);\n\t\tmove.ref = new std::atomic(1);\n\t}\n\n\tKafkaMeta& operator= (KafkaMeta&& move)\n\t{\n\t\tif (this != &move)\n\t\t{\n\t\t\tthis->~KafkaMeta();\n\t\t\tthis->ptr = move.ptr;\n\t\t\tthis->ref = move.ref;\n\t\t\tmove.ptr = new kafka_meta_t;\n\t\t\tkafka_meta_init(move.ptr);\n\t\t\tmove.ref = new std::atomic(1);\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tKafkaMeta(const KafkaMeta& copy)\n\t{\n\t\tthis->ptr = copy.ptr;\n\t\tthis->ref = copy.ref;\n\t\t++*this->ref;\n\t}\n\n\tKafkaMeta& operator= (const KafkaMeta& copy)\n\t{\n\t\tif (this != ©)\n\t\t{\n\t\t\tthis->~KafkaMeta();\n\t\t\tthis->ptr = copy.ptr;\n\t\t\tthis->ref = copy.ref;\n\t\t\t++*this->ref;\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tkafka_meta_t *get_raw_ptr() { return this->ptr; }\n\n\tbool set_topic(const std::string& topic)\n\t{\n\t\treturn kafka_meta_set_topic(topic.c_str(), this->ptr) == 0;\n\t}\n\n\tstruct list_head *get_list() { return &this->list; }\n\n\tint get_error() const { return this->ptr->error; }\n\n\tbool create_partitions(int partition_cnt);\n\n\tbool create_replica_nodes(int partition_idx, int replica_cnt)\n\t{\n\t\tint *replica_nodes = (int *)malloc(replica_cnt * 4);\n\n\t\tif (!replica_nodes)\n\t\t\treturn false;\n\n\t\tthis->ptr->partitions[partition_idx]->replica_nodes = replica_nodes;\n\t\tthis->ptr->partitions[partition_idx]->replica_node_elements = replica_cnt;\n\t\treturn true;\n\t}\n\n\tbool create_isr_nodes(int partition_idx, int isr_cnt)\n\t{\n\t\tint *isr_nodes = (int *)malloc(isr_cnt * 4);\n\n\t\tif (!isr_nodes)\n\t\t\treturn false;\n\n\t\tthis->ptr->partitions[partition_idx]->isr_nodes = isr_nodes;\n\t\tthis->ptr->partitions[partition_idx]->isr_node_elements = isr_cnt;\n\t\treturn true;\n\t}\n\nprivate:\n\tstruct list_head list;\n\tkafka_meta_t *ptr;\n\tstd::atomic *ref;\n\n\tfriend class KafkaList;\n\n\tfriend const KafkaMeta *get_meta(const char *topic, KafkaMetaList *meta_list);\n};\n\nclass KafkaMetaSubscriber\n{\npublic:\n\tvoid set_meta(KafkaMeta *meta)\n\t{\n\t\tthis->meta = meta;\n\t}\n\n\tconst KafkaMeta *get_meta() const\n\t{\n\t\treturn this->meta;\n\t}\n\n\tvoid add_member(kafka_member_t *member)\n\t{\n\t\tthis->member_vec.push_back(member);\n\t}\n\n\tconst std::vector *get_member() const\n\t{\n\t\treturn &this->member_vec;\n\t}\n\n\tvoid sort_by_member();\n\nprivate:\n\tKafkaMeta *meta;\n\tstd::vector member_vec;\n};\n\nclass KafkaCgroup\n{\npublic:\n\tconst char *get_group() const { return this->ptr->group_name; }\n\n\tconst char *get_protocol_type() const { return this->ptr->protocol_type; }\n\n\tconst char *get_protocol_name() const { return this->ptr->protocol_name; }\n\n\tint get_generation_id() const { return this->ptr->generation_id; }\n\n\tconst char *get_member_id() const { return this->ptr->member_id; }\n\npublic:\n\tKafkaCgroup();\n\n\t~KafkaCgroup();\n\n\tKafkaCgroup(KafkaCgroup&& move);\n\n\tKafkaCgroup& operator= (KafkaCgroup&& move);\n\n\tKafkaCgroup(const KafkaCgroup& copy);\n\n\tKafkaCgroup& operator= (const KafkaCgroup& copy);\n\n\tkafka_cgroup_t *get_raw_ptr() { return this->ptr; }\n\n\tvoid set_group(const std::string& group)\n\t{\n\t\tchar *p = new char[group.size() + 1];\n\t\tstrncpy(p, group.c_str(), group.size());\n\t\tp[group.size()] = 0;\n\t\tthis->ptr->group_name = p;\n\t}\n\n\tstruct list_head *get_list() { return &this->list; }\n\n\tint get_error() const { return this->ptr->error; }\n\n\tbool is_leader() const\n\t{\n\t\treturn strcmp(this->ptr->leader_id, this->ptr->member_id) == 0;\n\t}\n\n\tstruct list_head *get_group_protocol()\n\t{\n\t\treturn &this->ptr->group_protocol_list;\n\t}\n\n\tvoid set_member_id(const char *p)\n\t{\n\t\tfree(this->ptr->member_id);\n\t\tthis->ptr->member_id = strdup(p);\n\t}\n\n\tvoid set_error(short error) { this->ptr->error = error; }\n\n\tbool create_members(int member_cnt);\n\n\tkafka_member_t **get_members() const { return this->ptr->members; }\n\n\tint get_member_elements() { return this->ptr->member_elements; }\n\n\tvoid add_assigned_toppar(KafkaToppar *toppar);\n\n\tstruct list_head *get_assigned_toppar_list()\n\t{\n\t\treturn &this->ptr->assigned_toppar_list;\n\t}\n\n\tKafkaToppar *get_assigned_toppar_by_pos(struct list_head *pos)\n\t{\n\t\treturn list_entry(pos, KafkaToppar, list);\n\t}\n\n\tvoid assigned_toppar_rewind();\n\n\tKafkaToppar *get_assigned_toppar_next();\n\n\tvoid del_assigned_toppar_cur();\n\n\tKafkaBroker *get_coordinator()\n\t{\n\t\tif (!this->coordinator)\n\t\t\tthis->coordinator = new KafkaBroker(&this->ptr->coordinator);\n\n\t\treturn this->coordinator;\n\t}\n\n\tint run_assignor(KafkaMetaList *meta_list, const char *protocol_name);\n\n\tvoid add_subscriber(KafkaMetaList *meta_list,\n\t\t\t\t\t\tstd::vector *subscribers);\n\n\tstatic int kafka_range_assignor(kafka_member_t **members,\n\t\t\t\t\t\t\t\t\tint member_elements,\n\t\t\t\t\t\t\t\t\tvoid *meta_topic);\n\n\tstatic int kafka_roundrobin_assignor(kafka_member_t **members,\n\t\t\t\t\t\t\t\t\t\t int member_elements,\n\t\t\t\t\t\t\t\t\t\t void *meta_topic);\n\nprivate:\n\tstruct list_head list;\n\tkafka_cgroup_t *ptr;\n\tstd::atomic *ref;\n\tstruct list_head *curpos;\n\tKafkaBroker *coordinator;\n};\n\nclass KafkaBlock\n{\npublic:\n\tKafkaBlock()\n\t{\n\t\tthis->ptr = new kafka_block_t;\n\t\tkafka_block_init(this->ptr);\n\t}\n\n\t~KafkaBlock()\n\t{\n\t\tkafka_block_deinit(this->ptr);\n\t\tdelete this->ptr;\n\t}\n\n\tKafkaBlock(KafkaBlock&& move)\n\t{\n\t\tthis->ptr = move.ptr;\n\t\tmove.ptr = new kafka_block_t;\n\t\tkafka_block_init(move.ptr);\n\t}\n\n\tKafkaBlock& operator= (KafkaBlock&& move)\n\t{\n\t\tif (this != &move)\n\t\t{\n\t\t\tthis->~KafkaBlock();\n\t\t\tthis->ptr = move.ptr;\n\t\t\tmove.ptr = new kafka_block_t;\n\t\t\tkafka_block_init(move.ptr);\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tkafka_block_t *get_raw_ptr() const { return this->ptr; }\n\n\tstruct list_head *get_list() { return &this->list; }\n\n\tvoid *get_block() const { return this->ptr->buf; }\n\n\tsize_t get_len() const { return this->ptr->len; }\n\n\tbool allocate(size_t len)\n\t{\n\t\tvoid *p = malloc(len);\n\n\t\tif (!p)\n\t\t\treturn false;\n\n\t\tfree(this->ptr->buf);\n\t\tthis->ptr->buf = p;\n\t\tthis->ptr->len = len;\n\t\treturn true;\n\t}\n\n\tbool reallocate(size_t len)\n\t{\n\t\tvoid *p = realloc(this->ptr->buf, len);\n\n\t\tif (p)\n\t\t{\n\t\t\tthis->ptr->buf = p;\n\t\t\tthis->ptr->len = len;\n\t\t\treturn true;\n\t\t}\n\t\telse\n\t\t\treturn false;\n\t}\n\n\tbool set_block(void *buf, size_t len)\n\t{\n\t\tif (!this->allocate(len))\n\t\t\treturn false;\n\n\t\tmemcpy(this->ptr->buf, buf, len);\n\t\treturn true;\n\t}\n\n\tvoid set_block_nocopy(void *buf, size_t len)\n\t{\n\t\tthis->ptr->buf = buf;\n\t\tthis->ptr->len = len;\n\t}\n\n\tvoid set_len(size_t len) { this->ptr->len = len; }\n\nprivate:\n\tstruct list_head list;\n\tkafka_block_t *ptr;\n\n\tfriend class KafkaBuffer;\n\tfriend class KafkaList;\n};\n\nclass KafkaBuffer\n{\npublic:\n\tKafkaBuffer()\n\t{\n\t\tthis->insert_pos = NULL;\n\t\tthis->insert_curpos = NULL;\n\t\tthis->buf_size = 0;\n\t\tthis->inited = false;\n\t\tthis->insert_buf_size = 0;\n\t\tthis->insert_flag = false;\n\t}\n\n\tvoid backup(size_t n)\n\t{\n\t\tthis->buf_size -= n;\n\t}\n\n\tvoid list_splice(KafkaBuffer *buffer);\n\n\tvoid add_item(KafkaBlock block)\n\t{\n\t\tif (this->insert_flag)\n\t\t\tthis->insert_buf_size += block.get_len();\n\n\t\tthis->buf_size += block.get_len();\n\t\tthis->block_list.add_item(std::move(block));\n\t}\n\n\tvoid set_insert_pos()\n\t{\n\t\tthis->insert_pos = this->block_list.get_tail();\n\t\tthis->insert_flag = true;\n\t\tthis->insert_buf_size = 0;\n\t}\n\n\tvoid block_insert_rewind()\n\t{\n\t\tthis->insert_flag = false;\n\t\tthis->insert_curpos = this->insert_pos;\n\t}\n\n\tKafkaBlock *get_block_insert_next()\n\t{\n\t\tif (this->insert_curpos->next == this->block_list.get_head())\n\t\t\treturn NULL;\n\n\t\tthis->insert_curpos = this->insert_curpos->next;\n\t\treturn list_entry(this->insert_curpos, KafkaBlock, list);\n\t}\n\n\tKafkaBlock *get_block_tail()\n\t{\n\t\treturn this->block_list.get_tail_entry();\n\t}\n\n\tvoid insert_list(KafkaBlock *block)\n\t{\n\t\tthis->buf_size += block->get_len();\n\t\tthis->block_list.insert_pos(block->get_list(), this->insert_pos);\n\t\tthis->insert_pos = this->insert_pos->next;\n\t}\n\n\tKafkaBlock *get_block_first()\n\t{\n\t\tthis->block_list.rewind();\n\t\treturn this->block_list.get_next();\n\t}\n\n\tKafkaBlock *get_block_next()\n\t{\n\t\treturn this->block_list.get_next();\n\t}\n\n\tvoid append(const char *bytes, size_t n)\n\t{\n\t\tKafkaBlock block;\n\n\t\tblock.set_block((void *)bytes, n);\n\t\tthis->block_list.add_item(std::move(block));\n\t\tthis->buf_size += n;\n\t}\n\n\tsize_t get_size() const\n\t{\n\t\treturn this->buf_size;\n\t}\n\n\tsize_t peek(const char **buf);\n\n\tlong seek(long offset)\n\t{\n\t\tthis->cur_pos.second += offset;\n\t\treturn offset;\n\t}\n\n\tstruct list_head *get_head()\n\t{\n\t\treturn this->block_list.get_head();\n\t}\n\nprivate:\n\tKafkaList block_list;\n\tstd::pair cur_pos;\n\tstruct list_head *insert_pos;\n\tstruct list_head *insert_curpos;\n\tsize_t buf_size;\n\tsize_t insert_buf_size;\n\tbool inited;\n\tbool insert_flag;\n};\n\nclass KafkaSnappySink : public snappy::Sink\n{\npublic:\n\tKafkaSnappySink(KafkaBuffer *buffer)\n\t{\n\t\tthis->buffer = buffer;\n\t}\n\n\tvirtual void Append(const char *bytes, size_t n)\n\t{\n\t\tthis->buffer->append(bytes, n);\n\t}\n\n\tsize_t size() const\n\t{\n\t\treturn this->buffer->get_size();\n\t}\n\n\tKafkaBuffer *get_buffer() const\n\t{\n\t\treturn buffer;\n\t}\n\nprivate:\n\tKafkaBuffer *buffer;\n\npublic:\n\tvirtual ~KafkaSnappySink();\n};\n\nclass KafkaSnappySource : public snappy::Source\n{\npublic:\n\tKafkaSnappySource(KafkaBuffer *buffer)\n\t{\n\t\tthis->buffer = buffer;\n\t\tthis->buf_size = this->buffer->get_size();\n\t\tthis->pos = 0;\n\t}\n\n\tvirtual size_t Available() const\n\t{\n\t\treturn this->buf_size - this->pos;\n\t}\n\n\tvirtual const char *Peek(size_t *len)\n\t{\n\t\tconst char *pos;\n\n\t\t*len = this->buffer->peek(&pos);\n\t\treturn pos;\n\t}\n\n\tvirtual void Skip(size_t n)\n\t{\n\t\tthis->pos += this->buffer->seek(n);\n\t}\n\nprivate:\n\tKafkaBuffer *buffer;\n\tsize_t buf_size;\n\tsize_t pos;\n\npublic:\n\tvirtual ~KafkaSnappySource();\n};\n\n}\n\n#endif\n" }, { "path": "src/protocol/KafkaMessage.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n\t http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"crc32c.h\"\n#include \"EncodeStream.h\"\n#include \"KafkaMessage.h\"\n\nnamespace protocol\n{\n\n#define CHECK_RET(exp) \\\ndo { \\\n\tint tmp = exp; \\\n\tif (tmp < 0) \\\n\t\treturn tmp; \\\n} while (0)\n\n#ifndef htonll\nstatic uint64_t htonll(uint64_t x)\n{\n\tif (1 == htonl(1))\n\t\treturn x;\n\telse\n\t\treturn ((uint64_t)htonl(x & 0xFFFFFFFF) << 32) + htonl(x >> 32);\n}\n#endif\n\nstatic size_t append_bool(std::string& buf, bool val)\n{\n\tunsigned char v = 0;\n\n\tif (val)\n\t\tv = 1;\n\n\tbuf.append((char *)&v, 1);\n\treturn 1;\n}\n\nstatic size_t append_i8(std::string& buf, int8_t val)\n{\n\tbuf.append((char *)&val, 1);\n\treturn 1;\n}\n\nstatic size_t append_i8(void **buf, int8_t val)\n{\n\t*(char *)*buf = val;\n\t*buf = (char *)*buf + 1;\n\treturn 1;\n}\n\nstatic size_t append_i16(std::string& buf, int16_t val)\n{\n\tint16_t v = htons(val);\n\n\tbuf.append((char *)&v, 2);\n\treturn 2;\n}\n\nstatic size_t append_i32(std::string& buf, int32_t val)\n{\n\tint32_t v = htonl(val);\n\n\tbuf.append((char *)&v, 4);\n\treturn 4;\n}\n\nstatic size_t append_i32(void **buf, int32_t val)\n{\n\tint32_t v = htonl(val);\n\n\t*(int32_t *)*buf = v;\n\t*buf = (int32_t *)*buf + 1;\n\treturn 4;\n}\n\nstatic size_t append_i64(std::string& buf, int64_t val)\n{\n\tint64_t v = htonll(val);\n\n\tbuf.append((char *)&v, 8);\n\treturn 8;\n}\n\nstatic size_t append_i64(void **buf, int64_t val)\n{\n\tint64_t v = htonll(val);\n\n\t*(int64_t *)*buf = v;\n\t*buf = (int64_t *)*buf + 1;\n\treturn 8;\n}\n\nstatic size_t append_string(std::string& buf, const char *str, size_t len)\n{\n\tappend_i16(buf, len);\n\tbuf.append(str, len);\n\treturn len + 2;\n}\n\nstatic size_t append_string(std::string& buf, const char *str)\n{\n\tif (!str)\n\t\treturn append_string(buf, \"\", 0);\n\n\treturn append_string(buf, str, strlen(str));\n}\n\nstatic size_t append_string_raw(std::string& buf, const char *str, size_t len)\n{\n\tbuf.append(str, len);\n\treturn len;\n}\n\nstatic size_t append_nullable_string(std::string& buf, const char *str, size_t len)\n{\n\tif (len == 0)\n\t\treturn append_i16(buf, -1);\n\telse\n\t\treturn append_string(buf, str, len);\n}\n\nstatic size_t append_string_raw(void **buf, const char *str, size_t len)\n{\n\tmemcpy(*buf, str, len);\n\t*buf = (char *)*buf + len;\n\treturn len;\n}\n\nstatic size_t append_string_raw(void **buf, const std::string& str)\n{\n\treturn append_string_raw(buf, str.c_str(), str.size());\n}\n\nstatic size_t append_bytes(std::string& buf, const char *str, size_t len)\n{\n\tappend_i32(buf, len);\n\tbuf.append(str, len);\n\treturn 4 + len;\n}\n\nstatic size_t append_bytes(std::string& buf, const std::string& str)\n{\n\treturn append_bytes(buf, str.c_str(), str.size());\n}\n\nstatic size_t append_bytes(void **buf, const char *str, size_t len)\n{\n\t*((int32_t *)*buf) = htonl(len);\n\t*buf = (int32_t *)*buf + 1;\n\n\tmemcpy(*buf, str, len);\n\t*buf = (char *)*buf + len;\n\n\treturn len + 2;\n}\n\nstatic size_t append_nullable_bytes(void **buf, const char *str, size_t len)\n{\n\tif (len == 0)\n\t\treturn append_i32(buf, -1);\n\telse\n\t\treturn append_bytes(buf, str, len);\n}\n\nstatic size_t append_varint_u64(std::string& buf, uint64_t num)\n{\n\tsize_t len = 0;\n\n\tdo\n\t{\n\t\tunsigned char v = (num & 0x7f) | (num > 0x7f ? 0x80 : 0);\n\t\tbuf.append((char *)&v, 1);\n\t\tnum >>= 7;\n\t\t++len;\n\t} while (num);\n\treturn len;\n}\n\nstatic inline size_t append_varint_i64(std::string& buf, int64_t num)\n{\n\treturn append_varint_u64(buf, (num << 1) ^ (num >> 63));\n}\n\nstatic inline size_t append_varint_i32(std::string& buf, int32_t num)\n{\n\treturn append_varint_i64(buf, num);\n}\n\nstatic size_t append_compact_string(std::string& buf, const char *str)\n{\n\tif (!str || str[0] == '\\0')\n\t\tappend_string(buf, \"\");\n\n\tsize_t len = strlen(str);\n\tsize_t r = append_varint_u64(buf, len + 1);\n\tappend_string_raw(buf, str, len);\n\treturn r + len;\n}\n\nstatic inline int parse_i8(void **buf, size_t *size, int8_t *val)\n{\n\tif (*size >= 1)\n\t{\n\t\t*val = *(int8_t *)*buf;\n\t\t*size -= sizeof(int8_t);\n\t\t*buf = (int8_t *)*buf + 1;\n\t\treturn 0;\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstatic inline int parse_i16(void **buf, size_t *size, int16_t *val)\n{\n\tif (*size >= 2)\n\t{\n\t\t*val = ntohs(*(int16_t *)*buf);\n\t\t*size -= sizeof(int16_t);\n\t\t*buf = (int16_t *)*buf + 1;\n\t\treturn 0;\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstatic inline int parse_i32(void **buf, size_t *size, int32_t *val)\n{\n\tif (*size >= 4)\n\t{\n\t\t*val = ntohl(*(int32_t *)*buf);\n\t\t*size -= sizeof(int32_t);\n\t\t*buf = (int32_t *)*buf + 1;\n\t\treturn 0;\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstatic inline int parse_i64(void **buf, size_t *size, int64_t *val)\n{\n\tif (*size >= 8)\n\t{\n\t\t*val = htonll(*(int64_t *)*buf);\n\t\t*size -= sizeof(int64_t);\n\t\t*buf = (int64_t *)*buf + 1;\n\t\treturn 0;\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstatic int parse_string(void **buf, size_t *size, std::string& str);\n\nstatic int parse_string(void **buf, size_t *size, char **str);\n\nstatic int parse_bytes(void **buf, size_t *size, std::string& str);\n\nstatic int parse_bytes(void **buf, size_t *size,\n\t\t\t\t\t void **str, size_t *str_len);\n\nstatic int parse_varint_u64(void **buf, size_t *size, uint64_t *val);\n\nstatic int parse_varint_i64(void **buf, size_t *size, int64_t *val)\n{\n\tuint64_t n;\n\tint ret = parse_varint_u64(buf, size, &n);\n\n\tif (ret == 0)\n\t\t*val = (int64_t)(n >> 1) ^ -(int64_t)(n & 1);\n\n\treturn ret;\n}\n\nstatic int parse_varint_i32(void **buf, size_t *size, int32_t *val)\n{\n\tint64_t v = 0;\n\n\tif (parse_varint_i64(buf, size, &v) < 0)\n\t\treturn -1;\n\n\t*val = (int32_t)v;\n\treturn 0;\n}\n\nstatic const LZ4F_preferences_t kPrefs =\n{\n\t.frameInfo = {LZ4F_default, LZ4F_blockIndependent, },\n\t.compressionLevel = 0,\n};\n\nstatic int compress_buf(KafkaBlock *block, int compress_type, void *env)\n{\n\tz_stream *c_stream;\n\tsize_t total_in = 0, gzip_in, bound_size;\n\tKafkaBuffer *snappy_buffer;\n\tKafkaBlock nblock;\n\tLZ4F_errorCode_t lz4_r;\n\tLZ4F_cctx *lz4_cctx;\n\tZSTD_CStream *zstd_cctx;\n\tsize_t zstd_r;\n\tZSTD_outBuffer out;\n\tZSTD_inBuffer in;\n\n\tswitch (compress_type)\n\t{\n\tcase Kafka_Gzip:\n\t\tc_stream = static_cast(env);\n\t\tgzip_in = c_stream->total_in;\n\t\twhile (total_in < block->get_len())\n\t\t{\n\t\t\tif (c_stream->avail_in == 0)\n\t\t\t{\n\t\t\t\tc_stream->next_in = (Bytef *)block->get_block();\n\t\t\t\tc_stream->avail_in = block->get_len() - total_in;\n\t\t\t}\n\n\t\t\tif (c_stream->avail_out == 0)\n\t\t\t{\n\t\t\t\tbound_size = compressBound(c_stream->avail_in);\n\t\t\t\tif (!nblock.allocate(bound_size))\n\t\t\t\t{\n\t\t\t\t\tdelete c_stream;\n\t\t\t\t\treturn -1;\n\t\t\t\t}\n\n\t\t\t\tc_stream->next_out = (Bytef *)nblock.get_block();\n\t\t\t\tc_stream->avail_out = bound_size;\n\t\t\t}\n\n\t\t\tif (deflate(c_stream, Z_NO_FLUSH) != Z_OK)\n\t\t\t{\n\t\t\t\tdelete c_stream;\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\ttotal_in += c_stream->total_in - gzip_in;\n\t\t\tgzip_in = c_stream->total_in;\n\t\t}\n\n\t\t*block = std::move(nblock);\n\t\tbreak;\n\n\tcase Kafka_Snappy:\n\t\tsnappy_buffer = static_cast(env);\n\t\tsnappy_buffer->append((const char *)block->get_block(), block->get_len());\n\t\tbreak;\n\n\tcase Kafka_Lz4:\n\t\tlz4_cctx = static_cast(env);\n\t\tbound_size = LZ4F_compressBound(block->get_len(), &kPrefs);\n\t\tif (!nblock.allocate(bound_size))\n\t\t{\n\t\t\tLZ4F_freeCompressionContext(lz4_cctx);\n\t\t\treturn -1;\n\t\t}\n\n\t\tlz4_r = LZ4F_compressUpdate(lz4_cctx,\n\t\t\t\t\t\t\t\t\tnblock.get_block(), nblock.get_len(),\n\t\t\t\t\t\t\t\t\tblock->get_block(), block->get_len(),\n\t\t\t\t\t\t\t\t\tNULL);\n\n\t\tif (LZ4F_isError(lz4_r))\n\t\t{\n\t\t\tLZ4F_freeCompressionContext(lz4_cctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tnblock.set_len(lz4_r);\n\t\t*block = std::move(nblock);\n\t\tbreak;\n\n\tcase Kafka_Zstd:\n\t\tzstd_cctx = static_cast(env);\n\t\tbound_size = ZSTD_compressBound(block->get_len());\n\t\tif (!nblock.allocate(bound_size))\n\t\t{\n\t\t\tZSTD_freeCStream(zstd_cctx);\n\t\t\treturn -1;\n\t\t}\n\n\t\tin.src = block->get_block();\n\t\tin.pos = 0;\n\t\tin.size = block->get_len();\n\t\tout.dst = nblock.get_block();\n\t\tout.pos = 0;\n\t\tout.size = nblock.get_len();\n\t\tzstd_r = ZSTD_compressStream(zstd_cctx, &out, &in);\n\t\tif (ZSTD_isError(zstd_r) || in.pos < in.size)\n\t\t{\n\t\t\tZSTD_freeCStream(zstd_cctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tnblock.set_len(out.pos);\n\t\t*block = std::move(nblock);\n\t\tbreak;\n\n\tdefault:\n\t\treturn 0;\n\t}\n\n\treturn 0;\n}\n\nstatic int gzip_decompress(void *compressed, size_t n, KafkaBlock *block)\n{\n\tfor (int pass = 1; pass <= 2; pass++)\n\t{\n\t\tz_stream strm = {0};\n\t\tgz_header hdr;\n\t\tchar buf[512];\n\t\tchar *p;\n\t\tint len;\n\t\tint r;\n\n\t\tif ((r = inflateInit2(&strm, 15 | 32)) != Z_OK)\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tstrm.next_in = (Bytef *)compressed;\n\t\tstrm.avail_in = n;\n\n\t\tif ((r = inflateGetHeader(&strm, &hdr)) != Z_OK)\n\t\t{\n\t\t\tinflateEnd(&strm);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (pass == 1)\n\t\t{\n\t\t\tp = buf;\n\t\t\tlen = sizeof(buf);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tp = (char *)block->get_block();\n\t\t\tlen = block->get_len();\n\t\t}\n\n\t\tdo\n\t\t{\n\t\t\tstrm.next_out = (unsigned char *)p;\n\t\t\tstrm.avail_out = len;\n\n\t\t\tr = inflate(&strm, Z_NO_FLUSH);\n\t\t\tswitch (r)\n\t\t\t{\n\t\t\tcase Z_STREAM_ERROR:\n\t\t\tcase Z_NEED_DICT:\n\t\t\tcase Z_DATA_ERROR:\n\t\t\tcase Z_MEM_ERROR:\n\t\t\t\tinflateEnd(&strm);\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tif (pass == 2)\n\t\t\t{\n\t\t\t\tp += len - strm.avail_out;\n\t\t\t\tlen -= len - strm.avail_out;\n\t\t\t}\n\n\t\t} while (strm.avail_out == 0 && r != Z_STREAM_END);\n\n\n\t\tif (pass == 1)\n\t\t{\n\t\t\tif (!block->allocate(strm.total_out))\n\t\t\t{\n\t\t\t\tinflateEnd(&strm);\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t}\n\n\t\tinflateEnd(&strm);\n\n\t\tif (strm.total_in != n || r != Z_STREAM_END)\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nstatic int kafka_snappy_java_uncompress(const char *inbuf, size_t inlen, KafkaBlock *block)\n{\n\tchar *obuf = NULL;\n\n\tfor (int pass = 1; pass <= 2; pass++)\n\t{\n\t\tssize_t off = 0;\n\t\tssize_t uoff = 0;\n\n\t\twhile (off + 4 <= (ssize_t)inlen)\n\t\t{\n\t\t\tuint32_t clen;\n\t\t\tsize_t ulen;\n\n\t\t\tmemcpy(&clen, inbuf + off, 4);\n\t\t\tclen = ntohl(clen);\n\t\t\toff += 4;\n\n\t\t\tif (clen > inlen - off)\n\t\t\t{\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tif (snappy_uncompressed_length(inbuf + off, clen, &ulen) != SNAPPY_OK)\n\t\t\t{\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tif (pass == 1)\n\t\t\t{\n\t\t\t\toff += clen;\n\t\t\t\tuoff += ulen;\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tsize_t n = block->get_len() - uoff;\n\n\t\t\tif (snappy_uncompress(inbuf + off, clen, obuf + uoff, &n) != SNAPPY_OK)\n\t\t\t{\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\toff += clen;\n\t\t\tuoff += ulen;\n\t\t}\n\n\t\tif (off != (ssize_t)inlen)\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (pass == 1)\n\t\t{\n\t\t\tif (uoff <= 0)\n\t\t\t{\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tif (!block->allocate(uoff))\n\t\t\t\treturn -1;\n\n\t\t\tobuf = (char *)block->get_block();\n\t\t}\n\t\telse\n\t\t\tblock->set_len(uoff);\n\t}\n\n\treturn 0;\n}\n\nstatic int snappy_decompress(void *buf, size_t n, KafkaBlock *block)\n{\n\tconst char *inbuf = (const char *)buf;\n\tsize_t inlen = n;\n\tstatic const unsigned char snappy_java_magic[] = {\n\t\t0x82, 'S','N','A','P','P','Y', 0\n\t};\n\tstatic const size_t snappy_java_hdrlen = 8 + 4 + 4;\n\n\tif (!memcmp(buf, snappy_java_magic, 8))\n\t{\n\t\tinbuf = inbuf + snappy_java_hdrlen;\n\t\tinlen -= snappy_java_hdrlen;\n\t\treturn kafka_snappy_java_uncompress(inbuf, inlen, block);\n\t}\n\telse\n\t{\n\t\tsize_t uncompressed_len;\n\n\t\tif (snappy_uncompressed_length(inbuf, n, &uncompressed_len) != SNAPPY_OK)\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (!block->allocate(uncompressed_len))\n\t\t\treturn -1;\n\n\t\tsize_t nn = block->get_len();\n\n\t\treturn snappy_uncompress(inbuf, n, (char *)block->get_block(), &nn);\n\t}\n}\n\nstatic int lz4_decompress(void *buf, size_t n, KafkaBlock *block)\n{\n\tLZ4F_errorCode_t code;\n\tLZ4F_decompressionContext_t dctx;\n\tLZ4F_frameInfo_t fi;\n\tsize_t in_sz, out_sz;\n\tsize_t in_off, out_off;\n\tsize_t r;\n\tsize_t uncompressed_size;\n\tsize_t outlen;\n\tchar *out = NULL;\n\tsize_t inlen = n;\n\n\tconst char *inbuf = (const char *)buf;\n\n\tcode = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);\n\tif (LZ4F_isError(code))\n\t{\n\t\tcode = LZ4F_freeDecompressionContext(dctx);\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tin_sz = n;\n\tr = LZ4F_getFrameInfo(dctx, &fi, (const void *)buf, &in_sz);\n\tif (LZ4F_isError(r))\n\t{\n\t\tcode = LZ4F_freeDecompressionContext(dctx);\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tif (fi.contentSize == 0 || fi.contentSize > inlen * 255)\n\t\tuncompressed_size = inlen * 4;\n\telse\n\t\tuncompressed_size = (size_t)fi.contentSize;\n\n\tif (!block->allocate(uncompressed_size))\n\t{\n\t\tcode = LZ4F_freeDecompressionContext(dctx);\n\t\treturn -1;\n\t}\n\n\tout = (char *)block->get_block();\n\toutlen = block->get_len();\n\tin_off = in_sz;\n\tout_off = 0;\n\twhile (in_off < inlen)\n\t{\n\t\tout_sz = outlen - out_off;\n\t\tin_sz = inlen - in_off;\n\t\tr = LZ4F_decompress(dctx, out + out_off, &out_sz,\n\t\t\t\t\t\t\tinbuf + in_off, &in_sz, NULL);\n\t\tif (LZ4F_isError(r))\n\t\t{\n\t\t\tcode = LZ4F_freeDecompressionContext(dctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (!(out_off + out_sz <= outlen && in_off + in_sz <= inlen))\n\t\t{\n\t\t\tcode = LZ4F_freeDecompressionContext(dctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tout_off += out_sz;\n\t\tin_off += in_sz;\n\t\tif (r == 0)\n\t\t\tbreak;\n\n\t\tif (out_off == outlen)\n\t\t{\n\t\t\tsize_t extra = outlen * 3 / 4;\n\n\t\t\tif (!block->reallocate(outlen + extra))\n\t\t\t{\n\t\t\t\tcode = LZ4F_freeDecompressionContext(dctx);\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tout = (char *)block->get_block();\n\t\t\toutlen += extra;\n\t\t}\n\t}\n\n\n\tif (in_off < inlen)\n\t{\n\t\tcode = LZ4F_freeDecompressionContext(dctx);\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tLZ4F_freeDecompressionContext(dctx);\n\treturn 0;\n}\n\nstatic int zstd_decompress(void *buf, size_t n, KafkaBlock *block)\n{\n\tunsigned long long out_bufsize = ZSTD_getFrameContentSize(buf, n);\n\n\tswitch (out_bufsize)\n\t{\n\tcase ZSTD_CONTENTSIZE_UNKNOWN:\n\t\tout_bufsize = n * 2;\n\t\tbreak;\n\n\tcase ZSTD_CONTENTSIZE_ERROR:\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\twhile (1)\n\t{\n\t\tsize_t ret;\n\n\t\tif (!block->allocate(out_bufsize))\n\t\t\treturn -1;\n\n\t\tret = ZSTD_decompress(block->get_block(), out_bufsize,\n\t\t\t\t\t\t\t buf, n);\n\n\t\tif (!ZSTD_isError(ret))\n\t\t\treturn 0;\n\n\t\tif (ZSTD_getErrorCode(ret) == ZSTD_error_dstSize_tooSmall)\n\t\t{\n\t\t\tout_bufsize += out_bufsize * 2;\n\t\t}\n\t\telse\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\t}\n}\n\nstatic int uncompress_buf(void *buf, size_t size, KafkaBlock *block,\n\t\t\t\t\t\t int compress_type)\n{\n\tswitch(compress_type)\n\t{\n\tcase Kafka_Gzip:\n\t\treturn gzip_decompress(buf, size, block);\n\n\tcase Kafka_Snappy:\n\t\treturn snappy_decompress(buf, size, block);\n\n\tcase Kafka_Lz4:\n\t\treturn lz4_decompress(buf, size, block);\n\n\tcase Kafka_Zstd:\n\t\treturn zstd_decompress(buf, size, block);\n\n\tdefault:\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n}\n\nstatic int append_message_set(KafkaBlock *block,\n\t\t\t\t\t\t\t const KafkaRecord *record,\n\t\t\t\t\t\t\t int offset, int msg_version,\n\t\t\t\t\t\t\t const KafkaConfig& config, void *env,\n\t\t\t\t\t\t\t int cur_msg_size)\n{\n\tconst void *key;\n\tsize_t key_len;\n\trecord->get_key(&key, &key_len);\n\n\tconst void *value;\n\tsize_t value_len;\n\trecord->get_value(&value, &value_len);\n\n\tint message_size = 4 + 1 + 1 + 4 + 4 + key_len + value_len;\n\n\tif (msg_version == 1)\n\t\tmessage_size += 8;\n\n\tint max_msg_size = std::min(config.get_produce_msgset_max_bytes(),\n\t\t\t\t\t\t\t\tconfig.get_produce_msg_max_bytes());\n\tif (message_size + 8 + 4 + cur_msg_size > max_msg_size)\n\t\treturn 1;\n\n\tif (!block->allocate(message_size + 8 + 4))\n\t\treturn -1;\n\n\tvoid *cur = block->get_block();\n\n\tappend_i64(&cur, offset);\n\tappend_i32(&cur, message_size);\n\n\tint crc_32 = crc32(0, NULL, 0);\n\n\tappend_i32(&cur, crc_32); //need update\n\tappend_i8(&cur, msg_version);\n\tappend_i8(&cur, 0);\n\n\tif (msg_version == 1)\n\t\tappend_i64(&cur, record->get_timestamp());\n\n\tappend_bytes(&cur, (const char *)key, key_len);\n\tappend_nullable_bytes(&cur, (const char *)value, value_len);\n\n\tchar *crc_buf = (char *)block->get_block() + 8 + 4;\n\n\tcrc_32 = crc32(crc_32, (Bytef *)(crc_buf + 4), message_size - 4);\n\t*(uint32_t *)crc_buf = htonl(crc_32);\n\n\tif (compress_buf(block, config.get_compress_type(), env) < 0)\n\t\treturn -1;\n\n\treturn 0;\n}\n\nstatic int append_batch_record(KafkaBlock *block,\n\t\t\t\t\t\t\t const KafkaRecord *record,\n\t\t\t\t\t\t\t int offset, const KafkaConfig& config,\n\t\t\t\t\t\t\t int64_t first_timestamp, void *env,\n\t\t\t\t\t\t\t int cur_msg_size)\n{\n\tconst void *key;\n\tsize_t key_len;\n\trecord->get_key(&key, &key_len);\n\n\tconst void *value;\n\tsize_t value_len;\n\trecord->get_value(&value, &value_len);\n\n\tstd::string klen_str;\n\tstd::string vlen_str;\n\tstd::string timestamp_delta_str;\n\tint64_t timestamp_delta = record->get_timestamp() - first_timestamp;\n\n\tappend_varint_i64(timestamp_delta_str, timestamp_delta);\n\n\tstd::string offset_delta_str;\n\tappend_varint_i64(offset_delta_str, offset);\n\n\tif (key_len > 0)\n\t\tappend_varint_i32(klen_str, (int32_t)key_len);\n\telse\n\t\tappend_varint_i32(klen_str, (int32_t)-1);\n\n\tif (value)\n\t\tappend_varint_i32(vlen_str, (int32_t)value_len);\n\telse\n\t\tappend_varint_i32(vlen_str, -1);\n\n\tstruct list_head *pos;\n\tkafka_record_header_t *header;\n\tstd::string hdr_str;\n\tint hdr_cnt = 0;\n\n\tlist_for_each(pos, record->get_header_list())\n\t{\n\t\theader = list_entry(pos, kafka_record_header_t, list);\n\t\tappend_varint_i32(hdr_str, (int32_t)header->key_len);\n\t\tappend_string_raw(hdr_str, (const char *)header->key, header->key_len);\n\t\tappend_varint_i32(hdr_str, (int32_t)header->value_len);\n\t\tappend_string_raw(hdr_str, (const char *)header->value, header->value_len);\n\t\t++hdr_cnt;\n\t}\n\n\tstd::string hdr_cnt_str;\n\tappend_varint_i32(hdr_cnt_str, hdr_cnt);\n\n\tint length = 1 + timestamp_delta_str.size() + offset_delta_str.size() +\n\t\tklen_str.size() + key_len + vlen_str.size() + value_len +\n\t\thdr_cnt_str.size() + hdr_str.size();\n\n\tstd::string length_str;\n\tappend_varint_i32(length_str, length);\n\n\tint max_msg_size = std::min(config.get_produce_msgset_max_bytes(),\n\t\t\t\t\t\t\t\tconfig.get_produce_msg_max_bytes());\n\tif ((int)(length + length_str.size() + cur_msg_size) > max_msg_size)\n\t\treturn 1;\n\n\tif (!block->allocate(length + length_str.size()))\n\t\treturn false;\n\n\tvoid *cur = block->get_block();\n\n\tappend_string_raw(&cur, length_str);\n\tappend_i8(&cur, 0);\n\tappend_string_raw(&cur, timestamp_delta_str);\n\tappend_string_raw(&cur, offset_delta_str);\n\tappend_string_raw(&cur, klen_str);\n\n\tif (key_len > 0)\n\t\tappend_string_raw(&cur, (const char *)key, key_len);\n\n\tappend_string_raw(&cur, vlen_str);\n\tif (value_len > 0)\n\t\tappend_string_raw(&cur, (const char *)value, value_len);\n\n\tappend_string_raw(&cur, hdr_cnt_str);\n\tif (hdr_cnt > 0)\n\t\tappend_string_raw(&cur, hdr_str);\n\n\tif (compress_buf(block, config.get_compress_type(), env) < 0)\n\t\treturn -1;\n\n\treturn 0;\n}\n\nstatic int append_record(KafkaBlock *block,\n\t\t\t\t\t\t const KafkaRecord *record,\n\t\t\t\t\t\t int offset, int msg_version,\n\t\t\t\t\t\t const KafkaConfig& config,\n\t\t\t\t\t\t int64_t first_timestamp, void *env,\n\t\t\t\t\t\t int cur_msg_size)\n{\n\tif (config.get_produce_msgset_cnt() < offset)\n\t\treturn 1;\n\n\tint ret = 0;\n\n\tswitch (msg_version)\n\t{\n\tcase 0:\n\tcase 1:\n\t\tret = append_message_set(block, record, offset, msg_version,\n\t\t\t\t\t\t\t\t config, env, cur_msg_size);\n\t\tbreak;\n\n\tcase 2:\n\t\tret = append_batch_record(block, record, offset, config,\n\t\t\t\t\t\t\t\t first_timestamp, env, cur_msg_size);\n\t\tbreak;\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\treturn ret;\n}\n\nstatic int parse_string(void **buf, size_t *size, std::string& str)\n{\n\tif (*size >= 2)\n\t{\n\t\tint16_t len;\n\n\t\tif (parse_i16(buf, size, &len) >= 0)\n\t\t{\n\t\t\tif (len >= -1)\n\t\t\t{\n\t\t\t\tif (len == -1)\n\t\t\t\t\tlen = 0;\n\n\t\t\t\tif (*size >= (size_t)len)\n\t\t\t\t{\n\t\t\t\t\tstr.assign((char *)*buf, len);\n\t\t\t\t\t*size -= len;\n\t\t\t\t\t*buf = (char *)*buf + len;\n\t\t\t\t\treturn 0;\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\t*buf = (char *)*buf - 2;\n\t\t\t\t\t*size += 2;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstatic int parse_string(void **buf, size_t *size, char **str)\n{\n\tif (*size >= 2)\n\t{\n\t\tint16_t len;\n\n\t\tif (parse_i16(buf, size, &len) >= 0)\n\t\t{\n\t\t\tif (len >= -1)\n\t\t\t{\n\t\t\t\tif (len == -1)\n\t\t\t\t\tlen = 0;\n\n\t\t\t\tif (*size >= (size_t)len)\n\t\t\t\t{\n\t\t\t\t\tchar *p = (char *)malloc(len + 1);\n\n\t\t\t\t\tif (!p)\n\t\t\t\t\t{\n\t\t\t\t\t\t*buf = (char *)*buf - 2;\n\t\t\t\t\t\t*size += 2;\n\t\t\t\t\t\treturn -1;\n\t\t\t\t\t}\n\n\t\t\t\t\tfree(*str);\n\t\t\t\t\tmemcpy((void *)p, *buf, len);\n\t\t\t\t\tp[len] = 0;\n\t\t\t\t\t*size -= len;\n\t\t\t\t\t*buf = (char *)*buf + len;\n\t\t\t\t\t*str = p;\n\n\t\t\t\t\treturn 0;\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\t*buf = (char *)*buf - 2;\n\t\t\t\t\t*size += 2;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstatic int parse_bytes(void **buf, size_t *size, std::string& str)\n{\n\tif (*size >= 4)\n\t{\n\t\tint32_t len;\n\n\t\tif (parse_i32(buf, size, &len) >= 0)\n\t\t{\n\t\t\tif (len == -1)\n\t\t\t\tlen = 0;\n\n\t\t\tif (*size >= (size_t)len)\n\t\t\t{\n\t\t\t\tstr.assign((char *)*buf, len);\n\t\t\t\t*size -= len;\n\t\t\t\t*buf = (char *)*buf + len;\n\n\t\t\t\treturn 0;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t*buf = (char *)*buf - 4;\n\t\t\t\t*size += 4;\n\t\t\t}\n\t\t}\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstatic int parse_bytes(void **buf, size_t *size,\n\t\t\t\t\t void **str, size_t *str_len)\n{\n\tif (*size >= 4)\n\t{\n\t\tint32_t len;\n\n\t\tif (parse_i32(buf, size, &len) >= 0)\n\t\t{\n\t\t\tif (len == -1)\n\t\t\t\tlen = 0;\n\n\t\t\tif (*size >= (size_t)len)\n\t\t\t{\n\t\t\t\t*str = *buf;\n\t\t\t\t*str_len = len;\n\t\t\t\t*size -= len;\n\t\t\t\t*buf = (char *)*buf + len;\n\n\t\t\t\treturn 0;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t*buf = (char *)*buf - 4;\n\t\t\t\t*size += 4;\n\t\t\t}\n\t\t}\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstatic int parse_varint_u64(void **buf, size_t *size, uint64_t *val)\n{\n\tsize_t off = 0;\n\tuint64_t num = 0;\n\tint shift = 0;\n\tsize_t org_size = *size;\n\n\tdo\n\t{\n\t\tif (*size == 0)\n\t\t{\n\t\t\t*size = org_size;\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1; /* Underflow */\n\t\t}\n\n\t\tnum |= (uint64_t)(((char *)(*buf))[(int)off] & 0x7f) << shift;\n\t\tshift += 7;\n\t} while (((char *)(*buf))[(int)off++] & 0x80);\n\n\t*val = num;\n\t*buf = (char *)(*buf) + off;\n\t*size -= off;\n\treturn 0;\n}\n\nint KafkaMessage::parse_message_set(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t\tbool check_crcs, int msg_vers,\n\t\t\t\t\t\t\t\t\tstruct list_head *record_list,\n\t\t\t\t\t\t\t\t\tKafkaBuffer *uncompressed,\n\t\t\t\t\t\t\t\t\tKafkaToppar *toppar)\n{\n\tint64_t offset;\n\tint32_t message_size;\n\tint32_t crc;\n\n\tif (parse_i64(buf, size, &offset) < 0)\n\t\treturn -1;\n\n\tif (parse_i32(buf, size, &message_size) < 0)\n\t\treturn -1;\n\n\tif (*size < (size_t)(message_size - 8))\n\t\treturn 1;\n\n\tif (parse_i32(buf, size, &crc) < 0)\n\t\treturn -1;\n\n\tif (check_crcs)\n\t{\n\t\tint32_t crc_32 = crc32(0, NULL, 0);\n\t\tcrc_32 = crc32(crc_32, (Bytef *)*buf, message_size - 4);\n\t\tif (crc_32 != crc)\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\tint8_t magic;\n\tint8_t attributes;\n\n\tif (parse_i8(buf, size, &magic) < 0)\n\t\treturn -1;\n\n\tif (parse_i8(buf, size, &attributes) < 0)\n\t\treturn -1;\n\n\tint64_t timestamp = -1;\n\tif (msg_vers == 1 && parse_i64(buf, size, ×tamp) < 0)\n\t\treturn -1;\n\n\tvoid *key;\n\tsize_t key_len;\n\tif (parse_bytes(buf, size, &key, &key_len) < 0)\n\t\treturn -1;\n\n\tvoid *payload;\n\tsize_t payload_len;\n\n\tif (parse_bytes(buf, size, &payload, &payload_len) < 0)\n\t\treturn -1;\n\n\tif (offset >= toppar->get_offset())\n\t{\n\t\tint compress_type = attributes & 3;\n\t\tif (compress_type == 0)\n\t\t{\n\t\t\tKafkaRecord *kafka_record = new KafkaRecord;\n\t\t\tkafka_record_t *record = kafka_record->get_raw_ptr();\n\t\t\trecord->key = key;\n\t\t\trecord->key_len = key_len;\n\t\t\trecord->timestamp = timestamp;\n\t\t\trecord->offset = offset;\n\t\t\trecord->toppar = toppar->get_raw_ptr();\n\t\t\trecord->key_is_moved = 1;\n\t\t\trecord->value_is_moved = 1;\n\t\t\trecord->value = payload;\n\t\t\trecord->value_len = payload_len;\n\t\t\tlist_add_tail(kafka_record->get_list(), record_list);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tKafkaBlock block;\n\t\t\tif (uncompress_buf(payload, payload_len, &block, compress_type) < 0)\n\t\t\t\treturn -1;\n\n\t\t\tstruct list_head *record_head = record_list->prev;\n\t\t\tvoid *uncompressed_ptr = block.get_block();\n\t\t\tsize_t uncompressed_len = block.get_len();\n\t\t\tparse_message_set(&uncompressed_ptr, &uncompressed_len, check_crcs,\n\t\t\t\t\t\t\tmsg_vers, record_list, uncompressed, toppar);\n\n\t\t\tuncompressed->add_item(std::move(block));\n\n\t\t\tif (msg_vers == 1)\n\t\t\t{\n\t\t\t\tstruct list_head *pos;\n\t\t\t\tKafkaRecord *record;\n\t\t\t\tint n = 0;\n\n\t\t\t\tfor (pos = record_head->next; pos != record_list; pos = pos->next)\n\t\t\t\t\tn++;\n\n\t\t\t\tfor (pos = record_head->next; pos != record_list; pos = pos->next)\n\t\t\t\t{\n\t\t\t\t\tint64_t fix_offset;\n\n\t\t\t\t\trecord = list_entry(pos, KafkaRecord, list);\n\t\t\t\t\tfix_offset = offset + record->get_offset() - n + 1;\n\t\t\t\t\trecord->set_offset(fix_offset);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif (*size > 0)\n\t{\n\t\treturn parse_message_set(buf, size, check_crcs, msg_vers,\n\t\t\t\t\t\t\t\t record_list, uncompressed, toppar);\n\t}\n\n\treturn 0;\n}\n\nstatic int parse_varint_bytes(void **buf, size_t *size,\n\t\t\t\t\t\t\t void **str, size_t *str_len)\n{\n\tint64_t len = 0;\n\n\tif (parse_varint_i64(buf, size, &len) >= 0)\n\t{\n\t\tif (len >= -1)\n\t\t{\n\t\t\tif (len <= 0)\n\t\t\t{\n\t\t\t\t*str = NULL;\n\t\t\t\t*str_len = 0;\n\t\t\t\treturn 0;\n\t\t\t}\n\n\t\t\tif ((int64_t)*size >= len)\n\t\t\t{\n\t\t\t\t*str = *buf;\n\t\t\t\t*str_len = (size_t)len;\n\t\t\t\t*size -= len;\n\t\t\t\t*buf = (char *)*buf + len;\n\t\t\t\treturn 0;\n\t\t\t}\n\t\t}\n\t}\n\n\terrno = EBADMSG;\n\treturn -1;\n}\n\nstruct KafkaBatchRecordHeader\n{\n\tint64_t base_offset;\n\tint32_t length;\n\tint32_t partition_leader_epoch;\n\tint8_t\tmagic;\n\tint32_t crc;\n\tint16_t attributes;\n\tint32_t last_offset_delta;\n\tint64_t base_timestamp;\n\tint64_t max_timestamp;\n\tint64_t produce_id;\n\tint16_t producer_epoch;\n\tint32_t base_sequence;\n\tint32_t record_count;\n};\n\nint KafkaMessage::parse_message_record(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t\t kafka_record_t *record)\n{\n\tint64_t length;\n\tint8_t attributes;\n\tint64_t timestamp_delta;\n\tint64_t offset_delta;\n\tint32_t hdr_size;\n\n\tif (parse_varint_i64(buf, size, &length) < 0)\n\t\treturn -1;\n\n\tif (parse_i8(buf, size, &attributes) < 0)\n\t\treturn -1;\n\n\tif (parse_varint_i64(buf, size, ×tamp_delta) < 0)\n\t\treturn -1;\n\n\tif (parse_varint_i64(buf, size, &offset_delta) < 0)\n\t\treturn -1;\n\n\trecord->timestamp += timestamp_delta;\n\trecord->offset += offset_delta;\n\n\tif (parse_varint_bytes(buf, size, &record->key, &record->key_len) < 0)\n\t\treturn -1;\n\n\tif (parse_varint_bytes(buf, size, &record->value, &record->value_len) < 0)\n\t\treturn -1;\n\n\tif (parse_varint_i32(buf, size, &hdr_size) < 0)\n\t\treturn -1;\n\n\tfor (int i = 0; i < hdr_size; ++i)\n\t{\n\t\tkafka_record_header_t *header;\n\n\t\theader = (kafka_record_header_t *)malloc(sizeof(kafka_record_header_t));\n\t\tif (!header)\n\t\t\treturn -1;\n\n\t\tkafka_record_header_init(header);\n\t\tif (parse_varint_bytes(buf, size, &header->key, &header->key_len) < 0)\n\t\t{\n\t\t\tfree(header);\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (parse_varint_bytes(buf, size, &header->value, &header->value_len) < 0)\n\t\t{\n\t\t\tkafka_record_header_deinit(header);\n\t\t\tfree(header);\n\t\t\treturn -1;\n\t\t}\n\n\t\theader->key_is_moved = 1;\n\t\theader->value_is_moved = 1;\n\n\t\tlist_add_tail(&header->list, &record->header_list);\n\t}\n\n\treturn record->offset < record->toppar->offset ? 1 : 0;\n}\n\nint KafkaMessage::parse_record_batch(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t\t bool check_crcs,\n\t\t\t\t\t\t\t\t\t struct list_head *record_list,\n\t\t\t\t\t\t\t\t\t KafkaBuffer *uncompressed,\n\t\t\t\t\t\t\t\t\t KafkaToppar *toppar)\n{\n\tKafkaBatchRecordHeader hdr;\n\n\tif (parse_i64(buf, size, &hdr.base_offset) < 0)\n\t\treturn -1;\n\n\tif (parse_i32(buf, size, &hdr.length) < 0)\n\t\treturn -1;\n\n\tif (parse_i32(buf, size, &hdr.partition_leader_epoch) < 0)\n\t\treturn -1;\n\n\tif (parse_i8(buf, size, &hdr.magic) < 0)\n\t\treturn -1;\n\n\tif (parse_i32(buf, size, &hdr.crc) < 0)\n\t\treturn -1;\n\n\tif (check_crcs)\n\t{\n\t\tif (hdr.length > (int)*size + 9)\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif ((int)crc32c(*buf, hdr.length - 9) != hdr.crc)\n\t\t{\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\tif (parse_i16(buf, size, &hdr.attributes) < 0)\n\t\treturn -1;\n\n\tif (parse_i32(buf, size, &hdr.last_offset_delta) < 0)\n\t\treturn -1;\n\n\tif (parse_i64(buf, size, &hdr.base_timestamp) < 0)\n\t\treturn -1;\n\n\tif (parse_i64(buf, size, &hdr.max_timestamp) < 0)\n\t\treturn -1;\n\n\tif (parse_i64(buf, size, &hdr.produce_id) < 0)\n\t\treturn -1;\n\n\tif (parse_i16(buf, size, &hdr.producer_epoch) < 0)\n\t\treturn -1;\n\n\tif (parse_i32(buf, size, &hdr.base_sequence) < 0)\n\t\treturn -1;\n\n\tif (parse_i32(buf, size, &hdr.record_count) < 0)\n\t\treturn -1;\n\n\tif (*size < (size_t)(hdr.length - 61 + 12))\n\t\treturn 1;\n\n\tKafkaBlock block;\n\tint compress_type = hdr.attributes & 7;\n\n\tif (compress_type != 0)\n\t{\n\t\tif (uncompress_buf(*buf, hdr.length - 61 + 12, &block, compress_type) < 0)\n\t\t\treturn -1;\n\n\t\t*buf = (char *)*buf + hdr.length - 61 + 12;\n\t\t*size -= hdr.length - 61 + 12;\n\t}\n\n\tvoid *p = *buf;\n\tsize_t n = *size;\n\n\tif (block.get_len() > 0)\n\t{\n\t\tp = block.get_block();\n\t\tn = block.get_len();\n\t}\n\n\tfor (int i = 0; i < hdr.record_count; ++i)\n\t{\n\t\tKafkaRecord *record = new KafkaRecord;\n\t\trecord->set_offset(hdr.base_offset);\n\t\trecord->set_timestamp(hdr.base_timestamp);\n\t\trecord->get_raw_ptr()->key_is_moved = 1;\n\t\trecord->get_raw_ptr()->value_is_moved = 1;\n\t\trecord->get_raw_ptr()->toppar = toppar->get_raw_ptr();\n\n\t\tswitch (parse_message_record(&p, &n, record->get_raw_ptr()))\n\t\t{\n\t\t\tcase -1:\n\t\t\t\tdelete record;\n\t\t\t\treturn -1;\n\t\t\tcase 0:\n\t\t\t\tlist_add_tail(record->get_list(), record_list);\n\t\t\t\tbreak;\n\t\t\tdefault:\n\t\t\t\tdelete record;\n\t\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (compress_type == 0)\n\t{\n\t\t*buf = p;\n\t\t*size = n;\n\t}\n\n\tif (block.get_len() > 0)\n\t\tuncompressed->add_item(std::move(block));\n\n\treturn 0;\n}\n\nint KafkaMessage::parse_records(void **buf, size_t *size, bool check_crcs,\n\t\t\t\t\t\t\t\tKafkaBuffer *uncompressed, KafkaToppar *toppar)\n{\n\tstruct list_head *record_list = toppar->get_record();\n\tint msg_set_size = 0;\n\n\tif (parse_i32(buf, size, &msg_set_size) < 0)\n\t\treturn -1;\n\n\tif (msg_set_size == 0)\n\t\treturn 0;\n\n\tif (msg_set_size < 0)\n\t\treturn -1;\n\n\tif (*size < 17)\n\t\treturn -1;\n\n\tsize_t msg_size = msg_set_size;\n\n\twhile (msg_size > 16)\n\t{\n\t\tint ret = -1;\n\t\tchar magic = ((char *)(*buf))[16];\n\n\t\tswitch(magic)\n\t\t{\n\t\tcase 0:\n\t\tcase 1:\n\t\t\tret = parse_message_set(buf, &msg_size, check_crcs,\n\t\t\t\t\t\t\t\t\tmagic, record_list,\n\t\t\t\t\t\t\t\t\tuncompressed, toppar);\n\t\t\tbreak;\n\n\t\tcase 2:\n\t\t\tret = parse_record_batch(buf, &msg_size, check_crcs,\n\t\t\t\t\t\t\t\t\t record_list, uncompressed, toppar);\n\t\t\tbreak;\n\n\t\tdefault:\n\t\t\tbreak;\n\t\t}\n\n\t\tif (ret > 0)\n\t\t{\n\t\t\t*size -= msg_set_size;\n\t\t\t*buf = (char *)*buf + msg_size;\n\t\t\treturn 0;\n\t\t}\n\t\telse if (ret < 0)\n\t\t\treturn ret;\n\t}\n\n\t*size -= msg_set_size;\n\t*buf = (char *)*buf + msg_size;\n\treturn 0;\n}\n\nKafkaMessage::KafkaMessage()\n{\n\tthis->parser = new kafka_parser_t;\n\tkafka_parser_init(this->parser);\n\tthis->stream = new EncodeStream;\n\tthis->api_type = Kafka_Unknown;\n\tthis->correlation_id = 0;\n\tthis->cur_size = 0;\n}\n\nKafkaMessage::~KafkaMessage()\n{\n\tif (this->parser)\n\t{\n\t\tkafka_parser_deinit(this->parser);\n\t\tdelete this->parser;\n\t\tdelete this->stream;\n\t}\n}\n\nKafkaMessage::KafkaMessage(KafkaMessage&& msg) :\n\tProtocolMessage(std::move(msg))\n{\n\tthis->parser = msg.parser;\n\tthis->stream = msg.stream;\n\tmsg.parser = NULL;\n\tmsg.stream = NULL;\n\n\tthis->msgbuf = std::move(msg.msgbuf);\n\tthis->headbuf = std::move(msg.headbuf);\n\n\tthis->toppar_list = std::move(msg.toppar_list);\n\tthis->serialized = std::move(msg.serialized);\n\tthis->uncompressed = std::move(msg.uncompressed);\n\n\tthis->api_type = msg.api_type;\n\tmsg.api_type = Kafka_Unknown;\n\n\tthis->compress_env = msg.compress_env;\n\tmsg.compress_env = NULL;\n\n\tthis->cur_size = msg.cur_size;\n\tmsg.cur_size = 0;\n}\n\nKafkaMessage& KafkaMessage::operator= (KafkaMessage &&msg)\n{\n\tif (this != &msg)\n\t{\n\t\t*(ProtocolMessage *)this = std::move(msg);\n\n\t\tif (this->parser)\n\t\t{\n\t\t\tkafka_parser_deinit(this->parser);\n\t\t\tdelete this->parser;\n\t\t\tdelete this->stream;\n\t\t}\n\n\t\tthis->parser = msg.parser;\n\t\tthis->stream = msg.stream;\n\t\tmsg.parser = NULL;\n\t\tmsg.stream = NULL;\n\n\t\tthis->msgbuf = std::move(msg.msgbuf);\n\t\tthis->headbuf = std::move(msg.headbuf);\n\n\t\tthis->toppar_list = std::move(msg.toppar_list);\n\t\tthis->serialized = std::move(msg.serialized);\n\t\tthis->uncompressed = std::move(msg.uncompressed);\n\n\t\tthis->api_type = msg.api_type;\n\t\tmsg.api_type = Kafka_Unknown;\n\n\t\tthis->compress_env = msg.compress_env;\n\t\tmsg.compress_env = NULL;\n\n\t\tthis->cur_size = msg.cur_size;\n\t\tmsg.cur_size = 0;\n\t}\n\n\treturn *this;\n}\n\nint KafkaMessage::encode_message(int api_type, struct iovec vectors[], int max)\n{\n\tconst auto it = this->encode_func_map.find(api_type);\n\n\tif (it == this->encode_func_map.cend())\n\t\treturn -1;\n\n\treturn it->second(vectors, max);\n}\n\nstatic int kafka_api_get_max_ver(int api_type)\n{\n\tswitch (api_type)\n\t{\n\tcase Kafka_Metadata:\n\t\treturn 4;\n\tcase Kafka_Produce:\n\t\treturn 7;\n\tcase Kafka_Fetch:\n\t\treturn 11;\n\tcase Kafka_FindCoordinator:\n\t\treturn 2;\n\tcase Kafka_JoinGroup:\n\t\treturn 5;\n\tcase Kafka_SyncGroup:\n\t\treturn 3;\n\tcase Kafka_Heartbeat:\n\t\treturn 3;\n\tcase Kafka_OffsetFetch:\n\t\treturn 1;\n\tcase Kafka_OffsetCommit:\n\t\treturn 7;\n\tcase Kafka_ListOffsets:\n\t\treturn 1;\n\tcase Kafka_LeaveGroup:\n\t\treturn 1;\n\tcase Kafka_ApiVersions:\n\t\treturn 0;\n\tcase Kafka_SaslHandshake:\n\t\treturn 1;\n\tcase Kafka_SaslAuthenticate:\n\t\treturn 0;\n\tcase Kafka_DescribeGroups:\n\t\treturn 0;\n\tdefault:\n\t\treturn 0;\n\t}\n}\n\nstatic int kafka_get_api_version(const kafka_api_t *api, const KafkaConfig& conf,\n\t\t\t\t\t\t\t\t int api_type, int max_ver, int message_version)\n{\n\tint min_ver = 0;\n\n\tif (api_type == Kafka_Produce)\n\t{\n\t\tif (message_version == 2)\n\t\t\tmin_ver = 3;\n\t\telse if (message_version == 1)\n\t\t\tmin_ver = 1;\n\n\t\tif (conf.get_compress_type() == Kafka_Zstd)\n\t\t\tmin_ver = 7;\n\t}\n\n\treturn kafka_broker_get_api_version(api, api_type, min_ver, max_ver);\n}\n\nint KafkaMessage::encode_head()\n{\n\tif (this->api_type == Kafka_ApiVersions)\n\t\tthis->api_version = 0;\n\telse\n\t{\n\t\tint max_ver = kafka_api_get_max_ver(this->api_type);\n\n\t\tif (this->api->features & KAFKA_FEATURE_MSGVER2)\n\t\t\tthis->message_version = 2;\n\t\telse if (this->api->features & KAFKA_FEATURE_MSGVER1)\n\t\t\tthis->message_version = 1;\n\t\telse\n\t\t\tthis->message_version = 0;\n\n\t\tif (this->config.get_compress_type() == Kafka_Lz4 &&\n\t\t\t!(this->api->features & KAFKA_FEATURE_LZ4))\n\t\t{\n\t\t\tthis->config.set_compress_type(Kafka_NoCompress);\n\t\t}\n\n\t\tif (this->config.get_compress_type() == Kafka_Zstd &&\n\t\t\t!(this->api->features & KAFKA_FEATURE_ZSTD))\n\t\t{\n\t\t\tthis->config.set_compress_type(Kafka_NoCompress);\n\t\t}\n\n\t\tthis->api_version = kafka_get_api_version(this->api, this->config,\n\t\t\t\t\t\t\t\t\t\t\t\t this->api_type, max_ver,\n\t\t\t\t\t\t\t\t\t\t\t\t this->message_version);\n\t}\n\n\tif (this->api_version < 0)\n\t\treturn -1;\n\n\tappend_i32(this->headbuf, 0);\n\tappend_i16(this->headbuf, this->api_type);\n\tappend_i16(this->headbuf, this->api_version);\n\tappend_i32(this->headbuf, this->correlation_id);\n\tappend_string(this->headbuf, this->config.get_client_id());\n\n\treturn 0;\n}\n\nint KafkaMessage::encode(struct iovec vectors[], int max)\n{\n\tif (encode_head() < 0)\n\t\treturn -1;\n\n\tint n = encode_message(this->api_type, vectors + 1, max - 1);\n\tif (n < 0)\n\t\treturn -1;\n\n\tint msg_size = this->headbuf.size() + this->cur_size - 4;\n\t*(int32_t *)this->headbuf.c_str() = htonl(msg_size);\n\n\tvectors[0].iov_base = (void *)this->headbuf.c_str();\n\tvectors[0].iov_len = this->headbuf.size();\n\n\treturn n + 1;\n}\n\nint KafkaMessage::append(const void *buf, size_t *size)\n{\n\tint ret = kafka_parser_append_message(buf, size, this->parser);\n\n\tif (ret >= 0)\n\t{\n\t\tthis->cur_size += *size;\n\t\tif (this->cur_size > this->size_limit)\n\t\t{\n\t\t\terrno = EMSGSIZE;\n\t\t\tret = -1;\n\t\t}\n\t}\n\telse if (ret == -2)\n\t{\n\t\terrno = EBADMSG;\n\t\tret = -1;\n\t}\n\n\treturn ret;\n}\n\nstatic int kafka_compress_prepare(int compress_type, void **env,\n\t\t\t\t\t\t\t\t KafkaBlock *block)\n{\n\tz_stream *c_stream;\n\tKafkaBuffer *snappy_buffer;\n\tsize_t lz4_out_len;\n\tLZ4F_errorCode_t lz4_r;\n\tLZ4F_cctx *lz4_cctx = NULL;\n\tZSTD_CStream *zstd_cctx;\n\tsize_t zstd_r;\n\n\tswitch (compress_type)\n\t{\n\tcase Kafka_Gzip:\n\t\tc_stream = new z_stream;\n\t\tc_stream->zalloc = (alloc_func)0;\n\t\tc_stream->zfree = (free_func)0;\n\t\tc_stream->opaque = (voidpf)0;\n\t\tif (deflateInit2(c_stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16,\n\t\t\t\t\t\t 8, Z_DEFAULT_STRATEGY) != Z_OK)\n\t\t{\n\t\t\tdelete c_stream;\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tc_stream->avail_in = 0;\n\t\tc_stream->avail_out = 0;\n\t\tc_stream->total_in = 0;\n\n\t\t*env = (void *)c_stream;\n\t\tbreak;\n\n\tcase Kafka_Snappy:\n\t\tsnappy_buffer = new KafkaBuffer;\n\t\t*env = (void *)snappy_buffer;\n\t\tbreak;\n\n\tcase Kafka_Lz4:\n\t\tlz4_r = LZ4F_createCompressionContext(&lz4_cctx, LZ4F_VERSION);\n\t\tif (LZ4F_isError(lz4_r))\n\t\t{\n\t\t\tLZ4F_freeCompressionContext(lz4_cctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tlz4_out_len = LZ4F_HEADER_SIZE_MAX;\n\n\t\tif (!block->allocate(lz4_out_len))\n\t\t{\n\t\t\tLZ4F_freeCompressionContext(lz4_cctx);\n\t\t\treturn -1;\n\t\t}\n\n\t\tlz4_r = LZ4F_compressBegin(lz4_cctx, block->get_block(),\n\t\t\t\t\t\t\t\t block->get_len(), &kPrefs);\n\t\tif (LZ4F_isError(lz4_r))\n\t\t{\n\t\t\tLZ4F_freeCompressionContext(lz4_cctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tblock->set_len(lz4_r);\n\n\t\t*env = (void *)lz4_cctx;\n\t\tbreak;\n\n\tcase Kafka_Zstd:\n\t\tzstd_cctx = ZSTD_createCStream();\n\t\tif (!zstd_cctx)\n\t\t\treturn -1;\n\n\t\tzstd_r = ZSTD_initCStream(zstd_cctx, ZSTD_CLEVEL_DEFAULT);\n\t\tif (ZSTD_isError(zstd_r))\n\t\t{\n\t\t\tZSTD_freeCStream(zstd_cctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\t*env = (void *)zstd_cctx;\n\t\tbreak;\n\n\tdefault:\n\t\treturn 0;\n\t}\n\n\treturn 0;\n}\n\nstatic int kafka_compress_finish(int compress_type, void *env,\n\t\t\t\t\t\t\t\t KafkaBuffer *buffer, int *addon)\n{\n\tint gzip_err;\n\tLZ4F_cctx *lz4_cctx;\n\tz_stream *c_stream;\n\tsize_t out_len;\n\tKafkaBuffer *snappy_buffer;\n\tLZ4F_errorCode_t lz4_r;\n\tZSTD_CStream *zstd_cctx;\n\tsize_t zstd_r;\n\tZSTD_outBuffer out;\n\tKafkaBlock block;\n\tsize_t zstd_end_bufsize = ZSTD_compressBound(buffer->get_size());\n\n\tswitch (compress_type)\n\t{\n\tcase Kafka_Gzip:\n\t\tc_stream = static_cast(env);\n\t\tout_len = c_stream->total_out;\n\t\tfor(;;)\n\t\t{\n\t\t\tif (c_stream->avail_out == 0)\n\t\t\t{\n\t\t\t\tblock.allocate(1024);\n\t\t\t\tc_stream->next_out = (Bytef *)block.get_block();\n\t\t\t\tc_stream->avail_out = 1024;\n\t\t\t}\n\n\t\t\tgzip_err = deflate(c_stream, Z_FINISH);\n\n\t\t\tif (gzip_err == Z_STREAM_END)\n\t\t\t\tbreak;\n\n\t\t\tif (gzip_err != Z_OK)\n\t\t\t{\n\t\t\t\tdelete c_stream;\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t\telse if (block.get_len() > 0)\n\t\t\t{\n\t\t\t\tsize_t use_bytes = block.get_len() - c_stream->avail_out;\n\n\t\t\t\tblock.set_len(use_bytes);\n\t\t\t\tbuffer->add_item(std::move(block));\n\t\t\t\t*addon += use_bytes;\n\t\t\t}\n\t\t}\n\n\t\tif (deflateEnd(c_stream) != Z_OK)\n\t\t{\n\t\t\tdelete c_stream;\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tif (block.get_len() > 0)\n\t\t{\n\t\t\tsize_t use_bytes = block.get_len() - c_stream->avail_out;\n\t\t\tblock.set_len(use_bytes);\n\t\t\tbuffer->add_item(std::move(block));\n\t\t\t*addon += use_bytes;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tKafkaBlock *b = buffer->get_block_tail();\n\t\t\tsize_t use_bytes = b->get_len() - c_stream->avail_out;\n\t\t\tint remainer = b->get_len() - use_bytes;\n\n\t\t\tb->set_len(use_bytes);\n\t\t\t*addon += -remainer;\n\t\t}\n\n\t\tdelete c_stream;\n\t\tbreak;\n\n\tcase Kafka_Snappy:\n\t\tsnappy_buffer = static_cast(env);\n\t\t{\n\t\t\tKafkaBuffer kafka_buffer_sink;\n\t\t\tKafkaSnappySource source(snappy_buffer);\n\t\t\tKafkaSnappySink sink(&kafka_buffer_sink);\n\n\t\t\tif (snappy::Compress(&source, &sink) < 0)\n\t\t\t{\n\t\t\t\tdelete snappy_buffer;\n\t\t\t\terrno = EBADMSG;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tsize_t pre_n = buffer->get_size();\n\n\t\t\tbuffer->list_splice(&kafka_buffer_sink);\n\t\t\t*addon = buffer->get_size() - pre_n;\n\t\t}\n\n\t\tdelete snappy_buffer;\n\t\tbreak;\n\n\tcase Kafka_Lz4:\n\t\tlz4_cctx = static_cast(env);\n\t\tout_len = LZ4F_compressBound(0, &kPrefs);\n\t\tif (!block.allocate(out_len))\n\t\t{\n\t\t\tLZ4F_freeCompressionContext(lz4_cctx);\n\t\t\treturn -1;\n\t\t}\n\n\t\tlz4_r = LZ4F_compressEnd(lz4_cctx, block.get_block(), block.get_len(), NULL);\n\t\tif (LZ4F_isError(lz4_r))\n\t\t{\n\t\t\tLZ4F_freeCompressionContext(lz4_cctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tblock.set_len(lz4_r);\n\t\tbuffer->add_item(std::move(block));\n\t\t*addon = lz4_r;\n\t\tLZ4F_freeCompressionContext(lz4_cctx);\n\t\tbreak;\n\n\tcase Kafka_Zstd:\n\t\tzstd_cctx = static_cast(env);\n\t\tif (!block.allocate(zstd_end_bufsize))\n\t\t\treturn -1;\n\n\t\tout.dst = block.get_block();\n\t\tout.pos = 0;\n\t\tout.size = 1024000;\n\t\tzstd_r = ZSTD_endStream(zstd_cctx, &out);\n\t\tif (ZSTD_isError(zstd_r) || zstd_r > 0)\n\t\t{\n\t\t\tZSTD_freeCStream(zstd_cctx);\n\t\t\terrno = EBADMSG;\n\t\t\treturn -1;\n\t\t}\n\n\t\tblock.set_len(out.pos);\n\t\tbuffer->add_item(std::move(block));\n\t\t*addon = out.pos;\n\t\tZSTD_freeCStream(zstd_cctx);\n\t\tbreak;\n\n\tdefault:\n\t\treturn 0;\n\t}\n\n\treturn 0;\n}\n\nKafkaRequest::KafkaRequest()\n{\n\tusing namespace std::placeholders;\n\tthis->encode_func_map[Kafka_Metadata] = std::bind(&KafkaRequest::encode_metadata, this, _1, _2);\n\tthis->encode_func_map[Kafka_Produce] = std::bind(&KafkaRequest::encode_produce, this, _1, _2);\n\tthis->encode_func_map[Kafka_Fetch] = std::bind(&KafkaRequest::encode_fetch, this, _1, _2);\n\tthis->encode_func_map[Kafka_FindCoordinator] = std::bind(&KafkaRequest::encode_findcoordinator, this, _1, _2);\n\tthis->encode_func_map[Kafka_JoinGroup] = std::bind(&KafkaRequest::encode_joingroup, this, _1, _2);\n\tthis->encode_func_map[Kafka_SyncGroup] = std::bind(&KafkaRequest::encode_syncgroup, this, _1, _2);\n\tthis->encode_func_map[Kafka_Heartbeat] = std::bind(&KafkaRequest::encode_heartbeat, this, _1, _2);\n\tthis->encode_func_map[Kafka_OffsetFetch] = std::bind(&KafkaRequest::encode_offsetfetch, this, _1, _2);\n\tthis->encode_func_map[Kafka_OffsetCommit] = std::bind(&KafkaRequest::encode_offsetcommit, this, _1, _2);\n\tthis->encode_func_map[Kafka_ListOffsets] = std::bind(&KafkaRequest::encode_listoffset, this, _1, _2);\n\tthis->encode_func_map[Kafka_LeaveGroup] = std::bind(&KafkaRequest::encode_leavegroup, this, _1, _2);\n\tthis->encode_func_map[Kafka_ApiVersions] = std::bind(&KafkaRequest::encode_apiversions, this, _1, _2);\n\tthis->encode_func_map[Kafka_SaslHandshake] = std::bind(&KafkaRequest::encode_saslhandshake, this, _1, _2);\n\tthis->encode_func_map[Kafka_SaslAuthenticate] = std::bind(&KafkaRequest::encode_saslauthenticate, this, _1, _2);\n}\n\nint KafkaRequest::encode_produce(struct iovec vectors[], int max)\n{\n\tthis->stream->reset(vectors, max);\n\n\t//transaction_id\n\tif (this->api_version >= 3)\n\t\tappend_nullable_string(this->msgbuf, \"\", 0);\n\n\tappend_i16(this->msgbuf, this->config.get_produce_acks());\n\tappend_i32(this->msgbuf, this->config.get_produce_timeout());\n\n\tint topic_cnt = 0;\n\tthis->toppar_list.rewind();\n\tKafkaToppar *toppar;\n\tKafkaBlock *block;\n\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tstd::string topic_header;\n\t\tKafkaBlock header_block;\n\t\tint record_flag = -1;\n\n\t\tappend_string(topic_header, toppar->get_topic());\n\t\tappend_i32(topic_header, 1);\n\t\tappend_i32(topic_header, toppar->get_partition());\n\t\tappend_i32(topic_header, 0); // recordset length\n\n\t\tif (!header_block.set_block((void *)topic_header.c_str(),\n\t\t\t\t\t\t\t\t\ttopic_header.size()))\n\t\t{\n\t\t\treturn -1;\n\t\t}\n\n\t\tvoid *recordset_size_ptr = (void *)((char *)header_block.get_block() +\n\t\t\t\t\t\t\t\t\t\t\theader_block.get_len() - 4);\n\n\t\tint64_t first_timestamp = 0;\n\t\tint64_t max_timestamp = 0;\n\t\tconst int MSGV2HSize = (8 + 4 + 4 + 1 + 4 + 2 + 4 + 8 + 8 + 8 + 2 + 4 + 4);\n\t\tint batch_length = 0;\n\n\t\tif (this->message_version == 2)\n\t\t\tbatch_length = MSGV2HSize - (8 + 4);\n\n\t\tsize_t cur_serialized_len = this->serialized.get_size();\n\t\tint batch_cnt = 0;\n\n\t\ttoppar->save_record_startpos();\n\t\tKafkaRecord *record;\n\t\twhile ((record = toppar->get_record_next()) != NULL)\n\t\t{\n\t\t\tKafkaBlock compress_block;\n\t\t\tKafkaBlock record_block;\n\t\t\tstruct timespec ts;\n\n\t\t\tif (record->get_timestamp() == 0)\n\t\t\t{\n\t\t\t\tclock_gettime(CLOCK_REALTIME, &ts);\n\t\t\t\trecord->set_timestamp((ts.tv_sec * 1000000000 +\n\t\t\t\t\t\t\t\t\t ts.tv_nsec) / 1000 / 1000);\n\t\t\t}\n\n\t\t\tif (batch_cnt == 0)\n\t\t\t{\n\t\t\t\tif (kafka_compress_prepare(this->config.get_compress_type(),\n\t\t\t\t\t\t\t\t\t\t &this->compress_env,\n\t\t\t\t\t\t\t\t\t\t &compress_block) < 0)\n\t\t\t\t{\n\t\t\t\t\treturn -1;\n\t\t\t\t}\n\n\t\t\t\tfirst_timestamp = record->get_timestamp();\n\t\t\t}\n\n\t\t\tint ret = append_record(&record_block, record, batch_cnt,\n\t\t\t\t\t\t\t\t\tthis->message_version, this->config,\n\t\t\t\t\t\t\t\t\tfirst_timestamp, this->compress_env,\n\t\t\t\t\t\t\t\t\tbatch_length);\n\n\t\t\tif (ret < 0)\n\t\t\t\treturn -1;\n\n\t\t\tif (ret > 0)\n\t\t\t{\n\t\t\t\ttoppar->record_rollback();\n\t\t\t\ttoppar->save_record_endpos();\n\t\t\t\tif (record_flag < 0)\n\t\t\t\t{\n\t\t\t\t\terrno = EMSGSIZE;\n\t\t\t\t\treturn -1;\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t\trecord_flag = 1;\n\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (batch_cnt == 0)\n\t\t\t{\n\t\t\t\tthis->serialized.add_item(std::move(header_block));\n\t\t\t\tcur_serialized_len = this->serialized.get_size();\n\t\t\t\tthis->serialized.set_insert_pos();\n\n\t\t\t\tif (compress_block.get_len() > 0)\n\t\t\t\t\tthis->serialized.add_item(std::move(compress_block));\n\t\t\t}\n\n\t\t\tif (record_block.get_len() > 0)\n\t\t\t\tthis->serialized.add_item(std::move(record_block));\n\n\t\t\trecord_flag = 0;\n\t\t\ttoppar->save_record_endpos();\n\n\t\t\tmax_timestamp = record->get_timestamp();\n\t\t\t++batch_cnt;\n\n\t\t\tbatch_length += this->serialized.get_size() - cur_serialized_len;\n\t\t\tcur_serialized_len = this->serialized.get_size();\n\t\t}\n\n\t\tif (record_flag < 0)\n\t\t\tcontinue;\n\n\t\tif (this->message_version == 2)\n\t\t{\n\t\t\tif (this->config.get_compress_type() != Kafka_NoCompress)\n\t\t\t{\n\t\t\t\tint addon = 0;\n\n\t\t\t\tif (kafka_compress_finish(this->config.get_compress_type(),\n\t\t\t\t\t\t\t\t\t\t this->compress_env, &this->serialized, &addon) < 0)\n\t\t\t\t{\n\t\t\t\t\treturn -1;\n\t\t\t\t}\n\n\t\t\t\tbatch_length += addon;\n\t\t\t}\n\n\t\t\tstd::string record_header;\n\n\t\t\tappend_i64(record_header, 0);\n\t\t\tappend_i32(record_header, batch_length);\n\t\t\tappend_i32(record_header, 0);\n\t\t\tappend_i8(record_header, 2); //magic\n\t\t\tappend_i32(record_header, 0);\n\t\t\tappend_i16(record_header, this->config.get_compress_type());\n\t\t\tappend_i32(record_header, batch_cnt - 1);\n\t\t\tappend_i64(record_header, first_timestamp);\n\t\t\tappend_i64(record_header, max_timestamp);\n\t\t\tappend_i64(record_header, -1); //produce_id\n\t\t\tappend_i16(record_header, -1);\n\t\t\tappend_i32(record_header, -1);\n\t\t\tappend_i32(record_header, batch_cnt);\n\n\t\t\tblock = new KafkaBlock;\n\t\t\tif (!block->set_block((void *)record_header.c_str(), record_header.size()))\n\t\t\t{\n\t\t\t\tdelete block;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tsize_t crc32_offset = 8 + 4 + 4 + 1;\n\t\t\tchar *crc_ptr = (char *)block->get_block() + crc32_offset;\n\t\t\tuint32_t crc_32 = crc32c_start();\n\n\t\t\tthis->serialized.insert_list(block);\n\n\t\t\tcrc_32 = crc32c_continue(crc_ptr + 4, block->get_len() - crc32_offset - 4, crc_32);\n\n\t\t\tthis->serialized.block_insert_rewind();\n\t\t\twhile ((block = this->serialized.get_block_insert_next()) != NULL)\n\t\t\t\tcrc_32 = crc32c_continue(block->get_block(), block->get_len(), crc_32);\n\n\t\t\tcrc_32 = crc32c_finish(crc_32);\n\t\t\t*(uint32_t *)crc_ptr = htonl(crc_32);\n\t\t\t*(uint32_t *)recordset_size_ptr = htonl(batch_length + 4 + 8);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (this->config.get_compress_type() != Kafka_NoCompress)\n\t\t\t{\n\t\t\t\tint addon = 0;\n\n\t\t\t\tif (kafka_compress_finish(this->config.get_compress_type(),\n\t\t\t\t\t\t\t\t\t\t this->compress_env, &this->serialized, &addon) < 0)\n\t\t\t\t{\n\t\t\t\t\treturn -1;\n\t\t\t\t}\n\n\t\t\t\tbatch_length += addon;\n\n\t\t\t\tint message_size = 4 + 1 + 1 + 4 + 4 + batch_length;\n\n\t\t\t\tif (this->message_version == 1)\n\t\t\t\t\tmessage_size += 8;\n\n\t\t\t\tstd::string wrap_header;\n\n\t\t\t\tappend_i64(wrap_header, 0);\n\t\t\t\tappend_i32(wrap_header, message_size);\n\n\t\t\t\tint crc_32 = crc32(0, NULL, 0);\n\t\t\t\tsize_t crc32_offset = wrap_header.size();\n\n\t\t\t\tappend_i32(wrap_header, crc_32);\n\t\t\t\tappend_i8(wrap_header, this->message_version);\n\t\t\t\tappend_i8(wrap_header, this->config.get_compress_type());\n\n\t\t\t\tif (this->message_version == 1)\n\t\t\t\t\tappend_i64(wrap_header, first_timestamp);\n\n\t\t\t\tappend_bytes(wrap_header, \"\");\n\t\t\t\tappend_i32(wrap_header, batch_length);\n\t\t\t\tconst char *crc_ptr = (const char *)wrap_header.c_str() + crc32_offset;\n\n\t\t\t\tcrc_32 = crc32(crc_32, (Bytef *)(crc_ptr + 4),\n\t\t\t\t\t\t\t wrap_header.size() - crc32_offset - 4);\n\n\t\t\t\tthis->serialized.block_insert_rewind();\n\t\t\t\twhile ((block = this->serialized.get_block_insert_next()) != NULL)\n\t\t\t\t\tcrc_32 = crc32(crc_32, (Bytef *)block->get_block(), block->get_len());\n\n\t\t\t\t*(uint32_t *)crc_ptr = htonl(crc_32);\n\n\t\t\t\tblock = new KafkaBlock;\n\t\t\t\tif (!block->set_block((void *)wrap_header.c_str(), wrap_header.size()))\n\t\t\t\t{\n\t\t\t\t\tdelete block;\n\t\t\t\t\treturn -1;\n\t\t\t\t}\n\n\t\t\t\tthis->serialized.insert_list(block);\n\t\t\t\t*(uint32_t *)recordset_size_ptr = htonl(message_size + 8 + 4);\n\t\t\t}\n\t\t\telse\n\t\t\t\t*(uint32_t *)recordset_size_ptr = htonl(batch_length);\n\t\t}\n\n\t\t++topic_cnt;\n\t}\n\n\tappend_i32(this->msgbuf, topic_cnt);\n\tthis->cur_size += this->msgbuf.size();\n\tthis->stream->append_nocopy(this->msgbuf.c_str(), this->msgbuf.size());\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\n\tblock = this->serialized.get_block_first();\n\twhile (block)\n\t{\n\t\tthis->stream->append_nocopy((const char *)block->get_block(),\n\t\t\t\t\t\t\t\t\tblock->get_len());\n\t\tthis->cur_size += block->get_len();\n\t\tblock = this->serialized.get_block_next();\n\t}\n\n\treturn this->stream->size();\n}\n\nint KafkaRequest::encode_fetch(struct iovec vectors[], int max)\n{\n\tappend_i32(this->msgbuf, -1);\n\tappend_i32(this->msgbuf, this->config.get_fetch_timeout());\n\tappend_i32(this->msgbuf, this->config.get_fetch_min_bytes());\n\n\tif (this->api_version >= 3)\n\t\tappend_i32(this->msgbuf, this->config.get_fetch_max_bytes());\n\n\t//isolation_level\n\tif (this->api_version >= 4)\n\t\tappend_i8(this->msgbuf, 0);\n\n\tif (this->api_version >= 7)\n\t{\n\t\t//sessionid\n\t\tappend_i32(this->msgbuf, 0);\n\t\t//epoch\n\t\tappend_i32(this->msgbuf, -1);\n\t}\n\n\tint topic_cnt_pos = this->msgbuf.size();\n\n\tappend_i32(this->msgbuf, 0);\n\n\tint topic_cnt = 0;\n\tthis->toppar_list.rewind();\n\tKafkaToppar *toppar;\n\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tappend_string(this->msgbuf, toppar->get_topic());\n\t\tappend_i32(this->msgbuf, 1);\n\t\tappend_i32(this->msgbuf, toppar->get_partition());\n\n\t\t//CurrentLeaderEpoch\n\t\tif (this->api_version >= 9)\n\t\t\tappend_i32(this->msgbuf, -1);\n\n\t\tappend_i64(this->msgbuf, toppar->get_offset());\n\n\t\t//LogStartOffset\n\t\tif (this->api_version >= 5)\n\t\t\tappend_i64(this->msgbuf, -1);\n\n\t\tappend_i32(this->msgbuf, this->config.get_fetch_msg_max_bytes());\n\t\t++topic_cnt;\n\t}\n\n\t*(uint32_t *)(this->msgbuf.c_str() + topic_cnt_pos) = htonl(topic_cnt);\n\n\t//Length of the ForgottenTopics list\n\tif (this->api_version >= 7)\n\t\tappend_i32(this->msgbuf, 0);\n\n\t//rackid\n\tif (this->api_version >= 11)\n\t{\n\t\tif (this->config.get_rack_id())\n\t\t\tappend_compact_string(this->msgbuf, this->config.get_rack_id());\n\t\telse\n\t\t\tappend_string(this->msgbuf, \"\");\n\t}\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_metadata(struct iovec vectors[], int max)\n{\n\tint topic_cnt_pos = this->msgbuf.size();\n\n\tif (this->api_version >= 1)\n\t\tappend_i32(this->msgbuf, -1);\n\telse\n\t\tappend_i32(this->msgbuf, 0);\n\n\tthis->meta_list.rewind();\n\tKafkaMeta *meta;\n\tint topic_cnt = 0;\n\n\twhile ((meta = this->meta_list.get_next()) != NULL)\n\t{\n\t\tappend_string(this->msgbuf, meta->get_topic());\n\t\t++topic_cnt;\n\t}\n\n\tif (this->api_version >= 4)\n\t{\n\t\tappend_bool(this->msgbuf,\n\t\t\t\t\tthis->config.get_allow_auto_topic_creation());\n\t}\n\n\t*(uint32_t *)(this->msgbuf.c_str() + topic_cnt_pos) = htonl(topic_cnt);\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_findcoordinator(struct iovec vectors[], int max)\n{\n\tappend_string(this->msgbuf, this->cgroup.get_group());\n\n\t//coordinator key type\n\tif (this->api_version >= 1)\n\t\tappend_i8(this->msgbuf, 0);\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nstatic std::string kafka_cgroup_gen_metadata(KafkaMetaList& meta_list)\n{\n\tstd::string metadata;\n\tint meta_pos;\n\tint meta_cnt = 0;\n\tmeta_list.rewind();\n\tKafkaMeta *meta;\n\n\tappend_i16(metadata, 2); // version\n\tmeta_pos = metadata.size();\n\tappend_i32(metadata, 0);\n\n\twhile ((meta = meta_list.get_next()) != NULL)\n\t{\n\t\tappend_string(metadata, meta->get_topic());\n\t\tmeta_cnt++;\n\t}\n\n\t*(uint32_t *)(metadata.c_str() + meta_pos) = htonl(meta_cnt);\n\n\t//UserData empty\n\tappend_bytes(metadata, \"\");\n\n\treturn metadata;\n}\n\nint KafkaRequest::encode_joingroup(struct iovec vectors[], int max)\n{\n\tappend_string(this->msgbuf, this->cgroup.get_group());\n\tappend_i32(this->msgbuf, this->config.get_session_timeout());\n\n\tif (this->api_version >= 1)\n\t\tappend_i32(this->msgbuf, this->config.get_rebalance_timeout());\n\n\t//member_id\n\tappend_string(this->msgbuf, this->cgroup.get_member_id());\n\n\t//group_instance_id\n\tif (this->api_version >= 5)\n\t\tappend_nullable_string(this->msgbuf, \"\", 0);\n\n\tappend_string(this->msgbuf, this->cgroup.get_protocol_type());\n\tint protocol_pos = this->msgbuf.size();\n\tappend_i32(this->msgbuf, 0);\n\n\tint protocol_cnt = 0;\n\tstruct list_head *pos;\n\tkafka_group_protocol_t *group_protocol;\n\tlist_for_each(pos, this->cgroup.get_group_protocol())\n\t{\n\t\t++protocol_cnt;\n\t\tgroup_protocol = list_entry(pos, kafka_group_protocol_t, list);\n\t\tappend_string(this->msgbuf, group_protocol->protocol_name);\n\t\tappend_bytes(this->msgbuf,\n\t\t\t\t\t kafka_cgroup_gen_metadata(this->meta_list));\n\t}\n\n\t*(uint32_t *)(this->msgbuf.c_str() + protocol_pos) = htonl(protocol_cnt);\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nstd::string KafkaMessage::get_member_assignment(kafka_member_t *member)\n{\n\tstd::string assignment;\n\t//version\n\tappend_i16(assignment, 2);\n\n\tsize_t topic_cnt_pos = assignment.size();\n\tappend_i32(assignment, 0);\n\n\tstruct list_head *pos;\n\tKafkaToppar *toppar;\n\tint topic_cnt = 0;\n\n\tlist_for_each(pos, &member->assigned_toppar_list)\n\t{\n\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\tappend_string(assignment, toppar->get_topic());\n\t\tappend_i32(assignment, 1);\n\t\tappend_i32(assignment, toppar->get_partition());\n\t\t++topic_cnt;\n\t}\n\n\t//userdata\n\tappend_bytes(assignment, \"\");\n\n\t*(uint32_t *)(assignment.c_str() + topic_cnt_pos) = htonl(topic_cnt);\n\n\treturn assignment;\n}\n\nint KafkaRequest::encode_syncgroup(struct iovec vectors[], int max)\n{\n\tappend_string(this->msgbuf, this->cgroup.get_group());\n\tappend_i32(this->msgbuf, this->cgroup.get_generation_id());\n\tappend_string(this->msgbuf, this->cgroup.get_member_id());\n\n\t//group_instance_id\n\tif (this->api_version >= 3)\n\t\tappend_nullable_string(this->msgbuf, \"\", 0);\n\n\tif (this->cgroup.is_leader())\n\t{\n\t\tappend_i32(this->msgbuf, this->cgroup.get_member_elements());\n\t\tfor (int i = 0; i < this->cgroup.get_member_elements(); ++i)\n\t\t{\n\t\t\tkafka_member_t *member = this->cgroup.get_members()[i];\n\t\t\tappend_string(this->msgbuf, member->member_id);\n\t\t\tappend_bytes(this->msgbuf, std::move(get_member_assignment(member)));\n\t\t}\n\t}\n\telse\n\t\tappend_i32(this->msgbuf, 0);\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_leavegroup(struct iovec vectors[], int max)\n{\n\tappend_string(this->msgbuf, this->cgroup.get_group());\n\tappend_string(this->msgbuf, this->cgroup.get_member_id());\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_listoffset(struct iovec vectors[], int max)\n{\n\tappend_i32(this->msgbuf, -1);\n\n\tint topic_cnt = 0;\n\tint topic_cnt_pos = this->msgbuf.size();\n\tappend_i32(this->msgbuf, 0);\n\n\tstruct list_head *pos;\n\tKafkaToppar *toppar;\n\n\tlist_for_each(pos, this->toppar_list.get_head())\n\t{\n\t\ttoppar = this->toppar_list.get_entry(pos);\n\t\tappend_string(this->msgbuf, toppar->get_topic());\n\n\t\tappend_i32(this->msgbuf, 1);\n\t\tappend_i32(this->msgbuf, toppar->get_partition());\n\t\tappend_i64(this->msgbuf, toppar->get_offset_timestamp());\n\n\t\tif (this->api_version == 0)\n\t\t\tappend_i32(this->msgbuf, 1);\n\n\t\t++topic_cnt;\n\t}\n\n\t*(uint32_t *)(this->msgbuf.c_str() + topic_cnt_pos) = htonl(topic_cnt);\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_offsetfetch(struct iovec vectors[], int max)\n{\n\tappend_string(this->msgbuf, this->cgroup.get_group());\n\n\tint topic_cnt = 0;\n\tint topic_cnt_pos = this->msgbuf.size();\n\tappend_i32(this->msgbuf, 0);\n\n\tthis->cgroup.assigned_toppar_rewind();\n\tKafkaToppar *toppar;\n\n\twhile ((toppar = this->cgroup.get_assigned_toppar_next()) != NULL)\n\t{\n\t\tappend_string(this->msgbuf, toppar->get_topic());\n\t\tappend_i32(this->msgbuf, 1);\n\t\tappend_i32(this->msgbuf, toppar->get_partition());\n\t\t++topic_cnt;\n\t}\n\n\t*(uint32_t *)(this->msgbuf.c_str() + topic_cnt_pos) = htonl(topic_cnt);\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_offsetcommit(struct iovec vectors[], int max)\n{\n\tappend_string(this->msgbuf, this->cgroup.get_group());\n\n\tif (this->api_version >= 1)\n\t{\n\t\tappend_i32(this->msgbuf, this->cgroup.get_generation_id());\n\t\tappend_string(this->msgbuf, this->cgroup.get_member_id());\n\t}\n\n\t//GroupInstanceId\n\tif (this->api_version >= 7)\n\t\tappend_nullable_string(this->msgbuf, \"\", 0);\n\n\t//RetentionTime\n\tif (this->api_version >= 2 && this->api_version <= 4)\n\t\tappend_i64(this->msgbuf, -1);\n\n\tint toppar_cnt = 0;\n\tint toppar_cnt_pos = this->msgbuf.size();\n\tappend_i32(this->msgbuf, 0);\n\n\tthis->toppar_list.rewind();\n\tKafkaToppar *toppar;\n\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t{\n\t\tappend_string(this->msgbuf, toppar->get_topic());\n\t\tappend_i32(this->msgbuf, 1);\n\t\tappend_i32(this->msgbuf, toppar->get_partition());\n\t\tappend_i64(this->msgbuf, toppar->get_offset() + 1);\n\n\t\tif (this->api_version >= 6)\n\t\t\tappend_i32(this->msgbuf, -1);\n\n\t\tif (this->api_version == 1)\n\t\t\tappend_i64(this->msgbuf, -1);\n\n\t\tappend_nullable_string(this->msgbuf, \"\", 0);\n\t\t++toppar_cnt;\n\t}\n\n\t*(uint32_t *)(this->msgbuf.c_str() + toppar_cnt_pos) = htonl(toppar_cnt);\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_heartbeat(struct iovec vectors[], int max)\n{\n\tappend_string(this->msgbuf, this->cgroup.get_group());\n\tappend_i32(this->msgbuf, this->cgroup.get_generation_id());\n\tappend_string(this->msgbuf, this->cgroup.get_member_id());\n\n\t//group_instance_id\n\tif (this->api_version >= 3)\n\t\tappend_nullable_string(this->msgbuf, \"\", 0);\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_apiversions(struct iovec vectors[], int max)\n{\n\treturn 0;\n}\n\nint KafkaRequest::encode_saslhandshake(struct iovec vectors[], int max)\n{\n\tappend_string(this->msgbuf, this->config.get_sasl_mech());\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nint KafkaRequest::encode_saslauthenticate(struct iovec vectors[], int max)\n{\n\tappend_bytes(this->msgbuf, this->sasl->buf, this->sasl->bsize);\n\n\tthis->cur_size = this->msgbuf.size();\n\n\tvectors[0].iov_base = (void *)this->msgbuf.c_str();\n\tvectors[0].iov_len = this->msgbuf.size();\n\treturn 1;\n}\n\nKafkaResponse::KafkaResponse()\n{\n\tusing namespace std::placeholders;\n\tthis->parse_func_map[Kafka_Metadata] = std::bind(&KafkaResponse::parse_metadata, this, _1, _2);\n\tthis->parse_func_map[Kafka_Produce] = std::bind(&KafkaResponse::parse_produce, this, _1, _2);\n\tthis->parse_func_map[Kafka_Fetch] = std::bind(&KafkaResponse::parse_fetch, this, _1, _2);\n\tthis->parse_func_map[Kafka_FindCoordinator] = std::bind(&KafkaResponse::parse_findcoordinator, this, _1, _2);\n\tthis->parse_func_map[Kafka_JoinGroup] = std::bind(&KafkaResponse::parse_joingroup, this, _1, _2);\n\tthis->parse_func_map[Kafka_SyncGroup] = std::bind(&KafkaResponse::parse_syncgroup, this, _1, _2);\n\tthis->parse_func_map[Kafka_Heartbeat] = std::bind(&KafkaResponse::parse_heartbeat, this, _1, _2);\n\tthis->parse_func_map[Kafka_OffsetFetch] = std::bind(&KafkaResponse::parse_offsetfetch, this, _1, _2);\n\tthis->parse_func_map[Kafka_OffsetCommit] = std::bind(&KafkaResponse::parse_offsetcommit, this, _1, _2);\n\tthis->parse_func_map[Kafka_ListOffsets] = std::bind(&KafkaResponse::parse_listoffset, this, _1, _2);\n\tthis->parse_func_map[Kafka_LeaveGroup] = std::bind(&KafkaResponse::parse_leavegroup, this, _1, _2);\n\tthis->parse_func_map[Kafka_ApiVersions] = std::bind(&KafkaResponse::parse_apiversions, this, _1, _2);\n\tthis->parse_func_map[Kafka_SaslHandshake] = std::bind(&KafkaResponse::parse_saslhandshake, this, _1, _2);\n\tthis->parse_func_map[Kafka_SaslAuthenticate] = std::bind(&KafkaResponse::parse_saslauthenticate, this, _1, _2);\n}\n\nint KafkaResponse::parse_response()\n{\n\tauto it = this->parse_func_map.find(this->api_type);\n\n\tif (it == this->parse_func_map.end())\n\t{\n\t\terrno = EPROTO;\n\t\treturn -1;\n\t}\n\n\tvoid *buf = this->parser->msgbuf;\n\tsize_t size = this->parser->message_size;\n\tint32_t correlation_id;\n\n\tif (parse_i32(&buf, &size, &correlation_id) < 0)\n\t\treturn -1;\n\n\tthis->correlation_id = correlation_id;\n\n\tint ret = it->second(&buf, &size);\n\n\tif (ret < 0)\n\t\treturn -1;\n\n\tif (size != 0)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\treturn ret;\n}\n\nstatic int kafka_meta_parse_broker(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t int api_version,\n\t\t\t\t\t\t\t\t KafkaBrokerList *broker_list)\n{\n\tint32_t broker_cnt;\n\n\tCHECK_RET(parse_i32(buf, size, &broker_cnt));\n\n\tif (broker_cnt < 0)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tfor (int i = 0; i < broker_cnt; ++i)\n\t{\n\t\tKafkaBroker broker;\n\t\tkafka_broker_t *ptr = broker.get_raw_ptr();\n\n\t\tCHECK_RET(parse_i32(buf, size, &ptr->node_id));\n\t\tCHECK_RET(parse_string(buf, size, &ptr->host));\n\t\tCHECK_RET(parse_i32(buf, size, &ptr->port));\n\n\t\tif (api_version >= 1)\n\t\t\tCHECK_RET(parse_string(buf, size, &ptr->rack));\n\n\t\tbroker_list->rewind();\n\t\tKafkaBroker *last;\n\n\t\twhile ((last = broker_list->get_next()) != NULL)\n\t\t{\n\t\t\tif (last->get_node_id() == broker.get_node_id())\n\t\t\t{\n\t\t\t\tbroker_list->del_cur();\n\t\t\t\tdelete last;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tbroker_list->add_item(std::move(broker));\n\t}\n\n\treturn 0;\n}\n\nstatic bool kafka_broker_get_leader(int leader_id, KafkaBrokerList *broker_list,\n\t\t\t\t\t\t\t\t\tkafka_broker_t *leader)\n{\n\tKafkaBroker *bbroker;\n\n\tbroker_list->rewind();\n\twhile ((bbroker = broker_list->get_next()) != NULL)\n\t{\n\t\tif (bbroker->get_node_id() == leader_id)\n\t\t{\n\t\t\tkafka_broker_t *broker = bbroker->get_raw_ptr();\n\n\t\t\tchar *host = strdup(broker->host);\n\t\t\tif (host)\n\t\t\t{\n\t\t\t\tchar *rack = NULL;\n\n\t\t\t\tif (broker->rack)\n\t\t\t\t\track = strdup(broker->rack);\n\n\t\t\t\tif (!broker->rack || rack)\n\t\t\t\t{\n\t\t\t\t\tkafka_broker_deinit(leader);\n\t\t\t\t\t*leader = *broker;\n\t\t\t\t\tleader->host = host;\n\t\t\t\t\tleader->rack = rack;\n\t\t\t\t\treturn true;\n\t\t\t\t}\n\n\t\t\t\tfree(host);\n\t\t\t}\n\n\t\t\treturn false;\n\t\t}\n\t}\n\n\terrno = EBADMSG;\n\treturn false;\n}\n\nstatic int kafka_meta_parse_partition(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t\t KafkaMeta *meta,\n\t\t\t\t\t\t\t\t\t KafkaBrokerList *broker_list)\n{\n\tint32_t leader_id;\n\tint32_t replica_cnt, isr_cnt;\n\tint32_t partition_cnt;\n\tint32_t i, j;\n\n\tCHECK_RET(parse_i32(buf, size, &partition_cnt));\n\n\tif (partition_cnt < 0)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tif (!meta->create_partitions(partition_cnt))\n\t\treturn -1;\n\n\tkafka_partition_t **partition = meta->get_partitions();\n\n\tfor (i = 0; i < partition_cnt; ++i)\n\t{\n\t\tint16_t error;\n\t\tint32_t index;\n\n\t\tif (parse_i16(buf, size, &error) < 0)\n\t\t\tbreak;\n\n\t\tpartition[i]->error = error;\n\n\t\tif (parse_i32(buf, size, &index) < 0)\n\t\t\tbreak;\n\n\t\tpartition[i]->partition_index = index;\n\n\t\tif (parse_i32(buf, size, &leader_id) < 0)\n\t\t\tbreak;\n\n\t\tif (!kafka_broker_get_leader(leader_id, broker_list, &partition[i]->leader))\n\t\t\tbreak;\n\n\t\tif (parse_i32(buf, size, &replica_cnt) < 0)\n\t\t\tbreak;\n\n\t\tif (!meta->create_replica_nodes(i, replica_cnt))\n\t\t\tbreak;\n\n\t\tfor (j = 0; j < replica_cnt; ++j)\n\t\t{\n\t\t\tint32_t replica_node;\n\n\t\t\tif (parse_i32(buf, size, &replica_node) < 0)\n\t\t\t\tbreak;\n\n\t\t\tpartition[i]->replica_nodes[j] = replica_node;\n\t\t}\n\n\t\tif (j != replica_cnt)\n\t\t\tbreak;\n\n\t\tif (parse_i32(buf, size, &isr_cnt) < 0)\n\t\t\tbreak;\n\n\t\tif (!meta->create_isr_nodes(i, isr_cnt))\n\t\t\tbreak;\n\n\t\tfor (j = 0; j < isr_cnt; ++j)\n\t\t{\n\t\t\tint32_t isr_node;\n\n\t\t\tif (parse_i32(buf, size, &isr_node) < 0)\n\t\t\t\tbreak;\n\n\t\t\tpartition[i]->isr_nodes[j] = isr_node;\n\t\t}\n\n\t\tif (j != isr_cnt)\n\t\t\tbreak;\n\t}\n\n\tif (i != partition_cnt)\n\t\treturn -1;\n\n\treturn 0;\n}\n\nstatic KafkaMeta *find_meta_by_name(const std::string& topic, KafkaMetaList *meta_list)\n{\n\tmeta_list->rewind();\n\tKafkaMeta *meta;\n\n\twhile ((meta = meta_list->get_next()) != NULL)\n\t{\n\t\tif (meta->get_topic() == topic)\n\t\t\treturn meta;\n\t}\n\n\terrno = EBADMSG;\n\treturn NULL;\n}\n\nstatic int kafka_meta_parse_topic(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t int api_version,\n\t\t\t\t\t\t\t\t KafkaMetaList *meta_list,\n\t\t\t\t\t\t\t\t KafkaBrokerList *broker_list)\n{\n\tKafkaMetaList lst;\n\tint32_t topic_cnt;\n\n\tCHECK_RET(parse_i32(buf, size, &topic_cnt));\n\tfor (int32_t topic_idx = 0; topic_idx < topic_cnt; ++topic_idx)\n\t{\n\t\tint16_t error;\n\t\tCHECK_RET(parse_i16(buf, size, &error));\n\n\t\tstd::string topic_name;\n\t\tCHECK_RET(parse_string(buf, size, topic_name));\n\t\tKafkaMeta *meta = find_meta_by_name(topic_name, meta_list);\n\t\tif (!meta)\n\t\t\treturn -1;\n\n\t\tKafkaMeta new_mta;\n\t\tnew_mta.set_topic(topic_name);\n\n\t\tkafka_meta_t *ptr = new_mta.get_raw_ptr();\n\t\tptr->error = error;\n\n\t\tif (api_version >= 1)\n\t\t\tCHECK_RET(parse_i8(buf, size, &ptr->is_internal));\n\n\t\tCHECK_RET(kafka_meta_parse_partition(buf, size, &new_mta, broker_list));\n\n\t\tlst.add_item(std::move(new_mta));\n\t}\n\n\t*meta_list = std::move(lst);\n\treturn 0;\n}\n\nint KafkaResponse::parse_metadata(void **buf, size_t *size)\n{\n\tint32_t throttle_time, controller_id;\n\tstd::string cluster_id;\n\n\tif (this->api_version >= 3)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tCHECK_RET(kafka_meta_parse_broker(buf, size, this->api_version,\n\t\t\t\t\t\t\t\t\t &this->broker_list));\n\n\tif (this->api_version >= 2)\n\t\tCHECK_RET(parse_string(buf, size, cluster_id));\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_i32(buf, size, &controller_id));\n\n\tCHECK_RET(kafka_meta_parse_topic(buf, size, this->api_version,\n\t\t\t\t\t\t\t\t\t &this->meta_list, &this->broker_list));\n\n\treturn 0;\n}\n\nKafkaToppar *KafkaMessage::find_toppar_by_name(const std::string& topic, int partition,\n\t\t\t\t\t\t\t\t\t\t\t struct list_head *toppar_list)\n{\n\tKafkaToppar *toppar;\n\tstruct list_head *pos;\n\n\tlist_for_each(pos, toppar_list)\n\t{\n\t\ttoppar = list_entry(pos, KafkaToppar, list);\n\t\tif (toppar->get_topic() == topic && toppar->get_partition() == partition)\n\t\t\treturn toppar;\n\t}\n\n\terrno = EBADMSG;\n\treturn NULL;\n}\n\nKafkaToppar *KafkaMessage::find_toppar_by_name(const std::string& topic, int partition,\n\t\t\t\t\t\t\t\t\t\t\t KafkaTopparList *toppar_list)\n{\n\ttoppar_list->rewind();\n\tKafkaToppar *toppar;\n\n\twhile ((toppar = toppar_list->get_next()) != NULL)\n\t{\n\t\tif (toppar->get_topic() == topic &&\n\t\t\ttoppar->get_partition() == partition)\n\t\t\treturn toppar;\n\t}\n\n\terrno = EBADMSG;\n\treturn NULL;\n}\n\nint KafkaResponse::parse_produce(void **buf, size_t *size)\n{\n\tint32_t topic_cnt;\n\tstd::string topic_name;\n\tint32_t partition_cnt;\n\tint32_t partition;\n\tint64_t base_offset, log_append_time, log_start_offset;\n\tint32_t throttle_time;\n\tint produce_timeout = this->config.get_produce_timeout() * 2;\n\n\tCHECK_RET(parse_i32(buf, size, &topic_cnt));\n\tfor (int32_t topic_idx = 0; topic_idx < topic_cnt; ++topic_idx)\n\t{\n\t\tCHECK_RET(parse_string(buf, size, topic_name));\n\n\t\tCHECK_RET(parse_i32(buf, size, &partition_cnt));\n\t\tfor (int32_t i = 0; i < partition_cnt; ++i)\n\t\t{\n\t\t\tCHECK_RET(parse_i32(buf, size, &partition));\n\n\t\t\tKafkaToppar *toppar = find_toppar_by_name(topic_name, partition,\n\t\t\t\t\t\t\t\t\t\t\t\t\t &this->toppar_list);\n\t\t\tif (!toppar)\n\t\t\t\treturn -1;\n\n\t\t\tkafka_topic_partition_t *ptr = toppar->get_raw_ptr();\n\n\t\t\tCHECK_RET(parse_i16(buf, size, &ptr->error));\n\n\t\t\tlog_append_time = -1;\n\t\t\tCHECK_RET(parse_i64(buf, size, &base_offset));\n\n\t\t\tif (this->api_version >= 2)\n\t\t\t\tCHECK_RET(parse_i64(buf, size, &log_append_time));\n\n\t\t\tif (this->api_version >=5)\n\t\t\t\tCHECK_RET(parse_i64(buf, size, &log_start_offset));\n\n\t\t\tstruct list_head *pos;\n\t\t\tKafkaRecord *record;\n\n\t\t\tif (ptr->error == KAFKA_REQUEST_TIMED_OUT)\n\t\t\t{\n\t\t\t\ttoppar->restore_record_curpos();\n\t\t\t\tthis->config.set_produce_timeout(produce_timeout);\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tfor (pos = toppar->get_record_startpos()->next;\n\t\t\t\t pos != toppar->get_record_endpos(); pos = pos->next)\n\t\t\t{\n\t\t\t\trecord = list_entry(pos, KafkaRecord, list);\n\t\t\t\trecord->set_status(ptr->error);\n\n\t\t\t\tif (ptr->error)\n\t\t\t\t\tcontinue;\n\n\t\t\t\trecord->set_offset(base_offset++);\n\n\t\t\t\tif (log_append_time != -1)\n\t\t\t\t\trecord->set_timestamp(log_append_time);\n\t\t\t}\n\t\t}\n\n\t}\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_fetch(void **buf, size_t *size)\n{\n\tint32_t throttle_time;\n\n\tthis->toppar_list.rewind();\n\tKafkaToppar *toppar;\n\twhile ((toppar = this->toppar_list.get_next()) != NULL)\n\t\ttoppar->clear_records();\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tif (this->api_version >= 7)\n\t{\n\t\tint16_t error;\n\t\tint32_t sessionid;\n\t\tparse_i16(buf, size, &error);\n\t\tparse_i32(buf, size, &sessionid);\n\t}\n\n\tint32_t topic_cnt;\n\tstd::string topic_name;\n\tint32_t partition_cnt;\n\tint32_t partition;\n\tint32_t aborted_cnt;\n\tint32_t preferred_read_replica;\n\tint64_t producer_id, first_offset;\n\tint64_t high_watermark;\n\n\tCHECK_RET(parse_i32(buf, size, &topic_cnt));\n\tfor (int32_t topic_idx = 0; topic_idx < topic_cnt; ++topic_idx)\n\t{\n\t\tCHECK_RET(parse_string(buf, size, topic_name));\n\t\tCHECK_RET(parse_i32(buf, size, &partition_cnt));\n\n\t\tfor (int i = 0; i < partition_cnt; ++i)\n\t\t{\n\t\t\tCHECK_RET(parse_i32(buf, size, &partition));\n\n\t\t\tKafkaToppar *toppar = find_toppar_by_name(topic_name, partition,\n\t\t\t\t\t\t\t\t\t\t\t\t\t &this->toppar_list);\n\n\t\t\tif (!toppar)\n\t\t\t\treturn -1;\n\n\t\t\tkafka_topic_partition_t *ptr = toppar->get_raw_ptr();\n\n\t\t\tCHECK_RET(parse_i16(buf, size, &ptr->error));\n\t\t\tCHECK_RET(parse_i64(buf, size, &high_watermark));\n\n\t\t\tif (high_watermark > ptr->low_watermark)\n\t\t\t\tptr->high_watermark = high_watermark;\n\n\t\t\tif (this->api_version >= 4)\n\t\t\t{\n\t\t\t\tCHECK_RET(parse_i64(buf, size, (int64_t *)&ptr->last_stable_offset));\n\n\t\t\t\tif (this->api_version >= 5)\n\t\t\t\t\tCHECK_RET(parse_i64(buf, size, (int64_t *)&ptr->log_start_offset));\n\n\t\t\t\tCHECK_RET(parse_i32(buf, size, &aborted_cnt));\n\t\t\t\tfor (int32_t j = 0; j < aborted_cnt; ++j)\n\t\t\t\t{\n\t\t\t\t\tCHECK_RET(parse_i64(buf, size, &producer_id));\n\t\t\t\t\tCHECK_RET(parse_i64(buf, size, &first_offset));\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (this->api_version >= 11)\n\t\t\t{\n\t\t\t\tCHECK_RET(parse_i32(buf, size, &preferred_read_replica));\n\t\t\t\tptr->preferred_read_replica = preferred_read_replica;\n\t\t\t}\n\n\t\t\tif (parse_records(buf, size, this->config.get_check_crcs(),\n\t\t\t\t\t\t\t &this->uncompressed, toppar) != 0)\n\t\t\t{\n\t\t\t\tptr->error = KAFKA_CORRUPT_MESSAGE;\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_listoffset(void **buf, size_t *size)\n{\n\tint32_t throttle_time;\n\tint32_t topic_cnt;\n\tstd::string topic_name;\n\tint32_t partition_cnt;\n\tint32_t partition;\n\tint64_t offset_timestamp, offset;\n\tint32_t offset_cnt;\n\n\tif (this->api_version >= 2)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tCHECK_RET(parse_i32(buf, size, &topic_cnt));\n\tfor (int32_t topic_idx = 0; topic_idx < topic_cnt; ++topic_idx)\n\t{\n\t\tCHECK_RET(parse_string(buf, size, topic_name));\n\t\tCHECK_RET(parse_i32(buf, size, &partition_cnt));\n\n\t\tfor (int32_t i = 0; i < partition_cnt; ++i)\n\t\t{\n\t\t\tCHECK_RET(parse_i32(buf, size, &partition));\n\n\t\t\tKafkaToppar *toppar = find_toppar_by_name(topic_name, partition,\n\t\t\t\t\t\t\t\t\t\t\t\t\t &this->toppar_list);\n\t\t\tif (!toppar)\n\t\t\t\treturn -1;\n\n\t\t\tkafka_topic_partition_t *ptr = toppar->get_raw_ptr();\n\n\t\t\tCHECK_RET(parse_i16(buf, size, &ptr->error));\n\n\t\t\tif (this->api_version == 1)\n\t\t\t{\n\t\t\t\tCHECK_RET(parse_i64(buf, size, &offset_timestamp));\n\t\t\t\tCHECK_RET(parse_i64(buf, size, &offset));\n\t\t\t\tif (ptr->offset_timestamp == -1)\n\t\t\t\t\tptr->high_watermark = offset;\n\t\t\t\telse if (ptr->offset_timestamp == -2)\n\t\t\t\t\tptr->low_watermark = offset;\n\t\t\t\telse\n\t\t\t\t\tptr->offset = offset;\n\t\t\t}\n\t\t\telse if (this->api_version == 0)\n\t\t\t{\n\t\t\t\tCHECK_RET(parse_i32(buf, size, &offset_cnt));\n\t\t\t\tfor (int32_t j = 0; j < offset_cnt; ++j)\n\t\t\t\t{\n\t\t\t\t\tCHECK_RET(parse_i64(buf, size, &offset));\n\t\t\t\t\tptr->offset = offset;\n\t\t\t\t}\n\n\t\t\t\tptr->low_watermark = 0;\n\t\t\t}\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_findcoordinator(void **buf, size_t *size)\n{\n\tint32_t throttle_time;\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tkafka_cgroup_t *cgroup = this->cgroup.get_raw_ptr();\n\tCHECK_RET(parse_i16(buf, size, &cgroup->error));\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_string(buf, size, &cgroup->error_msg));\n\n\tCHECK_RET(parse_i32(buf, size, &cgroup->coordinator.node_id));\n\tCHECK_RET(parse_string(buf, size, &cgroup->coordinator.host));\n\tCHECK_RET(parse_i32(buf, size, &cgroup->coordinator.port));\n\n\treturn 0;\n}\n\nstatic bool kafka_meta_find_or_add_topic(const std::string& topic_name,\n\t\t\t\t\t\t\t\t\t\t KafkaMetaList *meta_list)\n{\n\tmeta_list->rewind();\n\tbool find = false;\n\tKafkaMeta *meta;\n\n\twhile ((meta = meta_list->get_next()) != NULL)\n\t{\n\t\tif (topic_name == meta->get_topic())\n\t\t{\n\t\t\tfind = true;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (!find)\n\t{\n\t\tKafkaMeta tmp;\n\t\tif (!tmp.set_topic(topic_name))\n\t\t\treturn false;\n\n\t\tmeta_list->add_item(tmp);\n\t}\n\n\treturn true;\n}\n\nstatic int kafka_cgroup_parse_member(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t\t KafkaCgroup *cgroup,\n\t\t\t\t\t\t\t\t\t KafkaMetaList *meta_list,\n\t\t\t\t\t\t\t\t\t int api_version)\n{\n\tint32_t member_cnt = 0;\n\tCHECK_RET(parse_i32(buf, size, &member_cnt));\n\n\tif (member_cnt < 0)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tif (!cgroup->create_members(member_cnt))\n\t\treturn -1;\n\n\tkafka_member_t **member = cgroup->get_members();\n\tint32_t i;\n\n\tfor (i = 0; i < member_cnt; ++i)\n\t{\n\t\tif (parse_string(buf, size, &member[i]->member_id) < 0)\n\t\t\tbreak;\n\n\t\tif (api_version >= 5)\n\t\t{\n\t\t\tstd::string group_instance_id;\n\t\t\tparse_string(buf, size, group_instance_id);\n\t\t}\n\n\t\tif (parse_bytes(buf, size, &member[i]->member_metadata,\n\t\t\t\t\t\t&member[i]->member_metadata_len) < 0)\n\t\t\tbreak;\n\n\t\tvoid *metadata = member[i]->member_metadata;\n\t\tsize_t metadata_len = member[i]->member_metadata_len;\n\t\tint16_t version;\n\t\tint32_t topic_cnt;\n\t\tstd::string topic_name;\n\t\tint32_t j;\n\n\t\tif (parse_i16(&metadata, &metadata_len, &version) < 0)\n\t\t\tbreak;\n\n\t\tif (parse_i32(&metadata, &metadata_len, &topic_cnt) < 0)\n\t\t\tbreak;\n\n\t\tfor (j = 0; j < topic_cnt; ++j)\n\t\t{\n\t\t\tif (parse_string(&metadata, &metadata_len, topic_name) < 0)\n\t\t\t\tbreak;\n\n\t\t\tKafkaToppar * toppar = new KafkaToppar;\n\t\t\tif (!toppar->set_topic(topic_name.c_str()))\n\t\t\t{\n\t\t\t\tdelete toppar;\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tlist_add_tail(toppar->get_list(), &member[i]->toppar_list);\n\n\t\t\tif (!kafka_meta_find_or_add_topic(topic_name, meta_list))\n\t\t\t\treturn -1;\n\t\t}\n\n\t\tif (j != topic_cnt)\n\t\t\tbreak;\n\t}\n\n\tif (i != member_cnt)\n\t\treturn -1;\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_joingroup(void **buf, size_t *size)\n{\n\tint32_t throttle_time;\n\n\tif (this->api_version >= 2)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tkafka_cgroup_t *cgroup = this->cgroup.get_raw_ptr();\n\tCHECK_RET(parse_i16(buf, size, &cgroup->error));\n\tCHECK_RET(parse_i32(buf, size, &cgroup->generation_id));\n\n\tCHECK_RET(parse_string(buf, size, &cgroup->protocol_name));\n\tCHECK_RET(parse_string(buf, size, &cgroup->leader_id));\n\tCHECK_RET(parse_string(buf, size, &cgroup->member_id));\n\tCHECK_RET(kafka_cgroup_parse_member(buf, size, &this->cgroup,\n\t\t\t\t\t\t\t\t\t\t&this->meta_list,\n\t\t\t\t\t\t\t\t\t\tthis->api_version));\n\n\treturn 0;\n}\n\nint KafkaMessage::kafka_parse_member_assignment(const char *bbuf, size_t n,\n\t\t\t\t\t\t\t\t\t\t\t\tKafkaCgroup *cgroup)\n{\n\tvoid **buf = (void **)&bbuf;\n\tsize_t *size = &n;\n\tint32_t topic_cnt;\n\tint32_t partition_cnt;\n\tint16_t version;\n\tstruct list_head *pos, *tmp;\n\tstd::string topic_name;\n\tint32_t partition;\n\n\tlist_for_each_safe(pos, tmp, cgroup->get_assigned_toppar_list())\n\t{\n\t\tKafkaToppar *toppar = list_entry(pos, KafkaToppar, list);\n\t\tlist_del(pos);\n\t\tdelete toppar;\n\t}\n\n\tCHECK_RET(parse_i16(buf, size, &version));\n\tCHECK_RET(parse_i32(buf, size, &topic_cnt));\n\tfor (int32_t i = 0; i < topic_cnt; ++i)\n\t{\n\t\tCHECK_RET(parse_string(buf, size, topic_name));\n\t\tCHECK_RET(parse_i32(buf, size, &partition_cnt));\n\n\t\tfor (int32_t j = 0; j < partition_cnt; ++j)\n\t\t{\n\t\t\tCHECK_RET(parse_i32(buf, size, &partition));\n\t\t\tKafkaToppar *toppar = new KafkaToppar;\n\n\t\t\tif (!toppar->set_topic_partition(topic_name, partition))\n\t\t\t{\n\t\t\t\tdelete toppar;\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t\tcgroup->add_assigned_toppar(toppar);\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_syncgroup(void **buf, size_t *size)\n{\n\tint32_t throttle_time;\n\tint16_t error;\n\tstd::string member_assignment;\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tCHECK_RET(parse_i16(buf, size, &error));\n\tthis->cgroup.set_error(error);\n\n\tCHECK_RET(parse_bytes(buf, size, member_assignment));\n\tif (!member_assignment.empty())\n\t{\n\t\tCHECK_RET(kafka_parse_member_assignment(member_assignment.c_str(),\n\t\t\t\t\t\t\t\t\t\t\t\tmember_assignment.size(),\n\t\t\t\t\t\t\t\t\t\t\t\t&this->cgroup));\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_leavegroup(void **buf, size_t *size)\n{\n\tint32_t throttle_time;\n\tint16_t error;\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tCHECK_RET(parse_i16(buf, size, &error));\n\tthis->cgroup.set_error(error);\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_offsetfetch(void **buf, size_t *size)\n{\n\tint32_t topic_cnt;\n\tstd::string topic_name;\n\tint32_t partition_cnt;\n\tint32_t partition;\n\n\tCHECK_RET(parse_i32(buf, size, &topic_cnt));\n\tfor (int32_t topic_idx = 0; topic_idx < topic_cnt; ++topic_idx)\n\t{\n\t\tCHECK_RET(parse_string(buf, size, topic_name));\n\n\t\tCHECK_RET(parse_i32(buf, size, &partition_cnt));\n\t\tfor (int32_t i = 0; i < partition_cnt; ++i)\n\t\t{\n\t\t\tCHECK_RET(parse_i32(buf, size, &partition));\n\t\t\tKafkaToppar *toppar = find_toppar_by_name(topic_name, partition,\n\t\t\t\t\t\t\t\t\t\t\t\t\t this->cgroup.get_assigned_toppar_list());\n\t\t\tif (!toppar)\n\t\t\t\treturn -1;\n\n\t\t\tkafka_topic_partition_t *ptr = toppar->get_raw_ptr();\n\n\t\t\tint64_t offset;\n\t\t\tCHECK_RET(parse_i64(buf, size, &offset));\n\t\t\tif (this->config.get_offset_store() != KAFKA_OFFSET_ASSIGN)\n\t\t\t\tptr->offset = offset;\n\n\t\t\tCHECK_RET(parse_string(buf, size, &ptr->committed_metadata));\n\t\t\tCHECK_RET(parse_i16(buf, size, &ptr->error));\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_offsetcommit(void **buf, size_t *size)\n{\n\tint32_t throttle_time;\n\tint32_t topic_cnt;\n\tstd::string topic_name;\n\tint32_t partition_cnt;\n\tint32_t partition;\n\n\tif (this->api_version >= 3)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tCHECK_RET(parse_i32(buf, size, &topic_cnt));\n\tfor (int32_t topic_idx = 0; topic_idx < topic_cnt; ++topic_idx)\n\t{\n\t\tCHECK_RET(parse_string(buf, size, topic_name));\n\t\tCHECK_RET(parse_i32(buf, size, &partition_cnt));\n\n\t\tfor (int32_t i = 0 ; i < partition_cnt; ++i)\n\t\t{\n\t\t\tCHECK_RET(parse_i32(buf, size, &partition));\n\t\t\tCHECK_RET(parse_i16(buf, size, &this->cgroup.get_raw_ptr()->error));\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_heartbeat(void **buf, size_t *size)\n{\n\tint32_t throttle_time;\n\tint16_t error;\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tCHECK_RET(parse_i16(buf, size, &error));\n\n\tthis->cgroup.set_error(error);\n\treturn 0;\n}\n\nstatic bool kafka_api_version_cmp(const kafka_api_version_t& api_ver1,\n\t\t\t\t\t\t\t\t const kafka_api_version_t& api_ver2)\n{\n\treturn api_ver1.api_key < api_ver2.api_key;\n}\n\nint KafkaResponse::parse_apiversions(void **buf, size_t *size)\n{\n\tint16_t error;\n\tint32_t api_cnt;\n\tint32_t throttle_time;\n\n\tCHECK_RET(parse_i16(buf, size, &error));\n\tCHECK_RET(parse_i32(buf, size, &api_cnt));\n\tif (api_cnt < 0)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tvoid *p = malloc(api_cnt * sizeof(kafka_api_version_t));\n\tif (!p)\n\t\treturn -1;\n\n\tthis->api->api = (kafka_api_version_t *)p;\n\tthis->api->elements = api_cnt;\n\n\tfor (int32_t i = 0; i < api_cnt; ++i)\n\t{\n\t\tCHECK_RET(parse_i16(buf, size, &this->api->api[i].api_key));\n\t\tCHECK_RET(parse_i16(buf, size, &this->api->api[i].min_ver));\n\t\tCHECK_RET(parse_i16(buf, size, &this->api->api[i].max_ver));\n\t}\n\n\tif (this->api_version >= 1)\n\t\tCHECK_RET(parse_i32(buf, size, &throttle_time));\n\n\tstd::sort(this->api->api, this->api->api + api_cnt, kafka_api_version_cmp);\n\tthis->api->features = kafka_get_features(this->api->api, api_cnt);\n\treturn 0;\n}\n\nint KafkaResponse::parse_saslhandshake(void **buf, size_t *size)\n{\n\tstd::string mechanism;\n\tint16_t error = 0;\n\tint32_t cnt, i;\n\n\tCHECK_RET(parse_i16(buf, size, &error));\n\tif (error != 0)\n\t{\n\t\tthis->broker.get_raw_ptr()->error = error;\n\t\treturn 1;\n\t}\n\n\tCHECK_RET(parse_i32(buf, size, &cnt));\n\n\tfor (i = 0; i < cnt; i++)\n\t{\n\t\tCHECK_RET(parse_string(buf, size, mechanism));\n\n\t\tif (strcasecmp(mechanism.c_str(), this->config.get_sasl_mech()) == 0)\n\t\t\tbreak;\n\t}\n\n\tif (i == cnt)\n\t{\n\t\tthis->broker.get_raw_ptr()->error = KAFKA_SASL_AUTHENTICATION_FAILED;\n\t\treturn 1;\n\t}\n\n\tfor (i++; i < cnt; i++)\n\t\tCHECK_RET(parse_string(buf, size, mechanism));\n\n\terrno = 0;\n\tif (!this->config.new_client(this->sasl))\n\t{\n\t\tif (errno)\n\t\t\treturn -1;\n\n\t\tthis->broker.get_raw_ptr()->error = KAFKA_SASL_AUTHENTICATION_FAILED;\n\t\treturn 1;\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::parse_saslauthenticate(void **buf, size_t *size)\n{\n\tstd::string error_message;\n\tstd::string auth_bytes;\n\tint16_t error = 0;\n\n\tCHECK_RET(parse_i16(buf, size, &error));\n\tCHECK_RET(parse_string(buf, size, error_message));\n\tCHECK_RET(parse_bytes(buf, size, auth_bytes));\n\n\tif (error != 0)\n\t{\n\t\tthis->broker.get_raw_ptr()->error = error;\n\t\treturn 1;\n\t}\n\n\terrno = 0;\n\tif (this->config.get_raw_ptr()->recv(auth_bytes.c_str(),\n\t\t\t\t\t\t\t\t\t\t auth_bytes.size(),\n\t\t\t\t\t\t\t\t\t\t this->config.get_raw_ptr(),\n\t\t\t\t\t\t\t\t\t\t this->sasl) != 0)\n\t{\n\t\tif (errno)\n\t\t\treturn -1;\n\n\t\tthis->broker.get_raw_ptr()->error = KAFKA_SASL_AUTHENTICATION_FAILED;\n\t\treturn 1;\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::handle_sasl_continue()\n{\n\tstruct iovec iovecs[64];\n\tint ret;\n\tint cnt = this->encode(iovecs, 64);\n\tif ((unsigned int)cnt > 64)\n\t{\n\t\tif (cnt > 64)\n\t\t\terrno = EOVERFLOW;\n\t\treturn -1;\n\t}\n\n\tfor (int i = 0; i < cnt; i++)\n\t{\n\t\tret = this->feedback(iovecs[i].iov_base, iovecs[i].iov_len);\n\t\tif (ret != (int)iovecs[i].iov_len)\n\t\t{\n\t\t\tif (ret >= 0)\n\t\t\t\terrno = ENOBUFS;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nint KafkaResponse::append(const void *buf, size_t *size)\n{\n\tint ret = KafkaMessage::append(buf, size);\n\n\tif (ret <= 0)\n\t\treturn ret;\n\n\tret = this->parse_response();\n\tif (ret != 0)\n\t\treturn ret;\n\n\tif (this->api_type == Kafka_SaslHandshake)\n\t{\n\t\tthis->api_type = Kafka_SaslAuthenticate;\n\t\tthis->clear_buf();\n\t\treturn this->handle_sasl_continue();\n\t}\n\telse if (this->api_type == Kafka_SaslAuthenticate)\n\t{\n\t\tif (strncasecmp(this->config.get_sasl_mech(), \"SCRAM\", 5) == 0)\n\t\t{\n\t\t\tthis->clear_buf();\n\t\t\tif (this->sasl->scram.state !=\n\t\t\t\t\tKAFKA_SASL_SCRAM_STATE_CLIENT_FINISHED)\n\t\t\t\treturn this->handle_sasl_continue();\n\t\t\telse\n\t\t\t\tthis->sasl->status = 1;\n\t\t}\n\t}\n\n\treturn 1;\n}\n\n}\n\n" }, { "path": "src/protocol/KafkaMessage.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n\t http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Wang Zhulei(wangzhulei@sogou-inc.com)\n*/\n\n#ifndef _KAFKAMESSAGE_H_\n#define _KAFKAMESSAGE_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \"kafka_parser.h\"\n#include \"ProtocolMessage.h\"\n#include \"KafkaDataTypes.h\"\n\nnamespace protocol\n{\n\nclass KafkaMessage : public ProtocolMessage\n{\npublic:\n\tKafkaMessage();\n\n\tvirtual ~KafkaMessage();\n\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\nprivate:\n\tint encode_head();\n\npublic:\n\tKafkaMessage(KafkaMessage&& msg);\n\tKafkaMessage& operator= (KafkaMessage&& msg);\n\npublic:\n\tint encode_message(int api_type, struct iovec vectors[], int max);\n\n\tvoid set_api_type(int api_type) { this->api_type = api_type; }\n\tint get_api_type() const { return this->api_type; }\n\n\tvoid set_api_version(int ver) { this->api_version = ver; }\n\tint get_api_version() const { return this->api_version; }\n\n\tvoid set_correlation_id(int id) { this->correlation_id = id; }\n\tint get_correlation_id() const { return this->correlation_id; }\n\n\tvoid set_config(const KafkaConfig& conf)\n\t{\n\t\tthis->config = conf;\n\t}\n\tconst KafkaConfig *get_config() const { return &this->config; }\n\n\tvoid set_cgroup(const KafkaCgroup& cgroup)\n\t{\n\t\tthis->cgroup = cgroup;\n\t}\n\tKafkaCgroup *get_cgroup()\n\t{\n\t\treturn &this->cgroup;\n\t}\n\n\tvoid set_broker(const KafkaBroker& broker)\n\t{\n\t\tthis->broker = broker;\n\t}\n\tKafkaBroker *get_broker()\n\t{\n\t\treturn &this->broker;\n\t}\n\n\tvoid set_meta_list(const KafkaMetaList& meta_list)\n\t{\n\t\tthis->meta_list = meta_list;\n\t}\n\tKafkaMetaList *get_meta_list()\n\t{\n\t\treturn &this->meta_list;\n\t}\n\n\tvoid set_toppar_list(const KafkaTopparList& toppar_list)\n\t{\n\t\tthis->toppar_list = toppar_list;\n\t}\n\tKafkaTopparList *get_toppar_list()\n\t{\n\t\treturn &this->toppar_list;\n\t}\n\n\tvoid set_broker_list(const KafkaBrokerList& broker_list)\n\t{\n\t\tthis->broker_list = broker_list;\n\t}\n\tKafkaBrokerList *get_broker_list()\n\t{\n\t\treturn &this->broker_list;\n\t}\n\n\tvoid set_sasl(kafka_sasl_t *sasl)\n\t{\n\t\tthis->sasl = sasl;\n\t}\n\n\tvoid set_api(kafka_api_t *api)\n\t{\n\t\tthis->api = api;\n\t}\n\n\tvoid duplicate(const KafkaMessage& msg)\n\t{\n\t\tthis->config = msg.config;\n\t\tthis->cgroup = msg.cgroup;\n\t\tthis->broker = msg.broker;\n\t\tthis->meta_list = msg.meta_list;\n\t\tthis->broker_list = msg.broker_list;\n\t\tthis->toppar_list = msg.toppar_list;\n\t\tthis->sasl = msg.sasl;\n\t\tthis->api = msg.api;\n\t}\n\n\tvoid clear_buf()\n\t{\n\t\tthis->msgbuf.clear();\n\t\tthis->headbuf.clear();\n\t\tkafka_parser_deinit(this->parser);\n\t\tkafka_parser_init(this->parser);\n\t\tthis->cur_size = 0;\n\t\tthis->serialized = KafkaBuffer();\n\t\tthis->uncompressed = KafkaBuffer();\n\t}\n\nprotected:\n\tstatic int parse_message_set(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t bool check_crcs, int msg_vers,\n\t\t\t\t\t\t\t\t struct list_head *record_list,\n\t\t\t\t\t\t\t\t KafkaBuffer *uncompressed,\n\t\t\t\t\t\t\t\t KafkaToppar *toppar);\n\n\tstatic int parse_message_record(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t\tkafka_record_t *kafka_record);\n\n\tstatic int parse_record_batch(void **buf, size_t *size,\n\t\t\t\t\t\t\t\t bool check_crcs,\n\t\t\t\t\t\t\t\t struct list_head *record_list,\n\t\t\t\t\t\t\t\t KafkaBuffer *uncompressed,\n\t\t\t\t\t\t\t\t KafkaToppar *toppar);\n\n\tstatic int parse_records(void **buf, size_t *size, bool check_crcs,\n\t\t\t\t\t\t\t KafkaBuffer *uncompressed, KafkaToppar *toppar);\n\n\tstatic std::string get_member_assignment(kafka_member_t *member);\n\n\tstatic KafkaToppar *find_toppar_by_name(const std::string& topic, int partition,\n\t\t\t\t\t\t\t\t\t\t\tstruct list_head *toppar_list);\n\n\tstatic KafkaToppar *find_toppar_by_name(const std::string& topic, int partition,\n\t\t\t\t\t\t\t\t\t\t\tKafkaTopparList *toppar_list);\n\n\tstatic int kafka_parse_member_assignment(const char *bbuf, size_t n,\n\t\t\t\t\t\t\t\t\t\t\t KafkaCgroup *cgroup);\n\nprotected:\n\tkafka_parser_t *parser;\n\tusing encode_func = std::function;\n\tstd::map encode_func_map;\n\n\tusing parse_func = std::function;\n\tstd::map parse_func_map;\n\n\tclass EncodeStream *stream;\n\tstd::string msgbuf;\n\tstd::string headbuf;\n\n\tKafkaConfig config;\n\tKafkaCgroup cgroup;\n\tKafkaBroker broker;\n\tKafkaMetaList meta_list;\n\tKafkaBrokerList broker_list;\n\tKafkaTopparList toppar_list;\n\tKafkaBuffer serialized;\n\tKafkaBuffer uncompressed;\n\n\tint api_type;\n\tint api_version;\n\tint correlation_id;\n\tint message_version;\n\n\tvoid *compress_env;\n\tsize_t cur_size;\n\n\tkafka_sasl_t *sasl;\n\tkafka_api_t *api;\n};\n\nclass KafkaRequest : public KafkaMessage\n{\npublic:\n\tKafkaRequest();\n\nprivate:\n\tint encode_produce(struct iovec vectors[], int max);\n\tint encode_fetch(struct iovec vectors[], int max);\n\tint encode_metadata(struct iovec vectors[], int max);\n\tint encode_findcoordinator(struct iovec vectors[], int max);\n\tint encode_listoffset(struct iovec vectors[], int max);\n\tint encode_joingroup(struct iovec vectors[], int max);\n\tint encode_syncgroup(struct iovec vectors[], int max);\n\tint encode_leavegroup(struct iovec vectors[], int max);\n\tint encode_heartbeat(struct iovec vectors[], int max);\n\tint encode_offsetcommit(struct iovec vectors[], int max);\n\tint encode_offsetfetch(struct iovec vectors[], int max);\n\tint encode_apiversions(struct iovec vectors[], int max);\n\tint encode_saslhandshake(struct iovec vectors[], int max);\n\tint encode_saslauthenticate(struct iovec vectors[], int max);\n};\n\nclass KafkaResponse : public KafkaRequest\n{\npublic:\n\tKafkaResponse();\n\n\tint parse_response();\n\nprotected:\n virtual int append(const void *buf, size_t *size);\n\nprivate:\n\tint parse_produce(void **buf, size_t *size);\n\tint parse_fetch(void **buf, size_t *size);\n\tint parse_metadata(void **buf, size_t *size);\n\tint parse_findcoordinator(void **buf, size_t *size);\n\tint parse_joingroup(void **buf, size_t *size);\n\tint parse_syncgroup(void **buf, size_t *size);\n\tint parse_leavegroup(void **buf, size_t *size);\n\tint parse_listoffset(void **buf, size_t *size);\n\tint parse_offsetcommit(void **buf, size_t *size);\n\tint parse_offsetfetch(void **buf, size_t *size);\n\tint parse_heartbeat(void **buf, size_t *size);\n\tint parse_apiversions(void **buf, size_t *size);\n\tint parse_saslhandshake(void **buf, size_t *size);\n\tint parse_saslauthenticate(void **buf, size_t *size);\n\n\tint handle_sasl_continue();\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/KafkaResult.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#include \"KafkaResult.h\"\n\nnamespace protocol\n{\n\nenum\n{\n\tKAFKA_STATUS_GET_RESULT,\n\tKAFKA_STATUS_END,\n};\n\nKafkaResult::~KafkaResult()\n{\n\tdelete []this->resp_vec;\n}\n\nKafkaResult& KafkaResult::operator= (KafkaResult&& move)\n{\n\tif (this != &move)\n\t{\n\t\tdelete []this->resp_vec;\n\n\t\tthis->resp_vec = move.resp_vec;\n\t\tmove.resp_vec = NULL;\n\n\t\tthis->resp_num = move.resp_num;\n\t\tmove.resp_num = 0;\n\t}\n\n\treturn *this;\n}\n\nKafkaResult::KafkaResult(KafkaResult&& move)\n{\n\tthis->resp_vec = move.resp_vec;\n\tmove.resp_vec = NULL;\n\n\tthis->resp_num = move.resp_num;\n\tmove.resp_num = 0;\n}\n\nvoid KafkaResult::create(size_t n)\n{\n\tdelete []this->resp_vec;\n\tthis->resp_vec = new KafkaResponse[n];\n\tthis->resp_num = n;\n}\n\nvoid KafkaResult::set_resp(KafkaResponse&& resp, size_t i)\n{\n\tassert(i < this->resp_num);\n\tthis->resp_vec[i] = std::move(resp);\n}\n\nvoid KafkaResult::fetch_toppars(std::vector& toppars)\n{\n\ttoppars.clear();\n\n\tKafkaToppar *toppar = NULL;\n\tfor (size_t i = 0; i < this->resp_num; ++i)\n\t{\n\t\tthis->resp_vec[i].get_toppar_list()->rewind();\n\n\t\twhile ((toppar = this->resp_vec[i].get_toppar_list()->get_next()) != NULL)\n\t\t\ttoppars.push_back(toppar);\n\t}\n}\n\nvoid KafkaResult::fetch_records(std::vector>& records)\n{\n\trecords.clear();\n\n\tKafkaToppar *toppar = NULL;\n\tKafkaRecord *record = NULL;\n\n\tfor (size_t i = 0; i < this->resp_num; ++i)\n\t{\n\t\tif (this->resp_vec[i].get_api_type() != Kafka_Produce &&\n\t\t\tthis->resp_vec[i].get_api_type() != Kafka_Fetch)\n\t\t\tcontinue;\n\n\t\tthis->resp_vec[i].get_toppar_list()->rewind();\n\n\t\twhile ((toppar = this->resp_vec[i].get_toppar_list()->get_next()) != NULL)\n\t\t{\n\t\t\tstd::vector tmp;\n\t\t\ttoppar->record_rewind();\n\n\t\t\twhile ((record = toppar->get_record_next()) != NULL)\n\t\t\t\ttmp.push_back(record);\n\n\t\t\tif (!tmp.empty())\n\t\t\t\trecords.emplace_back(std::move(tmp));\n\t\t}\n\t}\n}\n\n}\n\n" }, { "path": "src/protocol/KafkaResult.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#ifndef _KAFKARESULT_H_\n#define _KAFKARESULT_H_\n\n#include \n#include \n#include \n#include \"KafkaMessage.h\"\n#include \"KafkaDataTypes.h\"\n\nnamespace protocol\n{\n\nclass KafkaResult\n{\npublic:\n\t// for offsetcommit\n\tvoid fetch_toppars(std::vector& toppars);\n\n\t// for produce, fetch\n\tvoid fetch_records(std::vector>& records);\n\npublic:\n\tvoid create(size_t n);\n\n\tvoid set_resp(KafkaResponse&& resp, size_t i);\n\npublic:\n\tKafkaResult()\n\t{\n\t\tthis->resp_vec = NULL;\n\t\tthis->resp_num = 0;\n\t}\n\n\tvirtual ~KafkaResult();\n\n\tKafkaResult& operator= (KafkaResult&& move);\n\n\tKafkaResult(KafkaResult&& move);\n\nprivate:\n\tKafkaResponse *resp_vec;\n\tsize_t resp_num;\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/MySQLMessage.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"SSLWrapper.h\"\n#include \"mysql_byteorder.h\"\n#include \"mysql_types.h\"\n#include \"MySQLResult.h\"\n#include \"MySQLMessage.h\"\n\nnamespace protocol\n{\n\n#define MYSQL_PAYLOAD_MAX\t((1 << 24) - 1)\n\n#define MYSQL_NATIVE_PASSWORD\t\"mysql_native_password\"\n#define CACHING_SHA2_PASSWORD\t\"caching_sha2_password\"\n#define MYSQL_CLEAR_PASSWORD\t\"mysql_clear_password\"\n\nMySQLMessage::~MySQLMessage()\n{\n\tif (parser_)\n\t{\n\t\tmysql_parser_deinit(parser_);\n\t\tmysql_stream_deinit(stream_);\n\t\tdelete parser_;\n\t\tdelete stream_;\n\t}\n}\n\nMySQLMessage::MySQLMessage(MySQLMessage&& move) :\n\tProtocolMessage(std::move(move))\n{\n\tparser_ = move.parser_;\n\tstream_ = move.stream_;\n\tseqid_ = move.seqid_;\n\tcur_size_ = move.cur_size_;\n\n\tmove.parser_ = NULL;\n\tmove.stream_ = NULL;\n\tmove.seqid_ = 0;\n\tmove.cur_size_ = 0;\n}\n\nMySQLMessage& MySQLMessage::operator= (MySQLMessage&& move)\n{\n\tif (this != &move)\n\t{\n\t\t*(ProtocolMessage *)this = std::move(move);\n\n\t\tif (parser_)\n\t\t{\n\t\t\tmysql_parser_deinit(parser_);\n\t\t\tmysql_stream_deinit(stream_);\n\t\t\tdelete parser_;\n\t\t\tdelete stream_;\n\t\t}\n\n\t\tparser_ = move.parser_;\n\t\tstream_ = move.stream_;\n\t\tseqid_ = move.seqid_;\n\t\tcur_size_ = move.cur_size_;\n\n\t\tmove.parser_ = NULL;\n\t\tmove.stream_ = NULL;\n\t\tmove.seqid_ = 0;\n\t\tmove.cur_size_ = 0;\n\t}\n\n\treturn *this;\n}\n\nint MySQLMessage::append(const void *buf, size_t *size)\n{\n\tconst void *stream_buf;\n\tsize_t stream_len;\n\tsize_t nleft = *size;\n\tsize_t n;\n\tint ret;\n\n\tcur_size_ += *size;\n\tif (cur_size_ > this->size_limit)\n\t{\n\t\terrno = EMSGSIZE;\n\t\treturn -1;\n\t}\n\n\tdo\n\t{\n\t\tn = nleft;\n\t\tret = mysql_stream_write(buf, &n, stream_);\n\t\tif (ret > 0)\n\t\t{\n\t\t\tseqid_ = mysql_stream_get_seq(stream_);\n\t\t\tmysql_stream_get_buf(&stream_buf, &stream_len, stream_);\n\t\t\tret = decode_packet((const unsigned char *)stream_buf, stream_len);\n\t\t\tif (ret == -2)\n\t\t\t\terrno = EBADMSG;\n\t\t}\n\n\t\tif (ret < 0)\n\t\t\treturn -1;\n\n\t\tbuf = (const char *)buf + n;\n\t\tnleft -= n;\n\t} while (nleft > 0);\n\n\treturn ret;\n}\n\nint MySQLMessage::encode(struct iovec vectors[], int max)\n{\n\tconst unsigned char *p = (unsigned char *)buf_.c_str();\n\tsize_t nleft = buf_.size();\n\tuint8_t seqid_start = seqid_;\n\tuint8_t seqid = seqid_;\n\tunsigned char *head;\n\tuint32_t length;\n\tint i = 0;\n\n\tdo\n\t{\n\t\tlength = (nleft >= MYSQL_PAYLOAD_MAX ? MYSQL_PAYLOAD_MAX\n\t\t\t\t\t\t\t\t\t\t\t : (uint32_t)nleft);\n\t\thead = heads_[seqid];\n\t\tint3store(head, length);\n\t\thead[3] = seqid++;\n\t\tvectors[i].iov_base = head;\n\t\tvectors[i].iov_len = 4;\n\t\ti++;\n\t\tvectors[i].iov_base = const_cast(p);\n\t\tvectors[i].iov_len = length;\n\t\ti++;\n\n\t\tif (i > max)//overflow\n\t\t\tbreak;\n\n\t\tif (nleft < MYSQL_PAYLOAD_MAX)\n\t\t\treturn i;\n\n\t\tnleft -= MYSQL_PAYLOAD_MAX;\n\t\tp += length;\n\t} while (seqid != seqid_start);\n\n\terrno = EOVERFLOW;\n\treturn -1;\n}\n\nvoid MySQLRequest::set_query(const char *query, size_t length)\n{\n\tset_command(MYSQL_COM_QUERY);\n\tbuf_.resize(length + 1);\n\tchar *buffer = const_cast(buf_.c_str());\n\n\tbuffer[0] = MYSQL_COM_QUERY;\n\tif (length > 0)\n\t\tmemcpy(buffer + 1, query, length);\n}\n\nstd::string MySQLRequest::get_query() const\n{\n\tsize_t len = buf_.size();\n\tif (len <= 1 || buf_[0] != MYSQL_COM_QUERY)\n\t\treturn \"\";\n\n\treturn std::string(buf_.c_str() + 1);\n}\n\n#define MYSQL_CAPFLAG_CLIENT_SSL\t\t\t\t0x00000800\n#define MYSQL_CAPFLAG_CLIENT_PROTOCOL_41\t\t0x00000200\n#define MYSQL_CAPFLAG_CLIENT_SECURE_CONNECTION\t0x00008000\n#define MYSQL_CAPFLAG_CLIENT_CONNECT_WITH_DB\t0x00000008\n#define MYSQL_CAPFLAG_CLIENT_MULTI_STATEMENTS\t0x00010000\n#define MYSQL_CAPFLAG_CLIENT_MULTI_RESULTS\t\t0x00020000\n#define MYSQL_CAPFLAG_CLIENT_PS_MULTI_RESULTS\t0x00040000\n#define MYSQL_CAPFLAG_CLIENT_PLUGIN_AUTH\t\t0x00080000\n#define MYSQL_CAPFLAG_CLIENT_LOCAL_FILES\t\t0x00000080\n\nint MySQLHandshakeResponse::encode(struct iovec vectors[], int max)\n{\n\tconst char empty[10] = {0};\n\tuint16_t cap_flags_lower = capability_flags_ & 0xffffffff;\n\tuint16_t cap_flags_upper = capability_flags_ >> 16;\n\n\tbuf_.clear();\n\tbuf_.append((const char *)&protocol_version_, 1);\n\tbuf_.append(server_version_.c_str(), server_version_.size() + 1);\n\tbuf_.append((const char *)&connection_id_, 4);\n\tbuf_.append((const char *)auth_plugin_data_, 8);\n\tbuf_.append(empty, 1);\n\tbuf_.append((const char *)&cap_flags_lower, 2);\n\tbuf_.append((const char *)&character_set_, 1);\n\tbuf_.append((const char *)&status_flags_, 2);\n\tbuf_.append((const char *)&cap_flags_upper, 2);\n\tbuf_.push_back(21);\n\tbuf_.append(empty, 10);\n\tbuf_.append((const char *)auth_plugin_data_ + 8, 12);\n\tbuf_.push_back(0);\n\tif (capability_flags_ & MYSQL_CAPFLAG_CLIENT_PLUGIN_AUTH)\n\t\tbuf_.append(MYSQL_NATIVE_PASSWORD, strlen(MYSQL_NATIVE_PASSWORD) + 1);\n\n\treturn MySQLMessage::encode(vectors, max);\n}\n\nint MySQLHandshakeResponse::decode_packet(const unsigned char *buf, size_t buflen)\n{\n\tconst unsigned char *end = buf + buflen;\n\tconst unsigned char *pos;\n\tuint16_t cap_flags_lower;\n\tuint16_t cap_flags_upper;\n\n\tif (buflen == 0)\n\t\treturn -2;\n\n\tprotocol_version_ = *buf;\n\tif (protocol_version_ == 255)\n\t{\n\t\tif (buflen >= 4)\n\t\t{\n\t\t\tconst_cast(buf)[3] = '#';\n\t\t\tif (mysql_parser_parse(buf, buflen, parser_) == 1)\n\t\t\t{\n\t\t\t\tdisallowed_ = true;\n\t\t\t\treturn 1;\n\t\t\t}\n\t\t}\n\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tpos = ++buf;\n\twhile (pos < end && *pos)\n\t\tpos++;\n\n\tif (pos >= end || end - pos < 45)\n\t\treturn -2;\n\n\tserver_version_.assign((const char *)buf, pos - buf);\n\tbuf = pos + 1;\n\n\tconnection_id_ = uint4korr(buf);\n\tbuf += 4;\n\tmemcpy(auth_plugin_data_, buf, 8);\n\tbuf += 9;\n\tcap_flags_lower = uint2korr(buf);\n\tbuf += 2;\n\tcharacter_set_ = *buf++;\n\tstatus_flags_ = uint2korr(buf);\n\tbuf += 2;\n\tcap_flags_upper = uint2korr(buf);\n\tbuf += 2;\n\tcapability_flags_ = (cap_flags_upper << 16U) + cap_flags_lower;\n\tauth_plugin_data_len_ = *buf++;\n\t// 10 bytes reserved. All 0s.\n\tbuf += 10;\n\t// auth_plugin_data always 20 bytes\n\tif (auth_plugin_data_len_ > 21)\n\t\treturn -2;\n\n\tmemcpy(auth_plugin_data_ + 8, buf, 12);\n\tbuf += 13;\n\tif (capability_flags_ & MYSQL_CAPFLAG_CLIENT_PLUGIN_AUTH)\n\t{\n\t\tif (buf == end || *(end - 1) != '\\0')\n\t\t\treturn -2;\n\n\t\tauth_plugin_name_.assign((const char *)buf, end - 1 - buf);\n\t}\n\n\treturn 1;\n}\n\nstatic std::string __native_password_encrypt(const std::string& password,\n\t\t\t\t\t\t\t\t\t\t\t unsigned char seed[20])\n{\n\tunsigned char buf1[20];\n\tunsigned char buf2[40];\n\tint i;\n\n\t// SHA1( password ) ^ SHA1( seed + SHA1( SHA1( password ) ) )\n\tSHA1((unsigned char *)password.c_str(), password.size(), buf1);\n\tSHA1(buf1, 20, buf2 + 20);\n\tmemcpy(buf2, seed, 20);\n\tSHA1(buf2, 40, buf2);\n\tfor (i = 0; i < 20; i++)\n\t\tbuf1[i] ^= buf2[i];\n\n\treturn std::string((const char *)buf1, 20);\n}\n\nstatic std::string __caching_sha2_password_encrypt(const std::string& password,\n\t\t\t\t\t\t\t\t\t\t\t\t unsigned char seed[20])\n{\n\tunsigned char buf1[32];\n\tunsigned char buf2[52];\n\tint i;\n\n\t// SHA256( password ) ^ SHA256( SHA256( SHA256( password ) ) + seed)\n\tSHA256((unsigned char *)password.c_str(), password.size(), buf1);\n\tSHA256(buf1, 32, buf2);\n\tmemcpy(buf2 + 32, seed, 20);\n\tSHA256(buf2, 52, buf2);\n\tfor (i = 0; i < 32; i++)\n\t\tbuf1[i] ^= buf2[i];\n\n\treturn std::string((const char *)buf1, 32);\n}\n\nint MySQLSSLRequest::encode(struct iovec vectors[], int max)\n{\n\tunsigned char header[32] = {0};\n\tunsigned char *pos = header;\n\tint ret;\n\n\tint4store(pos, MYSQL_CAPFLAG_CLIENT_SSL |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_PROTOCOL_41 |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_SECURE_CONNECTION |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_CONNECT_WITH_DB |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_MULTI_RESULTS|\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_LOCAL_FILES |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_MULTI_STATEMENTS |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_PS_MULTI_RESULTS |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_PLUGIN_AUTH);\n\tpos += 4;\n\tint4store(pos, 0);\n\tpos += 4;\n\t*pos = (uint8_t)character_set_;\n\n\tbuf_.clear();\n\tbuf_.append((char *)header, 32);\n\tret = MySQLMessage::encode(vectors, max);\n\tif (ret >= 0)\n\t{\n\t\tmax -= ret;\n\t\tif (max >= 8) /* Indeed SSL handshaker needs only 1 vector. */\n\t\t{\n\t\t\tmax = ssl_handshaker_.encode(vectors + ret, max);\n\t\t\tif (max >= 0)\n\t\t\t\treturn max + ret;\n\t\t}\n\t\telse\n\t\t\terrno = EOVERFLOW;\n\t}\n\n\treturn -1;\n}\n\nint MySQLAuthRequest::encode(struct iovec vectors[], int max)\n{\n\tunsigned char header[32] = {0};\n\tunsigned char *pos = header;\n\tstd::string str;\n\n\tint4store(pos, MYSQL_CAPFLAG_CLIENT_PROTOCOL_41 |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_SECURE_CONNECTION |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_CONNECT_WITH_DB |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_MULTI_RESULTS|\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_LOCAL_FILES |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_MULTI_STATEMENTS |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_PS_MULTI_RESULTS |\n\t\t\t\t MYSQL_CAPFLAG_CLIENT_PLUGIN_AUTH);\n\tpos += 4;\n\tint4store(pos, 0);\n\tpos += 4;\n\t*pos = (uint8_t)character_set_;\n\n\tif (password_.empty())\n\t\tstr.push_back(0);\n\telse if (auth_plugin_name_ == CACHING_SHA2_PASSWORD)\n\t{\n\t\tstr.push_back(32);\n\t\tstr += __caching_sha2_password_encrypt(password_, seed_);\n\t}\n\telse\n\t{\n\t\tstr.push_back(20);\n\t\tstr += __native_password_encrypt(password_, seed_);\n\t}\n\n\tbuf_.clear();\n\tbuf_.append((char *)header, 32);\n\tbuf_.append(username_.c_str(), username_.size() + 1);\n\tbuf_.append(str);\n\tbuf_.append(db_.c_str(), db_.size() + 1);\n\tif (auth_plugin_name_.size() != 0)\n\t\tbuf_.append(auth_plugin_name_.c_str(), auth_plugin_name_.size() + 1);\n\n\treturn MySQLMessage::encode(vectors, max);\n}\n\nint MySQLAuthRequest::decode_packet(const unsigned char *buf, size_t buflen)\n{\n\tconst unsigned char *end = buf + buflen;\n\tconst unsigned char *pos;\n\n\tif (buflen < 32)\n\t\treturn -2;\n\n\tuint32_t flags = uint4korr(buf);\n\n\tif (!(flags & MYSQL_CAPFLAG_CLIENT_PROTOCOL_41))\n\t\treturn -2;\n\n\tbuf += 8;\n\tcharacter_set_ = *buf++;\n\tbuf += 23;\n\n\tpos = buf;\n\twhile (pos < end && *pos)\n\t\tpos++;\n\n\tif (pos >= end)\n\t\treturn -2;\n\n\tusername_.assign((const char *)buf, pos - buf);\n\tbuf = pos + 1;\n\n\treturn 1;\n}\n\nint MySQLAuthResponse::decode_packet(const unsigned char *buf, size_t buflen)\n{\n\tconst unsigned char *end = buf + buflen;\n\tconst unsigned char *pos;\n\tconst unsigned char *str;\n\tunsigned long long len;\n\n\tif (end == buf)\n\t\treturn -2;\n\n\tswitch (*buf)\n\t{\n\tcase 0x00:\n\tcase 0xff:\n\t\treturn MySQLResponse::decode_packet(buf, buflen);\n\n\tcase 0xfe:\n\t\tpos = ++buf;\n\t\twhile (pos < end && *pos)\n\t\t\tpos++;\n\n\t\tif (pos >= end)\n\t\t\treturn -2;\n\n\t\tauth_plugin_name_.assign((const char *)buf, pos - buf);\n\t\tbuf = pos + 1;\n\t\tif (buf == end || *(end - 1) != '\\0')\n\t\t\treturn -2;\n\n\t\tif (end - 1 - buf != 20)\n\t\t\treturn -2;\n\n\t\tmemcpy(seed_, buf, 20);\n\t\treturn 1;\n\n\tdefault:\n\t\tpos = buf;\n\t\tif (decode_string(&str, &len, &pos, end) > 0 && len == 1)\n\t\t{\n\t\t\tif (*str == 0x03)\n\t\t\t{\n\t\t\t\tif (end > pos)\n\t\t\t\t\treturn MySQLResponse::decode_packet(pos, end - pos);\n\t\t\t\telse\n\t\t\t\t\treturn 0;\n\t\t\t}\n\t\t\telse if (*str == 0x04)\n\t\t\t{\n\t\t\t\tcontinue_ = true;\n\t\t\t\treturn 1;\n\t\t\t}\n\t\t}\n\n\t\treturn -2;\n\t}\n}\n\nint MySQLAuthSwitchRequest::encode(struct iovec vectors[], int max)\n{\n\tif (password_.empty())\n\t{\n\t\tbuf_ = \"\\0\";\n\t}\n\telse if (auth_plugin_name_ == MYSQL_NATIVE_PASSWORD)\n\t{\n\t\tbuf_ = __native_password_encrypt(password_, seed_);\n\t}\n\telse if (auth_plugin_name_ == CACHING_SHA2_PASSWORD)\n\t{\n\t\tbuf_ = __caching_sha2_password_encrypt(password_, seed_);\n\t}\n\telse if (auth_plugin_name_ == MYSQL_CLEAR_PASSWORD)\n\t{\n\t\tbuf_ = password_;\n\t\tbuf_.push_back('\\0');\n\t}\n\telse\n\t{\n\t\terrno = EINVAL;\n\t\treturn -1;\n\t}\n\n\treturn MySQLMessage::encode(vectors, max);\n}\n\nint MySQLPublicKeyResponse::decode_packet(const unsigned char *buf,\n\t\t\t\t\t\t\t\t\t\t size_t buflen)\n{\n\tif (buflen == 0 || *buf != 0x01)\n\t\treturn -2;\n\n\tif (buflen == 1)\n\t\treturn 0;\n\n\tpublic_key_.assign((const char *)buf + 1, buflen - 1);\n\treturn 1;\n}\n\nint MySQLPublicKeyResponse::encode(struct iovec vectors[], int max)\n{\n\tbuf_.clear();\n\tbuf_.push_back(0x01);\n\tbuf_ += public_key_;\n\treturn MySQLMessage::encode(vectors, max);\n}\n\nint MySQLRSAAuthRequest::rsa_encrypt(void *ctx)\n{\n\tEVP_PKEY_CTX *pkey_ctx = (EVP_PKEY_CTX *)ctx;\n\tunsigned char out[256];\n\tsize_t outlen = 256;\n\tstd::string pass;\n\tunsigned char *p;\n\tsize_t i;\n\n\tif (EVP_PKEY_encrypt_init(pkey_ctx) > 0 &&\n\t\tEVP_PKEY_CTX_set_rsa_padding(pkey_ctx, RSA_PKCS1_OAEP_PADDING) > 0)\n\t{\n\t\tpass.reserve(password_.size() + 1);\n\t\tp = (unsigned char *)pass.c_str();\n\t\tfor (i = 0; i <= password_.size(); i++)\n\t\t\tp[i] = (unsigned char)password_[i] ^ seed_[i % 20];\n\n\t\tif (EVP_PKEY_encrypt(pkey_ctx, out, &outlen, p, i) > 0)\n\t\t{\n\t\t\tbuf_.assign((char *)out, 256);\n\t\t\treturn 0;\n\t\t}\n\t}\n\n\treturn -1;\n}\n\nint MySQLRSAAuthRequest::encode(struct iovec vectors[], int max)\n{\n\tBIO *bio;\n\tEVP_PKEY *pkey;\n\tEVP_PKEY_CTX *pkey_ctx;\n\tint ret = -1;\n\n\tbio = BIO_new_mem_buf(public_key_.c_str(), public_key_.size());\n\tif (bio)\n\t{\n\t\tpkey = PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL);\n\t\tif (pkey)\n\t\t{\n\t\t\tpkey_ctx = EVP_PKEY_CTX_new(pkey, NULL);\n\t\t\tif (pkey_ctx)\n\t\t\t{\n\t\t\t\tret = rsa_encrypt(pkey_ctx);\n\t\t\t\tEVP_PKEY_CTX_free(pkey_ctx);\n\t\t\t}\n\n\t\t\tEVP_PKEY_free(pkey);\n\t\t}\n\n\t\tBIO_free(bio);\n\t}\n\n\tif (ret < 0)\n\t\treturn ret;\n\n\treturn MySQLMessage::encode(vectors, max);\n}\n\nvoid MySQLResponse::set_ok_packet()\n{\n\tuint16_t zero16 = 0;\n\tbuf_.clear();\n\tbuf_.push_back(0x00);\n\tbuf_.append((const char *)&zero16, 2);\n\tbuf_.append((const char *)&zero16, 2);\n\tbuf_.append((const char *)&zero16, 2);\n}\n\nint MySQLResponse::decode_packet(const unsigned char *buf, size_t buflen)\n{\n\treturn mysql_parser_parse(buf, buflen, parser_);\n}\n\nunsigned long long MySQLResponse::get_affected_rows() const\n{\n\tunsigned long long affected_rows = 0;\n\tMySQLResultCursor cursor(this);\n\n\tdo {\n\t\taffected_rows += cursor.get_affected_rows();\n\t} while (cursor.next_result_set());\n\n\treturn affected_rows;\n}\n\n// return array api\nunsigned long long MySQLResponse::get_last_insert_id() const\n{\n\tunsigned long long insert_id = 0;\n\tMySQLResultCursor cursor(this);\n\n\tdo {\n\t\tif (cursor.get_insert_id())\n\t\t\tinsert_id = cursor.get_insert_id();\n\t} while (cursor.next_result_set());\n\n\treturn insert_id;\n}\n\nint MySQLResponse::get_warnings() const\n{\n\tint warning_count = 0;\n\tMySQLResultCursor cursor(this);\n\n\tdo {\n\t\twarning_count += cursor.get_warnings();\n\t} while (cursor.next_result_set());\n\n\treturn warning_count;\n}\n\nstd::string MySQLResponse::get_info() const\n{\n\tstd::string info;\n\tMySQLResultCursor cursor(this);\n\n\tdo {\n\t\tif (info.length() > 0)\n\t\t\tinfo += \" \";\n\t\tinfo += cursor.get_info();\n\t} while (cursor.next_result_set());\n\n\treturn info;\n}\n\nbool MySQLResponse::is_ok_packet() const\n{\n\treturn parser_->packet_type == MYSQL_PACKET_OK;\n}\n\nbool MySQLResponse::is_error_packet() const\n{\n\treturn parser_->packet_type == MYSQL_PACKET_ERROR;\n}\n\n}\n\n" }, { "path": "src/protocol/MySQLMessage.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _MYSQLMESSAGE_H_\n#define _MYSQLMESSAGE_H_\n\n#include \n#include \n#include \"ProtocolMessage.h\"\n#include \"mysql_stream.h\"\n#include \"mysql_parser.h\"\n\n/**\n * @file MySQLMessage.h\n * @brief MySQL Protocol Interface\n */\n\nnamespace protocol\n{\n\nclass MySQLMessage : public ProtocolMessage\n{\npublic:\n\tmysql_parser_t *get_parser() const;\n\tint get_seqid() const;\n\tvoid set_seqid(int seqid);\n\tint get_command() const;\n\nprotected:\n\tvirtual int append(const void *buf, size_t *size);\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen) { return 1; }\n\n\tvoid set_command(int cmd) const;\n\n\t//append\n\tmysql_stream_t *stream_;\n\tmysql_parser_t *parser_;\n\n\t//encode\n\tunsigned char heads_[256][4];\n\tuint8_t seqid_;\n\tstd::string buf_;\n\tsize_t cur_size_;\n\npublic:\n\tMySQLMessage();\n\tvirtual ~MySQLMessage();\n\t//move constructor\n\tMySQLMessage(MySQLMessage&& move);\n\t//move operator\n\tMySQLMessage& operator= (MySQLMessage&& move);\n};\n\nclass MySQLRequest : public MySQLMessage\n{\npublic:\n\tvoid set_query(const char *query);\n\tvoid set_query(const std::string& query);\n\tvoid set_query(const char *query, size_t length);\n\n\tstd::string get_query() const;\n\tbool query_is_unset() const;\n\npublic:\n\tMySQLRequest() = default;\n\t//move constructor\n\tMySQLRequest(MySQLRequest&& move) = default;\n\t//move operator\n\tMySQLRequest& operator= (MySQLRequest&& move) = default;\n};\n\nclass MySQLResponse : public MySQLMessage\n{\npublic:\n\tbool is_ok_packet() const;\n\tbool is_error_packet() const;\n\tint get_packet_type() const;\n\n\tunsigned long long get_affected_rows() const;\n\tunsigned long long get_last_insert_id() const;\n\tint get_warnings() const;\n\tint get_error_code() const;\n\tstd::string get_error_msg() const;\n\tstd::string get_sql_state() const;\n\tstd::string get_info() const;\n\n\tvoid set_ok_packet();\n\npublic:\n\tMySQLResponse() = default;\n\t//move constructor\n\tMySQLResponse(MySQLResponse&& move) = default;\n\t//move operator\n\tMySQLResponse& operator= (MySQLResponse&& move) = default;\n\nprotected:\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen);\n};\n\n}\n\n//impl. not for user\n#include \"MySQLMessage.inl\"\n\n#endif\n\n" }, { "path": "src/protocol/MySQLMessage.inl", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"SSLWrapper.h\"\n\nnamespace protocol\n{\n\nclass MySQLHandshakeRequest : public MySQLRequest\n{\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max) { return 0; }\n};\n\nclass MySQLHandshakeResponse : public MySQLResponse\n{\npublic:\n\tstd::string get_server_version() const { return server_version_; }\n\tstd::string get_auth_plugin_name() const { return auth_plugin_name_; }\n\n\tvoid get_seed(unsigned char seed[20]) const\n\t{\n\t\tmemcpy(seed, auth_plugin_data_, 20);\n\t}\n\n\tvirtual int encode(struct iovec vectors[], int max);\n\n\tvoid server_set(uint8_t protocol_version, const std::string server_version,\n\t\t\t\t\tuint32_t connection_id, const unsigned char seed[20],\n\t\t\t\t\tuint32_t capability_flags, uint8_t character_set,\n\t\t\t\t\tuint16_t status_flags)\n\t{\n\t\tprotocol_version_ = protocol_version;\n\t\tserver_version_ = server_version;\n\t\tconnection_id_ = connection_id;\n\t\tmemcpy(auth_plugin_data_, seed, 20);\n\t\tcapability_flags_ = capability_flags;\n\t\tcharacter_set_ = character_set;\n\t\tstatus_flags_ = status_flags;\n\t}\n\n\tbool host_disallowed() const { return disallowed_; }\n\tuint32_t get_capability_flags() const { return capability_flags_; }\n\tuint16_t get_status_flags() const { return status_flags_; }\n\nprivate:\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen);\n\n\tstd::string server_version_;\n\tstd::string auth_plugin_name_;\n\tunsigned char auth_plugin_data_[20];\n\tuint32_t connection_id_;\n\tuint32_t capability_flags_;\n\tuint16_t status_flags_;\n\tuint8_t character_set_;\n\tuint8_t auth_plugin_data_len_;\n\tuint8_t protocol_version_;\n\tbool disallowed_;\n\npublic:\n\tMySQLHandshakeResponse() : disallowed_(false) { }\n\t//move constructor\n\tMySQLHandshakeResponse(MySQLHandshakeResponse&& move) = default;\n\t//move operator\n\tMySQLHandshakeResponse& operator= (MySQLHandshakeResponse&& move) = default;\n};\n\nclass MySQLSSLRequest : public MySQLRequest\n{\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max);\n\n\t/* Do not support server side with SSL currently. */\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen)\n\t{\n\t\treturn -2;\n\t}\n\nprivate:\n\tint character_set_;\n\tSSLHandshaker ssl_handshaker_;\n\npublic:\n\tMySQLSSLRequest(int character_set, SSL *ssl) : ssl_handshaker_(ssl)\n\t{\n\t\tcharacter_set_ = character_set;\n\t}\n\n\tMySQLSSLRequest(MySQLSSLRequest&& move) = default;\n\tMySQLSSLRequest& operator= (MySQLSSLRequest&& move) = default;\n};\n\nclass MySQLAuthRequest : public MySQLRequest\n{\npublic:\n\tvoid set_auth(const std::string username,\n\t\t\t\t const std::string password,\n\t\t\t\t const std::string db,\n\t\t\t\t int character_set)\n\t{\n\t\tusername_ = std::move(username);\n\t\tpassword_ = std::move(password);\n\t\tdb_ = std::move(db);\n\t\tcharacter_set_ = character_set;\n\t}\n\n\tvoid set_auth_plugin_name(std::string name)\n\t{\n\t\tauth_plugin_name_ = std::move(name);\n\t}\n\n\tvoid set_seed(const unsigned char seed[20])\n\t{\n\t\tmemcpy(seed_, seed, 20);\n\t}\n\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen);\n\n\tstd::string username_;\n\tstd::string password_;\n\tstd::string db_;\n\tstd::string auth_plugin_name_;\n\tunsigned char seed_[20];\n\tint character_set_;\n\npublic:\n\tMySQLAuthRequest() : character_set_(33) { }\n\t//move constructor\n\tMySQLAuthRequest(MySQLAuthRequest&& move) = default;\n\t//move operator\n\tMySQLAuthRequest& operator= (MySQLAuthRequest&& move) = default;\n};\n\nclass MySQLAuthResponse : public MySQLResponse\n{\npublic:\n\tstd::string get_auth_plugin_name() const { return auth_plugin_name_; }\n\n\tvoid get_seed(unsigned char seed[20]) const\n\t{\n\t\tmemcpy(seed, seed_, 20);\n\t}\n\n\tbool is_continue() const\n\t{\n\t\treturn continue_;\n\t}\n\nprivate:\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen);\n\nprivate:\n\tstd::string auth_plugin_name_;\n\tunsigned char seed_[20];\n\tbool continue_;\n\npublic:\n\tMySQLAuthResponse() : continue_(false) { }\n\t//move constructor\n\tMySQLAuthResponse(MySQLAuthResponse&& move) = default;\n\t//move operator\n\tMySQLAuthResponse& operator= (MySQLAuthResponse&& move) = default;\n};\n\nclass MySQLAuthSwitchRequest : public MySQLRequest\n{\npublic:\n\tvoid set_password(std::string password)\n\t{\n\t\tpassword_ = std::move(password);\n\t}\n\n\tvoid set_auth_plugin_name(std::string name)\n\t{\n\t\tauth_plugin_name_ = std::move(name);\n\t}\n\n\tvoid set_seed(const unsigned char seed[20])\n\t{\n\t\tmemcpy(seed_, seed, 20);\n\t}\n\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max);\n\n\t/* Not implemented. */\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen)\n\t{\n\t\treturn -2;\n\t}\n\n\tstd::string password_;\n\tstd::string auth_plugin_name_;\n\tunsigned char seed_[20];\n\npublic:\n\tMySQLAuthSwitchRequest() { }\n\t//move constructor\n\tMySQLAuthSwitchRequest(MySQLAuthSwitchRequest&& move) = default;\n\t//move operator\n\tMySQLAuthSwitchRequest& operator= (MySQLAuthSwitchRequest&& move) = default;\n};\n\nclass MySQLPublicKeyRequest : public MySQLRequest\n{\npublic:\n\tvoid set_caching_sha2() { byte_ = 0x02; }\n\tvoid set_sha256() { byte_ = 0x01; }\n\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max)\n\t{\n\t\tbuf_.assign(&byte_, 1);\n\t\treturn MySQLRequest::encode(vectors, max);\n\t}\n\n\t/* Not implemented. */\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen)\n\t{\n\t\treturn -2;\n\t}\n\n\tchar byte_;\n\npublic:\n\tMySQLPublicKeyRequest() : byte_(0x01) { }\n\t//move constructor\n\tMySQLPublicKeyRequest(MySQLPublicKeyRequest&& move) = default;\n\t//move operator\n\tMySQLPublicKeyRequest& operator= (MySQLPublicKeyRequest&& move) = default;\n};\n\nclass MySQLPublicKeyResponse : public MySQLResponse\n{\npublic:\n\tstd::string get_public_key() const\n\t{\n\t\treturn public_key_;\n\t}\n\n\tvoid set_public_key(std::string key)\n\t{\n\t\tpublic_key_ = std::move(key);\n\t}\n\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen);\n\n\tstd::string public_key_;\n\npublic:\n\tMySQLPublicKeyResponse() { }\n\t//move constructor\n\tMySQLPublicKeyResponse(MySQLPublicKeyResponse&& move) = default;\n\t//move operator\n\tMySQLPublicKeyResponse& operator= (MySQLPublicKeyResponse&& move) = default;\n};\n\nclass MySQLRSAAuthRequest : public MySQLRequest\n{\npublic:\n\tvoid set_password(std::string password)\n\t{\n\t\tpassword_ = std::move(password);\n\t}\n\n\tvoid set_public_key(std::string key)\n\t{\n\t\tpublic_key_ = std::move(key);\n\t}\n\n\tvoid set_seed(const unsigned char seed[20])\n\t{\n\t\tmemcpy(seed_, seed, 20);\n\t}\n\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max);\n\n\t/* Not implemented. */\n\tvirtual int decode_packet(const unsigned char *buf, size_t buflen)\n\t{\n\t\treturn -2;\n\t}\n\n\tint rsa_encrypt(void *ctx);\n\n\tstd::string password_;\n\tstd::string public_key_;\n\tunsigned char seed_[20];\n\npublic:\n\tMySQLRSAAuthRequest() { }\n\t//move constructor\n\tMySQLRSAAuthRequest(MySQLRSAAuthRequest&& move) = default;\n\t//move operator\n\tMySQLRSAAuthRequest& operator= (MySQLRSAAuthRequest&& move) = default;\n};\n\n//////////\n\ninline mysql_parser_t *MySQLMessage::get_parser() const\n{\n\treturn parser_;\n}\n\ninline int MySQLMessage::get_seqid() const\n{\n\treturn seqid_;\n}\n\ninline void MySQLMessage::set_seqid(int seqid)\n{\n\tseqid_ = seqid;\n}\n\ninline int MySQLMessage::get_command() const\n{\n\treturn parser_->cmd;\n}\n\ninline void MySQLMessage::set_command(int cmd) const\n{\n\tmysql_parser_set_command(cmd, parser_);\n}\n\ninline MySQLMessage::MySQLMessage():\n\tstream_(new mysql_stream_t),\n\tparser_(new mysql_parser_t),\n\tseqid_(0),\n\tcur_size_(0)\n{\n\tmysql_stream_init(stream_);\n\tmysql_parser_init(parser_);\n}\n\ninline bool MySQLRequest::query_is_unset() const\n{\n\treturn buf_.empty();\n}\n\ninline void MySQLRequest::set_query(const char *query)\n{\n\tset_query(query, strlen(query));\n}\n\ninline void MySQLRequest::set_query(const std::string& query)\n{\n\tset_query(query.c_str(), query.size());\n}\n\ninline int MySQLResponse::get_packet_type() const\n{\n\treturn parser_->packet_type;\n}\n\ninline int MySQLResponse::get_error_code() const\n{\n\treturn is_error_packet() ? parser_->error : 0;\n}\n\ninline std::string MySQLResponse::get_error_msg() const\n{\n\tif (is_error_packet())\n\t{\n\t\tconst char *s;\n\t\tsize_t slen;\n\n\t\tmysql_parser_get_err_msg(&s, &slen, parser_);\n\t\tif (slen > 0)\n\t\t\treturn std::string(s, slen);\n\t}\n\n\treturn std::string();\n}\n\ninline std::string MySQLResponse::get_sql_state() const\n{\n\tif (is_error_packet())\n\t{\n\t\tconst char *s;\n\t\tsize_t slen;\n\n\t\tmysql_parser_get_net_state(&s, &slen, parser_);\n\t\tif (slen > 0)\n\t\t\treturn std::string(s, slen);\n\t}\n\n\treturn std::string();\n}\n\n}\n\n" }, { "path": "src/protocol/MySQLResult.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \"mysql_types.h\"\n#include \"mysql_byteorder.h\"\n#include \"MySQLMessage.h\"\n#include \"MySQLResult.h\"\n\nnamespace protocol\n{\n\nMySQLField::MySQLField(const void *buf, mysql_field_t *field)\n{\n\tconst char *p = (const char *)buf;\n\n\tthis->name = p + field->name_offset;\n\tthis->name_length = field->name_length;\t\n\n\tthis->org_name = p + field->org_name_offset;\n\tthis->org_name_length = field->org_name_length;\n\n\tthis->table = p + field->table_offset;\n\tthis->table_length = field->table_length;\n\n\tthis->org_table = p + field->org_table_offset;\n\tthis->org_table_length = field->org_table_length;\n\n\tthis->db = p + field->db_offset;\n\tthis->db_length = field->db_length;\n\n\tthis->catalog = p + field->catalog_offset;\n\tthis->catalog_length = field->catalog_length;\n\n\tif (field->def_offset == (size_t)-1 && field->def_length == 0)\n\t{\n\t\tthis->def = NULL;\n\t\tthis->def_length = 0;\n\t} else {\n\t\tthis->def = p + field->def_offset;\n\t\tthis->def_length = field->def_length;\n\t}\n\n\tthis->flags = field->flags;\n\tthis->length = field->length;\n\tthis->decimals = field->decimals;\n\tthis->charsetnr = field->charsetnr;\n\tthis->data_type = field->data_type;\n}\n\nMySQLResultCursor::MySQLResultCursor()\n{\n\tthis->init();\n}\n\nvoid MySQLResultCursor::init()\n{\n\tthis->current_field = 0;\n\tthis->current_row = 0;\n\tthis->field_count = 0;\n\tthis->fields = NULL;\n\tthis->parser = NULL;\n\tthis->status = MYSQL_STATUS_NOT_INIT;\n}\n\nMySQLResultCursor::MySQLResultCursor(const MySQLResponse *resp)\n{\n\tthis->init(resp);\n}\n\nvoid MySQLResultCursor::reset(MySQLResponse *resp)\n{\n\tthis->clear();\n\tthis->init(resp);\n}\n\nvoid MySQLResultCursor::fetch_result_set(const struct __mysql_result_set *result_set)\n{\n\tconst char *buf = (const char *)this->parser->buf;\n\tthis->server_status = result_set->server_status;\n\n\tswitch (result_set->type)\n\t{\n\tcase MYSQL_PACKET_GET_RESULT:\n\t\tthis->status = MYSQL_STATUS_GET_RESULT;\n\t\tthis->field_count = result_set->field_count;\n\t\tthis->start = buf + result_set->rows_begin_offset;\n\t\tthis->pos = this->start;\n\t\tthis->end = buf + result_set->rows_end_offset;\n\t\tthis->row_count = result_set->row_count;\n\n\t\tthis->fields = new MySQLField *[this->field_count];\n\t\tfor (int i = 0; i < this->field_count; i++)\n\t\t\tthis->fields[i] = new MySQLField(this->parser->buf, result_set->fields[i]);\n\t\tbreak;\n\n\tcase MYSQL_PACKET_OK:\n\t\tthis->status = MYSQL_STATUS_OK;\n\t\tthis->affected_rows = result_set->affected_rows;\n\t\tthis->insert_id = result_set->insert_id;\n\t\tthis->warning_count = result_set->warning_count;\n\t\tthis->start = buf + result_set->info_offset;\n\t\tthis->info_len = result_set->info_len;\n\t\tthis->field_count = 0;\n\t\tthis->fields = NULL;\n\t\tbreak;\n\n\tdefault:\n\t\tthis->status = MYSQL_STATUS_ERROR;\n\t\tbreak;\n\t}\n}\n\nvoid MySQLResultCursor::init(const MySQLResponse *resp)\n{\n\tthis->current_field = 0;\n\tthis->current_row = 0;\n\tthis->field_count = 0;\n\tthis->fields = NULL;\n\tthis->parser = resp->get_parser();\n\tthis->status = MYSQL_STATUS_NOT_INIT;\n\n\tif (!list_empty(&this->parser->result_set_list))\n\t{\n\t\tstruct __mysql_result_set *result_set;\n\n\t\tmysql_result_set_cursor_init(&this->cursor, this->parser);\n\t\tmysql_result_set_cursor_next(&result_set, &this->cursor);\n\n\t\tthis->fetch_result_set(result_set);\n\t}\n}\n\nbool MySQLResultCursor::next_result_set()\n{\n\tif (this->status == MYSQL_STATUS_NOT_INIT ||\n\t\tthis->status == MYSQL_STATUS_ERROR)\n\t{\n\t\treturn false;\n\t}\n\n\tstruct __mysql_result_set *result_set;\n\tif (mysql_result_set_cursor_next(&result_set, &this->cursor) == 0)\n\t{\n\t\tfor (int i = 0; i < this->field_count; i++)\n\t\t\tdelete this->fields[i];\n\n\t\tdelete []this->fields;\n\n\t\tthis->current_field = 0;\n\t\tthis->current_row = 0;\n\n\t\tthis->fetch_result_set(result_set);\n\t\treturn true;\n\t}\n\telse\n\t{\n\t\tthis->status = MYSQL_STATUS_END;\n\t\treturn false;\n\t}\n}\n\nbool MySQLResultCursor::fetch_row(std::vector& row_arr)\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT)\n\t\treturn false;\n\n\tunsigned long long len;\n\tconst unsigned char *data;\n\tint data_type;\n\n\tconst unsigned char *p = (const unsigned char *)this->pos;\n\tconst unsigned char *end = (const unsigned char *)this->end;\n\t\n\trow_arr.clear();\n\n\tfor (int i = 0; i < this->field_count; i++)\n\t{\n\t\tdata_type = this->fields[i]->get_data_type();\n\t\tif (*p == MYSQL_PACKET_HEADER_NULL)\n\t\t{\n\t\t\tdata = NULL;\n\t\t\tlen = 0;\n\t\t\tp++;\n\t\t\tdata_type = MYSQL_TYPE_NULL;\n\t\t} else if (decode_string(&data, &len, &p, end) == 0) {\n\t\t\tthis->status = MYSQL_STATUS_ERROR;\n\t\t\treturn false;\n\t\t}\n\n\t\trow_arr.emplace_back(data, len, data_type);\n\t}\n\n\tif (++this->current_row == this->row_count)\n\t\tthis->status = MYSQL_STATUS_END;\n\n\tthis->pos = p;\n\n\treturn true;\n}\n\nbool MySQLResultCursor::fetch_row(std::map& row_map)\n{\n\treturn this->fetch_row>(row_map);\n}\n\nbool MySQLResultCursor::fetch_row(std::unordered_map& row_map)\n{\n\treturn this->fetch_row>(row_map);\n}\n\nbool MySQLResultCursor::fetch_row_nocopy(const void **data, size_t *len, int *data_type)\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT)\n\t\treturn false;\n\n\tunsigned long long cell_len;\n\tconst unsigned char *cell_data;\n\n\tconst unsigned char *p = (const unsigned char *)this->pos;\n\tconst unsigned char *end = (const unsigned char *)this->end;\n\n\tfor (int i = 0; i < this->field_count; i++)\n\t{\t\n\t\tif (*p == MYSQL_PACKET_HEADER_NULL)\n\t\t{\n\t\t\tcell_data = NULL;\n\t\t\tcell_len = 0;\n\t\t\tp++;\n\t\t}\n\t\telse if (decode_string(&cell_data, &cell_len, &p, end) == 0)\n\t\t{\n\t\t\tthis->status = MYSQL_STATUS_ERROR;\n\t\t\treturn false;\n\t\t}\n\n\t\tdata[i] = cell_data;\n\t\tlen[i] = cell_len;\n\t\tdata_type[i] = this->fields[i]->get_data_type();\n\t}\n\n\tthis->pos = p;\n\n\tif (++this->current_row == this->row_count)\n\t\tthis->status = MYSQL_STATUS_END;\n\n\treturn true;\n}\n\nbool MySQLResultCursor::fetch_all(std::vector>& rows)\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT)\n\t\treturn false;\n\n\tunsigned long long len;\n\tconst unsigned char *data;\n\tint data_type;\n\n\tconst unsigned char *p = (const unsigned char *)this->pos;\n\tconst unsigned char *end = (const unsigned char *)this->end;\n\n\trows.clear();\n\n\tfor (int i = this->current_row; i < this->row_count; i++)\n\t{\n\t\tstd::vector tmp;\n\t\tfor (int j = 0; j < this->field_count; j++)\n\t\t{\n\t\t\tdata_type = this->fields[j]->get_data_type();\n\t\t\tif (*p == MYSQL_PACKET_HEADER_NULL)\n\t\t\t{\n\t\t\t\tdata = NULL;\n\t\t\t\tlen = 0;\n\t\t\t\tp++;\n\t\t\t\tdata_type = MYSQL_TYPE_NULL;\n\t\t\t}\n\t\t\telse if (decode_string(&data, &len, &p, end) == 0)\n\t\t\t{\n\t\t\t\tthis->status = MYSQL_STATUS_ERROR;\n\t\t\t\treturn false;\n\t\t\t}\n\n\t\t\ttmp.emplace_back(data, len, data_type);\n\t\t}\n\t\trows.emplace_back(std::move(tmp));\n\t}\n\n\tthis->current_row = this->row_count;\n\tthis->status = MYSQL_STATUS_END;\n\tthis->pos = p;\n\n\treturn true;\n}\n\nvoid MySQLResultCursor::first_result_set()\n{\n\tif (this->status == MYSQL_STATUS_NOT_INIT ||\n\t\tthis->status == MYSQL_STATUS_ERROR)\n\t{\n\t\treturn;\n\t}\n\n\tmysql_result_set_cursor_rewind(&this->cursor);\n\tstruct __mysql_result_set *result_set;\n\n\tif (mysql_result_set_cursor_next(&result_set, &this->cursor) == 0)\n\t{\n\t\tfor (int i = 0; i < this->field_count; i++)\n\t\t\tdelete this->fields[i];\n\n\t\tdelete []this->fields;\n\n\t\tthis->current_field = 0;\n\t\tthis->current_row = 0;\n\n\t\tthis->fetch_result_set(result_set);\n\t}\n}\n\nvoid MySQLResultCursor::rewind()\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT &&\n\t\tthis->status != MYSQL_STATUS_END)\n\t{\n\t\treturn;\n\t}\n\n\tthis->current_field = 0;\n\tthis->current_row = 0;\n\tthis->pos = this->start;\n}\n\n}\n\n" }, { "path": "src/protocol/MySQLResult.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#ifndef _MYSQLRESULT_H_\n#define _MYSQLRESULT_H_\n\n#include \n#include \n#include \n#include \n#include \"mysql_types.h\"\n#include \"mysql_parser.h\"\n#include \"MySQLMessage.h\"\n\n/**\n * @file MySQLResult.h\n * @brief MySQL toolbox for visit result\n */\n\nnamespace protocol\n{\n\nclass MySQLCell\n{\npublic:\n\tMySQLCell();\n\n\tMySQLCell(MySQLCell&& move);\n\tMySQLCell& operator=(MySQLCell&& move);\n\n\tMySQLCell(const void *data, size_t len, int data_type);\n\n\tint get_data_type() const;\n\n\tbool is_null() const;\n\tbool is_int() const;\n\tbool is_string() const;\n\tbool is_float() const;\n\tbool is_double() const;\n\tbool is_ulonglong() const;\n\tbool is_date() const;\n\tbool is_time() const;\n\tbool is_datetime() const;\n\n\t// for copy\n\tint as_int() const;\n\tstd::string as_string() const;\t\n\tstd::string as_binary_string() const;\t\n\tfloat as_float() const;\n\tdouble as_double() const;\n\tunsigned long long as_ulonglong() const;\n\tstd::string as_date() const;\n\tstd::string as_time() const;\n\tstd::string as_datetime() const;\n\n\t// for nocopy\n\tvoid get_cell_nocopy(const void **data, size_t *len, int *data_type) const;\n\nprivate:\n\tint data_type;\n\tvoid *data;\n\tsize_t len;\n};\n\nclass MySQLField\n{\npublic:\n\tMySQLField(const void *buf, mysql_field_t *field);\n\n\tstd::string get_name() const;\n\tstd::string get_org_name() const;\n\tstd::string get_table() const;\n\tstd::string get_org_table() const;\n\tstd::string get_db() const;\n\tstd::string get_catalog() const;\n\tstd::string get_def() const;\n\tint get_charsetnr() const;\n\tint get_length() const;\n\tint get_flags() const;\n\tint get_decimals() const;\n\tint get_data_type() const;\n\nprivate:\n\tconst char *name;\t\t\t/* Name of column */\n\tconst char *org_name;\t\t/* Original column name, if an alias */\n\tconst char *table;\t\t\t/* Table of column if column was a field */\n\tconst char *org_table;\t\t/* Org table name, if table was an alias */\n\tconst char *db;\t\t\t\t/* Database for table */\n\tconst char *catalog;\t\t/* Catalog for table */\n\tconst char *def;\t\t\t/* Default value (set by mysql_list_fields) */\n\tint length;\t\t\t\t\t/* Width of column (create length) */\n\tint name_length;\n\tint org_name_length;\n\tint table_length;\n\tint org_table_length;\n\tint db_length;\n\tint catalog_length;\n\tint def_length;\n\tint flags;\t\t\t\t\t/* Div flags */\n\tint decimals;\t\t\t\t/* Number of decimals in field */\n\tint charsetnr;\t\t\t\t/* Character set */\n\tint data_type;\t\t\t\t/* Type of field. See mysql_types.h for types */\n};\n\nclass MySQLResultCursor\n{\npublic:\n\tMySQLResultCursor(const MySQLResponse *resp);\n\n\tMySQLResultCursor(MySQLResultCursor&& move);\n\tMySQLResultCursor& operator=(MySQLResultCursor&& move);\n\n\tvirtual ~MySQLResultCursor();\n\n\tbool next_result_set();\n\tvoid first_result_set();\n\n\tconst MySQLField *fetch_field();\n\tconst MySQLField *const *fetch_fields() const;\n\n\tbool fetch_row(std::vector& row_arr);\n\tbool fetch_row(std::map& row_map);\n\tbool fetch_row(std::unordered_map& row_map);\n\n\tbool fetch_row_nocopy(const void **data, size_t *len, int *data_type);\n\tbool fetch_all(std::vector>& rows);\n\n\tint get_cursor_status() const;\n\tint get_server_status() const;\n\n\tint get_field_count() const;\n\tint get_rows_count() const;\n\tunsigned long long get_affected_rows() const;\n\tunsigned long long get_insert_id() const;\n\tint get_warnings() const;\n\tstd::string get_info() const;\n\n\tvoid rewind();\n\npublic:\n\tMySQLResultCursor();\n\tvoid reset(MySQLResponse *resp);\n\nprivate:\n\tvoid init(const MySQLResponse *resp);\n\tvoid init();\n\tvoid clear();\n\n\tvoid fetch_result_set(const struct __mysql_result_set *result_set);\n\n\ttemplate\n\tbool fetch_row(T& row_map); \n\n\tint status;\n\tint server_status;\n\tconst void *start;\n\tconst void *end;\n\tconst void *pos;\n\n\tconst void **row_data;\n\tMySQLField **fields;\n\tint row_count;\n\tint field_count;\n\tint current_row;\n\tint current_field;\n\n\tunsigned long long affected_rows;\n\tunsigned long long insert_id;\n\tint warning_count;\n\tint info_len;\n\n\tmysql_result_set_cursor_t cursor;\n\tmysql_parser_t *parser;\n};\n\n}\n\n#include \"MySQLResult.inl\"\n\n#endif\n\n" }, { "path": "src/protocol/MySQLResult.inl", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"mysql_byteorder.h\"\n\nnamespace protocol\n{\n\ninline std::string MySQLField::get_name() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn \"\";\n\treturn std::string(this->name, this->name_length);\n}\n\ninline std::string MySQLField::get_org_name() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn \"\";\n\treturn std::string(this->org_name, this->org_name_length);\n}\n\ninline std::string MySQLField::get_table() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn \"\";\n\treturn std::string(this->table, this->table_length);\n}\n\ninline std::string MySQLField::get_org_table() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn \"\";\n\treturn std::string(this->org_table, this->org_table_length);\n}\n\ninline std::string MySQLField::get_db() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn \"\";\n\treturn std::string(this->db, this->db_length);\n}\n\ninline std::string MySQLField::get_catalog() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn \"\";\n\treturn std::string(this->catalog, this->catalog_length);\n}\n\ninline std::string MySQLField::get_def() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn \"\";\n\treturn std::string(this->def, this->def_length);\n}\n\ninline int MySQLField::get_charsetnr() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn 0;\n\treturn this->charsetnr;\n}\n\ninline int MySQLField::get_length() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn 0;\n\treturn this->length;\n}\n\ninline int MySQLField::get_flags() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn 0;\n\treturn this->flags;\n}\n\ninline int MySQLField::get_decimals() const\n{\n\tif (this->data_type == MYSQL_TYPE_NULL)\n\t\treturn 0;\n\treturn this->decimals;\n}\n\ninline int MySQLField::get_data_type() const\n{\n\treturn this->data_type;\n}\n\ninline MySQLCell::MySQLCell(MySQLCell&& move)\n{\n\tthis->operator=(std::move(move));\n}\n\ninline MySQLCell& MySQLCell::operator=(MySQLCell&& move)\n{\n\tif (this != &move)\n\t{\n\t\tthis->data = move.data;\n\t\tthis->len = move.len;\n\t\tthis->data_type = move.data_type;\n\n\t\tmove.data = NULL;\n\t\tmove.len = 0;\n\t}\n\n\treturn *this;\n}\n\ninline MySQLCell::MySQLCell(const void *data, size_t len, int data_type)\n{\n\tthis->data_type = data_type;\n\tthis->data = const_cast(data);\n\tthis->len = len;\n}\n\ninline MySQLCell::MySQLCell()\n{\n\tthis->data = NULL;\n\tthis->len = 0;\n\tthis->data_type = MYSQL_TYPE_NULL;\n}\n\ninline int MySQLCell::get_data_type() const\n{\n\treturn this->data_type;\n}\n\ninline void MySQLCell::get_cell_nocopy(const void **data, size_t *len,\n\t\t\t\t\t\t\t\t\t int *data_type) const\n{\n\t*data = this->data;\n\t*len = this->len;\n\t*data_type = this->data_type;\n}\n\ninline bool MySQLCell::is_null() const\n{\n\treturn (this->data_type == MYSQL_TYPE_NULL);\n}\n\ninline std::string MySQLCell::as_binary_string() const\n{\n\treturn std::string((char *)this->data, this->len);\n}\n\ninline bool MySQLCell::is_int() const\n{\n\treturn (this->data_type == MYSQL_TYPE_TINY ||\n\t\t\tthis->data_type == MYSQL_TYPE_SHORT ||\n\t\t\tthis->data_type == MYSQL_TYPE_INT24 ||\n\t\t\tthis->data_type == MYSQL_TYPE_LONG);\n}\n\ninline int MySQLCell::as_int() const\n{\n\tif (!this->is_int())\n\t\treturn 0;\n\n\tstd::string num((char *)this->data, this->len);\n\treturn atoi(num.c_str());\n}\n\ninline bool MySQLCell::is_float() const\n{\n\treturn (this->data_type == MYSQL_TYPE_FLOAT);\n}\n\ninline float MySQLCell::as_float() const\n{\n\tif (!this->is_float())\n\t\treturn NAN;\n\n\tstd::string num((char *)this->data, this->len);\n\treturn strtof(num.c_str(), NULL);\n}\n\ninline bool MySQLCell::is_double() const\n{\n\treturn (this->data_type == MYSQL_TYPE_DOUBLE);\n}\n\ninline double MySQLCell::as_double() const\n{\n\tif (!this->is_double())\n\t\treturn NAN;\n\n\tstd::string num((char *)this->data, this->len);\n\treturn strtod(num.c_str(), NULL);\n}\n\ninline bool MySQLCell::is_ulonglong() const\n{\n\treturn (this->data_type == MYSQL_TYPE_LONGLONG);\n}\n\ninline unsigned long long MySQLCell::as_ulonglong() const\n{\n\tif (!this->is_ulonglong())\n\t\treturn (unsigned long long)-1;\n\n\tstd::string num((char *)this->data, this->len);\n\treturn strtoull(num.c_str(), NULL, 10);\n}\n\ninline bool MySQLCell::is_date() const\n{\n\treturn (this->data_type == MYSQL_TYPE_DATE);\n}\n\ninline std::string MySQLCell::as_date() const\n{\n\tif (!this->is_date())\n\t\treturn \"\";\n\n\treturn std::string((char *)this->data, this->len);\n}\n\ninline bool MySQLCell::is_time() const\n{\n\treturn (this->data_type == MYSQL_TYPE_TIME);\n}\n\ninline std::string MySQLCell::as_time() const\n{\n\tif (!this->is_time())\n\t\treturn \"\";\n\n\treturn std::string((char *)this->data, this->len);\n}\n\ninline bool MySQLCell::is_datetime() const\n{\n\treturn (this->data_type == MYSQL_TYPE_DATETIME ||\n\t\t\tthis->data_type == MYSQL_TYPE_TIMESTAMP);\n}\n\ninline std::string MySQLCell::as_datetime() const\n{\n\tif (!this->is_datetime())\n\t\treturn \"\";\n\n\treturn std::string((char *)this->data, this->len);\n}\n\ninline bool MySQLCell::is_string() const\n{\n\treturn (this->data_type == MYSQL_TYPE_DECIMAL ||\n\t\t\tthis->data_type == MYSQL_TYPE_NEWDECIMAL ||\n\t\t\tthis->data_type == MYSQL_TYPE_STRING ||\n\t\t\tthis->data_type == MYSQL_TYPE_VARCHAR ||\n\t\t\tthis->data_type == MYSQL_TYPE_VAR_STRING ||\n\t\t\tthis->data_type == MYSQL_TYPE_JSON);\n}\n\ninline std::string MySQLCell::as_string() const\n{\n\tif (!this->is_string() && !this->is_time() &&\n\t\t!this->is_date() && !this->is_datetime())\n\t\treturn \"\";\n\n\treturn std::string((char *)this->data, this->len);\n}\n\ntemplate\nbool MySQLResultCursor::fetch_row(T& row_map)\n{\t\n\tif (this->status != MYSQL_STATUS_GET_RESULT)\n\t\treturn false;\n\n\tunsigned long long len;\n\tconst unsigned char *data;\n\tint data_type;\n\n\tconst unsigned char *p = (const unsigned char *)this->pos;\n\tconst unsigned char *end = (const unsigned char *)this->end;\n\n\trow_map.clear();\n\n\tfor (int i = 0; i < this->field_count; i++)\n\t{\n\t\tdata_type = this->fields[i]->get_data_type();\n\t\tif (*p == MYSQL_PACKET_HEADER_NULL)\n\t\t{\n\t\t\tdata = NULL;\n\t\t\tlen = 0;\n\t\t\tp++;\n\t\t\tdata_type = MYSQL_TYPE_NULL;\n\t\t}\n\t\telse if (decode_string(&data, &len, &p, end) == 0)\n\t\t{\n\t\t\tthis->status = MYSQL_STATUS_ERROR;\n\t\t\treturn false;\n\t\t}\n\t\trow_map.emplace(this->fields[i]->get_name(), MySQLCell(data, len, data_type));\n\t}\n\n\tthis->pos = p;\n\n\tif (++this->current_row == this->row_count)\n\t\tthis->status = MYSQL_STATUS_END;\n\n\treturn true;\n}\n\ninline const MySQLField *MySQLResultCursor::fetch_field()\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT &&\n\t\tthis->status != MYSQL_STATUS_END)\n\t{\n\t\treturn NULL;\n\t}\n\n\tif (this->current_field >= this->field_count)\n\t\treturn NULL;\n\n\treturn this->fields[this->current_field++];\n}\n\ninline const MySQLField *const *MySQLResultCursor::fetch_fields() const\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT &&\n\t\tthis->status != MYSQL_STATUS_END)\n\t{\n\t\treturn NULL;\n\t}\n\n\treturn this->fields;\n}\n\ninline int MySQLResultCursor::get_cursor_status() const\n{\n\treturn this->status;\n}\n\ninline int MySQLResultCursor::get_server_status() const\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT &&\n\t\tthis->status != MYSQL_STATUS_END &&\n\t\tthis->status != MYSQL_STATUS_OK)\n\t{\n\t\treturn 0;\n\t}\n\n\treturn this->server_status;\n}\n\ninline int MySQLResultCursor::get_field_count() const\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT &&\n\t\tthis->status != MYSQL_STATUS_END)\n\t{\n\t\treturn 0;\n\t}\n\n\treturn this->field_count;\n}\n\ninline int MySQLResultCursor::get_rows_count() const\n{\n\tif (this->status != MYSQL_STATUS_GET_RESULT &&\n\t\tthis->status != MYSQL_STATUS_END)\n\t{\n\t\treturn 0;\n\t}\n\n\treturn this->row_count;\n}\n\ninline unsigned long long MySQLResultCursor::get_affected_rows() const\n{\n\tif (this->status != MYSQL_PACKET_OK)\n\t\treturn 0;\n\n\treturn this->affected_rows;\n}\n\ninline int MySQLResultCursor::get_warnings() const\n{\n\tif (this->status != MYSQL_PACKET_OK)\n\t\treturn 0;\n\n\treturn this->warning_count;\n}\n\ninline unsigned long long MySQLResultCursor::get_insert_id() const\n{\n\tif (this->status != MYSQL_PACKET_OK)\n\t\treturn 0;\n\n\treturn this->insert_id;\n}\n\ninline std::string MySQLResultCursor::get_info() const\n{\n\tif (this->status != MYSQL_PACKET_OK)\n\t\treturn \"\";\n\n\treturn std::string((char *)this->start, this->info_len);\n}\n\ninline void MySQLResultCursor::clear()\n{\n\tfor (int i = 0; i < this->field_count; i++)\n\t\tdelete this->fields[i];\n\n\tdelete []this->fields;\n}\n\ninline MySQLResultCursor::~MySQLResultCursor()\n{\n\tthis->clear();\n}\n\ninline MySQLResultCursor::MySQLResultCursor(MySQLResultCursor&& move)\n{\n\tthis->start = move.start;\n\tthis->end = move.end;\n\tthis->pos = move.pos;\n\tthis->status = move.status;\n\tthis->row_data = move.row_data;\n\tthis->fields = move.fields;\n\tthis->row_count = move.row_count;\n\tthis->field_count = move.field_count;\n\tthis->current_row = move.current_row;\n\tthis->current_field = move.current_field;\n\tthis->affected_rows = move.affected_rows;\n\tthis->insert_id = move.insert_id;\n\tthis->warning_count = move.warning_count;\n\tthis->info_len = move.info_len;\n\tthis->cursor = move.cursor;\n\tthis->parser = move.parser;\n\n\tmove.init();\n}\n\ninline MySQLResultCursor& MySQLResultCursor::operator=(MySQLResultCursor&& move)\n{\n\tif (this != &move)\n\t{\n\t\tthis->clear();\n\n\t\tthis->start = move.start;\n\t\tthis->end = move.end;\n\t\tthis->pos = move.pos;\n\t\tthis->status = move.status;\n\t\tthis->row_data = move.row_data;\n\t\tthis->fields = move.fields;\n\t\tthis->row_count = move.row_count;\n\t\tthis->field_count = move.field_count;\n\t\tthis->current_row = move.current_row;\n\t\tthis->current_field = move.current_field;\n\t\tthis->affected_rows = move.affected_rows;\n\t\tthis->insert_id = move.insert_id;\n\t\tthis->warning_count = move.warning_count;\n\t\tthis->info_len = move.info_len;\n\t\tthis->cursor = move.cursor;\n\t\tthis->parser = move.parser;\n\n\t\tmove.init();\n\t}\n\n\treturn *this;\n}\n\n}\n\n" }, { "path": "src/protocol/MySQLUtil.cc", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \"MySQLUtil.h\"\n\nnamespace protocol\n{\n\nstd::string MySQLUtil::escape_string(const std::string& str)\n{\n\tstd::string res;\n\tchar escape;\n\tsize_t i;\n\n\tfor (i = 0; i < str.size(); i++)\n\t{\n\t\tswitch (str[i])\n\t\t{\n\t\tcase '\\0':\n\t\t\tescape = '0';\n\t\t\tbreak;\n\t\tcase '\\n':\n\t\t\tescape = 'n';\n\t\t\tbreak;\n\t\tcase '\\r':\n\t\t\tescape = 'r';\n\t\t\tbreak;\n\t\tcase '\\\\':\n\t\t\tescape = '\\\\';\n\t\t\tbreak;\n\t\tcase '\\'':\n\t\t\tescape = '\\'';\n\t\t\tbreak;\n\t\tcase '\\\"':\n\t\t\tescape = '\\\"';\n\t\t\tbreak;\n\t\tcase '\\032':\n\t\t\tescape = 'Z';\n\t\t\tbreak;\n\t\tdefault:\n\t\t\tres.push_back(str[i]);\n\t\t\tcontinue;\n\t\t}\n\n\t\tres.push_back('\\\\');\n\t\tres.push_back(escape);\n\t}\n\n\treturn res;\n}\n\nstd::string MySQLUtil::escape_string_quote(const std::string& str, char quote)\n{\n\tstd::string res;\n\tsize_t i;\n\n\tfor (i = 0; i < str.size(); i++)\n\t{\n\t\tif (str[i] == quote)\n\t\t\tres.push_back(quote);\n\n\t\tres.push_back(str[i]);\n\t}\n\n\treturn res;\n}\n\n}\n\n" }, { "path": "src/protocol/MySQLUtil.h", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _MYSQLUTIL_H_\n#define _MYSQLUTIL_H_\n\n#include \n\nnamespace protocol\n{\n\nclass MySQLUtil\n{\npublic:\n\tstatic std::string escape_string(const std::string& str);\n\tstatic std::string escape_string_quote(const std::string& str, char quote);\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/PackageWrapper.cc", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \"PackageWrapper.h\"\n\nnamespace protocol\n{\n\nint PackageWrapper::encode(struct iovec vectors[], int max)\n{\n\tint cnt = 0;\n\tint ret;\n\n\twhile (max >= 8)\n\t{\n\t\tret = this->ProtocolWrapper::encode(vectors, max);\n\t\tif ((unsigned int)ret > (unsigned int)max)\n\t\t{\n\t\t\tif (ret < 0)\n\t\t\t\treturn ret;\n\n\t\t\tbreak;\n\t\t}\n\n\t\tcnt += ret;\n\t\tthis->set_message(this->next_out(this->message));\n\t\tif (!this->message)\n\t\t\treturn cnt;\n\n\t\tvectors += ret;\n\t\tmax -= ret;\n\t}\n\n\terrno = EOVERFLOW;\n\treturn -1;\n}\n\nint PackageWrapper::append(const void *buf, size_t *size)\n{\n\tint ret = this->ProtocolWrapper::append(buf, size);\n\n\tif (ret > 0)\n\t{\n\t\tthis->set_message(this->next_in(this->message));\n\t\tif (this->message)\n\t\t{\n\t\t\tthis->renew();\n\t\t\tret = 0;\n\t\t}\n\t}\n\n\treturn ret;\n}\n\n}\n\n" }, { "path": "src/protocol/PackageWrapper.h", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _PACKAGEWRAPPER_H_\n#define _PACKAGEWRAPPER_H_\n\n#include \"ProtocolMessage.h\"\n\nnamespace protocol\n{\n\nclass PackageWrapper : public ProtocolWrapper\n{\nprivate:\n\tvirtual ProtocolMessage *next_out(ProtocolMessage *message)\n\t{\n\t\treturn NULL;\n\t}\n\n\tvirtual ProtocolMessage *next_in(ProtocolMessage *message)\n\t{\n\t\treturn NULL;\n\t}\n\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\npublic:\n\tPackageWrapper(ProtocolMessage *message) : ProtocolWrapper(message)\n\t{\n\t}\n\npublic:\n\tPackageWrapper(PackageWrapper&& wrapper) = default;\n\tPackageWrapper& operator = (PackageWrapper&& wrapper) = default;\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/ProtocolMessage.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _PROTOCOLMESSAGE_H_\n#define _PROTOCOLMESSAGE_H_\n\n#include \n#include \n#include \n#include \"Communicator.h\"\n\n/**\n * @file ProtocolMessage.h\n * @brief General Protocol Interface\n */\n\nnamespace protocol\n{\n\nclass ProtocolMessage : public CommMessageOut, public CommMessageIn\n{\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max)\n\t{\n\t\terrno = ENOSYS;\n\t\treturn -1;\n\t}\n\n\t/* You have to implement one of the 'append' functions, and the first one\n\t * with arguement 'size_t *size' is recommmended. */\n\n\t/* Argument 'size' indicates bytes to append, and returns bytes used. */\n\tvirtual int append(const void *buf, size_t *size)\n\t{\n\t\treturn this->append(buf, *size);\n\t}\n\n\t/* When implementing this one, all bytes are consumed. Cannot support\n\t * streaming protocol. */\n\tvirtual int append(const void *buf, size_t size)\n\t{\n\t\terrno = ENOSYS;\n\t\treturn -1;\n\t}\n\npublic:\n\tvoid set_size_limit(size_t limit) { this->size_limit = limit; }\n\tsize_t get_size_limit() const { return this->size_limit; }\n\npublic:\n\tclass Attachment\n\t{\n\tpublic:\n\t\tvirtual ~Attachment() { }\n\t};\n\n\tvoid set_attachment(Attachment *att) { this->attachment = att; }\n\tAttachment *get_attachment() { return this->attachment; }\n\tconst Attachment *get_attachment() const { return this->attachment; }\n\nprotected:\n\tvirtual int feedback(const void *buf, size_t size)\n\t{\n\t\tif (this->wrapper)\n\t\t\treturn this->wrapper->feedback(buf, size);\n\t\telse\n\t\t\treturn this->CommMessageIn::feedback(buf, size);\n\t}\n\n\tvirtual void renew()\n\t{\n\t\tif (this->wrapper)\n\t\t\treturn this->wrapper->renew();\n\t\telse\n\t\t\treturn this->CommMessageIn::renew();\n\t}\n\n\tvirtual ProtocolMessage *inner() { return this; }\n\nprotected:\n\tsize_t size_limit;\n\nprivate:\n\tAttachment *attachment;\n\tProtocolMessage *wrapper;\n\npublic:\n\tProtocolMessage()\n\t{\n\t\tthis->size_limit = (size_t)-1;\n\t\tthis->attachment = NULL;\n\t\tthis->wrapper = NULL;\n\t}\n\n\tvirtual ~ProtocolMessage() { delete this->attachment; }\n\npublic:\n\tProtocolMessage(ProtocolMessage&& message)\n\t{\n\t\tthis->size_limit = message.size_limit;\n\t\tthis->attachment = message.attachment;\n\t\tmessage.attachment = NULL;\n\t\tthis->wrapper = NULL;\n\t}\n\n\tProtocolMessage& operator = (ProtocolMessage&& message)\n\t{\n\t\tif (&message != this)\n\t\t{\n\t\t\tthis->size_limit = message.size_limit;\n\t\t\tdelete this->attachment;\n\t\t\tthis->attachment = message.attachment;\n\t\t\tmessage.attachment = NULL;\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\tfriend class ProtocolWrapper;\n};\n\nclass ProtocolWrapper : public ProtocolMessage\n{\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max)\n\t{\n\t\treturn this->message->encode(vectors, max);\n\t}\n\n\tvirtual int append(const void *buf, size_t *size)\n\t{\n\t\treturn this->message->append(buf, size);\n\t}\n\nprotected:\n\tvirtual ProtocolMessage *inner()\n\t{\n\t\treturn this->message->inner();\n\t}\n\nprotected:\n\tvoid set_message(ProtocolMessage *message)\n\t{\n\t\tthis->message = message;\n\t\tif (message)\n\t\t\tmessage->wrapper = this;\n\t}\n\nprotected:\n\tProtocolMessage *message;\n\npublic:\n\tProtocolWrapper(ProtocolMessage *message)\n\t{\n\t\tthis->set_message(message);\n\t}\n\npublic:\n\tProtocolWrapper(ProtocolWrapper&& wrapper) :\n\t\tProtocolMessage(std::move(wrapper))\n\t{\n\t\tthis->set_message(wrapper.message);\n\t\twrapper.message = NULL;\n\t}\n\n\tProtocolWrapper& operator = (ProtocolWrapper&& wrapper)\n\t{\n\t\tif (&wrapper != this)\n\t\t{\n\t\t\t*(ProtocolMessage *)this = std::move(wrapper);\n\t\t\tthis->set_message(wrapper.message);\n\t\t\twrapper.message = NULL;\n\t\t}\n\n\t\treturn *this;\n\t}\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/RedisMessage.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"EncodeStream.h\"\n#include \"RedisMessage.h\"\n\nnamespace protocol\n{\n\ntypedef int64_t Rint;\ntypedef std::string Rstr;\ntypedef std::vector Rarr;\n\nRedisValue& RedisValue::operator= (const RedisValue& copy)\n{\n\tif (this != ©)\n\t{\n\t\tfree_data();\n\n\t\tswitch (copy.type_)\n\t\t{\n\t\tcase REDIS_REPLY_TYPE_INTEGER:\n\t\t\ttype_ = copy.type_;\n\t\t\tdata_ = new Rint(*((Rint*)(copy.data_)));\n\t\t\tbreak;\n\n\t\tcase REDIS_REPLY_TYPE_ERROR:\n\t\tcase REDIS_REPLY_TYPE_STATUS:\n\t\tcase REDIS_REPLY_TYPE_STRING:\n\t\t\ttype_ = copy.type_;\n\t\t\tdata_ = new Rstr(*((Rstr*)(copy.data_)));\n\t\t\tbreak;\n\n\t\tcase REDIS_REPLY_TYPE_ARRAY:\n\t\t\ttype_ = copy.type_;\n\t\t\tdata_ = new Rarr(*((Rarr*)(copy.data_)));\n\t\t\tbreak;\n\n\t\tdefault:\n\t\t\ttype_ = REDIS_REPLY_TYPE_NIL;\n\t\t\tdata_ = NULL;\n\t\t}\n\t}\n\n\treturn *this;\n}\n\nRedisValue& RedisValue::operator= (RedisValue&& move)\n{\n\tif (this != &move)\n\t{\n\t\tfree_data();\n\n\t\ttype_ = move.type_;\n\t\tdata_ = move.data_;\n\n\t\tmove.type_ = REDIS_REPLY_TYPE_NIL;\n\t\tmove.data_ = NULL;\n\t}\n\n\treturn *this;\n}\n\nvoid RedisValue::free_data()\n{\n\tif (data_)\n\t{\n\t\tswitch (type_)\n\t\t{\n\t\tcase REDIS_REPLY_TYPE_INTEGER:\n\t\t\tdelete (Rint *)data_;\n\t\t\tbreak;\n\n\t\tcase REDIS_REPLY_TYPE_ERROR:\n\t\tcase REDIS_REPLY_TYPE_STATUS:\n\t\tcase REDIS_REPLY_TYPE_STRING:\n\t\t\tdelete (Rstr *)data_;\n\t\t\tbreak;\n\n\t\tcase REDIS_REPLY_TYPE_ARRAY:\n\t\t\tdelete (Rarr *)data_;\n\t\t\tbreak;\n\t\t}\n\n\t\tdata_ = NULL;\n\t}\n}\n\nvoid RedisValue::only_set_string_data(const std::string& strv)\n{\n\tRstr *p = (Rstr *)(data_);\n\tp->assign(strv);\n}\n\nvoid RedisValue::only_set_string_data(const char *str, size_t len)\n{\n\tRstr *p = (Rstr *)(data_);\n\tif (str == NULL || len == 0)\n\t\tp->clear();\n\telse\n\t\tp->assign(str, len);\n}\n\nvoid RedisValue::only_set_string_data(const char *str)\n{\n\tRstr *p = (Rstr *)(data_);\n\tif (str == NULL)\n\t\tp->clear();\n\telse\n\t\tp->assign(str);\n}\n\nvoid RedisValue::set_int(int64_t intv)\n{\n\tif (type_ == REDIS_REPLY_TYPE_INTEGER)\n\t\t*((Rint *)data_) = intv;\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rint(intv);\n\t\ttype_ = REDIS_REPLY_TYPE_INTEGER;\n\t}\n}\n\nvoid RedisValue::set_string(const std::string& strv)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(strv);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(strv);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_STRING;\n}\n\nvoid RedisValue::set_status(const std::string& strv)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(strv);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(strv);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_STATUS;\n}\n\nvoid RedisValue::set_error(const std::string& strv)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(strv);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(strv);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_ERROR;\n}\n\nvoid RedisValue::set_string(const char *str, size_t len)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(str, len);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(str, len);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_STRING;\n}\n\nvoid RedisValue::set_status(const char *str, size_t len)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(str, len);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(str, len);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_STATUS;\n}\n\nvoid RedisValue::set_error(const char *str, size_t len)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(str, len);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(str, len);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_ERROR;\n}\n\nvoid RedisValue::set_string(const char *str)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(str);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(str);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_STRING;\n}\n\nvoid RedisValue::set_status(const char *str)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(str);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(str);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_STATUS;\n}\n\nvoid RedisValue::set_error(const char *str)\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\tonly_set_string_data(str);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rstr(str);\n\t}\n\n\ttype_ = REDIS_REPLY_TYPE_ERROR;\n}\n\nvoid RedisValue::set_array(size_t new_size)\n{\n\tif (type_ == REDIS_REPLY_TYPE_ARRAY)\n\t\t((Rarr *)data_)->resize(new_size);\n\telse\n\t{\n\t\tfree_data();\n\t\tdata_ = new Rarr(new_size);\n\t\ttype_ = REDIS_REPLY_TYPE_ARRAY;\n\t}\n}\n\nvoid RedisValue::set(const redis_reply_t *reply)\n{\n\tset_nil();\n\tswitch (reply->type)\n\t{\n\tcase REDIS_REPLY_TYPE_INTEGER:\n\t\tset_int(reply->integer);\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_ERROR:\n\t\tset_error(reply->str, reply->len);\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_STATUS:\n\t\tset_status(reply->str, reply->len);\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_STRING:\n\t\tset_string(reply->str, reply->len);\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_ARRAY:\n\t\tset_array(reply->elements);\n\n\t\tif (reply->elements > 0)\n\t\t{\n\t\t\tRarr *parr = (Rarr *)data_;\n\t\t\tfor (size_t i = 0; i < reply->elements; i++)\n\t\t\t\t(*parr)[i].set(reply->element[i]);\n\t\t}\n\n\t\tbreak;\n\t}\n}\n\nvoid RedisValue::arr_clear()\n{\n\tif (type_ == REDIS_REPLY_TYPE_ARRAY)\n\t\t((Rarr *)data_)->clear();\n}\n\nvoid RedisValue::arr_resize(size_t new_size)\n{\n\tif (type_ == REDIS_REPLY_TYPE_ARRAY)\n\t\t((Rarr *)data_)->resize(new_size);\n}\n\nbool RedisValue::transform(redis_reply_t *reply) const\n{\n\t//todo risk of stack overflow\n\tRarr *parr;\n\tRstr *pstr;\n\n\tredis_reply_set_null(reply);\n\tswitch (type_)\n\t{\n\tcase REDIS_REPLY_TYPE_INTEGER:\n\t\tredis_reply_set_integer(*((Rint *)data_), reply);\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_ARRAY:\n\t\tparr = (Rarr *)data_;\n\t\tif (redis_reply_set_array(parr->size(), reply) < 0)\n\t\t\treturn false;\n\n\t\tfor (size_t i = 0; i < reply->elements; i++)\n\t\t{\n\t\t\tif (!(*parr)[i].transform(reply->element[i]))\n\t\t\t\treturn false;\n\t\t}\n\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_STATUS:\n\t\tpstr = (Rstr *)data_;\n\t\tredis_reply_set_status(pstr->c_str(), pstr->size(), reply);\n\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_ERROR:\n\t\tpstr = (Rstr *)data_;\n\t\tredis_reply_set_error(pstr->c_str(), pstr->size(), reply);\n\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_STRING:\n\t\tpstr = (Rstr *)data_;\n\t\tredis_reply_set_string(pstr->c_str(), pstr->size(), reply);\n\n\t\tbreak;\n\t}\n\n\treturn true;\n}\n\nstd::string RedisValue::debug_string() const\n{\n\tstd::string ret;\n\n\tif (is_error())\n\t{\n\t\tret += \"ERROR: \";\n\t\tret += string_view()->c_str();\n\t}\n\telse if (is_int())\n\t{\n\t\tstd::ostringstream oss;\n\t\toss << int_value();\n\t\tret += oss.str();\n\t}\n\telse if (is_nil())\n\t{\n\t\tret += \"nil\";\n\t}\n\telse if (is_string())\n\t{\n\t\tret += '\\\"';\n\t\tret += string_view()->c_str();\n\t\tret += '\\\"';\n\t}\n\telse if (is_array())\n\t{\n\t\tret += '[';\n\t\tsize_t l = arr_size();\n\t\tfor (size_t i = 0; i < l; i++)\n\t\t{\n\t\t\tif (i)\n\t\t\t\tret += \", \";\n\n\t\t\tret += (*this)[i].debug_string();\n\t\t}\n\t\tret += ']';\n\t}\n\n\treturn ret;\n}\n\nRedisValue::~RedisValue()\n{\n\tfree_data();\n}\n\nRedisMessage::RedisMessage():\n\tparser_(new redis_parser_t),\n\tstream_(new EncodeStream),\n\tcur_size_(0),\n\tasking_(false)\n{\n\tredis_parser_init(parser_);\n}\n\nRedisMessage::~RedisMessage()\n{\n\tif (parser_)\n\t{\n\t\tredis_parser_deinit(parser_);\n\t\tdelete parser_;\n\t\tdelete stream_;\n\t}\n}\n\nRedisMessage::RedisMessage(RedisMessage&& move) :\n\tProtocolMessage(std::move(move))\n{\n\tparser_ = move.parser_;\n\tstream_ = move.stream_;\n\tcur_size_ = move.cur_size_;\n\tasking_ = move.asking_;\n\n\tmove.parser_ = NULL;\n\tmove.stream_ = NULL;\n\tmove.cur_size_ = 0;\n\tmove.asking_ = false;\n}\n\nRedisMessage& RedisMessage::operator= (RedisMessage &&move)\n{\n\tif (this != &move)\n\t{\n\t\t*(ProtocolMessage *)this = std::move(move);\n\n\t\tif (parser_)\n\t\t{\n\t\t\tredis_parser_deinit(parser_);\n\t\t\tdelete parser_;\n\t\t\tdelete stream_;\n\t\t}\n\n\t\tparser_ = move.parser_;\n\t\tstream_ = move.stream_;\n\t\tcur_size_ = move.cur_size_;\n\t\tasking_ = move.asking_;\n\n\t\tmove.parser_ = NULL;\n\t\tmove.stream_ = NULL;\n\t\tmove.cur_size_ = 0;\n\t\tmove.asking_ = false;\n\t}\n\n\treturn *this;\n}\n\nbool RedisMessage::encode_reply(redis_reply_t *reply)\n{\n\tEncodeStream& stream = *stream_;\n\tswitch (reply->type)\n\t{\n\tcase REDIS_REPLY_TYPE_STATUS:\n\t\tstream << \"+\" << std::make_pair(reply->str, reply->len) << \"\\r\\n\";\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_ERROR:\n\t\tstream << \"-\" << std::make_pair(reply->str, reply->len) << \"\\r\\n\";\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_NIL:\n\t\tstream << \"$-1\\r\\n\";\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_INTEGER:\n\t\tstream << \":\" << reply->integer << \"\\r\\n\";\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_STRING:\n\t\tstream << \"$\" << reply->len << \"\\r\\n\";\n\t\tstream << std::make_pair(reply->str, reply->len) << \"\\r\\n\";\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_ARRAY:\n\t\tstream << \"*\" << reply->elements << \"\\r\\n\";\n\t\tfor (size_t i = 0; i < reply->elements; i++)\n\t\t\tif (!encode_reply(reply->element[i]))\n\t\t\t\treturn false;\n\n\t\tbreak;\n\n\tdefault:\n\t\treturn false;\n\t}\n\n\treturn true;\n}\n\nint RedisMessage::encode(struct iovec vectors[], int max)\n{\n\tstream_->reset(vectors, max);\n\n\tif (encode_reply(&parser_->reply))\n\t\treturn stream_->size();\n\n\treturn 0;\n}\n\nint RedisMessage::append(const void *buf, size_t *size)\n{\n\tint ret = redis_parser_append_message(buf, size, parser_);\n\n\tif (ret >= 0)\n\t{\n\t\tcur_size_ += *size;\n\t\tif (cur_size_ > this->size_limit)\n\t\t{\n\t\t\terrno = EMSGSIZE;\n\t\t\tret = -1;\n\t\t}\n\t}\n\telse if (ret == -2)\n\t{\n\t\terrno = EBADMSG;\n\t\tret = -1;\n\t}\n\n\treturn ret;\n}\n\nvoid RedisRequest::set_request(const std::string& command,\n\t\t\t\t\t\t\t const std::vector& params)\n{\n\tsize_t n = params.size() + 1;\n\tuser_request_.reserve(n);\n\tuser_request_.clear();\n\tuser_request_.push_back(command);\n\tfor (size_t i = 0; i < params.size(); i++)\n\t\tuser_request_.push_back(params[i]);\n\n\tredis_reply_t *reply = &parser_->reply;\n\tredis_reply_set_array(n, reply);\n\tfor (size_t i = 0; i < n; i++)\n\t{\n\t\tredis_reply_set_string(user_request_[i].c_str(),\n\t\t\t\t\t\t\t user_request_[i].size(),\n\t\t\t\t\t\t\t reply->element[i]);\n\t}\n}\n\nbool RedisRequest::get_command(std::string& command) const\n{\n\tconst redis_reply_t *reply = &parser_->reply;\n\tif (reply->type == REDIS_REPLY_TYPE_ARRAY && reply->elements > 0)\n\t{\n\t\treply = reply->element[0];\n\t\tif (reply->type == REDIS_REPLY_TYPE_STRING)\n\t\t{\n\t\t\tcommand.assign(reply->str, reply->len);\n\t\t\treturn true;\n\t\t}\n\t}\n\n\treturn false;\n}\n\nbool RedisRequest::get_params(std::vector& params) const\n{\n\tconst redis_reply_t *reply = &parser_->reply;\n\tif (reply->type == REDIS_REPLY_TYPE_ARRAY && reply->elements > 0)\n\t{\n\t\tfor (size_t i = 1; i < reply->elements; i++)\n\t\t{\n\t\t\tif (reply->element[i]->type != REDIS_REPLY_TYPE_STRING &&\n\t\t\t\treply->element[i]->type != REDIS_REPLY_TYPE_NIL)\n\t\t\t{\n\t\t\t\treturn false;\n\t\t\t}\n\t\t}\n\n\t\tparams.reserve(reply->elements - 1);\n\t\tparams.clear();\n\t\tfor (size_t i = 1; i < reply->elements; i++)\n\t\t\tparams.emplace_back(reply->element[i]->str, reply->element[i]->len);\n\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\n#define REDIS_ASK_COMMAND\t\"ASKING\"\n#define REDIS_ASK_REQUEST\t\"*1\\r\\n$6\\r\\nASKING\\r\\n\"\n#define REDIS_OK_RESPONSE\t\"+OK\\r\\n\"\n\nint RedisRequest::encode(struct iovec vectors[], int max)\n{\n\tstream_->reset(vectors, max);\n\n\tif (is_asking())\n\t\t(*stream_) << REDIS_ASK_REQUEST;\n\tif (encode_reply(&parser_->reply))\n\t\treturn stream_->size();\n\n\treturn 0;\n}\n\nint RedisRequest::append(const void *buf, size_t *size)\n{\n\tint ret = RedisMessage::append(buf, size);\n\n\tif (ret > 0)\n\t{\n\t\tstd::string command;\n\n\t\tif (get_command(command) &&\n\t\t\tstrcasecmp(command.c_str(), REDIS_ASK_COMMAND) == 0)\n\t\t{\n\t\t\tredis_parser_deinit(parser_);\n\t\t\tredis_parser_init(parser_);\n\t\t\tset_asking(true);\n\n\t\t\tret = this->feedback(REDIS_OK_RESPONSE, strlen(REDIS_OK_RESPONSE));\n\t\t\tif (ret != strlen(REDIS_OK_RESPONSE))\n\t\t\t{\n\t\t\t\terrno = ENOBUFS;\n\t\t\t\tret = -1;\n\t\t\t}\n\t\t\telse\n\t\t\t\tret = 0;\n\t\t}\n\t}\n\n\treturn ret;\n}\n\nint RedisResponse::append(const void *buf, size_t *size)\n{\n\tint ret = RedisMessage::append(buf, size);\n\n\tif (ret > 0 && is_asking())\n\t{\n\t\tredis_parser_deinit(parser_);\n\t\tredis_parser_init(parser_);\n\t\tret = 0;\n\t\tset_asking(false);\n\t}\n\n\treturn ret;\n}\n\nbool RedisResponse::set_result(const RedisValue& value)\n{\n\tredis_reply_t *reply = &parser_->reply;\n\tredis_reply_deinit(reply);\n\tredis_reply_init(reply);\n\n\tvalue_ = value;\n\treturn value_.transform(reply);\n}\n\n}\n\n" }, { "path": "src/protocol/RedisMessage.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#ifndef _REDISMESSAGE_H_\n#define _REDISMESSAGE_H_\n\n#include \n#include \n#include \n#include \"ProtocolMessage.h\"\n#include \"redis_parser.h\"\n\n/**\n * @file RedisMessage.h\n * @brief Redis Protocol Interface\n */\n\nnamespace protocol\n{\n\nclass RedisValue\n{\npublic:\n\t// nil\n\tRedisValue();\n\tvirtual ~RedisValue();\n\n\t//copy constructor\n\tRedisValue(const RedisValue& copy);\n\t//copy operator\n\tRedisValue& operator= (const RedisValue& copy);\n\t//move constructor\n\tRedisValue(RedisValue&& move);\n\t//move operator\n\tRedisValue& operator= (RedisValue&& move);\n\n\t// release memory and change type to nil\n\tvoid set_nil();\n\tvoid set_int(int64_t intv);\n\tvoid set_string(const std::string& strv);\n\tvoid set_status(const std::string& strv);\n\tvoid set_error(const std::string& strv);\n\tvoid set_string(const char *str, size_t len);\n\tvoid set_status(const char *str, size_t len);\n\tvoid set_error(const char *str, size_t len);\n\tvoid set_string(const char *str);\n\tvoid set_status(const char *str);\n\tvoid set_error(const char *str);\n\t// array(resize)\n\tvoid set_array(size_t new_size);\n\t// set data by C style data struct\n\tvoid set(const redis_reply_t *reply);\n\n\t// Return true if not error\n\tbool is_ok() const;\n\t// Return true if error\n\tbool is_error() const;\n\t// Return true if nil\n\tbool is_nil() const;\n\t// Return true if integer\n\tbool is_int() const;\n\t// Return true if array\n\tbool is_array() const;\n\t// Return true if string/status\n\tbool is_string() const;\n\t// Return type of C style data struct\n\tint get_type() const;\n\n\t// Copy. If type isnot string/status/error, returns an empty std::string\n\tstd::string string_value() const;\n\t// No copy. If type isnot string/status/error, returns NULL.\n\tconst std::string *string_view() const;\n\t// If type isnot integer, returns 0\n\tint64_t int_value() const;\n\t// If type isnot array, returns 0\n\tsize_t arr_size() const;\n\t// If type isnot array, do nothing\n\tvoid arr_clear();\n\t// If type isnot array, do nothing\n\tvoid arr_resize(size_t new_size);\n\t// Always return std::vector.at(pos); notice overflow exception\n\tRedisValue& arr_at(size_t pos) const;\n\t// Always return std::vector[pos]; notice overflow exception\n\tRedisValue& operator[] (size_t pos) const;\n\n\t// transform data into C style data struct\n\tbool transform(redis_reply_t *reply) const;\n\t// equal to set_nil();\n\tvoid clear();\n\t// format data to text\n\tstd::string debug_string() const;\n\nprivate:\n\tvoid free_data();\n\tvoid only_set_string_data(const std::string& strv);\n\tvoid only_set_string_data(const char *str, size_t len);\n\tvoid only_set_string_data(const char *str);\n\n\tint type_;\n\tvoid *data_;\n};\n\nclass RedisMessage : public ProtocolMessage\n{\npublic:\n\tRedisMessage();\n\tvirtual ~RedisMessage();\n\t//move constructor\n\tRedisMessage(RedisMessage&& move);\n\t//move operator\n\tRedisMessage& operator= (RedisMessage&& move);\n\npublic:\n\t//peek after CommMessageIn append\n\t//not for users.\n\tbool parse_success() const;\n\tbool is_asking() const;\n\tvoid set_asking(bool asking);\n\nprotected:\n\tredis_parser_t *parser_;\n\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\tbool encode_reply(redis_reply_t *reply);\n\n\tclass EncodeStream *stream_;\n\nprivate:\n\tsize_t cur_size_;\n\tbool asking_;\n};\n\nclass RedisRequest : public RedisMessage\n{\npublic:\n\tRedisRequest() = default;\n\t//move constructor\n\tRedisRequest(RedisRequest&& move) = default;\n\t//move operator\n\tRedisRequest& operator= (RedisRequest&& move) = default;\n\npublic:// C++ style\n\t// Usually, client use set_request to (prepare)send request to server\n\t// Usually, server use get_command/get_params to get client request\n\n\t// set_request(\"HSET\", {\"keyname\", \"hashkey\", \"somevalue\"});\n\tvoid set_request(const std::string& command,\n\t\t\t\t\t const std::vector& params);\n\n\tbool get_command(std::string& command) const;\n\tbool get_params(std::vector& params) const;\n\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\nprivate:\n\tstd::vector user_request_;\n};\n\nclass RedisResponse : public RedisMessage\n{\npublic:\n\tRedisResponse() = default;\n\t//move constructor\n\tRedisResponse(RedisResponse&& move) = default;\n\t//move operator\n\tRedisResponse& operator= (RedisResponse&& move) = default;\n\npublic:// C++ style\n\t// client use get_result to get result from server, copy\n\tvoid get_result(RedisValue& value) const;\n\n\t// server use set_result to (prepare)send result to client, copy\n\tbool set_result(const RedisValue& value);\n\npublic:// C style\n\t// redis_parser_t is absolutely same as hiredis-redisReply in memory\n\t// If you include hiredis.h, redisReply* can cast to redis_reply_t* safely\n\t// BUT this function return not a copy, DONOT free the pointer by yourself\n\n\t// client read data from redis_reply_t by pointer of result_ptr\n\t// server write data into redis_reply_t by pointer of result_ptr\n\tredis_reply_t *result_ptr();\n\nprotected:\n\tvirtual int append(const void *buf, size_t *size);\n\nprivate:\n\tRedisValue value_;\n};\n\n////////////////////\n\ninline RedisValue::RedisValue():\n\ttype_(REDIS_REPLY_TYPE_NIL),\n\tdata_(NULL)\n{\n}\n\ninline RedisValue::RedisValue(const RedisValue& copy):\n\ttype_(REDIS_REPLY_TYPE_NIL),\n\tdata_(NULL)\n{\n\tthis->operator= (copy);\n}\n\ninline RedisValue::RedisValue(RedisValue&& move):\n\ttype_(REDIS_REPLY_TYPE_NIL),\n\tdata_(NULL)\n{\n\tthis->operator= (std::move(move));\n}\n\ninline bool RedisValue::is_ok() const { return type_ != REDIS_REPLY_TYPE_ERROR; }\ninline bool RedisValue::is_error() const { return type_ == REDIS_REPLY_TYPE_ERROR; }\ninline bool RedisValue::is_nil() const { return type_ == REDIS_REPLY_TYPE_NIL; }\ninline bool RedisValue::is_int() const { return type_ == REDIS_REPLY_TYPE_INTEGER; }\ninline bool RedisValue::is_array() const { return type_ == REDIS_REPLY_TYPE_ARRAY; }\ninline int RedisValue::get_type() const { return type_; }\n\ninline bool RedisValue::is_string() const\n{\n\treturn type_ == REDIS_REPLY_TYPE_STRING || type_ == REDIS_REPLY_TYPE_STATUS;\n}\n\ninline std::string RedisValue::string_value() const\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\treturn *((std::string *)data_);\n\telse\n\t\treturn \"\";\n}\n\ninline const std::string *RedisValue::string_view() const\n{\n\tif (type_ == REDIS_REPLY_TYPE_STRING ||\n\t\t\ttype_ == REDIS_REPLY_TYPE_STATUS ||\n\t\t\ttype_ == REDIS_REPLY_TYPE_ERROR)\n\t\treturn ((std::string *)data_);\n\telse\n\t\treturn NULL;\n}\n\ninline int64_t RedisValue::int_value() const\n{\n\tif (type_ == REDIS_REPLY_TYPE_INTEGER)\n\t\treturn *((int64_t *)data_);\n\telse\n\t\treturn 0;\n}\n\ninline size_t RedisValue::arr_size() const\n{\n\tif (type_ == REDIS_REPLY_TYPE_ARRAY)\n\t\treturn ((std::vector *)data_)->size();\n\telse\n\t\treturn 0;\n}\n\ninline RedisValue& RedisValue::arr_at(size_t pos) const\n{\n\treturn ((std::vector *)data_)->at(pos);\n}\n\ninline RedisValue& RedisValue::operator[] (size_t pos) const\n{\n\treturn (*((std::vector *)data_))[pos];\n}\n\ninline void RedisValue::set_nil()\n{\n\tfree_data();\n\ttype_ = REDIS_REPLY_TYPE_NIL;\n}\n\ninline void RedisValue::clear()\n{\n\tset_nil();\n}\n\ninline bool RedisMessage::parse_success() const { return parser_->parse_succ; }\n\ninline bool RedisMessage::is_asking() const { return asking_; }\n\ninline void RedisMessage::set_asking(bool asking) { asking_ = asking; }\n\ninline redis_reply_t *RedisResponse::result_ptr()\n{\n\treturn &parser_->reply;\n}\n\ninline void RedisResponse::get_result(RedisValue& value) const\n{\n\tif (parser_->parse_succ)\n\t\tvalue.set(&parser_->reply);\n\telse\n\t\tvalue.set_nil();\n}\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/SSLWrapper.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"SSLWrapper.h\"\n\nnamespace protocol\n{\n\nint SSLHandshaker::encode(struct iovec vectors[], int max)\n{\n\tBIO *wbio = SSL_get_wbio(this->ssl);\n\tchar *ptr;\n\tlong len;\n\tint ret;\n\n\tret = SSL_do_handshake(this->ssl);\n\tif (ret <= 0)\n\t{\n\t\tret = SSL_get_error(this->ssl, ret);\n\t\tif (ret != SSL_ERROR_WANT_READ)\n\t\t{\n\t\t\tif (ret != SSL_ERROR_SYSCALL)\n\t\t\t\terrno = -ret;\n\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\tlen = BIO_get_mem_data(wbio, &ptr);\n\tif (len > 0)\n\t{\n\t\tvectors[0].iov_base = ptr;\n\t\tvectors[0].iov_len = len;\n\t\treturn 1;\n\t}\n\telse if (len == 0)\n\t\treturn 0;\n\telse\n\t\treturn -1;\n}\n\nstatic int __ssl_handshake(const void *buf, size_t *size, SSL *ssl,\n\t\t\t\t\t\t char **ptr, long *len)\n{\n\tBIO *wbio = SSL_get_wbio(ssl);\n\tBIO *rbio = SSL_get_rbio(ssl);\n\tint ret;\n\n\tret = BIO_write(rbio, buf, *size);\n\tif (ret <= 0)\n\t\treturn -1;\n\n\t*size = ret;\n\tret = SSL_do_handshake(ssl);\n\tif (ret <= 0)\n\t{\n\t\tret = SSL_get_error(ssl, ret);\n\t\tif (ret != SSL_ERROR_WANT_READ)\n\t\t{\n\t\t\tif (ret != SSL_ERROR_SYSCALL)\n\t\t\t\terrno = -ret;\n\n\t\t\treturn -1;\n\t\t}\n\n\t\tret = 0;\n\t}\n\n\t*len = BIO_get_mem_data(wbio, ptr);\n\tif (*len < 0)\n\t\treturn -1;\n\n\treturn ret;\n}\n\nint SSLHandshaker::append(const void *buf, size_t *size)\n{\n\tBIO *wbio = SSL_get_wbio(this->ssl);\n\tchar *ptr;\n\tlong len;\n\tlong n;\n\tint ret;\n\n\tBIO_reset(wbio);\n\tret = __ssl_handshake(buf, size, this->ssl, &ptr, &len);\n\tif (ret != 0)\n\t\treturn ret;\n\n\tif (len > 0)\n\t{\n\t\tn = this->feedback(ptr, len);\n\t\tBIO_reset(wbio);\n\t}\n\telse\n\t\tn = 0;\n\n\tif (n == len)\n\t\treturn ret;\n\n\tif (n >= 0)\n\t\terrno = ENOBUFS;\n\n\treturn -1;\n}\n\nint SSLWrapper::encode(struct iovec vectors[], int max)\n{\n\tBIO *wbio = SSL_get_wbio(this->ssl);\n\tstruct iovec *iov;\n\tchar *ptr;\n\tlong len;\n\tint ret;\n\n\tret = this->ProtocolWrapper::encode(vectors, max);\n\tif ((unsigned int)ret > (unsigned int)max)\n\t\treturn ret;\n\n\tmax = ret;\n\tfor (iov = vectors; iov < vectors + max; iov++)\n\t{\n\t\tif (iov->iov_len > 0)\n\t\t{\n\t\t\tret = SSL_write(this->ssl, iov->iov_base, iov->iov_len);\n\t\t\tif (ret <= 0)\n\t\t\t{\n\t\t\t\tret = SSL_get_error(this->ssl, ret);\n\t\t\t\tif (ret != SSL_ERROR_SYSCALL)\n\t\t\t\t\terrno = -ret;\n\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t}\n\t}\n\n\tlen = BIO_get_mem_data(wbio, &ptr);\n\tif (len > 0)\n\t{\n\t\tvectors[0].iov_base = ptr;\n\t\tvectors[0].iov_len = len;\n\t\treturn 1;\n\t}\n\telse if (len == 0)\n\t\treturn 0;\n\telse\n\t\treturn -1;\n}\n\n#define BUFSIZE\t\t8192\n\nint SSLWrapper::append_message()\n{\n\tchar buf[BUFSIZE];\n\tint ret;\n\n\twhile ((ret = SSL_read(this->ssl, buf, BUFSIZE)) > 0)\n\t{\n\t\tsize_t nleft = ret;\n\t\tchar *p = buf;\n\t\tsize_t n;\n\n\t\tdo\n\t\t{\n\t\t\tn = nleft;\n\t\t\tret = this->ProtocolWrapper::append(p, &n);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tnleft -= n;\n\t\t\t\tp += n;\n\t\t\t}\n\t\t\telse\n\t\t\t\treturn ret;\n\n\t\t} while (nleft > 0);\n\t}\n\n\tret = SSL_get_error(this->ssl, ret);\n\tif (ret != SSL_ERROR_WANT_READ)\n\t{\n\t\tif (ret != SSL_ERROR_SYSCALL)\n\t\t\terrno = -ret;\n\n\t\treturn -1;\n\t}\n\n\treturn 0;\n}\n\nint SSLWrapper::append(const void *buf, size_t *size)\n{\n\tBIO *wbio = SSL_get_wbio(this->ssl);\n\tBIO *rbio = SSL_get_rbio(this->ssl);\n\tint ret;\n\n\tBIO_reset(wbio);\n\tret = BIO_write(rbio, buf, *size);\n\tif (ret <= 0)\n\t\treturn -1;\n\n\t*size = ret;\n\treturn this->append_message();\n}\n\nint SSLWrapper::feedback(const void *buf, size_t size)\n{\n\tBIO *wbio = SSL_get_wbio(this->ssl);\n\tchar *ptr;\n\tlong len;\n\tlong n;\n\tint ret;\n\n\tif (size == 0)\n\t\treturn 0;\n\n\tret = SSL_write(this->ssl, buf, size);\n\tif (ret <= 0)\n\t{\n\t\tret = SSL_get_error(this->ssl, ret);\n\t\tif (ret != SSL_ERROR_SYSCALL)\n\t\t\terrno = -ret;\n\n\t\treturn -1;\n\t}\n\n\tlen = BIO_get_mem_data(wbio, &ptr);\n\tif (len >= 0)\n\t{\n\t\tn = this->ProtocolWrapper::feedback(ptr, len);\n\t\tBIO_reset(wbio);\n\t\tif (n == len)\n\t\t\treturn size;\n\n\t\tif (ret > 0)\n\t\t\terrno = ENOBUFS;\n\t}\n\n\treturn -1;\n}\n\nint ServerSSLWrapper::append(const void *buf, size_t *size)\n{\n\tBIO *wbio = SSL_get_wbio(this->ssl);\n\tchar *ptr;\n\tlong len;\n\tlong n;\n\n\tBIO_reset(wbio);\n\tif (__ssl_handshake(buf, size, this->ssl, &ptr, &len) < 0)\n\t\treturn -1;\n\n\tif (len > 0)\n\t{\n\t\tn = this->ProtocolMessage::feedback(ptr, len);\n\t\tBIO_reset(wbio);\n\t}\n\telse\n\t\tn = 0;\n\n\tif (n == len)\n\t\treturn this->append_message();\n\n\tif (n >= 0)\n\t\terrno = ENOBUFS;\n\n\treturn -1;\n}\n\n}\n\n" }, { "path": "src/protocol/SSLWrapper.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _SSLWRAPPER_H_\n#define _SSLWRAPPER_H_\n\n#include \n#include \"ProtocolMessage.h\"\n\nnamespace protocol\n{\n\nclass SSLHandshaker : public ProtocolMessage\n{\npublic:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\nprotected:\n\tSSL *ssl;\n\npublic:\n\tSSLHandshaker(SSL *ssl)\n\t{\n\t\tthis->ssl = ssl;\n\t}\n\npublic:\n\tSSLHandshaker(SSLHandshaker&& handshaker) = default;\n\tSSLHandshaker& operator = (SSLHandshaker&& handshaker) = default;\n};\n\nclass SSLWrapper : public ProtocolWrapper\n{\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\nprotected:\n\tvirtual int feedback(const void *buf, size_t size);\n\nprotected:\n\tint append_message();\n\nprotected:\n\tSSL *ssl;\n\npublic:\n\tSSLWrapper(ProtocolMessage *message, SSL *ssl) :\n\t\tProtocolWrapper(message)\n\t{\n\t\tthis->ssl = ssl;\n\t}\n\npublic:\n\tSSLWrapper(SSLWrapper&& wrapper) = default;\n\tSSLWrapper& operator = (SSLWrapper&& wrapper) = default;\n};\n\nclass ServerSSLWrapper : public SSLWrapper\n{\nprotected:\n\tvirtual int append(const void *buf, size_t *size);\n\npublic:\n\tServerSSLWrapper(ProtocolMessage *message, SSL *ssl) :\n\t\tSSLWrapper(message, ssl)\n\t{\n\t}\n\npublic:\n\tServerSSLWrapper(ServerSSLWrapper&& wrapper) = default;\n\tServerSSLWrapper& operator = (ServerSSLWrapper&& wrapper) = default;\n};\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/TLVMessage.cc", "content": "/*\n Copyright (c) 2023 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"TLVMessage.h\"\n\nnamespace protocol\n{\n\nint TLVMessage::encode(struct iovec vectors[], int max)\n{\n\tthis->head[0] = htonl((uint32_t)this->type);\n\tthis->head[1] = htonl(this->value.size());\n\n\tvectors[0].iov_base = this->head;\n\tvectors[0].iov_len = 8;\n\tvectors[1].iov_base = (char *)this->value.data();\n\tvectors[1].iov_len = this->value.size();\n\treturn 2;\n}\n\nint TLVMessage::append(const void *buf, size_t *size)\n{\n\tsize_t n = *size;\n\tsize_t head_left;\n\n\thead_left = 8 - this->head_received;\n\tif (head_left > 0)\n\t{\n\t\tvoid *p = (char *)this->head + this->head_received;\n\n\t\tif (n < head_left)\n\t\t{\n\t\t\tmemcpy(p, buf, n);\n\t\t\tthis->head_received += n;\n\t\t\treturn 0;\n\t\t}\n\n\t\tmemcpy(p, buf, head_left);\n\t\tthis->head_received = 8;\n\t\tbuf = (const char *)buf + head_left;\n\t\tn -= head_left;\n\n\t\tthis->type = (int)ntohl(this->head[0]);\n\t\t*this->head = ntohl(this->head[1]);\n\t\tif (*this->head > this->size_limit)\n\t\t{\n\t\t\terrno = EMSGSIZE;\n\t\t\treturn -1;\n\t\t}\n\n\t\tthis->value.reserve(*this->head);\n\t}\n\n\tif (this->value.size() + n > *this->head)\n\t{\n\t\tn = *this->head - this->value.size();\n\t\t*size = n + head_left;\n\t}\n\n\tthis->value.append((const char *)buf, n);\n\treturn this->value.size() == *this->head;\n}\n\n}\n\n" }, { "path": "src/protocol/TLVMessage.h", "content": "/*\n Copyright (c) 2023 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _TLVMESSAGE_H_\n#define _TLVMESSAGE_H_\n\n#include \n#include \n#include \n#include \"ProtocolMessage.h\"\n\nnamespace protocol\n{\n\nclass TLVMessage : public ProtocolMessage\n{\npublic:\n\tint get_type() const { return this->type; }\n\tvoid set_type(int type) { this->type = type; }\n\n\tstd::string *get_value() { return &this->value; }\n\tvoid set_value(std::string value) { this->value = std::move(value); }\n\nprotected:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t *size);\n\nprotected:\n\tint type;\n\tstd::string value;\n\nprivate:\n\tuint32_t head[2];\n\tsize_t head_received;\n\npublic:\n\tTLVMessage()\n\t{\n\t\tthis->type = 0;\n\t\tthis->head_received = 0;\n\t}\n\npublic:\n\tTLVMessage(TLVMessage&& msg) = default;\n\tTLVMessage& operator = (TLVMessage&& msg) = default;\n};\n\nusing TLVRequest = TLVMessage;\nusing TLVResponse = TLVMessage;\n\n}\n\n#endif\n\n" }, { "path": "src/protocol/dns_parser.c", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"dns_types.h\"\n#include \"dns_parser.h\"\n\n#define DNS_LABELS_MAX\t\t\t63\n#define DNS_NAMES_MAX\t\t\t256\n#define DNS_MSGBASE_INIT_SIZE\t514 // 512 + 2(leading length)\n#define MAX(x, y) ((x) <= (y) ? (y) : (x))\n\nstruct __dns_record_entry\n{\n\tstruct list_head entry_list;\n\tstruct dns_record record;\n};\n\n\nstatic inline uint8_t __dns_parser_uint8(const char *ptr)\n{\n\treturn (unsigned char)ptr[0];\n}\n\nstatic inline uint16_t __dns_parser_uint16(const char *ptr)\n{\n\tconst unsigned char *p = (const unsigned char *)ptr;\n\treturn ((uint16_t)p[0] << 8) +\n\t\t ((uint16_t)p[1]);\n}\n\nstatic inline uint32_t __dns_parser_uint32(const char *ptr)\n{\n\tconst unsigned char *p = (const unsigned char *)ptr;\n\treturn ((uint32_t)p[0] << 24) +\n\t\t ((uint32_t)p[1] << 16) +\n\t\t ((uint32_t)p[2] << 8) +\n\t\t ((uint32_t)p[3]);\n}\n\n/*\n * Parse a single .\n * is a domain name represented as a series of labels, and\n * terminated by a label with zero length.\n * \n * phost must point to an char array with at least DNS_NAMES_MAX+1 size\n */\nstatic int __dns_parser_parse_host(char *phost, dns_parser_t *parser)\n{\n\tuint8_t len;\n\tuint16_t pointer;\n\tsize_t hcur;\n\tconst char *msgend;\n\tconst char **cur;\n\tconst char *curbackup; // backup cur when host label is pointer\n\n\tmsgend = (const char *)parser->msgbuf + parser->msgsize;\n\tcur = &(parser->cur);\n\tcurbackup = NULL;\n\thcur = 0;\n\n\tif (*cur >= msgend)\n\t\treturn -2;\n\n\twhile (*cur < msgend)\n\t{\n\t\tlen = __dns_parser_uint8(*cur);\n\n\t\tif ((len & 0xC0) == 0)\n\t\t{\n\t\t\t(*cur)++;\n\t\t\tif (len == 0)\n\t\t\t\tbreak;\n\t\t\tif (len > DNS_LABELS_MAX || *cur + len > msgend ||\n\t\t\t\thcur + len + 1 > DNS_NAMES_MAX)\n\t\t\t\treturn -2;\n\n\t\t\tmemcpy(phost + hcur, *cur, len);\n\t\t\t*cur += len;\n\t\t\thcur += len;\n\t\t\tphost[hcur++] = '.';\n\t\t}\n\t\t// RFC 1035, 4.1.4 Message compression\n\t\telse if ((len & 0xC0) == 0xC0)\n\t\t{\n\t\t\tpointer = __dns_parser_uint16(*cur) & 0x3FFF;\n\n\t\t\tif (pointer >= parser->msgsize)\n\t\t\t\treturn -2;\n\n\t\t\t// pointer must point to a prior position\n\t\t\tif ((const char *)parser->msgbase + pointer >= *cur)\n\t\t\t\treturn -2;\n\n\t\t\t*cur += 2;\n\n\t\t\t// backup cur only when the first pointer occurs\n\t\t\tif (curbackup == NULL)\n\t\t\t\tcurbackup = *cur;\n\t\t\t*cur = (const char *)parser->msgbase + pointer;\n\t\t}\n\t\telse\n\t\t\treturn -2;\n\t}\n\tif (curbackup != NULL)\n\t\t*cur = curbackup;\n\n\tif (hcur > 1 && phost[hcur - 1] == '.')\n\t\thcur--;\n\n\tif (hcur == 0)\n\t\tphost[hcur++] = '.';\n\tphost[hcur++] = '\\0';\n\n\treturn 0;\n}\n\nstatic void __dns_parser_free_record(struct __dns_record_entry *r)\n{\n\tswitch (r->record.type)\n\t{\n\tcase DNS_TYPE_SOA:\n\t{\n\t\tstruct dns_record_soa *soa;\n\t\tsoa = (struct dns_record_soa *)(r->record.rdata);\n\t\tfree(soa->mname);\n\t\tfree(soa->rname);\n\t\tbreak;\n\t}\n\tcase DNS_TYPE_SRV:\n\t{\n\t\tstruct dns_record_srv *srv;\n\t\tsrv = (struct dns_record_srv *)(r->record.rdata);\n\t\tfree(srv->target);\n\t\tbreak;\n\t}\n\tcase DNS_TYPE_MX:\n\t{\n\t\tstruct dns_record_mx *mx;\n\t\tmx = (struct dns_record_mx *)(r->record.rdata);\n\t\tfree(mx->exchange);\n\t\tbreak;\n\t}\n\t}\n\tfree(r->record.name);\n\tfree(r);\n}\n\nstatic void __dns_parser_free_record_list(struct list_head *head)\n{\n\tstruct list_head *pos, *tmp;\n\tstruct __dns_record_entry *entry;\n\n\tlist_for_each_safe(pos, tmp, head)\n\t{\n\t\tentry = list_entry(pos, struct __dns_record_entry, entry_list);\n\t\tlist_del(pos);\n\t\t__dns_parser_free_record(entry);\n\t}\n}\n\n/*\n * A RDATA format, from RFC 1035:\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | ADDRESS |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n *\n * ADDRESS: A 32 bit Internet address.\n * Hosts that have multiple Internet addresses will have multiple A records.\n */\nstatic int __dns_parser_parse_a(struct __dns_record_entry **r,\n\t\t\t\t\t\t\t\tuint16_t rdlength,\n\t\t\t\t\t\t\t\tdns_parser_t *parser)\n{\n\tconst char **cur;\n\tstruct __dns_record_entry *entry;\n\tsize_t entry_size;\n\n\tif (sizeof (struct in_addr) != rdlength)\n\t\treturn -2;\n\n\tcur = &(parser->cur);\n\tentry_size = sizeof (struct __dns_record_entry) + sizeof (struct in_addr);\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tmemcpy(entry->record.rdata, *cur, rdlength);\n\t*cur += rdlength;\n\t*r = entry;\n\n\treturn 0;\n}\n\n/*\n * AAAA RDATA format, from RFC 3596:\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | ADDRESS |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n *\n * ADDRESS: A 128 bit Internet address.\n * Hosts that have multiple addresses will have multiple AAAA records.\n */\nstatic int __dns_parser_parse_aaaa(struct __dns_record_entry **r,\n\t\t\t\t\t\t\t\t uint16_t rdlength,\n\t\t\t\t\t\t\t\t dns_parser_t *parser)\n{\n\tconst char **cur;\n\tstruct __dns_record_entry *entry;\n\tsize_t entry_size;\n\n\tif (sizeof (struct in6_addr) != rdlength)\n\t\treturn -2;\n\n\tcur = &(parser->cur);\n\tentry_size = sizeof (struct __dns_record_entry) + sizeof (struct in6_addr);\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tmemcpy(entry->record.rdata, *cur, rdlength);\n\t*cur += rdlength;\n\t*r = entry;\n\n\treturn 0;\n}\n\n/*\n * Parse any record.\n */\nstatic int __dns_parser_parse_names(struct __dns_record_entry **r,\n\t\t\t\t\t\t\t\t\tuint16_t rdlength,\n\t\t\t\t\t\t\t\t\tdns_parser_t *parser)\n{\n\tconst char *rcdend;\n\tconst char **cur;\n\tstruct __dns_record_entry *entry;\n\tsize_t entry_size;\n\tsize_t name_len;\n\tchar name[DNS_NAMES_MAX + 2];\n\tint ret;\n\n\tcur = &(parser->cur);\n\trcdend = *cur + rdlength;\n\tret = __dns_parser_parse_host(name, parser);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tif (*cur != rcdend)\n\t\treturn -2;\n\n\tname_len = strlen(name);\n\tentry_size = sizeof (struct __dns_record_entry) + name_len + 1;\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tmemcpy(entry->record.rdata, name, name_len + 1);\n\t*r = entry;\n\n\treturn 0;\n}\n\n/*\n * SOA RDATA format, from RFC 1035:\n * 1 1 1 1 1 1\n * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * / MNAME /\n * / /\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * / RNAME /\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | SERIAL |\n * | |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | REFRESH |\n * | |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | RETRY |\n * | |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | EXPIRE |\n * | |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | MINIMUM |\n * | |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n *\n * MNAME: \n * RNAME: \n * SERIAL: The unsigned 32 bit version number.\n * REFRESH: A 32 bit time interval.\n * RETRY: A 32 bit time interval.\n * EXPIRE: A 32 bit time value.\n * MINIMUM: The unsigned 32 bit integer.\n */\nstatic int __dns_parser_parse_soa(struct __dns_record_entry **r,\n\t\t\t\t\t\t\t\t uint16_t rdlength,\n\t\t\t\t\t\t\t\t dns_parser_t *parser)\n{\n\tconst char *rcdend;\n\tconst char **cur;\n\tstruct __dns_record_entry *entry;\n\tstruct dns_record_soa *soa;\n\tsize_t entry_size;\n\tchar mname[DNS_NAMES_MAX + 2];\n\tchar rname[DNS_NAMES_MAX + 2];\n\tint ret;\n\n\tcur = &(parser->cur);\n\trcdend = *cur + rdlength;\n\tret = __dns_parser_parse_host(mname, parser);\n\tif (ret < 0)\n\t\treturn ret;\n\tret = __dns_parser_parse_host(rname, parser);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tif (*cur + 20 != rcdend)\n\t\treturn -2;\n\n\tentry_size = sizeof (struct __dns_record_entry) +\n\t\t\t\t sizeof (struct dns_record_soa);\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tsoa = (struct dns_record_soa *)(entry->record.rdata);\n\n\tsoa->mname = strdup(mname);\n\tsoa->rname = strdup(rname);\n\tsoa->serial = __dns_parser_uint32(*cur);\n\tsoa->refresh = __dns_parser_uint32(*cur + 4);\n\tsoa->retry = __dns_parser_uint32(*cur + 8);\n\tsoa->expire = __dns_parser_uint32(*cur + 12);\n\tsoa->minimum = __dns_parser_uint32(*cur + 16);\n\n\tif (!soa->mname || !soa->rname)\n\t{\n\t\tfree(soa->mname);\n\t\tfree(soa->rname);\n\t\tfree(entry);\n\t\treturn -1;\n\t}\n\n\t*cur += 20;\n\t*r = entry;\n\n\treturn 0;\n}\n\n/*\n * SRV RDATA format, from RFC 2782:\n * 1 1 1 1 1 1\n * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | PRIORITY |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | WEIGHT |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | PORT |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * / TARGET /\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n *\n * PRIORITY: A 16 bit unsigned integer in network byte order.\n * WEIGHT: A 16 bit unsigned integer in network byte order.\n * PORT: A 16 bit unsigned integer in network byte order.\n * TARGET: \n */\nstatic int __dns_parser_parse_srv(struct __dns_record_entry **r,\n\t\t\t\t\t\t\t\t uint16_t rdlength,\n\t\t\t\t\t\t\t\t dns_parser_t *parser)\n{\n\tconst char *rcdend;\n\tconst char **cur;\n\tstruct __dns_record_entry *entry;\n\tstruct dns_record_srv *srv;\n\tsize_t entry_size;\n\tchar target[DNS_NAMES_MAX + 2];\n\tuint16_t priority;\n\tuint16_t weight;\n\tuint16_t port;\n\tint ret;\n\n\tcur = &(parser->cur);\n\trcdend = *cur + rdlength;\n\n\tif (*cur + 6 > rcdend)\n\t\treturn -2;\n\n\tpriority = __dns_parser_uint16(*cur);\n\tweight = __dns_parser_uint16(*cur + 2);\n\tport = __dns_parser_uint16(*cur + 4);\n\t*cur += 6;\n\n\tret = __dns_parser_parse_host(target, parser);\n\tif (ret < 0)\n\t\treturn ret;\n\tif (*cur != rcdend)\n\t\treturn -2;\n\n\tentry_size = sizeof (struct __dns_record_entry) +\n\t\t\t\t sizeof (struct dns_record_srv);\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tsrv = (struct dns_record_srv *)(entry->record.rdata);\n\n\tsrv->priority = priority;\n\tsrv->weight = weight;\n\tsrv->port = port;\n\tsrv->target = strdup(target);\n\n\tif (!srv->target)\n\t{\n\t\tfree(entry);\n\t\treturn -1;\n\t}\n\n\t*r = entry;\n\n\treturn 0;\n}\n\nstatic int __dns_parser_parse_mx(struct __dns_record_entry **r,\n\t\t\t\t\t\t\t\t uint16_t rdlength,\n\t\t\t\t\t\t\t\t dns_parser_t *parser)\n{\n\tconst char *rcdend;\n\tconst char **cur;\n\tstruct __dns_record_entry *entry;\n\tstruct dns_record_mx *mx;\n\tsize_t entry_size;\n\tchar exchange[DNS_NAMES_MAX + 2];\n\tint16_t preference;\n\tint ret;\n\n\tcur = &(parser->cur);\n\trcdend = *cur + rdlength;\n\n\tif (*cur + 2 > rcdend)\n\t\treturn -2;\n\tpreference = __dns_parser_uint16(*cur);\n\t*cur += 2;\n\n\tret = __dns_parser_parse_host(exchange, parser);\n\tif (ret < 0)\n\t\treturn ret;\n\tif (*cur != rcdend)\n\t\treturn -2;\n\n\tentry_size = sizeof (struct __dns_record_entry) +\n\t\t\t\t sizeof (struct dns_record_mx);\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tmx = (struct dns_record_mx *)(entry->record.rdata);\n\tmx->exchange = strdup(exchange);\n\tmx->preference = preference;\n\n\tif (!mx->exchange)\n\t{\n\t\tfree(entry);\n\t\treturn -1;\n\t}\n\n\t*r = entry;\n\n\treturn 0;\n}\n\nstatic int __dns_parser_parse_others(struct __dns_record_entry **r,\n\t\t\t\t\t\t\t\t\t uint16_t rdlength,\n\t\t\t\t\t\t\t\t\t dns_parser_t *parser)\n{\n\tconst char **cur;\n\tstruct __dns_record_entry *entry;\n\tsize_t entry_size;\n\n\tcur = &(parser->cur);\n\tentry_size = sizeof (struct __dns_record_entry) + rdlength;\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tmemcpy(entry->record.rdata, *cur, rdlength);\n\t*cur += rdlength;\n\t*r = entry;\n\n\treturn 0;\n}\n\n/*\n * RR format, from RFC 1035:\n * 1 1 1 1 1 1\n * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | |\n * / NAME /\n * / /\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | TYPE |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | CLASS |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | TTL |\n * | |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | RDLENGTH |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * / RDATA /\n * / /\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n *\n */\nstatic int __dns_parser_parse_record(int idx, dns_parser_t *parser)\n{\n\tuint16_t i;\n\tuint16_t type;\n\tuint16_t rclass;\n\tuint32_t ttl;\n\tuint16_t rdlength;\n\tuint16_t count;\n\tconst char *msgend;\n\tconst char **cur;\n\tint ret;\n\tstruct __dns_record_entry *entry;\n\tchar host[DNS_NAMES_MAX + 2];\n\tstruct list_head *list;\n\n\tswitch (idx)\n\t{\n\tcase 0:\n\t\tcount = parser->header.ancount;\n\t\tlist = &parser->answer_list;\n\t\tbreak;\n\tcase 1:\n\t\tcount = parser->header.nscount;\n\t\tlist = &parser->authority_list;\n\t\tbreak;\n\tcase 2:\n\t\tcount = parser->header.arcount;\n\t\tlist = &parser->additional_list;\n\t\tbreak;\n\tdefault:\n\t\treturn -2;\n\t}\n\n\tmsgend = (const char *)parser->msgbuf + parser->msgsize;\n\tcur = &(parser->cur);\n\n\tfor (i = 0; i < count; i++)\n\t{\n\t\tret = __dns_parser_parse_host(host, parser);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\t// TYPE(2) + CLASS(2) + TTL(4) + RDLENGTH(2) = 10\n\t\tif (*cur + 10 > msgend)\n\t\t\treturn -2;\n\t\ttype = __dns_parser_uint16(*cur);\n\t\trclass = __dns_parser_uint16(*cur + 2);\n\t\tttl = __dns_parser_uint32(*cur + 4);\n\t\trdlength = __dns_parser_uint16(*cur + 8);\n\t\t*cur += 10;\n\t\tif (*cur + rdlength > msgend)\n\t\t\treturn -2;\n\n\t\tentry = NULL;\n\t\tswitch (type)\n\t\t{\n\t\tcase DNS_TYPE_A:\n\t\t\tret = __dns_parser_parse_a(&entry, rdlength, parser);\n\t\t\tbreak;\n\t\tcase DNS_TYPE_AAAA:\n\t\t\tret = __dns_parser_parse_aaaa(&entry, rdlength, parser);\n\t\t\tbreak;\n\t\tcase DNS_TYPE_NS:\n\t\tcase DNS_TYPE_CNAME:\n\t\tcase DNS_TYPE_PTR:\n\t\t\tret = __dns_parser_parse_names(&entry, rdlength, parser);\n\t\t\tbreak;\n\t\tcase DNS_TYPE_SOA:\n\t\t\tret = __dns_parser_parse_soa(&entry, rdlength, parser);\n\t\t\tbreak;\n\t\tcase DNS_TYPE_SRV:\n\t\t\tret = __dns_parser_parse_srv(&entry, rdlength, parser);\n\t\t\tbreak;\n\t\tcase DNS_TYPE_MX:\n\t\t\tret = __dns_parser_parse_mx(&entry, rdlength, parser);\n\t\t\tbreak;\n\t\tdefault:\n\t\t\tret = __dns_parser_parse_others(&entry, rdlength, parser);\n\t\t}\n\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tentry->record.name = strdup(host);\n\t\tif (!entry->record.name)\n\t\t{\n\t\t\t__dns_parser_free_record(entry);\n\t\t\treturn -1;\n\t\t}\n\n\t\tentry->record.type = type;\n\t\tentry->record.rclass = rclass;\n\t\tentry->record.ttl = ttl;\n\t\tentry->record.rdlength = rdlength;\n\t\tlist_add_tail(&entry->entry_list, list);\n\t}\n\n\treturn 0;\n}\n\n/*\n * Question format, from RFC 1035:\n * 1 1 1 1 1 1\n * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | |\n * / QNAME /\n * / /\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | QTYPE |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n * | QCLASS |\n * +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+\n *\n * The query name is encoded as a series of labels, each represented\n * as a one-byte length (maximum 63) followed by the text of the\n * label. The list is terminated by a label of length zero (which can\n * be thought of as the root domain).\n */\nstatic int __dns_parser_parse_question(dns_parser_t *parser)\n{\n\tuint16_t qtype;\n\tuint16_t qclass;\n\tconst char *msgend;\n\tconst char **cur;\n\tint ret;\n\tchar host[DNS_NAMES_MAX + 2];\n\n\tmsgend = (const char *)parser->msgbuf + parser->msgsize;\n\tcur = &(parser->cur);\n\n\t// question count != 1 is an error\n\tif (parser->header.qdcount != 1)\n\t\treturn -2;\n\n\t// parse qname\n\tret = __dns_parser_parse_host(host, parser);\n\tif (ret < 0)\n\t\treturn ret;\n\n\t// parse qtype and qclass\n\tif (*cur + 4 > msgend)\n\t\treturn -2;\n\n\tqtype = __dns_parser_uint16(*cur);\n\tqclass = __dns_parser_uint16(*cur + 2);\n\t*cur += 4;\n\n\tif (parser->question.qname)\n\t\tfree(parser->question.qname);\n\n\tparser->question.qname = strdup(host);\n\tif (parser->question.qname == NULL)\n\t\treturn -1;\n\n\tparser->question.qtype = qtype;\n\tparser->question.qclass = qclass;\n\n\treturn 0;\n}\n\nvoid dns_parser_init(dns_parser_t *parser)\n{\n\tparser->msgbuf = NULL;\n\tparser->msgbase = NULL;\n\tparser->cur = NULL;\n\tparser->msgsize = 0;\n\tparser->bufsize = 0;\n\tparser->complete = 0;\n\tparser->single_packet = 0;\n\tmemset(&parser->header, 0, sizeof (struct dns_header));\n\tmemset(&parser->question, 0, sizeof (struct dns_question));\n\tINIT_LIST_HEAD(&parser->answer_list);\n\tINIT_LIST_HEAD(&parser->authority_list);\n\tINIT_LIST_HEAD(&parser->additional_list);\n}\n\nint dns_parser_set_question(const char *name,\n\t\t\t\t\t\t\tuint16_t qtype,\n\t\t\t\t\t\t\tuint16_t qclass,\n\t\t\t\t\t\t\tdns_parser_t *parser)\n{\n\tint ret;\n\n\tret = dns_parser_set_question_name(name, parser);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tparser->question.qtype = qtype;\n\tparser->question.qclass = qclass;\n\tparser->header.qdcount = 1;\n\n\treturn 0;\n}\n\nint dns_parser_set_question_name(const char *name, dns_parser_t *parser)\n{\n\tchar *newname;\n\tsize_t len;\n\n\tlen = strlen(name);\n\tnewname = (char *)malloc(len + 1);\n\n\tif (!newname)\n\t\treturn -1;\n\n\tmemcpy(newname, name, len + 1);\n\t// Remove trailing dot, except name is \".\"\n\tif (len > 1 && newname[len - 1] == '.')\n\t\tnewname[len - 1] = '\\0';\n\n\tif (parser->question.qname)\n\t\tfree(parser->question.qname);\n\tparser->question.qname = newname;\n\n\treturn 0;\n}\n\nvoid dns_parser_set_id(uint16_t id, dns_parser_t *parser)\n{\n\tparser->header.id = id;\n}\n\nint dns_parser_parse_all(dns_parser_t *parser)\n{\n\tstruct dns_header *h;\n\tint ret;\n\tint i;\n\n\tparser->complete = 1;\n\tparser->cur = (const char *)parser->msgbase;\n\th = &parser->header;\n\n\tif (parser->msgsize < sizeof (struct dns_header))\n\t\treturn -2;\n\n\tmemcpy(h, parser->msgbase, sizeof (struct dns_header));\n\th->id = ntohs(h->id);\n\th->qdcount = ntohs(h->qdcount);\n\th->ancount = ntohs(h->ancount);\n\th->nscount = ntohs(h->nscount);\n\th->arcount = ntohs(h->arcount);\n\tparser->cur += sizeof (struct dns_header);\n\n\tret = __dns_parser_parse_question(parser);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tfor (i = 0; i < 3; i++)\n\t{\n\t\tret = __dns_parser_parse_record(i, parser);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\t}\n\n\treturn 0;\n}\n\nint dns_parser_append_message(const void *buf,\n\t\t\t\t\t\t\t size_t *n,\n\t\t\t\t\t\t\t dns_parser_t *parser)\n{\n\tint ret;\n\tsize_t total;\n\tsize_t new_size;\n\tsize_t msgsize_bak;\n\tvoid *new_buf;\n\n\tif (parser->complete)\n\t{\n\t\t*n = 0;\n\t\treturn 1;\n\t}\n\n\tif (!parser->single_packet)\n\t{\n\t\tmsgsize_bak = parser->msgsize;\n\t\tif (parser->msgsize + *n > parser->bufsize)\n\t\t{\n\t\t\tnew_size = MAX(DNS_MSGBASE_INIT_SIZE, 2 * parser->bufsize);\n\n\t\t\twhile (new_size < parser->msgsize + *n)\n\t\t\t\tnew_size *= 2;\n\n\t\t\tnew_buf = realloc(parser->msgbuf, new_size);\n\t\t\tif (!new_buf)\n\t\t\t\treturn -1;\n\n\t\t\tparser->msgbuf = new_buf;\n\t\t\tparser->bufsize = new_size;\n\t\t}\n\n\t\tmemcpy((char*)parser->msgbuf + parser->msgsize, buf, *n);\n\t\tparser->msgsize += *n;\n\n\t\tif (parser->msgsize < 2)\n\t\t\treturn 0;\n\n\t\ttotal = __dns_parser_uint16((char*)parser->msgbuf);\n\t\tif (parser->msgsize < total + 2)\n\t\t\treturn 0;\n\n\t\t*n = total + 2 - msgsize_bak;\n\t\tparser->msgsize = total + 2;\n\t\tparser->msgbase = (char*)parser->msgbuf + 2;\n\t}\n\telse\n\t{\n\t\tparser->msgbuf = malloc(*n);\n\t\tmemcpy(parser->msgbuf, buf, *n);\n\t\tparser->msgbase = parser->msgbuf;\n\t\tparser->msgsize = *n;\n\t\tparser->bufsize = *n;\n\t}\n\n\tret = dns_parser_parse_all(parser);\n\tif (ret < 0)\n\t\treturn ret;\n\n\treturn 1;\n}\n\nvoid dns_parser_deinit(dns_parser_t *parser)\n{\n\tfree(parser->msgbuf);\n\tfree(parser->question.qname);\n\n\t__dns_parser_free_record_list(&parser->answer_list);\n\t__dns_parser_free_record_list(&parser->authority_list);\n\t__dns_parser_free_record_list(&parser->additional_list);\n}\n\nint dns_record_cursor_next(struct dns_record **record,\n\t\t\t\t\t\t dns_record_cursor_t *cursor)\n{\n\tstruct __dns_record_entry *e;\n\n\tif (cursor->next->next != cursor->head)\n\t{\n\t\tcursor->next = cursor->next->next;\n\t\te = list_entry(cursor->next, struct __dns_record_entry, entry_list);\n\t\t*record = &e->record;\n\t\treturn 0;\n\t}\n\n\treturn 1;\n}\n\nint dns_record_cursor_find_cname(const char *name,\n\t\t\t\t\t\t\t\t const char **cname,\n\t\t\t\t\t\t\t\t dns_record_cursor_t *cursor)\n{\n\tstruct __dns_record_entry *e;\n\n\tif (!name || !cname)\n\t\treturn 1;\n\n\tcursor->next = cursor->head;\n\twhile (cursor->next->next != cursor->head)\n\t{\n\t\tcursor->next = cursor->next->next;\n\t\te = list_entry(cursor->next, struct __dns_record_entry, entry_list);\n\n\t\tif (e->record.type == DNS_TYPE_CNAME &&\n\t\t\tstrcasecmp(name, e->record.name) == 0)\n\t\t{\n\t\t\t*cname = (const char *)e->record.rdata;\n\t\t\treturn 0;\n\t\t}\n\t}\n\n\treturn 1;\n}\n\nint dns_add_raw_record(const char *name, uint16_t type, uint16_t rclass,\n\t\t\t\t\t uint32_t ttl, uint16_t rlen, const void *rdata,\n\t\t\t\t\t struct list_head *list)\n{\n\tstruct __dns_record_entry *entry;\n\tsize_t entry_size = sizeof (struct __dns_record_entry) + rlen;\n\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.name = strdup(name);\n\tif (!entry->record.name)\n\t{\n\t\tfree(entry);\n\t\treturn -1;\n\t}\n\n\tentry->record.type = type;\n\tentry->record.rclass = rclass;\n\tentry->record.ttl = ttl;\n\tentry->record.rdlength = rlen;\n\tentry->record.rdata = (void *)(entry + 1);\n\tmemcpy(entry->record.rdata, rdata, rlen);\n\tlist_add_tail(&entry->entry_list, list);\n\n\treturn 0;\n}\n\nint dns_add_str_record(const char *name, uint16_t type, uint16_t rclass,\n\t\t\t\t\t uint32_t ttl, const char *rdata,\n\t\t\t\t\t struct list_head *list)\n{\n\tsize_t rlen = strlen(rdata);\n\t// record.rdlength has no meaning for parsed record types, ignore its\n\t// correctness, same for soa/srv/mx record\n\treturn dns_add_raw_record(name, type, rclass, ttl, rlen+1, rdata, list);\n}\n\nint dns_add_soa_record(const char *name, uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t const char *mname, const char *rname,\n\t\t\t\t\t uint32_t serial, int32_t refresh,\n\t\t\t\t\t int32_t retry, int32_t expire, uint32_t minimum,\n\t\t\t\t\t struct list_head *list)\n{\n\tstruct __dns_record_entry *entry;\n\tstruct dns_record_soa *soa;\n\tsize_t entry_size;\n\tchar *pname, *pmname, *prname;\n\n\tentry_size = sizeof (struct __dns_record_entry) +\n\t\t\t\t sizeof (struct dns_record_soa);\n\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tentry->record.rdlength = 0;\n\tsoa = (struct dns_record_soa *)(entry->record.rdata);\n\n\tpname = strdup(name);\n\tpmname = strdup(mname);\n\tprname = strdup(rname);\n\n\tif (!pname || !pmname || !prname)\n\t{\n\t\tfree(pname);\n\t\tfree(pmname);\n\t\tfree(prname);\n\t\tfree(entry);\n\t\treturn -1;\n\t}\n\n\tsoa->mname = pmname;\n\tsoa->rname = prname;\n\tsoa->serial = serial;\n\tsoa->refresh = refresh;\n\tsoa->retry = retry;\n\tsoa->expire = expire;\n\tsoa->minimum = minimum;\n\n\tentry->record.name = pname;\n\tentry->record.type = DNS_TYPE_SOA;\n\tentry->record.rclass = rclass;\n\tentry->record.ttl = ttl;\n\tlist_add_tail(&entry->entry_list, list);\n\n\treturn 0;\n}\n\nint dns_add_srv_record(const char *name, uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t uint16_t priority, uint16_t weight,\n\t\t\t\t\t uint16_t port, const char *target,\n\t\t\t\t\t struct list_head *list)\n{\n\tstruct __dns_record_entry *entry;\n\tstruct dns_record_srv *srv;\n\tsize_t entry_size;\n\tchar *pname, *ptarget;\n\n\tentry_size = sizeof (struct __dns_record_entry) +\n\t\t\t\t sizeof (struct dns_record_srv);\n\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tentry->record.rdlength = 0;\n\tsrv = (struct dns_record_srv *)(entry->record.rdata);\n\n\tpname = strdup(name);\n\tptarget = strdup(target);\n\n\tif (!pname || !ptarget)\n\t{\n\t\tfree(pname);\n\t\tfree(ptarget);\n\t\tfree(entry);\n\t\treturn -1;\n\t}\n\n\tsrv->priority = priority;\n\tsrv->weight = weight;\n\tsrv->port = port;\n\tsrv->target = ptarget;\n\n\tentry->record.name = pname;\n\tentry->record.type = DNS_TYPE_SRV;\n\tentry->record.rclass = rclass;\n\tentry->record.ttl = ttl;\n\tlist_add_tail(&entry->entry_list, list);\n\n\treturn 0;\n}\n\nint dns_add_mx_record(const char *name, uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t int16_t preference, const char *exchange,\n\t\t\t\t\t struct list_head *list)\n{\n\tstruct __dns_record_entry *entry;\n\tstruct dns_record_mx *mx;\n\tsize_t entry_size;\n\tchar *pname, *pexchange;\n\n\tentry_size = sizeof (struct __dns_record_entry) +\n\t\t\t\t sizeof (struct dns_record_mx);\n\n\tentry = (struct __dns_record_entry *)malloc(entry_size);\n\tif (!entry)\n\t\treturn -1;\n\n\tentry->record.rdata = (void *)(entry + 1);\n\tentry->record.rdlength = 0;\n\tmx = (struct dns_record_mx *)(entry->record.rdata);\n\n\tpname = strdup(name);\n\tpexchange = strdup(exchange);\n\n\tif (!pname || !pexchange)\n\t{\n\t\tfree(pname);\n\t\tfree(pexchange);\n\t\tfree(entry);\n\t\treturn -1;\n\t}\n\n\tmx->preference = preference;\n\tmx->exchange = pexchange;\n\n\tentry->record.name = pname;\n\tentry->record.type = DNS_TYPE_MX;\n\tentry->record.rclass = rclass;\n\tentry->record.ttl = ttl;\n\tlist_add_tail(&entry->entry_list, list);\n\n\treturn 0;\n}\n\nconst char *dns_type2str(int type)\n{\n\tswitch (type)\n\t{\n\tcase DNS_TYPE_A:\n\t\treturn \"A\";\n\tcase DNS_TYPE_NS:\n\t\treturn \"NS\";\n\tcase DNS_TYPE_MD:\n\t\treturn \"MD\";\n\tcase DNS_TYPE_MF:\n\t\treturn \"MF\";\n\tcase DNS_TYPE_CNAME:\n\t\treturn \"CNAME\";\n\tcase DNS_TYPE_SOA:\n\t\treturn \"SOA\";\n\tcase DNS_TYPE_MB:\n\t\treturn \"MB\";\n\tcase DNS_TYPE_MG:\n\t\treturn \"MG\";\n\tcase DNS_TYPE_MR:\n\t\treturn \"MR\";\n\tcase DNS_TYPE_NULL:\n\t\treturn \"NULL\";\n\tcase DNS_TYPE_WKS:\n\t\treturn \"WKS\";\n\tcase DNS_TYPE_PTR:\n\t\treturn \"PTR\";\n\tcase DNS_TYPE_HINFO:\n\t\treturn \"HINFO\";\n\tcase DNS_TYPE_MINFO:\n\t\treturn \"MINFO\";\n\tcase DNS_TYPE_MX:\n\t\treturn \"MX\";\n\tcase DNS_TYPE_AAAA:\n\t\treturn \"AAAA\";\n\tcase DNS_TYPE_SRV:\n\t\treturn \"SRV\";\n\tcase DNS_TYPE_TXT:\n\t\treturn \"TXT\";\n\tcase DNS_TYPE_AXFR:\n\t\treturn \"AXFR\";\n\tcase DNS_TYPE_MAILB:\n\t\treturn \"MAILB\";\n\tcase DNS_TYPE_MAILA:\n\t\treturn \"MAILA\";\n\tcase DNS_TYPE_ALL:\n\t\treturn \"ALL\";\n\tdefault:\n\t\treturn \"Unknown\";\n\t}\n}\n\nconst char *dns_class2str(int dnsclass)\n{\n\tswitch (dnsclass)\n\t{\n\tcase DNS_CLASS_IN:\n\t\treturn \"IN\";\n\tcase DNS_CLASS_CS:\n\t\treturn \"CS\";\n\tcase DNS_CLASS_CH:\n\t\treturn \"CH\";\n\tcase DNS_CLASS_HS:\n\t\treturn \"HS\";\n\tcase DNS_CLASS_ALL:\n\t\treturn \"ALL\";\n\tdefault:\n\t\treturn \"Unknown\";\n\t}\n}\n\nconst char *dns_opcode2str(int opcode)\n{\n\tswitch (opcode)\n\t{\n\tcase DNS_OPCODE_QUERY:\n\t\treturn \"QUERY\";\n\tcase DNS_OPCODE_IQUERY:\n\t\treturn \"IQUERY\";\n\tcase DNS_OPCODE_STATUS:\n\t\treturn \"STATUS\";\n\tdefault:\n\t\treturn \"Unknown\";\n\t}\n}\n\nconst char *dns_rcode2str(int rcode)\n{\n\tswitch (rcode)\n\t{\n\tcase DNS_RCODE_NO_ERROR:\n\t\treturn \"NO_ERROR\";\n\tcase DNS_RCODE_FORMAT_ERROR:\n\t\treturn \"FORMAT_ERROR\";\n\tcase DNS_RCODE_SERVER_FAILURE:\n\t\treturn \"SERVER_FAILURE\";\n\tcase DNS_RCODE_NAME_ERROR:\n\t\treturn \"NAME_ERROR\";\n\tcase DNS_RCODE_NOT_IMPLEMENTED:\n\t\treturn \"NOT_IMPLEMENTED\";\n\tcase DNS_RCODE_REFUSED:\n\t\treturn \"REFUSED\";\n\tdefault:\n\t\treturn \"Unknown\";\n\t}\n}\n\n" }, { "path": "src/protocol/dns_parser.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#ifndef _DNS_PARSER_H_\n#define _DNS_PARSER_H_\n\n#include \n#include \n#include \"list.h\"\n\n/**\n * dns_header_t is a struct to describe the header of a dns\n * request or response packet, but the byte order is not\n * transformed.\n */\n#pragma pack(1)\nstruct dns_header\n{\n\tuint16_t id;\n#if __BYTE_ORDER == __LITTLE_ENDIAN\n\tuint8_t rd : 1;\n\tuint8_t tc : 1;\n\tuint8_t aa : 1;\n\tuint8_t opcode : 4;\n\tuint8_t qr : 1;\n\tuint8_t rcode : 4;\n\tuint8_t z : 3;\n\tuint8_t ra : 1;\n#elif __BYTE_ORDER == __BIG_ENDIAN\n\tuint8_t qr : 1;\n\tuint8_t opcode : 4;\n\tuint8_t aa : 1;\n\tuint8_t tc : 1;\n\tuint8_t rd : 1;\n\tuint8_t ra : 1;\n\tuint8_t z : 3;\n\tuint8_t rcode : 4;\n#else\n# error \"unknown byte order\"\n#endif\n\tuint16_t qdcount;\n\tuint16_t ancount;\n\tuint16_t nscount;\n\tuint16_t arcount;\n};\n#pragma pack()\n\nstruct dns_question\n{\n\tchar *qname;\n\tuint16_t qtype;\n\tuint16_t qclass;\n};\n\nstruct dns_record_soa\n{\n\tchar *mname;\n\tchar *rname;\n\tuint32_t serial;\n\tint32_t refresh;\n\tint32_t retry;\n\tint32_t expire;\n\tuint32_t minimum;\n};\n\nstruct dns_record_srv\n{\n\tuint16_t priority;\n\tuint16_t weight;\n\tuint16_t port;\n\tchar *target;\n};\n\nstruct dns_record_mx\n{\n\tint16_t preference;\n\tchar *exchange;\n};\n\nstruct dns_record\n{\n\tchar *name;\n\tuint16_t type;\n\tuint16_t rclass;\n\tuint32_t ttl;\n\tuint16_t rdlength;\n\tvoid *rdata;\n};\n\ntypedef struct __dns_parser\n{\n\tvoid *msgbuf;\t\t\t\t// Message with leading length (TCP)\n\tvoid *msgbase;\t\t\t\t// Message without leading length\n\tconst char *cur;\t\t\t// Current parser position\n\tsize_t msgsize;\n\tsize_t bufsize;\n\tchar complete;\t\t\t\t// Whether parse completed\n\tchar single_packet;\t\t\t// Response without leading length When UDP\n\tstruct dns_header header;\n\tstruct dns_question question;\n\tstruct list_head answer_list;\n\tstruct list_head authority_list;\n\tstruct list_head additional_list;\n} dns_parser_t;\n\ntypedef struct __dns_record_cursor\n{\n\tconst struct list_head *head;\n\tconst struct list_head *next;\n} dns_record_cursor_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nvoid dns_parser_init(dns_parser_t *parser);\n\nvoid dns_parser_set_id(uint16_t id, dns_parser_t *parser);\nint dns_parser_set_question(const char *name,\n\t\t\t\t\t\t\tuint16_t qtype,\n\t\t\t\t\t\t\tuint16_t qclass,\n\t\t\t\t\t\t\tdns_parser_t *parser);\nint dns_parser_set_question_name(const char *name,\n\t\t\t\t\t\t\t\t dns_parser_t *parser);\n\nint dns_parser_parse_all(dns_parser_t *parser);\nint dns_parser_append_message(const void *buf, size_t *n,\n\t\t\t\t\t\t\t dns_parser_t *parser);\n\nvoid dns_parser_deinit(dns_parser_t *parser);\n\nint dns_record_cursor_next(struct dns_record **record,\n\t\t\t\t\t\t dns_record_cursor_t *cursor);\n\nint dns_record_cursor_find_cname(const char *name,\n\t\t\t\t\t\t\t\t const char **cname,\n\t\t\t\t\t\t\t\t dns_record_cursor_t *cursor);\n\nint dns_add_raw_record(const char *name, uint16_t type, uint16_t rclass,\n\t\t\t\t\t uint32_t ttl, uint16_t rlen, const void *rdata,\n\t\t\t\t\t struct list_head *list);\n\nint dns_add_str_record(const char *name, uint16_t type, uint16_t rclass,\n\t\t\t\t\t uint32_t ttl, const char *rdata,\n\t\t\t\t\t struct list_head *list);\n\nint dns_add_soa_record(const char *name, uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t const char *mname, const char *rname,\n\t\t\t\t\t uint32_t serial, int32_t refresh,\n\t\t\t\t\t int32_t retry, int32_t expire, uint32_t minimum,\n\t\t\t\t\t struct list_head *list);\n\nint dns_add_srv_record(const char *name, uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t uint16_t priority, uint16_t weight,\n\t\t\t\t\t uint16_t port, const char *target,\n\t\t\t\t\t struct list_head *list);\n\nint dns_add_mx_record(const char *name, uint16_t rclass, uint32_t ttl,\n\t\t\t\t\t int16_t preference, const char *exchange,\n\t\t\t\t\t struct list_head *list);\n\nconst char *dns_type2str(int type);\nconst char *dns_class2str(int dnsclass);\nconst char *dns_opcode2str(int opcode);\nconst char *dns_rcode2str(int rcode);\n\n#ifdef __cplusplus\n}\n#endif\n\nstatic inline void dns_answer_cursor_init(dns_record_cursor_t *cursor,\n\t\t\t\t\t\t\t\t\t\t const dns_parser_t *parser)\n{\n\tcursor->head = &parser->answer_list;\n\tcursor->next = cursor->head;\n}\n\nstatic inline void dns_authority_cursor_init(dns_record_cursor_t *cursor,\n\t\t\t\t\t\t\t\t\t\t\t const dns_parser_t *parser)\n{\n\tcursor->head = &parser->authority_list;\n\tcursor->next = cursor->head;\n}\n\nstatic inline void dns_additional_cursor_init(dns_record_cursor_t *cursor,\n\t\t\t\t\t\t\t\t\t\t\t const dns_parser_t *parser)\n{\n\tcursor->head = &parser->additional_list;\n\tcursor->next = cursor->head;\n}\n\nstatic inline void dns_record_cursor_deinit(dns_record_cursor_t *cursor)\n{\n}\n\n#endif\n\n" }, { "path": "src/protocol/dns_types.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#ifndef _DNS_TYPES_H_\n#define _DNS_TYPES_H_\n\nenum\n{\n\tDNS_TYPE_A = 1,\n\tDNS_TYPE_NS,\n\tDNS_TYPE_MD,\n\tDNS_TYPE_MF,\n\tDNS_TYPE_CNAME,\n\tDNS_TYPE_SOA = 6,\n\tDNS_TYPE_MB,\n\tDNS_TYPE_MG,\n\tDNS_TYPE_MR,\n\tDNS_TYPE_NULL,\n\tDNS_TYPE_WKS = 11,\n\tDNS_TYPE_PTR,\n\tDNS_TYPE_HINFO,\n\tDNS_TYPE_MINFO,\n\tDNS_TYPE_MX,\n\tDNS_TYPE_TXT = 16,\n\n\tDNS_TYPE_AAAA = 28,\n\tDNS_TYPE_SRV = 33,\n\n\tDNS_TYPE_AXFR = 252,\n\tDNS_TYPE_MAILB = 253,\n\tDNS_TYPE_MAILA = 254,\n\tDNS_TYPE_ALL = 255\n};\n\nenum\n{\n\tDNS_CLASS_IN = 1,\n\tDNS_CLASS_CS,\n\tDNS_CLASS_CH,\n\tDNS_CLASS_HS,\n\n\tDNS_CLASS_ALL = 255\n};\n\nenum\n{\n\tDNS_OPCODE_QUERY = 0,\n\tDNS_OPCODE_IQUERY,\n\tDNS_OPCODE_STATUS,\n};\n\nenum\n{\n\tDNS_RCODE_NO_ERROR = 0,\n\tDNS_RCODE_FORMAT_ERROR,\n\tDNS_RCODE_SERVER_FAILURE,\n\tDNS_RCODE_NAME_ERROR,\n\tDNS_RCODE_NOT_IMPLEMENTED,\n\tDNS_RCODE_REFUSED\n};\n\nenum\n{\n\tDNS_ANSWER_SECTION = 1,\n\tDNS_AUTHORITY_SECTION = 2,\n\tDNS_ADDITIONAL_SECTION = 3,\n};\n\n#endif\n\n" }, { "path": "src/protocol/http_parser.c", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"list.h\"\n#include \"http_parser.h\"\n\n#define MIN(x, y)\t((x) <= (y) ? (x) : (y))\n#define MAX(x, y)\t((x) >= (y) ? (x) : (y))\n\n#define HTTP_START_LINE_MAX\t\t8192\n#define HTTP_HEADER_VALUE_MAX\t8192\n#define HTTP_CHUNK_LINE_MAX\t\t1024\n#define HTTP_TRAILER_LINE_MAX\t8192\n#define HTTP_MSGBUF_INIT_SIZE\t2048\n\nenum\n{\n\tHPS_START_LINE,\n\tHPS_HEADER_NAME,\n\tHPS_HEADER_VALUE,\n\tHPS_HEADER_COMPLETE\n};\n\nenum\n{\n\tCPS_CHUNK_DATA,\n\tCPS_TRAILER_PART,\n\tCPS_CHUNK_COMPLETE\n};\n\nstruct __header_line\n{\n\tstruct list_head list;\n\tint name_len;\n\tint value_len;\n\tchar *buf;\n};\n\nstatic int __add_message_header(const char *name, size_t name_len,\n\t\t\t\t\t\t\t\tconst char *value, size_t value_len,\n\t\t\t\t\t\t\t\thttp_parser_t *parser)\n{\n\tsize_t size = sizeof (struct __header_line) + name_len + value_len + 4;\n\tstruct __header_line *line;\n\n\tline = (struct __header_line *)malloc(size);\n\tif (line)\n\t{\n\t\tline->buf = (char *)(line + 1);\n\t\tmemcpy(line->buf, name, name_len);\n\t\tline->buf[name_len] = ':';\n\t\tline->buf[name_len + 1] = ' ';\n\t\tmemcpy(line->buf + name_len + 2, value, value_len);\n\t\tline->buf[name_len + 2 + value_len] = '\\r';\n\t\tline->buf[name_len + 2 + value_len + 1] = '\\n';\n\t\tline->name_len = name_len;\n\t\tline->value_len = value_len;\n\t\tlist_add_tail(&line->list, &parser->header_list);\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nstatic int __set_message_header(const char *name, size_t name_len,\n\t\t\t\t\t\t\t\tconst char *value, size_t value_len,\n\t\t\t\t\t\t\t\thttp_parser_t *parser)\n{\n\tstruct __header_line *line;\n\tstruct list_head *pos;\n\tchar *buf;\n\n\tlist_for_each(pos, &parser->header_list)\n\t{\n\t\tline = list_entry(pos, struct __header_line, list);\n\t\tif (line->name_len == name_len &&\n\t\t\tstrncasecmp(line->buf, name, name_len) == 0)\n\t\t{\n\t\t\tif (value_len > line->value_len)\n\t\t\t{\n\t\t\t\tbuf = (char *)malloc(name_len + value_len + 4);\n\t\t\t\tif (!buf)\n\t\t\t\t\treturn -1;\n\n\t\t\t\tif (line->buf != (char *)(line + 1))\n\t\t\t\t\tfree(line->buf);\n\n\t\t\t\tline->buf = buf;\n\t\t\t\tmemcpy(buf, name, name_len);\n\t\t\t\tbuf[name_len] = ':';\n\t\t\t\tbuf[name_len + 1] = ' ';\n\t\t\t}\n\n\t\t\tmemcpy(line->buf + name_len + 2, value, value_len);\n\t\t\tline->buf[name_len + 2 + value_len] = '\\r';\n\t\t\tline->buf[name_len + 2 + value_len + 1] = '\\n';\n\t\t\tline->value_len = value_len;\n\t\t\treturn 0;\n\t\t}\n\t}\n\n\treturn __add_message_header(name, name_len, value, value_len, parser);\n}\n\nstatic int __match_request_line(const char *method, size_t method_len,\n\t\t\t\t\t\t\t\tconst char *uri, size_t uri_len,\n\t\t\t\t\t\t\t\tconst char *version, size_t version_len,\n\t\t\t\t\t\t\t\thttp_parser_t *parser)\n{\n\tmethod = strndup(method, method_len);\n\tif (method)\n\t{\n\t\turi = strndup(uri, uri_len);\n\t\tif (uri)\n\t\t{\n\t\t\tversion = strndup(version, version_len);\n\t\t\tif (version)\n\t\t\t{\n\t\t\t\tif (strcmp(version, \"HTTP/1.0\") == 0 ||\n\t\t\t\t\tstrncmp(version, \"HTTP/0\", 6) == 0)\n\t\t\t\t{\n\t\t\t\t\tparser->keep_alive = 0;\n\t\t\t\t}\n\n\t\t\t\tfree(parser->method);\n\t\t\t\tfree(parser->uri);\n\t\t\t\tfree(parser->version);\n\t\t\t\tparser->method = (char *)method;\n\t\t\t\tparser->uri = (char *)uri;\n\t\t\t\tparser->version = (char *)version;\n\t\t\t\treturn 0;\n\t\t\t}\n\n\t\t\tfree((char *)uri);\n\t\t}\n\n\t\tfree((char *)method);\n\t}\n\n\treturn -1;\n}\n\nstatic int __match_status_line(const char *version, size_t version_len,\n\t\t\t\t\t\t\t const char *code, size_t code_len,\n\t\t\t\t\t\t\t const char *phrase, size_t phrase_len,\n\t\t\t\t\t\t\t http_parser_t *parser)\n{\n\tversion = strndup(version, version_len);\n\tif (version)\n\t{\n\t\tcode = strndup(code, code_len);\n\t\tif (code)\n\t\t{\n\t\t\tphrase = strndup(phrase, phrase_len);\n\t\t\tif (phrase)\n\t\t\t{\n\t\t\t\tif (strcmp(version, \"HTTP/1.0\") == 0 ||\n\t\t\t\t\tstrncmp(version, \"HTTP/0\", 6) == 0)\n\t\t\t\t{\n\t\t\t\t\tparser->keep_alive = 0;\n\t\t\t\t}\n\n\t\t\t\tif (*code == '1' || strcmp(code, \"204\") == 0 ||\n\t\t\t\t\tstrcmp(code, \"304\") == 0)\n\t\t\t\t{\n\t\t\t\t\tparser->transfer_length = 0;\n\t\t\t\t}\n\n\t\t\t\tfree(parser->version);\n\t\t\t\tfree(parser->code);\n\t\t\t\tfree(parser->phrase);\n\t\t\t\tparser->version = (char *)version;\n\t\t\t\tparser->code = (char *)code;\n\t\t\t\tparser->phrase = (char *)phrase;\n\t\t\t\treturn 0;\n\t\t\t}\n\n\t\t\tfree((char *)code);\n\t\t}\n\n\t\tfree((char *)version);\n\t}\n\n\treturn -1;\n}\n\nstatic void __check_message_header(const char *name, size_t name_len,\n\t\t\t\t\t\t\t\t const char *value, size_t value_len,\n\t\t\t\t\t\t\t\t http_parser_t *parser)\n{\n\tswitch (name_len)\n\t{\n\tcase 6:\n\t\tif (strncasecmp(name, \"Expect\", 6) == 0)\n\t\t{\n\t\t\tif (value_len == 12 && strncasecmp(value, \"100-continue\", 12) == 0)\n\t\t\t\tparser->expect_continue = 1;\n\t\t}\n\n\t\tbreak;\n\n\tcase 10:\n\t\tif (strncasecmp(name, \"Connection\", 10) == 0)\n\t\t{\n\t\t\tparser->has_connection = 1;\n\t\t\tif (value_len == 10 && strncasecmp(value, \"Keep-Alive\", 10) == 0)\n\t\t\t\tparser->keep_alive = 1;\n\t\t\telse if (value_len == 5 && strncasecmp(value, \"close\", 5) == 0)\n\t\t\t\tparser->keep_alive = 0;\n\t\t}\n\t\telse if (strncasecmp(name, \"Keep-Alive\", 10) == 0)\n\t\t\tparser->has_keep_alive = 1;\n\n\t\tbreak;\n\n\tcase 14:\n\t\tif (strncasecmp(name, \"Content-Length\", 14) == 0)\n\t\t{\n\t\t\tparser->has_content_length = 1;\n\t\t\tif (*value >= '0' && *value <= '9' && value_len <= 15)\n\t\t\t{\n\t\t\t\tchar buf[16];\n\t\t\t\tmemcpy(buf, value, value_len);\n\t\t\t\tbuf[value_len] = '\\0';\n\t\t\t\tparser->content_length = atol(buf);\n\t\t\t}\n\t\t}\n\n\t\tbreak;\n\n\tcase 17:\n\t\tif (strncasecmp(name, \"Transfer-Encoding\", 17) == 0)\n\t\t{\n\t\t\tif (value_len != 8 || strncasecmp(value, \"identity\", 8) != 0)\n\t\t\t\tparser->chunked = 1;\n\t\t\telse\n\t\t\t\tparser->chunked = 0;\n\t\t}\n\n\t\tbreak;\n\t}\n}\n\nstatic int __parse_start_line(const char *ptr, size_t len,\n\t\t\t\t\t\t\t http_parser_t *parser)\n{\n\tsize_t min = MIN(HTTP_START_LINE_MAX, len);\n\tconst char *p1, *p2, *p3;\n\tsize_t l1, l2, l3;\n\tsize_t i;\n\tint ret;\n\n\tif (len >= 2 && ptr[0] == '\\r' && ptr[1] == '\\n')\n\t{\n\t\tparser->header_offset += 2;\n\t\treturn 1;\n\t}\n\n\tfor (i = 0; i < min; i++)\n\t{\n\t\tif (ptr[i] == '\\r')\n\t\t{\n\t\t\tif (i == len - 1)\n\t\t\t\treturn 0;\n\n\t\t\tif (ptr[i + 1] != '\\n')\n\t\t\t\treturn -2;\n\n\t\t\tlen = i;\n\t\t\twhile (len > 0 && ptr[len - 1] == ' ')\n\t\t\t\tlen--;\n\n\t\t\tp1 = ptr;\n\t\t\twhile (*p1 == ' ')\n\t\t\t\tp1++;\n\n\t\t\tp2 = p1;\n\t\t\twhile (p2 < ptr + len && *p2 != ' ')\n\t\t\t\tp2++;\n\n\t\t\tif (p2 == ptr + len)\n\t\t\t\treturn -2;\n\n\t\t\tl1 = p2++ - p1;\n\t\t\twhile (*p2 == ' ')\n\t\t\t\tp2++;\n\n\t\t\tp3 = p2;\n\t\t\twhile (p3 < ptr + len && *p3 != ' ')\n\t\t\t\tp3++;\n\n\t\t\tif (p3 == ptr + len)\n\t\t\t\treturn -2;\n\n\t\t\tl2 = p3++ - p2;\n\t\t\twhile (*p3 == ' ')\n\t\t\t\tp3++;\n\n\t\t\tl3 = ptr + len - p3;\n\t\t\tif (parser->is_resp)\n\t\t\t\tret = __match_status_line(p1, l1, p2, l2, p3, l3, parser);\n\t\t\telse\n\t\t\t\tret = __match_request_line(p1, l1, p2, l2, p3, l3, parser);\n\n\t\t\tif (ret < 0)\n\t\t\t\treturn ret;\n\n\t\t\tparser->header_offset += i + 2;\n\t\t\tparser->header_state = HPS_HEADER_NAME;\n\t\t\treturn 1;\n\t\t}\n\n\t\tif (ptr[i] == '\\0')\n\t\t\treturn -2;\n\t}\n\n\tif (i == HTTP_START_LINE_MAX)\n\t\treturn -2;\n\n\treturn 0;\n}\n\nstatic int __parse_header_name(const char *ptr, size_t len,\n\t\t\t\t\t\t\t http_parser_t *parser)\n{\n\tsize_t min = MIN(HTTP_HEADER_NAME_MAX, len);\n\tsize_t i;\n\n\tif (len >= 2 && ptr[0] == '\\r' && ptr[1] == '\\n')\n\t{\n\t\tparser->header_offset += 2;\n\t\tparser->header_state = HPS_HEADER_COMPLETE;\n\t\treturn 1;\n\t}\n\n\tfor (i = 0; i < min; i++)\n\t{\n\t\tif (ptr[i] == ':')\n\t\t{\n\t\t\tif (i == 0)\n\t\t\t\treturn -2;\n\n\t\t\tparser->namebuf[i] = '\\0';\n\t\t\tparser->header_offset += i + 1;\n\t\t\tparser->header_state = HPS_HEADER_VALUE;\n\t\t\treturn 1;\n\t\t}\n\n\t\tif ((signed char)ptr[i] <= ' ')\n\t\t\treturn -2;\n\n\t\tparser->namebuf[i] = ptr[i];\n\t}\n\n\tif (i == HTTP_HEADER_NAME_MAX)\n\t\treturn -2;\n\n\treturn 0;\n}\n\nstatic int __parse_header_value(const char *ptr, size_t len,\n\t\t\t\t\t\t\t\thttp_parser_t *parser)\n{\n\tchar header_value[HTTP_HEADER_VALUE_MAX];\n\tconst char *end = ptr + len;\n\tconst char *begin = ptr;\n\tsize_t i = 0;\n\tint ret;\n\n\twhile (1)\n\t{\n\t\twhile (1)\n\t\t{\n\t\t\tif (ptr == end)\n\t\t\t\treturn 0;\n\n\t\t\tif (*ptr == ' ' || *ptr == '\\t')\n\t\t\t\tptr++;\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\twhile (1)\n\t\t{\n\t\t\tif (i == HTTP_HEADER_VALUE_MAX)\n\t\t\t\treturn -2;\n\n\t\t\theader_value[i] = *ptr++;\n\t\t\tif (ptr == end)\n\t\t\t\treturn 0;\n\n\t\t\tif (header_value[i] == '\\r')\n\t\t\t\tbreak;\n\n\t\t\tif ((signed char)header_value[i] < ' ' && header_value[i] != '\\t')\n\t\t\t\treturn -2;\n\n\t\t\ti++;\n\t\t}\n\n\t\tif (*ptr == '\\n')\n\t\t\tptr++;\n\t\telse\n\t\t\treturn -2;\n\n\t\tif (ptr == end)\n\t\t\treturn 0;\n\n\t\twhile (i > 0)\n\t\t{\n\t\t\tif (header_value[i - 1] == ' ' || header_value[i - 1] == '\\t')\n\t\t\t\ti--;\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\tif (*ptr != ' ' && *ptr != '\\t')\n\t\t\tbreak;\n\n\t\tptr++;\n\t\theader_value[i++] = ' ';\n\t}\n\n\theader_value[i] = '\\0';\n\tret = http_parser_add_header(parser->namebuf, strlen(parser->namebuf),\n\t\t\t\t\t\t\t\t header_value, i, parser);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tparser->header_offset += ptr - begin;\n\tparser->header_state = HPS_HEADER_NAME;\n\treturn 1;\n}\n\nstatic int __parse_message_header(const void *message, size_t size,\n\t\t\t\t\t\t\t\t http_parser_t *parser)\n{\n\tconst char *ptr;\n\tsize_t len;\n\tint ret;\n\n\tdo\n\t{\n\t\tptr = (const char *)message + parser->header_offset;\n\t\tlen = size - parser->header_offset;\n\t\tif (parser->header_state == HPS_START_LINE)\n\t\t\tret = __parse_start_line(ptr, len, parser);\n\t\telse if (parser->header_state == HPS_HEADER_VALUE)\n\t\t\tret = __parse_header_value(ptr, len, parser);\n\t\telse /* if (parser->header_state == HPS_HEADER_NAME) */\n\t\t{\n\t\t\tret = __parse_header_name(ptr, len, parser);\n\t\t\tif (parser->header_state == HPS_HEADER_COMPLETE)\n\t\t\t\treturn 1;\n\t\t}\n\t} while (ret > 0);\n\n\treturn ret;\n}\n\n#define CHUNK_SIZE_MAX\t\t(2 * 1024 * 1024 * 1024U - HTTP_CHUNK_LINE_MAX - 4)\n\nstatic int __parse_chunk_data(const char *ptr, size_t len,\n\t\t\t\t\t\t\t http_parser_t *parser)\n{\n\tsize_t min = MIN(HTTP_CHUNK_LINE_MAX, len);\n\tsize_t chunk_size;\n\tchar *end;\n\tsize_t i;\n\n\tfor (i = 0; i < min; i++)\n\t{\n\t\tif (ptr[i] == '\\r')\n\t\t{\n\t\t\tif (i == len - 1)\n\t\t\t\treturn 0;\n\n\t\t\tif (ptr[i + 1] != '\\n')\n\t\t\t\treturn -2;\n\n\t\t\tchunk_size = strtoul(ptr, &end, 16);\n\t\t\tif (end == ptr)\n\t\t\t\treturn -2;\n\n\t\t\tif (chunk_size == 0)\n\t\t\t{\n\t\t\t\tchunk_size = i + 2;\n\t\t\t\tparser->chunk_state = CPS_TRAILER_PART;\n\t\t\t}\n\t\t\telse if (chunk_size < CHUNK_SIZE_MAX)\n\t\t\t{\n\t\t\t\tchunk_size += i + 4;\n\t\t\t\tif (len < chunk_size)\n\t\t\t\t\treturn 0;\n\t\t\t}\n\t\t\telse\n\t\t\t\treturn -2;\n\n\t\t\tparser->chunk_offset += chunk_size;\n\t\t\treturn 1;\n\t\t}\n\t}\n\n\tif (i == HTTP_CHUNK_LINE_MAX)\n\t\treturn -2;\n\n\treturn 0;\n}\n\nstatic int __parse_trailer_part(const char *ptr, size_t len,\n\t\t\t\t\t\t\t\thttp_parser_t *parser)\n{\n\tsize_t min = MIN(HTTP_TRAILER_LINE_MAX, len);\n\tsize_t i;\n\n\tfor (i = 0; i < min; i++)\n\t{\n\t\tif (ptr[i] == '\\r')\n\t\t{\n\t\t\tif (i == len - 1)\n\t\t\t\treturn 0;\n\n\t\t\tif (ptr[i + 1] != '\\n')\n\t\t\t\treturn -2;\n\n\t\t\tparser->chunk_offset += i + 2;\n\t\t\tif (i == 0)\n\t\t\t\tparser->chunk_state = CPS_CHUNK_COMPLETE;\n\n\t\t\treturn 1;\n\t\t}\n\t}\n\n\tif (i == HTTP_TRAILER_LINE_MAX)\n\t\treturn -2;\n\n\treturn 0;\n}\n\nstatic int __parse_chunk(const void *message, size_t size,\n\t\t\t\t\t\t http_parser_t *parser)\n{\n\tconst char *ptr;\n\tsize_t len;\n\tint ret;\n\n\tdo\n\t{\n\t\tptr = (const char *)message + parser->chunk_offset;\n\t\tlen = size - parser->chunk_offset;\n\t\tif (parser->chunk_state == CPS_CHUNK_DATA)\n\t\t\tret = __parse_chunk_data(ptr, len, parser);\n\t\telse /* if (parser->chunk_state == CPS_TRAILER_PART) */\n\t\t{\n\t\t\tret = __parse_trailer_part(ptr, len, parser);\n\t\t\tif (parser->chunk_state == CPS_CHUNK_COMPLETE)\n\t\t\t\treturn 1;\n\t\t}\n\t} while (ret > 0);\n\n\treturn ret;\n}\n\nvoid http_parser_init(int is_resp, http_parser_t *parser)\n{\n\tparser->header_state = HPS_START_LINE;\n\tparser->header_offset = 0;\n\tparser->transfer_length = (size_t)-1;\n\tparser->content_length = is_resp ? (size_t)-1 : 0;\n\tparser->version = NULL;\n\tparser->method = NULL;\n\tparser->uri = NULL;\n\tparser->code = NULL;\n\tparser->phrase = NULL;\n\tINIT_LIST_HEAD(&parser->header_list);\n\tparser->msgbuf = NULL;\n\tparser->msgsize = 0;\n\tparser->bufsize = 0;\n\tparser->has_connection = 0;\n\tparser->has_content_length = 0;\n\tparser->has_keep_alive = 0;\n\tparser->expect_continue = 0;\n\tparser->keep_alive = 1;\n\tparser->chunked = 0;\n\tparser->complete = 0;\n\tparser->is_resp = is_resp;\n}\n\nint http_parser_append_message(const void *buf, size_t *n,\n\t\t\t\t\t\t\t http_parser_t *parser)\n{\n\tint ret;\n\n\tif (parser->complete)\n\t{\n\t\t*n = 0;\n\t\treturn 1;\n\t}\n\n\tif (parser->msgsize + *n + 1 > parser->bufsize)\n\t{\n\t\tsize_t new_size = MAX(HTTP_MSGBUF_INIT_SIZE, 2 * parser->bufsize);\n\t\tvoid *new_base;\n\n\t\twhile (new_size < parser->msgsize + *n + 1)\n\t\t\tnew_size *= 2;\n\n\t\tnew_base = realloc(parser->msgbuf, new_size);\n\t\tif (!new_base)\n\t\t\treturn -1;\n\n\t\tparser->msgbuf = new_base;\n\t\tparser->bufsize = new_size;\n\t}\n\n\tmemcpy((char *)parser->msgbuf + parser->msgsize, buf, *n);\n\tparser->msgsize += *n;\n\tif (parser->header_state != HPS_HEADER_COMPLETE)\n\t{\n\t\tret = __parse_message_header(parser->msgbuf, parser->msgsize, parser);\n\t\tif (ret <= 0)\n\t\t\treturn ret;\n\n\t\tif (parser->chunked)\n\t\t{\n\t\t\tparser->chunk_offset = parser->header_offset;\n\t\t\tparser->chunk_state = CPS_CHUNK_DATA;\n\t\t}\n\t\telse if (parser->transfer_length == (size_t)-1)\n\t\t\tparser->transfer_length = parser->content_length;\n\t}\n\n\tif (parser->transfer_length != (size_t)-1)\n\t{\n\t\tsize_t total = parser->header_offset + parser->transfer_length;\n\n\t\tif (parser->msgsize >= total)\n\t\t{\n\t\t\t*n -= parser->msgsize - total;\n\t\t\tparser->msgsize = total;\n\t\t\tparser->complete = 1;\n\t\t\treturn 1;\n\t\t}\n\n\t\treturn 0;\n\t}\n\n\tif (!parser->chunked)\n\t\treturn 0;\n\n\tif (parser->chunk_state != CPS_CHUNK_COMPLETE)\n\t{\n\t\tret = __parse_chunk(parser->msgbuf, parser->msgsize, parser);\n\t\tif (ret <= 0)\n\t\t\treturn ret;\n\t}\n\n\t*n -= parser->msgsize - parser->chunk_offset;\n\tparser->msgsize = parser->chunk_offset;\n\tparser->complete = 1;\n\treturn 1;\n}\n\nint http_parser_header_complete(const http_parser_t *parser)\n{\n\treturn parser->header_state == HPS_HEADER_COMPLETE;\n}\n\nint http_parser_get_body(const void **body, size_t *size,\n\t\t\t\t\t\t const http_parser_t *parser)\n{\n\tif (parser->complete && parser->header_state == HPS_HEADER_COMPLETE)\n\t{\n\t\t*body = (char *)parser->msgbuf + parser->header_offset;\n\t\t*size = parser->msgsize - parser->header_offset;\n\t\t((char *)parser->msgbuf)[parser->msgsize] = '\\0';\n\t\treturn 0;\n\t}\n\n\treturn 1;\n}\n\nint http_parser_set_method(const char *method, http_parser_t *parser)\n{\n\tmethod = strdup(method);\n\tif (method)\n\t{\n\t\tfree(parser->method);\n\t\tparser->method = (char *)method;\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nint http_parser_set_uri(const char *uri, http_parser_t *parser)\n{\n\turi = strdup(uri);\n\tif (uri)\n\t{\n\t\tfree(parser->uri);\n\t\tparser->uri = (char *)uri;\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nint http_parser_set_version(const char *version, http_parser_t *parser)\n{\n\tversion = strdup(version);\n\tif (version)\n\t{\n\t\tfree(parser->version);\n\t\tparser->version = (char *)version;\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nint http_parser_set_code(const char *code, http_parser_t *parser)\n{\n\tcode = strdup(code);\n\tif (code)\n\t{\n\t\tfree(parser->code);\n\t\tparser->code = (char *)code;\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nint http_parser_set_phrase(const char *phrase, http_parser_t *parser)\n{\n\tphrase = strdup(phrase);\n\tif (phrase)\n\t{\n\t\tfree(parser->phrase);\n\t\tparser->phrase = (char *)phrase;\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nint http_parser_add_header(const void *name, size_t name_len,\n\t\t\t\t\t\t const void *value, size_t value_len,\n\t\t\t\t\t\t http_parser_t *parser)\n{\n\tif (__add_message_header((const char *)name, name_len,\n\t\t\t\t\t\t\t (const char *)value, value_len,\n\t\t\t\t\t\t\t parser) >= 0)\n\t{\n\t\t__check_message_header((const char *)name, name_len,\n\t\t\t\t\t\t\t (const char *)value, value_len,\n\t\t\t\t\t\t\t parser);\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nint http_parser_set_header(const void *name, size_t name_len,\n\t\t\t\t\t\t const void *value, size_t value_len,\n\t\t\t\t\t\t http_parser_t *parser)\n{\n\tif (__set_message_header((const char *)name, name_len,\n\t\t\t\t\t\t\t (const char *)value, value_len,\n\t\t\t\t\t\t\t parser) >= 0)\n\t{\n\t\t__check_message_header((const char *)name, name_len,\n\t\t\t\t\t\t\t (const char *)value, value_len,\n\t\t\t\t\t\t\t parser);\n\t\treturn 0;\n\t}\n\n\treturn -1;\n}\n\nvoid http_parser_deinit(http_parser_t *parser)\n{\n\tstruct __header_line *line;\n\tstruct list_head *pos, *tmp;\n\n\tlist_for_each_safe(pos, tmp, &parser->header_list)\n\t{\n\t\tline = list_entry(pos, struct __header_line, list);\n\t\tlist_del(pos);\n\t\tif (line->buf != (char *)(line + 1))\n\t\t\tfree(line->buf);\n\n\t\tfree(line);\n\t}\n\n\tfree(parser->version);\n\tfree(parser->method);\n\tfree(parser->uri);\n\tfree(parser->code);\n\tfree(parser->phrase);\n\tfree(parser->msgbuf);\n}\n\nint http_header_cursor_next(const void **name, size_t *name_len,\n\t\t\t\t\t\t\tconst void **value, size_t *value_len,\n\t\t\t\t\t\t\thttp_header_cursor_t *cursor)\n{\n\tstruct __header_line *line;\n\n\tif (cursor->next->next != cursor->head)\n\t{\n\t\tcursor->next = cursor->next->next;\n\t\tline = list_entry(cursor->next, struct __header_line, list);\n\t\t*name = line->buf;\n\t\t*name_len = line->name_len;\n\t\t*value = line->buf + line->name_len + 2;\n\t\t*value_len = line->value_len;\n\t\treturn 0;\n\t}\n\n\treturn 1;\n}\n\nint http_header_cursor_find(const void *name, size_t name_len,\n\t\t\t\t\t\t\tconst void **value, size_t *value_len,\n\t\t\t\t\t\t\thttp_header_cursor_t *cursor)\n{\n\tstruct __header_line *line;\n\n\twhile (cursor->next->next != cursor->head)\n\t{\n\t\tcursor->next = cursor->next->next;\n\t\tline = list_entry(cursor->next, struct __header_line, list);\n\t\tif (line->name_len == name_len)\n\t\t{\n\t\t\tif (strncasecmp(line->buf, (const char *)name, name_len) == 0)\n\t\t\t{\n\t\t\t\t*value = line->buf + name_len + 2;\n\t\t\t\t*value_len = line->value_len;\n\t\t\t\treturn 0;\n\t\t\t}\n\t\t}\n\t}\n\n\treturn 1;\n}\n\nint http_header_cursor_erase(http_header_cursor_t *cursor)\n{\n\tstruct __header_line *line;\n\n\tif (cursor->next != cursor->head)\n\t{\n\t\tline = list_entry(cursor->next, struct __header_line, list);\n\t\tcursor->next = cursor->next->prev;\n\t\tlist_del(&line->list);\n\t\tif (line->buf != (char *)(line + 1))\n\t\t\tfree(line->buf);\n\n\t\tfree(line);\n\t\treturn 0;\n\t}\n\n\treturn 1;\n}\n\n" }, { "path": "src/protocol/http_parser.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _HTTP_PARSER_H_\n#define _HTTP_PARSER_H_\n\n#include \n#include \"list.h\"\n\n#define HTTP_HEADER_NAME_MAX\t64\n\ntypedef struct __http_parser\n{\n\tint header_state;\n\tint chunk_state;\n\tsize_t header_offset;\n\tsize_t chunk_offset;\n\tsize_t content_length;\n\tsize_t transfer_length;\n\tchar *version;\n\tchar *method;\n\tchar *uri;\n\tchar *code;\n\tchar *phrase;\n\tstruct list_head header_list;\n\tchar namebuf[HTTP_HEADER_NAME_MAX];\n\tvoid *msgbuf;\n\tsize_t msgsize;\n\tsize_t bufsize;\n\tchar has_connection;\n\tchar has_content_length;\n\tchar has_keep_alive;\n\tchar expect_continue;\n\tchar keep_alive;\n\tchar chunked;\n\tchar complete;\n\tchar is_resp;\n} http_parser_t;\n\ntypedef struct __http_header_cursor\n{\n\tconst struct list_head *head;\n\tconst struct list_head *next;\n} http_header_cursor_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nvoid http_parser_init(int is_resp, http_parser_t *parser);\nint http_parser_append_message(const void *buf, size_t *n,\n\t\t\t\t\t\t\t http_parser_t *parser);\nint http_parser_get_body(const void **body, size_t *size,\n\t\t\t\t\t\t const http_parser_t *parser);\nint http_parser_header_complete(const http_parser_t *parser);\nint http_parser_set_method(const char *method, http_parser_t *parser);\nint http_parser_set_uri(const char *uri, http_parser_t *parser);\nint http_parser_set_version(const char *version, http_parser_t *parser);\nint http_parser_set_code(const char *code, http_parser_t *parser);\nint http_parser_set_phrase(const char *phrase, http_parser_t *parser);\nint http_parser_add_header(const void *name, size_t name_len,\n\t\t\t\t\t\t const void *value, size_t value_len,\n\t\t\t\t\t\t http_parser_t *parser);\nint http_parser_set_header(const void *name, size_t name_len,\n\t\t\t\t\t\t const void *value, size_t value_len,\n\t\t\t\t\t\t http_parser_t *parser);\nvoid http_parser_deinit(http_parser_t *parser);\n\nint http_header_cursor_next(const void **name, size_t *name_len,\n\t\t\t\t\t\t\tconst void **value, size_t *value_len,\n\t\t\t\t\t\t\thttp_header_cursor_t *cursor);\nint http_header_cursor_find(const void *name, size_t name_len,\n\t\t\t\t\t\t\tconst void **value, size_t *value_len,\n\t\t\t\t\t\t\thttp_header_cursor_t *cursor);\nint http_header_cursor_erase(http_header_cursor_t *cursor);\n\n#ifdef __cplusplus\n}\n#endif\n\nstatic inline const char *http_parser_get_method(const http_parser_t *parser)\n{\n\treturn parser->method;\n}\n\nstatic inline const char *http_parser_get_uri(const http_parser_t *parser)\n{\n\treturn parser->uri;\n}\n\nstatic inline const char *http_parser_get_version(const http_parser_t *parser)\n{\n\treturn parser->version;\n}\n\nstatic inline const char *http_parser_get_code(const http_parser_t *parser)\n{\n\treturn parser->code;\n}\n\nstatic inline const char *http_parser_get_phrase(const http_parser_t *parser)\n{\n\treturn parser->phrase;\n}\n\nstatic inline int http_parser_chunked(const http_parser_t *parser)\n{\n\treturn parser->chunked;\n}\n\nstatic inline int http_parser_keep_alive(const http_parser_t *parser)\n{\n\treturn parser->keep_alive;\n}\n\nstatic inline int http_parser_has_connection(const http_parser_t *parser)\n{\n\treturn parser->has_connection;\n}\n\nstatic inline int http_parser_has_content_length(const http_parser_t *parser)\n{\n\treturn parser->has_content_length;\n}\n\nstatic inline int http_parser_has_keep_alive(const http_parser_t *parser)\n{\n\treturn parser->has_keep_alive;\n}\n\nstatic inline void http_parser_close_message(http_parser_t *parser)\n{\n\tparser->complete = 1;\n}\n\nstatic inline void http_header_cursor_init(http_header_cursor_t *cursor,\n\t\t\t\t\t\t\t\t\t\t const http_parser_t *parser)\n{\n\tcursor->head = &parser->header_list;\n\tcursor->next = cursor->head;\n}\n\nstatic inline void http_header_cursor_rewind(http_header_cursor_t *cursor)\n{\n\tcursor->next = cursor->head;\n}\n\nstatic inline void http_header_cursor_deinit(http_header_cursor_t *cursor)\n{\n}\n\n#endif\n\n" }, { "path": "src/protocol/kafka_parser.c", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n\t http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"kafka_parser.h\"\n\nstatic kafka_api_version_t kafka_api_version_queryable[] = {\n\t{ Kafka_ApiVersions, 0, 0 }\n};\n\nstatic kafka_api_version_t kafka_api_version_0_9_0[] = {\n\t{ Kafka_Produce, 0, 1 },\n\t{ Kafka_Fetch, 0, 1 },\n\t{ Kafka_ListOffsets, 0, 0 },\n\t{ Kafka_Metadata, 0, 0 },\n\t{ Kafka_OffsetCommit, 0, 2 },\n\t{ Kafka_OffsetFetch, 0, 1 },\n\t{ Kafka_FindCoordinator, 0, 0 },\n\t{ Kafka_JoinGroup, 0, 0 },\n\t{ Kafka_Heartbeat, 0, 0 },\n\t{ Kafka_LeaveGroup, 0, 0 },\n\t{ Kafka_SyncGroup, 0, 0 },\n\t{ Kafka_DescribeGroups, 0, 0 },\n\t{ Kafka_ListGroups, 0, 0 }\n};\n\nstatic kafka_api_version_t kafka_api_version_0_8_2[] = {\n\t{ Kafka_Produce, 0, 0 },\n\t{ Kafka_Fetch, 0, 0 },\n\t{ Kafka_ListOffsets, 0, 0 },\n\t{ Kafka_Metadata, 0, 0 },\n\t{ Kafka_OffsetCommit, 0, 1 },\n\t{ Kafka_OffsetFetch, 0, 1 },\n\t{ Kafka_FindCoordinator, 0, 0 }\n};\n\nstatic kafka_api_version_t kafka_api_version_0_8_1[] = {\n\t{ Kafka_Produce, 0, 0 },\n\t{ Kafka_Fetch, 0, 0 },\n\t{ Kafka_ListOffsets, 0, 0 },\n\t{ Kafka_Metadata, 0, 0 },\n\t{ Kafka_OffsetCommit, 0, 1 },\n\t{ Kafka_OffsetFetch, 0, 0 }\n};\n\nstatic kafka_api_version_t kafka_api_version_0_8_0[] = {\n\t{ Kafka_Produce, 0, 0 },\n\t{ Kafka_Fetch, 0, 0 },\n\t{ Kafka_ListOffsets, 0, 0 },\n\t{ Kafka_Metadata, 0, 0 }\n};\n\nstatic const struct kafka_feature_map {\n\tunsigned feature;\n\tkafka_api_version_t depends[Kafka_ApiNums];\n} kafka_feature_map[] = {\n\t{\n\t\t.feature = KAFKA_FEATURE_MSGVER1,\n\t\t.depends = {\n\t\t\t{ Kafka_Produce, 2, 2 },\n\t\t\t{ Kafka_Fetch, 2, 2 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_MSGVER2,\n\t\t.depends = {\n\t\t\t{ Kafka_Produce, 3, 3 },\n\t\t\t{ Kafka_Fetch, 4, 4 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_APIVERSION,\n\t\t.depends = {\n\t\t\t{ Kafka_ApiVersions, 0, 0 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_BROKER_GROUP_COORD,\n\t\t.depends = {\n\t\t\t{ Kafka_FindCoordinator, 0, 0 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_BROKER_BALANCED_CONSUMER,\n\t\t.depends = {\n\t\t\t{ Kafka_FindCoordinator, 0, 0 },\n\t\t\t{ Kafka_OffsetCommit, 1, 2 },\n\t\t\t{ Kafka_OffsetFetch, 1, 1 },\n\t\t\t{ Kafka_JoinGroup, 0, 0 },\n\t\t\t{ Kafka_SyncGroup, 0, 0 },\n\t\t\t{ Kafka_Heartbeat, 0, 0 },\n\t\t\t{ Kafka_LeaveGroup, 0, 0 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_THROTTLETIME,\n\t\t.depends = {\n\t\t\t{ Kafka_Produce, 1, 2 },\n\t\t\t{ Kafka_Fetch, 1, 2 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_LZ4,\n\t\t.depends = {\n\t\t\t{ Kafka_FindCoordinator, 0, 0 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_OFFSET_TIME,\n\t\t.depends = {\n\t\t\t{ Kafka_ListOffsets, 1, 1 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t}\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_ZSTD,\n\t\t.depends = {\n\t\t\t{ Kafka_Produce, 7, 7 },\n\t\t\t{ Kafka_Fetch, 10, 10 },\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_SASL_GSSAPI,\n\t\t.depends = {\n\t\t\t{ Kafka_JoinGroup, 0, 0},\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_SASL_HANDSHAKE,\n\t\t.depends = {\n\t\t\t{ Kafka_SaslHandshake, 0, 0},\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = KAFKA_FEATURE_SASL_AUTH_REQ,\n\t\t.depends = {\n\t\t\t{ Kafka_SaslHandshake, 1, 1},\n\t\t\t{ Kafka_SaslAuthenticate, 0, 0},\n\t\t\t{ Kafka_Unknown, 0, 0 },\n\t\t},\n\t},\n\t{\n\t\t.feature = 0,\n\t},\n};\n\nstatic int kafka_get_legacy_api_version(const char *broker_version,\n\t\t\t\t\t\t\t\t\t\tkafka_api_version_t **api,\n\t\t\t\t\t\t\t\t\t\tsize_t *api_cnt)\n{\n\tstatic const struct {\n\t\tconst char *pfx;\n\t\tkafka_api_version_t *api;\n\t\tsize_t api_cnt;\n\t} vermap[] = {\n\t\t{ \"0.9.0\",\n\t\t kafka_api_version_0_9_0,\n\t\t sizeof(kafka_api_version_0_9_0) / sizeof(kafka_api_version_t)\n\t\t},\n\t\t{ \"0.8.2\",\n\t\t kafka_api_version_0_8_2,\n\t\t sizeof(kafka_api_version_0_8_2) / sizeof(kafka_api_version_t)\n\t\t},\n\t\t{ \"0.8.1\",\n\t\t kafka_api_version_0_8_1,\n\t\t sizeof(kafka_api_version_0_8_1) / sizeof(kafka_api_version_t)\n\t\t},\n\t\t{ \"0.8.0\",\n\t\t kafka_api_version_0_8_0,\n\t\t sizeof(kafka_api_version_0_8_0) / sizeof(kafka_api_version_t)\n\t\t},\n\t\t{ \"0.7.\", NULL, 0 },\n\t\t{ \"0.6\", NULL, 0 },\n\t\t{ \"\", kafka_api_version_queryable, 1 },\n\t\t{ NULL, NULL, 0 }\n\t};\n\tint i;\n\n\tfor (i = 0 ; vermap[i].pfx ; i++)\n\t{\n\t\tif (!strncmp(vermap[i].pfx, broker_version, strlen(vermap[i].pfx)))\n\t\t{\n\t\t\tif (!vermap[i].api)\n\t\t\t\treturn -1;\n\t\t\t*api = vermap[i].api;\n\t\t\t*api_cnt = vermap[i].api_cnt;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nint kafka_api_version_is_queryable(const char *broker_version,\n\t\t\t\t\t\t\t\t kafka_api_version_t **api,\n\t\t\t\t\t\t\t\t size_t *api_cnt)\n{\n\treturn kafka_get_legacy_api_version(broker_version, api, api_cnt);\n}\n\nstatic int kafka_api_version_key_cmp(const void *_a, const void *_b)\n{\n\tconst kafka_api_version_t *a = (const kafka_api_version_t *)_a;\n\tconst kafka_api_version_t *b = (const kafka_api_version_t *)_b;\n\n\tif (a->api_key > b->api_key)\n\t\treturn 1;\n\telse if (a->api_key == b->api_key)\n\t\treturn 0;\n\telse\n\t\treturn -1;\n}\n\nstatic int kafka_api_version_check(const kafka_api_version_t *apis,\n\t\t\t\t\t\t\t\t size_t api_cnt,\n\t\t\t\t\t\t\t\t const kafka_api_version_t *match)\n{\n\tconst kafka_api_version_t *api;\n\n\tapi = (const kafka_api_version_t *)bsearch(match, apis, api_cnt,\n\t\t\t\t\t\tsizeof(*apis), kafka_api_version_key_cmp);\n\tif (!api)\n\t\treturn 0;\n\n\treturn match->min_ver <= api->max_ver && api->min_ver <= match->max_ver;\n}\n\nunsigned kafka_get_features(kafka_api_version_t *api, size_t api_cnt)\n{\n\tunsigned features = 0;\n\tint i, fails, r;\n\tconst kafka_api_version_t *match;\n\n\tfor (i = 0; kafka_feature_map[i].feature != 0; i++)\n\t{\n\t\tfails = 0;\n\t\tfor (match = &kafka_feature_map[i].depends[0];\n\t\t\t\tmatch->api_key != -1 ; match++)\n\t\t{\n\t\t\tr = kafka_api_version_check(api, api_cnt, match);\n\t\t\tfails += !r;\n\t\t}\n\n\t\tif (!fails)\n\t\t\tfeatures |= kafka_feature_map[i].feature;\n\t}\n\n\treturn features;\n}\n\nint kafka_broker_get_api_version(const kafka_api_t *api, int api_key,\n\t\t\t\t\t\t\t\t int min_ver, int max_ver)\n{\n\tkafka_api_version_t sk = { .api_key = api_key };\n\tconst kafka_api_version_t *retp;\n\n\tretp = (const kafka_api_version_t *)bsearch(&sk, api->api, api->elements,\n\t\t\t\t\t\tsizeof(*api->api), kafka_api_version_key_cmp);\n\tif (!retp)\n\t\treturn -1;\n\n\tif (retp->max_ver < max_ver)\n\t{\n\t\tif (retp->max_ver < min_ver)\n\t\t\treturn -1;\n\t\telse\n\t\t\treturn retp->max_ver;\n\t}\n\telse if (retp->min_ver > min_ver)\n\t\treturn -1;\n\telse\n\t\treturn max_ver;\n}\n\nvoid kafka_parser_init(kafka_parser_t *parser)\n{\n\tparser->complete = 0;\n\tparser->message_size = 0;\n\tparser->msgbuf = NULL;\n\tparser->cur_size = 0;\n\tparser->hsize = 0;\n}\n\nvoid kafka_parser_deinit(kafka_parser_t *parser)\n{\n\tfree(parser->msgbuf);\n}\n\nvoid kafka_config_init(kafka_config_t *conf)\n{\n\tconf->produce_timeout = 100;\n\tconf->produce_msg_max_bytes = 1000000;\n\tconf->produce_msgset_cnt = 10000;\n\tconf->produce_msgset_max_bytes = 1000000;\n\tconf->fetch_timeout = 100;\n\tconf->fetch_min_bytes = 1;\n\tconf->fetch_max_bytes = 50 * 1024 * 1024;\n\tconf->fetch_msg_max_bytes = 10 * 1024 * 1024;\n\tconf->offset_timestamp = KAFKA_TIMESTAMP_LATEST;\n\tconf->commit_timestamp = 0;\n\tconf->session_timeout = 10*1000;\n\tconf->rebalance_timeout = 10000;\n\tconf->retention_time_period = 20000;\n\tconf->produce_acks = -1;\n\tconf->allow_auto_topic_creation = 1;\n\tconf->api_version_request = 0;\n\tconf->api_version_timeout = 10000;\n\tconf->broker_version = NULL;\n\tconf->compress_type = Kafka_NoCompress;\n\tconf->compress_level = 0;\n\tconf->client_id = NULL;\n\tconf->check_crcs = 0;\n\tconf->offset_store = KAFKA_OFFSET_AUTO;\n\tconf->rack_id = NULL;\n\tconf->mechanisms = NULL;\n\tconf->username = NULL;\n\tconf->password = NULL;\n\tconf->recv = NULL;\n\tconf->client_new = NULL;\n}\n\nvoid kafka_config_deinit(kafka_config_t *conf)\n{\n\tfree(conf->broker_version);\n\tfree(conf->client_id);\n\tfree(conf->rack_id);\n\tfree(conf->mechanisms);\n\tfree(conf->username);\n\tfree(conf->password);\n}\n\nvoid kafka_partition_init(kafka_partition_t *partition)\n{\n\tpartition->error = 0;\n\tpartition->partition_index = -1;\n\tkafka_broker_init(&partition->leader);\n\tpartition->replica_nodes = NULL;\n\tpartition->replica_node_elements = 0;\n\tpartition->isr_nodes = NULL;\n\tpartition->isr_node_elements = 0;\n}\n\nvoid kafka_partition_deinit(kafka_partition_t *partition)\n{\n\tkafka_broker_deinit(&partition->leader);\n\tfree(partition->replica_nodes);\n\tfree(partition->isr_nodes);\n}\n\nvoid kafka_api_init(kafka_api_t *api)\n{\n\tapi->features = 0;\n\tapi->api = NULL;\n\tapi->elements = 0;\n}\n\nvoid kafka_api_deinit(kafka_api_t *api)\n{\n\tfree(api->api);\n}\n\nvoid kafka_broker_init(kafka_broker_t *broker)\n{\n\tbroker->node_id = -1;\n\tbroker->port = 0;\n\tbroker->host = NULL;\n\tbroker->rack = NULL;\n\tbroker->error = 0;\n\tbroker->status = KAFKA_BROKER_UNINIT;\n}\n\nvoid kafka_broker_deinit(kafka_broker_t *broker)\n{\n\tfree(broker->host);\n\tfree(broker->rack);\n}\n\nvoid kafka_meta_init(kafka_meta_t *meta)\n{\n\tmeta->error = 0;\n\tmeta->topic_name = NULL;\n\tmeta->error_message = NULL;\n\tmeta->is_internal = 0;\n\tmeta->partitions = NULL;\n\tmeta->partition_elements = 0;\n}\n\nvoid kafka_meta_deinit(kafka_meta_t *meta)\n{\n\tint i;\n\n\tfree(meta->topic_name);\n\tfree(meta->error_message);\n\n\tfor (i = 0; i < meta->partition_elements; ++i)\n\t{\n\t\tkafka_partition_deinit(meta->partitions[i]);\n\t\tfree(meta->partitions[i]);\n\t}\n\tfree(meta->partitions);\n}\n\nvoid kafka_topic_partition_init(kafka_topic_partition_t *toppar)\n{\n\ttoppar->error = 0;\n\ttoppar->topic_name = NULL;\n\ttoppar->partition = -1;\n\ttoppar->preferred_read_replica = -1;\n\ttoppar->offset = KAFKA_OFFSET_UNINIT;\n\ttoppar->high_watermark = KAFKA_OFFSET_UNINIT;\n\ttoppar->low_watermark = KAFKA_OFFSET_UNINIT;\n\ttoppar->last_stable_offset = -1;\n\ttoppar->log_start_offset = -1;\n\ttoppar->offset_timestamp = KAFKA_TIMESTAMP_UNINIT;\n\ttoppar->committed_metadata = NULL;\n\tINIT_LIST_HEAD(&toppar->record_list);\n}\n\nvoid kafka_topic_partition_deinit(kafka_topic_partition_t *toppar)\n{\n\tfree(toppar->topic_name);\n\tfree(toppar->committed_metadata);\n}\n\nvoid kafka_record_header_init(kafka_record_header_t *header)\n{\n\theader->key = NULL;\n\theader->key_len = 0;\n\theader->key_is_moved = 0;\n\theader->value = NULL;\n\theader->value_len = 0;\n\theader->value_is_moved = 0;\n}\n\nvoid kafka_record_header_deinit(kafka_record_header_t *header)\n{\n\tif (!header->key_is_moved)\n\t\tfree(header->key);\n\n\tif (!header->value_is_moved)\n\t\tfree(header->value);\n}\n\nvoid kafka_record_init(kafka_record_t *record)\n{\n\trecord->key = NULL;\n\trecord->key_len = 0;\n\trecord->key_is_moved = 0;\n\trecord->value = NULL;\n\trecord->value_len = 0;\n\trecord->value_is_moved = 0;\n\trecord->timestamp = 0;\n\trecord->offset = 0;\n\tINIT_LIST_HEAD(&record->header_list);\n\trecord->status = 0;\n\trecord->toppar = NULL;\n}\n\nvoid kafka_record_deinit(kafka_record_t *record)\n{\n\tstruct list_head *tmp, *pos;\n\tkafka_record_header_t *header;\n\n\tif (!record->key_is_moved)\n\t\tfree(record->key);\n\n\tif (!record->value_is_moved)\n\t\tfree(record->value);\n\n\tlist_for_each_safe(pos, tmp, &record->header_list)\n\t{\n\t\theader = list_entry(pos, kafka_record_header_t, list);\n\t\tlist_del(pos);\n\t\tkafka_record_header_deinit(header);\n\t\tfree(header);\n\t}\n}\n\nvoid kafka_member_init(kafka_member_t *member)\n{\n\tmember->member_id = NULL;\n\tmember->client_id = NULL;\n\tmember->client_host = NULL;\n\tmember->member_metadata = NULL;\n\tmember->member_metadata_len = 0;\n}\n\nvoid kafka_member_deinit(kafka_member_t *member)\n{\n\tfree(member->member_id);\n\tfree(member->client_id);\n\tfree(member->client_host);\n\t//do not need free!\n\t//free(member->member_metadata);\n}\n\nvoid kafka_cgroup_init(kafka_cgroup_t *cgroup)\n{\n\tINIT_LIST_HEAD(&cgroup->assigned_toppar_list);\n\tcgroup->error = 0;\n\tcgroup->error_msg = NULL;\n\tkafka_broker_init(&cgroup->coordinator);\n\tcgroup->leader_id = NULL;\n\tcgroup->member_id = NULL;\n\tcgroup->members = NULL;\n\tcgroup->member_elements = 0;\n\tcgroup->generation_id = -1;\n\tcgroup->group_name = NULL;\n\tcgroup->protocol_type = (char *)\"consumer\";\n\tcgroup->protocol_name = NULL;\n\tINIT_LIST_HEAD(&cgroup->group_protocol_list);\n}\n\nvoid kafka_cgroup_deinit(kafka_cgroup_t *cgroup)\n{\n\tint i;\n\n\tfree(cgroup->error_msg);\n\tkafka_broker_deinit(&cgroup->coordinator);\n\tfree(cgroup->leader_id);\n\tfree(cgroup->member_id);\n\n\tfor (i = 0; i < cgroup->member_elements; ++i)\n\t{\n\t\tkafka_member_deinit(cgroup->members[i]);\n\t\tfree(cgroup->members[i]);\n\t}\n\n\tfree(cgroup->members);\n\tfree(cgroup->protocol_name);\n}\n\nvoid kafka_block_init(kafka_block_t *block)\n{\n\tblock->buf = NULL;\n\tblock->len = 0;\n\tblock->is_moved = 0;\n}\n\nvoid kafka_block_deinit(kafka_block_t *block)\n{\n\tif (!block->is_moved)\n\t\tfree(block->buf);\n}\n\nint kafka_parser_append_message(const void *buf, size_t *size,\n\t\t\t\t\t\t\t\tkafka_parser_t *parser)\n{\n\tsize_t s = *size;\n\tint totaln;\n\n\tif (parser->complete)\n\t{\n\t\t*size = 0;\n\t\treturn 1;\n\t}\n\n\tif (parser->hsize + s < 4)\n\t{\n\t\tmemcpy(parser->headbuf + parser->hsize, buf, s);\n\t\tparser->hsize += s;\n\t\treturn 0;\n\t}\n\telse if (!parser->msgbuf)\n\t{\n\t\tmemcpy(parser->headbuf + parser->hsize, buf, 4 - parser->hsize);\n\t\tbuf = (const char *)buf + 4 - parser->hsize;\n\t\ts -= 4 - parser->hsize;\n\t\tparser->hsize = 4;\n\t\tmemcpy(&totaln, parser->headbuf, 4);\n\t\tparser->message_size = ntohl(totaln);\n\t\tparser->msgbuf = malloc(parser->message_size);\n\t\tif (!parser->msgbuf)\n\t\t\treturn -1;\n\n\t\tparser->cur_size = 0;\n\t}\n\n\tif (s > parser->message_size - parser->cur_size)\n\t{\n\t\tmemcpy((char *)parser->msgbuf + parser->cur_size, buf,\n\t\t\t parser->message_size - parser->cur_size);\n\t\tparser->cur_size = parser->message_size;\n\t}\n\telse\n\t{\n\t\tmemcpy((char *)parser->msgbuf + parser->cur_size, buf, s);\n\t\tparser->cur_size += s;\n\t}\n\n\tif (parser->cur_size < parser->message_size)\n\t\treturn 0;\n\n\t*size -= parser->message_size - parser->cur_size;\n\treturn 1;\n}\n\nint kafka_topic_partition_set_tp(const char *topic_name, int partition,\n\t\t\t\t\t\t\t\t kafka_topic_partition_t *toppar)\n{\n\tchar *p = strdup(topic_name);\n\n\tif (!p)\n\t\treturn -1;\n\n\tfree(toppar->topic_name);\n\ttoppar->topic_name = p;\n\ttoppar->partition = partition;\n\treturn 0;\n}\n\nint kafka_record_set_key(const void *key, size_t key_len,\n\t\t\t\t\t\t kafka_record_t *record)\n{\n\tvoid *k = malloc(key_len);\n\n\tif (!k)\n\t\treturn -1;\n\n\tfree(record->key);\n\tmemcpy(k, key, key_len);\n\trecord->key = k;\n\trecord->key_len = key_len;\n\treturn 0;\n}\n\nint kafka_record_set_value(const void *val, size_t val_len,\n\t\t\t\t\t\t kafka_record_t *record)\n{\n\tvoid *v = malloc(val_len);\n\n\tif (!v)\n\t\treturn -1;\n\n\tfree(record->value);\n\tmemcpy(v, val, val_len);\n\trecord->value = v;\n\trecord->value_len = val_len;\n\treturn 0;\n}\n\nint kafka_record_header_set_kv(const void *key, size_t key_len,\n\t\t\t\t\t\t\t const void *val, size_t val_len,\n\t\t\t\t\t\t\t kafka_record_header_t *header)\n{\n\tvoid *k = malloc(key_len);\n\tvoid *v = malloc(val_len);\n\n\tif (!k || !v)\n\t{\n\t\tfree(k);\n\t\tfree(v);\n\t\treturn -1;\n\t}\n\n\tmemcpy(k, key, key_len);\n\tmemcpy(v, val, val_len);\n\theader->key = k;\n\theader->key_len = key_len;\n\theader->value = v;\n\theader->value_len = val_len;\n\treturn 0;\n}\n\nint kafka_meta_set_topic(const char *topic, kafka_meta_t *meta)\n{\n\tchar *t = strdup(topic);\n\n\tif (!t)\n\t\treturn -1;\n\n\tfree(meta->topic_name);\n\tmeta->topic_name = t;\n\treturn 0;\n}\n\nint kafka_cgroup_set_group(const char *group, kafka_cgroup_t *cgroup)\n{\n\tchar *t = strdup(group);\n\n\tif (!t)\n\t\treturn -1;\n\n\tfree(cgroup->group_name);\n\tcgroup->group_name = t;\n\treturn 0;\n}\n\nstatic int kafka_sasl_plain_recv(const char *buf, size_t len, void *conf, void *q)\n{\n\treturn 0;\n}\n\nstatic int kafka_sasl_plain_client_new(void *p, kafka_sasl_t *sasl)\n{\n\tkafka_config_t *conf = (kafka_config_t *)p;\n\tsize_t ulen = strlen(conf->username);\n\tsize_t plen = strlen(conf->password);\n\tsize_t blen = ulen + plen + 2;\n\tsize_t off = 0;\n\tchar *buf = (char *)malloc(blen);\n\n\tif (!buf)\n\t\treturn -1;\n\n\tbuf[off++] = '\\0';\n\n\tmemcpy(buf + off, conf->username, ulen);\n\toff += ulen;\n\tbuf[off++] = '\\0';\n\n\tmemcpy(buf + off, conf->password, plen);\n\n\tfree(sasl->buf);\n\tsasl->buf = buf;\n\tsasl->bsize = blen;\n\n\treturn 0;\n}\n\nstatic int scram_get_attr(const struct iovec *inbuf, char attr,\n\t\t\t\t\t\t struct iovec *outbuf)\n{\n\tconst char *td;\n\tsize_t len;\n\tsize_t of = 0;\n\tvoid *ptr;\n\tchar ochar, nchar;\n\n\tfor (of = 0; of < inbuf->iov_len;)\n\t{\n\t\tptr = (char *)inbuf->iov_base + of;\n\t\ttd = (char *)memchr(ptr, ',', inbuf->iov_len - of);\n\t\tif (td)\n\t\t\tlen = (size_t)((char *)td - (char *)inbuf->iov_base - of);\n\t\telse\n\t\t\tlen = inbuf->iov_len - of;\n\n\t\tochar = *((char *)inbuf->iov_base + of);\n\t\tnchar = *((char *)inbuf->iov_base + of + 1);\n\t\tif (ochar == attr && inbuf->iov_len > of + 1 && nchar == '=')\n\t\t{\n\t\t\toutbuf->iov_base = (char *)ptr + 2;\n\t\t\toutbuf->iov_len = len - 2;\n\t\t\treturn 0;\n\t\t}\n\n\t\tof += len + 1;\n\t}\n\n\treturn -1;\n}\n\nstatic char *scram_base64_encode(const struct iovec *in)\n{\n\tchar *ret;\n\tsize_t ret_len, max_len;\n\n\tif (in->iov_len > INT_MAX)\n\t\treturn NULL;\n\n\tmax_len = (((in->iov_len + 2) / 3) * 4) + 1;\n\tret = (char *)malloc(max_len);\n\tif (!ret)\n\t\treturn NULL;\n\n\tret_len = EVP_EncodeBlock((uint8_t *)ret, (uint8_t *)in->iov_base,\n\t\t\t\t\t\t\t (int)in->iov_len);\n\tif (ret_len >= max_len)\n\t{\n\t\tfree(ret);\n\t\treturn NULL;\n\t}\n\tret[ret_len] = 0;\n\n\treturn ret;\n}\n\nstatic int scram_base64_decode(const struct iovec *in, struct iovec *out)\n{\n\tsize_t ret_len;\n\n\tif (in->iov_len % 4 != 0 || in->iov_len > INT_MAX)\n\t\treturn -1;\n\n\tret_len = ((in->iov_len / 4) * 3);\n\tout->iov_base = malloc(ret_len + 1);\n\tif (!out->iov_base)\n\t\treturn -1;\n\n\tif (EVP_DecodeBlock((uint8_t*)out->iov_base, (uint8_t*)in->iov_base,\n\t\t\t\t\t\t(int)in->iov_len) == -1)\n\t{\n\t\tfree(out->iov_base);\n\t\tout->iov_base = NULL;\n\t\treturn -1;\n\t}\n\n\tif (in->iov_len > 1 && ((char *)(in->iov_base))[in->iov_len - 1] == '=')\n\t{\n\t\tif (in->iov_len > 2 && ((char *)(in->iov_base))[in->iov_len - 2] == '=')\n\t\t\tret_len -= 2;\n\t\telse\n\t\t\tret_len -= 1;\n\t}\n\n\t((char *)(out->iov_base))[ret_len] = '\\0';\n\tout->iov_len = ret_len;\n\n\treturn 0;\n}\n\nstatic int scram_hi(const EVP_MD *evp, int itcnt, const struct iovec *in,\n\t\t\t\t\tconst struct iovec *salt, struct iovec *out)\n{\n\tunsigned int ressize = 0;\n\tunsigned char tempres[EVP_MAX_MD_SIZE];\n\tunsigned char tempdest[EVP_MAX_MD_SIZE];\n\tunsigned char *saltplus;\n\tint i, j;\n\n\tsaltplus = (unsigned char *)alloca(salt->iov_len + 4);\n\tif (!saltplus)\n\t\treturn -1;\n\n\tmemcpy(saltplus, salt->iov_base, salt->iov_len);\n\tsaltplus[salt->iov_len]\t = '\\0';\n\tsaltplus[salt->iov_len + 1] = '\\0';\n\tsaltplus[salt->iov_len + 2] = '\\0';\n\tsaltplus[salt->iov_len + 3] = '\\1';\n\n\tif (!HMAC(evp, (const unsigned char *)in->iov_base, (int)in->iov_len,\n\t\t\t saltplus, salt->iov_len + 4, tempres, &ressize))\n\t{\n\t\treturn -1;\n\t}\n\n\tmemcpy(out->iov_base, tempres, ressize);\n\n\tfor (i = 1; i < itcnt; i++)\n\t{\n\t\tif (!HMAC(evp, (const unsigned char *)in->iov_base, (int)in->iov_len,\n\t\t\t\t tempres, ressize, tempdest, NULL))\n\t\t{\n\t\t\treturn -1;\n\t\t}\n\n\t\tfor (j = 0; j < (int)ressize; j++)\n\t\t{\n\t\t\t((char *)(out->iov_base))[j] ^= tempdest[j];\n\t\t\ttempres[j] = tempdest[j];\n\t\t}\n\t}\n\n\tout->iov_len = ressize;\n\treturn 0;\n}\n\nstatic int scram_hmac(const EVP_MD *evp, const struct iovec *key,\n\t\t\t\t\t const struct iovec *str, struct iovec *out)\n{\n\tunsigned int outsize;\n\n\tif (!HMAC(evp, (const unsigned char *)key->iov_base, (int)key->iov_len,\n\t\t\t (const unsigned char *)str->iov_base, (int)str->iov_len,\n\t\t\t (unsigned char *)out->iov_base, &outsize))\n\t{\n\t\treturn -1;\n\t}\n\n\tout->iov_len = outsize;\n\n\treturn 0;\n}\n\nstatic void scram_h(kafka_scram_t *scram, const struct iovec *str,\n\t\t\t\t\tstruct iovec *out)\n{\n\tscram->scram_h((const unsigned char *)str->iov_base, str->iov_len,\n\t\t\t\t (unsigned char *)out->iov_base);\n\tout->iov_len = scram->scram_h_size;\n}\n\nstatic void scram_build_client_final_message_wo_proof(\n\t\tkafka_scram_t *scram, const struct iovec *snonce, struct iovec *out)\n{\n\tconst char *attr_c = \"biws\";\n\n\tout->iov_len = 9 + scram->cnonce.iov_len + snonce->iov_len;\n\tout->iov_base = malloc(out->iov_len + 1);\n\tif (out->iov_base)\n\t{\n\t\tsnprintf((char *)out->iov_base, out->iov_len + 1, \"c=%s,r=%.*s%.*s\",\n\t\t\t\t attr_c, (int)scram->cnonce.iov_len,\n\t\t\t\t (char *)scram->cnonce.iov_base, (int)snonce->iov_len,\n\t\t\t\t (char *)snonce->iov_base);\n\t}\n}\n\nstatic int scram_build_client_final_message(kafka_scram_t *scram, int itcnt,\n\t\t\t\t\t\t\t\t\t\t\tconst struct iovec *salt,\n\t\t\t\t\t\t\t\t\t\t\tconst struct iovec *server_first_msg,\n\t\t\t\t\t\t\t\t\t\t\tconst struct iovec *server_nonce,\n\t\t\t\t\t\t\t\t\t\t\tstruct iovec *out,\n\t\t\t\t\t\t\t\t\t\t\tconst kafka_config_t *conf)\n{\n\tchar salted_pwd[EVP_MAX_MD_SIZE];\n\tchar client_key[EVP_MAX_MD_SIZE];\n\tchar server_key[EVP_MAX_MD_SIZE];\n\tchar stored_key[EVP_MAX_MD_SIZE];\n\tchar client_sign[EVP_MAX_MD_SIZE];\n\tchar server_sign[EVP_MAX_MD_SIZE];\n\tchar client_proof[EVP_MAX_MD_SIZE];\n\tstruct iovec password_iov = {conf->password, strlen(conf->password)};\n\tstruct iovec salted_pwd_iov = {salted_pwd, EVP_MAX_MD_SIZE};\n\tstruct iovec client_key_verbatim_iov = {(void *)\"Client Key\", 10};\n\tstruct iovec server_key_verbatim_iov = {(void *)\"Server Key\", 10};\n\tstruct iovec client_key_iov = {client_key, EVP_MAX_MD_SIZE};\n\tstruct iovec server_key_iov = {server_key, EVP_MAX_MD_SIZE};\n\tstruct iovec stored_key_iov = {stored_key, EVP_MAX_MD_SIZE};\n\tstruct iovec server_sign_iov = {server_sign, EVP_MAX_MD_SIZE};\n\tstruct iovec client_sign_iov = {client_sign, EVP_MAX_MD_SIZE};\n\tstruct iovec client_proof_iov = {client_proof, EVP_MAX_MD_SIZE};\n\tstruct iovec client_final_msg_wo_proof_iov;\n\tstruct iovec auth_message_iov;\n\tchar *server_sign_b64, *client_proof_b64 = NULL;\n\tint i;\n\n\tif (scram_hi((const EVP_MD *)scram->evp, itcnt, &password_iov, salt,\n\t\t\t\t &salted_pwd_iov) == -1)\n\t\treturn -1;\n\n\tif (scram_hmac((const EVP_MD *)scram->evp, &salted_pwd_iov,\n\t\t\t\t &client_key_verbatim_iov, &client_key_iov) == -1)\n\t\treturn -1;\n\n\tscram_h(scram, &client_key_iov, &stored_key_iov);\n\n\tscram_build_client_final_message_wo_proof(scram, server_nonce,\n\t\t\t\t\t\t\t\t\t\t\t &client_final_msg_wo_proof_iov);\n\n\tauth_message_iov.iov_len = scram->first_msg.iov_len + 1 +\n\t\tserver_first_msg->iov_len + 1 + client_final_msg_wo_proof_iov.iov_len;\n\tauth_message_iov.iov_base = alloca(auth_message_iov.iov_len + 1);\n\tif (auth_message_iov.iov_base)\n\t{\n\t\tsnprintf((char *)auth_message_iov.iov_base, auth_message_iov.iov_len + 1,\n\t\t\t\t \"%.*s,%.*s,%.*s\",\n\t\t\t\t (int)scram->first_msg.iov_len,\n\t\t\t\t (char *)scram->first_msg.iov_base,\n\t\t\t\t (int)server_first_msg->iov_len,\n\t\t\t\t (char *)server_first_msg->iov_base,\n\t\t\t\t (int)client_final_msg_wo_proof_iov.iov_len,\n\t\t\t\t (char *)client_final_msg_wo_proof_iov.iov_base);\n\n\t\tif (scram_hmac((const EVP_MD *)scram->evp, &salted_pwd_iov,\n\t\t\t\t\t &server_key_verbatim_iov, &server_key_iov) == 0 &&\n\t\t\tscram_hmac((const EVP_MD *)scram->evp, &server_key_iov,\n\t\t\t\t\t &auth_message_iov, &server_sign_iov) == 0)\n\t\t{\n\t\t\tserver_sign_b64 = scram_base64_encode(&server_sign_iov);\n\t\t\tif (server_sign_b64 &&\n\t\t\t\tscram_hmac((const EVP_MD *)scram->evp, &stored_key_iov,\n\t\t\t\t\t\t &auth_message_iov, &client_sign_iov) ==0 &&\n\t\t\t\tclient_key_iov.iov_len == client_sign_iov.iov_len)\n\t\t\t{\n\t\t\t\tscram->server_signature_b64.iov_base = server_sign_b64;\n\t\t\t\tscram->server_signature_b64.iov_len = strlen(server_sign_b64);\n\t\t\t\tfor (i = 0 ; i < (int)client_key_iov.iov_len; i++)\n\t\t\t\t\t((char *)(client_proof_iov.iov_base))[i] =\n\t\t\t\t\t\t((char *)(client_key_iov.iov_base))[i] ^\n\t\t\t\t\t\t((char *)(client_sign_iov.iov_base))[i];\n\t\t\t\tclient_proof_iov.iov_len = client_key_iov.iov_len;\n\n\t\t\t\tclient_proof_b64 = scram_base64_encode(&client_proof_iov);\n\t\t\t\tif (client_proof_b64)\n\t\t\t\t{\n\t\t\t\t\tout->iov_len = client_final_msg_wo_proof_iov.iov_len + 3 +\n\t\t\t\t\t\tstrlen(client_proof_b64);\n\t\t\t\t\tout->iov_base = malloc(out->iov_len + 1);\n\n\t\t\t\t\tsnprintf((char *)out->iov_base, out->iov_len + 1, \"%.*s,p=%s\",\n\t\t\t\t\t\t\t (int)client_final_msg_wo_proof_iov.iov_len,\n\t\t\t\t\t\t\t (char *)client_final_msg_wo_proof_iov.iov_base,\n\t\t\t\t\t\t\t client_proof_b64);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tfree(client_proof_b64);\n\tfree(client_final_msg_wo_proof_iov.iov_base);\n\treturn 0;\n}\n\nstatic int scram_handle_server_first_message(const char *buf, size_t len,\n\t\t\t\t\t\t\t\t\t\t\t kafka_config_t *conf,\n\t\t\t\t\t\t\t\t\t\t\t kafka_sasl_t *sasl)\n{\n\tint itcnt;\n\tint ret = -1;\n\tconst char *endptr;\n\tstruct iovec out, salt, server_nonce;\n\tconst struct iovec in = {(void *)buf, len};\n\n\tif (scram_get_attr(&in, 'm', &out) == 0)\n\t\treturn -1;\n\n\tif (scram_get_attr(&in, 'r', &server_nonce) != 0)\n\t\treturn -1;\n\n\tif (server_nonce.iov_len <= sasl->scram.cnonce.iov_len ||\n\t\tmemcmp(server_nonce.iov_base, sasl->scram.cnonce.iov_base,\n\t\t\t sasl->scram.cnonce.iov_len) != 0)\n\t{\n\t\treturn -1;\n\t}\n\n\tif (scram_get_attr(&in, 's', &out) != 0)\n\t\treturn -1;\n\n\tif (scram_base64_decode(&out, &salt) != 0)\n\t\treturn -1;\n\n\tif (scram_get_attr(&in, 'i', &out) == 0)\n\t{\n\t\titcnt = (int)strtoul((const char *)out.iov_base, (char **)&endptr, 10);\n\t\tif ((const char *)out.iov_base != endptr && *endptr == '\\0' &&\n\t\t\titcnt <= 1000000)\n\t\t{\n\t\t\tret = scram_build_client_final_message(&sasl->scram, itcnt, &salt,\n\t\t\t\t\t\t\t\t\t\t\t\t &in, &server_nonce, &out,\n\t\t\t\t\t\t\t\t\t\t\t\t conf);\n\t\t\tif (ret == 0)\n\t\t\t{\n\t\t\t\tfree(sasl->buf);\n\t\t\t\tsasl->buf = (char *)out.iov_base;\n\t\t\t\tsasl->bsize = out.iov_len;\n\t\t\t}\n\t\t}\n\t}\n\n\tfree(salt.iov_base);\n\treturn ret;\n}\n\nstatic int scram_handle_server_final_message(const char *buf, size_t len,\n\t\t\t\t\t\t\t\t\t\t\t kafka_config_t *conf,\n\t\t\t\t\t\t\t\t\t\t\t kafka_sasl_t *sasl)\n{\n\tstruct iovec attr_v, attr_e;\n\tconst struct iovec in = {(void *)buf, len};\n\n\tif (scram_get_attr(&in, 'm', &attr_e) == 0)\n\t\treturn -1;\n\n\tif (scram_get_attr(&in, 'v', &attr_v) == 0)\n\t{\n\t\tif (sasl->scram.server_signature_b64.iov_len == attr_v.iov_len &&\n\t\t\tstrncmp((const char *)sasl->scram.server_signature_b64.iov_base,\n\t\t\t\t\t(const char *)attr_v.iov_base, attr_v.iov_len) != 0)\n\t\t{\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\treturn 0;\n}\n\nstatic int kafka_sasl_scram_recv(const char *buf, size_t len, void *p, void *q)\n{\n\tkafka_config_t *conf = (kafka_config_t *)p;\n\tkafka_sasl_t *sasl = (kafka_sasl_t *)q;\n\tint ret = -1;\n\n\tswitch(sasl->scram.state)\n\t{\n\tcase KAFKA_SASL_SCRAM_STATE_SERVER_FIRST_MESSAGE:\n\t\tret = scram_handle_server_first_message(buf, len, conf, sasl);\n\t\tsasl->scram.state = KAFKA_SASL_SCRAM_STATE_CLIENT_FINAL_MESSAGE;\n\t\tbreak;\n\n\tcase KAFKA_SASL_SCRAM_STATE_CLIENT_FINAL_MESSAGE:\n\t\tret = scram_handle_server_final_message(buf, len, conf, sasl);\n\t\tsasl->scram.state = KAFKA_SASL_SCRAM_STATE_CLIENT_FINISHED;\n\t\tbreak;\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\treturn ret;\n}\n\nstatic int jitter(int low, int high)\n{\n\treturn (low + (rand() % ((high - low) + 1)));\n}\n\nstatic int scram_generate_nonce(struct iovec *iov)\n{\n\tint i;\n\tchar *ptr = (char *)malloc(33);\n\n\tif (!ptr)\n\t\treturn -1;\n\n\tfor (i = 0; i < 32; i++)\n\t\tptr[i] = jitter(0x2d, 0x7e);\n\tptr[32] = '\\0';\n\n\tiov->iov_base = ptr;\n\tiov->iov_len = 32;\n\treturn 0;\n}\n\nstatic int kafka_sasl_scram_client_new(void *p, kafka_sasl_t *sasl)\n{\n\tkafka_config_t *conf = (kafka_config_t *)p;\n\tsize_t ulen = strlen(conf->username);\n\tsize_t tlen = strlen(\"n,,n=,r=\");\n\tsize_t olen = ulen + tlen + 32;\n\tchar *ptr;\n\n\tif (sasl->scram.state != KAFKA_SASL_SCRAM_STATE_CLIENT_FIRST_MESSAGE)\n\t\treturn -1;\n\n\tif (scram_generate_nonce(&sasl->scram.cnonce) != 0)\n\t\treturn -1;\n\n\tptr = (char *)malloc(olen + 1);\n\tif (!ptr)\n\t\treturn -1;\n\n\tsnprintf(ptr, olen + 1, \"n,,n=%s,r=%.*s\", conf->username,\n\t\t\t(int)sasl->scram.cnonce.iov_len,\n\t\t\t(char *)sasl->scram.cnonce.iov_base);\n\tsasl->buf = ptr;\n\tsasl->bsize = olen;\n\n\tsasl->scram.first_msg.iov_base = ptr + 3;\n\tsasl->scram.first_msg.iov_len = olen - 3;\n\tsasl->scram.state = KAFKA_SASL_SCRAM_STATE_SERVER_FIRST_MESSAGE;\n\treturn 0;\n}\n\nint kafka_sasl_set_mechanisms(kafka_config_t *conf)\n{\n\tif (strcasecmp(conf->mechanisms, \"plain\") == 0)\n\t{\n\t\tconf->recv = kafka_sasl_plain_recv;\n\t\tconf->client_new = kafka_sasl_plain_client_new;\n\n\t\treturn 0;\n\t}\n\telse if (strncasecmp(conf->mechanisms, \"SCRAM\", 5) == 0)\n\t{\n\t\tconf->recv = kafka_sasl_scram_recv;\n\t\tconf->client_new = kafka_sasl_scram_client_new;\n\t}\n\n\treturn -1;\n}\n\nvoid kafka_sasl_init(kafka_sasl_t *sasl)\n{\n\tsasl->scram.evp = NULL;\n\tsasl->scram.scram_h = NULL;\n\tsasl->scram.scram_h_size = 0;\n\tsasl->scram.state = KAFKA_SASL_SCRAM_STATE_CLIENT_FIRST_MESSAGE;\n\tsasl->scram.cnonce.iov_base = NULL;\n\tsasl->scram.cnonce.iov_len = 0;\n\tsasl->scram.first_msg.iov_base = NULL;\n\tsasl->scram.first_msg.iov_len = 0;\n\tsasl->scram.server_signature_b64.iov_base = NULL;\n\tsasl->scram.server_signature_b64.iov_len = 0;\n\tsasl->buf = NULL;\n\tsasl->bsize = 0;\n\tsasl->status = 0;\n}\n\nvoid kafka_sasl_deinit(kafka_sasl_t *sasl)\n{\n\tfree(sasl->scram.cnonce.iov_base);\n\tfree(sasl->scram.server_signature_b64.iov_base);\n\tfree(sasl->buf);\n}\n\nint kafka_sasl_set_username(const char *username, kafka_config_t *conf)\n{\n\tchar *t = strdup(username);\n\n\tif (!t)\n\t\treturn -1;\n\n\tfree(conf->username);\n\tconf->username = t;\n\treturn 0;\n}\n\nint kafka_sasl_set_password(const char *password, kafka_config_t *conf)\n{\n\tchar *t = strdup(password);\n\n\tif (!t)\n\t\treturn -1;\n\n\tfree(conf->password);\n\tconf->password = t;\n\treturn 0;\n}\n\n" }, { "path": "src/protocol/kafka_parser.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n\t http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#ifndef _KAFKA_PARSER_H_\n#define _KAFKA_PARSER_H_\n\n#include \n#include \n#include \n#include \"list.h\"\n\nenum\n{\n\tKAFKA_UNKNOWN_SERVER_ERROR = -1,\n\tKAFKA_OFFSET_OUT_OF_RANGE = 1,\n\tKAFKA_CORRUPT_MESSAGE = 2,\n\tKAFKA_UNKNOWN_TOPIC_OR_PARTITION = 3,\n\tKAFKA_INVALID_FETCH_SIZE = 4,\n\tKAFKA_LEADER_NOT_AVAILABLE = 5,\n\tKAFKA_NOT_LEADER_FOR_PARTITION = 6,\n\tKAFKA_REQUEST_TIMED_OUT = 7,\n\tKAFKA_BROKER_NOT_AVAILABLE = 8,\n\tKAFKA_REPLICA_NOT_AVAILABLE = 9,\n\tKAFKA_MESSAGE_TOO_LARGE = 10,\n\tKAFKA_STALE_CONTROLLER_EPOCH = 11,\n\tKAFKA_OFFSET_METADATA_TOO_LARGE = 12,\n\tKAFKA_NETWORK_EXCEPTION = 13,\n\tKAFKA_COORDINATOR_LOAD_IN_PROGRESS = 14,\n\tKAFKA_COORDINATOR_NOT_AVAILABLE = 15,\n\tKAFKA_NOT_COORDINATOR = 16,\n\tKAFKA_INVALID_TOPIC_EXCEPTION = 17,\n\tKAFKA_RECORD_LIST_TOO_LARGE = 18,\n\tKAFKA_NOT_ENOUGH_REPLICAS = 19,\n\tKAFKA_NOT_ENOUGH_REPLICAS_AFTER_APPEND = 20,\n\tKAFKA_INVALID_REQUIRED_ACKS = 21,\n\tKAFKA_ILLEGAL_GENERATION = 22,\n\tKAFKA_INCONSISTENT_GROUP_PROTOCOL = 23,\n\tKAFKA_INVALID_GROUP_ID = 24,\n\tKAFKA_UNKNOWN_MEMBER_ID = 25,\n\tKAFKA_INVALID_SESSION_TIMEOUT = 26,\n\tKAFKA_REBALANCE_IN_PROGRESS = 27,\n\tKAFKA_INVALID_COMMIT_OFFSET_SIZE = 28,\n\tKAFKA_TOPIC_AUTHORIZATION_FAILED = 29,\n\tKAFKA_GROUP_AUTHORIZATION_FAILED = 30,\n\tKAFKA_CLUSTER_AUTHORIZATION_FAILED = 31,\n\tKAFKA_INVALID_TIMESTAMP = 32,\n\tKAFKA_UNSUPPORTED_SASL_MECHANISM = 33,\n\tKAFKA_ILLEGAL_SASL_STATE = 34,\n\tKAFKA_UNSUPPORTED_VERSION = 35,\n\tKAFKA_TOPIC_ALREADY_EXISTS = 36,\n\tKAFKA_INVALID_PARTITIONS = 37,\n\tKAFKA_INVALID_REPLICATION_FACTOR = 38,\n\tKAFKA_INVALID_REPLICA_ASSIGNMENT = 39,\n\tKAFKA_INVALID_CONFIG = 40,\n\tKAFKA_NOT_CONTROLLER = 41,\n\tKAFKA_INVALID_REQUEST = 42,\n\tKAFKA_UNSUPPORTED_FOR_MESSAGE_FORMAT = 43,\n\tKAFKA_POLICY_VIOLATION = 44,\n\tKAFKA_OUT_OF_ORDER_SEQUENCE_NUMBER = 45,\n\tKAFKA_DUPLICATE_SEQUENCE_NUMBER = 46,\n\tKAFKA_INVALID_PRODUCER_EPOCH = 47,\n\tKAFKA_INVALID_TXN_STATE = 48,\n\tKAFKA_INVALID_PRODUCER_ID_MAPPING = 49,\n\tKAFKA_INVALID_TRANSACTION_TIMEOUT = 50,\n\tKAFKA_CONCURRENT_TRANSACTIONS = 51,\n\tKAFKA_TRANSACTION_COORDINATOR_FENCED = 52,\n\tKAFKA_TRANSACTIONAL_ID_AUTHORIZATION_FAILED = 53,\n\tKAFKA_SECURITY_DISABLED = 54,\n\tKAFKA_OPERATION_NOT_ATTEMPTED = 55,\n\tKAFKA_KAFKA_STORAGE_ERROR = 56,\n\tKAFKA_LOG_DIR_NOT_FOUND = 57,\n\tKAFKA_SASL_AUTHENTICATION_FAILED = 58,\n\tKAFKA_UNKNOWN_PRODUCER_ID = 59,\n\tKAFKA_REASSIGNMENT_IN_PROGRESS = 60,\n\tKAFKA_DELEGATION_TOKEN_AUTH_DISABLED = 61,\n\tKAFKA_DELEGATION_TOKEN_NOT_FOUND = 62,\n\tKAFKA_DELEGATION_TOKEN_OWNER_MISMATCH = 63,\n\tKAFKA_DELEGATION_TOKEN_REQUEST_NOT_ALLOWED = 64,\n\tKAFKA_DELEGATION_TOKEN_AUTHORIZATION_FAILED = 65,\n\tKAFKA_DELEGATION_TOKEN_EXPIRED = 66,\n\tKAFKA_INVALID_PRINCIPAL_TYPE = 67,\n\tKAFKA_NON_EMPTY_GROUP = 68,\n\tKAFKA_GROUP_ID_NOT_FOUND = 69,\n\tKAFKA_FETCH_SESSION_ID_NOT_FOUND = 70,\n\tKAFKA_INVALID_FETCH_SESSION_EPOCH = 71,\n\tKAFKA_LISTENER_NOT_FOUND = 72,\n\tKAFKA_TOPIC_DELETION_DISABLED = 73,\n\tKAFKA_FENCED_LEADER_EPOCH = 74,\n\tKAFKA_UNKNOWN_LEADER_EPOCH = 75,\n\tKAFKA_UNSUPPORTED_COMPRESSION_TYPE = 76,\n\tKAFKA_STALE_BROKER_EPOCH = 77,\n\tKAFKA_OFFSET_NOT_AVAILABLE = 78,\n\tKAFKA_MEMBER_ID_REQUIRED = 79,\n\tKAFKA_PREFERRED_LEADER_NOT_AVAILABLE = 80,\n\tKAFKA_GROUP_MAX_SIZE_REACHED = 81,\n\tKAFKA_FENCED_INSTANCE_ID = 82,\n};\n\nenum\n{\n\tKafka_Unknown = -1,\n\tKafka_Produce = 0,\n\tKafka_Fetch = 1,\n\tKafka_ListOffsets = 2,\n\tKafka_Metadata = 3,\n\tKafka_LeaderAndIsr = 4,\n\tKafka_StopReplica = 5,\n\tKafka_UpdateMetadata = 6,\n\tKafka_ControlledShutdown = 7,\n\tKafka_OffsetCommit = 8,\n\tKafka_OffsetFetch = 9,\n\tKafka_FindCoordinator = 10,\n\tKafka_JoinGroup = 11,\n\tKafka_Heartbeat = 12,\n\tKafka_LeaveGroup = 13,\n\tKafka_SyncGroup = 14,\n\tKafka_DescribeGroups = 15,\n\tKafka_ListGroups = 16,\n\tKafka_SaslHandshake = 17,\n\tKafka_ApiVersions = 18,\n\tKafka_CreateTopics = 19,\n\tKafka_DeleteTopics = 20,\n\tKafka_DeleteRecords = 21,\n\tKafka_InitProducerId = 22,\n\tKafka_OffsetForLeaderEpoch = 23,\n\tKafka_AddPartitionsToTxn = 24,\n\tKafka_AddOffsetsToTxn = 25,\n\tKafka_EndTxn = 26,\n\tKafka_WriteTxnMarkers = 27,\n\tKafka_TxnOffsetCommit = 28,\n\tKafka_DescribeAcls = 29,\n\tKafka_CreateAcls = 30,\n\tKafka_DeleteAcls = 31,\n\tKafka_DescribeConfigs = 32,\n\tKafka_AlterConfigs = 33,\n\tKafka_AlterReplicaLogDirs = 34,\n\tKafka_DescribeLogDirs = 35,\n\tKafka_SaslAuthenticate = 36,\n\tKafka_CreatePartitions = 37,\n\tKafka_CreateDelegationToken = 38,\n\tKafka_RenewDelegationToken = 39,\n\tKafka_ExpireDelegationToken = 40,\n\tKafka_DescribeDelegationToken = 41,\n\tKafka_DeleteGroups = 42,\n\tKafka_ElectPreferredLeaders = 43,\n\tKafka_IncrementalAlterConfigs = 44,\n\tKafka_ApiNums,\n};\n\nenum\n{\n\tKafka_NoCompress,\n\tKafka_Gzip,\n\tKafka_Snappy,\n\tKafka_Lz4,\n\tKafka_Zstd,\n};\n\nenum\n{\n\tKAFKA_FEATURE_APIVERSION = 1<<0,\n\tKAFKA_FEATURE_BROKER_BALANCED_CONSUMER = 1<<1,\n\tKAFKA_FEATURE_THROTTLETIME = 1<<2,\n\tKAFKA_FEATURE_BROKER_GROUP_COORD = 1<<3,\n\tKAFKA_FEATURE_LZ4 = 1<<4,\n\tKAFKA_FEATURE_OFFSET_TIME = 1<<5,\n\tKAFKA_FEATURE_MSGVER2 = 1<<6,\n\tKAFKA_FEATURE_MSGVER1 = 1<<7,\n\tKAFKA_FEATURE_ZSTD = 1<<8,\n\tKAFKA_FEATURE_SASL_GSSAPI = 1<<9,\n\tKAFKA_FEATURE_SASL_HANDSHAKE = 1<<10,\n\tKAFKA_FEATURE_SASL_AUTH_REQ = 1<<11,\n};\n\nenum\n{\n\tKAFKA_OFFSET_AUTO,\n\tKAFKA_OFFSET_ASSIGN,\n};\n\nenum\n{\n\tKAFKA_BROKER_UNINIT,\n\tKAFKA_BROKER_DOING,\n\tKAFKA_BROKER_INITED,\n};\n\nenum\n{\n\tKAFKA_TIMESTAMP_EARLIEST = -2,\n\tKAFKA_TIMESTAMP_LATEST = -1,\n\tKAFKA_TIMESTAMP_UNINIT = 0,\n};\n\nenum\n{\n\tKAFKA_OFFSET_UNINIT = -2,\n\tKAFKA_OFFSET_OVERFLOW = -1,\n};\n\ntypedef struct __kafka_api_version\n{\n\tshort api_key;\n\tshort min_ver;\n\tshort max_ver;\n} kafka_api_version_t;\n\ntypedef struct __kafka_api_t\n{\n\tunsigned features;\n\tkafka_api_version_t *api;\n\tint elements;\n} kafka_api_t;\n\ntypedef struct __kafka_parser\n{\n\tint complete;\n\tsize_t message_size;\n\tvoid *msgbuf;\n\tsize_t cur_size;\n\tchar headbuf[4];\n\tsize_t hsize;\n} kafka_parser_t;\n\nenum __kafka_scram_state\n{\n\tKAFKA_SASL_SCRAM_STATE_CLIENT_FIRST_MESSAGE,\n\tKAFKA_SASL_SCRAM_STATE_SERVER_FIRST_MESSAGE,\n\tKAFKA_SASL_SCRAM_STATE_CLIENT_FINAL_MESSAGE,\n\tKAFKA_SASL_SCRAM_STATE_CLIENT_FINISHED,\n};\n\ntypedef struct __kafka_scram\n{\n\tconst void *evp;\n\tunsigned char *(*scram_h)(const unsigned char *d, size_t n,\n\t\t\t\t\t\t\t unsigned char *md);\n\tsize_t scram_h_size;\n\tenum __kafka_scram_state state;\n\tstruct iovec cnonce;\n\tstruct iovec first_msg;\n\tstruct iovec server_signature_b64;\n} kafka_scram_t;\n\ntypedef struct __kafka_sasl\n{\n\tkafka_scram_t scram;\n\tchar *buf;\n\tsize_t bsize;\n\tint status;\n} kafka_sasl_t;\n\ntypedef struct __kafka_config\n{\n\tint produce_timeout;\n\tint produce_msg_max_bytes;\n\tint produce_msgset_cnt;\n\tint produce_msgset_max_bytes;\n\tint fetch_timeout;\n\tint fetch_min_bytes;\n\tint fetch_max_bytes;\n\tint fetch_msg_max_bytes;\n\tlong long offset_timestamp;\n\tlong long commit_timestamp;\n\tint session_timeout;\n\tint rebalance_timeout;\n\tlong long retention_time_period;\n\tint produce_acks;\n\tint allow_auto_topic_creation;\n\tint api_version_request;\n\tint api_version_timeout;\n\tchar *broker_version;\n\tint compress_type;\n\tint compress_level;\n\tchar *client_id;\n\tint check_crcs;\n\tint offset_store;\n\tchar *rack_id;\n\n\tchar *mechanisms;\n\tchar *username;\n\tchar *password;\n\tint (*client_new)(void *conf, kafka_sasl_t *sasl);\n\tint (*recv)(const char *buf, size_t len, void *conf, void *sasl);\n} kafka_config_t;\n\ntypedef struct __kafka_broker\n{\n\tint node_id;\n\tint port;\n\tchar *host;\n\tchar *rack;\n\tshort error;\n\tint status;\n} kafka_broker_t;\n\ntypedef struct __kafka_partition\n{\n\tshort error;\n\tint partition_index;\n\tkafka_broker_t leader;\n\tint *replica_nodes;\n\tint replica_node_elements;\n\tint *isr_nodes;\n\tint isr_node_elements;\n} kafka_partition_t;\n\ntypedef struct __kafka_meta\n{\n\tshort error;\n\tchar *topic_name;\n\tchar *error_message;\n\tsigned char is_internal;\n\tkafka_partition_t **partitions;\n\tint partition_elements;\n} kafka_meta_t;\n\ntypedef struct __kafka_topic_partition\n{\n\tshort error;\n\tchar *topic_name;\n\tint partition;\n\tint preferred_read_replica;\n\tlong long offset;\n\tlong long high_watermark;\n\tlong long low_watermark;\n\tlong long last_stable_offset;\n\tlong long log_start_offset;\n\tlong long offset_timestamp;\n\tchar *committed_metadata;\n\tstruct list_head record_list;\n} kafka_topic_partition_t;\n\ntypedef struct __kafka_record_header\n{\n\tstruct list_head list;\n\tvoid *key;\n\tsize_t key_len;\n\tint key_is_moved;\n\tvoid *value;\n\tsize_t value_len;\n\tint value_is_moved;\n} kafka_record_header_t;\n\ntypedef struct __kafka_record\n{\n\tvoid *key;\n\tsize_t key_len;\n\tint key_is_moved;\n\tvoid *value;\n\tsize_t value_len;\n\tint value_is_moved;\n\tlong long timestamp;\n\tlong long offset;\n\tstruct list_head header_list;\n\tshort status;\n\tkafka_topic_partition_t *toppar;\n} kafka_record_t;\n\ntypedef struct __kafka_memeber\n{\n\tchar *member_id;\n\tchar *client_id;\n\tchar *client_host;\n\tvoid *member_metadata;\n\tsize_t member_metadata_len;\n\tstruct list_head toppar_list;\n\tstruct list_head assigned_toppar_list;\n} kafka_member_t;\n\ntypedef int (*kafka_assignor_t)(kafka_member_t **members, int member_elements,\n\t\t\t\t\t\t\t\tvoid *meta_topic);\n\ntypedef struct __kafka_group_protocol\n{\n\tstruct list_head list;\n\tchar *protocol_name;\n\tkafka_assignor_t assignor;\n} kafka_group_protocol_t;\n\ntypedef struct __kafka_cgroup\n{\n\tstruct list_head assigned_toppar_list;\n\tshort error;\n\tchar *error_msg;\n\tkafka_broker_t coordinator;\n\tchar *leader_id;\n\tchar *member_id;\n\tkafka_member_t **members;\n\tint member_elements;\n\tint generation_id;\n\tchar *group_name;\n\tchar *protocol_type;\n\tchar *protocol_name;\n\tstruct list_head group_protocol_list;\n} kafka_cgroup_t;\n\ntypedef struct __kafka_block\n{\n\tvoid *buf;\n\tsize_t len;\n\tint is_moved;\n} kafka_block_t;\n\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nint kafka_parser_append_message(const void *buf, size_t *size,\n\t\t\t\t\t\t\t\tkafka_parser_t *parser);\n\nvoid kafka_parser_init(kafka_parser_t *parser);\nvoid kafka_parser_deinit(kafka_parser_t *parser);\n\nvoid kafka_topic_partition_init(kafka_topic_partition_t *toppar);\nvoid kafka_topic_partition_deinit(kafka_topic_partition_t *toppar);\n\nvoid kafka_cgroup_init(kafka_cgroup_t *cgroup);\nvoid kafka_cgroup_deinit(kafka_cgroup_t *cgroup);\n\nvoid kafka_block_init(kafka_block_t *block);\nvoid kafka_block_deinit(kafka_block_t *block);\n\nvoid kafka_broker_init(kafka_broker_t *brock);\nvoid kafka_broker_deinit(kafka_broker_t *broker);\n\nvoid kafka_config_init(kafka_config_t *config);\nvoid kafka_config_deinit(kafka_config_t *config);\n\nvoid kafka_meta_init(kafka_meta_t *meta);\nvoid kafka_meta_deinit(kafka_meta_t *meta);\n\nvoid kafka_partition_init(kafka_partition_t *partition);\nvoid kafka_partition_deinit(kafka_partition_t *partition);\n\nvoid kafka_member_init(kafka_member_t *member);\nvoid kafka_member_deinit(kafka_member_t *member);\n\nvoid kafka_record_init(kafka_record_t *record);\nvoid kafka_record_deinit(kafka_record_t *record);\n\nvoid kafka_record_header_init(kafka_record_header_t *header);\nvoid kafka_record_header_deinit(kafka_record_header_t *header);\n\nvoid kafka_api_init(kafka_api_t *api);\nvoid kafka_api_deinit(kafka_api_t *api);\n\nvoid kafka_sasl_init(kafka_sasl_t *sasl);\nvoid kafka_sasl_deinit(kafka_sasl_t *sasl);\n\nint kafka_topic_partition_set_tp(const char *topic_name, int partition,\n\t\t\t\t\t\t\t\t kafka_topic_partition_t *toppar);\n\nint kafka_record_set_key(const void *key, size_t key_len,\n\t\t\t\t\t\t kafka_record_t *record);\n\nint kafka_record_set_value(const void *val, size_t val_len,\n\t\t\t\t\t\t kafka_record_t *record);\n\nint kafka_record_header_set_kv(const void *key, size_t key_len,\n\t\t\t\t\t\t\t const void *val, size_t val_len,\n\t\t\t\t\t\t\t kafka_record_header_t *header);\n\nint kafka_meta_set_topic(const char *topic_name, kafka_meta_t *meta);\n\nint kafka_cgroup_set_group(const char *group_name, kafka_cgroup_t *cgroup);\n\nint kafka_broker_get_api_version(const kafka_api_t *broker, int api_key,\n\t\t\t\t\t\t\t\t int min_ver, int max_ver);\n\nunsigned kafka_get_features(kafka_api_version_t *api, size_t api_cnt);\n\nint kafka_api_version_is_queryable(const char *broker_version,\n\t\t\t\t\t\t\t\t kafka_api_version_t **api,\n\t\t\t\t\t\t\t\t size_t *api_cnt);\n\nint kafka_sasl_set_mechanisms(kafka_config_t *conf);\nint kafka_sasl_set_username(const char *username, kafka_config_t *conf);\nint kafka_sasl_set_password(const char *passwd, kafka_config_t *conf);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n\n" }, { "path": "src/protocol/mysql_byteorder.c", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \"mysql_byteorder.h\"\n\nint decode_length_safe(unsigned long long *res, const unsigned char **pos,\n\t\t\t\t\t const unsigned char *end)\n{\n\tconst unsigned char *p = *pos;\n\n\tif (p >= end)\n\t\treturn 0;\n\n\tswitch (*p)\n\t{\n\tdefault:\n\t\t*res = *p;\n\t\t*pos = p + 1;\n\t\tbreak;\n\n\tcase 251:\n\t\t*res = (~0ULL);\n\t\t*pos = p + 1;\n\t\tbreak;\n\n\tcase 252:\n\t\tif (p + 2 > end)\n\t\t\treturn 0;\n\n\t\t*res = uint2korr(p + 1);\n\t\t*pos = p + 3;\n\t\tbreak;\n\n\tcase 253:\n\t\tif (p + 3 > end)\n\t\t\treturn 0;\n\n\t\t*res = uint3korr(p + 1);\n\t\t*pos = p + 4;\n\t\tbreak;\n\n\tcase 254:\n\t\tif (p + 8 > end)\n\t\t\treturn 0;\n\n\t\t*res = uint8korr(p + 1);\n\t\t*pos = p + 9;\n\t\tbreak;\n\n\tcase 255:\n\t\treturn -1;\n\t}\n\n\treturn 1;\n}\n\nint decode_string(const unsigned char **str, unsigned long long *len,\n\t\t\t\t const unsigned char **pos, const unsigned char *end)\n{\n\tunsigned long long length;\n\n\tif (decode_length_safe(&length, pos, end) <= 0)\n\t\treturn 0;\n\n\tif (length == (~0ULL))\n\t\tlength = 0;\n\n\tif (*pos + length > end)\n\t\treturn 0;\n\n\t*len = length;\n\t*str = *pos;\n\t*pos = *pos + length;\n\treturn 1;\n}\n\n" }, { "path": "src/protocol/mysql_byteorder.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#ifndef _MYSQL_BYTEORDER_H_\n#define _MYSQL_BYTEORDER_H_\n\n#include \n#include \n#include \n\n#if __BYTE_ORDER == __LITTLE_ENDIAN\n\nstatic inline uint16_t uint2korr(const unsigned char *A)\n{\n\tuint16_t ret;\n\tmemcpy(&ret, A, sizeof(ret));\n\treturn ret;\n}\n\nstatic inline uint32_t uint3korr(const unsigned char *A)\n{\n\tuint32_t ret = 0;\n\tmemcpy(&ret, A, 3);\n\treturn ret;\n}\n\nstatic inline uint32_t uint4korr(const unsigned char *A)\n{\n\tuint32_t ret;\n\tmemcpy(&ret, A, sizeof(ret));\n\treturn ret;\n}\n\nstatic inline uint64_t uint8korr(const unsigned char *A)\n{\n\tuint64_t ret;\n\tmemcpy(&ret, A, sizeof(ret));\n\treturn ret;\n}\n\nstatic inline void int2store(unsigned char *T, uint16_t A)\n{\n\tmemcpy(T, &A, sizeof(A));\n}\n\nstatic inline void int3store(unsigned char *T, uint32_t A)\n{\n\tmemcpy(T, &A, 3);\n}\n\nstatic inline void int4store(unsigned char *T, uint32_t A)\n{\n\tmemcpy(T, &A, sizeof(A));\n}\n\nstatic inline void int7store(unsigned char *T, uint64_t A)\n{\n\tmemcpy(T, &A, 7);\n}\n\nstatic inline void int8store(unsigned char *T, uint64_t A)\n{\n\tmemcpy(T, &A, sizeof(A));\n}\n\n#elif __BYTE_ORDER == __BIG_ENDIAN\n\nstatic inline uint16_t uint2korr(const unsigned char *A)\n{\n\treturn (uint16_t)(((uint16_t)(A[0])) + ((uint16_t)(A[1]) << 8));\n}\n\nstatic inline uint32_t uint3korr(const unsigned char *p)\n{\n\treturn (uint32_t)(((uint32_t)(p[0])) +\n\t\t (((uint32_t)(p[1])) << 8) +\n\t\t (((uint32_t)(p[2])) << 16));\n}\n\nstatic inline uint32_t uint4korr(const unsigned char *A)\n{\n\treturn (uint32_t)(((uint32_t)(A[0])) + (((uint32_t)(A[1])) << 8) +\n\t\t (((uint32_t)(A[2])) << 16) + (((uint32_t)(A[3])) << 24));\n}\n\nstatic inline uint64_t uint8korr(const unsigned char *A)\n{\n\treturn ((uint64_t)(((uint32_t)(A[0])) + (((uint32_t)(A[1])) << 8) +\n\t\t (((uint32_t)(A[2])) << 16) + (((uint32_t)(A[3])) << 24)) +\n\t\t (((uint64_t)(((uint32_t)(A[4])) + (((uint32_t)(A[5])) << 8) +\n\t\t (((uint32_t)(A[6])) << 16) + (((uint32_t)(A[7])) << 24))) << 32));\n}\n\nstatic inline void int2store(unsigned char *T, uint16_t A)\n{\n\tuint32_t def_temp = A;\n\t*(T) = (unsigned char)(def_temp);\n\t*(T + 1) = (unsigned char)(def_temp >> 8);\n}\n\nstatic inline void int3store(unsigned char *p, uint32_t x)\n{\n\t*(p) = (unsigned char)(x);\n\t*(p + 1) = (unsigned char)(x >> 8);\n\t*(p + 2) = (unsigned char)(x >> 16);\n}\n\nstatic inline void int4store(unsigned char *T, uint32_t A)\n{\n\t*(T) = (unsigned char)(A);\n\t*(T + 1) = (unsigned char)(A >> 8);\n\t*(T + 2) = (unsigned char)(A >> 16);\n\t*(T + 3) = (unsigned char)(A >> 24);\n}\n\nstatic inline void int7store(unsigned char *T, uint64_t A)\n{\n\t*(T) = (unsigned char)(A);\n\t*(T + 1) = (unsigned char)(A >> 8);\n\t*(T + 2) = (unsigned char)(A >> 16);\n\t*(T + 3) = (unsigned char)(A >> 24);\n\t*(T + 4) = (unsigned char)(A >> 32);\n\t*(T + 5) = (unsigned char)(A >> 40);\n\t*(T + 6) = (unsigned char)(A >> 48);\n}\n\nstatic inline void int8store(unsigned char *T, uint64_t A)\n{\n\tuint32_t def_temp = (uint32_t)A, def_temp2 = (uint32_t)(A >> 32);\n\tint4store(T, def_temp);\n\tint4store(T + 4, def_temp2);\n}\n\n#else\n# error \"unknown byte order\"\n#endif\n\n// length of buffer needed to store this number [1, 3, 4, 9].\nstatic inline unsigned int get_length_size(unsigned long long num)\n{\n\tif (num < (unsigned long long)252LL)\n\t\treturn 1;\n\n\tif (num < (unsigned long long)65536LL)\n\t\treturn 3;\n\n\tif (num < (unsigned long long)16777216LL)\n\t\treturn 4;\n\n\treturn 9;\n}\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\n// decode encoded length integer within *end, move pos forward\nint decode_length_safe(unsigned long long *res, const unsigned char **pos,\n\t\t\t\t\t const unsigned char *end);\n\n// decode encoded length string within *end, move pos forward\nint decode_string(const unsigned char **str, unsigned long long *len,\n\t\t\t\t const unsigned char **pos, const unsigned char *end);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n\n" }, { "path": "src/protocol/mysql_parser.c", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \"mysql_types.h\"\n#include \"mysql_byteorder.h\"\n#include \"mysql_parser.h\"\n\nstatic int parse_base_packet(const void *buf, size_t len, mysql_parser_t *parser);\n\nstatic int parse_error_packet(const void *buf, size_t len, mysql_parser_t *parser);\n\nstatic int parse_ok_packet(const void *buf, size_t len, mysql_parser_t *parser);\n\nstatic int parse_eof_packet(const void *buf, size_t len, mysql_parser_t *parser);\n\nstatic int parse_field_eof_packet(const void *buf, size_t len, mysql_parser_t *parser);\n\nstatic int parse_field_count(const void *buf, size_t len, mysql_parser_t *parser);\n\nstatic int parse_column_def_packet(const void *buf, size_t len, mysql_parser_t *parser);\n\nstatic int parse_local_inline(const void *buf, size_t len, mysql_parser_t *parser);\n\nstatic int parse_row_packet(const void *buf, size_t len, mysql_parser_t *parser);\n\nvoid mysql_parser_init(mysql_parser_t *parser)\n{\n\tparser->offset = 0;\n\tparser->cmd = MYSQL_COM_QUERY;\n\tparser->packet_type = MYSQL_PACKET_OTHER;\n\tparser->parse = parse_base_packet;\n\tparser->result_set_count = 0;\n\tINIT_LIST_HEAD(&parser->result_set_list);\n}\n\nvoid mysql_parser_deinit(mysql_parser_t *parser)\n{\n\tstruct __mysql_result_set *result_set;\n\tstruct list_head *pos, *tmp;\n\tint i;\n\n\tlist_for_each_safe(pos, tmp, &parser->result_set_list)\n\t{\n\t\tresult_set = list_entry(pos, struct __mysql_result_set, list);\n\t\tlist_del(pos);\n\n\t\tif (result_set->field_count)\n\t\t{\n\t\t\tfor (i = 0; i < result_set->field_count; i++)\n\t\t\t\tfree(result_set->fields[i]);\n\n\t\t\tfree(result_set->fields);\n\t\t}\n\n\t\tfree(result_set);\n\t}\n}\n\nint mysql_parser_parse(const void *buf, size_t len, mysql_parser_t *parser)\n{\n//\tconst char *end = (const char *)buf + len;\n\tint ret;\n\n\tdo {\n\t\tret = parser->parse(buf, len, parser);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tif (ret > 0 && parser->offset != len)\n\t\t\treturn -2;\n\n\t} while (parser->offset < len);\n\n\treturn ret;\n}\n\nvoid mysql_parser_get_net_state(const char **net_state_str,\n\t\t\t\t\t\t\t\tsize_t *net_state_len,\n\t\t\t\t\t\t\t\tmysql_parser_t *parser)\n{\n\t*net_state_str = (const char *)parser->buf + parser->net_state_offset;\n\t*net_state_len = MYSQL_STATE_LENGTH;\n}\n\nvoid mysql_parser_get_err_msg(const char **err_msg_str,\n\t\t\t\t\t\t\t size_t *err_msg_len,\n\t\t\t\t\t\t\t mysql_parser_t *parser)\n{\n\tif (parser->err_msg_offset == (size_t)-1 && parser->err_msg_len == 0)\n\t{\n\t\t*err_msg_str = MYSQL_STATE_DEFAULT;\n\t\t*err_msg_len = MYSQL_STATE_LENGTH;\n\t} else {\n\t\t*err_msg_str = (const char *)parser->buf + parser->err_msg_offset;\n\t\t*err_msg_len = parser->err_msg_len;\n\t}\n}\n\nstatic int parse_base_packet(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tconst unsigned char *p = (const unsigned char *)buf + parser->offset;\n\n\tswitch (*p)\n\t{\n\t// OK PACKET\n\tcase MYSQL_PACKET_HEADER_OK:\n\t\tparser->parse = parse_ok_packet;\n\t\tbreak;\n\t// ERR PACKET\n\tcase MYSQL_PACKET_HEADER_ERROR:\n\t\tparser->parse = parse_error_packet;\n\t\tbreak;\n\t// EOF PACKET\n\tcase MYSQL_PACKET_HEADER_EOF:\n\t\tparser->parse = parse_eof_packet;\n\t\tbreak;\n\t// LOCAL INFILE PACKET\n\tcase MYSQL_PACKET_HEADER_NULL:\n\t\t// if (field_count == -1)\n\t\tparser->parse = parse_local_inline;\n\t\tbreak;\n\tdefault:\n\t\tparser->parse = parse_field_count;\n\t\tbreak;\n\t}\n\n\treturn 0;\n}\n\n// 1:0xFF|2:err_no|1:#|5:server_state|0-512:err_msg\nstatic int parse_error_packet(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tconst unsigned char *p = (const unsigned char *)buf + parser->offset;\n\tconst unsigned char *buf_end = (const unsigned char *)buf + len;\n\n\tif (p + 9 > buf_end)\n\t\treturn -2;\n\n\tparser->error = uint2korr(p + 1);\n\tp += 3;\n\n\tif (*p == '#')\n\t{\n\t\tp += 1;\n\t\tparser->net_state_offset = p - (const unsigned char *)buf;\n\t\tp += MYSQL_STATE_LENGTH;\n\n\t\tsize_t msg_len = len - parser->offset - 9;\n\t\tparser->err_msg_offset = p - (const unsigned char *)buf;\n\t\tparser->err_msg_len = msg_len;\n\t} else {\n\t\tparser->err_msg_offset = (size_t)-1;\n\t\tparser->err_msg_len = 0;\n\t}\n\n\tparser->offset = len;\n\tparser->packet_type = MYSQL_PACKET_ERROR;\n\tparser->buf = buf;\n\treturn 1;\n}\n\n// 1:0x00|1-9:affect_row|1-9:insert_id|2:server_status|2:warning_count|0-n:server_msg\nstatic int parse_ok_packet(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tconst unsigned char *p = (const unsigned char *)buf + parser->offset;\n\tconst unsigned char *buf_end = (const unsigned char *)buf + len;\n\n\tunsigned long long affected_rows, insert_id, info_len;\n\tconst unsigned char *str;\n\tstruct __mysql_result_set *result_set;\n\tunsigned int warning_count;\n\tint server_status;\n\n\tp += 1;// 0x00\n\tif (decode_length_safe(&affected_rows, &p, buf_end) <= 0)\n\t\treturn -2;\n\n\tif (decode_length_safe(&insert_id, &p, buf_end) <= 0)\n\t\treturn -2;\n\n\tif (p + 4 > buf_end)\n\t\treturn -2;\n\n\tserver_status = uint2korr(p);\n\tp += 2;\n\twarning_count = uint2korr(p);\n\tp += 2;\n\n\tif (p != buf_end)\n\t{\n\t\tif (decode_string(&str, &info_len, &p, buf_end) == 0)\n\t\t\treturn -2;\n\n\t\tif (p != buf_end)\n\t\t{\n\t\t\tif (server_status & MYSQL_SERVER_SESSION_STATE_CHANGED)\n\t\t\t{\n\t\t\t\tconst unsigned char *tmp_str;\n\t\t\t\tunsigned long long tmp_len;\n\t\t\t\tif (decode_string(&tmp_str, &tmp_len, &p, buf_end) == 0)\n\t\t\t\t\treturn -2;\n\t\t\t} else\n\t\t\t\treturn -2;\n\t\t}\n\t} else {\n\t\tstr = p;\n\t\tinfo_len = 0;\n\t}\n\n\tresult_set = (struct __mysql_result_set *)malloc(sizeof(struct __mysql_result_set));\n\tif (result_set == NULL)\n\t\treturn -1;\n\n\tresult_set->info_offset = str - (const unsigned char *)buf;\n\tresult_set->info_len = info_len;\n\tresult_set->affected_rows = (affected_rows == ~0ULL) ? 0 : affected_rows;\n\tresult_set->insert_id = (insert_id == ~0ULL) ? 0 : insert_id;\n\tresult_set->server_status = server_status;\n\tresult_set->warning_count = warning_count;\n\tresult_set->type = MYSQL_PACKET_OK;\n\tresult_set->field_count = 0;\n\n\tlist_add_tail(&result_set->list, &parser->result_set_list);\n\tparser->current_result_set = result_set;\n\tparser->result_set_count++;\n\tparser->packet_type = MYSQL_PACKET_OK;\n\n\tparser->buf = buf;\n\tparser->offset = p - (const unsigned char *)buf;\n\n\tif (server_status & MYSQL_SERVER_MORE_RESULTS_EXIST)\n\t{\n\t\tparser->parse = parse_base_packet;\n\t\treturn 0;\n\t}\n\n\treturn 1;\n}\n\n// 1:0xfe|2:warnings|2:status_flag\nstatic int parse_eof_packet(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tconst unsigned char *p = (const unsigned char *)buf + parser->offset;\n\tconst unsigned char *buf_end = (const unsigned char *)buf + len;\n\n\tif (p + 5 > buf_end)\n\t\treturn -2;\n\n\tparser->offset += 5;\n\tparser->packet_type = MYSQL_PACKET_EOF;\n\tparser->buf = buf;\n\n\tint status_flag = uint2korr(p + 3);\n\tif (status_flag & MYSQL_SERVER_MORE_RESULTS_EXIST)\n\t{\n\t\tparser->parse = parse_base_packet;\n\t\treturn 0;\n\t}\n\n\treturn 1;\n}\n\nstatic int parse_field_eof_packet(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tconst unsigned char *p = (const unsigned char *)buf + parser->offset;\n\tconst unsigned char *buf_end = (const unsigned char *)buf + len;\n\n\tif (p + 5 > buf_end)\n\t\treturn -2;\n\n\tparser->offset += 5;\n\tparser->current_result_set->rows_begin_offset = parser->offset;\n\tparser->parse = parse_row_packet;\n\treturn 0;\n}\n\n//raw file data\nstatic int parse_local_inline(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tparser->local_inline_offset = parser->offset;\n\tparser->local_inline_length = len - parser->offset;\n\tparser->offset = len;\n\tparser->packet_type = MYSQL_PACKET_LOCAL_INLINE;\n\tparser->buf = buf;\n\treturn 1;\n}\n\n// for each field:\n// NULL as 0xfb, or a length-encoded-string\nstatic int parse_row_packet(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tconst unsigned char *p = (const unsigned char *)buf + parser->offset;\n\tconst unsigned char *buf_end = (const unsigned char *)buf + len;\n\n\tunsigned long long cell_len;\n\tconst unsigned char *cell_data;\n\n\tsize_t i;\n\n\tif (*p == MYSQL_PACKET_HEADER_ERROR)\n\t{\n\t\tparser->parse = parse_error_packet;\n\t\treturn 0;\n\t}\n\n\tif (*p == MYSQL_PACKET_HEADER_EOF)\n\t{\n\t\tparser->parse = parse_eof_packet;\n\t\tparser->current_result_set->rows_end_offset = parser->offset;\n\n\t\treturn 0;\n\t}\n\n\tfor (i = 0; i < parser->current_result_set->field_count; i++)\n\t{\n\t\tif (*p == MYSQL_PACKET_HEADER_NULL)\n\t\t{\n\t\t\tp++;\n\t\t} else {\n\t\t\tif (decode_string(&cell_data, &cell_len, &p, buf_end) == 0)\n\t\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (i != parser->current_result_set->field_count)\n\t\treturn -2;\n\n\tparser->current_result_set->row_count++;\n\tparser->offset = p - (const unsigned char *)buf;\n\treturn 0;\n}\n\nstatic int parse_field_count(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tconst unsigned char *p = (const unsigned char *)buf + parser->offset;\n\tconst unsigned char *buf_end = (const unsigned char *)buf + len;\n\n\tunsigned long long field_count;\n\tstruct __mysql_result_set *result_set;\n\n\tif (decode_length_safe(&field_count, &p, buf_end) <= 0)\n\t\treturn -2;\n\n\tfield_count = (field_count == ~0ULL) ? 0 : field_count;\n\n\tif (field_count)\n\t{\n\t\tresult_set = (struct __mysql_result_set *)malloc(sizeof (struct __mysql_result_set));\n\t\tif (result_set == NULL)\n\t\t\treturn -1;\n\n\t\tresult_set->fields = (mysql_field_t **)calloc(field_count, sizeof (mysql_field_t *));\n\t\tif (result_set->fields == NULL)\n\t\t{\n\t\t\tfree(result_set);\n\t\t\treturn -1;\n\t\t}\n\n\t\tresult_set->field_count = field_count;\n\t\tresult_set->row_count = 0;\n\t\tresult_set->type = MYSQL_PACKET_GET_RESULT;\n\n\t\tlist_add_tail(&result_set->list, &parser->result_set_list);\n\t\tparser->current_result_set = result_set;\n\t\tparser->current_field_count = 0;\n\t\tparser->result_set_count++;\n\t\tparser->packet_type = MYSQL_PACKET_GET_RESULT;\n\n\t\tparser->parse = parse_column_def_packet;\n\t\tparser->offset = p - (const unsigned char *)buf;\n\t} else {\n\t\tparser->parse = parse_ok_packet;\n\t}\n\treturn 0;\n}\n\n// COLUMN DEFINATION PACKET. for one field: (after protocol 41)\n// str:catalog|str:db|str:table|str:org_table|str:name|str:org_name|\n// 2:charsetnr|4:length|1:type|2:flags|1:decimals|1:0x00|1:0x00|n:str(if COM_FIELD_LIST)\nstatic int parse_column_def_packet(const void *buf, size_t len, mysql_parser_t *parser)\n{\n\tconst unsigned char *p = (const unsigned char *)buf + parser->offset;\n\tconst unsigned char *buf_end = (const unsigned char *)buf + len;\n\n\tint flag = 0;\n\tconst unsigned char *str;\n\tunsigned long long str_len;\n\tmysql_field_t *field = (mysql_field_t *)malloc(sizeof(mysql_field_t));\n\n\tif (!field)\n\t\treturn -1;\n\n\tdo {\n\t\tif (decode_string(&str, &str_len, &p, buf_end) == 0)\n\t\t\tbreak;\n\t\tfield->catalog_offset = str - (const unsigned char *)buf;\n\t\tfield->catalog_length = str_len;\n\n\t\tif (decode_string(&str, &str_len, &p, buf_end) == 0)\n\t\t\tbreak;\n\t\tfield->db_offset = str - (const unsigned char *)buf;\n\t\tfield->db_length = str_len;\n\n\t\tif (decode_string(&str, &str_len, &p, buf_end) == 0)\n\t\t\tbreak;\n\t\tfield->table_offset = str - (const unsigned char *)buf;\n\t\tfield->table_length = str_len;\n\n\t\tif (decode_string(&str, &str_len, &p, buf_end) == 0)\n\t\t\tbreak;\n\t\tfield->org_table_offset = str - (const unsigned char *)buf;\n\t\tfield->org_table_length = str_len;\n\n\t\tif (decode_string(&str, &str_len, &p, buf_end) == 0)\n\t\t\tbreak;\n\t\tfield->name_offset = str - (const unsigned char *)buf;\n\t\tfield->name_length = str_len;\n\n\t\tif (decode_string(&str, &str_len, &p, buf_end) == 0)\n\t\t\tbreak;\n\t\tfield->org_name_offset = str - (const unsigned char *)buf;\n\t\tfield->org_name_length = str_len;\n\n\t\t// the rest needs at least 13\n\t\tif (p + 13 > buf_end)\n\t\t\tbreak;\n\n\t\tp++; // length of the following fields (always 0x0c)\n\t\tfield->charsetnr = uint2korr(p);\n\t\tfield->length = uint4korr(p + 2);\n\t\tfield->data_type = *(p + 6);\n\t\tfield->flags = uint2korr(p + 7);\n\t\tfield->decimals = (int)p[9];\n\t\tp += 12;\n\t\t// if is COM_FIELD_LIST, the rest is a string\n\t\t// 0x03 for COM_QUERY\n\t\tif (parser->cmd == MYSQL_COM_FIELD_LIST)\n\t\t{\n\t\t\tif (decode_string(&str, &str_len, &p, buf_end) == 0)\n\t\t\t\tbreak;\n\t\t\tfield->def_offset = str - (const unsigned char *)buf;\n\t\t\tfield->def_length = str_len;\n\t\t} else {\n\t\t\tfield->def_offset = (size_t)-1;\n\t\t\tfield->def_length = 0;\n\t\t}\n\t\tflag = 1;\n\t} while (0);\n\n\tif (flag == 0)\n\t{\n\t\tfree(field);\n\t\treturn -2;\n\t}\n\n\t//parser->fields.emplace_back(std::move(field));\n\tparser->current_result_set->fields[parser->current_field_count] = field;\n\n\tparser->offset = p - (const unsigned char *)buf;\n\tif (++parser->current_field_count == parser->current_result_set->field_count)\n\t\tparser->parse = parse_field_eof_packet;\n\n\treturn 0;\n}\n\n" }, { "path": "src/protocol/mysql_parser.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#ifndef _MYSQL_PARSER_H_\n#define _MYSQL_PARSER_H_\n\n#include \n#include \"list.h\"\n\n// the first byte in response message\n// from 1 to 0xfa means result_set field or data_row\n// NULL is sent as 0xfb, will be treated as LOCAL_INLINE\n// MySQL MESSAGE STATUS\nenum\n{\n\tMYSQL_PACKET_HEADER_OK\t\t=\t0,\n\tMYSQL_PACKET_HEADER_NULL\t= 251, //0xfb\n\tMYSQL_PACKET_HEADER_EOF\t\t= 254, //0xfe\n\tMYSQL_PACKET_HEADER_ERROR\t= 255, //0xff\n};\n\ntypedef struct __mysql_field\n{\n\tsize_t name_offset;\t\t\t/* Name of column */\n\tsize_t org_name_offset;\t\t/* Original column name, if an alias */\n\tsize_t table_offset;\t\t/* Table of column if column was a field */\n\tsize_t org_table_offset;\t/* Org table name, if table was an alias */\n\tsize_t db_offset;\t\t\t/* Database for table */\n\tsize_t catalog_offset;\t\t/* Catalog for table */\n\tsize_t def_offset;\t\t\t/* Default value (set by mysql_list_fields) */\n\tint length;\t\t\t\t\t/* Width of column (create length) */\n\tint name_length;\n\tint org_name_length;\n\tint table_length;\n\tint org_table_length;\n\tint db_length;\n\tint catalog_length;\n\tint def_length;\n\tint flags;\t\t\t\t\t/* Div flags */\n\tint decimals;\t\t\t\t/* Number of decimals in field */\n\tint charsetnr;\t\t\t\t/* Character set */\n\tint data_type;\t\t\t\t/* Type of field. See mysql_types.h for types */\n//\tvoid *extension;\n} mysql_field_t;\n\nstruct __mysql_result_set\n{\n\tstruct list_head list;\n\tint type;\n\tint server_status;\n\n\tint field_count;\n\tint row_count;\n\tsize_t rows_begin_offset;\n\tsize_t rows_end_offset;\n\tmysql_field_t **fields;\n\n\tunsigned long long affected_rows;\n\tunsigned long long insert_id;\n\tint warning_count;\n\tsize_t info_offset;\n\tint info_len;\n};\n\ntypedef struct __mysql_result_set_cursor \n{\n\tconst struct list_head *head;\n\tconst struct list_head *current;\n} mysql_result_set_cursor_t;\n\ntypedef struct __mysql_parser\n{\n\tsize_t offset;\n\tint cmd;\n\tint packet_type;\n\tint (*parse)(const void *, size_t, struct __mysql_parser *);\n\n\tsize_t net_state_offset;\t\t// err packet server_state\n\tsize_t err_msg_offset; \t\t\t// -1 for default\n\tint err_msg_len;\t\t\t\t// -1 for default\n\n\tsize_t local_inline_offset; \t// local inline file name\n\tint local_inline_length;\n\tconst void *buf;\n\tint error;\n\n\tint result_set_count;\n\tstruct list_head result_set_list;\n\tstruct __mysql_result_set *current_result_set;\n\tint current_field_count;\n} mysql_parser_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nvoid mysql_parser_init(mysql_parser_t *parser);\nvoid mysql_parser_deinit(mysql_parser_t *parser);\nvoid mysql_parser_get_info(const char **info_str,\n\t\t\t\t\t\t\tsize_t *info_len,\n\t\t\t\t\t\t\tmysql_parser_t *parser);\nvoid mysql_parser_get_net_state(const char **net_state_str,\n\t\t\t\t\t\t\t\tsize_t *net_state_len,\n\t\t\t\t\t\t\t\tmysql_parser_t *parser);\nvoid mysql_parser_get_err_msg(const char **err_msg_str,\n\t\t\t\t\t\t\t size_t *err_msg_len,\n\t\t\t\t\t\t\t mysql_parser_t *parser);\n// if append check get 0, don`t need to parse()\n// if append check get 1, parse and tell them if this is all the package\n//\n// ret: 1: this ResultSet is received finished\n// 0: this ResultSet is not recieved finished\n//\t -1: system error\n//\t -2: bad message error\nint mysql_parser_parse(const void *buf, size_t len, mysql_parser_t *parser);\n\n#ifdef __cplusplus\n}\n#endif\n\nstatic inline void mysql_parser_set_command(int cmd, mysql_parser_t *parser)\n{\n\tparser->cmd = cmd;\n}\n\nstatic inline void mysql_parser_get_local_inline(const char **local_inline_name,\n\t\t\t\t\t\t\t\t\t\t\t\t size_t *local_inline_len,\n\t\t\t\t\t\t\t\t\t\t\t\t mysql_parser_t *parser)\n{\n\t*local_inline_name = (const char *)parser->buf + parser->local_inline_offset;\n\t*local_inline_len = parser->local_inline_length;\n}\n\nstatic inline void mysql_result_set_cursor_init(mysql_result_set_cursor_t *cursor,\n\t\t\t\t\t\t\t\t\t\t\t\tmysql_parser_t *parser)\n{\n\tcursor->head = &parser->result_set_list;\n\tcursor->current = cursor->head;\n}\n\nstatic inline void mysql_result_set_cursor_rewind(mysql_result_set_cursor_t *cursor)\n{\n\tcursor->current = cursor->head;\n}\n\nstatic inline void mysql_result_set_cursor_deinit(mysql_result_set_cursor_t *cursor)\n{\n}\n\nstatic inline int mysql_result_set_cursor_next(struct __mysql_result_set **result_set,\n\t\t\t\t\t\t\t\t\t\t\t mysql_result_set_cursor_t *cursor)\n{\n\tif (cursor->current->next != cursor->head)\n\t{\n\t\tcursor->current = cursor->current->next;\n\t\t*result_set = list_entry(cursor->current, struct __mysql_result_set, list);\n\t\treturn 0;\n\t}\n\treturn 1;\n}\n\n#endif\n" }, { "path": "src/protocol/mysql_stream.c", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"mysql_stream.h\"\n\n#define MAX(x, y)\t((x) >= (y) ? (x) : (y))\n\nstatic int __mysql_stream_write_payload(const void *buf, size_t *n,\n\t\t\t\t\t\t\t\t\t\tmysql_stream_t *stream);\n\nstatic int __mysql_stream_write_head(const void *buf, size_t *n,\n\t\t\t\t\t\t\t\t\t mysql_stream_t *stream)\n{\n\tvoid *p = &stream->head[4 - stream->head_left];\n\n\tif (*n < stream->head_left)\n\t{\n\t\tmemcpy(p, buf, *n);\n\t\tstream->head_left -= *n;\n\t\treturn 0;\n\t}\n\n\tmemcpy(p, buf, stream->head_left);\n\tstream->payload_length = (stream->head[2] << 16) +\n\t\t\t\t\t\t\t (stream->head[1] << 8) +\n\t\t\t\t\t\t\t stream->head[0];\n\tstream->payload_left = stream->payload_length;\n\tstream->sequence_id = stream->head[3];\n\tif (stream->bufsize < stream->length + stream->payload_left)\n\t{\n\t\tsize_t new_size = MAX(2048, 2 * stream->bufsize);\n\t\tvoid *new_base;\n\n\t\twhile (new_size < stream->length + stream->payload_left)\n\t\t\tnew_size *= 2;\n\n\t\tnew_base = realloc(stream->buf, new_size);\n\t\tif (!new_base)\n\t\t\treturn -1;\n\n\t\tstream->buf = new_base;\n\t\tstream->bufsize = new_size;\n\t}\n\n\t*n = stream->head_left;\n\tstream->write = __mysql_stream_write_payload;\n\treturn 0;\n}\n\nstatic int __mysql_stream_write_payload(const void *buf, size_t *n,\n\t\t\t\t\t\t\t\t\t\tmysql_stream_t *stream)\n{\n\tchar *p = (char *)stream->buf + stream->length;\n\n\tif (*n < stream->payload_left)\n\t{\n\t\tmemcpy(p, buf, *n);\n\t\tstream->length += *n;\n\t\tstream->payload_left -= *n;\n\t\treturn 0;\n\t}\n\n\tmemcpy(p, buf, stream->payload_left);\n\tstream->length += stream->payload_left;\n\n\t*n = stream->payload_left;\n\tstream->head_left = 4;\n\tstream->write = __mysql_stream_write_head;\n\treturn stream->payload_length != (1 << 24) - 1;\n}\n\nvoid mysql_stream_init(mysql_stream_t *stream)\n{\n\tstream->head_left = 4;\n\tstream->sequence_id = 0;\n\tstream->buf = NULL;\n\tstream->length = 0;\n\tstream->bufsize = 0;\n\tstream->write = __mysql_stream_write_head;\n}\n\n" }, { "path": "src/protocol/mysql_stream.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _MYSQL_STREAM_H_\n#define _MYSQL_STREAM_H_\n\n#include \n\ntypedef struct __mysql_stream\n{\n\tunsigned char head[4];\n\tunsigned char head_left;\n\tunsigned char sequence_id; \n\tint payload_length;\n\tint payload_left;\n\tvoid *buf;\n\tsize_t length;\n\tsize_t bufsize;\n\tint (*write)(const void *, size_t *, struct __mysql_stream *);\n} mysql_stream_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nvoid mysql_stream_init(mysql_stream_t *stream);\n\n#ifdef __cplusplus\n}\n#endif\n\nstatic inline int mysql_stream_write(const void *buf, size_t *n,\n\t\t\t\t\t\t\t\t\t mysql_stream_t *stream)\n{\n\treturn stream->write(buf, n, stream);\n}\n\nstatic inline int mysql_stream_get_seq(mysql_stream_t *stream)\n{\n\treturn stream->sequence_id;\n}\n\nstatic inline void mysql_stream_get_buf(const void **buf, size_t *length,\n\t\t\t\t\t\t\t\t\t\tmysql_stream_t *stream)\n{\n\t*buf = stream->buf;\n\t*length = stream->length;\n}\n\nstatic inline void mysql_stream_deinit(mysql_stream_t *stream)\n{\n\tfree(stream->buf);\n}\n\n#endif\n\n" }, { "path": "src/protocol/mysql_types.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Li Yingxin (liyingxin@sogou-inc.com)\n\t\t Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _MYSQL_TYPES_H_\n#define _MYSQL_TYPES_H_\n\n#define MYSQL_STATE_LENGTH\t\t\t5\n#define MYSQL_STATE_DEFAULT\t\t\t\"HY000\"\n\n#define MYSQL_SERVER_MORE_RESULTS_EXIST\t\t0x0008\n#define MYSQL_SERVER_SESSION_STATE_CHANGED\t0x4000\n\nenum\n{\n\tMYSQL_COM_SLEEP,\n\tMYSQL_COM_QUIT,\n\tMYSQL_COM_INIT_DB,\n\tMYSQL_COM_QUERY,\n\tMYSQL_COM_FIELD_LIST,\n\tMYSQL_COM_CREATE_DB,\n\tMYSQL_COM_DROP_DB,\n\tMYSQL_COM_REFRESH,\n\tMYSQL_COM_DEPRECATED_1,\n\tMYSQL_COM_STATISTICS,\n\tMYSQL_COM_PROCESS_INFO,\n\tMYSQL_COM_CONNECT,\n\tMYSQL_COM_PROCESS_KILL,\n\tMYSQL_COM_DEBUG,\n\tMYSQL_COM_PING,\n\tMYSQL_COM_TIME,\n\tMYSQL_COM_DELAYED_INSERT,\n\tMYSQL_COM_CHANGE_USER,\n\tMYSQL_COM_BINLOG_DUMP,\n\tMYSQL_COM_TABLE_DUMP,\n\tMYSQL_COM_CONNECT_OUT,\n\tMYSQL_COM_REGISTER_SLAVE,\n\tMYSQL_COM_STMT_PREPARE,\n\tMYSQL_COM_STMT_EXECUTE,\n\tMYSQL_COM_STMT_SEND_LONG_DATA,\n\tMYSQL_COM_STMT_CLOSE,\n\tMYSQL_COM_STMT_RESET,\n\tMYSQL_COM_SET_OPTION,\n\tMYSQL_COM_STMT_FETCH,\n\tMYSQL_COM_DAEMON,\n\tMYSQL_COM_BINLOG_DUMP_GTID,\n\tMYSQL_COM_RESET_CONNECTION,\n\tMYSQL_COM_CLONE,\n\tMYSQL_COM_END\n};\n\n// MySQL packet type\nenum\n{\n\tMYSQL_PACKET_OTHER\t=\t0,\n\tMYSQL_PACKET_OK,\n\tMYSQL_PACKET_NULL,\n\tMYSQL_PACKET_EOF,\n\tMYSQL_PACKET_ERROR,\n\tMYSQL_PACKET_GET_RESULT,\n\tMYSQL_PACKET_LOCAL_INLINE,\n};\n\n// MySQL cursor status\nenum\n{\n\tMYSQL_STATUS_NOT_INIT\t=\t0,\n\tMYSQL_STATUS_OK,\n\tMYSQL_STATUS_GET_RESULT,\n\tMYSQL_STATUS_ERROR,\n\tMYSQL_STATUS_END,\n};\n\n// Column types for MySQL\nenum\n{\n\tMYSQL_TYPE_DECIMAL\t=\t0,\n\tMYSQL_TYPE_TINY,\n\tMYSQL_TYPE_SHORT,\n\tMYSQL_TYPE_LONG,\n\tMYSQL_TYPE_FLOAT,\n\tMYSQL_TYPE_DOUBLE,\n\tMYSQL_TYPE_NULL,\n\tMYSQL_TYPE_TIMESTAMP,\n\tMYSQL_TYPE_LONGLONG,\n\tMYSQL_TYPE_INT24,\n\tMYSQL_TYPE_DATE,\n\tMYSQL_TYPE_TIME,\n\tMYSQL_TYPE_DATETIME,\n\tMYSQL_TYPE_YEAR,\n\tMYSQL_TYPE_NEWDATE,\t\t\t\t// Internal to MySQL. Not used in protocol\n\tMYSQL_TYPE_VARCHAR,\n\tMYSQL_TYPE_BIT,\n\tMYSQL_TYPE_TIMESTAMP2,\n\tMYSQL_TYPE_DATETIME2,\t\t\t// Internal to MySQL. Not used in protocol\n\tMYSQL_TYPE_TIME2,\t\t\t\t// Internal to MySQL. Not used in protocol\n\tMYSQL_TYPE_TYPED_ARRAY = 244,\t// Used for replication only\n\tMYSQL_TYPE_JSON = 245,\n\tMYSQL_TYPE_NEWDECIMAL = 246,\n\tMYSQL_TYPE_ENUM = 247,\n\tMYSQL_TYPE_SET = 248,\n\tMYSQL_TYPE_TINY_BLOB = 249,\n\tMYSQL_TYPE_MEDIUM_BLOB = 250,\n\tMYSQL_TYPE_LONG_BLOB = 251,\n\tMYSQL_TYPE_BLOB = 252,\n\tMYSQL_TYPE_VAR_STRING = 253,\n\tMYSQL_TYPE_STRING = 254,\n\tMYSQL_TYPE_GEOMETRY = 255\n};\n\nstatic inline const char *datatype2str(int data_type)\n{\n\tswitch (data_type)\n\t{\n\tcase MYSQL_TYPE_BIT:\n\t\treturn \"BIT\";\n\n\tcase MYSQL_TYPE_BLOB:\n\t\treturn \"BLOB\";\n\n\tcase MYSQL_TYPE_DATE:\n\t\treturn \"DATE\";\n\n\tcase MYSQL_TYPE_DATETIME:\n\t\treturn \"DATETIME\";\n\n\tcase MYSQL_TYPE_NEWDECIMAL:\n\t\treturn \"NEWDECIMAL\";\n\n\tcase MYSQL_TYPE_DECIMAL:\n\t\treturn \"DECIMAL\";\n\n\tcase MYSQL_TYPE_DOUBLE:\n\t\treturn \"DOUBLE\";\n\n\tcase MYSQL_TYPE_ENUM:\n\t\treturn \"ENUM\";\n\n\tcase MYSQL_TYPE_FLOAT:\n\t\treturn \"FLOAT\";\n\n\tcase MYSQL_TYPE_GEOMETRY:\n\t\treturn \"GEOMETRY\";\n\n\tcase MYSQL_TYPE_INT24:\n\t\treturn \"INT24\";\n\n\tcase MYSQL_TYPE_JSON:\n\t\treturn \"JSON\";\n\n\tcase MYSQL_TYPE_LONG:\n\t\treturn \"LONG\";\n\n\tcase MYSQL_TYPE_LONGLONG:\n\t\treturn \"LONGLONG\";\n\n\tcase MYSQL_TYPE_LONG_BLOB:\n\t\treturn \"LONG_BLOB\";\n\n\tcase MYSQL_TYPE_MEDIUM_BLOB:\n\t\treturn \"MEDIUM_BLOB\";\n\n\tcase MYSQL_TYPE_NEWDATE:\n\t\treturn \"NEWDATE\";\n\n\tcase MYSQL_TYPE_NULL:\n\t\treturn \"NULL\";\n\n\tcase MYSQL_TYPE_SET:\n\t\treturn \"SET\";\n\n\tcase MYSQL_TYPE_SHORT:\n\t\treturn \"SHORT\";\n\n\tcase MYSQL_TYPE_STRING:\n\t\treturn \"STRING\";\n\n\tcase MYSQL_TYPE_TIME:\n\t\treturn \"TIME\";\n\n\tcase MYSQL_TYPE_TIMESTAMP:\n\t\treturn \"TIMESTAMP\";\n\n\tcase MYSQL_TYPE_TINY:\n\t\treturn \"TINY\";\n\n\tcase MYSQL_TYPE_TINY_BLOB:\n\t\treturn \"TINY_BLOB\";\n\n\tcase MYSQL_TYPE_VAR_STRING:\n\t\treturn \"VAR_STRING\";\n\n\tcase MYSQL_TYPE_YEAR:\n\t\treturn \"YEAR\";\n\n\tdefault:\n\t\treturn \"?-unknown-?\";\n\n\t}\n}\n\n#endif\n\n" }, { "path": "src/protocol/redis_parser.c", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"list.h\"\n#include \"redis_parser.h\"\n\n#define MIN(x, y)\t((x) <= (y) ? (x) : (y))\n#define MAX(x, y)\t((x) >= (y) ? (x) : (y))\n\n#define REDIS_MSGBUF_INIT_SIZE\t\t8\n#define REDIS_REPLY_DEPTH_LIMIT\t\t64\n#define REDIS_ARRAY_SIZE_LIMIT\t\t(4 * 1024 * 1024)\n\nenum\n{\n\t//REDIS_PARSE_INIT = 0,\n\tREDIS_GET_CMD = 1,\n\tREDIS_GET_CR,\n\tREDIS_GET_LF,\n\tREDIS_UNTIL_CRLF,\n\tREDIS_GET_NCHAR,\n\tREDIS_PARSE_END\n};\n\nstruct __redis_read_record\n{\n\tstruct list_head list;\n\tredis_reply_t *reply;\n};\n\nvoid redis_reply_deinit(redis_reply_t *reply)\n{\n\tsize_t i;\n\n\tfor (i = 0; i < reply->elements; i++)\n\t{\n\t\tredis_reply_deinit(reply->element[i]);\n\t\tfree(reply->element[i]);\n\t}\n\n\tfree(reply->element);\n}\n\nstatic redis_reply_t **__redis_create_array(size_t size, redis_reply_t *reply)\n{\n\tsize_t elements = 0;\n\tredis_reply_t **element = (redis_reply_t **)malloc(size * sizeof (void *));\n\n\tif (element)\n\t{\n\t\tsize_t i;\n\n\t\tfor (i = 0; i < size; i++)\n\t\t{\n\t\t\telement[i] = (redis_reply_t *)malloc(sizeof (redis_reply_t));\n\t\t\tif (element[i])\n\t\t\t{\n\t\t\t\tredis_reply_init(element[i]);\n\t\t\t\telements++;\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tbreak;\n\t\t}\n\n\t\tif (elements == size)\n\t\t\treturn element;\n\n\t\twhile (elements > 0)\n\t\t\tfree(element[--elements]);\n\n\t\tfree(element);\n\t}\n\n\treturn NULL;\n}\n\nint redis_reply_set_array(size_t size, redis_reply_t *reply)\n{\n\tredis_reply_t **element = __redis_create_array(size, reply);\n\n\tif (element == NULL)\n\t\treturn -1;\n\n\tredis_reply_deinit(reply);\n\treply->element = element;\n\treply->elements = size;\n\treply->type = REDIS_REPLY_TYPE_ARRAY;\n\treturn 0;\n}\n\nstatic int __redis_parse_cmd(const char ch, redis_parser_t *parser)\n{\n\tswitch (ch)\n\t{\n\tcase '+':\n\tcase '-':\n\tcase ':':\n\tcase '$':\n\tcase '*':\n\t\tparser->cmd = ch;\n\t\tparser->status = REDIS_UNTIL_CRLF;\n\t\tparser->findidx = parser->msgidx;\n\t\treturn 0;\n\t}\n\n\treturn -2;\n}\n\nstatic int __redis_parse_cr(const char ch, redis_parser_t *parser)\n{\n\tif (ch != '\\r')\n\t\treturn -2;\n\n\tparser->status = REDIS_GET_LF;\n\treturn 0;\n}\n\nstatic int __redis_parse_lf(const char ch, redis_parser_t *parser)\n{\n\tif (ch != '\\n')\n\t\treturn -2;\n\n\treturn 1;\n}\n\nstatic int __redis_parse_line(redis_parser_t *parser)\n{\n\tchar *str = parser->msgbuf + parser->msgidx;\n\tsize_t slen = parser->findidx - parser->msgidx;\n\tchar data[32];\n\tint i, n;\n\tconst char *offset = (const char *)parser->msgidx;\n\tstruct __redis_read_record *node;\n\n\tparser->msgidx = parser->findidx + 2;\n\tswitch (parser->cmd)\n\t{\n\tcase '+':\n\t\tredis_reply_set_status(offset, slen, parser->cur);\n\t\treturn 1;\n\n\tcase '-':\n\t\tredis_reply_set_error(offset, slen, parser->cur);\n\t\treturn 1;\n\n\tcase ':':\n\t\tif (slen == 0 || slen > 30)\n\t\t\treturn -2;\n\n\t\tmemcpy(data, str, slen);\n\t\tdata[slen] = '\\0';\n\t\tredis_reply_set_integer(atoll(data), parser->cur);\n\t\treturn 1;\n\n\tcase '$':\n\t\tn = atoi(str);\n\t\tif (n < 0)\n\t\t{\n\t\t\tredis_reply_set_null(parser->cur);\n\t\t\treturn 1;\n\t\t}\n\t\telse if (n == 0)\n\t\t{\n\t\t\t/* \"-0\" not acceptable. */\n\t\t\tif (!isdigit(*str))\n\t\t\t\treturn -2;\n\n\t\t\tredis_reply_set_string(offset, 0, parser->cur);\n\t\t\tparser->status = REDIS_GET_CR;\n\t\t\treturn 0;\n\t\t}\n\n\t\tparser->nchar = n;\n\t\tparser->status = REDIS_GET_NCHAR;\n\t\treturn 0;\n\n\tcase '*':\n\t\tn = atoi(str);\n\t\tif (n < 0)\n\t\t{\n\t\t\tredis_reply_set_null(parser->cur);\n\t\t\treturn 1;\n\t\t}\n\n\t\tif (n == 0 && !isdigit(*str))\n\t\t\treturn -2;\n\n\t\tif (n > REDIS_ARRAY_SIZE_LIMIT)\n\t\t\treturn -2;\n\n\t\tparser->nleft += n;\n\t\tif (redis_reply_set_array(n, parser->cur) < 0)\n\t\t\treturn -1;\n\n\t\tif (n == 0)\n\t\t\treturn 1;\n\n\t\tparser->nleft--;\n\t\tfor (i = 0; i < n - 1; i++)\n\t\t{\n\t\t\tnode = (struct __redis_read_record *)malloc(sizeof *node);\n\t\t\tif (!node)\n\t\t\t\treturn -1;\n\n\t\t\tnode->reply = parser->cur->element[n - 1 - i];\n\t\t\tlist_add(&node->list, &parser->read_list);\n\t\t}\n\n\t\tparser->cur = parser->cur->element[0];\n\t\tparser->status = REDIS_GET_CMD;\n\t\treturn 0;\n\n\t}\n\n\treturn -2;\n}\n\nstatic int __redis_parse_crlf(redis_parser_t *parser)\n{\n\tchar *buf = parser->msgbuf;\n\n\tfor (; parser->findidx + 1 < parser->msgsize; parser->findidx++)\n\t{\n\t\tif (buf[parser->findidx] == '\\r' && buf[parser->findidx + 1] == '\\n')\n\t\t\treturn __redis_parse_line(parser);\n\t}\n\n\treturn 2;\n}\n\nstatic int __redis_parse_nchar(redis_parser_t *parser)\n{\n\tif (parser->nchar <= parser->msgsize - parser->msgidx)\n\t{\n\t\tredis_reply_set_string((const char *)parser->msgidx, parser->nchar,\n\t\t\t\t\t\t\t parser->cur);\n\n\t\tparser->msgidx += parser->nchar;\n\t\tparser->status = REDIS_GET_CR;\n\t\treturn 0;\n\t}\n\n\treturn 2;\n}\n\n//-1 error | 0 continue | 1 finish-one | 2 not-enough\nstatic int __redis_parser_forward(redis_parser_t *parser)\n{\n\tchar *buf = parser->msgbuf;\n\n\tif (parser->msgidx >= parser->msgsize)\n\t\treturn 2;\n\n\tswitch (parser->status)\n\t{\n\tcase REDIS_GET_CMD:\n\t\treturn __redis_parse_cmd(buf[parser->msgidx++], parser);\n\n\tcase REDIS_GET_CR:\n\t\treturn __redis_parse_cr(buf[parser->msgidx++], parser);\n\n\tcase REDIS_GET_LF:\n\t\treturn __redis_parse_lf(buf[parser->msgidx++], parser);\n\n\tcase REDIS_UNTIL_CRLF:\n\t\treturn __redis_parse_crlf(parser);\n\n\tcase REDIS_GET_NCHAR:\n\t\treturn __redis_parse_nchar(parser);\n\t}\n\n\treturn -2;\n}\n\nvoid redis_parser_init(redis_parser_t *parser)\n{\n\tredis_reply_init(&parser->reply);\n\tparser->parse_succ = 0;\n\tparser->msgbuf = NULL;\n\tparser->msgsize = 0;\n\tparser->bufsize = 0;\n\t//parser->status = REDIS_PARSE_INIT;\n\t//parser->nleft = 0;\n\tparser->status = REDIS_GET_CMD;\n\tparser->nleft = 1;\n\tparser->cur = &parser->reply;\n\tINIT_LIST_HEAD(&parser->read_list);\n\tparser->msgidx = 0;\n\tparser->cmd = '\\0';\n\tparser->nchar = 0;\n\tparser->findidx = 0;\n}\n\nvoid redis_parser_deinit(redis_parser_t *parser)\n{\n\tstruct list_head *pos, *tmp;\n\tstruct __redis_read_record *next;\n\n\tlist_for_each_safe(pos, tmp, &parser->read_list)\n\t{\n\t\tnext = list_entry(pos, struct __redis_read_record, list);\n\t\tlist_del(pos);\n\t\tfree(next);\n\t}\n\n\tredis_reply_deinit(&parser->reply);\n\tfree(parser->msgbuf);\n}\n\nstatic int __redis_parse_done(redis_reply_t *reply, char *buf, int depth)\n{\n\tsize_t i;\n\n\tif (depth == REDIS_REPLY_DEPTH_LIMIT)\n\t\treturn -2;\n\n\tswitch (reply->type)\n\t{\n\tcase REDIS_REPLY_TYPE_INTEGER:\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_ARRAY:\n\t\tfor (i = 0; i < reply->elements; i++)\n\t\t{\n\t\t\tif (__redis_parse_done(reply->element[i], buf, depth + 1) < 0)\n\t\t\t\treturn -2;\n\t\t}\n\n\t\tbreak;\n\n\tcase REDIS_REPLY_TYPE_STATUS:\n\tcase REDIS_REPLY_TYPE_ERROR:\n\tcase REDIS_REPLY_TYPE_STRING:\n\t\treply->str = buf + (size_t)reply->str;\n\t\tbreak;\n\t}\n\n\treturn 1;\n}\n\nstatic int __redis_split_inline_command(redis_parser_t *parser)\n{\n\tchar *msg = parser->msgbuf;\n\tchar *end = msg + parser->msgsize;\n\tsize_t arr_size = 0;\n\tredis_reply_t **ele;\n\tchar *cur;\n\tint ret;\n\n\twhile (msg != end)\n\t{\n\t\twhile (msg != end && isspace(*msg))\n\t\t\tmsg++;\n\n\t\tif (msg == end)\n\t\t\tbreak;\n\n\t\tarr_size++;\n\n\t\twhile (msg != end && !isspace(*msg))\n\t\t\tmsg++;\n\t}\n\n\tif (arr_size == 0)\n\t{\n\t\tparser->msgsize = 0;\n\t\tparser->msgidx = 0;\n\t\treturn 0;\n\t}\n\n\tret = redis_reply_set_array(arr_size, &parser->reply);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tele = parser->reply.element;\n\tmsg = parser->msgbuf;\n\n\twhile (msg != end)\n\t{\n\t\twhile (msg != end && isspace(*msg))\n\t\t\tmsg++;\n\n\t\tif (msg == end)\n\t\t\tbreak;\n\n\t\tcur = msg;\n\t\twhile (cur != end && !isspace(*cur))\n\t\t\tcur++;\n\n\t\tredis_reply_set_string(msg, cur - msg, *ele);\n\n\t\tmsg = cur;\n\t\tele++;\n\t}\n\n\tparser->status = REDIS_PARSE_END;\n\treturn 1;\n}\n\nint redis_parser_append_message(const void *buf, size_t *size,\n\t\t\t\t\t\t\t\tredis_parser_t *parser)\n{\n\tsize_t msgsize_bak = parser->msgsize;\n\n\tif (parser->status == REDIS_PARSE_END)\n\t{\n\t\t*size = 0;\n\t\treturn 1;\n\t}\n\n\tif (parser->msgsize + *size > parser->bufsize)\n\t{\n\t\tsize_t new_size = MAX(REDIS_MSGBUF_INIT_SIZE, 2 * parser->bufsize);\n\t\tvoid *new_base;\n\n\t\twhile (new_size < parser->msgsize + *size)\n\t\t\tnew_size *= 2;\n\n\t\tnew_base = realloc(parser->msgbuf, new_size);\n\t\tif (!new_base)\n\t\t\treturn -1;\n\n\t\tparser->msgbuf = (char *)new_base;\n\t\tparser->bufsize = new_size;\n\t}\n\n\tmemcpy(parser->msgbuf + parser->msgsize, buf, *size);\n\tparser->msgsize += *size;\n\tif (parser->msgsize > 0 && (isalpha(*parser->msgbuf) ||\n\t\t\t\t\t\t\t\tisspace(*parser->msgbuf)))\n\t{\n\t\twhile (parser->msgidx < parser->msgsize &&\n\t\t\t*(parser->msgbuf + parser->msgidx) != '\\n')\n\t\t{\n\t\t\tparser->msgidx++;\n\t\t}\n\n\t\tif (parser->msgidx == parser->msgsize)\n\t\t\treturn 0;\n\n\t\tparser->msgidx++;\n\t\tparser->msgsize = parser->msgidx;\n\t\t*size = parser->msgsize - msgsize_bak;\n\n\t\treturn __redis_split_inline_command(parser);\n\t}\n\n\tdo\n\t{\n\t\tint ret = __redis_parser_forward(parser);\n\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tif (ret == 1)\n\t\t{\n\t\t\tstruct list_head *lnext = parser->read_list.next;\n\t\t\tstruct __redis_read_record *next;\n\n\t\t\tparser->nleft--;\n\t\t\tif (lnext && lnext != &parser->read_list)\n\t\t\t{\n\t\t\t\tnext = list_entry(lnext, struct __redis_read_record, list);\n\t\t\t\tparser->cur = next->reply;\n\t\t\t\tlist_del(lnext);\n\t\t\t\tfree(next);\n\t\t\t}\n\n\t\t\tif (parser->nleft > 0)\n\t\t\t\tparser->status = REDIS_GET_CMD;\n\t\t\telse\n\t\t\t{\n\t\t\t\tparser->parse_succ = 1;\n\t\t\t\tparser->status = REDIS_PARSE_END;\n\t\t\t}\n\t\t}\n\t\telse if (ret == 2)\n\t\t\treturn 0;\n\n\t} while (parser->status != REDIS_PARSE_END);\n\n\t*size = parser->msgidx - msgsize_bak;\n\treturn __redis_parse_done(&parser->reply, parser->msgbuf, 0);\n}\n\n" }, { "path": "src/protocol/redis_parser.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _REDIS_PARSER_H_\n#define _REDIS_PARSER_H_\n\n#include \n#include \"list.h\"\n\n// redis_parser_t is absolutely same as hiredis-redisReply in memory\n// If you include hiredis.h, redisReply* can cast to redis_reply_t* safely\n\n#define REDIS_REPLY_TYPE_STRING 1\n#define REDIS_REPLY_TYPE_ARRAY 2\n#define REDIS_REPLY_TYPE_INTEGER 3\n#define REDIS_REPLY_TYPE_NIL 4\n#define REDIS_REPLY_TYPE_STATUS 5\n#define REDIS_REPLY_TYPE_ERROR 6\n\ntypedef struct __redis_reply {\n\tint type; /* REDIS_REPLY_TYPE_* */\n\tlong long integer; /* The integer when type is REDIS_REPLY_TYPE_INTEGER */\n\tsize_t len; /* Length of string */\n\tchar *str; /* Used for both REDIS_REPLY_TYPE_ERROR and REDIS_REPLY_TYPE_STRING */\n\tsize_t elements; /* number of elements, for REDIS_REPLY_TYPE_ARRAY */\n\tstruct __redis_reply **element; /* elements vector for REDIS_REPLY_TYPE_ARRAY */\n} redis_reply_t;\n\ntypedef struct __redis_parser\n{\n\tint parse_succ;//check first\n\tint status;\n\tchar *msgbuf;\n\tsize_t msgsize;\n\tsize_t bufsize;\n\tredis_reply_t *cur;\n\tstruct list_head read_list;\n\tsize_t msgidx;\n\tsize_t findidx;\n\tint nleft;\n\tint nchar;\n\tchar cmd;\n\tredis_reply_t reply;\n} redis_parser_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\nvoid redis_parser_init(redis_parser_t *parser);\nvoid redis_parser_deinit(redis_parser_t *parser);\nint redis_parser_append_message(const void *buf, size_t *size,\n\t\t\t\t\t\t\t\tredis_parser_t *parser);\n\nvoid redis_reply_deinit(redis_reply_t *reply);\n\nint redis_reply_set_array(size_t size, redis_reply_t *reply);\n\n#ifdef __cplusplus\n}\n#endif\n\nstatic inline void redis_reply_init(redis_reply_t *reply)\n{\n\treply->type = REDIS_REPLY_TYPE_NIL;\n\treply->integer = 0;\n\treply->len = 0;\n\treply->str = NULL;\n\treply->elements = 0;\n\treply->element = NULL;\n}\n\nstatic inline void redis_reply_set_string(const char *str, size_t len,\n\t\t\t\t\t\t\t\t\t\t redis_reply_t *reply)\n{\n\treply->type = REDIS_REPLY_TYPE_STRING;\n\treply->len = len;\n\treply->str = (char *)str;\n}\n\nstatic inline void redis_reply_set_integer(long long intv, redis_reply_t *reply)\n{\n\treply->type = REDIS_REPLY_TYPE_INTEGER;\n\treply->integer = intv;\n}\n\nstatic inline void redis_reply_set_null(redis_reply_t *reply)\n{\n\treply->type = REDIS_REPLY_TYPE_NIL;\n}\n\nstatic inline void redis_reply_set_error(const char *err, size_t len,\n\t\t\t\t\t\t\t\t\t\t redis_reply_t *reply)\n{\n\treply->type = REDIS_REPLY_TYPE_ERROR;\n\treply->len = len;\n\treply->str = (char *)err;\n}\n\nstatic inline void redis_reply_set_status(const char *str, size_t len,\n\t\t\t\t\t\t\t\t\t\t redis_reply_t *reply)\n{\n\treply->type = REDIS_REPLY_TYPE_STATUS;\n\treply->len = len;\n\treply->str = (char *)str;\n}\n\n#endif\n\n" }, { "path": "src/protocol/xmake.lua", "content": "target(\"basic_protocol\")\n set_kind(\"object\")\n add_files(\"PackageWrapper.cc\",\n \"SSLWrapper.cc\",\n \"dns_parser.c\",\n \"DnsMessage.cc\",\n \"DnsUtil.cc\",\n \"http_parser.c\",\n \"HttpMessage.cc\",\n \"HttpUtil.cc\")\n\ntarget(\"mysql_protocol\")\n if has_config(\"mysql\") then\n add_files(\"mysql_stream.c\",\n \"mysql_parser.c\",\n \"mysql_byteorder.c\",\n \"MySQLMessage.cc\",\n \"MySQLResult.cc\",\n \"MySQLUtil.cc\")\n set_kind(\"object\")\n add_deps(\"basic_protocol\")\n else\n set_kind(\"phony\")\n end\n\ntarget(\"redis_protocol\")\n if has_config(\"redis\") then\n add_files(\"redis_parser.c\", \"RedisMessage.cc\")\n set_kind(\"object\")\n add_deps(\"basic_protocol\")\n else\n set_kind(\"phony\")\n end\n\ntarget(\"protocol\")\n set_kind(\"object\")\n add_deps(\"basic_protocol\", \"mysql_protocol\", \"redis_protocol\")\n\ntarget(\"kafka_protocol\")\n if has_config(\"kafka\") then\n set_kind(\"object\")\n add_files(\"kafka_parser.c\",\n \"KafkaDataTypes.cc\",\n \"KafkaMessage.cc\",\n \"KafkaResult.cc\")\n add_deps(\"basic_protocol\")\n add_packages(\"zlib\", \"snappy\", \"zstd\", \"lz4\")\n else\n set_kind(\"phony\")\n end\n" }, { "path": "src/server/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\nproject(server)\n\nset(SRC\n\tWFServer.cc\n)\n\nif (NOT MYSQL STREQUAL \"n\")\n\tset(SRC\n\t\t${SRC}\n\t\tWFMySQLServer.cc\n\t)\nendif ()\n\nadd_library(${PROJECT_NAME} OBJECT ${SRC})\n" }, { "path": "src/server/WFDnsServer.h", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#ifndef _WFDNSSERVER_H_\n#define _WFDNSSERVER_H_\n\n#include \"dns_types.h\"\n#include \"DnsMessage.h\"\n#include \"WFServer.h\"\n#include \"WFTaskFactory.h\"\n\nusing dns_process_t = std::function;\nusing WFDnsServer = WFServer;\n\nstatic constexpr struct WFServerParams DNS_SERVER_PARAMS_DEFAULT =\n{\n\t.transport_type\t\t\t=\tTT_UDP,\n\t.max_connections\t\t=\t2000,\n\t.peer_response_timeout\t=\t10 * 1000,\n\t.receive_timeout\t\t=\t-1,\n\t.keep_alive_timeout\t\t=\t300 * 1000,\n\t.request_size_limit\t\t=\t(size_t)-1,\n\t.ssl_accept_timeout\t\t=\t5000,\n};\n\ntemplate<> inline\nWFDnsServer::WFServer(dns_process_t proc) :\n\tWFServerBase(&DNS_SERVER_PARAMS_DEFAULT),\n\tprocess(std::move(proc))\n{\n}\n\ntemplate<> inline\nCommSession *WFDnsServer::new_session(long long seq, CommConnection *conn)\n{\n\tWFDnsTask *task;\n\n\ttask = WFServerTaskFactory::create_dns_task(this, this->process);\n\ttask->set_keep_alive(this->params.keep_alive_timeout);\n\ttask->set_receive_timeout(this->params.receive_timeout);\n\ttask->get_req()->set_size_limit(this->params.request_size_limit);\n\n\treturn task;\n}\n\n#endif\n\n" }, { "path": "src/server/WFHttpServer.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFHTTPSERVER_H_\n#define _WFHTTPSERVER_H_\n\n#include \n#include \"HttpMessage.h\"\n#include \"WFServer.h\"\n#include \"WFTaskFactory.h\"\n\nusing http_process_t = std::function;\nusing WFHttpServer = WFServer;\n\nstatic constexpr struct WFServerParams HTTP_SERVER_PARAMS_DEFAULT =\n{\n\t.transport_type\t\t\t=\tTT_TCP,\n\t.max_connections\t\t=\t2000,\n\t.peer_response_timeout\t=\t10 * 1000,\n\t.receive_timeout\t\t=\t-1,\n\t.keep_alive_timeout\t\t=\t60 * 1000,\n\t.request_size_limit\t\t=\t(size_t)-1,\n\t.ssl_accept_timeout\t\t=\t10 * 1000,\n};\n\ntemplate<> inline\nWFHttpServer::WFServer(http_process_t proc) :\n\tWFServerBase(&HTTP_SERVER_PARAMS_DEFAULT),\n\tprocess(std::move(proc))\n{\n}\n\ntemplate<> inline\nCommSession *WFHttpServer::new_session(long long seq, CommConnection *conn)\n{\n\tWFHttpTask *task;\n\n\ttask = WFServerTaskFactory::create_http_task(this, this->process);\n\ttask->set_keep_alive(this->params.keep_alive_timeout);\n\ttask->set_receive_timeout(this->params.receive_timeout);\n\ttask->get_req()->set_size_limit(this->params.request_size_limit);\n\n\treturn task;\n}\n\n#endif\n\n" }, { "path": "src/server/WFMySQLServer.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \"WFMySQLServer.h\"\n\nWFConnection *WFMySQLServer::new_connection(int accept_fd)\n{\n\tWFConnection *conn = this->WFServer::new_connection(accept_fd);\n\n\tif (conn)\n\t{\n\t\tprotocol::MySQLHandshakeResponse resp;\n\t\tstruct iovec vec[8];\n\t\tint count;\n\n\t\tresp.server_set(0x0a, \"5.5\", 1, (const uint8_t *)\"12345678901234567890\",\n\t\t\t\t\t\t0, 33, 0);\n\t\tcount = resp.encode(vec, 8);\n\t\tif (count >= 0)\n\t\t{\n\t\t\tif (writev(accept_fd, vec, count) >= 0)\n\t\t\t\treturn conn;\n\t\t}\n\n\t\tthis->delete_connection(conn);\n\t}\n\n\treturn NULL;\n}\n\nCommSession *WFMySQLServer::new_session(long long seq, CommConnection *conn)\n{\n\tstatic mysql_process_t empty = [](WFMySQLTask *){ };\n\tWFMySQLTask *task;\n\n\ttask = WFServerTaskFactory::create_mysql_task(this, seq ? this->process :\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t empty);\n\ttask->set_keep_alive(this->params.keep_alive_timeout);\n\ttask->set_receive_timeout(this->params.receive_timeout);\n\ttask->get_req()->set_size_limit(this->params.request_size_limit);\n\n\treturn task;\n}\n\n" }, { "path": "src/server/WFMySQLServer.h", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _WFMYSQLSERVER_H_\n#define _WFMYSQLSERVER_H_\n\n#include \n#include \"MySQLMessage.h\"\n#include \"WFServer.h\"\n#include \"WFTaskFactory.h\"\n#include \"WFConnection.h\"\n\nusing mysql_process_t = std::function;\nclass MySQLServer;\n\nstatic constexpr struct WFServerParams MYSQL_SERVER_PARAMS_DEFAULT =\n{\n\t.transport_type\t\t\t=\tTT_TCP,\n\t.max_connections\t\t=\t2000,\n\t.peer_response_timeout\t=\t10 * 1000,\n\t.receive_timeout\t\t=\t-1,\n\t.keep_alive_timeout\t\t=\t28800 * 1000,\n\t.request_size_limit\t\t=\t(size_t)-1,\n\t.ssl_accept_timeout\t\t=\t10 * 1000,\n};\n\nclass WFMySQLServer : public WFServer\n{\npublic:\n\tWFMySQLServer(mysql_process_t proc):\n\t\tWFServer(&MYSQL_SERVER_PARAMS_DEFAULT, std::move(proc))\n\t{\n\t}\n\nprotected:\n\tvirtual WFConnection *new_connection(int accept_fd);\n\tvirtual CommSession *new_session(long long seq, CommConnection *conn);\n};\n\n#endif\n\n" }, { "path": "src/server/WFRedisServer.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _WFREDISSERVER_H_\n#define _WFREDISSERVER_H_\n\n#include \"RedisMessage.h\"\n#include \"WFServer.h\"\n#include \"WFTaskFactory.h\"\n\nusing redis_process_t = std::function;\nusing WFRedisServer = WFServer;\n\nstatic constexpr struct WFServerParams REDIS_SERVER_PARAMS_DEFAULT =\n{\n\t.transport_type\t\t\t=\tTT_TCP,\n\t.max_connections\t\t=\t2000,\n\t.peer_response_timeout\t=\t10 * 1000,\n\t.receive_timeout\t\t=\t-1,\n\t.keep_alive_timeout\t\t=\t300 * 1000,\n\t.request_size_limit\t\t=\t(size_t)-1,\n\t.ssl_accept_timeout\t\t=\t5000,\n};\n\ntemplate<> inline\nWFRedisServer::WFServer(redis_process_t proc) :\n\tWFServerBase(&REDIS_SERVER_PARAMS_DEFAULT),\n\tprocess(std::move(proc))\n{\n}\n\n#endif\n\n" }, { "path": "src/server/WFServer.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"CommScheduler.h\"\n#include \"EndpointParams.h\"\n#include \"WFConnection.h\"\n#include \"WFGlobal.h\"\n#include \"WFServer.h\"\n\n#define PORT_STR_MAX\t5\n\nclass WFServerConnection : public WFConnection\n{\npublic:\n\tWFServerConnection(std::atomic *conn_count)\n\t{\n\t\tthis->conn_count = conn_count;\n\t}\n\n\tvirtual ~WFServerConnection()\n\t{\n\t\t(*this->conn_count)--;\n\t}\n\nprivate:\n\tstd::atomic *conn_count;\n};\n\nint WFServerBase::ssl_ctx_callback(SSL *ssl, int *al, void *arg)\n{\n\tWFServerBase *server = (WFServerBase *)arg;\n\tconst char *servername = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);\n\tSSL_CTX *ssl_ctx = server->get_server_ssl_ctx(servername);\n\n\tif (!ssl_ctx)\n\t\treturn SSL_TLSEXT_ERR_NOACK;\n\n\tif (ssl_ctx != server->get_ssl_ctx())\n\t\tSSL_set_SSL_CTX(ssl, ssl_ctx);\n\n\treturn SSL_TLSEXT_ERR_OK;\n}\n\nSSL_CTX *WFServerBase::new_ssl_ctx(const char *cert_file, const char *key_file)\n{\n\tSSL_CTX *ssl_ctx = WFGlobal::new_ssl_server_ctx();\n\n\tif (!ssl_ctx)\n\t\treturn NULL;\n\n\tif (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file) > 0 &&\n\t\tSSL_CTX_use_PrivateKey_file(ssl_ctx, key_file, SSL_FILETYPE_PEM) > 0 &&\n\t\tSSL_CTX_check_private_key(ssl_ctx) > 0 &&\n\t\tSSL_CTX_set_tlsext_servername_callback(ssl_ctx, ssl_ctx_callback) > 0 &&\n\t\tSSL_CTX_set_tlsext_servername_arg(ssl_ctx, this) > 0)\n\t{\n\t\treturn ssl_ctx;\n\t}\n\n\tSSL_CTX_free(ssl_ctx);\n\treturn NULL;\n}\n\nint WFServerBase::init(const struct sockaddr *bind_addr, socklen_t addrlen,\n\t\t\t\t\t const char *cert_file, const char *key_file)\n{\n\tint timeout = this->params.peer_response_timeout;\n\n\tif (this->params.receive_timeout >= 0)\n\t{\n\t\tif ((unsigned int)timeout > (unsigned int)this->params.receive_timeout)\n\t\t\ttimeout = this->params.receive_timeout;\n\t}\n\n\tif (this->params.transport_type == TT_TCP_SSL ||\n\t\tthis->params.transport_type == TT_SCTP_SSL)\n\t{\n\t\tif (!cert_file || !key_file)\n\t\t{\n\t\t\terrno = EINVAL;\n\t\t\treturn -1;\n\t\t}\n\t}\n\n\tif (this->CommService::init(bind_addr, addrlen, -1, timeout) < 0)\n\t\treturn -1;\n\n\tif (cert_file && key_file && this->params.transport_type != TT_UDP)\n\t{\n\t\tSSL_CTX *ssl_ctx = this->new_ssl_ctx(cert_file, key_file);\n\n\t\tif (!ssl_ctx)\n\t\t{\n\t\t\tthis->deinit();\n\t\t\treturn -1;\n\t\t}\n\n\t\tthis->set_ssl(ssl_ctx, this->params.ssl_accept_timeout);\n\t}\n\n\tthis->scheduler = WFGlobal::get_scheduler();\n\treturn 0;\n}\n\nint WFServerBase::create_listen_fd()\n{\n\tif (this->listen_fd < 0)\n\t{\n\t\tconst struct sockaddr *bind_addr;\n\t\tsocklen_t addrlen;\n\t\tint type, protocol;\n\t\tint reuse = 1;\n\n\t\tswitch (this->params.transport_type)\n\t\t{\n\t\tcase TT_TCP:\n\t\tcase TT_TCP_SSL:\n\t\t\ttype = SOCK_STREAM;\n\t\t\tprotocol = 0;\n\t\t\tbreak;\n\t\tcase TT_UDP:\n\t\t\ttype = SOCK_DGRAM;\n\t\t\tprotocol = 0;\n\t\t\tbreak;\n#ifdef IPPROTO_SCTP\n\t\tcase TT_SCTP:\n\t\tcase TT_SCTP_SSL:\n\t\t\ttype = SOCK_STREAM;\n\t\t\tprotocol = IPPROTO_SCTP;\n\t\t\tbreak;\n#endif\n\t\tdefault:\n\t\t\terrno = EPROTONOSUPPORT;\n\t\t\treturn -1;\n\t\t}\n\n\t\tthis->get_addr(&bind_addr, &addrlen);\n\t\tthis->listen_fd = socket(bind_addr->sa_family, type, protocol);\n\t\tif (this->listen_fd >= 0)\n\t\t{\n\t\t\tsetsockopt(this->listen_fd, SOL_SOCKET, SO_REUSEADDR,\n\t\t\t\t\t &reuse, sizeof (int));\n\t\t}\n\t}\n\telse\n\t\tthis->listen_fd = dup(this->listen_fd);\n\n\treturn this->listen_fd;\n}\n\nWFConnection *WFServerBase::new_connection(int accept_fd)\n{\n\tif (++this->conn_count > this->params.max_connections &&\n\t\tthis->drain(1) <= 0)\n\t{\n\t\tthis->conn_count--;\n\t\terrno = EMFILE;\n\t\treturn NULL;\n\t}\n\n\treturn new WFServerConnection(&this->conn_count);\n}\n\nvoid WFServerBase::delete_connection(WFConnection *conn)\n{\n\tdelete (WFServerConnection *)conn;\n}\n\nvoid WFServerBase::handle_unbound()\n{\n\tthis->mutex.lock();\n\tthis->unbind_finish = true;\n\tthis->cond.notify_one();\n\tthis->mutex.unlock();\n}\n\nint WFServerBase::start(const struct sockaddr *bind_addr, socklen_t addrlen,\n\t\t\t\t\t\tconst char *cert_file, const char *key_file)\n{\n\tSSL_CTX *ssl_ctx;\n\n\tif (this->init(bind_addr, addrlen, cert_file, key_file) >= 0)\n\t{\n\t\tif (this->scheduler->bind(this) >= 0)\n\t\t\treturn 0;\n\n\t\tssl_ctx = this->get_ssl_ctx();\n\t\tthis->deinit();\n\t\tif (ssl_ctx)\n\t\t\tSSL_CTX_free(ssl_ctx);\n\t}\n\n\tthis->listen_fd = -1;\n\treturn -1;\n}\n\nint WFServerBase::start(int family, const char *host, unsigned short port,\n\t\t\t\t\t\tconst char *cert_file, const char *key_file)\n{\n\tstruct addrinfo hints = {\n\t\t.ai_flags\t\t=\tAI_PASSIVE,\n\t\t.ai_family\t\t=\tfamily,\n\t\t.ai_socktype\t=\tSOCK_STREAM,\n\t};\n\tstruct addrinfo *addrinfo;\n\tchar port_str[PORT_STR_MAX + 1];\n\tint ret;\n\n\tsnprintf(port_str, PORT_STR_MAX + 1, \"%d\", port);\n\tret = getaddrinfo(host, port_str, &hints, &addrinfo);\n\tif (ret == 0)\n\t{\n\t\tret = start(addrinfo->ai_addr, (socklen_t)addrinfo->ai_addrlen,\n\t\t\t\t\tcert_file, key_file);\n\t\tfreeaddrinfo(addrinfo);\n\t}\n\telse\n\t{\n\t\tif (ret != EAI_SYSTEM)\n\t\t\terrno = EINVAL;\n\t\tret = -1;\n\t}\n\n\treturn ret;\n}\n\nint WFServerBase::serve(int listen_fd,\n\t\t\t\t\t\tconst char *cert_file, const char *key_file)\n{\n\tstruct sockaddr_storage ss;\n\tsocklen_t len = sizeof ss;\n\n\tif (getsockname(listen_fd, (struct sockaddr *)&ss, &len) < 0)\n\t\treturn -1;\n\n\tthis->listen_fd = listen_fd;\n\treturn start((struct sockaddr *)&ss, len, cert_file, key_file);\n}\n\nvoid WFServerBase::shutdown()\n{\n\tthis->listen_fd = -1;\n\tthis->scheduler->unbind(this);\n}\n\nvoid WFServerBase::wait_finish()\n{\n\tSSL_CTX *ssl_ctx = this->get_ssl_ctx();\n\tstd::unique_lock lock(this->mutex);\n\n\twhile (!this->unbind_finish)\n\t\tthis->cond.wait(lock);\n\n\tthis->deinit();\n\tthis->unbind_finish = false;\n\tlock.unlock();\n\tif (ssl_ctx)\n\t\tSSL_CTX_free(ssl_ctx);\n}\n\n" }, { "path": "src/server/WFServer.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _WFSERVER_H_\n#define _WFSERVER_H_\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"EndpointParams.h\"\n#include \"WFTaskFactory.h\"\n\nstruct WFServerParams\n{\n\tenum TransportType transport_type;\n\tsize_t max_connections;\n\tint peer_response_timeout;\t/* timeout of each read or write operation */\n\tint receive_timeout;\t/* timeout of receiving the whole message */\n\tint keep_alive_timeout;\n\tsize_t request_size_limit;\n\tint ssl_accept_timeout;\t/* if not ssl, this will be ignored */\n};\n\nstatic constexpr struct WFServerParams SERVER_PARAMS_DEFAULT =\n{\n\t.transport_type\t\t\t=\tTT_TCP,\n\t.max_connections\t\t=\t2000,\n\t.peer_response_timeout\t=\t10 * 1000,\n\t.receive_timeout\t\t=\t-1,\n\t.keep_alive_timeout\t\t=\t60 * 1000,\n\t.request_size_limit\t\t=\t(size_t)-1,\n\t.ssl_accept_timeout\t\t=\t10 * 1000,\n};\n\nclass WFServerBase : protected CommService\n{\npublic:\n\tWFServerBase(const struct WFServerParams *params) :\n\t\tconn_count(0)\n\t{\n\t\tthis->params = *params;\n\t\tthis->unbind_finish = false;\n\t\tthis->listen_fd = -1;\n\t}\n\npublic:\n\t/* To start a TCP server */\n\n\t/* Start on port with IPv4. */\n\tint start(unsigned short port)\n\t{\n\t\treturn start(AF_INET, NULL, port, NULL, NULL);\n\t}\n\n\t/* Start with family. AF_INET or AF_INET6. */\n\tint start(int family, unsigned short port)\n\t{\n\t\treturn start(family, NULL, port, NULL, NULL);\n\t}\n\n\t/* Start with hostname and port. */\n\tint start(const char *host, unsigned short port)\n\t{\n\t\treturn start(AF_INET, host, port, NULL, NULL);\n\t}\n\n\t/* Start with family, hostname and port. */\n\tint start(int family, const char *host, unsigned short port)\n\t{\n\t\treturn start(family, host, port, NULL, NULL);\n\t}\n\n\t/* Start with binding address. */\n\tint start(const struct sockaddr *bind_addr, socklen_t addrlen)\n\t{\n\t\treturn start(bind_addr, addrlen, NULL, NULL);\n\t}\n\n\t/* To start an SSL server. */\n\n\tint start(unsigned short port, const char *cert_file, const char *key_file)\n\t{\n\t\treturn start(AF_INET, NULL, port, cert_file, key_file);\n\t}\n\n\tint start(int family, unsigned short port,\n\t\t\t const char *cert_file, const char *key_file)\n\t{\n\t\treturn start(family, NULL, port, cert_file, key_file);\n\t}\n\n\tint start(const char *host, unsigned short port,\n\t\t\t const char *cert_file, const char *key_file)\n\t{\n\t\treturn start(AF_INET, host, port, cert_file, key_file);\n\t}\n\n\tint start(int family, const char *host, unsigned short port,\n\t\t\t const char *cert_file, const char *key_file);\n\n\t/* This is the only necessary start function. */\n\tint start(const struct sockaddr *bind_addr, socklen_t addrlen,\n\t\t\t const char *cert_file, const char *key_file);\n\n\t/* To start with a specified fd. For graceful restart or SCTP server. */\n\tint serve(int listen_fd)\n\t{\n\t\treturn serve(listen_fd, NULL, NULL);\n\t}\n\n\tint serve(int listen_fd, const char *cert_file, const char *key_file);\n\n\t/* stop() is a blocking operation. */\n\tvoid stop()\n\t{\n\t\tthis->shutdown();\n\t\tthis->wait_finish();\n\t}\n\n\t/* Nonblocking terminating the server. For stopping multiple servers.\n\t * Typically, call shutdown() and then wait_finish().\n\t * But indeed wait_finish() can be called before shutdown(), even before\n\t * start() in another thread. */\n\tvoid shutdown();\n\tvoid wait_finish();\n\npublic:\n\tsize_t get_conn_count() const { return this->conn_count; }\n\n\t/* Get the listening address. This is often used after starting\n\t * server on a random port (start() with port == 0). */\n\tint get_listen_addr(struct sockaddr *addr, socklen_t *addrlen) const\n\t{\n\t\tif (this->listen_fd >= 0)\n\t\t\treturn getsockname(this->listen_fd, addr, addrlen);\n\n\t\terrno = ENOTCONN;\n\t\treturn -1;\n\t}\n\n\tconst struct WFServerParams *get_params() const { return &this->params; }\n\nprotected:\n\t/* Override this function to create the initial SSL CTX of the server */\n\tvirtual SSL_CTX *new_ssl_ctx(const char *cert_file, const char *key_file);\n\n\t/* Override this function to implement server that supports TLS SNI.\n\t * \"servername\" will be NULL if client does not set a host name.\n\t * Returning NULL to indicate that servername is not supported. */\n\tvirtual SSL_CTX *get_server_ssl_ctx(const char *servername)\n\t{\n\t\treturn this->get_ssl_ctx();\n\t}\n\n\t/* This can be used by the implementation of 'new_ssl_ctx'. */\n\tstatic int ssl_ctx_callback(SSL *ssl, int *al, void *arg);\n\nprotected:\n\tWFServerParams params;\n\nprotected:\n\tvirtual int create_listen_fd();\n\tvirtual WFConnection *new_connection(int accept_fd);\n\tvoid delete_connection(WFConnection *conn);\n\nprivate:\n\tint init(const struct sockaddr *bind_addr, socklen_t addrlen,\n\t\t\t const char *cert_file, const char *key_file);\n\tvirtual void handle_unbound();\n\nprotected:\n\tstd::atomic conn_count;\n\nprivate:\n\tint listen_fd;\n\tbool unbind_finish;\n\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\n\tclass CommScheduler *scheduler;\n};\n\ntemplate\nclass WFServer : public WFServerBase\n{\npublic:\n\tWFServer(const struct WFServerParams *params,\n\t\t\t std::function *)> proc) :\n\t\tWFServerBase(params),\n\t\tprocess(std::move(proc))\n\t{\n\t}\n\n\tWFServer(std::function *)> proc) :\n\t\tWFServerBase(&SERVER_PARAMS_DEFAULT),\n\t\tprocess(std::move(proc))\n\t{\n\t}\n\nprotected:\n\tvirtual CommSession *new_session(long long seq, CommConnection *conn);\n\nprotected:\n\tstd::function *)> process;\n};\n\ntemplate\nCommSession *WFServer::new_session(long long seq, CommConnection *conn)\n{\n\tusing factory = WFNetworkTaskFactory;\n\tWFNetworkTask *task;\n\n\ttask = factory::create_server_task(this, this->process);\n\ttask->set_keep_alive(this->params.keep_alive_timeout);\n\ttask->set_receive_timeout(this->params.receive_timeout);\n\ttask->get_req()->set_size_limit(this->params.request_size_limit);\n\n\treturn task;\n}\n\n#endif\n\n" }, { "path": "src/server/xmake.lua", "content": "target(\"server\")\n set_kind(\"object\")\n add_files(\"*.cc\")\n if not has_config(\"mysql\") then\n remove_files(\"WFMySQLServer.cc\")\n end\n" }, { "path": "src/util/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\nproject(util)\n\nset(SRC\n\tjson_parser.c\n\tEncodeStream.cc\n\tStringUtil.cc\n\tURIParser.cc\n)\n\nadd_library(${PROJECT_NAME} OBJECT ${SRC})\n\nif (KAFKA STREQUAL \"y\")\n\tset(SRC\n\t\tcrc32c.c\n\t)\n\tadd_library(\"util_kafka\" OBJECT ${SRC})\nendif ()\n" }, { "path": "src/util/EncodeStream.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \"list.h\"\n#include \"EncodeStream.h\"\n\n#define ALIGN(x,a) (((x)+(a)-1)&~((a)-1))\n#define ENCODE_BUF_SIZE\t\t1024\n\nstruct EncodeBuf\n{\n\tstruct list_head list;\n\tchar *pos;\n\tchar data[ENCODE_BUF_SIZE];\n};\n\nvoid EncodeStream::clear_buf_data()\n{\n\tstruct list_head *pos, *tmp;\n\tstruct EncodeBuf *entry;\n\n\tlist_for_each_safe(pos, tmp, &buf_list_)\n\t{\n\t\tentry = list_entry(pos, struct EncodeBuf, list);\n\t\tlist_del(pos);\n\t\tdelete [](char *)entry;\n\t}\n}\n\nvoid EncodeStream::merge()\n{\n\tsize_t len = bytes_ - merged_bytes_;\n\tstruct EncodeBuf *buf;\n\tsize_t n;\n\tchar *p;\n\tint i;\n\n\tif (len > ENCODE_BUF_SIZE)\n\t\tn = offsetof(struct EncodeBuf, data) + ALIGN(len, 8);\n\telse\n\t\tn = sizeof (struct EncodeBuf);\n\n\tbuf = (struct EncodeBuf *)new char[n];\n\tp = buf->data;\n\tfor (i = merged_size_; i < size_; i++)\n\t{\n\t\tmemcpy(p, vec_[i].iov_base, vec_[i].iov_len);\n\t\tp += vec_[i].iov_len;\n\t}\n\n\tbuf->pos = buf->data + ALIGN(len, 8);\n\tlist_add(&buf->list, &buf_list_);\n\n\tvec_[merged_size_].iov_base = buf->data;\n\tvec_[merged_size_].iov_len = len;\n\tmerged_size_++;\n\tmerged_bytes_ = bytes_;\n\tsize_ = merged_size_;\n}\n\nvoid EncodeStream::append_nocopy(const char *data, size_t len)\n{\n\tif (size_ >= max_)\n\t{\n\t\tif (merged_size_ + 1 < max_)\n\t\t\tmerge();\n\t\telse\n\t\t{\n\t\t\tsize_ = max_ + 1;\t/* Overflow */\n\t\t\treturn;\n\t\t}\n\t}\n\n\tvec_[size_].iov_base = (char *)data;\n\tvec_[size_].iov_len = len;\n\tsize_++;\n\tbytes_ += len;\n}\n\nvoid EncodeStream::append_copy(const char *data, size_t len)\n{\n\tif (size_ >= max_)\n\t{\n\t\tif (merged_size_ + 1 < max_)\n\t\t\tmerge();\n\t\telse\n\t\t{\n\t\t\tsize_ = max_ + 1;\t/* Overflow */\n\t\t\treturn;\n\t\t}\n\t}\n\n\tstruct EncodeBuf *buf = list_entry(buf_list_.prev, struct EncodeBuf, list);\n\n\tif (list_empty(&buf_list_) || buf->pos + len > buf->data + ENCODE_BUF_SIZE)\n\t{\n\t\tsize_t n;\n\n\t\tif (len > ENCODE_BUF_SIZE)\n\t\t\tn = offsetof(struct EncodeBuf, data) + ALIGN(len, 8);\n\t\telse\n\t\t\tn = sizeof (struct EncodeBuf);\n\n\t\tbuf = (struct EncodeBuf *)new char[n];\n\t\tbuf->pos = buf->data;\n\t\tlist_add_tail(&buf->list, &buf_list_);\n\t}\n\n\tmemcpy(buf->pos, data, len);\n\tvec_[size_].iov_base = buf->pos;\n\tvec_[size_].iov_len = len;\n\tsize_++;\n\tbytes_ += len;\n\n\tbuf->pos += ALIGN(len, 8);\n\tif (buf->pos >= buf->data + ENCODE_BUF_SIZE)\n\t\tlist_move(&buf->list, &buf_list_);\n}\n\n" }, { "path": "src/util/EncodeStream.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Xie Han (xiehan@sogou-inc.com)\n Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _ENCODESTREAM_H_\n#define _ENCODESTREAM_H_\n\n#include \n#include \n#include \n#include \n#include \"list.h\"\n\n/**\n * @file EncodeStream.h\n * @brief Encoder toolbox for protocol message encode\n */\n\n// make sure max > 0\nclass EncodeStream\n{\npublic:\n\tEncodeStream()\n\t{\n\t\tinit_vec(NULL, 0);\n\t\tINIT_LIST_HEAD(&buf_list_);\n\t}\n\n\tEncodeStream(struct iovec *vectors, int max)\n\t{\n\t\tinit_vec(vectors, max);\n\t\tINIT_LIST_HEAD(&buf_list_);\n\t}\n\n\t~EncodeStream() { clear_buf_data(); }\n\n\tvoid reset(struct iovec *vectors, int max)\n\t{\n\t\tclear_buf_data();\n\t\tinit_vec(vectors, max);\n\t}\n\n\tint size() const { return size_; }\n\tsize_t bytes() const { return bytes_; }\n\n\tvoid append_nocopy(const char *data, size_t len);\n\n\tvoid append_nocopy(const char *data)\n\t{\n\t\tappend_nocopy(data, strlen(data));\n\t}\n\n\tvoid append_nocopy(const std::string& data)\n\t{\n\t\tappend_nocopy(data.c_str(), data.size());\n\t}\n\n\tvoid append_copy(const char *data, size_t len);\n\n\tvoid append_copy(const char *data)\n\t{\n\t\tappend_copy(data, strlen(data));\n\t}\n\n\tvoid append_copy(const std::string& data)\n\t{\n\t\tappend_copy(data.c_str(), data.size());\n\t}\n\nprivate:\n\tvoid init_vec(struct iovec *vectors, int max)\n\t{\n\t\tvec_ = vectors;\n\t\tmax_ = max;\n\t\tbytes_ = 0;\n\t\tsize_ = 0;\n\t\tmerged_bytes_ = 0;\n\t\tmerged_size_ = 0;\n\t}\n\n\tvoid merge();\n\tvoid clear_buf_data();\n\nprivate:\n\tstruct iovec *vec_;\n\tint max_;\n\tint size_;\n\tsize_t bytes_;\n\tint merged_size_;\n\tsize_t merged_bytes_;\n\tstruct list_head buf_list_;\n};\n\nstatic inline EncodeStream& operator << (EncodeStream& stream,\n\t\t\t\t\t\t\t\t\t\t const char *data)\n{\n\tstream.append_nocopy(data, strlen(data));\n\treturn stream;\n}\n\nstatic inline EncodeStream& operator << (EncodeStream& stream,\n\t\t\t\t\t\t\t\t\t\t const std::string& data)\n{\n\tstream.append_nocopy(data.c_str(), data.size());\n\treturn stream;\n}\n\nstatic inline EncodeStream& operator << (EncodeStream& stream,\n\t\t\t\t\t\t\t\tconst std::pair& data)\n{\n\tstream.append_nocopy(data.first, data.second);\n\treturn stream;\n}\n\nstatic inline EncodeStream& operator << (EncodeStream& stream,\n\t\t\t\t\t\t\t\t\t\t int64_t intv)\n{\n\tstream.append_copy(std::to_string(intv));\n\treturn stream;\n}\n\n#endif\n\n" }, { "path": "src/util/LRUCache.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _LRUCACHE_H_\n#define _LRUCACHE_H_\n\n#include \n#include \"list.h\"\n#include \"rbtree.h\"\n\n/**\n * @file LRUCache.h\n * @brief Template LRU Cache\n */\n\n// RAII: NO. Release ref by LRUCache::release\n// Thread safety: NO.\n// DONOT change value by handler, use Cache::put instead\ntemplate\nclass LRUHandle\n{\npublic:\n\tVALUE value;\n\nprivate:\n\tLRUHandle(const KEY& k, const VALUE& v) :\n\t\tvalue(v), key(k)\n\t{\n\t}\n\n\tKEY key;\n\tstruct list_head list;\n\tstruct rb_node rb;\n\tbool in_cache;\n\tint ref;\n\n\ttemplate friend class LRUCache;\n};\n\n// RAII: NO. Release ref by LRUCache::release\n// Define ValueDeleter(VALUE& v) for value deleter\n// Thread safety: NO\n// Make sure KEY operator< usable\ntemplate\nclass LRUCache\n{\nprotected:\n\ttypedef LRUHandle\t\t\tHandle;\n\npublic:\n\tLRUCache()\n\t{\n\t\tINIT_LIST_HEAD(&this->not_use);\n\t\tINIT_LIST_HEAD(&this->in_use);\n\t\tthis->cache_map.rb_node = NULL;\n\t\tthis->max_size = 0;\n\t\tthis->size = 0;\n\t}\n\n\t~LRUCache()\n\t{\n\t\tstruct list_head *pos, *tmp;\n\t\tHandle *e;\n\n\t\t// Error if caller has an unreleased handle\n\t\tassert(list_empty(&this->in_use));\n\t\tlist_for_each_safe(pos, tmp, &this->not_use)\n\t\t{\n\t\t\te = list_entry(pos, Handle, list);\n\t\t\tassert(e->in_cache);\n\t\t\te->in_cache = false;\n\t\t\tassert(e->ref == 1);// Invariant for not_use_ list.\n\t\t\tthis->unref(e);\n\t\t}\n\t}\n\n\t// default max_size=0 means no-limit cache\n\t// max_size means max cache number of key-value pairs\n\tvoid set_max_size(size_t max_size)\n\t{\n\t\tthis->max_size = max_size;\n\t}\n\n\t// Remove all cache that are not actively in use.\n\tvoid prune()\n\t{\n\t\tstruct list_head *pos, *tmp;\n\t\tHandle *e;\n\n\t\tlist_for_each_safe(pos, tmp, &this->not_use)\n\t\t{\n\t\t\te = list_entry(pos, Handle, list);\n\t\t\tassert(e->ref == 1);\n\t\t\trb_erase(&e->rb, &this->cache_map);\n\t\t\tthis->erase_node(e);\n\t\t}\n\t}\n\n\t// release handle by get/put\n\tvoid release(const Handle *handle)\n\t{\n\t\tthis->unref(const_cast(handle));\n\t}\n\n\t// get handler\n\t// Need call release when handle no longer needed\n\tconst Handle *get(const KEY& key)\n\t{\n\t\tstruct rb_node *p = this->cache_map.rb_node;\n\t\tHandle *bound = NULL;\n\t\tHandle *e;\n\n\t\twhile (p)\n\t\t{\n\t\t\te = rb_entry(p, Handle, rb);\n\t\t\tif (!(e->key < key))\n\t\t\t{\n\t\t\t\tbound = e;\n\t\t\t\tp = p->rb_left;\n\t\t\t}\n\t\t\telse\n\t\t\t\tp = p->rb_right;\n\t\t}\n\n\t\tif (bound && !(key < bound->key))\n\t\t{\n\t\t\tthis->ref(bound);\n\t\t\treturn bound;\n\t\t}\n\n\t\treturn NULL;\n\t}\n\n\t// put copy\n\t// Need call release when handle no longer needed\n\tconst Handle *put(const KEY& key, VALUE value)\n\t{\n\t\tstruct rb_node **p = &this->cache_map.rb_node;\n\t\tstruct rb_node *parent = NULL;\n\t\tHandle *bound = NULL;\n\t\tHandle *e;\n\n\t\twhile (*p)\n\t\t{\n\t\t\tparent = *p;\n\t\t\te = rb_entry(*p, Handle, rb);\n\t\t\tif (!(e->key < key))\n\t\t\t{\n\t\t\t\tbound = e;\n\t\t\t\tp = &(*p)->rb_left;\n\t\t\t}\n\t\t\telse\n\t\t\t\tp = &(*p)->rb_right;\n\t\t}\n\n\t\te = new Handle(key, value);\n\t\te->in_cache = true;\n\t\te->ref = 2;\n\t\tlist_add_tail(&e->list, &this->in_use);\n\t\tthis->size++;\n\n\t\tif (bound && !(key < bound->key))\n\t\t{\n\t\t\trb_replace_node(&bound->rb, &e->rb, &this->cache_map);\n\t\t\tthis->erase_node(bound);\n\t\t}\n\t\telse\n\t\t{\n\t\t\trb_link_node(&e->rb, parent, p);\n\t\t\trb_insert_color(&e->rb, &this->cache_map);\n\t\t}\n\n\t\tif (this->max_size > 0)\n\t\t{\n\t\t\twhile (this->size > this->max_size && !list_empty(&this->not_use))\n\t\t\t{\n\t\t\t\tHandle *tmp = list_entry(this->not_use.next, Handle, list);\n\t\t\t\tassert(tmp->ref == 1);\n\t\t\t\trb_erase(&tmp->rb, &this->cache_map);\n\t\t\t\tthis->erase_node(tmp);\n\t\t\t}\n\t\t}\n\n\t\treturn e;\n\t}\n\n\t// delete from cache, deleter delay called when all inuse-handle release.\n\tvoid del(const KEY& key)\n\t{\n\t\tHandle *e = const_cast(this->get(key));\n\n\t\tif (e)\n\t\t{\n\t\t\tthis->unref(e);\n\t\t\trb_erase(&e->rb, &this->cache_map);\n\t\t\tthis->erase_node(e);\n\t\t}\n\t}\n\nprivate:\n\tvoid ref(Handle *e)\n\t{\n\t\tif (e->in_cache && e->ref == 1)\n\t\t\tlist_move_tail(&e->list, &this->in_use);\n\n\t\te->ref++;\n\t}\n\n\tvoid unref(Handle *e)\n\t{\n\t\tassert(e->ref > 0);\n\t\tif (--e->ref == 0)\n\t\t{\n\t\t\tassert(!e->in_cache);\n\t\t\tthis->value_deleter(e->value);\n\t\t\tdelete e;\n\t\t}\n\t\telse if (e->in_cache && e->ref == 1)\n\t\t\tlist_move_tail(&e->list, &this->not_use);\n\t}\n\n\tvoid erase_node(Handle *e)\n\t{\n\t\tassert(e->in_cache);\n\t\tlist_del(&e->list);\n\t\te->in_cache = false;\n\t\tthis->size--;\n\t\tthis->unref(e);\n\t}\n\n\tsize_t max_size;\n\tsize_t size;\n\n\tstruct list_head not_use;\n\tstruct list_head in_use;\n\tstruct rb_root cache_map;\n\n\tValueDeleter value_deleter;\n};\n\n#endif\n\n" }, { "path": "src/util/StringUtil.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"StringUtil.h\"\n\nstatic int __hex_to_int(const char s[2])\n{\n\tint value = 16;\n\n\tif (s[0] <= '9')\n\t\tvalue *= s[0] - '0';\n\telse\n\t\tvalue *= toupper(s[0]) - 'A' + 10;\n\n\tif (s[1] <= '9')\n\t\tvalue += s[1] - '0';\n\telse\n\t\tvalue += toupper(s[1]) - 'A' + 10;\n\n\treturn value;\n}\n\nstatic inline char __int_to_hex(int n)\n{\n\treturn n <= 9 ? n + '0' : n - 10 + 'A';\n}\n\nstatic size_t __url_decode(char *str)\n{\n\tchar *dest = str;\n\tchar *data = str;\n\n\twhile (*data)\n\t{\n\t\tif (*data == '%' && isxdigit(data[1]) && isxdigit(data[2]))\n\t\t{\n\t\t\t*dest = __hex_to_int(data + 1);\n\t\t\tdata += 2;\n\t\t}\n\t\telse if (*data == '+')\n\t\t\t*dest = ' ';\n\t\telse\n\t\t\t*dest = *data;\n\n\t\tdata++;\n\t\tdest++;\n\t}\n\n\t*dest = '\\0';\n\treturn dest - str;\n}\n\nvoid StringUtil::url_decode(std::string& str)\n{\n\tstr.resize(__url_decode(const_cast(str.c_str())));\n}\n\nstd::string StringUtil::url_encode(const std::string& str)\n{\n\tconst char *cur = str.c_str();\n\tconst char *end = cur + str.size();\n\tstd::string res;\n\n\twhile (cur < end)\n\t{\n\t\tif (isalnum(*cur) || *cur == '-' || *cur == '_' || *cur == '.' ||\n\t\t\t*cur == '!' || *cur == '~' || *cur == '*' || *cur == '\\'' ||\n\t\t\t*cur == '(' || *cur == ')' || *cur == ':' || *cur == '/' ||\n\t\t\t*cur == '@' || *cur == '?' || *cur == '#' || *cur == '&')\n\t\t{\n\t\t\tres += *cur;\n\t\t}\n\t\telse if (*cur == ' ')\n\t\t{\n\t\t\tres += '+';\n\t\t}\n\t\telse\n\t\t{\n\t\t\tres += '%';\n\t\t\tres += __int_to_hex(((const unsigned char)(*cur)) >> 4);\n\t\t\tres += __int_to_hex(((const unsigned char)(*cur)) % 16);\n\t\t}\n\n\t\tcur++;\n\t}\n\n\treturn res;\n}\n\nstd::string StringUtil::url_encode_component(const std::string& str)\n{\n\tconst char *cur = str.c_str();\n\tconst char *end = cur + str.size();\n\tstd::string res;\n\n\twhile (cur < end)\n\t{\n\t\tif (isalnum(*cur) || *cur == '-' || *cur == '_' || *cur == '.' ||\n\t\t\t*cur == '!' || *cur == '~' || *cur == '*' || *cur == '\\'' ||\n\t\t\t*cur == '(' || *cur == ')')\n\t\t{\n\t\t\tres += *cur;\n\t\t}\n\t\telse if (*cur == ' ')\n\t\t{\n\t\t\tres += '+';\n\t\t}\n\t\telse\n\t\t{\n\t\t\tres += '%';\n\t\t\tres += __int_to_hex(((const unsigned char)(*cur)) >> 4);\n\t\t\tres += __int_to_hex(((const unsigned char)(*cur)) % 16);\n\t\t}\n\n\t\tcur++;\n\t}\n\n\treturn res;\n}\n\nstd::vector StringUtil::split(const std::string& str, char sep)\n{\n\tstd::string::const_iterator cur = str.begin();\n\tstd::string::const_iterator end = str.end();\n\tstd::string::const_iterator next = find(cur, end, sep);\n\tstd::vector res;\n\n\twhile (next != end)\n\t{\n\t\tres.emplace_back(cur, next);\n\t\tcur = next + 1;\n\t\tnext = std::find(cur, end, sep);\n\t}\n\n\tres.emplace_back(cur, next);\n\treturn res;\n}\n\nstd::vector StringUtil::split_filter_empty(const std::string& str,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\tchar sep)\n{\n\tstd::vector res;\n\tstd::string::const_iterator cur = str.begin();\n\tstd::string::const_iterator end = str.end();\n\tstd::string::const_iterator next = find(cur, end, sep);\n\n\twhile (next != end)\n\t{\n\t\tif (cur < next)\n\t\t\tres.emplace_back(cur, next);\n\n\t\tcur = next + 1;\n\t\tnext = find(cur, end, sep);\n\t}\n\n\tif (cur < next)\n\t\tres.emplace_back(cur, next);\n\n\treturn res;\n}\n\nstd::string StringUtil::strip(const std::string& str)\n{\n\tstd::string res;\n\n\tif (!str.empty())\n\t{\n\t\tconst char *cur = str.c_str();\n\t\tconst char *end = cur + str.size();\n\n\t\twhile (cur < end)\n\t\t{\n\t\t\tif (!isspace(*cur))\n\t\t\t\tbreak;\n\n\t\t\tcur++;\n\t\t}\n\n\t\twhile (end > cur)\n\t\t{\n\t\t\tif (!isspace(*(end - 1)))\n\t\t\t\tbreak;\n\n\t\t\tend--;\n\t\t}\n\n\t\tif (end > cur)\n\t\t\tres.assign(cur, end - cur);\n\t}\n\n\treturn res;\n}\n\nbool StringUtil::start_with(const std::string& str, const std::string& prefix)\n{\n\tsize_t prefix_len = prefix.size();\n\n\tif (str.size() < prefix_len)\n\t\treturn false;\n\n\tfor (size_t i = 0; i < prefix_len; i++)\n\t{\n\t\tif (str[i] != prefix[i])\n\t\t\treturn false;\n\t}\n\n\treturn true;\n}\n\n" }, { "path": "src/util/StringUtil.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#ifndef _STRINGUTIL_H_\n#define _STRINGUTIL_H_\n\n#include \n#include \n\n/**\n * @file StringUtil.h\n * @brief String toolbox\n */\n\n// static class\nclass StringUtil\n{\npublic:\n\tstatic void url_decode(std::string& str);\n\tstatic std::string url_encode(const std::string& str);\n\tstatic std::string url_encode_component(const std::string& str);\n\tstatic std::vector split(const std::string& str, char sep);\n\tstatic std::string strip(const std::string& str);\n\tstatic bool start_with(const std::string& str, const std::string& prefix);\n\n\t//this will filter any empty result, so the result vector has no empty string\n\tstatic std::vector split_filter_empty(const std::string& str, char sep);\n};\n\n#endif\n\n" }, { "path": "src/util/URIParser.cc", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Wang Zhulei (wangzhulei@sogou-inc.com)\n Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"StringUtil.h\"\n#include \"URIParser.h\"\n\nenum\n{\n\tURI_SCHEME,\n\tURI_USERINFO,\n\tURI_HOST,\n\tURI_PORT,\n\tURI_QUERY,\n\tURI_FRAGMENT,\n\tURI_PATH,\n\tURI_PART_ELEMENTS,\n};\n\n//scheme://[userinfo@]host[:port][/path][?query][#fragment]\n//0-6 (scheme, userinfo, host, port, path, query, fragment)\nstatic constexpr unsigned char valid_char[4][256] = {\n\t{\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,\n\t\t0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,\n\t\t0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0\n\t},\n\t{\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0,\n\t\t0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,\n\t\t0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0\n\t},\n\t{\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0,\n\t\t0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1,\n\t\t0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0\n\t},\n\t{\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0\n\t},\n};\n\nstatic unsigned char authority_map[256] = {\n\tURI_PART_ELEMENTS, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, URI_FRAGMENT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, URI_PATH,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, URI_HOST, 0, 0, 0, 0, URI_QUERY,\n\tURI_USERINFO, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n};\n\nParsedURI::ParsedURI(ParsedURI&& uri)\n{\n\tscheme = uri.scheme;\n\tuserinfo = uri.userinfo;\n\thost = uri.host;\n\tport = uri.port;\n\tpath = uri.path;\n\tquery = uri.query;\n\tfragment = uri.fragment;\n\tstate = uri.state;\n\terror = uri.error;\n\turi.init();\n}\n\nParsedURI& ParsedURI::operator= (ParsedURI&& uri)\n{\n\tif (this != &uri)\n\t{\n\t\tdeinit();\n\t\tscheme = uri.scheme;\n\t\tuserinfo = uri.userinfo;\n\t\thost = uri.host;\n\t\tport = uri.port;\n\t\tpath = uri.path;\n\t\tquery = uri.query;\n\t\tfragment = uri.fragment;\n\t\tstate = uri.state;\n\t\terror = uri.error;\n\t\turi.init();\n\t}\n\n\treturn *this;\n}\n\nvoid ParsedURI::copy(const ParsedURI& uri)\n{\n\tinit();\n\tstate = uri.state;\n\terror = uri.error;\n\tif (state == URI_STATE_SUCCESS)\n\t{\n\t\tbool succ = false;\n\n\t\tdo\n\t\t{\n\t\t\tif (uri.scheme)\n\t\t\t{\n\t\t\t\tscheme = strdup(uri.scheme);\n\t\t\t\tif (!scheme)\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (uri.userinfo)\n\t\t\t{\n\t\t\t\tuserinfo = strdup(uri.userinfo);\n\t\t\t\tif (!userinfo)\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (uri.host)\n\t\t\t{\n\t\t\t\thost = strdup(uri.host);\n\t\t\t\tif (!host)\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (uri.port)\n\t\t\t{\n\t\t\t\tport = strdup(uri.port);\n\t\t\t\tif (!port)\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (uri.path)\n\t\t\t{\n\t\t\t\tpath = strdup(uri.path);\n\t\t\t\tif (!path)\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (uri.query)\n\t\t\t{\n\t\t\t\tquery = strdup(uri.query);\n\t\t\t\tif (!query)\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (uri.fragment)\n\t\t\t{\n\t\t\t\tfragment = strdup(uri.fragment);\n\t\t\t\tif (!fragment)\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tsucc = true;\n\t\t} while (0);\n\n\t\tif (!succ)\n\t\t{\n\t\t\tdeinit();\n\t\t\tinit();\n\t\t\tstate = URI_STATE_ERROR;\n\t\t\terror = errno;\n\t\t}\n\t}\n}\n\nint URIParser::parse(const char *str, ParsedURI& uri)\n{\n\turi.state = URI_STATE_INVALID;\n\n\tsize_t start_idx[URI_PART_ELEMENTS] = {0};\n\tsize_t end_idx[URI_PART_ELEMENTS] = {0};\n\tint pre_state = URI_SCHEME;\n\tbool in_ipv6 = false;\n\tsize_t i;\n\n\tfor (i = 0; str[i]; i++)\n\t{\n\t\tif (str[i] == ':')\n\t\t{\n\t\t\tend_idx[URI_SCHEME] = i++;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (end_idx[URI_SCHEME] == 0)\n\t\treturn -1;\n\n\tif (str[i] == '/' && str[i + 1] == '/')\n\t{\n\t\tpre_state = URI_HOST;\n\t\ti += 2;\n\t\tif (str[i] == '[')\n\t\t\tin_ipv6 = true;\n\t\telse\n\t\t\tstart_idx[URI_USERINFO] = i;\n\n\t\tstart_idx[URI_HOST] = i;\n\t}\n\telse\n\t{\n\t\tpre_state = URI_PATH;\n\t\tstart_idx[URI_PATH] = i;\n\t}\n\n\tbool skip_path = false;\n\tif (start_idx[URI_PATH] == 0)\n\t{\n\t\tfor (; ; i++)\n\t\t{\n\t\t\tswitch (authority_map[(unsigned char)str[i]])\n\t\t\t{\n\t\t\t\tcase 0:\n\t\t\t\t\tcontinue;\n\n\t\t\t\tcase URI_USERINFO:\n\t\t\t\t\tif (str[i + 1] == '[')\n\t\t\t\t\t\tin_ipv6 = true;\n\n\t\t\t\t\tend_idx[URI_USERINFO] = i;\n\t\t\t\t\tstart_idx[URI_HOST] = i + 1;\n\t\t\t\t\tpre_state = URI_HOST;\n\t\t\t\t\tcontinue;\n\n\t\t\t\tcase URI_HOST:\n\t\t\t\t\tif (str[i - 1] == ']')\n\t\t\t\t\t\tin_ipv6 = false;\n\n\t\t\t\t\tif (!in_ipv6)\n\t\t\t\t\t{\n\t\t\t\t\t\tend_idx[URI_HOST] = i;\n\t\t\t\t\t\tstart_idx[URI_PORT] = i + 1;\n\t\t\t\t\t\tpre_state = URI_PORT;\n\t\t\t\t\t}\n\t\t\t\t\tcontinue;\n\n\t\t\t\tcase URI_QUERY:\n\t\t\t\t\tend_idx[pre_state] = i;\n\t\t\t\t\tstart_idx[URI_QUERY] = i + 1;\n\t\t\t\t\tpre_state = URI_QUERY;\n\t\t\t\t\tskip_path = true;\n\t\t\t\t\tcontinue;\n\n\t\t\t\tcase URI_FRAGMENT:\n\t\t\t\t\tend_idx[pre_state] = i;\n\t\t\t\t\tstart_idx[URI_FRAGMENT] = i + 1;\n\t\t\t\t\tend_idx[URI_FRAGMENT] = i + strlen(str + i);\n\t\t\t\t\tpre_state = URI_PART_ELEMENTS;\n\t\t\t\t\tskip_path = true;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase URI_PATH:\n\t\t\t\t\tif (skip_path)\n\t\t\t\t\t\tcontinue;\n\n\t\t\t\t\tstart_idx[URI_PATH] = i;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase URI_PART_ELEMENTS:\n\t\t\t\t\tskip_path = true;\n\t\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tif (pre_state != URI_PART_ELEMENTS)\n\t\tend_idx[pre_state] = i;\n\n\tif (!skip_path)\n\t{\n\t\tpre_state = URI_PATH;\n\t\tfor (; str[i]; i++)\n\t\t{\n\t\t\tif (str[i] == '?')\n\t\t\t{\n\t\t\t\tend_idx[URI_PATH] = i;\n\t\t\t\tstart_idx[URI_QUERY] = i + 1;\n\t\t\t\tpre_state = URI_QUERY;\n\t\t\t\twhile (str[i + 1])\n\t\t\t\t{\n\t\t\t\t\tif (str[++i] == '#')\n\t\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (str[i] == '#')\n\t\t\t{\n\t\t\t\tend_idx[pre_state] = i;\n\t\t\t\tstart_idx[URI_FRAGMENT] = i + 1;\n\t\t\t\tpre_state = URI_FRAGMENT;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tend_idx[pre_state] = i + strlen(str + i);\n\t}\n\n\tfor (int i = 0; i < URI_QUERY; i++)\n\t{\n\t\tfor (size_t j = start_idx[i]; j < end_idx[i]; j++)\n\t\t{\n\t\t\tif (!valid_char[i][(unsigned char)str[j]])\n\t\t\t\treturn -1;//invalid char\n\t\t}\n\t}\n\n\tchar **dst[URI_PART_ELEMENTS] = {&uri.scheme, &uri.userinfo, &uri.host, &uri.port,\n\t\t\t\t\t &uri.query, &uri.fragment, &uri.path};\n\n\tfor (int i = 0; i < URI_PART_ELEMENTS; i++)\n\t{\n\t\tif (end_idx[i] > start_idx[i])\n\t\t{\n\t\t\tsize_t len = end_idx[i] - start_idx[i];\n\n\t\t\t*dst[i] = (char *)realloc(*dst[i], len + 1);\n\t\t\tif (*dst[i] == NULL)\n\t\t\t{\n\t\t\t\turi.state = URI_STATE_ERROR;\n\t\t\t\turi.error = errno;\n\t\t\t\treturn -1;\n\t\t\t}\n\n\t\t\tif (i == URI_HOST && str[start_idx[i]] == '[' &&\n\t\t\t\tstr[end_idx[i] - 1] == ']')\n\t\t\t{\n\t\t\t\tlen -= 2;\n\t\t\t\tmemcpy(*dst[i], str + start_idx[i] + 1, len);\n\t\t\t}\n\t\t\telse\n\t\t\t\tmemcpy(*dst[i], str + start_idx[i], len);\n\n\t\t\t(*dst[i])[len] = '\\0';\n\t\t}\n\t\telse\n\t\t{\n\t\t\tfree(*dst[i]);\n\t\t\t*dst[i] = NULL;\n\t\t}\n\t}\n\n\turi.state = URI_STATE_SUCCESS;\n\treturn 0;\n}\n\nstd::map>\nURIParser::split_query_strict(const std::string &query)\n{\n\tstd::map> res;\n\n\tif (query.empty())\n\t\treturn res;\n\n\tstd::vector arr = StringUtil::split(query, '&');\n\n\tif (arr.empty())\n\t\treturn res;\n\n\tfor (const auto& ele : arr)\n\t{\n\t\tif (ele.empty())\n\t\t\tcontinue;\n\n\t\tstd::vector kv = StringUtil::split(ele, '=');\n\t\tsize_t kv_size = kv.size();\n\t\tstd::string& key = kv[0];\n\n\t\tif (key.empty())\n\t\t\tcontinue;\n\n\t\tif (kv_size == 1)\n\t\t{\n\t\t\tres[key].emplace_back();\n\t\t\tcontinue;\n\t\t}\n\n\t\tstd::string& val = kv[1];\n\n\t\tif (val.empty())\n\t\t\tres[key].emplace_back();\n\t\telse\n\t\t\tres[key].emplace_back(std::move(val));\n\t}\n\n\treturn res;\n}\n\nstd::map URIParser::split_query(const std::string &query)\n{\n\tstd::map res;\n\n\tif (query.empty())\n\t\treturn res;\n\n\tstd::vector arr = StringUtil::split(query, '&');\n\n\tif (arr.empty())\n\t\treturn res;\n\n\tfor (const auto& ele : arr)\n\t{\n\t\tif (ele.empty())\n\t\t\tcontinue;\n\n\t\tstd::vector kv = StringUtil::split(ele, '=');\n\t\tsize_t kv_size = kv.size();\n\t\tstd::string& key = kv[0];\n\n\t\tif (key.empty() || res.count(key) > 0)\n\t\t\tcontinue;\n\n\t\tif (kv_size == 1)\n\t\t{\n\t\t\tres.emplace(std::move(key), \"\");\n\t\t\tcontinue;\n\t\t}\n\n\t\tstd::string& val = kv[1];\n\n\t\tif (val.empty())\n\t\t\tres.emplace(std::move(key), \"\");\n\t\telse\n\t\t\tres.emplace(std::move(key), std::move(val));\n\t}\n\n\treturn res;\n}\n\nstd::vector URIParser::split_path(const std::string &path)\n{\n\treturn StringUtil::split_filter_empty(path, '/');\n}\n\n" }, { "path": "src/util/URIParser.h", "content": "/*\n Copyright (c) 2019 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)\n Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#ifndef _URIPARSER_H_\n#define _URIPARSER_H_\n\n#include \n#include \n#include \n#include \n\n#define URI_STATE_INIT\t\t0\n#define URI_STATE_SUCCESS\t1\n#define URI_STATE_INVALID\t2\n#define URI_STATE_ERROR\t\t3\n\n/**\n * @file URIParser.h\n * @brief URI parser\n */\n\n// RAII: YES\nclass ParsedURI\n{\npublic:\n\tchar *scheme;\n\tchar *userinfo;\n\tchar *host;\n\tchar *port;\n\tchar *path;\n\tchar *query;\n\tchar *fragment;\n\tint state;\n\tint error;\n\n\tParsedURI() { init(); }\n\tvirtual ~ParsedURI() { deinit(); }\n\n\t//copy constructor\n\tParsedURI(const ParsedURI& uri) { copy(uri); }\n\t//copy operator\n\tParsedURI& operator= (const ParsedURI& uri)\n\t{\n\t\tif (this != &uri)\n\t\t{\n\t\t\tdeinit();\n\t\t\tcopy(uri);\n\t\t}\n\n\t\treturn *this;\n\t}\n\n\t//move constructor\n\tParsedURI(ParsedURI&& uri);\n\t//move operator\n\tParsedURI& operator= (ParsedURI&& uri);\n\nprivate:\n\tvoid init()\n\t{\n\t\tscheme = NULL;\n\t\tuserinfo = NULL;\n\t\thost = NULL;\n\t\tport = NULL;\n\t\tpath = NULL;\n\t\tquery = NULL;\n\t\tfragment = NULL;\n\t\tstate = URI_STATE_INIT;\n\t\terror = 0;\n\t}\n\n\tvoid deinit()\n\t{\n\t\tfree(scheme);\n\t\tfree(userinfo);\n\t\tfree(host);\n\t\tfree(port);\n\t\tfree(path);\n\t\tfree(query);\n\t\tfree(fragment);\n\t}\n\n\tvoid copy(const ParsedURI& uri);\n};\n\n// static class\nclass URIParser\n{\npublic:\n\t// return 0 mean succ, -1 mean fail\n\tstatic int parse(const char *str, ParsedURI& uri);\n\tstatic int parse(const std::string& str, ParsedURI& uri)\n\t{\n\t\treturn parse(str.c_str(), uri);\n\t}\n\n\tstatic std::map>\n\tsplit_query_strict(const std::string &query);\n\n\tstatic std::map\n\tsplit_query(const std::string &query);\n\n\tstatic std::vector split_path(const std::string &path);\n};\n\n#endif\n\n" }, { "path": "src/util/crc32c.c", "content": "/*\n ISC License\n\n Copyright (c) 2023, Antonio SJ Musumeci \n\n Permission to use, copy, modify, and/or distribute this software for any\n purpose with or without fee is hereby granted, provided that the above\n copyright notice and this permission notice appear in all copies.\n\n THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\n WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR\n ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN\n ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF\n OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n*/\n\n#include \"crc32c.h\"\n\n/*\n polynomial: 0x1EDC6F41\n reverse: true\n*/\nstatic\nconst\ncrc32c_t CRC32CTABLE[256] =\n {\n 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C,\n 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,\n 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, 0x105EC76F, 0xE235446C,\n 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,\n 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, 0x5D1D08BF, 0xAF768BBC,\n 0xBC267848, 0x4E4DFB4B, 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,\n 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, 0xAA64D611, 0x580F5512,\n 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,\n 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD,\n 0x1642AE59, 0xE4292D5A, 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,\n 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, 0x417B1DBC, 0xB3109EBF,\n 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,\n 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F,\n 0xED03A29B, 0x1F682198, 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,\n 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, 0xDBFC821C, 0x2997011F,\n 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,\n 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E,\n 0x4767748A, 0xB50CF789, 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,\n 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, 0x7198540D, 0x83F3D70E,\n 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,\n 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE,\n 0xDDE0EB2A, 0x2F8B6829, 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,\n 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, 0x082F63B7, 0xFA44E0B4,\n 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,\n 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B,\n 0xB4091BFF, 0x466298FC, 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,\n 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, 0xA24BB5A6, 0x502036A5,\n 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,\n 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975,\n 0x0E330A81, 0xFC588982, 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,\n 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, 0x38CC2A06, 0xCAA7A905,\n 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,\n 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8,\n 0xE52CC12C, 0x1747422F, 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,\n 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, 0xD3D3E1AB, 0x21B862A8,\n 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,\n 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78,\n 0x7FAB5E8C, 0x8DC0DD8F, 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,\n 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, 0x69E9F0D5, 0x9B8273D6,\n 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,\n 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, 0xC69F7B69,\n 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,\n 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351\n };\n\ncrc32c_t\ncrc32c_start(void)\n{\n return 0xFFFFFFFF;\n}\n\ncrc32c_t\ncrc32c_continue(const void *buf_,\n const crc32c_t len_,\n const crc32c_t crc_)\n{\n int i;\n char *buf;\n crc32c_t crc;\n\n crc = crc_;\n buf = (char*)buf_;\n for(i = 0; i < len_; i++)\n {\n crc = (CRC32CTABLE[(crc ^ buf[i]) & 0xFFL] ^ (crc >> 8));\n }\n\n return crc;\n}\n\ncrc32c_t\ncrc32c_finish(const crc32c_t crc_)\n{\n return (crc_ ^ 0xFFFFFFFF);\n}\n\ncrc32c_t\ncrc32c(const void *buf_,\n const crc32c_t len_)\n{\n crc32c_t crc;\n\n crc = crc32c_start();\n crc = crc32c_continue(buf_,len_,crc);\n crc = crc32c_finish(crc);\n\n return crc;\n}\n" }, { "path": "src/util/crc32c.h", "content": "/*\n ISC License\n\n Copyright (c) 2023, Antonio SJ Musumeci \n\n Permission to use, copy, modify, and/or distribute this software for any\n purpose with or without fee is hereby granted, provided that the above\n copyright notice and this permission notice appear in all copies.\n\n THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\n WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR\n ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN\n ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF\n OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n*/\n\n#ifndef CRC32C_H_INCLUDED\n#define CRC32C_H_INCLUDED\n\ntypedef unsigned int crc32c_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\ncrc32c_t crc32c_start(void);\ncrc32c_t crc32c_continue(const void *buf,\n const crc32c_t len,\n const crc32c_t crc);\ncrc32c_t crc32c_finish(const crc32c_t crc);\n\ncrc32c_t crc32c(const void *buf, crc32c_t len);\n\n#ifdef __cplusplus\n}\n#endif\n\n#endif\n" }, { "path": "src/util/json_parser.c", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"list.h\"\n#include \"json_parser.h\"\n\n#define JSON_DEPTH_LIMIT\t1024\n\nstruct __json_object\n{\n\tstruct list_head head;\n\tsize_t size;\n};\n\nstruct __json_array\n{\n\tstruct list_head head;\n\tsize_t size;\n};\n\nstruct __json_value\n{\n\tunion\n\t{\n\t\tchar *string;\n\t\tdouble number;\n\t\tjson_object_t object;\n\t\tjson_array_t array;\n\t} value;\n\tint type;\n};\n\nstruct __json_member\n{\n\tstruct list_head list;\n\tjson_value_t value;\n\tchar name[1];\n};\n\nstruct __json_element\n{\n\tstruct list_head list;\n\tjson_value_t value;\n};\n\ntypedef struct __json_member json_member_t;\ntypedef struct __json_element json_element_t;\n\nstatic const int __whitespace_map[256] = {\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t1,\n};\n\nstatic int __json_isspace(char c)\n{\n\treturn __whitespace_map[(unsigned char)c];\n}\n\nstatic int __json_isdigit(char c)\n{\n\treturn (unsigned int)(c - '0') <= 9;\n}\n\n#define isspace(c)\t__json_isspace(c)\n#define isdigit(c)\t__json_isdigit(c)\n\nstatic const int __character_map[256] = {\n\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n\t1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n\t0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\t0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n};\n\nstatic int __json_string_length(const char *cursor, size_t *escape, size_t *len)\n{\n\tsize_t esc = 0;\n\tsize_t n = 0;\n\n\twhile (1)\n\t{\n\t\twhile (__character_map[(unsigned char)cursor[n]] == 0)\n\t\t\tn++;\n\n\t\tif (cursor[n] == '\\\"')\n\t\t\tbreak;\n\n\t\tif (cursor[n] != '\\\\')\n\t\t\treturn -2;\n\n\t\tcursor++;\n\t\tif (cursor[n] == '\\0')\n\t\t\treturn -2;\n\n\t\tesc++;\n\t\tn++;\n\t}\n\n\t*escape = esc;\n\t*len = n;\n\treturn 0;\n}\n\nstatic int __parse_json_hex4(const char *cursor, const char **end,\n\t\t\t\t\t\t\t unsigned int *code)\n{\n\tint hex;\n\tint i;\n\n\t*code = 0;\n\tfor (i = 0; i < 4; i++)\n\t{\n\t\thex = *cursor;\n\t\tif (hex >= '0' && hex <= '9')\n\t\t\thex = hex - '0';\n\t\telse\n\t\t{\n\t\t\thex |= 0x20;\n\t\t\tif (hex >= 'a' && hex <= 'f')\n\t\t\t\thex = hex - 'a' + 10;\n\t\t\telse\n\t\t\t\treturn -2;\n\t\t}\n\n\t\t*code = (*code << 4) + hex;\n\t\tcursor++;\n\t}\n\n\t*end = cursor;\n\treturn 0;\n}\n\nstatic int __parse_json_unicode(const char *cursor, const char **end,\n\t\t\t\t\t\t\t\tchar *utf8)\n{\n\tunsigned int code;\n\tunsigned int next;\n\tint ret;\n\n\tret = __parse_json_hex4(cursor, end, &code);\n\tif (ret < 0)\n\t\treturn ret;\n\n\tif (code >= 0xdc00 && code <= 0xdfff)\n\t\treturn -2;\n\n\tif (code >= 0xd800 && code <= 0xdbff)\n\t{\n\t\tcursor = *end;\n\t\tif (*cursor != '\\\\')\n\t\t\treturn -2;\n\n\t\tcursor++;\n\t\tif (*cursor != 'u')\n\t\t\treturn -2;\n\n\t\tcursor++;\n\t\tret = __parse_json_hex4(cursor, end, &next);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tif (next < 0xdc00 || next > 0xdfff)\n\t\t\treturn -2;\n\n\t\tcode = (((code & 0x3ff) << 10) | (next & 0x3ff)) + 0x10000;\n\t}\n\n\tif (code <= 0x7f)\n\t{\n\t\tutf8[0] = code;\n\t\treturn 1;\n\t}\n\telse if (code <= 0x7ff)\n\t{\n\t\tutf8[0] = 0xc0 | (code >> 6);\n\t\tutf8[1] = 0x80 | (code & 0x3f);\n\t\treturn 2;\n\t}\n\telse if (code <= 0xffff)\n\t{\n\t\tutf8[0] = 0xe0 | (code >> 12);\n\t\tutf8[1] = 0x80 | ((code >> 6) & 0x3f);\n\t\tutf8[2] = 0x80 | (code & 0x3f);\n\t\treturn 3;\n\t}\n\telse\n\t{\n\t\tutf8[0] = 0xf0 | (code >> 18);\n\t\tutf8[1] = 0x80 | ((code >> 12) & 0x3f);\n\t\tutf8[2] = 0x80 | ((code >> 6) & 0x3f);\n\t\tutf8[3] = 0x80 | (code & 0x3f);\n\t\treturn 4;\n\t}\n}\n\nstatic int __parse_json_string(const char *cursor, const char **end,\n\t\t\t\t\t\t\t size_t escape, char *str)\n{\n\tint ret;\n\n\twhile (escape > 0)\n\t{\n\t\twhile (*cursor != '\\\\')\n\t\t{\n\t\t\t*str = *cursor;\n\t\t\tcursor++;\n\t\t\tstr++;\n\t\t}\n\n\t\tescape--;\n\t\tcursor++;\n\t\tswitch (*cursor)\n\t\t{\n\t\tcase '\\\"':\n\t\t\t*str = '\\\"';\n\t\t\tbreak;\n\t\tcase '\\\\':\n\t\t\t*str = '\\\\';\n\t\t\tbreak;\n\t\tcase '/':\n\t\t\t*str = '/';\n\t\t\tbreak;\n\t\tcase 'b':\n\t\t\t*str = '\\b';\n\t\t\tbreak;\n\t\tcase 'f':\n\t\t\t*str = '\\f';\n\t\t\tbreak;\n\t\tcase 'n':\n\t\t\t*str = '\\n';\n\t\t\tbreak;\n\t\tcase 'r':\n\t\t\t*str = '\\r';\n\t\t\tbreak;\n\t\tcase 't':\n\t\t\t*str = '\\t';\n\t\t\tbreak;\n\t\tcase 'u':\n\t\t\tcursor++;\n\t\t\tret = __parse_json_unicode(cursor, &cursor, str);\n\t\t\tif (ret < 0)\n\t\t\t\treturn ret;\n\n\t\t\tif (ret == 4)\n\t\t\t\tescape--;\n\n\t\t\tstr += ret;\n\t\t\tcontinue;\n\n\t\tdefault:\n\t\t\treturn -2;\n\t\t}\n\n\t\tcursor++;\n\t\tstr++;\n\t}\n\n\twhile (cursor[0] != '\\\"' && cursor[1] != '\\\"' &&\n\t\t cursor[2] != '\\\"' && cursor[3] != '\\\"')\n\t{\n\t\tmemcpy(str, cursor, 4);\n\t\tcursor += 4;\n\t\tstr += 4;\n\t}\n\n\tif (cursor[0] != '\\\"' && cursor[1] != '\\\"')\n\t{\n\t\tmemcpy(str, cursor, 2);\n\t\tcursor += 2;\n\t\tstr += 2;\n\t}\n\n\tif (*cursor != '\\\"')\n\t{\n\t\t*str = *cursor;\n\t\tcursor++;\n\t\tstr++;\n\t}\n\n\t*str = '\\0';\n\t*end = cursor + 1;\n\treturn 0;\n}\n\nstatic const double __power_of_10[309] = {\n\t1e0,\t1e1,\t1e2,\t1e3,\t1e4,\n\t1e5,\t1e6,\t1e7,\t1e8,\t1e9,\n\t1e10,\t1e11,\t1e12,\t1e13,\t1e14,\n\t1e15,\t1e16,\t1e17,\t1e18,\t1e19,\n\t1e20,\t1e21,\t1e22,\t1e23,\t1e24,\n\t1e25,\t1e26,\t1e27,\t1e28,\t1e29,\n\t1e30,\t1e31,\t1e32,\t1e33,\t1e34,\n\t1e35,\t1e36,\t1e37,\t1e38,\t1e39,\n\t1e40,\t1e41,\t1e42,\t1e43,\t1e44,\n\t1e45,\t1e46,\t1e47,\t1e48,\t1e49,\n\t1e50,\t1e51,\t1e52,\t1e53,\t1e54,\n\t1e55,\t1e56,\t1e57,\t1e58,\t1e59,\n\t1e60,\t1e61,\t1e62,\t1e63,\t1e64,\n\t1e65,\t1e66,\t1e67,\t1e68,\t1e69,\n\t1e70,\t1e71,\t1e72,\t1e73,\t1e74,\n\t1e75,\t1e76,\t1e77,\t1e78,\t1e79,\n\t1e80,\t1e81,\t1e82,\t1e83,\t1e84,\n\t1e85,\t1e86,\t1e87,\t1e88,\t1e89,\n\t1e90,\t1e91,\t1e92,\t1e93,\t1e94,\n\t1e95,\t1e96,\t1e97,\t1e98,\t1e99,\n\t1e100,\t1e101,\t1e102,\t1e103,\t1e104,\n\t1e105,\t1e106,\t1e107,\t1e108,\t1e109,\n\t1e110,\t1e111,\t1e112,\t1e113,\t1e114,\n\t1e115,\t1e116,\t1e117,\t1e118,\t1e119,\n\t1e120,\t1e121,\t1e122,\t1e123,\t1e124,\n\t1e125,\t1e126,\t1e127,\t1e128,\t1e129,\n\t1e130,\t1e131,\t1e132,\t1e133,\t1e134,\n\t1e135,\t1e136,\t1e137,\t1e138,\t1e139,\n\t1e140,\t1e141,\t1e142,\t1e143,\t1e144,\n\t1e145,\t1e146,\t1e147,\t1e148,\t1e149,\n\t1e150,\t1e151,\t1e152,\t1e153,\t1e154,\n\t1e155,\t1e156,\t1e157,\t1e158,\t1e159,\n\t1e160,\t1e161,\t1e162,\t1e163,\t1e164,\n\t1e165,\t1e166,\t1e167,\t1e168,\t1e169,\n\t1e170,\t1e171,\t1e172,\t1e173,\t1e174,\n\t1e175,\t1e176,\t1e177,\t1e178,\t1e179,\n\t1e180,\t1e181,\t1e182,\t1e183,\t1e184,\n\t1e185,\t1e186,\t1e187,\t1e188,\t1e189,\n\t1e190,\t1e191,\t1e192,\t1e193,\t1e194,\n\t1e195,\t1e196,\t1e197,\t1e198,\t1e199,\n\t1e200,\t1e201,\t1e202,\t1e203,\t1e204,\n\t1e205,\t1e206,\t1e207,\t1e208,\t1e209,\n\t1e210,\t1e211,\t1e212,\t1e213,\t1e214,\n\t1e215,\t1e216,\t1e217,\t1e218,\t1e219,\n\t1e220,\t1e221,\t1e222,\t1e223,\t1e224,\n\t1e225,\t1e226,\t1e227,\t1e228,\t1e229,\n\t1e230,\t1e231,\t1e232,\t1e233,\t1e234,\n\t1e235,\t1e236,\t1e237,\t1e238,\t1e239,\n\t1e240,\t1e241,\t1e242,\t1e243,\t1e244,\n\t1e245,\t1e246,\t1e247,\t1e248,\t1e249,\n\t1e250,\t1e251,\t1e252,\t1e253,\t1e254,\n\t1e255,\t1e256,\t1e257,\t1e258,\t1e259,\n\t1e260,\t1e261,\t1e262,\t1e263,\t1e264,\n\t1e265,\t1e266,\t1e267,\t1e268,\t1e269,\n\t1e270,\t1e271,\t1e272,\t1e273,\t1e274,\n\t1e275,\t1e276,\t1e277,\t1e278,\t1e279,\n\t1e280,\t1e281,\t1e282,\t1e283,\t1e284,\n\t1e285,\t1e286,\t1e287,\t1e288,\t1e289,\n\t1e290,\t1e291,\t1e292,\t1e293,\t1e294,\n\t1e295,\t1e296,\t1e297,\t1e298,\t1e299,\n\t1e300,\t1e301,\t1e302,\t1e303,\t1e304,\n\t1e305,\t1e306,\t1e307,\t1e308\n};\n\nstatic int __parse_json_number(const char *cursor, const char **end,\n\t\t\t\t\t\t\t double *num)\n{\n\tunsigned int digit;\n\tlong long exp = 0;\n\tlong long mant;\n\tint figures;\n\tint minus;\n\tdouble n;\n\n\tminus = (*cursor == '-');\n\tif (minus)\n\t\tcursor++;\n\n\tmant = *cursor - '0';\n\tif (mant >= 1 && mant <= 9)\n\t{\n\t\tfigures = 1;\n\t\tcursor++;\n\t\twhile (1)\n\t\t{\n\t\t\tdigit = (unsigned int)(*cursor - '0');\n\t\t\tif (digit <= 9 && figures < 18)\n\t\t\t{\n\t\t\t\tmant = 10 * mant + digit;\n\t\t\t\tfigures++;\n\t\t\t\tcursor++;\n\t\t\t}\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\twhile (isdigit(*cursor))\n\t\t{\n\t\t\texp++;\n\t\t\tcursor++;\n\t\t}\n\t}\n\telse if (mant == 0)\n\t{\n\t\tfigures = 0;\n\t\tcursor++;\n\t}\n\telse\n\t\treturn -2;\n\n\tif (*cursor == '.')\n\t{\n\t\tconst char *frac;\n\n\t\tcursor++;\n\t\tif (!isdigit(*cursor))\n\t\t\treturn -2;\n\n\t\tfrac = cursor;\n\t\tif (figures == 0)\n\t\t{\n\t\t\twhile (*cursor == '0')\n\t\t\t\tcursor++;\n\t\t}\n\n\t\twhile (1)\n\t\t{\n\t\t\tdigit = (unsigned int)(*cursor - '0');\n\t\t\tif (digit <= 9 && figures < 18)\n\t\t\t{\n\t\t\t\tmant = 10 * mant + digit;\n\t\t\t\tfigures++;\n\t\t\t\tcursor++;\n\t\t\t}\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\texp -= cursor - frac;\n\t\twhile (isdigit(*cursor))\n\t\t\tcursor++;\n\t}\n\n\tif (*cursor == 'E' || *cursor == 'e')\n\t{\n\t\tlong long e;\n\t\tchar sign;\n\n\t\tcursor++;\n\t\tsign = *cursor;\n\t\tif (sign == '+' || sign == '-')\n\t\t\tcursor++;\n\n\t\tif (!isdigit(*cursor))\n\t\t\treturn -2;\n\n\t\te = *cursor - '0';\n\t\tcursor++;\n\t\twhile (1)\n\t\t{\n\t\t\tdigit = (unsigned int)(*cursor - '0');\n\t\t\tif (digit <= 9 && e < 100000000000000000)\n\t\t\t{\n\t\t\t\te = 10 * e + digit;\n\t\t\t\tcursor++;\n\t\t\t}\n\t\t\telse\n\t\t\t\tbreak;\n\t\t}\n\n\t\twhile (isdigit(*cursor))\n\t\t\tcursor++;\n\n\t\tif (sign == '-')\n\t\t\te = -e;\n\n\t\texp += e;\n\t}\n\n\tif (exp != 0 && figures != 0)\n\t{\n\t\twhile (exp > 0 && figures < 18)\n\t\t{\n\t\t\tmant *= 10;\n\t\t\tfigures++;\n\t\t\texp--;\n\t\t}\n\n\t\twhile (exp < 0 && mant % 10 == 0)\n\t\t{\n\t\t\tmant /= 10;\n\t\t\tfigures--;\n\t\t\texp++;\n\t\t}\n\t}\n\n\tn = mant;\n\tif (exp != 0 && figures != 0)\n\t{\n\t\tif (exp > 0)\n\t\t{\n\t\t\tif (exp > 291)\n\t\t\t\tn = INFINITY;\n\t\t\telse\n\t\t\t\tn *= __power_of_10[exp];\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (exp > -309)\n\t\t\t\tn /= __power_of_10[-exp];\n\t\t\telse if (exp > -324 - figures)\n\t\t\t{\n\t\t\t\tn /= __power_of_10[-exp - 308];\n\t\t\t\tn /= __power_of_10[308];\n\t\t\t}\n\t\t\telse\n\t\t\t\tn = 0.0;\n\t\t}\n\t}\n\n\tif (minus)\n\t\tn = -n;\n\n\t*num = n;\n\t*end = cursor;\n\treturn 0;\n}\n\nstatic int __parse_json_value(const char *cursor, const char **end,\n\t\t\t\t\t\t\t int depth, json_value_t *val);\n\nstatic void __destroy_json_value(json_value_t *val);\n\nstatic int __parse_json_member(const char *cursor, const char **end,\n\t\t\t\t\t\t\t size_t escape, size_t len,\n\t\t\t\t\t\t\t int depth, json_member_t *memb)\n{\n\tint ret;\n\n\tif (escape != 0)\n\t{\n\t\tret = __parse_json_string(cursor, &cursor, escape, memb->name);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\t}\n\telse\n\t{\n\t\tmemcpy(memb->name, cursor, len);\n\t\tmemb->name[len] = '\\0';\n\t\tcursor += len + 1;\n\t}\n\n\twhile (isspace(*cursor))\n\t\tcursor++;\n\n\tif (*cursor != ':')\n\t\treturn -2;\n\n\tcursor++;\n\twhile (isspace(*cursor))\n\t\tcursor++;\n\n\tret = __parse_json_value(cursor, &cursor, depth, &memb->value);\n\tif (ret < 0)\n\t\treturn ret;\n\n\t*end = cursor;\n\treturn 0;\n}\n\nstatic int __parse_json_members(const char *cursor, const char **end,\n\t\t\t\t\t\t\t\tint depth, json_object_t *obj)\n{\n\tjson_member_t *memb;\n\tsize_t escape;\n\tsize_t len;\n\tint ret;\n\n\twhile (isspace(*cursor))\n\t\tcursor++;\n\n\tif (*cursor == '}')\n\t{\n\t\t*end = cursor + 1;\n\t\treturn 0;\n\t}\n\n\twhile (1)\n\t{\n\t\tif (*cursor != '\\\"')\n\t\t\treturn -2;\n\n\t\tcursor++;\n\t\tret = __json_string_length(cursor, &escape, &len);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tmemb = (json_member_t *)malloc(offsetof(json_member_t, name) + len + 1);\n\t\tif (!memb)\n\t\t\treturn -1;\n\n\t\tret = __parse_json_member(cursor, &cursor, escape, len, depth, memb);\n\t\tif (ret < 0)\n\t\t{\n\t\t\tfree(memb);\n\t\t\treturn ret;\n\t\t}\n\n\t\tlist_add_tail(&memb->list, &obj->head);\n\t\tobj->size++;\n\n\t\twhile (isspace(*cursor))\n\t\t\tcursor++;\n\n\t\tif (*cursor == ',')\n\t\t{\n\t\t\tcursor++;\n\t\t\twhile (isspace(*cursor))\n\t\t\t\tcursor++;\n\t\t}\n\t\telse if (*cursor == '}')\n\t\t\tbreak;\n\t\telse\n\t\t\treturn -2;\n\t}\n\n\t*end = cursor + 1;\n\treturn 0;\n}\n\nstatic void __destroy_json_members(json_object_t *obj)\n{\n\tstruct list_head *pos, *tmp;\n\tjson_member_t *memb;\n\n\tlist_for_each_safe(pos, tmp, &obj->head)\n\t{\n\t\tmemb = list_entry(pos, json_member_t, list);\n\t\t__destroy_json_value(&memb->value);\n\t\tfree(memb);\n\t}\n}\n\nstatic int __parse_json_object(const char *cursor, const char **end,\n\t\t\t\t\t\t\t int depth, json_object_t *obj)\n{\n\tint ret;\n\n\tif (depth == JSON_DEPTH_LIMIT)\n\t\treturn -3;\n\n\tINIT_LIST_HEAD(&obj->head);\n\tobj->size = 0;\n\tret = __parse_json_members(cursor, end, depth + 1, obj);\n\tif (ret < 0)\n\t{\n\t\t__destroy_json_members(obj);\n\t\treturn ret;\n\t}\n\n\treturn 0;\n}\n\nstatic int __parse_json_elements(const char *cursor, const char **end,\n\t\t\t\t\t\t\t\t int depth, json_array_t *arr)\n{\n\tjson_element_t *elem;\n\tint ret;\n\n\twhile (isspace(*cursor))\n\t\tcursor++;\n\n\tif (*cursor == ']')\n\t{\n\t\t*end = cursor + 1;\n\t\treturn 0;\n\t}\n\n\twhile (1)\n\t{\n\t\telem = (json_element_t *)malloc(sizeof (json_element_t));\n\t\tif (!elem)\n\t\t\treturn -1;\n\n\t\tret = __parse_json_value(cursor, &cursor, depth, &elem->value);\n\t\tif (ret < 0)\n\t\t{\n\t\t\tfree(elem);\n\t\t\treturn ret;\n\t\t}\n\n\t\tlist_add_tail(&elem->list, &arr->head);\n\t\tarr->size++;\n\n\t\twhile (isspace(*cursor))\n\t\t\tcursor++;\n\n\t\tif (*cursor == ',')\n\t\t{\n\t\t\tcursor++;\n\t\t\twhile (isspace(*cursor))\n\t\t\t\tcursor++;\n\t\t}\n\t\telse if (*cursor == ']')\n\t\t\tbreak;\n\t\telse\n\t\t\treturn -2;\n\t}\n\n\t*end = cursor + 1;\n\treturn 0;\n}\n\nstatic void __destroy_json_elements(json_array_t *arr)\n{\n\tstruct list_head *pos, *tmp;\n\tjson_element_t *elem;\n\n\tlist_for_each_safe(pos, tmp, &arr->head)\n\t{\n\t\telem = list_entry(pos, json_element_t, list);\n\t\t__destroy_json_value(&elem->value);\n\t\tfree(elem);\n\t}\n}\n\nstatic int __parse_json_array(const char *cursor, const char **end,\n\t\t\t\t\t\t\t int depth, json_array_t *arr)\n{\n\tint ret;\n\n\tif (depth == JSON_DEPTH_LIMIT)\n\t\treturn -3;\n\n\tINIT_LIST_HEAD(&arr->head);\n\tarr->size = 0;\n\tret = __parse_json_elements(cursor, end, depth + 1, arr);\n\tif (ret < 0)\n\t{\n\t\t__destroy_json_elements(arr);\n\t\treturn ret;\n\t}\n\n\treturn 0;\n}\n\nstatic int __parse_json_value(const char *cursor, const char **end,\n\t\t\t\t\t\t\t int depth, json_value_t *val)\n{\n\tsize_t escape;\n\tsize_t len;\n\tint ret;\n\n\tswitch (*cursor)\n\t{\n\tcase '\\\"':\n\t\tcursor++;\n\t\tret = __json_string_length(cursor, &escape, &len);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tval->value.string = (char *)malloc(len + 1);\n\t\tif (!val->value.string)\n\t\t\treturn -1;\n\n\t\tif (escape != 0)\n\t\t{\n\t\t\tret = __parse_json_string(cursor, end, escape, val->value.string);\n\t\t\tif (ret < 0)\n\t\t\t{\n\t\t\t\tfree(val->value.string);\n\t\t\t\treturn ret;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tmemcpy(val->value.string, cursor, len);\n\t\t\tval->value.string[len] = '\\0';\n\t\t\t*end = cursor + len + 1;\n\t\t}\n\n\t\tval->type = JSON_VALUE_STRING;\n\t\tbreak;\n\n\tcase '-':\n\tcase '0':\n\tcase '1':\n\tcase '2':\n\tcase '3':\n\tcase '4':\n\tcase '5':\n\tcase '6':\n\tcase '7':\n\tcase '8':\n\tcase '9':\n\t\tret = __parse_json_number(cursor, end, &val->value.number);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tval->type = JSON_VALUE_NUMBER;\n\t\tbreak;\n\n\tcase '{':\n\t\tcursor++;\n\t\tret = __parse_json_object(cursor, end, depth, &val->value.object);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tval->type = JSON_VALUE_OBJECT;\n\t\tbreak;\n\n\tcase '[':\n\t\tcursor++;\n\t\tret = __parse_json_array(cursor, end, depth, &val->value.array);\n\t\tif (ret < 0)\n\t\t\treturn ret;\n\n\t\tval->type = JSON_VALUE_ARRAY;\n\t\tbreak;\n\n\tcase 't':\n\t\tif (strncmp(cursor, \"true\", 4) != 0)\n\t\t\treturn -2;\n\n\t\t*end = cursor + 4;\n\t\tval->type = JSON_VALUE_TRUE;\n\t\tbreak;\n\n\tcase 'f':\n\t\tif (strncmp(cursor, \"false\", 5) != 0)\n\t\t\treturn -2;\n\n\t\t*end = cursor + 5;\n\t\tval->type = JSON_VALUE_FALSE;\n\t\tbreak;\n\n\tcase 'n':\n\t\tif (strncmp(cursor, \"null\", 4) != 0)\n\t\t\treturn -2;\n\n\t\t*end = cursor + 4;\n\t\tval->type = JSON_VALUE_NULL;\n\t\tbreak;\n\n\tdefault:\n\t\treturn -2;\n\t}\n\n\treturn 0;\n}\n\nstatic void __destroy_json_value(json_value_t *val)\n{\n\tswitch (val->type)\n\t{\n\tcase JSON_VALUE_STRING:\n\t\tfree(val->value.string);\n\t\tbreak;\n\n\tcase JSON_VALUE_OBJECT:\n\t\t__destroy_json_members(&val->value.object);\n\t\tbreak;\n\n\tcase JSON_VALUE_ARRAY:\n\t\t__destroy_json_elements(&val->value.array);\n\t\tbreak;\n\t}\n}\n\njson_value_t *json_value_parse(const char *cursor)\n{\n\tjson_value_t *val;\n\n\tval = (json_value_t *)malloc(sizeof (json_value_t));\n\tif (!val)\n\t\treturn NULL;\n\n\twhile (isspace(*cursor))\n\t\tcursor++;\n\n\tif (__parse_json_value(cursor, &cursor, 0, val) >= 0)\n\t{\n\t\twhile (isspace(*cursor))\n\t\t\tcursor++;\n\n\t\tif (*cursor == '\\0')\n\t\t\treturn val;\n\n\t\t__destroy_json_value(val);\n\t}\n\n\tfree(val);\n\treturn NULL;\n}\n\nstatic void __move_json_value(json_value_t *src, json_value_t *dest)\n{\n\tswitch (src->type)\n\t{\n\tcase JSON_VALUE_STRING:\n\t\tdest->value.string = src->value.string;\n\t\tbreak;\n\n\tcase JSON_VALUE_NUMBER:\n\t\tdest->value.number = src->value.number;\n\t\tbreak;\n\n\tcase JSON_VALUE_OBJECT:\n\t\tINIT_LIST_HEAD(&dest->value.object.head);\n\t\tlist_splice(&src->value.object.head, &dest->value.object.head);\n\t\tdest->value.object.size = src->value.object.size;\n\t\tbreak;\n\n\tcase JSON_VALUE_ARRAY:\n\t\tINIT_LIST_HEAD(&dest->value.array.head);\n\t\tlist_splice(&src->value.array.head, &dest->value.array.head);\n\t\tdest->value.array.size = src->value.array.size;\n\t\tbreak;\n\t}\n\n\tdest->type = src->type;\n}\n\nstatic int __set_json_value(int type, va_list ap, json_value_t *val)\n{\n\tjson_value_t *src;\n\tconst char *str;\n\tsize_t len;\n\n\tswitch (type)\n\t{\n\tcase 0:\n\t\tsrc = va_arg(ap, json_value_t *);\n\t\t__move_json_value(src, val);\n\t\tfree(src);\n\t\treturn 0;\n\n\tcase JSON_VALUE_STRING:\n\t\tstr = va_arg(ap, const char *);\n\t\tlen = strlen(str);\n\t\tval->value.string = (char *)malloc(len + 1);\n\t\tif (!val->value.string)\n\t\t\treturn -1;\n\n\t\tmemcpy(val->value.string, str, len + 1);\n\t\tbreak;\n\n\tcase JSON_VALUE_NUMBER:\n\t\tval->value.number = va_arg(ap, double);\n\t\tbreak;\n\n\tcase JSON_VALUE_OBJECT:\n\t\tINIT_LIST_HEAD(&val->value.object.head);\n\t\tval->value.object.size = 0;\n\t\tbreak;\n\n\tcase JSON_VALUE_ARRAY:\n\t\tINIT_LIST_HEAD(&val->value.array.head);\n\t\tval->value.array.size = 0;\n\t\tbreak;\n\t}\n\n\tval->type = type;\n\treturn 0;\n}\n\njson_value_t *json_value_create(int type, ...)\n{\n\tjson_value_t *val;\n\tva_list ap;\n\tint ret;\n\n\tval = (json_value_t *)malloc(sizeof (json_value_t));\n\tif (!val)\n\t\treturn NULL;\n\n\tva_start(ap, type);\n\tret = __set_json_value(type, ap, val);\n\tva_end(ap);\n\tif (ret >= 0)\n\t\treturn val;\n\n\tfree(val);\n\treturn NULL;\n}\n\nstatic int __copy_json_value(const json_value_t *src, json_value_t *dest);\n\nstatic int __copy_json_members(const json_object_t *src, json_object_t *dest)\n{\n\tstruct list_head *pos;\n\tjson_member_t *entry;\n\tjson_member_t *memb;\n\tsize_t len;\n\tint ret;\n\n\tlist_for_each(pos, &src->head)\n\t{\n\t\tentry = list_entry(pos, json_member_t, list);\n\t\tlen = strlen(entry->name);\n\t\tmemb = (json_member_t *)malloc(offsetof(json_member_t, name) + len + 1);\n\t\tif (!memb)\n\t\t\treturn -1;\n\n\t\tret = __copy_json_value(&entry->value, &memb->value);\n\t\tif (ret < 0)\n\t\t{\n\t\t\tfree(memb);\n\t\t\treturn ret;\n\t\t}\n\n\t\tmemcpy(memb->name, entry->name, len + 1);\n\t\tlist_add_tail(&memb->list, &dest->head);\n\t\tdest->size++;\n\t}\n\n\treturn 0;\n}\n\nstatic int __copy_json_elements(const json_array_t *src, json_array_t *dest)\n{\n\tstruct list_head *pos;\n\tjson_element_t *entry;\n\tjson_element_t *elem;\n\tint ret;\n\n\tlist_for_each(pos, &src->head)\n\t{\n\t\telem = (json_element_t *)malloc(sizeof (json_element_t));\n\t\tif (!elem)\n\t\t\treturn -1;\n\n\t\tentry = list_entry(pos, json_element_t, list);\n\t\tret = __copy_json_value(&entry->value, &elem->value);\n\t\tif (ret < 0)\n\t\t{\n\t\t\tfree(elem);\n\t\t\treturn ret;\n\t\t}\n\n\t\tlist_add_tail(&elem->list, &dest->head);\n\t\tdest->size++;\n\t}\n\n\treturn 0;\n}\n\nstatic int __copy_json_value(const json_value_t *src, json_value_t *dest)\n{\n\tsize_t len;\n\tint ret;\n\n\tswitch (src->type)\n\t{\n\tcase JSON_VALUE_STRING:\n\t\tlen = strlen(src->value.string);\n\t\tdest->value.string = (char *)malloc(len + 1);\n\t\tif (!dest->value.string)\n\t\t\treturn -1;\n\n\t\tmemcpy(dest->value.string, src->value.string, len + 1);\n\t\tbreak;\n\n\tcase JSON_VALUE_NUMBER:\n\t\tdest->value.number = src->value.number;\n\t\tbreak;\n\n\tcase JSON_VALUE_OBJECT:\n\t\tINIT_LIST_HEAD(&dest->value.object.head);\n\t\tdest->value.object.size = 0;\n\t\tret = __copy_json_members(&src->value.object, &dest->value.object);\n\t\tif (ret < 0)\n\t\t{\n\t\t\t__destroy_json_members(&dest->value.object);\n\t\t\treturn ret;\n\t\t}\n\n\t\tbreak;\n\n\tcase JSON_VALUE_ARRAY:\n\t\tINIT_LIST_HEAD(&dest->value.array.head);\n\t\tdest->value.array.size = 0;\n\t\tret = __copy_json_elements(&src->value.array, &dest->value.array);\n\t\tif (ret < 0)\n\t\t{\n\t\t\t__destroy_json_elements(&dest->value.array);\n\t\t\treturn ret;\n\t\t}\n\n\t\tbreak;\n\t}\n\n\tdest->type = src->type;\n\treturn 0;\n}\n\njson_value_t *json_value_copy(const json_value_t *val)\n{\n\tjson_value_t *copy;\n\n\tcopy = (json_value_t *)malloc(sizeof (json_value_t));\n\tif (!copy)\n\t\treturn NULL;\n\n\tif (__copy_json_value(val, copy) >= 0)\n\t\treturn copy;\n\n\tfree(copy);\n\treturn NULL;\n}\n\nvoid json_value_destroy(json_value_t *val)\n{\n\t__destroy_json_value(val);\n\tfree(val);\n}\n\nint json_value_type(const json_value_t *val)\n{\n\treturn val->type;\n}\n\nconst char *json_value_string(const json_value_t *val)\n{\n\tif (val->type != JSON_VALUE_STRING)\n\t\treturn NULL;\n\n\treturn val->value.string;\n}\n\ndouble json_value_number(const json_value_t *val)\n{\n\tif (val->type != JSON_VALUE_NUMBER)\n\t\treturn NAN;\n\n\treturn val->value.number;\n}\n\njson_object_t *json_value_object(const json_value_t *val)\n{\n\tif (val->type != JSON_VALUE_OBJECT)\n\t\treturn NULL;\n\n\treturn (json_object_t *)&val->value.object;\n}\n\njson_array_t *json_value_array(const json_value_t *val)\n{\n\tif (val->type != JSON_VALUE_ARRAY)\n\t\treturn NULL;\n\n\treturn (json_array_t *)&val->value.array;\n}\n\nconst json_value_t *json_object_find(const char *name,\n\t\t\t\t\t\t\t\t\t const json_object_t *obj)\n{\n\tstruct list_head *pos;\n\tjson_member_t *memb;\n\n\tlist_for_each(pos, &obj->head)\n\t{\n\t\tmemb = list_entry(pos, json_member_t, list);\n\t\tif (strcmp(name, memb->name) == 0)\n\t\t\treturn &memb->value;\n\t}\n\n\treturn NULL;\n}\n\nsize_t json_object_size(const json_object_t *obj)\n{\n\treturn obj->size;\n}\n\nconst char *json_object_next_name(const char *name,\n\t\t\t\t\t\t\t\t const json_object_t *obj)\n{\n\tconst struct list_head *pos;\n\n\tif (name)\n\t\tpos = &list_entry(name, json_member_t, name)->list;\n\telse\n\t\tpos = &obj->head;\n\n\tif (pos->next == &obj->head)\n\t\treturn NULL;\n\n\treturn list_entry(pos->next, json_member_t, list)->name;\n}\n\nconst json_value_t *json_object_next_value(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t const json_object_t *obj)\n{\n\tconst struct list_head *pos;\n\n\tif (val)\n\t\tpos = &list_entry(val, json_member_t, value)->list;\n\telse\n\t\tpos = &obj->head;\n\n\tif (pos->next == &obj->head)\n\t\treturn NULL;\n\n\treturn &list_entry(pos->next, json_member_t, list)->value;\n}\n\nconst char *json_object_prev_name(const char *name,\n\t\t\t\t\t\t\t\t const json_object_t *obj)\n{\n\tconst struct list_head *pos;\n\n\tif (name)\n\t\tpos = &list_entry(name, json_member_t, name)->list;\n\telse\n\t\tpos = &obj->head;\n\n\tif (pos->prev == &obj->head)\n\t\treturn NULL;\n\n\treturn list_entry(pos->prev, json_member_t, list)->name;\n}\n\nconst json_value_t *json_object_prev_value(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t const json_object_t *obj)\n{\n\tconst struct list_head *pos;\n\n\tif (val)\n\t\tpos = &list_entry(val, json_member_t, value)->list;\n\telse\n\t\tpos = &obj->head;\n\n\tif (pos->prev == &obj->head)\n\t\treturn NULL;\n\n\treturn &list_entry(pos->prev, json_member_t, list)->value;\n}\n\nconst char *json_object_value_name(const json_value_t *val,\n\t\t\t\t\t\t\t\t const json_object_t *obj)\n{\n\treturn list_entry(val, json_member_t, value)->name;\n}\n\nstatic const json_value_t *__json_object_insert(const char *name,\n\t\t\t\t\t\t\t\t\t\t\t\tint type, va_list ap,\n\t\t\t\t\t\t\t\t\t\t\t\tstruct list_head *pos,\n\t\t\t\t\t\t\t\t\t\t\t\tjson_object_t *obj)\n{\n\tjson_member_t *memb;\n\tsize_t len;\n\n\tlen = strlen(name);\n\tmemb = (json_member_t *)malloc(offsetof(json_member_t, name) + len + 1);\n\tif (!memb)\n\t\treturn NULL;\n\n\tmemcpy(memb->name, name, len + 1);\n\tif (__set_json_value(type, ap, &memb->value) < 0)\n\t{\n\t\tfree(memb);\n\t\treturn NULL;\n\t}\n\n\tlist_add(&memb->list, pos);\n\tobj->size++;\n\treturn &memb->value;\n}\n\nconst json_value_t *json_object_append(json_object_t *obj,\n\t\t\t\t\t\t\t\t\t const char *name,\n\t\t\t\t\t\t\t\t\t int type, ...)\n{\n\tconst json_value_t *val;\n\tva_list ap;\n\n\tva_start(ap, type);\n\tval = __json_object_insert(name, type, ap, obj->head.prev, obj);\n\tva_end(ap);\n\treturn val;\n}\n\nconst json_value_t *json_object_insert_after(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t\t json_object_t *obj,\n\t\t\t\t\t\t\t\t\t\t\t const char *name,\n\t\t\t\t\t\t\t\t\t\t\t int type, ...)\n{\n\tstruct list_head *pos;\n\tva_list ap;\n\n\tif (val)\n\t\tpos = &list_entry(val, json_member_t, value)->list;\n\telse\n\t\tpos = &obj->head;\n\n\tva_start(ap, type);\n\tval = __json_object_insert(name, type, ap, pos, obj);\n\tva_end(ap);\n\treturn val;\n}\n\nconst json_value_t *json_object_insert_before(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t\t json_object_t *obj,\n\t\t\t\t\t\t\t\t\t\t\t const char *name,\n\t\t\t\t\t\t\t\t\t\t\t int type, ...)\n{\n\tstruct list_head *pos;\n\tva_list ap;\n\n\tif (val)\n\t\tpos = &list_entry(val, json_member_t, value)->list;\n\telse\n\t\tpos = &obj->head;\n\n\tva_start(ap, type);\n\tval = __json_object_insert(name, type, ap, pos->prev, obj);\n\tva_end(ap);\n\treturn val;\n}\n\njson_value_t *json_object_remove(const json_value_t *val,\n\t\t\t\t\t\t\t\t json_object_t *obj)\n{\n\tjson_member_t *memb = list_entry(val, json_member_t, value);\n\n\tval = (json_value_t *)malloc(sizeof (json_value_t));\n\tif (!val)\n\t\treturn NULL;\n\n\tlist_del(&memb->list);\n\tobj->size--;\n\n\t__move_json_value(&memb->value, (json_value_t *)val);\n\tfree(memb);\n\treturn (json_value_t *)val;\n}\n\nsize_t json_array_size(const json_array_t *arr)\n{\n\treturn arr->size;\n}\n\nconst json_value_t *json_array_next_value(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t const json_array_t *arr)\n{\n\tconst struct list_head *pos;\n\n\tif (val)\n\t\tpos = &list_entry(val, json_element_t, value)->list;\n\telse\n\t\tpos = &arr->head;\n\n\tif (pos->next == &arr->head)\n\t\treturn NULL;\n\n\treturn &list_entry(pos->next, json_element_t, list)->value;\n}\n\nconst json_value_t *json_array_prev_value(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t const json_array_t *arr)\n{\n\tconst struct list_head *pos;\n\n\tif (val)\n\t\tpos = &list_entry(val, json_element_t, value)->list;\n\telse\n\t\tpos = &arr->head;\n\n\tif (pos->prev == &arr->head)\n\t\treturn NULL;\n\n\treturn &list_entry(pos->prev, json_element_t, list)->value;\n}\n\nstatic const json_value_t *__json_array_insert(int type, va_list ap,\n\t\t\t\t\t\t\t\t\t\t\t struct list_head *pos,\n\t\t\t\t\t\t\t\t\t\t\t json_array_t *arr)\n{\n\tjson_element_t *elem;\n\n\telem = (json_element_t *)malloc(sizeof (json_element_t));\n\tif (!elem)\n\t\treturn NULL;\n\n\tif (__set_json_value(type, ap, &elem->value) < 0)\n\t{\n\t\tfree(elem);\n\t\treturn NULL;\n\t}\n\n\tlist_add(&elem->list, pos);\n\tarr->size++;\n\treturn &elem->value;\n}\n\nconst json_value_t *json_array_append(json_array_t *arr,\n\t\t\t\t\t\t\t\t\t int type, ...)\n{\n\tconst json_value_t *val;\n\tva_list ap;\n\n\tva_start(ap, type);\n\tval = __json_array_insert(type, ap, arr->head.prev, arr);\n\tva_end(ap);\n\treturn val;\n}\n\nconst json_value_t *json_array_insert_after(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t\tjson_array_t *arr,\n\t\t\t\t\t\t\t\t\t\t\tint type, ...)\n{\n\tstruct list_head *pos;\n\tva_list ap;\n\n\tif (val)\n\t\tpos = &list_entry(val, json_element_t, value)->list;\n\telse\n\t\tpos = &arr->head;\n\n\tva_start(ap, type);\n\tval = __json_array_insert(type, ap, pos, arr);\n\tva_end(ap);\n\treturn val;\n}\n\nconst json_value_t *json_array_insert_before(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t\t json_array_t *arr,\n\t\t\t\t\t\t\t\t\t\t\t int type, ...)\n{\n\tstruct list_head *pos;\n\tva_list ap;\n\n\tif (val)\n\t\tpos = &list_entry(val, json_element_t, value)->list;\n\telse\n\t\tpos = &arr->head;\n\n\tva_start(ap, type);\n\tval = __json_array_insert(type, ap, pos->prev, arr);\n\tva_end(ap);\n\treturn val;\n}\n\njson_value_t *json_array_remove(const json_value_t *val,\n\t\t\t\t\t\t\t\tjson_array_t *arr)\n{\n\tjson_element_t *elem = list_entry(val, json_element_t, value);\n\n\tval = (json_value_t *)malloc(sizeof (json_value_t));\n\tif (!val)\n\t\treturn NULL;\n\n\tlist_del(&elem->list);\n\tarr->size--;\n\n\t__move_json_value(&elem->value, (json_value_t *)val);\n\tfree(elem);\n\treturn (json_value_t *)val;\n}\n\n" }, { "path": "src/util/json_parser.h", "content": "/*\n Copyright (c) 2022 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Xie Han (xiehan@sogou-inc.com)\n*/\n\n#ifndef _JSON_PARSER_H_\n#define _JSON_PARSER_H_\n\n#include \n\n#define JSON_VALUE_STRING\t1\n#define JSON_VALUE_NUMBER\t2\n#define JSON_VALUE_OBJECT\t3\n#define JSON_VALUE_ARRAY\t4\n#define JSON_VALUE_TRUE\t\t5\n#define JSON_VALUE_FALSE\t6\n#define JSON_VALUE_NULL\t\t7\n\ntypedef struct __json_value json_value_t;\ntypedef struct __json_object json_object_t;\ntypedef struct __json_array json_array_t;\n\n#ifdef __cplusplus\nextern \"C\"\n{\n#endif\n\njson_value_t *json_value_parse(const char *text);\njson_value_t *json_value_create(int type, ...);\njson_value_t *json_value_copy(const json_value_t *val);\nvoid json_value_destroy(json_value_t *val);\n\nint json_value_type(const json_value_t *val);\nconst char *json_value_string(const json_value_t *val);\ndouble json_value_number(const json_value_t *val);\njson_object_t *json_value_object(const json_value_t *val);\njson_array_t *json_value_array(const json_value_t *val);\n\nconst json_value_t *json_object_find(const char *name,\n\t\t\t\t\t\t\t\t\t const json_object_t *obj);\nsize_t json_object_size(const json_object_t *obj);\nconst char *json_object_next_name(const char *name,\n\t\t\t\t\t\t\t\t const json_object_t *obj);\nconst json_value_t *json_object_next_value(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t const json_object_t *obj);\nconst char *json_object_prev_name(const char *name,\n\t\t\t\t\t\t\t\t const json_object_t *obj);\nconst json_value_t *json_object_prev_value(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t const json_object_t *obj);\nconst char *json_object_value_name(const json_value_t *val,\n\t\t\t\t\t\t\t\t const json_object_t *obj);\nconst json_value_t *json_object_append(json_object_t *obj,\n\t\t\t\t\t\t\t\t\t const char *name,\n\t\t\t\t\t\t\t\t\t int type, ...);\nconst json_value_t *json_object_insert_after(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t\t json_object_t *obj,\n\t\t\t\t\t\t\t\t\t\t\t const char *name,\n\t\t\t\t\t\t\t\t\t\t\t int type, ...);\nconst json_value_t *json_object_insert_before(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t\t json_object_t *obj,\n\t\t\t\t\t\t\t\t\t\t\t const char *name,\n\t\t\t\t\t\t\t\t\t\t\t int type, ...);\njson_value_t *json_object_remove(const json_value_t *val,\n\t\t\t\t\t\t\t\t json_object_t *obj);\n\nsize_t json_array_size(const json_array_t *arr);\nconst json_value_t *json_array_next_value(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t const json_array_t *arr);\nconst json_value_t *json_array_prev_value(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t const json_array_t *arr);\nconst json_value_t *json_array_append(json_array_t *arry,\n\t\t\t\t\t\t\t\t\t int type, ...);\nconst json_value_t *json_array_insert_after(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t\tjson_array_t *arr,\n\t\t\t\t\t\t\t\t\t\t\tint type, ...);\nconst json_value_t *json_array_insert_before(const json_value_t *val,\n\t\t\t\t\t\t\t\t\t\t\t json_array_t *arr,\n\t\t\t\t\t\t\t\t\t\t\t int type, ...);\njson_value_t *json_array_remove(const json_value_t *val,\n\t\t\t\t\t\t\t\tjson_array_t *arr);\n\n#ifdef __cplusplus\n}\n#endif\n\n#define json_object_for_each(name, val, obj) \\\n\tfor (name = NULL, val = NULL; \\\n\t\t name = json_object_next_name(name, obj), \\\n\t\t val = json_object_next_value(val, obj), val; )\n\n#define json_object_for_each_prev(name, val, obj) \\\n\tfor (name = NULL, val = NULL; \\\n\t\t name = json_object_prev_name(name, obj), \\\n\t\t val = json_object_prev_value(val, obj), val; )\n\n#define json_array_for_each(val, arr) \\\n\tfor (val = NULL; val = json_array_next_value(val, arr), val; )\n\n#define json_array_for_each_prev(val, arr) \\\n\tfor (val = NULL; val = json_array_prev_value(val, arr), val; )\n\n#endif\n\n" }, { "path": "src/util/xmake.lua", "content": "target(\"util\")\n set_kind(\"object\")\n add_files(\"*.c\")\n add_files(\"*.cc\")\n remove_files(\"crc32c.c\")\n\ntarget(\"kafka_util\")\n if has_config(\"kafka\") then\n set_kind(\"object\")\n add_files(\"crc32c.c\")\n else\n set_kind(\"phony\")\n end\n" }, { "path": "src/xmake.lua", "content": "includes(\"**/xmake.lua\")\n\nafter_build(function (target)\n local lib_dir = get_config(\"workflow_lib\")\n if (not os.isdir(lib_dir)) then\n os.mkdir(lib_dir)\n end\n shared_suffix = \"*.so\"\n if is_plat(\"macosx\") then\n shared_suffix = \"*.dylib\"\n end\n if target:is_static() then\n os.mv(path.join(\"$(projectdir)\", target:targetdir(), \"*.a\"), lib_dir)\n else\n os.mv(path.join(\"$(projectdir)\", target:targetdir(), shared_suffix), lib_dir)\n end\nend)\n\ntarget(\"workflow\")\n set_kind(\"$(kind)\")\n add_deps(\"client\", \"factory\", \"kernel\", \"manager\",\n \"nameservice\", \"protocol\", \"server\", \"util\")\n\n on_load(function (package)\n local include_path = path.join(get_config(\"workflow_inc\"), \"workflow\")\n if (not os.isdir(include_path)) then\n os.mkdir(include_path)\n end\n\n os.cp(path.join(\"$(projectdir)\", \"src/include/**.h\"), include_path)\n os.cp(path.join(\"$(projectdir)\", \"src/include/**.inl\"), include_path)\n end)\n\n after_clean(function (target)\n os.rm(get_config(\"workflow_inc\"))\n os.rm(get_config(\"workflow_lib\"))\n os.rm(\"$(buildir)\")\n end)\n\n on_install(function (target)\n os.mkdir(path.join(target:installdir(), \"include/workflow\"))\n os.mkdir(path.join(target:installdir(), \"lib\"))\n os.cp(path.join(get_config(\"workflow_inc\"), \"workflow\"), path.join(target:installdir(), \"include\"))\n shared_suffix = \"*.so\"\n if is_plat(\"macosx\") then\n shared_suffix = \"*.dylib\"\n end\n if target:is_static() then\n os.cp(path.join(get_config(\"workflow_lib\"), \"*.a\"), path.join(target:installdir(), \"lib\"))\n else\n os.cp(path.join(get_config(\"workflow_lib\"), shared_suffix), path.join(target:installdir(), \"lib\"))\n end\n end)\n\ntarget(\"wfkafka\")\n if has_config(\"kafka\") then\n set_kind(\"$(kind)\")\n add_deps(\"kafka_client\", \"kafka_factory\", \"kafka_protocol\", \"kafka_util\", \"workflow\")\n else\n set_kind(\"phony\")\n end\n\n" }, { "path": "test/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\n\nset(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING \"build type\")\n\nproject(workflow_test\n\t\tLANGUAGES C CXX\n)\n\nfind_library(LIBRT rt)\nfind_package(OpenSSL REQUIRED)\nfind_package(workflow REQUIRED CONFIG HINTS ..)\ninclude_directories(${OPENSSL_INCLUDE_DIR} ${WORKFLOW_INCLUDE_DIR})\nlink_directories(${WORKFLOW_LIB_DIR})\nfind_library(WORKFLOW_LIB NAMES libworkflow.a workflow HINTS ${WORKFLOW_LIB_DIR})\n\nfind_program(CMAKE_MEMORYCHECK_COMMAND valgrind)\nset(memcheck_command ${CMAKE_MEMORYCHECK_COMMAND} ${CMAKE_MEMORYCHECK_COMMAND_OPTIONS} --error-exitcode=1 --leak-check=full)\n\nadd_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})\n\nenable_testing()\n\nset(CXX_STD \"c++17\")\n\nfind_package(GTest REQUIRED)\n\nset(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -Wall -fPIC -pipe -std=gnu90\")\nset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wall -fPIC -pipe -std=${CXX_STD} -fno-exceptions\")\nif (APPLE)\n\tset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations\")\nendif()\n\nset(TEST_LIST\n\ttask_unittest\n\talgo_unittest\n\thttp_unittest\n\tredis_unittest\n\tmysql_unittest\n\tfacilities_unittest\n\tgraph_unittest\n\tmemory_unittest\n\tupstream_unittest\n\tdns_unittest\n\tresource_unittest\n\turiparser_unittest\n)\n\nif (APPLE)\n\tset(LIB ${WORKFLOW_LIB} pthread OpenSSL::SSL OpenSSL::Crypto)\nelse ()\n\tset(LIB ${WORKFLOW_LIB} pthread OpenSSL::SSL OpenSSL::Crypto ${LIBRT})\nendif ()\n\nforeach(src ${TEST_LIST})\n\tadd_executable(${src} EXCLUDE_FROM_ALL ${src}.cc)\n\ttarget_link_libraries(${src} ${LIB} GTest::GTest GTest::Main)\n\tadd_test(${src} ${src})\n\tadd_dependencies(check ${src})\nendforeach()\n\nif (NOT ${CMAKE_MEMORYCHECK_COMMAND} STREQUAL \"CMAKE_MEMORYCHECK_COMMAND-NOTFOUND\")\n\tforeach(src ${TEST_LIST})\n\t\tadd_test(${src}-memory-check ${memcheck_command} ./${src})\n\tendforeach()\nendif ()\n" }, { "path": "test/GNUmakefile", "content": "ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))\nALL_TARGETS := all check clean\nMAKE_FILE := Makefile\n\nDEFAULT_BUILD_DIR := build.cmake\nBUILD_DIR := $(shell if [ -f $(MAKE_FILE) ]; then echo \".\"; else echo $(DEFAULT_BUILD_DIR); fi)\nCMAKE3 := $(shell if which cmake3 ; then echo cmake3; else echo cmake; fi;)\n\n.PHONY: $(ALL_TARGETS)\n\nall:\n\tmkdir -p $(BUILD_DIR)\nifeq ($(DEBUG),y)\n\tcd $(BUILD_DIR) && $(CMAKE3) -D CMAKE_BUILD_TYPE=Debug $(ROOT_DIR)\nelse\n\tcd $(BUILD_DIR) && $(CMAKE3) $(ROOT_DIR)\nendif\n\t$(MAKE) -C $(BUILD_DIR) -f Makefile\n\ncheck:\n\tmkdir -p $(BUILD_DIR)\n\tcd $(BUILD_DIR) && $(CMAKE3) $(ROOT_DIR)\n\t$(MAKE) -C $(BUILD_DIR) check CTEST_OUTPUT_ON_FAILURE=1\n\nclean:\nifeq ($(MAKE_FILE), $(wildcard $(MAKE_FILE)))\n\t-$(MAKE) -f Makefile clean\nendif\n\trm -rf $(DEFAULT_BUILD_DIR)\n\n" }, { "path": "test/algo_unittest.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"workflow/WFAlgoTaskFactory.h\"\n\nstatic void __arr_init(int *arr, int n)\n{\n\tsrand(time(NULL));\n\tfor (int i = 0; i < n; i++)\n\t\tarr[i] = rand() % 65536;\n}\n\nstatic void __arr_check(int *arr, int n)\n{\n\tfor (int i = 1; i < n; i++)\n\t\tEXPECT_LE(arr[i - 1], arr[i]);\n}\n\nTEST(algo_unittest, sort)\n{\n\tstatic constexpr int n = 100000;\n\tint *arr = new int[n];\n\t__arr_init(arr, n);\n\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tauto *task = WFAlgoTaskFactory::create_sort_task(\"sort\", arr, arr + n, [&mutex, &cond, &done](WFSortTask *task) {\n\t\tint *first = task->get_input()->first;\n\t\tint *last = task->get_input()->last;\n\t\t__arr_check(first, last - first);\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\ttask->start();\n\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\n\tdelete []arr;\n}\n\nTEST(algo_unittest, parallel_sort)\n{\n\tstatic constexpr int n = 100000;\n\tint *arr = new int[n];\n\t__arr_init(arr, n);\n\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tauto *task = WFAlgoTaskFactory::create_psort_task(\"psort\", arr, arr + n, [&mutex, &cond, &done](WFSortTask *task) {\n\t\tint *first = task->get_input()->first;\n\t\tint *last = task->get_input()->last;\n\t\t__arr_check(first, last - first);\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\ttask->start();\n\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\n\tdelete []arr;\n}\n\n" }, { "path": "test/dns_unittest.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Kai (liukaidx@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFDnsClient.h\"\n\n#define RETRY_MAX\t3\n\nTEST(dns_unittest, WFDnsTaskCreate1)\n{\n\tstd::string url = \"dns://119.29.29.29/www.sogou.com\";\n\tauto *task = WFTaskFactory::create_dns_task(url, 0, NULL);\n\ttask->dismiss();\n}\n\nTEST(dns_unittest, WFDnsTaskCreate2)\n{\n\tstd::string url = \"http://119.29.29.29:dns/\";\n\tstd::promise done;\n\tauto *task = WFTaskFactory::create_dns_task(url, 0,\n\t[&done] (WFDnsTask *task)\n\t{\n\t\tdone.set_value();\n\t});\n\ttask->start();\n\tdone.get_future().get();\n}\n\nTEST(dns_unittest, WFDnsTask)\n{\n\tstd::string url = \"dns://119.29.29.29/www.sogou.com\";\n\tunsigned short req_id = 0x1234;\n\tstd::promise done;\n\n\tauto *task = WFTaskFactory::create_dns_task(url, RETRY_MAX,\n\t[&done, req_id] (WFDnsTask *task)\n\t{\n\t\tint state = task->get_state();\n\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tunsigned short resp_id = task->get_resp()->get_id();\n\t\t\tEXPECT_TRUE(req_id == resp_id);\n\t\t}\n\n\t\tdone.set_value();\n\t});\n\n\tauto *req = task->get_req();\n\treq->set_id(req_id);\n\treq->set_rd(1);\n\treq->set_question_type(DNS_TYPE_A);\n\ttask->start();\n\n\tauto fut = done.get_future();\n\tfut.get();\n}\n\nTEST(dns_unittest, WFDnsClientInit1)\n{\n\tWFDnsClient client;\n\tif (client.init(\"bad\") >= 0)\n\t\tclient.deinit();\n}\n\nTEST(dns_unittest, WFDnsClientInit2)\n{\n\tWFDnsClient client;\n\tint ret = client.init(\"0.0.0.0,0.0.0.1:1,dns://0.0.0.2,dnss://0.0.0.3\");\n\tEXPECT_TRUE(ret >= 0);\n\tclient.deinit();\n}\n\nTEST(dns_unittest, WFDnsClient)\n{\n\tunsigned short req_id = 0x4321;\n\tstd::promise done;\n\tWFDnsClient client;\n\n\tclient.init(\"dns://119.29.29.29/\");\n\n\tauto *task = client.create_dns_task(\"www.sogou.com\",\n\t[&done, req_id] (WFDnsTask *task)\n\t{\n\t\tint state = task->get_state();\n\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tunsigned short resp_id = task->get_resp()->get_id();\n\t\t\tEXPECT_TRUE(req_id == resp_id);\n\t\t}\n\n\t\tdone.set_value();\n\t});\n\n\tclient.deinit();\n\n\tauto *req = task->get_req();\n\treq->set_id(req_id);\n\ttask->start();\n\n\tauto fut = done.get_future();\n\tfut.get();\n}\n\nint main(int argc, char *argv[])\n{\n\t::testing::InitGoogleTest(&argc, argv);\n\treturn RUN_ALL_TESTS();\n}\n" }, { "path": "test/facilities_unittest.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \"workflow/WFFacilities.h\"\n#include \"workflow/HttpUtil.h\"\n\n#define GET_CURRENT_MICRO\tstd::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()\n\nTEST(facilities_unittest, usleep)\n{\n\tint64_t st = GET_CURRENT_MICRO;\n\tWFFacilities::usleep(1000000);\n\tint64_t ed = GET_CURRENT_MICRO;\n\tEXPECT_LE(ed - st, 10000000) << \"usleep too slow\";\n}\n\nTEST(facilities_unittest, async_usleep)\n{\n\tint64_t st = GET_CURRENT_MICRO;\n\tWFFacilities::async_usleep(1000000).wait();\n\tint64_t ed = GET_CURRENT_MICRO;\n\tEXPECT_LE(ed - st, 10000000) << \"async_usleep too slow\";\n}\n\nTEST(facilities_unittest, request)\n{\n\tprotocol::HttpRequest req;\n\treq.set_method(HttpMethodGet);\n\treq.set_http_version(\"HTTP/1.1\");\n\treq.set_request_uri(\"/\");\n\treq.set_header_pair(\"Host\", \"github.com\");\n\tauto res = WFFacilities::request(TT_TCP, \"http://github.com\", std::move(req), 0);\n\t//EXPECT_EQ(res.task_state, WFT_STATE_SUCCESS);\n\tif (res.task_state == WFT_STATE_SUCCESS)\n\t{\n\t\tauto code = atoi(res.resp.get_status_code());\n\t\tEXPECT_TRUE(code == HttpStatusOK ||\n\t\t\t\t\tcode == HttpStatusMovedPermanently ||\n\t\t\t\t\tcode == HttpStatusFound ||\n\t\t\t\t\tcode == HttpStatusSeeOther ||\n\t\t\t\t\tcode == HttpStatusTemporaryRedirect ||\n\t\t\t\t\tcode == HttpStatusPermanentRedirect);\n\t}\n}\n\nTEST(facilities_unittest, async_request)\n{\n\tprotocol::HttpRequest req;\n\treq.set_method(HttpMethodGet);\n\treq.set_http_version(\"HTTP/1.1\");\n\treq.set_request_uri(\"/\");\n\treq.set_header_pair(\"Host\", \"github.com\");\n\tauto res = WFFacilities::request(TT_TCP_SSL, \"https://github.com\", std::move(req), 0);\n\t//EXPECT_EQ(res.task_state, WFT_STATE_SUCCESS);\n\tif (res.task_state == WFT_STATE_SUCCESS)\n\t{\n\t\tauto code = atoi(res.resp.get_status_code());\n\t\tEXPECT_TRUE(code == HttpStatusOK ||\n\t\t\t\t\tcode == HttpStatusMovedPermanently ||\n\t\t\t\t\tcode == HttpStatusFound ||\n\t\t\t\t\tcode == HttpStatusSeeOther ||\n\t\t\t\t\tcode == HttpStatusTemporaryRedirect ||\n\t\t\t\t\tcode == HttpStatusPermanentRedirect);\n\t}\n}\n\nTEST(facilities_unittest, fileIO)\n{\n\tuint64_t data = 0x1234;\n\tssize_t sz;\n\tint fd = open(\"test.test\", O_RDWR | O_TRUNC | O_CREAT, 0644);\n\n\tsz = WFFacilities::async_pwrite(fd, &data, 8, 0).get();\n\tEXPECT_EQ(sz, 8);\n\tdata = 0;\n\tsz = WFFacilities::async_pread(fd, &data, 8, 0).get();\n\tEXPECT_EQ(sz, 8);\n\tEXPECT_EQ(data, 0x1234);\n\tclose(fd);\n}\n\nstatic inline void f(int i, WFFacilities::WaitGroup *wg)\n{\n\twg->done();\n}\n\nTEST(facilities_unittest, WaitGroup)\n{\n\tWFFacilities::WaitGroup wg(100);\n\n\tfor (int i = 0; i < 100; i++)\n\t\tWFFacilities::go(\"facilities\", f, i, &wg);\n\n\twg.wait();\n\n\tWFFacilities::WaitGroup wg2(-100);\n\twg2.wait();\n\n\tWFFacilities::WaitGroup wg3(0);\n\twg3.wait();\n}\n\n#if OPENSSL_VERSION_NUMBER >= 0x10100000L\n\n#include \nint main(int argc, char* argv[])\n{\n\tOPENSSL_init_ssl(0, 0);\n\t::testing::InitGoogleTest(&argc, argv);\n\treturn RUN_ALL_TESTS();\n}\n\n#endif\n" }, { "path": "test/graph_unittest.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Yang (liuyang216492@sogou-inc.com)\n*/\n\n#include \n#include \n\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\nstatic SubTask *create_task(int& target)\n{\n\tstatic std::atomic generator;\n\treturn WFTaskFactory::create_timer_task(0, [&](WFTimerTask *)\n\t{\n\t\ttarget = generator++;\n\t});\n}\n\nTEST(graph_unittest, WFGraphTask1)\n{\n\tWFFacilities::WaitGroup wait_group(1);\n\n\tauto graph = WFTaskFactory::create_graph_task([&wait_group](WFGraphTask *){ wait_group.done(); });\n\n\tint ta, tb, tc, td;\n\n\tauto& a = graph->create_graph_node(create_task(ta));\n\tauto& b = graph->create_graph_node(create_task(tb));\n\tauto& c = graph->create_graph_node(create_task(tc));\n\tauto& d = graph->create_graph_node(create_task(td));\n\n\ta --> b <-- c --> d --> a;\n\tc --> a;\n\n\tgraph->start();\n\twait_group.wait();\n\n\tEXPECT_LT(ta, tb);\n\tEXPECT_LT(tc, tb);\n\tEXPECT_LT(tc, td);\n\tEXPECT_LT(td, ta);\n\tEXPECT_LT(tc, ta);\n}\n\nTEST(graph_unittest, WFGraphTask2)\n{\n\tWFFacilities::WaitGroup wait_group(1);\n\n\tauto graph = WFTaskFactory::create_graph_task([&wait_group](WFGraphTask *){ wait_group.done(); });\n\n\tconstexpr int N = 4096 - 1;\n\n\tauto target = new int[N];\n\tauto node = new WFGraphNode *[N];\n\n\tfor (int i = 0; i < N; i++)\n\t\tnode[i] = &graph->create_graph_node(create_task(target[i]));\n\n\tfor (int i = 1; i < N; i++)\n\t\tnode[i]->precede(*node[(i - 1) / 2]);\n\n\tgraph->start();\n\twait_group.wait();\n\n\tfor (int i = 1; i < N; i++)\n\t\tEXPECT_LT(target[i], target[(i - 1) / 2]);\n\n\tdelete[] target;\n\tdelete[] node;\n}\n" }, { "path": "test/http_unittest.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFOperator.h\"\n#include \"workflow/WFHttpServer.h\"\n#include \"workflow/HttpUtil.h\"\n\n#define RETRY_MAX 3\n\nstatic void __http_process(WFHttpTask *task)\n{\n\tauto *req = task->get_req();\n\tauto *resp = task->get_resp();\n\n\tEXPECT_TRUE(strcmp(req->get_request_uri(), \"/test\") == 0);\n\tresp->add_header_pair(\"Content-Type\", \"text/plain\");\n}\n\nTEST(http_unittest, WFHttpTask1)\n{\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tauto *task = WFTaskFactory::create_http_task(\"http://github.com\", 0, RETRY_MAX, [&mutex, &cond, &done](WFHttpTask *task) {\n\t\tauto state = task->get_state();\n\n\t\t//EXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tauto code = atoi(task->get_resp()->get_status_code());\n\t\t\tEXPECT_TRUE(code == HttpStatusOK ||\n\t\t\t\t\t\tcode == HttpStatusMovedPermanently ||\n\t\t\t\t\t\tcode == HttpStatusFound ||\n\t\t\t\t\t\tcode == HttpStatusSeeOther ||\n\t\t\t\t\t\tcode == HttpStatusTemporaryRedirect ||\n\t\t\t\t\t\tcode == HttpStatusPermanentRedirect);\n\t\t}\n\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\ttask->start();\n\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n}\n\nTEST(http_unittest, WFHttpTask2)\n{\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tauto *task = WFTaskFactory::create_http_task(\"http://github.com\", 1, RETRY_MAX, [&mutex, &cond, &done](WFHttpTask *task) {\n\t\tauto state = task->get_state();\n\n\t\t//EXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tauto code = atoi(task->get_resp()->get_status_code());\n\t\t\tEXPECT_TRUE(code == HttpStatusOK ||\n\t\t\t\t\t\tcode == HttpStatusMovedPermanently ||\n\t\t\t\t\t\tcode == HttpStatusFound ||\n\t\t\t\t\t\tcode == HttpStatusSeeOther ||\n\t\t\t\t\t\tcode == HttpStatusTemporaryRedirect ||\n\t\t\t\t\t\tcode == HttpStatusPermanentRedirect);\n\t\t}\n\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\ttask->start();\n\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n}\n\nTEST(http_unittest, WFHttpTask3)\n{\n\tFILE *f;\n\tf = fopen(\"server.crt\", \"w\");\n\tfputs(R\"(\n-----BEGIN CERTIFICATE-----\nMIIDrjCCApYCCQCzDnhp/eqaRTANBgkqhkiG9w0BAQUFADCBmDELMAkGA1UEBhMC\nQ04xEDAOBgNVBAgMB0JlaWppbmcxEDAOBgNVBAcMB0JlaWppbmcxFzAVBgNVBAoM\nDlNvZ291LmNvbSBJbmMuMRYwFAYDVQQLDA13d3cuc29nb3UuY29tMQ8wDQYDVQQD\nDAZ4aWVoYW4xIzAhBgkqhkiG9w0BCQEWFHhpZWhhbkBzb2dvdS1pbmMuY29tMB4X\nDTE5MDYxMTA5MjQxNloXDTIwMDYxMDA5MjQxNlowgZgxCzAJBgNVBAYTAkNOMRAw\nDgYDVQQIDAdCZWlqaW5nMRAwDgYDVQQHDAdCZWlqaW5nMRcwFQYDVQQKDA5Tb2dv\ndS5jb20gSW5jLjEWMBQGA1UECwwNd3d3LnNvZ291LmNvbTEPMA0GA1UEAwwGeGll\naGFuMSMwIQYJKoZIhvcNAQkBFhR4aWVoYW5Ac29nb3UtaW5jLmNvbTCCASIwDQYJ\nKoZIhvcNAQEBBQADggEPADCCAQoCggEBALB6E1+lnuey24j+BwcD21h5t/xD+K6I\nthHiyT3S8fztAd+BfyphT+KLhbHbJFUaz7tfoV8lyBDdyVlgfwlCLyCp2sNcaCwg\nTF+XjTWOkDtg5+rCgoHRUjLNIJ2auO/5780DZcaL41gwzAu5rwE3sOifIZ4XI5WO\n6zrd5MUFhpHy91Sz1sxcCLXwQEgPDsa10/6k5bSd8xYP29yZ80lZeJ++5fgOf/AU\nJkANXLjsHnfOFV42Je/6EEcqe0YM6kjA9d4d5TS+To5YPfObTTR21Cey4RD5Ijjg\n4/VGdtI6tDWa3+N/CVVc8CKLVGNCVyAGWoBXCZuzlfex9Z0jtY2dd1cCAwEAATAN\nBgkqhkiG9w0BAQUFAAOCAQEAoLALHvGt0xCsDsYxxQ3biioPa2djT5jN8/QI17QF\n7C+0IdFEJi6dwF/O0rPgHbVSMZB7pPl5gx/rC4bWg9CYvZmlptmDJym+SpR0CBLC\n/LXEFsA7VmkdAiG6CHLtg1uZy0LTN0sRMdLNIetm6PBcnr3JEB8erayRaYy1Qk7d\n6O+3KexviFX/dAJRj59AIYXoMwji2ZYowXH+InNVF8UEunynJGURJJGQXFh0R18Q\nSniEJZux/WkxaOkqMBHtXtdkowpSMjn/RUA5dVu5Zjyf8LL9cjBmyKMxLXKeQeKK\n0ylFmFZxY8GawFdCq4XUKzSuLw4/orfuKn/ViSSixuXL5A==\n-----END CERTIFICATE-----\n)\", f);\n\tfclose(f);\n\tf = fopen(\"server.key\", \"w\");\n\tfputs(R\"(\n-----BEGIN RSA PRIVATE KEY-----\nMIIEogIBAAKCAQEAsHoTX6We57LbiP4HBwPbWHm3/EP4roi2EeLJPdLx/O0B34F/\nKmFP4ouFsdskVRrPu1+hXyXIEN3JWWB/CUIvIKnaw1xoLCBMX5eNNY6QO2Dn6sKC\ngdFSMs0gnZq47/nvzQNlxovjWDDMC7mvATew6J8hnhcjlY7rOt3kxQWGkfL3VLPW\nzFwItfBASA8OxrXT/qTltJ3zFg/b3JnzSVl4n77l+A5/8BQmQA1cuOwed84VXjYl\n7/oQRyp7RgzqSMD13h3lNL5Ojlg985tNNHbUJ7LhEPkiOODj9UZ20jq0NZrf438J\nVVzwIotUY0JXIAZagFcJm7OV97H1nSO1jZ13VwIDAQABAoIBAFPW+yNCjLaouzFe\n9bm4dFmZIfZf2GIaotzmcBLGB57QfkZPwDlDF++Ztz9iy+T+otfyu7h3O4//veuP\nM2sTnU4YQ8zyNq9X/NChMD3UZ+M9y5A1Lkk8R5/I4gjd+6ROikVMqupjhPNd42Ji\nqaiba5loGFGBzq77wfcqece8M01cZTnCtZ5ZdFrxzWWd9EaKhXf6Mkibaf6Y4/Oi\nGVvhqKK7Yv4f+xX85GnZuBv8hau6nCfiC/5zYKm8SiAoWE1TikMZGd2+bwAE1COh\nqeVJyevA7XcP8z+dtqb0hBHqlm0DTyVmu/cuHAZHxYms7VvJ2isWKI4gl1MY3zD3\nODHEeHECgYEA36eVhGCAQeAP3eTtEq1dcSSsb3bEKTpZGxj6BT89HRp0qcw/dKQV\noITXMeSJpIRR879mi5FBFHlvTb0xkI96O5fXuAz/A7hSOtZpiJ4G3tAEplbPJhmB\n3km3syRXqXuv8m38Zjb9FOgu7D/OSWYe8QGWM/rrDjgBfJNveKlWn/kCgYEAyf/R\nheAvuFxqf77XRzjBhil1N09f9mw8yagFritNyy8Wb+SlNSHIBZ9WSKVdVxyA4GOe\nA/0yAY7r9i/Y1sMnCt0kL5UEwY2xlbA+Ld/B/5MjEN4mP9g5a2goj75w7CBT/YLh\ndAfNwN08wsTNl/53tovhqz1uvU+muAWQnAgURc8CgYAjqKOFHKG2XxQIi+RkkvGQ\nBYncp7H05NGqKVxLk96ZkktBe0guv66XDjcFRGvRqCss0rp1zC31JrthSKXrZ4TU\nlYwWUzQhkrTBnsfquU9dHQtwvex/JZf4Kga48DVt10OhQnn4jhHh0HcSwcWRHFAY\nmuko1nu9o55RD2y5bz5ZeQKBgFfzec/3n+9+1aQPfP52uNRogq/1cIwD7qfC7844\n7qNUOkm33TL4JXZFPTVeQvjl4TtSRH/qI3bIOvczOA+yYvJ4/QN2t95qinLpjPk+\nXuKftvnmL/NGeyHH9Tk5K0O0g71y2iVCLJUX/xeyxu2yD3+9AiIkGm51GtsvGRrG\n7cTDAoGAIlzSgiMSMkRUpzyJYvRd5o+Bt+v+SHDni40XrfZqc4cmh8MVPdVkNMFi\na/7MiJf+tw5lRG/Oks0pNOvFIpTXi8ncxW9tgQfy2hN6LMGD7uIu/X9uMJmwvNtj\nKZ1lOvb+vi3TLrQf4tfBekrXXe5tZK40QSJ7UdtY7HHrrbAXU+8=\n-----END RSA PRIVATE KEY-----\n)\", f);\n\tfclose(f);\n\n\tWFHttpServer http_server(__http_process);\n\tEXPECT_TRUE(http_server.start(\"127.0.0.1\", 8811) == 0) << \"http server start failed\";\n\n\tWFHttpServer https_server(__http_process);\n\tEXPECT_TRUE(https_server.start(\"127.0.0.1\", 8822, \"server.crt\", \"server.key\") == 0) << \"https server start failed\";\n\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tauto cb = [](WFHttpTask *task) {\n\t\tauto state = task->get_state();\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tauto *resp = task->get_resp();\n\t\t\tauto code = atoi(resp->get_status_code());\n\t\t\tEXPECT_EQ(code, HttpStatusOK);\n\t\t\tprotocol::HttpHeaderCursor cursor(resp);\n\t\t\tstd::string content_type;\n\t\t\tEXPECT_TRUE(cursor.find(\"Content-Type\", content_type));\n\t\t\tEXPECT_TRUE(content_type == \"text/plain\");\n\t\t}\n\t};\n\n\tauto *A = WFTaskFactory::create_http_task(\"http://127.0.0.1:8811/test\", 0, RETRY_MAX, cb);\n\tauto *B = WFTaskFactory::create_http_task(\"https://127.0.0.1:8822/test\", 0, RETRY_MAX, cb);\n\tauto& flow = *A > B;\n\n\tflow.set_callback([&mutex, &cond, &done](const SeriesWork *series) {\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\tflow.start();\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\thttp_server.stop();\n\thttps_server.stop();\n}\n\n#if OPENSSL_VERSION_NUMBER >= 0x10100000L\n\n#include \nint main(int argc, char* argv[])\n{\n\tOPENSSL_init_ssl(0, 0);\n\t::testing::InitGoogleTest(&argc, argv);\n\treturn RUN_ALL_TESTS();\n}\n\n#endif\n" }, { "path": "test/memory_unittest.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Liu Yang (liuyang216492@sogou-inc.com)\n*/\n\n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n\nTEST(memory_unittest, dismiss)\n{\n\tstd::vector tasks;\n\n\tauto *http_task = WFTaskFactory::create_http_task(\"http://www.sogou.com\", 0, 0, nullptr);\n\ttasks.push_back(http_task);\n\n\tauto *redis_task = WFTaskFactory::create_redis_task(\"redis://username:password@127.0.0.1:6676/1\", 0, nullptr);\n\ttasks.push_back(redis_task);\n\n\tauto *mysql_task = WFTaskFactory::create_mysql_task(\"mysql://username:password@127.0.0.1:8899/db\", 0, nullptr);\n\ttasks.push_back(mysql_task);\n\n\tauto *timer_task = WFTaskFactory::create_timer_task(0, nullptr);\n\ttasks.push_back(timer_task);\n\n\tauto *counter_task = WFTaskFactory::create_counter_task(\"\", 1, nullptr);\n\ttasks.push_back(counter_task);\n\n\tauto *go_task = WFTaskFactory::create_go_task(\"\", [](){});\n\ttasks.push_back(go_task);\n\n\tauto *thread_task = WFThreadTaskFactory::create_thread_task(\"\", [](int *, int *){}, nullptr);\n\ttasks.push_back(thread_task);\n\n\tauto *graph_task = WFTaskFactory::create_graph_task(nullptr);\n\tauto& node_a = graph_task->create_graph_node(WFTaskFactory::create_timer_task(0, nullptr));\n\tauto& node_b = graph_task->create_graph_node(WFTaskFactory::create_timer_task(0, nullptr));\n\tnode_a -->-- node_b;\n\ttasks.push_back(graph_task);\n\t\n\tauto *parallel_work = Workflow::create_parallel_work(nullptr);\n\tfor (auto *task : tasks)\n\t{\n\t\tauto *series_work = Workflow::create_series_work(task, nullptr);\n\t\tparallel_work->add_series(series_work);\n\t}\n\n\tparallel_work->dismiss();\n}\n" }, { "path": "test/mysql_unittest.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFMySQLServer.h\"\n\n#define RETRY_MAX 3\n\nstatic void __mysql_process(WFMySQLTask *task)\n{\n\t//auto *req = task->get_req();\n\tauto *resp = task->get_resp();\n\n\tresp->set_ok_packet();\n}\n\nstatic void test_client(const char *url, const char *sql, std::mutex& mutex, std::condition_variable& cond, bool& done)\n{\n\tauto *task = WFTaskFactory::create_mysql_task(url, RETRY_MAX, [&mutex, &cond, &done](WFMySQLTask *task) {\n\t\tauto state = task->get_state();\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\ttask->get_req()->set_query(sql);\n\ttask->start();\n}\n\nTEST(mysql_unittest, WFMySQLTask1)\n{\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tWFMySQLServer server(__mysql_process);\n\tEXPECT_TRUE(server.start(\"127.0.0.1\", 8899) == 0) << \"server start failed\";\n\n\ttest_client(\"mysql://testuser:testpass@127.0.0.1:8899/testdb\",\n\t\t\t\t\"select * from testtable limit 3\", mutex, cond, done);\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\tserver.stop();\n}\n\n#if OPENSSL_VERSION_NUMBER >= 0x10100000L\n\n#include \nint main(int argc, char* argv[])\n{\n\tOPENSSL_init_ssl(0, 0);\n\t::testing::InitGoogleTest(&argc, argv);\n\treturn RUN_ALL_TESTS();\n}\n\n#endif\n\n" }, { "path": "test/redis_unittest.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFRedisServer.h\"\n#include \"workflow/WFOperator.h\"\n\n#define RETRY_MAX 3\n\nstatic void __redis_process(WFRedisTask *task)\n{\n\tauto *req = task->get_req();\n\tauto *resp = task->get_resp();\n\n\tEXPECT_TRUE(req->parse_success());\n\n\tstd::string cmd;\n\tstd::vector params;\n\tprotocol::RedisValue val;\n\n\tEXPECT_TRUE(req->get_command(cmd));\n\tEXPECT_TRUE(req->get_params(params));\n\n\tif (strcasecmp(cmd.c_str(), \"SET\") == 0)\n\t{\n\t\tEXPECT_EQ(params.size(), 2);\n\t\tEXPECT_TRUE(params[0] == \"testkey\");\n\t\tEXPECT_TRUE(params[1] == \"testvalue\");\n\t\tval.set_status(\"OK\");\n\t}\n\telse if (strcasecmp(cmd.c_str(), \"GET\") == 0)\n\t{\n\t\tEXPECT_EQ(params.size(), 1);\n\t\tval.set_string(\"testvalue\");\n\t}\n\telse if (strcasecmp(cmd.c_str(), \"DEL\") == 0)\n\t{\n\t\tEXPECT_EQ(params.size(), 1);\n\t\tEXPECT_TRUE(params[0] == \"testkey\");\n\t\tval.set_status(\"OK\");\n\t}\n\telse if (strcasecmp(cmd.c_str(), \"SELECT\") == 0)\n\t{\n\t\tEXPECT_EQ(params.size(), 1);\n\t\tEXPECT_TRUE(params[0] == \"6\");\n\t\tval.set_status(\"OK\");\n\t}\n\telse if (strcasecmp(cmd.c_str(), \"AUTH\") == 0)\n\t{\n\t\tEXPECT_EQ(params.size(), 1);\n\t\tEXPECT_TRUE(params[0] == \"testpass\");\n\t\tval.set_status(\"OK\");\n\t}\n\telse\n\t{\n\t\tEXPECT_TRUE(0) << \"Command Not Support\";\n\t\tval.set_error(\"Command Not Support\");\n\t}\n\n\tresp->set_result(val);\n}\n\nstatic void test_client(const char *url, std::mutex& mutex, std::condition_variable& cond, bool& done)\n{\n\tauto&& set_cb = [](WFRedisTask *task) {\n\t\tauto state = task->get_state();\n\t\tauto *resp = task->get_resp();\n\t\tprotocol::RedisValue val;\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tEXPECT_TRUE(resp->parse_success());\n\t\tresp->get_result(val);\n\t\tEXPECT_TRUE(val.is_ok());\n\t};\n\n\tauto&& get_cb = [](WFRedisTask *task) {\n\t\tauto state = task->get_state();\n\t\tauto *resp = task->get_resp();\n\t\tprotocol::RedisValue val;\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tEXPECT_TRUE(resp->parse_success());\n\t\tresp->get_result(val);\n\t\tEXPECT_TRUE(val.is_string());\n\t\tEXPECT_TRUE(val.string_value() == \"testvalue\");\n\t};\n\n\tauto&& del_cb = [](WFRedisTask *task) {\n\t\tauto state = task->get_state();\n\t\tauto *resp = task->get_resp();\n\t\tprotocol::RedisValue val;\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tEXPECT_TRUE(resp->parse_success());\n\t\tresp->get_result(val);\n\t\tEXPECT_TRUE(val.is_ok());\n\t};\n\n\tauto *A = WFTaskFactory::create_redis_task(url, RETRY_MAX, std::move(set_cb));\n\tauto *B = WFTaskFactory::create_redis_task(url, RETRY_MAX, std::move(get_cb));\n\tauto *C = WFTaskFactory::create_redis_task(url, RETRY_MAX, std::move(del_cb));\n\tauto& flow = *A > B > C;\n\n\tA->get_req()->set_request(\"SET\", {\"testkey\", \"testvalue\"});\n\tB->get_req()->set_request(\"GET\", {\"testkey\"});\n\tC->get_req()->set_request(\"DEL\", {\"testkey\"});\n\tflow.set_callback([&mutex, &cond, &done](const SeriesWork *series) {\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\tflow.start();\n}\n\nTEST(redis_unittest, WFRedisTask1)\n{\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tWFRedisServer server(__redis_process);\n\tEXPECT_TRUE(server.start(\"127.0.0.1\", 6677) == 0) << \"server start failed\";\n\n\ttest_client(\"redis://:testpass@127.0.0.1:6677/6\", mutex, cond, done);\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\tserver.stop();\n}\n\n" }, { "path": "test/resource_unittest.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \"workflow/WFTask.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFResourcePool.h\"\n#include \"workflow/WFFacilities.h\"\n\nTEST(resource_unittest, resource_pool)\n{\n\tint res_concurrency = 3;\n\tint task_concurrency = 10;\n\tconst char *words[3] = {\"workflow\", \"srpc\", \"pyworkflow\"};\n\tWFResourcePool res_pool((void * const*)words, res_concurrency);\n\tWFFacilities::WaitGroup wg(task_concurrency);\n\n\tfor (int i = 0; i < task_concurrency; i++)\n\t{\n\t\tauto *user_task = WFTaskFactory::create_timer_task(0,\n\t\t[&wg, &res_pool](WFTimerTask *task) {\n\t\t\tuint64_t id = (uint64_t)series_of(task)->get_context();\n\t\t\tprintf(\"task-%lu get [%s]\\n\", id, (char *)task->user_data);\n\t\t\tres_pool.post(task->user_data);\n\t\t\twg.done();\n\t\t});\n\n\t\tauto *cond = res_pool.get(user_task, &user_task->user_data);\n\n\t\tSeriesWork *series = Workflow::create_series_work(cond, nullptr);\n\t\tseries->set_context(reinterpret_cast(i));\n\t\tseries->start();\n\t}\n\n\twg.wait();\n}\n\n" }, { "path": "test/task_unittest.cc", "content": "/*\n Copyright (c) 2020 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Wu Jiaxu (wujiaxu@sogou-inc.com)\n*/\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n\n#define GET_CURRENT_MICRO\tstd::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()\n\nTEST(task_unittest, WFTimerTask)\n{\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tauto *task = WFTaskFactory::create_timer_task(1000000, [&mutex, &cond, &done](WFTimerTask *task) {\n\t\tEXPECT_EQ(task->get_state(), WFT_STATE_SUCCESS);\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\tint64_t st = GET_CURRENT_MICRO;\n\ttask->start();\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\n\tint64_t ed = GET_CURRENT_MICRO;\n\tEXPECT_LE(ed - st, 10000000) << \"Timer Task too slow\";\n}\n\nTEST(task_unittest, WFCounterTask1)\n{\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tauto *task = WFTaskFactory::create_counter_task(\"abc\", 2, [&mutex, &cond, &done](WFCounterTask *task) {\n\t\tauto state = task->get_state();\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tWFTaskFactory::count_by_name(\"abc\", 0);\n\t\t\ttask->count();\n\t\t\tWFTaskFactory::count_by_name(\"abc\", 1);\n\t\t}\n\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\ttask->start();\n\tfor (int i = 0; i < 100; i++)\n\t{\n\t\tWFTaskFactory::count_by_name(\"abc\");\n\t\tWFTaskFactory::count_by_name(\"abc\");\n\t}\n\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n}\n\nTEST(task_unittest, WFCounterTask2)\n{\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tauto *task = WFTaskFactory::create_counter_task(\"def\", 2, [&mutex, &cond, &done](WFCounterTask *task) {\n\t\tauto state = task->get_state();\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tWFTaskFactory::count_by_name(\"def\", 0);\n\t\t\ttask->count();\n\t\t\tWFTaskFactory::count_by_name(\"def\", 1);\n\t\t}\n\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\ttask->count();\n\ttask->start();\n\ttask->count();\n\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n}\n\nTEST(task_unittest, WFGoTask)\n{\n\tsrand(time(NULL));\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tint target = rand() % 1024;\n\tint edit_inner = -1;\n\n\tauto&& f = [&mutex, &cond, &done, target, &edit_inner](int id) {\n\t\tEXPECT_EQ(target, id);\n\t\tedit_inner = 100;\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t};\n\n\tWFGoTask *task = WFTaskFactory::create_go_task(\"go\", std::move(f), target);\n\ttask->start();\n\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\n\tEXPECT_EQ(edit_inner, 100);\n}\n\nTEST(task_unittest, WFThreadTask)\n{\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\n\tusing MyTaskIn = std::pair;\n\tusing MyTaskOut = int;\n\tusing MyFactory = WFThreadTaskFactory;\n\tusing MyTask = WFThreadTask;\n\n\tauto&& calc_multi = [](MyTaskIn *in, MyTaskOut *out) {\n\t\t*out = in->first * in->second;\n\t};\n\n\tauto *task = MyFactory::create_thread_task(\"calc\", std::move(calc_multi), [&mutex, &cond, &done](MyTask *task) {\n\t\tauto state = task->get_state();\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tauto *in = task->get_input();\n\t\t\tauto *out = task->get_output();\n\t\t\tEXPECT_EQ(in->first * in->second, *out);\n\t\t}\n\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\ttask->start();\n\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n}\n\nTEST(task_unittest, WFFileIOTask)\n{\n\tsrand(time(NULL));\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tstd::string file_path = \"./\" + std::to_string(time(NULL)) +\n\t\t\t\t\t\t\t\"__\" + std::to_string(rand() % 4096);\n\n\tint fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644);\n\tEXPECT_TRUE(fd > 0);\n\n\tchar writebuf[] = \"testtest\";\n\tchar readbuf[16];\n\n\tauto *write = WFTaskFactory::create_pwrite_task(fd, writebuf, 8, 80, [fd](WFFileIOTask *task) {\n\t\tauto state = task->get_state();\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tauto *args = task->get_args();\n\t\t\tEXPECT_EQ(args->fd, fd);\n\t\t\tEXPECT_EQ(args->count, 8);\n\t\t\tEXPECT_EQ(args->offset, 80);\n\t\t\tEXPECT_TRUE(strncmp(\"testtest\", (char *)args->buf, 8) == 0);\n\t\t}\n\t});\n\n\tauto *read = WFTaskFactory::create_pread_task(fd, readbuf, 8, 80, [fd](WFFileIOTask *task) {\n\t\tauto state = task->get_state();\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tauto *args = task->get_args();\n\t\t\tEXPECT_EQ(args->fd, fd);\n\t\t\tEXPECT_EQ(args->count, 8);\n\t\t\tEXPECT_EQ(args->offset, 80);\n\t\t\tEXPECT_TRUE(strncmp(\"testtest\", (char *)args->buf, 8) == 0);\n\t\t}\n\t});\n\n\tauto *series = Workflow::create_series_work(write, [&mutex, &cond, &done](const SeriesWork *series) {\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\tseries->push_back(read);\n\tseries->start();\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\n\tclose(fd);\n\tremove(file_path.c_str());\n}\n\nTEST(task_unittest, WFFilePathIOTask)\n{\n\tsrand(time(NULL));\n\tstd::mutex mutex;\n\tstd::condition_variable cond;\n\tbool done = false;\n\tstd::string file_path = \"./\" + std::to_string(time(NULL)) +\n\t\t\t\t\t\t\t\"__\" + std::to_string(rand() % 4096);\n\n\tchar writebuf[] = \"testtest\";\n\tchar readbuf[16];\n\n\tauto *write = WFTaskFactory::create_pwrite_task(file_path, writebuf, 8, 80, [](WFFileIOTask *task) {\n\t\tauto state = task->get_state();\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tauto *args = task->get_args();\n\t\t\tEXPECT_EQ(args->count, 8);\n\t\t\tEXPECT_EQ(args->offset, 80);\n\t\t\tEXPECT_TRUE(strncmp(\"testtest\", (char *)args->buf, 8) == 0);\n\t\t}\n\t});\n\n\tauto *read = WFTaskFactory::create_pread_task(file_path, readbuf, 8, 80, [](WFFileIOTask *task) {\n\t\tauto state = task->get_state();\n\n\t\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\t\tif (state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tauto *args = task->get_args();\n\t\t\tEXPECT_EQ(args->count, 8);\n\t\t\tEXPECT_EQ(args->offset, 80);\n\t\t\tEXPECT_TRUE(strncmp(\"testtest\", (char *)args->buf, 8) == 0);\n\t\t}\n\t});\n\n\tauto *series = Workflow::create_series_work(write, [&mutex, &cond, &done](const SeriesWork *series) {\n\t\tmutex.lock();\n\t\tdone = true;\n\t\tmutex.unlock();\n\t\tcond.notify_one();\n\t});\n\n\tseries->push_back(read);\n\tseries->start();\n\tstd::unique_lock lock(mutex);\n\twhile (!done)\n\t\tcond.wait(lock);\n\n\tlock.unlock();\n\n\tremove(file_path.c_str());\n}\n" }, { "path": "test/upstream_unittest.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Li Yingxin (liyingxin@sogou-inc.com)\n*/\n\n#include \n#include \"workflow/UpstreamManager.h\"\n#include \"workflow/WFHttpServer.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"workflow/UpstreamPolicies.h\"\n\n#define REDIRECT_MAX\t2\n#define RETRY_MAX\t\t2\n#define MTTR\t\t\t2\n#define MAX_FAILS\t\t200\n\nstatic void __http_process(WFHttpTask *task, const char *name)\n{\n\tauto *resp = task->get_resp();\n\tresp->add_header_pair(\"Content-Type\", \"text/plain\");\n\tresp->append_output_body_nocopy(name, strlen(name));\n}\n\nWFHttpServer http_server1(std::bind(&__http_process,\n\t\t\t\t\t\t\t\t\tstd::placeholders::_1,\n\t\t\t\t\t\t\t\t\t\"server1\"));\nWFHttpServer http_server2(std::bind(&__http_process,\n\t\t\t\t\t\t\t\t\tstd::placeholders::_1,\n\t\t\t\t\t\t\t\t\t\"server2\"));\nWFHttpServer http_server3(std::bind(&__http_process,\n\t\t\t\t\t\t\t\t\tstd::placeholders::_1,\n\t\t\t\t\t\t\t\t\t\"server3\"));\n\nvoid register_upstream_hosts()\n{\n\tUpstreamManager::upstream_create_weighted_random(\"weighted.random\", false);\n\tAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\n\taddress_params.weight = 1000;\n\tUpstreamManager::upstream_add_server(\"weighted.random\",\n\t\t\t\t\t\t\t\t\t\t \"127.0.0.1:8001\",\n\t\t\t\t\t\t\t\t\t\t &address_params);\n\taddress_params.weight = 1;\n\tUpstreamManager::upstream_add_server(\"weighted.random\",\n\t\t\t\t\t\t\t\t\t\t \"127.0.0.1:8002\",\n\t\t\t\t\t\t\t\t\t\t &address_params);\n\n\tUpstreamManager::upstream_create_consistent_hash(\n\t\"hash\",\n\t[](const char *path, const char *query, const char *fragment) -> unsigned int {\n\t\treturn 4250947057; // test skip from the end to the begin, hit 8002\n\t});\n\tUpstreamManager::upstream_add_server(\"hash\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_add_server(\"hash\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_add_server(\"hash\", \"127.0.0.1:8002\");\n\n\tUpstreamManager::upstream_create_manual(\n\t\"manual\",\n\t[](const char *path, const char *query, const char *fragment) -> unsigned int {\n\t\treturn 0;\n\t},\n\ttrue,\n\t[](const char *path, const char *query, const char *fragment) -> unsigned int {\n\t\treturn 511702306; // test skip the non-alive server\n\t});\n\tUpstreamManager::upstream_add_server(\"manual\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_add_server(\"manual\", \"127.0.0.1:8002\");\n\n\tUpstreamManager::upstream_create_round_robin(\"round.robin\", true);\n\tUpstreamManager::upstream_add_server(\"round.robin\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_add_server(\"round.robin\", \"127.0.0.1:8002\");\n\n\tUpstreamManager::upstream_create_manual(\n\t\"try_another\",\n\t[](const char *path, const char *query, const char *fragment) -> unsigned int {\n\t\treturn 0;\n\t},\n\tfalse, nullptr);\n\tUpstreamManager::upstream_add_server(\"try_another\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_add_server(\"try_another\", \"127.0.0.1:8002\");\n\n\tUpstreamManager::upstream_create_weighted_random(\"test_tracing\", true);\n\taddress_params.weight = 1000;\n\tUpstreamManager::upstream_add_server(\"test_tracing\",\n\t\t\t\t\t\t\t\t\t\t \"127.0.0.1:8001\",\n\t\t\t\t\t\t\t\t\t\t &address_params);\n\taddress_params.weight = 1;\n\tUpstreamManager::upstream_add_server(\"test_tracing\",\n\t\t\t\t\t\t\t\t\t\t \"127.0.0.1:8002\",\n\t\t\t\t\t\t\t\t\t\t &address_params);\n\taddress_params.weight = 1000;\n\tUpstreamManager::upstream_add_server(\"test_tracing\",\n\t\t\t\t\t\t\t\t\t\t \"127.0.0.1:8003\",\n\t\t\t\t\t\t\t\t\t\t &address_params);\n}\n\nvoid basic_callback(WFHttpTask *task, std::string& message)\n{\n\tint state = task->get_state();\n\tEXPECT_EQ(state, WFT_STATE_SUCCESS);\n\tif (state == WFT_STATE_SUCCESS && message.compare(\"\"))\n\t{\n\t\tconst void *body;\n\t\tsize_t body_len;\n\t\ttask->get_resp()->get_parsed_body(&body, &body_len);\n\t\tstd::string buffer((char *)body, body_len);\n\t\tEXPECT_EQ(buffer, message);\n\t}\n\tWFFacilities::WaitGroup *wait_group = (WFFacilities::WaitGroup *)task->user_data;\n\twait_group->done();\n}\n\nTEST(upstream_unittest, BasicPolicy)\n{\n\tWFFacilities::WaitGroup wait_group(5);\n\tWFHttpTask *task1;\n\tWFHttpTask *task2;\n\n\tchar url[4][30] = {\"http://weighted.random\", \"http://manual\",\n\t\t\t\t\t\t\"http://hash\", \"http://round.robin\"};\n\n\thttp_callback_t cb1 = std::bind(basic_callback, std::placeholders::_1,\n\t\t\t\t\t\t\t\t std::string(\"server1\"));\n\tfor (int i = 0; i < 2; i++)\n\t{\n\t\ttask1 = WFTaskFactory::create_http_task(url[i], REDIRECT_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\tRETRY_MAX, cb1);\n\t\ttask1->user_data = &wait_group;\n\t\ttask1->start();\n\t}\n\n\thttp_callback_t cb2 = std::bind(basic_callback, std::placeholders::_1,\n\t\t\t\t\t\t\t\t std::string(\"server2\"));\n\n\ttask2 = WFTaskFactory::create_http_task(url[2], REDIRECT_MAX,\n\t\t\t\t\t\t\t\t\t\t\tRETRY_MAX, cb2);\n\ttask2->user_data = &wait_group;\n\ttask2->start();\n\n\ttask1 = WFTaskFactory::create_http_task(url[3], REDIRECT_MAX,\n\t\t\t\t\t\t\t\t\t\t\tRETRY_MAX, cb1);\n\ttask1->user_data = &wait_group;\n\n\ttask2 = WFTaskFactory::create_http_task(url[3], REDIRECT_MAX,\n\t\t\t\t\t\t\t\t\t\t\tRETRY_MAX, cb2);\n\ttask2->user_data = &wait_group;\n\n\tSeriesWork *series = Workflow::create_series_work(task1, nullptr);\n\tseries->push_back(task2);\n\tseries->start();\n\n\twait_group.wait();\n}\n\nTEST(upstream_unittest, EnableAndDisable)\n{\n\tWFFacilities::WaitGroup wait_group(1);\n\n\tUpstreamManager::upstream_disable_server(\"weighted.random\", \"127.0.0.1:8001\");\n\n\tstd::string url = \"http://weighted.random\";\n\tWFHttpTask *task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t[&wait_group, &url](WFHttpTask *task){\n\t\tint state = task->get_state();\n\t\tEXPECT_EQ(state, WFT_STATE_TASK_ERROR);\n\t\tEXPECT_EQ(task->get_error(), WFT_ERR_UPSTREAM_UNAVAILABLE);\n\t\tUpstreamManager::upstream_enable_server(\"weighted.random\", \"127.0.0.1:8001\");\n\t\tauto *task2 = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::bind(basic_callback,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tstd::placeholders::_1,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tstd::string(\"server1\")));\n\t\ttask2->user_data = &wait_group;\n\t\tseries_of(task)->push_back(task2);\n\t});\n\ttask->user_data = &wait_group;\n\ttask->start();\t\n\n\twait_group.wait();\n}\n\nTEST(upstream_unittest, AddAndRemove)\n{\n\tWFFacilities::WaitGroup wait_group(2);\n\tWFHttpTask *task;\n\tSeriesWork *series;\n\tprotocol::HttpRequest *req;\n\tint batch = MAX_FAILS + 50;\n\tstd::string url = \"http://add_and_remove\";\n\tstd::string name = \"add_and_remove\";\n\tUPSWeightedRandomPolicy test_policy(false);\n\n\tAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\n\n\taddress_params.weight = 1000;\n\ttest_policy.add_server(\"127.0.0.1:8001\", &address_params);\n\n\taddress_params.weight = 1;\n\ttest_policy.add_server(\"127.0.0.1:8002\", &address_params);\n\n\tauto *ns = WFGlobal::get_name_service();\n\tEXPECT_EQ(ns->add_policy(name.c_str(), &test_policy), 0);\n\n\tUpstreamManager::upstream_remove_server(name, \"127.0.0.1:8001\");\n\ttask = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t std::bind(basic_callback,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::placeholders::_1,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::string(\"server2\")));\n\ttask->user_data = &wait_group;\n\ttask->start();\n\n\t//test remove fused server\n\taddress_params.weight = 1000;\n\ttest_policy.add_server(\"127.0.0.1:8001\", &address_params);\n\thttp_server1.stop();\n\n\tfprintf(stderr, \"server 1 stopped start %d tasks to fuse it\\n\", batch);\n\tParallelWork *pwork = Workflow::create_parallel_work(\n\t\t\t\t\t\t\t\t\t\t[&wait_group, &name, &url](const ParallelWork *pwork) {\n\t\tfprintf(stderr, \"parallel finished and remove server1\\n\");\n\t\tUpstreamManager::upstream_remove_server(name, \"127.0.0.1:8001\");\n\t\tauto *task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::bind(basic_callback,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::placeholders::_1,\n\t\t\t\t\t\t\t\t \t\t\t\t\t std::string(\"server2\")));\n\t\ttask->user_data = &wait_group;\n\t\tseries_of(pwork)->push_back(task);\n\t});\n\n\tfor (int i = 0; i < batch; i++)\n\t{\n\t\ttask = WFTaskFactory::create_http_task(url, 0, 0, nullptr);\n\t\treq = task->get_req();\n\t\treq->add_header_pair(\"Connection\", \"keep-alive\");\n\t\tseries = Workflow::create_series_work(task, nullptr);\n\t\tpwork->add_series(series);\n\t}\n\n\tpwork->start();\n\twait_group.wait();\n\tEXPECT_TRUE(http_server1.start(\"127.0.0.1\", 8001) == 0)\n\t\t\t\t<< \"http server start failed\";\n\tns->del_policy(name.c_str());\n}\n\nTEST(upstream_unittest, FuseAndRecover)\n{\n\tWFFacilities::WaitGroup wait_group(1);\n\tWFHttpTask *task;\n\tSeriesWork *series;\n\tprotocol::HttpRequest *req;\n\tstd::string url = \"http://test_policy\";\n\tint batch = MAX_FAILS + 50;\n\tint timeout = (MTTR + 1) * 1000000;\n\n\tUPSWeightedRandomPolicy test_policy(false);\n\ttest_policy.set_mttr_seconds(MTTR);\n\tAddressParams address_params = ADDRESS_PARAMS_DEFAULT;\n\t\n\taddress_params.weight = 1000;\n\ttest_policy.add_server(\"127.0.0.1:8001\", &address_params);\n\n\taddress_params.weight = 1;\n\ttest_policy.add_server(\"127.0.0.1:8002\", &address_params);\n\n\tauto *ns = WFGlobal::get_name_service();\n\tEXPECT_EQ(ns->add_policy(\"test_policy\", &test_policy), 0);\n\n\thttp_server1.stop();\n\tfprintf(stderr, \"server 1 stopped start %d tasks to fuse it\\n\", batch);\n\tParallelWork *pwork = Workflow::create_parallel_work(\n\t\t\t\t\t\t\t\t\t\t[](const ParallelWork *pwork) {\n\t\tfprintf(stderr, \"parallel finished\\n\");\n\t});\n\n\tfor (int i = 0; i < batch; i++)\n\t{\n\t\ttask = WFTaskFactory::create_http_task(url, 0, 0, nullptr);\n\t\treq = task->get_req();\n\t\treq->add_header_pair(\"Connection\", \"keep-alive\");\n\t\tseries = Workflow::create_series_work(task, nullptr);\n\t\tpwork->add_series(series);\n\t}\n\n\tseries = Workflow::create_series_work(pwork, nullptr);\n\n\tWFTimerTask *timer = WFTaskFactory::create_timer_task(timeout,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t [](WFTimerTask *task) {\n\t\tfprintf(stderr, \"timer_finished and start server1\\n\");\n\t\tEXPECT_TRUE(http_server1.start(\"127.0.0.1\", 8001) == 0)\n\t\t\t\t\t<< \"http server start failed\";\n\t});\n\n\tseries->push_back(timer);\n\n\ttask = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t std::bind(basic_callback,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::placeholders::_1,\n\t\t\t\t\t\t\t\t \t\t\t\t\t std::string(\"server1\")));\n\ttask->user_data = &wait_group;\n\tseries->push_back(task);\n\n\tseries->start();\n\twait_group.wait();\n\tns->del_policy(\"test_policy\");\n}\n\nTEST(upstream_unittest, TryAnother)\n{\n\tWFFacilities::WaitGroup wait_group(3);\n\n\tUpstreamManager::upstream_disable_server(\"manual\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_disable_server(\"round.robin\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_disable_server(\"try_another\", \"127.0.0.1:8001\");\n\n\thttp_callback_t cb2 = std::bind(basic_callback, std::placeholders::_1,\n\t\t\t\t\t\t\t\t std::string(\"server2\"));\n\n\tWFHttpTask *task = WFTaskFactory::create_http_task(\"http://manual\",\n\t\t\t\t\t\t\t\t\t\t\t\t\t REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\t\t cb2);\n\ttask->user_data = &wait_group;\n\ttask->start();\n\n\t// this->cur_idx == 1. Will skip 8001 and try 8002.\n\ttask = WFTaskFactory::create_http_task(\"http://round.robin\",\n\t\t\t\t\t\t\t\t\t\t REDIRECT_MAX, RETRY_MAX, cb2);\n\ttask->user_data = &wait_group;\n\ttask->start();\n\n\ttask = WFTaskFactory::create_http_task(\"http://try_another\",\n\t\t\t\t\t\t\t\t\t\t REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t [&wait_group](WFHttpTask *task){\n\t\tint state = task->get_state();\n\t\tEXPECT_EQ(state, WFT_STATE_TASK_ERROR);\n\t\tEXPECT_EQ(task->get_error(), WFT_ERR_UPSTREAM_UNAVAILABLE);\n\t\twait_group.done();\n\t});\n\ttask->start();\n\n\twait_group.wait();\n\tUpstreamManager::upstream_enable_server(\"manual\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_enable_server(\"round.robin\", \"127.0.0.1:8001\");\n\tUpstreamManager::upstream_enable_server(\"try_another\", \"127.0.0.1:8001\");\n}\n\nTEST(upstream_unittest, Tracing)\n{\n\tWFFacilities::WaitGroup wait_group(2);\n\n\thttp_server1.stop();\n\n\t// test first_strategy()\n\tWFHttpTask *task = WFTaskFactory::create_http_task(\n\t\t\t\t\t\t\t\t\t\t\t\"http://weighted.random\",\n\t\t\t\t\t\t\t\t\t\t\tREDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\tstd::bind(basic_callback,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::placeholders::_1,\n\t\t\t\t\t\t\t\t \t\t\t\t\t std::string(\"server2\")));\n\ttask->user_data = &wait_group;\n\ttask->start();\n\n\t// test another_strategy()\n\tUpstreamManager::upstream_disable_server(\"test_tracing\",\n\t\t\t\t\t\t\t\t\t\t\t \"127.0.0.1:8003\");\n\tWFHttpTask *task2 = WFTaskFactory::create_http_task(\n\t\t\t\t\t\t\t\t\t\t\t\"http://test_tracing\",\n\t\t\t\t\t\t\t\t\t\t\tREDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\tstd::bind(basic_callback,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::placeholders::_1,\n\t\t\t\t\t\t\t\t \t\t\t\t\t std::string(\"server2\")));\n\ttask2->user_data = &wait_group;\n\ttask2->start();\n\n\twait_group.wait();\n\tEXPECT_TRUE(http_server1.start(\"127.0.0.1\", 8001) == 0)\n\t\t\t\t<< \"http server start failed\";\n\tUpstreamManager::upstream_enable_server(\"test_tracing\", \"127.0.0.1:8003\");\n}\n\n\nTEST(upstream_unittest, RoundRobin)\n{\n\tWFFacilities::WaitGroup wait_group(1);\n\n\t// this->cur_idx = 0. When 8002 is removed, we will try 8001.\n\tUpstreamManager::upstream_remove_server(\"round.robin\", \"127.0.0.1:8002\");\n\tWFHttpTask *task = WFTaskFactory::create_http_task(\"http://round.robin\",\n\t\t\t\t\t\t\t\t\t\t\t\t\tREDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::bind(basic_callback,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::placeholders::_1,\n\t\t\t\t\t\t\t\t\t\t\t\t\tstd::string(\"server1\")));\n\ttask->user_data = &wait_group;\n\ttask->start();\n\n\twait_group.wait();\n\tUpstreamManager::upstream_add_server(\"round.robin\", \"127.0.0.1:8002\");\n}\n\nint main(int argc, char* argv[])\n{\n\t::testing::InitGoogleTest(&argc, argv);\n\n\tregister_upstream_hosts();\n\n\tEXPECT_TRUE(http_server1.start(\"127.0.0.1\", 8001) == 0)\n\t\t\t\t<< \"http server start failed\";\n\n\tEXPECT_TRUE(http_server2.start(\"127.0.0.1\", 8002) == 0)\n\t\t\t\t<< \"http server start failed\";\n\n\tEXPECT_TRUE(http_server3.start(\"127.0.0.1\", 8003) == 0)\n\t\t\t\t<< \"http server start failed\";\n\t\t\n\tEXPECT_EQ(RUN_ALL_TESTS(), 0);\n\n\tEXPECT_EQ(UpstreamManager::upstream_delete(\"try_another\"), 0);\n\tEXPECT_EQ(UpstreamManager::upstream_delete(\"try_another\"), -1);\n\n\thttp_server1.stop();\n\thttp_server2.stop();\n\thttp_server3.stop();\n\n\treturn 0;\n}\n\n" }, { "path": "test/uriparser_unittest.cc", "content": "/*\n Copyright (c) 2021 Sogou, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\n Author: Wang Zhulei (wangzhulei@sogou-inc.com)\n*/\n\n#include \n#include \"workflow/URIParser.h\"\n\nTEST(uriparser_unittest, parse)\n{\n\tParsedURI uri;\n\n\tEXPECT_EQ(URIParser::parse(\"https://john.doe:pass@www.example.com:123/forum/questions/?tag=networking&order=newest#top\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"https\"), 0);\n\tEXPECT_EQ(strcmp(uri.userinfo, \"john.doe:pass\"), 0);\n\tEXPECT_EQ(strcmp(uri.host, \"www.example.com\"), 0);\n\tEXPECT_EQ(strcmp(uri.port, \"123\"), 0);\n\tEXPECT_EQ(strcmp(uri.path, \"/forum/questions/\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"tag=networking&order=newest\"), 0);\n\tEXPECT_EQ(strcmp(uri.fragment, \"top\"), 0);\n\n\tEXPECT_EQ(URIParser::parse(\"https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"https\"), 0);\n\tEXPECT_EQ(strcmp(uri.userinfo, \"john.doe\"), 0);\n\tEXPECT_EQ(strcmp(uri.host, \"www.example.com\"), 0);\n\tEXPECT_EQ(strcmp(uri.port, \"123\"), 0);\n\tEXPECT_EQ(strcmp(uri.path, \"/forum/questions/\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"tag=networking&order=newest\"), 0);\n\tEXPECT_EQ(strcmp(uri.fragment, \"top\"), 0);\n\n\tEXPECT_EQ(URIParser::parse(\"ldap://[2001:db8::7]/c=GB?objectClass?one\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"ldap\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"2001:db8::7\"), 0);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"/c=GB\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"objectClass?one\"), 0);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"ldap://user@[2001:db8::7]/c=GB?objectClass?one\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"ldap\"), 0);\n\tEXPECT_EQ(strcmp(uri.userinfo, \"user\"), 0);\n\tEXPECT_EQ(strcmp(uri.host, \"2001:db8::7\"), 0);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"/c=GB\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"objectClass?one\"), 0);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"ldap://user@[2001:db8::7]:12345/c=GB?objectClass?one\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"ldap\"), 0);\n\tEXPECT_EQ(strcmp(uri.userinfo, \"user\"), 0);\n\tEXPECT_EQ(strcmp(uri.host, \"2001:db8::7\"), 0);\n\tEXPECT_EQ(strcmp(uri.port, \"12345\"), 0);\n\tEXPECT_EQ(strcmp(uri.path, \"/c=GB\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"objectClass?one\"), 0);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"ldap://[2001:db8::7]:12345/c=GB?objectClass?one\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"ldap\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"2001:db8::7\"), 0);\n\tEXPECT_EQ(strcmp(uri.port, \"12345\"), 0);\n\tEXPECT_EQ(strcmp(uri.path, \"/c=GB\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"objectClass?one\"), 0);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"mailto:John.Doe@example.com\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"mailto\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(uri.host, nullptr);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"John.Doe@example.com\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"news:comp.infosystems.www.servers.unix\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"news\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(uri.host, nullptr);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"comp.infosystems.www.servers.unix\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"tel:+1-816-555-1212\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"tel\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(uri.host, nullptr);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"+1-816-555-1212\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"telnet://192.0.2.16:80/\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"telnet\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"192.0.2.16\"), 0);\n\tEXPECT_EQ(strcmp(uri.port, \"80\"), 0);\n\tEXPECT_EQ(strcmp(uri.path, \"/\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"urn:oasis:names:specification:docbook:dtd:xml:4.1.2\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"urn\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(uri.host, nullptr);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"oasis:names:specification:docbook:dtd:xml:4.1.2\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"https://www.example.com:123/forum/questions/?tag=networking&order=newest#top\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"https\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"www.example.com\"), 0);\n\tEXPECT_EQ(strcmp(uri.port, \"123\"), 0);\n\tEXPECT_EQ(strcmp(uri.path, \"/forum/questions/\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"tag=networking&order=newest\"), 0);\n\tEXPECT_EQ(strcmp(uri.fragment, \"top\"), 0);\n\n\tEXPECT_EQ(URIParser::parse(\"https://john.doe@www.example.com/forum/questions/?tag=networking&order=newest#top\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"https\"), 0);\n\tEXPECT_EQ(strcmp(uri.userinfo, \"john.doe\"), 0);\n\tEXPECT_EQ(strcmp(uri.host, \"www.example.com\"), 0);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"/forum/questions/\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"tag=networking&order=newest\"), 0);\n\tEXPECT_EQ(strcmp(uri.fragment, \"top\"), 0);\n\n\tEXPECT_EQ(URIParser::parse(\"foo:/index.html\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"foo\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(uri.host, nullptr);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"/index.html\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"http://www.test.cn/subject/ttt/index.html?abc-def-jki-lm-rstuvwxyz\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"http\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"www.test.cn\"), 0);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"/subject/ttt/index.html\"), 0);\n\tEXPECT_EQ(strcmp(uri.query, \"abc-def-jki-lm-rstuvwxyz\"), 0);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"http://sg.test1.com/zt/zz/#IJ\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"http\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"sg.test1.com\"), 0);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"/zt/zz/\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(strcmp(uri.fragment, \"IJ\"), 0);\n\n\tEXPECT_EQ(URIParser::parse(\"http://www.test2.com?sg_vid=R_3qHh9H471Ry8OtW5J9R10vc_QR6EQqgA6HHLO6666666qe0Co66666666\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"http\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"www.test2.com\"), 0);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(uri.path, nullptr);\n\tEXPECT_EQ(strcmp(uri.query, \"sg_vid=R_3qHh9H471Ry8OtW5J9R10vc_QR6EQqgA6HHLO6666666qe0Co66666666\"), 0);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"https://sgsares.test3.com/ttts/IJ_4115.apk\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"https\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"sgsares.test3.com\"), 0);\n\tEXPECT_EQ(uri.port, nullptr);\n\tEXPECT_EQ(strcmp(uri.path, \"/ttts/IJ_4115.apk\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"http://viptest.test5.com:8484?sg_vid=Rucnk5BKG81RcIVk7XySNhQtBODR6mKXA06PpWA66666663MTAfR6666666\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"http\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"viptest.test5.com\"), 0);\n\tEXPECT_EQ(strcmp(uri.port, \"8484\"), 0);\n\tEXPECT_EQ(uri.path, nullptr);\n\tEXPECT_EQ(strcmp(uri.query, \"sg_vid=Rucnk5BKG81RcIVk7XySNhQtBODR6mKXA06PpWA66666663MTAfR6666666\"), 0);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"http://viptest1.test6.com:84/abc#frag\", uri), 0);\n\tEXPECT_EQ(strcmp(uri.scheme, \"http\"), 0);\n\tEXPECT_EQ(uri.userinfo, nullptr);\n\tEXPECT_EQ(strcmp(uri.host, \"viptest1.test6.com\"), 0);\n\tEXPECT_EQ(strcmp(uri.port, \"84\"), 0);\n\tEXPECT_EQ(strcmp(uri.path, \"/abc\"), 0);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(strcmp(uri.fragment, \"frag\"), 0);\n\n\tEXPECT_EQ(URIParser::parse(\"http://www.sogou.com\", uri), 0);\n\tEXPECT_EQ(uri.path, nullptr);\n\tEXPECT_EQ(uri.query, nullptr);\n\tEXPECT_EQ(uri.fragment, nullptr);\n\n\tEXPECT_EQ(URIParser::parse(\"http://www.sogou.com?aaa/=bbb\", uri), 0);\n\tEXPECT_EQ(uri.path, nullptr);\n\tEXPECT_EQ(strcmp(uri.query, \"aaa/=bbb\"), 0);\n}\n\n" }, { "path": "test/xmake.lua", "content": "set_group(\"test\")\nset_default(false)\n\nadd_requires(\"gtest\")\n\nadd_deps(\"workflow\")\n\nadd_packages(\"gtest\")\nadd_links(\"gtest_main\")\n\nif not is_plat(\"macosx\") then\n add_ldflags(\"-lrt\")\nend\n\nfunction all_tests()\n local res = {}\n for _, x in ipairs(os.files(\"**.cc\")) do\n local item = {}\n local s = path.filename(x)\n if ((s == \"upstream_unittest.cc\" and not has_config(\"upstream\")) or\n (s == \"redis_unittest.cc\" and not has_config(\"redis\")) or\n (s == \"mysql_unittest.cc\" and not has_config(\"mysql\"))) then\n else\n table.insert(item, s:sub(1, #s - 3)) -- target\n table.insert(item, path.relative(x, \".\")) -- source\n table.insert(res, item)\n end\n end\n return res\nend\n\nfor _, test in ipairs(all_tests()) do\n target(test[1])\n set_kind(\"binary\")\n add_files(test[2])\n if has_config(\"memcheck\") then\n on_run(function (target)\n local argv = {}\n table.insert(argv, target:targetfile())\n table.insert(argv, \"--leak-check=full\")\n os.execv(\"valgrind\", argv)\n end)\n end\nend\n" }, { "path": "tutorial/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.10)\n\nset(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING \"build type\")\n\nproject(tutorial\n\t\tLANGUAGES C CXX\n)\n\nset(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR})\n\nif(ANDROID)\n\tlink_directories(${OPENSSL_LINK_DIR})\nelse()\n\tfind_library(LIBRT rt)\n\tfind_package(OpenSSL REQUIRED)\nendif()\n\nfind_package(workflow REQUIRED CONFIG HINTS ..)\ninclude_directories(${OPENSSL_INCLUDE_DIR} ${WORKFLOW_INCLUDE_DIR})\nlink_directories(${WORKFLOW_LIB_DIR})\n\nfind_library(WORKFLOW_LIB NAMES libworkflow.a workflow HINTS ${WORKFLOW_LIB_DIR})\n\nif (KAFKA STREQUAL \"y\")\n\tfind_library(WFKAFKA_LIB NAMES libwfkafka.a wfkafka HINTS ${WORKFLOW_LIB_DIR})\nendif ()\n\nset(CMAKE_C_FLAGS \"${CMAKE_C_FLAGS} -Wall -fPIC -pipe -std=gnu90\")\nset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wall -fPIC -pipe -std=c++11 -fno-exceptions\")\nif (APPLE)\n\tset(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations\")\nendif()\n\nset(TUTORIAL_LIST\n\ttutorial-00-helloworld\n\ttutorial-01-wget\n\ttutorial-04-http_echo_server\n\ttutorial-05-http_proxy\n\ttutorial-06-parallel_wget\n\ttutorial-07-sort_task\n\ttutorial-08-matrix_multiply\n\ttutorial-09-http_file_server\n\ttutorial-11-graph_task\n\ttutorial-15-name_service\n\ttutorial-17-dns_cli\n\ttutorial-19-dns_server\n)\n\nif (APPLE)\n\tset(LIB ${WORKFLOW_LIB} pthread OpenSSL::SSL OpenSSL::Crypto)\nelseif (ANDROID)\n\tset(LIB ${WORKFLOW_LIB} ssl crypto c)\nelse ()\n\tset(LIB ${WORKFLOW_LIB} pthread OpenSSL::SSL OpenSSL::Crypto ${LIBRT})\nendif ()\n\nforeach(src ${TUTORIAL_LIST})\n\tstring(REPLACE \"-\" \";\" arr ${src})\n\tlist(GET arr -1 bin_name)\n\tadd_executable(${bin_name} ${src}.cc)\n\ttarget_link_libraries(${bin_name} ${LIB})\nendforeach()\n\nif (NOT REDIS STREQUAL \"n\")\nset(TUTORIAL_LIST\n\ttutorial-02-redis_cli\n\ttutorial-03-wget_to_redis\n\ttutorial-18-redis_subscriber\n)\nforeach(src ${TUTORIAL_LIST})\n\tstring(REPLACE \"-\" \";\" arr ${src})\n\tlist(GET arr -1 bin_name)\n\tadd_executable(${bin_name} ${src}.cc)\n\ttarget_link_libraries(${bin_name} ${LIB})\nendforeach()\nendif()\n\nif (NOT MYSQL STREQUAL \"n\")\nset(TUTORIAL_LIST\n\ttutorial-12-mysql_cli\n)\nforeach(src ${TUTORIAL_LIST})\n\tstring(REPLACE \"-\" \";\" arr ${src})\n\tlist(GET arr -1 bin_name)\n\tadd_executable(${bin_name} ${src}.cc)\n\ttarget_link_libraries(${bin_name} ${LIB})\nendforeach()\nendif()\n\nif (NOT CONSUL STREQUAL \"n\")\nset(TUTORIAL_LIST\n\ttutorial-14-consul_cli\n)\nforeach(src ${TUTORIAL_LIST})\n\tstring(REPLACE \"-\" \";\" arr ${src})\n\tlist(GET arr -1 bin_name)\n\tadd_executable(${bin_name} ${src}.cc)\n\ttarget_link_libraries(${bin_name} ${LIB})\nendforeach()\nendif()\n\nif (KAFKA STREQUAL \"y\")\n\tadd_executable(\"kafka_cli\" \"tutorial-13-kafka_cli.cc\")\n\tfind_package(ZLIB REQUIRED)\n\tfind_path(SNAPPY_INCLUDE_PATH NAMES snappy.h)\n\tfind_library(SNAPPY_LIB NAMES snappy)\n\tif ((NOT SNAPPY_INCLUDE_PATH) OR (NOT SNAPPY_LIB))\n\t\tmessage(FATAL_ERROR \"Fail to find snappy with KAFKA=y\")\n\tendif ()\n\tinclude_directories(${SNAPPY_INCLUDE_PATH})\n\n\tfind_path(ZSTD_INCLUDE_PATH NAMES zstd.h)\n\tfind_library(ZSTD_LIB NAMES zstd)\n\tif ((NOT ZSTD_INCLUDE_PATH) OR (NOT ZSTD_LIB))\n\t\tmessage(FATAL_ERROR \"Fail to find zstd with KAFKA=y\")\n\tendif ()\n\tinclude_directories(${ZSTD_INCLUDE_PATH})\n\n\tfind_path(LZ4_INCLUDE_PATH NAMES lz4.h)\n\tfind_library(LZ4_LIB NAMES lz4)\n\tif ((NOT LZ4_INCLUDE_PATH) OR (NOT LZ4_LIB))\n\t\tmessage(FATAL_ERROR \"Fail to find lz4 with KAFKA=y\")\n\tendif ()\n\tinclude_directories(${LZ4_INCLUDE_PATH})\n\ttarget_link_libraries(\"kafka_cli\" ${WFKAFKA_LIB} ${LIB} ZLIB::ZLIB ${SNAPPY_LIB} ${LZ4_LIB} ${ZSTD_LIB})\nendif ()\n\nset(DIR10 tutorial-10-user_defined_protocol)\nadd_executable(server ${DIR10}/server.cc ${DIR10}/message.cc)\nadd_executable(client ${DIR10}/client.cc ${DIR10}/message.cc)\nadd_executable(server-uds ${DIR10}/server-uds.cc ${DIR10}/message.cc)\nadd_executable(client-uds ${DIR10}/client-uds.cc ${DIR10}/message.cc)\ntarget_link_libraries(server ${LIB})\ntarget_link_libraries(client ${LIB})\ntarget_link_libraries(server-uds ${LIB})\ntarget_link_libraries(client-uds ${LIB})\n\nset_target_properties(server PROPERTIES\n\tRUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/${DIR10})\nset_target_properties(client PROPERTIES\n\tRUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/${DIR10})\nset_target_properties(server-uds PROPERTIES\n\tRUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/${DIR10})\nset_target_properties(client-uds PROPERTIES\n\tRUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/${DIR10})\n\nset(DIR16 tutorial-16-graceful_restart)\nadd_executable(bootstrap ${DIR16}/bootstrap.c)\nadd_executable(bootstrap_server ${DIR16}/server.cc)\ntarget_link_libraries(bootstrap_server ${LIB})\n\nset_target_properties(bootstrap PROPERTIES\n\tRUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/${DIR16})\nset_target_properties(bootstrap_server PROPERTIES OUTPUT_NAME \"server\"\n\tRUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/${DIR16})\n" }, { "path": "tutorial/GNUmakefile", "content": "ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))\nALL_TARGETS := all clean\nMAKE_FILE := Makefile\n\nDEFAULT_BUILD_DIR := build.cmake\nBUILD_DIR := $(shell if [ -f $(MAKE_FILE) ]; then echo \".\"; else echo $(DEFAULT_BUILD_DIR); fi)\nCMAKE3 := $(shell if which cmake3>/dev/null ; then echo cmake3; else echo cmake; fi;)\n\n.PHONY: $(ALL_TARGETS)\n\nall:\n\tmkdir -p $(BUILD_DIR)\n\trm -rf $(DEFAULT_BUILD_DIR)/CMakeCache.txt\nifeq ($(DEBUG),y)\n\tcd $(BUILD_DIR) && $(CMAKE3) -D CMAKE_BUILD_TYPE=Debug -D CONSUL=$(CONSUL) -D KAFKA=$(KAFKA) -D MYSQL=$(MYSQL) -D REDIS=$(REDIS) $(ROOT_DIR)\nelse\n\tcd $(BUILD_DIR) && $(CMAKE3) -D CONSUL=$(CONSUL) -D KAFKA=$(KAFKA) -D MYSQL=$(MYSQL) -D REDIS=$(REDIS) $(ROOT_DIR)\nendif\n\t$(MAKE) -C $(BUILD_DIR) -f Makefile\n\nclean:\nifeq ($(MAKE_FILE), $(wildcard $(MAKE_FILE)))\n\t-$(MAKE) -f Makefile clean\nelse ifeq ($(DEFAULT_BUILD_DIR), $(wildcard $(DEFAULT_BUILD_DIR)))\n\t-$(MAKE) -C $(DEFAULT_BUILD_DIR) clean\nendif\n\trm -rf $(DEFAULT_BUILD_DIR)\n" }, { "path": "tutorial/tutorial-00-helloworld.cc", "content": "#include \n#include \"workflow/WFHttpServer.h\"\n\nint main()\n{\n WFHttpServer server([](WFHttpTask *task) {\n task->get_resp()->append_output_body(\"Hello World!\");\n });\n\n if (server.start(8888) == 0) { // start server on port 8888\n getchar(); // press \"Enter\" to end.\n server.stop();\n }\n\n return 0;\n}\n\n" }, { "path": "tutorial/tutorial-01-wget.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/HttpMessage.h\"\n#include \"workflow/HttpUtil.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\n#define REDIRECT_MAX 5\n#define RETRY_MAX 2\n\nvoid wget_callback(WFHttpTask *task)\n{\n\tprotocol::HttpRequest *req = task->get_req();\n\tprotocol::HttpResponse *resp = task->get_resp();\n\tint state = task->get_state();\n\tint error = task->get_error();\n\n\tswitch (state)\n\t{\n\tcase WFT_STATE_SYS_ERROR:\n\t\tfprintf(stderr, \"system error: %s\\n\", strerror(error));\n\t\tbreak;\n\tcase WFT_STATE_DNS_ERROR:\n\t\tfprintf(stderr, \"DNS error: %s\\n\", gai_strerror(error));\n\t\tbreak;\n\tcase WFT_STATE_SSL_ERROR:\n\t\tfprintf(stderr, \"SSL error: %d\\n\", error);\n\t\tbreak;\n\tcase WFT_STATE_TASK_ERROR:\n\t\tfprintf(stderr, \"Task error: %d\\n\", error);\n\t\tbreak;\n\tcase WFT_STATE_SUCCESS:\n\t\tbreak;\n\t}\n\n\tif (state != WFT_STATE_SUCCESS)\n\t{\n\t\tfprintf(stderr, \"Failed. Press Ctrl-C to exit.\\n\");\n\t\treturn;\n\t}\n\n\tstd::string name;\n\tstd::string value;\n\n\t/* Print request. */\n\tfprintf(stderr, \"%s %s %s\\r\\n\", req->get_method(),\n\t\t\t\t\t\t\t\t\treq->get_http_version(),\n\t\t\t\t\t\t\t\t\treq->get_request_uri());\n\n\tprotocol::HttpHeaderCursor req_cursor(req);\n\n\twhile (req_cursor.next(name, value))\n\t\tfprintf(stderr, \"%s: %s\\r\\n\", name.c_str(), value.c_str());\n\tfprintf(stderr, \"\\r\\n\");\n\n\t/* Print response header. */\n\tfprintf(stderr, \"%s %s %s\\r\\n\", resp->get_http_version(),\n\t\t\t\t\t\t\t\t\tresp->get_status_code(),\n\t\t\t\t\t\t\t\t\tresp->get_reason_phrase());\n\n\tprotocol::HttpHeaderCursor resp_cursor(resp);\n\n\twhile (resp_cursor.next(name, value))\n\t\tfprintf(stderr, \"%s: %s\\r\\n\", name.c_str(), value.c_str());\n\tfprintf(stderr, \"\\r\\n\");\n\n\t/* Print response body. */\n\tconst void *body;\n\tsize_t body_len;\n\n\tresp->get_parsed_body(&body, &body_len);\n\tfwrite(body, 1, body_len, stdout);\n\tfflush(stdout);\n\n\tfprintf(stderr, \"\\nSuccess. Press Ctrl-C to exit.\\n\");\n}\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tWFHttpTask *task;\n\n\tif (argc != 2)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tsignal(SIGINT, sig_handler);\n\n\tstd::string url = argv[1];\n\tif (strncasecmp(argv[1], \"http://\", 7) != 0 &&\n\t\tstrncasecmp(argv[1], \"https://\", 8) != 0)\n\t{\n\t\turl = \"http://\" + url;\n\t}\n\n\ttask = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t wget_callback);\n\tprotocol::HttpRequest *req = task->get_req();\n\treq->add_header_pair(\"Accept\", \"*/*\");\n\treq->add_header_pair(\"User-Agent\", \"Wget/1.14 (linux-gnu)\");\n\treq->add_header_pair(\"Connection\", \"close\");\n\ttask->start();\n\n\twait_group.wait();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-02-redis_cli.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/RedisMessage.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\n#define RETRY_MAX 2\n\nstruct tutorial_task_data\n{\n\tstd::string url;\n\tstd::string key;\n};\n\nvoid redis_callback(WFRedisTask *task)\n{\n\tprotocol::RedisRequest *req = task->get_req();\n\tprotocol::RedisResponse *resp = task->get_resp();\n\tint state = task->get_state();\n\tint error = task->get_error();\n\tprotocol::RedisValue val;\n\n\tswitch (state)\n\t{\n\tcase WFT_STATE_SYS_ERROR:\n\t\tfprintf(stderr, \"system error: %s\\n\", strerror(error));\n\t\tbreak;\n\tcase WFT_STATE_DNS_ERROR:\n\t\tfprintf(stderr, \"DNS error: %s\\n\", gai_strerror(error));\n\t\tbreak;\n\tcase WFT_STATE_SSL_ERROR:\n\t\tfprintf(stderr, \"SSL error: %d\\n\", error);\n\t\tbreak;\n\tcase WFT_STATE_TASK_ERROR:\n\t\tfprintf(stderr, \"Task error: %d\\n\", error);\n\t\tbreak;\n\tcase WFT_STATE_SUCCESS:\n\t\tresp->get_result(val);\n\t\tif (val.is_error())\n\t\t{\n\t\t\tfprintf(stderr, \"%*s\\n\", (int)val.string_view()->size(),\n\t\t\t\t\t\t\t\t\tval.string_view()->c_str());\n\t\t\tstate = WFT_STATE_TASK_ERROR;\n\t\t}\n\t\tbreak;\n\t}\n\n\tif (state != WFT_STATE_SUCCESS)\n\t{\n\t\tfprintf(stderr, \"Failed. Press Ctrl-C to exit.\\n\");\n\t\treturn;\n\t}\n\n\tstd::string cmd;\n\treq->get_command(cmd);\n\tif (cmd == \"SET\")\n\t{\n\t\ttutorial_task_data *data = (tutorial_task_data *)task->user_data;\n\t\tWFRedisTask *next = WFTaskFactory::create_redis_task(data->url,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t redis_callback);\n\n\t\tnext->get_req()->set_request(\"GET\", { data->key });\n\t\t/* Push next task(GET task) to current series. */\n\t\tseries_of(task)->push_back(next);\n\t\tfprintf(stderr, \"Redis SET request success. Trying to GET...\\n\");\n\t}\n\telse /* if (cmd == \"GET\") */\n\t{\n\t\tif (val.is_string())\n\t\t{\n\t\t\tfprintf(stderr, \"Redis GET success. value = %s\\n\",\n\t\t\t\t\tval.string_value().c_str());\n\t\t}\n\t\telse\n\t\t{\n\t\t\tfprintf(stderr, \"Error: Not a string value. \\n\");\n\t\t}\n\n\t\tfprintf(stderr, \"Finished. Press Ctrl-C to exit.\\n\");\n\t}\n}\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tWFRedisTask *task;\n\n\tif (argc != 4)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tsignal(SIGINT, sig_handler);\n\n\t/* This struct only used in this tutorial. */\n\tstruct tutorial_task_data data;\n\n\t/* Redis URL format: redis://:password@host:port/dbnum\n\t examples:\n\t redis://127.0.0.1\n\t redis://:12345@redis.sogou:6379/3\n\t*/\n\tdata.url = argv[1];\n\tif (strncasecmp(argv[1], \"redis://\", 8) != 0 &&\n\t\tstrncasecmp(argv[1], \"rediss://\", 9) != 0)\n\t{\n\t\tdata.url = \"redis://\" + data.url;\n\t}\n\n\tdata.key = argv[2];\n\n\ttask = WFTaskFactory::create_redis_task(data.url, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\tredis_callback);\n\tprotocol::RedisRequest *req = task->get_req();\n\treq->set_request(\"SET\", { data.key, argv[3] });\n\n\t/* task->user_data is a public (void *), can store anything. */\n\ttask->user_data = &data;\n\n\t/* task->start() equel to:\n\t * Workflow::start_series_work(task, nullptr) or\n\t * Workflow::create_series_work(task, nullptr)->start() */\n\ttask->start();\n\n\twait_group.wait();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-03-wget_to_redis.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \"workflow/HttpMessage.h\"\n#include \"workflow/HttpUtil.h\"\n#include \"workflow/RedisMessage.h\"\n#include \"workflow/Workflow.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\nusing namespace protocol;\n\n#define REDIRECT_MAX 5\n#define RETRY_MAX 2\n\nstruct tutorial_series_context\n{\n\tstd::string http_url;\n\tstd::string redis_url;\n\tsize_t body_len;\n\tbool success;\n};\n\nvoid redis_callback(WFRedisTask *task)\n{\n\tint state = task->get_state();\n\ttutorial_series_context *context =\n\t\t(tutorial_series_context *)series_of(task)->get_context();\n\tRedisValue value;\n\n\tif (state == WFT_STATE_SUCCESS)\n\t{\n\t\ttask->get_resp()->get_result(value);\n\t\tif (!value.is_error())\n\t\t{\n\t\t\tfprintf(stderr, \"redis SET success: key: %s, value size: %zu\\n\",\n\t\t\t\t\t\t\tcontext->http_url.c_str(), context->body_len);\n\t\t\tcontext->success = true;\n\t\t}\n\t\telse\n\t\t\tfprintf(stderr, \"redis error reply! Need password?\\n\");\n\t}\n\telse\n\t{\n\t\tfprintf(stderr, \"redis SET error: state = %d, error = %d\\n\",\n\t\t\t\tstate, task->get_error());\n\t}\n}\n\nvoid http_callback(WFHttpTask *task)\n{\n\tHttpResponse *resp = task->get_resp();\n\tint state = task->get_state();\n\tint error = task->get_error();\n\n\tif (state != WFT_STATE_SUCCESS)\n\t{\n\t\tfprintf(stderr, \"http task error: state = %d, error = %d\\n\",\n\t\t\t\t\t\tstate, error);\n\t\treturn;\n\t}\n\n\tSeriesWork *series = series_of(task); /* get the series of this task */\n\ttutorial_series_context *context =\n\t\t(tutorial_series_context *)series->get_context();\n\n\tconst void *body;\n\tsize_t body_len;\n\n\tresp->get_parsed_body(&body, &body_len);\n\tif (body_len == 0)\n\t{\n\t\tfprintf(stderr, \"Error: empty http body!\");\n\t\treturn;\n\t}\n\n\tcontext->body_len = body_len;\n\n\tWFRedisTask *redis_task =\n\t\tWFTaskFactory::create_redis_task(context->redis_url, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t redis_callback);\n\n\tstd::string value((char *)body, body_len);\n\tredis_task->get_req()->set_request(\"SET\", { context->http_url, value });\n\t*series << redis_task; /* equal to series->push_back(redis_task) */\n}\n\nint main(int argc, char *argv[])\n{\n\tWFHttpTask *http_task;\n\n\tif (argc != 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tstruct tutorial_series_context context;\n\n\tcontext.success = false;\n\tcontext.http_url = argv[1];\n\tif (strncasecmp(argv[1], \"http://\", 7) != 0 &&\n\t\tstrncasecmp(argv[1], \"https://\", 8) != 0)\n\t{\n\t\tcontext.http_url = \"http://\" + context.http_url;\n\t}\n\n\tcontext.redis_url = argv[2];\n\tif (strncasecmp(argv[2], \"redis://\", 8) != 0 &&\n\t\tstrncasecmp(argv[2], \"rediss://\", 9) != 0)\n\t{\n\t\tcontext.redis_url = \"redis://\" + context.redis_url;\n\t}\n\n\thttp_task = WFTaskFactory::create_http_task(context.http_url,\n\t\t\t\t\t\t\t\t\t\t\t\tREDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\thttp_callback);\n\tHttpRequest *req = http_task->get_req();\n\treq->add_header_pair(\"Accept\", \"*/*\");\n\treq->add_header_pair(\"User-Agent\", \"Wget/1.14 (linux-gnu)\");\n\treq->add_header_pair(\"Connection\", \"close\");\n\n\t/* Limit the http response size to 20M. */\n\thttp_task->get_resp()->set_size_limit(20 * 1024 * 1024);\n\n\t/* no more than 30 seconds receiving http response. */\n\thttp_task->set_receive_timeout(30 * 1000);\n\n\tWFFacilities::WaitGroup wait_group(1);\n\n\tauto series_callback = [&wait_group](const SeriesWork *series)\n\t{\n\t\ttutorial_series_context *context = (tutorial_series_context *)\n\t\t\t\t\t\t\t\t\t\t\tseries->get_context();\n\n\t\tif (context->success)\n\t\t\tfprintf(stderr, \"Series finished. all success!\\n\");\n\t\telse\n\t\t\tfprintf(stderr, \"Series finished. failed!\\n\");\n\n\t\t/* signal the main() to terminate */\n\t\twait_group.done();\n\t};\n\n\t/* Create a series */\n\tSeriesWork *series = Workflow::create_series_work(http_task,\n\t\t\t\t\t\t\t\t\t\t\t\t\t series_callback);\n\tseries->set_context(&context);\n\tseries->start();\n\n\twait_group.wait();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-04-http_echo_server.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/HttpMessage.h\"\n#include \"workflow/HttpUtil.h\"\n#include \"workflow/WFServer.h\"\n#include \"workflow/WFHttpServer.h\"\n#include \"workflow/WFFacilities.h\"\n\nvoid process(WFHttpTask *server_task)\n{\n\tprotocol::HttpRequest *req = server_task->get_req();\n\tprotocol::HttpResponse *resp = server_task->get_resp();\n\tlong long seq = server_task->get_task_seq();\n\tprotocol::HttpHeaderCursor cursor(req);\n\tstd::string name;\n\tstd::string value;\n\tchar buf[8192];\n\tint len;\n\n\t/* Set response message body. */\n\tresp->append_output_body_nocopy(\"\", 6);\n\tlen = snprintf(buf, 8192, \"

%s %s %s

\", req->get_method(),\n\t\t\t\t req->get_request_uri(), req->get_http_version());\n\tresp->append_output_body(buf, len);\n\n\twhile (cursor.next(name, value))\n\t{\n\t\tlen = snprintf(buf, 8192, \"

%s: %s

\", name.c_str(), value.c_str());\n\t\tresp->append_output_body(buf, len);\n\t}\n\n\tresp->append_output_body_nocopy(\"\", 7);\n\n\t/* Set status line if you like. */\n\tresp->set_http_version(\"HTTP/1.1\");\n\tresp->set_status_code(\"200\");\n\tresp->set_reason_phrase(\"OK\");\n\n\tresp->add_header_pair(\"Content-Type\", \"text/html\");\n\tresp->add_header_pair(\"Server\", \"Sogou WFHttpServer\");\n\tif (seq == 9) /* no more than 10 requests on the same connection. */\n\t\tresp->add_header_pair(\"Connection\", \"close\");\n\n\t/* print some log */\n\tchar addrstr[128];\n\tstruct sockaddr_storage addr;\n\tsocklen_t l = sizeof addr;\n\tunsigned short port = 0;\n\n\tserver_task->get_peer_addr((struct sockaddr *)&addr, &l);\n\tif (addr.ss_family == AF_INET)\n\t{\n\t\tstruct sockaddr_in *sin = (struct sockaddr_in *)&addr;\n\t\tinet_ntop(AF_INET, &sin->sin_addr, addrstr, 128);\n\t\tport = ntohs(sin->sin_port);\n\t}\n\telse if (addr.ss_family == AF_INET6)\n\t{\n\t\tstruct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&addr;\n\t\tinet_ntop(AF_INET6, &sin6->sin6_addr, addrstr, 128);\n\t\tport = ntohs(sin6->sin6_port);\n\t}\n\telse\n\t\tstrcpy(addrstr, \"Unknown\");\n\n\tfprintf(stderr, \"Peer address: %s:%d, seq: %lld.\\n\",\n\t\t\taddrstr, port, seq);\n}\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tunsigned short port;\n\n\tif (argc != 2)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tsignal(SIGINT, sig_handler);\n\n\tWFHttpServer server(process);\n\tport = atoi(argv[1]);\n\tif (server.start(port) == 0)\n\t{\n\t\twait_group.wait();\n\t\tserver.stop();\n\t}\n\telse\n\t{\n\t\tperror(\"Cannot start server\");\n\t\texit(1);\n\t}\n\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-05-http_proxy.cc", "content": "#include \n#include \n#include \n#include \n#include \"workflow/Workflow.h\"\n#include \"workflow/HttpMessage.h\"\n#include \"workflow/HttpUtil.h\"\n#include \"workflow/WFHttpServer.h\"\n#include \"workflow/WFFacilities.h\"\n\nstruct tutorial_series_context\n{\n\tstd::string url;\n\tWFHttpTask *proxy_task;\n\tbool is_keep_alive;\n};\n\nvoid reply_callback(WFHttpTask *proxy_task)\n{\n\tSeriesWork *series = series_of(proxy_task);\n\ttutorial_series_context *context =\n\t\t(tutorial_series_context *)series->get_context();\n\tauto *proxy_resp = proxy_task->get_resp();\n\tsize_t size = proxy_resp->get_output_body_size();\n\n\tif (proxy_task->get_state() == WFT_STATE_SUCCESS)\n\t\tfprintf(stderr, \"%s: Success. Http Status: %s, BodyLength: %zu\\n\",\n\t\t\t\tcontext->url.c_str(), proxy_resp->get_status_code(), size);\n\telse /* WFT_STATE_SYS_ERROR*/\n\t\tfprintf(stderr, \"%s: Reply failed: %s, BodyLength: %zu\\n\",\n\t\t\t\tcontext->url.c_str(), strerror(proxy_task->get_error()), size);\n}\n\nvoid http_callback(WFHttpTask *task)\n{\n\tint state = task->get_state();\n\tint error = task->get_error();\n\tauto *resp = task->get_resp();\n\tSeriesWork *series = series_of(task);\n\ttutorial_series_context *context =\n\t\t(tutorial_series_context *)series->get_context();\n\tauto *proxy_resp = context->proxy_task->get_resp();\n\n\tif (state == WFT_STATE_SUCCESS)\n\t{\n\t\tconst void *body;\n\t\tsize_t len;\n\n\t\t/* set a callback for getting reply status. */\n\t\tcontext->proxy_task->set_callback(reply_callback);\n\n\t\t/* Copy the remote webserver's response, to proxy response. */\n\t\tresp->get_parsed_body(&body, &len);\n\t\tresp->append_output_body_nocopy(body, len);\n\t\t*proxy_resp = std::move(*resp);\n\t\tif (!context->is_keep_alive)\n\t\t\tproxy_resp->set_header_pair(\"Connection\", \"close\");\n\t}\n\telse\n\t{\n\t\tconst char *err_string;\n\n\t\tif (state == WFT_STATE_SYS_ERROR)\n\t\t\terr_string = strerror(error);\n\t\telse if (state == WFT_STATE_DNS_ERROR)\n\t\t\terr_string = gai_strerror(error);\n\t\telse if (state == WFT_STATE_SSL_ERROR)\n\t\t\terr_string = \"SSL error\";\n\t\telse /* if (state == WFT_STATE_TASK_ERROR) */\n\t\t\terr_string = \"URL error (Cannot be a HTTPS proxy)\";\n\n\t\tfprintf(stderr, \"%s: Fetch failed. state = %d, error = %d: %s\\n\",\n\t\t\t\tcontext->url.c_str(), state, error, err_string);\n\n\t\t/* As a tutorial, make it simple. And ignore reply status. */\n\t\tproxy_resp->set_status_code(\"404\");\n\t\tproxy_resp->append_output_body_nocopy(\n\t\t\t\t\t\t\t\"404 Not Found.\", 27);\n\t}\n}\n\nvoid process(WFHttpTask *proxy_task)\n{\n\tauto *req = proxy_task->get_req();\n\tSeriesWork *series = series_of(proxy_task);\n\tWFHttpTask *http_task; /* for requesting remote webserver. */\n\n\ttutorial_series_context *context = new tutorial_series_context;\n\tcontext->url = req->get_request_uri();\n\tcontext->proxy_task = proxy_task;\n\n\tseries->set_context(context);\n\tseries->set_callback([](const SeriesWork *series) {\n\t\tdelete (tutorial_series_context *)series->get_context();\n\t});\n\n\tcontext->is_keep_alive = req->is_keep_alive();\n\thttp_task = WFTaskFactory::create_http_task(req->get_request_uri(), 0, 0,\n\t\t\t\t\t\t\t\t\t\t\t\thttp_callback);\n\n\tconst void *body;\n\tsize_t len;\n\n\t/* Copy user's request to the new task's reuqest using std::move() */\n\treq->set_request_uri(http_task->get_req()->get_request_uri());\n\treq->get_parsed_body(&body, &len);\n\treq->append_output_body_nocopy(body, len);\n\t*http_task->get_req() = std::move(*req);\n\n\t/* also, limit the remote webserver response size. */\n\thttp_task->get_resp()->set_size_limit(200 * 1024 * 1024);\n\n\t*series << http_task;\n}\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tunsigned short port;\n\n\tif (argc != 2)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tport = atoi(argv[1]);\n\tsignal(SIGINT, sig_handler);\n\n\tstruct WFServerParams params = HTTP_SERVER_PARAMS_DEFAULT;\n\t/* for safety, limit request size to 8MB. */\n\tparams.request_size_limit = 8 * 1024 * 1024;\n\n\tWFHttpServer server(¶ms, process);\n\n\tif (server.start(port) == 0)\n\t{\n\t\twait_group.wait();\n\t\tserver.stop();\n\t}\n\telse\n\t{\n\t\tperror(\"Cannot start server\");\n\t\texit(1);\n\t}\n\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-06-parallel_wget.cc", "content": "#include \n#include \n#include \n#include \n#include \"workflow/Workflow.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/HttpMessage.h\"\n#include \"workflow/HttpUtil.h\"\n#include \"workflow/WFFacilities.h\"\n\nusing namespace protocol;\n\n#define REDIRECT_MAX 5\n#define RETRY_MAX 2\n\nstruct tutorial_series_context\n{\n\tstd::string url;\n\tint state;\n\tint error;\n\tHttpResponse resp;\n};\n\nvoid callback(const ParallelWork *pwork)\n{\n\ttutorial_series_context *ctx;\n\tconst void *body;\n\tsize_t size;\n\n\tfor (const SeriesWork *series : *pwork)\n\t{\n\t\tctx = (tutorial_series_context *)series->get_context();\n\t\tprintf(\"%s\\n\", ctx->url.c_str());\n\t\tif (ctx->state == WFT_STATE_SUCCESS)\n\t\t{\n\t\t\tctx->resp.get_parsed_body(&body, &size);\n\t\t\tprintf(\"%zu%s\\n\", size, ctx->resp.is_chunked() ? \" chunked\" : \"\");\n\t\t\tfwrite(body, 1, size, stdout);\n\t\t\tprintf(\"\\n\");\n\t\t}\n\t\telse\n\t\t\tprintf(\"ERROR! state = %d, error = %d\\n\", ctx->state, ctx->error);\n\n\t\tdelete ctx;\n\t}\n}\n\nint main(int argc, char *argv[])\n{\n\tParallelWork *pwork = Workflow::create_parallel_work(callback);\n\tSeriesWork *series;\n\tWFHttpTask *task;\n\tHttpRequest *req;\n\ttutorial_series_context *ctx;\n\tint i;\n\n\tfor (i = 1; i < argc; i++)\n\t{\n\t\tstd::string url(argv[i]);\n\n\t\tif (strncasecmp(argv[i], \"http://\", 7) != 0 &&\n\t\t\tstrncasecmp(argv[i], \"https://\", 8) != 0)\n\t\t{\n\t\t\turl = \"http://\" +url;\n\t\t}\n\n\t\ttask = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,\n\t\t\t[](WFHttpTask *task)\n\t\t{\n\t\t\ttutorial_series_context *ctx =\n\t\t\t\t(tutorial_series_context *)series_of(task)->get_context();\n\t\t\tctx->state = task->get_state();\n\t\t\tctx->error = task->get_error();\n\t\t\tctx->resp = std::move(*task->get_resp());\n\t\t});\n\n\t\treq = task->get_req();\n\t\treq->add_header_pair(\"Accept\", \"*/*\");\n\t\treq->add_header_pair(\"User-Agent\", \"Wget/1.14 (linux-gnu)\");\n\t\treq->add_header_pair(\"Connection\", \"close\");\n\n\t\tctx = new tutorial_series_context;\n\t\tctx->url = std::move(url);\n\t\tseries = Workflow::create_series_work(task, nullptr);\n\t\tseries->set_context(ctx);\n\t\tpwork->add_series(series);\n\t}\n\n\tWFFacilities::WaitGroup wait_group(1);\n\n\tWorkflow::start_series_work(pwork, [&wait_group](const SeriesWork *) {\n\t\twait_group.done();\n\t});\n\n\twait_group.wait();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-07-sort_task.cc", "content": "#include \n#include \n#include \"workflow/WFAlgoTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\nusing namespace algorithm;\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nbool use_parallel_sort = false;\n\nvoid callback(WFSortTask *task)\n{\n\t/* Sort task's input and output are identical. */\n\tSortInput *input = task->get_input();\n\tint *first = input->first;\n\tint *last = input->last;\n\n\t/* You may remove this output to test speed. */\n\tint *p = first;\n\n\twhile (p < last)\n\t\tprintf(\"%d \", *p++);\n\n\tprintf(\"\\n\");\n\tif (task->user_data == NULL)\n\t{\n\t\tauto cmp = [](int a1, int a2)->bool{return a2 *reverse;\n\n\t\tif (use_parallel_sort)\n\t\t\treverse = WFAlgoTaskFactory::create_psort_task(\"sort\", first, last,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t cmp, callback);\n\t\telse\n\t\t\treverse = WFAlgoTaskFactory::create_sort_task(\"sort\", first, last,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t cmp, callback);\n\n\t\treverse->user_data = (void *)1;\t/* as a flag */\n\t\tseries_of(task)->push_back(reverse);\n\t\tprintf(\"Sort reversely:\\n\");\n\t}\n\telse\n\t\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tsize_t count;\n\tint *array;\n\tint *end;\n\tsize_t i;\n\n\tif (argc != 2 && argc != 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s [p]\\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tcount = atoi(argv[1]);\n\tarray = (int *)malloc(count * sizeof (int));\n\tif (!array)\n\t{\n\t\tperror(\"malloc\");\n\t\texit(1);\n\t}\n\n\tif (argc == 3 && (*argv[2] == 'p' || *argv[2] == 'P'))\n\t\tuse_parallel_sort = true;\n\n\tfor (i = 0; i < count; i++)\n\t\tarray[i] = rand() % 65536;\n\tend = &array[count];\n\n\tWFSortTask *task;\n\tif (use_parallel_sort)\n\t\ttask = WFAlgoTaskFactory::create_psort_task(\"sort\", array, end,\n\t\t\t\t\t\t\t\t\t\t\t\t\tcallback);\n\telse\n\t\ttask = WFAlgoTaskFactory::create_sort_task(\"sort\", array, end,\n\t\t\t\t\t\t\t\t\t\t\t\t\tcallback);\n\n\tif (use_parallel_sort)\n\t\tprintf(\"Start sorting parallelly...\\n\");\n\telse\n\t\tprintf(\"Start sorting...\\n\");\n\n\tprintf(\"Sort result:\\n\");\n\ttask->start();\n\n\twait_group.wait();\n\tfree(array);\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-08-matrix_multiply.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n\nnamespace algorithm\n{\n\ntypedef std::vector> Matrix;\n\nstruct MMInput\n{\n\tMatrix a;\n\tMatrix b;\n};\n\nstruct MMOutput\n{\n\tint error;\n\tsize_t m, n, k;\n\tMatrix c;\n};\n\nbool is_valid_matrix(const Matrix& matrix, size_t& m, size_t& n)\n{\n\tm = n = 0;\n\tif (matrix.size() == 0)\n\t\treturn true;\n\n\tm = matrix.size();\n\tn = matrix[0].size();\n\tif (n == 0)\n\t\treturn false;\n\n\tfor (const auto& row : matrix)\n\t\tif (row.size() != n)\n\t\t\treturn false;\n\n\treturn true;\n}\n\nvoid matrix_multiply(const MMInput *in, MMOutput *out)\n{\n\tsize_t m1, n1;\n\tsize_t m2, n2;\n\n\tif (!is_valid_matrix(in->a, m1, n1) || !is_valid_matrix(in->b, m2, n2))\n\t{\n\t\tout->error = EINVAL;\n\t\treturn;\n\t}\n\n\tif (n1 != m2)\n\t{\n\t\tout->error = EINVAL;\n\t\treturn;\n\t}\n\n\tout->error = 0;\n\tout->m = m1;\n\tout->n = n2;\n\tout->k = n1;\n\n\tout->c.resize(m1);\n\tfor (size_t i = 0; i < out->m; i++)\n\t{\n\t\tout->c[i].resize(n2);\n\t\tfor (size_t j = 0; j < out->n; j++)\n\t\t{\n\t\t\tout->c[i][j] = 0;\n\t\t\tfor (size_t k = 0; k < out->k; k++)\n\t\t\t\tout->c[i][j] += in->a[i][k] * in->b[k][j];\n\t\t}\n\t}\n}\n\n}\n\nusing MMTask = WFThreadTask;\n\nusing namespace algorithm;\n\nvoid print_matrix(const Matrix& matrix, size_t m, size_t n)\n{\n\tfor (size_t i = 0; i < m; i++)\n\t{\n\t\tfor (size_t j = 0; j < n; j++)\n\t\t\tprintf(\"\\t%8.2lf\", matrix[i][j]);\n\n\t\tprintf(\"\\n\");\n\t}\n}\n\nvoid callback(MMTask *task)\n{\n\tauto *input = task->get_input();\n\tauto *output = task->get_output();\n\n\tassert(task->get_state() == WFT_STATE_SUCCESS);\n\n\tif (output->error)\n\t\tprintf(\"Error: %d %s\\n\", output->error, strerror(output->error));\n\telse\n\t{\n\t\tprintf(\"Matrix A\\n\");\n\t\tprint_matrix(input->a, output->m, output->k);\n\t\tprintf(\"Matrix B\\n\");\n\t\tprint_matrix(input->b, output->k, output->n);\n\t\tprintf(\"Matrix A * Matrix B =>\\n\");\n\t\tprint_matrix(output->c, output->m, output->n);\n\t}\n}\n\nint main()\n{\n\tusing MMFactory = WFThreadTaskFactory;\n\tMMTask *task = MMFactory::create_thread_task(\"matrix_multiply_task\",\n\t\t\t\t\t\t\t\t\t\t\t\tmatrix_multiply,\n\t\t\t\t\t\t\t\t\t\t\t\tcallback);\n\tauto *input = task->get_input();\n\n\tinput->a = {{1, 2, 3}, {4, 5, 6}};\n\tinput->b = {{7, 8}, {9, 10}, {11, 12}};\n\n\tWFFacilities::WaitGroup wait_group(1);\n\n\tWorkflow::start_series_work(task, [&wait_group](const SeriesWork *) {\n\t\twait_group.done();\n\t});\n\n\twait_group.wait();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-09-http_file_server.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/HttpMessage.h\"\n#include \"workflow/HttpUtil.h\"\n#include \"workflow/WFHttpServer.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/Workflow.h\"\n#include \"workflow/WFFacilities.h\"\n\nusing namespace protocol;\n\nvoid pread_callback(WFFileIOTask *task)\n{\n\tFileIOArgs *args = task->get_args();\n\tlong ret = task->get_retval();\n\tHttpResponse *resp = (HttpResponse *)task->user_data;\n\n\tclose(args->fd);\n\tif (task->get_state() != WFT_STATE_SUCCESS || ret < 0)\n\t{\n\t\tresp->set_status_code(\"503\");\n\t\tresp->append_output_body(\"503 Internal Server Error.\");\n\t}\n\telse /* Use '_nocopy' carefully. */\n\t\tresp->append_output_body_nocopy(args->buf, ret);\n}\n\nvoid process(WFHttpTask *server_task, const char *root)\n{\n\tHttpRequest *req = server_task->get_req();\n\tHttpResponse *resp = server_task->get_resp();\n\tconst char *uri = req->get_request_uri();\n\tconst char *p = uri;\n\n\tprintf(\"Request-URI: %s\\n\", uri);\n\twhile (*p && *p != '?')\n\t\tp++;\n\n\tstd::string abs_path(uri, p - uri);\n\tabs_path = root + abs_path;\n\tif (abs_path.back() == '/')\n\t\tabs_path += \"index.html\";\n\n\tresp->add_header_pair(\"Server\", \"Sogou C++ Workflow Server\");\n\n\tint fd = open(abs_path.c_str(), O_RDONLY);\n\tif (fd >= 0)\n\t{\n\t\tsize_t size = lseek(fd, 0, SEEK_END);\n\t\tvoid *buf = malloc(size); /* As an example, assert(buf != NULL); */\n\t\tWFFileIOTask *pread_task;\n\n\t\tpread_task = WFTaskFactory::create_pread_task(fd, buf, size, 0,\n\t\t\t\t\t\t\t\t\t\t\t\t\t pread_callback);\n\t\t/* To implement a more complicated server, please use series' context\n\t\t * instead of tasks' user_data to pass/store internal data. */\n\t\tpread_task->user_data = resp;\t/* pass resp pointer to pread task. */\n\t\tserver_task->user_data = buf;\t/* to free() in callback() */\n\t\tserver_task->set_callback([](WFHttpTask *t){ free(t->user_data); });\n\t\tseries_of(server_task)->push_back(pread_task);\n\t}\n\telse\n\t{\n\t\tresp->set_status_code(\"404\");\n\t\tresp->append_output_body(\"404 Not Found.\");\n\t}\n}\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tif (argc != 2 && argc != 3 && argc != 5)\n\t{\n\t\tfprintf(stderr, \"%s [root path] [cert file] [key file]\\n\",\n\t\t\t\targv[0]);\n\t\texit(1);\n\t}\n\n\tsignal(SIGINT, sig_handler);\n\n\tunsigned short port = atoi(argv[1]);\n\tconst char *root = (argc >= 3 ? argv[2] : \".\");\n\tauto&& proc = std::bind(process, std::placeholders::_1, root);\n\tWFHttpServer server(proc);\n\tstd::string scheme;\n\tint ret;\n\n\tif (argc == 5)\n\t{\n\t\tret = server.start(port, argv[3], argv[4]);\t/* https server */\n\t\tscheme = \"https://\";\n\t}\n\telse\n\t{\n\t\tret = server.start(port);\n\t\tscheme = \"http://\";\n\t}\n\n\tif (ret < 0)\n\t{\n\t\tperror(\"start server\");\n\t\texit(1);\n\t}\n\n\t/* Test the server. */\n\tauto&& create = [&scheme, port](WFRepeaterTask *)->SubTask *{\n\t\tchar buf[1024];\n\t\t*buf = '\\0';\n\t\tprintf(\"Input file name: (Ctrl-D to exit): \");\n\t\tscanf(\"%1023s\", buf);\n\t\tif (*buf == '\\0')\n\t\t{\n\t\t\tprintf(\"\\n\");\n\t\t\treturn NULL;\n\t\t}\n\n\t\tstd::string url = scheme + \"127.0.0.1:\" + std::to_string(port) + \"/\" + buf;\n\t\tWFHttpTask *task = WFTaskFactory::create_http_task(url, 0, 0,\n\t\t\t\t\t\t\t\t\t[](WFHttpTask *task) {\n\t\t\tauto *resp = task->get_resp();\n\t\t\tif (strcmp(resp->get_status_code(), \"200\") == 0)\n\t\t\t{\n\t\t\t\tstd::string body = protocol::HttpUtil::decode_chunked_body(resp);\n\t\t\t\tfwrite(body.c_str(), body.size(), 1, stdout);\n\t\t\t\tprintf(\"\\n\");\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tprintf(\"%s %s\\n\", resp->get_status_code(), resp->get_reason_phrase());\n\t\t\t}\n\t\t});\n\n\t\treturn task;\n\t};\n\n\tWFFacilities::WaitGroup wg(1);\n\tWFRepeaterTask *repeater;\n\trepeater = WFTaskFactory::create_repeater_task(create, [&wg](WFRepeaterTask *) {\n\t\twg.done();\n\t});\n\n\trepeater->start();\n\twg.wait();\n\n\tserver.stop();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-10-user_defined_protocol/client-uds.cc", "content": "#include \n#include \n#include \n#include \n#include \"workflow/Workflow.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"message.h\"\n\nusing WFTutorialTask = WFNetworkTask;\nusing tutorial_callback_t = std::function;\n\nusing namespace protocol;\n\nclass MyFactory : public WFTaskFactory\n{\npublic:\n\tstatic WFTutorialTask *create_tutorial_task(const struct sockaddr *addr,\n\t\t\t\t\t\t\t\t\t\t\t\tsocklen_t addrlen,\n\t\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\ttutorial_callback_t callback)\n\t{\n\t\tusing NTF = WFNetworkTaskFactory;\n\t\tWFTutorialTask *task = NTF::create_client_task(TT_TCP, addr, addrlen,\n\t\t\t\t\t\t\t\t\t\t\t\t\t retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::move(callback));\n\t\ttask->set_keep_alive(30 * 1000);\n\t\treturn task;\n\t}\n};\n\nint main(int argc, char *argv[])\n{\n\tconst char *path;\n\tstd::string host;\n\n\tif (argc != 2)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tpath = argv[1];\n\n\tauto&& create = [path](WFRepeaterTask *)->SubTask *{\n\t\tchar buf[1024];\n\t\tprintf(\"Input next request string (Ctrl-D to exit): \");\n\t\t*buf = '\\0';\n\t\tscanf(\"%1023s\", buf);\n\t\tsize_t body_size = strlen(buf);\n\t\tif (body_size == 0)\n\t\t{\n\t\t\tprintf(\"\\n\");\n\t\t\treturn NULL;\n\t\t}\n\n\t\tstruct sockaddr_un sun = { };\n\t\tsun.sun_family = AF_UNIX;\n\t\tstrncpy(sun.sun_path, path, sizeof sun.sun_path - 1);\n\t\tWFTutorialTask *task = MyFactory::create_tutorial_task(\n\t\t\t\t\t\t\t\t\t\t\t(struct sockaddr *)&sun, sizeof sun,\n\t\t\t\t\t\t\t\t\t\t\t0,\n\t\t\t\t\t\t\t\t\t\t\t[](WFTutorialTask *task) {\n\t\t\tint state = task->get_state();\n\t\t\tint error = task->get_error();\n\t\t\tTutorialResponse *resp = task->get_resp();\n\t\t\tvoid *body;\n\t\t\tsize_t body_size;\n\n\t\t\tif (state == WFT_STATE_SUCCESS)\n\t\t\t{\n\t\t\t\tresp->get_message_body_nocopy(&body, &body_size);\n\t\t\t\tprintf(\"Server Response: %.*s\\n\", (int)body_size, (char *)body);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tconst char *str = WFGlobal::get_error_string(state, error);\n\t\t\t\tfprintf(stderr, \"Error: %s\\n\", str);\n\t\t\t}\n\t\t});\n\n\t\ttask->get_req()->set_message_body(buf, body_size);\n\t\ttask->get_resp()->set_size_limit(4 * 1024);\n\t\treturn task;\n\t};\n\n\tWFFacilities::WaitGroup wait_group(1);\n\n\tWFRepeaterTask *repeater;\n\trepeater = WFTaskFactory::create_repeater_task(std::move(create), nullptr);\n\tWorkflow::start_series_work(repeater, [&wait_group](const SeriesWork *) {\n\t\twait_group.done();\n\t});\n\n\twait_group.wait();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-10-user_defined_protocol/client.cc", "content": "#include \n#include \n#include \"workflow/Workflow.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"message.h\"\n\nusing WFTutorialTask = WFNetworkTask;\nusing tutorial_callback_t = std::function;\n\nusing namespace protocol;\n\nclass MyFactory : public WFTaskFactory\n{\npublic:\n\tstatic WFTutorialTask *create_tutorial_task(const std::string& host,\n\t\t\t\t\t\t\t\t\t\t\t\tunsigned short port,\n\t\t\t\t\t\t\t\t\t\t\t\tint retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\ttutorial_callback_t callback)\n\t{\n\t\tusing NTF = WFNetworkTaskFactory;\n\t\tWFTutorialTask *task = NTF::create_client_task(TT_TCP, host, port,\n\t\t\t\t\t\t\t\t\t\t\t\t\t retry_max,\n\t\t\t\t\t\t\t\t\t\t\t\t\t std::move(callback));\n\t\ttask->set_keep_alive(30 * 1000);\n\t\treturn task;\n\t}\n};\n\nint main(int argc, char *argv[])\n{\n\tunsigned short port;\n\tstd::string host;\n\n\tif (argc != 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\thost = argv[1];\n\tport = atoi(argv[2]);\n\n\tauto&& create = [host, port](WFRepeaterTask *)->SubTask *{\n\t\tchar buf[1024];\n\t\tprintf(\"Input next request string (Ctrl-D to exit): \");\n\t\t*buf = '\\0';\n\t\tscanf(\"%1023s\", buf);\n\t\tsize_t body_size = strlen(buf);\n\t\tif (body_size == 0)\n\t\t{\n\t\t\tprintf(\"\\n\");\n\t\t\treturn NULL;\n\t\t}\n\n\t\tWFTutorialTask *task = MyFactory::create_tutorial_task(host, port, 0,\n\t\t\t\t\t\t\t\t\t\t\t[](WFTutorialTask *task) {\n\t\t\tint state = task->get_state();\n\t\t\tint error = task->get_error();\n\t\t\tTutorialResponse *resp = task->get_resp();\n\t\t\tvoid *body;\n\t\t\tsize_t body_size;\n\n\t\t\tif (state == WFT_STATE_SUCCESS)\n\t\t\t{\n\t\t\t\tresp->get_message_body_nocopy(&body, &body_size);\n\t\t\t\tprintf(\"Server Response: %.*s\\n\", (int)body_size, (char *)body);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tconst char *str = WFGlobal::get_error_string(state, error);\n\t\t\t\tfprintf(stderr, \"Error: %s\\n\", str);\n\t\t\t}\n\t\t});\n\n\t\ttask->get_req()->set_message_body(buf, body_size);\n\t\ttask->get_resp()->set_size_limit(4 * 1024);\n\t\treturn task;\n\t};\n\n\tWFFacilities::WaitGroup wait_group(1);\n\n\tWFRepeaterTask *repeater;\n\trepeater = WFTaskFactory::create_repeater_task(std::move(create), nullptr);\n\tWorkflow::start_series_work(repeater, [&wait_group](const SeriesWork *) {\n\t\twait_group.done();\n\t});\n\n\twait_group.wait();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-10-user_defined_protocol/message.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \"message.h\"\n\nnamespace protocol\n{\n\nint TutorialMessage::encode(struct iovec vectors[], int max/*max==8192*/)\n{\n\tuint32_t n = htonl(this->body_size);\n\n\tmemcpy(this->head, &n, 4);\n\tvectors[0].iov_base = this->head;\n\tvectors[0].iov_len = 4;\n\tvectors[1].iov_base = this->body;\n\tvectors[1].iov_len = this->body_size;\n\n\treturn 2;\t/* return the number of vectors used, no more then max. */\n}\n\nint TutorialMessage::append(const void *buf, size_t size)\n{\n\tif (this->head_received < 4)\n\t{\n\t\tsize_t head_left;\n\t\tvoid *p;\n\n\t\tp = &this->head[head_received];\n\t\thead_left = 4 - this->head_received;\n\t\tif (size < 4 - this->head_received)\n\t\t{\n\t\t\tmemcpy(p, buf, size);\n\t\t\tthis->head_received += size;\n\t\t\treturn 0;\n\t\t}\n\t\tthis->head_received += head_left;\n\n\t\tmemcpy(p, buf, head_left);\n\t\tsize -= head_left;\n\t\tbuf = (const char *)buf + head_left;\n\n\t\tp = this->head;\n\t\tthis->body_size = ntohl(*(uint32_t *)p);\n\t\tif (this->body_size > this->size_limit)\n\t\t{\n\t\t\terrno = EMSGSIZE;\n\t\t\treturn -1;\n\t\t}\n\n\t\tthis->body = (char *)malloc(this->body_size);\n\t\tif (!this->body)\n\t\t\treturn -1;\n\n\t\tthis->body_received = 0;\n\t}\n\n\tsize_t body_left = this->body_size - this->body_received;\n\n\tif (size > body_left)\n\t{\n\t\terrno = EBADMSG;\n\t\treturn -1;\n\t}\n\n\tmemcpy(this->body + this->body_received, buf, size);\n\tthis->body_received += size;\n\tif (size < body_left)\n\t\treturn 0;\n\n\treturn 1;\n}\n\nint TutorialMessage::set_message_body(const void *body, size_t size)\n{\n\tvoid *p = malloc(size);\n\n\tif (!p)\n\t\treturn -1;\n\n\tmemcpy(p, body, size);\n\tfree(this->body);\n\tthis->body = (char *)p;\n\tthis->body_size = size;\n\n\tthis->head_received = 4;\n\tthis->body_received = size;\n\treturn 0;\n}\n\nTutorialMessage::TutorialMessage(TutorialMessage&& msg) :\n\tProtocolMessage(std::move(msg))\n{\n\tmemcpy(this->head, msg.head, 4);\n\tthis->head_received = msg.head_received;\n\tthis->body = msg.body;\n\tthis->body_received = msg.body_received;\n\tthis->body_size = msg.body_size;\n\n\tmsg.head_received = 0;\n\tmsg.body = NULL;\n\tmsg.body_size = 0;\n}\n\nTutorialMessage& TutorialMessage::operator = (TutorialMessage&& msg)\n{\n\tif (&msg != this)\n\t{\n\t\t*(ProtocolMessage *)this = std::move(msg);\n\n\t\tmemcpy(this->head, msg.head, 4);\n\t\tthis->head_received = msg.head_received;\n\t\tthis->body = msg.body;\n\t\tthis->body_received = msg.body_received;\n\t\tthis->body_size = msg.body_size;\n\n\t\tmsg.head_received = 0;\n\t\tmsg.body = NULL;\n\t\tmsg.body_size = 0;\n\t}\n\n\treturn *this;\n}\n\n}\n\n" }, { "path": "tutorial/tutorial-10-user_defined_protocol/message.h", "content": "#ifndef _TUTORIALMESSAGE_H_\n#define _TUTORIALMESSAGE_H_\n\n#include \n#include \"workflow/ProtocolMessage.h\"\n\nnamespace protocol\n{\n\nclass TutorialMessage : public ProtocolMessage\n{\nprivate:\n\tvirtual int encode(struct iovec vectors[], int max);\n\tvirtual int append(const void *buf, size_t size);\n\npublic:\n\tint set_message_body(const void *body, size_t size);\n\n\tvoid get_message_body_nocopy(void **body, size_t *size)\n\t{\n\t\t*body = this->body;\n\t\t*size = this->body_size;\n\t}\n\nprotected:\n\tchar head[4];\n\tsize_t head_received;\n\tchar *body;\n\tsize_t body_received;\n\tsize_t body_size;\n\npublic:\n\tTutorialMessage()\n\t{\n\t\tthis->head_received = 0;\n\t\tthis->body = NULL;\n\t\tthis->body_size = 0;\n\t}\n\n\tTutorialMessage(TutorialMessage&& msg);\n\tTutorialMessage& operator = (TutorialMessage&& msg);\n\n\tvirtual ~TutorialMessage()\n\t{\n\t\tfree(this->body);\n\t}\n};\n\nusing TutorialRequest = TutorialMessage;\nusing TutorialResponse = TutorialMessage;\n\n}\n\n#endif\n\n" }, { "path": "tutorial/tutorial-10-user_defined_protocol/server-uds.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/Workflow.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFServer.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"message.h\"\n\nusing WFTutorialTask = WFNetworkTask;\nusing WFTutorialServer = WFServer;\n\nusing namespace protocol;\n\nvoid process(WFTutorialTask *task)\n{\n\tTutorialRequest *req = task->get_req();\n\tTutorialResponse *resp = task->get_resp();\n\tvoid *body;\n\tsize_t size;\n\tsize_t i;\n\n\treq->get_message_body_nocopy(&body, &size);\n\tfor (i = 0; i < size; i++)\n\t\t((char *)body)[i] = toupper(((char *)body)[i]);\n\n\tresp->set_message_body(body, size);\n}\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tstruct sockaddr_un sun = { };\n\n\tif (argc != 2)\n\t{\n\t\tfprintf(stderr, \"USAGE %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tsun.sun_family = AF_UNIX;\n\tstrncpy(sun.sun_path, argv[1], sizeof sun.sun_path - 1);\n\n\tsignal(SIGINT, sig_handler);\n\n\tstruct WFServerParams params = SERVER_PARAMS_DEFAULT;\n\tparams.request_size_limit = 4 * 1024;\n\n\tWFTutorialServer server(¶ms, process);\n\tif (server.start((struct sockaddr *)&sun, sizeof sun) == 0)\n\t{\n\t\twait_group.wait();\n\t\tserver.stop();\n\t}\n\telse\n\t{\n\t\tperror(\"server.start\");\n\t\texit(1);\n\t}\n\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-10-user_defined_protocol/server.cc", "content": "#include \n#include \n#include \n#include \n#include \"workflow/Workflow.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFServer.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"message.h\"\n\nusing WFTutorialTask = WFNetworkTask;\nusing WFTutorialServer = WFServer;\n\nusing namespace protocol;\n\nvoid process(WFTutorialTask *task)\n{\n\tTutorialRequest *req = task->get_req();\n\tTutorialResponse *resp = task->get_resp();\n\tvoid *body;\n\tsize_t size;\n\tsize_t i;\n\n\treq->get_message_body_nocopy(&body, &size);\n\tfor (i = 0; i < size; i++)\n\t\t((char *)body)[i] = toupper(((char *)body)[i]);\n\n\tresp->set_message_body(body, size);\n}\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tunsigned short port;\n\n\tif (argc != 2)\n\t{\n\t\tfprintf(stderr, \"USAGE %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tport = atoi(argv[1]);\n\tsignal(SIGINT, sig_handler);\n\n\tstruct WFServerParams params = SERVER_PARAMS_DEFAULT;\n\tparams.request_size_limit = 4 * 1024;\n\n\tWFTutorialServer server(¶ms, process);\n\tif (server.start(AF_INET6, port) == 0 ||\n\t\tserver.start(AF_INET, port) == 0)\n\t{\n\t\twait_group.wait();\n\t\tserver.stop();\n\t}\n\telse\n\t{\n\t\tperror(\"server.start\");\n\t\texit(1);\n\t}\n\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-10-user_defined_protocol/xmake.lua", "content": "add_deps(\"workflow\")\n\ntarget(\"user_defined_message\")\n set_kind(\"object\")\n add_files(\"message.cc\")\n\ntarget(\"user_defined_server\")\n set_kind(\"binary\")\n add_files(\"server.cc\")\n add_deps(\"user_defined_message\")\n\ntarget(\"server-uds\")\n set_kind(\"binary\")\n add_files(\"server-uds.cc\")\n add_deps(\"user_defined_message\")\n\ntarget(\"user_defined_client\")\n set_kind(\"binary\")\n add_files(\"client.cc\")\n add_deps(\"user_defined_message\")\n\ntarget(\"client-uds\")\n set_kind(\"binary\")\n add_files(\"client-uds.cc\")\n add_deps(\"user_defined_message\")\n" }, { "path": "tutorial/tutorial-11-graph_task.cc", "content": "#include \n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFGraphTask.h\"\n#include \"workflow/HttpMessage.h\"\n#include \"workflow/WFFacilities.h\"\n\nusing namespace protocol;\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid go_func(const size_t *size1, const size_t *size2)\n{\n\tprintf(\"page1 size = %zu, page2 size = %zu\\n\", *size1, *size2);\n}\n\nvoid http_callback(WFHttpTask *task)\n{\n\tsize_t *size = (size_t *)task->user_data;\n\tconst void *body;\n\n\tif (task->get_state() == WFT_STATE_SUCCESS)\n\t\ttask->get_resp()->get_parsed_body(&body, size);\n\telse\n\t\t*size = (size_t)-1;\n}\n\n#define REDIRECT_MAX\t3\n#define RETRY_MAX\t\t1\n\nint main()\n{\n\tWFTimerTask *timer;\n\tWFHttpTask *http_task1;\n\tWFHttpTask *http_task2;\n\tWFGoTask *go_task;\n\tsize_t size1;\n\tsize_t size2;\n\n\ttimer = WFTaskFactory::create_timer_task(1000000, [](WFTimerTask *) {\n\t\tprintf(\"timer task complete(1s).\\n\");\n\t});\n\n\t/* Http task1 */\n\thttp_task1 = WFTaskFactory::create_http_task(\"https://www.sogou.com/\",\n\t\t\t\t\t\t\t\t\t\t\t\t REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\t http_callback);\n\thttp_task1->user_data = &size1;\n\n\t/* Http task2 */\n\thttp_task2 = WFTaskFactory::create_http_task(\"https://www.baidu.com/\",\n\t\t\t\t\t\t\t\t\t\t\t\t REDIRECT_MAX, RETRY_MAX,\n\t\t\t\t\t\t\t\t\t\t\t\t http_callback);\n\thttp_task2->user_data = &size2;\n\n\t/* go task will print the http pages size */\n\tgo_task = WFTaskFactory::create_go_task(\"go\", go_func, &size1, &size2);\n\n\t/* Create a graph. Graph is also a kind of task */\n\tWFGraphTask *graph = WFTaskFactory::create_graph_task([](WFGraphTask *) {\n\t\tprintf(\"Graph task complete. Wakeup main process\\n\");\n\t\twait_group.done();\n\t});\n\n\t/* Create graph nodes */\n\tWFGraphNode& a = graph->create_graph_node(timer);\n\tWFGraphNode& b = graph->create_graph_node(http_task1);\n\tWFGraphNode& c = graph->create_graph_node(http_task2);\n\tWFGraphNode& d = graph->create_graph_node(go_task);\n\n\t/* Build the graph */\n\ta-->b;\n\ta-->c;\n\tb-->d;\n\tc-->d;\n\n\tgraph->start();\n\twait_group.wait();\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-12-mysql_cli.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/Workflow.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/MySQLResult.h\"\n#include \"workflow/WFFacilities.h\"\n\nusing namespace protocol;\n\n#define RETRY_MAX 0\n\nvolatile bool stop_flag;\n\nvoid mysql_callback(WFMySQLTask *task);\n\nvoid get_next_cmd(WFMySQLTask *task)\n{\n\tint len;\n\tchar query[4096];\n\tWFMySQLTask *next_task;\n\n\tfprintf(stderr, \"mysql> \");\n\twhile ((fgets(query, 4096, stdin)) && stop_flag == false)\n\t{\n\t\tlen = strlen(query);\n\t\tif (len > 0 && query[len - 1] == '\\n')\n\t\t\tquery[len - 1] = '\\0';\n\n\t\tif (strncmp(query, \"quit\", len) == 0 || \n\t\t\tstrncmp(query, \"exit\", len) == 0)\n\t\t{\n\t\t\tfprintf(stderr, \"Bye\\n\");\n\t\t\treturn;\n\t\t}\n\n\t\tif (len == 0 || strncmp(query, \"\\0\", len) == 0)\n\t\t{\n\t\t\tfprintf(stderr, \"mysql> \");\n\t\t\tcontinue;\n\t\t}\n\n\t\tstd::string *url = (std::string *)series_of(task)->get_context();\n\t\tnext_task = WFTaskFactory::create_mysql_task(*url, RETRY_MAX, mysql_callback);\n\t\tnext_task->get_req()->set_query(query);\n\t\tseries_of(task)->push_back(next_task);\t\n\t\tbreak;\n\t}\n\treturn;\n}\n\nvoid mysql_callback(WFMySQLTask *task)\n{\n\tMySQLResponse *resp = task->get_resp();\n\n\tMySQLResultCursor cursor(resp);\n\tconst MySQLField *const *fields;\n\tstd::vector arr;\n\n\tif (task->get_state() != WFT_STATE_SUCCESS)\n\t{\n\t\tfprintf(stderr, \"error msg: %s\\n\",\n\t\t\t\tWFGlobal::get_error_string(task->get_state(),\n\t\t\t\t\t\t\t\t\t\t task->get_error()));\n\t\treturn;\n\t}\n\n\tdo {\n\t\tif (cursor.get_cursor_status() != MYSQL_STATUS_GET_RESULT &&\n\t\t\tcursor.get_cursor_status() != MYSQL_STATUS_OK)\n\t\t{\n\t\t\tbreak;\n\t\t}\n\n\t\tfprintf(stderr, \"---------------- RESULT SET ----------------\\n\");\n\n\t\tif (cursor.get_cursor_status() == MYSQL_STATUS_GET_RESULT)\n\t\t{\n\t\t\tfprintf(stderr, \"cursor_status=%d field_count=%u rows_count=%u\\n\",\n\t\t\t\t\tcursor.get_cursor_status(), cursor.get_field_count(),\n\t\t\t\t\tcursor.get_rows_count());\n\n\t\t\t//nocopy api\n\t\t\tfields = cursor.fetch_fields();\n\t\t\tfor (int i = 0; i < cursor.get_field_count(); i++)\n\t\t\t{\n\t\t\t\tif (i == 0)\n\t\t\t\t{\n\t\t\t\t\tfprintf(stderr, \"db=%s table=%s\\n\",\n\t\t\t\t\t\tfields[i]->get_db().c_str(), fields[i]->get_table().c_str());\n\t\t\t\t\tfprintf(stderr, \" ---------- COLUMNS ----------\\n\");\n\t\t\t\t}\n\t\t\t\tfprintf(stderr, \" name[%s] type[%s]\\n\",\n\t\t\t\t\t\tfields[i]->get_name().c_str(),\n\t\t\t\t\t\tdatatype2str(fields[i]->get_data_type()));\n\t\t\t}\n\t\t\tfprintf(stderr, \" _________ COLUMNS END _________\\n\\n\");\n\n\t\t\twhile (cursor.fetch_row(arr))\n\t\t\t{\n\t\t\t\tfprintf(stderr, \" ------------ ROW ------------\\n\");\n\t\t\t\tfor (size_t i = 0; i < arr.size(); i++)\n\t\t\t\t{\n\t\t\t\t\tfprintf(stderr, \" [%s][%s]\", fields[i]->get_name().c_str(),\n\t\t\t\t\t\t\tdatatype2str(arr[i].get_data_type()));\n\t\t\t\t\tif (arr[i].is_string())\n\t\t\t\t\t{\n\t\t\t\t\t\tstd::string res = arr[i].as_string();\n\t\t\t\t\t\tif (res.length() == 0)\n\t\t\t\t\t\t\tfprintf(stderr, \"[\\\"\\\"]\\n\");\n\t\t\t\t\t\telse \n\t\t\t\t\t\t\tfprintf(stderr, \"[%s]\\n\", res.c_str());\n\t\t\t\t\t} else if (arr[i].is_int()) {\n\t\t\t\t\t\tfprintf(stderr, \"[%d]\\n\", arr[i].as_int());\n\t\t\t\t\t} else if (arr[i].is_ulonglong()) {\n\t\t\t\t\t\tfprintf(stderr, \"[%llu]\\n\", arr[i].as_ulonglong());\n\t\t\t\t\t} else if (arr[i].is_float()) {\n\t\t\t\t\t\tconst void *ptr;\n\t\t\t\t\t\tsize_t len;\n\t\t\t\t\t\tint data_type;\n\t\t\t\t\t\tarr[i].get_cell_nocopy(&ptr, &len, &data_type);\n\t\t\t\t\t\tsize_t pos;\n\t\t\t\t\t\tfor (pos = 0; pos < len; pos++)\n\t\t\t\t\t\t\tif (*((const char *)ptr + pos) == '.')\n\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\tif (pos != len)\n\t\t\t\t\t\t\tpos = len - pos - 1;\n\t\t\t\t\t\telse\n\t\t\t\t\t\t\tpos = 0;\n\t\t\t\t\t\tfprintf(stderr, \"[%.*f]\\n\", (int)pos, arr[i].as_float());\n\t\t\t\t\t} else if (arr[i].is_double()) {\n\t\t\t\t\t\tconst void *ptr;\n\t\t\t\t\t\tsize_t len;\n\t\t\t\t\t\tint data_type;\n\t\t\t\t\t\tarr[i].get_cell_nocopy(&ptr, &len, &data_type);\n\t\t\t\t\t\tsize_t pos;\n\t\t\t\t\t\tfor (pos = 0; pos < len; pos++)\n\t\t\t\t\t\t\tif (*((const char *)ptr + pos) == '.')\n\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\tif (pos != len)\n\t\t\t\t\t\t\tpos = len - pos - 1;\n\t\t\t\t\t\telse\n\t\t\t\t\t\t\tpos= 0;\n\t\t\t\t\t\tfprintf(stderr, \"[%.*lf]\\n\", (int)pos, arr[i].as_double());\n\t\t\t\t\t} else if (arr[i].is_date()) {\n\t\t\t\t\t\tfprintf(stderr, \"[%s]\\n\", arr[i].as_string().c_str());\n\t\t\t\t\t} else if (arr[i].is_time()) {\n\t\t\t\t\t\tfprintf(stderr, \"[%s]\\n\", arr[i].as_string().c_str());\n\t\t\t\t\t} else if (arr[i].is_datetime()) {\n\t\t\t\t\t\tfprintf(stderr, \"[%s]\\n\", arr[i].as_string().c_str());\n\t\t\t\t\t} else if (arr[i].is_null()) {\n\t\t\t\t\t\tfprintf(stderr, \"[NULL]\\n\");\n\t\t\t\t\t} else {\n\t\t\t\t\t\tstd::string res = arr[i].as_binary_string();\n\t\t\t\t\t\tif (res.length() == 0)\n\t\t\t\t\t\t\tfprintf(stderr, \"[\\\"\\\"]\\n\");\n\t\t\t\t\t\telse \n\t\t\t\t\t\t\tfprintf(stderr, \"[%s]\\n\", res.c_str());\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfprintf(stderr, \" __________ ROW END __________\\n\");\n\t\t\t}\n\t\t}\n\t\telse if (cursor.get_cursor_status() == MYSQL_STATUS_OK)\n\t\t{\n\t\t\tfprintf(stderr, \" OK. %llu \", cursor.get_affected_rows());\n\t\t\tif (cursor.get_affected_rows() == 1)\n\t\t\t\tfprintf(stderr, \"row \");\n\t\t\telse\n\t\t\t\tfprintf(stderr, \"rows \");\n\t\t\tfprintf(stderr, \"affected. %d warnings. insert_id=%llu. %s\\n\",\n\t\t\t\t\tcursor.get_warnings(), cursor.get_insert_id(),\n\t\t\t\t\tcursor.get_info().c_str());\n\t\t}\n\n\t\tfprintf(stderr, \"________________ RESULT SET END ________________\\n\\n\");\n\t} while (cursor.next_result_set());\n\n\n\tif (resp->get_packet_type() == MYSQL_PACKET_ERROR)\n\t{\n\t\tfprintf(stderr, \"ERROR. error_code=%d %s\\n\",\n\t\t\t\ttask->get_resp()->get_error_code(),\n\t\t\t\ttask->get_resp()->get_error_msg().c_str());\n\t}\n\telse if (resp->get_packet_type() == MYSQL_PACKET_OK) // just check origin APIs\n\t{\n\t\tfprintf(stderr, \"OK. %llu \", task->get_resp()->get_affected_rows());\n\t\tif (task->get_resp()->get_affected_rows() == 1)\n\t\t\tfprintf(stderr, \"row \");\n\t\telse\n\t\t\tfprintf(stderr, \"rows \");\n\t\tfprintf(stderr, \"affected. %d warnings. insert_id=%llu. %s\\n\",\n\t\t\t\ttask->get_resp()->get_warnings(),\n\t\t\t\ttask->get_resp()->get_last_insert_id(),\n\t\t\t\ttask->get_resp()->get_info().c_str());\n\t}\n\n\tget_next_cmd(task);\n\treturn;\n}\n\nstatic void sighandler(int signo)\n{\n\tstop_flag = true;\n}\n\nint main(int argc, char *argv[])\n{\n\tWFMySQLTask *task;\n\n\tif (argc != 2)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\"\n\t\t\t\t\" url format: mysql://root:password@host:port/dbname?character_set=charset\\n\"\n\t\t\t\t\" example: mysql://root@test.mysql.com/test\\n\",\n\t\t\t\targv[0]);\n\t\treturn 0;\n\t}\n\n\tsignal(SIGINT, sighandler);\n\tsignal(SIGTERM, sighandler);\n\n\tstd::string url = argv[1];\n\tif (strncasecmp(argv[1], \"mysql://\", 8) != 0 &&\n\t\tstrncasecmp(argv[1], \"mysqls://\", 9) != 0)\n\t{\n\t\turl = \"mysql://\" + url;\n\t}\n\n\tconst char *query = \"show databases\";\n\tstop_flag = false;\n\n\ttask = WFTaskFactory::create_mysql_task(url, RETRY_MAX, mysql_callback);\n\ttask->get_req()->set_query(query);\n\n\tWFFacilities::WaitGroup wait_group(1);\n\tSeriesWork *series = Workflow::create_series_work(task,\n\t\t[&wait_group](const SeriesWork *series) {\n\t\t\twait_group.done();\n\t\t});\n\n\tseries->set_context(&url);\n\tseries->start();\n\n\twait_group.wait();\n\treturn 0;\n}\n" }, { "path": "tutorial/tutorial-13-kafka_cli.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/WFKafkaClient.h\"\n#include \"workflow/KafkaMessage.h\"\n#include \"workflow/KafkaResult.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"workflow/WFGlobal.h\"\n\nusing namespace protocol;\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nstd::string url;\nbool no_cgroup = false;\nWFKafkaClient client;\n\nvoid kafka_callback(WFKafkaTask *task)\n{\n\tint state = task->get_state();\n\tint error = task->get_error();\n\n\tif (state != WFT_STATE_SUCCESS)\n\t{\n\t\tfprintf(stderr, \"error msg: %s\\n\",\n\t\t\t\tWFGlobal::get_error_string(state, error));\n\t\tfprintf(stderr, \"Failed. Press Ctrl-C to exit.\\n\");\n\t\tclient.deinit();\n\t\twait_group.done();\n\t\treturn;\n\t}\n\n\tWFKafkaTask *next_task = NULL;\n\tstd::vector> records;\n\tstd::vector toppars;\n\tint api_type = task->get_api_type();\n\n\tprotocol::KafkaResult new_result;\n\n\tswitch (api_type)\n\t{\n\tcase Kafka_Produce:\n\t\ttask->get_result()->fetch_records(records);\n\n\t\tfor (const auto &v : records)\n\t\t{\n\t\t\tfor (const auto &w: v)\n\t\t\t{\n\t\t\t\tconst void *value;\n\t\t\t\tsize_t value_len;\n\t\t\t\tw->get_value(&value, &value_len);\n\t\t\t\tprintf(\"produce\\ttopic: %s, partition: %d, status: %d, \\\n\t\t\t\t\t\toffset: %lld, val_len: %zu\\n\",\n\t\t\t\t\t w->get_topic(), w->get_partition(), w->get_status(),\n\t\t\t\t\t w->get_offset(), value_len);\n\t\t\t}\n\t\t}\n\n\t\tbreak;\n\n\tcase Kafka_Fetch:\n\t\tnew_result = std::move(*task->get_result());\n\t\tnew_result.fetch_records(records);\n\n\t\tif (!records.empty())\n\t\t{\n\t\t\tif (!no_cgroup)\n\t\t\t\tnext_task = client.create_kafka_task(\"api=commit\", 3, kafka_callback);\n\n\t\t\tstd::string out;\n\n\t\t\tfor (const auto &v : records)\n\t\t\t{\n\t\t\t\tif (v.empty())\n\t\t\t\t\tcontinue;\n\n\t\t\t\tchar fn[1024];\n\t\t\t\tsnprintf(fn, 1024, \"/tmp/kafka.%s.%d.%llu\",\n\t\t\t\t\t\t v.back()->get_topic(), v.back()->get_partition(),\n\t\t\t\t\t\t v.back()->get_offset());\n\n\t\t\t\tFILE *fp = fopen(fn, \"w+\");\n\t\t\t\tlong long offset = 0;\n\t\t\t\tint partition = 0;\n\t\t\t\tstd::string topic;\n\n\t\t\t\tfor (const auto &w : v)\n\t\t\t\t{\n\t\t\t\t\tconst void *value;\n\t\t\t\t\tsize_t value_len;\n\t\t\t\t\tw->get_value(&value, &value_len);\n\t\t\t\t\tif (fp)\n\t\t\t\t\t\tfwrite(value, value_len, 1, fp);\n\n\t\t\t\t\toffset = w->get_offset();\n\t\t\t\t\tpartition = w->get_partition();\n\t\t\t\t\ttopic = w->get_topic();\n\n\t\t\t\t\tif (!no_cgroup)\n\t\t\t\t\t\tnext_task->add_commit_record(*w);\n\t\t\t\t}\n\n\t\t\t\tif (!topic.empty())\n\t\t\t\t{\n\t\t\t\t\tout += \"topic: \"\t + topic;\n\t\t\t\t\tout += \",partition: \" + std::to_string(partition);\n\t\t\t\t\tout += \",offset: \"\t + std::to_string(offset) + \";\";\n\t\t\t\t}\n\n\t\t\t\tif (fp)\n\t\t\t\t\tfclose(fp);\n\t\t\t}\n\n\t\t\tprintf(\"fetch\\t%s\\n\", out.c_str());\n\n\t\t\tif (!no_cgroup)\n\t\t\t\tseries_of(task)->push_back(next_task);\n\t\t}\n\n\t\tbreak;\n\n\tcase Kafka_OffsetCommit:\n\t\ttask->get_result()->fetch_toppars(toppars);\n\n\t\tif (!toppars.empty())\n\t\t{\n\t\t\tfor (const auto& v : toppars)\n\t\t\t{\n\t\t\t\tprintf(\"commit\\ttopic: %s, partition: %d, \\\n\t\t\t\t\t\toffset: %llu, error: %d\\n\",\n\t\t\t\t\t v->get_topic(), v->get_partition(),\n\t\t\t\t\t v->get_offset(), v->get_error());\n\t\t\t}\n\t\t}\n\n\t\tnext_task = client.create_leavegroup_task(3, kafka_callback);\n\n\t\tseries_of(task)->push_back(next_task);\n\n\t\tbreak;\n\n\tcase Kafka_LeaveGroup:\n\t\tprintf(\"leavegroup callback\\n\");\n\t\tbreak;\n\n\tdefault:\n\t\tbreak;\n\t}\n\n\tif (!next_task)\n\t{\n\t\tclient.deinit();\n\t\twait_group.done();\n\t}\n}\n\nvoid sig_handler(int signo) { }\n\nint main(int argc, char *argv[])\n{\n\tif (argc < 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s url

[compress_type/d]\\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tsignal(SIGINT, sig_handler);\n\n\turl = argv[1];\n\tif (strncmp(argv[1], \"kafka://\", 8) != 0 &&\n\t\tstrncmp(argv[1], \"kafkas://\", 9) != 0)\n\t{\n\t\turl = \"kafka://\" + url;\n\t}\n\n\tchar buf[512 * 1024];\n\tWFKafkaTask *task;\n\n\tif (argv[2][0] == 'p')\n\t{\n\t\tint compress_type = Kafka_NoCompress;\n\n\t\tif (argc > 3)\n\t\t\tcompress_type = atoi(argv[3]);\n\n\t\tif (compress_type > Kafka_Zstd)\n\t\t\texit(1);\n\n\t\tif (client.init(url) < 0)\n\t\t{\n\t\t\tperror(\"client.init\");\n\t\t\texit(1);\n\t\t}\n\n\t\ttask = client.create_kafka_task(\"api=produce\", 3, kafka_callback);\n\t\tKafkaConfig config;\n\t\tKafkaRecord record;\n\n\t\tconfig.set_compress_type(compress_type);\n\t\tconfig.set_client_id(\"workflow\");\n\t\ttask->set_config(std::move(config));\n\n\t\tfor (size_t i = 0; i < sizeof (buf); ++i)\n\t\t\tbuf[i] = '1' + rand() % 128;\n\n\t\trecord.set_key(\"key1\", strlen(\"key1\"));\n\t\trecord.set_value(buf, sizeof (buf));\n\t\trecord.add_header_pair(\"hk1\", 3, \"hv1\", 3);\n\t\ttask->add_produce_record(\"workflow_test1\", -1, std::move(record));\n\n\t\trecord.set_key(\"key2\", strlen(\"key2\"));\n\t\trecord.set_value(buf, sizeof (buf));\n\t\trecord.add_header_pair(\"hk2\", 3, \"hv2\", 3);\n\t\ttask->add_produce_record(\"workflow_test2\", -1, std::move(record));\n\t}\n\telse if (argv[2][0] == 'c')\n\t{\n\t\tif (argc > 3 && argv[3][0] == 'd')\n\t\t{\n\t\t\tif (client.init(url) < 0)\n\t\t\t{\n\t\t\t\tperror(\"client.init\");\n\t\t\t\texit(1);\n\t\t\t}\n\n\t\t\ttask = client.create_kafka_task(\"api=fetch\", 3, kafka_callback);\n\n\t\t\tKafkaToppar toppar;\n\t\t\ttoppar.set_topic_partition(\"workflow_test1\", 0);\n\t\t\ttoppar.set_offset(0);\n\t\t\ttask->add_toppar(toppar);\n\n\t\t\ttoppar.set_topic_partition(\"workflow_test2\", 0);\n\t\t\ttoppar.set_offset(1);\n\t\t\ttask->add_toppar(toppar);\n\n\t\t\tno_cgroup = true;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (client.init(url, \"workflow_group\") < 0)\n\t\t\t{\n\t\t\t\tperror(\"client.init\");\n\t\t\t\texit(1);\n\t\t\t}\n\n\t\t\ttask = client.create_kafka_task(\"topic=workflow_test1&topic=workflow_test2&api=fetch\",\n\t\t\t\t\t\t\t\t\t\t\t3, kafka_callback);\n\t\t}\n\n\t\tKafkaConfig config;\n\t\tconfig.set_client_id(\"workflow\");\n\t\ttask->set_config(std::move(config));\n\t}\n\telse\n\t{\n\t\tfprintf(stderr, \"USAGE: %s url

[compress_type/d]\\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\ttask->start();\n\n\twait_group.wait();\n\n\treturn 0;\n}\n" }, { "path": "tutorial/tutorial-14-consul_cli.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \"workflow/WFConsulClient.h\"\n#include \"workflow/ConsulDataTypes.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"workflow/HttpMessage.h\"\n#include \"workflow/WFGlobal.h\"\n\nusing namespace protocol;\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nstd::string url;\nWFConsulClient client;\n\nvoid print_discover_result(std::vector& discover_result)\n{\n\tfor (const auto& instance : discover_result)\n\t{\n\t\tfprintf(stderr, \"%s\", \"discover_instance\\n\");\n\t\t\n\t\tfprintf(stderr, \"node_id:%s\\n\", instance.node_id.c_str());\n\t\tfprintf(stderr, \"node_name:%s\\n\", instance.node_name.c_str());\n\t\tfprintf(stderr, \"node_address:%s\\n\", instance.node_address.c_str());\n\t\tfprintf(stderr, \"dc:%s\\n\", instance.dc.c_str());\n\t\tconst std::map& node_meta = instance.node_meta;\n\t\tfor (const auto& meta_kv : node_meta)\n\t\t{\n\t\t\tfprintf(stderr, \"node_meta:%s = %s\\n\", meta_kv.first.c_str(),\n\t\t\t\t\t\t\t\t\t\t\t\t meta_kv.second.c_str());\n\t\t}\n\t\tfprintf(stderr, \"create_index:%lld\\n\", instance.create_index);\n\t\tfprintf(stderr, \"modify_index:%lld\\n\", instance.modify_index);\n\n\t\tfprintf(stderr, \"service_id:%s\\n\", instance.service.service_id.c_str());\n\t\tfprintf(stderr, \"service_name:%s\\n\", instance.service.service_name.c_str());\n\t\tfprintf(stderr, \"service_namespace:%s\\n\", instance.service.service_namespace.c_str());\n\t\tfprintf(stderr, \"service_address:%s\\n\", instance.service.service_address.first.c_str());\n\t\tfprintf(stderr, \"service_port:%d\\n\", instance.service.service_address.second);\n\t\tfprintf(stderr, \"service_tag_override:%d\\n\", instance.service.tag_override);\n\t\tfprintf(stderr, \"%s\", \"service_tags:\");\n\t\tconst std::vector& tags = instance.service.tags;\n\t\tfor (const auto& tag : tags)\n\t\t{\n\t\t\tfprintf(stderr, \"%s,\", tag.c_str()); \n\t\t}\n\t\tfprintf(stderr, \"\\n\");\n\t\tconst std::map& service_meta = instance.service.meta;\n\t\tfor (const auto& meta_kv : service_meta)\n\t\t{\n\t\t\tfprintf(stderr, \"service_meta:%s = %s\\n\", meta_kv.first.c_str(),\n\t\t\t\t\t\t\t\t\t\t\t\t\t meta_kv.second.c_str());\n\t\t}\n\t\tfprintf(stderr, \"lan:%s:%d\\n\", instance.service.lan.first.c_str(),\n\t\t\t\t\t\t\t\t\t instance.service.lan.second);\n\t\tfprintf(stderr, \"lan_ipv4:%s:%d\\n\",\n\t\t\t\t\t\t\t\t\tinstance.service.lan_ipv4.first.c_str(),\n\t\t\t\t\t\t\t\t\tinstance.service.lan_ipv4.second);\n\t\tfprintf(stderr, \"lan_ipv6:%s:%d\\n\",\n\t\t\t\t\t\t\t\t\tinstance.service.lan_ipv6.first.c_str(),\n\t\t\t\t\t\t\t\t\tinstance.service.lan_ipv6.second);\n\t\tfprintf(stderr, \"wan:%s:%d\\n\", instance.service.wan.first.c_str(),\n\t\t\t\t\t\t\t\t\t instance.service.wan.second);\n\t\tfprintf(stderr, \"wan_ipv4:%s:%d\\n\",\n\t\t\t\t\t\t\t\t\tinstance.service.wan_ipv4.first.c_str(),\n\t\t\t\t\t\t\t\t\tinstance.service.wan_ipv4.second);\n\t\tfprintf(stderr, \"wan_ipv6:%s:%d\\n\",\n\t\t\t\t\t\t\t\t\tinstance.service.wan_ipv6.first.c_str(),\n\t\t\t\t\t\t\t\t\tinstance.service.wan_ipv6.second);\n\t\t\n\t\tfprintf(stderr, \"check_id:%s\\n\", instance.check_id.c_str());\n\t\tfprintf(stderr, \"check_name:%s\\n\", instance.check_name.c_str());\n\t\tfprintf(stderr, \"check_notes:%s\\n\", instance.check_notes.c_str());\n\t\tfprintf(stderr, \"check_output:%s\\n\", instance.check_output.c_str());\n\t\tfprintf(stderr, \"check_status:%s\\n\", instance.check_status.c_str());\n\t\tfprintf(stderr, \"check_type:%s\\n\", instance.check_type.c_str());\t\n\t}\n}\n\nvoid print_list_service_result(\n\tstd::vector& list_service_result)\n{\n\tfor (const auto& instance : list_service_result)\n\t{\n\t\tfprintf(stderr, \"service name:%s tags:\", instance.service_name.c_str());\n\t\tstd::string tags_log;\n\t\tfor (const auto& tag : instance.tags)\n\t\t{\n\t\t\ttags_log += tag;\n\t\t\ttags_log += \",\";\n\t\t}\n\t\tif (tags_log.size() > 0)\n\t\t\ttags_log.pop_back();\n\n\t\tfprintf(stderr, \"%s\\n\", tags_log.c_str());\n\t}\n}\n\nvoid consul_callback(WFConsulTask *task)\n{\n\tint state = task->get_state();\n\tint error = task->get_error();\n\n\tif (state != WFT_STATE_SUCCESS)\n\t{\n\t\tfprintf(stderr, \"error:%d, error msg:%s\\n\",\n\t\t\t\t error, WFGlobal::get_error_string(state, error));\n\t\tfprintf(stderr, \"Failed. Press Ctrl-C to exit.\\n\");\n\t\twait_group.done();\n\t\treturn;\n\t}\n\n\tint api_type = task->get_api_type();\n\tstd::vector dis_result;\n\tstd::vector list_service_result;\n\t\n\tswitch (api_type)\n\t{\n\tcase CONSUL_API_TYPE_DISCOVER:\n\t\tfprintf(stderr, \"discover ok\\n\");\n\t\tfprintf(stderr, \"consul-index:%lld\\n\", task->get_consul_index());\n\t\tif (task->get_discover_result(dis_result))\n\t\t\tprint_discover_result(dis_result);\n\t\telse\n\t\t\tfprintf(stderr, \"error:%d\\n\", task->get_error());\n\t\tbreak;\n\n\tcase CONSUL_API_TYPE_LIST_SERVICE:\n\t\tfprintf(stderr, \"list service ok\\n\");\n\t\tif (task->get_list_service_result(list_service_result))\n\t\t\tprint_list_service_result(list_service_result);\n\t\telse\n\t\t\tfprintf(stderr, \"error:%d\\n\", task->get_error());\n\t\tbreak;\n\n\tcase CONSUL_API_TYPE_REGISTER:\n\t\tfprintf(stderr, \"register ok\\n\");\n\t\tbreak;\n\n\tcase CONSUL_API_TYPE_DEREGISTER:\n\t\tfprintf(stderr, \"deregister ok\\n\");\n\t\tbreak;\n\n\tdefault:\n\t\tbreak;\n\t}\n\twait_group.done();\n}\n\nvoid sig_handler(int signo) { }\n\nint main(int argc, char *argv[])\n{\n\tif (argc != 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tsignal(SIGINT, sig_handler);\n\n\turl = argv[1];\n\tif (strncmp(argv[1], \"http://\", 7) != 0)\n\t\turl = \"http://\" + url;\n\n\tConsulConfig config;\n\tconfig.set_token(\"cd125427-3fd1-f326-bf46-fbce06cc9003\");\n\tconfig.set_health_check(true);\n\n\t// http health check\n\tconfig.set_check_http_url(\"http://127.0.0.1:8000/health_check/sd\");\n\t// config.add_http_header(\"Accept\", {\"text/html\", \"application/xml\"});\n\n\t// tcp health check\n\t//config.set_check_tcp(\"127.0.0.1:80\");\n\n\tclient.init(url, config);\n\n\tWFConsulTask *task;\n\n\tif (0 == strcmp(argv[2], \"discover\"))\n\t{\t\n\t\ttask = client.create_discover_task(\"\", \"dev-wf_test_service_1\", 3, consul_callback);\n\t\tconfig.set_blocking_query(true);\n\t}\n\telse if (0 == strcmp(argv[2], \"list_service\"))\n\t{\n\t\ttask = client.create_list_service_task(\"\", 3, consul_callback);\n\t}\n\telse if (0 == strcmp(argv[2], \"register\"))\n\t{\n\t\ttask = client.create_register_task(\"\", \"dev-wf_test_service_1\", \"wf_test_service_id_2\", 3, consul_callback);\n\t\tprotocol::ConsulService service;\n\t\tservice.tags.emplace_back(\"tag1\");\n\t\tservice.tags.emplace_back(\"tag2\");\n\t\tservice.service_address.first = \"127.0.0.1\";\n\t\tservice.service_address.second = 8000;\n\t\tservice.meta[\"mk1\"] = \"mv1\";\n\t\tservice.meta[\"mk2\"] = \"mv2\";\n\t\tservice.tag_override = true;\n\t\ttask->set_service(&service);\n\t}\n\telse if (0 == strcmp(argv[2], \"deregister\"))\n\t{\n\t\ttask = client.create_deregister_task(\"\", \"wf_test_service_id_2\", 3, consul_callback);\n\t}\n\telse\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\ttask->start();\n\n\twait_group.wait();\n\n\treturn 0;\n}\n" }, { "path": "tutorial/tutorial-15-name_service.cc", "content": "#include \n#include \n#include \n#include \n#include \n#include \"workflow/WFGlobal.h\"\n#include \"workflow/WFNameService.h\"\n#include \"workflow/WFTaskFactory.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"workflow/HttpUtil.h\"\n\n// The example domonstrate the simplest user defined naming policy.\n\n/* 'MyNSPolicy' is a naming policy, which use local file for naming.\n * The format of naming file is similar to 'hosts' file, but we allow\n * domain name and IP address as destination. For example:\n *\n * 127.0.0.1 localhost\n * 127.0.0.1 mydomain # another alias for 127.0.0.1\n * www.sogou.com sogou # sogou -> www.sogou.com\n */\nclass MyNSPolicy : public WFNSPolicy\n{\npublic:\n\tWFRouterTask *create_router_task(const struct WFNSParams *params,\n\t\t\t\t\t\t\t\t\t router_callback_t callback) override;\n\nprivate:\n\tstd::string path;\n\nprivate:\n\tstd::string read_from_fp(FILE *fp, const char *name);\n\tstd::string parse_line(char *p, const char *name);\n\npublic:\n\tMyNSPolicy(const char *naming_file) : path(naming_file) { }\n};\n\nstd::string MyNSPolicy::parse_line(char *p, const char *name)\n{\n\tconst char *dest = NULL;\n\tchar *start;\n\n\tstart = p;\n\twhile (*start != '\\0' && *start != '#')\n\t\tstart++;\n\t*start = '\\0';\n\n\twhile (1)\n\t{\n\t\twhile (isspace(*p))\n\t\t\tp++;\n\n\t\tstart = p;\n\t\twhile (*p != '\\0' && !isspace(*p))\n\t\t\tp++;\n\n\t\tif (start == p)\n\t\t\tbreak;\n\n\t\tif (*p != '\\0')\n\t\t\t*p++ = '\\0';\n\n\t\tif (dest == NULL)\n\t\t{\n\t\t\tdest = start;\n\t\t\tcontinue;\n\t\t}\n\n\t\tif (strcasecmp(name, start) == 0)\n\t\t\treturn std::string(dest);\n\t}\n\n\treturn std::string();\n}\n\nstd::string MyNSPolicy::read_from_fp(FILE *fp, const char *name)\n{\n\tchar *line = NULL;\n\tsize_t bufsize = 0;\n\tstd::string result;\n\n\twhile (getline(&line, &bufsize, fp) > 0)\n\t{\n\t\tresult = this->parse_line(line, name);\n\t\tif (result.size() > 0)\n\t\t\tbreak;\n\t}\n\n\tfree(line);\n\treturn result;\n}\n\nWFRouterTask *MyNSPolicy::create_router_task(const struct WFNSParams *params,\n\t\t\t\t\t\t\t\t\t\t\t router_callback_t callback)\n{\n\tWFDnsResolver *dns_resolver = WFGlobal::get_dns_resolver();\n\n\tif (params->uri.host)\n\t{\n\t\tFILE *fp = fopen(this->path.c_str(), \"r\");\n\t\tif (fp)\n\t\t{\n\t\t\tstd::string dest = this->read_from_fp(fp, params->uri.host);\n\t\t\tif (dest.size() > 0)\n\t\t\t{\n\t\t\t\t/* Update the uri structure's 'host' field directly.\n\t\t\t\t * You can also update the 'port' field if needed. */\n\t\t\t\tfree(params->uri.host);\n\t\t\t\tparams->uri.host = strdup(dest.c_str());\n\t\t\t}\n\n\t\t\tfclose(fp);\n\t\t}\n\t}\n\n\t/* Simply, use the global dns resolver to create a router task. */\n\treturn dns_resolver->create_router_task(params, std::move(callback));\n}\n\nint main(int argc, char *argv[])\n{\n\tif (argc != 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tParsedURI uri;\n\tURIParser::parse(argv[1], uri);\n\tchar *name = uri.host;\n\tif (name == NULL)\n\t{\n\t\tfprintf(stderr, \"Invalid http URI\\n\");\n\t\texit(1);\n\t}\n\n\t/* Create an naming policy. */\n\tMyNSPolicy *policy = new MyNSPolicy(argv[2]);\n\n\t/* Get the global name service object.*/\n\tWFNameService *ns = WFGlobal::get_name_service();\n\n\t/* Add the our name with policy to global name service.\n\t * You can add mutilply names with one policy object. */\n\tns->add_policy(name, policy);\n\n\tWFFacilities::WaitGroup wg(1);\n\tWFHttpTask *task = WFTaskFactory::create_http_task(argv[1], 2, 3,\n\t\t[&wg](WFHttpTask *task) {\n\t\t\tint state = task->get_state();\n\t\t\tint error = task->get_error();\n\t\t\tif (state != WFT_STATE_SUCCESS)\n\t\t\t{\n\t\t\t\tfprintf(stderr, \"error: %s\\n\",\n\t\t\t\t\t\tWFGlobal::get_error_string(state, error));\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tauto *r = task->get_resp();\n\t\t\t\tstd::string body = protocol::HttpUtil::decode_chunked_body(r);\n\t\t\t\tfwrite(body.c_str(), 1, body.size(), stdout);\n\t\t\t}\n\t\t\twg.done();\n\t\t});\n\n\ttask->start();\n\twg.wait();\n\n\t/* clean up */\n\tns->del_policy(name);\n\tdelete policy;\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-16-graceful_restart/bootstrap.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nint flag = 0;\n\nvoid sig_handler(int signo)\n{\n\tif (signo == SIGUSR1)\n\t\tflag = 1;\n\telse if (signo == SIGINT || signo == SIGTERM)\n\t\tflag = 2;\n}\n\nint main(int argc, const char *argv[])\n{\n\tif (argc != 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s EXEC_PROCESS PORT\\n\"\n\t\t\t\t\"Bootstrap for workflow server to restart gracefully.\\n\",\n\t\t\t\targv[0]);\n\t\texit(1);\n\t}\n\n\tunsigned short port = atoi(argv[2]);\n\tint listen_fd = socket(AF_INET, SOCK_STREAM, 0);\n\tstruct sockaddr_in sin;\n\n\tmemset(&sin, 0, sizeof sin);\n\tsin.sin_family = AF_INET;\n\tsin.sin_port = htons(port);\n\tsin.sin_addr.s_addr = htonl(INADDR_ANY);\n\n\tif (bind(listen_fd, (struct sockaddr *)&sin, sizeof sin) < 0)\n\t{\n\t\tclose(listen_fd);\n\t\tperror(\"bind error\");\n\t\texit(1);\n\t}\n\n\tpid_t pid;\n\tint pipe_fd[2];\n\tssize_t len;\n\tchar buf[100];\n\tint status;\n\tint ret;\n\n\tchar listen_fd_str[10] = { 0 };\n\tchar write_fd_str[10] = { 0 };\n\tsprintf(listen_fd_str, \"%d\", listen_fd);\n\n\tsignal(SIGINT, sig_handler);\n\tsignal(SIGTERM, sig_handler);\n\tsignal(SIGUSR1, sig_handler);\n\n\twhile (flag < 2)\n\t{\n\t\tif (pipe(pipe_fd) == -1)\n\t\t{\n\t\t\tperror(\"open pipe error\");\n\t\t\texit(1);\n\t\t}\n\n\t\tmemset(write_fd_str, 0, sizeof write_fd_str);\n\t\tsprintf(write_fd_str, \"%d\", pipe_fd[1]);\n\n\t\tpid = fork();\n\t\tif (pid < 0)\n\t\t{\n\t\t\tperror(\"fork error\");\n\t\t\tclose(pipe_fd[0]);\n\t\t\tclose(pipe_fd[1]);\n\t\t\tbreak;\n\t\t}\n\t\telse if (pid == 0)\n\t\t{\n\t\t\tclose(pipe_fd[0]);\n\t\t\texeclp(argv[1], argv[1], listen_fd_str, write_fd_str, NULL);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tclose(pipe_fd[1]);\n\n\t\t\tstatus = 0;\n\t\t\tret = 0;\n\t\t\tflag = 0;\n\t\t\tfprintf(stderr, \"Bootstrap daemon running with server pid-%d. \"\n\t\t\t\t\t\"Send SIGUSR1 to RESTART or SIGTERM to STOP.\\n\", pid);\n\n\t\t\twhile (1)\n\t\t\t{\n\t\t\t\tret = waitpid(pid, &status, WNOHANG);\n\t\t\t\tif (ret == -1 || !WIFEXITED(status) || flag != 0)\n\t\t\t\t\tbreak;\n\n\t\t\t\tsleep(3);\n\t\t\t}\n\n\t\t\tif (ret != -1 && WIFEXITED(status))\n\t\t\t{\n\t\t\t\tsignal(SIGCHLD, SIG_IGN);\n\n\t\t\t\tkill(pid, SIGUSR1);\n\t\t\t\tfprintf(stderr, \"Bootstrap daemon SIGUSR1 to pid-%ld %sing.\\n\",\n\t\t\t\t\t\t(long)pid, flag == 1 ? \"restart\" : \"stop\");\n\n\t\t\t\tlen = read(pipe_fd[0], buf, 7);\n\t\t\t\tfprintf(stderr, \"Bootstrap server served %*s.\\n\", (int)len, buf);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tfprintf(stderr, \"child exit. status = %d, waitpid ret = %d\\n\",\n\t\t\t\t\t\tWEXITSTATUS(status), ret);\n\t\t\t\tflag = 2;\n\t\t\t}\n\n\t\t\tclose(pipe_fd[0]);\n\t\t}\n\t}\n\n\tclose(listen_fd);\n\treturn 0;\n}\n\n" }, { "path": "tutorial/tutorial-16-graceful_restart/server.cc", "content": "#include \n#include \n#include \n#include \n#include \"workflow/WFFacilities.h\"\n#include \"workflow/WFHttpServer.h\"\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, const char *argv[])\n{\n\tif (argc != 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s listen_fd pipe_fd\\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tint listen_fd = atoi(argv[1]);\n\tint pipe_fd = atoi(argv[2]);\n\n\tsignal(SIGUSR1, sig_handler);\n\n\tWFHttpServer server([](WFHttpTask *task) {\n \ttask->get_resp()->append_output_body(\"Hello World!\");\n\t});\n\n\tif (server.serve(listen_fd) == 0)\n\t{\n\t\twait_group.wait();\n\t\tserver.shutdown();\n\t\twrite(pipe_fd, \"success\", strlen(\"success\"));\n\t\tserver.wait_finish();\n\t}\n\telse\n\t\twrite(pipe_fd, \"failed \", strlen(\"failed \"));\n\n\tclose(pipe_fd);\n\tclose(listen_fd);\n return 0;\n}\n\n" }, { "path": "tutorial/tutorial-16-graceful_restart/xmake.lua", "content": "target(\"bootstrap\")\n set_kind(\"binary\")\n add_files(\"bootstrap.c\")\n\ntarget(\"bootstrap_server\")\n set_kind(\"binary\")\n add_files(\"server.cc\")\n add_deps(\"workflow\")" }, { "path": "tutorial/tutorial-17-dns_cli.cc", "content": "#include \n#include \n#include \n#include \n#include \n\n#include \"workflow/DnsUtil.h\"\n#include \"workflow/WFDnsClient.h\"\n#include \"workflow/WFFacilities.h\"\n\nstatic const std::map qtype_map =\n{\n\t{\"A\",\t\tDNS_TYPE_A },\n\t{\"AAAA\",\tDNS_TYPE_AAAA },\n\t{\"CNAME\",\tDNS_TYPE_CNAME },\n\t{\"SOA\",\t\tDNS_TYPE_SOA },\n\t{\"NS\",\t\tDNS_TYPE_NS },\n\t{\"SRV\",\t\tDNS_TYPE_SRV },\n\t{\"MX\",\t\tDNS_TYPE_MX }\n};\nWFFacilities::WaitGroup wait_group(1);\n\nvoid show_result(protocol::DnsResultCursor& cursor)\n{\n\tchar information[1024];\n\tconst char *info;\n\tstruct dns_record *record;\n\tstruct dns_record_soa *soa;\n\tstruct dns_record_srv *srv;\n\tstruct dns_record_mx *mx;\n\n\twhile(cursor.next(&record))\n\t{\n\t\tswitch (record->type)\n\t\t{\n\t\tcase DNS_TYPE_A:\n\t\t\tinfo = inet_ntop(AF_INET, record->rdata, information, 64);\n\t\t\tbreak;\n\t\tcase DNS_TYPE_AAAA:\n\t\t\tinfo = inet_ntop(AF_INET6, record->rdata, information, 64);\n\t\t\tbreak;\n\t\tcase DNS_TYPE_NS:\n\t\tcase DNS_TYPE_CNAME:\n\t\tcase DNS_TYPE_PTR:\n\t\t\tinfo = (const char *)(record->rdata);\n\t\t\tbreak;\n\t\tcase DNS_TYPE_SOA:\n\t\t\tsoa = (struct dns_record_soa *)(record->rdata);\n\t\t\tsprintf(information, \"%s %s %u %d %d %d %u\",\n\t\t\t\tsoa->mname, soa->rname, soa->serial, soa->refresh,\n\t\t\t\tsoa->retry, soa->expire, soa->minimum\n\t\t\t);\n\t\t\tinfo = information;\n\t\t\tbreak;\n\t\tcase DNS_TYPE_SRV:\n\t\t\tsrv = (struct dns_record_srv *)(record->rdata);\n\t\t\tsprintf(information, \"%u %u %u %s\",\n\t\t\t\tsrv->priority, srv->weight, srv->port, srv->target\n\t\t\t);\n\t\t\tinfo = information;\n\t\t\tbreak;\n\t\tcase DNS_TYPE_MX:\n\t\t\tmx = (struct dns_record_mx *)(record->rdata);\n\t\t\tsprintf(information, \"%d %s\", mx->preference, mx->exchange);\n\t\t\tinfo = information;\n\t\t\tbreak;\n\t\tdefault:\n\t\t\tinfo = \"Unknown\";\n\t\t\tbreak;\n\t\t}\n\n\t\tprintf(\"%s\\t%d\\t%s\\t%s\\t%s\\n\",\n\t\t\trecord->name, record->ttl,\n\t\t\tdns_class2str(record->rclass),\n\t\t\tdns_type2str(record->type),\n\t\t\tinfo\n\t\t);\n\t}\n\tprintf(\"\\n\");\n}\n\nvoid dns_callback(WFDnsTask *task)\n{\n\tint state = task->get_state();\n\tint error = task->get_error();\n\tauto *resp = task->get_resp();\n\n\tif (state != WFT_STATE_SUCCESS)\n\t{\n\t\tprintf(\"State: %d, Error: %d\\n\", state, error);\n\t\tprintf(\"Error: %s\\n\", WFGlobal::get_error_string(state, error));\n\t\twait_group.done();\n\t\treturn;\n\t}\n\n\tprintf(\"; Workflow DNSResolver\\n\");\n\tprintf(\";; HEADER opcode:%s status:%s id:%d\\n\",\n\t\tdns_opcode2str(resp->get_opcode()),\n\t\tdns_rcode2str(resp->get_rcode()),\n\t\tresp->get_id()\n\t);\n\tprintf(\";; QUERY:%d ANSWER:%d AUTHORITY:%d ADDITIONAL:%d\\n\",\n\t\tresp->get_qdcount(), resp->get_ancount(),\n\t\tresp->get_nscount(), resp->get_arcount()\n\t);\n\n\tprintf(\"\\n\");\n\n\tprotocol::DnsResultCursor cursor(resp);\n\tif(resp->get_ancount() > 0)\n\t{\n\t\tcursor.reset_answer_cursor();\n\t\tprintf(\";; ANSWER SECTION:\\n\");\n\t\tshow_result(cursor);\n\t}\n\tif(resp->get_nscount() > 0)\n\t{\n\t\tcursor.reset_authority_cursor();\n\t\tprintf(\";; AUTHORITY SECTION\\n\");\n\t\tshow_result(cursor);\n\t}\n\tif(resp->get_arcount() > 0)\n\t{\n\t\tcursor.reset_additional_cursor();\n\t\tprintf(\";; ADDITIONAL SECTION\\n\");\n\t\tshow_result(cursor);\n\t}\n\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tint qtype = DNS_TYPE_A;\n\tconst char *domain;\n\n\tif (argc == 1 || argc > 3)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s [query type]\\n\", argv[0]);\n\t\treturn 1;\n\t}\n\n\tdomain = argv[1];\n\n\tif (argc == 3)\n\t{\n\t\tauto it = qtype_map.find(argv[2]);\n\t\tif (it != qtype_map.end())\n\t\t\tqtype = it->second;\n\t}\n\n\tstd::string url = \"dns://119.29.29.29\";\n\tWFDnsTask *task = WFTaskFactory::create_dns_task(url, 0, dns_callback);\n\n\tprotocol::DnsRequest *req = task->get_req();\n\treq->set_rd(1);\n\treq->set_question(domain, qtype, DNS_CLASS_IN);\n\n\ttask->start();\n\n\twait_group.wait();\n\treturn 0;\n}\n" }, { "path": "tutorial/tutorial-18-redis_subscriber.cc", "content": "#include \n#include \n#include \n#include \n#include \n\n#include \"workflow/WFRedisSubscriber.h\"\n#include \"workflow/WFFacilities.h\"\n#include \"workflow/StringUtil.h\"\n\nvoid extract(WFRedisSubscribeTask *task)\n{\n\tauto *resp = task->get_resp();\n\tprotocol::RedisValue value;\n\n\tresp->get_result(value);\n\n\tif (value.is_array())\n\t{\n\t\tfor (size_t i = 0; i < value.arr_size(); i++)\n\t\t{\n\t\t\tif (value[i].is_string())\n\t\t\t\tstd::cout << value[i].string_value();\n\t\t\telse if (value[i].is_int())\n\t\t\t\tstd::cout << value[i].int_value();\n\t\t\telse if (value[i].is_nil())\n\t\t\t\tstd::cout << \"nil\";\n\t\t\telse\n\t\t\t\tstd::cout << \"Unexpected value in array!\";\n\n\t\t\tstd::cout << \"\\n\";\n\t\t}\n\t}\n\telse\n\t\tstd::cout << \"Unexpected value!\\n\";\n}\n\nint main(int argc, char *argv[])\n{\n\tif (argc < 3)\n\t{\n\t\tstd::cerr << argv[0] << \" []...\" << std::endl;\n\t\texit(1);\n\t}\n\n\tstd::string url = argv[1];\n\tif (strncasecmp(argv[1], \"redis://\", 8) != 0 &&\n\t\tstrncasecmp(argv[1], \"rediss://\", 9) != 0)\n\t{\n\t\turl = \"redis://\" + url;\n\t}\n\n\tWFRedisSubscriber suber;\n\n\tif (suber.init(url) != 0)\n\t{\n\t\tstd::cerr << \"Subscriber init failed \" << strerror(errno) << std::endl;\n\t\texit(1);\n\t}\n\n\tstd::vector channels;\n\tfor (int i = 2; i < argc; i++)\n\t\tchannels.push_back(argv[i]);\n\n\tWFFacilities::WaitGroup wg(1);\n\tbool finished = false;\n\n\tauto callback = [&](WFRedisSubscribeTask *task)\n\t{\n\t\tstd::cout << \"state = \" << task->get_state()\n\t\t\t<< \", error = \" << task->get_error() << std::endl;\n\n\t\tfinished = true;\n\t\twg.done();\n\t};\n\n\tWFRedisSubscribeTask *task;\n\ttask = suber.create_subscribe_task(channels, extract, callback);\n\n\ttask->set_watch_timeout(1000000);\n\ttask->start();\n\n\tstd::string line;\n\n\twhile (!finished)\n\t{\n\t\tstd::string cmd;\n\t\tstd::vector params;\n\n\t\tif (std::getline(std::cin, line))\n\t\t{\n\t\t\tif (line.empty())\n\t\t\t\tcontinue;\n\n\t\t\tparams = StringUtil::split_filter_empty(line, ' ');\n\t\t}\n\n\t\tif (finished)\n\t\t\tbreak;\n\n\t\tif (params.empty())\n\t\t{\n\t\t\ttask->unsubscribe();\n\t\t\ttask->punsubscribe();\n\t\t\tbreak;\n\t\t}\n\n\t\tcmd = params[0];\n\t\tparams.erase(params.begin());\n\n\t\tfor (char &c : cmd)\n\t\t\tc = std::toupper(c);\n\n\t\tint ret;\n\t\tif (cmd == \"SUBSCRIBE\")\n\t\t\tret = task->subscribe(params);\n\t\telse if (cmd == \"UNSUBSCRIBE\")\n\t\t\tret = task->unsubscribe(params);\n\t\telse if (cmd == \"PSUBSCRIBE\")\n\t\t\tret = task->psubscribe(params);\n\t\telse if (cmd == \"PUNSUBSCRIBE\")\n\t\t\tret = task->punsubscribe(params);\n\t\telse if (cmd == \"PING\")\n\t\t{\n\t\t\tif (params.empty())\n\t\t\t\tret = task->ping();\n\t\t\telse\n\t\t\t\tret = task->ping(params[0]);\n\t\t}\n\t\telse if (cmd == \"QUIT\")\n\t\t\tret = task->quit();\n\t\telse\n\t\t{\n\t\t\tstd::cerr << \"Invalid command \" << cmd << std::endl;\n\t\t\tret = 0;\n\t\t}\n\n\t\tif (ret < 0)\n\t\t{\n\t\t\tstd::cerr << \"Send command failed \" << strerror(errno) << std::endl;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\ttask->release();\n\twg.wait();\n\tsuber.deinit();\n\n\treturn 0;\n}\n" }, { "path": "tutorial/tutorial-19-dns_server.cc", "content": "#include \n#include \n#include \n#include \n#include \n\n#include \"workflow/WFDnsServer.h\"\n#include \"workflow/WFFacilities.h\"\n\nvoid process(WFDnsTask *task)\n{\n\tprotocol::DnsRequest *req = task->get_req();\n\tprotocol::DnsResponse *resp = task->get_resp();\n\n\tstd::string name = req->get_question_name();\n\tint qtype = req->get_question_type();\n\tint qclass = req->get_question_class();\n\tint opcode = req->get_opcode();\n\n\tprintf(\"name:%s type:%s class:%s\\n\",\n\t\t name.c_str(), dns_type2str(qtype), dns_class2str(qclass));\n\n\tif (opcode != 0)\n\t{\n\t\tresp->set_rcode(DNS_RCODE_NOT_IMPLEMENTED);\n\t\treturn;\n\t}\n\n\tresp->set_rcode(DNS_RCODE_NO_ERROR);\n\tresp->set_aa(1);\n\n\tif (qtype == DNS_TYPE_A)\n\t{\n\t\tstd::string cname = \"cname.test\";\n\t\tresp->add_cname_record(DNS_ANSWER_SECTION,\n\t\t\t\t\t\t\t name.c_str(), DNS_CLASS_IN, 999, cname.c_str());\n\n\t\tstruct in_addr addr;\n\n\t\tinet_pton(AF_INET, \"192.168.0.1\", (void *)&addr);\n\t\tresp->add_a_record(DNS_ANSWER_SECTION,\n\t\t\t\t\t\t cname.c_str(), DNS_CLASS_IN, 600, &addr);\n\n\t\tinet_pton(AF_INET, \"192.168.0.2\", (void *)&addr);\n\t\tresp->add_a_record(DNS_ANSWER_SECTION,\n\t\t\t\t\t\t cname.c_str(), DNS_CLASS_IN, 600, &addr);\n\t}\n\telse if (qtype == DNS_TYPE_AAAA)\n\t{\n\t\tstruct in6_addr addr;\n\n\t\tinet_pton(AF_INET6, \"1234:5678:9abc:def0::\", (void *)&addr);\n\t\tresp->add_aaaa_record(DNS_ANSWER_SECTION,\n\t\t\t\t\t\t\t name.c_str(), DNS_CLASS_IN, 600, &addr);\n\t}\n\telse if (qtype == DNS_TYPE_SOA)\n\t{\n\t\tconst char *mname = \"mname.test\";\n\t\tconst char *rname = \"rname.test\";\n\n\t\tresp->add_soa_record(DNS_ANSWER_SECTION, name.c_str(), DNS_CLASS_IN,\n\t\t\t\t\t\t\t 60, mname, rname, 123, 86400, 3600, 2592000, 7200);\n\t}\n\telse if (qtype == DNS_TYPE_TXT)\n\t{\n\t\tconst char *raw_txt_data = \"\\x0dmy dns server\\x0fyour dns server\";\n\t\tuint16_t data_len = 30;\n\n\t\tresp->add_raw_record(DNS_ANSWER_SECTION, name.c_str(), DNS_TYPE_TXT,\n\t\t\t\t\t\t\t DNS_CLASS_IN, 1200, raw_txt_data, data_len);\n\t}\n\telse\n\t{\n\t\tresp->set_rcode(DNS_RCODE_NOT_IMPLEMENTED);\n\t}\n}\n\nstatic WFFacilities::WaitGroup wait_group(1);\n\nvoid sig_handler(int signo)\n{\n\twait_group.done();\n}\n\nint main(int argc, char *argv[])\n{\n\tunsigned short port;\n\n\tif (argc != 2)\n\t{\n\t\tfprintf(stderr, \"USAGE: %s \\n\", argv[0]);\n\t\texit(1);\n\t}\n\n\tsignal(SIGINT, sig_handler);\n\n\tWFDnsServer server(process);\n\tport = atoi(argv[1]);\n\tif (server.start(port) == 0)\n\t{\n\t\twait_group.wait();\n\t\tserver.stop();\n\t}\n\telse\n\t{\n\t\tperror(\"Cannot start server\");\n\t}\n\n\treturn 0;\n}\n" }, { "path": "tutorial/xmake.lua", "content": "set_group(\"tutorial\")\nset_default(false)\n\nif not is_plat(\"macosx\") then\n add_ldflags(\"-lrt\")\nend\n\nfunction all_examples()\n local res = {}\n for _, x in ipairs(os.files(\"*.cc\")) do\n local item = {}\n local s = path.filename(x)\n if ((s == \"upstream_unittest.cc\" and not has_config(\"upstream\")) or\n ((s == \"tutorial-02-redis_cli.cc\" or s == \"tutorial-03-wget_to_redis.cc\" or s == \"tutorial-18-redis_subscriber.cc\") and not has_config(\"redis\")) or\n (s == \"tutorial-12-mysql_cli.cc\" and not has_config(\"mysql\")) or\n (s == \"tutorial-14-consul_cli.cc\" and not has_config(\"consul\")) or\n (s == \"tutorial-13-kafka_cli.cc\")) then\n else\n table.insert(item, s:sub(1, #s - 3)) -- target\n table.insert(item, path.relative(x, \".\")) -- source\n table.insert(res, item)\n end\n end\n return res\nend\n\nfor _, example in ipairs(all_examples()) do\ntarget(example[1])\n set_kind(\"binary\")\n add_files(example[2])\n add_deps(\"workflow\")\nend\n\ntarget(\"tutorial-13-kafka_cli\")\n if has_config(\"kafka\") then\n set_kind(\"binary\")\n add_files(\"tutorial-13-kafka_cli.cc\")\n add_packages(\"zlib\", \"snappy\", \"zstd\", \"lz4\")\n add_deps(\"wfkafka\")\n else\n set_kind(\"phony\")\n end\n\nincludes(\"tutorial-10-user_defined_protocol\", \"tutorial-16-graceful_restart\")\n" }, { "path": "workflow-config.cmake.in", "content": "@PACKAGE_INIT@\n\nset(WORKFLOW_VERSION \"@workflow_VERSION@\")\nset_and_check(WORKFLOW_INCLUDE_DIR \"@PACKAGE_CONFIG_INC_DIR@\")\nset_and_check(WORKFLOW_LIB_DIR \"@PACKAGE_CONFIG_LIB_DIR@\")\n\nif (EXISTS \"${CMAKE_CURRENT_LIST_DIR}/workflow-targets.cmake\")\n include (\"${CMAKE_CURRENT_LIST_DIR}/workflow-targets.cmake\")\nendif ()\n\ncheck_required_components(workflow)\n" }, { "path": "xmake.lua", "content": "set_project(\"workflow\")\nset_version(\"1.0.0\")\n\noption(\"workflow_inc\", {description = \"workflow inc\", default = \"$(projectdir)/_include\"})\noption(\"workflow_lib\", {description = \"workflow lib\", default = \"$(projectdir)/_lib\"})\noption(\"kafka\", {description = \"build kafka component\", default = false})\noption(\"consul\", {description = \"build consul component\", default = true})\noption(\"mysql\", {description = \"build mysql component\", default = true})\noption(\"redis\", {description = \"build redis component\", default = true})\noption(\"upstream\", {description = \"build upstream component\", default = true})\noption(\"memcheck\", {description = \"valgrind memcheck\", default = false})\n\nif is_mode(\"release\") then\n set_optimize(\"faster\")\n set_strip(\"all\")\nelseif is_mode(\"debug\") then\n set_symbols(\"debug\")\n set_optimize(\"none\")\nend\n\nset_languages(\"gnu90\", \"c++11\")\nset_warnings(\"all\")\nset_exceptions(\"no-cxx\")\n\nadd_requires(\"openssl\")\nadd_packages(\"openssl\")\nadd_syslinks(\"pthread\")\n\nif has_config(\"kafka\") then\n add_requires(\"snappy\", \"lz4\", \"zstd\", \"zlib\")\nend\n\nadd_includedirs(get_config(\"workflow_inc\"))\nadd_includedirs(path.join(get_config(\"workflow_inc\"), \"workflow\"))\n\nset_config(\"buildir\", \"build.xmake\")\n\nadd_cflags(\"-fPIC\", \"-pipe\")\nadd_cxxflags(\"-fPIC\", \"-pipe\", \"-Wno-invalid-offsetof\")\nif (is_plat(\"macosx\")) then\n add_cxxflags(\"-Wno-deprecated-declarations\")\nend\n\nincludes(\"src\", \"test\", \"benchmark\", \"tutorial\")\n\n" } ]