Repository: baidu/unit-dmkit Branch: master Commit: 6b837ee07504 Files: 97 Total size: 849.3 KB Directory structure: gitextract_m7sg05uu/ ├── .gitignore ├── CMakeLists.txt ├── Dockerfile ├── LICENSE ├── NOTICE ├── README.md ├── conf/ │ ├── app/ │ │ ├── bot_tokens.json │ │ ├── demo/ │ │ │ ├── book_hotel.json │ │ │ ├── book_hotel.xml │ │ │ ├── cellular_data.json │ │ │ ├── cellular_data.xml │ │ │ ├── quota_adjust.json │ │ │ └── quota_adjust.xml │ │ ├── products.json │ │ └── remote_services.json │ └── gflags.conf ├── deps.sh ├── docs/ │ ├── demo_book_hotel_pattern.txt │ ├── demo_cellular_data_pattern.txt │ ├── demo_quota_adjust_pattern.txt │ ├── demo_skills.md │ ├── faq.md │ ├── tutorial.md │ └── visual_tool.md ├── language_compiler/ │ ├── compiler_xml.py │ ├── run.py │ └── settings.cfg ├── proto/ │ └── http.proto ├── src/ │ ├── app_container.cpp │ ├── app_container.h │ ├── app_log.h │ ├── application_base.h │ ├── brpc.h │ ├── butil.h │ ├── dialog_manager.cpp │ ├── dialog_manager.h │ ├── file_watcher.cpp │ ├── file_watcher.h │ ├── policy.cpp │ ├── policy.h │ ├── policy_manager.cpp │ ├── policy_manager.h │ ├── qu_result.cpp │ ├── qu_result.h │ ├── rapidjson.h │ ├── remote_service_manager.cpp │ ├── remote_service_manager.h │ ├── request_context.cpp │ ├── request_context.h │ ├── server.cpp │ ├── thirdparty/ │ │ └── rapidjson/ │ │ ├── allocators.h │ │ ├── document.h │ │ ├── encodedstream.h │ │ ├── encodings.h │ │ ├── error/ │ │ │ ├── en.h │ │ │ └── error.h │ │ ├── filereadstream.h │ │ ├── filewritestream.h │ │ ├── fwd.h │ │ ├── internal/ │ │ │ ├── biginteger.h │ │ │ ├── diyfp.h │ │ │ ├── dtoa.h │ │ │ ├── ieee754.h │ │ │ ├── itoa.h │ │ │ ├── meta.h │ │ │ ├── pow10.h │ │ │ ├── regex.h │ │ │ ├── stack.h │ │ │ ├── strfunc.h │ │ │ ├── strtod.h │ │ │ └── swap.h │ │ ├── istreamwrapper.h │ │ ├── memorybuffer.h │ │ ├── memorystream.h │ │ ├── msinttypes/ │ │ │ ├── inttypes.h │ │ │ └── stdint.h │ │ ├── ostreamwrapper.h │ │ ├── pointer.h │ │ ├── prettywriter.h │ │ ├── rapidjson.h │ │ ├── reader.h │ │ ├── schema.h │ │ ├── stream.h │ │ ├── stringbuffer.h │ │ └── writer.h │ ├── thread_data_base.h │ ├── token_manager.cpp │ ├── token_manager.h │ ├── user_function/ │ │ ├── demo.cpp │ │ ├── demo.h │ │ ├── shared.cpp │ │ └── shared.h │ ├── user_function_manager.cpp │ ├── user_function_manager.h │ └── utils.h └── tools/ ├── bot_emulator.py └── mock_api_server.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ /output /Makefile /brpc /_build /build /.vscode *.pyc .DS_Store ================================================ FILE: CMakeLists.txt ================================================ cmake_minimum_required(VERSION 2.8.10) project(dmkit C CXX) execute_process( COMMAND bash -c "find ${CMAKE_SOURCE_DIR}/brpc -type d -regex \".*output/include$\" | xargs dirname | tr -d '\n'" OUTPUT_VARIABLE OUTPUT_PATH ) set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) include(FindThreads) include(FindProtobuf) protobuf_generate_cpp(PROTO_SRC PROTO_HEADER proto/http.proto) # include PROTO_HEADER include_directories(${CMAKE_CURRENT_BINARY_DIR}) find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) find_library(BRPC_LIB NAMES libbrpc.a brpc) if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) message(FATAL_ERROR "Fail to find brpc") endif() include_directories(${BRPC_INCLUDE_PATH}) find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) find_library(GFLAGS_LIBRARY NAMES gflags libgflags) if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) message(FATAL_ERROR "Fail to find gflags") endif() include_directories(${GFLAGS_INCLUDE_PATH}) execute_process( COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" OUTPUT_VARIABLE GFLAGS_NS ) if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") execute_process( COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" OUTPUT_VARIABLE GFLAGS_NS ) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") include(CheckFunctionExists) CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME) if(NOT HAVE_CLOCK_GETTIME) set(DEFINE_CLOCK_GETTIME "-DNO_CLOCK_GETTIME_IN_MAC") endif() endif() set(CMAKE_CPP_FLAGS "${DEFINE_CLOCK_GETTIME} ${CMAKE_CPP_FLAGS} -DGFLAGS_NS=${GFLAGS_NS}") set(CMAKE_CXX_FLAGS "${CMAKE_CPP_FLAGS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") if(CMAKE_VERSION VERSION_LESS "3.1.3") if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") endif() else() set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) endif() find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) find_library(LEVELDB_LIB NAMES leveldb) if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) message(FATAL_ERROR "Fail to find leveldb") endif() include_directories(${LEVELDB_INCLUDE_PATH}) if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") set(OPENSSL_ROOT_DIR "/usr/local/opt/openssl" # Homebrew installed OpenSSL ) endif() include(FindOpenSSL) find_library(CURL_LIB NAMES curl) if (NOT CURL_LIB) message(FATAL_ERROR "Fail to find curl") endif() set(DYNAMIC_LIB ${CMAKE_THREAD_LIBS_INIT} ${GFLAGS_LIBRARY} ${PROTOBUF_LIBRARIES} ${LEVELDB_LIB} ${OPENSSL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARY} ${CURL_LIB} dl ) if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") set(DYNAMIC_LIB ${DYNAMIC_LIB} pthread "-framework CoreFoundation" "-framework CoreGraphics" "-framework CoreData" "-framework CoreText" "-framework Security" "-framework Foundation" "-Wl,-U,_MallocExtension_ReleaseFreeMemory" "-Wl,-U,_ProfilerStart" "-Wl,-U,_ProfilerStop" "-Wl,-U,_RegisterThriftProtocol") endif() file(GLOB DMKIT_SRC "src/*.cpp" "src/*/*.cpp" ) include_directories( ${CMAKE_SOURCE_DIR}/src ) add_executable(dmkit ${DMKIT_SRC} ${PROTO_SRC} ${PROTO_HEADER}) target_link_libraries(dmkit ${BRPC_LIB} ${DYNAMIC_LIB}) add_custom_command( TARGET dmkit POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/conf ${CMAKE_BINARY_DIR}/conf ) ================================================ FILE: Dockerfile ================================================ FROM ubuntu:18.04 WORKDIR /unit-dmkit COPY . /unit-dmkit RUN apt-get update && apt-get install -y --no-install-recommends sudo cmake wget vim curl ca-certificates RUN update-ca-certificates RUN sh deps.sh ubuntu RUN rm -rf _build && mkdir _build && cd _build && cmake .. && make -j8 EXPOSE 8010 ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: NOTICE ================================================ Copyright (c) 2018 Baidu, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # DMKit DMKit作为UNIT的开源对话管理模块，可以无缝对接UNIT的理解能力，并赋予开发者多状态的复杂对话流程管理能力，还可以低成本对接外部知识库，迅速丰富话术信息量。 ## 快速开始 ### 编译DMKit DMKit基于[brpc](https://github.com/brpc/brpc)开发并提供HTTP服务，支持MacOS，Ubuntu，Centos等系统环境，推荐使用Ubuntu 16.04或CentOS 7。在编译DMKit之前，需要先安装依赖并下载编译brpc： ```bash sh deps.sh [OS] ``` 其中[OS]参数指定系统类型用于安装对应系统依赖，支持取值包括ubuntu、mac、centos。如果已手动安装依赖，则传入none。 使用cmake编译DMKit： ```bash mkdir _build && cd _build && cmake .. && make ``` ### 运行示例技能 DMKit提供了示例场景技能，在运行示例技能之前，需要在UNIT平台配置实现技能的理解能力：[示例场景](docs/demo_skills.md) 根据UNIT平台创建的skill id修改编译产出_build目录下的conf/app/products.json文件，在其中配置所创建skill id与对应场景DMKit配置文件。例如，查询流量及续订场景，在UNIT平台创建skill id为12345，则对应的配置文件内容应为： ```JSON { "default": { "12345": { "score": 1, "conf_path": "conf/app/demo/cellular_data.json" } } } ``` 在_build目录下运行DMKit： ```bash ./dmkit ``` 可以通过tools目录下的bot_emulator.py程序模拟与技能进行交互，使用方法为： ```bash python bot_emulator.py [skill id] [access token] ``` ### 更多文档 * [DMKit快速上手](docs/tutorial.md) * [可视化配置工具](docs/visual_tool.md) * [常见问题](docs/faq.md) ### 多语言支持 * PHP：[PHP版本官方代码库](https://github.com/baidu/dm-kit-php) ## 如何贡献 * 提交issue可以是新需求也可以是bug，也可以是对某一个问题的讨论。 * 对于issues中的问题欢迎贡献并发起pull request。 ## 讨论 * 提issue发起问题讨论，如果是问题选择类型为问题即可。 * 欢迎加入UNIT QQ群（584835350）交流讨论。 ================================================ FILE: conf/app/bot_tokens.json ================================================ { "1234": { "api_key": "", "secret_key": "" } } ================================================ FILE: conf/app/demo/book_hotel.json ================================================ [ { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "请问您要预订哪里的酒店" } ], "session": { "context": {}, "state": "001" } } ], "params": [], "trigger": { "intent": "INTENT_BOOK_HOTEL", "slots": [ "user_time", "user_room_type" ], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "{%location%}附近的酒店有{%hotel_option%}，请问你要预订哪一个？" } ], "session": { "context": {}, "state": "003" } } ], "params": [ { "name": "location", "type": "slot_val", "value": "user_location" }, { "name": "hotel_option", "type": "func_val", "value": "service_http_get:hotel_service,/hotel/search?location={%location%}" } ], "trigger": { "intent": "INTENT_BOOK_HOTEL", "slots": [ "user_time", "user_room_type", "user_location" ], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "{%location%}附近的酒店有{%hotel_option%}，请问你要预订哪一个？" } ], "session": { "context": {}, "state": "003" } } ], "params": [ { "name": "location", "type": "slot_val", "value": "user_location" }, { "name": "hotel_option", "type": "func_val", "value": "service_http_get:hotel_service,/hotel/search?location={%location%}" } ], "trigger": { "intent": "INTENT_BOOK_HOTEL", "slots": [ "user_time", "user_room_type", "user_location" ], "state": "001" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "您选择了{%time%}{%hotel%}的{%room_type%}，是否确认预订？" } ], "session": { "context": {}, "state": "002" } } ], "params": [ { "name": "time", "type": "slot_val", "value": "user_time" }, { "name": "hotel", "type": "slot_val", "value": "user_hotel" }, { "name": "room_type", "type": "slot_val", "value": "user_room_type" } ], "trigger": { "intent": "INTENT_BOOK_HOTEL", "slots": [ "user_time", "user_room_type", "user_hotel" ], "state": "001" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "您选择了{%time%}{%hotel%}的{%room_type%}，是否确认预订？" } ], "session": { "context": {}, "state": "002" } } ], "params": [ { "name": "time", "type": "slot_val", "value": "user_time" }, { "name": "hotel", "type": "slot_val", "value": "user_hotel" }, { "name": "room_type", "type": "slot_val", "value": "user_room_type" } ], "trigger": { "intent": "INTENT_BOOK_HOTEL", "slots": [ "user_time", "user_room_type", "user_hotel" ], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "已为您预订{%time%}{%hotel%}的{%room_type%}" } ], "session": { "context": {}, "state": "004" } } ], "params": [ { "name": "time", "type": "slot_val", "value": "user_time" }, { "name": "hotel", "type": "slot_val", "value": "user_hotel" }, { "name": "room_type", "type": "slot_val", "value": "user_room_type" }, { "name": "hotel_option", "type": "func_val", "value": "service_http_get:hotel_service,/hotel/book?time={%time%}&hotel={%hotel%}&room_type={%room_type%}" } ], "trigger": { "intent": "INTENT_YES", "slots": [], "state": "002" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "请告诉我您的新需求" } ], "session": { "context": {}, "state": "005" } } ], "params": [], "trigger": { "intent": "INTENT_NO", "slots": [], "state": "002" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "您选择了{%time%}{%hotel%}的{%room_type%}，是否确认预订？" } ], "session": { "context": {}, "state": "002" } } ], "params": [ { "name": "time", "type": "slot_val", "value": "user_time" }, { "name": "hotel", "type": "slot_val", "value": "user_hotel" }, { "name": "room_type", "type": "slot_val", "value": "user_room_type" } ], "trigger": { "intent": "INTENT_BOOK_HOTEL", "slots": [ "user_time", "user_room_type", "user_location", "user_hotel" ], "state": "" } } ] ================================================ FILE: conf/app/demo/book_hotel.xml ================================================ ================================================ FILE: conf/app/demo/cellular_data.json ================================================ [ { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "您想查询几月份的流量？" } ], "session": { "context": {}, "state": "001" } } ], "params": [], "trigger": { "intent": "INTENT_CHECK_DATA_USAGE", "slots": [], "state": "" } }, { "output": [ { "assertion": [ { "type": "ge", "value": "{%left%},1" } ], "result": [ { "type": "tts", "value": "您的省内流量2GB,已用流量为{%usage%}GB；剩余流量为{%left%}G，全国流量包1GB，已用流量为1GB，剩余流量为0GB" } ], "session": { "context": {}, "state": "002" } }, { "assertion": [ { "type": "gt", "value": "1,{%left%}" } ], "result": [ { "type": "tts", "value": "您的省内流量2GB，已用流量为{%usage%}GB，剩余流量为{%left%}G；全国流量包1GB，已用流量为1GB，剩余流量为0GB。发现您的流量余额已经不足了，您是否需要续订流量包呢？" } ], "session": { "context": {}, "state": "003" } } ], "params": [ { "name": "time", "type": "slot_val", "value": "user_time" }, { "name": "usage", "type": "func_val", "value": "demo_get_cellular_data_usage:{%time%}" }, { "name": "left", "type": "func_val", "value": "demo_get_cellular_data_left:{%time%}" } ], "trigger": { "intent": "INTENT_CHECK_DATA_USAGE", "slots": [ "user_time" ], "state": "" } }, { "output": [ { "assertion": [ { "type": "ge", "value": "{%left%},1" } ], "result": [ { "type": "tts", "value": "您的省内流量2GB,已用流量为{%usage%}GB；剩余流量为{%left%}G，全国流量包1GB，已用流量为1GB，剩余流量为0GB" } ], "session": { "context": {}, "state": "002" } }, { "assertion": [ { "type": "gt", "value": "1,{%left%}" } ], "result": [ { "type": "tts", "value": "您的省内流量2GB，已用流量为{%usage%}GB，剩余流量为{%left%}G；全国流量包1GB，已用流量为1GB，剩余流量为0GB。发现您的流量余额已经不足了，您是否需要续订流量包呢？" } ], "session": { "context": {}, "state": "003" } } ], "params": [ { "name": "time", "type": "slot_val", "value": "user_time" }, { "name": "usage", "type": "func_val", "value": "demo_get_cellular_data_usage:{%time%}" }, { "name": "left", "type": "func_val", "value": "demo_get_cellular_data_left:{%time%}" } ], "trigger": { "intent": "INTENT_CHECK_DATA_USAGE", "slots": [ "user_time" ], "state": "001" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "王女士您好，我们有省内，全国，夜间三种流量包，您是要续订什么流量包？" } ], "session": { "context": {}, "state": "004" } } ], "params": [], "trigger": { "intent": "INTENT_YES", "slots": [], "state": "003" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的王女士，请问还有什么可以帮助您的吗" } ], "session": { "context": {}, "state": "007" } } ], "params": [], "trigger": { "intent": "INTENT_NO", "slots": [], "state": "003" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的，{%type%}有如下选择：{%options%}。请问您想续订哪种？" } ], "session": { "context": {}, "state": "005" } } ], "params": [ { "name": "type", "type": "slot_val", "value": "user_package_type" }, { "name": "options", "type": "func_val", "value": "demo_get_package_options:{%type%}" } ], "trigger": { "intent": "INTENT_BOOK_DATA_PACKAGE", "slots": [ "user_package_type" ], "state": "004" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的王女士，您是想续订{%name%}的{%type%}吗？" } ], "session": { "context": {}, "state": "006" } } ], "params": [ { "name": "type", "type": "slot_val", "value": "user_package_type" }, { "name": "name", "type": "slot_val", "value": "user_package_name" } ], "trigger": { "intent": "INTENT_BOOK_DATA_PACKAGE", "slots": [ "user_package_type", "user_package_name" ], "state": "005" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的王女士，请问还有什么可以帮助您的吗" } ], "session": { "context": {}, "state": "007" } } ], "params": [], "trigger": { "intent": "INTENT_YES", "slots": [], "state": "006" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "王女士您好，我们有省内，全国，夜间三种流量包，您是要续订什么流量包？" } ], "session": { "context": {}, "state": "004" } } ], "params": [], "trigger": { "intent": "INTENT_BOOK_DATA_PACKAGE", "slots": [], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "王女士您好，我们有省内，全国，夜间三种流量包，您是要续订什么流量包？" } ], "session": { "context": {}, "state": "004" } } ], "params": [], "trigger": { "intent": "INTENT_BOOK_DATA_PACKAGE", "slots": [], "state": "003" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的，很高兴为您服务" } ], "session": { "context": {}, "state": "008" } } ], "params": [], "trigger": { "intent": "INTENT_NO", "slots": [], "state": "007" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "{%dmkit_param_last_tts%}" } ], "session": { "context": {}, "state": "{%state%}" } } ], "params": [ { "name": "state", "type": "session_state", "value": "" } ], "trigger": { "intent": "INTENT_REPEAT", "slots": [], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "嗯，您说？" } ], "session": { "context": {}, "state": "009" } } ], "params": [ { "name": "dmkit_param_context_state", "type": "session_state", "value": "" }, { "name": "dmkit_param_context_tts", "type": "string", "value": "{%dmkit_param_last_tts%}" } ], "trigger": { "intent": "INTENT_WAIT", "slots": [], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "{%dmkit_param_context_tts%}" } ], "session": { "context": {}, "state": "{%dmkit_param_context_state%}" } } ], "params": [], "trigger": { "intent": "INTENT_CONTINUE", "slots": [], "state": "009" } } ] ================================================ FILE: conf/app/demo/cellular_data.xml ================================================ ================================================ FILE: conf/app/demo/quota_adjust.json ================================================ [ { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": [ "您需要调整临时额度还是固定额度？", "请问您要调整固定额度还是临时额度呢？" ] } ], "session": { "context": {}, "state": "001" } } ], "params": [], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [], "state": "" } }, { "output": [ { "assertion": [ { "type": "eq", "value": "{%type%},临时" } ], "result": [ { "type": "tts", "value": "临时额度只能提升，请问您是否需要提升临时额度？" } ], "session": { "context": {}, "state": "007" } }, { "assertion": [ { "type": "eq", "value": "{%type%},固定" } ], "result": [ { "type": "tts", "value": "您需要提升额度还是降低额度？" } ], "session": { "context": {}, "state": "002" } } ], "params": [ { "name": "type", "type": "slot_val", "value": "user_type" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_type" ], "state": "" } }, { "output": [ { "assertion": [ { "type": "eq", "value": "{%type%},临时" } ], "result": [ { "type": "tts", "value": "临时额度只能提升，请问您是否需要提升临时额度？" } ], "session": { "context": {}, "state": "007" } }, { "assertion": [ { "type": "eq", "value": "{%type%},固定" } ], "result": [ { "type": "tts", "value": "您需要提升额度还是降低额度？" } ], "session": { "context": {}, "state": "002" } } ], "params": [ { "name": "type", "type": "slot_val", "value": "user_type" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_type" ], "state": "001" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的王女士，请问您要{%method%}到多少呢？" } ], "session": { "context": {}, "state": "003" } } ], "params": [ { "name": "method", "type": "slot_val", "value": "user_method" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_type", "user_method" ], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的王女士，请问您要{%method%}到多少呢？" } ], "session": { "context": {}, "state": "003" } } ], "params": [ { "name": "method", "type": "slot_val", "value": "user_method" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_type", "user_method" ], "state": "010" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的王女士，请问您要{%method%}到多少呢？" } ], "session": { "context": {}, "state": "003" } } ], "params": [ { "name": "method", "type": "slot_val", "value": "user_method" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_type", "user_method" ], "state": "002" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "您确认要将银行卡{%type%}{%method%}至{%amount%}吗？" } ], "session": { "context": {}, "state": "004" } } ], "params": [ { "name": "method", "type": "slot_val", "value": "user_method" }, { "name": "amount", "type": "slot_val", "value": "user_amount" }, { "name": "type", "type": "slot_val", "value": "user_type" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_type", "user_method", "user_amount" ], "state": "003" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "您确认要将银行卡{%type%}{%method%}至{%amount%}吗？" } ], "session": { "context": {}, "state": "004" } } ], "params": [ { "name": "method", "type": "slot_val", "value": "user_method" }, { "name": "amount", "type": "slot_val", "value": "user_amount" }, { "name": "type", "type": "slot_val", "value": "user_type" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_type", "user_method", "user_amount" ], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": [ "很高兴为您服务，还有其他需要帮助的吗？", "好的，请问还有其他能帮到您的吗？" ] } ], "session": { "context": {}, "state": "005" } } ], "params": [], "trigger": { "intent": "INTENT_YES", "slots": [], "state": "004" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": [ "很高兴为您服务，还有其他需要帮助的吗？", "好的，请问还有其他能帮到您的吗？" ] } ], "session": { "context": {}, "state": "005" } } ], "params": [], "trigger": { "intent": "INTENT_YES", "slots": [], "state": "011" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "很高兴为您服务，王女士再见！" } ], "session": { "context": {}, "state": "006" } } ], "params": [], "trigger": { "intent": "INTENT_NO", "slots": [], "state": "009" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "很高兴为您服务，王女士再见！" } ], "session": { "context": {}, "state": "006" } } ], "params": [], "trigger": { "intent": "INTENT_NO", "slots": [], "state": "005" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "好的王女士，请问您要提升到多少呢？" } ], "session": { "context": {}, "state": "008" } } ], "params": [], "trigger": { "intent": "INTENT_YES", "slots": [], "state": "007" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "不好意思，临时额度只能提升，请问还有其他需要帮助的吗？" } ], "session": { "context": {}, "state": "009" } } ], "params": [], "trigger": { "intent": "INTENT_NO", "slots": [], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "您是要{%method%}固定额度还是临时额度？" } ], "session": { "context": {}, "state": "010" } } ], "params": [ { "name": "method", "type": "slot_val", "value": "user_method" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_method" ], "state": "" } }, { "output": [ { "assertion": [], "result": [ { "type": "tts", "value": "您确认要将银行卡临时额度提升至{%amount%}吗？" } ], "session": { "context": {}, "state": "011" } } ], "params": [ { "name": "amount", "type": "slot_val", "value": "user_amount" } ], "trigger": { "intent": "INTENT_ADJUST_QUOTA", "slots": [ "user_type", "user_amount" ], "state": "008" } } ] ================================================ FILE: conf/app/demo/quota_adjust.xml ================================================ ================================================ FILE: conf/app/products.json ================================================ { "default": { "4050": { "score": 1, "conf_path": "conf/app/demo/cellular_data.json" }, "1234": { "score": 1, "conf_path": "conf/app/demo/quota_adjust.json" }, "11669": { "score": 1, "conf_path": "conf/app/demo/book_hotel.json" } } } ================================================ FILE: conf/app/remote_services.json ================================================ { "unit_bot": { "naming_service_url": "https://aip.baidubce.com", "load_balancer_name": "", "protocol": "http", "client": "brpc", "timeout_ms": 3000, "retry": 1, "headers": { "Host": "aip.baidubce.com", "Content-Type": "application/json" } }, "token_auth": { "naming_service_url": "https://aip.baidubce.com", "load_balancer_name": "", "protocol": "http", "client": "brpc", "timeout_ms": 3000, "retry": 1, "headers": { "Host": "aip.baidubce.com" } }, "hotel_service": { "naming_service_url": "http://127.0.0.1:5000", "load_balancer_name": "random", "protocol": "http", "client": "brpc", "timeout_ms": 3000, "retry": 1, "headers": { } } } ================================================ FILE: conf/gflags.conf ================================================ # TCP Port of this server --port=8010 # Only allow builtin services at this port --internal_port=8011 # Connection will be closed if there is no read/write operations during this time --idle_timeout_s=-1 # Limit of requests processing in parallel --max_concurrency=0 # Url path of the app --url_path=/search # Log to file --log_to_file=true ================================================ FILE: deps.sh ================================================ #!/usr/bin/env bash cd "$(dirname "$0")" JOBS=8 OS=$1 case "$OS" in mac) ;; ubuntu) ;; centos) ;; none) ;; *) echo "Usage: $0 [ubuntu|mac|centos|none]" exit 1 esac if [ $OS = mac ]; then echo "Installing dependencies for MacOS..." brew install openssl git gnu-getopt gflags protobuf leveldb cmake openssl elif [ $OS = ubuntu ]; then echo "Installing dependencies for Ubuntu..." sudo apt-get install -y git \ g++ \ make \ libssl-dev \ coreutils \ libgflags-dev \ libprotobuf-dev \ libprotoc-dev \ protobuf-compiler \ libleveldb-dev \ libsnappy-dev \ libcurl4-openssl-dev \ libgoogle-perftools-dev elif [ $OS = centos ]; then echo "Installing dependencies for CentOS..." sudo yum install -y epel-release sudo yum install -y git gcc-c++ make openssl-devel libcurl-devel sudo yum install -y gflags-devel protobuf-devel protobuf-compiler leveldb-devel gperftools-devel else echo "Skipping dependencies installation..." fi if [ ! -e brpc ]; then echo "Cloning brpc..." git clone https://github.com/brpc/brpc.git fi cd brpc git checkout master git pull git checkout 2ae7f04ce513c6aee27545df49d5439a98ae3a3f #build brpc echo "Building brpc..." rm -rf _build mkdir -p _build cd _build cmake .. make -j$JOBS cd ../../ ================================================ FILE: docs/demo_book_hotel_pattern.txt ================================================ INTENT_BOOK_HOTEL 0.8 [D:user_hotel]#@##0#@##1 1 INTENT_YES 0.7 [D:kw_yes]#@##0#@##1 1 INTENT_NO 0.7 [D:kw_no]#@##0#@##1 1 INTENT_BOOK_HOTEL 0.7 [D:kw_book]#@##0#@##1#@@##[D:kw_hotel]#@##0#@##1#@@##[D:user_time]#@##0#@##0#@@##[D:user_room_type]#@##0#@##0#@@##[D:user_hotel]#@##0#@##0#@@##[D:user_location]#@##0#@##0 1 ================================================ FILE: docs/demo_cellular_data_pattern.txt ================================================ INTENT_CONTINUE 0.8 [D:kw_continue]#@##0#@##1 1 INTENT_WAIT 0.8 [D:kw_wait]#@##0#@##1 1 INTENT_CHECK_DATA_USAGE 0.8 [D:user_time]#@##0#@##1 1 INTENT_REPEAT 0.7 [D:kw_repeat]#@##0#@##1 1 INTENT_BOOK_DATA_PACKAGE 0.9 [D:user_package_type]#@##0#@##1 1 INTENT_BOOK_DATA_PACKAGE 0.9 [D:kw_book]#@##0#@##1#@@##[D:kw_package]#@##0#@##0#@@##[D:user_package_type]#@##0#@##0#@@##[D:user_package_name]#@##0#@##0 1 INTENT_CHECK_DATA_USAGE 0.9 [D:kw_check]#@##0#@##1#@@##[D:user_time]#@##0#@##0#@@##[D:kw_data_usage]#@##0#@##1 1 INTENT_NO 0.9 [D:kw_no]#@##0#@##1 1 INTENT_YES 0.9 [D:kw_yes]#@##0#@##1 1 ================================================ FILE: docs/demo_quota_adjust_pattern.txt ================================================ INTENT_NO 0.9 [D:kw_no]#@##0#@##1 1 INTENT_YES 0.9 [D:kw_yes]#@##0#@##1 1 INTENT_ADJUST_QUOTA 0.9 [D:kw_adjust]#@##0#@##0#@@##[D:kw_quota]#@##0#@##0#@@##[D:user_type]#@##0#@##0#@@##[D:user_method]#@##0#@##0#@@##[D:user_amount]#@##0#@##0 1 ================================================ FILE: docs/demo_skills.md ================================================ # DMKit示例场景 ## 查询流量及续订 该场景实现一个简单的手机流量查询及续订流量包的技能。 ### 查询流量及续订 - UNIT平台配置 一共分为以下几个步骤： * 创建技能 * 配置意图 * 配置词槽 * 添加词槽的词典值 * 添加特征词列表 * 导入对话模板 #### 创建BOT 1. 进入[百度理解与交互(unit)平台](http://ai.baidu.com/unit/v2#/sceneliblist) 2. 新建一个技能给查询流量及续订的demo使用。 #### 配置意图 1. 点击进入技能，新建对话意图，以INTENT_CHECK_DATA_USAGE为例。 2. 意图名称填写INTENT_CHECK_DATA_USAGE。 3. 意图别名可以根据自己偏好填写，比如填写查询手机流量。 4. 添加词槽。INTENT_CHECK_DATA_USAGE 所需的词槽为 user_time; 5. 词槽别名可以根据自己偏好填写，比如填写查询时间，查询月份等。 6. 添加完词槽名和词槽别名以后，点击下一步，进入到选择词典环节，user_time选择系统词槽词典中的 **sys_time(时间)** 词槽。 7. 点击下一步，词槽必填选项选择非必填，点击确认。 ** 其他两个user_package_type和user_package_name的词槽使用自定义词典，将下方词槽对应的词典内容自行粘贴到txt中上传即可** 全部意图包括列表如下： * INTENT_CHECK_DATA_USAGE，所需词槽为 user_time * INTENT_BOOK_DATA_PACKAGE，所需词槽为 user_package_type，user_package_name * INTENT_YES * INTENT_NO * INTENT_REPEAT * INTENT_WAIT * INTENT_CONTINUE 词槽列表： * user_time，使用系统时间词槽 * user_package_type ```text 省内 #省内流量 全国 #全国流量 夜间 #夜间流量 ``` * user_package_name ```text 10元100M #10元100兆 #十元一百M #十元一百兆 20元300M #20元300兆 #二十元三百M #二十元三百兆 50元1G #五十元1G ``` #### 新增特征词 点击**效果优化**中的**对话模板**，选择下方的新建特征词，将名称和词典值依次填入。eg：kw_check即为名称，名称下方为词典值，直接复制粘贴即可。 特征词列表如下： * kw_check ```text 查一查 查询 ``` * kw_data_usage ```text 流量 流量情况 流量使用情况 流量使用 ``` * kw_book ```text 续定 续订 预定 预订 ``` * kw_package ```text 流量包 流量套餐 ``` * kw_yes ```text 是 好 对 想 要 是的 好的 对的 我想 我要 可以 行的 需要 没问题 ``` * kw_no ```text 不 不想 不要 不行 别 没有 没了 没 不用 不需要 没有了 ``` * kw_repeat ```text 没听清楚 再说一次 ``` * kw_wait ```text 稍等 等等 稍等一下 等一下 等一等 ``` * kw_continue ```text 你继续 您继续 继续 ``` #### 导入对话模板 * 完成以上步骤后，再进行该步骤，不然系统会报错 * 将文件[demo_cellular_data_pattern.txt](demo_cellular_data_pattern.txt)下载导入即可 ### 查询流量及续订 - DMKit配置 该场景DMKit配置为conf/app/demo/cellular_data.json文件，该文件由同目录下对应的.xml文件生成，可以将.xml文件在 中导入查看。 ## 调整银行卡额度 该场景实现一个简单的银行卡固定额度及临时额度调整的技能。 ### 调整银行卡额度 - UNIT平台配置 平台配置参考查询流量的配置，所需的配置内容见下图。 所需意图包括列表： * INTENT_ADJUST_QUOTA, 所需词槽为user_type, user_method, user_amount * INTENT_YES * INTENT_NO 词槽列表： * user_amount, 复用系统sys_num词槽 * user_type ```text 固定 #固定额度 临时 #临时额度 ``` * user_method ```text 提升 #提高 #增加 降低 #减少 #下调 ``` 特征词列表： * kw_adjust ```text 调整 调整一下 改变 ``` * kw_quota ```text 额度 银行卡额度 信用卡额度 ``` * kw_yes ```text 是 好 对 想 要 是的 好的 对的 我想 我要 可以 行的 没问题 ``` * kw_no ```text 不 不想 不要 不对 错了 不是 别 没有 没了 没 不用 没有了 ``` 对话模板： * 将文件[demo_quota_adjust_pattern.txt](demo_quota_adjust_pattern.txt)下载导入即可 ### 调整银行卡额度 - DMKit配置 该场景DMKit配置为conf/app/demo/quota_adjust.json文件，该文件由同目录下对应的.xml文件生成，可以将.xml文件在 中导入查看。 ## 预订酒店 该场景实现一个简单的预订酒店的技能。 ### 预订酒店 - UNIT平台配置 平台配置参考查询流量的配置，所需的配置内容见下图。 所需意图包括列表： * INTENT_BOOK_HOTEL, 所需词槽为user_time, user_room_type, user_location, user_hotel * INTENT_YES * INTENT_NO 词槽列表： * user_time, 复用系统sys_time词槽。需要设置为必填词槽，澄清话术配置为『请问您要预订哪一天的酒店？』 * user_room_type。需要设置为必填词槽，澄清话术配置为『请问您要预订哪个房型？』 ```text 标间 大床房 单人间 双人间 双床房 标准房 标准间 ``` * user_hotel, 复用系统sys_loc_hotel词槽 * user_location, 复用系统sys_loc词槽 特征词列表： * kw_book ```text 定 预定 订 预订 ``` * kw_hotel ```text 旅馆 酒店 ``` * kw_yes ```text 是 好 对 想 要 是的 好的 对的 我想 我要 可以 行的 没问题 确认 ``` * kw_no ```text 不 不想 不要 不对 不是 不用 不行 不可以 别 没 没有 没了 没有了 错了 ``` 对话模板： * 将文件[demo_book_hotel_pattern.txt](demo_book_hotel_pattern.txt)下载导入即可 ### 预订酒店 - DMKit配置 该场景DMKit配置为conf/app/demo/book_hotel.json文件，该文件由同目录下对应的.xml文件生成，可以将.xml文件在 中导入查看。 ================================================ FILE: docs/faq.md ================================================ # DMKit 常见问题 ## 编译brpc失败 参考BRPC官方文档 ，检查是否已安装所需依赖库。建议使用系统Ubuntu 16.04或者CentOS 7。更多BRPC相关问题请在BRPC github库提起issue。 ## 返回错误信息 DM policy resolve failed DMKit通过UNIT云端技能解析出的用户query的意图和词槽之后，需要根据对话意图结合当前对话状态在DMKit配置中对应的policy处理对话流程。当用户query解析出的意图结合当前对话状态未能找到可选的policy或者选中policy执行流程出错时，DMKit返回错误信息DM policy resolve failed。开发者需要检查：1)当前技能配置是否在products.json文件中进行注册；2）当前query解析结果意图在技能配置中是否配置了policy，详细配置说明参考[DMKit快速上手](tutorial.md)；3）检查DMKit日志查看policy执行流程是否出错，。 ## 返回错误信息 Failed to call unit bot api DMKit访问UNIT云端失败。具体原因需要查看DMKit服务日志，常见原因是请求超时。 对于请求超时的情况，先检查DMKit所在服务器网络连接云端（默认地址为 aip.baidubce.com）是否畅通，尝试修改conf/app/remote_services.json文件中unit_bot服务对应超时时间。如果连接没有问题且增大超时时间无效，则尝试切换请求client：DMKit默认使用BRPC client请求UNIT云端，目前发现偶然情况下HTTPS访问云端出现卡死而返回超时错误。DMKit支持切换为curl方式访问云端，将conf/app/remote_services.json配置中client值由brpc修改为curl即可。需要注意使用curl方式时，建议升级openssl版本不低于1.1.0，libcurl版本不低于7.32。 ## 返回错误信息 Unsupported action type satisfy 使用DMKit需要将UNIT平台中【技能设置->高级设置】中【对话回应设置】一项设置为『使用DMKit配置』。设置该选项之后，UNIT云端使用DMKit支持的数据协议。如设置为『在UNIT平台上配置』, DMKit无法识别UNIT云端数据协议，将返回错误Unsupported action type satisfy。 ## DMKit如何支持FAQ问答对 目前UNIT平台中将【对话回应】设置为【使用DMKit配置】之后，如果对话触发了平台配置的FAQ问答集，平台返回结果不会将答案赋值给response中的say字段，但是会将答案赋值给名为qid的词槽值。因此，结合DMKit配置可以从词槽qid解析出问答回复后进行返回。例如，平台创建问题意图FAQ_HELLO之后，可以在DMKit对应技能的policy配置中添加一下policy支持FAQ_HELLO问答意图下的所有问答集： ```json { "trigger": { "intent": "FAQ_HELLO", "slots": [], "state": "" }, "params": [ { "name": "answer_list", "type": "slot_val", "value": "qid" }, { "name": "faq_answer", "type": "func_val", "value": "split_and_choose:{%answer_list%},|,random" } ], "output": [ { "assertion": [], "session": { "context": {}, "state": "001" }, "result": [ { "type": "tts", "value": "{%faq_answer%}" } ] } ] } ``` ================================================ FILE: docs/tutorial.md ================================================ # DMKit 快速上手 ## 简介 在任务型对话系统（Task-Oriented Dialogue System）中，一般包括了以下几个模块： * Automatic Speech Recognition（ASR），即语音识别模块，将音频转化为文本输入。 * Natural Language Understanding（NLU），即自然语言理解模块，通过分析文本输入，解析得到对话意图与槽位（Intent + Slots）。 * Dialog Manager（DM），即对话管理模块，根据NLU模块分析得到的意图+槽位值，结合当前对话状态，执行对应的动作并返回结果。其中执行的动作可能涉及到对内部或外部知识库的查询。 * Natural Language Generation（NLG），即自然语言生成。目前一般采用模板的形式。 * Text To Speech（TTS），即文字转语音模块，将对话系统的文本输出转化为音频。 DMKit关注其中的对话管理模块（Dialog Manager），解决对话系统中状态管理、对话逻辑处理等问题。在实际应用中，单个技能下对话逻辑一般都是根据NLU结果中意图与槽位值，结合当前对话状态，确定需要进行处理的子流程。子流程或者返回固定话术结果，或者根据NLU中槽位值与对话状态访问内部或外部知识库获取资源数据并生成话术结果返回，在返回结果的同时也对对话状态进行更新。我们将这部分对话处理逻辑进行抽象，提供一个通过配置快速构建对话流程，可复用的对话管理模块，即Reusable Dialog Manager。 ## 架构 ![DMKit架构](dmkit.png) 如上图所示，系统核心是一个对话管理引擎，在对话管理引擎的基础上，每个技能的实现都是通过一个配置文件来对对话逻辑和流程进行描述，这样每个技能仅需要关注自身对话逻辑，不需要重复开发框架代码。一个技能的配置包括了一系列的policy，每个policy包括三部分：触发条件（trigger），参数变量（params），以及输出（output）。 * 触发条件（trigger）包括了NLU解析得到的意图+槽位值，以及当前的对话状态，限定了该policy被触发的条件； * 参数变量（params）是该policy运行需要的一些数据变量的定义，可以包括NLU解析结果中的槽位值、session中的变量值以及函数运行结果等。这里的函数需要在系统中进行注册，例如发送网络请求获取数据这样的函数，这些通用的函数在各个技能间都能共享，特殊情况下个别技能会需要定制化注册自己的函数； * 输出结果（output）即为该policy运行返回作为对话系统的结果，可以包括话术tts及指令，同时还可对对话状态进行更新以及写入session变量。这里的结果可以使用已定义的参数变量进行模板填充。 在技能基础配置之上，还衍生出了一系列扩展功能。例如我们对一些仅需要触发条件及输出的技能，我们可以设计更精简的语法，使用更简洁的配置描述对话policy；对于多状态跳转的场景，我们引入了可视化的编辑工具，来描述对话跳转逻辑。精简语法表示及可视化编辑都可以自动转化为对话管理引擎可执行的配置，在系统中运行。 ## 使用DMKit搭建对话技能的一般步骤 DMKit依托UNIT提供的自然语言理解能力，在此基础上搭建对话技能的一般步骤为： * 通过UNIT平台创建技能，配置技能对话流程所需的意图解析能力； * 编写技能的policy配置，policy配置文件语法见[技能配置](#技能配置)。对于对话状态状态繁多，跳转复杂的技能，可以借助[可视化配置工具](visual_tool.md)进行可视化编辑并导出技能配置。 * 将UNIT平台所创建技能的id与其对应policy配置文件注册于DMKit全局注册文件，注册文件配置项见[技能注册](#技能注册)。完成之后编译运行DMKit主程序，访问DMKit[服务接口](#服务接口)即可测试对话效果。 ## 详细配置说明 本节详细介绍实现技能功能的配置语法。所有技能的配置均位于模块源码conf/app/目录下。 ### 技能注册 products.json为全局注册配置文件，默认采用以"default"为key的配置项，该配置项中每个技能以skill id为key，注册添加技能详细配置，配置字段解释如下： | 字段 |含义 | |-----------------|-----------------------------| |conf_path |技能policy配置文件地址 | |score |技能排序静态分数，可设置为固定值1 | ### 技能配置 单个技能配置文件包括了一系列policy，每个policy字段说明如下： | 字段 | 类型 |说明 | |---------------------------------|--------------|-------------------------------| |trigger |object | 触发节点，如果一个query满足多个policy的触发条件，则优先取state匹配的policy，再根据slot取覆盖个数最多的 | |+intent |string | 触发所需NLU意图，意图由UNIT云端对话理解获得。此外，DMKit定义了以下预留意图：
dmkit_intent_fallback： 当云端返回意图在DMKit配置中未找到匹配policy时，DMKit将尝试使用该意图触发policy | |+slot |list | 触发所需槽位值列表| |+state |string | 触发所需状态值，即上一轮对话session中保存的state字段值 | |params |list | 变量列表 | |+params[].name |string | 变量名，后定义的变量可以使用已定义的变量进行模板填充，result节点中的值也可以使用变量进行模板填充。变量的使用格式为{%name%}。当name=dmkit_param_context_xxx时可以直接将该变量以xxx为key存入本轮session结果context中 | |+params[].type |string | 变量类型，可能的类型为slot_val,request_param,session_context,func_val等，详细类型列表及说明可参照[params类型及说明](#params中变量类型列表及其说明) | |+params[].value |string | 变量定义值 | |+params[].required |bool | 是否必须，如果必须的变量为空值时，该policy将不会返回结果 | |output |list | 返回结果节点，可定义多个output，最终输出会按顺序选择第一个满足assertion条件的output | |+output[].assertion |list | 使用该output的前提条件列表 | |+output[].assertion[].type |string | 条件类型，详细列表及说明可参照[assertion类型及说明](#result中assertion类型说明) | |+output[].assertion[].value |string | 条件值 | |+output[].session |object | 需要保存的session数据，用于更新对话状态及记录上下文 | |+output[].session.state |string | 更新的对话状态值 | |+output[].session.context |kvdict | 写入session的变量节点，该节点下的key+value数据会存入session，再下一轮中可以在变量定义中使用 | |+output[].result |list | 返回结果中result节点，多个result作为数组元素一起返回 | |+output[].result[].type |string | result类型 | |+output[].result[].value |string | result值 | #### params中变量类型列表及其说明： | type |说明 | |----------|--------------| | slot_val | 从qu结果中取对应的slot值，有归一化值优先取归一化值。当对应tag值存在多个slot时，value值支持tag后按分隔符","添加下标i取对应tag的第i个值（索引从0开始） | | request_param | 取请求参数对应的字段。这里的请求参数对应请求数据request.client_session字段中包含的K-V对，仅支持V类型为string的参数。例如request.client_session字段值为{"param_name1": "param_value1", "param_name2": "param_value2"}，定义type为request_param，value为"param_name1"的变量，该变量将赋值为"param_value1" | | session_context | 上一轮对话session结果中context结构体中对应的字段，例如上一轮output中session结构体保存了变量： {"context": {"param_name": "{%param_name%}", "state": ""}}， 本轮可定义变量{"name": "param_name", "type": "session_context", "value": "param_name"} | | func_val | 调用开发者定义的函数。用户定义函数位于src/user_function目录下，并需要在user_function_manager.cpp文件中进行注册。value值为","连接的参数，其中第一个元素为函数名，第二个元素开始为函数参数 | | qu_intent | NLU结果中的intent值 | | session_state | 当前对话session中的state值 | | string | 字符串值，可以使用已定义变量进行模板填充 | 特别的，开发者可以添加注册自定义函数，定义func_val类型的变量调用自定义函数实现功能扩展、定制化对话逻辑。DMKit默认内置提供了包括以下函数： | 函数名 |函数说明 | 参数 | |----------------|--------------|----------------------| | service_http_get | 通过HTTP GET的方式请求知识库、第三方API等服务，服务地址需配置于conf/app/remote_services.json中 |参数1：remote_services.json中配置的服务名
参数2：服务请求的路径，例如"/baidu/unit-dmkit" | | service_http_post | 通过HTTP POST的方式请求知识库、第三方API等服务，服务地址需配置于conf/app/remote_services.json中。注意：如果请求路径包含中文，需要先对中文进行URL编码后再拼接URL |参数1：remote_services.json中配置的服务名
参数2：服务请求的路径，例如"/baidu/unit-dmkit"
参数3：POST数据内容 | | json_get_value | 根据提供的路径从json字符串中获取对应的字段值 |参数1：json字符串
参数2：所需获取的字段在json字符串中的路径。例如{"data":{"str":"hello", "arr":[{"str": "world"}]}}中路径data.str对应字段值为"hello", 路径data.arr.0.str对应字段值"world"。| | url_encode | 对输入字符串进行url编码操作 |参数1：进行编码的字符串| 另外，DMKit默认定义提供以下变量： | 变量名 | 说明 | |---------------------------------|--------------------------------------------------------------------------| | dmkit_param_last_tts | 上一轮返回结果result中第一个type为tts的元素value值，如果不存在则为空字符串 | | dmkit_param_context_xxxxx | 上一轮session结果context中key为xxxx的值，同时如果用户定义了名为dmkit_param_context_xxxxx的变量，dmkit自动将该变量以xxxxx为key存入本轮session结果context| | dmkit_param_slot_xxxxx | qu结果中tag为xxxxx的slot值, 如果存在多个相同tag的slot，则取第一个| #### result中assertion类型说明： | type |说明 | |----------|--------------------------------------------------| | empty | value值为空 | | not_empty| value值非空 | | in | value值以","切分，第一个元素在从第二个元素开始的列表中 | | not_in | value值以","切分，第一个元素不在从第二个元素开始的列表中 | | eq | value值以","切分，第一个元素等于第二个元素 | | gt | value值以","切分，第一个数字大于第二个数字 | | ge | value值以","切分，第一个数字大于等于第二个数字 | ### 精简语法及可视化配置 * 在默认基础配置之上，有能力的开发者可以自行设计使用更简洁的配置描述对话policy并转化为基础配置进行加载。 * 对于多状态跳转的场景，可以引入了可视化的编辑工具，来描述对话跳转逻辑。这里我们提供了一个使用[mxgraph](https://github.com/jgraph/mxgraph)进行可视化配置的样例，文档参考：[可视化配置工具](visual_tool.md) ## DMKit服务接口 * DMKit服务监听端口及访问路径等参数可通过conf/gflags.conf文件进行配置，默认请求链接为http://:8010/search, 其中为DMKit服务所在机器IP，请求方式为POST * 服务接收POST数据协议与[UNIT2.0接口协议](http://ai.baidu.com/docs#/UNIT-v2-API/top)兼容。开发者按照协议组装JSON数据请求DMKit，DMKit按照该协议返回JSON数据，同时DMKit定义返回结果action_list中custom_reply类型为DM_RESULT时，返回内容为DMKit输出的output结果。 ================================================ FILE: docs/visual_tool.md ================================================ # 对话流可视化配置 针对状态繁多、跳转复杂的垂类，DMKit支持通过可视化编辑工具进行状态跳转流程的编辑设计，并同步转化为对话基础配置供对话管理引擎加载执行。 ## 基于开源可视化工具的配置样例 这里的可视化编辑工具使用开源的[mxgraph](https://github.com/jgraph/mxgraph)可视化库，对话开发者可在可视化工具上进行图编辑，而该可视化库支持从图转化为xml文件，我们再利用转换框架实现对应的编译器将xml文件转化为对话基础配置加载执行。以demo场景【查询流量及续订】为例，步骤为： * 在[draw.io](https://www.draw.io/)中按照[编辑规则](#编辑规则)进行图编辑 * 在编辑好的图导出为xml文件，放置于conf/app/demo目录下 * 运行language_compiler/run.py程序，该程序调用对应的转换器将conf/app/demo目录下的xml文件转化为json文件 * 将json文件注册于conf/app/products.json文件后，运行DMKit加载执行 ### 编辑规则 规定使用以下构图元素进行编辑： * 单箭头连线，单箭头连线是路程图中最基本的元素之一，用来表示状态的跳转。注意在使用连线的时候，连线的两端需要出现蓝色的 x 标识，以确保这个连线成功连接了两个框。 * 椭圆，用户节点，椭圆中存放的是用户的意图，以及槽位值（可选），内部语言格式为： ```text INTENT: intent_xxx SLOT:user_a,user_b ``` 该节点表示用户输入query的NLU解析结果，结合指向该节点的BOT节点，构成了DMKit基础配置中一个完成trigger条件 * 圆角矩形，BOT节点，圆角矩形中存放的是BOT的回复，内部格式为： ```text PARAM:param_type:param_name1=param_value2 PARAM:param_type:param_name1=param_value2 BOT:XXXXXXX{%param_name1%}XXX{%param_name2%} ``` 该节点表示BOT应该执行的回复，同时节点中可以定义参数并对回复进行模板填充。 在这里定义`dmkit_param_context_xxxx`变量时，dmkit自动将该变量以`xxxx`为key存入本轮session结果context。下一轮可以定义type=session_context，value=xxxx的变量来读取，也可以直接使用value={%dmkit_param_context_xxxx%}来获取，具体可参考[params类型及说明](tutorial.md#params中变量类型列表及其说明) * 菱形，条件节点，在节点中可定义需要进行判断的变量: ```text PARAM:param_type:param_name1=param_value2 PARAM:param_type:param_name1=param_value2 ``` 同时对该节点连出的单箭头连线可以添加描述跳转条件，条件可使用在菱形中定义的变量，例如： ```text ge:{%param1%},1 ``` 跳转条件描述中，可用&&或||来连接多个条件以表达“和”或“或”，例如： ```text ge:{%param1%},1 || eq: {%param1%}, 10 ``` ||和&&不可同时出现在一个条件描述中。 另外规定： * 一个椭圆仅可以连向一个圆角矩形或者一个菱形 * 一个圆角矩形可以连向多个椭圆 * 一个菱形可以连向多个圆角矩形 详细使用示例参考conf/app/demo目录下demo场景的xml文件，该xml文件可在[draw.io](https://www.draw.io/)中导入查看 ================================================ FILE: language_compiler/compiler_xml.py ================================================ # -*- coding: utf-8 -*- # # Copyright (c) 2018 Baidu, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Parser for .xml file """ import xml.etree.ElementTree as ET import re import codecs import json import sys class Node(object): """ this class is used to denote the node in .xml file, which will be used in XmlParser """ def __init__(self, node_id, node_type, node_status, node_nlg, state, params): self.node_id = node_id self.node_type = node_type self.node_status = node_status # intent for user node, empty for server node self.node_nlg = node_nlg # slots for customer node, nlg for server node self.state = state self.from_nodes = [] # (node_type, node_id, arrow_value) self.to_nodes = [] # (node_type, node_id, arrow_value) self.params = params # parameters for server node class Arrow(object): """ this class is used to denote the arrow in .xml file, which will be used in XmlParser """ def __init__(self, arrow_source, arrow_target): self.arrow_source = arrow_source # cell id self.arrow_target = arrow_target # cell id self.arrow_text = "" class XmlParser(object): """ this class is used to represent the graph in .xml file as a class object, all necessary information about the graph will be extracted from the .xml file. """ def __init__(self, xml_data): # initialize the elementtree object self.root = ET.fromstring("".join(xml_data)) # collections of nodes, arrows, policies and nlg responses. self.__nodes = {} self.__arrows = {} self.__nlgs = [] self.__user_nodes = [] self.__policies = [] # the output json # methods to parse the element tree self.__parse_cells() # parse every cell to extract info of node and arrow self.__connect_nodes() # connect nodes with the info from arrow self.__extract_policy() def __clean_noise(self, value, stri='', noise=None): if not noise: noise = [' ', ' ', 'nbsp', '
', '
', '
', '', '', ''] for n in noise: value = value.replace(n, stri) if not stri: value = value.replace('&', '&') return value.strip() def __clean_span(self, value, stri=''): return self.__clean_noise(value, '', ['', '', '']) def __parse_cells(self): todo_cells = [] state = 0 for cell in self.root[0]: style = cell.get('style') cell_id = cell.get('id') # bot node if style and style.startswith('rounded=1'): value = cell.get('value') + '<' value = self.__clean_span(value) re_nlg = re.compile(r'BOT.*?<') re_params = re.compile(r'PARAM.*?<') re_state = re.compile(r'STATE.*?<') nlgs = [] if re_nlg.findall(value): for n in re_nlg.findall(value): n = self.__clean_noise(n[4:-1]) nlgs.append(n) params = [] if re_params.findall(value): for p in re_params.findall(value): p = self.__clean_noise(p[6:-1]) params.append(p) node_state = None if re_state.findall(value): for s in re_state.findall(value): node_state = self.__clean_noise(s[6:-1]) break if 'BOT' not in value: raise Exception('wrong shape is used in cell with text: ', nlg) if value: if not node_state: state += 1 node_state = (3 - len(str(state))) * '0' + str(state) node = Node(cell_id, 'server', '', nlgs, node_state, params) self.__nodes[cell_id] = node continue # customer node if style and style.startswith('ellipse'): value = cell.get('value') + '<' value = self.__clean_span(value) re_status = re.compile(r'INTENT.*?<') status = re_status.findall(value)[0] status = self.__clean_noise(status[status.find(':') + 1:-1]) re_nlg = re.compile(r'SLOT.*?<') if re_nlg.findall(value): nlg = self.__clean_noise(re_nlg.findall(value)[0][5:-1]) else: nlg = "" if 'INTENT' not in value: raise Exception('wrong shape is used in cell with text: ', nlg) if value: node = Node(cell_id, 'customer', status, nlg, '', []) self.__nodes[cell_id] = node self.__user_nodes.append(node) continue # arrow node if cell.get('edge'): source = cell.get('source') target = cell.get('target') text = cell.get('value') if not target: raise Exception('some arrow is not pointing to anything') else: arrow = Arrow(source, target) if text: arrow.arrow_text = self.__clean_noise(text) self.__arrows[cell_id] = arrow continue # judge node if style and style.startswith('rhombus'): value = cell.get('value') + '<' value = self.__clean_span(value) re_params = re.compile(r'PARAM.*?<') params = [] if re_params.findall(value): for p in re_params.findall(value): p = self.__clean_noise(p[6:-1]) params.append(p) if 'PARAM' not in value: raise Exception('wrong shape is used in cell with text: ', value) if value: node = Node(cell_id, 'judge', '', '', '', params) self.__nodes[cell_id] = node continue todo_cells.append(cell) # false initial self.__false_initial = Node(-1, "", "", "", "", "") self.__nodes[-1] = self.__false_initial for cell in todo_cells: style = cell.get('style') parent_id = cell.get('parent') value = cell.get('value') if style and style.startswith('text'): if parent_id in self.__arrows: self.__arrows[parent_id].arrow_text = self.__clean_noise(value) else: raise Exception("there is a bad cell with text", value) def __connect_nodes(self): for (arrow_id, arrow) in self.__arrows.items(): if not arrow.arrow_source: source_node = self.__false_initial else: source_node = self.__nodes[arrow.arrow_source] target_node = self.__nodes[arrow.arrow_target] # update source node and target node source_node.to_nodes.append((target_node.node_type, target_node.node_id, arrow.arrow_text)) self.__nodes[arrow.arrow_source] = source_node target_node.from_nodes.append((source_node.node_type, source_node.node_id, arrow.arrow_text)) self.__nodes[arrow.arrow_target] = target_node def __extract_policy(self): for node in self.__user_nodes: intent = node.node_status slots = node.node_nlg.replace(' ', '').split(',') if slots[0] == "": slots = [] params = [] output = [] if len(node.to_nodes) > 0: dir_to_node_info = node.to_nodes[0] dir_to_node = self.__nodes[dir_to_node_info[1]] # points to one branch if dir_to_node.node_type == "server": for p in dir_to_node.params: p = p.replace(" ", "") c1 = p.find(":") c2 = p.find("=") pname = p[c1 + 1:c2] ptype = p[0:c1] pvalue = p[c2 + 1:] param = {"name":pname, "type":ptype, "value":pvalue} params.append(param) next = {} next["assertion"] = [] next["session"] = {"state": dir_to_node.state, "context": {}} results = [] for n in dir_to_node.node_nlg: result = {} result["type"] = "tts" alt = n.replace(" ", "").split('|') if len(alt) == 1: result["value"] = alt[0] else: result["value"] = alt results.append(result) next["result"] = results output.append(next) # points to multiple branch elif dir_to_node.node_type == "judge": # add params in judge nodes for p in dir_to_node.params: p = p.replace(" ", "") c1 = p.find(":") c2 = p.find("=") pname = p[c1 + 1:c2] ptype = p[0:c1] pvalue = p[c2 + 1:] param = {"name":pname, "type":ptype, "value":pvalue} params.append(param) # extract policies from bot nodes pointed to by the judge node for to_node_info in dir_to_node.to_nodes: to_node = self.__nodes[to_node_info[1]] nexts = [] condition = to_node_info[2].replace(" ", "") # A condition can contain either '&&'s or '||'s, but not both if '&&' in condition: conditions = condition.split('&&') assertions = [] for cond in conditions: cut = cond.find(":") assertion = {} assertion["type"] = cond[0:cut] assertion["value"] = cond[cut + 1:] assertions.append(assertion) next = {} next["assertion"] = assertions next["session"] = {"state": to_node.state, "context": {}} nexts.append(next) elif '||' in condition: conditions = condition.split('||') for cond in conditions: next = {} assertion = {} cut = cond.find(":") assertion["type"] = cond[0:cut] assertion["value"] = cond[cut + 1:] next["assertion"] = [assertion] next["session"] = {"state": to_node.state, "context": {}} nexts.append(next) else: assertion = {} cut = condition.find(":") assertion["type"] = condition[0:cut] assertion["value"] = condition[cut + 1:] next = {} next["assertion"] = [assertion] next["session"] = {"state": to_node.state, "context": {}} nexts.append(next) results = [] for n in to_node.node_nlg: result = {} result["type"] = "tts" alt = n.replace(" ", "").split('|') if len(alt) == 1: result["value"] = alt[0] else: result["value"] = alt results.append(result) for nxt in nexts: nxt["result"] = results output.append(nxt) for p in to_node.params: p = p.replace(" ", "") c1 = p.find(":") c2 = p.find("=") pname = p[c1 + 1:c2] ptype = p[0:c1] pvalue = p[c2 + 1:] param = {"name":pname, "type":ptype, "value":pvalue} # merge if param not in params: params.append(param) for from_node_info in node.from_nodes: if from_node_info: policy = {} from_node = self.__nodes[from_node_info[1]] trigger = {} trigger["intent"] = intent trigger["slots"] = slots trigger["state"] = from_node.state policy["trigger"] = trigger policy["params"] = params policy["output"] = output self.__policies.append(policy) def write_json(self): """ this method is used to return the parsed json for the .xml input """ return [json.dumps(self.__policies, ensure_ascii=False,\ indent=2, sort_keys=True).encode('utf8')] def run(data): """ runs the parser and returns the parsed json """ ps = XmlParser(data) return ps.write_json() ================================================ FILE: language_compiler/run.py ================================================ # -*- coding: utf-8 -*- # # Copyright (c) 2018 Baidu, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Compiler Executor. """ import codecs import ConfigParser import importlib import os import sys def main(): """ Compiler Executor process files in data_path and generate system json configuration file """ current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.dirname(os.path.abspath(__file__))) config = ConfigParser.ConfigParser() config.read(current_dir + "/settings.cfg") compile_types = config.get('compiler', 'compile_types') compile_types = compile_types.replace(' ', '').split(',') data_path = current_dir + '/' + config.get('data', 'data_path') for dirpath, dirnames, filenames in os.walk(data_path): for filename in filenames: filepath = os.path.join(dirpath, filename) fname, extension = os.path.splitext(filename) extension = extension[1:] if extension not in compile_types: continue input_lines = [] with open(filepath, 'r') as f: input_lines = f.readlines() compiler_module = importlib.import_module('compiler_' + extension) compiler_function = getattr(compiler_module, 'run') output_lines = compiler_function(input_lines) with open(os.path.join(dirpath, fname + ".json"), 'w') as f: for line in output_lines: f.write("%s\n" % line) if __name__ == "__main__": main() ================================================ FILE: language_compiler/settings.cfg ================================================ [data] data_path=../conf/app [compiler] compile_types=xml exclude_file= ================================================ FILE: proto/http.proto ================================================ syntax="proto2"; package dmkit; option cc_generic_services = true; message HttpRequest { }; message HttpResponse { }; service HttpService { rpc run(HttpRequest) returns (HttpResponse); }; ================================================ FILE: src/app_container.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "app_container.h" #include #include "app_log.h" #include "brpc.h" #include "dialog_manager.h" namespace dmkit { ThreadLocalDataFactory::ThreadLocalDataFactory(ApplicationBase* application) { this->_application = application; } ThreadLocalDataFactory::~ThreadLocalDataFactory() { this->_application = nullptr; } void* ThreadLocalDataFactory::CreateData() const { return this->_application->create_thread_data(); } void ThreadLocalDataFactory::DestroyData(void* d) const { this->_application->destroy_thread_data(d); } AppContainer::AppContainer() { this->_application = nullptr; this->_data_factory = nullptr; } AppContainer::~AppContainer() { delete this->_application; this->_application = nullptr; delete this->_data_factory; this->_data_factory = nullptr; } int AppContainer::load_application() { if (nullptr != this->_application) { APP_LOG(ERROR) << "an application has already be loaded"; return -1; } // The real application is created here this->_application = new DialogManager(); if (nullptr == this->_application || 0 != this->_application->init()) { APP_LOG(ERROR) << "failed to init application!!!"; return -1; } this->_data_factory = new ThreadLocalDataFactory(this->_application); return 0; } ThreadLocalDataFactory* AppContainer::get_thread_local_data_factory() { if (nullptr == this->_data_factory) { APP_LOG(ERROR) << "Data factory has not been initialized!!!"; return nullptr; } return this->_data_factory; } int AppContainer::run(BRPC_NAMESPACE::Controller* cntl) { if (nullptr == this->_application) { APP_LOG(ERROR) << "No application is not loaded for processing!!!"; return -1; } auto time_start = std::chrono::steady_clock::now(); // Need to reset thread data status before running the application ThreadDataBase* tls = static_cast(BRPC_NAMESPACE::thread_local_data()); tls->reset(); APP_LOG(TRACE) << "Running application"; int result = this->_application->run(cntl); auto time_end = std::chrono::steady_clock::now(); std::chrono::duration diff = std::chrono::duration_cast>(time_end - time_start); double total_cost = diff.count() * 1000; APP_LOG(TRACE) << "Application run cost(ms): " << total_cost; tls->add_notice_log("tm", std::to_string(total_cost)); APP_LOG(NOTICE) << tls->get_notice_log(); return result; } } // namespace dmkit ================================================ FILE: src/app_container.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_APP_CONTAINER_H #define DMKIT_APP_CONTAINER_H #include "application_base.h" #include "brpc.h" namespace dmkit { class ThreadLocalDataFactory : public BRPC_NAMESPACE::DataFactory { public: ThreadLocalDataFactory(ApplicationBase* application); ~ThreadLocalDataFactory(); void* CreateData() const; void DestroyData(void* d) const; private: ApplicationBase* _application; }; // Container class which manages the instances of application, // as well as a thread data factory instance. class AppContainer { public: AppContainer(); ~AppContainer(); int load_application(); ThreadLocalDataFactory* get_thread_local_data_factory(); int run(BRPC_NAMESPACE::Controller* cntl); private: // The application instance is shared for all rpc threads ApplicationBase* _application; ThreadLocalDataFactory* _data_factory; }; } // namespace dmkit #endif //DMKIT_APP_CONTAINER_H ================================================ FILE: src/app_log.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_APP_LOG_H #define DMKIT_APP_LOG_H #include "brpc.h" #include "butil.h" #include "thread_data_base.h" namespace dmkit { // Wrapper for application logging to include trace id for each log during a request. #define APP_LOG(severity) \ LOG(severity) << "logid=" << (BRPC_NAMESPACE::thread_local_data() == nullptr ? "" : \ (static_cast(BRPC_NAMESPACE::thread_local_data()))->get_log_id()) \ << " " } // namespace dmkit #endif //DMKIT_APP_LOG_H ================================================ FILE: src/application_base.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_APPLICATION_BASE_H #define DMKIT_APPLICATION_BASE_H #include "brpc.h" #include "thread_data_base.h" namespace dmkit { // Base class for applications // Notice that the same application instance will be used for all request thread, // This call should be thread safe. class ApplicationBase { public: ApplicationBase() {}; virtual ~ApplicationBase() {}; // Interface for application to do global initialization. // This method will be invoke only once when server starts. virtual int init() = 0; // Interface for application to handle requests, it should be thread safe. virtual int run(BRPC_NAMESPACE::Controller* cntl) = 0; // Interface for application to register customized thread data. virtual void* create_thread_data() const { return new ThreadDataBase(); } // Interface to destroy thread data, the data instance was created by create_thread_data. virtual void destroy_thread_data(void* d) const { delete static_cast(d); } // Set log id for current request, // application should set log id as early as possible when processing request virtual void set_log_id(const std::string& log_id) { ThreadDataBase* tls = static_cast(BRPC_NAMESPACE::thread_local_data()); tls->set_log_id(log_id); } // Add a key/value notice log which will be logged when finish processing request virtual void add_notice_log(const std::string& key, const std::string& value) { ThreadDataBase* tls = static_cast(BRPC_NAMESPACE::thread_local_data()); tls->add_notice_log(key, value); } }; } // namespace dmkit #endif //DMKIT_APPLICATION_BASE_H ================================================ FILE: src/brpc.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_BRPC_H #define DMKIT_BRPC_H #ifndef BRPC_INCLUDE_PREFIX #define BRPC_INCLUDE_PREFIX #include BRPC_INCLUDE_PREFIX/channel.h> #include BRPC_INCLUDE_PREFIX/controller.h> #include BRPC_INCLUDE_PREFIX/restful.h> #include BRPC_INCLUDE_PREFIX/server.h> #endif //DMKIT_BRPC_H ================================================ FILE: src/butil.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_BUTIL_H #define DMKIT_BUTIL_H #ifndef BUTIL_INCLUDE_PREFIX #define BUTIL_INCLUDE_PREFIX #ifdef BUTIL_ENABLE_COMLOG_SINK #include BUTIL_INCLUDE_PREFIX/comlog_sink.h> #endif #include BUTIL_INCLUDE_PREFIX/containers/flat_map.h> #include BUTIL_INCLUDE_PREFIX/logging.h> #endif //DMKIT_BUTIL_H ================================================ FILE: src/dialog_manager.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "dialog_manager.h" #include #include #include "app_log.h" #include "butil.h" #include "rapidjson.h" #include "request_context.h" #include "utils.h" namespace dmkit { DialogManager::DialogManager() { this->_remote_service_manager = new RemoteServiceManager(); this->_policy_manager = new PolicyManager(); this->_token_manager = new TokenManager(); } DialogManager::~DialogManager() { delete this->_policy_manager; this->_policy_manager = nullptr; delete this->_remote_service_manager; this->_remote_service_manager = nullptr; delete this->_token_manager; this->_token_manager = nullptr; } int DialogManager::init() { if (0 != this->_remote_service_manager->init("conf/app", "remote_services.json")) { APP_LOG(ERROR) << "Failed to init _remote_service_manager"; return -1; } APP_LOG(TRACE) << "_remote_service_manager init done"; if (0 != this->_policy_manager->init("conf/app", "products.json")) { APP_LOG(ERROR) << "Failed to init _policy_manager"; return -1; } if (0 != this->_token_manager->init("conf/app", "bot_tokens.json")) { APP_LOG(ERROR) << "Failed to init _token_manager"; return -1; } APP_LOG(TRACE) << "_policy_manager init done"; return 0; } int DialogManager::run(BRPC_NAMESPACE::Controller* cntl) { std::string request_json = cntl->request_attachment().to_string(); APP_LOG(TRACE) << "received request: " << request_json; this->add_notice_log("req", request_json); rapidjson::Document request_doc; // In the case we cannot parse the request json, it is not a valid request. if (request_doc.Parse(request_json.c_str()).HasParseError() || !request_doc.IsObject()) { APP_LOG(WARNING) << "Failed to parse request data to json"; cntl->http_response().set_status_code(400); return 0; } // Need to set log_id as soon as we can get it to avoid missing log_id in logs. if (!request_doc.HasMember("log_id") || !request_doc["log_id"].IsString()) { APP_LOG(WARNING) << "Missing log_id"; this->send_json_response(cntl, this->get_error_response(-1, "Missing log_id")); return 0; } std::string log_id = request_doc["log_id"].GetString(); std::string dmkit_log_id_prefix = "dmkit_"; log_id = dmkit_log_id_prefix + log_id; this->set_log_id(log_id); request_doc["log_id"].SetString(log_id.c_str(), log_id.length(), request_doc.GetAllocator()); // Parsing bot_id from request json if (!request_doc.HasMember("bot_id") || !request_doc["bot_id"].IsString()) { APP_LOG(WARNING) << "Missing bot_id"; this->send_json_response(cntl, this->get_error_response(-1, "Missing bot_id")); return 0; } std::string bot_id = request_doc["bot_id"].GetString(); if (!request_doc.HasMember("request") || !request_doc["request"].HasMember("query") || !request_doc["request"]["query"].IsString()) { APP_LOG(WARNING) << "Missing query"; this->send_json_response(cntl, this->get_error_response(-1, "Missing query")); return 0; } std::string query = request_doc["request"]["query"].GetString(); std::string rewrite_query; if (request_doc["request"].HasMember("rewrite_query") && request_doc["request"]["rewrite_query"].IsString()) { rewrite_query = request_doc["request"]["rewrite_query"].GetString(); request_doc["request"].RemoveMember("rewrite_query"); } // Get dmkit session from request. We saved it in the bot session in latest response. std::string dm_session; if (request_doc.HasMember("bot_session")) { std::string request_bot_session = request_doc["bot_session"].GetString(); rapidjson::Document request_bot_session_doc; if (request_bot_session.empty() || request_bot_session_doc.Parse(request_bot_session.c_str()).HasParseError() || !request_bot_session_doc.IsObject() || !request_bot_session_doc.HasMember("bot_id") || !request_bot_session_doc["bot_id"].IsString() || request_bot_session_doc["bot_id"].GetString() != bot_id || !request_bot_session_doc.HasMember("session_id") || !request_bot_session_doc.HasMember("dialog_state") || !request_bot_session_doc["dialog_state"].HasMember("contexts") || !request_bot_session_doc["dialog_state"]["contexts"].HasMember("dmkit") || !request_bot_session_doc["dialog_state"]["contexts"]["dmkit"].HasMember("session")) { // Not a valid session from DMKit request_doc["bot_session"].SetString("", 0, request_doc.GetAllocator()); } else { dm_session = request_bot_session_doc["dialog_state"]["contexts"]["dmkit"]["session"].GetString(); } } APP_LOG(TRACE) << "dm session: " << dm_session; PolicyOutputSession session = PolicyOutputSession::from_json_str(dm_session); // Get access_token from request uri. std::string access_token; const std::string* access_token_ptr = cntl->http_request().uri().GetQuery("access_token"); if (access_token_ptr != nullptr) { access_token = *access_token_ptr; } if (access_token.empty() && this->_token_manager->get_access_token( bot_id, this->_remote_service_manager, access_token) != 0) { APP_LOG(ERROR) << "Failed to get access token"; this->send_json_response(cntl, this->get_error_response(-1, "Failed to get access token")); return 0; } std::string query_response; bool is_dmkit_response = false; this->process_request(request_doc, dm_session, access_token, query_response, is_dmkit_response); if (is_dmkit_response || rewrite_query.empty()) { this->send_json_response(cntl, query_response); return 0; } std::string rewrite_query_response; request_doc["request"]["query"].SetString(rewrite_query.c_str(), rewrite_query.length(), request_doc.GetAllocator()); this->process_request(request_doc, dm_session, access_token, rewrite_query_response, is_dmkit_response); if (is_dmkit_response) { this->send_json_response(cntl, rewrite_query_response); return 0; } this->send_json_response(cntl, query_response); return 0; } int DialogManager::process_request(const rapidjson::Document& request_doc, const std::string& dm_session, const std::string& access_token, std::string& json_response, bool& is_dmkit_response) { std::string bot_id = request_doc["bot_id"].GetString(); std::string log_id = request_doc["log_id"].GetString(); std::string query = request_doc["request"]["query"].GetString(); is_dmkit_response = false; std::string request_json = utils::json_to_string(request_doc); // Call unit bot api with the request json as dmkit use the same data contract. std::string unit_bot_result; if (this->call_unit_bot(access_token, request_json, unit_bot_result) != 0) { APP_LOG(ERROR) << "Failed to call unit bot api"; json_response = get_error_response(-1, "Failed to call unit bot api"); return 0; } APP_LOG(TRACE) << "unit bot result: " << unit_bot_result; // Parse unit bot response. // In the case something wrong with unit bot response, informs users. rapidjson::Document unit_response_doc; if (unit_response_doc.Parse(unit_bot_result.c_str()).HasParseError() || !unit_response_doc.IsObject()) { APP_LOG(ERROR) << "Failed to parse unit bot result: " << unit_bot_result; json_response = get_error_response(-1, "Failed to parse unit bot result"); return -1; } if (!unit_response_doc.HasMember("error_code") || !unit_response_doc["error_code"].IsInt() || unit_response_doc["error_code"].GetInt() != 0) { json_response = unit_bot_result; return 0; } // The bot status is included in bot_session std::string bot_session = unit_response_doc["result"]["bot_session"].GetString(); rapidjson::Document bot_session_doc; if (bot_session_doc.Parse(bot_session.c_str()).HasParseError() || !bot_session_doc.IsObject()) { APP_LOG(ERROR) << "Failed to parse bot session: " << bot_session; json_response = get_error_response(-1, "Failed to parse bot session"); return 0; } if (this->handle_unsatisfied_intent(unit_response_doc, bot_session_doc, dm_session, json_response) == 0) { return 0; } // Handle satify/understood intents QuResult* qu_result = QuResult::parse_from_dialog_state( bot_id, bot_session_doc["dialog_state"]); if (qu_result == nullptr) { json_response = this->get_error_response(-1, "Failed to parse qu_result"); return 0; } auto qu_map = new BUTIL_NAMESPACE::FlatMap(); // 2: bucket_count, initial count of buckets, big enough to avoid resize. // 50: load_factor, element_count * 100 / bucket_count. qu_map->init(2, 50); qu_map->insert(bot_id, qu_result); // Parsing request params from client_session, only string value is accepted std::unordered_map request_params; std::string product = "default"; if (request_doc["request"].HasMember("client_session")) { std::string client_session = request_doc["request"]["client_session"].GetString(); rapidjson::Document client_session_doc; if (!client_session_doc.Parse(client_session.c_str()).HasParseError() && client_session_doc.IsObject()) { for (auto& m_param: client_session_doc.GetObject()) { if (!m_param.value.IsString()) { continue; } std::string param_name = m_param.name.GetString(); std::string param_value = m_param.value.GetString(); if (param_name == "product") { product = param_value; } request_params[param_name] = param_value; } } } PolicyOutputSession session = PolicyOutputSession::from_json_str(dm_session); RequestContext context(this->_remote_service_manager, log_id, request_params); PolicyOutput* policy_output = this->_policy_manager->resolve( product, qu_map, session, context); for (auto iter = qu_map->begin(); iter != qu_map->end(); ++iter) { QuResult* qu_ptr = iter->second; if (qu_ptr != nullptr) { delete qu_ptr; } } delete qu_map; if (policy_output == nullptr) { json_response = this->get_error_response(-1, "DM policy resolve failed"); return 0; } bool has_query = false; for (auto const& meta: policy_output->meta) { if (meta.key == "query") { has_query = true; } } if (!has_query) { KVPair meta_query; meta_query.key = "query"; meta_query.value = query; policy_output->meta.push_back(meta_query); } this->set_dm_response(unit_response_doc, bot_session_doc, policy_output); delete policy_output; is_dmkit_response = true; json_response = utils::json_to_string(unit_response_doc); return 0; } int DialogManager::call_unit_bot(const std::string& access_token, const std::string& payload, std::string& result) { std::string url = "/rpc/2.0/unit/bot/chat?access_token="; url += access_token; RemoteServiceParam rsp = { url, HTTP_METHOD_POST, payload }; RemoteServiceResult rsr; APP_LOG(TRACE) << "Calling unit bot service, url: "<< url; APP_LOG(TRACE) << payload; // unit_bot is a remote service configured in conf/app/remote_services.json if (this->_remote_service_manager->call("unit_bot", rsp, rsr) !=0) { APP_LOG(ERROR) << "Failed to get unit bot result" ; return -1; } APP_LOG(TRACE) << "Got unit bot result"; result = rsr.result; return 0; } int DialogManager::handle_unsatisfied_intent(rapidjson::Document& unit_response_doc, rapidjson::Document& bot_session_doc, const std::string& dm_session, std::string& response) { std::string action_type; if (unit_response_doc.HasMember("result") && unit_response_doc["result"].HasMember("response") && unit_response_doc["result"]["response"].HasMember("action_list") && unit_response_doc["result"]["response"]["action_list"].Size() > 0 && unit_response_doc["result"]["response"]["action_list"][0].HasMember("type")) { action_type = unit_response_doc["result"]["response"]["action_list"][0]["type"].GetString(); } else { APP_LOG(WARNING) << "Failed to parse action type from unit bot response: " << utils::json_to_string(unit_response_doc); } if (action_type == "satisfy") { response = this->get_error_response(-1, "Unsupported action type satisfy"); return 0; } if (action_type != "understood") { // DM session should be saved rapidjson::Value dm_session_json; dm_session_json.SetString( dm_session.c_str(), dm_session.length(), bot_session_doc.GetAllocator()); if (!bot_session_doc.HasMember("dialog_state")) { rapidjson::Value dialog_state_json(rapidjson::kObjectType); bot_session_doc.AddMember("dialog_state", dialog_state_json, bot_session_doc.GetAllocator()); } if (!bot_session_doc["dialog_state"].HasMember("contexts")) { rapidjson::Value contexts_json(rapidjson::kObjectType); bot_session_doc["dialog_state"].AddMember("contexts", contexts_json, bot_session_doc.GetAllocator()); } if (!bot_session_doc["dialog_state"]["contexts"].HasMember("dmkit")) { rapidjson::Value contexts_json(rapidjson::kObjectType); bot_session_doc["dialog_state"]["contexts"].AddMember("dmkit", contexts_json, bot_session_doc.GetAllocator()); } if (bot_session_doc["dialog_state"]["contexts"]["dmkit"].HasMember("session")) { bot_session_doc["dialog_state"]["contexts"]["dmkit"].RemoveMember("session"); } bot_session_doc["dialog_state"]["contexts"]["dmkit"].AddMember( "session", dm_session_json, bot_session_doc.GetAllocator()); std::string bot_session = utils::json_to_string(bot_session_doc); //unit_response_doc.AddMember("debug", bot_session_doc, unit_response_doc.GetAllocator()); unit_response_doc["result"]["bot_session"].SetString( bot_session.c_str(), bot_session.length(), unit_response_doc.GetAllocator()); response = utils::json_to_string(unit_response_doc); return 0; } return -1; } void DialogManager::set_dm_response(rapidjson::Document& unit_response_doc, rapidjson::Document& bot_session_doc, const PolicyOutput* policy_output) { std::string session_str = PolicyOutputSession::to_json_str(policy_output->session); rapidjson::Value dm_session; dm_session.SetString(session_str.c_str(), session_str.length(), bot_session_doc.GetAllocator()); if (!bot_session_doc.HasMember("dialog_state")) { rapidjson::Value dialog_state_json(rapidjson::kObjectType); bot_session_doc.AddMember("dialog_state", dialog_state_json, bot_session_doc.GetAllocator()); } if (!bot_session_doc["dialog_state"].HasMember("contexts")) { rapidjson::Value contexts_json(rapidjson::kObjectType); bot_session_doc["dialog_state"].AddMember("contexts", contexts_json, bot_session_doc.GetAllocator()); } if (!bot_session_doc["dialog_state"]["contexts"].HasMember("dmkit")) { rapidjson::Value contexts_json(rapidjson::kObjectType); bot_session_doc["dialog_state"]["contexts"].AddMember("dmkit", contexts_json, bot_session_doc.GetAllocator()); } if (bot_session_doc["dialog_state"]["contexts"]["dmkit"].HasMember("session")) { bot_session_doc["dialog_state"]["contexts"]["dmkit"].RemoveMember("session"); } bot_session_doc["dialog_state"]["contexts"]["dmkit"].AddMember( "session", dm_session, bot_session_doc.GetAllocator()); // DMKit result as a custom reply rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); writer.StartObject(); writer.Key("event_name"); writer.String("DM_RESULT"); writer.Key("result"); std::string policy_output_str = PolicyOutput::to_json_str(*policy_output); writer.String(policy_output_str.c_str(), policy_output_str.length()); writer.EndObject(); std::string custom_reply = buffer.GetString(); APP_LOG(TRACE) << "custom_reply: " << custom_reply; bot_session_doc["interactions"][0]["response"]["action_list"][0]["type"] = "event"; bot_session_doc["interactions"][0]["response"]["action_list"][0]["say"] = ""; bot_session_doc["interactions"][0]["response"]["action_list"][0]["custom_reply"].SetString( custom_reply.c_str(), custom_reply.length(), unit_response_doc.GetAllocator()); unit_response_doc["result"]["response"]["action_list"][0]["type"] = "event"; unit_response_doc["result"]["response"]["action_list"][0]["say"] = ""; unit_response_doc["result"]["response"]["action_list"][0]["custom_reply"].SetString( custom_reply.c_str(), custom_reply.length(), unit_response_doc.GetAllocator()); std::string bot_session = utils::json_to_string(bot_session_doc); //unit_response_doc.AddMember("debug", bot_session_doc, unit_response_doc.GetAllocator()); unit_response_doc["result"]["bot_session"].SetString( bot_session.c_str(), bot_session.length(), unit_response_doc.GetAllocator()); } std::string DialogManager::get_error_response(int error_code, const std::string& error_msg) { rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); writer.StartObject(); writer.Key("error_code"); writer.Int(error_code); writer.Key("error_msg"); writer.String(error_msg.c_str(), error_msg.length()); writer.EndObject(); return buffer.GetString(); } void DialogManager::send_json_response(BRPC_NAMESPACE::Controller* cntl, const std::string& data) { cntl->http_response().set_content_type("application/json;charset=UTF-8"); this->add_notice_log("ret", data); cntl->response_attachment().append(data); } } // namespace dmkit ================================================ FILE: src/dialog_manager.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "application_base.h" #include "policy.h" #include "policy_manager.h" #include "qu_result.h" #include "rapidjson.h" #include "remote_service_manager.h" #include "token_manager.h" #ifndef DMKIT_DIALOG_MANAGER_H #define DMKIT_DIALOG_MANAGER_H namespace dmkit { // The Dialog Manager application class DialogManager : public ApplicationBase { public: DialogManager(); virtual ~DialogManager(); virtual int init(); virtual int run(BRPC_NAMESPACE::Controller* cntl); private: int process_request(const rapidjson::Document& request_doc, const std::string& dm_session, const std::string& access_token, std::string& json_response, bool& is_dmkit_response); int call_unit_bot(const std::string& access_token, const std::string& payload, std::string& result); int handle_unsatisfied_intent(rapidjson::Document& unit_response_doc, rapidjson::Document& bot_session_doc, const std::string& dm_session, std::string& response); std::string get_error_response(int error_code, const std::string& error_msg); void send_json_response(BRPC_NAMESPACE::Controller* cntl, const std::string& data); void set_dm_response(rapidjson::Document& unit_response_doc, rapidjson::Document& bot_session_doc, const PolicyOutput* policy_output); RemoteServiceManager* _remote_service_manager; PolicyManager* _policy_manager; TokenManager* _token_manager; }; } // namespace dmkit #endif //DMKIT_DIALOG_MANAGER_H ================================================ FILE: src/file_watcher.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "file_watcher.h" #include #include #include "app_log.h" #include "utils.h" namespace dmkit { const int FileWatcher::CHECK_INTERVAL_IN_MILLS; FileWatcher& FileWatcher::get_instance() { static FileWatcher instance; return instance; } FileWatcher::FileWatcher() { } FileWatcher::~FileWatcher() { this->_is_running = false; if (this->_watcher_thread.joinable()) { this->_watcher_thread.join(); } } static int get_file_last_modified_time(const std::string& file_path, std::string& mtime_str) { struct stat f_stat; if (stat(file_path.c_str(), &f_stat) != 0) { LOG(WARNING) << "Failed to get file modified time" << file_path; return -1; } mtime_str = ctime(&f_stat.st_mtime); return 0; } int FileWatcher::register_file(const std::string file_path, FileChangeCallback cb, void* param, bool level_trigger) { LOG(TRACE) << "FileWatcher registering file " << file_path; std::string last_modified_time; if (get_file_last_modified_time(file_path, last_modified_time) != 0) { return -1; } FileStatus file_status = {file_path, last_modified_time, cb, param, level_trigger}; std::lock_guard lock(this->_mutex); this->_file_info[file_path] = file_status; if (!this->_is_running) { this->_is_running = true; this->_watcher_thread = std::thread(&FileWatcher::watcher_thread_func, this); } return 0; } int FileWatcher::unregister_file(const std::string file_path) { LOG(TRACE) << "FileWatcher unregistering file " << file_path; std::lock_guard lock(this->_mutex); if (this->_file_info.erase(file_path) != 1) { return -1; } if (this->_file_info.size() == 0 && this->_is_running && this->_watcher_thread.joinable()) { this->_is_running = false; this->_watcher_thread.join(); } return 0; } void FileWatcher::watcher_thread_func() { LOG(TRACE) << "Watcher thread starting..."; while (this->_is_running) { { std::lock_guard lock(this->_mutex); for (const auto& file : this->_file_info) { std::string last_modified_time; get_file_last_modified_time(file.first, last_modified_time); if (last_modified_time != file.second.last_modified_time) { LOG(TRACE) << "File Changed. " << file.first << " modified time " << last_modified_time; if (file.second.callback(file.second.param) == 0 || !file.second.level_trigger) { if (this->_file_info.find(file.first) != this->_file_info.end()) { this->_file_info[file.first].last_modified_time = last_modified_time; } } } } } std::this_thread::sleep_for(std::chrono::milliseconds(FileWatcher::CHECK_INTERVAL_IN_MILLS)); } LOG(TRACE) << "Watcher thread stopping..."; } } // namespace dmkit ================================================ FILE: src/file_watcher.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_FILE_WATCHER_H #define DMKIT_FILE_WATCHER_H #include #include #include #include #include namespace dmkit { // Callback function when file changed typedef int (*FileChangeCallback)(void* param); struct FileStatus { std::string file_path; std::string last_modified_time; FileChangeCallback callback; void* param; bool level_trigger; }; // A file watcher singleton implemention class FileWatcher { public: static FileWatcher& get_instance(); int register_file(const std::string file_path, FileChangeCallback cb, void* param, bool level_trigger=false); int unregister_file(const std::string file_path); // Do not need copy constructor and assignment operator for a singleton class FileWatcher(FileWatcher const&) = delete; void operator=(FileWatcher const&) = delete; private: FileWatcher(); virtual ~FileWatcher(); void watcher_thread_func(); std::mutex _mutex; std::atomic _is_running; std::thread _watcher_thread; std::unordered_map _file_info; static const int CHECK_INTERVAL_IN_MILLS = 1000; }; } // namespace dmkit #endif //DMKIT_FILE_WATCHER_H ================================================ FILE: src/policy.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "policy.h" #include "app_log.h" #include "utils.h" namespace dmkit { Policy::Policy(const PolicyTrigger& trigger, const std::vector& params, const std::vector& outputs) : _trigger(trigger), _params(params), _outputs(outputs) { } const PolicyTrigger& Policy::trigger() const { return this->_trigger; } const std::vector& Policy::params() const { return this->_params; } const std::vector& Policy::outputs() const { return this->_outputs; } // Parse a policy from json configuration. // A sample policy is as following: // { // "trigger": { // "intent": "INTENT_XXX", // "slots": [ // "slot_1", // "slot_2" // ], // "state": "001" // }, // "params": [ // { // "name": "param_name", // "type": "slot_val", // "value": "slot_tag" // } // ], // "output": [ // { // "assertion": [ // { // "type": "eq", // "value": "1,{%param_name%}" // } // ], // "session": { // "state": "002" // "context": { // "key1": "value1", // "key2": "value2" // } // }, // "meta":{ // "key1": "value1", // "key2": "value2" // }, // "result": [ // { // "type":"tts", // "value": "Hello World" // } // ] // } // ] // } Policy* Policy::parse_from_json_value(const rapidjson::Value& value) { if (!value.IsObject()) { LOG(WARNING) << "Failed to parse policy from json, not an object: " << utils::json_to_string(value); return nullptr; } if (!value.HasMember("trigger") || !value["trigger"].IsObject() || !value["trigger"].HasMember("intent") || !value["trigger"]["intent"].IsString()) { LOG(WARNING) << "Failed to parse policy from json, invalid trigger"; return nullptr; } PolicyTrigger trigger; trigger.intent = value["trigger"]["intent"].GetString(); if (value["trigger"].HasMember("slots") && value["trigger"]["slots"].IsArray()) { for (auto& v : value["trigger"]["slots"].GetArray()) { trigger.slots.push_back(v.GetString()); } } if (value["trigger"].HasMember("state") && value["trigger"]["state"].IsString()) { trigger.state = value["trigger"]["state"].GetString(); } std::vector params; if (value.HasMember("params") && value["params"].IsArray()) { for (auto& v : value["params"].GetArray()) { PolicyParam param; param.name = v["name"].GetString(); param.type = v["type"].GetString(); param.value = v["value"].GetString(); if (v.HasMember("required") && v["required"].IsBool()) { param.required = v["required"].GetBool(); } else { param.required = false; } if (v.HasMember("default") && v["default"].IsString()) { param.default_value = v["default"].GetString(); } params.push_back(param); } } std::vector outputs; if (value.HasMember("output") && value["output"].IsArray()) { for (auto& v : value["output"].GetArray()) { PolicyOutput output; if (v.HasMember("assertion") && v["assertion"].IsArray()) { for (auto& v_assertion: v["assertion"].GetArray()) { std::string assertion_type = v_assertion["type"].GetString(); std::string assertion_value = v_assertion["value"].GetString(); PolicyOutputAssertion assertion = {assertion_type, assertion_value}; output.assertions.push_back(assertion); } } if (v.HasMember("session") && v["session"].IsObject()) { if (v["session"].HasMember("state") && v["session"]["state"].IsString()) { output.session.state = v["session"]["state"].GetString(); } if (v["session"].HasMember("context") && v["session"]["context"].IsObject()) { for (auto& m_context: v["session"]["context"].GetObject()) { std::string context_key = m_context.name.GetString(); std::string context_value = m_context.value.GetString(); KVPair context = {context_key, context_value}; output.session.context.push_back(context); } } } if (v.HasMember("meta") && v["meta"].IsObject()) { for (auto& m_meta: v["meta"].GetObject()) { std::string meta_key = m_meta.name.GetString(); std::string meta_value = m_meta.value.GetString(); KVPair meta = {meta_key, meta_value}; output.meta.push_back(meta); } } for (auto& v_result : v["result"].GetArray()) { PolicyOutputResult result; result.type = v_result["type"].GetString(); if (v_result["value"].IsString()) { result.values.push_back(v_result["value"].GetString()); } else if (v_result["value"].IsArray()) { for (auto& v_result_value : v_result["value"].GetArray()) { result.values.push_back(v_result_value.GetString()); } } if (v_result.HasMember("extra") && v_result["extra"].IsString()) { result.extra = v_result["extra"].GetString(); } output.results.push_back(result); } outputs.push_back(output); } } return new Policy(trigger, params, outputs);; } std::string PolicyOutputSession::to_json_str(const PolicyOutputSession& session) { rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); writer.StartObject(); writer.Key("domain"); writer.String(session.domain.c_str(), session.domain.length()); writer.Key("state"); writer.String(session.state.c_str(), session.state.length()); writer.Key("context"); writer.StartObject(); for (auto const& object: session.context) { writer.Key(object.key.c_str(), object.key.length()); writer.String(object.value.c_str(), object.value.length()); } writer.EndObject(); writer.EndObject(); return buffer.GetString(); } PolicyOutputSession PolicyOutputSession::from_json_str(const std::string& json_str) { PolicyOutputSession session; rapidjson::Document session_doc; if (session_doc.Parse(json_str.c_str()).HasParseError() || !session_doc.IsObject()) { return session; } if (session_doc.HasMember("domain")) { session.domain = session_doc["domain"].GetString(); } if (session_doc.HasMember("state")) { session.state = session_doc["state"].GetString(); } if (!session_doc.HasMember("context")) { return session; } for (auto& m_object: session_doc["context"].GetObject()) { std::string context_key = m_object.name.GetString(); std::string context_value = m_object.value.GetString(); KVPair context = {context_key, context_value}; session.context.push_back(context); } return session; } std::string PolicyOutput::to_json_str(const PolicyOutput& output) { rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); writer.StartObject(); writer.Key("meta"); writer.StartObject(); for (auto const& meta: output.meta) { writer.Key(meta.key.c_str(), meta.key.length()); writer.String(meta.value.c_str(), meta.value.length()); } writer.EndObject(); writer.Key("result"); writer.StartArray(); for (auto const& result: output.results) { writer.StartObject(); writer.Key("type"); writer.String(result.type.c_str(), result.type.length()); writer.Key("value"); writer.String(result.values[0].c_str(), result.values[0].length()); if (!result.extra.empty()) { rapidjson::Document extra_doc; if (!extra_doc.Parse(result.extra.c_str()).HasParseError() && extra_doc.IsObject()) { for (auto& v_extra: extra_doc.GetObject()) { std::string extra_key = v_extra.name.GetString(); if (extra_key == "type" || extra_key == "value") { LOG(WARNING) << "Unsupported extra key " << extra_key; } if (v_extra.value.IsString()) { std::string extra_value = v_extra.value.GetString(); writer.Key(extra_key.c_str()); writer.String(extra_value.c_str(), extra_value.length()); } else if (v_extra.value.IsBool()) { writer.Key(extra_key.c_str()); writer.Bool(v_extra.value.GetBool()); } else if (v_extra.value.IsInt()) { writer.Key(extra_key.c_str()); writer.Int(v_extra.value.GetInt()); } else if (v_extra.value.IsDouble()) { writer.Key(extra_key.c_str()); writer.Double(v_extra.value.GetDouble()); } else { LOG(WARNING) << "Unknown extra value type " << v_extra.value.GetType(); } } } else { LOG(WARNING) << "Failed to parse result extra json: " << result.extra; } } writer.EndObject(); } writer.EndArray(); writer.EndObject(); return buffer.GetString(); } } // namespace dmkit ================================================ FILE: src/policy.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_POLICY_H #define DMKIT_POLICY_H #include #include #include "rapidjson.h" namespace dmkit { struct KVPair { std::string key; std::string value; }; // A trigger define the condition under which a policy is choosen, // including intent and slots from qu result, as well as the current dm state. struct PolicyTrigger { std::string intent; std::vector slots; std::string state; }; // The parameter required in a policy. struct PolicyParam { std::string name; std::string type; std::string value; std::string default_value; bool required; }; // Session for policy output, including current domain, user defines contexts // and a state which DM will move to. struct PolicyOutputSession { std::string domain; std::string state; std::vector context; static std::string to_json_str(const PolicyOutputSession& session); static PolicyOutputSession from_json_str(const std::string& json_str); }; // A result item of dm output. struct PolicyOutputResult { std::string type; std::vector values; std::string extra; }; struct PolicyOutputQuSlot { std::string key; std::string value; std::string normalized_value; }; // In case the Qu result is required as well, currently not used. struct PolicyOutputQu { std::string domain; std::string intent; std::vector slots; }; // An assertion defines the condition under which a result is choosen. struct PolicyOutputAssertion { std::string type; std::string value; }; // Schema for DMKit output struct PolicyOutput { std::vector assertions; PolicyOutputQu qu; std::vector meta; PolicyOutputSession session; std::vector results; static std::string to_json_str(const PolicyOutput& output); }; // A policy defines a processing(params) & response(output), // given a trigger(intent+slots+state) condition. class Policy { public: Policy(const PolicyTrigger& trigger, const std::vector& params, const std::vector& outputs); const PolicyTrigger& trigger() const; const std::vector& params() const; const std::vector& outputs() const; static Policy* parse_from_json_value(const rapidjson::Value& value); private: PolicyTrigger _trigger; std::vector _params; std::vector _outputs; }; } // namespace dmkit #endif //DMKIT_POLICY_H ================================================ FILE: src/policy_manager.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "policy_manager.h" #include #include #include #include #include #include #include #include #include #include "app_log.h" #include "file_watcher.h" #include "utils.h" namespace dmkit { DomainPolicy::DomainPolicy(const std::string& name, int score, IntentPolicyMap* intent_policy_map) : _name(name), _score(score), _intent_policy_map(intent_policy_map) { } DomainPolicy::~DomainPolicy() { if (this->_intent_policy_map == nullptr) { return; } for (IntentPolicyMap::iterator iter = this->_intent_policy_map->begin(); iter != this->_intent_policy_map->end(); ++iter) { auto policy_vector = iter->second; if (policy_vector == nullptr) { continue; } for (PolicyVector::iterator iter2 = policy_vector->begin(); iter2 != policy_vector->end(); ++iter2) { auto policy = *iter2; if (policy == nullptr) { continue; } delete policy; *iter2 = nullptr; } delete policy_vector; iter->second = nullptr; } delete this->_intent_policy_map; this->_intent_policy_map = nullptr; } int DomainPolicy::score() { return this->_score; } const std::string& DomainPolicy::name() { return this->_name; } IntentPolicyMap* DomainPolicy::intent_policy_map() { return this->_intent_policy_map; } PolicyManager::PolicyManager() { this->_user_function_manager = nullptr; } PolicyManager::~PolicyManager() { if (this->_user_function_manager != nullptr) { delete this->_user_function_manager; this->_user_function_manager = nullptr; } FileWatcher::get_instance().unregister_file(this->_conf_file_path); this->_p_policy_dict.reset(); } static inline void destroy_policy_dict(ProductPolicyMap* policy_dict) { LOG(TRACE) << "Destroying policy dict"; if (policy_dict == nullptr) { return; } for (ProductPolicyMap::iterator iter = policy_dict->begin(); iter != policy_dict->end(); ++iter) { auto domain_policy_map = iter->second; if (domain_policy_map == nullptr) { continue; } for (DomainPolicyMap::iterator iter2 = domain_policy_map->begin(); iter2 != domain_policy_map->end(); ++iter2) { auto domain_policy = iter2->second; if (domain_policy == nullptr) { continue; } delete domain_policy; iter2->second = nullptr; } delete domain_policy_map; iter->second = nullptr; } delete policy_dict; policy_dict = nullptr; } // Loads policies from JSON configuration files. int PolicyManager::init(const char* dir_path, const char* conf_file) { std::string file_path; if (dir_path != nullptr) { file_path += dir_path; } if (!file_path.empty() && file_path[file_path.length() - 1] != '/') { file_path += '/'; } if (conf_file != nullptr) { file_path += conf_file; } this->_conf_file_path = file_path; ProductPolicyMap* policy_dict = this->load_policy_dict(); if (policy_dict == nullptr) { APP_LOG(ERROR) << "Failed to init policy dict"; return -1; } this->_p_policy_dict.reset(policy_dict, [](ProductPolicyMap* p) { destroy_policy_dict(p); }); FileWatcher::get_instance().register_file( this->_conf_file_path, PolicyManager::policy_conf_change_callback, this, true); this->_user_function_manager = new UserFunctionManager(); if (this->_user_function_manager->init() != 0) { APP_LOG(ERROR) << "Failed to init UserFunctionManager"; return -1; } return 0; } int PolicyManager::reload() { LOG(TRACE) << "Reloading policy dict"; ProductPolicyMap * policy_dict = this->load_policy_dict(); if (policy_dict == nullptr) { LOG(WARNING) << "Cannot reload policy! Policy dict load failed."; return -1; } this->_p_policy_dict.reset(policy_dict, [](ProductPolicyMap* p) { destroy_policy_dict(p); }); APP_LOG(TRACE) << "Reload finished."; return 0; } int PolicyManager::policy_conf_change_callback(void* param) { PolicyManager* pm = (PolicyManager*)param; return pm->reload(); } PolicyOutput* PolicyManager::resolve(const std::string& product, BUTIL_NAMESPACE::FlatMap* qu_result, const PolicyOutputSession& session, const RequestContext& context) { std::shared_ptr p_policy_map(this->_p_policy_dict); if (p_policy_map == nullptr) { APP_LOG(ERROR) << "Policy resolve failed, empty policy dict"; return nullptr; } DomainPolicyMap** seek_result = p_policy_map->seek(product); if (seek_result == nullptr) { APP_LOG(WARNING) << "unkown product " << product; return nullptr; } DomainPolicyMap* domain_policy_map = *seek_result; DomainPolicy* state_domain_policy = nullptr; Policy* state_policy = nullptr; std::forward_list> ranked_policies; std::vector empty_slots; QuResult empty_qu("", "", empty_slots); std::string request_domain; context.try_get_param("domain", request_domain); for (DomainPolicyMap::iterator iter = domain_policy_map->begin(); iter != domain_policy_map->end(); ++iter) { auto domain_policy = iter->second; const std::string& domain_name = domain_policy->name(); if (!request_domain.empty() && domain_name != request_domain) { continue; } QuResult** qu_seek_result = qu_result->seek(domain_name); QuResult* qu = qu_seek_result != nullptr ? *qu_seek_result : &empty_qu; // Find a best policy give the qu result in current domain Policy* find_result = this->find_best_policy(domain_policy, qu, session, context); if (find_result == nullptr) { continue; } // Policy with a domain and trigger state matches current DMKit domain&state is ranked top if (!session.domain.empty() && (domain_name == session.domain) && !session.state.empty() && (find_result->trigger().state == session.state)) { state_domain_policy = domain_policy; state_policy = find_result;; continue; } // In case there are multiple domain results, the policies a ranked by static domain score auto previous_it = ranked_policies.before_begin(); for (auto it = ranked_policies.begin(); it != ranked_policies.end(); ++it) { if (domain_policy->score() > it->first->score()) { break; } previous_it = it; } std::pair p(domain_policy, find_result); ranked_policies.insert_after(previous_it, p); } if (state_domain_policy != nullptr && state_policy != nullptr) { std::pair p(state_domain_policy, state_policy); ranked_policies.insert_after(ranked_policies.before_begin(), p); } // Resolve policy output and returns the first output resolved successfully. for (auto& p: ranked_policies) { const std::string& domain_name = p.first->name(); APP_LOG(TRACE) << "Resolving policy output for domain [" << domain_name << "]"; QuResult** qu_seek_result = qu_result->seek(domain_name); QuResult* qu = qu_seek_result != nullptr ? *qu_seek_result : &empty_qu; PolicyOutput* result = this->resolve_policy_output(domain_name, p.second, qu, session, context); if (result != nullptr) { APP_LOG(TRACE) << "Final result domain [" << domain_name << "]"; return result; } } return nullptr; } ProductPolicyMap* PolicyManager::load_policy_dict() { ProductPolicyMap* product_policy_map = new ProductPolicyMap(); // 10: bucket_count, initial count of buckets, big enough to avoid resize. // 80: load_factor, element_count * 100 / bucket_count. product_policy_map->init(10, 80); FILE* fp = fopen(this->_conf_file_path.c_str(), "r"); if (fp == nullptr) { APP_LOG(ERROR) << "Failed to open file " << this->_conf_file_path; destroy_policy_dict(product_policy_map); return nullptr; } char read_buffer[1024]; rapidjson::FileReadStream is(fp, read_buffer, sizeof(read_buffer)); rapidjson::Document doc; doc.ParseStream(is); fclose(fp); if (doc.HasParseError() || !doc.IsObject()) { APP_LOG(ERROR) << "Failed to parse products.json file"; destroy_policy_dict(product_policy_map); return nullptr; } for (rapidjson::Value::ConstMemberIterator prod_iter = doc.MemberBegin(); prod_iter != doc.MemberEnd(); ++prod_iter) { std::string prod_name = prod_iter->name.GetString(); if (!prod_iter->value.IsObject()) { APP_LOG(ERROR) << "Invalid product conf for " << prod_name; destroy_policy_dict(product_policy_map); return nullptr; } DomainPolicyMap* domain_policy_map = this->load_domain_policy_map(prod_name, prod_iter->value); if (domain_policy_map == nullptr) { APP_LOG(ERROR) << "Failed to load policies for product " << prod_name; destroy_policy_dict(product_policy_map); return nullptr; } product_policy_map->insert(prod_name, domain_policy_map); } return product_policy_map; } DomainPolicyMap* PolicyManager::load_domain_policy_map(const std::string& product_name, const rapidjson::Value& product_json) { DomainPolicyMap* domain_policy_map = new DomainPolicyMap(); // 10: bucket_count, initial count of buckets, big enough to avoid resize. // 80: load_factor, element_count * 100 / bucket_count. domain_policy_map->init(10, 80); APP_LOG(TRACE) << "Loading policies for product: " << product_name; for (rapidjson::Value::ConstMemberIterator domain_iter = product_json.MemberBegin(); domain_iter != product_json.MemberEnd(); ++domain_iter) { std::string domain_name = domain_iter->name.GetString(); const rapidjson::Value& domain_json = domain_iter->value; rapidjson::Value::ConstMemberIterator setting_iter; setting_iter = domain_json.FindMember("score"); if (setting_iter == domain_json.MemberEnd() || !setting_iter->value.IsInt()) { APP_LOG(WARNING) << "Failed to parse score for domain " << domain_name << " in product " << product_name << ", skipped"; continue; } int score = setting_iter->value.GetInt(); setting_iter = domain_json.FindMember("conf_path"); if (setting_iter == domain_json.MemberEnd() || !setting_iter->value.IsString()) { APP_LOG(WARNING) << "Failed to parse conf_path for domain " << domain_name << " in product " << product_name << ", skipped"; continue; } std::string conf_path = setting_iter->value.GetString(); APP_LOG(TRACE) << "Loading policies for domain " << domain_name << " from " << conf_path; DomainPolicy* domain_policy = this->load_domain_policy(domain_name, score, conf_path); if (domain_policy == nullptr) { APP_LOG(WARNING) << "Failed to load policy for domain " << domain_name << " in product " << product_name << ", skipped"; continue; } APP_LOG(TRACE) << "Loaded policies for domain " << domain_name; domain_policy_map->insert(domain_name, domain_policy); } return domain_policy_map; } DomainPolicy* PolicyManager::load_domain_policy(const std::string& domain_name, int score, const std::string& conf_path) { FILE* fp = fopen(conf_path.c_str(), "r"); if (fp == nullptr) { APP_LOG(ERROR) << "Failed to open file " << conf_path; return nullptr; } char read_buffer[1024]; rapidjson::FileReadStream is(fp, read_buffer, sizeof(read_buffer)); rapidjson::Document doc; doc.ParseStream(is); fclose(fp); if (doc.HasParseError() || !doc.IsArray()) { APP_LOG(ERROR) << "Failed to parse domain conf " << conf_path; return nullptr; } IntentPolicyMap* intent_policy_map = new IntentPolicyMap(); // 10: bucket_count, initial count of buckets, big enough to avoid resize. // 80: load_factor, element_count * 100 / bucket_count. intent_policy_map->init(10, 80); for (rapidjson::Value::ConstValueIterator policy_iter = doc.Begin(); policy_iter != doc.End(); ++policy_iter) { APP_LOG(TRACE) << "loading policy..."; Policy* policy = Policy::parse_from_json_value(*policy_iter); if (policy == nullptr) { APP_LOG(WARNING) << "Found invalid policy conf in path " << conf_path << ", skipped"; continue; } const std::string& trigger_intent = policy->trigger().intent; if (intent_policy_map->seek(trigger_intent) == nullptr) { intent_policy_map->insert(trigger_intent, new PolicyVector); } (*intent_policy_map)[trigger_intent]->push_back(policy); } APP_LOG(TRACE) << "initializing domain policy..."; DomainPolicy* domain_policy = new DomainPolicy(domain_name, score, intent_policy_map); APP_LOG(TRACE) << "finish initializing domain policy..."; return domain_policy; } // When multiple policies satisfy the current intent, // the following strategy is applies to find the best one: // 1. The policies with none empty trigger state, it should match current DM state. // 2. Policy with maximum number of matched trigger slot is ranked top. static Policy* find_best_policy_from_candidates(PolicyVector& policy_vector, std::string& state, std::unordered_multiset& qu_slot_set) { Policy* policy_result = nullptr; for (auto const& policy: policy_vector) { if (!policy->trigger().state.empty() && policy->trigger().state != state) { continue; } bool missing_slot = false; std::unordered_multiset trigger_slot_set; for (auto const& slot: policy->trigger().slots) { trigger_slot_set.insert(slot); } for (auto iter = trigger_slot_set.begin(); iter != trigger_slot_set.end();) { int trigger_slot_cnt = trigger_slot_set.count(*iter); int qu_slot_cnt = qu_slot_set.count(*iter); if (qu_slot_cnt < trigger_slot_cnt) { missing_slot = true; break; } std::advance(iter, trigger_slot_cnt); } if (missing_slot) { continue; } if (policy_result == nullptr) { policy_result = policy; continue; } if (!policy_result->trigger().state.empty() && policy->trigger().state.empty()) { continue; } if (policy_result->trigger().state.empty() && !policy->trigger().state.empty()) { policy_result = policy; continue; } if (policy_result->trigger().slots.size() < policy->trigger().slots.size()) { policy_result = policy; } } return policy_result; } Policy* PolicyManager::find_best_policy(DomainPolicy* domain_policy, QuResult* qu_result, const PolicyOutputSession& session, const RequestContext& context) { (void) context; std::string state; if (domain_policy->name() == session.domain) { state = session.state; } std::unordered_multiset qu_slot_set; IntentPolicyMap* intent_policy_map = domain_policy->intent_policy_map(); PolicyVector** policy_vector_seek = nullptr; Policy* policy_result = nullptr; for (auto const& slot: qu_result->slots()) { qu_slot_set.insert(slot.key()); } // Policy with matching intent. const std::string& intent = qu_result->intent(); policy_vector_seek = intent_policy_map->seek(intent); if (policy_vector_seek != nullptr) { APP_LOG(TRACE) << "intent [" << intent << "] candidate count [" << (*policy_vector_seek)->size() << "]"; policy_result = find_best_policy_from_candidates(**policy_vector_seek, state, qu_slot_set); } // Fallback policy when none of the policies match intent. if (policy_result == nullptr) { const std::string fallback_intent = "dmkit_intent_fallback"; policy_vector_seek = intent_policy_map->seek(fallback_intent); if (policy_vector_seek != nullptr) { APP_LOG(TRACE) << "dmkit_intent_fallback candidate count [" << (*policy_vector_seek)->size() << "]"; policy_result = find_best_policy_from_candidates(**policy_vector_seek, state, qu_slot_set); } } return policy_result; } // Resolve a string with params in it. static bool try_resolve_params(std::string& unresolved, const std::unordered_map& param_map) { if (unresolved.empty()) { return true; } std::string resolved; bool is_param = false; unsigned int last_index = 0; for (unsigned int i = 0; i < unresolved.length(); ++i) { if (unresolved[i] == '{' && i + 1 < unresolved.length() && unresolved[i + 1] == '%' ) { resolved += unresolved.substr(last_index, i - last_index); last_index = i + 2; is_param = true; ++i; } if (unresolved[i] == '%' && i + 1 < unresolved.length() && unresolved[i + 1] == '}') { if (!is_param) { APP_LOG(WARNING) << "Cannot resolve params in string, invalid format. " << unresolved; return false; } std::string param_name = unresolved.substr(last_index, i - last_index); std::unordered_map::const_iterator find_res = param_map.find(param_name); if (find_res == param_map.end()) { APP_LOG(WARNING) << "Cannot resolve params in string, unknow param. " << unresolved << " " << param_name; return false; } resolved += find_res->second; last_index = i + 2; is_param = false; ++i; } } if (is_param) { APP_LOG(WARNING) << "Cannot resolve params in string, invalid format. " << unresolved; return false; } if (last_index < unresolved.length()) { resolved += unresolved.substr(last_index); } unresolved = resolved; return true; } // Resolve a string with delimiter and params in it static bool try_resolve_param_list(const std::string& unresolved, const char delimiter, const std::unordered_map& param_map, std::vector &result) { std::size_t pos = 0; std::size_t last_pos = 0; result.clear(); while (last_pos < unresolved.length() && (pos = unresolved.find(delimiter, last_pos)) != std::string::npos) { std::string part = unresolved.substr(last_pos, pos - last_pos); utils::trim(part); if (!try_resolve_params(part, param_map)) { result.clear(); return false; } result.push_back(part); last_pos = pos + 1; } if (last_pos < unresolved.length()) { std::string part = unresolved.substr(last_pos); utils::trim(part); if (!try_resolve_params(part, param_map)) { result.clear(); return false; } result.push_back(part); } return true; } PolicyOutput* PolicyManager::resolve_policy_output(const std::string& domain, Policy* policy, QuResult* qu_result, const PolicyOutputSession& session, const RequestContext& context) { // Process parameters std::unordered_map param_map; // Default parameters for (auto const& context: session.context) { if (context.key == "dmkit_param_last_tts") { param_map[context.key] = context.value; continue; } std::string param_key = "dmkit_param_context_"; param_key += context.key; param_map[param_key] = context.value; } for (auto const& slot: qu_result->slots()) { std::string param_key = "dmkit_param_slot_"; param_key += slot.key(); if (param_map.find(param_key) != param_map.end()) { continue; } std::string param_value = slot.normalized_value(); if (param_value.empty()) { param_value = slot.value(); } param_map[param_key] = param_value; } for (auto const& param: policy->params()) { APP_LOG(TRACE) << "resolving parameter [" << param.name << "]"; std::string value; if (param.type == "slot_val" || param.type == "slot_val_ori") { bool success = false; std::vector args; utils::split(param.value, ',', args); int index = 0; if (args.size() >= 2 && !utils::try_atoi(args[1], index)) { APP_LOG(WARNING) << "Invalid index for slot_val parameter: " << param.value; index = -1; } for (auto const& slot: qu_result->slots()) { if (index < 0) { break; } if (slot.key() == args[0]) { if (index > 0) { index--; continue; } if (param.type == "slot_val" && !slot.normalized_value().empty()) { value = slot.normalized_value(); success = true; break; } value = slot.value(); success = true; break; } } if (!success) { if (param.required) { return nullptr; } value = param.default_value; } } else if (param.type == "qu_intent") { value = qu_result->intent(); } else if (param.type == "session_state") { value = session.state; } else if (param.type == "session_context") { bool success = false; for (auto const& obj: session.context) { if (obj.key == param.value) { value = obj.value; success = true; break; } } if (!success) { if (param.required) { return nullptr; } value = param.default_value; } }else if (param.type == "const") { value = param.value; } else if (param.type == "string") { value = param.value; if (!try_resolve_params(value, param_map)) { if (param.required) { return nullptr; } value = param.default_value; } } else if (param.type == "request_param") { const std::unordered_map request_params = context.params(); std::unordered_map::const_iterator find_res = request_params.find(param.value); bool success = false; if (find_res != request_params.end()) { value = find_res->second; success = true; } if (!success) { if (param.required) { return nullptr; } value = param.default_value; } } else if (param.type == "func_val") { std::string func_val = param.value; std::string func_name; std::vector args; std::size_t pos = func_val.find(':'); bool has_error = false; if (pos == std::string::npos || pos == func_val.length() - 1) { func_name = func_val; if (!try_resolve_params(func_name, param_map)) { has_error = true; } } else { func_name = func_val.substr(0, pos); if (!try_resolve_params(func_name, param_map)) { has_error = true;; } std::string arg_list = func_val.substr(pos + 1); if (!try_resolve_param_list(arg_list, ',', param_map, args)) { has_error = true; } } utils::trim(func_name); if (has_error || this->_user_function_manager->call_user_function(func_name, args, context, value) != 0) { has_error = true; } if (has_error) { if (param.required) { return nullptr; } value = param.default_value; } } else { APP_LOG(WARNING) << "Unknown param type " << param.type; if (param.required) { return nullptr; } value = param.default_value; } param_map[param.name] = value; APP_LOG(TRACE) << "Parameter value [" << value << "]"; } int selected_output_index = -1; for (unsigned int i = 0; i < policy->outputs().size(); ++i) { bool failed = false; // Process assertions APP_LOG(TRACE) << "Candidate output size [" << policy->outputs().size() << "]"; for (unsigned int j = 0; j < policy->outputs()[i].assertions.size(); ++j) { std::string assertion_type = policy->outputs()[i].assertions[j].type; std::string assertion_value = policy->outputs()[i].assertions[j].value; if (!try_resolve_params(assertion_value, param_map)) { failed = true; break; } APP_LOG(TRACE) << "evaluating assertion, type[" << assertion_type << "] value[" << assertion_value << "]"; if (assertion_type == "not_empty") { if (assertion_value.empty()) { failed = true; break; } } else if (assertion_type == "empty") { if (!assertion_value.empty()) { failed = true; break; } } else if (assertion_type == "in") { std::vector value_list; bool has_match = false; if (try_resolve_param_list(assertion_value, ',', param_map, value_list)) { for (unsigned int k = 1; k < value_list.size(); k++) { if (value_list[0] == value_list[k]) { has_match = true; break; } } } if (!has_match) { failed = true; } } else if (assertion_type == "not_in") { std::vector value_list; if (try_resolve_param_list(assertion_value, ',', param_map, value_list)) { for (unsigned int k = 1; k < value_list.size(); k++) { if (value_list[0] == value_list[k]) { failed = true; break; } } } } else if (assertion_type == "eq") { std::vector value_list; if (!try_resolve_param_list(assertion_value, ',', param_map, value_list) || value_list.size() < 2 || value_list[0] != value_list[1]) { failed = true; } } else if (assertion_type == "gt") { std::vector value_list; double left_val = 0; double right_val = 0; if (!try_resolve_param_list(assertion_value, ',', param_map, value_list) || value_list.size() < 2 || !utils::try_atof(value_list[0], left_val) || !utils::try_atof(value_list[1], right_val) || left_val <= right_val) { failed = true; } } else if (assertion_type == "ge") { std::vector value_list; double left_val = 0; double right_val = 0; if (!try_resolve_param_list(assertion_value, ',', param_map, value_list) || value_list.size() < 2 || !utils::try_atof(value_list[0], left_val) || !utils::try_atof(value_list[1], right_val) || left_val < right_val) { failed = true; } } else { APP_LOG(WARNING) << "Unknown assertion type " << assertion_type; failed = true; break; } } if (!failed) { LOG(TRACE) << "selected output at index [" << i << "]"; selected_output_index = i; break; } } if (selected_output_index == -1) { return nullptr; } PolicyOutput output; output.meta = policy->outputs()[selected_output_index].meta; output.session = policy->outputs()[selected_output_index].session; output.qu = policy->outputs()[selected_output_index].qu; output.results = policy->outputs()[selected_output_index].results; for (unsigned int i = 0; i < output.meta.size(); ++i) { if (!try_resolve_params(output.meta[i].key, param_map)) { return nullptr; } if (!try_resolve_params(output.meta[i].value, param_map)) { return nullptr; } } if (!try_resolve_params(output.session.domain, param_map)) { return nullptr; } if (!try_resolve_params(output.session.state, param_map)) { return nullptr; } for (unsigned int i = 0; i < output.session.context.size(); ++i) { if (!try_resolve_params(output.session.context[i].key, param_map)) { return nullptr; } if (!try_resolve_params(output.session.context[i].value, param_map)) { return nullptr; } } if (!try_resolve_params(output.qu.domain, param_map)) { return nullptr; } if (!try_resolve_params(output.qu.intent, param_map)) { return nullptr; } for (unsigned int i = 0; i < output.qu.slots.size(); ++i) { if (!try_resolve_params(output.qu.slots[i].key, param_map)) { return nullptr; } if (!try_resolve_params(output.qu.slots[i].value, param_map)) { return nullptr; } if (!try_resolve_params(output.qu.slots[i].normalized_value, param_map)) { return nullptr; } } for (unsigned int i = 0; i < output.results.size(); ++i) { if (output.results[i].values.empty()) { APP_LOG(WARNING) << "empty result value!"; return nullptr; } if (output.results[i].values.size() > 1) { int size = output.results[i].values.size(); int index = std::time(nullptr) % size; if (index < 0 || index >= size) { index = 0; } std::string value = output.results[i].values[index]; output.results[i].values.clear(); output.results[i].values.push_back(value); } if (!try_resolve_params(output.results[i].values[0], param_map)) { return nullptr; } } PolicyOutput* output_ptr = new PolicyOutput(); *output_ptr = output; output_ptr->session.domain = domain; // Saved parameters for (auto const& param: param_map) { if (param.first.find("dmkit_param_context_") != 0) { continue; } std::string context_key = param.first.substr(20); if (context_key.empty()) { continue; } KVPair context = {context_key, param.second}; output_ptr->session.context.push_back(context); } std::string first_tts; for (unsigned int i = 0; i < output_ptr->results.size(); ++i) { if (output_ptr->results[i].type == "tts") { first_tts = output_ptr->results[i].values[0]; break; } } KVPair last_tts_context = {"dmkit_param_last_tts", first_tts}; output_ptr->session.context.push_back(last_tts_context); return output_ptr; } } // namespace dmkit ================================================ FILE: src/policy_manager.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_POLICY_MANAGER_H #define DMKIT_POLICY_MANAGER_H #include #include #include #include #include #include "butil.h" #include "policy.h" #include "qu_result.h" #include "request_context.h" #include "user_function_manager.h" namespace dmkit { typedef std::vector PolicyVector; typedef BUTIL_NAMESPACE::FlatMap IntentPolicyMap; // Holds all policies given a domain. class DomainPolicy { public: DomainPolicy(const std::string& name, int score, IntentPolicyMap* intent_policy_map); ~DomainPolicy(); const std::string& name(); int score(); // Maps a intent to a vector of policies IntentPolicyMap* intent_policy_map(); private: std::string _name; int _score; IntentPolicyMap* _intent_policy_map; }; typedef BUTIL_NAMESPACE::FlatMap DomainPolicyMap; typedef BUTIL_NAMESPACE::FlatMap ProductPolicyMap; class PolicyManager { public: PolicyManager(); ~PolicyManager(); int init(const char* dir_path, const char* conf_file); int reload(); // Resolve a policy output given a qu result and current dm session PolicyOutput* resolve(const std::string& product, BUTIL_NAMESPACE::FlatMap* qu_result, const PolicyOutputSession& session, const RequestContext& context); static int policy_conf_change_callback(void* param); private: DomainPolicyMap* load_domain_policy_map(const std::string& product_name, const rapidjson::Value& product_json); DomainPolicy* load_domain_policy(const std::string& domain_name, int score, const std::string& conf_path); Policy* find_best_policy(DomainPolicy* domain_policy, QuResult* qu_result, const PolicyOutputSession& session, const RequestContext& context); PolicyOutput* resolve_policy_output(const std::string& domain, Policy* policy, QuResult* qu_result, const PolicyOutputSession& session, const RequestContext& context); ProductPolicyMap* load_policy_dict(); std::string _conf_file_path; std::shared_ptr _p_policy_dict; UserFunctionManager* _user_function_manager; }; } // namespace dmkit #endif //DMKIT_POLICY_MANAGER_H ================================================ FILE: src/qu_result.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "qu_result.h" #include "app_log.h" #include "utils.h" namespace dmkit { Slot::Slot(const std::string& key, const std::string& value, const std::string& normalized_value) : _key(key), _value(value), _normalized_value(normalized_value) { } const std::string& Slot::key() const { return this->_key; } const std::string& Slot::value() const { return this->_value; } const std::string& Slot::normalized_value() const { return this->_normalized_value; } QuResult::QuResult(const std::string& domain, const std::string& intent, const std::vector& slots) : _domain(domain), _intent(intent), _slots(slots) { } // Parse QU result from dialog state in unit response QuResult* QuResult::parse_from_dialog_state(const std::string& domain, const rapidjson::Value& value) { if (!value.IsObject()) { APP_LOG(WARNING) << "Failed to parse qu result from json, not an object"; return nullptr; } if (!value.HasMember("intents") || !value["intents"].IsArray() || value["intents"].Size() < 1) { APP_LOG(WARNING) << "Failed to parse qu result from json"; } int intents_size = value["intents"].Size(); std::string intent = value["intents"][intents_size - 1]["name"].GetString(); std::vector slots; for (auto& m_slot: value["user_slots"].GetObject()) { std::string tag = m_slot.name.GetString(); for (auto& m_value: m_slot.value.GetObject()["values"].GetObject()) { int slot_state = m_value.value.GetObject()["state"].GetInt(); // Slot state possible values are // 0: slot not filled // 1: slot is filled by default value // 2: slot is filled by SLU // 4: slot was filled but has been replaced by other value if (slot_state == 0 || slot_state == 4) { continue; } std::string normalized_value = m_value.name.GetString(); std::string value = m_value.value.GetObject()["original_name"].GetString(); Slot slot(tag, value, normalized_value); slots.push_back(slot); } } QuResult* result = new QuResult(domain, intent, slots); APP_LOG(TRACE) << result->to_string(); return result; } const std::string& QuResult::domain() const { return this->_domain; } const std::string& QuResult::intent() const { return this->_intent; } const std::vector& QuResult::slots() const { return this->_slots; } std::string QuResult::to_string() const { std::string result; result += "domain:"; result += this->domain(); result += " intent:"; result += this->intent(); result += " slots: {"; for (auto iter = this->slots().begin(); iter != this->slots().end(); ++iter) { result += iter->key(); result += ":"; result += iter->value(); if (!iter->normalized_value().empty()) { result += "("; result += iter->normalized_value(); result += ")"; } result += " "; } result += "}"; return result; } } // namespace dmkit ================================================ FILE: src/qu_result.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_QU_RESULT_H #define DMKIT_QU_RESULT_H #include #include #include "rapidjson.h" namespace dmkit { class Slot { public: Slot(const std::string& key, const std::string& value, const std::string& normalized_value); const std::string& key() const; const std::string& value() const; const std::string& normalized_value() const; private: std::string _key; std::string _value; std::string _normalized_value; }; // Query Understanding, or natural language understanding (NLU) results for user input, // generally includes domain, intent and slots. class QuResult { public: QuResult(const std::string& domain, const std::string& intent, const std::vector& slots); static QuResult* parse_from_dialog_state(const std::string& domain, const rapidjson::Value& value); const std::string& domain() const; const std::string& intent() const; const std::vector& slots() const; std::string to_string() const; private: std::string _domain; std::string _intent; std::vector _slots; }; } // namespace dmkit #endif //DMKIT_QU_RESULT_H ================================================ FILE: src/rapidjson.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_RAPIDJSON_H #define DMKIT_RAPIDJSON_H #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-local-typedefs" #endif #include "thirdparty/rapidjson/allocators.h" #include "thirdparty/rapidjson/document.h" #include "thirdparty/rapidjson/encodedstream.h" #include "thirdparty/rapidjson/encodings.h" #include "thirdparty/rapidjson/filereadstream.h" #include "thirdparty/rapidjson/filewritestream.h" #include "thirdparty/rapidjson/fwd.h" #include "thirdparty/rapidjson/istreamwrapper.h" #include "thirdparty/rapidjson/memorybuffer.h" #include "thirdparty/rapidjson/memorystream.h" #include "thirdparty/rapidjson/ostreamwrapper.h" #include "thirdparty/rapidjson/pointer.h" #include "thirdparty/rapidjson/prettywriter.h" #include "thirdparty/rapidjson/rapidjson.h" #include "thirdparty/rapidjson/reader.h" #include "thirdparty/rapidjson/schema.h" #include "thirdparty/rapidjson/stream.h" #include "thirdparty/rapidjson/stringbuffer.h" #include "thirdparty/rapidjson/writer.h" #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) #pragma GCC diagnostic pop #endif #endif //DMKIT_RAPIDJSON_H ================================================ FILE: src/remote_service_manager.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "remote_service_manager.h" #include #include #include #include "app_log.h" #include "file_watcher.h" #include "rapidjson.h" #include "thread_data_base.h" namespace dmkit { RemoteServiceManager::RemoteServiceManager() { } RemoteServiceManager::~RemoteServiceManager() { FileWatcher::get_instance().unregister_file(this->_conf_file_path); this->_p_channel_map.reset(); } static inline void destroy_channel_map(ChannelMap* p) { APP_LOG(TRACE) << "Destroying service map..."; if (nullptr == p) { return; } for (auto& channel : *p) { if (nullptr != channel.second.channel) { delete channel.second.channel; channel.second.channel = nullptr; APP_LOG(TRACE) << "Destroyed service" << channel.first; } } delete p; } int RemoteServiceManager::init(const char* path, const char* conf) { std::string file_path; if (path != nullptr) { file_path += path; } if (!file_path.empty() && file_path[file_path.length() - 1] != '/') { file_path += '/'; } if (conf != nullptr) { file_path += conf; } this->_conf_file_path = file_path; ChannelMap* channel_map = this->load_channel_map(); if (channel_map == nullptr) { APP_LOG(ERROR) << "Failed to init RemoteServiceManager, cannot load channel map"; return -1; } this->_p_channel_map.reset(channel_map, [](ChannelMap* p) { destroy_channel_map(p); }); FileWatcher::get_instance().register_file( this->_conf_file_path, RemoteServiceManager::service_conf_change_callback, this, true); return 0; } int RemoteServiceManager::reload() { APP_LOG(TRACE) << "Reloading RemoteServiceManager..."; ChannelMap* channel_map = this->load_channel_map(); if (channel_map == nullptr) { APP_LOG(ERROR) << "Failed to reload RemoteServiceManager, cannot load channel map"; return -1; } this->_p_channel_map.reset(channel_map, [](ChannelMap* p) { destroy_channel_map(p); }); APP_LOG(TRACE) << "Reload finished."; return 0; } int RemoteServiceManager::service_conf_change_callback(void* param) { RemoteServiceManager* rsm = (RemoteServiceManager*)param; return rsm->reload(); } int RemoteServiceManager::call(const std::string& service_name, const RemoteServiceParam& params, RemoteServiceResult& result) const { std::shared_ptr p_channel_map(this->_p_channel_map); if (p_channel_map == nullptr) { APP_LOG(ERROR) << "Remote service call failed, channel map is null"; return -1; } if (p_channel_map->find(service_name) == p_channel_map->end()) { APP_LOG(ERROR) << "Remote service call failed, cannot find service " << service_name; return -1; } RemoteServiceChannel& service_channel = (*p_channel_map)[service_name]; APP_LOG(TRACE) << "Calling service " << service_name; int ret = 0; std::string remote_side; int latency = 0; if (service_channel.protocol == "http") { if (service_channel.channel != nullptr) { ret = this->call_http_by_BRPC_NAMESPACE(service_channel.channel, params.url, params.http_method, service_channel.headers, params.payload, result.result, remote_side, latency); } else { ret = this->call_http_by_curl(params.url, params.http_method, service_channel.headers, params.payload, service_channel.timeout_ms, service_channel.max_retry, result.result, remote_side, latency); } } else { APP_LOG(ERROR) << "Remote service call failed. Unknown protocol" << service_channel.protocol; ret = -1; } std::string log_str; log_str += "remote:"; log_str += remote_side; log_str += "|tm:"; log_str += std::to_string(latency); log_str += "|ret:"; log_str += std::to_string(ret); std::string log_key = "service_"; log_key += service_name; APP_LOG(TRACE) << "remote_side=" << remote_side << ", cost=" << latency; ThreadDataBase* tls = static_cast(BRPC_NAMESPACE::thread_local_data()); // All backend requests are logged. tls->add_notice_log(log_key, log_str); return ret; } ChannelMap* RemoteServiceManager::load_channel_map() { APP_LOG(TRACE) << "Loading channel map..."; FILE* fp = fopen(this->_conf_file_path.c_str(), "r"); if (fp == nullptr) { APP_LOG(ERROR) << "Failed to open file " << this->_conf_file_path; return nullptr; } char read_buffer[1024]; rapidjson::FileReadStream is(fp, read_buffer, sizeof(read_buffer)); rapidjson::Document doc; doc.ParseStream(is); fclose(fp); if (doc.HasParseError() || !doc.IsObject()) { APP_LOG(ERROR) << "Failed to parse RemoteServiceManager settings"; return nullptr; } ChannelMap* channel_map = new ChannelMap(); rapidjson::Value::ConstMemberIterator service_iter; for (service_iter = doc.MemberBegin(); service_iter != doc.MemberEnd(); ++service_iter) { // Service name as the key for the channel. std::string service_name = service_iter->name.GetString(); if (!service_iter->value.IsObject()) { APP_LOG(ERROR) << "Invalid service settings for " << service_name << ", expecting type object for service setting."; destroy_channel_map(channel_map); return nullptr; } const rapidjson::Value& settings = service_iter->value; rapidjson::Value::ConstMemberIterator setting_iter; // Naming service url such as https://www.baidu.com. // All supported url format can be found in BRPC docs. setting_iter = settings.FindMember("naming_service_url"); if (setting_iter == settings.MemberEnd() || !setting_iter->value.IsString()) { APP_LOG(ERROR) << "Invalid service settings for " << service_name << ", expecting type String for property naming_service_url."; destroy_channel_map(channel_map); return nullptr; } std::string naming_service_url = setting_iter->value.GetString(); // Load balancer name such as random, rr. // All supported balancer can be found in BRPC docs. setting_iter = settings.FindMember("load_balancer_name"); if (setting_iter == settings.MemberEnd() || !setting_iter->value.IsString()) { APP_LOG(ERROR) << "Invalid service settings for " << service_name << ", expecting type String for property load_balancer_name."; destroy_channel_map(channel_map); return nullptr; } std::string load_balancer_name = setting_iter->value.GetString(); // Protocol for the channel. // Currently we support http. setting_iter = settings.FindMember("protocol"); if (setting_iter == settings.MemberEnd() || !setting_iter->value.IsString()) { APP_LOG(ERROR) << "Invalid service settings for " << service_name << ", expecting type String for property protocol."; destroy_channel_map(channel_map); return nullptr; } std::string protocol = setting_iter->value.GetString(); // Client to use for sending request. setting_iter = settings.FindMember("client"); std::string client; if (setting_iter != settings.MemberEnd() && setting_iter->value.IsString()) { client = setting_iter->value.GetString(); } // Timeout value in millisecond. setting_iter = settings.FindMember("timeout_ms"); if (setting_iter == settings.MemberEnd() || !setting_iter->value.IsInt()) { APP_LOG(ERROR) << "Invalid service settings for " << service_name << ", expecting type Int for property timeout_ms."; destroy_channel_map(channel_map); return nullptr; } int timeout_ms = setting_iter->value.GetInt(); // Retry count. setting_iter = settings.FindMember("retry"); if (setting_iter == settings.MemberEnd() || !setting_iter->value.IsInt()) { APP_LOG(ERROR) << "Invalid service settings for " << service_name << ", expecting type Int for property retry."; destroy_channel_map(channel_map); return nullptr; } int retry = setting_iter->value.GetInt(); // Headers for a http request. std::vector> headers; setting_iter = settings.FindMember("headers"); if (setting_iter != settings.MemberEnd() && setting_iter->value.IsObject()) { const rapidjson::Value& obj_headers = setting_iter->value; rapidjson::Value::ConstMemberIterator header_iter; for (header_iter = obj_headers.MemberBegin(); header_iter != obj_headers.MemberEnd(); ++header_iter) { std::string header_key = header_iter->name.GetString(); if (!header_iter->value.IsString()) { APP_LOG(ERROR) << "Invalid header value for " << header_key << ", expecting type String for header value."; destroy_channel_map(channel_map); return nullptr; } std::string header_value = header_iter->value.GetString(); headers.push_back(std::make_pair(header_key, header_value)); } } BRPC_NAMESPACE::Channel* rpc_channel = nullptr; if (protocol == "http") { if (client.empty() || client == "brpc") { rpc_channel = new BRPC_NAMESPACE::Channel(); BRPC_NAMESPACE::ChannelOptions options; options.protocol = BRPC_NAMESPACE::PROTOCOL_HTTP; options.timeout_ms = timeout_ms; options.max_retry = retry; int ret = rpc_channel->Init(naming_service_url.c_str(), load_balancer_name.c_str(), &options); if (ret != 0) { APP_LOG(ERROR) << "Failed to init channel."; delete rpc_channel; destroy_channel_map(channel_map); return nullptr; } } else if (client == "curl") { // curl does not need to init rpc channel } else { APP_LOG(ERROR) << "Unsupported client value [" << client << "]."; destroy_channel_map(channel_map); return nullptr; } } else { APP_LOG(ERROR) << "Unsupported protocol [" << protocol << "] for service [" << service_name << "], skipped..."; destroy_channel_map(channel_map); return nullptr; } RemoteServiceChannel service_channel { .name = service_name, .protocol = protocol, .channel = rpc_channel, .timeout_ms = timeout_ms, .max_retry = retry, .headers = headers }; channel_map->insert({service_name, service_channel}); APP_LOG(TRACE) << "Loaded service " << service_name; } return channel_map; } int RemoteServiceManager::call_http_by_BRPC_NAMESPACE(BRPC_NAMESPACE::Channel* channel, const std::string& url, const HttpMethod method, const std::vector>& headers, const std::string& payload, std::string& result, std::string& remote_side, int& latency) const { BRPC_NAMESPACE::Controller cntl; cntl.http_request().uri() = url.c_str(); if (method == HTTP_METHOD_POST) { cntl.http_request().set_method(BRPC_NAMESPACE::HTTP_METHOD_POST); cntl.request_attachment().append(payload); } for (auto const& header: headers) { if (header.first == "Content-Type" || header.first == "content-type") { cntl.http_request().set_content_type(header.second); continue; } cntl.http_request().SetHeader(header.first, header.second); } channel->CallMethod(NULL, &cntl, NULL, NULL, NULL); if (cntl.Failed()) { APP_LOG(WARNING) << "Call failed, error: " << cntl.ErrorText(); remote_side = BUTIL_NAMESPACE::endpoint2str(cntl.remote_side()).c_str(); latency = cntl.latency_us() / 1000; return -1; } result = cntl.response_attachment().to_string(); remote_side = BUTIL_NAMESPACE::endpoint2str(cntl.remote_side()).c_str(); latency = cntl.latency_us() / 1000; return 0; } static size_t curl_write_callback(void *contents, size_t size, size_t nmemb, void *userp) { BUTIL_NAMESPACE::IOBuf* buffer = static_cast(userp); size_t realsize = size * nmemb; buffer->append(contents, realsize); return realsize; } int RemoteServiceManager::call_http_by_curl(const std::string& url, const HttpMethod method, const std::vector>& headers, const std::string& payload, const int timeout_ms, const int max_retry, std::string& result, std::string& remote_side, int& latency) const { // curl does not support retry now (void)max_retry; CURL *curl; CURLcode res; struct curl_slist *curl_headers = nullptr; curl = curl_easy_init(); if (!curl) { APP_LOG(ERROR) << "Failed to init curl"; return -1; } curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); BUTIL_NAMESPACE::IOBuf response_buffer; curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast(&response_buffer)); if (method == HTTP_METHOD_POST) { curl_easy_setopt(curl, CURLOPT_POST, 1L); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, payload.length()); } for (auto const& header: headers) { std::string header_value = header.first; header_value += ": "; header_value += header.second; curl_headers = curl_slist_append(curl_headers, header_value.c_str()); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout_ms); //curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); res = curl_easy_perform(curl); curl_slist_free_all(curl_headers); if(res != CURLE_OK) { APP_LOG(ERROR) << "curl failed, error: " << curl_easy_strerror(res); curl_easy_cleanup(curl); return -1; } double total_time; res = curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time); if (CURLE_OK == res) { latency = total_time * 1000; } char *ip = nullptr; res = curl_easy_getinfo(curl, CURLINFO_PRIMARY_IP, &ip); if (CURLE_OK == res && ip != nullptr) { remote_side = ip; } curl_easy_cleanup(curl); result = response_buffer.to_string(); return 0; } } // namespace dmkit ================================================ FILE: src/remote_service_manager.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_REMOTE_SERVICE_MANAGER_H #define DMKIT_REMOTE_SERVICE_MANAGER_H #include #include #include #include #include #include "brpc.h" #include "butil.h" namespace dmkit { enum HttpMethod { // Set to the same number as the method defined in baidu::rpc::HttpMethod HTTP_METHOD_DELETE = 0, HTTP_METHOD_GET = 1, HTTP_METHOD_POST = 3, HTTP_METHOD_PUT = 4 }; struct RemoteServiceParam { // Url for http service std::string url; // Http method HttpMethod http_method; // Request body std::string payload; }; struct RemoteServiceResult { // Response body std::string result; }; struct RemoteServiceChannel { // Name of the channel for rpc call std::string name; // Protocol such as http std::string protocol; // Rpc channel instance BRPC_NAMESPACE::Channel *channel; // timeout in milliseconds int timeout_ms; // retry count int max_retry; // Headers for http procotol std::vector> headers; }; // Type for channel map typedef std::unordered_map ChannelMap; // A configurable remote service manager class. // All remote service channels are created with configuration file // when initialization. Caller calls a remote service by supplying // the service name and other parameters. class RemoteServiceManager { public: RemoteServiceManager(); ~RemoteServiceManager(); // Initalization with a json configuration file. int init(const char *path, const char *conf); // Reload config. int reload(); // Callback when service conf changed. static int service_conf_change_callback(void* param); // Call a remote service with specifid service name. int call(const std::string& servie_name, const RemoteServiceParam& params, RemoteServiceResult &result) const; private: // Http is the most common protocol. int call_http_by_BRPC_NAMESPACE(BRPC_NAMESPACE::Channel* channel, const std::string& url, const HttpMethod method, const std::vector>& headers, const std::string& payload, std::string& result, std::string& remote_side, int& latency) const; int call_http_by_curl(const std::string& url, const HttpMethod method, const std::vector>& headers, const std::string& payload, const int timeout_ms, const int max_retry, std::string& result, std::string& remote_side, int& latency) const; ChannelMap* load_channel_map(); std::string _conf_file_path; std::shared_ptr _p_channel_map; }; } // namespace dmkit #endif //DMKIT_THREAD_DATA_BASE_H ================================================ FILE: src/request_context.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "request_context.h" namespace dmkit { RequestContext::RequestContext(RemoteServiceManager* remote_service_manager, const std::string& qid, const std::unordered_map& params) : _remote_service_manager(remote_service_manager), _qid(qid), _params(params) { } RequestContext::~RequestContext() { } const RemoteServiceManager* RequestContext::remote_service_manager() const { return _remote_service_manager; } const std::string& RequestContext::qid() const { return _qid; } const std::unordered_map& RequestContext::params() const { return _params; } bool RequestContext::set_param_value(const std::string& param_name, const std::string& value) { this->_params[param_name] = value; return true; } bool RequestContext::try_get_param(const std::string& param_name, std::string& value) const { value.clear(); auto search = this->_params.find(param_name); if (search == this->_params.end()) { return false; } value = search->second; return true; } } // namespace dmkit ================================================ FILE: src/request_context.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_REQUEST_CONTEXT_H #define DMKIT_REQUEST_CONTEXT_H #include #include #include "remote_service_manager.h" namespace dmkit { // Request context for user functions to access, // including request parameters and an remote_service_manager instance class RequestContext { public: RequestContext(RemoteServiceManager* remote_service_manager, const std::string& qid, const std::unordered_map& params); ~RequestContext(); const RemoteServiceManager* remote_service_manager() const; const std::string& qid() const; const std::unordered_map& params() const; bool set_param_value(const std::string& param_name, const std::string& value); bool try_get_param(const std::string& param_name, std::string& value) const; private: RemoteServiceManager* _remote_service_manager; std::string _qid; std::unordered_map _params; }; } // namespace dmkit #endif //DMKIT_THREAD_DATA_BASE_H ================================================ FILE: src/server.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include "app_container.h" #include "app_log.h" #include "brpc.h" #include "http.pb.h" DEFINE_int32(port, -1, "TCP Port of this server"); DEFINE_int32(internal_port, -1, "Only allow builtin services at this port"); DEFINE_int32(idle_timeout_s, -1, "Connection will be closed if there is no read/write operations during this time"); DEFINE_int32(max_concurrency, 0, "Limit of requests processing in parallel"); DEFINE_string(url_path, "", "Url path of the app"); DEFINE_bool(log_to_file, false, "Log to file"); namespace dmkit { // Service with static path. class HttpServiceImpl : public HttpService { public: HttpServiceImpl() {}; virtual ~HttpServiceImpl() noexcept {}; int init() { return this->_app_container.load_application(); } ThreadLocalDataFactory* get_thread_local_data_factory() { return this->_app_container.get_thread_local_data_factory(); } void run(google::protobuf::RpcController* cntl_base, const HttpRequest*, HttpResponse*, google::protobuf::Closure* done) { // This object helps you to call done->Run() in RAII style. If you need // to process the request asynchronously, pass done_guard.release(). BRPC_NAMESPACE::ClosureGuard done_guard(done); BRPC_NAMESPACE::Controller* cntl = static_cast(cntl_base); this->_app_container.run(cntl); } private: AppContainer _app_container; }; } // namespace dmkit int main(int argc, char* argv[]) { // Parse gflags. We recommend you to use gflags as well. GFLAGS_NS::SetCommandLineOption("flagfile", "conf/gflags.conf"); GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true); #ifdef BUTIL_ENABLE_COMLOG_SINK if (FLAGS_log_to_file) { if (logging::ComlogSink::GetInstance()->SetupFromConfig("conf/log.conf") != 0) { APP_LOG(ERROR) << "Fail to setup comlog from conf/log.conf"; return -1; } } #endif // Generally you only need one Server. BRPC_NAMESPACE::Server server; dmkit::HttpServiceImpl http_svc; if (http_svc.init() != 0) { APP_LOG(ERROR) << "Fail to init http_svc"; return -1; } // Add services into server. Notice the second parameter, because the // service is put on stack, we don't want server to delete it, otherwise // use baidu::rpc::SERVER_OWNS_SERVICE. std::string mapping = FLAGS_url_path + " => run"; if (server.AddService(&http_svc, BRPC_NAMESPACE::SERVER_DOESNT_OWN_SERVICE, mapping.c_str()) != 0) { APP_LOG(ERROR) << "Fail to add http_svc"; return -1; } // Start the server. BRPC_NAMESPACE::ServerOptions options; options.idle_timeout_sec = FLAGS_idle_timeout_s; options.max_concurrency = FLAGS_max_concurrency; options.internal_port = FLAGS_internal_port; options.thread_local_data_factory = http_svc.get_thread_local_data_factory(); APP_LOG(TRACE) << "Starting server..."; if (server.Start(FLAGS_port, &options) != 0) { APP_LOG(ERROR) << "Fail to start server"; return -1; } // Wait until Ctrl-C is pressed, then Stop() and Join() the server. server.RunUntilAskedToQuit(); return 0; } ================================================ FILE: src/thirdparty/rapidjson/allocators.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ALLOCATORS_H_ #define RAPIDJSON_ALLOCATORS_H_ #include "rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // Allocator /*! \class rapidjson::Allocator \brief Concept for allocating, resizing and freeing memory block. Note that Malloc() and Realloc() are non-static but Free() is static. So if an allocator need to support Free(), it needs to put its pointer in the header of memory block. \code concept Allocator { static const bool kNeedFree; //!< Whether this allocator needs to call Free(). // Allocate a memory block. // \param size of the memory block in bytes. // \returns pointer to the memory block. void* Malloc(size_t size); // Resize a memory block. // \param originalPtr The pointer to current memory block. Null pointer is permitted. // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) // \param newSize the new size in bytes. void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); // Free a memory block. // \param pointer to the memory block. Null pointer is permitted. static void Free(void *ptr); }; \endcode */ /////////////////////////////////////////////////////////////////////////////// // CrtAllocator //! C-runtime library allocator. /*! This class is just wrapper for standard C library memory routines. \note implements Allocator concept */ class CrtAllocator { public: static const bool kNeedFree = true; void* Malloc(size_t size) { if (size) // behavior of malloc(0) is implementation defined. return std::malloc(size); else return NULL; // standardize to returning NULL. } void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { (void)originalSize; if (newSize == 0) { std::free(originalPtr); return NULL; } return std::realloc(originalPtr, newSize); } static void Free(void *ptr) { std::free(ptr); } }; /////////////////////////////////////////////////////////////////////////////// // MemoryPoolAllocator //! Default memory allocator used by the parser and DOM. /*! This allocator allocate memory blocks from pre-allocated memory chunks. It does not free memory blocks. And Realloc() only allocate new memory. The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. User may also supply a buffer as the first chunk. If the user-buffer is full then additional chunks are allocated by BaseAllocator. The user-buffer is not deallocated by this allocator. \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. \note implements Allocator concept */ template class MemoryPoolAllocator { public: static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) //! Constructor with chunkSize. /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. \param baseAllocator The allocator for allocating memory chunks. */ MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) { } //! Constructor with user-supplied buffer. /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. The user buffer will not be deallocated when this allocator is destructed. \param buffer User supplied buffer. \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. \param baseAllocator The allocator for allocating memory chunks. */ MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) { RAPIDJSON_ASSERT(buffer != 0); RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); chunkHead_ = reinterpret_cast(buffer); chunkHead_->capacity = size - sizeof(ChunkHeader); chunkHead_->size = 0; chunkHead_->next = 0; } //! Destructor. /*! This deallocates all memory chunks, excluding the user-supplied buffer. */ ~MemoryPoolAllocator() { Clear(); RAPIDJSON_DELETE(ownBaseAllocator_); } //! Deallocates all memory chunks, excluding the user-supplied buffer. void Clear() { while (chunkHead_ && chunkHead_ != userBuffer_) { ChunkHeader* next = chunkHead_->next; baseAllocator_->Free(chunkHead_); chunkHead_ = next; } if (chunkHead_ && chunkHead_ == userBuffer_) chunkHead_->size = 0; // Clear user buffer } //! Computes the total capacity of allocated memory chunks. /*! \return total capacity in bytes. */ size_t Capacity() const { size_t capacity = 0; for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) capacity += c->capacity; return capacity; } //! Computes the memory blocks allocated. /*! \return total used bytes. */ size_t Size() const { size_t size = 0; for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) size += c->size; return size; } //! Allocates a memory block. (concept Allocator) void* Malloc(size_t size) { if (!size) return NULL; size = RAPIDJSON_ALIGN(size); if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) return NULL; void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; chunkHead_->size += size; return buffer; } //! Resizes a memory block (concept Allocator) void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { if (originalPtr == 0) return Malloc(newSize); if (newSize == 0) return NULL; originalSize = RAPIDJSON_ALIGN(originalSize); newSize = RAPIDJSON_ALIGN(newSize); // Do not shrink if new size is smaller than original if (originalSize >= newSize) return originalPtr; // Simply expand it if it is the last allocation and there is sufficient space if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { size_t increment = static_cast(newSize - originalSize); if (chunkHead_->size + increment <= chunkHead_->capacity) { chunkHead_->size += increment; return originalPtr; } } // Realloc process: allocate and copy memory, do not free original buffer. if (void* newBuffer = Malloc(newSize)) { if (originalSize) std::memcpy(newBuffer, originalPtr, originalSize); return newBuffer; } else return NULL; } //! Frees a memory block (concept Allocator) static void Free(void *ptr) { (void)ptr; } // Do nothing private: //! Copy constructor is not permitted. MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; //! Copy assignment operator is not permitted. MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; //! Creates a new chunk. /*! \param capacity Capacity of the chunk in bytes. \return true if success. */ bool AddChunk(size_t capacity) { if (!baseAllocator_) ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { chunk->capacity = capacity; chunk->size = 0; chunk->next = chunkHead_; chunkHead_ = chunk; return true; } else return false; } static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. //! Chunk header for perpending to each chunk. /*! Chunks are stored as a singly linked list. */ struct ChunkHeader { size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). size_t size; //!< Current size of allocated memory in bytes. ChunkHeader *next; //!< Next chunk in the linked list. }; ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. void *userBuffer_; //!< User supplied buffer. BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. }; RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_ENCODINGS_H_ ================================================ FILE: src/thirdparty/rapidjson/document.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_DOCUMENT_H_ #define RAPIDJSON_DOCUMENT_H_ /*! \file document.h */ #include "reader.h" #include "internal/meta.h" #include "internal/strfunc.h" #include "memorystream.h" #include "encodedstream.h" #include // placement new #include RAPIDJSON_DIAG_PUSH #ifdef _MSC_VER RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data #endif #ifdef __clang__ RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(switch-enum) RAPIDJSON_DIAG_OFF(c++98-compat) #endif #ifdef __GNUC__ RAPIDJSON_DIAG_OFF(effc++) #if __GNUC__ >= 6 RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions #endif #endif // __GNUC__ #ifndef RAPIDJSON_NOMEMBERITERATORCLASS #include // std::iterator, std::random_access_iterator_tag #endif #if RAPIDJSON_HAS_CXX11_RVALUE_REFS #include // std::move #endif RAPIDJSON_NAMESPACE_BEGIN // Forward declaration. template class GenericValue; template class GenericDocument; //! Name-value pair in a JSON object value. /*! This class was internal to GenericValue. It used to be a inner struct. But a compiler (IBM XL C/C++ for AIX) have reported to have problem with that so it moved as a namespace scope struct. https://code.google.com/p/rapidjson/issues/detail?id=64 */ template struct GenericMember { GenericValue name; //!< name of member (must be a string) GenericValue value; //!< value of member. }; /////////////////////////////////////////////////////////////////////////////// // GenericMemberIterator #ifndef RAPIDJSON_NOMEMBERITERATORCLASS //! (Constant) member iterator for a JSON object value /*! \tparam Const Is this a constant iterator? \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) \tparam Allocator Allocator type for allocating memory of object, array and string. This class implements a Random Access Iterator for GenericMember elements of a GenericValue, see ISO/IEC 14882:2003(E) C++ standard, 24.1 [lib.iterator.requirements]. \note This iterator implementation is mainly intended to avoid implicit conversions from iterator values to \c NULL, e.g. from GenericValue::FindMember. \note Define \c RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a pointer-based implementation, if your platform doesn't provide the C++ header. \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator */ template class GenericMemberIterator : public std::iterator >::Type> { friend class GenericValue; template friend class GenericMemberIterator; typedef GenericMember PlainType; typedef typename internal::MaybeAddConst::Type ValueType; typedef std::iterator BaseType; public: //! Iterator type itself typedef GenericMemberIterator Iterator; //! Constant iterator type typedef GenericMemberIterator ConstIterator; //! Non-constant iterator type typedef GenericMemberIterator NonConstIterator; //! Pointer to (const) GenericMember typedef typename BaseType::pointer Pointer; //! Reference to (const) GenericMember typedef typename BaseType::reference Reference; //! Signed integer type (e.g. \c ptrdiff_t) typedef typename BaseType::difference_type DifferenceType; //! Default constructor (singular value) /*! Creates an iterator pointing to no element. \note All operations, except for comparisons, are undefined on such values. */ GenericMemberIterator() : ptr_() {} //! Iterator conversions to more const /*! \param it (Non-const) iterator to copy from Allows the creation of an iterator from another GenericMemberIterator that is "less const". Especially, creating a non-constant iterator from a constant iterator are disabled: \li const -> non-const (not ok) \li const -> const (ok) \li non-const -> const (ok) \li non-const -> non-const (ok) \note If the \c Const template parameter is already \c false, this constructor effectively defines a regular copy-constructor. Otherwise, the copy constructor is implicitly defined. */ GenericMemberIterator(const NonConstIterator & it) : ptr_(it.ptr_) {} Iterator& operator=(const NonConstIterator & it) { ptr_ = it.ptr_; return *this; } //! @name stepping //@{ Iterator& operator++(){ ++ptr_; return *this; } Iterator& operator--(){ --ptr_; return *this; } Iterator operator++(int){ Iterator old(*this); ++ptr_; return old; } Iterator operator--(int){ Iterator old(*this); --ptr_; return old; } //@} //! @name increment/decrement //@{ Iterator operator+(DifferenceType n) const { return Iterator(ptr_+n); } Iterator operator-(DifferenceType n) const { return Iterator(ptr_-n); } Iterator& operator+=(DifferenceType n) { ptr_+=n; return *this; } Iterator& operator-=(DifferenceType n) { ptr_-=n; return *this; } //@} //! @name relations //@{ bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } //@} //! @name dereference //@{ Reference operator*() const { return *ptr_; } Pointer operator->() const { return ptr_; } Reference operator[](DifferenceType n) const { return ptr_[n]; } //@} //! Distance DifferenceType operator-(ConstIterator that) const { return ptr_-that.ptr_; } private: //! Internal constructor from plain pointer explicit GenericMemberIterator(Pointer p) : ptr_(p) {} Pointer ptr_; //!< raw pointer }; #else // RAPIDJSON_NOMEMBERITERATORCLASS // class-based member iterator implementation disabled, use plain pointers template struct GenericMemberIterator; //! non-const GenericMemberIterator template struct GenericMemberIterator { //! use plain pointer as iterator type typedef GenericMember* Iterator; }; //! const GenericMemberIterator template struct GenericMemberIterator { //! use plain const pointer as iterator type typedef const GenericMember* Iterator; }; #endif // RAPIDJSON_NOMEMBERITERATORCLASS /////////////////////////////////////////////////////////////////////////////// // GenericStringRef //! Reference to a constant string (not taking a copy) /*! \tparam CharType character type of the string This helper class is used to automatically infer constant string references for string literals, especially from \c const \b (!) character arrays. The main use is for creating JSON string values without copying the source string via an \ref Allocator. This requires that the referenced string pointers have a sufficient lifetime, which exceeds the lifetime of the associated GenericValue. \b Example \code Value v("foo"); // ok, no need to copy & calculate length const char foo[] = "foo"; v.SetString(foo); // ok const char* bar = foo; // Value x(bar); // not ok, can't rely on bar's lifetime Value x(StringRef(bar)); // lifetime explicitly guaranteed by user Value y(StringRef(bar, 3)); // ok, explicitly pass length \endcode \see StringRef, GenericValue::SetString */ template struct GenericStringRef { typedef CharType Ch; //!< character type of the string //! Create string reference from \c const character array #ifndef __clang__ // -Wdocumentation /*! This constructor implicitly creates a constant string reference from a \c const character array. It has better performance than \ref StringRef(const CharType*) by inferring the string \ref length from the array length, and also supports strings containing null characters. \tparam N length of the string, automatically inferred \param str Constant character array, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \post \ref s == str \note Constant complexity. \note There is a hidden, private overload to disallow references to non-const character arrays to be created via this constructor. By this, e.g. function-scope arrays used to be filled via \c snprintf are excluded from consideration. In such cases, the referenced string should be \b copied to the GenericValue instead. */ #endif template GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT : s(str), length(N-1) {} //! Explicitly create string reference from \c const character pointer #ifndef __clang__ // -Wdocumentation /*! This constructor can be used to \b explicitly create a reference to a constant string pointer. \see StringRef(const CharType*) \param str Constant character pointer, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \post \ref s == str \note There is a hidden, private overload to disallow references to non-const character arrays to be created via this constructor. By this, e.g. function-scope arrays used to be filled via \c snprintf are excluded from consideration. In such cases, the referenced string should be \b copied to the GenericValue instead. */ #endif explicit GenericStringRef(const CharType* str) : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != 0); } //! Create constant string reference from pointer and length #ifndef __clang__ // -Wdocumentation /*! \param str constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \param len length of the string, excluding the trailing NULL terminator \post \ref s == str && \ref length == len \note Constant complexity. */ #endif GenericStringRef(const CharType* str, SizeType len) : s(str), length(len) { RAPIDJSON_ASSERT(s != 0); } GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; } //! implicit conversion to plain CharType pointer operator const Ch *() const { return s; } const Ch* const s; //!< plain CharType pointer const SizeType length; //!< length of the string (excluding the trailing NULL terminator) private: //! Disallow construction from non-const array template GenericStringRef(CharType (&str)[N]) /* = delete */; }; //! Mark a character pointer as constant string /*! Mark a plain character pointer as a "string literal". This function can be used to avoid copying a character string to be referenced as a value in a JSON GenericValue object, if the string's lifetime is known to be valid long enough. \tparam CharType Character type of the string \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \return GenericStringRef string reference object \relatesalso GenericStringRef \see GenericValue::GenericValue(StringRefType), GenericValue::operator=(StringRefType), GenericValue::SetString(StringRefType), GenericValue::PushBack(StringRefType, Allocator&), GenericValue::AddMember */ template inline GenericStringRef StringRef(const CharType* str) { return GenericStringRef(str, internal::StrLen(str)); } //! Mark a character pointer as constant string /*! Mark a plain character pointer as a "string literal". This function can be used to avoid copying a character string to be referenced as a value in a JSON GenericValue object, if the string's lifetime is known to be valid long enough. This version has better performance with supplied length, and also supports string containing null characters. \tparam CharType character type of the string \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \param length The length of source string. \return GenericStringRef string reference object \relatesalso GenericStringRef */ template inline GenericStringRef StringRef(const CharType* str, size_t length) { return GenericStringRef(str, SizeType(length)); } #if RAPIDJSON_HAS_STDSTRING //! Mark a string object as constant string /*! Mark a string object (e.g. \c std::string) as a "string literal". This function can be used to avoid copying a string to be referenced as a value in a JSON GenericValue object, if the string's lifetime is known to be valid long enough. \tparam CharType character type of the string \param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue \return GenericStringRef string reference object \relatesalso GenericStringRef \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ template inline GenericStringRef StringRef(const std::basic_string& str) { return GenericStringRef(str.data(), SizeType(str.size())); } #endif /////////////////////////////////////////////////////////////////////////////// // GenericValue type traits namespace internal { template struct IsGenericValueImpl : FalseType {}; // select candidates according to nested encoding and allocator types template struct IsGenericValueImpl::Type, typename Void::Type> : IsBaseOf, T>::Type {}; // helper to match arbitrary GenericValue instantiations, including derived classes template struct IsGenericValue : IsGenericValueImpl::Type {}; } // namespace internal /////////////////////////////////////////////////////////////////////////////// // TypeHelper namespace internal { template struct TypeHelper {}; template struct TypeHelper { static bool Is(const ValueType& v) { return v.IsBool(); } static bool Get(const ValueType& v) { return v.GetBool(); } static ValueType& Set(ValueType& v, bool data) { return v.SetBool(data); } static ValueType& Set(ValueType& v, bool data, typename ValueType::AllocatorType&) { return v.SetBool(data); } }; template struct TypeHelper { static bool Is(const ValueType& v) { return v.IsInt(); } static int Get(const ValueType& v) { return v.GetInt(); } static ValueType& Set(ValueType& v, int data) { return v.SetInt(data); } static ValueType& Set(ValueType& v, int data, typename ValueType::AllocatorType&) { return v.SetInt(data); } }; template struct TypeHelper { static bool Is(const ValueType& v) { return v.IsUint(); } static unsigned Get(const ValueType& v) { return v.GetUint(); } static ValueType& Set(ValueType& v, unsigned data) { return v.SetUint(data); } static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } }; template struct TypeHelper { static bool Is(const ValueType& v) { return v.IsInt64(); } static int64_t Get(const ValueType& v) { return v.GetInt64(); } static ValueType& Set(ValueType& v, int64_t data) { return v.SetInt64(data); } static ValueType& Set(ValueType& v, int64_t data, typename ValueType::AllocatorType&) { return v.SetInt64(data); } }; template struct TypeHelper { static bool Is(const ValueType& v) { return v.IsUint64(); } static uint64_t Get(const ValueType& v) { return v.GetUint64(); } static ValueType& Set(ValueType& v, uint64_t data) { return v.SetUint64(data); } static ValueType& Set(ValueType& v, uint64_t data, typename ValueType::AllocatorType&) { return v.SetUint64(data); } }; template struct TypeHelper { static bool Is(const ValueType& v) { return v.IsDouble(); } static double Get(const ValueType& v) { return v.GetDouble(); } static ValueType& Set(ValueType& v, double data) { return v.SetDouble(data); } static ValueType& Set(ValueType& v, double data, typename ValueType::AllocatorType&) { return v.SetDouble(data); } }; template struct TypeHelper { static bool Is(const ValueType& v) { return v.IsFloat(); } static float Get(const ValueType& v) { return v.GetFloat(); } static ValueType& Set(ValueType& v, float data) { return v.SetFloat(data); } static ValueType& Set(ValueType& v, float data, typename ValueType::AllocatorType&) { return v.SetFloat(data); } }; template struct TypeHelper { typedef const typename ValueType::Ch* StringType; static bool Is(const ValueType& v) { return v.IsString(); } static StringType Get(const ValueType& v) { return v.GetString(); } static ValueType& Set(ValueType& v, const StringType data) { return v.SetString(typename ValueType::StringRefType(data)); } static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } }; #if RAPIDJSON_HAS_STDSTRING template struct TypeHelper > { typedef std::basic_string StringType; static bool Is(const ValueType& v) { return v.IsString(); } static StringType Get(const ValueType& v) { return StringType(v.GetString(), v.GetStringLength()); } static ValueType& Set(ValueType& v, const StringType& data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); } }; #endif template struct TypeHelper { typedef typename ValueType::Array ArrayType; static bool Is(const ValueType& v) { return v.IsArray(); } static ArrayType Get(ValueType& v) { return v.GetArray(); } static ValueType& Set(ValueType& v, ArrayType data) { return v = data; } static ValueType& Set(ValueType& v, ArrayType data, typename ValueType::AllocatorType&) { return v = data; } }; template struct TypeHelper { typedef typename ValueType::ConstArray ArrayType; static bool Is(const ValueType& v) { return v.IsArray(); } static ArrayType Get(const ValueType& v) { return v.GetArray(); } }; template struct TypeHelper { typedef typename ValueType::Object ObjectType; static bool Is(const ValueType& v) { return v.IsObject(); } static ObjectType Get(ValueType& v) { return v.GetObject(); } static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { v = data; } }; template struct TypeHelper { typedef typename ValueType::ConstObject ObjectType; static bool Is(const ValueType& v) { return v.IsObject(); } static ObjectType Get(const ValueType& v) { return v.GetObject(); } }; } // namespace internal // Forward declarations template class GenericArray; template class GenericObject; /////////////////////////////////////////////////////////////////////////////// // GenericValue //! Represents a JSON value. Use Value for UTF8 encoding and default allocator. /*! A JSON value can be one of 7 types. This class is a variant type supporting these types. Use the Value if UTF8 and default allocator \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) \tparam Allocator Allocator type for allocating memory of object, array and string. */ template > class GenericValue { public: //! Name-value pair in an object. typedef GenericMember Member; typedef Encoding EncodingType; //!< Encoding type from template parameter. typedef Allocator AllocatorType; //!< Allocator type from template parameter. typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. typedef GenericStringRef StringRefType; //!< Reference to a constant string typedef typename GenericMemberIterator::Iterator MemberIterator; //!< Member iterator for iterating in object. typedef typename GenericMemberIterator::Iterator ConstMemberIterator; //!< Constant member iterator for iterating in object. typedef GenericValue* ValueIterator; //!< Value iterator for iterating in array. typedef const GenericValue* ConstValueIterator; //!< Constant value iterator for iterating in array. typedef GenericValue ValueType; //!< Value type of itself. typedef GenericArray Array; typedef GenericArray ConstArray; typedef GenericObject Object; typedef GenericObject ConstObject; //!@name Constructors and destructor. //@{ //! Default constructor creates a null value. GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) { rhs.data_.f.flags = kNullFlag; // give up contents } #endif private: //! Copy constructor is not permitted. GenericValue(const GenericValue& rhs); #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Moving from a GenericDocument is not permitted. template GenericValue(GenericDocument&& rhs); //! Move assignment from a GenericDocument is not permitted. template GenericValue& operator=(GenericDocument&& rhs); #endif public: //! Constructor with JSON value type. /*! This creates a Value of specified type with default content. \param type Type of the value. \note Default content for number is zero. */ explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { static const uint16_t defaultFlags[7] = { kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, kNumberAnyFlag }; RAPIDJSON_ASSERT(type <= kNumberType); data_.f.flags = defaultFlags[type]; // Use ShortString to store empty string. if (type == kStringType) data_.ss.SetLength(0); } //! Explicit copy constructor (with allocator) /*! Creates a copy of a Value by using the given Allocator \tparam SourceAllocator allocator of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). \see CopyFrom() */ template< typename SourceAllocator > GenericValue(const GenericValue& rhs, Allocator & allocator); //! Constructor for boolean value. /*! \param b Boolean value \note This constructor is limited to \em real boolean values and rejects implicitly converted types like arbitrary pointers. Use an explicit cast to \c bool, if you want to construct a boolean JSON value in such cases. */ #ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen template explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472 #else explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT #endif : data_() { // safe-guard against failing SFINAE RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value)); data_.f.flags = b ? kTrueFlag : kFalseFlag; } //! Constructor for int value. explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() { data_.n.i64 = i; data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag; } //! Constructor for unsigned value. explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() { data_.n.u64 = u; data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag); } //! Constructor for int64_t value. explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() { data_.n.i64 = i64; data_.f.flags = kNumberInt64Flag; if (i64 >= 0) { data_.f.flags |= kNumberUint64Flag; if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) data_.f.flags |= kUintFlag; if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) data_.f.flags |= kIntFlag; } else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) data_.f.flags |= kIntFlag; } //! Constructor for uint64_t value. explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() { data_.n.u64 = u64; data_.f.flags = kNumberUint64Flag; if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000))) data_.f.flags |= kInt64Flag; if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000))) data_.f.flags |= kUintFlag; if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000))) data_.f.flags |= kIntFlag; } //! Constructor for double value. explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } //! Constructor for constant string (i.e. do not make a copy of string) GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } //! Constructor for constant string (i.e. do not make a copy of string) explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); } //! Constructor for copy-string (i.e. do make a copy of string) GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); } //! Constructor for copy-string (i.e. do make a copy of string) GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } #if RAPIDJSON_HAS_STDSTRING //! Constructor for copy-string from a string object (i.e. do make a copy of string) /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); } #endif //! Constructor for Array. /*! \param a An array obtained by \c GetArray(). \note \c Array is always pass-by-value. \note the source array is moved into this value and the sourec array becomes empty. */ GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) { a.value_.data_ = Data(); a.value_.data_.f.flags = kArrayFlag; } //! Constructor for Object. /*! \param o An object obtained by \c GetObject(). \note \c Object is always pass-by-value. \note the source object is moved into this value and the sourec object becomes empty. */ GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) { o.value_.data_ = Data(); o.value_.data_.f.flags = kObjectFlag; } //! Destructor. /*! Need to destruct elements of array, members of object, or copy-string. */ ~GenericValue() { if (Allocator::kNeedFree) { // Shortcut by Allocator's trait switch(data_.f.flags) { case kArrayFlag: { GenericValue* e = GetElementsPointer(); for (GenericValue* v = e; v != e + data_.a.size; ++v) v->~GenericValue(); Allocator::Free(e); } break; case kObjectFlag: for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) m->~Member(); Allocator::Free(GetMembersPointer()); break; case kCopyStringFlag: Allocator::Free(const_cast(GetStringPointer())); break; default: break; // Do nothing for other types. } } } //@} //!@name Assignment operators //@{ //! Assignment with move semantics. /*! \param rhs Source of the assignment. It will become a null value after assignment. */ GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { RAPIDJSON_ASSERT(this != &rhs); this->~GenericValue(); RawAssign(rhs); return *this; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move assignment in C++11 GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT { return *this = rhs.Move(); } #endif //! Assignment of constant string reference (no copy) /*! \param str Constant string reference to be assigned \note This overload is needed to avoid clashes with the generic primitive type assignment overload below. \see GenericStringRef, operator=(T) */ GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT { GenericValue s(str); return *this = s; } //! Assignment with primitive types. /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t \param value The value to be assigned. \note The source type \c T explicitly disallows all pointer types, especially (\c const) \ref Ch*. This helps avoiding implicitly referencing character strings with insufficient lifetime, use \ref SetString(const Ch*, Allocator&) (for copying) or \ref StringRef() (to explicitly mark the pointer as constant) instead. All other pointer types would implicitly convert to \c bool, use \ref SetBool() instead. */ template RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&)) operator=(T value) { GenericValue v(value); return *this = v; } //! Deep-copy assignment from Value /*! Assigns a \b copy of the Value to the current Value object \tparam SourceAllocator Allocator type of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator to use for copying */ template GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); this->~GenericValue(); new (this) GenericValue(rhs, allocator); return *this; } //! Exchange the contents of this value with those of other. /*! \param other Another value. \note Constant complexity. */ GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT { GenericValue temp; temp.RawAssign(*this); RawAssign(other); other.RawAssign(temp); return *this; } //! free-standing swap function helper /*! Helper function to enable support for common swap implementation pattern based on \c std::swap: \code void swap(MyClass& a, MyClass& b) { using std::swap; swap(a.value, b.value); // ... } \endcode \see Swap() */ friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } //! Prepare Value for move semantics /*! \return *this */ GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; } //@} //!@name Equal-to and not-equal-to operators //@{ //! Equal-to operator /*! \note If an object contains duplicated named member, comparing equality with any object is always \c false. \note Linear time complexity (number of all values in the subtree and total lengths of all strings). */ template bool operator==(const GenericValue& rhs) const { typedef GenericValue RhsType; if (GetType() != rhs.GetType()) return false; switch (GetType()) { case kObjectType: // Warning: O(n^2) inner-loop if (data_.o.size != rhs.data_.o.size) return false; for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) return false; } return true; case kArrayType: if (data_.a.size != rhs.data_.a.size) return false; for (SizeType i = 0; i < data_.a.size; i++) if ((*this)[i] != rhs[i]) return false; return true; case kStringType: return StringEqual(rhs); case kNumberType: if (IsDouble() || rhs.IsDouble()) { double a = GetDouble(); // May convert from integer to double. double b = rhs.GetDouble(); // Ditto return a >= b && a <= b; // Prevent -Wfloat-equal } else return data_.n.u64 == rhs.data_.n.u64; default: return true; } } //! Equal-to operator with const C-string pointer bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); } #if RAPIDJSON_HAS_STDSTRING //! Equal-to operator with string object /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); } #endif //! Equal-to operator with primitive types /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } //! Not-equal-to operator /*! \return !(*this == rhs) */ template bool operator!=(const GenericValue& rhs) const { return !(*this == rhs); } //! Not-equal-to operator with const C-string pointer bool operator!=(const Ch* rhs) const { return !(*this == rhs); } //! Not-equal-to operator with arbitrary types /*! \return !(*this == rhs) */ template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } //! Equal-to operator with arbitrary types (symmetric version) /*! \return (rhs == lhs) */ template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; } //! Not-Equal-to operator with arbitrary types (symmetric version) /*! \return !(rhs == lhs) */ template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } //@} //!@name Type //@{ Type GetType() const { return static_cast(data_.f.flags & kTypeMask); } bool IsNull() const { return data_.f.flags == kNullFlag; } bool IsFalse() const { return data_.f.flags == kFalseFlag; } bool IsTrue() const { return data_.f.flags == kTrueFlag; } bool IsBool() const { return (data_.f.flags & kBoolFlag) != 0; } bool IsObject() const { return data_.f.flags == kObjectFlag; } bool IsArray() const { return data_.f.flags == kArrayFlag; } bool IsNumber() const { return (data_.f.flags & kNumberFlag) != 0; } bool IsInt() const { return (data_.f.flags & kIntFlag) != 0; } bool IsUint() const { return (data_.f.flags & kUintFlag) != 0; } bool IsInt64() const { return (data_.f.flags & kInt64Flag) != 0; } bool IsUint64() const { return (data_.f.flags & kUint64Flag) != 0; } bool IsDouble() const { return (data_.f.flags & kDoubleFlag) != 0; } bool IsString() const { return (data_.f.flags & kStringFlag) != 0; } // Checks whether a number can be losslessly converted to a double. bool IsLosslessDouble() const { if (!IsNumber()) return false; if (IsUint64()) { uint64_t u = GetUint64(); volatile double d = static_cast(u); return (d >= 0.0) && (d < static_cast(std::numeric_limits::max())) && (u == static_cast(d)); } if (IsInt64()) { int64_t i = GetInt64(); volatile double d = static_cast(i); return (d >= static_cast(std::numeric_limits::min())) && (d < static_cast(std::numeric_limits::max())) && (i == static_cast(d)); } return true; // double, int, uint are always lossless } // Checks whether a number is a float (possible lossy). bool IsFloat() const { if ((data_.f.flags & kDoubleFlag) == 0) return false; double d = GetDouble(); return d >= -3.4028234e38 && d <= 3.4028234e38; } // Checks whether a number can be losslessly converted to a float. bool IsLosslessFloat() const { if (!IsNumber()) return false; double a = GetDouble(); if (a < static_cast(-std::numeric_limits::max()) || a > static_cast(std::numeric_limits::max())) return false; double b = static_cast(static_cast(a)); return a >= b && a <= b; // Prevent -Wfloat-equal } //@} //!@name Null //@{ GenericValue& SetNull() { this->~GenericValue(); new (this) GenericValue(); return *this; } //@} //!@name Bool //@{ bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; } //!< Set boolean value /*! \post IsBool() == true */ GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; } //@} //!@name Object //@{ //! Set this value as an empty object. /*! \post IsObject() == true */ GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; } //! Get the number of members in the object. SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } //! Check whether the object is empty. bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } //! Get a value from an object associated with the name. /*! \pre IsObject() == true \tparam T Either \c Ch or \c const \c Ch (template used for disambiguation with \ref operator[](SizeType)) \note In version 0.1x, if the member is not found, this function returns a null value. This makes issue 7. Since 0.2, if the name is not correct, it will assert. If user is unsure whether a member exists, user should use HasMember() first. A better approach is to use FindMember(). \note Linear time complexity. */ template RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) { GenericValue n(StringRef(name)); return (*this)[n]; } template RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; } //! Get a value from an object associated with the name. /*! \pre IsObject() == true \tparam SourceAllocator Allocator of the \c name value \note Compared to \ref operator[](T*), this version is faster because it does not need a StrLen(). And it can also handle strings with embedded null characters. \note Linear time complexity. */ template GenericValue& operator[](const GenericValue& name) { MemberIterator member = FindMember(name); if (member != MemberEnd()) return member->value; else { RAPIDJSON_ASSERT(false); // see above note // This will generate -Wexit-time-destructors in clang // static GenericValue NullValue; // return NullValue; // Use static buffer and placement-new to prevent destruction static char buffer[sizeof(GenericValue)]; return *new (buffer) GenericValue(); } } template const GenericValue& operator[](const GenericValue& name) const { return const_cast(*this)[name]; } #if RAPIDJSON_HAS_STDSTRING //! Get a value from an object associated with name (string object). GenericValue& operator[](const std::basic_string& name) { return (*this)[GenericValue(StringRef(name))]; } const GenericValue& operator[](const std::basic_string& name) const { return (*this)[GenericValue(StringRef(name))]; } #endif //! Const member iterator /*! \pre IsObject() == true */ ConstMemberIterator MemberBegin() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer()); } //! Const \em past-the-end member iterator /*! \pre IsObject() == true */ ConstMemberIterator MemberEnd() const { RAPIDJSON_ASSERT(IsObject()); return ConstMemberIterator(GetMembersPointer() + data_.o.size); } //! Member iterator /*! \pre IsObject() == true */ MemberIterator MemberBegin() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer()); } //! \em Past-the-end member iterator /*! \pre IsObject() == true */ MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } //! Check whether a member exists in the object. /*! \param name Member name to be searched. \pre IsObject() == true \return Whether a member with that name exists. \note It is better to use FindMember() directly if you need the obtain the value as well. \note Linear time complexity. */ bool HasMember(const Ch* name) const { return FindMember(name) != MemberEnd(); } #if RAPIDJSON_HAS_STDSTRING //! Check whether a member exists in the object with string object. /*! \param name Member name to be searched. \pre IsObject() == true \return Whether a member with that name exists. \note It is better to use FindMember() directly if you need the obtain the value as well. \note Linear time complexity. */ bool HasMember(const std::basic_string& name) const { return FindMember(name) != MemberEnd(); } #endif //! Check whether a member exists in the object with GenericValue name. /*! This version is faster because it does not need a StrLen(). It can also handle string with null character. \param name Member name to be searched. \pre IsObject() == true \return Whether a member with that name exists. \note It is better to use FindMember() directly if you need the obtain the value as well. \note Linear time complexity. */ template bool HasMember(const GenericValue& name) const { return FindMember(name) != MemberEnd(); } //! Find member by name. /*! \param name Member name to be searched. \pre IsObject() == true \return Iterator to member, if it exists. Otherwise returns \ref MemberEnd(). \note Earlier versions of Rapidjson returned a \c NULL pointer, in case the requested member doesn't exist. For consistency with e.g. \c std::map, this has been changed to MemberEnd() now. \note Linear time complexity. */ MemberIterator FindMember(const Ch* name) { GenericValue n(StringRef(name)); return FindMember(n); } ConstMemberIterator FindMember(const Ch* name) const { return const_cast(*this).FindMember(name); } //! Find member by name. /*! This version is faster because it does not need a StrLen(). It can also handle string with null character. \param name Member name to be searched. \pre IsObject() == true \return Iterator to member, if it exists. Otherwise returns \ref MemberEnd(). \note Earlier versions of Rapidjson returned a \c NULL pointer, in case the requested member doesn't exist. For consistency with e.g. \c std::map, this has been changed to MemberEnd() now. \note Linear time complexity. */ template MemberIterator FindMember(const GenericValue& name) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); MemberIterator member = MemberBegin(); for ( ; member != MemberEnd(); ++member) if (name.StringEqual(member->name)) break; return member; } template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } #if RAPIDJSON_HAS_STDSTRING //! Find member by string object name. /*! \param name Member name to be searched. \pre IsObject() == true \return Iterator to member, if it exists. Otherwise returns \ref MemberEnd(). */ MemberIterator FindMember(const std::basic_string& name) { return FindMember(GenericValue(StringRef(name))); } ConstMemberIterator FindMember(const std::basic_string& name) const { return FindMember(GenericValue(StringRef(name))); } #endif //! Add a member (name-value pair) to the object. /*! \param name A string value as name of member. \param value Value of any type. \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \note The ownership of \c name and \c value will be transferred to this object on success. \pre IsObject() && name.IsString() \post name.IsNull() && value.IsNull() \note Amortized Constant time complexity. */ GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); ObjectData& o = data_.o; if (o.size >= o.capacity) { if (o.capacity == 0) { o.capacity = kDefaultObjectCapacity; SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); } else { SizeType oldCapacity = o.capacity; o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); } } Member* members = GetMembersPointer(); members[o.size].name.RawAssign(name); members[o.size].value.RawAssign(value); o.size++; return *this; } //! Add a constant string value as member (name-value pair) to the object. /*! \param name A string value as name of member. \param value constant string reference as value of member. \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \pre IsObject() \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. \note Amortized Constant time complexity. */ GenericValue& AddMember(GenericValue& name, StringRefType value, Allocator& allocator) { GenericValue v(value); return AddMember(name, v, allocator); } #if RAPIDJSON_HAS_STDSTRING //! Add a string object as member (name-value pair) to the object. /*! \param name A string value as name of member. \param value constant string reference as value of member. \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \pre IsObject() \note This overload is needed to avoid clashes with the generic primitive type AddMember(GenericValue&,T,Allocator&) overload below. \note Amortized Constant time complexity. */ GenericValue& AddMember(GenericValue& name, std::basic_string& value, Allocator& allocator) { GenericValue v(value, allocator); return AddMember(name, v, allocator); } #endif //! Add any primitive value as member (name-value pair) to the object. /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t \param name A string value as name of member. \param value Value of primitive type \c T as value of member \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \pre IsObject() \note The source type \c T explicitly disallows all pointer types, especially (\c const) \ref Ch*. This helps avoiding implicitly referencing character strings with insufficient lifetime, use \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref AddMember(StringRefType, StringRefType, Allocator&). All other pointer types would implicitly convert to \c bool, use an explicit cast instead, if needed. \note Amortized Constant time complexity. */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) AddMember(GenericValue& name, T value, Allocator& allocator) { GenericValue v(value); return AddMember(name, v, allocator); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS GenericValue& AddMember(GenericValue&& name, GenericValue&& value, Allocator& allocator) { return AddMember(name, value, allocator); } GenericValue& AddMember(GenericValue&& name, GenericValue& value, Allocator& allocator) { return AddMember(name, value, allocator); } GenericValue& AddMember(GenericValue& name, GenericValue&& value, Allocator& allocator) { return AddMember(name, value, allocator); } GenericValue& AddMember(StringRefType name, GenericValue&& value, Allocator& allocator) { GenericValue n(name); return AddMember(n, value, allocator); } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Add a member (name-value pair) to the object. /*! \param name A constant string reference as name of member. \param value Value of any type. \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \note The ownership of \c value will be transferred to this object on success. \pre IsObject() \post value.IsNull() \note Amortized Constant time complexity. */ GenericValue& AddMember(StringRefType name, GenericValue& value, Allocator& allocator) { GenericValue n(name); return AddMember(n, value, allocator); } //! Add a constant string value as member (name-value pair) to the object. /*! \param name A constant string reference as name of member. \param value constant string reference as value of member. \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \pre IsObject() \note This overload is needed to avoid clashes with the generic primitive type AddMember(StringRefType,T,Allocator&) overload below. \note Amortized Constant time complexity. */ GenericValue& AddMember(StringRefType name, StringRefType value, Allocator& allocator) { GenericValue v(value); return AddMember(name, v, allocator); } //! Add any primitive value as member (name-value pair) to the object. /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t \param name A constant string reference as name of member. \param value Value of primitive type \c T as value of member \param allocator Allocator for reallocating memory. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \pre IsObject() \note The source type \c T explicitly disallows all pointer types, especially (\c const) \ref Ch*. This helps avoiding implicitly referencing character strings with insufficient lifetime, use \ref AddMember(StringRefType, GenericValue&, Allocator&) or \ref AddMember(StringRefType, StringRefType, Allocator&). All other pointer types would implicitly convert to \c bool, use an explicit cast instead, if needed. \note Amortized Constant time complexity. */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) AddMember(StringRefType name, T value, Allocator& allocator) { GenericValue n(name); return AddMember(n, value, allocator); } //! Remove all members in the object. /*! This function do not deallocate memory in the object, i.e. the capacity is unchanged. \note Linear time complexity. */ void RemoveAllMembers() { RAPIDJSON_ASSERT(IsObject()); for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) m->~Member(); data_.o.size = 0; } //! Remove a member in object by its name. /*! \param name Name of member to be removed. \return Whether the member existed. \note This function may reorder the object members. Use \ref EraseMember(ConstMemberIterator) if you need to preserve the relative order of the remaining members. \note Linear time complexity. */ bool RemoveMember(const Ch* name) { GenericValue n(StringRef(name)); return RemoveMember(n); } #if RAPIDJSON_HAS_STDSTRING bool RemoveMember(const std::basic_string& name) { return RemoveMember(GenericValue(StringRef(name))); } #endif template bool RemoveMember(const GenericValue& name) { MemberIterator m = FindMember(name); if (m != MemberEnd()) { RemoveMember(m); return true; } else return false; } //! Remove a member in object by iterator. /*! \param m member iterator (obtained by FindMember() or MemberBegin()). \return the new iterator after removal. \note This function may reorder the object members. Use \ref EraseMember(ConstMemberIterator) if you need to preserve the relative order of the remaining members. \note Constant time complexity. */ MemberIterator RemoveMember(MemberIterator m) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(data_.o.size > 0); RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); if (data_.o.size > 1 && m != last) *m = *last; // Move the last one to this place else m->~Member(); // Only one left, just destroy --data_.o.size; return m; } //! Remove a member from an object by iterator. /*! \param pos iterator to the member to remove \pre IsObject() == true && \ref MemberBegin() <= \c pos < \ref MemberEnd() \return Iterator following the removed element. If the iterator \c pos refers to the last element, the \ref MemberEnd() iterator is returned. \note This function preserves the relative order of the remaining object members. If you do not need this, use the more efficient \ref RemoveMember(MemberIterator). \note Linear time complexity. */ MemberIterator EraseMember(ConstMemberIterator pos) { return EraseMember(pos, pos +1); } //! Remove members in the range [first, last) from an object. /*! \param first iterator to the first member to remove \param last iterator following the last member to remove \pre IsObject() == true && \ref MemberBegin() <= \c first <= \c last <= \ref MemberEnd() \return Iterator following the last removed element. \note This function preserves the relative order of the remaining object members. \note Linear time complexity. */ MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(data_.o.size > 0); RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(first >= MemberBegin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= MemberEnd()); MemberIterator pos = MemberBegin() + (first - MemberBegin()); for (MemberIterator itr = pos; itr != last; ++itr) itr->~Member(); std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); data_.o.size -= static_cast(last - first); return pos; } //! Erase a member in object by its name. /*! \param name Name of member to be removed. \return Whether the member existed. \note Linear time complexity. */ bool EraseMember(const Ch* name) { GenericValue n(StringRef(name)); return EraseMember(n); } #if RAPIDJSON_HAS_STDSTRING bool EraseMember(const std::basic_string& name) { return EraseMember(GenericValue(StringRef(name))); } #endif template bool EraseMember(const GenericValue& name) { MemberIterator m = FindMember(name); if (m != MemberEnd()) { EraseMember(m); return true; } else return false; } Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } //@} //!@name Array //@{ //! Set this value as an empty array. /*! \post IsArray == true */ GenericValue& SetArray() { this->~GenericValue(); new (this) GenericValue(kArrayType); return *this; } //! Get the number of elements in array. SizeType Size() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size; } //! Get the capacity of array. SizeType Capacity() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.capacity; } //! Check whether the array is empty. bool Empty() const { RAPIDJSON_ASSERT(IsArray()); return data_.a.size == 0; } //! Remove all elements in the array. /*! This function do not deallocate memory in the array, i.e. the capacity is unchanged. \note Linear time complexity. */ void Clear() { RAPIDJSON_ASSERT(IsArray()); GenericValue* e = GetElementsPointer(); for (GenericValue* v = e; v != e + data_.a.size; ++v) v->~GenericValue(); data_.a.size = 0; } //! Get an element from array by index. /*! \pre IsArray() == true \param index Zero-based index of element. \see operator[](T*) */ GenericValue& operator[](SizeType index) { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(index < data_.a.size); return GetElementsPointer()[index]; } const GenericValue& operator[](SizeType index) const { return const_cast(*this)[index]; } //! Element iterator /*! \pre IsArray() == true */ ValueIterator Begin() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer(); } //! \em Past-the-end element iterator /*! \pre IsArray() == true */ ValueIterator End() { RAPIDJSON_ASSERT(IsArray()); return GetElementsPointer() + data_.a.size; } //! Constant element iterator /*! \pre IsArray() == true */ ConstValueIterator Begin() const { return const_cast(*this).Begin(); } //! Constant \em past-the-end element iterator /*! \pre IsArray() == true */ ConstValueIterator End() const { return const_cast(*this).End(); } //! Request the array to have enough capacity to store elements. /*! \param newCapacity The capacity that the array at least need to have. \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \note Linear time complexity. */ GenericValue& Reserve(SizeType newCapacity, Allocator &allocator) { RAPIDJSON_ASSERT(IsArray()); if (newCapacity > data_.a.capacity) { SetElementsPointer(reinterpret_cast(allocator.Realloc(GetElementsPointer(), data_.a.capacity * sizeof(GenericValue), newCapacity * sizeof(GenericValue)))); data_.a.capacity = newCapacity; } return *this; } //! Append a GenericValue at the end of the array. /*! \param value Value to be appended. \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). \pre IsArray() == true \post value.IsNull() == true \return The value itself for fluent API. \note The ownership of \c value will be transferred to this array on success. \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. \note Amortized constant time complexity. */ GenericValue& PushBack(GenericValue& value, Allocator& allocator) { RAPIDJSON_ASSERT(IsArray()); if (data_.a.size >= data_.a.capacity) Reserve(data_.a.capacity == 0 ? kDefaultArrayCapacity : (data_.a.capacity + (data_.a.capacity + 1) / 2), allocator); GetElementsPointer()[data_.a.size++].RawAssign(value); return *this; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS GenericValue& PushBack(GenericValue&& value, Allocator& allocator) { return PushBack(value, allocator); } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Append a constant string reference at the end of the array. /*! \param value Constant string reference to be appended. \param allocator Allocator for reallocating memory. It must be the same one used previously. Commonly use GenericDocument::GetAllocator(). \pre IsArray() == true \return The value itself for fluent API. \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. \note Amortized constant time complexity. \see GenericStringRef */ GenericValue& PushBack(StringRefType value, Allocator& allocator) { return (*this).template PushBack(value, allocator); } //! Append a primitive value at the end of the array. /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t \param value Value of primitive type T to be appended. \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). \pre IsArray() == true \return The value itself for fluent API. \note If the number of elements to be appended is known, calls Reserve() once first may be more efficient. \note The source type \c T explicitly disallows all pointer types, especially (\c const) \ref Ch*. This helps avoiding implicitly referencing character strings with insufficient lifetime, use \ref PushBack(GenericValue&, Allocator&) or \ref PushBack(StringRefType, Allocator&). All other pointer types would implicitly convert to \c bool, use an explicit cast instead, if needed. \note Amortized constant time complexity. */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericValue&)) PushBack(T value, Allocator& allocator) { GenericValue v(value); return PushBack(v, allocator); } //! Remove the last element in the array. /*! \note Constant time complexity. */ GenericValue& PopBack() { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(!Empty()); GetElementsPointer()[--data_.a.size].~GenericValue(); return *this; } //! Remove an element of array by iterator. /*! \param pos iterator to the element to remove \pre IsArray() == true && \ref Begin() <= \c pos < \ref End() \return Iterator following the removed element. If the iterator pos refers to the last element, the End() iterator is returned. \note Linear time complexity. */ ValueIterator Erase(ConstValueIterator pos) { return Erase(pos, pos + 1); } //! Remove elements in the range [first, last) of the array. /*! \param first iterator to the first element to remove \param last iterator following the last element to remove \pre IsArray() == true && \ref Begin() <= \c first <= \c last <= \ref End() \return Iterator following the last removed element. \note Linear time complexity. */ ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) { RAPIDJSON_ASSERT(IsArray()); RAPIDJSON_ASSERT(data_.a.size > 0); RAPIDJSON_ASSERT(GetElementsPointer() != 0); RAPIDJSON_ASSERT(first >= Begin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= End()); ValueIterator pos = Begin() + (first - Begin()); for (ValueIterator itr = pos; itr != last; ++itr) itr->~GenericValue(); std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); data_.a.size -= static_cast(last - first); return pos; } Array GetArray() { RAPIDJSON_ASSERT(IsArray()); return Array(*this); } ConstArray GetArray() const { RAPIDJSON_ASSERT(IsArray()); return ConstArray(*this); } //@} //!@name Number //@{ int GetInt() const { RAPIDJSON_ASSERT(data_.f.flags & kIntFlag); return data_.n.i.i; } unsigned GetUint() const { RAPIDJSON_ASSERT(data_.f.flags & kUintFlag); return data_.n.u.u; } int64_t GetInt64() const { RAPIDJSON_ASSERT(data_.f.flags & kInt64Flag); return data_.n.i64; } uint64_t GetUint64() const { RAPIDJSON_ASSERT(data_.f.flags & kUint64Flag); return data_.n.u64; } //! Get the value as double type. /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessDouble() to check whether the converison is lossless. */ double GetDouble() const { RAPIDJSON_ASSERT(IsNumber()); if ((data_.f.flags & kDoubleFlag) != 0) return data_.n.d; // exact type, no conversion. if ((data_.f.flags & kIntFlag) != 0) return data_.n.i.i; // int -> double if ((data_.f.flags & kUintFlag) != 0) return data_.n.u.u; // unsigned -> double if ((data_.f.flags & kInt64Flag) != 0) return static_cast(data_.n.i64); // int64_t -> double (may lose precision) RAPIDJSON_ASSERT((data_.f.flags & kUint64Flag) != 0); return static_cast(data_.n.u64); // uint64_t -> double (may lose precision) } //! Get the value as float type. /*! \note If the value is 64-bit integer type, it may lose precision. Use \c IsLosslessFloat() to check whether the converison is lossless. */ float GetFloat() const { return static_cast(GetDouble()); } GenericValue& SetInt(int i) { this->~GenericValue(); new (this) GenericValue(i); return *this; } GenericValue& SetUint(unsigned u) { this->~GenericValue(); new (this) GenericValue(u); return *this; } GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(f); return *this; } //@} //!@name String //@{ const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } //! Get the length of string. /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). */ SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } //! Set this value as a string without copying source string. /*! This version has better performance with supplied length, and also support string containing null character. \param s source string pointer. \param length The length of source string, excluding the trailing null terminator. \return The value itself for fluent API. \post IsString() == true && GetString() == s && GetStringLength() == length \see SetString(StringRefType) */ GenericValue& SetString(const Ch* s, SizeType length) { return SetString(StringRef(s, length)); } //! Set this value as a string without copying source string. /*! \param s source string reference \return The value itself for fluent API. \post IsString() == true && GetString() == s && GetStringLength() == s.length */ GenericValue& SetString(StringRefType s) { this->~GenericValue(); SetStringRaw(s); return *this; } //! Set this value as a string by copying from source string. /*! This version has better performance with supplied length, and also support string containing null character. \param s source string. \param length The length of source string, excluding the trailing null terminator. \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } //! Set this value as a string by copying from source string. /*! \param s source string. \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } #if RAPIDJSON_HAS_STDSTRING //! Set this value as a string by copying from source string. /*! \param s source string. \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). \return The value itself for fluent API. \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } #endif //@} //!@name Array //@{ //! Templated version for checking whether this value is type T. /*! \tparam T Either \c bool, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c float, \c const \c char*, \c std::basic_string */ template bool Is() const { return internal::TypeHelper::Is(*this); } template T Get() const { return internal::TypeHelper::Get(*this); } template T Get() { return internal::TypeHelper::Get(*this); } template ValueType& Set(const T& data) { return internal::TypeHelper::Set(*this, data); } template ValueType& Set(const T& data, AllocatorType& allocator) { return internal::TypeHelper::Set(*this, data, allocator); } //@} //! Generate events of this value to a Handler. /*! This function adopts the GoF visitor pattern. Typical usage is to output this JSON value as JSON text via Writer, which is a Handler. It can also be used to deep clone this value via GenericDocument, which is also a Handler. \tparam Handler type of handler. \param handler An object implementing concept Handler. */ template bool Accept(Handler& handler) const { switch(GetType()) { case kNullType: return handler.Null(); case kFalseType: return handler.Bool(false); case kTrueType: return handler.Bool(true); case kObjectType: if (RAPIDJSON_UNLIKELY(!handler.StartObject())) return false; for (ConstMemberIterator m = MemberBegin(); m != MemberEnd(); ++m) { RAPIDJSON_ASSERT(m->name.IsString()); // User may change the type of name by MemberIterator. if (RAPIDJSON_UNLIKELY(!handler.Key(m->name.GetString(), m->name.GetStringLength(), (m->name.data_.f.flags & kCopyFlag) != 0))) return false; if (RAPIDJSON_UNLIKELY(!m->value.Accept(handler))) return false; } return handler.EndObject(data_.o.size); case kArrayType: if (RAPIDJSON_UNLIKELY(!handler.StartArray())) return false; for (const GenericValue* v = Begin(); v != End(); ++v) if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) return false; return handler.EndArray(data_.a.size); case kStringType: return handler.String(GetString(), GetStringLength(), (data_.f.flags & kCopyFlag) != 0); default: RAPIDJSON_ASSERT(GetType() == kNumberType); if (IsDouble()) return handler.Double(data_.n.d); else if (IsInt()) return handler.Int(data_.n.i.i); else if (IsUint()) return handler.Uint(data_.n.u.u); else if (IsInt64()) return handler.Int64(data_.n.i64); else return handler.Uint64(data_.n.u64); } } private: template friend class GenericValue; template friend class GenericDocument; enum { kBoolFlag = 0x0008, kNumberFlag = 0x0010, kIntFlag = 0x0020, kUintFlag = 0x0040, kInt64Flag = 0x0080, kUint64Flag = 0x0100, kDoubleFlag = 0x0200, kStringFlag = 0x0400, kCopyFlag = 0x0800, kInlineStrFlag = 0x1000, // Initial flags of different types. kNullFlag = kNullType, kTrueFlag = kTrueType | kBoolFlag, kFalseFlag = kFalseType | kBoolFlag, kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, kConstStringFlag = kStringType | kStringFlag, kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, kObjectFlag = kObjectType, kArrayFlag = kArrayType, kTypeMask = 0x07 }; static const SizeType kDefaultArrayCapacity = 16; static const SizeType kDefaultObjectCapacity = 16; struct Flag { #if RAPIDJSON_48BITPOINTER_OPTIMIZATION char payload[sizeof(SizeType) * 2 + 6]; // 2 x SizeType + lower 48-bit pointer #elif RAPIDJSON_64BIT char payload[sizeof(SizeType) * 2 + sizeof(void*) + 6]; // 6 padding bytes #else char payload[sizeof(SizeType) * 2 + sizeof(void*) + 2]; // 2 padding bytes #endif uint16_t flags; }; struct String { SizeType length; SizeType hashcode; //!< reserved const Ch* str; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // implementation detail: ShortString can represent zero-terminated strings up to MaxSize chars // (excluding the terminating zero) and store a value to determine the length of the contained // string in the last character str[LenPos] by storing "MaxSize - length" there. If the string // to store has the maximal length of MaxSize then str[LenPos] will be 0 and therefore act as // the string terminator as well. For getting the string length back from that value just use // "MaxSize - str[LenPos]". // This allows to store 13-chars strings in 32-bit mode, 21-chars strings in 64-bit mode, // 13-chars strings for RAPIDJSON_48BITPOINTER_OPTIMIZATION=1 inline (for `UTF8`-encoded strings). struct ShortString { enum { MaxChars = sizeof(static_cast(0)->payload) / sizeof(Ch), MaxSize = MaxChars - 1, LenPos = MaxSize }; Ch str[MaxChars]; inline static bool Usable(SizeType len) { return (MaxSize >= len); } inline void SetLength(SizeType len) { str[LenPos] = static_cast(MaxSize - len); } inline SizeType GetLength() const { return static_cast(MaxSize - str[LenPos]); } }; // at most as many bytes as "String" above => 12 bytes in 32-bit mode, 16 bytes in 64-bit mode // By using proper binary layout, retrieval of different integer types do not need conversions. union Number { #if RAPIDJSON_ENDIAN == RAPIDJSON_LITTLEENDIAN struct I { int i; char padding[4]; }i; struct U { unsigned u; char padding2[4]; }u; #else struct I { char padding[4]; int i; }i; struct U { char padding2[4]; unsigned u; }u; #endif int64_t i64; uint64_t u64; double d; }; // 8 bytes struct ObjectData { SizeType size; SizeType capacity; Member* members; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode struct ArrayData { SizeType size; SizeType capacity; GenericValue* elements; }; // 12 bytes in 32-bit mode, 16 bytes in 64-bit mode union Data { String s; ShortString ss; Number n; ObjectData o; ArrayData a; Flag f; }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } RAPIDJSON_FORCEINLINE GenericValue* SetElementsPointer(GenericValue* elements) { return RAPIDJSON_SETPOINTER(GenericValue, data_.a.elements, elements); } RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } // Initialize this value as array with initial data, without calling destructor. void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { data_.f.flags = kArrayFlag; if (count) { GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); SetElementsPointer(e); std::memcpy(e, values, count * sizeof(GenericValue)); } else SetElementsPointer(0); data_.a.size = data_.a.capacity = count; } //! Initialize this value as object with initial data, without calling destructor. void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { data_.f.flags = kObjectFlag; if (count) { Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); SetMembersPointer(m); std::memcpy(m, members, count * sizeof(Member)); } else SetMembersPointer(0); data_.o.size = data_.o.capacity = count; } //! Initialize this value as constant string, without calling destructor. void SetStringRaw(StringRefType s) RAPIDJSON_NOEXCEPT { data_.f.flags = kConstStringFlag; SetStringPointer(s); data_.s.length = s.length; } //! Initialize this value as copy string with initial data, without calling destructor. void SetStringRaw(StringRefType s, Allocator& allocator) { Ch* str = 0; if (ShortString::Usable(s.length)) { data_.f.flags = kShortStringFlag; data_.ss.SetLength(s.length); str = data_.ss.str; } else { data_.f.flags = kCopyStringFlag; data_.s.length = s.length; str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); SetStringPointer(str); } std::memcpy(str, s, s.length * sizeof(Ch)); str[s.length] = '\0'; } //! Assignment without calling destructor void RawAssign(GenericValue& rhs) RAPIDJSON_NOEXCEPT { data_ = rhs.data_; // data_.f.flags = rhs.data_.f.flags; rhs.data_.f.flags = kNullFlag; } template bool StringEqual(const GenericValue& rhs) const { RAPIDJSON_ASSERT(IsString()); RAPIDJSON_ASSERT(rhs.IsString()); const SizeType len1 = GetStringLength(); const SizeType len2 = rhs.GetStringLength(); if(len1 != len2) { return false; } const Ch* const str1 = GetString(); const Ch* const str2 = rhs.GetString(); if(str1 == str2) { return true; } // fast path for constant string return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); } Data data_; }; //! GenericValue with UTF8 encoding typedef GenericValue > Value; /////////////////////////////////////////////////////////////////////////////// // GenericDocument //! A document for parsing JSON text as DOM. /*! \note implements Handler concept \tparam Encoding Encoding for both parsing and string storage. \tparam Allocator Allocator for allocating memory for the DOM \tparam StackAllocator Allocator for allocating memory for stack during parsing. \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. */ template , typename StackAllocator = CrtAllocator> class GenericDocument : public GenericValue { public: typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. typedef GenericValue ValueType; //!< Value type of the document. typedef Allocator AllocatorType; //!< Allocator type from template parameter. //! Constructor /*! Creates an empty document of specified type. \param type Mandatory type of object to create. \param allocator Optional allocator for allocating memory. \param stackCapacity Optional initial capacity of stack in bytes. \param stackAllocator Optional allocator for allocating memory for stack. */ explicit GenericDocument(Type type, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() { if (!allocator_) ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); } //! Constructor /*! Creates an empty document which type is Null. \param allocator Optional allocator for allocating memory. \param stackCapacity Optional initial capacity of stack in bytes. \param stackAllocator Optional allocator for allocating memory for stack. */ GenericDocument(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity, StackAllocator* stackAllocator = 0) : allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() { if (!allocator_) ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 GenericDocument(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT : ValueType(std::forward(rhs)), // explicit cast to avoid prohibited move from Document allocator_(rhs.allocator_), ownAllocator_(rhs.ownAllocator_), stack_(std::move(rhs.stack_)), parseResult_(rhs.parseResult_) { rhs.allocator_ = 0; rhs.ownAllocator_ = 0; rhs.parseResult_ = ParseResult(); } #endif ~GenericDocument() { Destroy(); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move assignment in C++11 GenericDocument& operator=(GenericDocument&& rhs) RAPIDJSON_NOEXCEPT { // The cast to ValueType is necessary here, because otherwise it would // attempt to call GenericValue's templated assignment operator. ValueType::operator=(std::forward(rhs)); // Calling the destructor here would prematurely call stack_'s destructor Destroy(); allocator_ = rhs.allocator_; ownAllocator_ = rhs.ownAllocator_; stack_ = std::move(rhs.stack_); parseResult_ = rhs.parseResult_; rhs.allocator_ = 0; rhs.ownAllocator_ = 0; rhs.parseResult_ = ParseResult(); return *this; } #endif //! Exchange the contents of this document with those of another. /*! \param rhs Another document. \note Constant complexity. \see GenericValue::Swap */ GenericDocument& Swap(GenericDocument& rhs) RAPIDJSON_NOEXCEPT { ValueType::Swap(rhs); stack_.Swap(rhs.stack_); internal::Swap(allocator_, rhs.allocator_); internal::Swap(ownAllocator_, rhs.ownAllocator_); internal::Swap(parseResult_, rhs.parseResult_); return *this; } //! free-standing swap function helper /*! Helper function to enable support for common swap implementation pattern based on \c std::swap: \code void swap(MyClass& a, MyClass& b) { using std::swap; swap(a.doc, b.doc); // ... } \endcode \see Swap() */ friend inline void swap(GenericDocument& a, GenericDocument& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } //! Populate this document by a generator which produces SAX events. /*! \tparam Generator A functor with bool f(Handler) prototype. \param g Generator functor which sends SAX events to the parameter. \return The document itself for fluent API. */ template GenericDocument& Populate(Generator& g) { ClearStackOnExit scope(*this); if (g(*this)) { RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document } return *this; } //!@name Parse from stream //!@{ //! Parse JSON text from an input stream (with Encoding conversion) /*! \tparam parseFlags Combination of \ref ParseFlag. \tparam SourceEncoding Encoding of input stream \tparam InputStream Type of input stream, implementing Stream concept \param is Input stream to be parsed. \return The document itself for fluent API. */ template GenericDocument& ParseStream(InputStream& is) { GenericReader reader( stack_.HasAllocator() ? &stack_.GetAllocator() : 0); ClearStackOnExit scope(*this); parseResult_ = reader.template Parse(is, *this); if (parseResult_) { RAPIDJSON_ASSERT(stack_.GetSize() == sizeof(ValueType)); // Got one and only one root object ValueType::operator=(*stack_.template Pop(1));// Move value from stack to document } return *this; } //! Parse JSON text from an input stream /*! \tparam parseFlags Combination of \ref ParseFlag. \tparam InputStream Type of input stream, implementing Stream concept \param is Input stream to be parsed. \return The document itself for fluent API. */ template GenericDocument& ParseStream(InputStream& is) { return ParseStream(is); } //! Parse JSON text from an input stream (with \ref kParseDefaultFlags) /*! \tparam InputStream Type of input stream, implementing Stream concept \param is Input stream to be parsed. \return The document itself for fluent API. */ template GenericDocument& ParseStream(InputStream& is) { return ParseStream(is); } //!@} //!@name Parse in-place from mutable string //!@{ //! Parse JSON text from a mutable string /*! \tparam parseFlags Combination of \ref ParseFlag. \param str Mutable zero-terminated string to be parsed. \return The document itself for fluent API. */ template GenericDocument& ParseInsitu(Ch* str) { GenericInsituStringStream s(str); return ParseStream(s); } //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) /*! \param str Mutable zero-terminated string to be parsed. \return The document itself for fluent API. */ GenericDocument& ParseInsitu(Ch* str) { return ParseInsitu(str); } //!@} //!@name Parse from read-only string //!@{ //! Parse JSON text from a read-only string (with Encoding conversion) /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). \tparam SourceEncoding Transcoding from input Encoding \param str Read-only zero-terminated string to be parsed. */ template GenericDocument& Parse(const typename SourceEncoding::Ch* str) { RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); GenericStringStream s(str); return ParseStream(s); } //! Parse JSON text from a read-only string /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). \param str Read-only zero-terminated string to be parsed. */ template GenericDocument& Parse(const Ch* str) { return Parse(str); } //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) /*! \param str Read-only zero-terminated string to be parsed. */ GenericDocument& Parse(const Ch* str) { return Parse(str); } template GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); MemoryStream ms(static_cast(str), length * sizeof(typename SourceEncoding::Ch)); EncodedInputStream is(ms); ParseStream(is); return *this; } template GenericDocument& Parse(const Ch* str, size_t length) { return Parse(str, length); } GenericDocument& Parse(const Ch* str, size_t length) { return Parse(str, length); } #if RAPIDJSON_HAS_STDSTRING template GenericDocument& Parse(const std::basic_string& str) { // c_str() is constant complexity according to standard. Should be faster than Parse(const char*, size_t) return Parse(str.c_str()); } template GenericDocument& Parse(const std::basic_string& str) { return Parse(str.c_str()); } GenericDocument& Parse(const std::basic_string& str) { return Parse(str); } #endif // RAPIDJSON_HAS_STDSTRING //!@} //!@name Handling parse errors //!@{ //! Whether a parse error has occured in the last parsing. bool HasParseError() const { return parseResult_.IsError(); } //! Get the \ref ParseErrorCode of last parsing. ParseErrorCode GetParseError() const { return parseResult_.Code(); } //! Get the position of last parsing error in input, 0 otherwise. size_t GetErrorOffset() const { return parseResult_.Offset(); } //! Implicit conversion to get the last parse result #ifndef __clang // -Wdocumentation /*! \return \ref ParseResult of the last parse operation \code Document doc; ParseResult ok = doc.Parse(json); if (!ok) printf( "JSON parse error: %s (%u)\n", GetParseError_En(ok.Code()), ok.Offset()); \endcode */ #endif operator ParseResult() const { return parseResult_; } //!@} //! Get the allocator of this document. Allocator& GetAllocator() { RAPIDJSON_ASSERT(allocator_); return *allocator_; } //! Get the capacity of stack in bytes. size_t GetStackCapacity() const { return stack_.GetCapacity(); } private: // clear stack on any exit from ParseStream, e.g. due to exception struct ClearStackOnExit { explicit ClearStackOnExit(GenericDocument& d) : d_(d) {} ~ClearStackOnExit() { d_.ClearStack(); } private: ClearStackOnExit(const ClearStackOnExit&); ClearStackOnExit& operator=(const ClearStackOnExit&); GenericDocument& d_; }; // callers of the following private Handler functions // template friend class GenericReader; // for parsing template friend class GenericValue; // for deep copying public: // Implementation of Handler bool Null() { new (stack_.template Push()) ValueType(); return true; } bool Bool(bool b) { new (stack_.template Push()) ValueType(b); return true; } bool Int(int i) { new (stack_.template Push()) ValueType(i); return true; } bool Uint(unsigned i) { new (stack_.template Push()) ValueType(i); return true; } bool Int64(int64_t i) { new (stack_.template Push()) ValueType(i); return true; } bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } bool RawNumber(const Ch* str, SizeType length, bool copy) { if (copy) new (stack_.template Push()) ValueType(str, length, GetAllocator()); else new (stack_.template Push()) ValueType(str, length); return true; } bool String(const Ch* str, SizeType length, bool copy) { if (copy) new (stack_.template Push()) ValueType(str, length, GetAllocator()); else new (stack_.template Push()) ValueType(str, length); return true; } bool StartObject() { new (stack_.template Push()) ValueType(kObjectType); return true; } bool Key(const Ch* str, SizeType length, bool copy) { return String(str, length, copy); } bool EndObject(SizeType memberCount) { typename ValueType::Member* members = stack_.template Pop(memberCount); stack_.template Top()->SetObjectRaw(members, memberCount, GetAllocator()); return true; } bool StartArray() { new (stack_.template Push()) ValueType(kArrayType); return true; } bool EndArray(SizeType elementCount) { ValueType* elements = stack_.template Pop(elementCount); stack_.template Top()->SetArrayRaw(elements, elementCount, GetAllocator()); return true; } private: //! Prohibit copying GenericDocument(const GenericDocument&); //! Prohibit assignment GenericDocument& operator=(const GenericDocument&); void ClearStack() { if (Allocator::kNeedFree) while (stack_.GetSize() > 0) // Here assumes all elements in stack array are GenericValue (Member is actually 2 GenericValue objects) (stack_.template Pop(1))->~ValueType(); else stack_.Clear(); stack_.ShrinkToFit(); } void Destroy() { RAPIDJSON_DELETE(ownAllocator_); } static const size_t kDefaultStackCapacity = 1024; Allocator* allocator_; Allocator* ownAllocator_; internal::Stack stack_; ParseResult parseResult_; }; //! GenericDocument with UTF8 encoding typedef GenericDocument > Document; // defined here due to the dependency on GenericDocument template template inline GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) { switch (rhs.GetType()) { case kObjectType: case kArrayType: { // perform deep copy via SAX Handler GenericDocument d(&allocator); rhs.Accept(d); RawAssign(*d.stack_.template Pop(1)); } break; case kStringType: if (rhs.data_.f.flags == kConstStringFlag) { data_.f.flags = rhs.data_.f.flags; data_ = *reinterpret_cast(&rhs.data_); } else { SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); } break; default: data_.f.flags = rhs.data_.f.flags; data_ = *reinterpret_cast(&rhs.data_); break; } } //! Helper class for accessing Value of array type. /*! Instance of this helper class is obtained by \c GenericValue::GetArray(). In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. */ template class GenericArray { public: typedef GenericArray ConstArray; typedef GenericArray Array; typedef ValueT PlainType; typedef typename internal::MaybeAddConst::Type ValueType; typedef ValueType* ValueIterator; // This may be const or non-const iterator typedef const ValueT* ConstValueIterator; typedef typename ValueType::AllocatorType AllocatorType; typedef typename ValueType::StringRefType StringRefType; template friend class GenericValue; GenericArray(const GenericArray& rhs) : value_(rhs.value_) {} GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } ~GenericArray() {} SizeType Size() const { return value_.Size(); } SizeType Capacity() const { return value_.Capacity(); } bool Empty() const { return value_.Empty(); } void Clear() const { value_.Clear(); } ValueType& operator[](SizeType index) const { return value_[index]; } ValueIterator Begin() const { return value_.Begin(); } ValueIterator End() const { return value_.End(); } GenericArray Reserve(SizeType newCapacity, AllocatorType &allocator) const { value_.Reserve(newCapacity, allocator); return *this; } GenericArray PushBack(ValueType& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS GenericArray PushBack(ValueType&& value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS GenericArray PushBack(StringRefType value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (const GenericArray&)) PushBack(T value, AllocatorType& allocator) const { value_.PushBack(value, allocator); return *this; } GenericArray PopBack() const { value_.PopBack(); return *this; } ValueIterator Erase(ConstValueIterator pos) const { return value_.Erase(pos); } ValueIterator Erase(ConstValueIterator first, ConstValueIterator last) const { return value_.Erase(first, last); } #if RAPIDJSON_HAS_CXX11_RANGE_FOR ValueIterator begin() const { return value_.Begin(); } ValueIterator end() const { return value_.End(); } #endif private: GenericArray(); GenericArray(ValueType& value) : value_(value) {} ValueType& value_; }; //! Helper class for accessing Value of object type. /*! Instance of this helper class is obtained by \c GenericValue::GetObject(). In addition to all APIs for array type, it provides range-based for loop if \c RAPIDJSON_HAS_CXX11_RANGE_FOR=1. */ template class GenericObject { public: typedef GenericObject ConstObject; typedef GenericObject Object; typedef ValueT PlainType; typedef typename internal::MaybeAddConst::Type ValueType; typedef GenericMemberIterator MemberIterator; // This may be const or non-const iterator typedef GenericMemberIterator ConstMemberIterator; typedef typename ValueType::AllocatorType AllocatorType; typedef typename ValueType::StringRefType StringRefType; typedef typename ValueType::EncodingType EncodingType; typedef typename ValueType::Ch Ch; template friend class GenericValue; GenericObject(const GenericObject& rhs) : value_(rhs.value_) {} GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } ~GenericObject() {} SizeType MemberCount() const { return value_.MemberCount(); } bool ObjectEmpty() const { return value_.ObjectEmpty(); } template ValueType& operator[](T* name) const { return value_[name]; } template ValueType& operator[](const GenericValue& name) const { return value_[name]; } #if RAPIDJSON_HAS_STDSTRING ValueType& operator[](const std::basic_string& name) const { return value_[name]; } #endif MemberIterator MemberBegin() const { return value_.MemberBegin(); } MemberIterator MemberEnd() const { return value_.MemberEnd(); } bool HasMember(const Ch* name) const { return value_.HasMember(name); } #if RAPIDJSON_HAS_STDSTRING bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } #endif template bool HasMember(const GenericValue& name) const { return value_.HasMember(name); } MemberIterator FindMember(const Ch* name) const { return value_.FindMember(name); } template MemberIterator FindMember(const GenericValue& name) const { return value_.FindMember(name); } #if RAPIDJSON_HAS_STDSTRING MemberIterator FindMember(const std::basic_string& name) const { return value_.FindMember(name); } #endif GenericObject AddMember(ValueType& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } GenericObject AddMember(ValueType& name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } #if RAPIDJSON_HAS_STDSTRING GenericObject AddMember(ValueType& name, std::basic_string& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) AddMember(ValueType& name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS GenericObject AddMember(ValueType&& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } GenericObject AddMember(ValueType&& name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } GenericObject AddMember(ValueType& name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } GenericObject AddMember(StringRefType name, ValueType&& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } void RemoveAllMembers() { return value_.RemoveAllMembers(); } bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } #if RAPIDJSON_HAS_STDSTRING bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } #endif template bool RemoveMember(const GenericValue& name) const { return value_.RemoveMember(name); } MemberIterator RemoveMember(MemberIterator m) const { return value_.RemoveMember(m); } MemberIterator EraseMember(ConstMemberIterator pos) const { return value_.EraseMember(pos); } MemberIterator EraseMember(ConstMemberIterator first, ConstMemberIterator last) const { return value_.EraseMember(first, last); } bool EraseMember(const Ch* name) const { return value_.EraseMember(name); } #if RAPIDJSON_HAS_STDSTRING bool EraseMember(const std::basic_string& name) const { return EraseMember(ValueType(StringRef(name))); } #endif template bool EraseMember(const GenericValue& name) const { return value_.EraseMember(name); } #if RAPIDJSON_HAS_CXX11_RANGE_FOR MemberIterator begin() const { return value_.MemberBegin(); } MemberIterator end() const { return value_.MemberEnd(); } #endif private: GenericObject(); GenericObject(ValueType& value) : value_(value) {} ValueType& value_; }; RAPIDJSON_NAMESPACE_END RAPIDJSON_DIAG_POP #endif // RAPIDJSON_DOCUMENT_H_ ================================================ FILE: src/thirdparty/rapidjson/encodedstream.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ENCODEDSTREAM_H_ #define RAPIDJSON_ENCODEDSTREAM_H_ #include "stream.h" #include "memorystream.h" #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) #endif RAPIDJSON_NAMESPACE_BEGIN //! Input byte stream wrapper with a statically bound encoding. /*! \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. \tparam InputByteStream Type of input byte stream. For example, FileReadStream. */ template class EncodedInputStream { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); public: typedef typename Encoding::Ch Ch; EncodedInputStream(InputByteStream& is) : is_(is) { current_ = Encoding::TakeBOM(is_); } Ch Peek() const { return current_; } Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } size_t Tell() const { return is_.Tell(); } // Not implemented void Put(Ch) { RAPIDJSON_ASSERT(false); } void Flush() { RAPIDJSON_ASSERT(false); } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: EncodedInputStream(const EncodedInputStream&); EncodedInputStream& operator=(const EncodedInputStream&); InputByteStream& is_; Ch current_; }; //! Specialized for UTF8 MemoryStream. template <> class EncodedInputStream, MemoryStream> { public: typedef UTF8<>::Ch Ch; EncodedInputStream(MemoryStream& is) : is_(is) { if (static_cast(is_.Peek()) == 0xEFu) is_.Take(); if (static_cast(is_.Peek()) == 0xBBu) is_.Take(); if (static_cast(is_.Peek()) == 0xBFu) is_.Take(); } Ch Peek() const { return is_.Peek(); } Ch Take() { return is_.Take(); } size_t Tell() const { return is_.Tell(); } // Not implemented void Put(Ch) {} void Flush() {} Ch* PutBegin() { return 0; } size_t PutEnd(Ch*) { return 0; } MemoryStream& is_; private: EncodedInputStream(const EncodedInputStream&); EncodedInputStream& operator=(const EncodedInputStream&); }; //! Output byte stream wrapper with statically bound encoding. /*! \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream. */ template class EncodedOutputStream { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); public: typedef typename Encoding::Ch Ch; EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { if (putBOM) Encoding::PutBOM(os_); } void Put(Ch c) { Encoding::Put(os_, c); } void Flush() { os_.Flush(); } // Not implemented Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} Ch Take() { RAPIDJSON_ASSERT(false); return 0;} size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: EncodedOutputStream(const EncodedOutputStream&); EncodedOutputStream& operator=(const EncodedOutputStream&); OutputByteStream& os_; }; #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x //! Input stream wrapper with dynamically bound encoding and automatic encoding detection. /*! \tparam CharType Type of character for reading. \tparam InputByteStream type of input byte stream to be wrapped. */ template class AutoUTFInputStream { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); public: typedef CharType Ch; //! Constructor. /*! \param is input stream to be wrapped. \param type UTF encoding type if it is not detected from the stream. */ AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); DetectType(); static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; takeFunc_ = f[type_]; current_ = takeFunc_(*is_); } UTFType GetType() const { return type_; } bool HasBOM() const { return hasBOM_; } Ch Peek() const { return current_; } Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } size_t Tell() const { return is_->Tell(); } // Not implemented void Put(Ch) { RAPIDJSON_ASSERT(false); } void Flush() { RAPIDJSON_ASSERT(false); } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: AutoUTFInputStream(const AutoUTFInputStream&); AutoUTFInputStream& operator=(const AutoUTFInputStream&); // Detect encoding type with BOM or RFC 4627 void DetectType() { // BOM (Byte Order Mark): // 00 00 FE FF UTF-32BE // FF FE 00 00 UTF-32LE // FE FF UTF-16BE // FF FE UTF-16LE // EF BB BF UTF-8 const unsigned char* c = reinterpret_cast(is_->Peek4()); if (!c) return; unsigned bom = static_cast(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24)); hasBOM_ = false; if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } // RFC 4627: Section 3 // "Since the first two characters of a JSON text will always be ASCII // characters [RFC0020], it is possible to determine whether an octet // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking // at the pattern of nulls in the first four octets." // 00 00 00 xx UTF-32BE // 00 xx 00 xx UTF-16BE // xx 00 00 00 UTF-32LE // xx 00 xx 00 UTF-16LE // xx xx xx xx UTF-8 if (!hasBOM_) { unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); switch (pattern) { case 0x08: type_ = kUTF32BE; break; case 0x0A: type_ = kUTF16BE; break; case 0x01: type_ = kUTF32LE; break; case 0x05: type_ = kUTF16LE; break; case 0x0F: type_ = kUTF8; break; default: break; // Use type defined by user. } } // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); } typedef Ch (*TakeFunc)(InputByteStream& is); InputByteStream* is_; UTFType type_; Ch current_; TakeFunc takeFunc_; bool hasBOM_; }; //! Output stream wrapper with dynamically bound encoding and automatic encoding detection. /*! \tparam CharType Type of character for writing. \tparam OutputByteStream type of output byte stream to be wrapped. */ template class AutoUTFOutputStream { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); public: typedef CharType Ch; //! Constructor. /*! \param os output stream to be wrapped. \param type UTF encoding type. \param putBOM Whether to write BOM at the beginning of the stream. */ AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE); // Runtime check whether the size of character type is sufficient. It only perform checks with assertion. if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2); if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4); static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; putFunc_ = f[type_]; if (putBOM) PutBOM(); } UTFType GetType() const { return type_; } void Put(Ch c) { putFunc_(*os_, c); } void Flush() { os_->Flush(); } // Not implemented Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;} Ch Take() { RAPIDJSON_ASSERT(false); return 0;} size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } private: AutoUTFOutputStream(const AutoUTFOutputStream&); AutoUTFOutputStream& operator=(const AutoUTFOutputStream&); void PutBOM() { typedef void (*PutBOMFunc)(OutputByteStream&); static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; f[type_](*os_); } typedef void (*PutFunc)(OutputByteStream&, Ch); OutputByteStream* os_; UTFType type_; PutFunc putFunc_; }; #undef RAPIDJSON_ENCODINGS_FUNC RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_FILESTREAM_H_ ================================================ FILE: src/thirdparty/rapidjson/encodings.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ENCODINGS_H_ #define RAPIDJSON_ENCODINGS_H_ #include "rapidjson.h" #ifdef _MSC_VER RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data RAPIDJSON_DIAG_OFF(4702) // unreachable code #elif defined(__GNUC__) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) RAPIDJSON_DIAG_OFF(overflow) #endif RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // Encoding /*! \class rapidjson::Encoding \brief Concept for encoding of Unicode characters. \code concept Encoding { typename Ch; //! Type of character. A "character" is actually a code unit in unicode's definition. enum { supportUnicode = 1 }; // or 0 if not supporting unicode //! \brief Encode a Unicode codepoint to an output stream. //! \param os Output stream. //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. template static void Encode(OutputStream& os, unsigned codepoint); //! \brief Decode a Unicode codepoint from an input stream. //! \param is Input stream. //! \param codepoint Output of the unicode codepoint. //! \return true if a valid codepoint can be decoded from the stream. template static bool Decode(InputStream& is, unsigned* codepoint); //! \brief Validate one Unicode codepoint from an encoded stream. //! \param is Input stream to obtain codepoint. //! \param os Output for copying one codepoint. //! \return true if it is valid. //! \note This function just validating and copying the codepoint without actually decode it. template static bool Validate(InputStream& is, OutputStream& os); // The following functions are deal with byte streams. //! Take a character from input byte stream, skip BOM if exist. template static CharType TakeBOM(InputByteStream& is); //! Take a character from input byte stream. template static Ch Take(InputByteStream& is); //! Put BOM to output byte stream. template static void PutBOM(OutputByteStream& os); //! Put a character to output byte stream. template static void Put(OutputByteStream& os, Ch c); }; \endcode */ /////////////////////////////////////////////////////////////////////////////// // UTF8 //! UTF-8 encoding. /*! http://en.wikipedia.org/wiki/UTF-8 http://tools.ietf.org/html/rfc3629 \tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char. \note implements Encoding concept */ template struct UTF8 { typedef CharType Ch; enum { supportUnicode = 1 }; template static void Encode(OutputStream& os, unsigned codepoint) { if (codepoint <= 0x7F) os.Put(static_cast(codepoint & 0xFF)); else if (codepoint <= 0x7FF) { os.Put(static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); os.Put(static_cast(0x80 | ((codepoint & 0x3F)))); } else if (codepoint <= 0xFFFF) { os.Put(static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); os.Put(static_cast(0x80 | (codepoint & 0x3F))); } else { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); os.Put(static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); os.Put(static_cast(0x80 | ((codepoint >> 12) & 0x3F))); os.Put(static_cast(0x80 | ((codepoint >> 6) & 0x3F))); os.Put(static_cast(0x80 | (codepoint & 0x3F))); } } template static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { if (codepoint <= 0x7F) PutUnsafe(os, static_cast(codepoint & 0xFF)); else if (codepoint <= 0x7FF) { PutUnsafe(os, static_cast(0xC0 | ((codepoint >> 6) & 0xFF))); PutUnsafe(os, static_cast(0x80 | ((codepoint & 0x3F)))); } else if (codepoint <= 0xFFFF) { PutUnsafe(os, static_cast(0xE0 | ((codepoint >> 12) & 0xFF))); PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); } else { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); PutUnsafe(os, static_cast(0xF0 | ((codepoint >> 18) & 0xFF))); PutUnsafe(os, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); PutUnsafe(os, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); PutUnsafe(os, static_cast(0x80 | (codepoint & 0x3F))); } } template static bool Decode(InputStream& is, unsigned* codepoint) { #define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) #define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) typename InputStream::Ch c = is.Take(); if (!(c & 0x80)) { *codepoint = static_cast(c); return true; } unsigned char type = GetRange(static_cast(c)); if (type >= 32) { *codepoint = 0; } else { *codepoint = (0xFF >> type) & static_cast(c); } bool result = true; switch (type) { case 2: TAIL(); return result; case 3: TAIL(); TAIL(); return result; case 4: COPY(); TRANS(0x50); TAIL(); return result; case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; case 6: TAIL(); TAIL(); TAIL(); return result; case 10: COPY(); TRANS(0x20); TAIL(); return result; case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; default: return false; } #undef COPY #undef TRANS #undef TAIL } template static bool Validate(InputStream& is, OutputStream& os) { #define COPY() os.Put(c = is.Take()) #define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define TAIL() COPY(); TRANS(0x70) Ch c; COPY(); if (!(c & 0x80)) return true; bool result = true; switch (GetRange(static_cast(c))) { case 2: TAIL(); return result; case 3: TAIL(); TAIL(); return result; case 4: COPY(); TRANS(0x50); TAIL(); return result; case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; case 6: TAIL(); TAIL(); TAIL(); return result; case 10: COPY(); TRANS(0x20); TAIL(); return result; case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; default: return false; } #undef COPY #undef TRANS #undef TAIL } static unsigned char GetRange(unsigned char c) { // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. static const unsigned char type[] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, 0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, }; return type[c]; } template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); typename InputByteStream::Ch c = Take(is); if (static_cast(c) != 0xEFu) return c; c = is.Take(); if (static_cast(c) != 0xBBu) return c; c = is.Take(); if (static_cast(c) != 0xBFu) return c; c = is.Take(); return c; } template static Ch Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); return static_cast(is.Take()); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(0xEFu)); os.Put(static_cast(0xBBu)); os.Put(static_cast(0xBFu)); } template static void Put(OutputByteStream& os, Ch c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(c)); } }; /////////////////////////////////////////////////////////////////////////////// // UTF16 //! UTF-16 encoding. /*! http://en.wikipedia.org/wiki/UTF-16 http://tools.ietf.org/html/rfc2781 \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. \note implements Encoding concept \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. For streaming, use UTF16LE and UTF16BE, which handle endianness. */ template struct UTF16 { typedef CharType Ch; RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); enum { supportUnicode = 1 }; template static void Encode(OutputStream& os, unsigned codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); if (codepoint <= 0xFFFF) { RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair os.Put(static_cast(codepoint)); } else { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); unsigned v = codepoint - 0x10000; os.Put(static_cast((v >> 10) | 0xD800)); os.Put((v & 0x3FF) | 0xDC00); } } template static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); if (codepoint <= 0xFFFF) { RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair PutUnsafe(os, static_cast(codepoint)); } else { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); unsigned v = codepoint - 0x10000; PutUnsafe(os, static_cast((v >> 10) | 0xD800)); PutUnsafe(os, (v & 0x3FF) | 0xDC00); } } template static bool Decode(InputStream& is, unsigned* codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); typename InputStream::Ch c = is.Take(); if (c < 0xD800 || c > 0xDFFF) { *codepoint = static_cast(c); return true; } else if (c <= 0xDBFF) { *codepoint = (static_cast(c) & 0x3FF) << 10; c = is.Take(); *codepoint |= (static_cast(c) & 0x3FF); *codepoint += 0x10000; return c >= 0xDC00 && c <= 0xDFFF; } return false; } template static bool Validate(InputStream& is, OutputStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); typename InputStream::Ch c; os.Put(static_cast(c = is.Take())); if (c < 0xD800 || c > 0xDFFF) return true; else if (c <= 0xDBFF) { os.Put(c = is.Take()); return c >= 0xDC00 && c <= 0xDFFF; } return false; } }; //! UTF-16 little endian encoding. template struct UTF16LE : UTF16 { template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); return static_cast(c) == 0xFEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); unsigned c = static_cast(is.Take()); c |= static_cast(static_cast(is.Take())) << 8; return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(0xFFu)); os.Put(static_cast(0xFEu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(static_cast(c) & 0xFFu)); os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); } }; //! UTF-16 big endian encoding. template struct UTF16BE : UTF16 { template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); return static_cast(c) == 0xFEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); unsigned c = static_cast(static_cast(is.Take())) << 8; c |= static_cast(is.Take()); return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(0xFEu)); os.Put(static_cast(0xFFu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast((static_cast(c) >> 8) & 0xFFu)); os.Put(static_cast(static_cast(c) & 0xFFu)); } }; /////////////////////////////////////////////////////////////////////////////// // UTF32 //! UTF-32 encoding. /*! http://en.wikipedia.org/wiki/UTF-32 \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. \note implements Encoding concept \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. For streaming, use UTF32LE and UTF32BE, which handle endianness. */ template struct UTF32 { typedef CharType Ch; RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); enum { supportUnicode = 1 }; template static void Encode(OutputStream& os, unsigned codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); os.Put(codepoint); } template static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); PutUnsafe(os, codepoint); } template static bool Decode(InputStream& is, unsigned* codepoint) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); Ch c = is.Take(); *codepoint = c; return c <= 0x10FFFF; } template static bool Validate(InputStream& is, OutputStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); Ch c; os.Put(c = is.Take()); return c <= 0x10FFFF; } }; //! UTF-32 little endian enocoding. template struct UTF32LE : UTF32 { template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); return static_cast(c) == 0x0000FEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); unsigned c = static_cast(is.Take()); c |= static_cast(static_cast(is.Take())) << 8; c |= static_cast(static_cast(is.Take())) << 16; c |= static_cast(static_cast(is.Take())) << 24; return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(0xFFu)); os.Put(static_cast(0xFEu)); os.Put(static_cast(0x00u)); os.Put(static_cast(0x00u)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(c & 0xFFu)); os.Put(static_cast((c >> 8) & 0xFFu)); os.Put(static_cast((c >> 16) & 0xFFu)); os.Put(static_cast((c >> 24) & 0xFFu)); } }; //! UTF-32 big endian encoding. template struct UTF32BE : UTF32 { template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); CharType c = Take(is); return static_cast(c) == 0x0000FEFFu ? Take(is) : c; } template static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); unsigned c = static_cast(static_cast(is.Take())) << 24; c |= static_cast(static_cast(is.Take())) << 16; c |= static_cast(static_cast(is.Take())) << 8; c |= static_cast(static_cast(is.Take())); return static_cast(c); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(0x00u)); os.Put(static_cast(0x00u)); os.Put(static_cast(0xFEu)); os.Put(static_cast(0xFFu)); } template static void Put(OutputByteStream& os, CharType c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast((c >> 24) & 0xFFu)); os.Put(static_cast((c >> 16) & 0xFFu)); os.Put(static_cast((c >> 8) & 0xFFu)); os.Put(static_cast(c & 0xFFu)); } }; /////////////////////////////////////////////////////////////////////////////// // ASCII //! ASCII encoding. /*! http://en.wikipedia.org/wiki/ASCII \tparam CharType Code unit for storing 7-bit ASCII data. Default is char. \note implements Encoding concept */ template struct ASCII { typedef CharType Ch; enum { supportUnicode = 0 }; template static void Encode(OutputStream& os, unsigned codepoint) { RAPIDJSON_ASSERT(codepoint <= 0x7F); os.Put(static_cast(codepoint & 0xFF)); } template static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { RAPIDJSON_ASSERT(codepoint <= 0x7F); PutUnsafe(os, static_cast(codepoint & 0xFF)); } template static bool Decode(InputStream& is, unsigned* codepoint) { uint8_t c = static_cast(is.Take()); *codepoint = c; return c <= 0X7F; } template static bool Validate(InputStream& is, OutputStream& os) { uint8_t c = static_cast(is.Take()); os.Put(static_cast(c)); return c <= 0x7F; } template static CharType TakeBOM(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); uint8_t c = static_cast(Take(is)); return static_cast(c); } template static Ch Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); return static_cast(is.Take()); } template static void PutBOM(OutputByteStream& os) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); (void)os; } template static void Put(OutputByteStream& os, Ch c) { RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); os.Put(static_cast(c)); } }; /////////////////////////////////////////////////////////////////////////////// // AutoUTF //! Runtime-specified UTF encoding type of a stream. enum UTFType { kUTF8 = 0, //!< UTF-8. kUTF16LE = 1, //!< UTF-16 little endian. kUTF16BE = 2, //!< UTF-16 big endian. kUTF32LE = 3, //!< UTF-32 little endian. kUTF32BE = 4 //!< UTF-32 big endian. }; //! Dynamically select encoding according to stream's runtime-specified UTF encoding type. /*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). */ template struct AutoUTF { typedef CharType Ch; enum { supportUnicode = 1 }; #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x template RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { typedef void (*EncodeFunc)(OutputStream&, unsigned); static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; (*f[os.GetType()])(os, codepoint); } template RAPIDJSON_FORCEINLINE static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { typedef void (*EncodeFunc)(OutputStream&, unsigned); static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; (*f[os.GetType()])(os, codepoint); } template RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { typedef bool (*DecodeFunc)(InputStream&, unsigned*); static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; return (*f[is.GetType()])(is, codepoint); } template RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { typedef bool (*ValidateFunc)(InputStream&, OutputStream&); static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; return (*f[is.GetType()])(is, os); } #undef RAPIDJSON_ENCODINGS_FUNC }; /////////////////////////////////////////////////////////////////////////////// // Transcoder //! Encoding conversion. template struct Transcoder { //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. template RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { unsigned codepoint; if (!SourceEncoding::Decode(is, &codepoint)) return false; TargetEncoding::Encode(os, codepoint); return true; } template RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { unsigned codepoint; if (!SourceEncoding::Decode(is, &codepoint)) return false; TargetEncoding::EncodeUnsafe(os, codepoint); return true; } //! Validate one Unicode codepoint from an encoded stream. template RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { return Transcode(is, os); // Since source/target encoding is different, must transcode. } }; // Forward declaration. template inline void PutUnsafe(Stream& stream, typename Stream::Ch c); //! Specialization of Transcoder with same source and target encoding. template struct Transcoder { template RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. return true; } template RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. return true; } template RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { return Encoding::Validate(is, os); // source/target encoding are the same } }; RAPIDJSON_NAMESPACE_END #if defined(__GNUC__) || defined(_MSC_VER) RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_ENCODINGS_H_ ================================================ FILE: src/thirdparty/rapidjson/error/en.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ERROR_EN_H_ #define RAPIDJSON_ERROR_EN_H_ #include "error.h" #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(switch-enum) RAPIDJSON_DIAG_OFF(covered-switch-default) #endif RAPIDJSON_NAMESPACE_BEGIN //! Maps error code of parsing into error message. /*! \ingroup RAPIDJSON_ERRORS \param parseErrorCode Error code obtained in parsing. \return the error message. \note User can make a copy of this function for localization. Using switch-case is safer for future modification of error codes. */ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { switch (parseErrorCode) { case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); case kParseErrorStringUnicodeSurrogateInvalid: return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); case kParseErrorStringEscapeInvalid: return RAPIDJSON_ERROR_STRING("Invalid escape character in string."); case kParseErrorStringMissQuotationMark: return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); case kParseErrorStringInvalidEncoding: return RAPIDJSON_ERROR_STRING("Invalid encoding in string."); case kParseErrorNumberTooBig: return RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); case kParseErrorNumberMissFraction: return RAPIDJSON_ERROR_STRING("Miss fraction part in number."); case kParseErrorNumberMissExponent: return RAPIDJSON_ERROR_STRING("Miss exponent in number."); case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); default: return RAPIDJSON_ERROR_STRING("Unknown error."); } } RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_ERROR_EN_H_ ================================================ FILE: src/thirdparty/rapidjson/error/error.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ERROR_ERROR_H_ #define RAPIDJSON_ERROR_ERROR_H_ #include "../rapidjson.h" #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) #endif /*! \file error.h */ /*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */ /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_ERROR_CHARTYPE //! Character type of error messages. /*! \ingroup RAPIDJSON_ERRORS The default character type is \c char. On Windows, user can define this macro as \c TCHAR for supporting both unicode/non-unicode settings. */ #ifndef RAPIDJSON_ERROR_CHARTYPE #define RAPIDJSON_ERROR_CHARTYPE char #endif /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_ERROR_STRING //! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. /*! \ingroup RAPIDJSON_ERRORS By default this conversion macro does nothing. On Windows, user can define this macro as \c _T(x) for supporting both unicode/non-unicode settings. */ #ifndef RAPIDJSON_ERROR_STRING #define RAPIDJSON_ERROR_STRING(x) x #endif RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // ParseErrorCode //! Error code of parsing. /*! \ingroup RAPIDJSON_ERRORS \see GenericReader::Parse, GenericReader::GetParseErrorCode */ enum ParseErrorCode { kParseErrorNone = 0, //!< No error. kParseErrorDocumentEmpty, //!< The document is empty. kParseErrorDocumentRootNotSingular, //!< The document root must not follow by other values. kParseErrorValueInvalid, //!< Invalid value. kParseErrorObjectMissName, //!< Missing a name for object member. kParseErrorObjectMissColon, //!< Missing a colon after a name of object member. kParseErrorObjectMissCommaOrCurlyBracket, //!< Missing a comma or '}' after an object member. kParseErrorArrayMissCommaOrSquareBracket, //!< Missing a comma or ']' after an array element. kParseErrorStringUnicodeEscapeInvalidHex, //!< Incorrect hex digit after \\u escape in string. kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid. kParseErrorStringEscapeInvalid, //!< Invalid escape character in string. kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string. kParseErrorStringInvalidEncoding, //!< Invalid encoding in string. kParseErrorNumberTooBig, //!< Number too big to be stored in double. kParseErrorNumberMissFraction, //!< Miss fraction part in number. kParseErrorNumberMissExponent, //!< Miss exponent in number. kParseErrorTermination, //!< Parsing was terminated. kParseErrorUnspecificSyntaxError //!< Unspecific syntax error. }; //! Result of parsing (wraps ParseErrorCode) /*! \ingroup RAPIDJSON_ERRORS \code Document doc; ParseResult ok = doc.Parse("[42]"); if (!ok) { fprintf(stderr, "JSON parse error: %s (%u)", GetParseError_En(ok.Code()), ok.Offset()); exit(EXIT_FAILURE); } \endcode \see GenericReader::Parse, GenericDocument::Parse */ struct ParseResult { public: //! Default constructor, no error. ParseResult() : code_(kParseErrorNone), offset_(0) {} //! Constructor to set an error. ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {} //! Get the error code. ParseErrorCode Code() const { return code_; } //! Get the error offset, if \ref IsError(), 0 otherwise. size_t Offset() const { return offset_; } //! Conversion to \c bool, returns \c true, iff !\ref IsError(). operator bool() const { return !IsError(); } //! Whether the result is an error. bool IsError() const { return code_ != kParseErrorNone; } bool operator==(const ParseResult& that) const { return code_ == that.code_; } bool operator==(ParseErrorCode code) const { return code_ == code; } friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } //! Reset error code. void Clear() { Set(kParseErrorNone); } //! Update error code and offset. void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; } private: ParseErrorCode code_; size_t offset_; }; //! Function pointer type of GetParseError(). /*! \ingroup RAPIDJSON_ERRORS This is the prototype for \c GetParseError_X(), where \c X is a locale. User can dynamically change locale in runtime, e.g.: \code GetParseErrorFunc GetParseError = GetParseError_En; // or whatever const RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode()); \endcode */ typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_ERROR_ERROR_H_ ================================================ FILE: src/thirdparty/rapidjson/filereadstream.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_FILEREADSTREAM_H_ #define RAPIDJSON_FILEREADSTREAM_H_ #include "stream.h" #include #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(unreachable-code) RAPIDJSON_DIAG_OFF(missing-noreturn) #endif RAPIDJSON_NAMESPACE_BEGIN //! File byte stream for input using fread(). /*! \note implements Stream concept */ class FileReadStream { public: typedef char Ch; //!< Character type (byte). //! Constructor. /*! \param fp File pointer opened for read. \param buffer user-supplied buffer. \param bufferSize size of buffer in bytes. Must >=4 bytes. */ FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { RAPIDJSON_ASSERT(fp_ != 0); RAPIDJSON_ASSERT(bufferSize >= 4); Read(); } Ch Peek() const { return *current_; } Ch Take() { Ch c = *current_; Read(); return c; } size_t Tell() const { return count_ + static_cast(current_ - buffer_); } // Not implemented void Put(Ch) { RAPIDJSON_ASSERT(false); } void Flush() { RAPIDJSON_ASSERT(false); } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } // For encoding detection only. const Ch* Peek4() const { return (current_ + 4 <= bufferLast_) ? current_ : 0; } private: void Read() { if (current_ < bufferLast_) ++current_; else if (!eof_) { count_ += readCount_; readCount_ = fread(buffer_, 1, bufferSize_, fp_); bufferLast_ = buffer_ + readCount_ - 1; current_ = buffer_; if (readCount_ < bufferSize_) { buffer_[readCount_] = '\0'; ++bufferLast_; eof_ = true; } } } std::FILE* fp_; Ch *buffer_; size_t bufferSize_; Ch *bufferLast_; Ch *current_; size_t readCount_; size_t count_; //!< Number of characters read bool eof_; }; RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_FILESTREAM_H_ ================================================ FILE: src/thirdparty/rapidjson/filewritestream.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_FILEWRITESTREAM_H_ #define RAPIDJSON_FILEWRITESTREAM_H_ #include "stream.h" #include #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(unreachable-code) #endif RAPIDJSON_NAMESPACE_BEGIN //! Wrapper of C file stream for input using fread(). /*! \note implements Stream concept */ class FileWriteStream { public: typedef char Ch; //!< Character type. Only support char. FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { RAPIDJSON_ASSERT(fp_ != 0); } void Put(char c) { if (current_ >= bufferEnd_) Flush(); *current_++ = c; } void PutN(char c, size_t n) { size_t avail = static_cast(bufferEnd_ - current_); while (n > avail) { std::memset(current_, c, avail); current_ += avail; Flush(); n -= avail; avail = static_cast(bufferEnd_ - current_); } if (n > 0) { std::memset(current_, c, n); current_ += n; } } void Flush() { if (current_ != buffer_) { size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); if (result < static_cast(current_ - buffer_)) { // failure deliberately ignored at this time // added to avoid warn_unused_result build errors } current_ = buffer_; } } // Not implemented char Peek() const { RAPIDJSON_ASSERT(false); return 0; } char Take() { RAPIDJSON_ASSERT(false); return 0; } size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } private: // Prohibit copy constructor & assignment operator. FileWriteStream(const FileWriteStream&); FileWriteStream& operator=(const FileWriteStream&); std::FILE* fp_; char *buffer_; char *bufferEnd_; char *current_; }; //! Implement specialized version of PutN() with memset() for better performance. template<> inline void PutN(FileWriteStream& stream, char c, size_t n) { stream.PutN(c, n); } RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_FILESTREAM_H_ ================================================ FILE: src/thirdparty/rapidjson/fwd.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_FWD_H_ #define RAPIDJSON_FWD_H_ #include "rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN // encodings.h template struct UTF8; template struct UTF16; template struct UTF16BE; template struct UTF16LE; template struct UTF32; template struct UTF32BE; template struct UTF32LE; template struct ASCII; template struct AutoUTF; template struct Transcoder; // allocators.h class CrtAllocator; template class MemoryPoolAllocator; // stream.h template struct GenericStringStream; typedef GenericStringStream > StringStream; template struct GenericInsituStringStream; typedef GenericInsituStringStream > InsituStringStream; // stringbuffer.h template class GenericStringBuffer; typedef GenericStringBuffer, CrtAllocator> StringBuffer; // filereadstream.h class FileReadStream; // filewritestream.h class FileWriteStream; // memorybuffer.h template struct GenericMemoryBuffer; typedef GenericMemoryBuffer MemoryBuffer; // memorystream.h struct MemoryStream; // reader.h template struct BaseReaderHandler; template class GenericReader; typedef GenericReader, UTF8, CrtAllocator> Reader; // writer.h template class Writer; // prettywriter.h template class PrettyWriter; // document.h template struct GenericMember; template class GenericMemberIterator; template struct GenericStringRef; template class GenericValue; typedef GenericValue, MemoryPoolAllocator > Value; template class GenericDocument; typedef GenericDocument, MemoryPoolAllocator, CrtAllocator> Document; // pointer.h template class GenericPointer; typedef GenericPointer Pointer; // schema.h template class IGenericRemoteSchemaDocumentProvider; template class GenericSchemaDocument; typedef GenericSchemaDocument SchemaDocument; typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; template < typename SchemaDocumentType, typename OutputHandler, typename StateAllocator> class GenericSchemaValidator; typedef GenericSchemaValidator, void>, CrtAllocator> SchemaValidator; RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_RAPIDJSONFWD_H_ ================================================ FILE: src/thirdparty/rapidjson/internal/biginteger.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_BIGINTEGER_H_ #define RAPIDJSON_BIGINTEGER_H_ #include "../rapidjson.h" #if defined(_MSC_VER) && defined(_M_AMD64) #include // for _umul128 #pragma intrinsic(_umul128) #endif RAPIDJSON_NAMESPACE_BEGIN namespace internal { class BigInteger { public: typedef uint64_t Type; BigInteger(const BigInteger& rhs) : count_(rhs.count_) { std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); } explicit BigInteger(uint64_t u) : count_(1) { digits_[0] = u; } BigInteger(const char* decimals, size_t length) : count_(1) { RAPIDJSON_ASSERT(length > 0); digits_[0] = 0; size_t i = 0; const size_t kMaxDigitPerIteration = 19; // 2^64 = 18446744073709551616 > 10^19 while (length >= kMaxDigitPerIteration) { AppendDecimal64(decimals + i, decimals + i + kMaxDigitPerIteration); length -= kMaxDigitPerIteration; i += kMaxDigitPerIteration; } if (length > 0) AppendDecimal64(decimals + i, decimals + i + length); } BigInteger& operator=(const BigInteger &rhs) { if (this != &rhs) { count_ = rhs.count_; std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type)); } return *this; } BigInteger& operator=(uint64_t u) { digits_[0] = u; count_ = 1; return *this; } BigInteger& operator+=(uint64_t u) { Type backup = digits_[0]; digits_[0] += u; for (size_t i = 0; i < count_ - 1; i++) { if (digits_[i] >= backup) return *this; // no carry backup = digits_[i + 1]; digits_[i + 1] += 1; } // Last carry if (digits_[count_ - 1] < backup) PushBack(1); return *this; } BigInteger& operator*=(uint64_t u) { if (u == 0) return *this = 0; if (u == 1) return *this; if (*this == 1) return *this = u; uint64_t k = 0; for (size_t i = 0; i < count_; i++) { uint64_t hi; digits_[i] = MulAdd64(digits_[i], u, k, &hi); k = hi; } if (k > 0) PushBack(k); return *this; } BigInteger& operator*=(uint32_t u) { if (u == 0) return *this = 0; if (u == 1) return *this; if (*this == 1) return *this = u; uint64_t k = 0; for (size_t i = 0; i < count_; i++) { const uint64_t c = digits_[i] >> 32; const uint64_t d = digits_[i] & 0xFFFFFFFF; const uint64_t uc = u * c; const uint64_t ud = u * d; const uint64_t p0 = ud + k; const uint64_t p1 = uc + (p0 >> 32); digits_[i] = (p0 & 0xFFFFFFFF) | (p1 << 32); k = p1 >> 32; } if (k > 0) PushBack(k); return *this; } BigInteger& operator<<=(size_t shift) { if (IsZero() || shift == 0) return *this; size_t offset = shift / kTypeBit; size_t interShift = shift % kTypeBit; RAPIDJSON_ASSERT(count_ + offset <= kCapacity); if (interShift == 0) { std::memmove(&digits_[count_ - 1 + offset], &digits_[count_ - 1], count_ * sizeof(Type)); count_ += offset; } else { digits_[count_] = 0; for (size_t i = count_; i > 0; i--) digits_[i + offset] = (digits_[i] << interShift) | (digits_[i - 1] >> (kTypeBit - interShift)); digits_[offset] = digits_[0] << interShift; count_ += offset; if (digits_[count_]) count_++; } std::memset(digits_, 0, offset * sizeof(Type)); return *this; } bool operator==(const BigInteger& rhs) const { return count_ == rhs.count_ && std::memcmp(digits_, rhs.digits_, count_ * sizeof(Type)) == 0; } bool operator==(const Type rhs) const { return count_ == 1 && digits_[0] == rhs; } BigInteger& MultiplyPow5(unsigned exp) { static const uint32_t kPow5[12] = { 5, 5 * 5, 5 * 5 * 5, 5 * 5 * 5 * 5, 5 * 5 * 5 * 5 * 5, 5 * 5 * 5 * 5 * 5 * 5, 5 * 5 * 5 * 5 * 5 * 5 * 5, 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 }; if (exp == 0) return *this; for (; exp >= 27; exp -= 27) *this *= RAPIDJSON_UINT64_C2(0X6765C793, 0XFA10079D); // 5^27 for (; exp >= 13; exp -= 13) *this *= static_cast(1220703125u); // 5^13 if (exp > 0) *this *= kPow5[exp - 1]; return *this; } // Compute absolute difference of this and rhs. // Assume this != rhs bool Difference(const BigInteger& rhs, BigInteger* out) const { int cmp = Compare(rhs); RAPIDJSON_ASSERT(cmp != 0); const BigInteger *a, *b; // Makes a > b bool ret; if (cmp < 0) { a = &rhs; b = this; ret = true; } else { a = this; b = &rhs; ret = false; } Type borrow = 0; for (size_t i = 0; i < a->count_; i++) { Type d = a->digits_[i] - borrow; if (i < b->count_) d -= b->digits_[i]; borrow = (d > a->digits_[i]) ? 1 : 0; out->digits_[i] = d; if (d != 0) out->count_ = i + 1; } return ret; } int Compare(const BigInteger& rhs) const { if (count_ != rhs.count_) return count_ < rhs.count_ ? -1 : 1; for (size_t i = count_; i-- > 0;) if (digits_[i] != rhs.digits_[i]) return digits_[i] < rhs.digits_[i] ? -1 : 1; return 0; } size_t GetCount() const { return count_; } Type GetDigit(size_t index) const { RAPIDJSON_ASSERT(index < count_); return digits_[index]; } bool IsZero() const { return count_ == 1 && digits_[0] == 0; } private: void AppendDecimal64(const char* begin, const char* end) { uint64_t u = ParseUint64(begin, end); if (IsZero()) *this = u; else { unsigned exp = static_cast(end - begin); (MultiplyPow5(exp) <<= exp) += u; // *this = *this * 10^exp + u } } void PushBack(Type digit) { RAPIDJSON_ASSERT(count_ < kCapacity); digits_[count_++] = digit; } static uint64_t ParseUint64(const char* begin, const char* end) { uint64_t r = 0; for (const char* p = begin; p != end; ++p) { RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); r = r * 10u + static_cast(*p - '0'); } return r; } // Assume a * b + k < 2^128 static uint64_t MulAdd64(uint64_t a, uint64_t b, uint64_t k, uint64_t* outHigh) { #if defined(_MSC_VER) && defined(_M_AMD64) uint64_t low = _umul128(a, b, outHigh) + k; if (low < k) (*outHigh)++; return low; #elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) __extension__ typedef unsigned __int128 uint128; uint128 p = static_cast(a) * static_cast(b); p += k; *outHigh = static_cast(p >> 64); return static_cast(p); #else const uint64_t a0 = a & 0xFFFFFFFF, a1 = a >> 32, b0 = b & 0xFFFFFFFF, b1 = b >> 32; uint64_t x0 = a0 * b0, x1 = a0 * b1, x2 = a1 * b0, x3 = a1 * b1; x1 += (x0 >> 32); // can't give carry x1 += x2; if (x1 < x2) x3 += (static_cast(1) << 32); uint64_t lo = (x1 << 32) + (x0 & 0xFFFFFFFF); uint64_t hi = x3 + (x1 >> 32); lo += k; if (lo < k) hi++; *outHigh = hi; return lo; #endif } static const size_t kBitCount = 3328; // 64bit * 54 > 10^1000 static const size_t kCapacity = kBitCount / sizeof(Type); static const size_t kTypeBit = sizeof(Type) * 8; Type digits_[kCapacity]; size_t count_; }; } // namespace internal RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_BIGINTEGER_H_ ================================================ FILE: src/thirdparty/rapidjson/internal/diyfp.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. // This is a C++ header-only implementation of Grisu2 algorithm from the publication: // Loitsch, Florian. "Printing floating-point numbers quickly and accurately with // integers." ACM Sigplan Notices 45.6 (2010): 233-243. #ifndef RAPIDJSON_DIYFP_H_ #define RAPIDJSON_DIYFP_H_ #include "../rapidjson.h" #if defined(_MSC_VER) && defined(_M_AMD64) #include #pragma intrinsic(_BitScanReverse64) #pragma intrinsic(_umul128) #endif RAPIDJSON_NAMESPACE_BEGIN namespace internal { #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) #endif struct DiyFp { DiyFp() : f(), e() {} DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {} explicit DiyFp(double d) { union { double d; uint64_t u64; } u = { d }; int biased_e = static_cast((u.u64 & kDpExponentMask) >> kDpSignificandSize); uint64_t significand = (u.u64 & kDpSignificandMask); if (biased_e != 0) { f = significand + kDpHiddenBit; e = biased_e - kDpExponentBias; } else { f = significand; e = kDpMinExponent + 1; } } DiyFp operator-(const DiyFp& rhs) const { return DiyFp(f - rhs.f, e); } DiyFp operator*(const DiyFp& rhs) const { #if defined(_MSC_VER) && defined(_M_AMD64) uint64_t h; uint64_t l = _umul128(f, rhs.f, &h); if (l & (uint64_t(1) << 63)) // rounding h++; return DiyFp(h, e + rhs.e + 64); #elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) __extension__ typedef unsigned __int128 uint128; uint128 p = static_cast(f) * static_cast(rhs.f); uint64_t h = static_cast(p >> 64); uint64_t l = static_cast(p); if (l & (uint64_t(1) << 63)) // rounding h++; return DiyFp(h, e + rhs.e + 64); #else const uint64_t M32 = 0xFFFFFFFF; const uint64_t a = f >> 32; const uint64_t b = f & M32; const uint64_t c = rhs.f >> 32; const uint64_t d = rhs.f & M32; const uint64_t ac = a * c; const uint64_t bc = b * c; const uint64_t ad = a * d; const uint64_t bd = b * d; uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32); tmp += 1U << 31; /// mult_round return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64); #endif } DiyFp Normalize() const { #if defined(_MSC_VER) && defined(_M_AMD64) unsigned long index; _BitScanReverse64(&index, f); return DiyFp(f << (63 - index), e - (63 - index)); #elif defined(__GNUC__) && __GNUC__ >= 4 int s = __builtin_clzll(f); return DiyFp(f << s, e - s); #else DiyFp res = *this; while (!(res.f & (static_cast(1) << 63))) { res.f <<= 1; res.e--; } return res; #endif } DiyFp NormalizeBoundary() const { DiyFp res = *this; while (!(res.f & (kDpHiddenBit << 1))) { res.f <<= 1; res.e--; } res.f <<= (kDiySignificandSize - kDpSignificandSize - 2); res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2); return res; } void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const { DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary(); DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1); mi.f <<= mi.e - pl.e; mi.e = pl.e; *plus = pl; *minus = mi; } double ToDouble() const { union { double d; uint64_t u64; }u; const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : static_cast(e + kDpExponentBias); u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize); return u.d; } static const int kDiySignificandSize = 64; static const int kDpSignificandSize = 52; static const int kDpExponentBias = 0x3FF + kDpSignificandSize; static const int kDpMaxExponent = 0x7FF - kDpExponentBias; static const int kDpMinExponent = -kDpExponentBias; static const int kDpDenormalExponent = -kDpExponentBias + 1; static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); uint64_t f; int e; }; inline DiyFp GetCachedPowerByIndex(size_t index) { // 10^-348, 10^-340, ..., 10^340 static const uint64_t kCachedPowers_F[] = { RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76), RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea), RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df), RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f), RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c), RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5), RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d), RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637), RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7), RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5), RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b), RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996), RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6), RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8), RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053), RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd), RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94), RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b), RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac), RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3), RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb), RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c), RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000), RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984), RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70), RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245), RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8), RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a), RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea), RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85), RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2), RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3), RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25), RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece), RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5), RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a), RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c), RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a), RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129), RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429), RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d), RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841), RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9), RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b) }; static const int16_t kCachedPowers_E[] = { -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, -157, -130, -103, -77, -50, -24, 3, 30, 56, 83, 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, 375, 402, 428, 455, 481, 508, 534, 561, 588, 614, 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, 907, 933, 960, 986, 1013, 1039, 1066 }; return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); } inline DiyFp GetCachedPower(int e, int* K) { //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; double dk = (-61 - e) * 0.30102999566398114 + 347; // dk must be positive, so can do ceiling in positive int k = static_cast(dk); if (dk - k > 0.0) k++; unsigned index = static_cast((k >> 3) + 1); *K = -(-348 + static_cast(index << 3)); // decimal exponent no need lookup table return GetCachedPowerByIndex(index); } inline DiyFp GetCachedPower10(int exp, int *outExp) { unsigned index = (static_cast(exp) + 348u) / 8u; *outExp = -348 + static_cast(index) * 8; return GetCachedPowerByIndex(index); } #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif #ifdef __clang__ RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_OFF(padded) #endif } // namespace internal RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_DIYFP_H_ ================================================ FILE: src/thirdparty/rapidjson/internal/dtoa.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. // This is a C++ header-only implementation of Grisu2 algorithm from the publication: // Loitsch, Florian. "Printing floating-point numbers quickly and accurately with // integers." ACM Sigplan Notices 45.6 (2010): 233-243. #ifndef RAPIDJSON_DTOA_ #define RAPIDJSON_DTOA_ #include "itoa.h" // GetDigitsLut() #include "diyfp.h" #include "ieee754.h" RAPIDJSON_NAMESPACE_BEGIN namespace internal { #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) RAPIDJSON_DIAG_OFF(array-bounds) // some gcc versions generate wrong warnings https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 #endif inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uint64_t ten_kappa, uint64_t wp_w) { while (rest < wp_w && delta - rest >= ten_kappa && (rest + ten_kappa < wp_w || /// closer wp_w - rest > rest + ten_kappa - wp_w)) { buffer[len - 1]--; rest += ten_kappa; } } inline unsigned CountDecimalDigit32(uint32_t n) { // Simple pure C++ implementation was faster than __builtin_clz version in this situation. if (n < 10) return 1; if (n < 100) return 2; if (n < 1000) return 3; if (n < 10000) return 4; if (n < 100000) return 5; if (n < 1000000) return 6; if (n < 10000000) return 7; if (n < 100000000) return 8; // Will not reach 10 digits in DigitGen() //if (n < 1000000000) return 9; //return 10; return 9; } inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); const DiyFp wp_w = Mp - W; uint32_t p1 = static_cast(Mp.f >> -one.e); uint64_t p2 = Mp.f & (one.f - 1); unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9] *len = 0; while (kappa > 0) { uint32_t d = 0; switch (kappa) { case 9: d = p1 / 100000000; p1 %= 100000000; break; case 8: d = p1 / 10000000; p1 %= 10000000; break; case 7: d = p1 / 1000000; p1 %= 1000000; break; case 6: d = p1 / 100000; p1 %= 100000; break; case 5: d = p1 / 10000; p1 %= 10000; break; case 4: d = p1 / 1000; p1 %= 1000; break; case 3: d = p1 / 100; p1 %= 100; break; case 2: d = p1 / 10; p1 %= 10; break; case 1: d = p1; p1 = 0; break; default:; } if (d || *len) buffer[(*len)++] = static_cast('0' + static_cast(d)); kappa--; uint64_t tmp = (static_cast(p1) << -one.e) + p2; if (tmp <= delta) { *K += kappa; GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); return; } } // kappa = 0 for (;;) { p2 *= 10; delta *= 10; char d = static_cast(p2 >> -one.e); if (d || *len) buffer[(*len)++] = static_cast('0' + d); p2 &= one.f - 1; kappa--; if (p2 < delta) { *K += kappa; int index = -static_cast(kappa); GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[-static_cast(kappa)] : 0)); return; } } } inline void Grisu2(double value, char* buffer, int* length, int* K) { const DiyFp v(value); DiyFp w_m, w_p; v.NormalizedBoundaries(&w_m, &w_p); const DiyFp c_mk = GetCachedPower(w_p.e, K); const DiyFp W = v.Normalize() * c_mk; DiyFp Wp = w_p * c_mk; DiyFp Wm = w_m * c_mk; Wm.f++; Wp.f--; DigitGen(W, Wp, Wp.f - Wm.f, buffer, length, K); } inline char* WriteExponent(int K, char* buffer) { if (K < 0) { *buffer++ = '-'; K = -K; } if (K >= 100) { *buffer++ = static_cast('0' + static_cast(K / 100)); K %= 100; const char* d = GetDigitsLut() + K * 2; *buffer++ = d[0]; *buffer++ = d[1]; } else if (K >= 10) { const char* d = GetDigitsLut() + K * 2; *buffer++ = d[0]; *buffer++ = d[1]; } else *buffer++ = static_cast('0' + static_cast(K)); return buffer; } inline char* Prettify(char* buffer, int length, int k, int maxDecimalPlaces) { const int kk = length + k; // 10^(kk-1) <= v < 10^kk if (0 <= k && kk <= 21) { // 1234e7 -> 12340000000 for (int i = length; i < kk; i++) buffer[i] = '0'; buffer[kk] = '.'; buffer[kk + 1] = '0'; return &buffer[kk + 2]; } else if (0 < kk && kk <= 21) { // 1234e-2 -> 12.34 std::memmove(&buffer[kk + 1], &buffer[kk], static_cast(length - kk)); buffer[kk] = '.'; if (0 > k + maxDecimalPlaces) { // When maxDecimalPlaces = 2, 1.2345 -> 1.23, 1.102 -> 1.1 // Remove extra trailing zeros (at least one) after truncation. for (int i = kk + maxDecimalPlaces; i > kk + 1; i--) if (buffer[i] != '0') return &buffer[i + 1]; return &buffer[kk + 2]; // Reserve one zero } else return &buffer[length + 1]; } else if (-6 < kk && kk <= 0) { // 1234e-6 -> 0.001234 const int offset = 2 - kk; std::memmove(&buffer[offset], &buffer[0], static_cast(length)); buffer[0] = '0'; buffer[1] = '.'; for (int i = 2; i < offset; i++) buffer[i] = '0'; if (length - kk > maxDecimalPlaces) { // When maxDecimalPlaces = 2, 0.123 -> 0.12, 0.102 -> 0.1 // Remove extra trailing zeros (at least one) after truncation. for (int i = maxDecimalPlaces + 1; i > 2; i--) if (buffer[i] != '0') return &buffer[i + 1]; return &buffer[3]; // Reserve one zero } else return &buffer[length + offset]; } else if (kk < -maxDecimalPlaces) { // Truncate to zero buffer[0] = '0'; buffer[1] = '.'; buffer[2] = '0'; return &buffer[3]; } else if (length == 1) { // 1e30 buffer[1] = 'e'; return WriteExponent(kk - 1, &buffer[2]); } else { // 1234e30 -> 1.234e33 std::memmove(&buffer[2], &buffer[1], static_cast(length - 1)); buffer[1] = '.'; buffer[length + 1] = 'e'; return WriteExponent(kk - 1, &buffer[0 + length + 2]); } } inline char* dtoa(double value, char* buffer, int maxDecimalPlaces = 324) { RAPIDJSON_ASSERT(maxDecimalPlaces >= 1); Double d(value); if (d.IsZero()) { if (d.Sign()) *buffer++ = '-'; // -0.0, Issue #289 buffer[0] = '0'; buffer[1] = '.'; buffer[2] = '0'; return &buffer[3]; } else { if (value < 0) { *buffer++ = '-'; value = -value; } int length, K; Grisu2(value, buffer, &length, &K); return Prettify(buffer, length, K, maxDecimalPlaces); } } #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif } // namespace internal RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_DTOA_ ================================================ FILE: src/thirdparty/rapidjson/internal/ieee754.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_IEEE754_ #define RAPIDJSON_IEEE754_ #include "../rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN namespace internal { class Double { public: Double() {} Double(double d) : d_(d) {} Double(uint64_t u) : u_(u) {} double Value() const { return d_; } uint64_t Uint64Value() const { return u_; } double NextPositiveDouble() const { RAPIDJSON_ASSERT(!Sign()); return Double(u_ + 1).Value(); } bool Sign() const { return (u_ & kSignMask) != 0; } uint64_t Significand() const { return u_ & kSignificandMask; } int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } bool IsNanOrInf() const { return (u_ & kExponentMask) == kExponentMask; } bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } static unsigned EffectiveSignificandSize(int order) { if (order >= -1021) return 53; else if (order <= -1074) return 0; else return static_cast(order) + 1074; } private: static const int kSignificandSize = 52; static const int kExponentBias = 0x3FF; static const int kDenormalExponent = 1 - kExponentBias; static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); union { double d_; uint64_t u_; }; }; } // namespace internal RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_IEEE754_ ================================================ FILE: src/thirdparty/rapidjson/internal/itoa.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ITOA_ #define RAPIDJSON_ITOA_ #include "../rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN namespace internal { inline const char* GetDigitsLut() { static const char cDigitsLut[200] = { '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' }; return cDigitsLut; } inline char* u32toa(uint32_t value, char* buffer) { const char* cDigitsLut = GetDigitsLut(); if (value < 10000) { const uint32_t d1 = (value / 100) << 1; const uint32_t d2 = (value % 100) << 1; if (value >= 1000) *buffer++ = cDigitsLut[d1]; if (value >= 100) *buffer++ = cDigitsLut[d1 + 1]; if (value >= 10) *buffer++ = cDigitsLut[d2]; *buffer++ = cDigitsLut[d2 + 1]; } else if (value < 100000000) { // value = bbbbcccc const uint32_t b = value / 10000; const uint32_t c = value % 10000; const uint32_t d1 = (b / 100) << 1; const uint32_t d2 = (b % 100) << 1; const uint32_t d3 = (c / 100) << 1; const uint32_t d4 = (c % 100) << 1; if (value >= 10000000) *buffer++ = cDigitsLut[d1]; if (value >= 1000000) *buffer++ = cDigitsLut[d1 + 1]; if (value >= 100000) *buffer++ = cDigitsLut[d2]; *buffer++ = cDigitsLut[d2 + 1]; *buffer++ = cDigitsLut[d3]; *buffer++ = cDigitsLut[d3 + 1]; *buffer++ = cDigitsLut[d4]; *buffer++ = cDigitsLut[d4 + 1]; } else { // value = aabbbbcccc in decimal const uint32_t a = value / 100000000; // 1 to 42 value %= 100000000; if (a >= 10) { const unsigned i = a << 1; *buffer++ = cDigitsLut[i]; *buffer++ = cDigitsLut[i + 1]; } else *buffer++ = static_cast('0' + static_cast(a)); const uint32_t b = value / 10000; // 0 to 9999 const uint32_t c = value % 10000; // 0 to 9999 const uint32_t d1 = (b / 100) << 1; const uint32_t d2 = (b % 100) << 1; const uint32_t d3 = (c / 100) << 1; const uint32_t d4 = (c % 100) << 1; *buffer++ = cDigitsLut[d1]; *buffer++ = cDigitsLut[d1 + 1]; *buffer++ = cDigitsLut[d2]; *buffer++ = cDigitsLut[d2 + 1]; *buffer++ = cDigitsLut[d3]; *buffer++ = cDigitsLut[d3 + 1]; *buffer++ = cDigitsLut[d4]; *buffer++ = cDigitsLut[d4 + 1]; } return buffer; } inline char* i32toa(int32_t value, char* buffer) { uint32_t u = static_cast(value); if (value < 0) { *buffer++ = '-'; u = ~u + 1; } return u32toa(u, buffer); } inline char* u64toa(uint64_t value, char* buffer) { const char* cDigitsLut = GetDigitsLut(); const uint64_t kTen8 = 100000000; const uint64_t kTen9 = kTen8 * 10; const uint64_t kTen10 = kTen8 * 100; const uint64_t kTen11 = kTen8 * 1000; const uint64_t kTen12 = kTen8 * 10000; const uint64_t kTen13 = kTen8 * 100000; const uint64_t kTen14 = kTen8 * 1000000; const uint64_t kTen15 = kTen8 * 10000000; const uint64_t kTen16 = kTen8 * kTen8; if (value < kTen8) { uint32_t v = static_cast(value); if (v < 10000) { const uint32_t d1 = (v / 100) << 1; const uint32_t d2 = (v % 100) << 1; if (v >= 1000) *buffer++ = cDigitsLut[d1]; if (v >= 100) *buffer++ = cDigitsLut[d1 + 1]; if (v >= 10) *buffer++ = cDigitsLut[d2]; *buffer++ = cDigitsLut[d2 + 1]; } else { // value = bbbbcccc const uint32_t b = v / 10000; const uint32_t c = v % 10000; const uint32_t d1 = (b / 100) << 1; const uint32_t d2 = (b % 100) << 1; const uint32_t d3 = (c / 100) << 1; const uint32_t d4 = (c % 100) << 1; if (value >= 10000000) *buffer++ = cDigitsLut[d1]; if (value >= 1000000) *buffer++ = cDigitsLut[d1 + 1]; if (value >= 100000) *buffer++ = cDigitsLut[d2]; *buffer++ = cDigitsLut[d2 + 1]; *buffer++ = cDigitsLut[d3]; *buffer++ = cDigitsLut[d3 + 1]; *buffer++ = cDigitsLut[d4]; *buffer++ = cDigitsLut[d4 + 1]; } } else if (value < kTen16) { const uint32_t v0 = static_cast(value / kTen8); const uint32_t v1 = static_cast(value % kTen8); const uint32_t b0 = v0 / 10000; const uint32_t c0 = v0 % 10000; const uint32_t d1 = (b0 / 100) << 1; const uint32_t d2 = (b0 % 100) << 1; const uint32_t d3 = (c0 / 100) << 1; const uint32_t d4 = (c0 % 100) << 1; const uint32_t b1 = v1 / 10000; const uint32_t c1 = v1 % 10000; const uint32_t d5 = (b1 / 100) << 1; const uint32_t d6 = (b1 % 100) << 1; const uint32_t d7 = (c1 / 100) << 1; const uint32_t d8 = (c1 % 100) << 1; if (value >= kTen15) *buffer++ = cDigitsLut[d1]; if (value >= kTen14) *buffer++ = cDigitsLut[d1 + 1]; if (value >= kTen13) *buffer++ = cDigitsLut[d2]; if (value >= kTen12) *buffer++ = cDigitsLut[d2 + 1]; if (value >= kTen11) *buffer++ = cDigitsLut[d3]; if (value >= kTen10) *buffer++ = cDigitsLut[d3 + 1]; if (value >= kTen9) *buffer++ = cDigitsLut[d4]; if (value >= kTen8) *buffer++ = cDigitsLut[d4 + 1]; *buffer++ = cDigitsLut[d5]; *buffer++ = cDigitsLut[d5 + 1]; *buffer++ = cDigitsLut[d6]; *buffer++ = cDigitsLut[d6 + 1]; *buffer++ = cDigitsLut[d7]; *buffer++ = cDigitsLut[d7 + 1]; *buffer++ = cDigitsLut[d8]; *buffer++ = cDigitsLut[d8 + 1]; } else { const uint32_t a = static_cast(value / kTen16); // 1 to 1844 value %= kTen16; if (a < 10) *buffer++ = static_cast('0' + static_cast(a)); else if (a < 100) { const uint32_t i = a << 1; *buffer++ = cDigitsLut[i]; *buffer++ = cDigitsLut[i + 1]; } else if (a < 1000) { *buffer++ = static_cast('0' + static_cast(a / 100)); const uint32_t i = (a % 100) << 1; *buffer++ = cDigitsLut[i]; *buffer++ = cDigitsLut[i + 1]; } else { const uint32_t i = (a / 100) << 1; const uint32_t j = (a % 100) << 1; *buffer++ = cDigitsLut[i]; *buffer++ = cDigitsLut[i + 1]; *buffer++ = cDigitsLut[j]; *buffer++ = cDigitsLut[j + 1]; } const uint32_t v0 = static_cast(value / kTen8); const uint32_t v1 = static_cast(value % kTen8); const uint32_t b0 = v0 / 10000; const uint32_t c0 = v0 % 10000; const uint32_t d1 = (b0 / 100) << 1; const uint32_t d2 = (b0 % 100) << 1; const uint32_t d3 = (c0 / 100) << 1; const uint32_t d4 = (c0 % 100) << 1; const uint32_t b1 = v1 / 10000; const uint32_t c1 = v1 % 10000; const uint32_t d5 = (b1 / 100) << 1; const uint32_t d6 = (b1 % 100) << 1; const uint32_t d7 = (c1 / 100) << 1; const uint32_t d8 = (c1 % 100) << 1; *buffer++ = cDigitsLut[d1]; *buffer++ = cDigitsLut[d1 + 1]; *buffer++ = cDigitsLut[d2]; *buffer++ = cDigitsLut[d2 + 1]; *buffer++ = cDigitsLut[d3]; *buffer++ = cDigitsLut[d3 + 1]; *buffer++ = cDigitsLut[d4]; *buffer++ = cDigitsLut[d4 + 1]; *buffer++ = cDigitsLut[d5]; *buffer++ = cDigitsLut[d5 + 1]; *buffer++ = cDigitsLut[d6]; *buffer++ = cDigitsLut[d6 + 1]; *buffer++ = cDigitsLut[d7]; *buffer++ = cDigitsLut[d7 + 1]; *buffer++ = cDigitsLut[d8]; *buffer++ = cDigitsLut[d8 + 1]; } return buffer; } inline char* i64toa(int64_t value, char* buffer) { uint64_t u = static_cast(value); if (value < 0) { *buffer++ = '-'; u = ~u + 1; } return u64toa(u, buffer); } } // namespace internal RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_ITOA_ ================================================ FILE: src/thirdparty/rapidjson/internal/meta.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_META_H_ #define RAPIDJSON_INTERNAL_META_H_ #include "../rapidjson.h" #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif #if defined(_MSC_VER) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(6334) #endif #if RAPIDJSON_HAS_CXX11_TYPETRAITS #include #endif //@cond RAPIDJSON_INTERNAL RAPIDJSON_NAMESPACE_BEGIN namespace internal { // Helper to wrap/convert arbitrary types to void, useful for arbitrary type matching template struct Void { typedef void Type; }; /////////////////////////////////////////////////////////////////////////////// // BoolType, TrueType, FalseType // template struct BoolType { static const bool Value = Cond; typedef BoolType Type; }; typedef BoolType TrueType; typedef BoolType FalseType; /////////////////////////////////////////////////////////////////////////////// // SelectIf, BoolExpr, NotExpr, AndExpr, OrExpr // template struct SelectIfImpl { template struct Apply { typedef T1 Type; }; }; template <> struct SelectIfImpl { template struct Apply { typedef T2 Type; }; }; template struct SelectIfCond : SelectIfImpl::template Apply {}; template struct SelectIf : SelectIfCond {}; template struct AndExprCond : FalseType {}; template <> struct AndExprCond : TrueType {}; template struct OrExprCond : TrueType {}; template <> struct OrExprCond : FalseType {}; template struct BoolExpr : SelectIf::Type {}; template struct NotExpr : SelectIf::Type {}; template struct AndExpr : AndExprCond::Type {}; template struct OrExpr : OrExprCond::Type {}; /////////////////////////////////////////////////////////////////////////////// // AddConst, MaybeAddConst, RemoveConst template struct AddConst { typedef const T Type; }; template struct MaybeAddConst : SelectIfCond {}; template struct RemoveConst { typedef T Type; }; template struct RemoveConst { typedef T Type; }; /////////////////////////////////////////////////////////////////////////////// // IsSame, IsConst, IsMoreConst, IsPointer // template struct IsSame : FalseType {}; template struct IsSame : TrueType {}; template struct IsConst : FalseType {}; template struct IsConst : TrueType {}; template struct IsMoreConst : AndExpr::Type, typename RemoveConst::Type>, BoolType::Value >= IsConst::Value> >::Type {}; template struct IsPointer : FalseType {}; template struct IsPointer : TrueType {}; /////////////////////////////////////////////////////////////////////////////// // IsBaseOf // #if RAPIDJSON_HAS_CXX11_TYPETRAITS template struct IsBaseOf : BoolType< ::std::is_base_of::value> {}; #else // simplified version adopted from Boost template struct IsBaseOfImpl { RAPIDJSON_STATIC_ASSERT(sizeof(B) != 0); RAPIDJSON_STATIC_ASSERT(sizeof(D) != 0); typedef char (&Yes)[1]; typedef char (&No) [2]; template static Yes Check(const D*, T); static No Check(const B*, int); struct Host { operator const B*() const; operator const D*(); }; enum { Value = (sizeof(Check(Host(), 0)) == sizeof(Yes)) }; }; template struct IsBaseOf : OrExpr, BoolExpr > >::Type {}; #endif // RAPIDJSON_HAS_CXX11_TYPETRAITS ////////////////////////////////////////////////////////////////////////// // EnableIf / DisableIf // template struct EnableIfCond { typedef T Type; }; template struct EnableIfCond { /* empty */ }; template struct DisableIfCond { typedef T Type; }; template struct DisableIfCond { /* empty */ }; template struct EnableIf : EnableIfCond {}; template struct DisableIf : DisableIfCond {}; // SFINAE helpers struct SfinaeTag {}; template struct RemoveSfinaeTag; template struct RemoveSfinaeTag { typedef T Type; }; #define RAPIDJSON_REMOVEFPTR_(type) \ typename ::RAPIDJSON_NAMESPACE::internal::RemoveSfinaeTag \ < ::RAPIDJSON_NAMESPACE::internal::SfinaeTag&(*) type>::Type #define RAPIDJSON_ENABLEIF(cond) \ typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ ::Type * = NULL #define RAPIDJSON_DISABLEIF(cond) \ typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ ::Type * = NULL #define RAPIDJSON_ENABLEIF_RETURN(cond,returntype) \ typename ::RAPIDJSON_NAMESPACE::internal::EnableIf \ ::Type #define RAPIDJSON_DISABLEIF_RETURN(cond,returntype) \ typename ::RAPIDJSON_NAMESPACE::internal::DisableIf \ ::Type } // namespace internal RAPIDJSON_NAMESPACE_END //@endcond #if defined(__GNUC__) || defined(_MSC_VER) RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_INTERNAL_META_H_ ================================================ FILE: src/thirdparty/rapidjson/internal/pow10.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_POW10_ #define RAPIDJSON_POW10_ #include "../rapidjson.h" RAPIDJSON_NAMESPACE_BEGIN namespace internal { //! Computes integer powers of 10 in double (10.0^n). /*! This function uses lookup table for fast and accurate results. \param n non-negative exponent. Must <= 308. \return 10.0^n */ inline double Pow10(int n) { static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes 1e+0, 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 }; RAPIDJSON_ASSERT(n >= 0 && n <= 308); return e[n]; } } // namespace internal RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_POW10_ ================================================ FILE: src/thirdparty/rapidjson/internal/regex.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_REGEX_H_ #define RAPIDJSON_INTERNAL_REGEX_H_ #include "../allocators.h" #include "../stream.h" #include "stack.h" #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(switch-enum) RAPIDJSON_DIAG_OFF(implicit-fallthrough) #endif #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif #ifdef _MSC_VER RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif #ifndef RAPIDJSON_REGEX_VERBOSE #define RAPIDJSON_REGEX_VERBOSE 0 #endif RAPIDJSON_NAMESPACE_BEGIN namespace internal { /////////////////////////////////////////////////////////////////////////////// // GenericRegex static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 static const SizeType kRegexInvalidRange = ~SizeType(0); //! Regular expression engine with subset of ECMAscript grammar. /*! Supported regular expression syntax: - \c ab Concatenation - \c a|b Alternation - \c a? Zero or one - \c a* Zero or more - \c a+ One or more - \c a{3} Exactly 3 times - \c a{3,} At least 3 times - \c a{3,5} 3 to 5 times - \c (ab) Grouping - \c ^a At the beginning - \c a$ At the end - \c . Any character - \c [abc] Character classes - \c [a-c] Character class range - \c [a-z0-9_] Character class combination - \c [^abc] Negated character classes - \c [^a-c] Negated character class range - \c [\b] Backspace (U+0008) - \c \\| \\\\ ... Escape characters - \c \\f Form feed (U+000C) - \c \\n Line feed (U+000A) - \c \\r Carriage return (U+000D) - \c \\t Tab (U+0009) - \c \\v Vertical tab (U+000B) \note This is a Thompson NFA engine, implemented with reference to Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", https://swtch.com/~rsc/regexp/regexp1.html */ template class GenericRegex { public: typedef typename Encoding::Ch Ch; GenericRegex(const Ch* source, Allocator* allocator = 0) : states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), stateSet_(), state0_(allocator, 0), state1_(allocator, 0), anchorBegin_(), anchorEnd_() { GenericStringStream ss(source); DecodedStream > ds(ss); Parse(ds); } ~GenericRegex() { Allocator::Free(stateSet_); } bool IsValid() const { return root_ != kRegexInvalidState; } template bool Match(InputStream& is) const { return SearchWithAnchoring(is, true, true); } bool Match(const Ch* s) const { GenericStringStream is(s); return Match(is); } template bool Search(InputStream& is) const { return SearchWithAnchoring(is, anchorBegin_, anchorEnd_); } bool Search(const Ch* s) const { GenericStringStream is(s); return Search(is); } private: enum Operator { kZeroOrOne, kZeroOrMore, kOneOrMore, kConcatenation, kAlternation, kLeftParenthesis }; static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' static const unsigned kRangeCharacterClass = 0xFFFFFFFE; static const unsigned kRangeNegationFlag = 0x80000000; struct Range { unsigned start; // unsigned end; SizeType next; }; struct State { SizeType out; //!< Equals to kInvalid for matching state SizeType out1; //!< Equals to non-kInvalid for split SizeType rangeStart; unsigned codepoint; }; struct Frag { Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} SizeType start; SizeType out; //!< link-list of all output states SizeType minIndex; }; template class DecodedStream { public: DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } unsigned Peek() { return codepoint_; } unsigned Take() { unsigned c = codepoint_; if (c) // No further decoding when '\0' Decode(); return c; } private: void Decode() { if (!Encoding::Decode(ss_, &codepoint_)) codepoint_ = 0; } SourceStream& ss_; unsigned codepoint_; }; State& GetState(SizeType index) { RAPIDJSON_ASSERT(index < stateCount_); return states_.template Bottom()[index]; } const State& GetState(SizeType index) const { RAPIDJSON_ASSERT(index < stateCount_); return states_.template Bottom()[index]; } Range& GetRange(SizeType index) { RAPIDJSON_ASSERT(index < rangeCount_); return ranges_.template Bottom()[index]; } const Range& GetRange(SizeType index) const { RAPIDJSON_ASSERT(index < rangeCount_); return ranges_.template Bottom()[index]; } template void Parse(DecodedStream& ds) { Allocator allocator; Stack operandStack(&allocator, 256); // Frag Stack operatorStack(&allocator, 256); // Operator Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) *atomCountStack.template Push() = 0; unsigned codepoint; while (ds.Peek() != 0) { switch (codepoint = ds.Take()) { case '^': anchorBegin_ = true; break; case '$': anchorEnd_ = true; break; case '|': while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) if (!Eval(operandStack, *operatorStack.template Pop(1))) return; *operatorStack.template Push() = kAlternation; *atomCountStack.template Top() = 0; break; case '(': *operatorStack.template Push() = kLeftParenthesis; *atomCountStack.template Push() = 0; break; case ')': while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) if (!Eval(operandStack, *operatorStack.template Pop(1))) return; if (operatorStack.Empty()) return; operatorStack.template Pop(1); atomCountStack.template Pop(1); ImplicitConcatenation(atomCountStack, operatorStack); break; case '?': if (!Eval(operandStack, kZeroOrOne)) return; break; case '*': if (!Eval(operandStack, kZeroOrMore)) return; break; case '+': if (!Eval(operandStack, kOneOrMore)) return; break; case '{': { unsigned n, m; if (!ParseUnsigned(ds, &n)) return; if (ds.Peek() == ',') { ds.Take(); if (ds.Peek() == '}') m = kInfinityQuantifier; else if (!ParseUnsigned(ds, &m) || m < n) return; } else m = n; if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') return; ds.Take(); } break; case '.': PushOperand(operandStack, kAnyCharacterClass); ImplicitConcatenation(atomCountStack, operatorStack); break; case '[': { SizeType range; if (!ParseRange(ds, &range)) return; SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); GetState(s).rangeStart = range; *operandStack.template Push() = Frag(s, s, s); } ImplicitConcatenation(atomCountStack, operatorStack); break; case '\\': // Escape character if (!CharacterEscape(ds, &codepoint)) return; // Unsupported escape character // fall through to default default: // Pattern character PushOperand(operandStack, codepoint); ImplicitConcatenation(atomCountStack, operatorStack); } } while (!operatorStack.Empty()) if (!Eval(operandStack, *operatorStack.template Pop(1))) return; // Link the operand to matching state. if (operandStack.GetSize() == sizeof(Frag)) { Frag* e = operandStack.template Pop(1); Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); root_ = e->start; #if RAPIDJSON_REGEX_VERBOSE printf("root: %d\n", root_); for (SizeType i = 0; i < stateCount_ ; i++) { State& s = GetState(i); printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); } printf("\n"); #endif } // Preallocate buffer for SearchWithAnchoring() RAPIDJSON_ASSERT(stateSet_ == 0); if (stateCount_ > 0) { stateSet_ = static_cast(states_.GetAllocator().Malloc(GetStateSetSize())); state0_.template Reserve(stateCount_); state1_.template Reserve(stateCount_); } } SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { State* s = states_.template Push(); s->out = out; s->out1 = out1; s->codepoint = codepoint; s->rangeStart = kRegexInvalidRange; return stateCount_++; } void PushOperand(Stack& operandStack, unsigned codepoint) { SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); *operandStack.template Push() = Frag(s, s, s); } void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { if (*atomCountStack.template Top()) *operatorStack.template Push() = kConcatenation; (*atomCountStack.template Top())++; } SizeType Append(SizeType l1, SizeType l2) { SizeType old = l1; while (GetState(l1).out != kRegexInvalidState) l1 = GetState(l1).out; GetState(l1).out = l2; return old; } void Patch(SizeType l, SizeType s) { for (SizeType next; l != kRegexInvalidState; l = next) { next = GetState(l).out; GetState(l).out = s; } } bool Eval(Stack& operandStack, Operator op) { switch (op) { case kConcatenation: RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); { Frag e2 = *operandStack.template Pop(1); Frag e1 = *operandStack.template Pop(1); Patch(e1.out, e2.start); *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); } return true; case kAlternation: if (operandStack.GetSize() >= sizeof(Frag) * 2) { Frag e2 = *operandStack.template Pop(1); Frag e1 = *operandStack.template Pop(1); SizeType s = NewState(e1.start, e2.start, 0); *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); return true; } return false; case kZeroOrOne: if (operandStack.GetSize() >= sizeof(Frag)) { Frag e = *operandStack.template Pop(1); SizeType s = NewState(kRegexInvalidState, e.start, 0); *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); return true; } return false; case kZeroOrMore: if (operandStack.GetSize() >= sizeof(Frag)) { Frag e = *operandStack.template Pop(1); SizeType s = NewState(kRegexInvalidState, e.start, 0); Patch(e.out, s); *operandStack.template Push() = Frag(s, s, e.minIndex); return true; } return false; default: RAPIDJSON_ASSERT(op == kOneOrMore); if (operandStack.GetSize() >= sizeof(Frag)) { Frag e = *operandStack.template Pop(1); SizeType s = NewState(kRegexInvalidState, e.start, 0); Patch(e.out, s); *operandStack.template Push() = Frag(e.start, s, e.minIndex); return true; } return false; } } bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { RAPIDJSON_ASSERT(n <= m); RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); if (n == 0) { if (m == 0) // a{0} not support return false; else if (m == kInfinityQuantifier) Eval(operandStack, kZeroOrMore); // a{0,} -> a* else { Eval(operandStack, kZeroOrOne); // a{0,5} -> a? for (unsigned i = 0; i < m - 1; i++) CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? for (unsigned i = 0; i < m - 1; i++) Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? } return true; } for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a CloneTopOperand(operandStack); if (m == kInfinityQuantifier) Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ else if (m > n) { CloneTopOperand(operandStack); // a{3,5} -> a a a a Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? for (unsigned i = n; i < m - 1; i++) CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? for (unsigned i = n; i < m; i++) Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? } for (unsigned i = 0; i < n - 1; i++) Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? return true; } static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } void CloneTopOperand(Stack& operandStack) { const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) State* s = states_.template Push(count); memcpy(s, &GetState(src.minIndex), count * sizeof(State)); for (SizeType j = 0; j < count; j++) { if (s[j].out != kRegexInvalidState) s[j].out += count; if (s[j].out1 != kRegexInvalidState) s[j].out1 += count; } *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); stateCount_ += count; } template bool ParseUnsigned(DecodedStream& ds, unsigned* u) { unsigned r = 0; if (ds.Peek() < '0' || ds.Peek() > '9') return false; while (ds.Peek() >= '0' && ds.Peek() <= '9') { if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 return false; // overflow r = r * 10 + (ds.Take() - '0'); } *u = r; return true; } template bool ParseRange(DecodedStream& ds, SizeType* range) { bool isBegin = true; bool negate = false; int step = 0; SizeType start = kRegexInvalidRange; SizeType current = kRegexInvalidRange; unsigned codepoint; while ((codepoint = ds.Take()) != 0) { if (isBegin) { isBegin = false; if (codepoint == '^') { negate = true; continue; } } switch (codepoint) { case ']': if (start == kRegexInvalidRange) return false; // Error: nothing inside [] if (step == 2) { // Add trailing '-' SizeType r = NewRange('-'); RAPIDJSON_ASSERT(current != kRegexInvalidRange); GetRange(current).next = r; } if (negate) GetRange(start).start |= kRangeNegationFlag; *range = start; return true; case '\\': if (ds.Peek() == 'b') { ds.Take(); codepoint = 0x0008; // Escape backspace character } else if (!CharacterEscape(ds, &codepoint)) return false; // fall through to default default: switch (step) { case 1: if (codepoint == '-') { step++; break; } // fall through to step 0 for other characters case 0: { SizeType r = NewRange(codepoint); if (current != kRegexInvalidRange) GetRange(current).next = r; if (start == kRegexInvalidRange) start = r; current = r; } step = 1; break; default: RAPIDJSON_ASSERT(step == 2); GetRange(current).end = codepoint; step = 0; } } } return false; } SizeType NewRange(unsigned codepoint) { Range* r = ranges_.template Push(); r->start = r->end = codepoint; r->next = kRegexInvalidRange; return rangeCount_++; } template bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { unsigned codepoint; switch (codepoint = ds.Take()) { case '^': case '$': case '|': case '(': case ')': case '?': case '*': case '+': case '.': case '[': case ']': case '{': case '}': case '\\': *escapedCodepoint = codepoint; return true; case 'f': *escapedCodepoint = 0x000C; return true; case 'n': *escapedCodepoint = 0x000A; return true; case 'r': *escapedCodepoint = 0x000D; return true; case 't': *escapedCodepoint = 0x0009; return true; case 'v': *escapedCodepoint = 0x000B; return true; default: return false; // Unsupported escape character } } template bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) const { RAPIDJSON_ASSERT(IsValid()); DecodedStream ds(is); state0_.Clear(); Stack *current = &state0_, *next = &state1_; const size_t stateSetSize = GetStateSetSize(); std::memset(stateSet_, 0, stateSetSize); bool matched = AddState(*current, root_); unsigned codepoint; while (!current->Empty() && (codepoint = ds.Take()) != 0) { std::memset(stateSet_, 0, stateSetSize); next->Clear(); matched = false; for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { const State& sr = GetState(*s); if (sr.codepoint == codepoint || sr.codepoint == kAnyCharacterClass || (sr.codepoint == kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) { matched = AddState(*next, sr.out) || matched; if (!anchorEnd && matched) return true; } if (!anchorBegin) AddState(*next, root_); } internal::Swap(current, next); } return matched; } size_t GetStateSetSize() const { return (stateCount_ + 31) / 32 * 4; } // Return whether the added states is a match state bool AddState(Stack& l, SizeType index) const { RAPIDJSON_ASSERT(index != kRegexInvalidState); const State& s = GetState(index); if (s.out1 != kRegexInvalidState) { // Split bool matched = AddState(l, s.out); return AddState(l, s.out1) || matched; } else if (!(stateSet_[index >> 5] & (1 << (index & 31)))) { stateSet_[index >> 5] |= (1 << (index & 31)); *l.template PushUnsafe() = index; } return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. } bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { bool yes = (GetRange(rangeIndex).start & kRangeNegationFlag) == 0; while (rangeIndex != kRegexInvalidRange) { const Range& r = GetRange(rangeIndex); if (codepoint >= (r.start & ~kRangeNegationFlag) && codepoint <= r.end) return yes; rangeIndex = r.next; } return !yes; } Stack states_; Stack ranges_; SizeType root_; SizeType stateCount_; SizeType rangeCount_; static const unsigned kInfinityQuantifier = ~0u; // For SearchWithAnchoring() uint32_t* stateSet_; // allocated by states_.GetAllocator() mutable Stack state0_; mutable Stack state1_; bool anchorBegin_; bool anchorEnd_; }; typedef GenericRegex > Regex; } // namespace internal RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #ifdef _MSC_VER RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_INTERNAL_REGEX_H_ ================================================ FILE: src/thirdparty/rapidjson/internal/stack.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_STACK_H_ #define RAPIDJSON_INTERNAL_STACK_H_ #include "../allocators.h" #include "swap.h" #if defined(__clang__) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(c++98-compat) #endif RAPIDJSON_NAMESPACE_BEGIN namespace internal { /////////////////////////////////////////////////////////////////////////////// // Stack //! A type-unsafe stack for storing different types of data. /*! \tparam Allocator Allocator for allocating stack memory. */ template class Stack { public: // Optimization note: Do not allocate memory for stack_ in constructor. // Do it lazily when first Push() -> Expand() -> Resize(). Stack(Allocator* allocator, size_t stackCapacity) : allocator_(allocator), ownAllocator_(0), stack_(0), stackTop_(0), stackEnd_(0), initialCapacity_(stackCapacity) { } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS Stack(Stack&& rhs) : allocator_(rhs.allocator_), ownAllocator_(rhs.ownAllocator_), stack_(rhs.stack_), stackTop_(rhs.stackTop_), stackEnd_(rhs.stackEnd_), initialCapacity_(rhs.initialCapacity_) { rhs.allocator_ = 0; rhs.ownAllocator_ = 0; rhs.stack_ = 0; rhs.stackTop_ = 0; rhs.stackEnd_ = 0; rhs.initialCapacity_ = 0; } #endif ~Stack() { Destroy(); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS Stack& operator=(Stack&& rhs) { if (&rhs != this) { Destroy(); allocator_ = rhs.allocator_; ownAllocator_ = rhs.ownAllocator_; stack_ = rhs.stack_; stackTop_ = rhs.stackTop_; stackEnd_ = rhs.stackEnd_; initialCapacity_ = rhs.initialCapacity_; rhs.allocator_ = 0; rhs.ownAllocator_ = 0; rhs.stack_ = 0; rhs.stackTop_ = 0; rhs.stackEnd_ = 0; rhs.initialCapacity_ = 0; } return *this; } #endif void Swap(Stack& rhs) RAPIDJSON_NOEXCEPT { internal::Swap(allocator_, rhs.allocator_); internal::Swap(ownAllocator_, rhs.ownAllocator_); internal::Swap(stack_, rhs.stack_); internal::Swap(stackTop_, rhs.stackTop_); internal::Swap(stackEnd_, rhs.stackEnd_); internal::Swap(initialCapacity_, rhs.initialCapacity_); } void Clear() { stackTop_ = stack_; } void ShrinkToFit() { if (Empty()) { // If the stack is empty, completely deallocate the memory. Allocator::Free(stack_); stack_ = 0; stackTop_ = 0; stackEnd_ = 0; } else Resize(GetSize()); } // Optimization note: try to minimize the size of this function for force inline. // Expansion is run very infrequently, so it is moved to another (probably non-inline) function. template RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { // Expand the stack if needed if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) Expand(count); } template RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) { Reserve(count); return PushUnsafe(count); } template RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); T* ret = reinterpret_cast(stackTop_); stackTop_ += sizeof(T) * count; return ret; } template T* Pop(size_t count) { RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); stackTop_ -= count * sizeof(T); return reinterpret_cast(stackTop_); } template T* Top() { RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); return reinterpret_cast(stackTop_ - sizeof(T)); } template const T* Top() const { RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); return reinterpret_cast(stackTop_ - sizeof(T)); } template T* End() { return reinterpret_cast(stackTop_); } template const T* End() const { return reinterpret_cast(stackTop_); } template T* Bottom() { return reinterpret_cast(stack_); } template const T* Bottom() const { return reinterpret_cast(stack_); } bool HasAllocator() const { return allocator_ != 0; } Allocator& GetAllocator() { RAPIDJSON_ASSERT(allocator_); return *allocator_; } bool Empty() const { return stackTop_ == stack_; } size_t GetSize() const { return static_cast(stackTop_ - stack_); } size_t GetCapacity() const { return static_cast(stackEnd_ - stack_); } private: template void Expand(size_t count) { // Only expand the capacity if the current stack exists. Otherwise just create a stack with initial capacity. size_t newCapacity; if (stack_ == 0) { if (!allocator_) ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); newCapacity = initialCapacity_; } else { newCapacity = GetCapacity(); newCapacity += (newCapacity + 1) / 2; } size_t newSize = GetSize() + sizeof(T) * count; if (newCapacity < newSize) newCapacity = newSize; Resize(newCapacity); } void Resize(size_t newCapacity) { const size_t size = GetSize(); // Backup the current size stack_ = static_cast(allocator_->Realloc(stack_, GetCapacity(), newCapacity)); stackTop_ = stack_ + size; stackEnd_ = stack_ + newCapacity; } void Destroy() { Allocator::Free(stack_); RAPIDJSON_DELETE(ownAllocator_); // Only delete if it is owned by the stack } // Prohibit copy constructor & assignment operator. Stack(const Stack&); Stack& operator=(const Stack&); Allocator* allocator_; Allocator* ownAllocator_; char *stack_; char *stackTop_; char *stackEnd_; size_t initialCapacity_; }; } // namespace internal RAPIDJSON_NAMESPACE_END #if defined(__clang__) RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_STACK_H_ ================================================ FILE: src/thirdparty/rapidjson/internal/strfunc.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ #define RAPIDJSON_INTERNAL_STRFUNC_H_ #include "../stream.h" RAPIDJSON_NAMESPACE_BEGIN namespace internal { //! Custom strlen() which works on different character types. /*! \tparam Ch Character type (e.g. char, wchar_t, short) \param s Null-terminated input string. \return Number of characters in the string. \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. */ template inline SizeType StrLen(const Ch* s) { const Ch* p = s; while (*p) ++p; return SizeType(p - s); } //! Returns number of code points in a encoded string. template bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { GenericStringStream is(s); const typename Encoding::Ch* end = s + length; SizeType count = 0; while (is.src_ < end) { unsigned codepoint; if (!Encoding::Decode(is, &codepoint)) return false; count++; } *outCount = count; return true; } } // namespace internal RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_INTERNAL_STRFUNC_H_ ================================================ FILE: src/thirdparty/rapidjson/internal/strtod.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_STRTOD_ #define RAPIDJSON_STRTOD_ #include "ieee754.h" #include "biginteger.h" #include "diyfp.h" #include "pow10.h" RAPIDJSON_NAMESPACE_BEGIN namespace internal { inline double FastPath(double significand, int exp) { if (exp < -308) return 0.0; else if (exp >= 0) return significand * internal::Pow10(exp); else return significand / internal::Pow10(-exp); } inline double StrtodNormalPrecision(double d, int p) { if (p < -308) { // Prevent expSum < -308, making Pow10(p) = 0 d = FastPath(d, -308); d = FastPath(d, p + 308); } else d = FastPath(d, p); return d; } template inline T Min3(T a, T b, T c) { T m = a; if (m > b) m = b; if (m > c) m = c; return m; } inline int CheckWithinHalfULP(double b, const BigInteger& d, int dExp) { const Double db(b); const uint64_t bInt = db.IntegerSignificand(); const int bExp = db.IntegerExponent(); const int hExp = bExp - 1; int dS_Exp2 = 0, dS_Exp5 = 0, bS_Exp2 = 0, bS_Exp5 = 0, hS_Exp2 = 0, hS_Exp5 = 0; // Adjust for decimal exponent if (dExp >= 0) { dS_Exp2 += dExp; dS_Exp5 += dExp; } else { bS_Exp2 -= dExp; bS_Exp5 -= dExp; hS_Exp2 -= dExp; hS_Exp5 -= dExp; } // Adjust for binary exponent if (bExp >= 0) bS_Exp2 += bExp; else { dS_Exp2 -= bExp; hS_Exp2 -= bExp; } // Adjust for half ulp exponent if (hExp >= 0) hS_Exp2 += hExp; else { dS_Exp2 -= hExp; bS_Exp2 -= hExp; } // Remove common power of two factor from all three scaled values int common_Exp2 = Min3(dS_Exp2, bS_Exp2, hS_Exp2); dS_Exp2 -= common_Exp2; bS_Exp2 -= common_Exp2; hS_Exp2 -= common_Exp2; BigInteger dS = d; dS.MultiplyPow5(static_cast(dS_Exp5)) <<= static_cast(dS_Exp2); BigInteger bS(bInt); bS.MultiplyPow5(static_cast(bS_Exp5)) <<= static_cast(bS_Exp2); BigInteger hS(1); hS.MultiplyPow5(static_cast(hS_Exp5)) <<= static_cast(hS_Exp2); BigInteger delta(0); dS.Difference(bS, &delta); return delta.Compare(hS); } inline bool StrtodFast(double d, int p, double* result) { // Use fast path for string-to-double conversion if possible // see http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ if (p > 22 && p < 22 + 16) { // Fast Path Cases In Disguise d *= internal::Pow10(p - 22); p = 22; } if (p >= -22 && p <= 22 && d <= 9007199254740991.0) { // 2^53 - 1 *result = FastPath(d, p); return true; } else return false; } // Compute an approximation and see if it is within 1/2 ULP inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) { uint64_t significand = 0; size_t i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 for (; i < length; i++) { if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) break; significand = significand * 10u + static_cast(decimals[i] - '0'); } if (i < length && decimals[i] >= '5') // Rounding significand++; size_t remaining = length - i; const unsigned kUlpShift = 3; const unsigned kUlp = 1 << kUlpShift; int64_t error = (remaining == 0) ? 0 : kUlp / 2; DiyFp v(significand, 0); v = v.Normalize(); error <<= -v.e; const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; int actualExp; DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); if (actualExp != dExp) { static const DiyFp kPow10[] = { DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60), // 10^1 DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57), // 10^2 DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54), // 10^3 DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50), // 10^4 DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47), // 10^5 DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 }; int adjustment = dExp - actualExp - 1; RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); v = v * kPow10[adjustment]; if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit error += kUlp / 2; } v = v * cachedPower; error += kUlp + (error == 0 ? 0 : 1); const int oldExp = v.e; v = v.Normalize(); error <<= oldExp - v.e; const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); unsigned precisionSize = 64 - effectiveSignificandSize; if (precisionSize + kUlpShift >= 64) { unsigned scaleExp = (precisionSize + kUlpShift) - 63; v.f >>= scaleExp; v.e += scaleExp; error = (error >> scaleExp) + 1 + static_cast(kUlp); precisionSize -= scaleExp; } DiyFp rounded(v.f >> precisionSize, v.e + static_cast(precisionSize)); const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; if (precisionBits >= halfWay + static_cast(error)) { rounded.f++; if (rounded.f & (DiyFp::kDpHiddenBit << 1)) { // rounding overflows mantissa (issue #340) rounded.f >>= 1; rounded.e++; } } *result = rounded.ToDouble(); return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); } inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { const BigInteger dInt(decimals, length); const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; Double a(approx); int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); if (cmp < 0) return a.Value(); // within half ULP else if (cmp == 0) { // Round towards even if (a.Significand() & 1) return a.NextPositiveDouble(); else return a.Value(); } else // adjustment return a.NextPositiveDouble(); } inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { RAPIDJSON_ASSERT(d >= 0.0); RAPIDJSON_ASSERT(length >= 1); double result; if (StrtodFast(d, p, &result)) return result; // Trim leading zeros while (*decimals == '0' && length > 1) { length--; decimals++; decimalPosition--; } // Trim trailing zeros while (decimals[length - 1] == '0' && length > 1) { length--; decimalPosition--; exp++; } // Trim right-most digits const int kMaxDecimalDigit = 780; if (static_cast(length) > kMaxDecimalDigit) { int delta = (static_cast(length) - kMaxDecimalDigit); exp += delta; decimalPosition -= static_cast(delta); length = kMaxDecimalDigit; } // If too small, underflow to zero if (int(length) + exp < -324) return 0.0; if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result)) return result; // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison return StrtodBigInteger(result, decimals, length, decimalPosition, exp); } } // namespace internal RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_STRTOD_ ================================================ FILE: src/thirdparty/rapidjson/internal/swap.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_INTERNAL_SWAP_H_ #define RAPIDJSON_INTERNAL_SWAP_H_ #include "../rapidjson.h" #if defined(__clang__) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(c++98-compat) #endif RAPIDJSON_NAMESPACE_BEGIN namespace internal { //! Custom swap() to avoid dependency on C++ header /*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. \note This has the same semantics as std::swap(). */ template inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { T tmp = a; a = b; b = tmp; } } // namespace internal RAPIDJSON_NAMESPACE_END #if defined(__clang__) RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_INTERNAL_SWAP_H_ ================================================ FILE: src/thirdparty/rapidjson/istreamwrapper.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ISTREAMWRAPPER_H_ #define RAPIDJSON_ISTREAMWRAPPER_H_ #include "stream.h" #include #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) #endif #ifdef _MSC_VER RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized #endif RAPIDJSON_NAMESPACE_BEGIN //! Wrapper of \c std::basic_istream into RapidJSON's Stream concept. /*! The classes can be wrapped including but not limited to: - \c std::istringstream - \c std::stringstream - \c std::wistringstream - \c std::wstringstream - \c std::ifstream - \c std::fstream - \c std::wifstream - \c std::wfstream \tparam StreamType Class derived from \c std::basic_istream. */ template class BasicIStreamWrapper { public: typedef typename StreamType::char_type Ch; BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} Ch Peek() const { typename StreamType::int_type c = stream_.peek(); return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : '\0'; } Ch Take() { typename StreamType::int_type c = stream_.get(); if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { count_++; return static_cast(c); } else return '\0'; } // tellg() may return -1 when failed. So we count by ourself. size_t Tell() const { return count_; } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } void Put(Ch) { RAPIDJSON_ASSERT(false); } void Flush() { RAPIDJSON_ASSERT(false); } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } // For encoding detection only. const Ch* Peek4() const { RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. int i; bool hasError = false; for (i = 0; i < 4; ++i) { typename StreamType::int_type c = stream_.get(); if (c == StreamType::traits_type::eof()) { hasError = true; stream_.clear(); break; } peekBuffer_[i] = static_cast(c); } for (--i; i >= 0; --i) stream_.putback(peekBuffer_[i]); return !hasError ? peekBuffer_ : 0; } private: BasicIStreamWrapper(const BasicIStreamWrapper&); BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); StreamType& stream_; size_t count_; //!< Number of characters read. Note: mutable Ch peekBuffer_[4]; }; typedef BasicIStreamWrapper IStreamWrapper; typedef BasicIStreamWrapper WIStreamWrapper; #if defined(__clang__) || defined(_MSC_VER) RAPIDJSON_DIAG_POP #endif RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_ISTREAMWRAPPER_H_ ================================================ FILE: src/thirdparty/rapidjson/memorybuffer.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_MEMORYBUFFER_H_ #define RAPIDJSON_MEMORYBUFFER_H_ #include "stream.h" #include "internal/stack.h" RAPIDJSON_NAMESPACE_BEGIN //! Represents an in-memory output byte stream. /*! This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. Differences between MemoryBuffer and StringBuffer: 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. \tparam Allocator type for allocating memory buffer. \note implements Stream concept */ template struct GenericMemoryBuffer { typedef char Ch; // byte GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} void Put(Ch c) { *stack_.template Push() = c; } void Flush() {} void Clear() { stack_.Clear(); } void ShrinkToFit() { stack_.ShrinkToFit(); } Ch* Push(size_t count) { return stack_.template Push(count); } void Pop(size_t count) { stack_.template Pop(count); } const Ch* GetBuffer() const { return stack_.template Bottom(); } size_t GetSize() const { return stack_.GetSize(); } static const size_t kDefaultCapacity = 256; mutable internal::Stack stack_; }; typedef GenericMemoryBuffer<> MemoryBuffer; //! Implement specialized version of PutN() with memset() for better performance. template<> inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); } RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_MEMORYBUFFER_H_ ================================================ FILE: src/thirdparty/rapidjson/memorystream.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_MEMORYSTREAM_H_ #define RAPIDJSON_MEMORYSTREAM_H_ #include "stream.h" #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(unreachable-code) RAPIDJSON_DIAG_OFF(missing-noreturn) #endif RAPIDJSON_NAMESPACE_BEGIN //! Represents an in-memory input byte stream. /*! This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. Differences between MemoryStream and StringStream: 1. StringStream has encoding but MemoryStream is a byte stream. 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). \note implements Stream concept */ struct MemoryStream { typedef char Ch; // byte MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} Ch Peek() const { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_; } Ch Take() { return RAPIDJSON_UNLIKELY(src_ == end_) ? '\0' : *src_++; } size_t Tell() const { return static_cast(src_ - begin_); } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } void Put(Ch) { RAPIDJSON_ASSERT(false); } void Flush() { RAPIDJSON_ASSERT(false); } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } // For encoding detection only. const Ch* Peek4() const { return Tell() + 4 <= size_ ? src_ : 0; } const Ch* src_; //!< Current read position. const Ch* begin_; //!< Original head of the string. const Ch* end_; //!< End of stream. size_t size_; //!< Size of the stream. }; RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_MEMORYBUFFER_H_ ================================================ FILE: src/thirdparty/rapidjson/msinttypes/inttypes.h ================================================ // ISO C9x compliant inttypes.h for Microsoft Visual Studio // Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 // // Copyright (c) 2006-2013 Alexander Chemeris // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // 3. Neither the name of the product nor the names of its contributors may // be used to endorse or promote products derived from this software // without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // /////////////////////////////////////////////////////////////////////////////// // The above software in this distribution may have been modified by // THL A29 Limited ("Tencent Modifications"). // All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. #ifndef _MSC_VER // [ #error "Use this header only with Microsoft Visual C++ compilers!" #endif // _MSC_VER ] #ifndef _MSC_INTTYPES_H_ // [ #define _MSC_INTTYPES_H_ #if _MSC_VER > 1000 #pragma once #endif #include "stdint.h" // miloyip: VC supports inttypes.h since VC2013 #if _MSC_VER >= 1800 #include #else // 7.8 Format conversion of integer types typedef struct { intmax_t quot; intmax_t rem; } imaxdiv_t; // 7.8.1 Macros for format specifiers #if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 // The fprintf macros for signed integers are: #define PRId8 "d" #define PRIi8 "i" #define PRIdLEAST8 "d" #define PRIiLEAST8 "i" #define PRIdFAST8 "d" #define PRIiFAST8 "i" #define PRId16 "hd" #define PRIi16 "hi" #define PRIdLEAST16 "hd" #define PRIiLEAST16 "hi" #define PRIdFAST16 "hd" #define PRIiFAST16 "hi" #define PRId32 "I32d" #define PRIi32 "I32i" #define PRIdLEAST32 "I32d" #define PRIiLEAST32 "I32i" #define PRIdFAST32 "I32d" #define PRIiFAST32 "I32i" #define PRId64 "I64d" #define PRIi64 "I64i" #define PRIdLEAST64 "I64d" #define PRIiLEAST64 "I64i" #define PRIdFAST64 "I64d" #define PRIiFAST64 "I64i" #define PRIdMAX "I64d" #define PRIiMAX "I64i" #define PRIdPTR "Id" #define PRIiPTR "Ii" // The fprintf macros for unsigned integers are: #define PRIo8 "o" #define PRIu8 "u" #define PRIx8 "x" #define PRIX8 "X" #define PRIoLEAST8 "o" #define PRIuLEAST8 "u" #define PRIxLEAST8 "x" #define PRIXLEAST8 "X" #define PRIoFAST8 "o" #define PRIuFAST8 "u" #define PRIxFAST8 "x" #define PRIXFAST8 "X" #define PRIo16 "ho" #define PRIu16 "hu" #define PRIx16 "hx" #define PRIX16 "hX" #define PRIoLEAST16 "ho" #define PRIuLEAST16 "hu" #define PRIxLEAST16 "hx" #define PRIXLEAST16 "hX" #define PRIoFAST16 "ho" #define PRIuFAST16 "hu" #define PRIxFAST16 "hx" #define PRIXFAST16 "hX" #define PRIo32 "I32o" #define PRIu32 "I32u" #define PRIx32 "I32x" #define PRIX32 "I32X" #define PRIoLEAST32 "I32o" #define PRIuLEAST32 "I32u" #define PRIxLEAST32 "I32x" #define PRIXLEAST32 "I32X" #define PRIoFAST32 "I32o" #define PRIuFAST32 "I32u" #define PRIxFAST32 "I32x" #define PRIXFAST32 "I32X" #define PRIo64 "I64o" #define PRIu64 "I64u" #define PRIx64 "I64x" #define PRIX64 "I64X" #define PRIoLEAST64 "I64o" #define PRIuLEAST64 "I64u" #define PRIxLEAST64 "I64x" #define PRIXLEAST64 "I64X" #define PRIoFAST64 "I64o" #define PRIuFAST64 "I64u" #define PRIxFAST64 "I64x" #define PRIXFAST64 "I64X" #define PRIoMAX "I64o" #define PRIuMAX "I64u" #define PRIxMAX "I64x" #define PRIXMAX "I64X" #define PRIoPTR "Io" #define PRIuPTR "Iu" #define PRIxPTR "Ix" #define PRIXPTR "IX" // The fscanf macros for signed integers are: #define SCNd8 "d" #define SCNi8 "i" #define SCNdLEAST8 "d" #define SCNiLEAST8 "i" #define SCNdFAST8 "d" #define SCNiFAST8 "i" #define SCNd16 "hd" #define SCNi16 "hi" #define SCNdLEAST16 "hd" #define SCNiLEAST16 "hi" #define SCNdFAST16 "hd" #define SCNiFAST16 "hi" #define SCNd32 "ld" #define SCNi32 "li" #define SCNdLEAST32 "ld" #define SCNiLEAST32 "li" #define SCNdFAST32 "ld" #define SCNiFAST32 "li" #define SCNd64 "I64d" #define SCNi64 "I64i" #define SCNdLEAST64 "I64d" #define SCNiLEAST64 "I64i" #define SCNdFAST64 "I64d" #define SCNiFAST64 "I64i" #define SCNdMAX "I64d" #define SCNiMAX "I64i" #ifdef _WIN64 // [ # define SCNdPTR "I64d" # define SCNiPTR "I64i" #else // _WIN64 ][ # define SCNdPTR "ld" # define SCNiPTR "li" #endif // _WIN64 ] // The fscanf macros for unsigned integers are: #define SCNo8 "o" #define SCNu8 "u" #define SCNx8 "x" #define SCNX8 "X" #define SCNoLEAST8 "o" #define SCNuLEAST8 "u" #define SCNxLEAST8 "x" #define SCNXLEAST8 "X" #define SCNoFAST8 "o" #define SCNuFAST8 "u" #define SCNxFAST8 "x" #define SCNXFAST8 "X" #define SCNo16 "ho" #define SCNu16 "hu" #define SCNx16 "hx" #define SCNX16 "hX" #define SCNoLEAST16 "ho" #define SCNuLEAST16 "hu" #define SCNxLEAST16 "hx" #define SCNXLEAST16 "hX" #define SCNoFAST16 "ho" #define SCNuFAST16 "hu" #define SCNxFAST16 "hx" #define SCNXFAST16 "hX" #define SCNo32 "lo" #define SCNu32 "lu" #define SCNx32 "lx" #define SCNX32 "lX" #define SCNoLEAST32 "lo" #define SCNuLEAST32 "lu" #define SCNxLEAST32 "lx" #define SCNXLEAST32 "lX" #define SCNoFAST32 "lo" #define SCNuFAST32 "lu" #define SCNxFAST32 "lx" #define SCNXFAST32 "lX" #define SCNo64 "I64o" #define SCNu64 "I64u" #define SCNx64 "I64x" #define SCNX64 "I64X" #define SCNoLEAST64 "I64o" #define SCNuLEAST64 "I64u" #define SCNxLEAST64 "I64x" #define SCNXLEAST64 "I64X" #define SCNoFAST64 "I64o" #define SCNuFAST64 "I64u" #define SCNxFAST64 "I64x" #define SCNXFAST64 "I64X" #define SCNoMAX "I64o" #define SCNuMAX "I64u" #define SCNxMAX "I64x" #define SCNXMAX "I64X" #ifdef _WIN64 // [ # define SCNoPTR "I64o" # define SCNuPTR "I64u" # define SCNxPTR "I64x" # define SCNXPTR "I64X" #else // _WIN64 ][ # define SCNoPTR "lo" # define SCNuPTR "lu" # define SCNxPTR "lx" # define SCNXPTR "lX" #endif // _WIN64 ] #endif // __STDC_FORMAT_MACROS ] // 7.8.2 Functions for greatest-width integer types // 7.8.2.1 The imaxabs function #define imaxabs _abs64 // 7.8.2.2 The imaxdiv function // This is modified version of div() function from Microsoft's div.c found // in %MSVC.NET%\crt\src\div.c #ifdef STATIC_IMAXDIV // [ static #else // STATIC_IMAXDIV ][ _inline #endif // STATIC_IMAXDIV ] imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) { imaxdiv_t result; result.quot = numer / denom; result.rem = numer % denom; if (numer < 0 && result.rem > 0) { // did division wrong; must fix up ++result.quot; result.rem -= denom; } return result; } // 7.8.2.3 The strtoimax and strtoumax functions #define strtoimax _strtoi64 #define strtoumax _strtoui64 // 7.8.2.4 The wcstoimax and wcstoumax functions #define wcstoimax _wcstoi64 #define wcstoumax _wcstoui64 #endif // _MSC_VER >= 1800 #endif // _MSC_INTTYPES_H_ ] ================================================ FILE: src/thirdparty/rapidjson/msinttypes/stdint.h ================================================ // ISO C9x compliant stdint.h for Microsoft Visual Studio // Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 // // Copyright (c) 2006-2013 Alexander Chemeris // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // 3. Neither the name of the product nor the names of its contributors may // be used to endorse or promote products derived from this software // without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // /////////////////////////////////////////////////////////////////////////////// // The above software in this distribution may have been modified by // THL A29 Limited ("Tencent Modifications"). // All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. #ifndef _MSC_VER // [ #error "Use this header only with Microsoft Visual C++ compilers!" #endif // _MSC_VER ] #ifndef _MSC_STDINT_H_ // [ #define _MSC_STDINT_H_ #if _MSC_VER > 1000 #pragma once #endif // miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. #if _MSC_VER >= 1600 // [ #include #if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 #undef INT8_C #undef INT16_C #undef INT32_C #undef INT64_C #undef UINT8_C #undef UINT16_C #undef UINT32_C #undef UINT64_C // 7.18.4.1 Macros for minimum-width integer constants #define INT8_C(val) val##i8 #define INT16_C(val) val##i16 #define INT32_C(val) val##i32 #define INT64_C(val) val##i64 #define UINT8_C(val) val##ui8 #define UINT16_C(val) val##ui16 #define UINT32_C(val) val##ui32 #define UINT64_C(val) val##ui64 // 7.18.4.2 Macros for greatest-width integer constants // These #ifndef's are needed to prevent collisions with . // Check out Issue 9 for the details. #ifndef INTMAX_C // [ # define INTMAX_C INT64_C #endif // INTMAX_C ] #ifndef UINTMAX_C // [ # define UINTMAX_C UINT64_C #endif // UINTMAX_C ] #endif // __STDC_CONSTANT_MACROS ] #else // ] _MSC_VER >= 1700 [ #include // For Visual Studio 6 in C++ mode and for many Visual Studio versions when // compiling for ARM we have to wrap include with 'extern "C++" {}' // or compiler would give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed #if defined(__cplusplus) && !defined(_M_ARM) extern "C" { #endif # include #if defined(__cplusplus) && !defined(_M_ARM) } #endif // Define _W64 macros to mark types changing their size, like intptr_t. #ifndef _W64 # if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 # define _W64 __w64 # else # define _W64 # endif #endif // 7.18.1 Integer types // 7.18.1.1 Exact-width integer types // Visual Studio 6 and Embedded Visual C++ 4 doesn't // realize that, e.g. char has the same size as __int8 // so we give up on __intX for them. #if (_MSC_VER < 1300) typedef signed char int8_t; typedef signed short int16_t; typedef signed int int32_t; typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; #else typedef signed __int8 int8_t; typedef signed __int16 int16_t; typedef signed __int32 int32_t; typedef unsigned __int8 uint8_t; typedef unsigned __int16 uint16_t; typedef unsigned __int32 uint32_t; #endif typedef signed __int64 int64_t; typedef unsigned __int64 uint64_t; // 7.18.1.2 Minimum-width integer types typedef int8_t int_least8_t; typedef int16_t int_least16_t; typedef int32_t int_least32_t; typedef int64_t int_least64_t; typedef uint8_t uint_least8_t; typedef uint16_t uint_least16_t; typedef uint32_t uint_least32_t; typedef uint64_t uint_least64_t; // 7.18.1.3 Fastest minimum-width integer types typedef int8_t int_fast8_t; typedef int16_t int_fast16_t; typedef int32_t int_fast32_t; typedef int64_t int_fast64_t; typedef uint8_t uint_fast8_t; typedef uint16_t uint_fast16_t; typedef uint32_t uint_fast32_t; typedef uint64_t uint_fast64_t; // 7.18.1.4 Integer types capable of holding object pointers #ifdef _WIN64 // [ typedef signed __int64 intptr_t; typedef unsigned __int64 uintptr_t; #else // _WIN64 ][ typedef _W64 signed int intptr_t; typedef _W64 unsigned int uintptr_t; #endif // _WIN64 ] // 7.18.1.5 Greatest-width integer types typedef int64_t intmax_t; typedef uint64_t uintmax_t; // 7.18.2 Limits of specified-width integer types #if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 // 7.18.2.1 Limits of exact-width integer types #define INT8_MIN ((int8_t)_I8_MIN) #define INT8_MAX _I8_MAX #define INT16_MIN ((int16_t)_I16_MIN) #define INT16_MAX _I16_MAX #define INT32_MIN ((int32_t)_I32_MIN) #define INT32_MAX _I32_MAX #define INT64_MIN ((int64_t)_I64_MIN) #define INT64_MAX _I64_MAX #define UINT8_MAX _UI8_MAX #define UINT16_MAX _UI16_MAX #define UINT32_MAX _UI32_MAX #define UINT64_MAX _UI64_MAX // 7.18.2.2 Limits of minimum-width integer types #define INT_LEAST8_MIN INT8_MIN #define INT_LEAST8_MAX INT8_MAX #define INT_LEAST16_MIN INT16_MIN #define INT_LEAST16_MAX INT16_MAX #define INT_LEAST32_MIN INT32_MIN #define INT_LEAST32_MAX INT32_MAX #define INT_LEAST64_MIN INT64_MIN #define INT_LEAST64_MAX INT64_MAX #define UINT_LEAST8_MAX UINT8_MAX #define UINT_LEAST16_MAX UINT16_MAX #define UINT_LEAST32_MAX UINT32_MAX #define UINT_LEAST64_MAX UINT64_MAX // 7.18.2.3 Limits of fastest minimum-width integer types #define INT_FAST8_MIN INT8_MIN #define INT_FAST8_MAX INT8_MAX #define INT_FAST16_MIN INT16_MIN #define INT_FAST16_MAX INT16_MAX #define INT_FAST32_MIN INT32_MIN #define INT_FAST32_MAX INT32_MAX #define INT_FAST64_MIN INT64_MIN #define INT_FAST64_MAX INT64_MAX #define UINT_FAST8_MAX UINT8_MAX #define UINT_FAST16_MAX UINT16_MAX #define UINT_FAST32_MAX UINT32_MAX #define UINT_FAST64_MAX UINT64_MAX // 7.18.2.4 Limits of integer types capable of holding object pointers #ifdef _WIN64 // [ # define INTPTR_MIN INT64_MIN # define INTPTR_MAX INT64_MAX # define UINTPTR_MAX UINT64_MAX #else // _WIN64 ][ # define INTPTR_MIN INT32_MIN # define INTPTR_MAX INT32_MAX # define UINTPTR_MAX UINT32_MAX #endif // _WIN64 ] // 7.18.2.5 Limits of greatest-width integer types #define INTMAX_MIN INT64_MIN #define INTMAX_MAX INT64_MAX #define UINTMAX_MAX UINT64_MAX // 7.18.3 Limits of other integer types #ifdef _WIN64 // [ # define PTRDIFF_MIN _I64_MIN # define PTRDIFF_MAX _I64_MAX #else // _WIN64 ][ # define PTRDIFF_MIN _I32_MIN # define PTRDIFF_MAX _I32_MAX #endif // _WIN64 ] #define SIG_ATOMIC_MIN INT_MIN #define SIG_ATOMIC_MAX INT_MAX #ifndef SIZE_MAX // [ # ifdef _WIN64 // [ # define SIZE_MAX _UI64_MAX # else // _WIN64 ][ # define SIZE_MAX _UI32_MAX # endif // _WIN64 ] #endif // SIZE_MAX ] // WCHAR_MIN and WCHAR_MAX are also defined in #ifndef WCHAR_MIN // [ # define WCHAR_MIN 0 #endif // WCHAR_MIN ] #ifndef WCHAR_MAX // [ # define WCHAR_MAX _UI16_MAX #endif // WCHAR_MAX ] #define WINT_MIN 0 #define WINT_MAX _UI16_MAX #endif // __STDC_LIMIT_MACROS ] // 7.18.4 Limits of other integer types #if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 // 7.18.4.1 Macros for minimum-width integer constants #define INT8_C(val) val##i8 #define INT16_C(val) val##i16 #define INT32_C(val) val##i32 #define INT64_C(val) val##i64 #define UINT8_C(val) val##ui8 #define UINT16_C(val) val##ui16 #define UINT32_C(val) val##ui32 #define UINT64_C(val) val##ui64 // 7.18.4.2 Macros for greatest-width integer constants // These #ifndef's are needed to prevent collisions with . // Check out Issue 9 for the details. #ifndef INTMAX_C // [ # define INTMAX_C INT64_C #endif // INTMAX_C ] #ifndef UINTMAX_C // [ # define UINTMAX_C UINT64_C #endif // UINTMAX_C ] #endif // __STDC_CONSTANT_MACROS ] #endif // _MSC_VER >= 1600 ] #endif // _MSC_STDINT_H_ ] ================================================ FILE: src/thirdparty/rapidjson/ostreamwrapper.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_OSTREAMWRAPPER_H_ #define RAPIDJSON_OSTREAMWRAPPER_H_ #include "stream.h" #include #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) #endif RAPIDJSON_NAMESPACE_BEGIN //! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. /*! The classes can be wrapped including but not limited to: - \c std::ostringstream - \c std::stringstream - \c std::wpstringstream - \c std::wstringstream - \c std::ifstream - \c std::fstream - \c std::wofstream - \c std::wfstream \tparam StreamType Class derived from \c std::basic_ostream. */ template class BasicOStreamWrapper { public: typedef typename StreamType::char_type Ch; BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} void Put(Ch c) { stream_.put(c); } void Flush() { stream_.flush(); } // Not implemented char Peek() const { RAPIDJSON_ASSERT(false); return 0; } char Take() { RAPIDJSON_ASSERT(false); return 0; } size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } private: BasicOStreamWrapper(const BasicOStreamWrapper&); BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); StreamType& stream_; }; typedef BasicOStreamWrapper OStreamWrapper; typedef BasicOStreamWrapper WOStreamWrapper; #ifdef __clang__ RAPIDJSON_DIAG_POP #endif RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_OSTREAMWRAPPER_H_ ================================================ FILE: src/thirdparty/rapidjson/pointer.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_POINTER_H_ #define RAPIDJSON_POINTER_H_ #include "document.h" #include "internal/itoa.h" #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(switch-enum) #endif #ifdef _MSC_VER RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif RAPIDJSON_NAMESPACE_BEGIN static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token //! Error code of parsing. /*! \ingroup RAPIDJSON_ERRORS \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode */ enum PointerParseErrorCode { kPointerParseErrorNone = 0, //!< The parse is successful kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' kPointerParseErrorInvalidEscape, //!< Invalid escape kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment }; /////////////////////////////////////////////////////////////////////////////// // GenericPointer //! Represents a JSON Pointer. Use Pointer for UTF8 encoding and default allocator. /*! This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" (https://tools.ietf.org/html/rfc6901). A JSON pointer is for identifying a specific value in a JSON document (GenericDocument). It can simplify coding of DOM tree manipulation, because it can access multiple-level depth of DOM tree with single API call. After it parses a string representation (e.g. "/foo/0" or URI fragment representation (e.g. "#/foo/0") into its internal representation (tokens), it can be used to resolve a specific value in multiple documents, or sub-tree of documents. Contrary to GenericValue, Pointer can be copy constructed and copy assigned. Apart from assignment, a Pointer cannot be modified after construction. Although Pointer is very convenient, please aware that constructing Pointer involves parsing and dynamic memory allocation. A special constructor with user- supplied tokens eliminates these. GenericPointer depends on GenericDocument and GenericValue. \tparam ValueType The value type of the DOM tree. E.g. GenericValue > \tparam Allocator The allocator type for allocating memory for internal representation. \note GenericPointer uses same encoding of ValueType. However, Allocator of GenericPointer is independent of Allocator of Value. */ template class GenericPointer { public: typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value typedef typename ValueType::Ch Ch; //!< Character type from Value //! A token is the basic units of internal representation. /*! A JSON pointer string representation "/foo/123" is parsed to two tokens: "foo" and 123. 123 will be represented in both numeric form and string form. They are resolved according to the actual value type (object or array). For token that are not numbers, or the numeric value is out of bound (greater than limits of SizeType), they are only treated as string form (i.e. the token's index will be equal to kPointerInvalidIndex). This struct is public so that user can create a Pointer without parsing and allocation, using a special constructor. */ struct Token { const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. SizeType length; //!< Length of the name. SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. }; //!@name Constructors and destructor. //@{ //! Default constructor. GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Constructor that parses a string or URI fragment representation. /*! \param source A null-terminated, string or URI fragment representation of JSON pointer. \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. */ explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { Parse(source, internal::StrLen(source)); } #if RAPIDJSON_HAS_STDSTRING //! Constructor that parses a string or URI fragment representation. /*! \param source A string or URI fragment representation of JSON pointer. \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ explicit GenericPointer(const std::basic_string& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { Parse(source.c_str(), source.size()); } #endif //! Constructor that parses a string or URI fragment representation, with length of the source string. /*! \param source A string or URI fragment representation of JSON pointer. \param length Length of source. \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. \note Slightly faster than the overload without length. */ GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { Parse(source, length); } //! Constructor with user-supplied tokens. /*! This constructor let user supplies const array of tokens. This prevents the parsing process and eliminates allocation. This is preferred for memory constrained environments. \param tokens An constant array of tokens representing the JSON pointer. \param tokenCount Number of tokens. \b Example \code #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } #define INDEX(i) { #i, sizeof(#i) - 1, i } static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); // Equivalent to static const Pointer p("/foo/123"); #undef NAME #undef INDEX \endcode */ GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Copy constructor. GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { *this = rhs; } //! Destructor. ~GenericPointer() { if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. Allocator::Free(tokens_); RAPIDJSON_DELETE(ownAllocator_); } //! Assignment operator. GenericPointer& operator=(const GenericPointer& rhs) { if (this != &rhs) { // Do not delete ownAllcator if (nameBuffer_) Allocator::Free(tokens_); tokenCount_ = rhs.tokenCount_; parseErrorOffset_ = rhs.parseErrorOffset_; parseErrorCode_ = rhs.parseErrorCode_; if (rhs.nameBuffer_) CopyFromRaw(rhs); // Normally parsed tokens. else { tokens_ = rhs.tokens_; // User supplied const tokens. nameBuffer_ = 0; } } return *this; } //@} //!@name Append token //@{ //! Append a token and return a new Pointer /*! \param token Token to be appended. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ GenericPointer Append(const Token& token, Allocator* allocator = 0) const { GenericPointer r; r.allocator_ = allocator; Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); r.tokens_[tokenCount_].name = p; r.tokens_[tokenCount_].length = token.length; r.tokens_[tokenCount_].index = token.index; return r; } //! Append a name token with length, and return a new Pointer /*! \param name Name to be appended. \param length Length of name. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const { Token token = { name, length, kPointerInvalidIndex }; return Append(token, allocator); } //! Append a name token without length, and return a new Pointer /*! \param name Name (const Ch*) to be appended. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ template RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) Append(T* name, Allocator* allocator = 0) const { return Append(name, StrLen(name), allocator); } #if RAPIDJSON_HAS_STDSTRING //! Append a name token, and return a new Pointer /*! \param name Name to be appended. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ GenericPointer Append(const std::basic_string& name, Allocator* allocator = 0) const { return Append(name.c_str(), static_cast(name.size()), allocator); } #endif //! Append a index token, and return a new Pointer /*! \param index Index to be appended. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ GenericPointer Append(SizeType index, Allocator* allocator = 0) const { char buffer[21]; char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); SizeType length = static_cast(end - buffer); buffer[length] = '\0'; if (sizeof(Ch) == 1) { Token token = { reinterpret_cast(buffer), length, index }; return Append(token, allocator); } else { Ch name[21]; for (size_t i = 0; i <= length; i++) name[i] = buffer[i]; Token token = { name, length, index }; return Append(token, allocator); } } //! Append a token by value, and return a new Pointer /*! \param token token to be appended. \param allocator Allocator for the newly return Pointer. \return A new Pointer with appended token. */ GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const { if (token.IsString()) return Append(token.GetString(), token.GetStringLength(), allocator); else { RAPIDJSON_ASSERT(token.IsUint64()); RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); return Append(static_cast(token.GetUint64()), allocator); } } //!@name Handling Parse Error //@{ //! Check whether this is a valid pointer. bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } //! Get the parsing error offset in code unit. size_t GetParseErrorOffset() const { return parseErrorOffset_; } //! Get the parsing error code. PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } //@} //! Get the allocator of this pointer. Allocator& GetAllocator() { return *allocator_; } //!@name Tokens //@{ //! Get the token array (const version only). const Token* GetTokens() const { return tokens_; } //! Get the number of tokens. size_t GetTokenCount() const { return tokenCount_; } //@} //!@name Equality/inequality operators //@{ //! Equality operator. /*! \note When any pointers are invalid, always returns false. */ bool operator==(const GenericPointer& rhs) const { if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) return false; for (size_t i = 0; i < tokenCount_; i++) { if (tokens_[i].index != rhs.tokens_[i].index || tokens_[i].length != rhs.tokens_[i].length || (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) { return false; } } return true; } //! Inequality operator. /*! \note When any pointers are invalid, always returns true. */ bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } //@} //!@name Stringify //@{ //! Stringify the pointer into string representation. /*! \tparam OutputStream Type of output stream. \param os The output stream. */ template bool Stringify(OutputStream& os) const { return Stringify(os); } //! Stringify the pointer into URI fragment representation. /*! \tparam OutputStream Type of output stream. \param os The output stream. */ template bool StringifyUriFragment(OutputStream& os) const { return Stringify(os); } //@} //!@name Create value //@{ //! Create a value in a subtree. /*! If the value is not exist, it creates all parent values and a JSON Null value. So it always succeed and return the newly created or existing value. Remind that it may change types of parents according to tokens, so it potentially removes previously stored values. For example, if a document was an array, and "/foo" is used to create a value, then the document will be changed to an object, and all existing array elements are lost. \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. \param allocator Allocator for creating the values if the specified value or its parents are not exist. \param alreadyExist If non-null, it stores whether the resolved value is already exist. \return The resolved newly created (a JSON Null value), or already exists value. */ ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const { RAPIDJSON_ASSERT(IsValid()); ValueType* v = &root; bool exist = true; for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { if (v->IsArray() && t->name[0] == '-' && t->length == 1) { v->PushBack(ValueType().Move(), allocator); v = &((*v)[v->Size() - 1]); exist = false; } else { if (t->index == kPointerInvalidIndex) { // must be object name if (!v->IsObject()) v->SetObject(); // Change to Object } else { // object name or array index if (!v->IsArray() && !v->IsObject()) v->SetArray(); // Change to Array } if (v->IsArray()) { if (t->index >= v->Size()) { v->Reserve(t->index + 1, allocator); while (t->index >= v->Size()) v->PushBack(ValueType().Move(), allocator); exist = false; } v = &((*v)[t->index]); } else { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) { v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end exist = false; } else v = &m->value; } } } if (alreadyExist) *alreadyExist = exist; return *v; } //! Creates a value in a document. /*! \param document A document to be resolved. \param alreadyExist If non-null, it stores whether the resolved value is already exist. \return The resolved newly created, or already exists value. */ template ValueType& Create(GenericDocument& document, bool* alreadyExist = 0) const { return Create(document, document.GetAllocator(), alreadyExist); } //@} //!@name Query value //@{ //! Query a value in a subtree. /*! \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. \return Pointer to the value if it can be resolved. Otherwise null. \note There are only 3 situations when a value cannot be resolved: 1. A value in the path is not an array nor object. 2. An object value does not contain the token. 3. A token is out of range of an array value. Use unresolvedTokenIndex to retrieve the token index. */ ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { RAPIDJSON_ASSERT(IsValid()); ValueType* v = &root; for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { switch (v->GetType()) { case kObjectType: { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) break; v = &m->value; } continue; case kArrayType: if (t->index == kPointerInvalidIndex || t->index >= v->Size()) break; v = &((*v)[t->index]); continue; default: break; } // Error: unresolved token if (unresolvedTokenIndex) *unresolvedTokenIndex = static_cast(t - tokens_); return 0; } return v; } //! Query a const value in a const subtree. /*! \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \return Pointer to the value if it can be resolved. Otherwise null. */ const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { return Get(const_cast(root), unresolvedTokenIndex); } //@} //!@name Query a value with default //@{ //! Query a value in a subtree with default value. /*! Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. So that this function always succeed. \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \param defaultValue Default value to be cloned if the value was not exists. \param allocator Allocator for creating the values if the specified value or its parents are not exist. \see Create() */ ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { bool alreadyExist; Value& v = Create(root, allocator, &alreadyExist); return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); } //! Query a value in a subtree with default null-terminated string. ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { bool alreadyExist; Value& v = Create(root, allocator, &alreadyExist); return alreadyExist ? v : v.SetString(defaultValue, allocator); } #if RAPIDJSON_HAS_STDSTRING //! Query a value in a subtree with default std::basic_string. ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const { bool alreadyExist; Value& v = Create(root, allocator, &alreadyExist); return alreadyExist ? v : v.SetString(defaultValue, allocator); } #endif //! Query a value in a subtree with default primitive value. /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const { return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); } //! Query a value in a document with default value. template ValueType& GetWithDefault(GenericDocument& document, const ValueType& defaultValue) const { return GetWithDefault(document, defaultValue, document.GetAllocator()); } //! Query a value in a document with default null-terminated string. template ValueType& GetWithDefault(GenericDocument& document, const Ch* defaultValue) const { return GetWithDefault(document, defaultValue, document.GetAllocator()); } #if RAPIDJSON_HAS_STDSTRING //! Query a value in a document with default std::basic_string. template ValueType& GetWithDefault(GenericDocument& document, const std::basic_string& defaultValue) const { return GetWithDefault(document, defaultValue, document.GetAllocator()); } #endif //! Query a value in a document with default primitive value. /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) GetWithDefault(GenericDocument& document, T defaultValue) const { return GetWithDefault(document, defaultValue, document.GetAllocator()); } //@} //!@name Set a value //@{ //! Set a value in a subtree, with move semantics. /*! It creates all parents if they are not exist or types are different to the tokens. So this function always succeeds but potentially remove existing values. \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \param value Value to be set. \param allocator Allocator for creating the values if the specified value or its parents are not exist. \see Create() */ ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { return Create(root, allocator) = value; } //! Set a value in a subtree, with copy semantics. ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const { return Create(root, allocator).CopyFrom(value, allocator); } //! Set a null-terminated string in a subtree. ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const { return Create(root, allocator) = ValueType(value, allocator).Move(); } #if RAPIDJSON_HAS_STDSTRING //! Set a std::basic_string in a subtree. ValueType& Set(ValueType& root, const std::basic_string& value, typename ValueType::AllocatorType& allocator) const { return Create(root, allocator) = ValueType(value, allocator).Move(); } #endif //! Set a primitive value in a subtree. /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const { return Create(root, allocator) = ValueType(value).Move(); } //! Set a value in a document, with move semantics. template ValueType& Set(GenericDocument& document, ValueType& value) const { return Create(document) = value; } //! Set a value in a document, with copy semantics. template ValueType& Set(GenericDocument& document, const ValueType& value) const { return Create(document).CopyFrom(value, document.GetAllocator()); } //! Set a null-terminated string in a document. template ValueType& Set(GenericDocument& document, const Ch* value) const { return Create(document) = ValueType(value, document.GetAllocator()).Move(); } #if RAPIDJSON_HAS_STDSTRING //! Sets a std::basic_string in a document. template ValueType& Set(GenericDocument& document, const std::basic_string& value) const { return Create(document) = ValueType(value, document.GetAllocator()).Move(); } #endif //! Set a primitive value in a document. /*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) Set(GenericDocument& document, T value) const { return Create(document) = value; } //@} //!@name Swap a value //@{ //! Swap a value with a value in a subtree. /*! It creates all parents if they are not exist or types are different to the tokens. So this function always succeeds but potentially remove existing values. \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \param value Value to be swapped. \param allocator Allocator for creating the values if the specified value or its parents are not exist. \see Create() */ ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { return Create(root, allocator).Swap(value); } //! Swap a value with a value in a document. template ValueType& Swap(GenericDocument& document, ValueType& value) const { return Create(document).Swap(value); } //@} //! Erase a value in a subtree. /*! \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. \return Whether the resolved value is found and erased. \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. */ bool Erase(ValueType& root) const { RAPIDJSON_ASSERT(IsValid()); if (tokenCount_ == 0) // Cannot erase the root return false; ValueType* v = &root; const Token* last = tokens_ + (tokenCount_ - 1); for (const Token *t = tokens_; t != last; ++t) { switch (v->GetType()) { case kObjectType: { typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); if (m == v->MemberEnd()) return false; v = &m->value; } break; case kArrayType: if (t->index == kPointerInvalidIndex || t->index >= v->Size()) return false; v = &((*v)[t->index]); break; default: return false; } } switch (v->GetType()) { case kObjectType: return v->EraseMember(GenericStringRef(last->name, last->length)); case kArrayType: if (last->index == kPointerInvalidIndex || last->index >= v->Size()) return false; v->Erase(v->Begin() + last->index); return true; default: return false; } } private: //! Clone the content from rhs to this. /*! \param rhs Source pointer. \param extraToken Extra tokens to be allocated. \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. \return Start of non-occupied name buffer, for storing extra names. */ Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { if (!allocator_) // allocator is independently owned. ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) nameBufferSize += t->length; tokenCount_ = rhs.tokenCount_ + extraToken; tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); if (rhs.tokenCount_ > 0) { std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); } if (nameBufferSize > 0) { std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); } // Adjust pointers to name buffer std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) t->name += diff; return nameBuffer_ + nameBufferSize; } //! Check whether a character should be percent-encoded. /*! According to RFC 3986 2.3 Unreserved Characters. \param c The character (code unit) to be tested. */ bool NeedPercentEncode(Ch c) const { return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); } //! Parse a JSON String or its URI fragment representation into tokens. #ifndef __clang__ // -Wdocumentation /*! \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. \param length Length of the source string. \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. */ #endif void Parse(const Ch* source, size_t length) { RAPIDJSON_ASSERT(source != NULL); RAPIDJSON_ASSERT(nameBuffer_ == 0); RAPIDJSON_ASSERT(tokens_ == 0); // Create own allocator if user did not supply. if (!allocator_) ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); // Count number of '/' as tokenCount tokenCount_ = 0; for (const Ch* s = source; s != source + length; s++) if (*s == '/') tokenCount_++; Token* token = tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); Ch* name = nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); size_t i = 0; // Detect if it is a URI fragment bool uriFragment = false; if (source[i] == '#') { uriFragment = true; i++; } if (i != length && source[i] != '/') { parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; goto error; } while (i < length) { RAPIDJSON_ASSERT(source[i] == '/'); i++; // consumes '/' token->name = name; bool isNumber = true; while (i < length && source[i] != '/') { Ch c = source[i]; if (uriFragment) { // Decoding percent-encoding for URI fragment if (c == '%') { PercentDecodeStream is(&source[i], source + length); GenericInsituStringStream os(name); Ch* begin = os.PutBegin(); if (!Transcoder, EncodingType>().Validate(is, os) || !is.IsValid()) { parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; goto error; } size_t len = os.PutEnd(begin); i += is.Tell() - 1; if (len == 1) c = *name; else { name += len; isNumber = false; i++; continue; } } else if (NeedPercentEncode(c)) { parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; goto error; } } i++; // Escaping "~0" -> '~', "~1" -> '/' if (c == '~') { if (i < length) { c = source[i]; if (c == '0') c = '~'; else if (c == '1') c = '/'; else { parseErrorCode_ = kPointerParseErrorInvalidEscape; goto error; } i++; } else { parseErrorCode_ = kPointerParseErrorInvalidEscape; goto error; } } // First check for index: all of characters are digit if (c < '0' || c > '9') isNumber = false; *name++ = c; } token->length = static_cast(name - token->name); if (token->length == 0) isNumber = false; *name++ = '\0'; // Null terminator // Second check for index: more than one digit cannot have leading zero if (isNumber && token->length > 1 && token->name[0] == '0') isNumber = false; // String to SizeType conversion SizeType n = 0; if (isNumber) { for (size_t j = 0; j < token->length; j++) { SizeType m = n * 10 + static_cast(token->name[j] - '0'); if (m < n) { // overflow detection isNumber = false; break; } n = m; } } token->index = isNumber ? n : kPointerInvalidIndex; token++; } RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer parseErrorCode_ = kPointerParseErrorNone; return; error: Allocator::Free(tokens_); nameBuffer_ = 0; tokens_ = 0; tokenCount_ = 0; parseErrorOffset_ = i; return; } //! Stringify to string or URI fragment representation. /*! \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. \tparam OutputStream type of output stream. \param os The output stream. */ template bool Stringify(OutputStream& os) const { RAPIDJSON_ASSERT(IsValid()); if (uriFragment) os.Put('#'); for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { os.Put('/'); for (size_t j = 0; j < t->length; j++) { Ch c = t->name[j]; if (c == '~') { os.Put('~'); os.Put('0'); } else if (c == '/') { os.Put('~'); os.Put('1'); } else if (uriFragment && NeedPercentEncode(c)) { // Transcode to UTF8 sequence GenericStringStream source(&t->name[j]); PercentEncodeStream target(os); if (!Transcoder >().Validate(source, target)) return false; j += source.Tell() - 1; } else os.Put(c); } } return true; } //! A helper stream for decoding a percent-encoded sequence into code unit. /*! This stream decodes %XY triplet into code unit (0-255). If it encounters invalid characters, it sets output code unit as 0 and mark invalid, and to be checked by IsValid(). */ class PercentDecodeStream { public: typedef typename ValueType::Ch Ch; //! Constructor /*! \param source Start of the stream \param end Past-the-end of the stream. */ PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} Ch Take() { if (*src_ != '%' || src_ + 3 > end_) { // %XY triplet valid_ = false; return 0; } src_++; Ch c = 0; for (int j = 0; j < 2; j++) { c = static_cast(c << 4); Ch h = *src_; if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); else { valid_ = false; return 0; } src_++; } return c; } size_t Tell() const { return static_cast(src_ - head_); } bool IsValid() const { return valid_; } private: const Ch* src_; //!< Current read position. const Ch* head_; //!< Original head of the string. const Ch* end_; //!< Past-the-end position. bool valid_; //!< Whether the parsing is valid. }; //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. template class PercentEncodeStream { public: PercentEncodeStream(OutputStream& os) : os_(os) {} void Put(char c) { // UTF-8 must be byte unsigned char u = static_cast(c); static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; os_.Put('%'); os_.Put(hexDigits[u >> 4]); os_.Put(hexDigits[u & 15]); } private: OutputStream& os_; }; Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. Allocator* ownAllocator_; //!< Allocator owned by this Pointer. Ch* nameBuffer_; //!< A buffer containing all names in tokens. Token* tokens_; //!< A list of tokens. size_t tokenCount_; //!< Number of tokens in tokens_. size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. PointerParseErrorCode parseErrorCode_; //!< Parsing error code. }; //! GenericPointer for Value (UTF-8, default allocator). typedef GenericPointer Pointer; //!@name Helper functions for GenericPointer //@{ ////////////////////////////////////////////////////////////////////////////// template typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer& pointer, typename T::AllocatorType& a) { return pointer.Create(root, a); } template typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) { return GenericPointer(source, N - 1).Create(root, a); } // No allocator parameter template typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer& pointer) { return pointer.Create(document); } template typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) { return GenericPointer(source, N - 1).Create(document); } ////////////////////////////////////////////////////////////////////////////// template typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { return pointer.Get(root, unresolvedTokenIndex); } template const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { return pointer.Get(root, unresolvedTokenIndex); } template typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); } template const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); } ////////////////////////////////////////////////////////////////////////////// template typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { return pointer.GetWithDefault(root, defaultValue, a); } template typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) { return pointer.GetWithDefault(root, defaultValue, a); } #if RAPIDJSON_HAS_STDSTRING template typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const std::basic_string& defaultValue, typename T::AllocatorType& a) { return pointer.GetWithDefault(root, defaultValue, a); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, T2 defaultValue, typename T::AllocatorType& a) { return pointer.GetWithDefault(root, defaultValue, a); } template typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); } template typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); } #if RAPIDJSON_HAS_STDSTRING template typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string& defaultValue, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); } // No allocator parameter template typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& defaultValue) { return pointer.GetWithDefault(document, defaultValue); } template typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* defaultValue) { return pointer.GetWithDefault(document, defaultValue); } #if RAPIDJSON_HAS_STDSTRING template typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const std::basic_string& defaultValue) { return pointer.GetWithDefault(document, defaultValue); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, T2 defaultValue) { return pointer.GetWithDefault(document, defaultValue); } template typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) { return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); } template typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) { return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); } #if RAPIDJSON_HAS_STDSTRING template typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string& defaultValue) { return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) { return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); } ////////////////////////////////////////////////////////////////////////////// template typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { return pointer.Set(root, value, a); } template typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) { return pointer.Set(root, value, a); } template typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::Ch* value, typename T::AllocatorType& a) { return pointer.Set(root, value, a); } #if RAPIDJSON_HAS_STDSTRING template typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const std::basic_string& value, typename T::AllocatorType& a) { return pointer.Set(root, value, a); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) SetValueByPointer(T& root, const GenericPointer& pointer, T2 value, typename T::AllocatorType& a) { return pointer.Set(root, value, a); } template typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).Set(root, value, a); } template typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).Set(root, value, a); } template typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).Set(root, value, a); } #if RAPIDJSON_HAS_STDSTRING template typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string& value, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).Set(root, value, a); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).Set(root, value, a); } // No allocator parameter template typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { return pointer.Set(document, value); } template typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& value) { return pointer.Set(document, value); } template typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* value) { return pointer.Set(document, value); } #if RAPIDJSON_HAS_STDSTRING template typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const std::basic_string& value) { return pointer.Set(document, value); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) SetValueByPointer(DocumentType& document, const GenericPointer& pointer, T2 value) { return pointer.Set(document, value); } template typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { return GenericPointer(source, N - 1).Set(document, value); } template typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) { return GenericPointer(source, N - 1).Set(document, value); } template typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) { return GenericPointer(source, N - 1).Set(document, value); } #if RAPIDJSON_HAS_STDSTRING template typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string& value) { return GenericPointer(source, N - 1).Set(document, value); } #endif template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) { return GenericPointer(source, N - 1).Set(document, value); } ////////////////////////////////////////////////////////////////////////////// template typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { return pointer.Swap(root, value, a); } template typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { return GenericPointer(source, N - 1).Swap(root, value, a); } template typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { return pointer.Swap(document, value); } template typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { return GenericPointer(source, N - 1).Swap(document, value); } ////////////////////////////////////////////////////////////////////////////// template bool EraseValueByPointer(T& root, const GenericPointer& pointer) { return pointer.Erase(root); } template bool EraseValueByPointer(T& root, const CharType(&source)[N]) { return GenericPointer(source, N - 1).Erase(root); } //@} RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #ifdef _MSC_VER RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_POINTER_H_ ================================================ FILE: src/thirdparty/rapidjson/prettywriter.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_PRETTYWRITER_H_ #define RAPIDJSON_PRETTYWRITER_H_ #include "writer.h" #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif RAPIDJSON_NAMESPACE_BEGIN //! Combination of PrettyWriter format flags. /*! \see PrettyWriter::SetFormatOptions */ enum PrettyFormatOptions { kFormatDefault = 0, //!< Default pretty formatting. kFormatSingleLineArray = 1 //!< Format arrays on a single line. }; //! Writer with indentation and spacing. /*! \tparam OutputStream Type of ouptut os. \tparam SourceEncoding Encoding of source string. \tparam TargetEncoding Encoding of output stream. \tparam StackAllocator Type of allocator for allocating memory of stack. */ template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> class PrettyWriter : public Writer { public: typedef Writer Base; typedef typename Base::Ch Ch; //! Constructor /*! \param os Output stream. \param allocator User supplied allocator. If it is null, it will create a private one. \param levelDepth Initial capacity of stack. */ explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} //! Set custom indentation. /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). \param indentCharCount Number of indent characters for each indentation level. \note The default indentation is 4 spaces. */ PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); indentChar_ = indentChar; indentCharCount_ = indentCharCount; return *this; } //! Set pretty writer formatting options. /*! \param options Formatting options. */ PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { formatOptions_ = options; return *this; } /*! @name Implementation of Handler \see Handler */ //@{ bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } bool RawNumber(const Ch* str, SizeType length, bool copy = false) { (void)copy; PrettyPrefix(kNumberType); return Base::WriteString(str, length); } bool String(const Ch* str, SizeType length, bool copy = false) { (void)copy; PrettyPrefix(kStringType); return Base::WriteString(str, length); } #if RAPIDJSON_HAS_STDSTRING bool String(const std::basic_string& str) { return String(str.data(), SizeType(str.size())); } #endif bool StartObject() { PrettyPrefix(kObjectType); new (Base::level_stack_.template Push()) typename Base::Level(false); return Base::WriteStartObject(); } bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } #if RAPIDJSON_HAS_STDSTRING bool Key(const std::basic_string& str) { return Key(str.data(), SizeType(str.size())); } #endif bool EndObject(SizeType memberCount = 0) { (void)memberCount; RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; if (!empty) { Base::os_->Put('\n'); WriteIndent(); } bool ret = Base::WriteEndObject(); (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text Base::os_->Flush(); return true; } bool StartArray() { PrettyPrefix(kArrayType); new (Base::level_stack_.template Push()) typename Base::Level(true); return Base::WriteStartArray(); } bool EndArray(SizeType memberCount = 0) { (void)memberCount; RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { Base::os_->Put('\n'); WriteIndent(); } bool ret = Base::WriteEndArray(); (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text Base::os_->Flush(); return true; } //@} /*! @name Convenience extensions */ //@{ //! Simpler but slower overload. bool String(const Ch* str) { return String(str, internal::StrLen(str)); } bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } //@} //! Write a raw JSON value. /*! For user to write a stringified JSON as a value. \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. \param length Length of the json. \param type Type of the root of json. \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. */ bool RawValue(const Ch* json, size_t length, Type type) { PrettyPrefix(type); return Base::WriteRawValue(json, length); } protected: void PrettyPrefix(Type type) { (void)type; if (Base::level_stack_.GetSize() != 0) { // this value is not at root typename Base::Level* level = Base::level_stack_.template Top(); if (level->inArray) { if (level->valueCount > 0) { Base::os_->Put(','); // add comma if it is not the first element in array if (formatOptions_ & kFormatSingleLineArray) Base::os_->Put(' '); } if (!(formatOptions_ & kFormatSingleLineArray)) { Base::os_->Put('\n'); WriteIndent(); } } else { // in object if (level->valueCount > 0) { if (level->valueCount % 2 == 0) { Base::os_->Put(','); Base::os_->Put('\n'); } else { Base::os_->Put(':'); Base::os_->Put(' '); } } else Base::os_->Put('\n'); if (level->valueCount % 2 == 0) WriteIndent(); } if (!level->inArray && level->valueCount % 2 == 0) RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name level->valueCount++; } else { RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. Base::hasRoot_ = true; } } void WriteIndent() { size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; PutN(*Base::os_, static_cast(indentChar_), count); } Ch indentChar_; unsigned indentCharCount_; PrettyFormatOptions formatOptions_; private: // Prohibit copy constructor & assignment operator. PrettyWriter(const PrettyWriter&); PrettyWriter& operator=(const PrettyWriter&); }; RAPIDJSON_NAMESPACE_END #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_RAPIDJSON_H_ ================================================ FILE: src/thirdparty/rapidjson/rapidjson.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_RAPIDJSON_H_ #define RAPIDJSON_RAPIDJSON_H_ /*!\file rapidjson.h \brief common definitions and configuration \see RAPIDJSON_CONFIG */ /*! \defgroup RAPIDJSON_CONFIG RapidJSON configuration \brief Configuration macros for library features Some RapidJSON features are configurable to adapt the library to a wide variety of platforms, environments and usage scenarios. Most of the features can be configured in terms of overriden or predefined preprocessor macros at compile-time. Some additional customization is available in the \ref RAPIDJSON_ERRORS APIs. \note These macros should be given on the compiler command-line (where applicable) to avoid inconsistent values when compiling different translation units of a single application. */ #include // malloc(), realloc(), free(), size_t #include // memset(), memcpy(), memmove(), memcmp() /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_VERSION_STRING // // ALWAYS synchronize the following 3 macros with corresponding variables in /CMakeLists.txt. // //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN // token stringification #define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) #define RAPIDJSON_DO_STRINGIFY(x) #x //!@endcond /*! \def RAPIDJSON_MAJOR_VERSION \ingroup RAPIDJSON_CONFIG \brief Major version of RapidJSON in integer. */ /*! \def RAPIDJSON_MINOR_VERSION \ingroup RAPIDJSON_CONFIG \brief Minor version of RapidJSON in integer. */ /*! \def RAPIDJSON_PATCH_VERSION \ingroup RAPIDJSON_CONFIG \brief Patch version of RapidJSON in integer. */ /*! \def RAPIDJSON_VERSION_STRING \ingroup RAPIDJSON_CONFIG \brief Version of RapidJSON in ".." string format. */ #define RAPIDJSON_MAJOR_VERSION 1 #define RAPIDJSON_MINOR_VERSION 1 #define RAPIDJSON_PATCH_VERSION 0 #define RAPIDJSON_VERSION_STRING \ RAPIDJSON_STRINGIFY(RAPIDJSON_MAJOR_VERSION.RAPIDJSON_MINOR_VERSION.RAPIDJSON_PATCH_VERSION) /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NAMESPACE_(BEGIN|END) /*! \def RAPIDJSON_NAMESPACE \ingroup RAPIDJSON_CONFIG \brief provide custom rapidjson namespace In order to avoid symbol clashes and/or "One Definition Rule" errors between multiple inclusions of (different versions of) RapidJSON in a single binary, users can customize the name of the main RapidJSON namespace. In case of a single nesting level, defining \c RAPIDJSON_NAMESPACE to a custom name (e.g. \c MyRapidJSON) is sufficient. If multiple levels are needed, both \ref RAPIDJSON_NAMESPACE_BEGIN and \ref RAPIDJSON_NAMESPACE_END need to be defined as well: \code // in some .cpp file #define RAPIDJSON_NAMESPACE my::rapidjson #define RAPIDJSON_NAMESPACE_BEGIN namespace my { namespace rapidjson { #define RAPIDJSON_NAMESPACE_END } } #include "rapidjson/..." \endcode \see rapidjson */ /*! \def RAPIDJSON_NAMESPACE_BEGIN \ingroup RAPIDJSON_CONFIG \brief provide custom rapidjson namespace (opening expression) \see RAPIDJSON_NAMESPACE */ /*! \def RAPIDJSON_NAMESPACE_END \ingroup RAPIDJSON_CONFIG \brief provide custom rapidjson namespace (closing expression) \see RAPIDJSON_NAMESPACE */ #ifndef RAPIDJSON_NAMESPACE #define RAPIDJSON_NAMESPACE rapidjson #endif #ifndef RAPIDJSON_NAMESPACE_BEGIN #define RAPIDJSON_NAMESPACE_BEGIN namespace RAPIDJSON_NAMESPACE { #endif #ifndef RAPIDJSON_NAMESPACE_END #define RAPIDJSON_NAMESPACE_END } #endif /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_HAS_STDSTRING #ifndef RAPIDJSON_HAS_STDSTRING #ifdef RAPIDJSON_DOXYGEN_RUNNING #define RAPIDJSON_HAS_STDSTRING 1 // force generation of documentation #else #define RAPIDJSON_HAS_STDSTRING 0 // no std::string support by default #endif /*! \def RAPIDJSON_HAS_STDSTRING \ingroup RAPIDJSON_CONFIG \brief Enable RapidJSON support for \c std::string By defining this preprocessor symbol to \c 1, several convenience functions for using \ref rapidjson::GenericValue with \c std::string are enabled, especially for construction and comparison. \hideinitializer */ #endif // !defined(RAPIDJSON_HAS_STDSTRING) #if RAPIDJSON_HAS_STDSTRING #include #endif // RAPIDJSON_HAS_STDSTRING /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NO_INT64DEFINE /*! \def RAPIDJSON_NO_INT64DEFINE \ingroup RAPIDJSON_CONFIG \brief Use external 64-bit integer types. RapidJSON requires the 64-bit integer types \c int64_t and \c uint64_t types to be available at global scope. If users have their own definition, define RAPIDJSON_NO_INT64DEFINE to prevent RapidJSON from defining its own types. */ #ifndef RAPIDJSON_NO_INT64DEFINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 #include "msinttypes/stdint.h" #include "msinttypes/inttypes.h" #else // Other compilers should have this. #include #include #endif //!@endcond #ifdef RAPIDJSON_DOXYGEN_RUNNING #define RAPIDJSON_NO_INT64DEFINE #endif #endif // RAPIDJSON_NO_INT64TYPEDEF /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_FORCEINLINE #ifndef RAPIDJSON_FORCEINLINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #if defined(_MSC_VER) && defined(NDEBUG) #define RAPIDJSON_FORCEINLINE __forceinline #elif defined(__GNUC__) && __GNUC__ >= 4 && defined(NDEBUG) #define RAPIDJSON_FORCEINLINE __attribute__((always_inline)) #else #define RAPIDJSON_FORCEINLINE #endif //!@endcond #endif // RAPIDJSON_FORCEINLINE /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_ENDIAN #define RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine #define RAPIDJSON_BIGENDIAN 1 //!< Big endian machine //! Endianness of the machine. /*! \def RAPIDJSON_ENDIAN \ingroup RAPIDJSON_CONFIG GCC 4.6 provided macro for detecting endianness of the target machine. But other compilers may not have this. User can define RAPIDJSON_ENDIAN to either \ref RAPIDJSON_LITTLEENDIAN or \ref RAPIDJSON_BIGENDIAN. Default detection implemented with reference to \li https://gcc.gnu.org/onlinedocs/gcc-4.6.0/cpp/Common-Predefined-Macros.html \li http://www.boost.org/doc/libs/1_42_0/boost/detail/endian.hpp */ #ifndef RAPIDJSON_ENDIAN // Detect with GCC 4.6's macro # ifdef __BYTE_ORDER__ # if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # else # error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. # endif // __BYTE_ORDER__ // Detect with GLIBC's endian.h # elif defined(__GLIBC__) # include # if (__BYTE_ORDER == __LITTLE_ENDIAN) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif (__BYTE_ORDER == __BIG_ENDIAN) # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # else # error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. # endif // __GLIBC__ // Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro # elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN // Detect with architecture macros # elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif defined(_MSC_VER) && defined(_M_ARM) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif defined(RAPIDJSON_DOXYGEN_RUNNING) # define RAPIDJSON_ENDIAN # else # error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. # endif #endif // RAPIDJSON_ENDIAN /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_64BIT //! Whether using 64-bit architecture #ifndef RAPIDJSON_64BIT #if defined(__LP64__) || (defined(__x86_64__) && defined(__ILP32__)) || defined(_WIN64) || defined(__EMSCRIPTEN__) #define RAPIDJSON_64BIT 1 #else #define RAPIDJSON_64BIT 0 #endif #endif // RAPIDJSON_64BIT /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_ALIGN //! Data alignment of the machine. /*! \ingroup RAPIDJSON_CONFIG \param x pointer to align Some machines require strict data alignment. Currently the default uses 4 bytes alignment on 32-bit platforms and 8 bytes alignment for 64-bit platforms. User can customize by defining the RAPIDJSON_ALIGN function macro. */ #ifndef RAPIDJSON_ALIGN #if RAPIDJSON_64BIT == 1 #define RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) #else #define RAPIDJSON_ALIGN(x) (((x) + 3u) & ~3u) #endif #endif /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_UINT64_C2 //! Construct a 64-bit literal by a pair of 32-bit integer. /*! 64-bit literal with or without ULL suffix is prone to compiler warnings. UINT64_C() is C macro which cause compilation problems. Use this macro to define 64-bit constants by a pair of 32-bit integer. */ #ifndef RAPIDJSON_UINT64_C2 #define RAPIDJSON_UINT64_C2(high32, low32) ((static_cast(high32) << 32) | static_cast(low32)) #endif /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_48BITPOINTER_OPTIMIZATION //! Use only lower 48-bit address for some pointers. /*! \ingroup RAPIDJSON_CONFIG This optimization uses the fact that current X86-64 architecture only implement lower 48-bit virtual address. The higher 16-bit can be used for storing other data. \c GenericValue uses this optimization to reduce its size form 24 bytes to 16 bytes in 64-bit architecture. */ #ifndef RAPIDJSON_48BITPOINTER_OPTIMIZATION #if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) #define RAPIDJSON_48BITPOINTER_OPTIMIZATION 1 #else #define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 #endif #endif // RAPIDJSON_48BITPOINTER_OPTIMIZATION #if RAPIDJSON_48BITPOINTER_OPTIMIZATION == 1 #if RAPIDJSON_64BIT != 1 #error RAPIDJSON_48BITPOINTER_OPTIMIZATION can only be set to 1 when RAPIDJSON_64BIT=1 #endif #define RAPIDJSON_SETPOINTER(type, p, x) (p = reinterpret_cast((reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0xFFFF0000, 0x00000000))) | reinterpret_cast(reinterpret_cast(x)))) #define RAPIDJSON_GETPOINTER(type, p) (reinterpret_cast(reinterpret_cast(p) & static_cast(RAPIDJSON_UINT64_C2(0x0000FFFF, 0xFFFFFFFF)))) #else #define RAPIDJSON_SETPOINTER(type, p, x) (p = (x)) #define RAPIDJSON_GETPOINTER(type, p) (p) #endif /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD /*! \def RAPIDJSON_SIMD \ingroup RAPIDJSON_CONFIG \brief Enable SSE2/SSE4.2 optimization. RapidJSON supports optimized implementations for some parsing operations based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible processors. To enable these optimizations, two different symbols can be defined; \code // Enable SSE2 optimization. #define RAPIDJSON_SSE2 // Enable SSE4.2 optimization. #define RAPIDJSON_SSE42 \endcode \c RAPIDJSON_SSE42 takes precedence, if both are defined. If any of these symbols is defined, RapidJSON defines the macro \c RAPIDJSON_SIMD to indicate the availability of the optimized code. */ #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ || defined(RAPIDJSON_DOXYGEN_RUNNING) #define RAPIDJSON_SIMD #endif /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NO_SIZETYPEDEFINE #ifndef RAPIDJSON_NO_SIZETYPEDEFINE /*! \def RAPIDJSON_NO_SIZETYPEDEFINE \ingroup RAPIDJSON_CONFIG \brief User-provided \c SizeType definition. In order to avoid using 32-bit size types for indexing strings and arrays, define this preprocessor symbol and provide the type rapidjson::SizeType before including RapidJSON: \code #define RAPIDJSON_NO_SIZETYPEDEFINE namespace rapidjson { typedef ::std::size_t SizeType; } #include "rapidjson/..." \endcode \see rapidjson::SizeType */ #ifdef RAPIDJSON_DOXYGEN_RUNNING #define RAPIDJSON_NO_SIZETYPEDEFINE #endif RAPIDJSON_NAMESPACE_BEGIN //! Size type (for string lengths, array sizes, etc.) /*! RapidJSON uses 32-bit array/string indices even on 64-bit platforms, instead of using \c size_t. Users may override the SizeType by defining \ref RAPIDJSON_NO_SIZETYPEDEFINE. */ typedef unsigned SizeType; RAPIDJSON_NAMESPACE_END #endif // always import std::size_t to rapidjson namespace RAPIDJSON_NAMESPACE_BEGIN using std::size_t; RAPIDJSON_NAMESPACE_END /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_ASSERT //! Assertion. /*! \ingroup RAPIDJSON_CONFIG By default, rapidjson uses C \c assert() for internal assertions. User can override it by defining RAPIDJSON_ASSERT(x) macro. \note Parsing errors are handled and can be customized by the \ref RAPIDJSON_ERRORS APIs. */ #ifndef RAPIDJSON_ASSERT #include #define RAPIDJSON_ASSERT(x) assert(x) #endif // RAPIDJSON_ASSERT /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_STATIC_ASSERT // Adopt from boost #ifndef RAPIDJSON_STATIC_ASSERT #ifndef __clang__ //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #endif RAPIDJSON_NAMESPACE_BEGIN template struct STATIC_ASSERTION_FAILURE; template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; template struct StaticAssertTest {}; RAPIDJSON_NAMESPACE_END #define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) #define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) #define RAPIDJSON_DO_JOIN2(X, Y) X##Y #if defined(__GNUC__) #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) #else #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE #endif #ifndef __clang__ //!@endcond #endif /*! \def RAPIDJSON_STATIC_ASSERT \brief (Internal) macro to check for conditions at compile-time \param x compile-time condition \hideinitializer */ #define RAPIDJSON_STATIC_ASSERT(x) \ typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest< \ sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE)> \ RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE #endif /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_LIKELY, RAPIDJSON_UNLIKELY //! Compiler branching hint for expression with high probability to be true. /*! \ingroup RAPIDJSON_CONFIG \param x Boolean expression likely to be true. */ #ifndef RAPIDJSON_LIKELY #if defined(__GNUC__) || defined(__clang__) #define RAPIDJSON_LIKELY(x) __builtin_expect(!!(x), 1) #else #define RAPIDJSON_LIKELY(x) (x) #endif #endif //! Compiler branching hint for expression with low probability to be true. /*! \ingroup RAPIDJSON_CONFIG \param x Boolean expression unlikely to be true. */ #ifndef RAPIDJSON_UNLIKELY #if defined(__GNUC__) || defined(__clang__) #define RAPIDJSON_UNLIKELY(x) __builtin_expect(!!(x), 0) #else #define RAPIDJSON_UNLIKELY(x) (x) #endif #endif /////////////////////////////////////////////////////////////////////////////// // Helpers //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define RAPIDJSON_MULTILINEMACRO_BEGIN do { #define RAPIDJSON_MULTILINEMACRO_END \ } while((void)0, 0) // adopted from Boost #define RAPIDJSON_VERSION_CODE(x,y,z) \ (((x)*100000) + ((y)*100) + (z)) /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_DIAG_PUSH/POP, RAPIDJSON_DIAG_OFF #if defined(__GNUC__) #define RAPIDJSON_GNUC \ RAPIDJSON_VERSION_CODE(__GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__) #endif #if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,2,0)) #define RAPIDJSON_PRAGMA(x) _Pragma(RAPIDJSON_STRINGIFY(x)) #define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(GCC diagnostic x) #define RAPIDJSON_DIAG_OFF(x) \ RAPIDJSON_DIAG_PRAGMA(ignored RAPIDJSON_STRINGIFY(RAPIDJSON_JOIN(-W,x))) // push/pop support in Clang and GCC>=4.6 #if defined(__clang__) || (defined(RAPIDJSON_GNUC) && RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) #define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) #define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) #else // GCC >= 4.2, < 4.6 #define RAPIDJSON_DIAG_PUSH /* ignored */ #define RAPIDJSON_DIAG_POP /* ignored */ #endif #elif defined(_MSC_VER) // pragma (MSVC specific) #define RAPIDJSON_PRAGMA(x) __pragma(x) #define RAPIDJSON_DIAG_PRAGMA(x) RAPIDJSON_PRAGMA(warning(x)) #define RAPIDJSON_DIAG_OFF(x) RAPIDJSON_DIAG_PRAGMA(disable: x) #define RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_PRAGMA(push) #define RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_PRAGMA(pop) #else #define RAPIDJSON_DIAG_OFF(x) /* ignored */ #define RAPIDJSON_DIAG_PUSH /* ignored */ #define RAPIDJSON_DIAG_POP /* ignored */ #endif // RAPIDJSON_DIAG_* /////////////////////////////////////////////////////////////////////////////// // C++11 features #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS #if defined(__clang__) #if __has_feature(cxx_rvalue_references) && \ (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 #else #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 #endif #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ (defined(_MSC_VER) && _MSC_VER >= 1600) #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 #else #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 #endif #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS #ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT #if defined(__clang__) #define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) // (defined(_MSC_VER) && _MSC_VER >= ????) // not yet supported #define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 #else #define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 #endif #endif #if RAPIDJSON_HAS_CXX11_NOEXCEPT #define RAPIDJSON_NOEXCEPT noexcept #else #define RAPIDJSON_NOEXCEPT /* noexcept */ #endif // RAPIDJSON_HAS_CXX11_NOEXCEPT // no automatic detection, yet #ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS #define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 #endif #ifndef RAPIDJSON_HAS_CXX11_RANGE_FOR #if defined(__clang__) #define RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ (defined(_MSC_VER) && _MSC_VER >= 1700) #define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 #else #define RAPIDJSON_HAS_CXX11_RANGE_FOR 0 #endif #endif // RAPIDJSON_HAS_CXX11_RANGE_FOR //!@endcond /////////////////////////////////////////////////////////////////////////////// // new/delete #ifndef RAPIDJSON_NEW ///! customization point for global \c new #define RAPIDJSON_NEW(x) new x #endif #ifndef RAPIDJSON_DELETE ///! customization point for global \c delete #define RAPIDJSON_DELETE(x) delete x #endif /////////////////////////////////////////////////////////////////////////////// // Type /*! \namespace rapidjson \brief main RapidJSON namespace \see RAPIDJSON_NAMESPACE */ RAPIDJSON_NAMESPACE_BEGIN //! Type of JSON value enum Type { kNullType = 0, //!< null kFalseType = 1, //!< false kTrueType = 2, //!< true kObjectType = 3, //!< object kArrayType = 4, //!< array kStringType = 5, //!< string kNumberType = 6 //!< number }; RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_RAPIDJSON_H_ ================================================ FILE: src/thirdparty/rapidjson/reader.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_READER_H_ #define RAPIDJSON_READER_H_ /*! \file reader.h */ #include "allocators.h" #include "stream.h" #include "encodedstream.h" #include "internal/meta.h" #include "internal/stack.h" #include "internal/strtod.h" #include #if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) #include #pragma intrinsic(_BitScanForward) #endif #ifdef RAPIDJSON_SSE42 #include #elif defined(RAPIDJSON_SSE2) #include #endif #ifdef _MSC_VER RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant RAPIDJSON_DIAG_OFF(4702) // unreachable code #endif #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(old-style-cast) RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(switch-enum) #endif #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define RAPIDJSON_NOTHING /* deliberately empty */ #ifndef RAPIDJSON_PARSE_ERROR_EARLY_RETURN #define RAPIDJSON_PARSE_ERROR_EARLY_RETURN(value) \ RAPIDJSON_MULTILINEMACRO_BEGIN \ if (RAPIDJSON_UNLIKELY(HasParseError())) { return value; } \ RAPIDJSON_MULTILINEMACRO_END #endif #define RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID \ RAPIDJSON_PARSE_ERROR_EARLY_RETURN(RAPIDJSON_NOTHING) //!@endcond /*! \def RAPIDJSON_PARSE_ERROR_NORETURN \ingroup RAPIDJSON_ERRORS \brief Macro to indicate a parse error. \param parseErrorCode \ref rapidjson::ParseErrorCode of the error \param offset position of the error in JSON input (\c size_t) This macros can be used as a customization point for the internal error handling mechanism of RapidJSON. A common usage model is to throw an exception instead of requiring the caller to explicitly check the \ref rapidjson::GenericReader::Parse's return value: \code #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode,offset) \ throw ParseException(parseErrorCode, #parseErrorCode, offset) #include // std::runtime_error #include "rapidjson/error/error.h" // rapidjson::ParseResult struct ParseException : std::runtime_error, rapidjson::ParseResult { ParseException(rapidjson::ParseErrorCode code, const char* msg, size_t offset) : std::runtime_error(msg), ParseResult(code, offset) {} }; #include "rapidjson/reader.h" \endcode \see RAPIDJSON_PARSE_ERROR, rapidjson::GenericReader::Parse */ #ifndef RAPIDJSON_PARSE_ERROR_NORETURN #define RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset) \ RAPIDJSON_MULTILINEMACRO_BEGIN \ RAPIDJSON_ASSERT(!HasParseError()); /* Error can only be assigned once */ \ SetParseError(parseErrorCode, offset); \ RAPIDJSON_MULTILINEMACRO_END #endif /*! \def RAPIDJSON_PARSE_ERROR \ingroup RAPIDJSON_ERRORS \brief (Internal) macro to indicate and handle a parse error. \param parseErrorCode \ref rapidjson::ParseErrorCode of the error \param offset position of the error in JSON input (\c size_t) Invokes RAPIDJSON_PARSE_ERROR_NORETURN and stops the parsing. \see RAPIDJSON_PARSE_ERROR_NORETURN \hideinitializer */ #ifndef RAPIDJSON_PARSE_ERROR #define RAPIDJSON_PARSE_ERROR(parseErrorCode, offset) \ RAPIDJSON_MULTILINEMACRO_BEGIN \ RAPIDJSON_PARSE_ERROR_NORETURN(parseErrorCode, offset); \ RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; \ RAPIDJSON_MULTILINEMACRO_END #endif #include "error/error.h" // ParseErrorCode, ParseResult RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // ParseFlag /*! \def RAPIDJSON_PARSE_DEFAULT_FLAGS \ingroup RAPIDJSON_CONFIG \brief User-defined kParseDefaultFlags definition. User can define this as any \c ParseFlag combinations. */ #ifndef RAPIDJSON_PARSE_DEFAULT_FLAGS #define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseNoFlags #endif //! Combination of parseFlags /*! \see Reader::Parse, Document::Parse, Document::ParseInsitu, Document::ParseStream */ enum ParseFlag { kParseNoFlags = 0, //!< No flags are set. kParseInsituFlag = 1, //!< In-situ(destructive) parsing. kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; /////////////////////////////////////////////////////////////////////////////// // Handler /*! \class rapidjson::Handler \brief Concept for receiving events from GenericReader upon parsing. The functions return true if no error occurs. If they return false, the event publisher should terminate the process. \code concept Handler { typename Ch; bool Null(); bool Bool(bool b); bool Int(int i); bool Uint(unsigned i); bool Int64(int64_t i); bool Uint64(uint64_t i); bool Double(double d); /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) bool RawNumber(const Ch* str, SizeType length, bool copy); bool String(const Ch* str, SizeType length, bool copy); bool StartObject(); bool Key(const Ch* str, SizeType length, bool copy); bool EndObject(SizeType memberCount); bool StartArray(); bool EndArray(SizeType elementCount); }; \endcode */ /////////////////////////////////////////////////////////////////////////////// // BaseReaderHandler //! Default implementation of Handler. /*! This can be used as base class of any reader handler. \note implements Handler concept */ template, typename Derived = void> struct BaseReaderHandler { typedef typename Encoding::Ch Ch; typedef typename internal::SelectIf, BaseReaderHandler, Derived>::Type Override; bool Default() { return true; } bool Null() { return static_cast(*this).Default(); } bool Bool(bool) { return static_cast(*this).Default(); } bool Int(int) { return static_cast(*this).Default(); } bool Uint(unsigned) { return static_cast(*this).Default(); } bool Int64(int64_t) { return static_cast(*this).Default(); } bool Uint64(uint64_t) { return static_cast(*this).Default(); } bool Double(double) { return static_cast(*this).Default(); } /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool StartObject() { return static_cast(*this).Default(); } bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } bool EndObject(SizeType) { return static_cast(*this).Default(); } bool StartArray() { return static_cast(*this).Default(); } bool EndArray(SizeType) { return static_cast(*this).Default(); } }; /////////////////////////////////////////////////////////////////////////////// // StreamLocalCopy namespace internal { template::copyOptimization> class StreamLocalCopy; //! Do copy optimization. template class StreamLocalCopy { public: StreamLocalCopy(Stream& original) : s(original), original_(original) {} ~StreamLocalCopy() { original_ = s; } Stream s; private: StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; Stream& original_; }; //! Keep reference. template class StreamLocalCopy { public: StreamLocalCopy(Stream& original) : s(original) {} Stream& s; private: StreamLocalCopy& operator=(const StreamLocalCopy&) /* = delete */; }; } // namespace internal /////////////////////////////////////////////////////////////////////////////// // SkipWhitespace //! Skip the JSON white spaces in a stream. /*! \param is A input stream for skipping white spaces. \note This function has SSE2/SSE4.2 specialization. */ template void SkipWhitespace(InputStream& is) { internal::StreamLocalCopy copy(is); InputStream& s(copy.s); typename InputStream::Ch c; while ((c = s.Peek()) == ' ' || c == '\n' || c == '\r' || c == '\t') s.Take(); } inline const char* SkipWhitespace(const char* p, const char* end) { while (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) ++p; return p; } #ifdef RAPIDJSON_SSE42 //! Skip whitespace with SSE 4.2 pcmpistrm instruction, testing 16 8-byte characters at once. inline const char *SkipWhitespace_SIMD(const char* p) { // Fast return for single non-whitespace if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; else return p; // 16-byte align to the next boundary const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; else return p; // The rest of string using SIMD static const char whitespace[16] = " \n\r\t"; const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); for (;; p += 16) { const __m128i s = _mm_load_si128(reinterpret_cast(p)); const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); if (r != 0) { // some of characters is non-whitespace #ifdef _MSC_VER // Find the index of first non-whitespace unsigned long offset; _BitScanForward(&offset, r); return p + offset; #else return p + __builtin_ffs(r) - 1; #endif } } } inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { // Fast return for single non-whitespace if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) ++p; else return p; // The middle of string using SIMD static const char whitespace[16] = " \n\r\t"; const __m128i w = _mm_loadu_si128(reinterpret_cast(&whitespace[0])); for (; p <= end - 16; p += 16) { const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); if (r != 0) { // some of characters is non-whitespace #ifdef _MSC_VER // Find the index of first non-whitespace unsigned long offset; _BitScanForward(&offset, r); return p + offset; #else return p + __builtin_ffs(r) - 1; #endif } } return SkipWhitespace(p, end); } #elif defined(RAPIDJSON_SSE2) //! Skip whitespace with SSE2 instructions, testing 16 8-byte characters at once. inline const char *SkipWhitespace_SIMD(const char* p) { // Fast return for single non-whitespace if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; else return p; // 16-byte align to the next boundary const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); while (p != nextAligned) if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') ++p; else return p; // The rest of string #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; #undef C16 const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); for (;; p += 16) { const __m128i s = _mm_load_si128(reinterpret_cast(p)); __m128i x = _mm_cmpeq_epi8(s, w0); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); unsigned short r = static_cast(~_mm_movemask_epi8(x)); if (r != 0) { // some of characters may be non-whitespace #ifdef _MSC_VER // Find the index of first non-whitespace unsigned long offset; _BitScanForward(&offset, r); return p + offset; #else return p + __builtin_ffs(r) - 1; #endif } } } inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { // Fast return for single non-whitespace if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) ++p; else return p; // The rest of string #define C16(c) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c } static const char whitespaces[4][16] = { C16(' '), C16('\n'), C16('\r'), C16('\t') }; #undef C16 const __m128i w0 = _mm_loadu_si128(reinterpret_cast(&whitespaces[0][0])); const __m128i w1 = _mm_loadu_si128(reinterpret_cast(&whitespaces[1][0])); const __m128i w2 = _mm_loadu_si128(reinterpret_cast(&whitespaces[2][0])); const __m128i w3 = _mm_loadu_si128(reinterpret_cast(&whitespaces[3][0])); for (; p <= end - 16; p += 16) { const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); __m128i x = _mm_cmpeq_epi8(s, w0); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w1)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w2)); x = _mm_or_si128(x, _mm_cmpeq_epi8(s, w3)); unsigned short r = static_cast(~_mm_movemask_epi8(x)); if (r != 0) { // some of characters may be non-whitespace #ifdef _MSC_VER // Find the index of first non-whitespace unsigned long offset; _BitScanForward(&offset, r); return p + offset; #else return p + __builtin_ffs(r) - 1; #endif } } return SkipWhitespace(p, end); } #endif // RAPIDJSON_SSE2 #ifdef RAPIDJSON_SIMD //! Template function specialization for InsituStringStream template<> inline void SkipWhitespace(InsituStringStream& is) { is.src_ = const_cast(SkipWhitespace_SIMD(is.src_)); } //! Template function specialization for StringStream template<> inline void SkipWhitespace(StringStream& is) { is.src_ = SkipWhitespace_SIMD(is.src_); } template<> inline void SkipWhitespace(EncodedInputStream, MemoryStream>& is) { is.is_.src_ = SkipWhitespace_SIMD(is.is_.src_, is.is_.end_); } #endif // RAPIDJSON_SIMD /////////////////////////////////////////////////////////////////////////////// // GenericReader //! SAX-style JSON parser. Use \ref Reader for UTF8 encoding and default allocator. /*! GenericReader parses JSON text from a stream, and send events synchronously to an object implementing Handler concept. It needs to allocate a stack for storing a single decoded string during non-destructive parsing. For in-situ parsing, the decoded string is directly written to the source text string, no temporary buffer is required. A GenericReader object can be reused for parsing multiple JSON text. \tparam SourceEncoding Encoding of the input stream. \tparam TargetEncoding Encoding of the parse output. \tparam StackAllocator Allocator type for stack. */ template class GenericReader { public: typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type //! Constructor. /*! \param stackAllocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) */ GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} //! Parse JSON text. /*! \tparam parseFlags Combination of \ref ParseFlag. \tparam InputStream Type of input stream, implementing Stream concept. \tparam Handler Type of handler, implementing Handler concept. \param is Input stream to be parsed. \param handler The handler to receive events. \return Whether the parsing is successful. */ template ParseResult Parse(InputStream& is, Handler& handler) { if (parseFlags & kParseIterativeFlag) return IterativeParse(is, handler); parseResult_.Clear(); ClearStackOnExit scope(*this); SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } else { ParseValue(is, handler); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (!(parseFlags & kParseStopWhenDoneFlag)) { SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (RAPIDJSON_UNLIKELY(is.Peek() != '\0')) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } } } return parseResult_; } //! Parse JSON text (with \ref kParseDefaultFlags) /*! \tparam InputStream Type of input stream, implementing Stream concept \tparam Handler Type of handler, implementing Handler concept. \param is Input stream to be parsed. \param handler The handler to receive events. \return Whether the parsing is successful. */ template ParseResult Parse(InputStream& is, Handler& handler) { return Parse(is, handler); } //! Whether a parse error has occured in the last parsing. bool HasParseError() const { return parseResult_.IsError(); } //! Get the \ref ParseErrorCode of last parsing. ParseErrorCode GetParseErrorCode() const { return parseResult_.Code(); } //! Get the position of last parsing error in input, 0 otherwise. size_t GetErrorOffset() const { return parseResult_.Offset(); } protected: void SetParseError(ParseErrorCode code, size_t offset) { parseResult_.Set(code, offset); } private: // Prohibit copy constructor & assignment operator. GenericReader(const GenericReader&); GenericReader& operator=(const GenericReader&); void ClearStack() { stack_.Clear(); } // clear stack on any exit from ParseStream, e.g. due to exception struct ClearStackOnExit { explicit ClearStackOnExit(GenericReader& r) : r_(r) {} ~ClearStackOnExit() { r_.ClearStack(); } private: GenericReader& r_; ClearStackOnExit(const ClearStackOnExit&); ClearStackOnExit& operator=(const ClearStackOnExit&); }; template void SkipWhitespaceAndComments(InputStream& is) { SkipWhitespace(is); if (parseFlags & kParseCommentsFlag) { while (RAPIDJSON_UNLIKELY(Consume(is, '/'))) { if (Consume(is, '*')) { while (true) { if (RAPIDJSON_UNLIKELY(is.Peek() == '\0')) RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); else if (Consume(is, '*')) { if (Consume(is, '/')) break; } else is.Take(); } } else if (RAPIDJSON_LIKELY(Consume(is, '/'))) while (is.Peek() != '\0' && is.Take() != '\n'); else RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); SkipWhitespace(is); } } } // Parse object: { string : value, ... } template void ParseObject(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == '{'); is.Take(); // Skip '{' if (RAPIDJSON_UNLIKELY(!handler.StartObject())) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (Consume(is, '}')) { if (RAPIDJSON_UNLIKELY(!handler.EndObject(0))) // empty object RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } for (SizeType memberCount = 0;;) { if (RAPIDJSON_UNLIKELY(is.Peek() != '"')) RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); ParseString(is, handler, true); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (RAPIDJSON_UNLIKELY(!Consume(is, ':'))) RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ParseValue(is, handler); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++memberCount; switch (is.Peek()) { case ',': is.Take(); SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; break; case '}': is.Take(); if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; default: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy } if (parseFlags & kParseTrailingCommasFlag) { if (is.Peek() == '}') { if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); is.Take(); return; } } } } // Parse array: [ value, ... ] template void ParseArray(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == '['); is.Take(); // Skip '[' if (RAPIDJSON_UNLIKELY(!handler.StartArray())) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (Consume(is, ']')) { if (RAPIDJSON_UNLIKELY(!handler.EndArray(0))) // empty array RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } for (SizeType elementCount = 0;;) { ParseValue(is, handler); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++elementCount; SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (Consume(is, ',')) { SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; } else if (Consume(is, ']')) { if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; } else RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); if (parseFlags & kParseTrailingCommasFlag) { if (is.Peek() == ']') { if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); is.Take(); return; } } } } template void ParseNull(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == 'n'); is.Take(); if (RAPIDJSON_LIKELY(Consume(is, 'u') && Consume(is, 'l') && Consume(is, 'l'))) { if (RAPIDJSON_UNLIKELY(!handler.Null())) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); } template void ParseTrue(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == 't'); is.Take(); if (RAPIDJSON_LIKELY(Consume(is, 'r') && Consume(is, 'u') && Consume(is, 'e'))) { if (RAPIDJSON_UNLIKELY(!handler.Bool(true))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); } template void ParseFalse(InputStream& is, Handler& handler) { RAPIDJSON_ASSERT(is.Peek() == 'f'); is.Take(); if (RAPIDJSON_LIKELY(Consume(is, 'a') && Consume(is, 'l') && Consume(is, 's') && Consume(is, 'e'))) { if (RAPIDJSON_UNLIKELY(!handler.Bool(false))) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); } template RAPIDJSON_FORCEINLINE static bool Consume(InputStream& is, typename InputStream::Ch expect) { if (RAPIDJSON_LIKELY(is.Peek() == expect)) { is.Take(); return true; } else return false; } // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). template unsigned ParseHex4(InputStream& is, size_t escapeOffset) { unsigned codepoint = 0; for (int i = 0; i < 4; i++) { Ch c = is.Peek(); codepoint <<= 4; codepoint += static_cast(c); if (c >= '0' && c <= '9') codepoint -= '0'; else if (c >= 'A' && c <= 'F') codepoint -= 'A' - 10; else if (c >= 'a' && c <= 'f') codepoint -= 'a' - 10; else { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStringUnicodeEscapeInvalidHex, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(0); } is.Take(); } return codepoint; } template class StackStream { public: typedef CharType Ch; StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} RAPIDJSON_FORCEINLINE void Put(Ch c) { *stack_.template Push() = c; ++length_; } RAPIDJSON_FORCEINLINE void* Push(SizeType count) { length_ += count; return stack_.template Push(count); } size_t Length() const { return length_; } Ch* Pop() { return stack_.template Pop(length_); } private: StackStream(const StackStream&); StackStream& operator=(const StackStream&); internal::Stack& stack_; SizeType length_; }; // Parse string and generate String event. Different code paths for kParseInsituFlag. template void ParseString(InputStream& is, Handler& handler, bool isKey = false) { internal::StreamLocalCopy copy(is); InputStream& s(copy.s); RAPIDJSON_ASSERT(s.Peek() == '\"'); s.Take(); // Skip '\"' bool success = false; if (parseFlags & kParseInsituFlag) { typename InputStream::Ch *head = s.PutBegin(); ParseStringToStream(s, s); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; size_t length = s.PutEnd(head) - 1; RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); const typename TargetEncoding::Ch* const str = reinterpret_cast(head); success = (isKey ? handler.Key(str, SizeType(length), false) : handler.String(str, SizeType(length), false)); } else { StackStream stackStream(stack_); ParseStringToStream(s, stackStream); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; SizeType length = static_cast(stackStream.Length()) - 1; const typename TargetEncoding::Ch* const str = stackStream.Pop(); success = (isKey ? handler.Key(str, length, true) : handler.String(str, length, true)); } if (RAPIDJSON_UNLIKELY(!success)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, s.Tell()); } // Parse string to an output is // This function handles the prefix/suffix double quotes, escaping, and optional encoding validation. template RAPIDJSON_FORCEINLINE void ParseStringToStream(InputStream& is, OutputStream& os) { //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 static const char escape[256] = { Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 }; #undef Z16 //!@endcond for (;;) { // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. if (!(parseFlags & kParseValidateEncodingFlag)) ScanCopyUnescapedString(is, os); Ch c = is.Peek(); if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset is.Take(); Ch e = is.Peek(); if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) { is.Take(); os.Put(static_cast(escape[static_cast(e)])); } else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode is.Take(); unsigned codepoint = ParseHex4(is, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { // Handle UTF-16 surrogate pair if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); unsigned codepoint2 = ParseHex4(is, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; } TEncoding::Encode(os, codepoint); } else RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, escapeOffset); } else if (RAPIDJSON_UNLIKELY(c == '"')) { // Closing double quote is.Take(); os.Put('\0'); // null-terminate the string return; } else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF if (c == '\0') RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); else RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell()); } else { size_t offset = is.Tell(); if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? !Transcoder::Validate(is, os) : !Transcoder::Transcode(is, os)))) RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, offset); } } } template static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { // Do nothing for generic version } #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) // StringStream -> StackStream static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { const char* p = is.src_; // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); while (p != nextAligned) if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { is.src_ = p; return; } else os.Put(*p++); // The rest of string using SIMD static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); for (;; p += 16) { const __m128i s = _mm_load_si128(reinterpret_cast(p)); const __m128i t1 = _mm_cmpeq_epi8(s, dq); const __m128i t2 = _mm_cmpeq_epi8(s, bs); const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); unsigned short r = static_cast(_mm_movemask_epi8(x)); if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped SizeType length; #ifdef _MSC_VER // Find the index of first escaped unsigned long offset; _BitScanForward(&offset, r); length = offset; #else length = static_cast(__builtin_ffs(r) - 1); #endif char* q = reinterpret_cast(os.Push(length)); for (size_t i = 0; i < length; i++) q[i] = p[i]; p += length; break; } _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); } is.src_ = p; } // InsituStringStream -> InsituStringStream static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { RAPIDJSON_ASSERT(&is == &os); (void)os; if (is.src_ == is.dst_) { SkipUnescapedString(is); return; } char* p = is.src_; char *q = is.dst_; // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); while (p != nextAligned) if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { is.src_ = p; is.dst_ = q; return; } else *q++ = *p++; // The rest of string using SIMD static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); for (;; p += 16, q += 16) { const __m128i s = _mm_load_si128(reinterpret_cast(p)); const __m128i t1 = _mm_cmpeq_epi8(s, dq); const __m128i t2 = _mm_cmpeq_epi8(s, bs); const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); unsigned short r = static_cast(_mm_movemask_epi8(x)); if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped size_t length; #ifdef _MSC_VER // Find the index of first escaped unsigned long offset; _BitScanForward(&offset, r); length = offset; #else length = static_cast(__builtin_ffs(r) - 1); #endif for (const char* pend = p + length; p != pend; ) *q++ = *p++; break; } _mm_storeu_si128(reinterpret_cast<__m128i *>(q), s); } is.src_ = p; is.dst_ = q; } // When read/write pointers are the same for insitu stream, just skip unescaped characters static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { RAPIDJSON_ASSERT(is.src_ == is.dst_); char* p = is.src_; // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); for (; p != nextAligned; p++) if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { is.src_ = is.dst_ = p; return; } // The rest of string using SIMD static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); for (;; p += 16) { const __m128i s = _mm_load_si128(reinterpret_cast(p)); const __m128i t1 = _mm_cmpeq_epi8(s, dq); const __m128i t2 = _mm_cmpeq_epi8(s, bs); const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); unsigned short r = static_cast(_mm_movemask_epi8(x)); if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped size_t length; #ifdef _MSC_VER // Find the index of first escaped unsigned long offset; _BitScanForward(&offset, r); length = offset; #else length = static_cast(__builtin_ffs(r) - 1); #endif p += length; break; } } is.src_ = is.dst_ = p; } #endif template class NumberStream; template class NumberStream { public: typedef typename InputStream::Ch Ch; NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } RAPIDJSON_FORCEINLINE void Push(char) {} size_t Tell() { return is.Tell(); } size_t Length() { return 0; } const char* Pop() { return 0; } protected: NumberStream& operator=(const NumberStream&); InputStream& is; }; template class NumberStream : public NumberStream { typedef NumberStream Base; public: NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch TakePush() { stackStream.Put(static_cast(Base::is.Peek())); return Base::is.Take(); } RAPIDJSON_FORCEINLINE void Push(char c) { stackStream.Put(c); } size_t Length() { return stackStream.Length(); } const char* Pop() { stackStream.Put('\0'); return stackStream.Pop(); } private: StackStream stackStream; }; template class NumberStream : public NumberStream { typedef NumberStream Base; public: NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } }; template void ParseNumber(InputStream& is, Handler& handler) { internal::StreamLocalCopy copy(is); NumberStream s(*this, copy.s); size_t startOffset = s.Tell(); double d = 0.0; bool useNanOrInf = false; // Parse minus bool minus = Consume(s, '-'); // Parse int: zero / ( digit1-9 *DIGIT ) unsigned i = 0; uint64_t i64 = 0; bool use64bit = false; int significandDigit = 0; if (RAPIDJSON_UNLIKELY(s.Peek() == '0')) { i = 0; s.TakePush(); } else if (RAPIDJSON_LIKELY(s.Peek() >= '1' && s.Peek() <= '9')) { i = static_cast(s.TakePush() - '0'); if (minus) while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (RAPIDJSON_UNLIKELY(i >= 214748364)) { // 2^31 = 2147483648 if (RAPIDJSON_LIKELY(i != 214748364 || s.Peek() > '8')) { i64 = i; use64bit = true; break; } } i = i * 10 + static_cast(s.TakePush() - '0'); significandDigit++; } else while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (RAPIDJSON_UNLIKELY(i >= 429496729)) { // 2^32 - 1 = 4294967295 if (RAPIDJSON_LIKELY(i != 429496729 || s.Peek() > '5')) { i64 = i; use64bit = true; break; } } i = i * 10 + static_cast(s.TakePush() - '0'); significandDigit++; } } // Parse NaN or Infinity here else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { useNanOrInf = true; if (RAPIDJSON_LIKELY(Consume(s, 'N') && Consume(s, 'a') && Consume(s, 'N'))) { d = std::numeric_limits::quiet_NaN(); } else if (RAPIDJSON_LIKELY(Consume(s, 'I') && Consume(s, 'n') && Consume(s, 'f'))) { d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); // Parse 64bit int bool useDouble = false; if (use64bit) { if (minus) while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { d = static_cast(i64); useDouble = true; break; } i64 = i64 * 10 + static_cast(s.TakePush() - '0'); significandDigit++; } else while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { d = static_cast(i64); useDouble = true; break; } i64 = i64 * 10 + static_cast(s.TakePush() - '0'); significandDigit++; } } // Force double for big integer if (useDouble) { while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); d = d * 10 + (s.TakePush() - '0'); } } // Parse frac = decimal-point 1*DIGIT int expFrac = 0; size_t decimalPosition; if (Consume(s, '.')) { decimalPosition = s.Length(); if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissFraction, s.Tell()); if (!useDouble) { #if RAPIDJSON_64BIT // Use i64 to store significand in 64-bit architecture if (!use64bit) i64 = i; while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (i64 > RAPIDJSON_UINT64_C2(0x1FFFFF, 0xFFFFFFFF)) // 2^53 - 1 for fast path break; else { i64 = i64 * 10 + static_cast(s.TakePush() - '0'); --expFrac; if (i64 != 0) significandDigit++; } } d = static_cast(i64); #else // Use double to store significand in 32-bit architecture d = static_cast(use64bit ? i64 : i); #endif useDouble = true; } while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (significandDigit < 17) { d = d * 10.0 + (s.TakePush() - '0'); --expFrac; if (RAPIDJSON_LIKELY(d > 0.0)) significandDigit++; } else s.TakePush(); } } else decimalPosition = s.Length(); // decimal position at the end of integer. // Parse exp = e [ minus / plus ] 1*DIGIT int exp = 0; if (Consume(s, 'e') || Consume(s, 'E')) { if (!useDouble) { d = static_cast(use64bit ? i64 : i); useDouble = true; } bool expMinus = false; if (Consume(s, '+')) ; else if (Consume(s, '-')) expMinus = true; if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { exp = static_cast(s.Take() - '0'); if (expMinus) { while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { exp = exp * 10 + static_cast(s.Take() - '0'); if (exp >= 214748364) { // Issue #313: prevent overflow exponent while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent s.Take(); } } } else { // positive exp int maxExp = 308 - expFrac; while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { exp = exp * 10 + static_cast(s.Take() - '0'); if (RAPIDJSON_UNLIKELY(exp > maxExp)) RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); } } } else RAPIDJSON_PARSE_ERROR(kParseErrorNumberMissExponent, s.Tell()); if (expMinus) exp = -exp; } // Finish parsing, call event according to the type of number. bool cont = true; if (parseFlags & kParseNumbersAsStringsFlag) { if (parseFlags & kParseInsituFlag) { s.Pop(); // Pop stack no matter if it will be used or not. typename InputStream::Ch* head = is.PutBegin(); const size_t length = s.Tell() - startOffset; RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); // unable to insert the \0 character here, it will erase the comma after this number const typename TargetEncoding::Ch* const str = reinterpret_cast(head); cont = handler.RawNumber(str, SizeType(length), false); } else { SizeType numCharsToCopy = static_cast(s.Length()); StringStream srcStream(s.Pop()); StackStream dstStream(stack_); while (numCharsToCopy--) { Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); } dstStream.Put('\0'); const typename TargetEncoding::Ch* str = dstStream.Pop(); const SizeType length = static_cast(dstStream.Length()) - 1; cont = handler.RawNumber(str, SizeType(length), true); } } else { size_t length = s.Length(); const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. if (useDouble) { int p = exp + expFrac; if (parseFlags & kParseFullPrecisionFlag) d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); else d = internal::StrtodNormalPrecision(d, p); cont = handler.Double(minus ? -d : d); } else if (useNanOrInf) { cont = handler.Double(d); } else { if (use64bit) { if (minus) cont = handler.Int64(static_cast(~i64 + 1)); else cont = handler.Uint64(i64); } else { if (minus) cont = handler.Int(static_cast(~i + 1)); else cont = handler.Uint(i); } } } if (RAPIDJSON_UNLIKELY(!cont)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); } // Parse any JSON value template void ParseValue(InputStream& is, Handler& handler) { switch (is.Peek()) { case 'n': ParseNull (is, handler); break; case 't': ParseTrue (is, handler); break; case 'f': ParseFalse (is, handler); break; case '"': ParseString(is, handler); break; case '{': ParseObject(is, handler); break; case '[': ParseArray (is, handler); break; default : ParseNumber(is, handler); break; } } // Iterative Parsing // States enum IterativeParsingState { IterativeParsingStartState = 0, IterativeParsingFinishState, IterativeParsingErrorState, // Object states IterativeParsingObjectInitialState, IterativeParsingMemberKeyState, IterativeParsingKeyValueDelimiterState, IterativeParsingMemberValueState, IterativeParsingMemberDelimiterState, IterativeParsingObjectFinishState, // Array states IterativeParsingArrayInitialState, IterativeParsingElementState, IterativeParsingElementDelimiterState, IterativeParsingArrayFinishState, // Single value state IterativeParsingValueState }; enum { cIterativeParsingStateCount = IterativeParsingValueState + 1 }; // Tokens enum Token { LeftBracketToken = 0, RightBracketToken, LeftCurlyBracketToken, RightCurlyBracketToken, CommaToken, ColonToken, StringToken, FalseToken, TrueToken, NullToken, NumberToken, kTokenCount }; RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define N NumberToken #define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N // Maps from ASCII to Token static const unsigned char tokenMap[256] = { N16, // 00~0F N16, // 10~1F N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F N16, // 40~4F N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF }; #undef N #undef N16 //!@endcond if (sizeof(Ch) == 1 || static_cast(c) < 256) return static_cast(tokenMap[static_cast(c)]); else return NumberToken; } RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { // current state x one lookahead token -> new state static const char G[cIterativeParsingStateCount][kTokenCount] = { // Start { IterativeParsingArrayInitialState, // Left bracket IterativeParsingErrorState, // Right bracket IterativeParsingObjectInitialState, // Left curly bracket IterativeParsingErrorState, // Right curly bracket IterativeParsingErrorState, // Comma IterativeParsingErrorState, // Colon IterativeParsingValueState, // String IterativeParsingValueState, // False IterativeParsingValueState, // True IterativeParsingValueState, // Null IterativeParsingValueState // Number }, // Finish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState }, // Error(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState }, // ObjectInitial { IterativeParsingErrorState, // Left bracket IterativeParsingErrorState, // Right bracket IterativeParsingErrorState, // Left curly bracket IterativeParsingObjectFinishState, // Right curly bracket IterativeParsingErrorState, // Comma IterativeParsingErrorState, // Colon IterativeParsingMemberKeyState, // String IterativeParsingErrorState, // False IterativeParsingErrorState, // True IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, // MemberKey { IterativeParsingErrorState, // Left bracket IterativeParsingErrorState, // Right bracket IterativeParsingErrorState, // Left curly bracket IterativeParsingErrorState, // Right curly bracket IterativeParsingErrorState, // Comma IterativeParsingKeyValueDelimiterState, // Colon IterativeParsingErrorState, // String IterativeParsingErrorState, // False IterativeParsingErrorState, // True IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, // KeyValueDelimiter { IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) IterativeParsingErrorState, // Right bracket IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) IterativeParsingErrorState, // Right curly bracket IterativeParsingErrorState, // Comma IterativeParsingErrorState, // Colon IterativeParsingMemberValueState, // String IterativeParsingMemberValueState, // False IterativeParsingMemberValueState, // True IterativeParsingMemberValueState, // Null IterativeParsingMemberValueState // Number }, // MemberValue { IterativeParsingErrorState, // Left bracket IterativeParsingErrorState, // Right bracket IterativeParsingErrorState, // Left curly bracket IterativeParsingObjectFinishState, // Right curly bracket IterativeParsingMemberDelimiterState, // Comma IterativeParsingErrorState, // Colon IterativeParsingErrorState, // String IterativeParsingErrorState, // False IterativeParsingErrorState, // True IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, // MemberDelimiter { IterativeParsingErrorState, // Left bracket IterativeParsingErrorState, // Right bracket IterativeParsingErrorState, // Left curly bracket IterativeParsingObjectFinishState, // Right curly bracket IterativeParsingErrorState, // Comma IterativeParsingErrorState, // Colon IterativeParsingMemberKeyState, // String IterativeParsingErrorState, // False IterativeParsingErrorState, // True IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, // ObjectFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState }, // ArrayInitial { IterativeParsingArrayInitialState, // Left bracket(push Element state) IterativeParsingArrayFinishState, // Right bracket IterativeParsingObjectInitialState, // Left curly bracket(push Element state) IterativeParsingErrorState, // Right curly bracket IterativeParsingErrorState, // Comma IterativeParsingErrorState, // Colon IterativeParsingElementState, // String IterativeParsingElementState, // False IterativeParsingElementState, // True IterativeParsingElementState, // Null IterativeParsingElementState // Number }, // Element { IterativeParsingErrorState, // Left bracket IterativeParsingArrayFinishState, // Right bracket IterativeParsingErrorState, // Left curly bracket IterativeParsingErrorState, // Right curly bracket IterativeParsingElementDelimiterState, // Comma IterativeParsingErrorState, // Colon IterativeParsingErrorState, // String IterativeParsingErrorState, // False IterativeParsingErrorState, // True IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, // ElementDelimiter { IterativeParsingArrayInitialState, // Left bracket(push Element state) IterativeParsingArrayFinishState, // Right bracket IterativeParsingObjectInitialState, // Left curly bracket(push Element state) IterativeParsingErrorState, // Right curly bracket IterativeParsingErrorState, // Comma IterativeParsingErrorState, // Colon IterativeParsingElementState, // String IterativeParsingElementState, // False IterativeParsingElementState, // True IterativeParsingElementState, // Null IterativeParsingElementState // Number }, // ArrayFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState }, // Single Value (sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState } }; // End of G return static_cast(G[state][token]); } // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). // May return a new state on state pop. template RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { (void)token; switch (dst) { case IterativeParsingErrorState: return dst; case IterativeParsingObjectInitialState: case IterativeParsingArrayInitialState: { // Push the state(Element or MemeberValue) if we are nested in another array or value of member. // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. IterativeParsingState n = src; if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) n = IterativeParsingElementState; else if (src == IterativeParsingKeyValueDelimiterState) n = IterativeParsingMemberValueState; // Push current state. *stack_.template Push(1) = n; // Initialize and push the member/element count. *stack_.template Push(1) = 0; // Call handler bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); // On handler short circuits the parsing. if (!hr) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); return IterativeParsingErrorState; } else { is.Take(); return dst; } } case IterativeParsingMemberKeyState: ParseString(is, handler, true); if (HasParseError()) return IterativeParsingErrorState; else return dst; case IterativeParsingKeyValueDelimiterState: RAPIDJSON_ASSERT(token == ColonToken); is.Take(); return dst; case IterativeParsingMemberValueState: // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. ParseValue(is, handler); if (HasParseError()) { return IterativeParsingErrorState; } return dst; case IterativeParsingElementState: // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. ParseValue(is, handler); if (HasParseError()) { return IterativeParsingErrorState; } return dst; case IterativeParsingMemberDelimiterState: case IterativeParsingElementDelimiterState: is.Take(); // Update member/element count. *stack_.template Top() = *stack_.template Top() + 1; return dst; case IterativeParsingObjectFinishState: { // Transit from delimiter is only allowed when trailing commas are enabled if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell()); return IterativeParsingErrorState; } // Get member count. SizeType c = *stack_.template Pop(1); // If the object is not empty, count the last member. if (src == IterativeParsingMemberValueState) ++c; // Restore the state. IterativeParsingState n = static_cast(*stack_.template Pop(1)); // Transit to Finish state if this is the topmost scope. if (n == IterativeParsingStartState) n = IterativeParsingFinishState; // Call handler bool hr = handler.EndObject(c); // On handler short circuits the parsing. if (!hr) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); return IterativeParsingErrorState; } else { is.Take(); return n; } } case IterativeParsingArrayFinishState: { // Transit from delimiter is only allowed when trailing commas are enabled if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell()); return IterativeParsingErrorState; } // Get element count. SizeType c = *stack_.template Pop(1); // If the array is not empty, count the last element. if (src == IterativeParsingElementState) ++c; // Restore the state. IterativeParsingState n = static_cast(*stack_.template Pop(1)); // Transit to Finish state if this is the topmost scope. if (n == IterativeParsingStartState) n = IterativeParsingFinishState; // Call handler bool hr = handler.EndArray(c); // On handler short circuits the parsing. if (!hr) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); return IterativeParsingErrorState; } else { is.Take(); return n; } } default: // This branch is for IterativeParsingValueState actually. // Use `default:` rather than // `case IterativeParsingValueState:` is for code coverage. // The IterativeParsingStartState is not enumerated in this switch-case. // It is impossible for that case. And it can be caught by following assertion. // The IterativeParsingFinishState is not enumerated in this switch-case either. // It is a "derivative" state which cannot triggered from Predict() directly. // Therefore it cannot happen here. And it can be caught by following assertion. RAPIDJSON_ASSERT(dst == IterativeParsingValueState); // Must be non-compound value. Or it would be ObjectInitial or ArrayInitial state. ParseValue(is, handler); if (HasParseError()) { return IterativeParsingErrorState; } return IterativeParsingFinishState; } } template void HandleError(IterativeParsingState src, InputStream& is) { if (HasParseError()) { // Error flag has been set. return; } switch (src) { case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); return; case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); return; case IterativeParsingObjectInitialState: case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; case IterativeParsingKeyValueDelimiterState: case IterativeParsingArrayInitialState: case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; } } template ParseResult IterativeParse(InputStream& is, Handler& handler) { parseResult_.Clear(); ClearStackOnExit scope(*this); IterativeParsingState state = IterativeParsingStartState; SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); while (is.Peek() != '\0') { Token t = Tokenize(is.Peek()); IterativeParsingState n = Predict(state, t); IterativeParsingState d = Transit(state, t, n, is, handler); if (d == IterativeParsingErrorState) { HandleError(state, is); break; } state = d; // Do not further consume streams if a root JSON has been parsed. if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) break; SkipWhitespaceAndComments(is); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } // Handle the end of file. if (state != IterativeParsingFinishState) HandleError(state, is); return parseResult_; } static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. ParseResult parseResult_; }; // class GenericReader //! Reader with UTF8 encoding and default allocator. typedef GenericReader, UTF8<> > Reader; RAPIDJSON_NAMESPACE_END #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif #ifdef _MSC_VER RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_READER_H_ ================================================ FILE: src/thirdparty/rapidjson/schema.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available-> // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License-> You may obtain a copy of the License at // // http://opensource->org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied-> See the License for the // specific language governing permissions and limitations under the License-> #ifndef RAPIDJSON_SCHEMA_H_ #define RAPIDJSON_SCHEMA_H_ #include "document.h" #include "pointer.h" #include // abs, floor #if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) #define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 #else #define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 #endif #if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) #define RAPIDJSON_SCHEMA_USE_STDREGEX 1 #else #define RAPIDJSON_SCHEMA_USE_STDREGEX 0 #endif #if RAPIDJSON_SCHEMA_USE_INTERNALREGEX #include "internal/regex.h" #elif RAPIDJSON_SCHEMA_USE_STDREGEX #include #endif #if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX #define RAPIDJSON_SCHEMA_HAS_REGEX 1 #else #define RAPIDJSON_SCHEMA_HAS_REGEX 0 #endif #ifndef RAPIDJSON_SCHEMA_VERBOSE #define RAPIDJSON_SCHEMA_VERBOSE 0 #endif #if RAPIDJSON_SCHEMA_VERBOSE #include "stringbuffer.h" #endif RAPIDJSON_DIAG_PUSH #if defined(__GNUC__) RAPIDJSON_DIAG_OFF(effc++) #endif #ifdef __clang__ RAPIDJSON_DIAG_OFF(weak-vtables) RAPIDJSON_DIAG_OFF(exit-time-destructors) RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) RAPIDJSON_DIAG_OFF(variadic-macros) #endif #ifdef _MSC_VER RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // Verbose Utilities #if RAPIDJSON_SCHEMA_VERBOSE namespace internal { inline void PrintInvalidKeyword(const char* keyword) { printf("Fail keyword: %s\n", keyword); } inline void PrintInvalidKeyword(const wchar_t* keyword) { wprintf(L"Fail keyword: %ls\n", keyword); } inline void PrintInvalidDocument(const char* document) { printf("Fail document: %s\n\n", document); } inline void PrintInvalidDocument(const wchar_t* document) { wprintf(L"Fail document: %ls\n\n", document); } inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); } inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); } } // namespace internal #endif // RAPIDJSON_SCHEMA_VERBOSE /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_INVALID_KEYWORD_RETURN #if RAPIDJSON_SCHEMA_VERBOSE #define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) #else #define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) #endif #define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ RAPIDJSON_MULTILINEMACRO_BEGIN\ context.invalidKeyword = keyword.GetString();\ RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ return false;\ RAPIDJSON_MULTILINEMACRO_END /////////////////////////////////////////////////////////////////////////////// // Forward declarations template class GenericSchemaDocument; namespace internal { template class Schema; /////////////////////////////////////////////////////////////////////////////// // ISchemaValidator class ISchemaValidator { public: virtual ~ISchemaValidator() {} virtual bool IsValid() const = 0; }; /////////////////////////////////////////////////////////////////////////////// // ISchemaStateFactory template class ISchemaStateFactory { public: virtual ~ISchemaStateFactory() {} virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; virtual void* CreateHasher() = 0; virtual uint64_t GetHashCode(void* hasher) = 0; virtual void DestroryHasher(void* hasher) = 0; virtual void* MallocState(size_t size) = 0; virtual void FreeState(void* p) = 0; }; /////////////////////////////////////////////////////////////////////////////// // Hasher // For comparison of compound value template class Hasher { public: typedef typename Encoding::Ch Ch; Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} bool Null() { return WriteType(kNullType); } bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } bool Double(double d) { Number n; if (d < 0) n.u.i = static_cast(d); else n.u.u = static_cast(d); n.d = d; return WriteNumber(n); } bool RawNumber(const Ch* str, SizeType len, bool) { WriteBuffer(kNumberType, str, len * sizeof(Ch)); return true; } bool String(const Ch* str, SizeType len, bool) { WriteBuffer(kStringType, str, len * sizeof(Ch)); return true; } bool StartObject() { return true; } bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } bool EndObject(SizeType memberCount) { uint64_t h = Hash(0, kObjectType); uint64_t* kv = stack_.template Pop(memberCount * 2); for (SizeType i = 0; i < memberCount; i++) h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive *stack_.template Push() = h; return true; } bool StartArray() { return true; } bool EndArray(SizeType elementCount) { uint64_t h = Hash(0, kArrayType); uint64_t* e = stack_.template Pop(elementCount); for (SizeType i = 0; i < elementCount; i++) h = Hash(h, e[i]); // Use hash to achieve element order sensitive *stack_.template Push() = h; return true; } bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } uint64_t GetHashCode() const { RAPIDJSON_ASSERT(IsValid()); return *stack_.template Top(); } private: static const size_t kDefaultSize = 256; struct Number { union U { uint64_t u; int64_t i; }u; double d; }; bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } bool WriteBuffer(Type type, const void* data, size_t len) { // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); const unsigned char* d = static_cast(data); for (size_t i = 0; i < len; i++) h = Hash(h, d[i]); *stack_.template Push() = h; return true; } static uint64_t Hash(uint64_t h, uint64_t d) { static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); h ^= d; h *= kPrime; return h; } Stack stack_; }; /////////////////////////////////////////////////////////////////////////////// // SchemaValidationContext template struct SchemaValidationContext { typedef Schema SchemaType; typedef ISchemaStateFactory SchemaValidatorFactoryType; typedef typename SchemaType::ValueType ValueType; typedef typename ValueType::Ch Ch; enum PatternValidatorType { kPatternValidatorOnly, kPatternValidatorWithProperty, kPatternValidatorWithAdditionalProperty }; SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : factory(f), schema(s), valueSchema(), invalidKeyword(), hasher(), arrayElementHashCodes(), validators(), validatorCount(), patternPropertiesValidators(), patternPropertiesValidatorCount(), patternPropertiesSchemas(), patternPropertiesSchemaCount(), valuePatternValidatorType(kPatternValidatorOnly), propertyExist(), inArray(false), valueUniqueness(false), arrayUniqueness(false) { } ~SchemaValidationContext() { if (hasher) factory.DestroryHasher(hasher); if (validators) { for (SizeType i = 0; i < validatorCount; i++) factory.DestroySchemaValidator(validators[i]); factory.FreeState(validators); } if (patternPropertiesValidators) { for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) factory.DestroySchemaValidator(patternPropertiesValidators[i]); factory.FreeState(patternPropertiesValidators); } if (patternPropertiesSchemas) factory.FreeState(patternPropertiesSchemas); if (propertyExist) factory.FreeState(propertyExist); } SchemaValidatorFactoryType& factory; const SchemaType* schema; const SchemaType* valueSchema; const Ch* invalidKeyword; void* hasher; // Only validator access void* arrayElementHashCodes; // Only validator access this ISchemaValidator** validators; SizeType validatorCount; ISchemaValidator** patternPropertiesValidators; SizeType patternPropertiesValidatorCount; const SchemaType** patternPropertiesSchemas; SizeType patternPropertiesSchemaCount; PatternValidatorType valuePatternValidatorType; PatternValidatorType objectPatternValidatorType; SizeType arrayElementIndex; bool* propertyExist; bool inArray; bool valueUniqueness; bool arrayUniqueness; }; /////////////////////////////////////////////////////////////////////////////// // Schema template class Schema { public: typedef typename SchemaDocumentType::ValueType ValueType; typedef typename SchemaDocumentType::AllocatorType AllocatorType; typedef typename SchemaDocumentType::PointerType PointerType; typedef typename ValueType::EncodingType EncodingType; typedef typename EncodingType::Ch Ch; typedef SchemaValidationContext Context; typedef Schema SchemaType; typedef GenericValue SValue; friend class GenericSchemaDocument; Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : allocator_(allocator), enum_(), enumCount_(), not_(), type_((1 << kTotalSchemaType) - 1), // typeless validatorCount_(), properties_(), additionalPropertiesSchema_(), patternProperties_(), patternPropertyCount_(), propertyCount_(), minProperties_(), maxProperties_(SizeType(~0)), additionalProperties_(true), hasDependencies_(), hasRequired_(), hasSchemaDependencies_(), additionalItemsSchema_(), itemsList_(), itemsTuple_(), itemsTupleCount_(), minItems_(), maxItems_(SizeType(~0)), additionalItems_(true), uniqueItems_(false), pattern_(), minLength_(0), maxLength_(~SizeType(0)), exclusiveMinimum_(false), exclusiveMaximum_(false) { typedef typename SchemaDocumentType::ValueType ValueType; typedef typename ValueType::ConstValueIterator ConstValueIterator; typedef typename ValueType::ConstMemberIterator ConstMemberIterator; if (!value.IsObject()) return; if (const ValueType* v = GetMember(value, GetTypeString())) { type_ = 0; if (v->IsString()) AddType(*v); else if (v->IsArray()) for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) AddType(*itr); } if (const ValueType* v = GetMember(value, GetEnumString())) if (v->IsArray() && v->Size() > 0) { enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { typedef Hasher > EnumHasherType; char buffer[256 + 24]; MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); EnumHasherType h(&hasherAllocator, 256); itr->Accept(h); enum_[enumCount_++] = h.GetHashCode(); } } if (schemaDocument) { AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); } if (const ValueType* v = GetMember(value, GetNotString())) { schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); notValidatorIndex_ = validatorCount_; validatorCount_++; } // Object const ValueType* properties = GetMember(value, GetPropertiesString()); const ValueType* required = GetMember(value, GetRequiredString()); const ValueType* dependencies = GetMember(value, GetDependenciesString()); { // Gather properties from properties/required/dependencies SValue allProperties(kArrayType); if (properties && properties->IsObject()) for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) AddUniqueElement(allProperties, itr->name); if (required && required->IsArray()) for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) if (itr->IsString()) AddUniqueElement(allProperties, *itr); if (dependencies && dependencies->IsObject()) for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { AddUniqueElement(allProperties, itr->name); if (itr->value.IsArray()) for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) if (i->IsString()) AddUniqueElement(allProperties, *i); } if (allProperties.Size() > 0) { propertyCount_ = allProperties.Size(); properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); for (SizeType i = 0; i < propertyCount_; i++) { new (&properties_[i]) Property(); properties_[i].name = allProperties[i]; properties_[i].schema = GetTypeless(); } } } if (properties && properties->IsObject()) { PointerType q = p.Append(GetPropertiesString(), allocator_); for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { SizeType index; if (FindPropertyIndex(itr->name, &index)) schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); } } if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { PointerType q = p.Append(GetPatternPropertiesString(), allocator_); patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); patternPropertyCount_ = 0; for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { new (&patternProperties_[patternPropertyCount_]) PatternProperty(); patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); patternPropertyCount_++; } } if (required && required->IsArray()) for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) if (itr->IsString()) { SizeType index; if (FindPropertyIndex(*itr, &index)) { properties_[index].required = true; hasRequired_ = true; } } if (dependencies && dependencies->IsObject()) { PointerType q = p.Append(GetDependenciesString(), allocator_); hasDependencies_ = true; for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { SizeType sourceIndex; if (FindPropertyIndex(itr->name, &sourceIndex)) { if (itr->value.IsArray()) { properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { SizeType targetIndex; if (FindPropertyIndex(*targetItr, &targetIndex)) properties_[sourceIndex].dependencies[targetIndex] = true; } } else if (itr->value.IsObject()) { hasSchemaDependencies_ = true; schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; validatorCount_++; } } } } if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { if (v->IsBool()) additionalProperties_ = v->GetBool(); else if (v->IsObject()) schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); } AssignIfExist(minProperties_, value, GetMinPropertiesString()); AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); // Array if (const ValueType* v = GetMember(value, GetItemsString())) { PointerType q = p.Append(GetItemsString(), allocator_); if (v->IsObject()) // List validation schemaDocument->CreateSchema(&itemsList_, q, *v, document); else if (v->IsArray()) { // Tuple validation itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); SizeType index = 0; for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); } } AssignIfExist(minItems_, value, GetMinItemsString()); AssignIfExist(maxItems_, value, GetMaxItemsString()); if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { if (v->IsBool()) additionalItems_ = v->GetBool(); else if (v->IsObject()) schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); } AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); // String AssignIfExist(minLength_, value, GetMinLengthString()); AssignIfExist(maxLength_, value, GetMaxLengthString()); if (const ValueType* v = GetMember(value, GetPatternString())) pattern_ = CreatePattern(*v); // Number if (const ValueType* v = GetMember(value, GetMinimumString())) if (v->IsNumber()) minimum_.CopyFrom(*v, *allocator_); if (const ValueType* v = GetMember(value, GetMaximumString())) if (v->IsNumber()) maximum_.CopyFrom(*v, *allocator_); AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); if (const ValueType* v = GetMember(value, GetMultipleOfString())) if (v->IsNumber() && v->GetDouble() > 0.0) multipleOf_.CopyFrom(*v, *allocator_); } ~Schema() { if (allocator_) { allocator_->Free(enum_); } if (properties_) { for (SizeType i = 0; i < propertyCount_; i++) properties_[i].~Property(); AllocatorType::Free(properties_); } if (patternProperties_) { for (SizeType i = 0; i < patternPropertyCount_; i++) patternProperties_[i].~PatternProperty(); AllocatorType::Free(patternProperties_); } AllocatorType::Free(itemsTuple_); #if RAPIDJSON_SCHEMA_HAS_REGEX if (pattern_) { pattern_->~RegexType(); allocator_->Free(pattern_); } #endif } bool BeginValue(Context& context) const { if (context.inArray) { if (uniqueItems_) context.valueUniqueness = true; if (itemsList_) context.valueSchema = itemsList_; else if (itemsTuple_) { if (context.arrayElementIndex < itemsTupleCount_) context.valueSchema = itemsTuple_[context.arrayElementIndex]; else if (additionalItemsSchema_) context.valueSchema = additionalItemsSchema_; else if (additionalItems_) context.valueSchema = GetTypeless(); else RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); } else context.valueSchema = GetTypeless(); context.arrayElementIndex++; } return true; } RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { if (context.patternPropertiesValidatorCount > 0) { bool otherValid = false; SizeType count = context.patternPropertiesValidatorCount; if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) otherValid = context.patternPropertiesValidators[--count]->IsValid(); bool patternValid = true; for (SizeType i = 0; i < count; i++) if (!context.patternPropertiesValidators[i]->IsValid()) { patternValid = false; break; } if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { if (!patternValid) RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); } else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { if (!patternValid || !otherValid) RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); } else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); } if (enum_) { const uint64_t h = context.factory.GetHashCode(context.hasher); for (SizeType i = 0; i < enumCount_; i++) if (enum_[i] == h) goto foundEnum; RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); foundEnum:; } if (allOf_.schemas) for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) if (!context.validators[i]->IsValid()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); if (anyOf_.schemas) { for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) if (context.validators[i]->IsValid()) goto foundAny; RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); foundAny:; } if (oneOf_.schemas) { bool oneValid = false; for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) if (context.validators[i]->IsValid()) { if (oneValid) RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); else oneValid = true; } if (!oneValid) RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); } if (not_ && context.validators[notValidatorIndex_]->IsValid()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); return true; } bool Null(Context& context) const { if (!(type_ & (1 << kNullSchemaType))) RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); return CreateParallelValidator(context); } bool Bool(Context& context, bool) const { if (!(type_ & (1 << kBooleanSchemaType))) RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); return CreateParallelValidator(context); } bool Int(Context& context, int i) const { if (!CheckInt(context, i)) return false; return CreateParallelValidator(context); } bool Uint(Context& context, unsigned u) const { if (!CheckUint(context, u)) return false; return CreateParallelValidator(context); } bool Int64(Context& context, int64_t i) const { if (!CheckInt(context, i)) return false; return CreateParallelValidator(context); } bool Uint64(Context& context, uint64_t u) const { if (!CheckUint(context, u)) return false; return CreateParallelValidator(context); } bool Double(Context& context, double d) const { if (!(type_ & (1 << kNumberSchemaType))) RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) return false; if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) return false; if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) return false; return CreateParallelValidator(context); } bool String(Context& context, const Ch* str, SizeType length, bool) const { if (!(type_ & (1 << kStringSchemaType))) RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); if (minLength_ != 0 || maxLength_ != SizeType(~0)) { SizeType count; if (internal::CountStringCodePoint(str, length, &count)) { if (count < minLength_) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); if (count > maxLength_) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); } } if (pattern_ && !IsPatternMatch(pattern_, str, length)) RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); return CreateParallelValidator(context); } bool StartObject(Context& context) const { if (!(type_ & (1 << kObjectSchemaType))) RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); if (hasDependencies_ || hasRequired_) { context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); } if (patternProperties_) { // pre-allocate schema array SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); context.patternPropertiesSchemaCount = 0; std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); } return CreateParallelValidator(context); } bool Key(Context& context, const Ch* str, SizeType len, bool) const { if (patternProperties_) { context.patternPropertiesSchemaCount = 0; for (SizeType i = 0; i < patternPropertyCount_; i++) if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; } SizeType index; if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { if (context.patternPropertiesSchemaCount > 0) { context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; context.valueSchema = GetTypeless(); context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; } else context.valueSchema = properties_[index].schema; if (context.propertyExist) context.propertyExist[index] = true; return true; } if (additionalPropertiesSchema_) { if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; context.valueSchema = GetTypeless(); context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; } else context.valueSchema = additionalPropertiesSchema_; return true; } else if (additionalProperties_) { context.valueSchema = GetTypeless(); return true; } if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); return true; } bool EndObject(Context& context, SizeType memberCount) const { if (hasRequired_) for (SizeType index = 0; index < propertyCount_; index++) if (properties_[index].required) if (!context.propertyExist[index]) RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); if (memberCount < minProperties_) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); if (memberCount > maxProperties_) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); if (hasDependencies_) { for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) if (context.propertyExist[sourceIndex]) { if (properties_[sourceIndex].dependencies) { for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); } else if (properties_[sourceIndex].dependenciesSchema) if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); } } return true; } bool StartArray(Context& context) const { if (!(type_ & (1 << kArraySchemaType))) RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); context.arrayElementIndex = 0; context.inArray = true; return CreateParallelValidator(context); } bool EndArray(Context& context, SizeType elementCount) const { context.inArray = false; if (elementCount < minItems_) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); if (elementCount > maxItems_) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); return true; } // Generate functions for string literal according to Ch #define RAPIDJSON_STRING_(name, ...) \ static const ValueType& Get##name##String() {\ static const Ch s[] = { __VA_ARGS__, '\0' };\ static const ValueType v(s, sizeof(s) / sizeof(Ch) - 1);\ return v;\ } RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') RAPIDJSON_STRING_(Not, 'n', 'o', 't') RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') #undef RAPIDJSON_STRING_ private: enum SchemaValueType { kNullSchemaType, kBooleanSchemaType, kObjectSchemaType, kArraySchemaType, kStringSchemaType, kNumberSchemaType, kIntegerSchemaType, kTotalSchemaType }; #if RAPIDJSON_SCHEMA_USE_INTERNALREGEX typedef internal::GenericRegex RegexType; #elif RAPIDJSON_SCHEMA_USE_STDREGEX typedef std::basic_regex RegexType; #else typedef char RegexType; #endif struct SchemaArray { SchemaArray() : schemas(), count() {} ~SchemaArray() { AllocatorType::Free(schemas); } const SchemaType** schemas; SizeType begin; // begin index of context.validators SizeType count; }; static const SchemaType* GetTypeless() { static SchemaType typeless(0, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); return &typeless; } template void AddUniqueElement(V1& a, const V2& v) { for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) if (*itr == v) return; V1 c(v, *allocator_); a.PushBack(c, *allocator_); } static const ValueType* GetMember(const ValueType& value, const ValueType& name) { typename ValueType::ConstMemberIterator itr = value.FindMember(name); return itr != value.MemberEnd() ? &(itr->value) : 0; } static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { if (const ValueType* v = GetMember(value, name)) if (v->IsBool()) out = v->GetBool(); } static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { if (const ValueType* v = GetMember(value, name)) if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) out = static_cast(v->GetUint64()); } void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { if (const ValueType* v = GetMember(value, name)) { if (v->IsArray() && v->Size() > 0) { PointerType q = p.Append(name, allocator_); out.count = v->Size(); out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); memset(out.schemas, 0, sizeof(Schema*)* out.count); for (SizeType i = 0; i < out.count; i++) schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); out.begin = validatorCount_; validatorCount_ += out.count; } } } #if RAPIDJSON_SCHEMA_USE_INTERNALREGEX template RegexType* CreatePattern(const ValueType& value) { if (value.IsString()) { RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); if (!r->IsValid()) { r->~RegexType(); AllocatorType::Free(r); r = 0; } return r; } return 0; } static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { return pattern->Search(str); } #elif RAPIDJSON_SCHEMA_USE_STDREGEX template RegexType* CreatePattern(const ValueType& value) { if (value.IsString()) try { return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); } catch (const std::regex_error&) { } return 0; } static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { std::match_results r; return std::regex_search(str, str + length, r, *pattern); } #else template RegexType* CreatePattern(const ValueType&) { return 0; } static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } #endif // RAPIDJSON_SCHEMA_USE_STDREGEX void AddType(const ValueType& type) { if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); } bool CreateParallelValidator(Context& context) const { if (enum_ || context.arrayUniqueness) context.hasher = context.factory.CreateHasher(); if (validatorCount_) { RAPIDJSON_ASSERT(context.validators == 0); context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); context.validatorCount = validatorCount_; if (allOf_.schemas) CreateSchemaValidators(context, allOf_); if (anyOf_.schemas) CreateSchemaValidators(context, anyOf_); if (oneOf_.schemas) CreateSchemaValidators(context, oneOf_); if (not_) context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); if (hasSchemaDependencies_) { for (SizeType i = 0; i < propertyCount_; i++) if (properties_[i].dependenciesSchema) context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); } } return true; } void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { for (SizeType i = 0; i < schemas.count; i++) context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); } // O(n) bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { SizeType len = name.GetStringLength(); const Ch* str = name.GetString(); for (SizeType index = 0; index < propertyCount_; index++) if (properties_[index].name.GetStringLength() == len && (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) { *outIndex = index; return true; } return false; } bool CheckInt(Context& context, int64_t i) const { if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); if (!minimum_.IsNull()) { if (minimum_.IsInt64()) { if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); } else if (minimum_.IsUint64()) { RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() } else if (!CheckDoubleMinimum(context, static_cast(i))) return false; } if (!maximum_.IsNull()) { if (maximum_.IsInt64()) { if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); } else if (maximum_.IsUint64()) /* do nothing */; // i <= max(int64_t) < maximum_.GetUint64() else if (!CheckDoubleMaximum(context, static_cast(i))) return false; } if (!multipleOf_.IsNull()) { if (multipleOf_.IsUint64()) { if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); } else if (!CheckDoubleMultipleOf(context, static_cast(i))) return false; } return true; } bool CheckUint(Context& context, uint64_t i) const { if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); if (!minimum_.IsNull()) { if (minimum_.IsUint64()) { if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); } else if (minimum_.IsInt64()) /* do nothing */; // i >= 0 > minimum.Getint64() else if (!CheckDoubleMinimum(context, static_cast(i))) return false; } if (!maximum_.IsNull()) { if (maximum_.IsUint64()) { if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); } else if (maximum_.IsInt64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ else if (!CheckDoubleMaximum(context, static_cast(i))) return false; } if (!multipleOf_.IsNull()) { if (multipleOf_.IsUint64()) { if (i % multipleOf_.GetUint64() != 0) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); } else if (!CheckDoubleMultipleOf(context, static_cast(i))) return false; } return true; } bool CheckDoubleMinimum(Context& context, double d) const { if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); return true; } bool CheckDoubleMaximum(Context& context, double d) const { if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); return true; } bool CheckDoubleMultipleOf(Context& context, double d) const { double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); double q = std::floor(a / b); double r = a - q * b; if (r > 0.0) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); return true; } struct Property { Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} ~Property() { AllocatorType::Free(dependencies); } SValue name; const SchemaType* schema; const SchemaType* dependenciesSchema; SizeType dependenciesValidatorIndex; bool* dependencies; bool required; }; struct PatternProperty { PatternProperty() : schema(), pattern() {} ~PatternProperty() { if (pattern) { pattern->~RegexType(); AllocatorType::Free(pattern); } } const SchemaType* schema; RegexType* pattern; }; AllocatorType* allocator_; uint64_t* enum_; SizeType enumCount_; SchemaArray allOf_; SchemaArray anyOf_; SchemaArray oneOf_; const SchemaType* not_; unsigned type_; // bitmask of kSchemaType SizeType validatorCount_; SizeType notValidatorIndex_; Property* properties_; const SchemaType* additionalPropertiesSchema_; PatternProperty* patternProperties_; SizeType patternPropertyCount_; SizeType propertyCount_; SizeType minProperties_; SizeType maxProperties_; bool additionalProperties_; bool hasDependencies_; bool hasRequired_; bool hasSchemaDependencies_; const SchemaType* additionalItemsSchema_; const SchemaType* itemsList_; const SchemaType** itemsTuple_; SizeType itemsTupleCount_; SizeType minItems_; SizeType maxItems_; bool additionalItems_; bool uniqueItems_; RegexType* pattern_; SizeType minLength_; SizeType maxLength_; SValue minimum_; SValue maximum_; SValue multipleOf_; bool exclusiveMinimum_; bool exclusiveMaximum_; }; template struct TokenHelper { RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { *documentStack.template Push() = '/'; char buffer[21]; size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); for (size_t i = 0; i < length; i++) *documentStack.template Push() = buffer[i]; } }; // Partial specialized version for char to prevent buffer copying. template struct TokenHelper { RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { if (sizeof(SizeType) == 4) { char *buffer = documentStack.template Push(1 + 10); // '/' + uint *buffer++ = '/'; const char* end = internal::u32toa(index, buffer); documentStack.template Pop(static_cast(10 - (end - buffer))); } else { char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 *buffer++ = '/'; const char* end = internal::u64toa(index, buffer); documentStack.template Pop(static_cast(20 - (end - buffer))); } } }; } // namespace internal /////////////////////////////////////////////////////////////////////////////// // IGenericRemoteSchemaDocumentProvider template class IGenericRemoteSchemaDocumentProvider { public: typedef typename SchemaDocumentType::Ch Ch; virtual ~IGenericRemoteSchemaDocumentProvider() {} virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; }; /////////////////////////////////////////////////////////////////////////////// // GenericSchemaDocument //! JSON schema document. /*! A JSON schema document is a compiled version of a JSON schema. It is basically a tree of internal::Schema. \note This is an immutable class (i.e. its instance cannot be modified after construction). \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. \tparam Allocator Allocator type for allocating memory of this document. */ template class GenericSchemaDocument { public: typedef ValueT ValueType; typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; typedef Allocator AllocatorType; typedef typename ValueType::EncodingType EncodingType; typedef typename EncodingType::Ch Ch; typedef internal::Schema SchemaType; typedef GenericPointer PointerType; friend class internal::Schema; template friend class GenericSchemaValidator; //! Constructor. /*! Compile a JSON document into schema document. \param document A JSON document as source. \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. \param allocator An optional allocator instance for allocating memory. Can be null. */ explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : remoteProvider_(remoteProvider), allocator_(allocator), ownAllocator_(), root_(), schemaMap_(allocator, kInitialSchemaMapSize), schemaRef_(allocator, kInitialSchemaRefSize) { if (!allocator_) ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); // Generate root schema, it will call CreateSchema() to create sub-schemas, // And call AddRefSchema() if there are $ref. CreateSchemaRecursive(&root_, PointerType(), document, document); // Resolve $ref while (!schemaRef_.Empty()) { SchemaRefEntry* refEntry = schemaRef_.template Pop(1); if (const SchemaType* s = GetSchema(refEntry->target)) { if (refEntry->schema) *refEntry->schema = s; // Create entry in map if not exist if (!GetSchema(refEntry->source)) { new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); } } refEntry->~SchemaRefEntry(); } RAPIDJSON_ASSERT(root_ != 0); schemaRef_.ShrinkToFit(); // Deallocate all memory for ref } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS //! Move constructor in C++11 GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : remoteProvider_(rhs.remoteProvider_), allocator_(rhs.allocator_), ownAllocator_(rhs.ownAllocator_), root_(rhs.root_), schemaMap_(std::move(rhs.schemaMap_)), schemaRef_(std::move(rhs.schemaRef_)) { rhs.remoteProvider_ = 0; rhs.allocator_ = 0; rhs.ownAllocator_ = 0; } #endif //! Destructor ~GenericSchemaDocument() { while (!schemaMap_.Empty()) schemaMap_.template Pop(1)->~SchemaEntry(); RAPIDJSON_DELETE(ownAllocator_); } //! Get the root schema. const SchemaType& GetRoot() const { return *root_; } private: //! Prohibit copying GenericSchemaDocument(const GenericSchemaDocument&); //! Prohibit assignment GenericSchemaDocument& operator=(const GenericSchemaDocument&); struct SchemaRefEntry { SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} PointerType source; PointerType target; const SchemaType** schema; }; struct SchemaEntry { SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} ~SchemaEntry() { if (owned) { schema->~SchemaType(); Allocator::Free(schema); } } PointerType pointer; SchemaType* schema; bool owned; }; void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { if (schema) *schema = SchemaType::GetTypeless(); if (v.GetType() == kObjectType) { const SchemaType* s = GetSchema(pointer); if (!s) CreateSchema(schema, pointer, v, document); for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); } else if (v.GetType() == kArrayType) for (SizeType i = 0; i < v.Size(); i++) CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); } void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { RAPIDJSON_ASSERT(pointer.IsValid()); if (v.IsObject()) { if (!HandleRefSchema(pointer, schema, v, document)) { SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); if (schema) *schema = s; } } } bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; static const ValueType kRefValue(kRefString, 4); typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); if (itr == v.MemberEnd()) return false; if (itr->value.IsString()) { SizeType len = itr->value.GetStringLength(); if (len > 0) { const Ch* s = itr->value.GetString(); SizeType i = 0; while (i < len && s[i] != '#') // Find the first # i++; if (i > 0) { // Remote reference, resolve immediately if (remoteProvider_) { if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i - 1)) { PointerType pointer(&s[i], len - i, allocator_); if (pointer.IsValid()) { if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { if (schema) *schema = sc; return true; } } } } } else if (s[i] == '#') { // Local reference, defer resolution PointerType pointer(&s[i], len - i, allocator_); if (pointer.IsValid()) { if (const ValueType* nv = pointer.Get(document)) if (HandleRefSchema(source, schema, *nv, document)) return true; new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); return true; } } } } return false; } const SchemaType* GetSchema(const PointerType& pointer) const { for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) if (pointer == target->pointer) return target->schema; return 0; } PointerType GetPointer(const SchemaType* schema) const { for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) if (schema == target->schema) return target->pointer; return PointerType(); } static const size_t kInitialSchemaMapSize = 64; static const size_t kInitialSchemaRefSize = 64; IRemoteSchemaDocumentProviderType* remoteProvider_; Allocator *allocator_; Allocator *ownAllocator_; const SchemaType* root_; //!< Root schema. internal::Stack schemaMap_; // Stores created Pointer -> Schemas internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref }; //! GenericSchemaDocument using Value type. typedef GenericSchemaDocument SchemaDocument; //! IGenericRemoteSchemaDocumentProvider using SchemaDocument. typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; /////////////////////////////////////////////////////////////////////////////// // GenericSchemaValidator //! JSON Schema Validator. /*! A SAX style JSON schema validator. It uses a \c GenericSchemaDocument to validate SAX events. It delegates the incoming SAX events to an output handler. The default output handler does nothing. It can be reused multiple times by calling \c Reset(). \tparam SchemaDocumentType Type of schema document. \tparam OutputHandler Type of output handler. Default handler does nothing. \tparam StateAllocator Allocator for storing the internal validation states. */ template < typename SchemaDocumentType, typename OutputHandler = BaseReaderHandler, typename StateAllocator = CrtAllocator> class GenericSchemaValidator : public internal::ISchemaStateFactory, public internal::ISchemaValidator { public: typedef typename SchemaDocumentType::SchemaType SchemaType; typedef typename SchemaDocumentType::PointerType PointerType; typedef typename SchemaType::EncodingType EncodingType; typedef typename EncodingType::Ch Ch; //! Constructor without output handler. /*! \param schemaDocument The schema document to conform to. \param allocator Optional allocator for storing internal validation states. \param schemaStackCapacity Optional initial capacity of schema path stack. \param documentStackCapacity Optional initial capacity of document path stack. */ GenericSchemaValidator( const SchemaDocumentType& schemaDocument, StateAllocator* allocator = 0, size_t schemaStackCapacity = kDefaultSchemaStackCapacity, size_t documentStackCapacity = kDefaultDocumentStackCapacity) : schemaDocument_(&schemaDocument), root_(schemaDocument.GetRoot()), outputHandler_(GetNullHandler()), stateAllocator_(allocator), ownStateAllocator_(0), schemaStack_(allocator, schemaStackCapacity), documentStack_(allocator, documentStackCapacity), valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE , depth_(0) #endif { } //! Constructor with output handler. /*! \param schemaDocument The schema document to conform to. \param allocator Optional allocator for storing internal validation states. \param schemaStackCapacity Optional initial capacity of schema path stack. \param documentStackCapacity Optional initial capacity of document path stack. */ GenericSchemaValidator( const SchemaDocumentType& schemaDocument, OutputHandler& outputHandler, StateAllocator* allocator = 0, size_t schemaStackCapacity = kDefaultSchemaStackCapacity, size_t documentStackCapacity = kDefaultDocumentStackCapacity) : schemaDocument_(&schemaDocument), root_(schemaDocument.GetRoot()), outputHandler_(outputHandler), stateAllocator_(allocator), ownStateAllocator_(0), schemaStack_(allocator, schemaStackCapacity), documentStack_(allocator, documentStackCapacity), valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE , depth_(0) #endif { } //! Destructor. ~GenericSchemaValidator() { Reset(); RAPIDJSON_DELETE(ownStateAllocator_); } //! Reset the internal states. void Reset() { while (!schemaStack_.Empty()) PopSchema(); documentStack_.Clear(); valid_ = true; } //! Checks whether the current state is valid. // Implementation of ISchemaValidator virtual bool IsValid() const { return valid_; } //! Gets the JSON pointer pointed to the invalid schema. PointerType GetInvalidSchemaPointer() const { return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); } //! Gets the keyword of invalid schema. const Ch* GetInvalidSchemaKeyword() const { return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; } //! Gets the JSON pointer pointed to the invalid value. PointerType GetInvalidDocumentPointer() const { return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); } #if RAPIDJSON_SCHEMA_VERBOSE #define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ RAPIDJSON_MULTILINEMACRO_BEGIN\ *documentStack_.template Push() = '\0';\ documentStack_.template Pop(1);\ internal::PrintInvalidDocument(documentStack_.template Bottom());\ RAPIDJSON_MULTILINEMACRO_END #else #define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() #endif #define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ if (!valid_) return false; \ if (!BeginValue() || !CurrentSchema().method arg1) {\ RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ return valid_ = false;\ } #define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ if (context->hasher)\ static_cast(context->hasher)->method arg2;\ if (context->validators)\ for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ static_cast(context->validators[i_])->method arg2;\ if (context->patternPropertiesValidators)\ for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ static_cast(context->patternPropertiesValidators[i_])->method arg2;\ } #define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ return valid_ = EndValue() && outputHandler_.method arg2 #define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } bool RawNumber(const Ch* str, SizeType length, bool copy) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } bool String(const Ch* str, SizeType length, bool copy) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } bool StartObject() { RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); return valid_ = outputHandler_.StartObject(); } bool Key(const Ch* str, SizeType len, bool copy) { if (!valid_) return false; AppendToken(str, len); if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); return valid_ = outputHandler_.Key(str, len, copy); } bool EndObject(SizeType memberCount) { if (!valid_) return false; RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); } bool StartArray() { RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); return valid_ = outputHandler_.StartArray(); } bool EndArray(SizeType elementCount) { if (!valid_) return false; RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); } #undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ #undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ #undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ #undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ // Implementation of ISchemaStateFactory virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, #if RAPIDJSON_SCHEMA_VERBOSE depth_ + 1, #endif &GetStateAllocator()); } virtual void DestroySchemaValidator(ISchemaValidator* validator) { GenericSchemaValidator* v = static_cast(validator); v->~GenericSchemaValidator(); StateAllocator::Free(v); } virtual void* CreateHasher() { return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); } virtual uint64_t GetHashCode(void* hasher) { return static_cast(hasher)->GetHashCode(); } virtual void DestroryHasher(void* hasher) { HasherType* h = static_cast(hasher); h->~HasherType(); StateAllocator::Free(h); } virtual void* MallocState(size_t size) { return GetStateAllocator().Malloc(size); } virtual void FreeState(void* p) { return StateAllocator::Free(p); } private: typedef typename SchemaType::Context Context; typedef GenericValue, StateAllocator> HashCodeArray; typedef internal::Hasher HasherType; GenericSchemaValidator( const SchemaDocumentType& schemaDocument, const SchemaType& root, #if RAPIDJSON_SCHEMA_VERBOSE unsigned depth, #endif StateAllocator* allocator = 0, size_t schemaStackCapacity = kDefaultSchemaStackCapacity, size_t documentStackCapacity = kDefaultDocumentStackCapacity) : schemaDocument_(&schemaDocument), root_(root), outputHandler_(GetNullHandler()), stateAllocator_(allocator), ownStateAllocator_(0), schemaStack_(allocator, schemaStackCapacity), documentStack_(allocator, documentStackCapacity), valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE , depth_(depth) #endif { } StateAllocator& GetStateAllocator() { if (!stateAllocator_) stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator()); return *stateAllocator_; } bool BeginValue() { if (schemaStack_.Empty()) PushSchema(root_); else { if (CurrentContext().inArray) internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); if (!CurrentSchema().BeginValue(CurrentContext())) return false; SizeType count = CurrentContext().patternPropertiesSchemaCount; const SchemaType** sa = CurrentContext().patternPropertiesSchemas; typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; bool valueUniqueness = CurrentContext().valueUniqueness; if (CurrentContext().valueSchema) PushSchema(*CurrentContext().valueSchema); if (count > 0) { CurrentContext().objectPatternValidatorType = patternValidatorType; ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); for (SizeType i = 0; i < count; i++) va[validatorCount++] = CreateSchemaValidator(*sa[i]); } CurrentContext().arrayUniqueness = valueUniqueness; } return true; } bool EndValue() { if (!CurrentSchema().EndValue(CurrentContext())) return false; #if RAPIDJSON_SCHEMA_VERBOSE GenericStringBuffer sb; schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); *documentStack_.template Push() = '\0'; documentStack_.template Pop(1); internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); #endif uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; PopSchema(); if (!schemaStack_.Empty()) { Context& context = CurrentContext(); if (context.valueUniqueness) { HashCodeArray* a = static_cast(context.arrayElementHashCodes); if (!a) CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) if (itr->GetUint64() == h) RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); a->PushBack(h, GetStateAllocator()); } } // Remove the last token of document pointer while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') ; return true; } void AppendToken(const Ch* str, SizeType len) { documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters *documentStack_.template PushUnsafe() = '/'; for (SizeType i = 0; i < len; i++) { if (str[i] == '~') { *documentStack_.template PushUnsafe() = '~'; *documentStack_.template PushUnsafe() = '0'; } else if (str[i] == '/') { *documentStack_.template PushUnsafe() = '~'; *documentStack_.template PushUnsafe() = '1'; } else *documentStack_.template PushUnsafe() = str[i]; } } RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } RAPIDJSON_FORCEINLINE void PopSchema() { Context* c = schemaStack_.template Pop(1); if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { a->~HashCodeArray(); StateAllocator::Free(a); } c->~Context(); } const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } Context& CurrentContext() { return *schemaStack_.template Top(); } const Context& CurrentContext() const { return *schemaStack_.template Top(); } static OutputHandler& GetNullHandler() { static OutputHandler nullHandler; return nullHandler; } static const size_t kDefaultSchemaStackCapacity = 1024; static const size_t kDefaultDocumentStackCapacity = 256; const SchemaDocumentType* schemaDocument_; const SchemaType& root_; OutputHandler& outputHandler_; StateAllocator* stateAllocator_; StateAllocator* ownStateAllocator_; internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) bool valid_; #if RAPIDJSON_SCHEMA_VERBOSE unsigned depth_; #endif }; typedef GenericSchemaValidator SchemaValidator; /////////////////////////////////////////////////////////////////////////////// // SchemaValidatingReader //! A helper class for parsing with validation. /*! This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). \tparam parseFlags Combination of \ref ParseFlag. \tparam InputStream Type of input stream, implementing Stream concept. \tparam SourceEncoding Encoding of the input stream. \tparam SchemaDocumentType Type of schema document. \tparam StackAllocator Allocator type for stack. */ template < unsigned parseFlags, typename InputStream, typename SourceEncoding, typename SchemaDocumentType = SchemaDocument, typename StackAllocator = CrtAllocator> class SchemaValidatingReader { public: typedef typename SchemaDocumentType::PointerType PointerType; typedef typename InputStream::Ch Ch; //! Constructor /*! \param is Input stream. \param sd Schema document. */ SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} template bool operator()(Handler& handler) { GenericReader reader; GenericSchemaValidator validator(sd_, handler); parseResult_ = reader.template Parse(is_, validator); isValid_ = validator.IsValid(); if (isValid_) { invalidSchemaPointer_ = PointerType(); invalidSchemaKeyword_ = 0; invalidDocumentPointer_ = PointerType(); } else { invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); } return parseResult_; } const ParseResult& GetParseResult() const { return parseResult_; } bool IsValid() const { return isValid_; } const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } private: InputStream& is_; const SchemaDocumentType& sd_; ParseResult parseResult_; PointerType invalidSchemaPointer_; const Ch* invalidSchemaKeyword_; PointerType invalidDocumentPointer_; bool isValid_; }; RAPIDJSON_NAMESPACE_END RAPIDJSON_DIAG_POP #endif // RAPIDJSON_SCHEMA_H_ ================================================ FILE: src/thirdparty/rapidjson/stream.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "rapidjson.h" #ifndef RAPIDJSON_STREAM_H_ #define RAPIDJSON_STREAM_H_ #include "encodings.h" RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // Stream /*! \class rapidjson::Stream \brief Concept for reading and writing characters. For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). For write-only stream, only need to implement Put() and Flush(). \code concept Stream { typename Ch; //!< Character type of the stream. //! Read the current character from stream without moving the read cursor. Ch Peek() const; //! Read the current character from stream and moving the read cursor to next character. Ch Take(); //! Get the current read cursor. //! \return Number of characters read from start. size_t Tell(); //! Begin writing operation at the current read pointer. //! \return The begin writer pointer. Ch* PutBegin(); //! Write a character. void Put(Ch c); //! Flush the buffer. void Flush(); //! End the writing operation. //! \param begin The begin write pointer returned by PutBegin(). //! \return Number of characters written. size_t PutEnd(Ch* begin); } \endcode */ //! Provides additional information for stream. /*! By using traits pattern, this type provides a default configuration for stream. For custom stream, this type can be specialized for other configuration. See TEST(Reader, CustomStringStream) in readertest.cpp for example. */ template struct StreamTraits { //! Whether to make local copy of stream for optimization during parsing. /*! By default, for safety, streams do not use local copy optimization. Stream that can be copied fast should specialize this, like StreamTraits. */ enum { copyOptimization = 0 }; }; //! Reserve n characters for writing to a stream. template inline void PutReserve(Stream& stream, size_t count) { (void)stream; (void)count; } //! Write character to a stream, presuming buffer is reserved. template inline void PutUnsafe(Stream& stream, typename Stream::Ch c) { stream.Put(c); } //! Put N copies of a character to a stream. template inline void PutN(Stream& stream, Ch c, size_t n) { PutReserve(stream, n); for (size_t i = 0; i < n; i++) PutUnsafe(stream, c); } /////////////////////////////////////////////////////////////////////////////// // StringStream //! Read-only string stream. /*! \note implements Stream concept */ template struct GenericStringStream { typedef typename Encoding::Ch Ch; GenericStringStream(const Ch *src) : src_(src), head_(src) {} Ch Peek() const { return *src_; } Ch Take() { return *src_++; } size_t Tell() const { return static_cast(src_ - head_); } Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } void Put(Ch) { RAPIDJSON_ASSERT(false); } void Flush() { RAPIDJSON_ASSERT(false); } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } const Ch* src_; //!< Current read position. const Ch* head_; //!< Original head of the string. }; template struct StreamTraits > { enum { copyOptimization = 1 }; }; //! String stream with UTF8 encoding. typedef GenericStringStream > StringStream; /////////////////////////////////////////////////////////////////////////////// // InsituStringStream //! A read-write string stream. /*! This string stream is particularly designed for in-situ parsing. \note implements Stream concept */ template struct GenericInsituStringStream { typedef typename Encoding::Ch Ch; GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} // Read Ch Peek() { return *src_; } Ch Take() { return *src_++; } size_t Tell() { return static_cast(src_ - head_); } // Write void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } Ch* PutBegin() { return dst_ = src_; } size_t PutEnd(Ch* begin) { return static_cast(dst_ - begin); } void Flush() {} Ch* Push(size_t count) { Ch* begin = dst_; dst_ += count; return begin; } void Pop(size_t count) { dst_ -= count; } Ch* src_; Ch* dst_; Ch* head_; }; template struct StreamTraits > { enum { copyOptimization = 1 }; }; //! Insitu string stream with UTF8 encoding. typedef GenericInsituStringStream > InsituStringStream; RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_STREAM_H_ ================================================ FILE: src/thirdparty/rapidjson/stringbuffer.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_STRINGBUFFER_H_ #define RAPIDJSON_STRINGBUFFER_H_ #include "stream.h" #include "internal/stack.h" #if RAPIDJSON_HAS_CXX11_RVALUE_REFS #include // std::move #endif #include "internal/stack.h" #if defined(__clang__) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(c++98-compat) #endif RAPIDJSON_NAMESPACE_BEGIN //! Represents an in-memory output stream. /*! \tparam Encoding Encoding of the stream. \tparam Allocator type for allocating memory buffer. \note implements Stream concept */ template class GenericStringBuffer { public: typedef typename Encoding::Ch Ch; GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} #if RAPIDJSON_HAS_CXX11_RVALUE_REFS GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { if (&rhs != this) stack_ = std::move(rhs.stack_); return *this; } #endif void Put(Ch c) { *stack_.template Push() = c; } void PutUnsafe(Ch c) { *stack_.template PushUnsafe() = c; } void Flush() {} void Clear() { stack_.Clear(); } void ShrinkToFit() { // Push and pop a null terminator. This is safe. *stack_.template Push() = '\0'; stack_.ShrinkToFit(); stack_.template Pop(1); } void Reserve(size_t count) { stack_.template Reserve(count); } Ch* Push(size_t count) { return stack_.template Push(count); } Ch* PushUnsafe(size_t count) { return stack_.template PushUnsafe(count); } void Pop(size_t count) { stack_.template Pop(count); } const Ch* GetString() const { // Push and pop a null terminator. This is safe. *stack_.template Push() = '\0'; stack_.template Pop(1); return stack_.template Bottom(); } size_t GetSize() const { return stack_.GetSize(); } static const size_t kDefaultCapacity = 256; mutable internal::Stack stack_; private: // Prohibit copy constructor & assignment operator. GenericStringBuffer(const GenericStringBuffer&); GenericStringBuffer& operator=(const GenericStringBuffer&); }; //! String buffer with UTF8 encoding typedef GenericStringBuffer > StringBuffer; template inline void PutReserve(GenericStringBuffer& stream, size_t count) { stream.Reserve(count); } template inline void PutUnsafe(GenericStringBuffer& stream, typename Encoding::Ch c) { stream.PutUnsafe(c); } //! Implement specialized version of PutN() with memset() for better performance. template<> inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { std::memset(stream.stack_.Push(n), c, n * sizeof(c)); } RAPIDJSON_NAMESPACE_END #if defined(__clang__) RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_STRINGBUFFER_H_ ================================================ FILE: src/thirdparty/rapidjson/writer.h ================================================ // Tencent is pleased to support the open source community by making RapidJSON available. // // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // // Unless required by applicable law or agreed to in writing, software distributed // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_WRITER_H_ #define RAPIDJSON_WRITER_H_ #include "stream.h" #include "internal/stack.h" #include "internal/strfunc.h" #include "internal/dtoa.h" #include "internal/itoa.h" #include "stringbuffer.h" #include // placement new #if defined(RAPIDJSON_SIMD) && defined(_MSC_VER) #include #pragma intrinsic(_BitScanForward) #endif #ifdef RAPIDJSON_SSE42 #include #elif defined(RAPIDJSON_SSE2) #include #endif #ifdef _MSC_VER RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant #endif #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(unreachable-code) #endif RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// // WriteFlag /*! \def RAPIDJSON_WRITE_DEFAULT_FLAGS \ingroup RAPIDJSON_CONFIG \brief User-defined kWriteDefaultFlags definition. User can define this as any \c WriteFlag combinations. */ #ifndef RAPIDJSON_WRITE_DEFAULT_FLAGS #define RAPIDJSON_WRITE_DEFAULT_FLAGS kWriteNoFlags #endif //! Combination of writeFlags enum WriteFlag { kWriteNoFlags = 0, //!< No flags are set. kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS }; //! JSON writer /*! Writer implements the concept Handler. It generates JSON text by events to an output os. User may programmatically calls the functions of a writer to generate JSON text. On the other side, a writer can also be passed to objects that generates events, for example Reader::Parse() and Document::Accept(). \tparam OutputStream Type of output stream. \tparam SourceEncoding Encoding of source string. \tparam TargetEncoding Encoding of output stream. \tparam StackAllocator Type of allocator for allocating memory of stack. \note implements Handler concept */ template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> class Writer { public: typedef typename SourceEncoding::Ch Ch; static const int kDefaultMaxDecimalPlaces = 324; //! Constructor /*! \param os Output stream. \param stackAllocator User supplied allocator. If it is null, it will create a private one. \param levelDepth Initial capacity of stack. */ explicit Writer(OutputStream& os, StackAllocator* stackAllocator = 0, size_t levelDepth = kDefaultLevelDepth) : os_(&os), level_stack_(stackAllocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} explicit Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} //! Reset the writer with a new stream. /*! This function reset the writer with a new stream and default settings, in order to make a Writer object reusable for output multiple JSONs. \param os New output stream. \code Writer writer(os1); writer.StartObject(); // ... writer.EndObject(); writer.Reset(os2); writer.StartObject(); // ... writer.EndObject(); \endcode */ void Reset(OutputStream& os) { os_ = &os; hasRoot_ = false; level_stack_.Clear(); } //! Checks whether the output is a complete JSON. /*! A complete JSON has a complete root object or array. */ bool IsComplete() const { return hasRoot_ && level_stack_.Empty(); } int GetMaxDecimalPlaces() const { return maxDecimalPlaces_; } //! Sets the maximum number of decimal places for double output. /*! This setting truncates the output with specified number of decimal places. For example, \code writer.SetMaxDecimalPlaces(3); writer.StartArray(); writer.Double(0.12345); // "0.123" writer.Double(0.0001); // "0.0" writer.Double(1.234567890123456e30); // "1.234567890123456e30" (do not truncate significand for positive exponent) writer.Double(1.23e-4); // "0.0" (do truncate significand for negative exponent) writer.EndArray(); \endcode The default setting does not truncate any decimal places. You can restore to this setting by calling \code writer.SetMaxDecimalPlaces(Writer::kDefaultMaxDecimalPlaces); \endcode */ void SetMaxDecimalPlaces(int maxDecimalPlaces) { maxDecimalPlaces_ = maxDecimalPlaces; } /*!@name Implementation of Handler \see Handler */ //@{ bool Null() { Prefix(kNullType); return EndValue(WriteNull()); } bool Bool(bool b) { Prefix(b ? kTrueType : kFalseType); return EndValue(WriteBool(b)); } bool Int(int i) { Prefix(kNumberType); return EndValue(WriteInt(i)); } bool Uint(unsigned u) { Prefix(kNumberType); return EndValue(WriteUint(u)); } bool Int64(int64_t i64) { Prefix(kNumberType); return EndValue(WriteInt64(i64)); } bool Uint64(uint64_t u64) { Prefix(kNumberType); return EndValue(WriteUint64(u64)); } //! Writes the given \c double value to the stream /*! \param d The value to be written. \return Whether it is succeed. */ bool Double(double d) { Prefix(kNumberType); return EndValue(WriteDouble(d)); } bool RawNumber(const Ch* str, SizeType length, bool copy = false) { (void)copy; Prefix(kNumberType); return EndValue(WriteString(str, length)); } bool String(const Ch* str, SizeType length, bool copy = false) { (void)copy; Prefix(kStringType); return EndValue(WriteString(str, length)); } #if RAPIDJSON_HAS_STDSTRING bool String(const std::basic_string& str) { return String(str.data(), SizeType(str.size())); } #endif bool StartObject() { Prefix(kObjectType); new (level_stack_.template Push()) Level(false); return WriteStartObject(); } bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } bool EndObject(SizeType memberCount = 0) { (void)memberCount; RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); level_stack_.template Pop(1); return EndValue(WriteEndObject()); } bool StartArray() { Prefix(kArrayType); new (level_stack_.template Push()) Level(true); return WriteStartArray(); } bool EndArray(SizeType elementCount = 0) { (void)elementCount; RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); level_stack_.template Pop(1); return EndValue(WriteEndArray()); } //@} /*! @name Convenience extensions */ //@{ //! Simpler but slower overload. bool String(const Ch* str) { return String(str, internal::StrLen(str)); } bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } //@} //! Write a raw JSON value. /*! For user to write a stringified JSON as a value. \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. \param length Length of the json. \param type Type of the root of json. */ bool RawValue(const Ch* json, size_t length, Type type) { Prefix(type); return EndValue(WriteRawValue(json, length)); } protected: //! Information for each nested level struct Level { Level(bool inArray_) : valueCount(0), inArray(inArray_) {} size_t valueCount; //!< number of values in this level bool inArray; //!< true if in array, otherwise in object }; static const size_t kDefaultLevelDepth = 32; bool WriteNull() { PutReserve(*os_, 4); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; } bool WriteBool(bool b) { if (b) { PutReserve(*os_, 4); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'r'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'e'); } else { PutReserve(*os_, 5); PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 's'); PutUnsafe(*os_, 'e'); } return true; } bool WriteInt(int i) { char buffer[11]; const char* end = internal::i32toa(i, buffer); PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteUint(unsigned u) { char buffer[10]; const char* end = internal::u32toa(u, buffer); PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteInt64(int64_t i64) { char buffer[21]; const char* end = internal::i64toa(i64, buffer); PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteUint64(uint64_t u64) { char buffer[20]; char* end = internal::u64toa(u64, buffer); PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteDouble(double d) { if (internal::Double(d).IsNanOrInf()) { if (!(writeFlags & kWriteNanAndInfFlag)) return false; if (internal::Double(d).IsNan()) { PutReserve(*os_, 3); PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); return true; } if (internal::Double(d).Sign()) { PutReserve(*os_, 9); PutUnsafe(*os_, '-'); } else PutReserve(*os_, 8); PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); return true; } char buffer[25]; char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteString(const Ch* str, SizeType length) { static const typename TargetEncoding::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; static const char escape[256] = { #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 //0 1 2 3 4 5 6 7 8 9 A B C D E F 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 Z16, Z16, // 30~4F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF #undef Z16 }; if (TargetEncoding::supportUnicode) PutReserve(*os_, 2 + length * 6); // "\uxxxx..." else PutReserve(*os_, 2 + length * 12); // "\uxxxx\uyyyy..." PutUnsafe(*os_, '\"'); GenericStringStream is(str); while (ScanWriteUnescapedString(is, length)) { const Ch c = is.Peek(); if (!TargetEncoding::supportUnicode && static_cast(c) >= 0x80) { // Unicode escaping unsigned codepoint; if (RAPIDJSON_UNLIKELY(!SourceEncoding::Decode(is, &codepoint))) return false; PutUnsafe(*os_, '\\'); PutUnsafe(*os_, 'u'); if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)) { PutUnsafe(*os_, hexDigits[(codepoint >> 12) & 15]); PutUnsafe(*os_, hexDigits[(codepoint >> 8) & 15]); PutUnsafe(*os_, hexDigits[(codepoint >> 4) & 15]); PutUnsafe(*os_, hexDigits[(codepoint ) & 15]); } else { RAPIDJSON_ASSERT(codepoint >= 0x010000 && codepoint <= 0x10FFFF); // Surrogate pair unsigned s = codepoint - 0x010000; unsigned lead = (s >> 10) + 0xD800; unsigned trail = (s & 0x3FF) + 0xDC00; PutUnsafe(*os_, hexDigits[(lead >> 12) & 15]); PutUnsafe(*os_, hexDigits[(lead >> 8) & 15]); PutUnsafe(*os_, hexDigits[(lead >> 4) & 15]); PutUnsafe(*os_, hexDigits[(lead ) & 15]); PutUnsafe(*os_, '\\'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, hexDigits[(trail >> 12) & 15]); PutUnsafe(*os_, hexDigits[(trail >> 8) & 15]); PutUnsafe(*os_, hexDigits[(trail >> 4) & 15]); PutUnsafe(*os_, hexDigits[(trail ) & 15]); } } else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { is.Take(); PutUnsafe(*os_, '\\'); PutUnsafe(*os_, static_cast(escape[static_cast(c)])); if (escape[static_cast(c)] == 'u') { PutUnsafe(*os_, '0'); PutUnsafe(*os_, '0'); PutUnsafe(*os_, hexDigits[static_cast(c) >> 4]); PutUnsafe(*os_, hexDigits[static_cast(c) & 0xF]); } } else if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? Transcoder::Validate(is, *os_) : Transcoder::TranscodeUnsafe(is, *os_)))) return false; } PutUnsafe(*os_, '\"'); return true; } bool ScanWriteUnescapedString(GenericStringStream& is, size_t length) { return RAPIDJSON_LIKELY(is.Tell() < length); } bool WriteStartObject() { os_->Put('{'); return true; } bool WriteEndObject() { os_->Put('}'); return true; } bool WriteStartArray() { os_->Put('['); return true; } bool WriteEndArray() { os_->Put(']'); return true; } bool WriteRawValue(const Ch* json, size_t length) { PutReserve(*os_, length); for (size_t i = 0; i < length; i++) { RAPIDJSON_ASSERT(json[i] != '\0'); PutUnsafe(*os_, json[i]); } return true; } void Prefix(Type type) { (void)type; if (RAPIDJSON_LIKELY(level_stack_.GetSize() != 0)) { // this value is not at root Level* level = level_stack_.template Top(); if (level->valueCount > 0) { if (level->inArray) os_->Put(','); // add comma if it is not the first element in array else // in object os_->Put((level->valueCount % 2 == 0) ? ',' : ':'); } if (!level->inArray && level->valueCount % 2 == 0) RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name level->valueCount++; } else { RAPIDJSON_ASSERT(!hasRoot_); // Should only has one and only one root. hasRoot_ = true; } } // Flush the value if it is the top level one. bool EndValue(bool ret) { if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text os_->Flush(); return ret; } OutputStream* os_; internal::Stack level_stack_; int maxDecimalPlaces_; bool hasRoot_; private: // Prohibit copy constructor & assignment operator. Writer(const Writer&); Writer& operator=(const Writer&); }; // Full specialization for StringStream to prevent memory copying template<> inline bool Writer::WriteInt(int i) { char *buffer = os_->Push(11); const char* end = internal::i32toa(i, buffer); os_->Pop(static_cast(11 - (end - buffer))); return true; } template<> inline bool Writer::WriteUint(unsigned u) { char *buffer = os_->Push(10); const char* end = internal::u32toa(u, buffer); os_->Pop(static_cast(10 - (end - buffer))); return true; } template<> inline bool Writer::WriteInt64(int64_t i64) { char *buffer = os_->Push(21); const char* end = internal::i64toa(i64, buffer); os_->Pop(static_cast(21 - (end - buffer))); return true; } template<> inline bool Writer::WriteUint64(uint64_t u) { char *buffer = os_->Push(20); const char* end = internal::u64toa(u, buffer); os_->Pop(static_cast(20 - (end - buffer))); return true; } template<> inline bool Writer::WriteDouble(double d) { if (internal::Double(d).IsNanOrInf()) { // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) return false; if (internal::Double(d).IsNan()) { PutReserve(*os_, 3); PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); return true; } if (internal::Double(d).Sign()) { PutReserve(*os_, 9); PutUnsafe(*os_, '-'); } else PutReserve(*os_, 8); PutUnsafe(*os_, 'I'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'f'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'i'); PutUnsafe(*os_, 't'); PutUnsafe(*os_, 'y'); return true; } char *buffer = os_->Push(25); char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); os_->Pop(static_cast(25 - (end - buffer))); return true; } #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) template<> inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { if (length < 16) return RAPIDJSON_LIKELY(is.Tell() < length); if (!RAPIDJSON_LIKELY(is.Tell() < length)) return false; const char* p = is.src_; const char* end = is.head_ + length; const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); if (nextAligned > end) return true; while (p != nextAligned) if (*p < 0x20 || *p == '\"' || *p == '\\') { is.src_ = p; return RAPIDJSON_LIKELY(is.Tell() < length); } else os_->PutUnsafe(*p++); // The rest of string using SIMD static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); for (; p != endAligned; p += 16) { const __m128i s = _mm_load_si128(reinterpret_cast(p)); const __m128i t1 = _mm_cmpeq_epi8(s, dq); const __m128i t2 = _mm_cmpeq_epi8(s, bs); const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); unsigned short r = static_cast(_mm_movemask_epi8(x)); if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped SizeType len; #ifdef _MSC_VER // Find the index of first escaped unsigned long offset; _BitScanForward(&offset, r); len = offset; #else len = static_cast(__builtin_ffs(r) - 1); #endif char* q = reinterpret_cast(os_->PushUnsafe(len)); for (size_t i = 0; i < len; i++) q[i] = p[i]; p += len; break; } _mm_storeu_si128(reinterpret_cast<__m128i *>(os_->PushUnsafe(16)), s); } is.src_ = p; return RAPIDJSON_LIKELY(is.Tell() < length); } #endif // defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) RAPIDJSON_NAMESPACE_END #ifdef _MSC_VER RAPIDJSON_DIAG_POP #endif #ifdef __clang__ RAPIDJSON_DIAG_POP #endif #endif // RAPIDJSON_RAPIDJSON_H_ ================================================ FILE: src/thread_data_base.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_THREAD_DATA_BASE_H #define DMKIT_THREAD_DATA_BASE_H #include #include namespace dmkit { class ThreadDataBase { public: ThreadDataBase() {} virtual ~ThreadDataBase() {} virtual void reset() { this->_log_id.clear(); this->_notice_log.clear(); } // Set a logid for current request, this logid will show in all logs for current request void set_log_id(const std::string& log_id) { this->_log_id = log_id; } const std::string get_log_id() { return this->_log_id; } // Add and save notice log inside thread data so that each request will log only one notice log void add_notice_log(const std::string& key, const std::string& value) { std::string log; log += key; log += "="; log += value; this->_notice_log.push_back(log); } // Get notice log as a string in the format "key1=value1 key2=value2" const std::string get_notice_log() { std::string log_str; for (auto v: this->_notice_log) { log_str += v; log_str += " "; } return log_str; } private: std::string _log_id; std::vector _notice_log; }; } // namespace dmkit #endif //DMKIT_THREAD_DATA_BASE_H ================================================ FILE: src/token_manager.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "token_manager.h" #include "file_watcher.h" #include "rapidjson.h" namespace dmkit { TokenManager::TokenManager() { this->_token_cache = new std::unordered_map(); } TokenManager::~TokenManager() { delete this->_token_cache; this->_token_cache = nullptr; FileWatcher::get_instance().unregister_file(this->_client_key_conf_path); this->_p_client_key_map.reset(); } int TokenManager::init(const char* dir_path, const char* conf_file) { std::string file_path; if (dir_path != nullptr) { file_path += dir_path; } if (!file_path.empty() && file_path[file_path.length() - 1] != '/') { file_path += '/'; } if (conf_file != nullptr) { file_path += conf_file; } this->_client_key_conf_path = file_path; ClientKeyMap* client_key_map = this->load_client_key_map(); if (client_key_map == nullptr) { APP_LOG(ERROR) << "Failed to init TokenManager, cannot load client key map"; return -1; } this->_p_client_key_map.reset(client_key_map); FileWatcher::get_instance().register_file( this->_client_key_conf_path, TokenManager::client_key_conf_change_callback, this, true); return 0; } int TokenManager::reload() { APP_LOG(TRACE) << "Reloading TokenManager..."; ClientKeyMap* client_key_map = this->load_client_key_map(); if (client_key_map == nullptr) { APP_LOG(ERROR) << "Failed to reload TokenManager, cannot load client key map"; return -1; } this->_p_client_key_map.reset(client_key_map); std::lock_guard lock(this->_token_cache_mutex); this->_token_cache->clear(); APP_LOG(TRACE) << "Reload finished."; return 0; } int TokenManager::client_key_conf_change_callback(void* param) { TokenManager* tm = (TokenManager*)param; return tm->reload(); } int TokenManager::get_access_token(const std::string bot_id, const RemoteServiceManager* remote_service_manager, std::string& access_token) { TokenValue token_value; if (this->get_token_from_cache(bot_id, token_value) == 0) { access_token = token_value.access_token; return 0; } std::shared_ptr p_client_key_map(this->_p_client_key_map); auto client_key_iter = p_client_key_map->find(bot_id); if (client_key_iter == p_client_key_map->end()) { return -1; } ClientKey client_key = client_key_iter->second; if (this->get_token_from_remote(client_key, remote_service_manager, token_value) != 0) { return -1; } this->update_token_cache(bot_id, token_value); access_token = token_value.access_token; return 0; } ClientKeyMap* TokenManager::load_client_key_map() { FILE* fp = fopen(this->_client_key_conf_path.c_str(), "r"); if (fp == nullptr) { APP_LOG(ERROR) << "Failed to open file " << this->_client_key_conf_path; return nullptr; } char read_buffer[1024]; rapidjson::FileReadStream is(fp, read_buffer, sizeof(read_buffer)); rapidjson::Document doc; doc.ParseStream(is); fclose(fp); if (doc.HasParseError() || !doc.IsObject()) { APP_LOG(ERROR) << "Failed to parse Token settings"; return nullptr; } ClientKeyMap* client_key_map = new ClientKeyMap(); for (rapidjson::Value::ConstMemberIterator bot_iter = doc.MemberBegin(); bot_iter != doc.MemberEnd(); ++bot_iter) { std::string bot_id = bot_iter->name.GetString(); if (!bot_iter->value.IsObject()) { APP_LOG(ERROR) << "Invalid token conf for " << bot_id << ", invalid format"; delete client_key_map; return nullptr; } const rapidjson::Value& val_token = bot_iter->value; if (!val_token.HasMember("api_key") || !val_token["api_key"].IsString()) { APP_LOG(ERROR) << "Invalid token conf for " << bot_id << ", missing api_key"; delete client_key_map; return nullptr; } if (!val_token.HasMember("secret_key") || !val_token["secret_key"].IsString()) { APP_LOG(ERROR) << "Invalid token conf for " << bot_id << ", missing secret_key"; delete client_key_map; return nullptr; } ClientKey client_key; client_key.api_key = val_token["api_key"].GetString(); client_key.secret_key = val_token["secret_key"].GetString(); (*client_key_map)[bot_id] = client_key; } return client_key_map; } int TokenManager::get_token_from_cache(const std::string bot_id, TokenValue& token_value) { time_t expire_guard = time(nullptr) + 60; std::lock_guard lock(this->_token_cache_mutex); auto token_iter = this->_token_cache->find(bot_id); if (token_iter == this->_token_cache->end()) { return -1; } TokenValue value = token_iter->second; if (value.expire_time < expire_guard) { return -1; } token_value = value; return 0; } int TokenManager::update_token_cache(const std::string bot_id, const TokenValue token_value) { std::lock_guard lock(this->_token_cache_mutex); (*this->_token_cache)[bot_id] = token_value; return 0; } int TokenManager::get_token_from_remote(const ClientKey client_key, const RemoteServiceManager* remote_service_manager, TokenValue& token_value) { std::string url = "/oauth/2.0/token?grant_type=client_credentials"; url += "&client_id=" + client_key.api_key + "&client_secret=" + client_key.secret_key; RemoteServiceParam rsp = { url, HTTP_METHOD_GET, "" }; RemoteServiceResult rsr; if (remote_service_manager->call("token_auth", rsp, rsr) != 0) { APP_LOG(ERROR) << "Failed to get authorization result"; return -1; } rapidjson::Document json; if (json.Parse(rsr.result.c_str()).HasParseError()) { APP_LOG(ERROR) << "Failed to parse authorization result to json"; return -1; } token_value.access_token = json["access_token"].GetString(); token_value.expire_time = time(0) + json["expires_in"].GetInt(); return 0; } } //namespace dmkit ================================================ FILE: src/token_manager.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include "app_log.h" #include "remote_service_manager.h" #ifndef DMKIT_TOKEN_MANAGER_H #define DMKIT_TOKEN_MANAGER_H namespace dmkit { struct ClientKey { std::string api_key; std::string secret_key; }; struct TokenValue { // access token std::string access_token; // timestamp when the access token expires std::time_t expire_time; }; // Type for client key map. typedef std::unordered_map ClientKeyMap; // A Manager class to manage and cache access token accessing unit bot api. class TokenManager { public: TokenManager(); ~TokenManager(); // Initialization with a json configuration file. int init(const char* dir_path, const char* conf_file); // Reload config. int reload(); // Callback when conf change. static int client_key_conf_change_callback(void* param); // Get access token with bot id. int get_access_token(const std::string bot_id, const RemoteServiceManager* remote_service_manager, std::string& access_token); private: int get_token_from_cache(const std::string bot_id, TokenValue& token_value); int update_token_cache(const std::string bot_id, const TokenValue token_value); int get_token_from_remote(const ClientKey client_key, const RemoteServiceManager* remote_service_manager, TokenValue& token_value); ClientKeyMap* load_client_key_map(); std::string _client_key_conf_path; std::shared_ptr _p_client_key_map; std::unordered_map* _token_cache; std::mutex _token_cache_mutex; }; } #endif ================================================ FILE: src/user_function/demo.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "demo.h" #include "../utils.h" namespace dmkit { namespace user_function { namespace demo { // These functions are for demo purpose which returns mock data. // In real application, these might need to invoke a remote service call. // Http GET/POST calls are already included as shared function. int get_cellular_data_usage(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() < 1) { return -1; } std::vector elements; utils::split(args[0], '-', elements); if (elements.size() > 1 && elements[1] == "01") { result = "0"; } else { result = "2"; } return 0; } int get_cellular_data_left(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() < 1) { return -1; } std::vector elements; utils::split(args[0], '-', elements); if (elements.size() > 1 && elements[1] == "01") { result = "2"; } else { result = "0"; } return 0; } int get_package_options(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() < 1) { return -1; } if (args[0] == "省内流量包") { result = "10元100M，20元300M"; } else if (args[0] == "全国流量包") { result = "10元100M，50元1G"; } else { result = "20元300M，50元1G"; } return 0; } } // namespace demo } // namespace user_function } // namespace dmkit ================================================ FILE: src/user_function/demo.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_USER_FUNCTION_DEMO_H #define DMKIT_USER_FUNCTION_DEMO_H #include #include #include "../request_context.h" namespace dmkit { namespace user_function { namespace demo { // Functions used in our demo scenario int get_cellular_data_usage(const std::vector& args, const RequestContext& context, std::string& result); int get_cellular_data_left(const std::vector& args, const RequestContext& context, std::string& result); int get_package_options(const std::vector& args, const RequestContext& context, std::string& result); } // namespace demo } // namespace user_function } // namespace dmkit #endif //DMKIT_USER_FUNCTION_DEMO_H ================================================ FILE: src/user_function/shared.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "shared.h" #include #include #include #include #include #include #include #include #include "../app_log.h" #include "../rapidjson.h" #include "../utils.h" namespace dmkit { namespace user_function { // All user functions share the same signature, // arg: // args: arguements supplied in configuration // context: current request context // result: value assigned to parameter value in configuration // return: // 0 if function process success // -1 if function process fail // Get value from a JSON string with supplied path. // args[0]: JSON string // args[1]: path to search for, split with a dot sign int json_get_value(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; result = ""; if (args.size() < 2) { return -1; } std::string data = args[0]; std::string search = args[1]; if (search.empty()) { return -1; } rapidjson::Document doc; if (doc.Parse(data.c_str()).HasParseError() || (!doc.IsObject() && !doc.IsArray())) { APP_LOG(WARNING) << "Failed to load json data: " << data; return -1; } std::size_t last_pos = 0; const rapidjson::Value* value = &doc; while (last_pos < search.length()) { std::size_t pos = search.find('.', last_pos); if (pos == std::string::npos) { pos = search.length(); } std::string key = search.substr(last_pos, pos - last_pos); last_pos = pos + 1; utils::trim(key); if (key.empty()) { return -1; } if (value->IsObject()) { const rapidjson::Value::ConstMemberIterator miter = value->FindMember(key.c_str()); if (miter == value->MemberEnd()) { return -1; } value = &(miter->value); continue; } else if (value->IsArray()) { if (key.empty()) { return -1; } unsigned index = atoi(key.c_str()); if (index == 0 && key[0] != '0') { return -1; } if (value->Size() <= index) { return -1; } value = &((*value)[index]); } else { return -1; } } if (value->IsNull()) { return -1; } if (value->IsUint()) { unsigned num = value->GetUint(); result = std::to_string(num); } else if (value->IsInt()) { int num = value->GetInt(); result = std::to_string(num); } else if (value->IsUint64()) { unsigned long num = value->GetUint64(); result = std::to_string(num); } else if (value->IsInt64()) { long num = value->GetInt64(); result = std::to_string(num); } else if (value->IsDouble()) { double num = value->GetDouble(); result = std::to_string(num); } else if (value->IsString()) { result = value->GetString(); } else { rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); value->Accept(writer); result = buffer.GetString(); } return 0; } // String replacement. // args[0]: string // args[1]: sub string to be replaced // args[2]: a new string to replace with int replace(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() != 3) { return -1; } std::string str = args[0]; std::string old_substr = args[1]; std::string new_substr = args[2]; size_t pos = str.find(old_substr); if (pos != std::string::npos) { str.replace(pos, old_substr.length(), new_substr); } result = str; return 0; } // split string with delimiter and choose one element from resulting list // args[0]: string // args[1]: delimeter to split // args[2]: "random" to choose randomly or a number indicate the index to choose int split_and_choose(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() != 3) { return -1; } std::string str = args[0]; std::string delimiter = args[1]; std::string choose = args[2]; std::vector elements; if (delimiter.length() != 1) { APP_LOG(WARNING) << "Delimiter only support one character"; return -1; } ::dmkit::utils::split(str, delimiter[0], elements); if (elements.empty()) { return -1; } int size = elements.size(); int index = 0; if (choose == "random") { index = std::time(nullptr) % size; } else if (!::dmkit::utils::try_atoi(choose, index) || index >= size) { return -1; } result = elements[index]; return 0; } // Add all integer numbers supplied in args. int number_add(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() < 1) { return -1; } long res = 0; for (auto & arg : args) { if (arg.empty()) { return -1; } int num = atoi(arg.c_str()); if (num == 0 && arg[0] != '0') { return -1; } res += num; } result = std::to_string(res); return 0; } // Multiply all float numbers supplied in args. int float_mul(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() < 1) { return -1; } double res = 1; for (auto & arg : args) { if (arg.empty()) { return -1; } double num = atof(arg.c_str()); const double EPSILON = 1e-9; if (fabs(num) < EPSILON && arg[0] != '0') { return -1; } res *= num; } result = std::to_string(res); return 0; } // Return args[2] if args[0] equals to args[1], // otherwise return args[3] int choose_if_equal(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() < 4) { return -1; } if (args[0] == args[1]) { result = args[2]; return 0; } result = args[3]; return 0; } // Url encode args[0] int url_encode(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() < 1) { return -1; } if (args[0].empty()) { result = ""; return 0; } CURL *curl = curl_easy_init(); if (!curl) { return -1; } char *output = curl_easy_escape(curl, args[0].c_str(), args[0].length()); if (!output) { curl_easy_cleanup(curl); return -1; } result.assign(output); curl_free(output); curl_easy_cleanup(curl); return 0; } // Make a http get request. // args[0]: service name // args[1]: url int service_http_get(const std::vector& args, const RequestContext& context, std::string& result) { if (args.size() < 2) { return -1; } const RemoteServiceManager* rsm = context.remote_service_manager(); RemoteServiceParam rsm_param = { args[1], HTTP_METHOD_GET, "", }; RemoteServiceResult rsm_result; if (rsm->call(args[0], rsm_param, rsm_result) != 0) { return -1; } result = rsm_result.result; return 0; } // Make a http post request. // args[0]: service name // args[1]: url // args[2], ...: these arguments will be concatenated and used as the http post body. int service_http_post(const std::vector& args, const RequestContext& context, std::string& result) { if (args.size() < 2) { return -1; } const RemoteServiceManager* rsm = context.remote_service_manager(); std::string post_data = ""; for (size_t i = 2; i < args.size(); i++) { if (i == 2) { post_data = args[i]; } else { post_data = post_data + "," + args[i]; } } APP_LOG(TRACE) << "post data request:" << post_data; RemoteServiceParam rsm_param = { args[1], HTTP_METHOD_POST, post_data, }; RemoteServiceResult rsm_result; if (rsm->call(args[0], rsm_param, rsm_result) != 0) { return -1; } result = rsm_result.result; APP_LOG(TRACE) << "post data result:" << result; return 0; } // Get strftime result for current time // args[0]: strftime used format string int now_strftime(const std::vector& args, const RequestContext& context, std::string& result) { (void)context; if (args.size() < 1) { return -1; } time_t t = time(0); struct tm now; localtime_r(&t, &now); char buffer [80]; if (strftime(buffer, 80, args[0].c_str(), &now) == 0) { APP_LOG(WARNING) << "strftime returns 0!"; return -1; } result = buffer; return 0; } } // namespace user_function } // namespace dmkit ================================================ FILE: src/user_function/shared.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_USER_FUNCTION_SHARED_H #define DMKIT_USER_FUNCTION_SHARED_H #include #include #include "../request_context.h" namespace dmkit { namespace user_function { // JSON Operations int json_get_value(const std::vector& args, const RequestContext& context, std::string& result); // String Operations int replace(const std::vector& args, const RequestContext& context, std::string& result); int split_and_choose(const std::vector& args, const RequestContext& context, std::string& result); // Arithmetic Operations int number_add(const std::vector& args, const RequestContext& context, std::string& result); int float_mul(const std::vector& args, const RequestContext& context, std::string& result); // Logical Operations int choose_if_equal(const std::vector& args, const RequestContext& context, std::string& result); // Network Operations int url_encode(const std::vector& args, const RequestContext& context, std::string& result); int service_http_get(const std::vector& args, const RequestContext& context, std::string& result); int service_http_post(const std::vector& args, const RequestContext& context, std::string& result); // Time Related int now_strftime(const std::vector& args, const RequestContext& context, std::string& result); } // namespace user_function } // namespace dmkit #endif //DMKIT_USER_FUNCTION_SHARED_H ================================================ FILE: src/user_function_manager.cpp ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include "app_log.h" #include "user_function_manager.h" #include "user_function/shared.h" #include "user_function/demo.h" namespace dmkit { UserFunctionManager::UserFunctionManager() { this->_user_function_map = new std::unordered_map(); this->_init = false; } UserFunctionManager::~UserFunctionManager() { delete this->_user_function_map; this->_user_function_map = nullptr; if (this->_init) { curl_global_cleanup(); this->_init = false; } } int UserFunctionManager::init() { // All user functions should be registered here. // Shared functions. (*_user_function_map)["json_get_value"] = user_function::json_get_value; (*_user_function_map)["replace"] = user_function::replace; (*_user_function_map)["split_and_choose"] = user_function::split_and_choose; (*_user_function_map)["number_add"] = user_function::number_add; (*_user_function_map)["float_mul"] = user_function::float_mul; (*_user_function_map)["choose_if_equal"] = user_function::choose_if_equal; (*_user_function_map)["url_encode"] = user_function::url_encode; (*_user_function_map)["service_http_get"] = user_function::service_http_get; (*_user_function_map)["service_http_post"] = user_function::service_http_post; (*_user_function_map)["now_strftime"] = user_function::now_strftime; // Scenario specific functions. (*_user_function_map)["demo_get_cellular_data_usage"] = user_function::demo::get_cellular_data_usage; (*_user_function_map)["demo_get_cellular_data_left"] = user_function::demo::get_cellular_data_left; (*_user_function_map)["demo_get_package_options"] = user_function::demo::get_package_options; // Curl init is required for url_encode function. if (curl_global_init(CURL_GLOBAL_DEFAULT) != 0) { APP_LOG(WARNING) << "curl_global_init returns non-zeros"; return -1; } this->_init = true; return 0; } int UserFunctionManager::call_user_function(const std::string& func_name, const std::vector& args, const RequestContext& context, std::string& result) { auto find_res = this->_user_function_map->find(func_name); if (find_res == this->_user_function_map->end()) { APP_LOG(WARNING) << "call to undefined user function: " << func_name; return -1; } UserFunctionPtr func_ptr = find_res->second; int res = -1; try { res = (*func_ptr)(args, context, result); } catch (const char* msg) { APP_LOG(WARNING) << "Exception calling user function [" << func_name << "], exception: " << msg; res = -1; } return res; } } // namespace dmkit ================================================ FILE: src/user_function_manager.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_USER_FUNCTION_MANAGER_H #define DMKIT_USER_FUNCTION_MANAGER_H #include #include #include #include "request_context.h" namespace dmkit { typedef int (*UserFunctionPtr)(const std::vector& args, const RequestContext& context, std::string& result); class UserFunctionManager { public: UserFunctionManager(); ~UserFunctionManager(); int init(); int call_user_function(const std::string& func_name, const std::vector& args, const RequestContext& context, std::string& result); private: std::unordered_map* _user_function_map; bool _init; }; } // namespace dmkit #endif //DMKIT_USER_FUNCTION_MANAGER_H ================================================ FILE: src/utils.h ================================================ // Copyright (c) 2018 Baidu, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef DMKIT_UTILS_H #define DMKIT_UTILS_H #include #include #include "app_log.h" #include "rapidjson.h" namespace dmkit { namespace utils { static inline void ltrim(std::string &s) { s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { return !std::isspace(ch); })); } static inline void rtrim(std::string &s) { s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end()); } static inline void trim(std::string &s) { ltrim(s); rtrim(s); } static inline bool try_atoi(const std::string& str, int& value) { if (str.empty()) { return false; } value = atoi(str.c_str()); if (value == 0 && str[0] != '0') { return false; } if (value < 0 && str[0] != '-') { return false; } return true; } static inline bool try_atof(const std::string& str, double& value) { if (str.empty()) { return false; } value = atof(str.c_str()); const double EPSILON = 1e-9; if (fabs(value) < EPSILON && str[0] != '0') { return false; } return true; } static inline bool split(const std::string& str, const char sep, std::vector& elems) { elems.clear(); if (str.empty()) { return false; } std::istringstream ss(str); std::string item; while (std::getline(ss, item, sep)) { elems.push_back(item); } return true; } static inline std::string json_to_string(const rapidjson::Value& value) { rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); value.Accept(writer); return buffer.GetString(); } } // namespace utils } // namespace dmkit #endif //DMKIT_UTILS_H ================================================ FILE: tools/bot_emulator.py ================================================ # -*- coding: utf-8 -*- # # Copyright (c) 2018 Baidu, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ A simple console program to emulate a bot for user to interact with """ import json import os import random import requests import sys def main(bot_id, token): """ Bot emulator Read user input and get bot response """ # Url of dmkit, change this if you are runnning dmkit in a different ip/port. url = "http://127.0.0.1:8010/search?access_token=" + token json_header = {'Content-Type': 'application/json'} user_id = str(random.randint(1, 100000)) session = "" while True: print "-----------------------------------------------------------------" user_input = raw_input("input:\n") user_input = user_input.strip() if not user_input: continue logid = str(random.randint(100000,999999)) payload = { "version": "2.0", "bot_id": bot_id, "log_id": logid, "request": { "user_id": user_id, "query": user_input, "query_info": { "type": "TEXT", "source": "ASR", "asr_candidates": [] }, "updates": "", "client_session": "{\"client_results\":\"\", \"candidate_options\":[]}", "bernard_level": 1 }, "bot_session": session } obj = requests.post(url, data=json.dumps(payload), headers=json_header).json() print "BOT:" # Not a valid response. if obj['error_code'] != 0: print "ERROR: %s" % obj['error_msg'] continue session = obj['result']['bot_session'] response = obj['result']['response'] action = response['action_list'][0] # For clarify or failed responses which dmkit does not take actions. if action['type'] != 'event': print action['say'] continue custom_reply = json.loads(action['custom_reply']) if custom_reply['event_name'] != 'DM_RESULT': print 'ERROR:unknown event name %s' % custom_reply['event_name'] continue # This is a dmkit response dm_result = json.loads(custom_reply['result']) for item in dm_result['result']: if item['type'] == 'tts': print item['value'] if __name__ == "__main__": if len(sys.argv) < 2: print "Usage: %s %s %s" % (sys.argv[0], "[bot_id]", "[access_token]") else: main(sys.argv[1], sys.argv[2]) ================================================ FILE: tools/mock_api_server.py ================================================ # -*- coding: utf-8 -*- # # Copyright (c) 2018 Baidu, Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ A mock server to provide api which DMKit can access to retrieve resources """ from flask import Flask from flask import request app = Flask(__name__) @app.route("/hotel/search") def hotel_search(): location = request.args.get('location') return "如家酒店、四季酒店和香格里拉酒店" @app.route("/hotel/book") def hotel_book(): time = request.args.get('time') hotel = request.args.get('hotel') room_type = request.args.get('room_type') return "OK"