Repository: alibaba/sentinel-cpp Branch: master Commit: 36871db9cc0c Files: 305 Total size: 1.1 MB Directory structure: gitextract_c4eu14oz/ ├── .bazelrc ├── .clang-format ├── .github/ │ ├── ISSUE_TEMPLATE.md │ ├── PULL_REQUEST_TEMPLATE.md │ └── workflows/ │ └── main.yml ├── .gitignore ├── BUILD ├── CMakeLists.txt ├── LICENSE ├── README.md ├── WORKSPACE ├── bazel/ │ ├── BUILD │ ├── copts.bzl │ ├── fmtlib.BUILD │ ├── osx.tbb.BUILD │ ├── spdlog.BUILD │ ├── tbb.BUILD │ └── third_party_repositories.bzl ├── cmake/ │ ├── common.cmake │ └── third_party.cmake ├── examples/ │ ├── CMakeLists.txt │ ├── abseil/ │ │ ├── BUILD │ │ └── abseil_string.cc │ ├── benchmark/ │ │ ├── BUILD │ │ └── benchamrk_test.cc │ ├── cache/ │ │ ├── BUILD │ │ └── cache_test.cc │ ├── fmt/ │ │ ├── BUILD │ │ └── fmt_test.cc │ ├── gtest/ │ │ ├── BUILD │ │ └── hello_test.cc │ ├── json/ │ │ ├── BUILD │ │ └── nlohmann_json_example.cc │ ├── libevent/ │ │ ├── BUILD │ │ └── libevent_echosrv1.c │ ├── log/ │ │ ├── BUILD │ │ └── log_test.cc │ ├── sentinel-cpp/ │ │ ├── BUILD │ │ ├── basic_concurrency_limit.cc │ │ ├── basic_param_limit.cc │ │ ├── basic_qps_limit.cc │ │ └── basic_system_limit.cc │ └── tbb/ │ ├── BUILD │ └── tbb_test.cc ├── format.sh ├── sentinel-core/ │ ├── BUILD │ ├── circuitbreaker/ │ │ ├── BUILD │ │ ├── circuit_breaker.cc │ │ ├── circuit_breaker.h │ │ ├── error_circuit_breaker.cc │ │ ├── error_circuit_breaker.h │ │ ├── rt_circuit_breaker.cc │ │ ├── rt_circuit_breaker.h │ │ ├── rule.cc │ │ ├── rule.h │ │ ├── rule_manager.cc │ │ ├── rule_manager.h │ │ ├── slot.cc │ │ ├── slot.h │ │ └── slot_test.cc │ ├── common/ │ │ ├── BUILD │ │ ├── constants.h │ │ ├── entry.h │ │ ├── entry_context.cc │ │ ├── entry_context.h │ │ ├── entry_node.h │ │ ├── entry_result.cc │ │ ├── entry_result.h │ │ ├── entry_type.h │ │ ├── global_status.cc │ │ ├── global_status.h │ │ ├── resource_wrapper.h │ │ ├── rule.h │ │ ├── string_resource_wrapper.h │ │ ├── tracer.cc │ │ ├── tracer.h │ │ └── tracer_test.cc │ ├── config/ │ │ ├── BUILD │ │ ├── config_constants.h │ │ ├── local_config.cc │ │ ├── local_config.h │ │ └── local_config_test.cc │ ├── flow/ │ │ ├── BUILD │ │ ├── default_traffic_shaping_calculator.cc │ │ ├── default_traffic_shaping_calculator.h │ │ ├── default_traffic_shaping_checker.cc │ │ ├── default_traffic_shaping_checker.h │ │ ├── flow_rule.cc │ │ ├── flow_rule.h │ │ ├── flow_rule_checker.cc │ │ ├── flow_rule_checker.h │ │ ├── flow_rule_constants.h │ │ ├── flow_rule_manager.cc │ │ ├── flow_rule_manager.h │ │ ├── flow_slot.cc │ │ ├── flow_slot.h │ │ ├── flow_slot_test.cc │ │ ├── throttling_traffic_shaping_checker.cc │ │ ├── throttling_traffic_shaping_checker.h │ │ ├── traffic_shaping_calculator.h │ │ ├── traffic_shaping_checker.h │ │ ├── traffic_shaping_controller.cc │ │ ├── traffic_shaping_controller.h │ │ └── traffic_shaping_controller_test.cc │ ├── init/ │ │ ├── BUILD │ │ ├── init_target.h │ │ ├── init_target_registry.h │ │ └── init_target_registry_test.cc │ ├── log/ │ │ ├── BUILD │ │ ├── block/ │ │ │ ├── BUILD │ │ │ ├── block_log_task.cc │ │ │ ├── block_log_task.h │ │ │ └── block_log_task_test.cc │ │ ├── log_base.cc │ │ ├── log_base.h │ │ ├── log_base_test.cc │ │ ├── logger.cc │ │ ├── logger.h │ │ ├── logger_test.cc │ │ └── metric/ │ │ ├── BUILD │ │ ├── metric_log_task.cc │ │ ├── metric_log_task.h │ │ ├── metric_reader.cc │ │ ├── metric_reader.h │ │ ├── metric_reader_test.cc │ │ ├── metric_searcher.cc │ │ ├── metric_searcher.h │ │ ├── metric_searcher_test.cc │ │ ├── metric_test_utils.h │ │ ├── metric_writer.cc │ │ ├── metric_writer.h │ │ └── metric_writer_test.cc │ ├── param/ │ │ ├── BUILD │ │ ├── param_flow_item.cc │ │ ├── param_flow_item.h │ │ ├── param_flow_rule.cc │ │ ├── param_flow_rule.h │ │ ├── param_flow_rule_checker.cc │ │ ├── param_flow_rule_checker.h │ │ ├── param_flow_rule_constants.h │ │ ├── param_flow_rule_manager.cc │ │ ├── param_flow_rule_manager.h │ │ ├── param_flow_slot.cc │ │ ├── param_flow_slot.h │ │ ├── param_flow_slot_test.cc │ │ └── statistic/ │ │ ├── BUILD │ │ ├── any_cmp.cc │ │ ├── any_cmp.h │ │ ├── any_cmp_test.cc │ │ ├── lru_cache.h │ │ ├── param_bucket.cc │ │ ├── param_bucket.h │ │ ├── param_event.h │ │ ├── param_leap_array.cc │ │ ├── param_leap_array.h │ │ ├── param_leap_array_key.h │ │ ├── param_metric.cc │ │ ├── param_metric.h │ │ ├── param_metric_test.cc │ │ └── scalable_cache.h │ ├── property/ │ │ ├── BUILD │ │ ├── dynamic_sentinel_property.h │ │ ├── dynamic_sentinel_property_test.cc │ │ ├── property_listener.h │ │ └── sentinel_property.h │ ├── public/ │ │ ├── BUILD │ │ ├── sph_u.h │ │ └── sph_u_test.cc │ ├── slot/ │ │ ├── BUILD │ │ ├── base/ │ │ │ ├── BUILD │ │ │ ├── default_slot_chain_impl.cc │ │ │ ├── default_slot_chain_impl.h │ │ │ ├── default_slot_chain_impl_test.cc │ │ │ ├── rule_checker_slot.h │ │ │ ├── slot.h │ │ │ ├── slot_base.h │ │ │ ├── slot_chain.h │ │ │ ├── stats_slot.h │ │ │ ├── token_result.cc │ │ │ └── token_result.h │ │ ├── global_slot_chain.cc │ │ ├── global_slot_chain.h │ │ ├── log_slot.cc │ │ ├── log_slot.h │ │ ├── resource_node_builder_slot.cc │ │ ├── resource_node_builder_slot.h │ │ ├── resource_node_builder_slot_test.cc │ │ ├── statistic_slot.cc │ │ ├── statistic_slot.h │ │ └── statistic_slot_test.cc │ ├── statistic/ │ │ ├── base/ │ │ │ ├── BUILD │ │ │ ├── bucket_leap_array.cc │ │ │ ├── bucket_leap_array.h │ │ │ ├── bucket_leap_array_test.cc │ │ │ ├── leap_array.h │ │ │ ├── metric.h │ │ │ ├── metric_bucket.cc │ │ │ ├── metric_bucket.h │ │ │ ├── metric_event.h │ │ │ ├── metric_item.cc │ │ │ ├── metric_item.h │ │ │ ├── metric_item_test.cc │ │ │ ├── sliding_window_metric.cc │ │ │ ├── sliding_window_metric.h │ │ │ ├── sliding_window_metric_test.cc │ │ │ ├── stat_config.h │ │ │ ├── stat_config_manager.cc │ │ │ ├── stat_config_manager.h │ │ │ └── window_wrap.h │ │ └── node/ │ │ ├── BUILD │ │ ├── cluster_node.cc │ │ ├── cluster_node.h │ │ ├── node.h │ │ ├── resource_node_storage.cc │ │ ├── resource_node_storage.h │ │ ├── statistic_node.cc │ │ └── statistic_node.h │ ├── system/ │ │ ├── BUILD │ │ ├── system_rule.cc │ │ ├── system_rule.h │ │ ├── system_rule_manager.cc │ │ ├── system_rule_manager.h │ │ ├── system_slot.cc │ │ ├── system_slot.h │ │ ├── system_slot_mock.h │ │ ├── system_slot_test.cc │ │ ├── system_status_listener.cc │ │ ├── system_status_listener.h │ │ └── system_status_listener_test.cc │ ├── test/ │ │ └── mock/ │ │ ├── common/ │ │ │ ├── BUILD │ │ │ ├── mock.cc │ │ │ └── mock.h │ │ ├── flow/ │ │ │ ├── BUILD │ │ │ ├── mock.cc │ │ │ └── mock.h │ │ ├── init/ │ │ │ ├── BUILD │ │ │ └── mock.h │ │ ├── property/ │ │ │ ├── BUILD │ │ │ ├── mock.cc │ │ │ └── mock.h │ │ ├── slot/ │ │ │ ├── BUILD │ │ │ ├── mock.cc │ │ │ └── mock.h │ │ └── statistic/ │ │ ├── base/ │ │ │ ├── BUILD │ │ │ ├── mock.cc │ │ │ └── mock.h │ │ └── node/ │ │ ├── BUILD │ │ ├── mock.cc │ │ └── mock.h │ ├── transport/ │ │ ├── BUILD │ │ ├── command/ │ │ │ ├── BUILD │ │ │ ├── command_handler.h │ │ │ ├── command_request.cc │ │ │ ├── command_request.h │ │ │ ├── command_request_test.cc │ │ │ ├── command_response.h │ │ │ ├── handler/ │ │ │ │ ├── BUILD │ │ │ │ ├── fetch_cluster_node_handler.cc │ │ │ │ ├── fetch_cluster_node_handler.h │ │ │ │ ├── fetch_cluster_node_handler_test.cc │ │ │ │ ├── fetch_metric_log_handler.cc │ │ │ │ ├── fetch_metric_log_handler.h │ │ │ │ ├── get_switch_status_handler.cc │ │ │ │ ├── get_switch_status_handler.h │ │ │ │ ├── set_switch_status_handler.cc │ │ │ │ ├── set_switch_status_handler.h │ │ │ │ ├── set_switch_status_handler_test.cc │ │ │ │ ├── version_handler.cc │ │ │ │ ├── version_handler.h │ │ │ │ └── vo/ │ │ │ │ ├── BUILD │ │ │ │ ├── statistic_node_vo.cc │ │ │ │ └── statistic_node_vo.h │ │ │ ├── http_command_center.cc │ │ │ ├── http_command_center.h │ │ │ ├── http_command_utils.cc │ │ │ ├── http_command_utils.h │ │ │ ├── http_server.cc │ │ │ ├── http_server.h │ │ │ ├── http_server_init_target.cc │ │ │ └── http_server_init_target.h │ │ ├── common/ │ │ │ ├── BUILD │ │ │ ├── event_loop_thread.cc │ │ │ ├── event_loop_thread.h │ │ │ └── event_loop_thread_test.cc │ │ └── constants.h │ └── utils/ │ ├── BUILD │ ├── file_utils.cc │ ├── file_utils.h │ ├── macros.h │ ├── time_utils.cc │ ├── time_utils.h │ ├── utils.cc │ └── utils.h ├── sentinel-datasource-extension/ │ ├── datasource/ │ │ ├── BUILD │ │ ├── abstract_readable_data_source.h │ │ ├── abstract_readable_data_source_unittests.cc │ │ ├── converter.h │ │ └── readable_data_source.h │ └── test/ │ └── mock/ │ └── datasource/ │ ├── BUILD │ ├── mock.cc │ └── mock.h ├── tests/ │ ├── BUILD │ └── tsan-flow.cc └── third_party/ └── nlohmann/ ├── BUILD └── json.hpp ================================================ FILE CONTENTS ================================================ ================================================ FILE: .bazelrc ================================================ # Clang TSAN build:clang-tsan --copt -fsanitize=thread build:clang-tsan --linkopt -fsanitize=thread ================================================ FILE: .clang-format ================================================ # Use the Google style in this project. BasedOnStyle: Google ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ ## Issue Description Type: *bug report* or *feature request* ### Describe what happened (or what feature you want) ### Describe what you expected to happen ### How to reproduce it (as minimally and precisely as possible) 1. 2. 3. ### Tell us your environment ### Anything else we need to know? ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ ### Describe what this PR does / why we need it ### Does this pull request fix one issue? ### Describe how you did it ### Describe how to verify it ### Special notes for reviews ================================================ FILE: .github/workflows/main.yml ================================================ name: CI on: push: branches: master pull_request: branches: "*" jobs: build: runs-on: ${{ matrix.os }} strategy: matrix: java: [8] os: [ubuntu-18.04] steps: - uses: actions/checkout@v2 - name: Install bazel run: | wget https://github.com/bazelbuild/bazel/releases/download/3.7.0/bazel_3.7.0-linux-x86_64.deb sudo dpkg -i bazel_3.7.0-linux-x86_64.deb - name: Install clang8 run: | echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" | sudo tee -a /etc/apt/sources.list echo "deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" | sudo tee -a /etc/apt/sources.list wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo apt-get update sudo apt-get -y install clang-8 clang-format-8 - name: Clang format checker run: | bash format.sh - name: Library build run: | bazel build //sentinel-core/... && bazel build //sentinel-datasource-extension/... - name: Sentinel core unit tests run: | bazel test --test_filter=*-ParamMetricTest.TestOperateMetric --test_output=errors --strategy=TestRunner=standalone //sentinel-core/... - name: Sentinel datasource extension tests run: | bazel build //sentinel-core/... && bazel build //sentinel-datasource-extension/... - name: tsan for flow control run: | bazel build -c dbg --config=clang-tsan //tests/... && ./bazel-bin/tests/tsan-flow ================================================ FILE: .gitignore ================================================ /bazel-* BROWSE /build /build_* .cache /ci/bazel-* /ci/prebuilt/thirdparty /ci/prebuilt/thirdparty_build compile_commands.json cscope.* .deps /docs/landing_source/.bundle /generated *.pyc **/pyformat SOURCE_VERSION *.sw* tags TAGS /test/coverage/BUILD /tools/.aspell.en.pws .vimrc .vs .vscode .DS_Store .gdb_history ================================================ FILE: BUILD ================================================ load("@rules_foreign_cc//tools/build_defs:configure.bzl", "configure_make") # load("@rules_foreign_cc//tools/build_defs:make.bzl", "make") configure_make( name = "libevent", visibility = ["//visibility:public"], configure_options = [ "--enable-shared=no", "--disable-libevent-regress", "--disable-openssl", ], lib_source = "@com_github_libevent//:all", out_lib_dir = "lib", ) # make( # name = "com_github_libtbb", # visibility = ["//visibility:public"], # out_lib_dir = "lib", # lib_source = "@com_github_libtbb//:all", # ) ================================================ FILE: CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.14) project(sentinel-cpp VERSION 1.0.1 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 14) set(SENTINEL_ROOT_DIR "${CMAKE_SOURCE_DIR}") set(SENTINEL_CORE_ROOT_DIR "${CMAKE_SOURCE_DIR}/sentinel-core") include( cmake/common.cmake ) ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner]. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ Sentinel Logo [![Build Status](https://travis-ci.org/alibaba/sentinel-cpp.svg?branch=master)](https://travis-ci.org/alibaba/sentinel-cpp) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Gitter](https://badges.gitter.im/alibaba/Sentinel.svg)](https://gitter.im/alibaba/Sentinel) # Sentinel: The Sentinel of Your Microservices ## Build 1. Install the latest version of [Bazel](https://bazel.build/versions/master/docs/install.html) in your environment. 2. Build the project. ```sh bazel build -c opt //... ``` 3. Run the [QPS limiting demo](https://github.com/alibaba/sentinel-cpp/blob/master/examples/sentinel-cpp/basic_qps_limit.cc): ```sh CSP_SENTINEL_APP_NAME=my-demo ./bazel-bin/examples/sentinel-cpp/sentinel_basic_qps_limit ``` ================================================ FILE: WORKSPACE ================================================ # Copyright 2019 Alibaba Inc. All rights reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. workspace(name = "com_alibaba_sentinel") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Rule repository http_archive( name = "rules_foreign_cc", strip_prefix = "rules_foreign_cc-master", url = "https://github.com/bazelbuild/rules_foreign_cc/archive/master.zip", ) load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") rules_foreign_cc_dependencies() # abseil-cpp http_archive( name = "com_google_absl", urls = ["https://github.com/abseil/abseil-cpp/archive/61c9bf3e3e1c28a4aa6d7f1be4b37fd473bb5529.tar.gz"], # 2019-06-05 strip_prefix = "abseil-cpp-61c9bf3e3e1c28a4aa6d7f1be4b37fd473bb5529", sha256 = "7ddf863ddced6fa5bf7304103f9c7aa619c20a2fcf84475512c8d3834b9d14fa", ) # Google Test http_archive( name = "com_google_googletest", urls = ["https://github.com/google/googletest/archive/8b6d3f9c4a774bef3081195d422993323b6bb2e0.zip"], # 2019-03-05 strip_prefix = "googletest-8b6d3f9c4a774bef3081195d422993323b6bb2e0", sha256 = "d21ba93d7f193a9a0ab80b96e8890d520b25704a6fac976fe9da81fffb3392e3", ) # Google Benchmark http_archive( name = "com_google_benchmark", urls = ["https://github.com/google/benchmark/archive/505be96ab23056580a3a2315abba048f4428b04e.tar.gz"], strip_prefix = "benchmark-505be96ab23056580a3a2315abba048f4428b04e", sha256 = "0de43b6eaddd356f1d6cd164f73f37faf2f6c96fd684e1f7ea543ce49c1d144e", ) load("//bazel:third_party_repositories.bzl", "include_third_party_repositories") include_third_party_repositories() ================================================ FILE: bazel/BUILD ================================================ licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:public"]) config_setting( name = "llvm_compiler", values = { "compiler": "llvm", }, ) config_setting( name = "windows", values = { "cpu": "x64_windows", }, ) config_setting( name = "is_osx", define_values = {"os":"osx"}, ) ================================================ FILE: bazel/copts.bzl ================================================ """ Flags specified here must not impact ABI. Code compiled with and without these opts will be linked together, and in some cases headers compiled with and without these options will be part of the same program. We use the same flags as absl. """ load( "@com_google_absl//absl:copts/GENERATED_copts.bzl", "ABSL_GCC_EXCEPTIONS_FLAGS", "ABSL_GCC_FLAGS", "ABSL_GCC_TEST_FLAGS", "ABSL_LLVM_EXCEPTIONS_FLAGS", "ABSL_LLVM_FLAGS", "ABSL_LLVM_TEST_FLAGS", "ABSL_MSVC_EXCEPTIONS_FLAGS", "ABSL_MSVC_FLAGS", "ABSL_MSVC_LINKOPTS", "ABSL_MSVC_TEST_FLAGS", ) WERROR = ["-Werror=return-type", "-Werror=switch"] DEFAULT_COPTS = select({ "//bazel:windows": ABSL_MSVC_FLAGS, "//bazel:llvm_compiler": ABSL_LLVM_FLAGS, "//conditions:default": ABSL_GCC_FLAGS + WERROR + ["-std=c++14"], }) TEST_COPTS = DEFAULT_COPTS + select({ "//bazel:windows": ABSL_MSVC_TEST_FLAGS, "//bazel:llvm_compiler": ABSL_LLVM_TEST_FLAGS, "//conditions:default": ABSL_GCC_TEST_FLAGS + WERROR + ["-std=c++14"], }) ================================================ FILE: bazel/fmtlib.BUILD ================================================ licenses(["notice"]) # Apache 2 cc_library( name = "fmtlib", srcs = glob([ "fmt/*.cc", ]), hdrs = glob([ "include/fmt/*.h", ]), defines = ["FMT_HEADER_ONLY"], includes = ["include"], visibility = ["//visibility:public"], ) ================================================ FILE: bazel/osx.tbb.BUILD ================================================ licenses(["notice"]) # Apache 2 package(default_visibility = ["//visibility:public"]) genrule( name = "build_tbb_osx", srcs = glob(["**"]) + [ "@local_config_cc//:toolchain", ], cmd = """ set -e set -u WORK_DIR=$$PWD DEST_DIR=$$PWD/$(@D) cd $$(dirname $(location :Makefile)) make files=build/*/*.dylib; echo cp $$files $$DEST_DIR cp $$files $$DEST_DIR cd $$WORK_DIR """, outs = [ "libtbb.dylib", "libtbbmalloc.dylib", "libtbbmalloc_proxy.dylib", ], ) cc_library( name = "tbb_osx", hdrs = glob([ "include/serial/**", "include/tbb/**/**", ]), srcs = [ "libtbb.dylib", "libtbbmalloc.dylib", "libtbbmalloc_proxy.dylib", ], includes = ["include"], visibility = ["//visibility:public"], ) ================================================ FILE: bazel/spdlog.BUILD ================================================ licenses(["notice"]) # Apache 2 cc_library( name = "spdlog", hdrs = glob([ "include/**/*.cc", "include/**/*.h", ]), defines = ["SPDLOG_FMT_EXTERNAL"], includes = ["include"], visibility = ["//visibility:public"], deps = ["@com_github_fmtlib_fmt//:fmtlib"], ) ================================================ FILE: bazel/tbb.BUILD ================================================ licenses(["notice"]) # Apache 2 package(default_visibility = ["//visibility:public"]) genrule( name = "build_tbb", srcs = glob(["**"]) + [ "@local_config_cc//:toolchain", ], cmd = """ set -e set -u WORK_DIR=$$PWD DEST_DIR=$$PWD/$(@D) cd $$(dirname $(location :Makefile)) make if [[ $$(echo `uname` | grep 'Dar') != "" ]]; then files=build/*/*.dylib; else files=build/*/*.so*; fi echo cp $$files $$DEST_DIR cp $$files $$DEST_DIR cd $$WORK_DIR """, outs = [ "libtbb.so", "libtbbmalloc.so", "libtbbmalloc_proxy.so", "libtbb.so.2", "libtbbmalloc.so.2", "libtbbmalloc_proxy.so.2", ], ) cc_library( name = "tbb", hdrs = glob([ "include/serial/**", "include/tbb/**/**", ]), srcs = [ "libtbb.so", "libtbbmalloc.so", "libtbbmalloc_proxy.so", "libtbb.so.2", "libtbbmalloc.so.2", "libtbbmalloc_proxy.so.2", ], includes = ["include"], visibility = ["//visibility:public"], ) ================================================ FILE: bazel/third_party_repositories.bzl ================================================ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])""" def include_third_party_repositories(): http_archive( name = "com_github_libevent", build_file_content = all_content, strip_prefix = "libevent-2.1.8-stable", urls = ["https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz"], ) http_archive( name = "com_github_libtbb", build_file = "//bazel:tbb.BUILD", strip_prefix = "oneTBB-2020.3", urls = ["https://github.com/oneapi-src/oneTBB/archive/v2020.3.tar.gz"], sha256 = "ebc4f6aa47972daed1f7bf71d100ae5bf6931c2e3144cf299c8cc7d041dca2f3", ) http_archive( name = "com_github_libtbb_osx", build_file = "//bazel:osx.tbb.BUILD", strip_prefix = "oneTBB-2020.3", urls = ["https://github.com/oneapi-src/oneTBB/archive/v2020.3.tar.gz"], sha256 = "ebc4f6aa47972daed1f7bf71d100ae5bf6931c2e3144cf299c8cc7d041dca2f3", ) http_archive( name = "com_github_fmtlib_fmt", sha256 = "4c0741e10183f75d7d6f730b8708a99b329b2f942dad5a9da3385ab92bb4a15c", strip_prefix = "fmt-5.3.0", urls = ["https://github.com/fmtlib/fmt/releases/download/5.3.0/fmt-5.3.0.zip"], build_file = "//bazel:fmtlib.BUILD", ) http_archive( name = "com_github_gabime_spdlog", build_file = "//bazel:spdlog.BUILD", sha256 = "160845266e94db1d4922ef755637f6901266731c4cb3b30b45bf41efa0e6ab70", strip_prefix = "spdlog-1.3.1", urls = ["https://github.com/gabime/spdlog/archive/v1.3.1.tar.gz"], ) ================================================ FILE: cmake/common.cmake ================================================ cmake_minimum_required( VERSION 3.14 ) find_package(Libevent REQUIRED) include_directories(${LIBEVENT_INCLUDE_DIR}) include( ${SENTINEL_ROOT_DIR}/cmake/third_party.cmake ) include_directories( ${SENTINEL_ROOT_DIR} ) include_directories(${SENTINEL_CORE_ROOT_DIR}/common) include_directories(${SENTINEL_CORE_ROOT_DIR}/config) include_directories(${SENTINEL_CORE_ROOT_DIR}/property) include_directories(${SENTINEL_CORE_ROOT_DIR}/public) # common file(GLOB COMMON_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/common/*.cc") file(GLOB COMMON_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/common/*_test.cc") list(REMOVE_ITEM COMMON_SOURCE_FILES ${COMMON_TEST_SOURCE_FILES}) # config file(GLOB CONFIG_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/config/*.cc") file(GLOB CONFIG_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/config/*_test.cc") list(REMOVE_ITEM CONFIG_SOURCE_FILES ${CONFIG_TEST_SOURCE_FILES}) # log file(GLOB_RECURSE LOG_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/log/*.cc") file(GLOB_RECURSE LOG_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/log/*_test.cc") list(REMOVE_ITEM LOG_SOURCE_FILES ${LOG_TEST_SOURCE_FILES}) # utils file(GLOB UTILS_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/utils/*.cc") # file(GLOB UTILS_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/utils/*test.cc") # list(REMOVE_ITEM UTILS_SOURCE_FILES ${UTILS_TEST_SOURCE_FILES}) # slot file(GLOB_RECURSE SLOT_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/slot/*.cc") file(GLOB_RECURSE SLOT_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/slot/*_test.cc") list(REMOVE_ITEM SLOT_SOURCE_FILES ${SLOT_TEST_SOURCE_FILES}) # statistic file(GLOB_RECURSE STATISTIC_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/statistic/*.cc") file(GLOB_RECURSE STATISTIC_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/statistic/*_test.cc") list(REMOVE_ITEM STATISTIC_SOURCE_FILES ${STATISTIC_TEST_SOURCE_FILES}) # transport file(GLOB_RECURSE TRANSPORT_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/transport/*.cc") file(GLOB_RECURSE TRANSPORT_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/transport/*_test.cc") list(REMOVE_ITEM TRANSPORT_SOURCE_FILES ${TRANSPORT_TEST_SOURCE_FILES}) # flow file(GLOB FLOW_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/flow/*.cc") file(GLOB FLOW_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/flow/*_test.cc") list(REMOVE_ITEM FLOW_SOURCE_FILES ${FLOW_TEST_SOURCE_FILES}) # circuitbreaker file(GLOB CIRCUIT_BREAKER_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/circuitbreaker/*.cc") file(GLOB CIRCUIT_BREAKER_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/circuitbreaker/*_test.cc") list(REMOVE_ITEM CIRCUIT_BREAKER_SOURCE_FILES ${CIRCUIT_BREAKER_TEST_SOURCE_FILES}) # param file(GLOB_RECURSE PARAM_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/param/*.cc") file(GLOB_RECURSE PARAM_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/param/*_test.cc") list(REMOVE_ITEM PARAM_SOURCE_FILES ${PARAM_TEST_SOURCE_FILES}) # system file(GLOB_RECURSE SYSTEM_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/system/*.cc") file(GLOB_RECURSE SYSTEM_TEST_SOURCE_FILES "${SENTINEL_CORE_ROOT_DIR}/system/*_test.cc") list(REMOVE_ITEM SYSTEM_SOURCE_FILES ${SYSTEM_TEST_SOURCE_FILES}) list(APPEND SENTINEL_SOURCE_FILES ${COMMON_SOURCE_FILES} ${CONFIG_SOURCE_FILES} ${LOG_SOURCE_FILES} ${UTILS_SOURCE_FILES} ${SLOT_SOURCE_FILES} ${STATISTIC_SOURCE_FILES} ${TRANSPORT_SOURCE_FILES} ${FLOW_SOURCE_FILES} ${CIRCUIT_BREAKER_SOURCE_FILES} ${PARAM_SOURCE_FILES} ${SYSTEM_SOURCE_FILES}) add_library(sentinel STATIC ${SENTINEL_SOURCE_FILES}) target_link_libraries(sentinel PUBLIC libevent::core libevent::extra libevent::pthreads TBB::tbb spdlog absl::flat_hash_set absl::str_format absl::synchronization absl::strings absl::any) ================================================ FILE: cmake/third_party.cmake ================================================ cmake_minimum_required( VERSION 3.14 ) include( FetchContent ) ####################################################################### # Declare project dependencies ####################################################################### FetchContent_Declare( abseil GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git GIT_TAG 20230125.0 ) FetchContent_Declare( spdlog GIT_REPOSITORY https://github.com/gabime/spdlog.git GIT_TAG v1.11.0 ) FetchContent_Declare( onetbb GIT_REPOSITORY https://github.com/oneapi-src/oneTBB.git GIT_TAG v2021.9.0 ) FetchContent_MakeAvailable(abseil) FetchContent_MakeAvailable(spdlog) FetchContent_MakeAvailable(onetbb) ================================================ FILE: examples/CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.14) project(sentinel-cpp VERSION 1.0.1 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 14) set(SENTINEL_CORE_ROOT_DIR "${CMAKE_SOURCE_DIR}/../sentinel-core") set(SENTINEL_ROOT_DIR "${CMAKE_SOURCE_DIR}/..") include( ${CMAKE_SOURCE_DIR}/../cmake/common.cmake ) # basic_qps_limit add_executable(basic_qps_limit ${CMAKE_SOURCE_DIR}/../examples/sentinel-cpp/basic_qps_limit.cc) target_link_libraries(basic_qps_limit sentinel) # basic_concurrency_limit add_executable(basic_concurrency_limit ${CMAKE_SOURCE_DIR}/../examples/sentinel-cpp/basic_concurrency_limit.cc) target_link_libraries(basic_concurrency_limit sentinel) # basic_param_limit add_executable(basic_param_limit ${CMAKE_SOURCE_DIR}/../examples/sentinel-cpp/basic_param_limit.cc) target_link_libraries(basic_param_limit sentinel) # basic_system_limit add_executable(basic_system_limit ${CMAKE_SOURCE_DIR}/../examples/sentinel-cpp/basic_system_limit.cc) target_link_libraries(basic_system_limit sentinel) ================================================ FILE: examples/abseil/BUILD ================================================ cc_binary( name = "asbeil_string", srcs = ["abseil_string.cc"], deps = [ "@com_google_absl//absl/strings", ], ) ================================================ FILE: examples/abseil/abseil_string.cc ================================================ #include #include #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" std::string Greet(absl::string_view person) { return absl::StrCat("Hello ", person); } int main(int argc, char* argv[]) { std::cout << Greet(argc < 2 ? "world" : argv[1]) << std::endl; } ================================================ FILE: examples/benchmark/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") cc_binary( name = "benchmark_test", srcs = ["benchamrk_test.cc"], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/public:sph_u_lib", "@com_google_benchmark//:benchmark", "//sentinel-core/log/metric:metric_log_task_lib", ], ) ================================================ FILE: examples/benchmark/benchamrk_test.cc ================================================ #include #include #include #include #include #include "sentinel-core/log/metric/metric_log_task.h" #include "sentinel-core/param/param_flow_rule_constants.h" #include "sentinel-core/param/param_flow_rule_manager.h" #include "sentinel-core/public/sph_u.h" static void BM_StringCreation(benchmark::State& state) { for (auto _ : state) std::string empty_string; } // Register the function as a benchmark // BENCHMARK(BM_StringCreation); static void ParamRun(benchmark::State& state) { std::string myResource("some_param_test"); Sentinel::Param::ParamFlowRule rule0, rule1, rule12; rule0.set_resource(myResource); rule0.set_metric_type(Sentinel::Param::ParamFlowMetricType::kQps); rule0.set_threshold(10000); rule0.set_cache_size(200); rule0.set_interval_in_ms(1000); rule0.set_param_idx(0); rule1.set_resource(myResource); rule1.set_metric_type(Sentinel::Param::ParamFlowMetricType::kQps); rule1.set_threshold(INT_MAX); rule1.set_param_idx(1); rule1.set_interval_in_ms(1000); Sentinel::Param::ParamFlowItem item0(std::string("nonexisting-str"), Sentinel::Param::ParamItemType::kString, 100); rule1.set_param_flow_item_list({item0}); rule12.set_resource("non-existing-resource"); // should not work rule12.set_metric_type(Sentinel::Param::ParamFlowMetricType::kQps); rule12.set_threshold(1); rule12.set_param_idx(1); rule12.set_interval_in_ms(1000); Sentinel::Param::ParamFlowRuleManager::GetInstance().LoadRules( {rule1, rule0, rule12}); for (auto _ : state) { int randParam = rand() % 10; auto r = Sentinel::SphU::Entry(myResource.c_str(), Sentinel::EntryType::IN, 1, 0, randParam); //, std::string("example")); r->Exit(); } } static void ParamNotRun(benchmark::State& state) { std::string myResource("some_param_test"); Sentinel::Log::Logger::InitDefaultLogger(); Sentinel::Log::MetricLogTask metric_log_task; metric_log_task.Initialize(); for (auto _ : state) { int randParam = rand() % 10; auto r = Sentinel::SphU::Entry(myResource.c_str(), Sentinel::EntryType::IN, 1); r->Exit(); } } static void NoCache(benchmark::State& state) { std::unordered_map m; int i = 0; for (auto _ : state) { m.insert(std::make_pair<>(i, i + 1)); i++; } } enum class ParamItemType { kInt32 = 0, kInt64, kString }; class MyAny : public absl::any { public: // std::atomic type_; ParamItemType my_type_; // MyAny(int v) : absl::any(v), type_(ParamItemType::kInt) {} MyAny(int32_t v) : absl::any(v), my_type_(ParamItemType::kInt32) {} MyAny(int64_t v) : absl::any(v), my_type_(ParamItemType::kInt64) {} MyAny(std::string v) : absl::any(v), my_type_(ParamItemType::kString) {} ParamItemType my_type() const noexcept { return my_type_; } operator int32_t() { assert(my_type_ == ParamItemType::kInt32); return absl::any_cast(*this); } operator int64_t() { assert(my_type_ == ParamItemType::kInt64); return absl::any_cast(*this); } operator std::string() { assert(my_type_ == ParamItemType::kString); return absl::any_cast(*this); } friend bool operator==(const MyAny& a0, const MyAny& a1) { std::cout << "==," << std::endl; if (a0.my_type_ == a1.my_type_) { switch (a0.my_type_) { case ParamItemType::kInt32: return absl::any_cast(a0) == absl::any_cast(a1); case ParamItemType::kInt64: return absl::any_cast(a0) == absl::any_cast(a1); case ParamItemType::kString: return absl::any_cast(a0) == absl::any_cast(a1); default: return false; } } if (a0.my_type_ == ParamItemType::kInt32 && a1.my_type_ == ParamItemType::kInt64) { return absl::any_cast(a0) == absl::any_cast(a1); } else if (a0.my_type_ == ParamItemType::kInt64 && a1.my_type_ == ParamItemType::kInt32) { return absl::any_cast(a0) == absl::any_cast(a1); } return false; } }; namespace std { template <> struct hash { size_t operator()(const MyAny& any) const { // std::cout << "hash of " << static_cast(any.my_type()) << ", " << // any.type().name() << ", " << any.has_value() << std::endl; int val = -1; switch (any.my_type()) { case ParamItemType::kInt32: return hash{}(absl::any_cast(any)); case ParamItemType::kInt64: return hash{}(absl::any_cast(any)); case ParamItemType::kString: return hash{}(absl::any_cast(any)); default: return -1; } } }; } // namespace std static void WithCache(benchmark::State& state) { std::unordered_map m; int32_t i = 0; for (auto _ : state) { m.insert(std::make_pair<>(i, i + 1)); i++; } } // BENCHMARK(ParamRun)->MinTime(8); BENCHMARK(NoCache)->MinTime(4); BENCHMARK(WithCache)->MinTime(4); BENCHMARK_MAIN(); ================================================ FILE: examples/cache/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_binary( name = "cache_test", srcs = ["cache_test.cc"], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/param/statistic:scalable_cache_lib", "//sentinel-core/param/statistic:any_cmp_lib", "//sentinel-core/log:logger_lib", ], ) ================================================ FILE: examples/cache/cache_test.cc ================================================ #include #include #include #include #include #include "sentinel-core/log/logger.h" #include "sentinel-core/param/statistic/any_cmp.h" #include "sentinel-core/param/statistic/scalable_cache.h" using ScalableCache = Sentinel::Param::ThreadSafeLRUCache; using ScalableCacheUniquePtr = std::unique_ptr; ScalableCache cache(100); std::atomic winStart(0); std::atomic stop(false); constexpr int THREAD_NUM = 10; int64_t CurrentTimeMillis() { return std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); } void Increase(int id, int cnt) { while (!stop.load()) { int64_t curTime = CurrentTimeMillis(); if (curTime - 500 > winStart) { // int64_t sz = cache.size(); // Sentinel::Log::Logger::Log(Sentinel::Log::Logger::kDefaultFileLogger, // Sentinel::info, "size={}", sz); cache.clear(); winStart.store(curTime - curTime % 500); } cache.increase(id, cnt); } } void Timer() { int cnt = 6; while (!stop.load()) { std::cout << "[" << cnt << "] " << cache.size() << std::endl; std::this_thread::sleep_for(std::chrono::milliseconds(1000)); if (cnt-- <= 0) { stop.store(true); } } } int main() { Sentinel::Log::Logger::InitDefaultLogger(); std::thread myThreads[THREAD_NUM]; std::thread t0(Timer); for (int i = 0; i < THREAD_NUM; i++) { myThreads[i] = std::thread(Increase, 11000 + 100 * i, 1); } t0.join(); for (int i = 0; i < THREAD_NUM; i++) { myThreads[i].join(); } // std::vector> pairs; // cache.snapshotPairs(pairs); // for (const auto& pair : pairs) { // std::cout << pair.second << " "; // } std::cout << "\n"; return 0; } ================================================ FILE: examples/fmt/BUILD ================================================ cc_binary( name = "fmt_test", srcs = ["fmt_test.cc"], deps = [ "@com_github_fmtlib_fmt//:fmtlib", ], ) ================================================ FILE: examples/fmt/fmt_test.cc ================================================ #include #include #include "fmt/format.h" int main() { std::string s = fmt::format("I'd rather be {1} than {0}.", "right", "happy"); std::cout << s << std::endl; return 0; } ================================================ FILE: examples/gtest/BUILD ================================================ cc_test( name = "hello_test", srcs = ["hello_test.cc"], deps = [ "@com_google_googletest//:gtest_main", ], ) ================================================ FILE: examples/gtest/hello_test.cc ================================================ #include #include "gtest/gtest.h" TEST(HelloTest, Basic) { EXPECT_EQ("Hello tester", "Hello tester"); } ================================================ FILE: examples/json/BUILD ================================================ cc_binary( name = "nlohmann_json_example", srcs = ["nlohmann_json_example.cc"], deps = [ "//third_party/nlohmann:nlohmann_json_lib", ], ) ================================================ FILE: examples/json/nlohmann_json_example.cc ================================================ #include #include #include "third_party/nlohmann/json.hpp" int main() { try { auto j2 = nlohmann::json::parse("sa"); } catch (std::exception& ex) { std::cerr << "Exception: " << ex.what() << std::endl; } // create a JSON array auto j3 = nlohmann::json::parse("{ \"happy\": true, \"pi\": 3.141 }"); auto c = j3["pi"]; if (c.is_number()) { int d = static_cast(c); std::cout << d << std::endl; } std::cout << typeid(c).name() << std::endl; return 0; } ================================================ FILE: examples/libevent/BUILD ================================================ cc_binary( name = "libevent_echosrv1", srcs = ["libevent_echosrv1.c"], deps = [ "//:libevent", ], ) ================================================ FILE: examples/libevent/libevent_echosrv1.c ================================================ // Copied from // https://github.com/jasonish/libevent-examples/blob/6d20f0d86c2cd263f5edff28862bc09ce4a3220f/echo-server/libevent_echosrv1.c // for testing purposes only. /* * Copyright (c) 2011, Jason Ish * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * - Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * - Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * libevent echo server example. */ #include #include #include /* For inet_ntoa. */ #include /* Required by event.h. */ #include #include #include #include #include #include #include #include /* Libevent. */ #include /* Port to listen on. */ #define SERVER_PORT 5555 /** * A struct for client specific data, in this simple case the only * client specific data is the read event. */ struct client { struct event ev_read; }; /** * Set a socket to non-blocking mode. */ int setnonblock(int fd) { int flags; flags = fcntl(fd, F_GETFL); if (flags < 0) return flags; flags |= O_NONBLOCK; if (fcntl(fd, F_SETFL, flags) < 0) return -1; return 0; } /** * This function will be called by libevent when the client socket is * ready for reading. */ void on_read(int fd, short ev, void *arg) { struct client *client = (struct client *)arg; u_char buf[8196]; int len, wlen; len = read(fd, buf, sizeof(buf)); if (len == 0) { /* Client disconnected, remove the read event and the * free the client structure. */ printf("Client disconnected.\n"); close(fd); event_del(&client->ev_read); free(client); return; } else if (len < 0) { /* Some other error occurred, close the socket, remove * the event and free the client structure. */ printf("Socket failure, disconnecting client: %s", strerror(errno)); close(fd); event_del(&client->ev_read); free(client); return; } /* XXX For the sake of simplicity we'll echo the data write * back to the client. Normally we shouldn't do this in a * non-blocking app, we should queue the data and wait to be * told that we can write. */ wlen = write(fd, buf, len); if (wlen < len) { /* We didn't write all our data. If we had proper * queueing/buffering setup, we'd finish off the write * when told we can write again. For this simple case * we'll just lose the data that didn't make it in the * write. */ printf("Short write, not all data echoed back to client.\n"); } } /** * This function will be called by libevent when there is a connection * ready to be accepted. */ void on_accept(int fd, short ev, void *arg) { int client_fd; struct sockaddr_in client_addr; socklen_t client_len = sizeof(client_addr); struct client *client; /* Accept the new connection. */ client_fd = accept(fd, (struct sockaddr *)&client_addr, &client_len); if (client_fd == -1) { warn("accept failed"); return; } /* Set the client socket to non-blocking mode. */ if (setnonblock(client_fd) < 0) warn("failed to set client socket non-blocking"); /* We've accepted a new client, allocate a client object to * maintain the state of this client. */ client = calloc(1, sizeof(*client)); if (client == NULL) err(1, "malloc failed"); /* Setup the read event, libevent will call on_read() whenever * the clients socket becomes read ready. We also make the * read event persistent so we don't have to re-add after each * read. */ event_set(&client->ev_read, client_fd, EV_READ | EV_PERSIST, on_read, client); /* Setting up the event does not activate, add the event so it * becomes active. */ event_add(&client->ev_read, NULL); printf("Accepted connection from %s\n", inet_ntoa(client_addr.sin_addr)); } int main(int argc, char **argv) { int listen_fd; struct sockaddr_in listen_addr; int reuseaddr_on = 1; /* The socket accept event. */ struct event ev_accept; /* Initialize libevent. */ event_init(); // comment out since it is called from test and should terminate // /* Create our listening socket. This is largely boiler plate // * code that I'll abstract away in the future. */ // listen_fd = socket(AF_INET, SOCK_STREAM, 0); // if (listen_fd < 0) // err(1, "listen failed"); // if (setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuseaddr_on, // sizeof(reuseaddr_on)) == -1) // err(1, "setsockopt failed"); // memset(&listen_addr, 0, sizeof(listen_addr)); // listen_addr.sin_family = AF_INET; // listen_addr.sin_addr.s_addr = INADDR_ANY; // listen_addr.sin_port = htons(SERVER_PORT); // if (bind(listen_fd, (struct sockaddr *)&listen_addr, sizeof(listen_addr)) // < 0) // err(1, "bind failed"); // if (listen(listen_fd, 5) < 0) // err(1, "listen failed"); // // /* Set the socket to non-blocking, this is essential in event // * based programming with libevent. */ // if (setnonblock(listen_fd) < 0) // err(1, "failed to set server socket to non-blocking"); // // /* We now have a listening socket, we create a read event to // * be notified when a client connects. */ // event_set(&ev_accept, listen_fd, EV_READ | EV_PERSIST, on_accept, NULL); // event_add(&ev_accept, NULL); // // /* Start the libevent event loop. */ // event_dispatch(); return 0; } ================================================ FILE: examples/log/BUILD ================================================ cc_binary( name = "log_test", srcs = ["log_test.cc"], deps = [ "@com_github_gabime_spdlog//:spdlog", ], ) ================================================ FILE: examples/log/log_test.cc ================================================ #include "spdlog/spdlog.h" int main() { spdlog::info("Welcome to spdlog!"); spdlog::error("Some error message with arg: {}", 1); spdlog::warn("Easy padding in numbers like {:08d}", 12); spdlog::critical( "Support for int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}", 42); spdlog::info("Support for floats {:03.2f}", 1.23456); spdlog::info("Positional args are {1} {0}..", "too", "supported"); spdlog::info("{:<30}", "left aligned"); spdlog::set_level(spdlog::level::debug); // Set global log level to debug spdlog::debug("This message should be displayed.."); // change log pattern spdlog::set_pattern("[%H:%M:%S %z] [%n] [%^---%L---%$] [thread %t] %v"); // Compile time log levels // define SPDLOG_ACTIVE_LEVEL to desired level SPDLOG_TRACE("Some trace message with param {}", {}); SPDLOG_DEBUG("Some debug message"); // Set the default logger to file logger // auto file_logger = spdlog::basic_logger_mt("basic_logger", // "logs/basic.txt"); spdlog::set_default_logger(file_logger); } ================================================ FILE: examples/sentinel-cpp/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) # https://docs.bazel.build/versions/master/be/c-cpp.html#cc_binary cc_binary( name = "sentinel_basic_qps_limit", srcs = ["basic_qps_limit.cc"], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/public:sph_u_lib", "//sentinel-core/log/metric:metric_log_task_lib", "//sentinel-core/transport/command:http_command_center_init_target", ], ) cc_binary( name = "sentinel_basic_concurrency_limit", srcs = ["basic_concurrency_limit.cc"], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/public:sph_u_lib", "//sentinel-core/log/metric:metric_log_task_lib", "//sentinel-core/transport/command:http_command_center_init_target", ], ) # https://docs.bazel.build/versions/master/be/c-cpp.html#cc_binary cc_binary( name = "sentinel_basic_system_limit", srcs = ["basic_system_limit.cc"], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/public:sph_u_lib", "//sentinel-core/log/metric:metric_log_task_lib", "//sentinel-core/transport/command:http_command_center_init_target", ], ) cc_binary( name = "sentinel_basic_param_limit", srcs = ["basic_param_limit.cc"], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/public:sph_u_lib", "//sentinel-core/log/metric:metric_log_task_lib", "//sentinel-core/transport/command:http_command_center_init_target", ], ) ================================================ FILE: examples/sentinel-cpp/basic_concurrency_limit.cc ================================================ #include #include #include #include #include #include "sentinel-core/flow/flow_rule_manager.h" #include "sentinel-core/init/init_target_registry.h" #include "sentinel-core/log/metric/metric_log_task.h" #include "sentinel-core/public/sph_u.h" #include "sentinel-core/transport/command/http_server_init_target.h" Sentinel::Init::InitTargetRegister< Sentinel::Transport::HttpCommandCenterInitTarget> HCCIT_registered; void doEntry(const char* resource) { while (true) { auto r = Sentinel::SphU::Entry(resource); if (r->IsBlocked()) { std::cout << "b"; std::this_thread::sleep_for(std::chrono::milliseconds(10)); } else { std::cout << "\n~~passed~~\n"; std::this_thread::sleep_for(std::chrono::milliseconds(10)); r->Exit(); } } } void doOneEntry() { doEntry("my_open_api_abc"); } void doAnotherEntry() { doEntry("big_brother_service:foo()"); } /* * Run the demo: CSP_SENTINEL_APP_NAME=your-app-name * ./bazel-bin/examples/sentinel-cpp/sentinel_basic_concurrency_limit */ int main() { // Initialize for Sentinel. Sentinel::Log::Logger::InitDefaultLogger(); Sentinel::Transport::HttpCommandCenterInitTarget command_center_init; command_center_init.Initialize(); Sentinel::Log::MetricLogTask metric_log_task; metric_log_task.Initialize(); Sentinel::Flow::FlowRule rule1{"my_open_api_abc"}; rule1.set_metric_type(Sentinel::Flow::FlowMetricType::kThreadCount); rule1.set_count(2); Sentinel::Flow::FlowRuleManager::GetInstance().LoadRules({rule1}); std::thread t1(doOneEntry); std::thread t2(doOneEntry); std::this_thread::sleep_for(std::chrono::milliseconds(5)); std::thread t3(doOneEntry); std::thread t4(doAnotherEntry); std::thread t5(doOneEntry); std::thread t6(doOneEntry); t1.join(); t2.join(); t3.join(); t4.join(); t5.join(); t6.join(); return 0; } ================================================ FILE: examples/sentinel-cpp/basic_param_limit.cc ================================================ #include #include #include #include #include #include #include #include #include "sentinel-core/init/init_target_registry.h" #include "sentinel-core/log/metric/metric_log_task.h" #include "sentinel-core/param/param_flow_rule_constants.h" #include "sentinel-core/param/param_flow_rule_manager.h" #include "sentinel-core/public/sph_u.h" #include "sentinel-core/transport/command/http_server_init_target.h" std::mutex mtx; std::atomic stop(false); constexpr int SECONDS = 60, THREAD_NUM = 8; std::atomic pass, block, seconds(SECONDS); std::vector> pPassCnt(10); std::vector> pBlockCnt(10); std::thread myThreads[THREAD_NUM]; int threadCnt = 0; int64_t CurrentTimeMillis() { return std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); } void RunTask() { double ans = 1.001; for (int i = 0; i < 3000; i++) { double index = (rand() % 5) * ((CurrentTimeMillis() / 1000) % 3); ans = pow(ans, index); } } void DoEntry(const char* resource) { int cnt = 0; while (!stop.load()) { int32_t randParam = (rand() + (CurrentTimeMillis() / 10)) % 10; randParam = ((cnt++)) % 50; auto r = Sentinel::SphU::Entry(resource, Sentinel::EntryType::IN, 1, 0, randParam, std::string("example")); if (r->IsBlocked()) { // pBlockCnt[randParam].fetch_add(1); block.fetch_add(1); // std::this_thread::sleep_for(std::chrono::milliseconds(100)); } else { pass.fetch_add(1); // pPassCnt[randParam].fetch_add(1); RunTask(); // std::this_thread::sleep_for(std::chrono::milliseconds(100)); r->Exit(); } } } void TimerTask() { int oldTotal = 0, oldPass = 0, oldBlock = 0; int oldPTotal[10] = {0}, oldPPass[10] = {0}, oldPBlock[10] = {0}; std::cerr << "Begin to statistic!!!" << std::endl; int cnt = 0; while (true) { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); int globalPass = pass.load(); int globalBlock = block.load(); int globalTotal = globalBlock + globalPass; int oneSecondPass = globalPass - oldPass; int oneSecondBlock = globalBlock - oldBlock; int oneSecondTotal = globalTotal - oldTotal; oldPass = globalPass; oldBlock = globalBlock; oldTotal = globalTotal; std::cerr << "[" << seconds.load() << "] total=" << oneSecondTotal << ", pass=" << oneSecondPass << ", block=" << oneSecondBlock << std::endl; int leftSec = seconds.fetch_sub(1); if (leftSec <= 0) { stop.store(true); break; } } } /* * Run the demo: CSP_SENTINEL_APP_NAME=your-app-name * ./bazel-bin/examples/sentinel-cpp/sentinel_basic_param_limit */ int main() { // Initialize for Sentinel. Sentinel::Log::Logger::InitDefaultLogger(); Sentinel::Transport::HttpCommandCenterInitTarget command_center_init; command_center_init.Initialize(); Sentinel::Log::MetricLogTask metric_log_task; metric_log_task.Initialize(); std::string myResource("some_param_test"); Sentinel::Param::ParamFlowRule rule0, rule1, rule12, rule11; rule0.set_resource(myResource); rule0.set_metric_type(Sentinel::Param::ParamFlowMetricType::kQps); rule0.set_threshold(2000); rule0.set_cache_size(200); rule0.set_interval_in_ms(1000); rule0.set_param_idx(0); rule1.set_resource(myResource); rule1.set_metric_type(Sentinel::Param::ParamFlowMetricType::kQps); rule1.set_threshold(6000); rule1.set_param_idx(1); rule1.set_interval_in_ms(1000); Sentinel::Param::ParamFlowItem item0(std::string("nonexisting-str"), Sentinel::Param::ParamItemType::kString, 100); rule1.set_param_flow_item_list({item0}); rule12.set_resource("non-existing-resource"); // should not work rule12.set_metric_type(Sentinel::Param::ParamFlowMetricType::kQps); rule12.set_threshold(1); rule12.set_param_idx(1); rule12.set_interval_in_ms(1000); Sentinel::Param::ParamFlowRuleManager::GetInstance().LoadRules( {rule1, rule0, rule12}); std::thread t0(TimerTask); std::this_thread::sleep_for(std::chrono::milliseconds(1000)); std::cerr << "Thread num is " << THREAD_NUM << std::endl; for (int i = 0; i < THREAD_NUM; i++) { myThreads[i] = std::thread(DoEntry, myResource.c_str()); } for (int i = 0; i < THREAD_NUM; i++) { myThreads[i].join(); } t0.join(); return 0; } ================================================ FILE: examples/sentinel-cpp/basic_qps_limit.cc ================================================ #include #include #include #include #include #include "sentinel-core/flow/flow_rule_manager.h" #include "sentinel-core/init/init_target_registry.h" #include "sentinel-core/log/metric/metric_log_task.h" #include "sentinel-core/public/sph_u.h" #include "sentinel-core/transport/command/http_server_init_target.h" void DoEntry(const char* resource) { while (true) { auto r = Sentinel::SphU::Entry(resource); if (r->IsBlocked()) { // Indicating the request is blocked. We can do something for this. std::cout << "b"; std::this_thread::sleep_for(std::chrono::milliseconds(8)); } else { std::cout << "\n~~passed~~\n"; std::this_thread::sleep_for(std::chrono::milliseconds(8)); r->Exit(); } std::cout.flush(); } } void DoOneEntry() { DoEntry("my_open_api_abc"); } void DoAnotherEntry() { DoEntry("m1:my_another_api_233"); } /* * Run the demo: CSP_SENTINEL_APP_NAME=your-app-name * ./bazel-bin/examples/sentinel-cpp/sentinel_basic_qps_limit */ int main() { // Initialize for Sentinel. Sentinel::Log::Logger::InitDefaultLogger(); Sentinel::Transport::HttpCommandCenterInitTarget command_center_init; command_center_init.Initialize(); Sentinel::Log::MetricLogTask metric_log_task; metric_log_task.Initialize(); Sentinel::Flow::FlowRule rule1{"my_open_api_abc"}; rule1.set_count(10); rule1.set_control_behavior(Sentinel::Flow::FlowControlBehavior::kThrotting); Sentinel::Flow::FlowRule rule2{"m1:my_another_api_233"}; rule2.set_count(5); Sentinel::Flow::FlowRuleManager::GetInstance().LoadRules({rule1, rule2}); std::thread t1(DoOneEntry); std::thread t2(DoOneEntry); std::thread t21(DoOneEntry); std::thread t22(DoOneEntry); std::thread t23(DoOneEntry); std::this_thread::sleep_for(std::chrono::milliseconds(5)); std::thread t3(DoOneEntry); std::thread t4(DoAnotherEntry); std::this_thread::sleep_for(std::chrono::milliseconds(9)); std::thread t5(DoOneEntry); std::thread t6(DoOneEntry); t1.join(); t2.join(); t3.join(); t4.join(); t5.join(); t6.join(); return 0; } ================================================ FILE: examples/sentinel-cpp/basic_system_limit.cc ================================================ #include #include #include #include #include #include #include #include "sentinel-core/init/init_target_registry.h" #include "sentinel-core/log/metric/metric_log_task.h" #include "sentinel-core/public/sph_u.h" #include "sentinel-core/system/system_rule_manager.h" #include "sentinel-core/transport/command/http_server_init_target.h" std::atomic pass, block, seconds(1200); std::atomic stop(false); int64_t CurrentTimeMillis() { return std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); } void RunTask() { double ans = 1.001; for (int i = 0; i < 400; i++) { double index = (rand() % 5) * ((CurrentTimeMillis() / 1000) % 3); ans = pow(ans, index); } } void DoEntry(const char* resource, Sentinel::EntryType trafficType) { while (true) { auto r = Sentinel::SphU::Entry(resource, trafficType); if (r->IsBlocked()) { // Indicating the request is blocked. We can do something for this. block.fetch_add(1); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } else { pass.fetch_add(1); std::this_thread::sleep_for(std::chrono::milliseconds(10)); r->Exit(); } std::cout.flush(); } } void TimerTask() { int oldTotal = 0, oldPass = 0, oldBlock = 0; std::cout << "Begin to statistic!!!" << std::endl; while (true) { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); int globalPass = pass.load(); int globalBlock = block.load(); int globalTotal = globalBlock + globalPass; int oneSecondPass = globalPass - oldPass; int oneSecondBlock = globalBlock - oldBlock; int oneSecondTotal = globalTotal - oldTotal; oldPass = globalPass; oldBlock = globalBlock; oldTotal = globalTotal; std::cout << "[" << seconds.load() << "] total=" << oneSecondTotal << ", pass=" << oneSecondPass << ", block=" << oneSecondBlock << std::endl; int leftSec = seconds.fetch_sub(1); if (leftSec <= 0) { stop.store(true); break; } } } /* * Run the demo: CSP_SENTINEL_APP_NAME=your-app-name * ./bazel-bin/examples/sentinel-cpp/sentinel_basic_qps_limit */ int main() { // Initialize for Sentinel. Sentinel::Log::Logger::InitDefaultLogger(); Sentinel::Transport::HttpCommandCenterInitTarget command_center_init; command_center_init.Initialize(); Sentinel::Log::MetricLogTask metric_log_task; metric_log_task.Initialize(); Sentinel::System::SystemStatusListener::GetInstance().Initialize(); Sentinel::System::SystemRule rule1, rule2, rule3; rule1.set_rule_type(Sentinel::System::MetricType::kQps); rule1.set_threshold(static_cast(250)); rule2.set_rule_type(Sentinel::System::MetricType::kConcurrency); rule2.set_threshold(4); rule3.set_rule_type(Sentinel::System::MetricType::kCpuUsage); rule3.set_threshold(static_cast(0.8)); Sentinel::System::SystemRule rule4{Sentinel::System::MetricType::kCpuUsage, 0.7}; Sentinel::System::SystemRule badRule{Sentinel::System::MetricType::kRt, -2}; Sentinel::System::SystemRuleManager::GetInstance().LoadRules( {rule1, rule2, rule3, rule4, badRule}); std::thread t0(TimerTask); std::thread t1(DoEntry, "my_open_api_abc", Sentinel::EntryType::IN); std::this_thread::sleep_for(std::chrono::milliseconds(47)); std::thread t2(DoEntry, "my_open_api_abc", Sentinel::EntryType::IN); std::this_thread::sleep_for(std::chrono::milliseconds(23)); std::thread t3(DoEntry, "my_open_api_abc", Sentinel::EntryType::IN); // std::this_thread::sleep_for(std::chrono::milliseconds(19)); std::thread t4(DoEntry, "my_open_api_abc", Sentinel::EntryType::IN); // std::thread t5(DoEntry, "foo", Sentinel::EntryType::OUT); t0.join(); t1.join(); t2.join(); t3.join(); t4.join(); // t5.join(); return 0; } ================================================ FILE: examples/tbb/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_binary( name = "tbb_test", srcs = ["tbb_test.cc"], # srcs = select({ # "//bazel:is_android": [], # "//bazel:is_wsl": [ "tbb_test.cc" ], # "//conditions:default": [ "tbb_test.cc" ], # }), copts = DEFAULT_COPTS, deps = select({ "//bazel:is_osx": [ "//sentinel-core/log:logger_lib", "@com_github_libtbb_osx//:tbb_osx", ], "//conditions:default": [ "//sentinel-core/log:logger_lib", "@com_github_libtbb//:tbb", ], }), ) ================================================ FILE: examples/tbb/tbb_test.cc ================================================ #include #include #include #include "sentinel-core/log/logger.h" #include "tbb/concurrent_hash_map.h" constexpr int NUM_THREADS = 10000; class Stu { public: int id; std::string name; ~Stu() { std::cout << "Destructor" << std::endl; } }; tbb::concurrent_hash_map> stuMap; int main() { Sentinel::Log::Logger::InitDefaultLogger(); std::cout << "loop begin\n"; decltype(stuMap)::accessor ac; if (stuMap.insert(ac, std::make_pair<>(1, nullptr))) { ac->second = std::make_unique(); std::cout << stuMap.size() << std::endl; } ac.release(); decltype(stuMap)::accessor ac2; stuMap.insert(ac2, std::make_pair<>(1, std::make_unique())); std::cout << "===== END =====" << stuMap.size() << std::endl; return 0; } ================================================ FILE: format.sh ================================================ #!/usr/bin/env bash CMD=clang-format-8 $CMD -version $CMD -i -style=Google $(git ls-files|grep -E ".*\.(cc|h)$") CHANGED="$(git ls-files --modified)" if [[ ! -z "$CHANGED" ]]; then echo "The following files have changes due to incrrect format:" echo "$CHANGED" echo "please use format.sh script fix it" exit 1 else echo "No changes." fi ================================================ FILE: sentinel-core/BUILD ================================================ ================================================ FILE: sentinel-core/circuitbreaker/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "rule_lib", srcs = [ "rule.h", "rule.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/common:rule_lib", "@com_google_absl//absl/strings:str_format", ] ) cc_library( name = "circuit_breaker_interface", srcs = [ "circuit_breaker.h", "circuit_breaker.cc", ], copts = DEFAULT_COPTS, deps = [ ":rule_lib", "//sentinel-core/statistic/node:node_interface", "//sentinel-core/utils:utils_lib", ] ) cc_library( name = "rt_circuit_breaker_lib", srcs = [ "rt_circuit_breaker.h", "rt_circuit_breaker.cc", ], copts = DEFAULT_COPTS, deps = [ ":circuit_breaker_interface", ":rule_lib", "//sentinel-core/statistic/base:leap_array_lib", ] ) cc_library( name = "error_circuit_breaker_lib", srcs = [ "error_circuit_breaker.h", "error_circuit_breaker.cc", ], copts = DEFAULT_COPTS, deps = [ ":circuit_breaker_interface", ":rule_lib", "//sentinel-core/statistic/base:leap_array_lib", ] ) cc_library( name = "rule_manager_lib", srcs = [ "rule_manager.h", "rule_manager.cc", ], copts = DEFAULT_COPTS, deps = [ ":error_circuit_breaker_lib", ":rt_circuit_breaker_lib", ":rule_lib", "//sentinel-core/log:logger_lib", "//sentinel-core/property:dynamic_sentinel_property_lib", "//sentinel-core/property:property_listener_interface", "//sentinel-core/property:sentinel_property_interface", "@com_google_absl//absl/synchronization", ] ) cc_library( name = "rule_slot_lib", srcs = [ "slot.h", "slot.cc", ], copts = DEFAULT_COPTS, deps = [ ":rule_manager_lib", "//sentinel-core/slot/base:rule_checker_slot_interface", "//sentinel-core/slot/base:stats_slot_interface", ] ) cc_test( name = "slot_unittests", srcs = [ "slot_test.cc", ], copts = TEST_COPTS, deps = [ ":rule_slot_lib", "//sentinel-core/common:string_resource_wrapper_lib", "//sentinel-core/test/mock/statistic/node:mock_lib", "@com_google_googletest//:gtest_main", ] ) # cc_library( # name = "flow_rule_manager_lib", # srcs = [ # "flow_rule_manager.h", # "flow_rule_manager.cc", # ], # copts = DEFAULT_COPTS, # deps = [ # ":flow_rule_constants_lib", # ":flow_rule_lib", # ":traffic_shaping_controller_lib", # ":default_traffic_shaping_calculator_lib", # ":default_traffic_shaping_checker_lib", # "//sentinel-core/property:property_listener_interface", # "//sentinel-core/log:logger_lib", # "//sentinel-core/property:sentinel_property_interface", # "//sentinel-core/property:dynamic_sentinel_property_lib", # "@com_google_absl//absl/synchronization", # ] # ) # cc_test( # name = "flow_slot_unittests", # srcs = [ # "flow_slot_test.cc", # ], # copts = TEST_COPTS, # deps = [ # ":flow_slot_lib", # "//sentinel-core/common:string_resource_wrapper_lib", # "//sentinel-core/test/mock/statistic/node:mock_lib", # "@com_google_googletest//:gtest_main", # ] # ) ================================================ FILE: sentinel-core/circuitbreaker/circuit_breaker.cc ================================================ #include #include #include "sentinel-core/circuitbreaker/circuit_breaker.h" #include "sentinel-core/utils/time_utils.h" namespace Sentinel { namespace CircuitBreaker { State AbstractCircuitBreaker::CurrentState() { return current_state_.load(); } const Rule& AbstractCircuitBreaker::GetRule() const { return rule_; } bool AbstractCircuitBreaker::RetryTimeoutArrived() { return Utils::TimeUtils::CurrentTimeMillis().count() >= next_retry_timestamp_; } void AbstractCircuitBreaker::UpdateNextRetryTimestamp() { this->next_retry_timestamp_ = Utils::TimeUtils::CurrentTimeMillis().count() + retry_timeout_ms_; } bool AbstractCircuitBreaker::FromCloseToOpen(double snapshot) { auto expected = State::kClosed; if (current_state_.compare_exchange_strong(expected, State::kOpen)) { UpdateNextRetryTimestamp(); return true; } return false; } bool AbstractCircuitBreaker::FromOpenToHalfOpen() { auto expected = State::kOpen; return current_state_.compare_exchange_strong(expected, State::kHalfOpen); } bool AbstractCircuitBreaker::FromHalfOpenToOpen(double snapshot) { auto expected = State::kHalfOpen; if (current_state_.compare_exchange_strong(expected, State::kOpen)) { UpdateNextRetryTimestamp(); return true; } return false; } bool AbstractCircuitBreaker::FromHalfOpenToClose() { auto expected = State::kHalfOpen; if (current_state_.compare_exchange_strong(expected, State::kClosed)) { Reset(); return true; } return false; } } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/circuit_breaker.h ================================================ #pragma once #include #include #include "sentinel-core/circuitbreaker/rule.h" #include "sentinel-core/statistic/node/node.h" namespace Sentinel { namespace CircuitBreaker { enum class State { kClosed = 0, kOpen = 1, kHalfOpen = 2 }; class CircuitBreaker { public: CircuitBreaker() = default; virtual ~CircuitBreaker() = default; virtual const Rule& GetRule() const = 0; virtual State CurrentState() = 0; virtual bool TryPass(Stat::NodeSharedPtr& node) = 0; virtual void Reset() = 0; virtual void RecordComplete(int64_t rt, const std::string& error) = 0; }; using CircuitBreakerSharedPtr = std::shared_ptr; class AbstractCircuitBreaker : public CircuitBreaker { public: explicit AbstractCircuitBreaker(const Rule& rule) : rule_(rule), retry_timeout_ms_(rule.retry_timeout_sec() * 1000) {} virtual ~AbstractCircuitBreaker() = default; State CurrentState() override; const Rule& GetRule() const override; protected: bool RetryTimeoutArrived(); void UpdateNextRetryTimestamp(); bool FromCloseToOpen(double snapshot); bool FromOpenToHalfOpen(); bool FromHalfOpenToOpen(double snapshot); bool FromHalfOpenToClose(); const Rule rule_; const int32_t retry_timeout_ms_; std::atomic current_state_{State::kClosed}; int64_t next_retry_timestamp_ = 0; }; } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/error_circuit_breaker.cc ================================================ #include #include "sentinel-core/circuitbreaker/error_circuit_breaker.h" namespace Sentinel { namespace CircuitBreaker { std::shared_ptr SimpleErrorCounterLeapArray::NewEmptyBucket( int64_t) { return std::make_shared(); } void SimpleErrorCounterLeapArray::ResetWindowTo( const Stat::WindowWrapSharedPtr& w, int64_t start_time) { // Update the start time and reset value. w->ResetTo(start_time); w->Value()->Reset(); } bool ErrorCircuitBreaker::TryPass(Stat::NodeSharedPtr&) { State state = current_state_.load(); if (state == State::kClosed) { return true; } if (state == State::kOpen) { return RetryTimeoutArrived() && FromOpenToHalfOpen(); } return false; } void ErrorCircuitBreaker::Reset() { // NOTE: only reset the current bucket as the sampleCount=1 sliding_counter_->CurrentWindow()->Value()->Reset(); } void ErrorCircuitBreaker::RecordComplete(int64_t, const std::string& err) { auto counter = sliding_counter_->CurrentWindow()->Value(); if (!err.empty()) { counter->AddErrorCount(1); } counter->AddTotalCount(1); RecordAndHandleStateChange(err); } void ErrorCircuitBreaker::RecordAndHandleStateChange(const std::string& err) { if (current_state_.load() == State::kOpen) { return; } if (current_state_.load() == State::kHalfOpen) { if (!err.empty()) { FromHalfOpenToOpen(1); } else { FromHalfOpenToClose(); } return; } auto counters = sliding_counter_->Values(); int64_t err_count = 0, total_count = 0; for (auto& c : counters) { err_count += c->error_count(); total_count += c->total_count(); } if (total_count < rule_.min_request_amount()) { return; } double cur_value = strategy_ == Strategy::kErrorRatio ? err_count * 1.0 / total_count : err_count; if (cur_value > threshold_) { auto cs = current_state_.load(); switch (cs) { case State::kClosed: FromCloseToOpen(cur_value); break; case State::kHalfOpen: FromHalfOpenToOpen(cur_value); break; default: break; } } } } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/error_circuit_breaker.h ================================================ #pragma once #include #include #include "sentinel-core/circuitbreaker/circuit_breaker.h" #include "sentinel-core/circuitbreaker/rule.h" #include "sentinel-core/statistic/base/leap_array.h" namespace Sentinel { namespace CircuitBreaker { class SimpleErrorCounter { public: explicit SimpleErrorCounter() = default; virtual ~SimpleErrorCounter() = default; int64_t AddErrorCount(int64_t count) { return error_count_.fetch_add(count) + count; }; int64_t AddTotalCount(int64_t count) { return total_count_.fetch_add(count) + count; }; void Reset() { error_count_ = 0; total_count_ = 0; }; int64_t error_count() { return error_count_.load(); }; int64_t total_count() { return total_count_.load(); }; private: std::atomic error_count_{0}; std::atomic total_count_{0}; }; class SimpleErrorCounterLeapArray : public Stat::LeapArray { public: explicit SimpleErrorCounterLeapArray(int32_t sample_count, int32_t interval_ms) : LeapArray(sample_count, interval_ms) {} virtual ~SimpleErrorCounterLeapArray() {} std::shared_ptr NewEmptyBucket( int64_t time_millis) override; void ResetWindowTo(const Stat::WindowWrapSharedPtr& wrap, int64_t start_time) override; }; class ErrorCircuitBreaker : public AbstractCircuitBreaker { public: explicit ErrorCircuitBreaker(const Rule& rule) : AbstractCircuitBreaker(rule), sliding_counter_(std::make_unique( 1, rule.stat_interval_ms())) { this->strategy_ = rule.strategy(); this->threshold_ = rule.threshold(); } virtual ~ErrorCircuitBreaker() = default; bool TryPass(Stat::NodeSharedPtr& node) override; void Reset() override; void RecordComplete(int64_t rt, const std::string& error) override; private: void RecordAndHandleStateChange(const std::string& err); Strategy strategy_; double threshold_; const std::unique_ptr sliding_counter_; }; } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/rt_circuit_breaker.cc ================================================ #include #include "sentinel-core/circuitbreaker/rt_circuit_breaker.h" namespace Sentinel { namespace CircuitBreaker { std::shared_ptr SlowRequestLeapArray::NewEmptyBucket( int64_t) { return std::make_shared(); } void SlowRequestLeapArray::ResetWindowTo( const Stat::WindowWrapSharedPtr& w, int64_t start_time) { // Update the start time and reset value. w->ResetTo(start_time); w->Value()->Reset(); } bool ResponseTimeCircuitBreaker::TryPass(Stat::NodeSharedPtr&) { State state = current_state_.load(); if (state == State::kClosed) { return true; } if (state == State::kOpen) { return RetryTimeoutArrived() && FromOpenToHalfOpen(); } return false; } void ResponseTimeCircuitBreaker::Reset() { sliding_counter_->CurrentWindow()->Value()->Reset(); } void ResponseTimeCircuitBreaker::RecordComplete(int64_t rt, const std::string&) { auto counter = sliding_counter_->CurrentWindow()->Value(); if (rt > max_allowed_rt_) { counter->AddSlowCount(1); } counter->AddTotalCount(1); RecordAndHandleStateChange(rt); } void ResponseTimeCircuitBreaker::RecordAndHandleStateChange(int64_t rt) { if (current_state_.load() == State::kOpen) { return; } if (current_state_.load() == State::kHalfOpen) { if (rt > max_allowed_rt_) { FromHalfOpenToOpen(1.0); } else { FromHalfOpenToClose(); } return; } auto counters = sliding_counter_->Values(); int64_t slow_count = 0, total_count = 0; for (auto& c : counters) { slow_count += c->slow_count(); total_count += c->total_count(); } if (total_count < rule_.min_request_amount()) { return; } double current_ratio = slow_count * 1.0 / total_count; if (current_ratio > max_slow_request_ratio_) { auto cs = current_state_.load(); switch (cs) { case State::kClosed: FromCloseToOpen(current_ratio); break; case State::kHalfOpen: FromHalfOpenToOpen(current_ratio); break; default: break; } } } } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/rt_circuit_breaker.h ================================================ #pragma once #include #include #include "sentinel-core/circuitbreaker/circuit_breaker.h" #include "sentinel-core/circuitbreaker/rule.h" #include "sentinel-core/statistic/base/leap_array.h" namespace Sentinel { namespace CircuitBreaker { class SlowRequestCounter { public: explicit SlowRequestCounter() = default; virtual ~SlowRequestCounter() = default; int64_t AddSlowCount(int64_t count) { return slow_count_.fetch_add(count) + count; }; int64_t AddTotalCount(int64_t count) { return total_count_.fetch_add(count) + count; }; void Reset() { slow_count_ = 0; total_count_ = 0; }; int64_t slow_count() { return slow_count_.load(); }; int64_t total_count() { return total_count_.load(); }; private: std::atomic slow_count_{0}; std::atomic total_count_{0}; }; class SlowRequestLeapArray : public Stat::LeapArray { public: explicit SlowRequestLeapArray(int32_t sample_count, int32_t interval_ms) : LeapArray(sample_count, interval_ms) {} virtual ~SlowRequestLeapArray() {} std::shared_ptr NewEmptyBucket( int64_t time_millis) override; void ResetWindowTo(const Stat::WindowWrapSharedPtr& wrap, int64_t start_time) override; }; class ResponseTimeCircuitBreaker : public AbstractCircuitBreaker { public: explicit ResponseTimeCircuitBreaker(const Rule& rule) : AbstractCircuitBreaker(rule), sliding_counter_(std::make_unique( 1, rule.stat_interval_ms())) { this->max_allowed_rt_ = rule.max_allowed_rt(); this->max_slow_request_ratio_ = rule.threshold(); } virtual ~ResponseTimeCircuitBreaker() = default; bool TryPass(Stat::NodeSharedPtr& node) override; void Reset() override; void RecordComplete(int64_t rt, const std::string& error) override; private: void RecordAndHandleStateChange(int64_t rt); int32_t max_allowed_rt_; double max_slow_request_ratio_; const std::unique_ptr sliding_counter_; }; } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/rule.cc ================================================ #include #include "sentinel-core/circuitbreaker/rule.h" #include "absl/strings/str_format.h" namespace Sentinel { namespace CircuitBreaker { bool CircuitBreaker::Rule::operator==(const CircuitBreaker::Rule& rule) const { return resource_ == rule.resource() && threshold_ == rule.threshold() && strategy_ == rule.strategy() && retry_timeout_sec_ == rule.retry_timeout_sec() && min_request_amount_ == rule.min_request_amount() && stat_interval_ms_ == rule.stat_interval_ms() && max_allowed_rt_ == rule.max_allowed_rt(); } std::string CircuitBreaker::Rule::ToString() const { return absl::StrFormat( "CircuitBreakerRule{resource=%s, strategy=%d, threshold=%.2f, " "retry_timeout_sec=%d, min_request_amount=%d, stat_interval_ms=%d, " "max_allowed_rt=%d}", resource_, static_cast(strategy_), threshold_, retry_timeout_sec_, min_request_amount_, stat_interval_ms_, max_allowed_rt_); } } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/rule.h ================================================ #pragma once #include #include #include #include #include #include "sentinel-core/common/rule.h" namespace Sentinel { namespace CircuitBreaker { enum class MetricType { kResponseTime = 0, kException = 1 }; enum class Strategy { kSlowRtRatio = 0, kErrorRatio = 1, kErrorCount = 2 }; struct Rule : public Sentinel::Rule { public: Rule() = default; virtual ~Rule() = default; explicit Rule(const std::string& resource) : resource_(resource) {} const std::string& resource() const { return resource_; } Strategy strategy() const { return strategy_; } double threshold() const { return threshold_; } int32_t retry_timeout_sec() const { return retry_timeout_sec_; } int32_t min_request_amount() const { return min_request_amount_; } int32_t stat_interval_ms() const { return stat_interval_ms_; } int32_t max_allowed_rt() const { return max_allowed_rt_; } void set_resource(const std::string& resource) { resource_ = resource; } void set_strategy(Strategy s) { strategy_ = s; } void set_threshold(double threshold) { threshold_ = threshold; } void set_retry_timeout_sec(int32_t retry_timeout_sec) { retry_timeout_sec_ = retry_timeout_sec; } void set_min_request_amount(int32_t min_request_amount) { min_request_amount_ = min_request_amount; } void set_stat_interval_ms(int32_t stat_interval_ms) { stat_interval_ms_ = stat_interval_ms; } void set_max_allowed_rt(int32_t max_allowed_rt) { max_allowed_rt_ = max_allowed_rt; } bool operator==(const CircuitBreaker::Rule& rule) const; std::string ToString() const; private: std::string resource_; Strategy strategy_{Strategy::kSlowRtRatio}; double threshold_ = 0; int32_t retry_timeout_sec_ = 0; int32_t min_request_amount_{10}; int32_t stat_interval_ms_{1000}; int32_t max_allowed_rt_{0}; }; struct RuleHash { std::size_t operator()(const CircuitBreaker::Rule& rule) const noexcept { std::size_t result = std::hash{}(rule.resource()); result = 31 * result + static_cast(rule.strategy()); result = 31 * result + std::hash{}(rule.threshold()); result = 31 * result + rule.retry_timeout_sec(); result = 31 * result + rule.min_request_amount(); result = 31 * result + rule.stat_interval_ms(); result = 31 * result + rule.max_allowed_rt(); return result; } }; using RuleList = std::vector; using RuleSet = std::unordered_set; } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/rule_manager.cc ================================================ #include #include #include #include #include "sentinel-core/circuitbreaker/error_circuit_breaker.h" #include "sentinel-core/circuitbreaker/rt_circuit_breaker.h" #include "sentinel-core/circuitbreaker/rule_manager.h" #include "sentinel-core/log/logger.h" #include "sentinel-core/property/dynamic_sentinel_property.h" namespace Sentinel { namespace CircuitBreaker { constexpr auto kPropertyListenerName = "CircuitBreakerRulePropertyListener"; bool IsValidRule(const Rule& rule) { auto threshold = rule.threshold(); bool base_valid = !rule.resource().empty() && threshold >= 0; if (!base_valid) { return false; } if (rule.retry_timeout_sec() <= 0 || rule.stat_interval_ms() <= 0 || rule.min_request_amount() < 0) { return false; } switch (rule.strategy()) { case Strategy::kSlowRtRatio: return rule.max_allowed_rt() >= 0 && threshold <= 1; case Strategy::kErrorRatio: return threshold <= 1; case Strategy::kErrorCount: return true; default: return false; } } // RuleManager RuleManager::RuleManager() { cur_property_ = std::make_shared>>(); cur_property_->AddListener(std::make_unique()); } bool RuleManager::LoadRules(const RuleList& rules) { return cur_property_->UpdateValue(rules); } bool RuleManager::HasRules(const std::string& resource) { absl::ReaderMutexLock lck(&update_mtx_); return cb_map_.find(resource) != cb_map_.end(); } RuleList RuleManager::GetRules() const { absl::ReaderMutexLock lck(&update_mtx_); RuleList list{}; for (const auto& e : rule_map_) { list.insert(std::end(list), std::begin(e.second), std::end(e.second)); } return list; } RuleSet RuleManager::GetRulesOfResource(const std::string& resource) const { absl::ReaderMutexLock lck(&update_mtx_); auto it = rule_map_.find(resource); if (it == rule_map_.end()) { return {}; } return it->second; } std::vector RuleManager::GetCircuitBreakers( const std::string& resource) const { absl::ReaderMutexLock lck(&update_mtx_); auto it = cb_map_.find(resource); if (it == cb_map_.end()) { return {}; } return it->second; } void RuleManager::RegisterToProperty(const RulePropertySharedPtr& property) { if (property == nullptr) { return; } std::lock_guard lck(property_mtx_); SENTINEL_LOG(info, "Registering new property to CircuitBreakerRuleManager"); cur_property_->RemoveListener(kPropertyListenerName); cur_property_ = property; cur_property_->AddListener(std::make_unique()); } // If the rule already exists, reuse the circuit breaker instance directly, // otherwise generate a new instance. CircuitBreakerSharedPtr RuleManager::GetExistingSameCbOrNew(const Rule& rule) { auto it = cb_map_.find(rule.resource()); if (it == cb_map_.end()) { return NewCircuitBreaker(rule); } auto cbs = it->second; if (cbs.empty()) { return NewCircuitBreaker(rule); } for (auto& cb : cbs) { if (rule == cb->GetRule()) { // Reuse the circuit breaker if the rule remains unchanged. return cb; } } return NewCircuitBreaker(rule); } CircuitBreakerSharedPtr RuleManager::NewCircuitBreaker(const Rule& rule) { switch (rule.strategy()) { case Strategy::kSlowRtRatio: return std::make_shared(rule); case Strategy::kErrorRatio: return std::make_shared(rule); default: return nullptr; } } void LogRuleMap(const std::unordered_map& map) { std::string s("["); for (const auto& e : map) { for (const auto& rule : e.second) { s += rule.ToString(); s += ","; } } s[s.size() - 1] = ']'; SENTINEL_LOG(info, "[CircuitBreakerRuleManager] Rules received: {}", s); } // RulePropertyListener CircuitBreakerMap RulePropertyListener::BuildCircuitBreakerMap( const RuleList& list) { CircuitBreakerMap m{}; if (list.empty()) { return m; } for (auto& rule : list) { if (!IsValidRule(rule)) { SENTINEL_LOG(warn, "[CircuitBreakerRuleManager] Ignoring invalid rule when " "loading new circuit breaker rules: {}", rule.ToString()); continue; } CircuitBreakerSharedPtr cb = RuleManager::GetInstance().GetExistingSameCbOrNew(rule); if (cb == nullptr) { SENTINEL_LOG(warn, "[CircuitBreakerRuleManager] Unknown circuit breaking " "strategy, ignoring: {}", rule.ToString()); continue; } auto it = m.find(rule.resource()); if (it == m.end()) { m.insert({rule.resource(), {cb}}); } else { auto& vec = it->second; vec.push_back(std::move(cb)); } } return m; } void RulePropertyListener::ConfigUpdate(const RuleList& value, bool) { RuleManager& m = RuleManager::GetInstance(); if (value.empty()) { absl::WriterMutexLock lck(&(m.update_mtx_)); m.rule_map_.clear(); m.cb_map_.clear(); SENTINEL_LOG(info, "[CircuitBreakerRuleManager] Rules received: []"); return; } absl::WriterMutexLock lck(&(m.update_mtx_)); auto cbs = BuildCircuitBreakerMap(value); std::unordered_map rule_map; for (const auto& kv : cbs) { if (kv.second.empty()) { continue; } RuleSet rules{}; for (const auto& cb : kv.second) { rules.insert(cb->GetRule()); } rule_map.insert({kv.first, rules}); } m.rule_map_ = std::move(rule_map); m.cb_map_ = std::move(cbs); LogRuleMap(m.rule_map_); } const std::string RulePropertyListener::Name() const { return kPropertyListenerName; } } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/rule_manager.h ================================================ #pragma once #include #include #include #include #include "absl/synchronization/mutex.h" #include "sentinel-core/circuitbreaker/circuit_breaker.h" #include "sentinel-core/circuitbreaker/rule.h" #include "sentinel-core/property/property_listener.h" #include "sentinel-core/property/sentinel_property.h" namespace Sentinel { namespace CircuitBreaker { using CircuitBreakerMap = std::unordered_map>; using RulePropertySharedPtr = Property::SentinelPropertySharedPtr; class RuleManager { public: static RuleManager& GetInstance() { static RuleManager* instance = new RuleManager(); return *instance; } friend class RulePropertyListener; void RegisterToProperty(const RulePropertySharedPtr& property); bool LoadRules(const RuleList& rules); bool HasRules(const std::string& resource); RuleList GetRules() const; RuleSet GetRulesOfResource(const std::string& resource) const; std::vector GetCircuitBreakers( const std::string& resource) const; private: RuleManager(); virtual ~RuleManager() = default; RulePropertySharedPtr cur_property_; std::unordered_map rule_map_{}; CircuitBreakerMap cb_map_{}; mutable std::mutex property_mtx_; mutable absl::Mutex update_mtx_; CircuitBreakerSharedPtr GetExistingSameCbOrNew(const Rule& rule); CircuitBreakerSharedPtr NewCircuitBreaker(const Rule& rule); }; class RulePropertyListener : public Property::PropertyListener { public: RulePropertyListener() = default; ~RulePropertyListener() = default; void ConfigUpdate(const RuleList& value, bool first_load) override; const std::string Name() const override; private: CircuitBreakerMap BuildCircuitBreakerMap(const RuleList& list); }; bool IsValidRule(const Rule& rule); } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/slot.cc ================================================ #include "sentinel-core/circuitbreaker/slot.h" #include "sentinel-core/circuitbreaker/rule_manager.h" #include "sentinel-core/utils/time_utils.h" namespace Sentinel { namespace CircuitBreaker { // CheckerSlot Sentinel::Slot::TokenResultSharedPtr CheckerSlot::Entry( const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { auto cbs = RuleManager::GetInstance().GetCircuitBreakers(entry->resource()->name()); if (cbs.empty()) { return Sentinel::Slot::TokenResult::Ok(); } for (auto& cb : cbs) { if (!cb->TryPass(node)) { return Sentinel::Slot::TokenResult::Blocked("DegradeException"); } } return Sentinel::Slot::TokenResult::Ok(); } void CheckerSlot::Exit(const EntrySharedPtr& entry, int count, const std::vector& params) {} // CompleteStatSlot Sentinel::Slot::TokenResultSharedPtr CompleteStatSlot::Entry( const EntrySharedPtr& entry, /*const*/ Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { if (entry == nullptr || entry->context() == nullptr) { return Sentinel::Slot::TokenResult::Ok(); } auto prev_result = entry->context()->last_token_result(); if (prev_result == nullptr) { return Sentinel::Slot::TokenResult::Ok(); } return prev_result; } void CompleteStatSlot::Exit(const EntrySharedPtr& entry, int count, const std::vector& params) { if (entry == nullptr || entry->HasBlockError()) { return; } auto cbs = RuleManager::GetInstance().GetCircuitBreakers(entry->resource()->name()); if (cbs.empty()) { return; } auto rt = entry->rt(); if (rt < 0) { rt = std::max(int64_t(0), Utils::TimeUtils::CurrentTimeMillis().count() - entry->create_time().count()); } for (auto& cb : cbs) { cb->RecordComplete(rt, entry->error()); } } } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/slot.h ================================================ #pragma once #include "sentinel-core/slot/base/rule_checker_slot.h" #include "sentinel-core/slot/base/stats_slot.h" namespace Sentinel { namespace CircuitBreaker { constexpr auto kCheckerSlotName = "CircuitBreakerCheckerSlot"; constexpr auto kCompleteStatSlotName = "CircuitBreakerCompleteStatSlot"; class CheckerSlot : public Slot::RuleCheckerSlot { public: CheckerSlot() = default; virtual ~CheckerSlot() = default; Sentinel::Slot::TokenResultSharedPtr Entry( const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; const std::string& Name() const override { return name_; }; private: const std::string name_{kCheckerSlotName}; }; class CompleteStatSlot : public Slot::StatsSlot { public: CompleteStatSlot() = default; virtual ~CompleteStatSlot() = default; const std::string& Name() const override { return name_; }; Sentinel::Slot::TokenResultSharedPtr Entry( const EntrySharedPtr& entry, /*const*/ Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; private: const std::string name_{kCompleteStatSlotName}; }; } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/circuitbreaker/slot_test.cc ================================================ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/test/mock/statistic/node/mock.h" #include "sentinel-core/circuitbreaker/rule.h" #include "sentinel-core/circuitbreaker/rule_manager.h" #include "sentinel-core/circuitbreaker/slot.h" #include "sentinel-core/common/string_resource_wrapper.h" using testing::_; using testing::InSequence; using testing::Mock; using testing::Return; namespace Sentinel { namespace CircuitBreaker { Sentinel::Slot::TokenResultSharedPtr Entry_And_Exit( CheckerSlot& slot_checker, CompleteStatSlot& slot_complete, const EntrySharedPtr& entry, const ResourceWrapperSharedPtr& resource, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { auto result = slot_checker.Entry(entry, node, count, flag, params); EXPECT_EQ(Sentinel::Slot::TokenStatus::RESULT_STATUS_OK, result->status()); slot_complete.Entry(entry, node, count, flag, params); slot_complete.Exit(entry, count, params); return result; } TEST(CircuitBreakerSlotTest, CircuitBreakerErrorRatioTest) { std::string resource_name{"test_resource"}; EntryContextSharedPtr context = std::make_shared("test_context"); Stat::NodeSharedPtr node = std::make_shared(); auto resource = std::make_shared(resource_name, EntryType::OUT); auto entry = std::make_shared(resource, context); entry->set_cur_node(node); auto entry_error = std::make_shared(resource, context); entry_error->set_cur_node(node); entry_error->set_error("test_error"); const std::vector myParams; CheckerSlot slot_checker; CompleteStatSlot slot_complete; // Test breaker checking when no rule exists. for (int i = 0; i < 10; i++) { Entry_And_Exit(slot_checker, slot_complete, entry, resource, node, 1, 0, myParams); } Rule rule{resource_name}; rule.set_strategy(Strategy::kErrorRatio); rule.set_threshold(0.5); rule.set_retry_timeout_sec(1); rule.set_stat_interval_ms(2000); rule.set_min_request_amount(10); RuleList rules{rule}; RuleManager& m = RuleManager::GetInstance(); m.LoadRules(rules); auto cbs = m.GetCircuitBreakers(resource->name()); EXPECT_EQ(cbs[0]->CurrentState(), State::kClosed); // Test breaker checking when error entry happens. for (int i = 0; i < 10; i++) { Entry_And_Exit(slot_checker, slot_complete, entry_error, resource, node, 1, 0, myParams); } EXPECT_EQ(cbs[0]->CurrentState(), State::kOpen); // skip break time span sleep(1); // Switch state to kHalfOpen auto result = slot_checker.Entry(entry, node, 1, 0, myParams); EXPECT_EQ(Sentinel::Slot::TokenStatus::RESULT_STATUS_OK, result->status()); EXPECT_EQ(cbs[0]->CurrentState(), State::kHalfOpen); // Switch state to kClosed slot_complete.Entry(entry, node, 1, 0, myParams); slot_complete.Exit(entry, 1, myParams); EXPECT_EQ(cbs[0]->CurrentState(), State::kClosed); m.LoadRules({}); } TEST(CircuitBreakerSlotTest, CircuitBreakerSlowRatioTest) { std::string resource_name{"test_resource"}; EntryContextSharedPtr context = std::make_shared("test_context"); Stat::NodeSharedPtr node = std::make_shared(); auto resource = std::make_shared(resource_name, EntryType::OUT); auto entry = std::make_shared(resource, context); entry->set_cur_node(node); entry->set_rt(10); auto entry_slow = std::make_shared(resource, context); entry_slow->set_cur_node(node); entry_slow->set_rt(500); const std::vector myParams; CheckerSlot slot_checker; CompleteStatSlot slot_complete; // Test breaker checking when no rule exists. for (int i = 0; i < 10; i++) { Entry_And_Exit(slot_checker, slot_complete, entry, resource, node, 1, 0, myParams); } Rule rule{resource_name}; rule.set_strategy(Strategy::kSlowRtRatio); rule.set_threshold(0.5); rule.set_retry_timeout_sec(1); rule.set_stat_interval_ms(10000); rule.set_min_request_amount(10); rule.set_max_allowed_rt(200); RuleList rules{rule}; RuleManager& m = RuleManager::GetInstance(); m.LoadRules(rules); auto cbs = m.GetCircuitBreakers(resource->name()); EXPECT_EQ(cbs[0]->CurrentState(), State::kClosed); // Test breaker checking when slow entry happens. for (int i = 0; i < 10; i++) { Entry_And_Exit(slot_checker, slot_complete, entry_slow, resource, node, 1, 0, myParams); } EXPECT_EQ(cbs[0]->CurrentState(), State::kOpen); // skip break time span sleep(1); // Switch state to kHalfOpen auto result = slot_checker.Entry(entry, node, 1, 0, myParams); EXPECT_EQ(Sentinel::Slot::TokenStatus::RESULT_STATUS_OK, result->status()); EXPECT_EQ(cbs[0]->CurrentState(), State::kHalfOpen); // Switch state to kClosed slot_complete.Entry(entry, node, 1, 0, myParams); slot_complete.Exit(entry, 1, myParams); EXPECT_EQ(cbs[0]->CurrentState(), State::kClosed); m.LoadRules({}); } } // namespace CircuitBreaker } // namespace Sentinel ================================================ FILE: sentinel-core/common/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "constants", srcs = [ "constants.h", ], copts = DEFAULT_COPTS, ) cc_library( name = "global_status", srcs = [ "global_status.h", "global_status.cc", ], copts = DEFAULT_COPTS, ) cc_library( name = "entry_type_enum", srcs = [ "entry_type.h", ], copts = DEFAULT_COPTS, ) cc_library( name = "resource_wrapper_interface", srcs = [ "resource_wrapper.h", ], copts = DEFAULT_COPTS, deps = [ ":entry_type_enum", "@com_google_absl//absl/types:any", ] ) cc_library( name = "string_resource_wrapper_lib", srcs = [ "string_resource_wrapper.h", ], copts = DEFAULT_COPTS, deps = [ ":resource_wrapper_interface", "@com_google_absl//absl/types:any", ] ) cc_library( name = "entry_context_lib", srcs = [ "entry_context.h", "entry_context.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/statistic/node:node_interface", "//sentinel-core/slot/base:token_result_lib", ] ) cc_library( name = "entry_lib", srcs = [ "entry.h", ], copts = DEFAULT_COPTS, deps = [ ":entry_context_lib", ":resource_wrapper_interface", "//sentinel-core/utils:utils_lib", ] ) cc_library( name = "rule_lib", srcs = [ "rule.h", ], copts = DEFAULT_COPTS, deps = [ ":constants", ] ) cc_library( name = "entry_result_lib", srcs = [ "entry_result.h", "entry_result.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/slot:global_slot_chain_header", "@com_google_absl//absl/types:optional", ] ) cc_library( name = "tracer_lib", srcs = [ "tracer.h", "tracer.cc", ], copts = DEFAULT_COPTS, deps = [ ":entry_lib", ] ) cc_test( name = "tracer_lib_unittests", srcs = [ "tracer_test.cc", ], copts = TEST_COPTS, deps = [ ":tracer_lib", ":entry_lib", ":string_resource_wrapper_lib", "//sentinel-core/test/mock/statistic/node:mock_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/common/constants.h ================================================ #pragma once namespace Sentinel { namespace Constants { static constexpr int kDefaultSampleCount = 2; static constexpr int kDefaultIntervalMs = 1000; static constexpr const char* kLimitOriginDefault = "default"; static constexpr const char* kLimitOriginOther = "other"; static constexpr const char* kDefaultContextName = "sentinel_default_context"; static constexpr int kMaxAllowedRt = 4900; static constexpr int kMaxResourceSize = 10000; static constexpr int kMaxTagSize = 1000; }; // namespace Constants } // namespace Sentinel ================================================ FILE: sentinel-core/common/entry.h ================================================ #pragma once #include #include #include #include "sentinel-core/common/entry_context.h" #include "sentinel-core/common/resource_wrapper.h" #include "sentinel-core/utils/time_utils.h" namespace Sentinel { class Entry { public: explicit Entry(const ResourceWrapperSharedPtr& resource, const EntryContextSharedPtr& context) : resource_(resource), context_(context), create_time_(Utils::TimeUtils::CurrentTimeMillis()) {} virtual ~Entry() = default; friend class EntryResult; friend class SphU; ResourceWrapperSharedPtr resource() const { return resource_; } std::chrono::milliseconds create_time() const { return create_time_; } EntryContextSharedPtr context() const { return context_; } Stat::NodeSharedPtr cur_node() const { return cur_node_; } int64_t rt() const { return rt_; } std::vector params() const { return params_; } bool exited() const { return exited_; } std::string error() const { return error_; } bool HasError() const { return !error_.empty(); }; bool HasBlockError() const { return !block_error_.empty(); }; void set_rt(int64_t rt) { rt_ = rt; } void set_error(const std::string& message) { error_ = message; } void set_block_error(const std::string& message) { block_error_ = message; } void set_cur_node(const Stat::NodeSharedPtr& node) { cur_node_ = node; } void set_params(const std::vector&& params) { params_ = params; } private: const ResourceWrapperSharedPtr resource_; EntryContextSharedPtr context_; const std::chrono::milliseconds create_time_; bool exited_{false}; int64_t rt_{-1}; std::string error_{}; std::string block_error_{}; Stat::NodeSharedPtr cur_node_; std::vector params_; }; using EntrySharedPtr = std::shared_ptr; } // namespace Sentinel ================================================ FILE: sentinel-core/common/entry_context.cc ================================================ #include #include "sentinel-core/common/entry_context.h" namespace Sentinel {} // namespace Sentinel ================================================ FILE: sentinel-core/common/entry_context.h ================================================ #pragma once #include #include #include "sentinel-core/slot/base/token_result.h" #include "sentinel-core/statistic/node/node.h" namespace Sentinel { class EntryContext { public: explicit EntryContext(const std::string& name) : EntryContext(name, "") {} EntryContext(const std::string& name, const std::string& tag) : name_(name), tag_(tag) {} const std::string& name() const { return name_; }; const std::string& tag() const { return tag_; }; Stat::NodeSharedPtr tag_node() { return tag_node_; } const Slot::TokenResultSharedPtr& last_token_result() const { return last_token_result_; } /** * Users should not invoke this. */ void set_last_token_result(const Slot::TokenResultSharedPtr& r) { last_token_result_ = r; } void set_tag_node(Stat::NodeSharedPtr& node) { tag_node_ = node; } private: const std::string name_; // Maybe multiple tags in the future, using vector instead of string. const std::string tag_; Stat::NodeSharedPtr tag_node_; Slot::TokenResultSharedPtr last_token_result_; }; using EntryContextSharedPtr = std::shared_ptr; } // namespace Sentinel ================================================ FILE: sentinel-core/common/entry_node.h ================================================ ================================================ FILE: sentinel-core/common/entry_result.cc ================================================ #include "sentinel-core/common/entry_result.h" #include "sentinel-core/slot/global_slot_chain.h" namespace Sentinel { bool EntryResult::IsBlocked() const { return blocked_reason_.has_value(); } bool EntryResult::Exit() { return Exit(1); } bool EntryResult::Exit(int count) { if (entry_ == nullptr) { return false; } const std::vector params = entry_->params(); if (!entry_->exited()) { Slot::SlotChainSharedPtr chain = Slot::GetGlobalSlotChain(); if (chain != nullptr) { // NOTE: keep consistent with exit operation in SphU::Entry when blocked. chain->Exit(entry_, count, params); } entry_->exited_ = true; return true; } return false; } void EntryResult::SetError(const std::string& err) { if (entry_ != nullptr) { entry_->set_error(err); } } } // namespace Sentinel ================================================ FILE: sentinel-core/common/entry_result.h ================================================ #pragma once #include #include #include #include "absl/types/optional.h" #include "sentinel-core/common/entry_result.h" #include "sentinel-core/slot/global_slot_chain.h" namespace Sentinel { class EntryResult { public: explicit EntryResult(const EntrySharedPtr& entry) : entry_(entry) {} explicit EntryResult(const std::string& reason) : entry_(nullptr), blocked_reason_(reason) {} ~EntryResult() = default; EntrySharedPtr entry() const { return entry_; }; absl::optional blocked_reason() const { return blocked_reason_; }; bool IsBlocked() const; bool Exit(); bool Exit(int count); bool Exit(int count, const std::vector& params); void SetError(const std::string& err); private: const EntrySharedPtr entry_; const absl::optional blocked_reason_; }; using EntryResultPtr = std::unique_ptr; } // namespace Sentinel ================================================ FILE: sentinel-core/common/entry_type.h ================================================ #pragma once namespace Sentinel { enum class EntryType { IN, OUT }; } // namespace Sentinel ================================================ FILE: sentinel-core/common/global_status.cc ================================================ namespace Sentinel { namespace GlobalStatus { bool activated = true; }; // namespace GlobalStatus } // namespace Sentinel ================================================ FILE: sentinel-core/common/global_status.h ================================================ #pragma once namespace Sentinel { namespace GlobalStatus { extern bool activated; }; // namespace GlobalStatus } // namespace Sentinel ================================================ FILE: sentinel-core/common/resource_wrapper.h ================================================ #pragma once #include #include #include "absl/types/any.h" #include "sentinel-core/common/entry_type.h" namespace Sentinel { class ResourceWrapper; using ResourceWrapperSharedPtr = std::shared_ptr; class ResourceWrapper { public: virtual ~ResourceWrapper() = default; virtual const std::string& name() const = 0; virtual EntryType entry_type() const = 0; }; } // namespace Sentinel ================================================ FILE: sentinel-core/common/rule.h ================================================ #pragma once #include #include "sentinel-core/common/constants.h" namespace Sentinel { class Rule { public: Rule() = default; virtual ~Rule() = default; static bool LimitOriginEquals(const std::string& lhs, const std::string& rhs) { if (lhs.empty()) { return rhs.empty() || rhs == Constants::kLimitOriginDefault; } else if (lhs == Constants::kLimitOriginDefault) { return rhs.empty() || rhs == Constants::kLimitOriginDefault; } return lhs == rhs; } }; } // namespace Sentinel ================================================ FILE: sentinel-core/common/string_resource_wrapper.h ================================================ #pragma once #include #include "sentinel-core/common/resource_wrapper.h" namespace Sentinel { class StringResourceWrapper : public ResourceWrapper { public: StringResourceWrapper(const std::string& name, EntryType type) : name_(name), entry_type_(type) {} virtual ~StringResourceWrapper() = default; const std::string& name() const override { return name_; } EntryType entry_type() const override { return entry_type_; } private: std::string name_; EntryType entry_type_; }; } // namespace Sentinel ================================================ FILE: sentinel-core/common/tracer.cc ================================================ #include #include "sentinel-core/common/tracer.h" #include "sentinel-core/statistic/node/node.h" namespace Sentinel { void Tracer::Trace(const EntrySharedPtr entry, const std::string&, int count) { if (entry == nullptr || count <= 0) { return; } // TODO: check BlockException? Stat::NodeSharedPtr node = entry->cur_node(); if (node != nullptr) { node->AddExceptionRequest(count); } } void Tracer::Trace(const EntrySharedPtr entry, const std::string& message) { Tracer::Trace(entry, message, 1); } } // namespace Sentinel ================================================ FILE: sentinel-core/common/tracer.h ================================================ #pragma once #include #include "sentinel-core/common/entry.h" namespace Sentinel { class Tracer { public: Tracer() = delete; static void Trace(const EntrySharedPtr entry, const std::string& message); static void Trace(const EntrySharedPtr entry, const std::string& message, int count); }; } // namespace Sentinel ================================================ FILE: sentinel-core/common/tracer_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/common/entry.h" #include "sentinel-core/common/string_resource_wrapper.h" #include "sentinel-core/common/tracer.h" #include "sentinel-core/test/mock/statistic/node/mock.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { TEST(TracerTest, TraceExceptionTest) { { auto node = std::make_shared(); int n = 10; EXPECT_CALL(*node.get(), AddExceptionRequest(n)).Times(1); EntrySharedPtr entry = std::make_shared( std::make_shared("abc", EntryType::IN), nullptr); entry->set_cur_node(node); Tracer::Trace(entry, "some_exception", n); } } } // namespace Sentinel ================================================ FILE: sentinel-core/config/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "config_constants", srcs = [ "config_constants.h", ], copts = DEFAULT_COPTS, ) cc_library( name = "local_config_lib", srcs = [ "local_config.h", "local_config.cc", ], copts = DEFAULT_COPTS, deps = [ ":config_constants", "//sentinel-core/init:init_target_interface", "//sentinel-core/log:logger_lib", "@com_google_absl//absl/strings", ] ) cc_test( name = "local_config_unittests", srcs = [ "local_config_test.cc", ], copts = TEST_COPTS, deps = [ ":local_config_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/config/config_constants.h ================================================ #pragma once #include namespace Sentinel { namespace Config { const auto kUnknownAppName = "unknown_cpp_service"; constexpr uint32_t kDefaultAppType = 0; constexpr auto kDefaultCharset = "UTF-8"; constexpr uint32_t kDefaultSingleMetricFileSize = 1024 * 1024 * 50; const uint32_t kDefaultTotalMetricFileCount = 6; const uint32_t kDefaultWarmUpColdFactor = 3; const uint32_t kDefaultStatisticMaxRt = 4900; namespace Env { constexpr auto kAppNameKey = "CSP_SENTINEL_APP_NAME"; constexpr auto kAppTypeKey = "CSP_SENTINEL_APP_TYPE"; constexpr auto kCharsetKey = "CSP_SENTINEL_CHARSET"; constexpr auto kSingleMetricFileSizeKey = "CSP_SENTINEL_METRIC_FILE_SINGLE_SIZE"; constexpr auto kTotalMetricFileCountKey = "CSP_SENTINEL_METRIC_FILE_TOTAL_COUNT"; constexpr auto kWarmUpColdFactorKey = "CSP_SENTINEL_FLOW_WARMUP_COLD_FACTOR"; constexpr auto kStatisticMaxRtKey = "CSP_SENTINEL_STATISTIC_MAX_RT"; } // namespace Env namespace Args { constexpr auto kAppNameKey = "project.name"; constexpr auto kAppTypeKey = "csp.sentinel.app.type"; constexpr auto kCharsetKey = "csp.sentinel.charset"; constexpr auto kSingleMetricFileSizeKey = "csp.sentinel.metric.file.single.size"; constexpr auto kTotalMetricFileCountKey = "csp.sentinel.metric.file.total.count"; constexpr auto kWarmUpColdFactorKey = "csp.sentinel.flow.cold.factor"; constexpr auto kStatisticMaxRtKey = "csp.sentinel.statistic.max.rt"; } // namespace Args } // namespace Config } // namespace Sentinel ================================================ FILE: sentinel-core/config/local_config.cc ================================================ #include #include "absl/strings/numbers.h" #include "sentinel-core/config/config_constants.h" #include "sentinel-core/config/local_config.h" #include "sentinel-core/log/logger.h" namespace Sentinel { namespace Config { LocalConfig::LocalConfig() { // Initialize on create. this->Initialize(); } void LocalConfig::SetConfig(const std::string& key, const std::string& value) { config_map_.emplace(std::make_pair(key, value)); } void LocalConfig::SetConfigIfNotExists(const std::string& key, const std::string& value) { auto iter = config_map_.find(key); if (iter != config_map_.end()) { return; } SetConfig(key, value); } void LocalConfig::RemoveConfig(const std::string& key) { config_map_.erase(key); } const std::string LocalConfig::GetConfig(const std::string& key) const { auto iter = config_map_.find(key); if (iter == config_map_.end()) { return std::string(); } return iter->second; } int32_t LocalConfig::GetInt32(const std::string& key, int32_t default_value) const { auto v = GetConfig(key); if (v.empty()) { return default_value; } int32_t x; if (!absl::SimpleAtoi(v, &x)) { x = default_value; } return x; } int64_t LocalConfig::GetInt64(const std::string& key, int64_t default_value) const { auto v = GetConfig(key); if (v.empty()) { return default_value; } int64_t x; if (!absl::SimpleAtoi(v, &x)) { x = default_value; } return x; } int32_t LocalConfig::WarmUpColdFactor() const { int cold_factor = GetInt32(Env::kWarmUpColdFactorKey, kDefaultWarmUpColdFactor); if (cold_factor <= 1) { SENTINEL_LOG( info, "Invalid cold_factor <{}>, fallback with the default cold factor <{}>", cold_factor, kDefaultWarmUpColdFactor); cold_factor = kDefaultWarmUpColdFactor; } return cold_factor; } int32_t LocalConfig::StatisticMaxRt() const { int max_rt = GetInt32(Env::kStatisticMaxRtKey, kDefaultStatisticMaxRt); if (max_rt < 0) { max_rt = kDefaultStatisticMaxRt; } return max_rt; } int32_t LocalConfig::TotalMetricFileCount() const { return GetInt32(Env::kTotalMetricFileCountKey, kDefaultTotalMetricFileCount); } int64_t LocalConfig::SingleMetricFileSize() const { return GetInt64(Env::kSingleMetricFileSizeKey, kDefaultSingleMetricFileSize); } const std::string LocalConfig::Charset() const { return GetConfig(Env::kCharsetKey); } void LocalConfig::ResolveAppName() { const char* app_name_env = std::getenv(Env::kAppNameKey); if (app_name_env) { app_name_ = app_name_env; SENTINEL_LOG(info, "App name resolved: {}", app_name_); } else { app_name_ = kUnknownAppName; SENTINEL_LOG(warn, "No {} configured, using the fallback app name: {}", Env::kAppNameKey, kUnknownAppName); } } void LocalConfig::Initialize() { ResolveAppName(); config_map_.emplace(std::make_pair(Env::kCharsetKey, kDefaultCharset)); config_map_.emplace( std::make_pair(Env::kSingleMetricFileSizeKey, std::to_string(kDefaultSingleMetricFileSize))); config_map_.emplace( std::make_pair(Env::kTotalMetricFileCountKey, std::to_string(kDefaultTotalMetricFileCount))); config_map_.emplace(std::make_pair(Env::kWarmUpColdFactorKey, std::to_string(kDefaultWarmUpColdFactor))); config_map_.emplace(std::make_pair(Env::kStatisticMaxRtKey, std::to_string(kDefaultStatisticMaxRt))); } } // namespace Config } // namespace Sentinel ================================================ FILE: sentinel-core/config/local_config.h ================================================ #pragma once #include #include #include "sentinel-core/init/init_target.h" namespace Sentinel { namespace Config { class LocalConfig { public: ~LocalConfig() = default; static LocalConfig& GetInstance() { static LocalConfig* instance = new LocalConfig(); return *instance; } const std::string GetConfig(const std::string& key) const; void SetConfig(const std::string& key, const std::string& value); void SetConfigIfNotExists(const std::string& key, const std::string& value); void RemoveConfig(const std::string& key); int32_t GetInt32(const std::string& key, int32_t default_value) const; int64_t GetInt64(const std::string& key, int64_t default_value) const; const std::string& app_name() const { return app_name_; } void set_app_name(const std::string& app_name) { app_name_ = app_name; } int32_t WarmUpColdFactor() const; int32_t StatisticMaxRt() const; int32_t TotalMetricFileCount() const; int64_t SingleMetricFileSize() const; const std::string Charset() const; private: std::unordered_map config_map_; std::string app_name_; LocalConfig(); void ResolveAppName(); void Initialize(); }; } // namespace Config } // namespace Sentinel ================================================ FILE: sentinel-core/config/local_config_test.cc ================================================ #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/config/config_constants.h" #include "sentinel-core/config/local_config.h" namespace Sentinel { namespace Config { TEST(LocalConfigTest, TestResolveNormalAppName) { auto app_name = "test_app"; setenv(Env::kAppNameKey, app_name, 1); LocalConfig config = LocalConfig::GetInstance(); EXPECT_EQ(app_name, config.app_name()); unsetenv(Env::kAppNameKey); } TEST(LocalConfigTest, TestGetIntOfInvalidValue) { LocalConfig config = LocalConfig::GetInstance(); constexpr auto key = "some_key_bad"; config.SetConfig(key, "a32"); constexpr int32_t d = 32; EXPECT_EQ(d, config.GetInt32(key, d)); } TEST(LocalConfigTest, TestGetIntOfNormalValue) { LocalConfig config = LocalConfig::GetInstance(); constexpr auto key = "some_key_good"; config.SetConfig(key, "64"); constexpr int32_t d = 32; EXPECT_EQ(64, config.GetInt32(key, d)); } } // namespace Config } // namespace Sentinel ================================================ FILE: sentinel-core/flow/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "flow_rule_constants_lib", srcs = [ "flow_rule_constants.h", ], copts = DEFAULT_COPTS ) cc_library( name = "flow_rule_lib", srcs = [ "flow_rule.h", "flow_rule.cc", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/strings:str_format", ":flow_rule_constants_lib", "//sentinel-core/common:rule_lib", ] ) cc_library( name = "flow_rule_manager_lib", srcs = [ "flow_rule_manager.h", "flow_rule_manager.cc", ], copts = DEFAULT_COPTS, deps = [ ":flow_rule_constants_lib", ":flow_rule_lib", ":traffic_shaping_controller_lib", ":default_traffic_shaping_calculator_lib", ":default_traffic_shaping_checker_lib", "//sentinel-core/property:property_listener_interface", "//sentinel-core/property:dynamic_sentinel_property_lib", "@com_google_absl//absl/synchronization", ] ) cc_library( name = "flow_rule_checker_lib", srcs = [ "flow_rule_checker.h", "flow_rule_checker.cc", ], copts = DEFAULT_COPTS, deps = [ ":flow_rule_manager_lib", "//sentinel-core/common:entry_lib", "//sentinel-core/statistic/node:resource_node_storage_lib", ] ) cc_library( name = "flow_slot_lib", srcs = [ "flow_slot.h", "flow_slot.cc", ], copts = DEFAULT_COPTS, deps = [ ":flow_rule_checker_lib", "//sentinel-core/slot/base:rule_checker_slot_interface", ] ) cc_library( name = "traffic_shaping_calculator_interface", srcs = [ "traffic_shaping_calculator.h", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/statistic/node:node_interface", ] ) cc_library( name = "traffic_shaping_checker_interface", srcs = [ "traffic_shaping_checker.h", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/statistic/node:node_interface", "//sentinel-core/slot/base:token_result_lib" ] ) cc_library( name = "default_traffic_shaping_calculator_lib", srcs = [ "default_traffic_shaping_calculator.h", "default_traffic_shaping_calculator.cc", ], copts = DEFAULT_COPTS, deps = [ ":traffic_shaping_calculator_interface", ] ) cc_library( name = "default_traffic_shaping_checker_lib", srcs = [ "default_traffic_shaping_checker.h", "default_traffic_shaping_checker.cc", ], copts = DEFAULT_COPTS, deps = [ ":flow_rule_constants_lib", ":traffic_shaping_checker_interface", ] ) cc_library( name = "throttling_traffic_shaping_checker_lib", srcs = [ "throttling_traffic_shaping_checker.h", "throttling_traffic_shaping_checker.cc", ], copts = DEFAULT_COPTS, deps = [ ":flow_rule_constants_lib", ":traffic_shaping_checker_interface", "//sentinel-core/utils:utils_lib", ] ) cc_library( name = "traffic_shaping_controller_lib", srcs = [ "traffic_shaping_controller.h", "traffic_shaping_controller.cc", ], copts = DEFAULT_COPTS, deps = [ ":flow_rule_constants_lib", ":traffic_shaping_checker_interface", ":traffic_shaping_calculator_interface", ] ) cc_test( name = "traffic_shaping_controller_unittests", srcs = [ "traffic_shaping_controller_test.cc", ], copts = TEST_COPTS, deps = [ ":default_traffic_shaping_calculator_lib", ":default_traffic_shaping_checker_lib", ":traffic_shaping_controller_lib", "//sentinel-core/test/mock/flow:flow_mock_lib", "//sentinel-core/test/mock/statistic/node:mock_lib", "@com_google_googletest//:gtest_main", ] ) cc_test( name = "flow_slot_unittests", srcs = [ "flow_slot_test.cc", ], copts = TEST_COPTS, deps = [ ":flow_slot_lib", "//sentinel-core/common:string_resource_wrapper_lib", "//sentinel-core/test/mock/statistic/node:mock_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/flow/default_traffic_shaping_calculator.cc ================================================ #include "sentinel-core/flow/default_traffic_shaping_calculator.h" namespace Sentinel { namespace Flow { double DefaultTrafficShapingCalculator::CalculateAllowedTokens( const Stat::NodeSharedPtr&, int, int) { return threshold_; } } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/default_traffic_shaping_calculator.h ================================================ #pragma once #include "sentinel-core/flow/traffic_shaping_calculator.h" namespace Sentinel { namespace Flow { class DefaultTrafficShapingCalculator : public TrafficShapingCalculator { public: DefaultTrafficShapingCalculator(double t) : threshold_(t) {} virtual ~DefaultTrafficShapingCalculator() = default; double CalculateAllowedTokens(const Stat::NodeSharedPtr& node, int acquire_count, int flag) override; private: const double threshold_; }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/default_traffic_shaping_checker.cc ================================================ #include "sentinel-core/flow/default_traffic_shaping_checker.h" #include "sentinel-core/flow/flow_rule_constants.h" namespace Sentinel { namespace Flow { Slot::TokenResultSharedPtr DefaultTrafficShapingChecker::DoCheck( const Stat::NodeSharedPtr& node, int acquire_count, double threshold) { double cur_pass = 0; if (node != nullptr) { cur_pass = mode_ == (int)FlowMetricType::kThreadCount ? node->CurThreadNum() : node->PassQps(); } if (cur_pass + acquire_count > threshold) { return Slot::TokenResult::Blocked("FlowException"); } return Slot::TokenResult::Ok(); } } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/default_traffic_shaping_checker.h ================================================ #pragma once #include "sentinel-core/flow/flow_rule_constants.h" #include "sentinel-core/flow/traffic_shaping_checker.h" namespace Sentinel { namespace Flow { class DefaultTrafficShapingChecker : public TrafficShapingChecker { public: explicit DefaultTrafficShapingChecker(int mode) : mode_(mode) {} explicit DefaultTrafficShapingChecker(FlowMetricType mode) : mode_((int)mode) {} virtual ~DefaultTrafficShapingChecker() = default; Slot::TokenResultSharedPtr DoCheck(const Stat::NodeSharedPtr& node, int acquire_count, double threshold) override; private: const int mode_; }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_rule.cc ================================================ #include #include "sentinel-core/flow/flow_rule.h" #include "absl/strings/str_format.h" namespace Sentinel { namespace Flow { bool FlowRule::operator==(const FlowRule& rule) const { return resource_ == rule.resource() && Rule::LimitOriginEquals(limit_origin_, rule.limit_origin()) && metric_type_ == rule.metric_type() && count_ == rule.count() && strategy_ == rule.strategy() && ref_resource_ == rule.ref_resource() && control_behavior_ == rule.control_behavior() && warm_up_period_sec_ == rule.warm_up_period_sec() && max_queueing_time_ms_ == rule.max_queueing_time_ms() && cluster_mode_ == rule.cluster_mode(); } std::string FlowRule::ToString() const { return absl::StrFormat( "FlowRule{resource=%s, limit_origin=%s, metric_type=%d, count=%.2f, " "strategy=%d, ref_resource=%s, control_behavior=%d, " "warm_up_period_sec=%d, max_queueing_time_ms=%d, cluster_mode=%d}", resource_, limit_origin_, static_cast(metric_type_), count_, static_cast(strategy_), ref_resource_, static_cast(control_behavior_), warm_up_period_sec_, max_queueing_time_ms_, cluster_mode_); } } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_rule.h ================================================ #pragma once #include #include #include #include #include "sentinel-core/common/constants.h" #include "sentinel-core/common/rule.h" #include "sentinel-core/flow/flow_rule_constants.h" namespace Sentinel { namespace Flow { struct FlowRule : public Rule { public: FlowRule() = default; virtual ~FlowRule() = default; explicit FlowRule(const std::string& resource) : resource_(resource), limit_origin_(Constants::kLimitOriginDefault) {} FlowRule(const std::string& resource, const std::string& limit_origin) : resource_(resource), limit_origin_(limit_origin) {} const std::string& resource() const { return resource_; } const std::string& limit_origin() const { return limit_origin_; } FlowMetricType metric_type() const { return metric_type_; } double count() const { return count_; } FlowRelationStrategy strategy() const { return strategy_; } const std::string& ref_resource() const { return ref_resource_; } FlowControlBehavior control_behavior() const { return control_behavior_; } int32_t warm_up_period_sec() const { return warm_up_period_sec_; } int32_t max_queueing_time_ms() const { return max_queueing_time_ms_; } bool cluster_mode() const { return cluster_mode_; } void set_resource(const std::string& resource) { resource_ = resource; } void set_limit_origin(const std::string& limit_origin) { limit_origin_ = limit_origin; } void set_limit_origin(const char* limit_origin) { if (limit_origin != nullptr) { limit_origin_ = limit_origin; } } void set_metric_type(FlowMetricType metric_type) { metric_type_ = metric_type; } void set_count(double count) { count_ = count; } void set_strategy(FlowRelationStrategy strategy) { strategy_ = strategy; } void set_ref_resource(const std::string& r) { ref_resource_ = r; } void set_control_behavior(FlowControlBehavior cb) { control_behavior_ = cb; } void set_warm_up_period_sec(int32_t w) { warm_up_period_sec_ = w; } void set_max_queueing_time_ms(int32_t q) { max_queueing_time_ms_ = q; } void set_cluster_mode(bool cluster_mode) { cluster_mode_ = cluster_mode; } bool operator==(const FlowRule& rule) const; std::string ToString() const; private: std::string resource_; // resource std::string limit_origin_{Constants::kLimitOriginDefault}; // limitApp FlowMetricType metric_type_{FlowMetricType::kQps}; // grade double count_ = 0; // count FlowRelationStrategy strategy_{FlowRelationStrategy::kDirect}; // strategy FlowControlBehavior control_behavior_{ FlowControlBehavior::kReject}; // controlBehavior std::string ref_resource_{}; // refResource int32_t warm_up_period_sec_ = 10; // warmUpPeriodSec int32_t max_queueing_time_ms_ = 500; // maxQueueingTimeMs bool cluster_mode_ = false; // clusterMode }; using FlowRulePtr = std::shared_ptr; using FlowRuleList = std::vector; struct FlowRuleHash { std::size_t operator()(const FlowRule& rule) const noexcept { std::size_t result = std::hash{}(rule.resource()); const std::string& limit_origin = rule.limit_origin(); if (!limit_origin.empty() && limit_origin != Constants::kLimitOriginDefault) { result = 31 * result + std::hash{}(limit_origin); } result = 31 * result + static_cast(rule.metric_type()); result = 31 * result + std::hash{}(rule.count()); result = 31 * result + static_cast(rule.strategy()); result = 31 * result + static_cast(rule.control_behavior()); result = 31 * result + std::hash{}(rule.ref_resource()); result = 31 * result + rule.warm_up_period_sec(); result = 31 * result + rule.max_queueing_time_ms(); result = 31 * result + std::hash{}(rule.cluster_mode()); return result; } }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_rule_checker.cc ================================================ #include "sentinel-core/flow/flow_rule_checker.h" #include "sentinel-core/flow/flow_rule_constants.h" #include "sentinel-core/flow/flow_rule_manager.h" #include "sentinel-core/statistic/node/resource_node_storage.h" namespace Sentinel { namespace Flow { Slot::TokenResultSharedPtr FlowRuleChecker::CanPassCheck( const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node, int count, int flag) { if (rule.limit_origin().empty()) { return Slot::TokenResult::Ok(); } // if (rule.cluster_mode()) { // return PassClusterCheck(); // } return PassLocalCheck(rule, entry, node, count, flag); } Slot::TokenResultSharedPtr FlowRuleChecker::CanPassCheck( const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node, int count) { return CanPassCheck(rule, entry, node, count, 0); } Slot::TokenResultSharedPtr FlowRuleChecker::PassLocalCheck( const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node, int count, int flag) { Stat::NodeSharedPtr selected_node = selectNodeByRequesterAndStrategy(rule, entry, node); if (selected_node == nullptr) { return Slot::TokenResult::Ok(); } auto controller = FlowRuleManager::GetInstance().GetTrafficControllerFor(rule); if (controller == nullptr) { return Slot::TokenResult::Ok(); } return controller->CanPass(selected_node, count, flag); } Stat::NodeSharedPtr FlowRuleChecker::selectNodeByRequesterAndStrategy( const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node) { FlowRuleManager& m = FlowRuleManager::GetInstance(); std::string tag = entry->context()->tag(); std::string limit_origin = rule.limit_origin(); FlowRelationStrategy strategy = rule.strategy(); Stat::NodeSharedPtr tag_node = entry->context()->tag_node(); if ((tag == limit_origin) && IsValidTag(tag)) { if (strategy == FlowRelationStrategy::kDirect) { // When tag matches, return tag node. return tag_node; } return SelectNodeByRelStrategy(rule, entry, node); } else if (limit_origin == Constants::kLimitOriginDefault) { if (strategy == FlowRelationStrategy::kDirect) { // When rule contains default tag, which means all request should follow // rule's limit count. return node; } return SelectNodeByRelStrategy(rule, entry, node); } else if ((limit_origin == Constants::kLimitOriginOther) && m.IsTagNotInFlowRuleList(rule.resource(), tag)) { if (strategy == FlowRelationStrategy::kDirect) { // When rule contains other tag, which means all request except this tag // should follow this rule. return tag_node; } return SelectNodeByRelStrategy(rule, entry, node); } return nullptr; } Stat::NodeSharedPtr FlowRuleChecker::SelectNodeByRelStrategy( const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node) { const std::string& ref_resource = rule.ref_resource(); auto rel_strategy = rule.strategy(); if (!ref_resource.empty() && rel_strategy == FlowRelationStrategy::kAssociatedResource) { return Stat::ResourceNodeStorage::GetInstance().GetClusterNode( ref_resource); } return node; } bool FlowRuleChecker::IsValidTag(const std::string& tag) { return !tag.empty() && (tag != Constants::kLimitOriginDefault) && (tag != Constants::kLimitOriginOther); } } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_rule_checker.h ================================================ #pragma once #include "sentinel-core/common/entry.h" #include "sentinel-core/flow/flow_rule.h" #include "sentinel-core/slot/base/token_result.h" namespace Sentinel { namespace Flow { class FlowRuleChecker { public: FlowRuleChecker() = default; ~FlowRuleChecker() = default; Slot::TokenResultSharedPtr CanPassCheck(const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node, int count); Slot::TokenResultSharedPtr CanPassCheck(const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node, int count, int flag); private: Slot::TokenResultSharedPtr PassLocalCheck(const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node, int count, int flag); Stat::NodeSharedPtr selectNodeByRequesterAndStrategy( const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node); Stat::NodeSharedPtr SelectNodeByRelStrategy(const FlowRule& rule, const EntrySharedPtr& entry, const Stat::NodeSharedPtr& node); bool IsValidTag(const std::string& tag); }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_rule_constants.h ================================================ #pragma once namespace Sentinel { namespace Flow { enum class FlowMetricType { kThreadCount = 0, kQps = 1 // default mode }; enum class FlowRelationStrategy { kDirect = 0, // default relation strategy kAssociatedResource = 1, kInvocationChainEntrance = 2 }; enum class FlowControlBehavior { kReject = 0, // default behavior kWarmUp = 1, kThrotting = 2, kWarmUpThrottling = 3 }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_rule_manager.cc ================================================ #include #include #include #include #include "sentinel-core/flow/default_traffic_shaping_calculator.h" #include "sentinel-core/flow/default_traffic_shaping_checker.h" #include "sentinel-core/flow/flow_rule_manager.h" #include "sentinel-core/log/logger.h" #include "sentinel-core/property/dynamic_sentinel_property.h" namespace Sentinel { namespace Flow { constexpr auto kFlowPropertyListenerName = "FlowPropertyListener"; bool IsValidRule(const FlowRule& rule) { bool base_valid = !rule.resource().empty() && rule.count() >= 0; if (!base_valid) { return false; } // Check rel strategy. bool rel = rule.strategy() == FlowRelationStrategy::kAssociatedResource || rule.strategy() == FlowRelationStrategy::kInvocationChainEntrance; bool rel_valid = !rel || !rule.ref_resource().empty(); // Check control behavior. bool cb_valid; switch (rule.control_behavior()) { case FlowControlBehavior::kWarmUp: cb_valid = rule.warm_up_period_sec() > 0; break; case FlowControlBehavior::kThrotting: cb_valid = rule.max_queueing_time_ms() > 0; break; default: cb_valid = true; } return rel_valid && cb_valid; } std::shared_ptr CreateDefaultController( const FlowRule& rule) { return std::make_shared( std::make_unique(rule.count()), std::make_unique(rule.metric_type())); } // FlowRuleManager FlowRuleManager::FlowRuleManager() { cur_property_ = std::make_shared< Property::DynamicSentinelProperty>>(); cur_property_->AddListener(std::make_unique()); } bool FlowRuleManager::LoadRules(const FlowRuleList& rules) { return cur_property_->UpdateValue(rules); } bool FlowRuleManager::HasRules(const std::string& resource) { absl::ReaderMutexLock lck(&update_mtx_); return rule_map_.find(resource) != rule_map_.end(); } FlowRuleList FlowRuleManager::GetRules() const { absl::ReaderMutexLock lck(&update_mtx_); FlowRuleList list{}; for (const auto& e : rule_map_) { list.insert(std::end(list), std::begin(e.second), std::end(e.second)); } return list; } FlowRuleList FlowRuleManager::GetRulesForResource( const std::string& resource) const { absl::ReaderMutexLock lck(&update_mtx_); auto it = rule_map_.find(resource); if (it == rule_map_.end()) { return {}; } return it->second; } std::shared_ptr FlowRuleManager::GetTrafficControllerFor(const FlowRule& rule) const { absl::ReaderMutexLock lck(&update_mtx_); auto it = traffic_controller_map_.find(rule); if (it == traffic_controller_map_.end()) { return nullptr; } return it->second; } void FlowRuleManager::RegisterToProperty( const FlowRulePropertySharedPtr& property) { if (property == nullptr) { return; } std::lock_guard lck(property_mtx_); SENTINEL_LOG(info, "Registering new property to FlowRuleManager"); cur_property_->RemoveListener(kFlowPropertyListenerName); cur_property_ = property; cur_property_->AddListener(std::make_unique()); } std::shared_ptr FlowRuleManager::GenerateController( const FlowRule& rule) { if (rule.metric_type() == FlowMetricType::kQps) { switch (rule.control_behavior()) { case FlowControlBehavior::kWarmUp: // return (WarmUpCalculator, DefaultChecker); case FlowControlBehavior::kThrotting: // return (DefaultCalculator, ThrottlingChecker); default: // Default mode or unknown mode: default traffic shaping controller return CreateDefaultController(rule); } } return CreateDefaultController(rule); } bool FlowRuleManager::IsTagNotInFlowRuleList(const std::string& resource_name, const std::string& tag) { if (tag.empty()) { return false; } absl::ReaderMutexLock lck(&update_mtx_); auto got = rule_map_.find(resource_name); if (got == rule_map_.end()) { return true; } for (const auto rule : got->second) { if (rule.limit_origin() == tag) { return false; } } return true; } // FlowPropertyListener void LogFlowMap(const std::unordered_map& map) { std::string s("["); for (const auto& e : map) { for (const auto& rule : e.second) { s += rule.ToString(); s += ","; } } s[s.size() - 1] = ']'; SENTINEL_LOG(info, "[FlowRuleManager] Flow rules received: {}", s); } void FlowPropertyListener::ConfigUpdate(const FlowRuleList& value, bool) { FlowRuleManager& m = FlowRuleManager::GetInstance(); if (value.empty()) { absl::WriterMutexLock lck(&(m.update_mtx_)); m.rule_map_.clear(); m.traffic_controller_map_.clear(); SENTINEL_LOG(info, "[FlowRuleManager] Flow rules received: []"); return; } std::unordered_set tmp_set; for (const auto& rule : value) { if (!IsValidRule(rule)) { SENTINEL_LOG( info, "[FlowRuleManager] Ignoring invalid flow rule when loading new flow " "rules: {}", rule.ToString()); continue; } FlowRule f_rule = rule; if (f_rule.limit_origin().empty()) { f_rule.set_limit_origin(Constants::kLimitOriginDefault); } tmp_set.insert(std::move(f_rule)); } std::unordered_map new_rule_map; TrafficControllerMap new_controller_map; for (const auto& rule : tmp_set) { new_controller_map.insert({rule, m.GenerateController(rule)}); auto it = new_rule_map.find(rule.resource()); if (it == new_rule_map.end()) { new_rule_map.insert({rule.resource(), {rule}}); } else { auto& vec = it->second; vec.push_back(std::move(rule)); } } absl::WriterMutexLock lck(&(m.update_mtx_)); m.rule_map_ = std::move(new_rule_map); m.traffic_controller_map_ = std::move(new_controller_map); LogFlowMap(m.rule_map_); } const std::string FlowPropertyListener::Name() const { return kFlowPropertyListenerName; } } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_rule_manager.h ================================================ #pragma once #include #include #include #include #include "absl/synchronization/mutex.h" #include "sentinel-core/flow/flow_rule.h" #include "sentinel-core/flow/traffic_shaping_controller.h" #include "sentinel-core/property/property_listener.h" #include "sentinel-core/property/sentinel_property.h" namespace Sentinel { namespace Flow { using TrafficControllerMap = std::unordered_map, FlowRuleHash>; using FlowRulePropertySharedPtr = Property::SentinelPropertySharedPtr; class FlowRuleManager { public: static FlowRuleManager& GetInstance() { static FlowRuleManager* instance = new FlowRuleManager(); return *instance; } friend class FlowPropertyListener; void RegisterToProperty(const FlowRulePropertySharedPtr& property); bool LoadRules(const FlowRuleList& rules); bool HasRules(const std::string& resource); FlowRuleList GetRules() const; FlowRuleList GetRulesForResource(const std::string& resource) const; std::shared_ptr GetTrafficControllerFor( const FlowRule& rule) const; bool IsTagNotInFlowRuleList(const std::string& resource_name, const std::string& tag); private: FlowRuleManager(); FlowRulePropertySharedPtr cur_property_; std::unordered_map rule_map_{}; TrafficControllerMap traffic_controller_map_{}; mutable std::mutex property_mtx_; mutable absl::Mutex update_mtx_; std::shared_ptr GenerateController( const FlowRule& rule); }; class FlowPropertyListener : public Property::PropertyListener { public: FlowPropertyListener() = default; ~FlowPropertyListener() = default; void ConfigUpdate(const FlowRuleList& value, bool first_load) override; const std::string Name() const override; }; bool IsValidRule(const FlowRule& rule); std::shared_ptr CreateDefaultController( const FlowRule& rule); } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_slot.cc ================================================ #include #include "sentinel-core/flow/flow_rule_manager.h" #include "sentinel-core/flow/flow_slot.h" namespace Sentinel { namespace Slot { const std::string& FlowSlot::Name() const { return name_; } TokenResultSharedPtr FlowSlot::Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { std::vector rules = Flow::FlowRuleManager::GetInstance().GetRulesForResource( entry->resource()->name()); if (!rules.empty()) { for (const auto& rule : rules) { // check in order const TokenResultSharedPtr res = checker_.CanPassCheck(rule, entry, node, count, flag); if (res->status() == TokenStatus::RESULT_STATUS_BLOCKED) { return res; } if (res->status() == TokenStatus::RESULT_STATUS_SHOULD_WAIT) { if (res->wait_ms().has_value() && res->wait_ms().value().count() > 0) { std::this_thread::sleep_for(res->wait_ms().value()); continue; } } } } return TokenResult::Ok(); } void FlowSlot::Exit(const EntrySharedPtr&, int, const std::vector& params) { // Do nothing } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_slot.h ================================================ #pragma once #include #include "sentinel-core/flow/flow_rule_checker.h" #include "sentinel-core/slot/base/rule_checker_slot.h" namespace Sentinel { namespace Slot { constexpr auto kFlowSlotName = "FlowSlot"; class FlowSlot : public RuleCheckerSlot { public: FlowSlot() = default; virtual ~FlowSlot() = default; TokenResultSharedPtr Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; const std::string& Name() const override; private: const std::string name_{kFlowSlotName}; Flow::FlowRuleChecker checker_{}; }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/flow/flow_slot_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/test/mock/statistic/node/mock.h" #include "sentinel-core/common/string_resource_wrapper.h" #include "sentinel-core/flow/flow_rule.h" #include "sentinel-core/flow/flow_rule_manager.h" #include "sentinel-core/flow/flow_slot.h" using testing::_; using testing::InSequence; using testing::Mock; using testing::Return; namespace Sentinel { namespace Slot { TEST(FlowSlotTest, FlowControlSingleThreadIntegrationTest) { std::string resource_name{"test_resource"}; EntryContextSharedPtr context = std::make_shared("test_context"); Stat::NodeSharedPtr node = std::make_shared(); auto resource = std::make_shared(resource_name, EntryType::OUT); auto entry = std::make_shared(resource, context); entry->set_cur_node(node); const std::vector myParams; FlowSlot slot; { // Test flow checking when no rule exists. auto result = slot.Entry(entry, node, 1000, 0, myParams); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, result->status()); } Flow::FlowRule rule{resource_name}; rule.set_count(1); Flow::FlowRuleList rules{rule}; Flow::FlowRuleManager& m = Flow::FlowRuleManager::GetInstance(); m.LoadRules(rules); { InSequence s; EXPECT_CALL(*(static_cast(node.get())), PassQps()) .WillOnce(Return(0)); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry, node, 1, 0, myParams)->status()); Mock::VerifyAndClearExpectations(node.get()); } { EXPECT_CALL(*(static_cast(node.get())), PassQps()) .WillOnce(Return(0)); EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, slot.Entry(entry, node, 2, 0, myParams)->status()); Mock::VerifyAndClearExpectations(node.get()); } { EXPECT_CALL(*(static_cast(node.get())), PassQps()) .WillOnce(Return(1)); EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, slot.Entry(entry, node, 1, 0, myParams)->status()); Mock::VerifyAndClearExpectations(node.get()); } m.LoadRules({}); } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/flow/throttling_traffic_shaping_checker.cc ================================================ #include #include "sentinel-core/flow/throttling_traffic_shaping_checker.h" #include "sentinel-core/utils/time_utils.h" namespace Sentinel { namespace Flow { Slot::TokenResultSharedPtr ThrottlingTrafficShapingChecker::DoCheck( const Stat::NodeSharedPtr&, int acquire_count, double threshold) { if (acquire_count < 0) { return Slot::TokenResult::Ok(); } if (threshold <= 0) { return Slot::TokenResult::Blocked("blocked"); } int64_t cur_time = Utils::TimeUtils::CurrentTimeMillis().count(); // Calculate the interval between every two requests. int64_t cost_time = std::round(acquire_count / threshold * 1000); // Expected pass time of this request. int64_t expected_time = cost_time + last_passed_time_.load(); if (expected_time <= cur_time) { last_passed_time_.store(cur_time); return Slot::TokenResult::Ok(); } int64_t wait_time = cost_time + last_passed_time_.load() - Utils::TimeUtils::CurrentTimeMillis().count(); if (wait_time > max_timeout_) { return Slot::TokenResult::Blocked("timeout"); } int64_t old_time = cost_time + last_passed_time_.fetch_add(cost_time); wait_time = old_time - Utils::TimeUtils::CurrentTimeMillis().count(); if (wait_time > max_timeout_) { last_passed_time_.fetch_sub(cost_time); return Slot::TokenResult::Blocked("timeout"); } if (wait_time > 0) { return Slot::TokenResult::ShouldWait(std::chrono::milliseconds(wait_time)); } return Slot::TokenResult::Ok(); } } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/throttling_traffic_shaping_checker.h ================================================ #pragma once #include #include "sentinel-core/flow/flow_rule_constants.h" #include "sentinel-core/flow/traffic_shaping_checker.h" namespace Sentinel { namespace Flow { class ThrottlingTrafficShapingChecker : public TrafficShapingChecker { public: explicit ThrottlingTrafficShapingChecker(int32_t timeout) : max_timeout_(timeout) {} virtual ~ThrottlingTrafficShapingChecker() = default; Slot::TokenResultSharedPtr DoCheck(const Stat::NodeSharedPtr& node, int acquire_count, double threshold) override; private: const int32_t max_timeout_; std::atomic last_passed_time_; }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/traffic_shaping_calculator.h ================================================ #pragma once #include "sentinel-core/statistic/node/node.h" namespace Sentinel { namespace Flow { class TrafficShapingCalculator { public: TrafficShapingCalculator() = default; virtual ~TrafficShapingCalculator() = default; virtual double CalculateAllowedTokens(const Stat::NodeSharedPtr& node, int acquire_count, int flag) = 0; }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/traffic_shaping_checker.h ================================================ #pragma once #include "sentinel-core/slot/base/token_result.h" #include "sentinel-core/statistic/node/node.h" namespace Sentinel { namespace Flow { class TrafficShapingChecker { public: TrafficShapingChecker() = default; virtual ~TrafficShapingChecker() = default; virtual Slot::TokenResultSharedPtr DoCheck(const Stat::NodeSharedPtr& node, int acquire_count, double threshold) = 0; }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/traffic_shaping_controller.cc ================================================ #include #include "sentinel-core/flow/traffic_shaping_controller.h" namespace Sentinel { namespace Flow { Slot::TokenResultSharedPtr TrafficShapingController::CanPass( const Stat::NodeSharedPtr& node, int acquire_count, int flag) { double allowed_tokens = calculator_->CalculateAllowedTokens(node, acquire_count, flag); return action_checker_->DoCheck(node, acquire_count, allowed_tokens); } } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/traffic_shaping_controller.h ================================================ #pragma once #include #include "sentinel-core/flow/traffic_shaping_calculator.h" #include "sentinel-core/flow/traffic_shaping_checker.h" #include "sentinel-core/statistic/node/node.h" namespace Sentinel { namespace Flow { class TrafficShapingController { public: TrafficShapingController( std::unique_ptr&& calculator, std::unique_ptr&& checker) : calculator_(std::move(calculator)), action_checker_(std::move(checker)) {} ~TrafficShapingController() = default; Slot::TokenResultSharedPtr CanPass(const Stat::NodeSharedPtr& node, int acquire_count, int flag); private: const std::unique_ptr calculator_; const std::unique_ptr action_checker_; }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/flow/traffic_shaping_controller_test.cc ================================================ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/test/mock/flow/mock.h" #include "sentinel-core/test/mock/statistic/node/mock.h" #include "sentinel-core/flow/default_traffic_shaping_calculator.h" #include "sentinel-core/flow/default_traffic_shaping_checker.h" #include "sentinel-core/flow/flow_rule_constants.h" #include "sentinel-core/flow/traffic_shaping_controller.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { namespace Flow { TEST(TrafficShapingControllerTest, TestBasicPassCheck) { auto node = std::make_shared(); TrafficShapingController always_pass_controller{ std::make_unique(), std::make_unique()}; TrafficShapingController always_block_controller{ std::make_unique(), std::make_unique()}; auto pass_result = always_pass_controller.CanPass(node, 1, 0); auto block_result = always_block_controller.CanPass(node, 1, 0); EXPECT_EQ(Slot::TokenStatus::RESULT_STATUS_OK, pass_result->status()); EXPECT_EQ(Slot::TokenStatus::RESULT_STATUS_BLOCKED, block_result->status()); EXPECT_TRUE(block_result->blocked_reason().has_value()); double threshold = 10; TrafficShapingController default_qps_controller{ std::make_unique(threshold), std::make_unique(FlowMetricType::kQps)}; ON_CALL(*node.get(), PassQps()).WillByDefault(Return(9)); EXPECT_EQ(Slot::TokenStatus::RESULT_STATUS_OK, default_qps_controller.CanPass(node, 1, 0)->status()); ON_CALL(*node.get(), PassQps()).WillByDefault(Return(9)); EXPECT_EQ(Slot::TokenStatus::RESULT_STATUS_BLOCKED, default_qps_controller.CanPass(node, 2, 0)->status()); ON_CALL(*node.get(), PassQps()).WillByDefault(Return(10)); EXPECT_EQ(Slot::TokenStatus::RESULT_STATUS_BLOCKED, default_qps_controller.CanPass(node, 1, 0)->status()); } } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/init/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "init_target_interface", srcs = [ "init_target.h", ], copts = DEFAULT_COPTS, ) cc_library( name = "init_target_registry_lib", srcs = [ "init_target_registry.h", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/utils:utils_lib", ] ) cc_test( name = "init_target_registry_unittests", srcs = [ "init_target_registry_test.cc", ], copts = TEST_COPTS, deps = [ ":init_target_interface", ":init_target_registry_lib", "//sentinel-core/test/mock/init:mock_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/init/init_target.h ================================================ #pragma once namespace Sentinel { namespace Init { class Target { public: virtual ~Target() = default; virtual void Initialize() = 0; }; } // namespace Init } // namespace Sentinel ================================================ FILE: sentinel-core/init/init_target_registry.h ================================================ #pragma once #include "sentinel-core/utils/utils.h" namespace Sentinel { namespace Init { template class InitTargetRegister { public: InitTargetRegister() { instance_.Initialize(); } T& GetInstance() { return instance_; } private: T instance_{}; }; template class InitTargetRegister> { public: InitTargetRegister(const Utils::Singleton& s) { s.get().Initialize(); } }; /** * Macro used for static registration. * Static variable initialization does not guarantee order, so each target here * cannot have mutual dependencies. */ #ifndef REGISTER_INIT_TARGET #define REGISTER_INIT_TARGET(TARGET_OBJ_TYPE) \ static Sentinel::Init::InitTargetRegister \ TARGET_OBJ_TYPE##_registered #endif #ifndef REGISTER_SINGLETON_INIT_TARGET #define REGISTER_SINGLETON_INIT_TARGET(TARGET_OBJ_TYPE, o) \ static Sentinel::Init::InitTargetRegister> \ TARGET_OBJ_TYPE##_sgt_registered(o) #endif } // namespace Init } // namespace Sentinel ================================================ FILE: sentinel-core/init/init_target_registry_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/test/mock/init/mock.h" #include "sentinel-core/init/init_target.h" #include "sentinel-core/init/init_target_registry.h" namespace Sentinel { namespace Init { TEST(InitTargetRegisterTest, TestCommonInitTargetInvoked) { REGISTER_INIT_TARGET(FakeInitTarget); EXPECT_EQ(1, FakeInitTarget_registered.GetInstance().count()); } } // namespace Init } // namespace Sentinel ================================================ FILE: sentinel-core/log/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "logger_lib", srcs = [ "logger.h", "logger.cc", ], copts = DEFAULT_COPTS, deps = [ ":log_base_impl_lib", "//sentinel-core/utils:macros_lib", "@com_github_gabime_spdlog//:spdlog" ] ) cc_library( name = "log_base_interface", srcs = [ "log_base.h", ], copts = DEFAULT_COPTS, deps = [ ] ) cc_library( name = "log_base_impl_lib", srcs = [ "log_base.h", "log_base.cc", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/strings", "//sentinel-core/utils:file_utils_lib", "//sentinel-core/init:init_target_registry_lib", ] ) cc_test( name = "log_base_unittests", srcs = [ "log_base_test.cc", ], copts = TEST_COPTS, deps = [ ":log_base_impl_lib", "@com_google_googletest//:gtest_main", ] ) cc_test( name = "logger_unittests", srcs = [ "logger_unittests.cc", ], copts = TEST_COPTS, deps = [ ":logger_lib", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/log/block/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "block_log_task_lib", srcs = [ "block_log_task.h", "block_log_task.cc", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", "@com_github_gabime_spdlog//:spdlog", "//sentinel-core/log:logger_lib", ] ) cc_test( name = "block_log_task_unittests", srcs = [ "block_log_task_test.cc", ], copts = TEST_COPTS, deps = [ ":block_log_task_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/log/block/block_log_task.cc ================================================ #include "sentinel-core/log/block/block_log_task.h" #include #include #include #include "sentinel-core/log/logger.h" #include "sentinel-core/utils/time_utils.h" #include "absl/strings/str_format.h" #include "absl/time/time.h" #include "spdlog/sinks/rotating_file_sink.h" using namespace Sentinel::Utils; namespace Sentinel { namespace Log { BlockLogTask::BlockLogTask(const std::string& log_path) { try { logger_ = spdlog::rotating_logger_mt(kBlockLoggerName, log_path, kDefaultBlockLogMaxSize, 3); logger_->set_pattern("%v"); } catch (const spdlog::spdlog_ex& ex) { std::cerr << "Sentinel block log initialization failed: " << ex.what() << std::endl; } } BlockLogTask::~BlockLogTask() { Stop(); spdlog::drop(kBlockLoggerName); } void BlockLogTask::LoopWriteBlockLog() { while (started()) { if (logger_ != nullptr) { absl::WriterMutexLock lck(&mtx_); for (auto& e : map_) { if (e.second.last_block_ - e.second.last_write_ > 0) { int64_t cur_time = TimeUtils::CurrentTimeMillis().count(); auto time_str = absl::FormatTime("%Y-%m-%d %H:%M:%S", absl::FromUnixMillis(cur_time), absl::LocalTimeZone()); // format: time|resource|exception logger_->info("{}|{}", time_str, e.first); e.second.last_write_ = cur_time; } } logger_->flush(); map_.clear(); } // sleep for 1s std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } } void BlockLogTask::Start() { if (logger_ == nullptr) { SENTINEL_LOG( error, "Block logger failed to initialize, the block log task won't start"); return; } bool expected = false; if (started_.compare_exchange_strong(expected, true)) { thd_.reset(new std::thread(&BlockLogTask::LoopWriteBlockLog, this)); } } void BlockLogTask::Stop() { started_.store(false); thd_->join(); } void BlockLogTask::Log(const std::string& resource, const std::string& cause) { if (logger_ == nullptr) { return; } auto key = absl::StrFormat("%s|%s", resource, cause); { absl::ReaderMutexLock lck(&mtx_); auto it = map_.find(key); if (it != map_.end()) { it->second.last_block_ = TimeUtils::CurrentTimeMillis().count(); return; } } absl::WriterMutexLock lck(&mtx_); map_.emplace(std::make_pair( key, BlockLogRecord{0, TimeUtils::CurrentTimeMillis().count()})); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/block/block_log_task.h ================================================ #pragma once #include #include #include #include "absl/container/flat_hash_map.h" #include "spdlog/spdlog.h" #include "sentinel-core/log/log_base.h" namespace Sentinel { namespace Log { static constexpr const char* kBlockLoggerName = "sentinel_block_logger"; static constexpr const char* kBlockLogFilename = "sentinel-block.log"; static constexpr uint32_t kDefaultBlockLogMaxSize = 1024 * 1024 * 300; struct BlockLogRecord { BlockLogRecord() = default; BlockLogRecord(int64_t lw, int64_t lb) : last_write_(lw), last_block_(lb) {} int64_t last_write_{0}; int64_t last_block_{0}; }; class BlockLogTask { public: BlockLogTask() : BlockLogTask(LogBase::GetLogBaseDir() + kBlockLogFilename) {} explicit BlockLogTask(const std::string& log_path); ~BlockLogTask(); void Start(); void Stop(); void Log(const std::string& resource, const std::string& cause); bool started() const { return started_.load(); } private: std::atomic_bool started_{false}; std::unique_ptr thd_; std::shared_ptr logger_; absl::flat_hash_map map_; mutable absl::Mutex mtx_; void LoopWriteBlockLog(); }; } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/block/block_log_task_test.cc ================================================ #include #include #include #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/log/block/block_log_task.h" namespace Sentinel { namespace Log { namespace { size_t GetLineNumberFromFilePath(const std::string& filename) { std::ifstream inFile(filename); return std::count(std::istreambuf_iterator(inFile), std::istreambuf_iterator(), '\n'); } } // namespace TEST(BlockLogTaskTest, TestWriteBlockLog) { std::string log_file_name = std::tmpnam(nullptr); ASSERT_TRUE(!log_file_name.empty()); BlockLogTask task(log_file_name); task.Start(); EXPECT_TRUE(task.started()); auto res1 = "some_resource"; auto res2 = "another_resource"; auto flow_exception = "FlowException"; task.Log(res1, flow_exception); task.Log(res2, flow_exception); std::this_thread::sleep_for(std::chrono::milliseconds(1100)); EXPECT_EQ(2, GetLineNumberFromFilePath(log_file_name)); task.Log(res1, flow_exception); std::this_thread::sleep_for(std::chrono::milliseconds(1100)); EXPECT_EQ(3, GetLineNumberFromFilePath(log_file_name)); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/log_base.cc ================================================ #include "sentinel-core/log/log_base.h" #include #include #include "absl/strings/match.h" #include "absl/strings/string_view.h" #include "sentinel-core/utils/file_utils.h" using namespace Sentinel::Utils; namespace Sentinel { namespace Log { static constexpr auto kFileSeparator = "/"; LogBase::LogBase() { InitializeInternal(); } void LogBase::InitializeInternal() { // First use config from ENV. // If absent, then use `~/logs/csp/`. std::string str_log_dir; auto log_dir = std::getenv(kEnvLogDir); if (log_dir == nullptr) { auto home_dir = std::getenv("HOME"); if (home_dir != nullptr) { str_log_dir = std::string(home_dir); str_log_dir = AddSeparator(str_log_dir) + kDirName; } else { std::cout << "INFO: home_dir is null" << std::endl; str_log_dir = "./"; } } log_base_dir_ = AddSeparator(str_log_dir); if (!FileUtils::DirExists(str_log_dir)) { auto ret = FileUtils::CreateDir(str_log_dir); if (!ret) { // log error } } const char *use_pid = std::getenv(kEnvLogNameUsrPid); if (use_pid != nullptr && std::string(use_pid) == "true") { log_name_use_pid_ = true; } else { log_name_use_pid_ = false; } std::cout << "INFO: log base dir is: " << log_base_dir_ << std::endl; std::cout << "INFO: log name use pid is: " << log_name_use_pid_ << std::endl; } std::string LogBase::AddSeparator(const std::string &dir) { if (!absl::EndsWith(dir, kFileSeparator)) { return dir + kFileSeparator; } return dir; } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/log_base.h ================================================ #pragma once #include namespace Sentinel { namespace Log { static constexpr auto kEnvLogDir = "CSP_SENTINEL_LOG_DIR"; static constexpr auto kEnvLogNameUsrPid = "CSP_SENTINEL_LOG_USE_PID"; static constexpr auto kDirName = "logs/csp"; class LogBase { public: ~LogBase() = default; static LogBase& GetInstance() { static LogBase* instance = new LogBase(); return *instance; } static bool IsLogNameUsePid() { return GetInstance().log_name_use_pid_; } static std::string GetLogBaseDir() { return GetInstance().log_base_dir_; } private: LogBase(); void InitializeInternal(); static std::string AddSeparator(const std::string& dir); std::string log_base_dir_{}; bool log_name_use_pid_{false}; }; } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/log_base_test.cc ================================================ #include "gtest/gtest.h" #define private public #include "sentinel-core/log/log_base.h" namespace Sentinel { namespace Log { TEST(LogBaseTest, TestAddSeparator) { auto ret = LogBase::AddSeparator("test_path"); EXPECT_EQ(ret, "test_path/"); ret = LogBase::AddSeparator("test_path/"); EXPECT_EQ(ret, "test_path/"); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/logger.cc ================================================ #include "logger.h" #include #include #include namespace Sentinel { namespace Log { static constexpr const char* kDefaultRecordLogFormat = "[%H:%M:%S] [%l] %v"; const char Logger::kDefaultFileLogger[] = "default_sentinel_logger"; bool Logger::InitDefaultLogger() { return Logger::InitDefaultLogger(Logger::GetDefaultLogPath()); } bool Logger::InitDefaultLogger(const std::string& file_path) { return Logger::InitDefaultLogger(file_path, kDefaultRecordLogFormat); } bool Logger::InitDefaultLogger(const std::string& file_path, const std::string& log_format) { try { auto logger = spdlog::daily_logger_mt( kDefaultFileLogger, file_path); if (!logger) { return false; } // https://github.com/gabime/spdlog/wiki/3.-Custom-formatting if (!log_format.empty()) { logger->set_pattern(log_format); } logger->set_level(spdlog::level::info); logger->flush_on(spdlog::level::info); } catch (const spdlog::spdlog_ex& ex) { std::cerr << "Log initialization failed: " << ex.what() << std::endl; return false; } return true; } void Logger::Uninitialization() { spdlog::drop(kDefaultFileLogger); } void Logger::SetAllLoggerLevel(levels level) { spdlog::apply_all([&](std::shared_ptr l) { l->set_level(static_cast(level)); }); } void Logger::FlushAllLogger() { spdlog::apply_all([&](std::shared_ptr l) { l->flush(); }); } std::string Logger::GetDefaultLogPath() { return LogBase::GetLogBaseDir() + kRecordLogFilename; } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/logger.h ================================================ #pragma once #include #include "sentinel-core/log/log_base.h" #include "sentinel-core/utils/macros.h" #include "spdlog/spdlog.h" namespace Sentinel { typedef enum { trace = spdlog::level::trace, debug = spdlog::level::debug, info = spdlog::level::info, warn = spdlog::level::warn, error = spdlog::level::err, critical = spdlog::level::critical, } levels; namespace Log { static constexpr const char* kRecordLogFilename = "sentinel-record.log"; class Logger { public: Logger() = delete; static bool InitDefaultLogger(); static bool InitDefaultLogger(const std::string& file_path); static bool InitDefaultLogger(const std::string& file_path, const std::string& log_format); static void Uninitialization(); static void SetAllLoggerLevel(levels level); static void FlushAllLogger(); static std::string GetDefaultLogPath(); template static void Log(const std::string& logger_name, levels level, const char* format, const Args&... args) { auto logger = spdlog::get(logger_name); if (!logger) { return; } switch (level) { case trace: { logger->trace(format, args...); break; } case debug: { logger->debug(format, args...); break; } case info: { logger->info(format, args...); break; } case warn: { logger->warn(format, args...); break; } case error: { logger->error(format, args...); break; } case critical: { logger->critical(format, args...); break; } default: { SENTINEL_NOT_REACHED_GCOVR_EXCL_LINE } } } static const char kDefaultFileLogger[]; }; #define SENTINEL_LOG(LEVEL, ...) \ Log::Logger::Log(Log::Logger::kDefaultFileLogger, LEVEL, ##__VA_ARGS__) } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/logger_test.cc ================================================ #include #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include #include #include #include "sentinel-core/log/logger.h" #include "sentinel-core/utils/time_utils.h" #include "absl/strings/str_format.h" #include "absl/time/clock.h" #include "absl/time/time.h" using namespace Sentinel::Utils; namespace Sentinel { namespace Log { namespace { size_t GetLineNumberFromFilePathWithDate(const std::string& filename) { auto real_filename = absl::StrFormat( "%s_%s", filename, absl::FormatTime("%Y-%m-%d", absl::Now(), absl::LocalTimeZone())); std::ifstream inFile(real_filename); return std::count(std::istreambuf_iterator(inFile), std::istreambuf_iterator(), '\n'); } } // namespace TEST(LoggerTest, BasicConstruct) { std::string log_file_name = std::tmpnam(nullptr); ASSERT_TRUE(!log_file_name.empty()); EXPECT_TRUE(Logger::InitDefaultLogger(log_file_name, "")); } TEST(LoggerTest, BasicLogger) { Logger::Uninitialization(); std::string log_file_name = absl::StrFormat("./test-%d", TimeUtils::CurrentTimeMillis().count()); ASSERT_TRUE(!log_file_name.empty()); ASSERT_TRUE(Logger::InitDefaultLogger(log_file_name, "")); SENTINEL_LOG(info, "test one"); SENTINEL_LOG(info, "test two"); Logger::FlushAllLogger(); sync(); EXPECT_EQ(2, GetLineNumberFromFilePathWithDate(log_file_name)); SENTINEL_LOG(warn, "test one"); SENTINEL_LOG(warn, "test two"); Logger::FlushAllLogger(); sync(); EXPECT_EQ(4, GetLineNumberFromFilePathWithDate(log_file_name)); // Default log level is warn SENTINEL_LOG(debug, "test one"); SENTINEL_LOG(debug, "test two"); Logger::FlushAllLogger(); sync(); EXPECT_EQ(4, GetLineNumberFromFilePathWithDate(log_file_name)); Logger::SetAllLoggerLevel(trace); SENTINEL_LOG(trace, "test one"); SENTINEL_LOG(trace, "test two"); Logger::FlushAllLogger(); sync(); EXPECT_EQ(6, GetLineNumberFromFilePathWithDate(log_file_name)); remove(log_file_name.c_str()); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "metric_log_lib", srcs = [ "metric_writer.h", "metric_writer.cc", "metric_reader.h", "metric_reader.cc", "metric_searcher.h", "metric_searcher.cc", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/strings", "@com_google_absl//absl/time", "//sentinel-core/statistic/base:metric_item_lib", "//sentinel-core/config:local_config_lib", ], ) cc_library( name = "metric_log_task_lib", srcs = [ "metric_log_task.h", "metric_log_task.cc", ], copts = DEFAULT_COPTS, deps = [ ":metric_log_lib", "//sentinel-core/init:init_target_interface", "//sentinel-core/statistic/node:resource_node_storage_lib", ] ) cc_test( name = "metric_writer_unittests", srcs = [ "metric_test_utils.h", "metric_writer_test.cc", ], copts = TEST_COPTS, deps = [ ":metric_log_lib", "@com_google_googletest//:gtest_main", ] ) cc_test( name = "metric_reader_unittests", srcs = [ "metric_test_utils.h", "metric_reader_test.cc", ], copts = TEST_COPTS, deps = [ ":metric_log_lib", "@com_google_absl//absl/time", "@com_google_googletest//:gtest_main", ], linkstatic = 1, ) cc_test( name = "metric_searcher_unittests", srcs = [ "metric_test_utils.h", "metric_searcher_test.cc", ], copts = TEST_COPTS, deps = [ ":metric_log_lib", "@com_google_googletest//:gtest_main", ], linkstatic = 1, ) ================================================ FILE: sentinel-core/log/metric/metric_log_task.cc ================================================ #include #include #include #include #include #include "sentinel-core/config/local_config.h" #include "sentinel-core/log/metric/metric_log_task.h" #include "sentinel-core/log/metric/metric_writer.h" #include "sentinel-core/statistic/base/metric_item.h" #include "sentinel-core/statistic/node/resource_node_storage.h" namespace Sentinel { namespace Log { MetricLogTask::MetricLogTask() { Config::LocalConfig& conf = Config::LocalConfig::GetInstance(); writer_ = std::make_unique(conf.SingleMetricFileSize(), conf.TotalMetricFileCount()); } MetricLogTask::~MetricLogTask() { Stop(); } void MetricLogTask::AggregateMetrics( MetricItemTimeMap& map, std::unordered_map&& metrics, const std::string& resource) { for (const auto& e : metrics) { int64_t t = e.first; auto item = e.second; item->set_resource(resource); if (map.find(t) == map.end()) { map.insert(std::make_pair(t, std::vector{})); } auto iter = map.find(t); if (iter != map.end()) { iter->second.push_back(item); } } } void MetricLogTask::RunLogTask() { while (!stopped_.load()) { std::map> map; const auto resource_node_map = Stat::ResourceNodeStorage::GetInstance().GetNodeMap(); for (const auto& e : resource_node_map) { auto resource_node = e.second; assert(resource_node != nullptr); if (resource_node != nullptr) { AggregateMetrics(map, resource_node->Metrics(), e.first); } } if (!map.empty()) { for (auto& e : map) { writer_->Write(e.first, e.second); } } // sleep for 1s std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } } void MetricLogTask::Initialize() { thd_.reset(new std::thread(&MetricLogTask::RunLogTask, this)); } void MetricLogTask::Stop() { stopped_.store(true); thd_->join(); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_log_task.h ================================================ #pragma once #include #include #include #include #include #include "sentinel-core/init/init_target.h" #include "sentinel-core/log/metric/metric_writer.h" #include "sentinel-core/statistic/base/metric_item.h" namespace Sentinel { namespace Log { using MetricItemTimeMap = std::map>; class MetricLogTask : public Init::Target { public: MetricLogTask(); virtual ~MetricLogTask(); void Initialize() override; void Stop(); private: std::unique_ptr writer_; std::atomic stopped_{false}; std::unique_ptr thd_; void RunLogTask(); void AggregateMetrics( MetricItemTimeMap& map, std::unordered_map&& metrics, const std::string& resource); }; } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_reader.cc ================================================ #include "sentinel-core/log/metric/metric_reader.h" #include #include #include namespace Sentinel { namespace Log { std::vector MetricReader::ReadMetrics( const std::vector &file_names, int pos, int64_t offset, int recommend_lines_) { std::vector metric_vec; if (pos >= file_names.size()) { return metric_vec; } ReadMetricsInOneFile(metric_vec, file_names[pos++], offset, recommend_lines_); while (pos < file_names.size() && metric_vec.size() < recommend_lines_) { ReadMetricsInOneFile(metric_vec, file_names[pos++], 0, recommend_lines_); } return metric_vec; } void MetricReader::ReadMetricsInOneFile( std::vector &metric_vec, const std::string &file_name, int64_t offset, int recommend_lines) { int64_t last_second = -1; if (metric_vec.size() > 0) { last_second = metric_vec[metric_vec.size() - 1]->timestamp() / 1000; } std::ifstream in_file(file_name.c_str(), std::ios::in); if (!in_file.is_open()) { return; } in_file.seekg(offset); std::string line; while (getline(in_file, line)) { auto node = Stat::MetricItem::FromFatString(line); int64_t cur_second = node->timestamp() / 1000; if (metric_vec.size() < recommend_lines) { metric_vec.emplace_back(std::move(node)); } else if (cur_second == last_second) { metric_vec.emplace_back(std::move(node)); } else { break; } last_second = cur_second; } in_file.close(); } std::vector MetricReader::ReadMetricsByEndTime( const std::vector file_names, int pos, int64_t offset, int64_t begin_time_ms, int64_t end_time_ms, const std::string &identity) { std::vector metric_vec; if (pos >= file_names.size()) { return metric_vec; } if (!ReadMetricsInOneFileByEndTime(metric_vec, file_names[pos++], offset, begin_time_ms, end_time_ms, identity)) { return metric_vec; } while (pos < file_names.size() && ReadMetricsInOneFileByEndTime(metric_vec, file_names[pos++], 0, begin_time_ms, end_time_ms, identity)) { } return metric_vec; } bool MetricReader::ReadMetricsInOneFileByEndTime( std::vector &metric_vec, const std::string &file_name, int64_t offset, int64_t begin_time_ms, int64_t end_time_ms, const std::string &identity) { std::ifstream in_file(file_name.c_str(), std::ios::in | std::ios::binary); if (!in_file.is_open()) { return false; } long begin_second = begin_time_ms / 1000; long end_second = end_time_ms / 1000; in_file.seekg(offset); std::string line; while (getline(in_file, line)) { auto node = Stat::MetricItem::FromFatString(line); auto cur_second = node->timestamp() / 1000; // currentSecond should >= beginSecond, otherwise a wrong metric file must // occur if (cur_second < begin_second) { return false; } if (cur_second <= end_second) { // read all if (identity.size() == 0) { metric_vec.emplace_back(std::move(node)); } else if (node->resource() == identity) { metric_vec.emplace_back(std::move(node)); } } else { return false; } if (metric_vec.size() >= kMaxLinesReturn) { return false; } } return true; } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_reader.h ================================================ #pragma once #include #include #include "sentinel-core/statistic/base/metric_item.h" namespace Sentinel { namespace Log { class MetricReader { public: std::vector ReadMetrics( const std::vector &file_names, int pos, int64_t offset, int recommend_lines_); void ReadMetricsInOneFile(std::vector &metric_vec, const std::string &file_name, int64_t offset, int recommend_lines); std::vector ReadMetricsByEndTime( const std::vector file_names, int pos, int64_t offset, int64_t begin_time_ms, int64_t end_time_ms, const std::string &identity); bool ReadMetricsInOneFileByEndTime( std::vector &metric_vec, const std::string &file_name, int64_t offset, int64_t begin_time_ms, int64_t end_time_ms, const std::string &identity); private: static const int kMaxLinesReturn = 100000; }; } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_reader_test.cc ================================================ #include "gmock/gmock.h" #include "gtest/gtest.h" #include #include #include #define private public #include "absl/time/time.h" #include "sentinel-core/config/local_config.h" #include "sentinel-core/log/log_base.h" #include "sentinel-core/log/metric/metric_reader.h" #include "sentinel-core/log/metric/metric_test_utils.h" #include "sentinel-core/log/metric/metric_writer.h" #include "sentinel-core/utils/file_utils.h" #include "sentinel-core/utils/time_utils.h" namespace Sentinel { namespace Log { TEST(MetricReaderTest, TestReadMetrics) { int64_t time = Sentinel::Utils::TimeUtils::CurrentTimeMillis().count(); MetricTestUtils::TestWriteMetricLog(time); auto date_str = absl::FormatTime("%Y-%m-%d", absl::FromUnixMillis(time), absl::LocalTimeZone()); auto log_file_name = LogBase::GetLogBaseDir() + MetricTestUtils::GetAppName() + "-metrics.log." + date_str; std::vector file_names{log_file_name}; MetricReader reader; auto ret1 = reader.ReadMetrics(file_names, 0, 0, 10); EXPECT_EQ(ret1.size(), 2); auto ret2 = reader.ReadMetrics(file_names, 0, 0, 1); EXPECT_EQ(ret2.size(), 1); MetricTestUtils::RemoveTestLogFile(); } TEST(MetricReaderTest, TestReadMetricsByEndTime) { int64_t time = Sentinel::Utils::TimeUtils::CurrentTimeMillis().count(); MetricTestUtils::TestWriteMetricLog(time); auto date_str = absl::FormatTime("%Y-%m-%d", absl::FromUnixMillis(time), absl::LocalTimeZone()); auto log_file_name = LogBase::GetLogBaseDir() + MetricTestUtils::GetAppName() + "-metrics.log." + date_str; std::vector file_names{log_file_name}; MetricReader reader; auto begin_time_ms = time; auto end_time_ms = time; auto ret1 = reader.ReadMetricsByEndTime(file_names, 0, 0, begin_time_ms, end_time_ms, ""); EXPECT_EQ(ret1.size(), 1); auto ret2 = reader.ReadMetricsByEndTime(file_names, 0, 0, begin_time_ms, end_time_ms + 1000, ""); EXPECT_EQ(ret2.size(), 2); MetricTestUtils::RemoveTestLogFile(); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_searcher.cc ================================================ #include "sentinel-core/log/metric/metric_searcher.h" #include #include #include "sentinel-core/log/log_base.h" #include "sentinel-core/log/metric/metric_writer.h" #include "sentinel-core/utils/file_utils.h" namespace Sentinel { namespace Log { bool MetricSearcher::ValidPosition(int64_t begin_time_ms) { if (begin_time_ms / 1000 < last_pos_.second) { return false; } if (last_pos_.index_file_name.size() == 0) { return false; } // index file dose not exits if (!Sentinel::Utils::FileUtils::FileExists(last_pos_.index_file_name)) { return false; } std::ifstream in(last_pos_.index_file_name); in.seekg(last_pos_.offset_in_index); int64_t sec; in >> sec; in.close(); // timestamp(second) in the specific position == that we cached return sec == last_pos_.second; } int64_t MetricSearcher::FindOffset(int64_t begin_time_ms, const std::string &metric_file_name, const std::string &idx_file_name, int64_t offset_in_index) { last_pos_.metric_file_name = ""; last_pos_.index_file_name = ""; if (!Sentinel::Utils::FileUtils::FileExists(idx_file_name)) { return -1; } auto begin_second = begin_time_ms / 1000; std::ifstream in(idx_file_name); in.seekg(offset_in_index); int64_t second; int64_t tmp_offset = -1; int64_t offset = -1; while (!in.eof() && !in.fail()) { in >> second >> tmp_offset; if (second < begin_second) { last_pos_.offset_in_index = in.tellg(); } else { offset = tmp_offset; break; } } in.close(); if (offset != -1) { last_pos_.metric_file_name = metric_file_name; last_pos_.index_file_name = idx_file_name; } return offset; } MetricSearcher::MetricSearcher(const std::string &base_dir, const std::string &base_file_name) : base_dir_(base_dir), base_file_name_(base_file_name) {} std::vector MetricSearcher::Find( int64_t begin_time_ms, int recommend_lines) { std::lock_guard guard(lock_); auto file_names = MetricWriter::ListMetricFiles(base_dir_, base_file_name_); auto it = file_names.begin(); int64_t offset_in_index = 0; if (ValidPosition(begin_time_ms)) { auto lit = std::find(file_names.begin(), file_names.end(), last_pos_.metric_file_name); if (lit != file_names.end()) { it = lit; offset_in_index = last_pos_.offset_in_index; } } for (; it < file_names.end(); ++it) { auto file_name = *it; long offset = FindOffset(begin_time_ms, file_name, MetricWriter::FormIndexFileName(file_name), offset_in_index); offset_in_index = 0; if (offset != -1) { return metric_reader_.ReadMetrics(file_names, it - file_names.begin(), offset, recommend_lines); } } return std::vector(); } std::vector MetricSearcher::FindByTimeAndResource( int64_t begin_time_ms, int64_t end_time_ms, const std::string &identity) { std::lock_guard guard(lock_); auto file_names = MetricWriter::ListMetricFiles(base_dir_, base_file_name_); auto it = file_names.begin(); int64_t offset_in_index = 0; if (ValidPosition(begin_time_ms)) { auto lit = std::find(file_names.begin(), file_names.end(), last_pos_.metric_file_name); if (lit != file_names.end()) { it = lit; offset_in_index = last_pos_.offset_in_index; } } for (; it < file_names.end(); ++it) { auto file_name = *it; long offset = FindOffset(begin_time_ms, file_name, MetricWriter::FormIndexFileName(file_name), offset_in_index); offset_in_index = 0; if (offset != -1) { return metric_reader_.ReadMetricsByEndTime( file_names, it - file_names.begin(), offset, begin_time_ms, end_time_ms, identity); } } return std::vector(); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_searcher.h ================================================ #pragma once #include #include #include #include "sentinel-core/log/metric/metric_reader.h" #include "sentinel-core/statistic/base/metric_item.h" namespace Sentinel { namespace Log { struct Position { std::string metric_file_name = ""; std::string index_file_name = ""; int64_t offset_in_index = -1; int64_t second = -1; }; class MetricSearcher { public: MetricSearcher(const std::string &base_dir, const std::string &base_file_name); std::vector Find(int64_t begin_time_ms, int recommend_lines); std::vector FindByTimeAndResource( int64_t begin_time_ms, int64_t end_time_ms, const std::string &identity); private: bool ValidPosition(int64_t begin_time_ms); int64_t FindOffset(int64_t begin_time, const std::string &metric_file_name, const std::string &idx_file_name, int64_t offset_in_index); private: std::string base_dir_; std::string base_file_name_; Position last_pos_; MetricReader metric_reader_; std::mutex lock_; }; } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_searcher_test.cc ================================================ #include "gmock/gmock.h" #include "gtest/gtest.h" #include #include #include #include #define private public #include "sentinel-core/config/local_config.h" #include "sentinel-core/log/log_base.h" #include "sentinel-core/log/metric/metric_searcher.h" #include "sentinel-core/log/metric/metric_test_utils.h" #include "sentinel-core/log/metric/metric_writer.h" #include "sentinel-core/utils/file_utils.h" #include "sentinel-core/utils/time_utils.h" #include namespace Sentinel { namespace Log { TEST(MetricSearcherTest, TestFind) { int64_t time = Sentinel::Utils::TimeUtils::CurrentTimeMillis().count(); MetricTestUtils::TestWriteMetricLog(time); auto base_dir_ = LogBase::GetLogBaseDir(); auto base_file_name_ = MetricWriter::FormMetricFileName(MetricTestUtils::GetAppName(), 0); MetricSearcher searcher(base_dir_, base_file_name_); auto ret1 = searcher.Find(0, 100); EXPECT_EQ(ret1.size(), 2); auto ret2 = searcher.Find(0, 1); EXPECT_EQ(ret2.size(), 1); auto ret3 = searcher.Find(time, 100); EXPECT_EQ(ret3.size(), 2); auto ret4 = searcher.Find(time + 1000, 100); EXPECT_EQ(ret4.size(), 1); MetricTestUtils::RemoveTestLogFile(); } TEST(MetricSearcherTest, TestFindInMultiFile) { int64_t time = Sentinel::Utils::TimeUtils::CurrentTimeMillis().count(); MetricTestUtils::TestWriteMetricLog(time); MetricTestUtils::TestWriteMetricLog(time + 10 * 1000); auto base_dir_ = LogBase::GetLogBaseDir(); auto base_file_name_ = MetricWriter::FormMetricFileName(MetricTestUtils::GetAppName(), 0); MetricSearcher searcher(base_dir_, base_file_name_); auto ret1 = searcher.Find(0, 100); EXPECT_EQ(ret1.size(), 4); auto ret2 = searcher.Find(0, 1); EXPECT_EQ(ret2.size(), 1); auto ret3 = searcher.Find(time, 100); EXPECT_EQ(ret3.size(), 4); auto ret4 = searcher.Find(time + 1000, 100); EXPECT_EQ(ret4.size(), 3); auto ret5 = searcher.Find(time + 3000, 100); EXPECT_EQ(ret5.size(), 2); MetricTestUtils::RemoveTestLogFile(); } TEST(MetricSearcherTest, TestFindByTimeAndResource) { int64_t time = Sentinel::Utils::TimeUtils::CurrentTimeMillis().count(); MetricTestUtils::TestWriteMetricLog(time); MetricTestUtils::TestWriteMetricLog(time + 10 * 1000); auto base_dir_ = LogBase::GetLogBaseDir(); auto base_file_name_ = MetricWriter::FormMetricFileName(MetricTestUtils::GetAppName(), 0); MetricSearcher searcher(base_dir_, base_file_name_); auto ret1 = searcher.FindByTimeAndResource(0, time, ""); EXPECT_EQ(ret1.size(), 1); auto ret2 = searcher.FindByTimeAndResource(time, time + 11 * 1000, ""); EXPECT_EQ(ret2.size(), 4); auto ret3 = searcher.FindByTimeAndResource(time, time + 5 * 1000, ""); EXPECT_EQ(ret3.size(), 2); auto ret4 = searcher.FindByTimeAndResource(0, time + 11 * 1000, "resource_context"); EXPECT_EQ(ret4.size(), 4); auto ret5 = searcher.FindByTimeAndResource(0, time + 11 * 1000, "error_resource_context"); EXPECT_EQ(ret5.size(), 0); MetricTestUtils::RemoveTestLogFile(); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_test_utils.h ================================================ #pragma once #include #include #include #include #include #include "sentinel-core/config/local_config.h" #include "sentinel-core/log/log_base.h" #include "sentinel-core/log/metric/metric_writer.h" #include "sentinel-core/utils/file_utils.h" namespace Sentinel { namespace Log { namespace MetricTestUtils { constexpr int64_t kSingleFileSize = 10000; constexpr int64_t kTotalFileCount = 10000; void TestWriteMetricLog(int64_t time) { MetricWriter writer(kSingleFileSize, kTotalFileCount); std::vector nodes; Stat::MetricItemSharedPtr node = std::make_shared(); node->set_timestamp(123456); node->set_pass_qps(123); node->set_block_qps(456); node->set_complete_qps(789); node->set_exception_qps(13323); node->set_rt(11); node->set_resource("resource_context"); nodes.push_back(node); writer.Write(time, nodes); writer.Write(time + 1000, nodes); writer.Close(); } std::string GetAppName() { auto app_name = Config::LocalConfig::GetInstance().app_name(); return app_name; } void RemoveTestLogFile() { auto files = Utils::FileUtils::ListFiles(LogBase::GetLogBaseDir()); for (auto &file_name : files) { auto path = LogBase::GetLogBaseDir() + file_name; remove(path.c_str()); } } } // namespace MetricTestUtils } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_writer.cc ================================================ #include "sentinel-core/log/metric/metric_writer.h" #include #include #include #include #include #include #include #include #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "absl/strings/str_replace.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" #include "sentinel-core/config/local_config.h" #include "sentinel-core/log/logger.h" #include "sentinel-core/utils/file_utils.h" #include "sentinel-core/utils/time_utils.h" using namespace Sentinel::Utils; namespace Sentinel { namespace Log { MetricWriter::MetricWriter(int64_t single_file_size, int32_t total_file_count) : single_file_size_(single_file_size), total_file_count_(total_file_count) { SENTINEL_LOG(info, "[MetricWriter] Creating new MetricWriter, singleFileSize={}, " "totalFileCount={}", std::to_string(single_file_size_), std::to_string(total_file_count_)); base_dir_ = LogBase::GetLogBaseDir(); auto time = TimeUtils::CurrentTimeMillis(); last_second_ = time.count() / 1000; single_file_size_ = single_file_size; total_file_count_ = total_file_count; time_second_base_ = absl::ToUnixSeconds( absl::FromDateTime(1970, 1, 1, 0, 0, 0, absl::LocalTimeZone())); pid_ = ::getpid(); auto app_name = Config::LocalConfig::GetInstance().app_name(); base_file_name_ = FormMetricFileName(app_name, pid_); } void MetricWriter::Write(int64_t time, std::vector &nodes) { std::lock_guard lk(lock_); if (time != -1) { for (auto &node : nodes) { node->set_timestamp(time); } } auto app_name = Config::LocalConfig::GetInstance().app_name(); // first write, should create file if (!metric_out_.is_open()) { base_file_name_ = FormMetricFileName(app_name, pid_); CloseAndNewFile(NextFileNameOfDay(time)); } auto second = time / 1000; if (second < last_second_) { } else if (second == last_second_) { if (first_write_) { first_write_ = false; WriteIndex(second, metric_out_.tellp()); } DoWrite(time, nodes); } else { WriteIndex(second, metric_out_.tellp()); if (IsNewDay(last_second_, second)) { CloseAndNewFile(NextFileNameOfDay(time)); DoWrite(time, nodes); } else { DoWrite(time, nodes); } last_second_ = second; } } void MetricWriter::DoWrite( int64_t time, const std::vector &nodes) { for (auto &node : nodes) { metric_out_ << node->ToFatString() << "\n"; } metric_out_.flush(); if (IsExceedMaxSingleFileSize()) { CloseAndNewFile(NextFileNameOfDay(time)); } } bool MetricWriter::IsNewDay(int64_t last_second, int64_t second) { int64_t last_day = (last_second - time_second_base_) / 86400; int64_t new_day = (second - time_second_base_) / 86400; return new_day > last_day; } void MetricWriter::WriteIndex(int64_t time, int64_t offset) { metric_index_out_ << time << " "; metric_index_out_ << offset << "\n"; metric_index_out_.flush(); } bool MetricWriter::IsExceedMaxSingleFileSize() { auto size = metric_out_.tellp(); return size >= single_file_size_; } std::string MetricWriter::FormSelfMetricFileName(const std::string &app_name) { return FormMetricFileName(app_name, ::getpid()); } std::string MetricWriter::FormMetricFileName(const std::string &app_name, int pid) { // dot is special char that should be replaced. static std::string dot = "."; static std::string separator = "-"; auto form_app_name = app_name; if (app_name.find('.') != std::string::npos) { form_app_name = absl::StrReplaceAll(app_name, {{dot, separator}}); } std::string file_name = form_app_name + separator + kMetricFile; if (LogBase::IsLogNameUsePid()) { file_name = file_name + ".pid" + std::to_string(pid); } return file_name; } std::string MetricWriter::NextFileNameOfDay(int64_t time) { auto date_str = absl::FormatTime("%Y-%m-%d", absl::FromUnixMillis(time), absl::LocalTimeZone()); auto file_name_model = base_file_name_ + "." + date_str; std::vector files; for (auto &file_name : Sentinel::Utils::FileUtils::ListFiles(base_dir_)) { if (file_name.find(file_name_model) != std::string::npos && !absl::EndsWith(file_name, kMetricIndexFileSuffix) && !absl::EndsWith(file_name, ".lck")) { files.push_back(base_dir_ + file_name); } } if (files.size() == 0) { SENTINEL_LOG(info, "NextFileNameOfDay: {} {}", base_dir_, file_name_model); return base_dir_ + file_name_model; } std::sort(files.begin(), files.end(), &MetricWriter::MetricFileNameComparator); auto last_file_name = files[files.size() - 1]; int n = 0; const std::vector strs = absl::StrSplit(last_file_name, "."); auto last_field = strs[strs.size() - 1]; std::regex re("[0-9]{1,10}"); if (strs.size() > 0 && std::regex_match(last_field, re)) { int64_t val = 0; if (absl::SimpleAtoi(last_field, &val)) { n = val; } } auto file_name = base_dir_ + file_name_model + "." + std::to_string(n + 1); SENTINEL_LOG(info, "NextFileNameOfDay: {}", file_name); return file_name; } bool MetricWriter::MetricFileNameComparator(const std::string &name1, const std::string &name2) { const std::vector l1 = absl::StrSplit(name1, "."); const std::vector l2 = absl::StrSplit(name2, "."); auto date_str1 = l1[2]; auto date_str2 = l2[2]; // in case of file name contains pid, skip it, like // Sentinel-Admin-metrics.log.pid22568.2018-12-24 if (absl::StartsWith(date_str1, "pid")) { date_str1 = l1[3]; date_str2 = l2[3]; } // compare date first int t = date_str1.compare(date_str2); if (t != 0) { return t < 0; } // same date, compare file number t = name1.length() - name2.length(); if (t != 0) { return t < 0; } return name1.compare(name2) < 0; } std::string MetricWriter::FormIndexFileName( const std::string &metric_file_name) { return metric_file_name + kMetricIndexFileSuffix; } void MetricWriter::CloseAndNewFile(const std::string &file_name) { RemoveMoreFiles(); DoClose(); metric_out_.open(file_name, std::ios::out); auto idx_file_name = FormIndexFileName(file_name); metric_index_out_.open(idx_file_name, std::ios::out); SENTINEL_LOG(info, "[MetricWriter] New metric file created: {}", file_name); SENTINEL_LOG(info, "[MetricWriter] New metric index file created: {}", idx_file_name); } void MetricWriter::Close() { std::lock_guard lk(lock_); DoClose(); } void MetricWriter::DoClose() { if (metric_out_.is_open()) { metric_out_.close(); } if (metric_index_out_.is_open()) { metric_index_out_.close(); } } bool MetricWriter::FileNameMatches(const std::string &file_name, const std::string &base_file_name) { if (absl::StartsWith(file_name, base_file_name)) { auto part = file_name.substr(base_file_name.size()); // part is like: ".yyyy-MM-dd.number", eg. ".2018-12-24.11" std::regex re("\\.[0-9]{4}-[0-9]{2}-[0-9]{2}(\\.[0-9]*)?"); return std::regex_match(part, re); } else { return false; } } std::vector MetricWriter::ListMetricFiles( const std::string &base_dir, const std::string &base_file_name) { std::vector vec; auto files = Sentinel::Utils::FileUtils::ListFiles(base_dir); for (auto &file_name : files) { if (FileNameMatches(file_name, base_file_name) && !absl::EndsWith(file_name, kMetricIndexFileSuffix) && !absl::EndsWith(file_name, ".lck")) { vec.push_back(base_dir + file_name); } } std::sort(vec.begin(), vec.end(), &MetricWriter::MetricFileNameComparator); return vec; } void MetricWriter::RemoveMoreFiles() { auto list = ListMetricFiles(base_dir_, base_file_name_); if (list.empty() || list.size() <= total_file_count_) { return; } auto diff = int(list.size() - total_file_count_); for (int i = 0; i < diff; i++) { auto &file_name = list[i]; auto index_file = FormIndexFileName(file_name); remove(file_name.c_str()); SENTINEL_LOG(info, "[MetricWriter] Removing metric file: {}", file_name); remove(index_file.c_str()); SENTINEL_LOG(info, "[MetricWriter] Removing metric index file: {}", index_file); } } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_writer.h ================================================ #pragma once #include #include #include #include #include #include "sentinel-core/statistic/base/metric_item.h" namespace Sentinel { namespace Log { static constexpr auto kMetricFile = "metrics.log"; static constexpr auto kMetricIndexFileSuffix = ".idx"; class MetricWriter { public: MetricWriter(int64_t single_file_size, int32_t total_file_count); virtual ~MetricWriter() {} void Write(int64_t time, std::vector& nodes); void Close(); static std::string FormSelfMetricFileName(const std::string& app_name); static std::string FormMetricFileName(const std::string& app_name, int pid); static std::string FormIndexFileName(const std::string& metric_file_name); static std::vector ListMetricFiles( const std::string& base_dir, const std::string& base_file_name); static bool MetricFileNameComparator(const std::string& name1, const std::string& name2); static bool FileNameMatches(const std::string& file_name, const std::string& base_file_name); private: void DoWrite(int64_t time, const std::vector& nodes); void WriteIndex(int64_t time, int64_t offset); std::string NextFileNameOfDay(int64_t time); void CloseAndNewFile(const std::string& file_name); bool IsNewDay(int64_t last_second, int64_t second); bool IsExceedMaxSingleFileSize(); void RemoveMoreFiles(); void DoClose(); private: int64_t single_file_size_; int32_t total_file_count_; std::string base_dir_; std::string base_file_name_; std::ofstream metric_out_; std::ofstream metric_index_out_; int64_t time_second_base_; int64_t last_second_ = -1; bool first_write_ = true; int pid_; std::mutex lock_; }; } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/log/metric/metric_writer_test.cc ================================================ #include "gmock/gmock.h" #include "gtest/gtest.h" #include #include #define private public #include "sentinel-core/log/log_base.h" #include "sentinel-core/log/metric/metric_test_utils.h" #include "sentinel-core/log/metric/metric_writer.h" #include "sentinel-core/utils/file_utils.h" #include "sentinel-core/utils/time_utils.h" #include namespace Sentinel { namespace Log { constexpr int64_t kSingleFileSize = 10000; constexpr int64_t kTotalFileCount = 10000; TEST(MetricWriterTest, TestFormMetricFileName) { MetricWriter writer(kSingleFileSize, kTotalFileCount); auto pid = 1001; LogBase::GetInstance().log_name_use_pid_ = true; auto name = writer.FormMetricFileName("appname", pid); EXPECT_EQ(name, "appname-metrics.log.pid1001"); LogBase::GetInstance().log_name_use_pid_ = false; name = writer.FormMetricFileName("appname", pid); EXPECT_EQ(name, "appname-metrics.log"); } TEST(MetricWriterTest, TestFormIndexFileName) { MetricWriter writer(kSingleFileSize, kTotalFileCount); auto name = writer.FormIndexFileName("appname-metrics.log"); EXPECT_EQ(name, "appname-metrics.log.idx"); } TEST(MetricWriterTest, TestMetricFileNameComparator) { MetricWriter writer(kSingleFileSize, kTotalFileCount); std::vector vec{ "testappname-metrics.log.2019-05-20", "testappname-metrics.log.2019-05-20.2", "testappname-metrics.log.2019-05-20.1", "testappname-metrics.log.2019-05-19.2", "testappname-metrics.log.2019-05-19", "testappname-metrics.log.2019-05-21", "testappname-metrics.log.2019-05-19.1", "testappname-metrics.log.2019-05-21.1", }; std::sort(vec.begin(), vec.end(), &MetricWriter::MetricFileNameComparator); std::vector res{ "testappname-metrics.log.2019-05-19", "testappname-metrics.log.2019-05-19.1", "testappname-metrics.log.2019-05-19.2", "testappname-metrics.log.2019-05-20", "testappname-metrics.log.2019-05-20.1", "testappname-metrics.log.2019-05-20.2", "testappname-metrics.log.2019-05-21", "testappname-metrics.log.2019-05-21.1", }; for (int i = 0; i < vec.size(); ++i) { EXPECT_EQ(vec[i], res[i]); } } TEST(MetricWriterTest, TestFileNameMatches) { auto base_file_name = "appname-metric.log"; auto ret = MetricWriter::FileNameMatches( base_file_name + std::string(".2019-05-19"), base_file_name); EXPECT_EQ(ret, true); ret = MetricWriter::FileNameMatches(std::string("pre") + base_file_name, base_file_name); EXPECT_EQ(ret, false); ret = MetricWriter::FileNameMatches( base_file_name + std::string(".2019.05.19"), base_file_name); EXPECT_EQ(ret, false); ret = MetricWriter::FileNameMatches( base_file_name + std::string(".2019-05-19.2"), base_file_name); EXPECT_EQ(ret, true); } TEST(MetricWriterTest, TestNextFileNameOfDay) { MetricWriter writer(kSingleFileSize, kTotalFileCount); writer.base_file_name_ = "teatappname-metrics.log"; writer.base_dir_ = "./"; auto time = 1558583341420; // 2019/5/23 11:49:1 auto file_name = writer.NextFileNameOfDay(time); EXPECT_EQ(file_name, "./teatappname-metrics.log.2019-05-23"); std::ofstream metric_out; metric_out.open(file_name, std::ios::out); metric_out.close(); file_name = writer.NextFileNameOfDay(time); remove("./teatappname-metrics.log.2019-05-23"); EXPECT_EQ(file_name, "./teatappname-metrics.log.2019-05-23.1"); file_name = writer.NextFileNameOfDay(time + 3600 * 24 * 1000); EXPECT_EQ(file_name, "./teatappname-metrics.log.2019-05-24"); } TEST(MetricWriterTest, TestIsNextDay) { MetricWriter writer(kSingleFileSize, kTotalFileCount); auto time1 = 1558583341; // 2019/5/23 11:49:1 auto ret = writer.IsNewDay(time1, time1); EXPECT_EQ(ret, false); ret = writer.IsNewDay(time1, time1 + 100); EXPECT_EQ(ret, false); ret = writer.IsNewDay(time1, time1 + 3600 * 24); EXPECT_EQ(ret, true); ret = writer.IsNewDay(time1, time1 + 3600 * 24 * 2); EXPECT_EQ(ret, true); } TEST(MetricWriterTest, TestIsExceedMaxSingleFileSize) { MetricWriter writer(kSingleFileSize, kTotalFileCount); auto file_name = "./test_max_single_file_size"; writer.metric_out_.open(file_name, std::ios::out); writer.metric_out_ << "text"; auto ret = writer.IsExceedMaxSingleFileSize(); EXPECT_EQ(ret, false); std::string content("a", kSingleFileSize); writer.metric_out_ << content; ret = writer.IsExceedMaxSingleFileSize(); EXPECT_EQ(ret, true); writer.metric_out_.close(); remove(file_name); } TEST(MetricWriterTest, TestRemoveMoreFiles) { auto total_file_count = 2; MetricWriter writer(kSingleFileSize, total_file_count); int64_t time = Sentinel::Utils::TimeUtils::CurrentTimeMillis().count(); MetricTestUtils::TestWriteMetricLog(time); auto list1 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list1.size(), 1); MetricTestUtils::TestWriteMetricLog(time + 10 * 1000); auto list2 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list2.size(), 2); MetricTestUtils::TestWriteMetricLog(time + 20 * 1000); auto list3 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list3.size(), 3); //移除多于的文件 writer.RemoveMoreFiles(); auto list4 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list4.size(), 2); MetricTestUtils::TestWriteMetricLog(time + 30 * 1000); auto list5 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list5.size(), 3); //移除多于的文件 writer.RemoveMoreFiles(); auto list6 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list6.size(), 2); MetricTestUtils::TestWriteMetricLog(time + 40 * 1000); auto list7 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list7.size(), 3); MetricTestUtils::TestWriteMetricLog(time + 50 * 1000); auto list8 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list8.size(), 4); //移除多于的文件 writer.RemoveMoreFiles(); auto list9 = writer.ListMetricFiles(writer.base_dir_, writer.base_file_name_); EXPECT_EQ(list9.size(), 2); MetricTestUtils::RemoveTestLogFile(); } } // namespace Log } // namespace Sentinel ================================================ FILE: sentinel-core/param/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "param_flow_rule_constants_lib", srcs = [ "param_flow_rule_constants.h", ], copts = DEFAULT_COPTS ) cc_library( name = "param_flow_item_lib", srcs = [ "param_flow_item.cc", "param_flow_item.h", ], deps = [ ":param_flow_rule_constants_lib", "//sentinel-core/param/statistic:any_cmp_lib", "@com_google_absl//absl/types:any", "@com_google_absl//absl/strings:str_format", ], copts = DEFAULT_COPTS, ) cc_library( name = "param_flow_rule_checker_lib", srcs = [ "param_flow_rule_checker.h", "param_flow_rule_checker.cc", ], copts = DEFAULT_COPTS, deps = [ ":param_flow_rule_lib", "//sentinel-core/common:entry_lib", "//sentinel-core/param/statistic:param_metric_lib", ] ) cc_library( name = "param_flow_rule_lib", srcs = [ "param_flow_rule.cc", "param_flow_rule.h", ], copts = DEFAULT_COPTS, deps = select({ "//bazel:is_osx": [ ":param_flow_rule_constants_lib", ":param_flow_item_lib", "//sentinel-core/common:rule_lib", "//sentinel-core/param/statistic:param_leap_array_key_lib", "@com_google_absl//absl/strings:str_format", "@com_github_libtbb_osx//:tbb_osx", ], "//conditions:default": [ ":param_flow_rule_constants_lib", ":param_flow_item_lib", "//sentinel-core/common:rule_lib", "//sentinel-core/param/statistic:param_leap_array_key_lib", "@com_google_absl//absl/strings:str_format", "@com_github_libtbb//:tbb", ], }), ) # https://docs.bazel.build/versions/master/be/c-cpp.html#cc_library cc_library( name = "param_flow_rule_manager_lib", srcs = [ "param_flow_rule_manager.cc", "param_flow_rule_manager.h", ], copts = DEFAULT_COPTS, deps = [ ":param_flow_rule_lib", "//sentinel-core/property:dynamic_sentinel_property_lib", "@com_google_absl//absl/synchronization", ], ) cc_library( name = "param_flow_slot_lib", srcs = [ "param_flow_slot.cc", "param_flow_slot.h", ], copts = DEFAULT_COPTS, deps = [ ":param_flow_rule_manager_lib", ":param_flow_rule_checker_lib", "//sentinel-core/slot/base:rule_checker_slot_interface", "@com_google_absl//absl/strings:str_format", ], ) cc_test( name = "param_slot_unittests", srcs = [ "param_flow_slot_test.cc", ], copts = TEST_COPTS, deps = [ "//sentinel-core/common:string_resource_wrapper_lib", "//sentinel-core/param:param_flow_slot_lib", "//sentinel-core/test/mock/statistic/node:mock_lib", "@com_google_googletest//:gtest_main", ], ) ================================================ FILE: sentinel-core/param/param_flow_item.cc ================================================ #include "sentinel-core/param/param_flow_item.h" #include "absl/strings/str_format.h" namespace Sentinel { namespace Param { bool ParamFlowItem::operator==(const ParamFlowItem& item) const noexcept { return param_type_ == item.param_type_ && param_value_ == item.param_value_ && threshold_ == item.threshold_; } std::string ParamFlowItem::ToString() const { std::string typeName; if (IsInt32(param_value_)) { typeName = "int32_t"; } else if (IsInt64(param_value_)) { typeName = "int64_t"; } else if (IsString(param_value_)) { typeName = "String"; } else { typeName = "unknown"; } return absl::StrFormat("ParamFlowItem{threshold=%.2lf, type=%s}", threshold_, typeName); } ParamFlowItemList::ParamFlowItemList( std::initializer_list args) { for (const auto& arg : args) { this->push_back(arg); } } bool ParamFlowItemList::operator==(const ParamFlowItemList& list) const noexcept { if (this->size() != list.size()) { return false; } for (int i = 0; i < this->size(); i++) { if (!(list[i] == this->at(i))) { return false; } } return true; } std::string ParamFlowItemList::ToString() const { std::string str("["); for (const auto& item : *this) { str += item.ToString(); str += ","; } str += "]"; return str; } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_item.h ================================================ #pragma once #include #include #include #include "absl/types/any.h" #include "sentinel-core/param/param_flow_rule_constants.h" #include "sentinel-core/param/statistic/any_cmp.h" namespace Sentinel { namespace Param { class ParamFlowItem { public: ParamFlowItem() : threshold_(-1) {} ParamFlowItem(absl::any param_value, ParamItemType param_type, double threshold = -1) : param_value_(param_value), threshold_(threshold), param_type_(param_type) {} ParamItemType param_type() const { return param_type_; } absl::any param_value() const { return param_value_; } double threshold() const { return threshold_; } void set_param_type(ParamItemType param_type) { param_type_ = param_type; } void set_param_value(absl::any param_value) { param_value_ = param_value; } void set_threshold(double threshold) { threshold_ = threshold; } bool operator==(const ParamFlowItem& item) const noexcept; std::string ToString() const; private: ParamItemType param_type_; absl::any param_value_; double threshold_ = -1; }; class ParamFlowItemList : public std::vector { public: ParamFlowItemList() = default; ParamFlowItemList(std::initializer_list args); bool operator==(const ParamFlowItemList& list) const noexcept; std::string ToString() const; }; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_rule.cc ================================================ #include "sentinel-core/param/param_flow_rule.h" #include "absl/strings/str_format.h" namespace Sentinel { namespace Param { void ParamFlowRule::FillExceptionFlowItems() const { if (specific_item_list_.size() == 0) { return; } for (auto& item : specific_item_list_) { parsed_hot_items_->insert( std::make_pair<>(item.param_value(), item.threshold())); } } bool ParamFlowRule::operator==(const ParamFlowRule& rule) const { if (!(resource_ == rule.resource() && param_idx_ == rule.param_idx() && metric_type_ == rule.metric_type() && threshold_ == rule.threshold() && interval_in_ms_ == rule.interval_in_ms() && sample_count_ == rule.sample_count() && cache_size_ == rule.cache_size() && cluster_mode_ == rule.cluster_mode() && specific_item_list_.size() == rule.specific_item_list_.size())) { return false; } for (int i = 0; i < specific_item_list_.size(); i++) { if (!(rule.specific_item_list_[i] == specific_item_list_[i])) { return false; } } return true; } std::string ParamFlowRule::ToString() const { return absl::StrFormat( "ParamFlowRule{resource=%s, param_idx=%d, metric_type=%d, " "threshold=%.2f, interval_in_ms=%d, sample_count=%d, " "cache_size=%d, cluster_mode=%d, specific_item_list=%s}", resource_, param_idx_, static_cast(metric_type_), threshold_, interval_in_ms_, sample_count_, cache_size_, cluster_mode_, specific_item_list_.ToString()); } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_rule.h ================================================ #pragma once #include #include #include #include #include "absl/types/any.h" #include "sentinel-core/common/rule.h" #include "sentinel-core/log/logger.h" #include "sentinel-core/param/param_flow_item.h" #include "sentinel-core/param/param_flow_rule_constants.h" #include "sentinel-core/param/statistic/param_leap_array_key.h" #include "tbb/concurrent_hash_map.h" namespace Sentinel { namespace Param { using HotItemsMap = std::unordered_map; using HotItemsMapSharedPtr = std::shared_ptr; // Deep copy safe class ParamFlowRule : public Rule { public: ParamFlowRule() : ParamFlowRule("") {} explicit ParamFlowRule(const std::string& resource) : resource_(resource), metric_key_(std::make_shared()), parsed_hot_items_(std::make_shared(1)) {} virtual ~ParamFlowRule() = default; const std::string& resource() const noexcept { return resource_; } ParamFlowMetricType metric_type() const noexcept { return metric_type_; } double threshold() const noexcept { return threshold_; } int32_t param_idx() const noexcept { return param_idx_; } int32_t cache_size() const noexcept { return cache_size_; } int32_t interval_in_ms() const noexcept { return interval_in_ms_; } int32_t sample_count() const noexcept { return sample_count_; } bool cluster_mode() const noexcept { return cluster_mode_; } ParamFlowItemList specific_item_list() const { return specific_item_list_; } HotItemsMapSharedPtr parsed_hot_items() const noexcept { return parsed_hot_items_; } ParamLeapArrayKeySharedPtr metric_key() const noexcept { return metric_key_; } void set_resource(const std::string& resource) noexcept { this->resource_ = resource; } void set_metric_type(ParamFlowMetricType metric_type) noexcept { this->metric_type_ = metric_type; } void set_threshold(double threshold) noexcept { this->threshold_ = threshold; } void set_param_idx(int32_t index) noexcept { this->param_idx_ = index; this->metric_key_->param_idx_ = index; } void set_cache_size(int32_t cache_size) noexcept { this->cache_size_ = cache_size; this->metric_key_->cache_size_ = cache_size; } void set_sample_count(int32_t sample_count) noexcept { this->sample_count_ = sample_count; this->metric_key_->sample_count_ = sample_count; } void set_interval_in_ms(int32_t interval) noexcept { this->interval_in_ms_ = interval; this->metric_key_->interval_in_ms_ = interval; } void set_cluster_mode(bool cluster_mode) noexcept { this->cluster_mode_ = cluster_mode; } void set_param_flow_item_list(ParamFlowItemList&& list) noexcept { this->specific_item_list_ = std::move(list); } void FillExceptionFlowItems() const; bool operator==(const ParamFlowRule& rule) const; std::string ToString() const; const static int32_t DEFAULT_CACHE_SIZE = 200; private: std::string resource_; // resource int32_t param_idx_ = -1; ParamFlowMetricType metric_type_{ParamFlowMetricType::kQps}; // grade double threshold_ = 0; // threshold int32_t interval_in_ms_ = 1000; int32_t sample_count_ = 1; int32_t cache_size_ = DEFAULT_CACHE_SIZE; mutable ParamFlowItemList specific_item_list_; bool cluster_mode_ = false; // reserved field ParamLeapArrayKeySharedPtr metric_key_; // internal key for metric storage mutable HotItemsMapSharedPtr parsed_hot_items_; // parsed param items }; using ParamFlowRuleSharedPtr = std::shared_ptr; using ParamFlowRuleList = std::vector; using ParamFlowRulePtrList = std::vector; using ParamFlowRulePtrListSharedPtr = std::shared_ptr; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_rule_checker.cc ================================================ #include "sentinel-core/param/param_flow_rule_checker.h" namespace Sentinel { namespace Param { bool ParamFlowChecker::PassCheck(ParamMetricSharedPtr& metric, const ParamFlowRuleSharedPtr& rule, int count, const std::vector& params) { return PassLocalCheck(metric, rule, count, params); } bool ParamFlowChecker::PassLocalCheck(ParamMetricSharedPtr& metric, const ParamFlowRuleSharedPtr& rule, int count, const std::vector& params) { for (const auto& param : params) { if (!PassSingleValueCheck(metric, rule, count, param)) { return false; } } return true; } bool ParamFlowChecker::PassSingleValueCheck(ParamMetricSharedPtr& metric, const ParamFlowRuleSharedPtr& rule, int count, const absl::any& param) { bool result = true; auto item_map = rule->parsed_hot_items(); if (rule->metric_type() == ParamFlowMetricType::kQps) { int threshold = static_cast(rule->threshold()); int curCount = metric->PassInterval(rule->metric_key(), param); auto it = item_map->find(param); if (it != item_map->end()) { threshold = it->second; } result = (count + curCount <= threshold); } else if (rule->metric_type() == ParamFlowMetricType::kThreadCount) { int threshold = static_cast(rule->threshold()); int threadCount = metric->GetThreadCount(rule->param_idx(), param); auto it = item_map->find(param); if (it != item_map->end()) { threshold = it->second; } result = (++threadCount <= threshold); } else { SENTINEL_LOG(error, "[ParamFlowChecker] Illegal metricType: {}", static_cast(rule->metric_type())); } return result; } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_rule_checker.h ================================================ #pragma once #include "sentinel-core/common/entry.h" #include "sentinel-core/param/param_flow_rule.h" #include "sentinel-core/param/statistic/param_metric.h" #include "sentinel-core/slot/base/token_result.h" namespace Sentinel { namespace Param { class ParamFlowChecker { public: ParamFlowChecker() = default; ~ParamFlowChecker() = default; static bool PassCheck(ParamMetricSharedPtr& metric, const ParamFlowRuleSharedPtr& rule, int count, const std::vector& params); private: static bool PassLocalCheck(ParamMetricSharedPtr& metric, const ParamFlowRuleSharedPtr& rule, int count, const std::vector& params); static bool PassSingleValueCheck(ParamMetricSharedPtr& metric, const ParamFlowRuleSharedPtr& rule, int count, const absl::any& param); }; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_rule_constants.h ================================================ #pragma once namespace Sentinel { namespace Param { enum class ParamFlowMetricType { kThreadCount = 0, kQps = 1, // default mode kNum }; enum class ParamFlowRelationStrategy { kDirect = 0, // default relation strategy kAssociatedResource = 1, kInvocationChainEntrance = 2 }; enum class ParamFlowControlBehavior { kReject = 0, // default behavior kWarmUp = 1, kThrotting = 2, kWarmUpThrottling = 3 }; enum class ParamItemType { kString = 0, // Default type kInt32 = 1, kInt64 = 2, kNum }; static constexpr const char* kString = "String"; static constexpr const char* kInt32 = "int"; static constexpr const char* kInt64 = "long"; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_rule_manager.cc ================================================ #include "sentinel-core/param/param_flow_rule_manager.h" namespace Sentinel { namespace Param { constexpr auto kParamPropertyListenerName = "ParamPropertyListener"; bool IsValidRule(const ParamFlowRule& rule) { bool isOK = static_cast(rule.metric_type()) >= 0 && static_cast(rule.metric_type()) < static_cast(ParamFlowMetricType::kNum); isOK = isOK && rule.threshold() >= 0 && rule.param_idx() >= 0 && rule.cache_size() > 0 && rule.sample_count() > 0 && rule.interval_in_ms() > 0; if (!isOK) { return false; } // Check each item for (const auto& item : rule.specific_item_list()) { if (item.threshold() < 0 || !item.param_value().has_value()) { return false; } if (item.param_type() > ParamItemType::kNum) { return false; } } return true; } void LogParamMap(const ParamFlowRulePtrMapSharedPtr map) { std::string s("["); for (const auto& rule_set_pair : *map) { for (const ParamFlowRuleSharedPtr& rule : *(rule_set_pair.second)) { s += rule->ToString(); s += ","; } } s[s.size() - 1] = ']'; SENTINEL_LOG(info, "[ParamFlowRuleManager] Param flow rules received: {}", s); } void ParamFlowRuleManager::RegisterToProperty( const ParamFlowRulePropertySharedPtr& property) { if (property == nullptr) { return; } std::lock_guard lck(property_mtx_); SENTINEL_LOG(info, "Registering new property to ParamFlowRuleManager"); cur_property_->RemoveListener(kParamPropertyListenerName); cur_property_ = property; cur_property_->AddListener(std::make_unique()); } ParamFlowRulePtrListSharedPtr ParamFlowRuleManager::GetRuleOfResource( const std::string& resource) const { absl::ReaderMutexLock lck(&update_mtx_); auto it = rule_map_->find(resource); if (it == rule_map_->end()) { return nullptr; } return it->second; } ParamFlowRulePtrMapSharedPtr ParamFlowRuleManager::GetRuleMap() const { absl::ReaderMutexLock lck(&update_mtx_); return rule_map_; } ParamFlowRuleManager::ParamFlowRuleManager() { cur_property_ = std::make_shared>(); cur_property_->AddListener(std::make_unique()); } ParamFlowRulePtrMapSharedPtr ParamPropertyListener::AggregatedHotParamRules( const ParamFlowRuleList& list) const { ParamFlowRulePtrMapSharedPtr new_map = std::make_shared(); for (const ParamFlowRule& rule : list) { if (!IsValidRule(rule)) { SENTINEL_LOG(error, "[ParamFlowRuleManager] Invalid param flow rule: {}", rule.ToString()); continue; } ParamFlowRuleSharedPtr p = std::make_shared(rule); rule.FillExceptionFlowItems(); // Insert directly. Existing key will not disturb auto pair = new_map->insert(std::make_pair<>( p->resource(), std::make_shared())); ParamFlowRulePtrMap::iterator it = pair.first; it->second->push_back(p); } return new_map; } void ParamPropertyListener::ConfigUpdate(const ParamFlowRuleList& value, bool first_load) { ParamFlowRuleManager& m = ParamFlowRuleManager::GetInstance(); ParamFlowRulePtrMapSharedPtr new_map = AggregatedHotParamRules(value); absl::WriterMutexLock lck(&(m.update_mtx_)); m.rule_map_ = new_map; LogParamMap(m.rule_map_); } const std::string ParamPropertyListener::Name() const { return kParamPropertyListenerName; } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_rule_manager.h ================================================ #pragma once #include #include #include #include #include "absl/synchronization/mutex.h" #include "absl/types/any.h" #include "sentinel-core/param/param_flow_rule.h" #include "sentinel-core/property/dynamic_sentinel_property.h" #include "sentinel-core/property/property_listener.h" namespace Sentinel { namespace Param { using ParamFlowRulePropertySharedPtr = Property::SentinelPropertySharedPtr; using ParamFlowRulePtrMap = std::unordered_map; using ParamFlowRulePtrMapSharedPtr = std::shared_ptr; class ParamFlowRuleManager { public: static ParamFlowRuleManager& GetInstance() { static ParamFlowRuleManager* instance = new ParamFlowRuleManager(); return *instance; } friend class ParamPropertyListener; void RegisterToProperty(const ParamFlowRulePropertySharedPtr& property); bool LoadRules(const ParamFlowRuleList& rules) { return cur_property_->UpdateValue(rules); } ParamFlowRulePtrListSharedPtr GetRuleOfResource( const std::string& resource) const; ParamFlowRulePtrMapSharedPtr GetRuleMap() const; ParamFlowRuleList GetRules() const; ParamFlowRuleList GetRulesForResource(const std::string& resource) const; private: ParamFlowRuleManager(); ParamFlowRulePropertySharedPtr cur_property_; ParamFlowRulePtrMapSharedPtr rule_map_; mutable std::mutex property_mtx_; mutable absl::Mutex update_mtx_; }; class ParamPropertyListener : public Property::PropertyListener { public: ParamPropertyListener() = default; ~ParamPropertyListener() = default; ParamFlowRulePtrMapSharedPtr AggregatedHotParamRules( const ParamFlowRuleList& list) const; void ConfigUpdate(const ParamFlowRuleList& value, bool first_load) override; const std::string Name() const override; }; bool IsValidRule(const ParamFlowRule& rule); } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_slot.cc ================================================ #include "sentinel-core/param/param_flow_slot.h" #include namespace Sentinel { namespace Slot { MetricMapSharedPtr ParamFlowSlot::metric_map_ = std::make_shared< tbb::concurrent_hash_map>(); const std::string& ParamFlowSlot::Name() const { return name_; } TokenResultSharedPtr ParamFlowSlot::Entry( const EntrySharedPtr& entry, Stat::NodeSharedPtr&, int count, int flag, const std::vector& params) { TokenResultSharedPtr res = TokenResult::Ok(); auto rule_map_ = Param::ParamFlowRuleManager::GetInstance().GetRuleMap(); if (rule_map_ && rule_map_->size() > 0) { res = CheckFlow(entry->resource()->name(), count, params); } return res; } // NOTE: Although `initializeForIndex` is not a const method, we can safely // use const_accessor here while put concurrency control in inner method void ParamFlowSlot::initHotParamMetricsFor( const std::string& resource, const Param::ParamFlowRuleSharedPtr& rule) { MetricMap::const_accessor cac; // Do insert if this key does not exist, do nothing otherwise. if (!metric_map_->find(cac, resource)) { metric_map_->insert( cac, std::make_pair<>(resource, std::make_shared())); } if (cac->second) { cac->second->initializeForRule(rule->metric_key()); } } Param::ParamMetricSharedPtr ParamFlowSlot::GetParamMetric( const std::string& resource) { if (resource.size() == 0) { return std::shared_ptr(nullptr); } MetricMap::const_accessor cac; if (metric_map_->find(cac, resource)) { return cac->second; } else { return std::shared_ptr(nullptr); } } TokenResultSharedPtr ParamFlowSlot::CheckFlow( const std::string& resource, int count, const std::vector& params) { Param::ParamFlowRulePtrListSharedPtr rule_list = Param::ParamFlowRuleManager::GetInstance().GetRuleOfResource(resource); if (rule_list) { for (const Param::ParamFlowRuleSharedPtr& rule : *rule_list) { if (!rule) { SENTINEL_LOG(error, "[ParamFlowSlot] rule in vector is nullptr"); continue; } initHotParamMetricsFor(resource, rule); auto metric = GetParamMetric(resource); if (!checker_.PassCheck(metric, rule, count, params)) { metric->AddBlock(count, params); return TokenResult::Blocked("param exception"); } } } return TokenResult::Ok(); } void ParamFlowSlot::Exit(const EntrySharedPtr&, int, const std::vector& params) { // Do nothing } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_slot.h ================================================ #pragma once #include "sentinel-core/param/param_flow_rule_checker.h" #include "sentinel-core/param/param_flow_rule_manager.h" #include "sentinel-core/slot/base/rule_checker_slot.h" namespace Sentinel { namespace Slot { constexpr auto kParamFlowSlotName = "ParamFlowSlot"; using MetricMap = tbb::concurrent_hash_map; using MetricMapSharedPtr = std::shared_ptr; class ParamFlowSlot : public RuleCheckerSlot { public: ParamFlowSlot() = default; virtual ~ParamFlowSlot() = default; TokenResultSharedPtr Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr&, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; const std::string& Name() const override; void initHotParamMetricsFor(const std::string& resource, const Param::ParamFlowRuleSharedPtr& rule); TokenResultSharedPtr CheckFlow(const std::string& resource, int count, const std::vector& params); friend class Param::ParamFlowRuleManager; static Param::ParamMetricSharedPtr GetParamMetric( const std::string& resource); private: static MetricMapSharedPtr metric_map_; const std::string name_{kParamFlowSlotName}; Param::ParamFlowChecker checker_{}; }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/param/param_flow_slot_test.cc ================================================ #include #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/test/mock/statistic/node/mock.h" #include "sentinel-core/common/string_resource_wrapper.h" #include "sentinel-core/param/param_flow_rule_constants.h" #include "sentinel-core/param/param_flow_slot.h" using testing::_; using testing::InSequence; using testing::Mock; using testing::Return; namespace Sentinel { namespace Slot { TEST(ParamFlowSlotTest, ParamFlowControlSingleThreadIntegrationTest) { Sentinel::Log::Logger::InitDefaultLogger(); std::string resourceName{"testResource"}; std::string anotherResourceName{"anotherTestResource"}; EntryContextSharedPtr context = std::make_shared("test_context"); Stat::NodeSharedPtr uselessNode = std::make_shared(); auto resource = std::make_shared(resourceName, EntryType::OUT); auto entry = std::make_shared(resource, context); entry->set_cur_node(uselessNode); std::vector myParams; ParamFlowSlot slot; { // Pass since no rule exists. auto result = slot.Entry(entry, uselessNode, 1000, 0, myParams); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, result->status()); } Param::ParamFlowRule rule0{resourceName}, rule1{resourceName}, rule2{anotherResourceName}, rule3{resourceName}, rule4{resourceName}; rule0.set_metric_type(Param::ParamFlowMetricType::kQps); rule0.set_param_idx(0); rule0.set_threshold(10); rule0.set_interval_in_ms(5000); // limit 10 Qs in 5s, QPS=2 Param::ParamFlowItem item0(78, Param::ParamItemType::kInt32, 100); rule0.set_param_flow_item_list({item0}); rule1.set_param_idx(1); rule1.set_threshold(1); rule1.set_metric_type( Param::ParamFlowMetricType::kThreadCount); // limit 10 Qs in 5s, QPS=2 // rule2 should not work (resource name not match) rule2.set_param_idx(1); rule2.set_threshold(1); rule2.set_metric_type(Param::ParamFlowMetricType::kQps); // rule3 and rule4 are neither valid rule rule3.set_param_idx(1); rule3.set_threshold(-1); rule3.set_metric_type(Param::ParamFlowMetricType::kQps); rule4.set_param_idx(1); rule4.set_threshold(1); rule4.set_sample_count(0); rule4.set_metric_type(Param::ParamFlowMetricType::kQps); Param::ParamFlowRuleManager& m = Param::ParamFlowRuleManager::GetInstance(); m.LoadRules({rule0, rule1, rule2, rule3, rule4}); EXPECT_EQ(m.GetRuleOfResource(resourceName)->size(), 2); EXPECT_EQ(m.GetRuleOfResource(anotherResourceName)->size(), 1); myParams.push_back(15213); // NOTE: If the time of invoking Add and Get of counter are unluckly in two // different windows, the test will fail. Longer window interval can reduce // but not eliminate this possibility. // If this occurs, retry until the test passes... { EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry, uselessNode, 9, 0, myParams)->status()); slot.GetParamMetric(resource->name())->AddPass(9, myParams); EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, slot.Entry(entry, uselessNode, 2, 0, myParams)->status()); EXPECT_EQ(slot.GetParamMetric(resource->name()) ->PassInterval(rule0.metric_key(), myParams[0]), 9); EXPECT_EQ(slot.GetParamMetric(resource->name()) ->BlockInterval(rule0.metric_key(), myParams[0]), 2); myParams.pop_back(); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry, uselessNode, 100, 0, myParams)->status()); EXPECT_EQ(slot.GetParamMetric(resource->name()) ->PassInterval(rule0.metric_key(), 15213), 9); EXPECT_EQ(slot.GetParamMetric(resource->name()) ->BlockInterval(rule0.metric_key(), 15213), 2); } { // Exception hot item rule myParams.push_back(78); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry, uselessNode, 50, 0, myParams)->status()); slot.GetParamMetric(resource->name())->AddPass(50, myParams); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry, uselessNode, 50, 0, myParams)->status()); EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, slot.Entry(entry, uselessNode, 51, 0, myParams)->status()); myParams.pop_back(); } { // thread limit myParams.push_back(13239); myParams.push_back(std::string("justAnExample")); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry, uselessNode, 1, 0, myParams)->status()); slot.GetParamMetric(resource->name())->AddThreadCount(myParams); // Blocked due to thread count exceeded on idx 1 EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, slot.Entry(entry, uselessNode, 1, 0, myParams)->status()); EXPECT_EQ(1, slot.GetParamMetric(resource->name()) ->GetThreadCount(1, std::string("justAnExample"))); slot.GetParamMetric(resource->name())->DecreaseThreadCount(myParams); EXPECT_EQ(0, slot.GetParamMetric(resource->name()) ->GetThreadCount(1, std::string("justAnExample"))); // Pass since value on idx 1 has changed myParams[1] = std::string("anotherExample"); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry, uselessNode, 1, 0, myParams)->status()); } } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "any_cmp_lib", srcs = [ "any_cmp.cc", "any_cmp.h", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/types:any", "//sentinel-core/log:logger_lib", ], ) cc_library( name = "scalable_cache_lib", srcs = [ "scalable_cache.h", "lru_cache.h", ], copts = DEFAULT_COPTS, deps = select({ "//bazel:is_osx": [ "@com_github_libtbb_osx//:tbb_osx", ], "//conditions:default": [ "@com_github_libtbb//:tbb", ], }), ) cc_library( name = "param_events_lib", srcs = ["param_event.h",], copts = DEFAULT_COPTS, ) cc_library( name = "param_leap_array_key_lib", srcs = ["param_leap_array_key.h"], copts = DEFAULT_COPTS, ) cc_library( name = "param_leap_array_lib", srcs = [ "param_leap_array.h", "param_leap_array.cc", ], copts = DEFAULT_COPTS, deps = [ ":param_bucket_lib", "//sentinel-core/statistic/base:leap_array_lib", ], ) cc_library( name = "param_bucket_lib", srcs = [ "param_bucket.h", "param_bucket.cc", ], copts = DEFAULT_COPTS, deps = [ ":any_cmp_lib", ":param_events_lib", ":scalable_cache_lib", "@com_google_absl//absl/types:any", ], # linkopts = select({ # "//conditions:default": ["-ltbb"], # }), ) cc_library( name = "param_metric_lib", srcs = [ "param_metric.h", "param_metric.cc", ], copts = DEFAULT_COPTS, deps = [ ":param_leap_array_lib", ":param_leap_array_key_lib", ], ) cc_test( name = "param_metric_unittests", srcs = [ "param_metric_test.cc", ], copts = TEST_COPTS, deps = [ ":param_metric_lib", "//sentinel-core/param:param_flow_rule_manager_lib", "@com_google_googletest//:gtest_main", ], ) # https://docs.bazel.build/versions/master/be/c-cpp.html#cc_binary cc_binary( name = "any_cmp_test", srcs = ["any_cmp_test.cc"], copts = DEFAULT_COPTS, deps = [ ":any_cmp_lib", ], ) ================================================ FILE: sentinel-core/param/statistic/any_cmp.cc ================================================ #include "sentinel-core/param/statistic/any_cmp.h" #include "sentinel-core/log/logger.h" namespace Sentinel { namespace Param { const std::string INT_TYPE_STR(typeid(3).name()); const std::string INT32_TYPE_STR(typeid(static_cast(3)).name()); const std::string INT64_TYPE_STR(typeid(static_cast(3L)).name()); const std::string STRING_TYPE_STR(typeid(std::string("example_str")).name()); bool IsInt32(const absl::any& a) { return INT_TYPE_STR.compare(a.type().name()) == 0; } bool IsInt64(const absl::any& a) { return INT64_TYPE_STR.compare(a.type().name()) == 0; } bool IsString(const absl::any& a) { std::string type_str(a.type().name()); return type_str.find(STRING_TYPE_STR) != type_str.npos; } } // namespace Param } // namespace Sentinel namespace absl { using Sentinel::Param::IsInt32; using Sentinel::Param::IsInt64; using Sentinel::Param::IsString; bool operator==(const absl::any& any1, const absl::any& any2) { if (IsInt32(any1) && IsInt32(any2)) { return absl::any_cast(any1) == absl::any_cast(any2); } else if (IsInt64(any1) && IsInt64(any2)) { return absl::any_cast(any1) == absl::any_cast(any2); } else if (IsString(any1) && IsString(any2)) { return absl::any_cast(any1).compare( absl::any_cast(any2)) == 0; } return false; } } // namespace absl ================================================ FILE: sentinel-core/param/statistic/any_cmp.h ================================================ #pragma once #include #include "absl/types/any.h" namespace Sentinel { namespace Param { extern const std::string INT_TYPE_STR; extern const std::string INT64_TYPE_STR; extern const std::string STRING_TYPE_STR; bool IsInt32(const absl::any& a); bool IsInt64(const absl::any& a); bool IsString(const absl::any& a); } // namespace Param } // namespace Sentinel namespace std { using Sentinel::Param::IsInt32; using Sentinel::Param::IsInt64; using Sentinel::Param::IsString; template <> struct hash { size_t operator()(const absl::any& any) const { if (IsInt32(any)) { return 31 * absl::any_cast(any); } else if (IsInt64(any)) { return 31 * absl::any_cast(any); } else if (IsString(any)) { return std::hash{}(absl::any_cast(any)); } return -1; } }; } // namespace std namespace absl { using Sentinel::Param::IsInt32; using Sentinel::Param::IsInt64; using Sentinel::Param::IsString; bool operator==(const absl::any& any1, const absl::any& any2); } // namespace absl namespace Sentinel { namespace Param { struct AnyCmp { static size_t hash(const absl::any& a) { return std::hash{}(a); } static bool equal(const absl::any& a0, const absl::any& a1) { return a0 == a1; } }; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/any_cmp_test.cc ================================================ #include "sentinel-core/param/statistic/any_cmp.h" #include int main() { absl::any a; std::cout << a.has_value(); std::cout << a.type().name() << std::endl; const std::string INT_TYPE_STR(typeid(3).name()); return 0; } ================================================ FILE: sentinel-core/param/statistic/lru_cache.h ================================================ /* * Copyright (c) 2014 Tim Starling * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include #include #include #include #include "tbb/concurrent_hash_map.h" namespace Sentinel { namespace Param { using AtomicIntSharedPtr = std::shared_ptr>; /** * ThreadSafeLRUCache is a thread-safe hashtable with a limited size. When * it is full, insert() evicts the least recently used item from the cache. * * The find() operation fills a ConstAccessor object, which is a smart pointer * similar to TBB's const_accessor. After eviction, destruction of the value is * deferred until all ConstAccessor objects are destroyed. * * The implementation is generally conservative, relying on the documented * behaviour of tbb::concurrent_hash_map. LRU list transactions are protected * with a single mutex. Having our own doubly-linked list implementation helps * to ensure that list transactions are sufficiently brief, consisting of only * a few loads and stores. User code is not executed while the lock is held. * * The acquisition of the list mutex during find() is non-blocking (try_lock), * so under heavy lookup load, the container will not stall, instead some LRU * update operations will be omitted. * * Insert performance was observed to degrade rapidly when there is a heavy * concurrent insert/evict load, mostly due to locks in the underlying * TBB::CHM. So if that is a possibility for your workload, * ThreadSafeScalableCache is recommended instead. */ template > class ThreadSafeLRUCache { /** * The LRU list node. * * We make a copy of the key in the list node, allowing us to find the * TBB::CHM element from the list node. TBB::CHM invalidates iterators * on most operations, even find(), ruling out more efficient * implementations. */ struct ListNode { ListNode() : m_prev(nullptr), m_next(nullptr) {} ListNode(const TKey& key) : m_key(key), m_prev(nullptr), m_next(nullptr) {} TKey m_key; std::shared_ptr m_prev; std::shared_ptr m_next; bool isInList() const { if (m_prev) return true; else return false; } }; static std::shared_ptr OutOfListMarker; public: /** * The value that we store in the hashtable. The list node is allocated from * an internal object_pool. The ListNode* is owned by the list. */ struct HashMapValue { HashMapValue() : m_listNode(nullptr) {} HashMapValue(int32_t value, std::shared_ptr node) : m_listNode(node) { m_value = std::make_shared>(value); } AtomicIntSharedPtr m_value; std::shared_ptr m_listNode; }; typedef tbb::concurrent_hash_map HashMap; typedef std::shared_ptr HashMapSharedPtr; typedef typename HashMap::const_accessor HashMapConstAccessor; typedef typename HashMap::accessor HashMapAccessor; typedef typename HashMap::value_type HashMapKVPair; // typedef std::pair> SnapshotValue; public: /** * Create a container with a given maximum size */ explicit ThreadSafeLRUCache(size_t maxSize); ThreadSafeLRUCache(const ThreadSafeLRUCache& other) = delete; ThreadSafeLRUCache& operator=(const ThreadSafeLRUCache&) = delete; ~ThreadSafeLRUCache(); /** * Find a value by key, and return it by filling the ConstAccessor, which * can be default-constructed. Returns true if the element was found, false * otherwise. Updates the eviction list, making the element the * most-recently used. */ bool find(HashMapAccessor& ac, const TKey& key); bool find(HashMapConstAccessor& ac, const TKey& key); /** * Find a value by key without disturbing the eviction list. */ bool silentFind(HashMapAccessor& ac, const TKey& key); bool silentFind(HashMapConstAccessor& ac, const TKey& key); /** * Increase the value of the givin key by value. * Require that the value type should be addable. * If the givin key does not exist, just insert pair and * return false. */ bool increase(const TKey& key, int value); bool increase(HashMapAccessor& ac, const TKey& key, int value); /** * Insert a value into the container. Both the key and value will be copied. * The new element will put into the eviction list as the most-recently * used. * * If there was already an element in the container with the same key, it * will not be updated, and false will be returned. Otherwise, true will be * returned. */ bool insert(const TKey& key, int value); bool insert(HashMapAccessor& ac, const TKey& key, int value); /** * Clear the container. */ void clear(); /** * Get a snapshot of the keys in the container by copying them into the * supplied vector. This will block inserts and prevent LRU updates while it * completes. The keys will be inserted in order from most-recently used to * least-recently used. */ void snapshotKeys(std::vector& keys); void snapshotPairs(std::vector>& pairs); /** * Get the approximate size of the container. May be slightly too low when * insertion is in progress. */ size_t size() const { return m_size.load(); } private: /** * Unlink a node from the list. The caller must lock the list mutex while * this is called. */ void delink(std::shared_ptr node); /** * Add a new node to the list in the most-recently used position. The caller * must lock the list mutex while this is called. */ void pushFront(std::shared_ptr node); /** * Evict the least-recently used item from the container. This function does * its own locking. */ void evict(const TKey& key); /** * The maximum number of elements in the container. */ size_t m_maxSize; /** * This atomic variable is used to signal to all threads whether or not * eviction should be done on insert. It is approximately equal to the * number of elements in the container. */ std::atomic m_size; /** * The underlying TBB hash map. */ HashMapSharedPtr m_map; /** * The linked list. The "head" is the most-recently used node, and the * "tail" is the least-recently used node. The list mutex must be held * during both read and write. */ std::shared_ptr m_head = std::make_shared(); std::shared_ptr m_tail = std::make_shared(); std::mutex m_listMutex; std::mutex m_mapMutex; }; // template // typename ThreadSafeLRUCache::std::shared_ptr const // ThreadSafeLRUCache::OutOfListMarker = nullptr; template ThreadSafeLRUCache::ThreadSafeLRUCache(size_t maxSize) : m_maxSize(maxSize), m_size(0), m_map(std::make_unique(std::thread::hardware_concurrency() * 4)) // it will automatically grow { m_head->m_prev = nullptr; m_head->m_next = m_tail; m_tail->m_prev = m_head; m_tail->m_next = nullptr; } template ThreadSafeLRUCache::~ThreadSafeLRUCache() { clear(); m_head->m_next.reset(); m_tail->m_prev.reset(); } template bool ThreadSafeLRUCache::silentFind(HashMapConstAccessor& ac, const TKey& key) { std::unique_lock lck(m_mapMutex); HashMapSharedPtr local_map = m_map; lck.unlock(); if (!local_map->find(ac, key)) { return false; } return true; } template bool ThreadSafeLRUCache::silentFind(HashMapAccessor& ac, const TKey& key) { std::unique_lock lck(m_mapMutex); HashMapSharedPtr local_map = m_map; lck.unlock(); if (!local_map->find(ac, key)) { return false; } return true; } template bool ThreadSafeLRUCache::find(HashMapAccessor& ac, const TKey& key) { std::unique_lock lck(m_mapMutex); HashMapSharedPtr local_map = m_map; lck.unlock(); if (!local_map->find(ac, key)) { return false; } // Acquire the lock, but don't block if it is already held std::unique_lock lock(m_listMutex); if (lock) { std::shared_ptr node = ac->second.m_listNode; // The list node may be out of the list if it is in the process of being // inserted or evicted. Doing this check allows us to lock the list for // shorter periods of time. if (node->isInList()) { delink(node); pushFront(node); } lock.unlock(); } return true; } template bool ThreadSafeLRUCache::find(HashMapConstAccessor& ac, const TKey& key) { std::unique_lock lck(m_mapMutex); HashMapSharedPtr local_map = m_map; lck.unlock(); if (!local_map->find(ac, key)) { return false; } std::unique_lock lock(m_listMutex); if (lock) { std::shared_ptr node = ac->second.m_listNode; if (node->isInList()) { delink(node); pushFront(node); } lock.unlock(); } return true; } template bool ThreadSafeLRUCache::increase(const TKey& key, int value) { HashMapAccessor ac; return increase(ac, key, value); } template bool ThreadSafeLRUCache::increase(HashMapAccessor& ac, const TKey& key, int value) { std::unique_lock lck(m_mapMutex); HashMapSharedPtr local_map = m_map; lck.unlock(); if (!local_map->find(ac, key)) { // Key not found, insert the pair. // When multiple threads reach here, only one thread successfully // inserts the key while others just increase the value on the // exist key. if (insert(ac, key, value)) { // Successfully inserted return false; } // On fail return, `ac` is emplaced with ptr to the existing pair } ac->second.m_value->fetch_add(value); std::unique_lock lock(m_listMutex); if (lock) { std::shared_ptr node = ac->second.m_listNode; // The list node may be out of the list if it is in the process of being // inserted or evicted. Doing this check allows us to lock the list for // shorter periods of time. if (node->isInList()) { delink(node); pushFront(node); } lock.unlock(); } return true; } template bool ThreadSafeLRUCache::insert(const TKey& key, int value) { HashMapAccessor ac; return insert(ac, key, value); } template bool ThreadSafeLRUCache::insert(HashMapAccessor& ac, const TKey& key, int value) { // Insert into the CHM std::shared_ptr node = std::make_shared(key); HashMapKVPair hashMapKV(key, HashMapValue(value, node)); std::unique_lock lck(m_mapMutex); HashMapSharedPtr local_map = m_map; lck.unlock(); if (!local_map->insert(ac, hashMapKV)) { node.reset(); return false; } // Evict if necessary, now that we know the hashmap insertion was successful. size_t size = m_size.load(); bool evictionDone = false; if (size >= m_maxSize) { // The container is at (or over) capacity, so eviction needs to be done. // Do not decrement m_size, since that would cause other threads to // inappropriately omit eviction during their own inserts. evict(key); evictionDone = true; } // Note that we have to update the LRU list before we increment m_size, so // that other threads don't attempt to evict list items before they even // exist. std::unique_lock lock(m_listMutex); pushFront(node); lock.unlock(); if (!evictionDone) { size = m_size++; } if (size > m_maxSize) { // It is possible for the size to temporarily exceed the maximum if there is // a heavy insert() load, once only as the cache fills. In this situation, // we have to be careful not to have every thread simultaneously attempt to // evict the extra entries, since we could end up underfilled. Instead we do // a compare-and-exchange to acquire an exclusive right to reduce the size // to a particular value. // // We could continue to evict in a loop, but if there are a lot of threads // here at the same time, that could lead to spinning. So we will just evict // one extra element per insert() until the overfill is rectified. if (m_size.compare_exchange_strong(size, size - 1)) { evict(key); } } return true; } template void ThreadSafeLRUCache::clear() { std::unique_lock mapLck(m_mapMutex, std::defer_lock); std::unique_lock listLck(m_listMutex, std::defer_lock); lock(mapLck, listLck); m_map = std::make_unique(std::thread::hardware_concurrency() * 4); std::shared_ptr node = m_head->m_next; std::shared_ptr next; while (node->m_next) { next = node->m_next; node->m_prev.reset(); node->m_next.reset(); node = next; } m_head->m_next = m_tail; m_tail->m_prev = m_head; mapLck.unlock(); listLck.unlock(); m_size.store(0); } template void ThreadSafeLRUCache::snapshotKeys(std::vector& keys) { keys.reserve(keys.size() + m_size.load()); std::unique_lock lock(m_listMutex); for (std::shared_ptr node = m_head->m_next; node != m_tail; node = node->m_next) { keys.push_back(node->m_key); } } template void ThreadSafeLRUCache::snapshotPairs( std::vector>& pairs) { pairs.reserve(pairs.size() + m_size.load()); std::unique_lock lck(m_mapMutex); HashMapSharedPtr local_map = m_map; lck.unlock(); std::lock_guard lock(m_listMutex); for (std::shared_ptr node = m_head->m_next; node != m_tail; node = node->m_next) { HashMapConstAccessor cac; if (!local_map->find(cac, node->m_key)) { // may have already been delinked continue; } pairs.push_back(std::make_pair<>(node->m_key, cac->second.m_value->load())); } } template inline void ThreadSafeLRUCache::delink( std::shared_ptr node) { std::shared_ptr prev = node->m_prev; std::shared_ptr next = node->m_next; prev->m_next = next; next->m_prev = prev; node->m_prev = nullptr; } template inline void ThreadSafeLRUCache::pushFront( std::shared_ptr node) { std::shared_ptr oldRealHead = m_head->m_next; node->m_prev = m_head; node->m_next = oldRealHead; oldRealHead->m_prev = node; m_head->m_next = node; } template void ThreadSafeLRUCache::evict(const TKey& key) { std::unique_lock lock(m_listMutex); std::shared_ptr moribund = m_tail->m_prev; if (!(moribund->m_prev) || THash().equal(moribund->m_key, key)) { // List is empty, can't evict return; } delink(moribund); lock.unlock(); std::unique_lock lck(m_mapMutex); HashMapSharedPtr local_map = m_map; lck.unlock(); HashMapAccessor hashAccessor; if (!local_map->find(hashAccessor, moribund->m_key)) { // Presumably unreachable return; } local_map->erase(hashAccessor); } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_bucket.cc ================================================ #include "sentinel-core/param/statistic/param_bucket.h" namespace Sentinel { namespace Param { ParamBucket::ParamBucket(int capacity) { for (int i = 0; i < static_cast(ParamMetricEvent::Count); i++) { counters_.push_back(std::make_unique(capacity)); } } int ParamBucket::Get(const ParamMetricEvent& e, const absl::any& value) const { int i = static_cast(e); ScalableCache::HashMapConstAccessor cac; if (counters_[i]->find(cac, value)) { return cac->second.m_value->load(); } else { return 0; } } void ParamBucket::Add(const ParamMetricEvent& e, int count, const absl::any& value) { int i = static_cast(e); counters_[i]->increase(value, count); // Create a new pair if not present } void ParamBucket::Reset() { for (int i = 0; i < static_cast(ParamMetricEvent::Count); i++) { counters_[i]->clear(); } } void ParamBucket::GetPairSet(const ParamMetricEvent& e, HotPairList& pairs) const { int i = static_cast(e); counters_[i]->snapshotPairs(pairs); } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_bucket.h ================================================ #pragma once #include #include #include #include #include "absl/types/any.h" #include "any_cmp.h" #include "sentinel-core/param/statistic/param_event.h" #include "sentinel-core/param/statistic/scalable_cache.h" namespace Sentinel { namespace Param { using ScalableCache = ThreadSafeScalableCache; using ScalableCacheUniquePtr = std::unique_ptr; using HotPair = std::pair; using HotPairList = std::vector; class ParamBucket { public: ParamBucket(int capacity); int Get(const ParamMetricEvent& e, const absl::any& value) const; void Add(const ParamMetricEvent& e, int count, const absl::any& value); void Reset(); void GetPairSet(const ParamMetricEvent& e, HotPairList& pairs) const; private: std::vector counters_; }; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_event.h ================================================ #pragma once namespace Sentinel { namespace Param { enum class ParamMetricEvent { PASS = 0, BLOCK, Count, // hack for getting length of enum }; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_leap_array.cc ================================================ #include "sentinel-core/param/statistic/param_leap_array.h" namespace Sentinel { namespace Param { int ParamLeapArray::cache_size() const noexcept { return cache_size_; } void ParamLeapArray::set_cache_size(int cache_size) noexcept { cache_size_ = cache_size; } ParamLeapArray::ParamLeapArray(int32_t sample_count, int32_t interval_ms, int32_t cache_size) : cache_size_(cache_size), Stat::LeapArray(sample_count, interval_ms) {} std::shared_ptr ParamLeapArray::NewEmptyBucket( int64_t time_millis) { return std::make_shared(cache_size_); } void ParamLeapArray::ResetWindowTo( const Stat::WindowWrapSharedPtr& wrap, int64_t start_time) { wrap->ResetTo(start_time); wrap->Value()->Reset(); } HotPairList&& ParamLeapArray::GetTopValues(const ParamMetricEvent& e, int number) { this->CurrentWindow(); auto v = this->Values(); HotPairList pairs; for (const auto& bucket : v) { bucket->GetPairSet(e, pairs); } std::sort(pairs.begin(), pairs.end(), [](const HotPair& p0, const HotPair& p1) -> bool { return p0.second < p1.second; }); int size = number < pairs.size() ? number : pairs.size(); return HotPairList(pairs.begin(), pairs.begin() + size); } HotPairList&& ParamLeapArray::GetTopPassValues(int number) { return GetTopValues(ParamMetricEvent::PASS, number); } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_leap_array.h ================================================ #pragma once #include #include "sentinel-core/param/statistic/param_bucket.h" #include "sentinel-core/statistic/base/leap_array.h" namespace Sentinel { namespace Param { class ParamLeapArray : public Stat::LeapArray { public: ParamLeapArray(int32_t sample_count, int32_t interval_ms, int32_t cache_size = DEFAULT_CACHE_SIZE); std::shared_ptr NewEmptyBucket(int64_t time_millis); void ResetWindowTo(const Stat::WindowWrapSharedPtr& wrap, int64_t start_time); int cache_size() const noexcept; void set_cache_size(int cache_size) noexcept; HotPairList&& GetTopValues(const ParamMetricEvent& e, int number); HotPairList&& GetTopPassValues(int number); private: int cache_size_; const static int DEFAULT_CACHE_SIZE = 200; }; using ParamLeapArraySharedPtr = std::shared_ptr; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_leap_array_key.h ================================================ #pragma once #include namespace Sentinel { namespace Param { class ParamLeapArrayKey { public: int32_t param_idx_ = -1; int32_t interval_in_ms_ = 1000; int32_t sample_count_ = 1; int32_t cache_size_; int32_t param_idx() const noexcept { return param_idx_; } int32_t interval_in_ms() const noexcept { return interval_in_ms_; } int32_t sample_count() const noexcept { return sample_count_; } int32_t cache_size() const noexcept { return cache_size_; } }; using ParamLeapArrayKeySharedPtr = std::shared_ptr; class ParamLeapArrayKeyPtrHashEq { public: static size_t hash(const ParamLeapArrayKeySharedPtr& key) { if (!key) { return 0; } size_t result = key->interval_in_ms_; result = 31 * result + key->sample_count_; result = 31 * result + key->param_idx_; result = 31 * result + key->cache_size_; return result; } static bool equal(const ParamLeapArrayKeySharedPtr& k0, const ParamLeapArrayKeySharedPtr& k1) { if (!k0 && !k1) { return true; } else if (!k0 || !k1) { return false; } return k0->interval_in_ms_ == k1->interval_in_ms_ && k0->sample_count_ == k1->sample_count_ && k0->param_idx_ == k1->param_idx_ && k0->cache_size_ == k1->cache_size_; } }; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_metric.cc ================================================ #include "sentinel-core/param/statistic/param_metric.h" namespace Sentinel { namespace Param { void ParamMetric::AddThreadCount(const std::vector& params) { int idx = 0; for (const auto& param : params) { decltype(thread_count_map_)::const_accessor cac; if (thread_count_map_.find(cac, idx)) { // Due to partial initializing problem, is possible to get a // nullptr here. See ParamMetric::initializeForRule for detail. if (!(cac->second)) { continue; } cac->second->increase(param, 1); } // else : do nothing idx++; } } void ParamMetric::DecreaseThreadCount(const std::vector& params) { int idx = 0; for (const auto& param : params) { decltype(thread_count_map_)::const_accessor cac; if (thread_count_map_.find(cac, idx)) { cac->second->increase(param, -1); } // else : do nothing idx++; } } int ParamMetric::GetThreadCount(int index, const absl::any& param) const { decltype(thread_count_map_)::const_accessor cacheCac; if (!thread_count_map_.find(cacheCac, index)) { return 0; } ScalableCache::HashMapConstAccessor counterCac; // Due to partial initializing problem, is possible to get a // nullptr here. See ParamMetric::initializeForRule for detail. if (!(cacheCac->second)) { return 0; } if (!cacheCac->second->find(counterCac, param)) { return 0; } return counterCac->second.m_value->load(); } void ParamMetric::Add(const ParamMetricEvent& e, int count, const std::vector& params) { int idx = 0; for (const auto& param : params) { auto range = index_map_.equal_range(idx); for_each(range.first, range.second, [¶m, count, &e](decltype(index_map_)::value_type& x) { x.second->CurrentWindow()->Value()->Add(e, count, param); }); idx++; } } void ParamMetric::AddPass(int count, const std::vector& params) { Add(ParamMetricEvent::PASS, count, params); } void ParamMetric::AddBlock(int count, const std::vector& params) { Add(ParamMetricEvent::BLOCK, count, params); } int ParamMetric::GetSum(int index, const ParamMetricEvent& e, const absl::any& param) const { int sum = 0; auto it = index_map_.find(index); // Anyone of LeapArray on this index is ok if (it != index_map_.end()) { it->second->CurrentWindow(); auto v = it->second->Values(); for (const auto& bucket : v) { sum += bucket->Get(e, param); } } return sum; } int ParamMetric::GetAvg(int index, const ParamMetricEvent& e, const absl::any& param) const { tbb::concurrent_hash_map::const_accessor cac; int sum = 0; auto it = index_map_.find(index); // Anyone of LeapArray on this index is ok if (it != index_map_.end()) { it->second->CurrentWindow(); auto v = it->second->Values(); for (const auto& bucket : v) { sum += bucket->Get(e, param); } } return sum / (it->second->IntervalInMs() / 1000.0); } int ParamMetric::GetInterval(const ParamLeapArrayKeySharedPtr& key, const ParamMetricEvent& e, const absl::any& param) const { int sum = 0; decltype(rolling_params_)::const_accessor cac; if (rolling_params_.find(cac, key)) { cac->second->CurrentWindow(); auto v = cac->second->Values(); for (const auto& bucket : v) { sum += bucket->Get(e, param); } } return sum; } int ParamMetric::PassInterval(const ParamLeapArrayKeySharedPtr& key, const absl::any& param) const { return GetInterval(key, ParamMetricEvent::PASS, param); } int ParamMetric::BlockInterval(const ParamLeapArrayKeySharedPtr& key, const absl::any& param) const { return GetInterval(key, ParamMetricEvent::BLOCK, param); } int ParamMetric::PassQps(int index, const absl::any& param) const { return this->GetAvg(index, ParamMetricEvent::PASS, param); } int ParamMetric::BlockQps(int index, const absl::any& param) const { return this->GetAvg(index, ParamMetricEvent::BLOCK, param); } HotPairList&& ParamMetric::GetTopPassParamCount( const ParamLeapArrayKeySharedPtr& key, int number) { decltype(rolling_params_)::const_accessor cac; if (!rolling_params_.find(cac, key)) { return {}; } return cac->second->GetTopPassValues(number); } void ParamMetric::initializeForRule(const ParamLeapArrayKeySharedPtr& k) { if (!k) { SENTINEL_LOG(error, "[ParamMetric::initializeForRule] LeapArrayKey is nullptr"); return; } // Here we find then insert the ScalableCache. // On failed insertation cases, this can avoid the unnecessary and // heavy contruction work of ScalableCache. decltype(thread_count_map_)::const_accessor cac0; if (!thread_count_map_.find(cac0, k->param_idx())) { // Partial initialization: other thread may query this cache before // initialization work ends here thread_count_map_.insert(std::make_pair<>( k->param_idx(), std::make_unique(k->cache_size()))); } tbb::concurrent_hash_map::const_accessor cac1; // Partial initialization problem may arise here: other threads may // fail to find an entry in `index_map_` with this index at `AddPass` // as they're expected. // However, it doesn't matter to miss some QPS. if (rolling_params_.insert( cac1, std::make_pair<>(k, std::make_shared( k->sample_count(), k->interval_in_ms(), k->cache_size())))) { // Dangerous interval! index_map_.insert(std::make_pair<>(k->param_idx(), cac1->second)); } } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_metric.h ================================================ #pragma once #include #include #include #include "sentinel-core/param/statistic/param_leap_array.h" #include "sentinel-core/param/statistic/param_leap_array_key.h" #include "tbb/concurrent_unordered_map.h" namespace Sentinel { namespace Param { class ParamMetric { public: ParamMetric() {} void initializeForRule(const ParamLeapArrayKeySharedPtr& k); void AddThreadCount(const std::vector& params); void DecreaseThreadCount(const std::vector& params); int GetThreadCount(int index, const absl::any& param) const; void Add(const ParamMetricEvent& e, int count, const std::vector& params); void AddPass(int count, const std::vector& params); void AddBlock(int count, const std::vector& params); int GetSum(int index, const ParamMetricEvent& e, const absl::any& param) const; int GetInterval(const ParamLeapArrayKeySharedPtr& key, const ParamMetricEvent& e, const absl::any& param) const; int PassInterval(const ParamLeapArrayKeySharedPtr& key, const absl::any& param) const; int BlockInterval(const ParamLeapArrayKeySharedPtr& key, const absl::any& param) const; int GetAvg(int index, const ParamMetricEvent& e, const absl::any& param) const; int PassQps(int index, const absl::any& param) const; int BlockQps(int index, const absl::any& param) const; HotPairList&& GetTopPassParamCount(const ParamLeapArrayKeySharedPtr& key, int number); // WARNING: Not thread safe void Clear() { thread_count_map_.clear(); index_map_.clear(); rolling_params_.clear(); } private: tbb::concurrent_hash_map thread_count_map_; // NOTE: The key of `rolling_params_` is combination of some fields in rule // type, which means that several rules may share the same metric type // instance. tbb::concurrent_hash_map rolling_params_; tbb::concurrent_unordered_multimap index_map_; }; using ParamMetricSharedPtr = std::shared_ptr; } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/param_metric_test.cc ================================================ #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/param/param_flow_rule_manager.h" #include "sentinel-core/param/statistic/param_metric.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { namespace Param { TEST(ParamMetricTest, TestOperateMetric) { ParamFlowRuleSharedPtr rule0 = std::make_shared(); ParamFlowRuleSharedPtr rule1 = std::make_shared(); rule0->set_param_idx(0); rule0->set_interval_in_ms(2000); rule1->set_param_idx(1); rule1->set_interval_in_ms(2000); auto metric = std::make_shared(); metric->initializeForRule(rule0->metric_key()); metric->initializeForRule(rule1->metric_key()); metric->AddPass(2, {12313, std::string("exampleString"), 456}); metric->AddPass(3, {12313, std::string("anotherExampleString")}); metric->AddBlock(1, {456, std::string("exampleString"), 456}); // NOTE: If the time of invoking Add and Get of counter are unluckly in two // different windows, the test will fail. Longer window interval can reduce // but cannot eliminate this possibility. // If this occurs, retry until the test passes... EXPECT_EQ(5, metric->PassInterval(rule0->metric_key(), 12313)); EXPECT_EQ(0, metric->BlockInterval(rule0->metric_key(), 12313)); EXPECT_EQ(0, metric->PassInterval(rule0->metric_key(), 456)); EXPECT_EQ(1, metric->BlockInterval(rule0->metric_key(), 456)); // Should not segmentation fault on nullptr key EXPECT_EQ(metric->PassInterval(nullptr, 456), 0); EXPECT_EQ(metric->BlockInterval(nullptr, 456), 0); // '456' didn't appear on rule1->param_idx() EXPECT_EQ(metric->BlockInterval(rule1->metric_key(), 456), 0); EXPECT_EQ(2, metric->PassInterval(rule1->metric_key(), std::string("exampleString"))); EXPECT_EQ(1, metric->BlockInterval(rule1->metric_key(), std::string("exampleString"))); EXPECT_EQ(3, metric->PassInterval(rule1->metric_key(), std::string("anotherExampleString"))); EXPECT_EQ(0, metric->BlockInterval(rule1->metric_key(), std::string("anotherExampleString"))); } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/param/statistic/scalable_cache.h ================================================ /* * Copyright (c) 2014 Tim Starling * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include "lru_cache.h" namespace Sentinel { namespace Param { /** * ThreadSafeScalableCache is a thread-safe sharded hashtable with limited * size. When it is full, it evicts a rough approximation to the least recently * used item. * * The find() operation fills a ConstAccessor object, which is a smart pointer * similar to TBB's const_accessor. After eviction, destruction of the value is * deferred until all ConstAccessor objects are destroyed. * * Since the hash value of each key is requested multiple times, you should use * a key with a memoized hash function. ThreadSafeStringKey is provided for * this purpose. */ template > class ThreadSafeScalableCache { public: using Shard = ThreadSafeLRUCache; typedef typename Shard::HashMapValue HashMapValue; typedef typename Shard::HashMapAccessor HashMapAccessor; typedef typename Shard::HashMapConstAccessor HashMapConstAccessor; typedef tbb::concurrent_hash_map HashMap; /** * Constructor * - maxSize: the maximum number of items in the container * - numShards: the number of child containers. If this is zero, the * "hardware concurrency" will be used (typically the logical processor * count). */ explicit ThreadSafeScalableCache(size_t maxSize, size_t numShards = 0); ThreadSafeScalableCache(const ThreadSafeScalableCache&) = delete; ThreadSafeScalableCache& operator=(const ThreadSafeScalableCache&) = delete; /** * Find a value by key, and return it by filling the ConstAccessor, which * can be default-constructed. Returns true if the element was found, false * otherwise. Updates the eviction list, making the element the * most-recently used. */ bool find(HashMapAccessor& ac, const TKey& key); bool find(HashMapConstAccessor& ac, const TKey& key); bool silentFind(HashMapAccessor& ac, const TKey& key); bool silentFind(HashMapConstAccessor& ac, const TKey& key); bool increase(const TKey& key, int value); bool increase(HashMapAccessor& ac, const TKey& key, int value); /** * Insert a value into the container. Both the key and value will be copied. * The new element will put into the eviction list as the most-recently * used. * * If there was already an element in the container with the same key, it * will not be updated, and false will be returned. Otherwise, true will be * returned. */ bool insert(const TKey& key, int value); bool insert(HashMapAccessor& ac, const TKey& key, int value); /** * Clear the container. NOT THREAD SAFE -- do not use while other threads * are accessing the container. */ void clear(); /** * Get a snapshot of the keys in the container by copying them into the * supplied vector. This will block inserts and prevent LRU updates while it * completes. The keys will be inserted in a random order. */ void snapshotKeys(std::vector& keys); void snapshotPairs(std::vector>& pairs); /** * Get the approximate size of the container. May be slightly too low when * insertion is in progress. */ size_t size() const; private: /** * Get the child container for a given key */ Shard& getShard(const TKey& key); /** * The maximum number of elements in the container. */ size_t m_maxSize; /** * The child containers */ size_t m_numShards; typedef std::shared_ptr ShardPtr; std::vector m_shards; }; /** * A specialisation of ThreadSafeScalableCache providing a cache with efficient * string keys. */ template ThreadSafeScalableCache::ThreadSafeScalableCache(size_t maxSize, size_t numShards) : m_maxSize(maxSize), m_numShards(numShards) { if (m_numShards == 0) { m_numShards = std::thread::hardware_concurrency(); } for (size_t i = 0; i < m_numShards; i++) { size_t s = maxSize / m_numShards; if (i == 0) { s += maxSize % m_numShards; } m_shards.emplace_back(std::make_shared(s)); } } template typename ThreadSafeScalableCache::Shard& ThreadSafeScalableCache::getShard(const TKey& key) { THash hashObj; constexpr int shift = std::numeric_limits::digits - 16; size_t h = (hashObj.hash(key) >> shift) % m_numShards; return *m_shards.at(h); } template bool ThreadSafeScalableCache::silentFind(HashMapConstAccessor& ac, const TKey& key) { return getShard(key).silentFind(ac, key); } template bool ThreadSafeScalableCache::silentFind(HashMapAccessor& ac, const TKey& key) { return getShard(key).silentFind(ac, key); } template bool ThreadSafeScalableCache::find(HashMapConstAccessor& ac, const TKey& key) { return getShard(key).find(ac, key); } template bool ThreadSafeScalableCache::find(HashMapAccessor& ac, const TKey& key) { return getShard(key).find(ac, key); } template bool ThreadSafeScalableCache::increase(const TKey& key, int value) { return getShard(key).increase(key, value); } template bool ThreadSafeScalableCache::increase(HashMapAccessor& ac, const TKey& key, int value) { return getShard(key).increase(ac, key, value); } template bool ThreadSafeScalableCache::insert(const TKey& key, int value) { return getShard(key).insert(key, value); } template bool ThreadSafeScalableCache::insert(HashMapAccessor& ac, const TKey& key, int value) { return getShard(key).insert(ac, key, value); } template void ThreadSafeScalableCache::clear() { for (size_t i = 0; i < m_numShards; i++) { m_shards[i]->clear(); } } template void ThreadSafeScalableCache::snapshotKeys( std::vector& keys) { for (size_t i = 0; i < m_numShards; i++) { m_shards[i]->snapshotKeys(keys); } } template void ThreadSafeScalableCache::snapshotPairs( std::vector>& pairs) { for (size_t i = 0; i < m_numShards; i++) { m_shards[i]->snapshotPairs(pairs); } } template size_t ThreadSafeScalableCache::size() const { size_t size; for (size_t i = 0; i < m_numShards; i++) { size += m_shards[i]->size(); } return size; } } // namespace Param } // namespace Sentinel ================================================ FILE: sentinel-core/property/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "property_listener_interface", srcs = [ "property_listener.h", ], copts = DEFAULT_COPTS, deps = [] ) cc_library( name = "sentinel_property_interface", srcs = [ "sentinel_property.h", ], copts = DEFAULT_COPTS, deps = [ ":property_listener_interface", "//sentinel-core/log:logger_lib", ] ) cc_library( name = "dynamic_sentinel_property_lib", srcs = [ "dynamic_sentinel_property.h", ], copts = DEFAULT_COPTS, deps = [ ":sentinel_property_interface", "@com_google_absl//absl/container:flat_hash_map", ] ) cc_test( name = "dynamic_sentinel_property_unittests", srcs = [ "dynamic_sentinel_property_unittests.cc", ], copts = TEST_COPTS, deps = [ ":dynamic_sentinel_property_lib", "//sentinel-core/test/mock/property:mock_lib", "//sentinel-core/test/mock/common:mock_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/property/dynamic_sentinel_property.h ================================================ #pragma once #include #include #include #include "sentinel-core/log/logger.h" #include "sentinel-core/property/sentinel_property.h" namespace Sentinel { namespace Property { template class DynamicSentinelProperty : public SentinelProperty { public: DynamicSentinelProperty() = default; virtual ~DynamicSentinelProperty() = default; void AddListener(PropertyListenerPtr&& listener) override { auto name = listener->Name(); listeners_.emplace(std::make_pair(std::string(name), std::move(listener))); } void RemoveListener(const std::string& listener_name) override { listeners_.erase(listener_name); } bool UpdateValue(const T& value) override { if (last_value_ == value) { return false; } last_value_ = value; SENTINEL_LOG(info, "starting to config update"); for (auto it = listeners_.begin(); it != listeners_.end(); ++it) { it->second->ConfigUpdate(value, false); } return true; } void Clear() { listeners_.clear(); } private: T last_value_; std::unordered_map> listeners_; }; } // namespace Property } // namespace Sentinel ================================================ FILE: sentinel-core/property/dynamic_sentinel_property_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/property/dynamic_sentinel_property.h" #include "sentinel-core/test/mock/common/mock.h" #include "sentinel-core/test/mock/property/mock.h" using testing::_; using testing::InSequence; using testing::Matcher; using testing::Return; using testing::ReturnRef; namespace Sentinel { namespace Property { TEST(DynamicSentinelPropertyTest, Basic) { FakeRule rule; rule.set_fake_rule_data("test"); DynamicSentinelProperty ds; auto pl = std::make_unique>(); auto pl_point = pl.get(); EXPECT_CALL(*pl_point, ConfigUpdate(Matcher(rule), false)) .Times(1); ds.AddListener(std::move(pl)); ds.UpdateValue(rule); } } // namespace Property } // namespace Sentinel ================================================ FILE: sentinel-core/property/property_listener.h ================================================ #pragma once #include #include namespace Sentinel { namespace Property { template class PropertyListener { public: virtual ~PropertyListener() = default; virtual void ConfigUpdate(const T& value, bool first_load) = 0; virtual const std::string Name() const = 0; }; template using PropertyListenerPtr = std::unique_ptr>; } // namespace Property } // namespace Sentinel ================================================ FILE: sentinel-core/property/sentinel_property.h ================================================ #pragma once #include #include #include "sentinel-core/property/property_listener.h" namespace Sentinel { namespace Property { template class SentinelProperty { public: virtual ~SentinelProperty() = default; virtual void AddListener(PropertyListenerPtr&& listener) = 0; virtual void RemoveListener(const std::string& listener_name) = 0; virtual bool UpdateValue(const T& value) = 0; }; template using SentinelPropertySharedPtr = std::shared_ptr>; } // namespace Property } // namespace Sentinel ================================================ FILE: sentinel-core/public/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "sph_u_lib", srcs = [ "sph_u.h", # "sph_u.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/common:global_status", "//sentinel-core/common:entry_result_lib", "//sentinel-core/common:string_resource_wrapper_lib", ] ) cc_test( name = "sph_u_unittests", srcs = [ "sph_u_test.cc", ], copts = TEST_COPTS, deps = [ ":sph_u_lib", "//sentinel-core/log:logger_lib", "//sentinel-core/flow:flow_rule_manager_lib", "//sentinel-core/statistic/node:resource_node_storage_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/public/sph_u.h ================================================ #pragma once #include #include "sentinel-core/common/constants.h" #include "sentinel-core/common/entry.h" #include "sentinel-core/common/entry_context.h" #include "sentinel-core/common/entry_result.h" #include "sentinel-core/common/entry_type.h" #include "sentinel-core/common/global_status.h" #include "sentinel-core/common/resource_wrapper.h" #include "sentinel-core/common/string_resource_wrapper.h" #include "sentinel-core/log/logger.h" namespace Sentinel { class SphU { public: ~SphU() = default; static void FetchParams(std::vector& params) {} template static void FetchParams(std::vector& params, T arg); template static void FetchParams(std::vector& params, T arg, Ts... args); template static EntryResultPtr Entry(const std::string& r, EntryType t, int count, int flag, Ts... args); static EntryResultPtr Entry(const std::string& r, EntryType t, int count); static EntryResultPtr Entry(const std::string& r, EntryType t); static EntryResultPtr Entry(const std::string& r); template static EntryResultPtr Entry(const EntryContextSharedPtr& context, const std::string& r, EntryType t, int count, int flag, Ts... args); private: SphU() = default; }; template void SphU::FetchParams(std::vector& params, T arg) { params.push_back(std::move(arg)); } template void SphU::FetchParams(std::vector& params, T arg, Ts... args) { params.push_back(std::move(arg)); FetchParams(params, args...); } template EntryResultPtr SphU::Entry(const EntryContextSharedPtr& context, const std::string& r, EntryType t, int count, int flag, Ts... args) { auto resource = std::make_shared(r, t); EntrySharedPtr e = std::make_shared(resource, context); if (!GlobalStatus::activated) { e->exited_ = true; return std::make_unique(e); } Slot::SlotChainSharedPtr chain = Slot::GetGlobalSlotChain(); if (chain == nullptr) { SENTINEL_LOG(warn, "[EntryResultPtr] SlotChain is nullptr"); return std::make_unique(e); } std::vector params; FetchParams(params, args...); Stat::NodeSharedPtr empty_node = nullptr; auto result = chain->Entry(e, empty_node, count, flag, params); if (result->status() == Slot::TokenStatus::RESULT_STATUS_BLOCKED) { // NOTE: keep consistent with EntryResult::exit. if (chain != nullptr) { chain->Exit(e, count, params); } return std::make_unique(result->blocked_reason().value()); } e->set_params(std::move(params)); return std::make_unique(e); } template EntryResultPtr SphU::Entry(const std::string& r, EntryType t, int count, int flag, Ts... args) { return Entry(std::make_shared(Constants::kDefaultContextName), r, t, count, flag, args...); } EntryResultPtr SphU::Entry(const std::string& r, EntryType t, int count) { return Entry<>(r, t, count, 0); } EntryResultPtr SphU::Entry(const std::string& r, EntryType t) { return Entry(r, t, 1); } EntryResultPtr SphU::Entry(const std::string& r) { return Entry(r, EntryType::OUT, 1); } } // namespace Sentinel ================================================ FILE: sentinel-core/public/sph_u_test.cc ================================================ #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/flow/flow_rule_manager.h" #include "sentinel-core/public/sph_u.h" #include "sentinel-core/statistic/node/resource_node_storage.h" namespace Sentinel { TEST(SphUTest, TestEntryBlockSimple) { auto resource_name = "SphUTest::TestEntryBlockSimple"; Flow::FlowRule rule{resource_name}; rule.set_count(0); Flow::FlowRuleManager& m = Flow::FlowRuleManager::GetInstance(); m.LoadRules({rule}); auto r = SphU::Entry(resource_name); EXPECT_TRUE(r->IsBlocked()); auto entry = r->entry(); EXPECT_TRUE(entry == nullptr); auto& s = Stat::ResourceNodeStorage::GetInstance(); auto node = s.GetClusterNode(resource_name); EXPECT_FALSE(node == nullptr); EXPECT_DOUBLE_EQ(0, node->PassQps()); EXPECT_DOUBLE_EQ(1, node->BlockQps()); EXPECT_EQ(0, node->CurThreadNum()); EXPECT_DOUBLE_EQ(0, node->CompleteQps()); } TEST(SphUTest, TestEntryPassWithoutRules) { Flow::FlowRuleManager& m = Flow::FlowRuleManager::GetInstance(); m.LoadRules({}); auto resource_name = "SphUTest::TestEntryPassWithoutRules"; auto r = SphU::Entry(resource_name); EXPECT_FALSE(r->IsBlocked()); auto entry = r->entry(); EXPECT_FALSE(entry == nullptr); EXPECT_FALSE(entry->exited()); auto& s = Stat::ResourceNodeStorage::GetInstance(); auto node = s.GetClusterNode(resource_name); EXPECT_FALSE(node == nullptr); EXPECT_DOUBLE_EQ(1, node->PassQps()); EXPECT_DOUBLE_EQ(0, node->BlockQps()); EXPECT_EQ(1, node->CurThreadNum()); EXPECT_DOUBLE_EQ(0, node->CompleteQps()); r->Exit(); EXPECT_TRUE(entry->exited()); EXPECT_DOUBLE_EQ(1, node->PassQps()); EXPECT_DOUBLE_EQ(0, node->BlockQps()); EXPECT_EQ(0, node->CurThreadNum()); EXPECT_DOUBLE_EQ(1, node->CompleteQps()); } } // namespace Sentinel ================================================ FILE: sentinel-core/slot/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "log_slot_lib", srcs = [ "log_slot.h", "log_slot.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/slot/base:stats_slot_interface", "//sentinel-core/log/block:block_log_task_lib", ] ) cc_library( name = "resource_node_builder_slot_lib", srcs = [ "resource_node_builder_slot.h", "resource_node_builder_slot.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/slot/base:stats_slot_interface", "//sentinel-core/statistic/node:resource_node_storage_lib", ] ) cc_library( name = "statistic_slot_lib", srcs = [ "statistic_slot.h", "statistic_slot.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/param:param_flow_slot_lib", "//sentinel-core/statistic/node:resource_node_storage_lib", "//sentinel-core/slot/base:stats_slot_interface", ] ) cc_library( name = "global_slot_chain_header", srcs = [ "global_slot_chain.h", "global_slot_chain.cc", ], copts = DEFAULT_COPTS, deps = [ ":resource_node_builder_slot_lib", ":statistic_slot_lib", ":log_slot_lib", "//sentinel-core/flow:flow_slot_lib", "//sentinel-core/circuitbreaker:rule_slot_lib", "//sentinel-core/slot/base:default_slot_chain_impl_lib", "//sentinel-core/system:system_slot_lib", ] ) cc_test( name = "resource_node_builder_slot_unittests", srcs = [ "resource_node_builder_slot_test.cc", ], copts = TEST_COPTS, deps = [ ":resource_node_builder_slot_lib", "//sentinel-core/common:string_resource_wrapper_lib", "//sentinel-core/statistic/node:cluster_node_lib", "//sentinel-core/slot/base:default_slot_chain_impl_lib", "@com_google_googletest//:gtest_main", ] ) cc_test( name = "statistic_slot_unittests", srcs = [ "statistic_slot_test.cc", ], copts = TEST_COPTS, deps = [ ":statistic_slot_lib", "//sentinel-core/common:string_resource_wrapper_lib", "//sentinel-core/statistic/node:cluster_node_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/slot/base/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "token_result_lib", srcs = [ "token_result.h", "token_result.cc", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/types:optional", ] ) cc_library( name = "rule_checker_slot_interface", srcs = [ "rule_checker_slot.h", ], copts = DEFAULT_COPTS, deps = [ ":slot_interface", ] ) cc_library( name = "stats_slot_interface", srcs = [ "stats_slot.h", ], copts = DEFAULT_COPTS, deps = [ ":slot_interface", ] ) cc_library( name = "slot_interface", srcs = [ "slot.h", ], copts = DEFAULT_COPTS, deps = [ ":slot_base", ] ) cc_library( name = "slot_chain_interface", srcs = [ "slot_chain.h", ], copts = DEFAULT_COPTS, deps = [ ":slot_interface", ] ) cc_library( name = "slot_base", srcs = [ "slot_base.h", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/common:entry_lib", ] ) cc_library( name = "default_slot_chain_impl_lib", srcs = [ "default_slot_chain_impl.h", "default_slot_chain_impl.cc", ], copts = DEFAULT_COPTS, deps = [ ":slot_chain_interface", ] ) cc_test( name = "default_slot_chain_impl_unittests", srcs = [ "default_slot_chain_impl_unittests.cc", ], copts = TEST_COPTS, deps = [ ":rule_checker_slot_interface", ":stats_slot_interface", ":slot_interface", ":default_slot_chain_impl_lib", "//sentinel-core/common:string_resource_wrapper_lib", "//sentinel-core/test/mock/slot:mock_lib", "//sentinel-core/test/mock/statistic/node:mock_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/slot/base/default_slot_chain_impl.cc ================================================ #include "sentinel-core/slot/base/default_slot_chain_impl.h" #include namespace Sentinel { namespace Slot { void DefaultSlotChainImpl::AddFirst(std::unique_ptr&& slot) { // The StatsSlot type slot is forbidden to be placed in the first position assert(slot->Type() != SlotType::STATS_SLOT); slots_.emplace_front(std::move(slot)); } void DefaultSlotChainImpl::AddLast(std::unique_ptr&& slot) { slots_.emplace_back(std::move(slot)); } TokenResultSharedPtr DefaultSlotChainImpl::Entry( const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { auto context = entry != nullptr ? entry->context() : nullptr; auto token_result = TokenResult::Ok(); for (auto elem = slots_.begin(); elem != slots_.end(); ++elem) { if ((*elem)->IsContinue(token_result, context)) { token_result = (*elem)->Entry(entry, node, count, flag, params); } } return token_result; } void DefaultSlotChainImpl::Exit(const EntrySharedPtr& entry, int count, const std::vector& params) { for (auto elem = slots_.begin(); elem != slots_.end(); ++elem) { (*elem)->Exit(entry, count, params); } } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/default_slot_chain_impl.h ================================================ #pragma once #include #include #include #include "sentinel-core/slot/base/slot_chain.h" #include "sentinel-core/slot/base/token_result.h" namespace Sentinel { namespace Slot { class DefaultSlotChainImpl : public SlotChain { public: DefaultSlotChainImpl() = default; virtual ~DefaultSlotChainImpl() = default; // SlotChain void AddFirst(std::unique_ptr&& slot) override; void AddLast(std::unique_ptr&& slot) override; TokenResultSharedPtr Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; private: std::deque> slots_; }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/default_slot_chain_impl_test.cc ================================================ #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/common/string_resource_wrapper.h" #include "sentinel-core/common/entry.h" #include "sentinel-core/slot/base/default_slot_chain_impl.h" #include "sentinel-core/test/mock/slot/mock.h" #include "sentinel-core/test/mock/statistic/node/mock.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { namespace Slot { TEST(DefaultSlotChainImplTest, Basic) { const std::vector myParams; { DefaultSlotChainImpl slot_chain; Stat::NodeSharedPtr node = std::make_shared(); auto mock_rule_checker_slot = std::make_unique(); auto mock_stat_slot = std::make_unique(); InSequence s; EXPECT_CALL(*mock_rule_checker_slot.get(), Entry(_, _, _, _, _)) .WillOnce(Return(TokenResult::Blocked("test"))); EXPECT_CALL(*mock_stat_slot.get(), Entry(_, _, _, _, _)).Times(1); slot_chain.AddLast(std::move(mock_rule_checker_slot)); slot_chain.AddLast(std::move(mock_stat_slot)); auto context = std::make_shared("my_context"); ResourceWrapperSharedPtr test_resource = std::make_shared("test_resource", EntryType::IN); auto entry = std::make_shared(test_resource, context); slot_chain.Entry(entry, node, 1, 1, myParams); } { DefaultSlotChainImpl slot_chain; Stat::NodeSharedPtr node = std::make_shared(); auto mock_rule_checker_slot1 = std::make_unique(); auto mock_rule_checker_slot2 = std::make_unique(); auto mock_stat_slot1 = std::make_unique(); auto mock_stat_slot2 = std::make_unique(); InSequence s; EXPECT_CALL(*mock_rule_checker_slot1.get(), Entry(_, _, _, _, _)) .WillOnce(Return(TokenResult::Blocked("test"))); EXPECT_CALL(*mock_rule_checker_slot2.get(), Entry(_, _, _, _, _)).Times(0); EXPECT_CALL(*mock_stat_slot1.get(), Entry(_, _, _, _, _)).Times(1); EXPECT_CALL(*mock_stat_slot2.get(), Entry(_, _, _, _, _)).Times(1); slot_chain.AddLast(std::move(mock_rule_checker_slot1)); slot_chain.AddLast(std::move(mock_rule_checker_slot2)); slot_chain.AddLast(std::move(mock_stat_slot1)); slot_chain.AddLast(std::move(mock_stat_slot2)); auto context = std::make_shared("my_context"); ResourceWrapperSharedPtr test_resource = std::make_shared("test_resource", EntryType::IN); auto entry = std::make_shared(test_resource, context); slot_chain.Entry(entry, node, 1, 1, myParams); } } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/rule_checker_slot.h ================================================ #pragma once #include #include "sentinel-core/slot/base/slot.h" namespace Sentinel { namespace Slot { class RuleCheckerSlot : public Slot { public: virtual ~RuleCheckerSlot() = default; bool IsContinue(const TokenResultSharedPtr& token, const EntryContextSharedPtr&) override { if (token->status() != TokenStatus::RESULT_STATUS_OK) { return false; } return true; } SlotType Type() const override { static constexpr SlotType type = SlotType::RULE_CHECKER_SLOT; return type; } }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/slot.h ================================================ #pragma once #include "sentinel-core/slot/base/slot_base.h" namespace Sentinel { namespace Slot { enum class SlotType { RULE_CHECKER_SLOT, STATS_SLOT, STATS_PREPARE_SLOT, }; class Slot : public SlotBase { public: virtual ~Slot() = default; virtual bool IsContinue(const TokenResultSharedPtr& token, const EntryContextSharedPtr& context) = 0; virtual const std::string& Name() const = 0; virtual SlotType Type() const = 0; }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/slot_base.h ================================================ #pragma once #include #include #include "sentinel-core/common/entry.h" #include "sentinel-core/common/resource_wrapper.h" #include "sentinel-core/slot/base/slot_base.h" #include "sentinel-core/slot/base/token_result.h" namespace Sentinel { namespace Slot { class SlotBase { public: virtual ~SlotBase() = default; virtual TokenResultSharedPtr Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) = 0; virtual void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) = 0; }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/slot_chain.h ================================================ #pragma once #include #include "sentinel-core/slot/base/slot.h" namespace Sentinel { namespace Slot { class SlotChain : public SlotBase { public: virtual ~SlotChain() = default; virtual void AddFirst(std::unique_ptr&& slot) = 0; virtual void AddLast(std::unique_ptr&& slot) = 0; }; using SlotChainSharedPtr = std::shared_ptr; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/stats_slot.h ================================================ #pragma once #include #include #include "sentinel-core/slot/base/slot.h" namespace Sentinel { namespace Slot { class StatsSlot : public Slot { public: virtual ~StatsSlot() = default; /* * Statistics class slot default always continue the rest of the slot * TODO(tianqian.zyf): TokenResultSharedPtr should be passed over the entire * slot chain using the SlotChainContext method. */ bool IsContinue(const TokenResultSharedPtr& token, const EntryContextSharedPtr& context) override { assert(context != nullptr); // We need to check nullptr to prevent unexpected circumstances. if (context != nullptr) { context->set_last_token_result(token); } return true; } SlotType Type() const override { static constexpr SlotType type = SlotType::STATS_SLOT; return type; } }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/token_result.cc ================================================ #include "sentinel-core/slot/base/token_result.h" namespace Sentinel { namespace Slot { TokenResultSharedPtr TokenResult::cachedOkPtr = std::make_shared(TokenStatus::RESULT_STATUS_OK); TokenResultSharedPtr TokenResult::Ok() { return cachedOkPtr; } TokenResultSharedPtr TokenResult::Blocked(const std::string& blocked_reason) { return std::make_shared(TokenStatus::RESULT_STATUS_BLOCKED, blocked_reason); } TokenResultSharedPtr TokenResult::ShouldWait( std::chrono::milliseconds wait_ms) { return std::make_shared(TokenStatus::RESULT_STATUS_SHOULD_WAIT, wait_ms); } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/base/token_result.h ================================================ #pragma once #include #include #include "absl/types/optional.h" namespace Sentinel { namespace Slot { enum class TokenStatus { RESULT_STATUS_OK = 0, RESULT_STATUS_BLOCKED = 1, RESULT_STATUS_SHOULD_WAIT = 2, }; class TokenResult; using TokenResultSharedPtr = std::shared_ptr; class TokenResult { public: TokenResult(TokenStatus status) : status_(status) {} TokenResult(TokenStatus status, const std::string& blocked_reason) : status_(status), blocked_reason_(blocked_reason) {} TokenResult(TokenStatus status, std::chrono::milliseconds wait_ms) : status_(status), wait_ms_(wait_ms) {} static TokenResultSharedPtr Ok(); static TokenResultSharedPtr Blocked(const std::string& blocked_reason); static TokenResultSharedPtr ShouldWait(std::chrono::milliseconds wait_ms); TokenStatus status() const { return status_; } absl::optional blocked_reason() const { return blocked_reason_; }; absl::optional wait_ms() const { return wait_ms_; }; private: const TokenStatus status_; static TokenResultSharedPtr cachedOkPtr; absl::optional blocked_reason_; absl::optional wait_ms_; }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/global_slot_chain.cc ================================================ #include #include "sentinel-core/circuitbreaker/slot.h" #include "sentinel-core/flow/flow_slot.h" #include "sentinel-core/param/param_flow_slot.h" #include "sentinel-core/slot/base/default_slot_chain_impl.h" #include "sentinel-core/slot/base/slot_chain.h" #include "sentinel-core/slot/log_slot.h" #include "sentinel-core/slot/resource_node_builder_slot.h" #include "sentinel-core/slot/statistic_slot.h" #include "sentinel-core/system/system_slot.h" namespace Sentinel { namespace Slot { namespace { SlotChainSharedPtr BuildDefaultSlotChain() { auto chain = std::make_shared(); chain->AddLast(std::make_unique()); chain->AddLast(std::make_unique()); chain->AddLast(std::make_unique()); chain->AddLast(std::make_unique()); chain->AddLast(std::make_unique()); chain->AddLast(std::make_unique()); chain->AddLast(std::make_unique()); chain->AddLast(std::make_unique()); return chain; } // namespace } // namespace SlotChainSharedPtr GetGlobalSlotChain() { static SlotChainSharedPtr GlobalSlotChain = BuildDefaultSlotChain(); return GlobalSlotChain; } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/global_slot_chain.h ================================================ #pragma once #include #include "sentinel-core/slot/base/slot_chain.h" namespace Sentinel { namespace Slot { SlotChainSharedPtr GetGlobalSlotChain(); } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/log_slot.cc ================================================ #include "sentinel-core/slot/log_slot.h" #include "sentinel-core/log/log_base.h" #include "sentinel-core/utils/time_utils.h" using namespace Sentinel::Utils; namespace Sentinel { namespace Slot { LogSlot::LogSlot() { log_task_ = std::make_unique(); log_task_->Start(); } TokenResultSharedPtr LogSlot::Entry(const EntrySharedPtr& entry, /*const*/ Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { if (entry == nullptr || entry->context() == nullptr) { return TokenResult::Ok(); } TokenResultSharedPtr prev_result = entry->context()->last_token_result(); if (prev_result == nullptr) { return TokenResult::Ok(); } if (prev_result->status() == TokenStatus::RESULT_STATUS_BLOCKED) { log_task_->Log(entry->resource()->name(), prev_result->blocked_reason().value_or("unknown")); } return prev_result; } void LogSlot::Exit(const EntrySharedPtr&, int, const std::vector& params) { // Do nothing } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/log_slot.h ================================================ #pragma once #include #include "sentinel-core/log/block/block_log_task.h" #include "sentinel-core/log/log_base.h" #include "sentinel-core/slot/base/stats_slot.h" #include "sentinel-core/slot/log_slot.h" #include "sentinel-core/utils/time_utils.h" using namespace Sentinel::Utils; namespace Sentinel { namespace Slot { constexpr auto kLogSlotName = "LogSlot"; class LogSlot : public StatsSlot { public: LogSlot(); virtual ~LogSlot() = default; const std::string& Name() const override { return name_; }; TokenResultSharedPtr Entry(const EntrySharedPtr& entry, /*const*/ Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; private: const std::string name_{kLogSlotName}; std::unique_ptr log_task_; }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/resource_node_builder_slot.cc ================================================ #include #include #include "sentinel-core/slot/resource_node_builder_slot.h" namespace Sentinel { namespace Slot { const std::string& ResourceNodeBuilderSlot::Name() const { return name_; } TokenResultSharedPtr ResourceNodeBuilderSlot::Entry( const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int, int, const std::vector& params) { auto cluster_node = node_storage_.GetOrCreateClusterNode(entry->resource()->name()); entry->set_cur_node(cluster_node); if (cluster_node && !entry->context()->tag().empty()) { Stat::NodeSharedPtr tag_node = cluster_node->GetOrCreateTagNode(entry->context()->tag()); entry->context()->set_tag_node(tag_node); } node = cluster_node; return TokenResult::Ok(); } void ResourceNodeBuilderSlot::Exit(const EntrySharedPtr&, int, const std::vector& params) { // Do nothing. } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/resource_node_builder_slot.h ================================================ #pragma once #include #include #include "sentinel-core/slot/base/stats_slot.h" #include "sentinel-core/statistic/node/resource_node_storage.h" namespace Sentinel { namespace Slot { constexpr auto kResourceNodeBuilderSlotName = "ResourceNodeBuilderSlot"; class ResourceNodeBuilderSlot : public StatsSlot { public: ResourceNodeBuilderSlot() = default; virtual ~ResourceNodeBuilderSlot() = default; TokenResultSharedPtr Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; const std::string& Name() const override; private: const std::string name_{kResourceNodeBuilderSlotName}; Stat::ResourceNodeStorage& node_storage_ = Stat::ResourceNodeStorage::GetInstance(); }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/resource_node_builder_slot_test.cc ================================================ #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/common/entry_context.h" #include "sentinel-core/common/string_resource_wrapper.h" #include "sentinel-core/slot/base/default_slot_chain_impl.h" #include "sentinel-core/slot/resource_node_builder_slot.h" #include "sentinel-core/statistic/node/cluster_node.h" #include "sentinel-core/statistic/node/resource_node_storage.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { namespace Slot { class RequireNodeFakeStatsSlot : public StatsSlot { public: RequireNodeFakeStatsSlot() = default; virtual ~RequireNodeFakeStatsSlot() = default; TokenResultSharedPtr Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { if (node == nullptr) { return TokenResult::Blocked("null"); } return TokenResult::Ok(); }; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params){}; const std::string& Name() const { return name_; }; private: const std::string name_{"RequireNodeFakeStatsSlot"}; }; TEST(ResourceNodeBuilderSlotTest, TestEntrySingleThread) { DefaultSlotChainImpl slot_chain; std::unique_ptr resource_node_slot = std::make_unique(); std::unique_ptr fake_stat_slot = std::make_unique(); slot_chain.AddLast(std::move(resource_node_slot)); slot_chain.AddLast(std::move(fake_stat_slot)); std::string resource_name{ "ResourceNodeBuilderSlotTest:TestEntrySingleThread"}; EntryContextSharedPtr context = std::make_shared("test_context", "test_tag"); auto resource = std::make_shared(resource_name, EntryType::OUT); auto entry = std::make_shared(resource, context); Stat::ResourceNodeStorage& s = Stat::ResourceNodeStorage::GetInstance(); EXPECT_TRUE(s.GetClusterNode(resource_name) == nullptr); Stat::NodeSharedPtr empty_node = nullptr; const std::vector myParams; auto result = slot_chain.Entry(entry, empty_node, 1, 0, myParams); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, result->status()); auto res_node = s.GetClusterNode(resource_name); EXPECT_FALSE(res_node == nullptr); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot_chain.Entry(entry, empty_node, 1, 0, myParams)->status()); EXPECT_EQ(res_node, s.GetClusterNode(resource_name)); } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/statistic_slot.cc ================================================ #include "sentinel-core/slot/statistic_slot.h" #include "sentinel-core/common/constants.h" #include "sentinel-core/common/entry.h" #include "sentinel-core/slot/base/token_result.h" #include "sentinel-core/statistic/node/node.h" #include "sentinel-core/utils/time_utils.h" #include namespace Sentinel { namespace Slot { const std::string& StatisticSlot::Name() const { return name_; } void StatisticSlot::RecordPassFor(const Stat::NodeSharedPtr& node, int count) { if (node != nullptr) { node->IncreaseThreadNum(); node->AddPassRequest(count); } } void StatisticSlot::RecordBlockFor(const Stat::NodeSharedPtr& node, int count) { if (node != nullptr) { node->AddBlockRequest(count); } } void StatisticSlot::RecordCompleteFor(const Stat::NodeSharedPtr& node, int rt, const std::string& err, int count) { if (node != nullptr) { // Record response time and success count. node->AddRtAndCompleteRequest(rt, count); if (!err.empty()) { node->AddExceptionRequest(count); } node->DecreaseThreadNum(); } } TokenResultSharedPtr StatisticSlot::OnPass( const EntrySharedPtr& entry, const ResourceWrapperSharedPtr& resource, const Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { this->RecordPassFor(node, count); if (resource->entry_type() == EntryType::IN) { this->RecordPassFor(Stat::ResourceNodeStorage::GetInstance().GetEntryNode(), count); } if (entry != nullptr) { this->RecordPassFor(entry->context()->tag_node(), count); } auto metric = ParamFlowSlot::GetParamMetric(resource->name()); if (metric) { metric->AddPass(count, params); metric->AddThreadCount(params); } return TokenResult::Ok(); } TokenResultSharedPtr StatisticSlot::OnBlock( const TokenResultSharedPtr& prev_result, const EntrySharedPtr& entry, const ResourceWrapperSharedPtr& resource, const Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { if (entry == nullptr) { return prev_result; } entry->set_block_error( prev_result->blocked_reason().value_or("unexpected_blocked")); this->RecordBlockFor(node, count); this->RecordBlockFor(entry->context()->tag_node(), count); if (resource->entry_type() == EntryType::IN) { this->RecordBlockFor( Stat::ResourceNodeStorage::GetInstance().GetEntryNode(), count); } return prev_result; } TokenResultSharedPtr StatisticSlot::Entry( const EntrySharedPtr& entry, /*const*/ Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) { if (entry == nullptr || entry->context() == nullptr) { return OnPass(entry, entry->resource(), node, count, flag, params); } TokenResultSharedPtr prev_result = entry->context()->last_token_result(); if (prev_result == nullptr) { return OnPass(entry, entry->resource(), node, count, flag, params); } switch (prev_result->status()) { case TokenStatus::RESULT_STATUS_BLOCKED: return OnBlock(prev_result, entry, entry->resource(), node, count, flag, params); case TokenStatus::RESULT_STATUS_OK: default: return OnPass(entry, entry->resource(), node, count, flag, params); } } void StatisticSlot::Exit(const EntrySharedPtr& entry, int count, const std::vector& params) { if (entry == nullptr) { return; } Stat::NodeSharedPtr node = entry->cur_node(); if (node == nullptr) { return; // Should not happen. } if (entry->HasBlockError()) { return; } long rt = Utils::TimeUtils::CurrentTimeMillis().count() - entry->create_time().count(); entry->set_rt(rt); this->RecordCompleteFor(node, rt, entry->error(), count); this->RecordCompleteFor(entry->context()->tag_node(), rt, entry->error(), count); if (entry->resource()->entry_type() == EntryType::IN) { this->RecordCompleteFor( Stat::ResourceNodeStorage::GetInstance().GetEntryNode(), rt, entry->error(), count); } auto metric = ParamFlowSlot::GetParamMetric(entry->resource()->name()); if (metric) { metric->DecreaseThreadCount(params); } } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/statistic_slot.h ================================================ #pragma once #include "sentinel-core/common/constants.h" #include "sentinel-core/param/param_flow_slot.h" #include "sentinel-core/slot/base/stats_slot.h" #include "sentinel-core/slot/base/token_result.h" #include "sentinel-core/statistic/node/node.h" #include "sentinel-core/statistic/node/resource_node_storage.h" namespace Sentinel { namespace Slot { constexpr auto kStatisticSlotName = "StatisticSlot"; class StatisticSlot : public StatsSlot { public: StatisticSlot() = default; virtual ~StatisticSlot() = default; const std::string& Name() const override; TokenResultSharedPtr Entry(const EntrySharedPtr& entry, /*const*/ Stat::NodeSharedPtr&, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; private: const std::string name_{kStatisticSlotName}; TokenResultSharedPtr OnPass(const EntrySharedPtr& entry, const ResourceWrapperSharedPtr& resource, const Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params); TokenResultSharedPtr OnBlock(const TokenResultSharedPtr& prev_result, const EntrySharedPtr& entry, const ResourceWrapperSharedPtr& resource, const Stat::NodeSharedPtr&, int count, int flag, const std::vector& params); void RecordPassFor(const Stat::NodeSharedPtr& node, int count); void RecordBlockFor(const Stat::NodeSharedPtr& node, int count); void RecordCompleteFor(const Stat::NodeSharedPtr& node, int rt, const std::string& err, int count); }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/slot/statistic_slot_test.cc ================================================ #include #include #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/common/entry_context.h" #include "sentinel-core/common/string_resource_wrapper.h" #include "sentinel-core/slot/statistic_slot.h" #include "sentinel-core/statistic/node/cluster_node.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { namespace Slot { TEST(StatisticSlotTest, TestEntryAndExitSingleThread) { EntryContextSharedPtr context = std::make_shared("test_context"); Stat::NodeSharedPtr node = std::make_shared(); auto resource = std::make_shared("test_resource", EntryType::OUT); auto entry = std::make_shared(resource, context); entry->set_cur_node(node); StatisticSlot slot; std::vector myParams; // Make the slot pass. auto pass_result = TokenResult::Ok(); slot.IsContinue(pass_result, context); auto result1 = slot.Entry(entry, node, 1, 0, myParams); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, result1->status()); EXPECT_EQ(1, node->CurThreadNum()); EXPECT_DOUBLE_EQ(1, node->PassQps()); EXPECT_DOUBLE_EQ(0, node->BlockQps()); EXPECT_DOUBLE_EQ(0, node->CompleteQps()); std::this_thread::sleep_for(std::chrono::milliseconds(200)); slot.Exit(entry, 1, myParams); EXPECT_EQ(0, node->CurThreadNum()); EXPECT_DOUBLE_EQ(1, node->CompleteQps()); EXPECT_NEAR(200, node->AvgRt(), 50); // Make the slot block. const std::string error_msg = "SomeBlockException"; auto block_result = TokenResult::Blocked(error_msg); slot.IsContinue(block_result, context); auto result2 = slot.Entry(entry, node, 1, 0, myParams); EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, result2->status()); EXPECT_TRUE(result2->blocked_reason().has_value()); // The pass count should remain unchanged. EXPECT_DOUBLE_EQ(1, node->PassQps()); EXPECT_DOUBLE_EQ(1, node->BlockQps()); } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "stat_config_lib", srcs = [ "stat_config.h", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/common:constants", ] ) cc_library( name = "stat_config_manager_lib", srcs = [ "stat_config_manager.h", "stat_config_manager.cc", ], copts = DEFAULT_COPTS, deps = [ ":stat_config_lib", "//sentinel-core/log:logger_lib", "//sentinel-core/property:sentinel_property_interface", "//sentinel-core/statistic/node:resource_node_storage_lib", ] ) cc_library( name = "metric_event_enum", srcs = [ "metric_event.h", ], copts = DEFAULT_COPTS, ) cc_library( name = "metric_bucket_lib", srcs = [ "metric_bucket.h", "metric_bucket.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/common:constants", ":metric_event_enum", ] ) cc_library( name = "window_wrap_lib", srcs = [ "window_wrap.h", ], copts = DEFAULT_COPTS ) cc_library( name = "metric_item_lib", srcs = [ "metric_item.h", "metric_item.cc", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", ] ) cc_library( name = "leap_array_lib", srcs = [ "leap_array.h", ], copts = DEFAULT_COPTS, deps = [ ":window_wrap_lib", "//sentinel-core/log:logger_lib", ] ) cc_library( name = "bucket_leap_array_lib", srcs = [ "bucket_leap_array.h", "bucket_leap_array.cc", ], copts = DEFAULT_COPTS, deps = [ ":leap_array_lib", ":metric_bucket_lib" ] ) cc_library( name = "metric_interface", srcs = [ "metric.h", ], copts = DEFAULT_COPTS, deps = [ ":metric_bucket_lib", ":metric_item_lib", ] ) cc_library( name = "sliding_window_metric_lib", srcs = [ "sliding_window_metric.h", "sliding_window_metric.cc" ], copts = DEFAULT_COPTS, deps = [ ":metric_interface", ":bucket_leap_array_lib", ":window_wrap_lib", ] ) cc_test( name = "sliding_window_metric_unittests", srcs = [ "sliding_window_metric_test.cc", ], copts = TEST_COPTS, deps = [ ":sliding_window_metric_lib", "@com_google_googletest//:gtest_main", ] ) cc_test( name = "bucket_leap_array_unittests", srcs = [ "bucket_leap_array_test.cc", ], copts = TEST_COPTS, deps = [ ":bucket_leap_array_lib", "@com_google_googletest//:gtest_main", ] ) cc_test( name = "metric_item_unittests", srcs = [ "metric_item_test.cc", ], copts = TEST_COPTS, deps = [ ":metric_item_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/statistic/base/bucket_leap_array.cc ================================================ #include "sentinel-core/statistic/base/bucket_leap_array.h" namespace Sentinel { namespace Stat { std::shared_ptr BucketLeapArray::NewEmptyBucket(int64_t) { return std::make_shared(); } void BucketLeapArray::ResetWindowTo(const WindowWrapSharedPtr& w, int64_t start_time) { // Update the start time and reset value. w->ResetTo(start_time); w->Value()->Reset(); } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/bucket_leap_array.h ================================================ #pragma once #include #include "sentinel-core/statistic/base/leap_array.h" #include "sentinel-core/statistic/base/metric_bucket.h" namespace Sentinel { namespace Stat { class BucketLeapArray : public LeapArray { public: explicit BucketLeapArray(int32_t sample_count, int32_t interval_ms) : LeapArray(sample_count, interval_ms) {} virtual ~BucketLeapArray() {} std::shared_ptr NewEmptyBucket(int64_t time_millis) override; void ResetWindowTo(const WindowWrapSharedPtr& wrap, int64_t start_time) override; }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/bucket_leap_array_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/statistic/base/bucket_leap_array.h" #include "sentinel-core/utils/time_utils.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { namespace Stat { constexpr int kBucketLengthInMs = 1000; constexpr int kIntervalInSec = 2; constexpr int kIntervalInMs = kIntervalInSec * 1000; constexpr int kSampleCount = kIntervalInMs / kBucketLengthInMs; TEST(BucketLeapArrayTest, TestNewWindow) { auto leap_array = std::make_shared(kSampleCount, kIntervalInMs); int64_t t = Utils::TimeUtils::CurrentTimeMillis().count(); WindowWrapSharedPtr bucket = leap_array->CurrentWindow(t); EXPECT_EQ(kBucketLengthInMs, bucket->BucketLengthInMs()); EXPECT_EQ(bucket->BucketStart(), t - t % kBucketLengthInMs); EXPECT_TRUE(bucket->Value() != nullptr); EXPECT_EQ(0, bucket->Value()->Get(MetricEvent::PASS)); } TEST(BucketLeapArrayTest, TestLeapArrayWindowStart) { auto leap_array = std::make_shared(kSampleCount, kIntervalInMs); int64_t first_time = Utils::TimeUtils::CurrentTimeMillis().count(); int64_t previous_window_start = first_time - first_time % kBucketLengthInMs; WindowWrapSharedPtr bucket = leap_array->CurrentWindow(first_time); EXPECT_EQ(kBucketLengthInMs, bucket->BucketLengthInMs()); EXPECT_EQ(previous_window_start, bucket->BucketStart()); } TEST(BucketLeapArrayTest, TestWindowAfterOneInterval) { auto leap_array = std::make_shared(kIntervalInSec, kIntervalInMs); int64_t first_time = Utils::TimeUtils::CurrentTimeMillis().count(); int64_t previous_window_start = first_time - first_time % kBucketLengthInMs; WindowWrapSharedPtr bucket = leap_array->CurrentWindow(previous_window_start); EXPECT_EQ(kBucketLengthInMs, bucket->BucketLengthInMs()); EXPECT_EQ(previous_window_start, bucket->BucketStart()); std::shared_ptr current_bucket = bucket->Value(); EXPECT_TRUE(current_bucket != nullptr); current_bucket->Add(MetricEvent::PASS, 1); current_bucket->Add(MetricEvent::BLOCK, 1); int64_t middle_time = previous_window_start + kBucketLengthInMs / 2; bucket = leap_array->CurrentWindow(middle_time); EXPECT_EQ(previous_window_start, bucket->BucketStart()); EXPECT_TRUE(current_bucket == bucket->Value()); int64_t next_time = middle_time + kBucketLengthInMs / 2; bucket = leap_array->CurrentWindow(next_time); EXPECT_EQ(kBucketLengthInMs, bucket->BucketLengthInMs()); EXPECT_EQ(kBucketLengthInMs, bucket->BucketStart() - previous_window_start); current_bucket = bucket->Value(); EXPECT_TRUE(current_bucket != nullptr); EXPECT_EQ(0L, current_bucket->Get(MetricEvent::PASS)); EXPECT_EQ(0L, current_bucket->Get(MetricEvent::BLOCK)); } TEST(BucketLeapArrayTest, TestListWindowsResetOld) { int windowLengthInMs = 100; int intervalInMs = 1000; int sampleCount = intervalInMs / windowLengthInMs; auto leap_array = std::make_shared(sampleCount, intervalInMs); int64_t time = Utils::TimeUtils::CurrentTimeMillis().count(); } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/leap_array.h ================================================ #pragma once #include #include #include #include #include "sentinel-core/log/logger.h" #include "sentinel-core/statistic/base/window_wrap.h" #include "sentinel-core/utils/time_utils.h" namespace Sentinel { namespace Stat { template class LeapArray { public: explicit LeapArray(int32_t sample_count, int32_t interval_ms) : interval_ms_(interval_ms), sample_count_(sample_count), bucket_length_ms_(interval_ms / sample_count), array_(std::make_unique[]>(sample_count)) {} virtual ~LeapArray() = default; int32_t SampleCount() const; int32_t IntervalInMs() const; virtual std::shared_ptr NewEmptyBucket(int64_t time_millis) = 0; virtual void ResetWindowTo(const WindowWrapSharedPtr& wrap, int64_t start_time) = 0; virtual WindowWrapSharedPtr CurrentWindow(); virtual WindowWrapSharedPtr CurrentWindow(int64_t time_millis); std::vector> Buckets() const; std::vector> Buckets(int64_t time_millis) const; std::vector> Values() const; std::vector> Values(int64_t time_millis) const; bool IsBucketDeprecated(const WindowWrapSharedPtr& wrap) const; bool IsBucketDeprecated(int64_t time_millis, const WindowWrapSharedPtr& wrap) const; protected: const int32_t interval_ms_; // total time length of the sliding window const int32_t sample_count_; // divide the sliding window into n parts const int32_t bucket_length_ms_; // time length of each bucket private: const std::unique_ptr[]> array_; std::mutex mtx_; int32_t CalculateTimeIdx(/*@Valid*/ int64_t time_millis) const; int64_t CalculateWindowStart(/*@Valid*/ int64_t time_millis) const; }; template int32_t LeapArray::SampleCount() const { return this->sample_count_; } template int32_t LeapArray::IntervalInMs() const { return this->interval_ms_; } template WindowWrapSharedPtr LeapArray::CurrentWindow() { return this->CurrentWindow(Utils::TimeUtils::CurrentTimeMillis().count()); } template WindowWrapSharedPtr LeapArray::CurrentWindow(int64_t time_millis) { if (time_millis < 0) { return nullptr; } uint32_t idx = CalculateTimeIdx(time_millis); int64_t bucket_start = CalculateWindowStart(time_millis); while (true) { WindowWrapSharedPtr old = array_[idx]; if (old == nullptr) { std::unique_lock lck(mtx_, std::defer_lock); if (lck.try_lock() && array_[idx] == nullptr) { WindowWrapSharedPtr bucket = std::make_shared>( bucket_length_ms_, bucket_start, NewEmptyBucket(time_millis)); array_[idx] = bucket; return bucket; } } else if (bucket_start == old->BucketStart()) { return old; } else if (bucket_start > old->BucketStart()) { std::unique_lock lck(mtx_, std::defer_lock); if (lck.try_lock()) { ResetWindowTo(old, bucket_start); return old; } } else if (bucket_start < old->BucketStart()) { // Should not go through here, as the provided time is already behind. return std::make_shared>(bucket_length_ms_, bucket_start, NewEmptyBucket(time_millis)); } } } template int32_t LeapArray::CalculateTimeIdx(int64_t time_millis) const { int64_t time_id = time_millis / bucket_length_ms_; // Calculate current index so we can map the timestamp to the leap array. int32_t size = sample_count_; // array_.size() return static_cast(time_id % size); } template int64_t LeapArray::CalculateWindowStart(int64_t time_millis) const { return time_millis - time_millis % bucket_length_ms_; } template bool LeapArray::IsBucketDeprecated( const WindowWrapSharedPtr& wrap) const { return this->IsBucketDeprecated(Utils::TimeUtils::CurrentTimeMillis().count(), wrap); } template bool LeapArray::IsBucketDeprecated( int64_t time_millis, const WindowWrapSharedPtr& wrap) const { return time_millis - wrap->BucketStart() > interval_ms_; } template std::vector> LeapArray::Buckets() const { return this->Buckets(Utils::TimeUtils::CurrentTimeMillis().count()); } template std::vector> LeapArray::Values() const { return this->Values(Utils::TimeUtils::CurrentTimeMillis().count()); } template std::vector> LeapArray::Buckets( int64_t time_millis) const { std::vector> result{}; if (time_millis < 0) { return result; } int size = sample_count_; // array_.size() for (int i = 0; i < size; i++) { auto w = array_[i]; if (w == nullptr || IsBucketDeprecated(time_millis, w)) { continue; } result.push_back(std::move(w)); } return result; } template std::vector> LeapArray::Values( int64_t time_millis) const { std::vector> result{}; if (time_millis < 0) { return result; } int size = sample_count_; // array_.size() for (int i = 0; i < size; i++) { WindowWrapSharedPtr w = array_[i]; if (w == nullptr || IsBucketDeprecated(time_millis, w)) { continue; } result.push_back(std::move(w->Value())); } return result; } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/metric.h ================================================ #pragma once #include #include "sentinel-core/statistic/base/metric_bucket.h" #include "sentinel-core/statistic/base/metric_item.h" namespace Sentinel { namespace Stat { class Metric { public: virtual ~Metric() = default; virtual long Complete() = 0; /** * Get max success count. * * @return max success count */ virtual long MaxComplete() = 0; /** * Get total exception count. * * @return exception count */ virtual long Exception() = 0; /** * Get total block count. * * @return block count */ virtual long Block() = 0; /** * Get total pass count. not include {@link #occupiedPass()} * * @return pass count */ virtual long Pass() = 0; /** * Get total response time. * * @return total RT */ virtual long Rt() = 0; /** * Get the minimal RT. * * @return minimal RT */ virtual long MinRt() = 0; /** * Get aggregated metric nodes of all resources. * * @return metric node list of all resources */ virtual std::vector Details() = 0; /** * Add current exception count. * * @param n count to add */ virtual void AddException(int n) = 0; /** * Add current block count. * * @param n count to add */ virtual void AddBlock(int n) = 0; /** * Add current completed count. * * @param n count to add */ virtual void AddComplete(int n) = 0; /** * Add current pass count. * * @param n count to add */ virtual void AddPass(int n) = 0; /** * Add given RT to current total RT. * * @param rt RT */ virtual void AddRt(long rt) = 0; /** * Get the sliding window length in seconds. * * @return the sliding window length */ virtual double WindowIntervalInSec() const = 0; virtual int SampleCount() const = 0; }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/metric_bucket.cc ================================================ #include #include "sentinel-core/statistic/base/metric_bucket.h" namespace Sentinel { namespace Stat { MetricBucket& MetricBucket::Reset() { int size = (int)MetricEvent::Count; for (int i = 0; i < size; i++) { counters_[i].store(0); } InitMinRt(); return *this; } int64_t MetricBucket::Get(const MetricEvent& event) const { int i = (int)event; return counters_[i].load(); } int64_t MetricBucket::MinRt() const { return min_rt_; } void MetricBucket::Add(const MetricEvent& event, int64_t n) { int i = (int)event; counters_[i].fetch_add(n); } void MetricBucket::AddRt(int64_t rt) { Add(MetricEvent::RT, rt); // Not thread-safe, but it's okay. if (rt < min_rt_) { min_rt_ = rt; } } void MetricBucket::InitMinRt() { min_rt_ = Constants::kMaxAllowedRt; } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/metric_bucket.h ================================================ #pragma once #include #include #include "sentinel-core/common/constants.h" #include "sentinel-core/statistic/base/metric_event.h" namespace Sentinel { namespace Stat { class MetricBucket { public: explicit MetricBucket() { InitMinRt(); } ~MetricBucket() = default; MetricBucket& Reset(); int64_t Get(const MetricEvent& event) const; int64_t MinRt() const; void Add(const MetricEvent& event, int64_t n); void AddRt(int64_t rt); private: const std::unique_ptr[]> counters_ = std::make_unique[]>( static_cast(MetricEvent::Count)); long min_rt_; void InitMinRt(); }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/metric_event.h ================================================ #pragma once namespace Sentinel { namespace Stat { enum class MetricEvent { PASS, BLOCK, EXCEPTION, COMPLETE, RT, Count, // hack for getting length of enum }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/metric_item.cc ================================================ #include #include #include #include #include "absl/strings/numbers.h" #include "absl/strings/str_format.h" #include "absl/strings/str_split.h" #include "absl/time/time.h" #include "sentinel-core/statistic/base/metric_item.h" namespace Sentinel { namespace Stat { int64_t ParseOrDefault(const std::string& s, int64_t default_value) { int64_t x; if (!absl::SimpleAtoi(s, &x)) { x = default_value; } return x; } std::string MetricItem::ToThinString() const { std::string legal_name = resource_; std::replace(legal_name.begin(), legal_name.end(), '|', '_'); return absl::StrFormat("%d|%s|%d|%d|%d|%d|%d", timestamp_, legal_name, pass_qps_, block_qps_, complete_qps_, exception_qps_, rt_); } std::string MetricItem::ToFatString() const { auto time_str = absl::FormatTime("%Y-%m-%d %H:%M:%S", absl::FromUnixMillis(timestamp_), absl::LocalTimeZone()); std::string legal_name = resource_; std::replace(legal_name.begin(), legal_name.end(), '|', '_'); return absl::StrFormat("%d|%s|%s|%d|%d|%d|%d|%d", timestamp_, time_str, legal_name, pass_qps_, block_qps_, complete_qps_, exception_qps_, rt_); } MetricItemSharedPtr MetricItem::FromThinString(const std::string& thin_str) { std::vector vec = absl::StrSplit(thin_str, '|'); if (vec.size() < 7) { return nullptr; } MetricItemSharedPtr item = std::make_shared(); item->set_timestamp(ParseOrDefault(vec[0], 0)); item->set_resource(vec[1]); item->set_pass_qps(ParseOrDefault(vec[2], 0)); item->set_block_qps(ParseOrDefault(vec[3], 0)); item->set_complete_qps(ParseOrDefault(vec[4], 0)); item->set_exception_qps(ParseOrDefault(vec[5], 0)); item->set_rt(ParseOrDefault(vec[6], 0)); return item; } MetricItemSharedPtr MetricItem::FromFatString(const std::string& fat_str) { std::vector vec = absl::StrSplit(fat_str, '|'); if (vec.size() < 8) { return nullptr; } MetricItemSharedPtr item = std::make_shared(); item->set_timestamp(ParseOrDefault(vec[0], 0)); item->set_resource(vec[2]); item->set_pass_qps(ParseOrDefault(vec[3], 0)); item->set_block_qps(ParseOrDefault(vec[4], 0)); item->set_complete_qps(ParseOrDefault(vec[5], 0)); item->set_exception_qps(ParseOrDefault(vec[6], 0)); item->set_rt(ParseOrDefault(vec[7], 0)); return item; } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/metric_item.h ================================================ #pragma once #include #include namespace Sentinel { namespace Stat { class MetricItem; using MetricItemSharedPtr = std::shared_ptr; class MetricItem { public: MetricItem() = default; static MetricItemSharedPtr FromThinString(const std::string& thin_str); static MetricItemSharedPtr FromFatString(const std::string& fat_str); const std::string& resource() const { return resource_; }; int64_t timestamp() const { return timestamp_; }; int64_t pass_qps() const { return pass_qps_; }; int64_t block_qps() const { return block_qps_; }; int64_t complete_qps() const { return complete_qps_; }; int64_t exception_qps() const { return exception_qps_; }; int64_t rt() const { return rt_; }; void set_resource(const std::string& resource) { resource_ = resource; }; void set_timestamp(int64_t t) { timestamp_ = t; }; void set_pass_qps(int64_t p) { pass_qps_ = p; }; void set_block_qps(int64_t b) { block_qps_ = b; }; void set_complete_qps(int64_t c) { complete_qps_ = c; }; void set_exception_qps(int64_t e) { exception_qps_ = e; }; void set_rt(int64_t rt) { rt_ = rt; }; std::string ToThinString() const; std::string ToFatString() const; private: std::string resource_; int64_t timestamp_; int64_t pass_qps_; int64_t block_qps_; int64_t complete_qps_; int64_t exception_qps_; int64_t rt_; }; // MetricNode in Java version } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/metric_item_test.cc ================================================ #include "gtest/gtest.h" #include "sentinel-core/statistic/base/metric_item.h" namespace Sentinel { namespace Stat { TEST(MetricItemTest, TestToThinString) { MetricItem item; item.set_resource("MetricItemTest|TestToThinString"); item.set_timestamp(1529998904000); item.set_pass_qps(10); item.set_block_qps(1); item.set_complete_qps(10); item.set_exception_qps(0); item.set_rt(3); auto str = "1529998904000|MetricItemTest_TestToThinString|10|1|10|0|3"; EXPECT_EQ(str, item.ToThinString()); } TEST(MetricItemTest, TestFromThinString) { auto str = "1529998908000|MetricItemTest::TestFromThinString|10|1|10|0|3"; MetricItemSharedPtr item = MetricItem::FromThinString(str); EXPECT_TRUE(item != nullptr); EXPECT_EQ(item->resource(), "MetricItemTest::TestFromThinString"); EXPECT_EQ(item->timestamp(), 1529998908000); EXPECT_EQ(item->pass_qps(), 10); EXPECT_EQ(item->block_qps(), 1); EXPECT_EQ(item->complete_qps(), 10); EXPECT_EQ(item->exception_qps(), 0); EXPECT_EQ(item->rt(), 3); } TEST(MetricItemTest, TestFromFatString) { auto str = "1529998913000|2018-06-26 " "15:41:53|MetricItemTest::TestFromFatString|10|1|10|2|25"; MetricItemSharedPtr item = MetricItem::FromFatString(str); EXPECT_TRUE(item != nullptr); EXPECT_EQ(item->resource(), "MetricItemTest::TestFromFatString"); EXPECT_EQ(item->timestamp(), 1529998913000); EXPECT_EQ(item->pass_qps(), 10); EXPECT_EQ(item->block_qps(), 1); EXPECT_EQ(item->complete_qps(), 10); EXPECT_EQ(item->exception_qps(), 2); EXPECT_EQ(item->rt(), 25); } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/sliding_window_metric.cc ================================================ #include #include #include "sentinel-core/common/constants.h" #include "sentinel-core/statistic/base/metric_event.h" #include "sentinel-core/statistic/base/sliding_window_metric.h" namespace Sentinel { namespace Stat { long SlidingWindowMetric::GetSum(const MetricEvent &event) { // Refresh current bucket for sliding window. sliding_window_->CurrentWindow(); long sum = 0; auto list = sliding_window_->Values(); for (const auto &bucket : list) { sum += bucket->Get(event); } return sum; } double SlidingWindowMetric::GetAvg(const MetricEvent &event) { return this->GetSum(event) / this->WindowIntervalInSec(); } long SlidingWindowMetric::Complete() { return this->GetSum(MetricEvent::COMPLETE); } long SlidingWindowMetric::MaxComplete() { sliding_window_->CurrentWindow(); long max = 0; auto list = sliding_window_->Values(); for (const auto &bucket : list) { long c = bucket->Get(MetricEvent::COMPLETE); if (c > max) { max = c; } } return std::max(1L, max); } long SlidingWindowMetric::Exception() { return this->GetSum(MetricEvent::EXCEPTION); } long SlidingWindowMetric::Block() { return this->GetSum(MetricEvent::BLOCK); } long SlidingWindowMetric::Pass() { return this->GetSum(MetricEvent::PASS); } long SlidingWindowMetric::Rt() { return this->GetSum(MetricEvent::RT); } long SlidingWindowMetric::MinRt() { sliding_window_->CurrentWindow(); long rt = Constants::kMaxAllowedRt; auto list = sliding_window_->Values(); for (const auto &bucket : list) { long minRt = bucket->MinRt(); if (minRt < rt) { rt = minRt; } } return std::max(1L, rt); } void SlidingWindowMetric::AddException(int n) { WindowWrapSharedPtr wrap = sliding_window_->CurrentWindow(); wrap->Value()->Add(MetricEvent::EXCEPTION, n); } void SlidingWindowMetric::AddBlock(int n) { WindowWrapSharedPtr wrap = sliding_window_->CurrentWindow(); wrap->Value()->Add(MetricEvent::BLOCK, n); } void SlidingWindowMetric::AddComplete(int n) { WindowWrapSharedPtr wrap = sliding_window_->CurrentWindow(); wrap->Value()->Add(MetricEvent::COMPLETE, n); } void SlidingWindowMetric::AddPass(int n) { WindowWrapSharedPtr wrap = sliding_window_->CurrentWindow(); wrap->Value()->Add(MetricEvent::PASS, n); } void SlidingWindowMetric::AddRt(long rt) { WindowWrapSharedPtr wrap = sliding_window_->CurrentWindow(); wrap->Value()->AddRt(rt); } std::vector SlidingWindowMetric::Details() { std::vector items; sliding_window_->CurrentWindow(); std::vector> list = sliding_window_->Buckets(); for (const auto &wrap : list) { if (wrap == nullptr) { continue; } auto bucket = wrap->Value(); if (bucket == nullptr) { continue; } MetricItemSharedPtr item = std::make_shared(); item->set_block_qps(bucket->Get(MetricEvent::BLOCK)); item->set_pass_qps(bucket->Get(MetricEvent::PASS)); item->set_exception_qps(bucket->Get(MetricEvent::EXCEPTION)); auto complete = bucket->Get(MetricEvent::COMPLETE); item->set_complete_qps(complete); if (complete <= 0) { item->set_rt(bucket->Get(MetricEvent::RT)); } else { item->set_rt(bucket->Get(MetricEvent::RT) / complete); } item->set_timestamp(wrap->BucketStart()); items.push_back(std::move(item)); } return items; } double SlidingWindowMetric::WindowIntervalInSec() const { return sliding_window_->IntervalInMs() / 1000.0; } int SlidingWindowMetric::SampleCount() const { return sliding_window_->SampleCount(); } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/sliding_window_metric.h ================================================ #pragma once #include #include "sentinel-core/statistic/base/bucket_leap_array.h" #include "sentinel-core/statistic/base/metric.h" #include "sentinel-core/statistic/base/metric_bucket.h" #include "sentinel-core/statistic/base/metric_event.h" namespace Sentinel { namespace Stat { class SlidingWindowMetric : public Metric { public: explicit SlidingWindowMetric(int32_t sample_count, int32_t interval_ms) : sliding_window_( std::make_unique(sample_count, interval_ms)) {} virtual ~SlidingWindowMetric() = default; long GetSum(const MetricEvent& event); double GetAvg(const MetricEvent& event); long Complete() override; long MaxComplete() override; long Exception() override; long Block() override; long Pass() override; long Rt() override; long MinRt() override; void AddException(int n) override; void AddBlock(int n) override; void AddComplete(int n) override; void AddPass(int n) override; void AddRt(long rt) override; std::vector Details() override; double WindowIntervalInSec() const override; int SampleCount() const override; private: const std::unique_ptr> sliding_window_; }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/sliding_window_metric_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/statistic/base/sliding_window_metric.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { namespace Stat { TEST(SlidingWindowMetricTest, TestOperateMetric) { auto metric = std::make_shared(2, 1000); metric->AddBlock(3); metric->AddBlock(1); metric->AddPass(2); EXPECT_EQ(4, metric->Block()); EXPECT_EQ(2, metric->Pass()); } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/stat_config.h ================================================ #pragma once #include #include "sentinel-core/common/constants.h" namespace Sentinel { namespace Stat { class StatConfig { public: static StatConfig& GetInstance() { static StatConfig* instance = new StatConfig(); return *instance; } friend class StatConfigManager; int32_t SampleCount() const { return sample_count_; }; int32_t IntervalMs() const { return interval_ms_; }; private: int32_t sample_count_ = Constants::kDefaultSampleCount; int32_t interval_ms_ = Constants::kDefaultIntervalMs; StatConfig() = default; }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/stat_config_manager.cc ================================================ #include "sentinel-core/statistic/base/stat_config_manager.h" #include "sentinel-core/log/logger.h" #include "sentinel-core/statistic/node/resource_node_storage.h" namespace Sentinel { namespace Stat { void StatConfigManager::UpdateSampleCount(int32_t new_sample_count) { StatConfig& config = StatConfig::GetInstance(); if (config.sample_count_ == new_sample_count) { return; } if (new_sample_count <= 0 || config.interval_ms_ % new_sample_count != 0) { SENTINEL_LOG(warn, "Invalid sample_count <{}> does not satisfy the restriction: " "sample_count > 0 && interval\%sample_count != 0"); return; } config.sample_count_ = new_sample_count; Stat::ResourceNodeStorage::GetInstance().ResetClusterNodes(); } void StatConfigManager::UpdateInterval(int32_t new_interval_ms) { StatConfig& config = StatConfig::GetInstance(); if (config.interval_ms_ == new_interval_ms) { return; } if (new_interval_ms <= 0 || new_interval_ms % config.sample_count_ != 0) { SENTINEL_LOG(warn, "Invalid interval_ms <{}> does not satisfy the restriction: " "interval_ms > 0 && interval_ms\%sample_count != 0"); return; } config.interval_ms_ = new_interval_ms; Stat::ResourceNodeStorage::GetInstance().ResetClusterNodes(); } void StatConfigManager::RegisterSampleCountProperty( const Property::SentinelPropertySharedPtr& property) { property->AddListener(std::make_unique()); } void StatConfigManager::RegisterIntervalProperty( const Property::SentinelPropertySharedPtr& property) { property->AddListener(std::make_unique()); } void SampleCountPropertyListener::ConfigUpdate(const int32_t& value, bool) { StatConfigManager::GetInstance().UpdateSampleCount(value); } void IntervalPropertyListener::ConfigUpdate(const int32_t& value, bool) { StatConfigManager::GetInstance().UpdateInterval(value); } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/stat_config_manager.h ================================================ #pragma once #include #include "sentinel-core/property/sentinel_property.h" #include "sentinel-core/statistic/base/stat_config.h" namespace Sentinel { namespace Stat { class StatConfigManager { public: static StatConfigManager& GetInstance() { static StatConfigManager* instance = new StatConfigManager(); return *instance; } void UpdateSampleCount(int32_t new_sample_count); void UpdateInterval(int32_t new_interval_ms); void RegisterSampleCountProperty( const Property::SentinelPropertySharedPtr& property); void RegisterIntervalProperty( const Property::SentinelPropertySharedPtr& property); private: StatConfigManager() = default; }; class SampleCountPropertyListener : public Property::PropertyListener { public: SampleCountPropertyListener() = default; ~SampleCountPropertyListener() = default; void ConfigUpdate(const int32_t& value, bool first_load) override; const std::string Name() const override { return "SampleCountPropertyListener"; } }; class IntervalPropertyListener : public Property::PropertyListener { public: IntervalPropertyListener() = default; ~IntervalPropertyListener() = default; void ConfigUpdate(const int32_t& value, bool first_load) override; const std::string Name() const override { return "IntervalPropertyListener"; }; }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/base/window_wrap.h ================================================ #pragma once #include namespace Sentinel { namespace Stat { template class WindowWrap { public: explicit WindowWrap(int64_t length_ms, int64_t start, const std::shared_ptr& value) : bucket_start_(start), bucket_length_ms_(length_ms), value_(value) {} ~WindowWrap() = default; int64_t BucketLengthInMs() const; int64_t BucketStart() const; std::shared_ptr Value() const; void ResetTo(int64_t start_time); bool IsTimeInBucket(int64_t time_millis) const; private: int64_t bucket_start_; const int64_t bucket_length_ms_; const std::shared_ptr value_; }; template using WindowWrapSharedPtr = std::shared_ptr>; template int64_t WindowWrap::BucketLengthInMs() const { return bucket_length_ms_; } template int64_t WindowWrap::BucketStart() const { return bucket_start_; } template std::shared_ptr WindowWrap::Value() const { return value_; } template void WindowWrap::ResetTo(int64_t start_time) { this->bucket_start_ = start_time; } template bool WindowWrap::IsTimeInBucket(int64_t time_millis) const { return bucket_start_ <= time_millis && time_millis < bucket_start_ + bucket_length_ms_; } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/node/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "node_interface", srcs = [ "node.h", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/statistic/base:metric_item_lib", ] ) cc_library( name = "statistic_node_lib", srcs = [ "statistic_node.h", "statistic_node.cc", ], copts = DEFAULT_COPTS, deps = [ ":node_interface", "//sentinel-core/statistic/base:sliding_window_metric_lib", "//sentinel-core/statistic/base:stat_config_lib", ] ) cc_library( name = "cluster_node_lib", srcs = [ "cluster_node.h", "cluster_node.cc", ], copts = DEFAULT_COPTS, deps = [ ":statistic_node_lib", "@com_google_absl//absl/container:flat_hash_map", ] ) cc_library( name = "resource_node_storage_lib", srcs = [ "resource_node_storage.h", "resource_node_storage.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/statistic/node:cluster_node_lib", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/synchronization", ] ) ================================================ FILE: sentinel-core/statistic/node/cluster_node.cc ================================================ #include "sentinel-core/statistic/node/cluster_node.h" namespace Sentinel { namespace Stat { StatisticNodeSharedPtr ClusterNode::GetTagNode(const std::string& tag) const { absl::ReaderMutexLock lck(&node_map_mtx_); auto got = tag_node_map_.find(tag); if (got == tag_node_map_.end()) { return nullptr; } return got->second; } StatisticNodeSharedPtr ClusterNode::GetOrCreateTagNode(const std::string& tag) { auto tag_node = GetTagNode(tag); if (tag_node == nullptr) { absl::WriterMutexLock lck(&node_map_mtx_); if (tag_node_map_.size() > Constants::kMaxTagSize) { SENTINEL_LOG(warn, "Tag node size exceeds the threshold {}", Constants::kMaxTagSize); return nullptr; } else { // The node is absent, create a new node for the classification_id. tag_node = std::make_shared(); tag_node_map_.insert(std::make_pair(tag, tag_node)); } } return tag_node; } void ClusterNode::TraceException(int count) { if (count > 0) { this->AddExceptionRequest(count); } } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/node/cluster_node.h ================================================ #pragma once #include #include #include #include "sentinel-core/statistic/node/statistic_node.h" #include "absl/container/flat_hash_map.h" namespace Sentinel { namespace Stat { class ClusterNode : public StatisticNode { public: explicit ClusterNode() = default; virtual ~ClusterNode() {} StatisticNodeSharedPtr GetTagNode(const std::string& tag) const; StatisticNodeSharedPtr GetOrCreateTagNode(const std::string& tag); void TraceException(int32_t count); public: absl::flat_hash_map tag_node_map_; mutable absl::Mutex node_map_mtx_; }; using ClusterNodeSharedPtr = std::shared_ptr; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/node/node.h ================================================ #pragma once #include #include #include "sentinel-core/statistic/base/metric_item.h" namespace Sentinel { namespace Stat { class Node { public: virtual ~Node() = default; virtual int64_t TotalCountInMinute() = 0; virtual int64_t PassCountInMinute() = 0; virtual int64_t CompleteCountInMinute() = 0; virtual int64_t BlockCountInMinute() = 0; virtual int64_t ExceptionCountInMinute() = 0; virtual double PassQps() = 0; virtual double BlockQps() = 0; virtual double TotalQps() = 0; virtual double CompleteQps() = 0; virtual double MaxCompleteQps() = 0; virtual double ExceptionQps() = 0; virtual double AvgRt() = 0; virtual double MinRt() = 0; virtual uint32_t CurThreadNum() const = 0; virtual double PreviousBlockQps() = 0; virtual double PreviousPassQps() = 0; virtual std::unordered_map Metrics() = 0; virtual void AddPassRequest(int32_t count) = 0; virtual void AddRtAndCompleteRequest(int32_t rt, int32_t completeCount) = 0; virtual void AddBlockRequest(int32_t count) = 0; virtual void AddExceptionRequest(int32_t count) = 0; virtual void IncreaseThreadNum() = 0; virtual void DecreaseThreadNum() = 0; virtual void Reset() = 0; }; using NodeSharedPtr = std::shared_ptr; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/node/resource_node_storage.cc ================================================ #include #include #include "sentinel-core/common/constants.h" #include "sentinel-core/log/logger.h" #include "sentinel-core/statistic/node/resource_node_storage.h" namespace Sentinel { namespace Stat { Stat::ClusterNodeSharedPtr ResourceNodeStorage::GetClusterNode( const std::string& resource_name) const { absl::ReaderMutexLock lck(&node_map_mtx_); auto got = node_map_.find(resource_name); if (got == node_map_.end()) { return nullptr; } return got->second; } Stat::ClusterNodeSharedPtr ResourceNodeStorage::GetOrCreateClusterNode( const std::string& resource_name) { auto cluster_node = GetClusterNode(resource_name); if (cluster_node == nullptr) { absl::WriterMutexLock lck(&node_map_mtx_); auto got = node_map_.find(resource_name); if (got == node_map_.end()) { if (node_map_.size() >= Constants::kMaxResourceSize) { SENTINEL_LOG(warn, "Resource node size exceeds the threshold {}", Constants::kMaxResourceSize); } // Resource node not found, so we create a new node. cluster_node = std::make_shared(); node_map_.insert(std::make_pair(resource_name, cluster_node)); SENTINEL_LOG(info, "Creating resource node for <{}>", resource_name); } else { cluster_node = got->second; } } // write lock end return cluster_node; } void ResourceNodeStorage::ResetClusterNodes() { absl::WriterMutexLock lck(&node_map_mtx_); for (const auto& e : node_map_) { e.second->Reset(); } } const std::unordered_map ResourceNodeStorage::GetNodeMap() const { absl::ReaderMutexLock lck(&node_map_mtx_); return this->node_map_; } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/node/resource_node_storage.h ================================================ #pragma once #include #include #include #include "sentinel-core/statistic/node/cluster_node.h" #include "absl/synchronization/mutex.h" namespace Sentinel { namespace Stat { class ResourceNodeStorage { public: ~ResourceNodeStorage() = default; static ResourceNodeStorage& GetInstance() { static ResourceNodeStorage* instance = new ResourceNodeStorage(); return *instance; } Stat::ClusterNodeSharedPtr GetClusterNode( const std::string& resource_name) const; Stat::ClusterNodeSharedPtr GetOrCreateClusterNode( const std::string& resource_name); void ResetClusterNodes(); Stat::ClusterNodeSharedPtr GetEntryNode() { return entry_node_; } const std::unordered_map GetNodeMap() const; private: ResourceNodeStorage() { entry_node_ = std::make_shared(); } std::unordered_map node_map_; // Global statistic node for inbound traffic. Usually used for SystemRule // checking. ClusterNodeSharedPtr entry_node_; mutable absl::Mutex node_map_mtx_; // protect `node_map_` }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/node/statistic_node.cc ================================================ #include #include #include #include "sentinel-core/statistic/node/statistic_node.h" namespace Sentinel { namespace Stat { int64_t StatisticNode::TotalCountInMinute() { return rolling_counter_minute_->Pass() + rolling_counter_minute_->Block(); } int64_t StatisticNode::PassCountInMinute() { return rolling_counter_minute_->Pass(); } int64_t StatisticNode::CompleteCountInMinute() { return rolling_counter_minute_->Complete(); } int64_t StatisticNode::BlockCountInMinute() { return rolling_counter_minute_->Block(); } int64_t StatisticNode::ExceptionCountInMinute() { return rolling_counter_minute_->Exception(); } double StatisticNode::PassQps() { return rolling_counter_second_->Pass() / rolling_counter_second_->WindowIntervalInSec(); } double StatisticNode::BlockQps() { return rolling_counter_second_->Block() / rolling_counter_second_->WindowIntervalInSec(); } double StatisticNode::TotalQps() { return this->PassQps() + this->BlockQps(); } double StatisticNode::CompleteQps() { return rolling_counter_second_->Complete() / rolling_counter_second_->WindowIntervalInSec(); } double StatisticNode::MaxCompleteQps() { return rolling_counter_second_->MaxComplete() * rolling_counter_second_->SampleCount(); } double StatisticNode::ExceptionQps() { return rolling_counter_second_->Exception() / rolling_counter_second_->WindowIntervalInSec(); } double StatisticNode::AvgRt() { long completed = rolling_counter_second_->Complete(); if (completed == 0) { return 0; } return rolling_counter_second_->Rt() * 1.0 / completed; } double StatisticNode::MinRt() { return rolling_counter_second_->MinRt(); } uint32_t StatisticNode::CurThreadNum() const { return this->cur_thread_num_.load(); } bool StatisticNode::IsValidMetricItem(const MetricItemSharedPtr& item) const { return item != nullptr && (item->pass_qps() > 0 || item->block_qps() > 0 || item->complete_qps() > 0 || item->exception_qps() > 0 || item->rt() > 0); } bool StatisticNode::IsNodeInTime(const MetricItemSharedPtr& item, int64_t cur_time) const { return item != nullptr && item->timestamp() > last_fetch_timestamp_ && item->timestamp() < cur_time; } std::unordered_map StatisticNode::Metrics() { int64_t cur_time = Utils::TimeUtils::CurrentTimeMillis().count(); cur_time = cur_time - cur_time % 1000; std::unordered_map map; std::vector items_of_second = rolling_counter_minute_->Details(); int64_t new_last_fetch_time = last_fetch_timestamp_; // Iterate metrics of all resources, filter valid metrics (not-empty and // up-to-date). for (const auto& item : items_of_second) { if (IsNodeInTime(item, cur_time) && IsValidMetricItem(item)) { new_last_fetch_time = std::max(new_last_fetch_time, item->timestamp()); map.insert(std::make_pair(item->timestamp(), item)); } } this->last_fetch_timestamp_ = new_last_fetch_time; return map; } double StatisticNode::PreviousBlockQps() { return 0; // TODO } double StatisticNode::PreviousPassQps() { return 0; // TODO } void StatisticNode::AddPassRequest(int32_t count) { this->rolling_counter_second_->AddPass(count); this->rolling_counter_minute_->AddPass(count); } void StatisticNode::AddRtAndCompleteRequest(int32_t rt, int32_t complete_count) { this->rolling_counter_second_->AddComplete(complete_count); this->rolling_counter_second_->AddRt(rt); this->rolling_counter_minute_->AddComplete(complete_count); this->rolling_counter_minute_->AddRt(rt); } void StatisticNode::AddBlockRequest(int32_t count) { this->rolling_counter_second_->AddBlock(count); this->rolling_counter_minute_->AddBlock(count); } void StatisticNode::AddExceptionRequest(int32_t count) { this->rolling_counter_second_->AddException(count); this->rolling_counter_minute_->AddException(count); } void StatisticNode::IncreaseThreadNum() { this->cur_thread_num_++; } void StatisticNode::DecreaseThreadNum() { this->cur_thread_num_--; } void StatisticNode::Reset() { StatConfig& c = StatConfig::GetInstance(); this->rolling_counter_second_ = std::make_unique(c.SampleCount(), c.IntervalMs()); } } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/statistic/node/statistic_node.h ================================================ #pragma once #include #include "sentinel-core/statistic/base/metric.h" #include "sentinel-core/statistic/base/sliding_window_metric.h" #include "sentinel-core/statistic/base/stat_config.h" #include "sentinel-core/statistic/node/node.h" namespace Sentinel { namespace Stat { class StatisticNode : public Node { public: explicit StatisticNode() = default; virtual ~StatisticNode() = default; // Node virtual int64_t TotalCountInMinute() override; virtual int64_t PassCountInMinute() override; virtual int64_t CompleteCountInMinute() override; virtual int64_t BlockCountInMinute() override; virtual int64_t ExceptionCountInMinute() override; virtual double PassQps() override; virtual double BlockQps() override; virtual double TotalQps() override; virtual double CompleteQps() override; virtual double MaxCompleteQps() override; virtual double ExceptionQps() override; virtual double AvgRt() override; virtual double MinRt() override; virtual uint32_t CurThreadNum() const override; virtual double PreviousBlockQps() override; virtual double PreviousPassQps() override; virtual std::unordered_map Metrics() override; virtual void AddPassRequest(int32_t count) override; virtual void AddRtAndCompleteRequest(int32_t rt, int32_t complete_count) override; virtual void AddBlockRequest(int32_t count) override; virtual void AddExceptionRequest(int32_t count) override; virtual void IncreaseThreadNum() override; virtual void DecreaseThreadNum() override; virtual void Reset() override; private: std::unique_ptr rolling_counter_second_ = std::make_unique( StatConfig::GetInstance().SampleCount(), StatConfig::GetInstance().IntervalMs()); std::unique_ptr rolling_counter_minute_ = std::make_unique(60, 60 * 1000); std::atomic cur_thread_num_{0}; int64_t last_fetch_timestamp_ = -1; bool IsValidMetricItem(const MetricItemSharedPtr& item) const; bool IsNodeInTime(const MetricItemSharedPtr& item, int64_t cur_time) const; }; using StatisticNodeSharedPtr = std::shared_ptr; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/system/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "system_status_listener_lib", srcs = [ "system_status_listener.h", "system_status_listener.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/log:logger_lib", ], ) cc_library( name = "system_rule_lib", srcs = [ "system_rule.cc", "system_rule.h", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/strings:str_format", "//sentinel-core/common:rule_lib", "@com_google_absl//absl/synchronization", ], ) # https://docs.bazel.build/versions/master/be/c-cpp.html#cc_library cc_library( name = "system_rule_manager_lib", srcs = [ "system_rule_manager.cc", "system_rule_manager.h", ], copts = DEFAULT_COPTS, deps = [ ":system_rule_lib", ":system_status_listener_lib", "//sentinel-core/property:dynamic_sentinel_property_lib", "//sentinel-core/slot/base:token_result_lib", "//sentinel-core/statistic/node:resource_node_storage_lib", "@com_google_absl//absl/synchronization", ], ) cc_library( name = "system_slot_lib", srcs = [ "system_slot.cc", "system_slot.h", ], copts = DEFAULT_COPTS, deps = [ ":system_rule_manager_lib", "//sentinel-core/slot/base:rule_checker_slot_interface", "@com_google_absl//absl/strings:str_format", ] ) cc_test( name = "system_slot_unittests", srcs = [ "system_slot_test.cc", "system_slot_mock.h", ], copts = TEST_COPTS, deps = [ ":system_slot_lib", ":system_rule_lib", ":system_rule_manager_lib", "//sentinel-core/common:string_resource_wrapper_lib", "//sentinel-core/test/mock/statistic/node:mock_lib", "@com_google_googletest//:gtest_main", ] ) cc_test( name = "system_status_listener_unittests", srcs = [ "system_status_listener_test.cc", ], copts = TEST_COPTS, deps = [ ":system_status_listener_lib", ":system_rule_manager_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-core/system/system_rule.cc ================================================ #include "sentinel-core/system/system_rule.h" #include "absl/strings/str_format.h" namespace Sentinel { namespace System { std::string GetMetricTypeString(MetricType ruleType) { switch (ruleType) { case MetricType::kQps: return std::string("qps"); case MetricType::kConcurrency: return std::string("concurrency"); case MetricType::kRt: return std::string("rt"); case MetricType::kCpuUsage: return std::string("cpu_usage"); case MetricType::kSystemLoad: return std::string("system_load"); default: return absl::StrFormat("unknown_type(%d)", ruleType); } } // Sentinel::Property::SentinelProperty. need this bool SystemRule::operator==(const SystemRule &rule) const { return metric_type_ == rule.metric_type() && threshold_ == rule.threshold(); } std::string SystemRule::ToString() const { return absl::StrFormat("SystemRule{metric_type=%s, threshold=%.2lf}", GetMetricTypeString(metric_type_).c_str(), threshold_); } } // namespace System } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_rule.h ================================================ #pragma once #include #include #include #include #include #include "absl/strings/str_format.h" #include "absl/synchronization/mutex.h" #include "sentinel-core/common/rule.h" namespace Sentinel { namespace System { enum MetricType { kSystemLoad = 0, kRt, kConcurrency, kQps, kCpuUsage }; std::string GetMetricTypeString(MetricType ruleType); struct SystemRule : public Rule { public: SystemRule() = default; SystemRule(MetricType metric_type, double threshold) : metric_type_(metric_type), threshold_(threshold) {} virtual ~SystemRule() = default; void set_rule_type(MetricType r_t_) { metric_type_ = r_t_; } void set_threshold(double t_) { threshold_ = t_; } MetricType metric_type() const { return metric_type_; } double threshold() const { return threshold_; } bool operator==(const SystemRule& rule) const; std::string ToString() const; private: MetricType metric_type_; double threshold_; }; using SystemRuleSharedPtr = std::shared_ptr; using SystemRuleList = std::vector; } // namespace System } // namespace Sentinel namespace std { template <> struct hash { size_t operator()(const Sentinel::System::MetricType& t) const { return hash{}(static_cast(t)); } }; } // namespace std ================================================ FILE: sentinel-core/system/system_rule_manager.cc ================================================ #include "sentinel-core/system/system_rule_manager.h" #include "sentinel-core/log/logger.h" namespace Sentinel { namespace System { constexpr auto kSystemPropertyListenerName = "SystemPropertyListener"; bool IsValidRule(const SystemRule &rule) { bool is_valid = true; switch (rule.metric_type()) { case MetricType::kCpuUsage: is_valid = (rule.threshold() >= 0 && rule.threshold() <= 1) ? true : false; break; default: is_valid = (rule.threshold() >= 0) ? true : false; break; } return is_valid; } void LogSystemMap(const SystemRuleMapSharedPtr map) { std::string s("["); for (const auto &rule : *map) { s += rule.second.ToString(); s += ","; } s[s.size() - 1] = ']'; SENTINEL_LOG(info, "[SystemRuleManager] System rules received: {}", s); } SystemRuleManager::SystemRuleManager() { cur_property_ = std::make_shared>(); cur_property_->AddListener(std::make_unique()); } bool SystemRuleManager::LoadRules(const SystemRuleList &rules) { return cur_property_->UpdateValue(rules); } void SystemRuleManager::RegisterToProperty( const DynamicSystemRulePropertySharedPtr &property) { if (property == nullptr) { return; } std::lock_guard lck(property_mtx_); SENTINEL_LOG(info, "Registering new property to SystemRuleManager"); cur_property_->RemoveListener(kSystemPropertyListenerName); cur_property_ = property; cur_property_->AddListener(std::make_unique()); } // Reentrant, since we use lock to protect `m.rule_map_`. // This ensure that `values` are updated atomically. // Only the most strict rule of each type are loaded. void SystemPropertyListener::ConfigUpdate(const SystemRuleList &values, bool) { SystemRuleMapSharedPtr new_rule_map = std::make_shared(); for (const auto &rule : values) { if (!IsValidRule(rule)) { SENTINEL_LOG(info, "[SystemRuleManager] Ignoring invalid system rule when " "loading new flow " "rules: {}", rule.ToString()); continue; } auto it = new_rule_map->find(rule.metric_type()); if (it == new_rule_map->end()) { // Add a new rule new_rule_map->insert(std::make_pair( rule.metric_type(), SystemRule(rule.metric_type(), rule.threshold()))); } else { // Change to a more strict threshold, or do nothing if (rule.threshold() < it->second.threshold()) { it->second.set_threshold(rule.threshold()); } } } SystemRuleManager &m = SystemRuleManager::GetInstance(); absl::WriterMutexLock lck(&(m.update_mtx_)); m.rule_map_ = new_rule_map; LogSystemMap(m.rule_map_); } const std::string SystemPropertyListener::Name() const { return kSystemPropertyListenerName; } } // namespace System } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_rule_manager.h ================================================ #pragma once #include #include #include #include #include #include #include #include "absl/synchronization/mutex.h" // #include "sentinel-core/common/entry.h" #include "sentinel-core/property/dynamic_sentinel_property.h" #include "sentinel-core/property/property_listener.h" #include "sentinel-core/slot/base/token_result.h" #include "sentinel-core/statistic/node/cluster_node.h" #include "sentinel-core/statistic/node/resource_node_storage.h" #include "sentinel-core/system/system_rule.h" #include "sentinel-core/system/system_status_listener.h" namespace Sentinel { namespace System { using DynamicSystemRulePropertySharedPtr = std::shared_ptr>; using SystemRuleMap = std::unordered_map; using SystemRuleMapSharedPtr = std::shared_ptr; class SystemRuleManager { public: static SystemRuleManager& GetInstance() { static SystemRuleManager* instance = new SystemRuleManager(); return *instance; } void RegisterToProperty(const DynamicSystemRulePropertySharedPtr& property); bool LoadRules(const SystemRuleList& rules); SystemRuleMapSharedPtr rule_map() { absl::ReaderMutexLock lck(&(update_mtx_)); return rule_map_; } // TODO: Use atomic here instead? friend class SystemPropertyListener; private: SystemRuleManager(); DynamicSystemRulePropertySharedPtr cur_property_; SystemRuleMapSharedPtr rule_map_; mutable std::mutex property_mtx_; // protect cur_property_ mutable absl::Mutex update_mtx_; // protect rule_map_ std::atomic check_system_status_{false}; }; class SystemPropertyListener : public Property::PropertyListener { public: SystemPropertyListener() = default; ~SystemPropertyListener() = default; void ConfigUpdate(const SystemRuleList& values, bool first_load) override; const std::string Name() const override; }; bool IsValidRule(const SystemRule& rule); } // namespace System } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_slot.cc ================================================ #include "sentinel-core/system/system_slot.h" #include "absl/strings/str_format.h" #include "sentinel-core/common/resource_wrapper.h" #include "sentinel-core/slot/base/token_result.h" #include "sentinel-core/statistic/node/resource_node_storage.h" #include "sentinel-core/system/system_rule_manager.h" namespace Sentinel { namespace Slot { const std::string& SystemSlot::Name() const { return name_; } TokenResultSharedPtr SystemSlot::Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr&, int count, int, const std::vector&) { // Fetch global statistic node for inbound traffic Stat::NodeSharedPtr entryNode = Stat::ResourceNodeStorage::GetInstance().GetEntryNode(); TokenResultSharedPtr res = TokenResult::Ok(); if (entry->resource()->entry_type() == EntryType::IN && entryNode != nullptr) { System::SystemRuleMapSharedPtr ruleMap = sysMgr.rule_map(); if (ruleMap && ruleMap->size() > 0) { return CheckSystem(ruleMap, entryNode, count); } else { return TokenResult::Ok(); } } return TokenResult::Ok(); } // In fact, except `acquire_count`, other arguments do not need to // be passed into `CheckSystem`, since they can be directly obtained // from global variables. // However, for convenience of passing mocked objects in unit test, // we still put them in function signature. TokenResultSharedPtr SystemSlot::CheckSystem( const System::SystemRuleMapSharedPtr sysRuleMap, Stat::NodeSharedPtr& node, int acquire_count) const { double curQps, concurrency, rt, load, cpuUsage; for (const auto& e : *sysRuleMap) { switch (e.first) { case System::MetricType::kQps: curQps = node->PassQps(); if (curQps + acquire_count > e.second.threshold()) { return TokenResult::Blocked("SystemException"); } break; case System::MetricType::kConcurrency: concurrency = static_cast(node->CurThreadNum()); if (concurrency + acquire_count > e.second.threshold()) { return TokenResult::Blocked("SystemException"); } break; case System::MetricType::kRt: rt = node->AvgRt(); if (rt > e.second.threshold()) { return TokenResult::Blocked("SystemException"); } break; case System::MetricType::kSystemLoad: load = GetCurLoad(); concurrency = static_cast(node->CurThreadNum()); if (load > e.second.threshold()) { if (!CheckBbr(concurrency, node)) { return TokenResult::Blocked("SystemException"); } } break; case System::MetricType::kCpuUsage: cpuUsage = GetCurCpuUsage(); concurrency = static_cast(node->CurThreadNum()); if (cpuUsage > e.second.threshold()) { if (!CheckBbr(concurrency, node)) { return TokenResult::Blocked("SystemException"); } } break; default: break; } } return TokenResult::Ok(); } bool SystemSlot::CheckBbr(double concurrency, Stat::NodeSharedPtr& node) const { if (concurrency > 1 && concurrency > node->MaxCompleteQps() * node->MinRt() / 1000) { return false; } return true; } void SystemSlot::Exit(const EntrySharedPtr&, int, const std::vector& params) { // Do nothing } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_slot.h ================================================ #pragma once #include "sentinel-core/slot/base/rule_checker_slot.h" #include "sentinel-core/system/system_rule_manager.h" namespace Sentinel { namespace Slot { constexpr auto kSystemSlotName = "SystemSlot"; class SystemSlot : public RuleCheckerSlot { public: SystemSlot() = default; virtual ~SystemSlot() = default; TokenResultSharedPtr Entry(const EntrySharedPtr& entry, Stat::NodeSharedPtr& node, int count, int flag, const std::vector& params) override; void Exit(const EntrySharedPtr& entry, int count, const std::vector& params) override; const std::string& Name() const override; TokenResultSharedPtr CheckSystem( const System::SystemRuleMapSharedPtr sysRuleMap, Stat::NodeSharedPtr& node, int acquire_count) const; virtual double GetCurCpuUsage() const { return System::SystemStatusListener::GetInstance().GetCurCpuUsage(); } virtual double GetCurLoad() const { return System::SystemStatusListener::GetInstance().GetCurLoad(); } friend class System::SystemRuleManager; private: const std::string name_{kSystemSlotName}; System::SystemRuleManager& sysMgr = System::SystemRuleManager::GetInstance(); bool CheckBbr(double curThread, Stat::NodeSharedPtr& node) const; }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_slot_mock.h ================================================ #pragma once #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/common/string_resource_wrapper.h" #include "sentinel-core/slot/base/token_result.h" #include "sentinel-core/statistic/node/node.h" #include "sentinel-core/statistic/node/statistic_node.h" #include "sentinel-core/system/system_rule.h" #include "sentinel-core/system/system_rule_manager.h" #include "sentinel-core/system/system_slot.h" namespace Sentinel { namespace Slot { class MockSystemSlot : public SystemSlot { public: MockSystemSlot() = default; ~MockSystemSlot() = default; MOCK_METHOD6(Entry, TokenResultSharedPtr(const EntrySharedPtr&, const ResourceWrapperSharedPtr&, Stat::NodeSharedPtr&, int, int, const std::vector&)); MOCK_METHOD4(Exit, void(const EntrySharedPtr&, const ResourceWrapperSharedPtr&, int, const std::vector&)); MOCK_CONST_METHOD3(CheckSystem, TokenResultSharedPtr(const System::SystemRuleMapSharedPtr, Stat::NodeSharedPtr&, int)); MOCK_CONST_METHOD0(GetCurCpuUsage, double(void)); MOCK_CONST_METHOD0(GetCurLoad, double(void)); MOCK_CONST_METHOD0(Name, const std::string&(void)); TokenResultSharedPtr OriginalCheckSystem( const System::SystemRuleMapSharedPtr sysRuleMap, Stat::NodeSharedPtr& node, int acquire_count) const { return SystemSlot::CheckSystem(sysRuleMap, node, acquire_count); } }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_slot_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/system/system_slot_mock.h" #include "sentinel-core/test/mock/statistic/node/mock.h" using testing::_; using testing::Mock; using testing::Return; namespace Sentinel { namespace Slot { // 3 pass test cases and 3 block test cases TEST(SystemSlotTest, SystemRuleSingleThreadTest) { std::string resource_name{"test_resource"}; EntryContextSharedPtr context = std::make_shared("test_resource"); Stat::NodeSharedPtr node = std::make_shared(); auto resource_in = std::make_shared(resource_name, EntryType::IN); auto resource_out = std::make_shared(resource_name, EntryType::OUT); auto entry_in = std::make_shared(resource_in, context); auto entry_out = std::make_shared(resource_out, context); SystemSlot slot; std::vector myParams; // No rule set EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry_in, node, 1000, 0, myParams)->status()); System::SystemRule rule1(System::MetricType::kCpuUsage, 0.6); System::SystemRule rule2(System::MetricType::kQps, 1); System::SystemRule rule3(System::MetricType::kSystemLoad, 0.8); System::SystemRuleManager::GetInstance().LoadRules({rule1, rule2, rule3}); System::SystemRuleMapSharedPtr p = std::make_shared(); p->insert(std::make_pair<>(System::MetricType::kCpuUsage, std::move(rule1))); // OUT entry type should not be blocked by system rule EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, slot.Entry(entry_out, node, 1000, 0, myParams)->status()); EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, slot.Entry(entry_in, node, 10, 0, myParams)->status()); MockSystemSlot mock_system_slot; ON_CALL(mock_system_slot, CheckSystem(_, _, _)) .WillByDefault( [&mock_system_slot](const System::SystemRuleMapSharedPtr sysRuleMap, Stat::NodeSharedPtr& node, int acquire_count) { return mock_system_slot.OriginalCheckSystem(sysRuleMap, node, acquire_count); }); ON_CALL(*(static_cast(node.get())), CurThreadNum()) .WillByDefault(Return(2)); EXPECT_CALL(*(static_cast(node.get())), CurThreadNum()) .Times(testing::AnyNumber()); // SystemRule: CPU usage // SystemSlot and ClusterNode are mocked here { EXPECT_CALL(mock_system_slot, CheckSystem(_, _, _)).Times(2); EXPECT_CALL(mock_system_slot, GetCurCpuUsage()) .WillOnce(Return(0.7)) .WillOnce(Return(0.9)); EXPECT_CALL(*(static_cast(node.get())), MaxCompleteQps()) .WillOnce(Return(1000)) .WillOnce(Return(2500)); EXPECT_CALL(*(static_cast(node.get())), MinRt()) .WillOnce(Return(1)) .WillOnce(Return(1)); // Block since cpu load excceeds threshold EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, mock_system_slot.CheckSystem(p, node, 1)->status()); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, mock_system_slot.CheckSystem(p, node, 1)->status()); Mock::VerifyAndClearExpectations(&mock_system_slot); } // SystemRule: QPS p->insert(std::make_pair( System::MetricType::kQps, std::move(rule2))); { EXPECT_CALL(mock_system_slot, CheckSystem(_, _, _)).Times(2); EXPECT_CALL(mock_system_slot, GetCurCpuUsage()).Times(testing::AnyNumber()); EXPECT_CALL(*(static_cast(node.get())), PassQps()) .WillOnce(Return(0)) .WillOnce(Return(1)); // Block since `pass + required > threshold` EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, mock_system_slot.CheckSystem(p, node, 1)->status()); EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, mock_system_slot.CheckSystem(p, node, 1)->status()); Mock::VerifyAndClearExpectations(&mock_system_slot); } // SystemRule: System Load p->insert(std::make_pair( System::MetricType::kSystemLoad, std::move(rule3))); { EXPECT_CALL(mock_system_slot, CheckSystem(_, _, _)).Times(2); EXPECT_CALL(*(static_cast(node.get())), PassQps()) .Times(testing::AnyNumber()); EXPECT_CALL(mock_system_slot, GetCurCpuUsage()).Times(testing::AnyNumber()); EXPECT_CALL(*(static_cast(node.get())), MaxCompleteQps()) .WillOnce(Return(1000)) .WillOnce(Return(2500)); EXPECT_CALL(*(static_cast(node.get())), MinRt()) .WillOnce(Return(1)) .WillOnce(Return(1)); EXPECT_CALL(mock_system_slot, GetCurLoad()) .WillOnce(Return(0.9)) .WillOnce(Return(1.1)); // Block since system load excceeds threshold EXPECT_EQ(TokenStatus::RESULT_STATUS_BLOCKED, mock_system_slot.CheckSystem(p, node, 1)->status()); EXPECT_EQ(TokenStatus::RESULT_STATUS_OK, mock_system_slot.CheckSystem(p, node, 1)->status()); Mock::VerifyAndClearExpectations(&mock_system_slot); } } } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_status_listener.cc ================================================ #include "sentinel-core/system/system_status_listener.h" #include "sentinel-core/log/logger.h" namespace Sentinel { namespace System { // Unless both of two /proc/* files read error, the listener thread should be // started SystemStatusListener::SystemStatusListener() { file_stat_ = std::ifstream("/proc/stat"); if (file_stat_.is_open()) { usage_info_p1_ = std::make_unique(); usage_info_p2_ = std::make_unique(); } else { SENTINEL_LOG(error, "[SystemStatusListener] Open /proc/stat error, cpu usage " "listener not enabled"); } file_load_ = std::ifstream("/proc/loadavg"); if (file_load_.is_open()) { load_info_p_ = std::make_unique(); } else { SENTINEL_LOG(error, "[SystemStatusListener] Open /proc/loadavg error, system load " "listener not enabled"); } } void SystemStatusListener::ReadCpuUsageFromProc() { std::string line; file_stat_.seekg(std::ios::beg); if (file_stat_.fail() || file_stat_.bad()) { SENTINEL_LOG(error, "[SystemStatusListener] Read /proc/stat error, cpu usage " "listener disabled"); // `UpdateCpuUsage` will be skipped since next listener loop usage_info_p1_.reset(); usage_info_p2_.reset(); return; } getline(file_stat_, line); int len_cpu = strlen(STR_CPU); // cpu stats line found if (line.compare(0, len_cpu, STR_CPU)) { SENTINEL_LOG( error, "[SystemStatusListener] /proc/stat format error: not started with ", STR_CPU); return; } std::istringstream ss(line); ss >> usage_info_p2_->cpu; for (int i = 0; i < NUM_CPU_STATES; ++i) { ss >> usage_info_p2_->times[i]; } } void SystemStatusListener::UpdateCpuUsage() { if (!usage_info_p1_ || !usage_info_p2_) { return; } usage_info_p1_.swap(usage_info_p2_); ReadCpuUsageFromProc(); // 100ms pause int64_t active_time = usage_info_p2_->GetActiveTime() - usage_info_p1_->GetActiveTime(); int64_t idle_time = usage_info_p2_->GetIdleTime() - usage_info_p1_->GetIdleTime(); int64_t total_time = active_time + idle_time; cur_cpu_usage_.store(1.f * active_time / total_time); } void SystemStatusListener::UpdateSystemLoad() { if (!load_info_p_) { return; } std::string line; file_load_.seekg(std::ios::beg); if (file_load_.fail() || file_load_.bad()) { SENTINEL_LOG(error, "[SystemStatusListener] Read /proc/loadavg error, system load " "listener disabled"); // `UpdateSystemLoad` will be skipped since next listener loop load_info_p_.reset(); return; } getline(file_load_, line); std::istringstream ss(line); // read times for (int i = 0; i < 3; ++i) { ss >> load_info_p_->usage[i]; } ss >> load_info_p_->curProNum; ss >> load_info_p_->curProId; cur_load_.store(load_info_p_->usage[0]); } void SystemStatusListener::RunCpuListener() { // In order to shorten the buzy waiting time in `stopListner` // we make the listener as if has stopped the loop while sleeping while (!stopped_cmd_.load()) { stopped_.store(false); UpdateCpuUsage(); UpdateSystemLoad(); stopped_.store(true); std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } stopped_.store(true); } // `Initialize()` should be called explicitly after construction void SystemStatusListener::Initialize() { bool b = false; if (inited_.compare_exchange_strong(b, true)) { if (load_info_p_ || usage_info_p1_) { stopped_cmd_.store(false); std::thread listen_task(&SystemStatusListener::RunCpuListener, this); listen_task.detach(); } } } } // namespace System } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_status_listener.h ================================================ #pragma once #include #include #include #include #include #include #include #include static constexpr const char* STR_CPU = "cpu"; namespace Sentinel { namespace System { constexpr size_t NUM_CPU_STATES = 10; enum CpuStates { S_USER = 0, S_NICE, S_SYSTEM, S_IDLE, S_IOWAIT, S_IRQ, S_SOFTIRQ, S_STEAL, S_GUEST, S_GUEST_NICE }; class CpuLoadInfo { public: double usage[3]; std::string curProNum; size_t curProId; }; class CpuUsageInfo { public: std::string cpu; int64_t times[10]; int64_t GetIdleTime() { return times[S_IDLE] + times[S_IOWAIT]; } int64_t GetActiveTime() { return times[S_USER] + times[S_NICE] + times[S_SYSTEM] + times[S_IRQ] + times[S_SOFTIRQ] + times[S_STEAL] + times[S_GUEST] + times[S_GUEST_NICE]; } }; class SystemStatusListener { public: SystemStatusListener(); static SystemStatusListener& GetInstance() { static SystemStatusListener* instance = new SystemStatusListener(); return *instance; } virtual ~SystemStatusListener() { StopListner(); file_stat_.close(); file_load_.close(); } void RunCpuListener(); void Initialize(); double GetCurLoad() { return cur_load_.load(); } double GetCurCpuUsage() { return cur_cpu_usage_.load(); } private: void ReadCpuUsageFromProc(); void UpdateCpuUsage(); void UpdateSystemLoad(); std::ifstream file_stat_, file_load_; // Two snapshots of cpu stat are required to get // usage percentage. `p2` is newer than `p1` std::unique_ptr usage_info_p1_, usage_info_p2_; std::unique_ptr load_info_p_; std::atomic cur_load_{-1}; std::atomic cur_cpu_usage_{-1}; std::atomic stopped_cmd_{false}; std::atomic stopped_{false}; std::atomic inited_{false}; // wait until loop in RunCpuListener stop void StopListner() { stopped_cmd_.store(true); while (!stopped_.load()) ; } }; } // namespace System } // namespace Sentinel ================================================ FILE: sentinel-core/system/system_status_listener_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/system/system_rule_manager.h" #include "sentinel-core/system/system_status_listener.h" namespace Sentinel { namespace System { TEST(SystemStatusListenerTest, SystemStatusListenerSingleThreadTest) { SystemStatusListener::GetInstance().Initialize(); // If the listening thread hasn't finished its first loop when code gets here, // the read-out properties still remain the initial value(-1) EXPECT_GE(SystemStatusListener::GetInstance().GetCurCpuUsage(), -1); EXPECT_GE(SystemStatusListener::GetInstance().GetCurLoad(), -1); } } // namespace System } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/common/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "mock_lib", srcs = [ "mock.h", "mock.cc", ], copts = TEST_COPTS, deps = [ "//sentinel-core/common:rule_lib", "@com_google_googletest//:gtest", ] ) ================================================ FILE: sentinel-core/test/mock/common/mock.cc ================================================ ================================================ FILE: sentinel-core/test/mock/common/mock.h ================================================ #pragma once #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/common/rule.h" namespace Sentinel { class FakeRule : public Rule { public: FakeRule() = default; void set_fake_rule_data(const std::string& data) { fake_rule_data_ = data; } const std::string& fake_rule_data() const { return fake_rule_data_; } bool operator==(const FakeRule& other) const { return fake_rule_data_ == other.fake_rule_data(); } private: std::string fake_rule_data_; }; } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/flow/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "flow_mock_lib", srcs = [ "mock.h", "mock.cc", ], copts = TEST_COPTS, deps = [ "//sentinel-core/flow:traffic_shaping_calculator_interface", "//sentinel-core/flow:traffic_shaping_checker_interface", "@com_google_googletest//:gtest", ] ) ================================================ FILE: sentinel-core/test/mock/flow/mock.cc ================================================ #include "sentinel-core/test/mock/flow/mock.h" namespace Sentinel { namespace Flow { MockTrafficShapingChecker::MockTrafficShapingChecker() = default; MockTrafficShapingChecker::~MockTrafficShapingChecker() = default; MockTrafficShapingCalculator::MockTrafficShapingCalculator() = default; MockTrafficShapingCalculator::~MockTrafficShapingCalculator() = default; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/flow/mock.h ================================================ #pragma once #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/flow/traffic_shaping_calculator.h" #include "sentinel-core/flow/traffic_shaping_checker.h" namespace Sentinel { namespace Flow { class AlwaysPassChecker : public TrafficShapingChecker { public: AlwaysPassChecker() = default; ~AlwaysPassChecker() = default; Slot::TokenResultSharedPtr DoCheck(const Stat::NodeSharedPtr& node, int acquire_count, double threshold) { return Slot::TokenResult::Ok(); } }; class AlwaysBlockChecker : public TrafficShapingChecker { public: AlwaysBlockChecker() = default; ~AlwaysBlockChecker() = default; Slot::TokenResultSharedPtr DoCheck(const Stat::NodeSharedPtr& node, int acquire_count, double threshold) { return Slot::TokenResult::Blocked("block"); } }; class MockTrafficShapingChecker : public TrafficShapingChecker { public: MockTrafficShapingChecker(); ~MockTrafficShapingChecker(); MOCK_METHOD3(DoCheck, Slot::TokenResultSharedPtr(const Stat::NodeSharedPtr&, int, double)); }; class MockTrafficShapingCalculator : public TrafficShapingCalculator { public: MockTrafficShapingCalculator(); ~MockTrafficShapingCalculator(); MOCK_METHOD3(CalculateAllowedTokens, double(const Stat::NodeSharedPtr&, int, int)); }; } // namespace Flow } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/init/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "mock_lib", srcs = [ "mock.h", ], copts = TEST_COPTS, deps = [ "//sentinel-core/init:init_target_interface", "@com_google_googletest//:gtest", ] ) ================================================ FILE: sentinel-core/test/mock/init/mock.h ================================================ #pragma once #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/init/init_target.h" namespace Sentinel { namespace Init { class FakeInitTarget : public Target { public: FakeInitTarget() = default; ~FakeInitTarget() = default; void Initialize() { count_ = 1; } int count() const { return count_; } private: int count_; }; class MockInitTarget : public Target { public: MockInitTarget() = default; ~MockInitTarget() = default; MOCK_METHOD0(Initialize, void()); }; } // namespace Init } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/property/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "mock_lib", srcs = [ "mock.h", "mock.cc", ], copts = TEST_COPTS, deps = [ "//sentinel-core/property:property_listener_interface", "@com_google_googletest//:gtest", ] ) ================================================ FILE: sentinel-core/test/mock/property/mock.cc ================================================ #include "sentinel-core/test/mock/property/mock.h" namespace Sentinel { namespace Property {} // namespace Property } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/property/mock.h ================================================ #pragma once #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/property/property_listener.h" namespace Sentinel { namespace Property { template class MockPropertyListener : public PropertyListener { public: MockPropertyListener() = default; ~MockPropertyListener() = default; MOCK_METHOD2_T(ConfigUpdate, void(const T&, bool)); const std::string Name() const override { return "MockPropertyListener"; } }; } // namespace Property } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/slot/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "mock_lib", srcs = [ "mock.h", "mock.cc", ], copts = TEST_COPTS, deps = [ "//sentinel-core/slot/base:rule_checker_slot_interface", "//sentinel-core/slot/base:stats_slot_interface", "//sentinel-core/slot/base:slot_interface", "//sentinel-core/slot/base:default_slot_chain_impl_lib", "@com_google_googletest//:gtest", ] ) ================================================ FILE: sentinel-core/test/mock/slot/mock.cc ================================================ #include "sentinel-core/test/mock/slot/mock.h" namespace Sentinel { namespace Slot { MockRuleCheckerSlot::MockRuleCheckerSlot() = default; MockRuleCheckerSlot::~MockRuleCheckerSlot() = default; MockStatsSlot::MockStatsSlot() = default; MockStatsSlot::~MockStatsSlot() = default; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/slot/mock.h ================================================ #pragma once #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/slot/base/rule_checker_slot.h" #include "sentinel-core/slot/base/stats_slot.h" #include "sentinel-core/slot/base/token_result.h" namespace Sentinel { namespace Slot { class MockRuleCheckerSlot : public RuleCheckerSlot { public: MockRuleCheckerSlot(); ~MockRuleCheckerSlot(); MOCK_METHOD5(Entry, TokenResultSharedPtr(const EntrySharedPtr &, Stat::NodeSharedPtr &, int, int, const std::vector &)); MOCK_METHOD3(Exit, void(const EntrySharedPtr &, int, const std::vector &)); MOCK_CONST_METHOD0(Name, const std::string &(void)); }; class MockStatsSlot : public StatsSlot { public: MockStatsSlot(); ~MockStatsSlot(); MOCK_METHOD5(Entry, TokenResultSharedPtr(const EntrySharedPtr &, Stat::NodeSharedPtr &, int, int, const std::vector &)); MOCK_METHOD3(Exit, void(const EntrySharedPtr &, int, const std::vector &)); MOCK_CONST_METHOD0(Name, const std::string &(void)); }; } // namespace Slot } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/statistic/base/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "mock_lib", srcs = [ "mock.h", "mock.cc", ], copts = TEST_COPTS, deps = [ "//sentinel-core/statistic/base:sliding_window_metric_lib", "@com_google_googletest//:gtest", ] ) ================================================ FILE: sentinel-core/test/mock/statistic/base/mock.cc ================================================ #include "sentinel-core/test/mock/statistic/base/mock.h" namespace Sentinel { namespace Stat {} // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/statistic/base/mock.h ================================================ #pragma once #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/statistic/base/bucket_leap_array.h" namespace Sentinel { namespace Stat {} // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/statistic/node/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "mock_lib", srcs = [ "mock.h", "mock.cc", ], copts = TEST_COPTS, deps = [ "//sentinel-core/statistic/node:node_interface", "@com_google_googletest//:gtest", ] ) ================================================ FILE: sentinel-core/test/mock/statistic/node/mock.cc ================================================ #include "sentinel-core/test/mock/statistic/node/mock.h" namespace Sentinel { namespace Stat { MockNode::MockNode() = default; MockNode::~MockNode() = default; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/test/mock/statistic/node/mock.h ================================================ #pragma once #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/statistic/node/node.h" namespace Sentinel { namespace Stat { class MockNode : public Node { public: MockNode(); ~MockNode(); MOCK_METHOD0(TotalCountInMinute, int64_t(void)); MOCK_METHOD0(PassCountInMinute, int64_t(void)); MOCK_METHOD0(CompleteCountInMinute, int64_t(void)); MOCK_METHOD0(BlockCountInMinute, int64_t(void)); MOCK_METHOD0(ExceptionCountInMinute, int64_t(void)); MOCK_METHOD0(PassQps, double(void)); MOCK_METHOD0(BlockQps, double(void)); MOCK_METHOD0(TotalQps, double(void)); MOCK_METHOD0(CompleteQps, double(void)); MOCK_METHOD0(MaxCompleteQps, double(void)); MOCK_METHOD0(ExceptionQps, double(void)); MOCK_METHOD0(AvgRt, double(void)); MOCK_METHOD0(MinRt, double(void)); MOCK_CONST_METHOD0(CurThreadNum, uint32_t(void)); MOCK_METHOD0(Metrics, std::unordered_map(void)); MOCK_METHOD0(PreviousBlockQps, double(void)); MOCK_METHOD0(PreviousPassQps, double(void)); MOCK_METHOD1(AddPassRequest, void(int32_t)); MOCK_METHOD2(AddRtAndCompleteRequest, void(int32_t, int)); MOCK_METHOD1(AddBlockRequest, void(int32_t)); MOCK_METHOD1(AddExceptionRequest, void(int32_t)); MOCK_METHOD0(IncreaseThreadNum, void(void)); MOCK_METHOD0(DecreaseThreadNum, void(void)); MOCK_METHOD0(Reset, void(void)); }; } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/transport/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "transport_constants", copts = DEFAULT_COPTS, srcs = [ "constants.h", ], visibility = ["//visibility:public"], ) ================================================ FILE: sentinel-core/transport/command/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "command_request_lib", copts = DEFAULT_COPTS, srcs = [ "command_request.h", "command_request.cc", ], deps = [], ) cc_library( name = "command_response_lib", copts = DEFAULT_COPTS, srcs = [ "command_response.h", ], deps = [], ) cc_library( name = "command_handler_interface", copts = DEFAULT_COPTS, srcs = [ "command_handler.h", ], deps = [ ":command_request_lib", ":command_response_lib", ], ) cc_library( name = "http_command_utils_lib", copts = DEFAULT_COPTS, srcs = [ "http_command_utils.h", "http_command_utils.cc", ], deps = [ "//:libevent", ":command_request_lib", ], ) cc_library( name = "transport_httpserver_lib", copts = DEFAULT_COPTS, srcs = [ "http_server.h", "http_server.cc", ], deps = [ "//:libevent", "//sentinel-core/log:logger_lib", "//sentinel-core/transport/common:transport_common", ], ) cc_library( name = "http_command_center", copts = DEFAULT_COPTS, srcs = [ "http_command_center.h", "http_command_center.cc", ], deps = [ ":command_handler_interface", ":transport_httpserver_lib", ":http_command_utils_lib", "//:libevent", "//sentinel-core/transport/common:transport_common", ], ) cc_library( name = "http_command_center_init_target", copts = DEFAULT_COPTS, srcs = [ "http_server_init_target.h", "http_server_init_target.cc", ], deps = [ ":http_command_center", "//sentinel-core/init:init_target_interface", "//sentinel-core/transport:transport_constants", "//sentinel-core/transport/command/handler:fetch_metric_log_handler_lib", "//sentinel-core/transport/command/handler:fetch_cluster_node_handler_lib", "//sentinel-core/transport/command/handler:sentinel_on_off_switch_handler_lib", "//sentinel-core/transport/command/handler:version_command_handler_lib", "@com_google_absl//absl/strings", ], ) cc_test( name = "command_request_unittests", srcs = [ "command_request_test.cc", ], copts = TEST_COPTS, deps = [ ":command_request_lib", "@com_google_googletest//:gtest_main", ], linkstatic = 1, ) ================================================ FILE: sentinel-core/transport/command/command_handler.h ================================================ #pragma once #include #include "sentinel-core/transport/command/command_request.h" #include "sentinel-core/transport/command/command_response.h" namespace Sentinel { namespace Transport { class CommandHandler { public: CommandHandler(const std::string& name) : command_name_(name) {} virtual ~CommandHandler() = default; virtual CommandResponsePtr Handle(const CommandRequest& request) = 0; const std::string& command_name() const { return command_name_; } protected: std::string command_name_; }; using CommandHandlerPtr = std::unique_ptr; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/command_request.cc ================================================ #include "sentinel-core/transport/command/command_request.h" namespace Sentinel { namespace Transport { const std::string& CommandRequest::body() const { return body_; } CommandRequest& CommandRequest::set_body(const std::string& body) { body_ = body; return *this; } const std::unordered_map& CommandRequest::GetParameters() const { return parameters_; } std::string CommandRequest::GetParam(const std::string& key) const { auto it = parameters_.find(key); if (it == parameters_.end()) { return std::string(""); } return it->second; } std::string CommandRequest::GetParam(const std::string& key, const std::string& defaultValue) const { auto it = parameters_.find(key); if (it == parameters_.end()) { return defaultValue; } return it->second; } CommandRequest& CommandRequest::AddParam(const std::string& key, const std::string& value) { if (key.size() == 0) { // logerror } else { parameters_[key] = value; } return *this; } CommandRequest& CommandRequest::AddMetadata(const std::string& key, const std::string& value) { if (key.size() == 0) { // logerror } else { metadata_[key] = value; } return *this; } std::string CommandRequest::GetMetadata(const std::string& key) const { auto it = metadata_.find(key); if (it == metadata_.end()) { return std::string(""); } return it->second; } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/command_request.h ================================================ #pragma once #include #include namespace Sentinel { namespace Transport { class CommandRequest { public: const std::string& body() const; CommandRequest& set_body(const std::string& body); CommandRequest& AddParam(const std::string& key, const std::string& value); std::string GetParam(const std::string& key) const; std::string GetParam(const std::string& key, const std::string& defaultValue) const; const std::unordered_map& GetParameters() const; CommandRequest& AddMetadata(const std::string& key, const std::string& value); std::string GetMetadata(const std::string& key) const; private: std::unordered_map metadata_; std::unordered_map parameters_; std::string body_; }; constexpr char kRequestTarget[] = "command-target"; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/command_request_test.cc ================================================ #include "gmock/gmock.h" #include "gtest/gtest.h" #define private public #include "sentinel-core/transport/command/command_request.h" namespace Sentinel { namespace Transport { TEST(CommandRequestTest, TestBody) { CommandRequest request; request.set_body("testbody"); auto ret = request.body(); EXPECT_EQ(ret, "testbody"); } TEST(CommandRequestTest, TestParam) { CommandRequest request; request.AddParam("key1", "val1"); request.AddParam("key2", "val2"); auto v1 = request.GetParam("key1"); EXPECT_EQ(v1, "val1"); auto v2 = request.GetParam("key2"); EXPECT_EQ(v2, "val2"); auto v3 = request.GetParam("key3"); EXPECT_EQ(v3, ""); auto v4 = request.GetParam("key4", "defaultVal"); EXPECT_EQ(v4, "defaultVal"); } TEST(CommandRequestTest, TestMetadata) { CommandRequest request; request.AddMetadata("key1", "val1"); request.AddMetadata("key2", "val2"); auto v1 = request.GetMetadata("key1"); EXPECT_EQ(v1, "val1"); auto v2 = request.GetMetadata("key2"); EXPECT_EQ(v2, "val2"); auto v3 = request.GetMetadata("key3"); EXPECT_EQ(v3, ""); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/command_response.h ================================================ #pragma once #include #include namespace Sentinel { namespace Transport { class CommandResponse; using CommandResponseSharedPtr = std::shared_ptr; using CommandResponsePtr = std::unique_ptr; class CommandResponse { public: CommandResponse(bool success, const std::string& result) : success_(success), result_(result) {} static CommandResponsePtr OfSuccess(const std::string& result) { return std::make_unique(true, result); } static CommandResponsePtr OfFailure(const std::string& result) { return std::make_unique(false, result); } bool success() const { return success_; } const std::string& result() const { return result_; } private: bool success_; std::string result_; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "fetch_metric_log_handler_lib", srcs = [ "fetch_metric_log_handler.h", "fetch_metric_log_handler.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/log/metric:metric_log_lib", "//sentinel-core/transport/command:command_handler_interface", ] ) cc_library( name = "fetch_cluster_node_handler_lib", srcs = [ "fetch_cluster_node_handler.h", "fetch_cluster_node_handler.cc", ], copts = DEFAULT_COPTS, deps = [ "//third_party/nlohmann:nlohmann_json_lib", "//sentinel-core/statistic/node:resource_node_storage_lib", "//sentinel-core/transport/command:command_handler_interface", "//sentinel-core/transport/command/handler/vo:statistic_node_vo_lib", ] ) cc_library( name = "sentinel_on_off_switch_handler_lib", srcs = [ "get_switch_status_handler.h", "get_switch_status_handler.cc", "set_switch_status_handler.h", "set_switch_status_handler.cc", ], copts = DEFAULT_COPTS, deps = [ "@com_google_absl//absl/strings:str_format", "//sentinel-core/common:global_status", "//sentinel-core/transport/command:command_handler_interface", ] ) cc_library( name = "version_command_handler_lib", srcs = [ "version_handler.h", "version_handler.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/transport/command:command_handler_interface", ] ) cc_test( name = "sentinel_on_off_switch_handler_unittests", srcs = [ "set_switch_status_handler_test.cc", ], copts = TEST_COPTS, deps = [ ":sentinel_on_off_switch_handler_lib", "//sentinel-core/common:global_status", "@com_google_googletest//:gtest_main", ], ) ================================================ FILE: sentinel-core/transport/command/handler/fetch_cluster_node_handler.cc ================================================ #include #include #include #include "third_party/nlohmann/json.hpp" #include "sentinel-core/statistic/node/resource_node_storage.h" #include "sentinel-core/transport/command/handler/fetch_cluster_node_handler.h" #include "sentinel-core/transport/command/handler/vo/statistic_node_vo.h" namespace Sentinel { namespace Transport { nlohmann::json ConvertNodeVoToJson( const std::shared_ptr& node) { return nlohmann::json{{"id", node->id()}, {"parentId", node->parent_id()}, {"resource", node->resource()}, {"threadNum", node->thread_num()}, {"passQps", node->pass_qps()}, {"blockQps", node->block_qps()}, {"totalQps", node->total_qps()}, {"averageRt", node->avg_rt()}, {"successQps", node->complete_qps()}, {"exceptionQps", node->exception_qps()}, {"oneMinutePass", node->pass_per_min()}, {"oneMinuteBlock", node->block_per_min()}, {"oneMinuteException", node->exception_per_min()}, {"oneMinuteTotal", node->total_per_min()}, {"timestamp", node->timestamp()}}; } CommandResponsePtr FetchClusterNodeCommandHandler::Handle( const CommandRequest& request) { std::string type = request.GetParam("type"); std::vector> vec; const auto& node_map = Stat::ResourceNodeStorage::GetInstance().GetNodeMap(); for (const auto& e : node_map) { if (type == "notZero") { auto node = e.second; if (node != nullptr && node->TotalCountInMinute() > 0) { vec.emplace_back(StatisticNodeVO::FromResourceNode(e.first, node)); } } else { auto vo = StatisticNodeVO::FromResourceNode(e.first, e.second); if (vo != nullptr) { vec.emplace_back(std::move(vo)); } } } auto j = nlohmann::json::array(); for (const auto& v : vec) { j.emplace_back(ConvertNodeVoToJson(v)); } return CommandResponse::OfSuccess(j.dump()); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/fetch_cluster_node_handler.h ================================================ #pragma once #include #include "sentinel-core/transport/command/command_handler.h" namespace Sentinel { namespace Transport { class FetchClusterNodeCommandHandler : public CommandHandler { public: FetchClusterNodeCommandHandler() : CommandHandler("clusterNode") {} virtual ~FetchClusterNodeCommandHandler() = default; CommandResponsePtr Handle(const CommandRequest& request) override; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/fetch_cluster_node_handler_test.cc ================================================ #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-core/transport/command/handler/fetch_cluster_node_handler.h" using testing::_; using testing::InSequence; using testing::Return; namespace Sentinel { namespace Stat { TEST(FetchClusterNodeHandlerTest, TestBasic) {} } // namespace Stat } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/fetch_metric_log_handler.cc ================================================ #include #include #include "absl/strings/numbers.h" #include "sentinel-core/statistic/base/metric_item.h" #include "sentinel-core/transport/command/handler/fetch_metric_log_handler.h" namespace Sentinel { namespace Transport { namespace { constexpr auto kStartTimeKey = "startTime"; constexpr auto kEndTimeKey = "endTime"; constexpr auto kMaxLinesKey = "maxLines"; constexpr auto kResourceKey = "identity"; constexpr int32_t kMaxLines = 12000; } // namespace int64_t ParseOrDefault(const std::string& s, int64_t default_value) { int64_t x; if (!absl::SimpleAtoi(s, &x)) { x = default_value; } return x; } CommandResponsePtr FetchMetricLogCommandHandler::Handle( const CommandRequest& request) { std::string start_time_str = request.GetParam(kStartTimeKey); std::string end_time_str = request.GetParam(kEndTimeKey); std::string max_lines_str = request.GetParam(kMaxLinesKey); std::string identity = request.GetParam(kResourceKey); if (start_time_str.empty()) { return CommandResponse::OfSuccess(""); } std::vector vec; int64_t start_time; if (!absl::SimpleAtoi(start_time_str, &start_time)) { return CommandResponse::OfFailure("Invalid start_time"); } if (!end_time_str.empty()) { // Find by end time if set. int64_t end_time; if (!absl::SimpleAtoi(end_time_str, &end_time)) { return CommandResponse::OfFailure("Invalid end_time"); } vec = searcher_->FindByTimeAndResource(start_time, end_time, identity); } else { int32_t max_lines; if (!max_lines_str.empty()) { if (!absl::SimpleAtoi(max_lines_str, &max_lines)) { return CommandResponse::OfFailure("Invalid max_lines"); } max_lines = std::max(max_lines, kMaxLines); vec = searcher_->Find(start_time, max_lines); } } std::string result(""); for (auto& item : vec) { result += item->ToThinString(); result += '\n'; } return CommandResponse::OfSuccess(result); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/fetch_metric_log_handler.h ================================================ #pragma once #include #include #include "sentinel-core/config/local_config.h" #include "sentinel-core/log/log_base.h" #include "sentinel-core/log/metric/metric_searcher.h" #include "sentinel-core/log/metric/metric_writer.h" #include "sentinel-core/transport/command/command_handler.h" namespace Sentinel { namespace Transport { class FetchMetricLogCommandHandler : public CommandHandler { public: FetchMetricLogCommandHandler() : CommandHandler("metric") { const std::string& app_name = Config::LocalConfig::GetInstance().app_name(); searcher_ = std::make_unique( Log::LogBase::GetLogBaseDir(), Log::MetricWriter::FormSelfMetricFileName(app_name)); } virtual ~FetchMetricLogCommandHandler() = default; CommandResponsePtr Handle(const CommandRequest& request) override; private: std::unique_ptr searcher_; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/get_switch_status_handler.cc ================================================ #include "sentinel-core/transport/command/handler/get_switch_status_handler.h" #include #include "absl/strings/str_format.h" #include "sentinel-core/common/global_status.h" namespace Sentinel { namespace Transport { CommandResponsePtr GetSwitchStatusCommandHandler::Handle( const CommandRequest&) { const char* status = GlobalStatus::activated ? "enabled" : "disabled"; return CommandResponse::OfSuccess( absl::StrFormat("Sentinel status: %s", status)); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/get_switch_status_handler.h ================================================ #pragma once #include #include "sentinel-core/transport/command/command_handler.h" namespace Sentinel { namespace Transport { class GetSwitchStatusCommandHandler : public CommandHandler { public: GetSwitchStatusCommandHandler() : CommandHandler("getSwitch") {} virtual ~GetSwitchStatusCommandHandler() = default; CommandResponsePtr Handle(const CommandRequest& request) override; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/set_switch_status_handler.cc ================================================ #include "sentinel-core/transport/command/handler/set_switch_status_handler.h" #include "sentinel-core/common/global_status.h" namespace Sentinel { namespace Transport { CommandResponsePtr SetSwitchStatusCommandHandler::Handle( const CommandRequest& request) { auto v = request.GetParam("value"); if (v == "true") { GlobalStatus::activated = true; // SENTINEL_LOG(info, "[SwitchOnOffHandler] Sentinel has been activated"); return CommandResponse::OfSuccess("Sentinel has been enabled"); } if (v == "false") { GlobalStatus::activated = false; // SENTINEL_LOG(info, "[SwitchOnOffHandler] Sentinel has been disabled"); return CommandResponse::OfSuccess("Sentinel has been disabled"); } return CommandResponse::OfFailure("bad new status"); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/set_switch_status_handler.h ================================================ #pragma once #include #include "sentinel-core/transport/command/command_handler.h" namespace Sentinel { namespace Transport { class SetSwitchStatusCommandHandler : public CommandHandler { public: SetSwitchStatusCommandHandler() : CommandHandler("setSwitch") {} virtual ~SetSwitchStatusCommandHandler() = default; CommandResponsePtr Handle(const CommandRequest& request) override; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/set_switch_status_handler_test.cc ================================================ #include #include "sentinel-core/common/global_status.h" #include "sentinel-core/transport/command/handler/set_switch_status_handler.h" #include "gmock/gmock.h" #include "gtest/gtest.h" namespace Sentinel { namespace Transport { TEST(SetSwitchStatusCommandHandlerTest, TestHandleRequest) { SetSwitchStatusCommandHandler handler; CommandRequest request; EXPECT_FALSE(handler.Handle(request)->success()); EXPECT_TRUE(GlobalStatus::activated); request.AddParam("value", "false"); EXPECT_TRUE(handler.Handle(request)->success()); EXPECT_FALSE(GlobalStatus::activated); request.AddParam("value", "true"); EXPECT_TRUE(handler.Handle(request)->success()); EXPECT_TRUE(GlobalStatus::activated); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/version_handler.cc ================================================ #include "sentinel-core/transport/command/handler/version_handler.h" #include namespace Sentinel { namespace Transport { CommandResponsePtr VersionCommandHandler::Handle( const CommandRequest& request) { return CommandResponse::OfSuccess("0.1.0"); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/version_handler.h ================================================ #pragma once #include #include "sentinel-core/transport/command/command_handler.h" namespace Sentinel { namespace Transport { class VersionCommandHandler : public CommandHandler { public: VersionCommandHandler() : CommandHandler("version") {} virtual ~VersionCommandHandler() = default; CommandResponsePtr Handle(const CommandRequest& request) override; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/vo/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "statistic_node_vo_lib", srcs = [ "statistic_node_vo.h", "statistic_node_vo.cc", ], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/statistic/node:cluster_node_lib", ] ) ================================================ FILE: sentinel-core/transport/command/handler/vo/statistic_node_vo.cc ================================================ #include #include #include "sentinel-core/transport/command/handler/vo/statistic_node_vo.h" namespace Sentinel { namespace Transport { std::shared_ptr StatisticNodeVO::FromResourceNode( const std::string& name, const Stat::ClusterNodeSharedPtr& node) { if (node == nullptr) { return nullptr; } std::shared_ptr vo = std::make_shared(); vo->set_resource(name); vo->set_thread_num(node->CurThreadNum()); vo->set_pass_qps(node->PassQps()); vo->set_block_qps(node->BlockQps()); vo->set_total_qps(node->TotalQps()); vo->set_avg_rt(node->AvgRt()); vo->set_complete_qps(node->CompleteQps()); vo->set_exception_qps(node->ExceptionQps()); vo->set_pass_per_min(node->PassCountInMinute()); vo->set_block_per_min(node->BlockCountInMinute()); vo->set_total_per_min(node->TotalCountInMinute()); vo->set_exception_per_min(node->ExceptionCountInMinute()); vo->set_timestamp(Utils::TimeUtils::CurrentTimeMillis().count()); return vo; } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/handler/vo/statistic_node_vo.h ================================================ #include #include #include "sentinel-core/statistic/node/cluster_node.h" namespace Sentinel { namespace Transport { class StatisticNodeVO { public: StatisticNodeVO() = default; ~StatisticNodeVO() = default; static std::shared_ptr FromResourceNode( const std::string& name, const Stat::ClusterNodeSharedPtr& node); const std::string& id() const { return id_; } const std::string& parent_id() const { return parent_id_; } const std::string& resource() const { return resource_; }; int32_t thread_num() const { return thread_num_; } int64_t pass_qps() const { return pass_qps_; } int64_t block_qps() const { return block_qps_; } int64_t total_qps() const { return total_qps_; } int64_t complete_qps() const { return complete_qps_; } int64_t avg_rt() const { return avg_rt_; } int64_t exception_qps() const { return exception_qps_; } int64_t pass_per_min() const { return pass_per_min_; } int64_t block_per_min() const { return block_per_min_; } int64_t exception_per_min() const { return exception_per_min_; } int64_t total_per_min() const { return total_per_min_; } int64_t timestamp() const { return timestamp_; } void set_id(const std::string& id) { id_ = id; } void set_parent_id(const std::string& id) { parent_id_ = id; } void set_resource(const std::string& resource) { resource_ = resource; } void set_thread_num(int32_t v) { thread_num_ = v; } void set_pass_qps(int64_t v) { pass_qps_ = v; } void set_block_qps(int64_t v) { block_qps_ = v; } void set_total_qps(int64_t v) { total_qps_ = v; } void set_complete_qps(int64_t v) { complete_qps_ = v; } void set_avg_rt(int64_t v) { avg_rt_ = v; } void set_exception_qps(int64_t v) { exception_qps_ = v; } void set_pass_per_min(int64_t v) { pass_per_min_ = v; } void set_block_per_min(int64_t v) { block_per_min_ = v; } void set_exception_per_min(int64_t v) { exception_per_min_ = v; } void set_total_per_min(int64_t v) { total_per_min_ = v; } void set_timestamp(int64_t v) { timestamp_ = v; } private: std::string id_; std::string parent_id_; std::string resource_; int32_t thread_num_; int64_t pass_qps_; int64_t block_qps_; int64_t total_qps_; int64_t complete_qps_; int64_t avg_rt_; int64_t exception_qps_; int64_t pass_per_min_; int64_t block_per_min_; int64_t exception_per_min_; int64_t total_per_min_; int64_t timestamp_; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/http_command_center.cc ================================================ #include "sentinel-core/transport/command/http_command_center.h" #include #include #include #include "sentinel-core/transport/command/http_command_utils.h" namespace Sentinel { namespace Transport { HttpCommandCenter::~HttpCommandCenter() { Stop(); } bool HttpCommandCenter::Start(int port) { if (http_server_) { // log warn return false; } http_server_ = std::make_unique(std::bind( &HttpCommandCenter::OnHttpRequest, this, std::placeholders::_1)); return http_server_->Start(port); } void HttpCommandCenter::Stop() { if (http_server_) { http_server_->Stop(); } } bool HttpCommandCenter::RegisterCommand(CommandHandlerPtr&& handler) { if (handler == nullptr) { return false; } auto command_name = handler->command_name(); if (handler_map_.find(command_name) != handler_map_.end()) { // log warn return false; } handler_map_.emplace(std::make_pair(command_name, std::move(handler))); return true; } void HttpCommandCenter::OnHttpRequest(struct evhttp_request* http_req) { auto request = HttpCommandUtils::ParseHttpRequest(http_req); auto target = request.GetMetadata(kRequestTarget); auto it = handler_map_.find(target); if (it == handler_map_.end()) { HandleResponse(http_req, CommandResponse::OfFailure("Unknown command: " + target)); return; } auto response = it->second->Handle(request); HandleResponse(http_req, std::move(response)); return; } void HttpCommandCenter::HandleResponse(struct evhttp_request* http_req, CommandResponsePtr&& response) { if (response->success()) { HttpCommandUtils::SucessRequest(http_req, response->result()); } else { HttpCommandUtils::BadRequest(http_req, response->result()); } } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/http_command_center.h ================================================ #pragma once #include #include #include "sentinel-core/transport/command/command_handler.h" #include "sentinel-core/transport/command/command_response.h" #include "sentinel-core/transport/command/http_server.h" namespace Sentinel { namespace Transport { class HttpCommandCenter { public: explicit HttpCommandCenter() = default; ~HttpCommandCenter(); bool Start(int port); void Stop(); bool RegisterCommand(CommandHandlerPtr&& handler); private: void OnHttpRequest(struct evhttp_request* http_req); void HandleResponse(struct evhttp_request* http_req, CommandResponsePtr&& response); private: std::unique_ptr http_server_; std::unordered_map handler_map_; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/http_command_utils.cc ================================================ #include "sentinel-core/transport/command/http_command_utils.h" #include #include #include #include #include #include namespace Sentinel { namespace Transport { void HttpCommandUtils::SucessRequest(struct evhttp_request *http_req, const std::string &msg) { auto evb = evbuffer_new(); evbuffer_add_printf(evb, msg.c_str()); evhttp_send_reply(http_req, 200, "OK", evb); evbuffer_free(evb); } void HttpCommandUtils::BadRequest(struct evhttp_request *http_req, const std::string &msg) { auto evb = evbuffer_new(); evbuffer_add_printf(evb, msg.c_str()); evhttp_send_reply(http_req, 400, "Bad Request", evb); evbuffer_free(evb); } void HttpCommandUtils::InternalError(struct evhttp_request *http_req, const std::string &msg) { auto evb = evbuffer_new(); evbuffer_add_printf(evb, msg.c_str()); evhttp_send_reply(http_req, 500, "Internal Server Error", evb); evbuffer_free(evb); } CommandRequest HttpCommandUtils::ParseHttpRequest( struct evhttp_request *http_req) { std::string uri = evhttp_request_uri(http_req); // std::string decoded_uri = evhttp_decode_uri(uri.c_str()); CommandRequest request; ParseRequestUri(uri, &request); // parse post auto cmd_type = evhttp_request_get_command(http_req); if (cmd_type == EVHTTP_REQ_POST) { ParsePostData(http_req, &request); } return request; } void HttpCommandUtils::ParseRequestUri(const std::string &uri, CommandRequest *request) { request->set_body(uri); // parse target auto start = uri.find('/'); auto ask = uri.find('?') == std::string::npos ? uri.find_last_of(' ') : uri.find('?'); auto target = uri.substr(start != std::string::npos ? start + 1 : 0, ask != std::string::npos ? ask - 1 : uri.length()); request->AddMetadata(kRequestTarget, target); // parse params struct evkeyvalq params; evhttp_parse_query(uri.c_str(), ¶ms); struct evkeyval *param; for (param = params.tqh_first; param; param = param->next.tqe_next) { auto key = std::string(param->key); auto value = std::string(param->value); request->AddParam(key, value); } } void HttpCommandUtils::ParsePostData( struct evhttp_request *http_req, Sentinel::Transport::CommandRequest *request) { auto data_len = EVBUFFER_LENGTH(http_req->input_buffer); std::vector post_data; memcpy(post_data.data(), EVBUFFER_DATA(http_req->input_buffer), data_len); ParseRequestUri("/?%" + std::string(post_data.begin(), post_data.end()), request); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/http_command_utils.h ================================================ #pragma once #include #include #include #include #include "sentinel-core/transport/command/command_request.h" namespace Sentinel { namespace Transport { class HttpCommandUtils { public: HttpCommandUtils() = delete; static void SucessRequest(struct evhttp_request *http_req, const std::string &msg); static void BadRequest(struct evhttp_request *http_req, const std::string &msg); static void InternalError(struct evhttp_request *http_req, const std::string &msg); static CommandRequest ParseHttpRequest(struct evhttp_request *http_req); private: static void ParseRequestUri(const std::string &uri, CommandRequest *request); static void ParsePostData(struct evhttp_request *http_req, Sentinel::Transport::CommandRequest *request); }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/http_server.cc ================================================ #include "sentinel-core/transport/command/http_server.h" #include "sentinel-core/log/logger.h" #ifndef WIN32 #include #endif // !WIN32 namespace Sentinel { namespace Transport { HttpServer::HttpServer(http_request_callback_t callback) : request_callback_(callback) {} HttpServer::~HttpServer() { if (http_) { evhttp_free(http_); http_ = nullptr; } } bool HttpServer::Start(int port) { auto ret = event_loop_thread_.Start(); if (!ret) { return false; } port_ = port; std::promise start_promise; auto start_future = start_promise.get_future(); auto task = [&start_promise, this]() { InternalStart(start_promise); }; event_loop_thread_.RunTask(task); return start_future.get(); } void HttpServer::Stop() { event_loop_thread_.Stop(); if (http_) { evhttp_free(http_); http_ = nullptr; } } void HttpServer::InternalStart(std::promise &promise) { http_ = evhttp_new(event_loop_thread_.GetEventBase()); if (!http_) { SENTINEL_LOG(error, "HttpServer evhttp_new() failed, pending port={}", port_); promise.set_value(false); return; } evhttp_set_gencb(http_, &HttpServer::HttpGenCallback, this); auto handle = evhttp_bind_socket_with_handle(http_, "0.0.0.0", port_); if (!handle) { SENTINEL_LOG( error, "HttpServer evhttp_bind_socket_with_handle() failed, pending port={}", port_); promise.set_value(false); return; } promise.set_value(true); SENTINEL_LOG(info, "HttpServer is running at port {}", port_); return; } void HttpServer::HttpGenCallback(struct evhttp_request *req, void *arg) { HttpServer *server = static_cast(arg); server->OnHttpRequest(req); } void HttpServer::OnHttpRequest(struct evhttp_request *req) { request_callback_(req); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/http_server.h ================================================ #pragma once #include #include #include //#include //#include #include #include "sentinel-core/transport/common/event_loop_thread.h" namespace Sentinel { namespace Transport { void HttpCallback(struct evhttp_request *req, void *arg); using http_request_callback_t = std::function; class HttpServer { public: HttpServer(http_request_callback_t callback); ~HttpServer(); bool Start(int port); void Stop(); private: void InternalStart(std::promise &promise); void OnHttpRequest(struct evhttp_request *req); private: EventLoopThread event_loop_thread_; struct evhttp *http_ = nullptr; http_request_callback_t request_callback_; static void HttpGenCallback(struct evhttp_request *req, void *arg); private: int port_; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/http_server_init_target.cc ================================================ #include #include #include "absl/strings/numbers.h" #include "sentinel-core/transport/command/handler/fetch_cluster_node_handler.h" #include "sentinel-core/transport/command/handler/fetch_metric_log_handler.h" #include "sentinel-core/transport/command/handler/get_switch_status_handler.h" #include "sentinel-core/transport/command/handler/set_switch_status_handler.h" #include "sentinel-core/transport/command/handler/version_handler.h" #include "sentinel-core/transport/command/http_server_init_target.h" #include "sentinel-core/transport/constants.h" namespace Sentinel { namespace Transport { void HttpCommandCenterInitTarget::Close() { if (!closed_ && command_center_ != nullptr) { // Not thread-safe closed_ = true; command_center_->Stop(); } } uint32_t HttpCommandCenterInitTarget::GetAvailablePort() { const char* command_port_env = std::getenv(Constants::kCommandPortKey); uint32_t port; if (command_port_env != nullptr) { if (!absl::SimpleAtoi(command_port_env, &port)) { port = Constants::kDefaultCommandPort; } } else { port = Constants::kDefaultCommandPort; } return port; } void HttpCommandCenterInitTarget::Initialize() { // Register commands. command_center_->RegisterCommand( std::make_unique()); command_center_->RegisterCommand( std::make_unique()); command_center_->RegisterCommand( std::make_unique()); command_center_->RegisterCommand( std::make_unique()); command_center_->RegisterCommand(std::make_unique()); uint32_t port = GetAvailablePort(); command_center_->Start(port); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/command/http_server_init_target.h ================================================ #pragma once #include #include #include "sentinel-core/init/init_target.h" #include "sentinel-core/transport/command/http_command_center.h" namespace Sentinel { namespace Transport { class HttpCommandCenterInitTarget : public Init::Target { public: HttpCommandCenterInitTarget() : command_center_(std::make_unique()) {} virtual ~HttpCommandCenterInitTarget() = default; void Initialize() override; void Close(); private: uint32_t GetAvailablePort(); const std::unique_ptr command_center_; bool closed_{false}; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/common/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "transport_common", copts = DEFAULT_COPTS, srcs = [ "event_loop_thread.h", "event_loop_thread.cc", ], deps = [ "//:libevent", ], visibility = ["//visibility:public"], ) cc_test( name = "event_loop_thread_unittests", srcs = [ "event_loop_thread.cc", ], copts = TEST_COPTS, deps = [ ":transport_common", "@com_google_googletest//:gtest_main", ], linkstatic = 1, ) ================================================ FILE: sentinel-core/transport/common/event_loop_thread.cc ================================================ #include "sentinel-core/transport/common/event_loop_thread.h" #ifndef WIN32 #include #endif // !WIN32 namespace Sentinel { namespace Transport { EventLoopThread::EventLoopThread() : stoped_(true) {} bool EventLoopThread::Start() { std::promise start_promise; auto start_future = start_promise.get_future(); thd_.reset( new std::thread([this, &start_promise] { this->Work(start_promise); })); return start_future.get(); } void EventLoopThread::Stop() { if (stoped_.load()) { return; } stoped_ = true; Wakeup(); thd_->join(); } void EventLoopThread::Work(std::promise& promise) { auto ret = InitEventBase(); if (!ret) { promise.set_value(false); return; } promise.set_value(true); Dispatch(); ClearEventBase(); } bool EventLoopThread::InitEventBase() { base_ = event_base_new(); if (!base_) { // log error return false; } evutil_socket_t pair[2] = {-1, -1}; if (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, pair) < 0) { // log error return false; } if (evutil_make_socket_nonblocking(pair[0]) < 0) { // log error return false; } if (evutil_make_socket_closeonexec(pair[0]) < 0) { // log error return false; } if (evutil_make_socket_nonblocking(pair[1]) < 0) { // log error return false; } if (evutil_make_socket_closeonexec(pair[1]) < 0) { // log error return false; } wakeup_fd_[0] = pair[0]; wakeup_fd_[1] = pair[1]; auto wake_read_event = event_new(base_, wakeup_fd_[0], EV_READ | EV_PERSIST, &EventLoopThread::OnWakeupFdCallback, this); event_add(wake_read_event, nullptr); return true; } void EventLoopThread::ClearEventBase() { if (base_) { event_base_free(base_); base_ = nullptr; } } void EventLoopThread::Dispatch() { stoped_.store(false); while (!stoped_.load()) { event_base_loop(base_, EVLOOP_ONCE); DoPendingTasks(); } } void EventLoopThread::RunTask(Functor func) { if (IsInLoopThread()) { func(); return; } { std::lock_guard lock(task_mutex_); pending_tasks_.emplace_back(func); } Wakeup(); } void EventLoopThread::Wakeup() { uint64_t chr = 1; #ifndef WIN32 ::write(wakeup_fd_[1], &chr, sizeof chr); #else send(wakeup_fd_[1], (const char*)&chr, sizeof chr, 0); #endif // !WIN32 } void EventLoopThread::OnWakeupFdCallback(evutil_socket_t fd, short, void*) { uint64_t chr; #ifndef WIN32 ::read(fd, &chr, sizeof chr); #else while (recv(fd, (char*)&chr, sizeof chr, 0) > 0) ; #endif // !WIN32 } struct event_base* EventLoopThread::GetEventBase() { return base_; } void EventLoopThread::DoPendingTasks() { std::vector functors; { std::lock_guard lock(task_mutex_); functors.swap(pending_tasks_); } for (const Functor& functor : functors) { functor(); } } bool EventLoopThread::IsInLoopThread() const { return thd_->get_id() == std::this_thread::get_id(); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/common/event_loop_thread.h ================================================ #pragma once #include #include #include #include #include namespace Sentinel { namespace Transport { class EventLoopThread { using Functor = std::function; public: EventLoopThread(); ~EventLoopThread() = default; bool Start(); void Stop(); struct event_base *GetEventBase(); void RunTask(Functor func); bool IsInLoopThread() const; private: bool InitEventBase(); void ClearEventBase(); void Dispatch(); void Work(std::promise &promise); void Wakeup(); void DoPendingTasks(); static void OnWakeupFdCallback(evutil_socket_t fd, short events, void *userdata); private: struct event_base *base_ = nullptr; std::unique_ptr thd_; std::atomic stoped_; evutil_socket_t wakeup_fd_[2]; // 0:read 1:write std::mutex task_mutex_; std::vector pending_tasks_; }; } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/common/event_loop_thread_test.cc ================================================ #include "gmock/gmock.h" #include "gtest/gtest.h" #define private public #include "sentinel-core/transport/common/event_loop_thread.h" namespace Sentinel { namespace Transport { TEST(CommandRequestTest, TestStart) { EventLoopThread loop; auto ret = loop.Start(); EXPECT_EQ(ret, true); EXPECT_EQ(loop.stoped_, false); EXPECT_NE(loop.thd_->get_id(), std::this_thread::get_id()); } TEST(CommandRequestTest, TestStop) { EventLoopThread loop; auto ret = loop.Start(); EXPECT_EQ(ret, true); loop.Stop(); EXPECT_EQ(loop.stoped_, true); EXPECT_EQ(loop.thd_->joinable(), false); } TEST(CommandRequestTest, TestRunTask) { EventLoopThread loop; auto ret = loop.Start(); EXPECT_EQ(ret, true); std::promise promise; auto future = promise.get_future(); auto task = [&promise]() { promise.set_value(std::this_thread::get_id()); }; loop.RunTask(task); auto run_task_thread_id = future.get(); EXPECT_EQ(run_task_thread_id, loop.thd_->get_id()); loop.Stop(); EXPECT_EQ(loop.stoped_, true); EXPECT_EQ(loop.thd_->joinable(), false); } } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/transport/constants.h ================================================ #pragma once #include namespace Sentinel { namespace Transport { namespace Constants { constexpr auto kCommandPortKey = "CSP_SENTINEL_API_PORT"; constexpr uint32_t kDefaultCommandPort = 8718; } // namespace Constants } // namespace Transport } // namespace Sentinel ================================================ FILE: sentinel-core/utils/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "utils_lib", srcs = [ "utils.h", "utils.cc", "time_utils.h", "time_utils.cc", ], copts = DEFAULT_COPTS, ) cc_library( name = "file_utils_lib", srcs = [ "file_utils.h", "file_utils.cc", ], copts = DEFAULT_COPTS, ) cc_library( name = "macros_lib", srcs = [ "macros.h", ], copts = DEFAULT_COPTS, ) ================================================ FILE: sentinel-core/utils/file_utils.cc ================================================ #include "sentinel-core/utils/file_utils.h" #include #include #include #include #include #include #include namespace Sentinel { namespace Utils { bool FileUtils::FileExists(const std::string& path) { std::ifstream input_file(path); return input_file.is_open(); } bool FileUtils::DirExists(const std::string& path) { DIR* const dir = ::opendir(path.c_str()); bool dir_exists = (nullptr != dir); if (dir_exists) { ::closedir(dir); } return dir_exists; } bool FileUtils::CreateDir(const std::string& path) { auto s = path; size_t pos = 0; std::string dir; int ret; if (s[s.size() - 1] != '/') { s += '/'; } while ((pos = s.find_first_of('/', pos)) != std::string::npos) { dir = s.substr(0, pos++); if (dir.size() == 0) { continue; // if leading / first time is 0 length } if ((ret = mkdir(dir.c_str(), S_IRWXU)) && errno != EEXIST) { return ret == 0; } } return ret == 0; } std::vector FileUtils::ListFiles(const std::string& path) { std::vector files; auto* dir = opendir(path.c_str()); if (dir == nullptr) { return files; } struct dirent* ent; while ((ent = readdir(dir)) != nullptr) { files.emplace_back(std::move(ent->d_name)); } closedir(dir); return files; } std::string FileUtils::GetAbsolutePath(const std::string& path) { char abs_path[8192] = {0}; if (realpath(path.c_str(), abs_path) != nullptr) { return std::string(abs_path); } return ""; } } // namespace Utils } // namespace Sentinel ================================================ FILE: sentinel-core/utils/file_utils.h ================================================ #pragma once #include #include namespace Sentinel { namespace Utils { class FileUtils { public: FileUtils() = delete; static bool FileExists(const std::string& path); static bool DirExists(const std::string& path); static bool CreateDir(const std::string& path); static std::vector ListFiles(const std::string& path); static std::string GetAbsolutePath(const std::string& path); }; } // namespace Utils } // namespace Sentinel ================================================ FILE: sentinel-core/utils/macros.h ================================================ #pragma once #include namespace Sentinel { #define SENTINEL_PANIC(X) \ std::cerr << "panic: " << X << std::endl; \ abort(); #define SENTINEL_NOT_IMPLEMENTED_GCOVR_EXCL_LINE \ SENTINEL_PANIC("not implemented") #define SENTINEL_NOT_REACHED_GCOVR_EXCL_LINE SENTINEL_PANIC("not reached") } // namespace Sentinel ================================================ FILE: sentinel-core/utils/time_utils.cc ================================================ #include "sentinel-core/utils/time_utils.h" namespace Sentinel { namespace Utils { std::chrono::milliseconds TimeUtils::CurrentTimeMillis() { return std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()); } } // namespace Utils } // namespace Sentinel ================================================ FILE: sentinel-core/utils/time_utils.h ================================================ #pragma once #include namespace Sentinel { namespace Utils { class TimeUtils { public: TimeUtils() = delete; static std::chrono::milliseconds CurrentTimeMillis(); }; } // namespace Utils } // namespace Sentinel ================================================ FILE: sentinel-core/utils/utils.cc ================================================ ================================================ FILE: sentinel-core/utils/utils.h ================================================ #pragma once namespace Sentinel { namespace Utils { template class Singleton { public: /** * Obtain an instance of the singleton for class T. * @return const T& a reference to the singleton for class T. */ static T& get() { static T* instance = new T(); return *instance; } }; } // namespace Utils } // namespace Sentinel ================================================ FILE: sentinel-datasource-extension/datasource/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "converter_interface", srcs = [ "converter.h" ], copts = DEFAULT_COPTS, deps = ["@com_google_absl//absl/types:optional",] ) cc_library( name = "readable_data_source_interface", srcs = [ "readable_data_source.h", "abstract_readable_data_source.h" ], copts = DEFAULT_COPTS, deps = [ ":converter_interface", "//sentinel-core/property:dynamic_sentinel_property_lib" ] ) cc_test( name = "abstract_readable_data_source_unittests", srcs = [ "abstract_readable_data_source_unittests.cc", ], copts = TEST_COPTS, deps = [ ":readable_data_source_interface", ":converter_interface", "//sentinel-core/test/mock/common:mock_lib", "//sentinel-core/test/mock/property:mock_lib", "//sentinel-core/property:dynamic_sentinel_property_lib", "//sentinel-datasource-extension/test/mock/datasource:mock_lib", "@com_google_googletest//:gtest_main", ] ) ================================================ FILE: sentinel-datasource-extension/datasource/abstract_readable_data_source.h ================================================ #pragma once #include "sentinel-core/property/dynamic_sentinel_property.h" #include "sentinel-datasource-extension/datasource/converter.h" #include "sentinel-datasource-extension/datasource/readable_data_source.h" #include #include namespace Sentinel { namespace DataSource { template class AbstractReadableDataSource : public ReadableDataSource { public: explicit AbstractReadableDataSource(ConverterSharedPtr converter) : parser_(converter), property_(std::make_shared>()) {} virtual ~AbstractReadableDataSource() = default; absl::optional LoadConfig() override { return Convert(this->ReadSource()); } Property::SentinelPropertySharedPtr GetProperty() override { return property_; } absl::optional Convert(const S& s) { return parser_->Convert(s); } private: ConverterSharedPtr parser_; Property::SentinelPropertySharedPtr property_; }; } // namespace DataSource } // namespace Sentinel ================================================ FILE: sentinel-datasource-extension/datasource/abstract_readable_data_source_unittests.cc ================================================ #include "sentinel-datasource-extension/datasource/abstract_readable_data_source.h" #include "sentinel-core/test/mock/common/mock.h" #include "sentinel-core/test/mock/property/mock.h" #include "sentinel-datasource-extension/test/mock/datasource/mock.h" namespace Sentinel { namespace DataSource { using testing::_; using testing::An; using testing::InSequence; using testing::Matcher; TEST(AbstractReadableDataSourceTest, Basic) { auto *convert = new MockConverter(); ConverterSharedPtr convert_shared_ptr; convert_shared_ptr.reset(convert); MockAbstractReabableDataSource fake_data_source( convert_shared_ptr); FakeRule rule; rule.set_fake_rule_data("test"); auto pl = std::make_unique>(); auto pl_point = pl.get(); EXPECT_CALL(*pl_point, ConfigUpdate(Matcher(rule), false)) .Times(1); fake_data_source.GetProperty()->AddListener(std::move(pl)); fake_data_source.GetProperty()->UpdateValue(rule); EXPECT_CALL(*convert, Convert(_)).Times(1); EXPECT_CALL(fake_data_source, ReadSource()).Times(1); fake_data_source.LoadConfig(); }; } // namespace DataSource } // namespace Sentinel ================================================ FILE: sentinel-datasource-extension/datasource/converter.h ================================================ #pragma once #include #include "absl/types/optional.h" namespace Sentinel { namespace DataSource { template class Converter { public: virtual ~Converter() = default; virtual absl::optional Convert(const S& source) = 0; }; template using ConverterSharedPtr = std::shared_ptr>; } // namespace DataSource } // namespace Sentinel ================================================ FILE: sentinel-datasource-extension/datasource/readable_data_source.h ================================================ #pragma once #include "absl/types/optional.h" #include "sentinel-core/property/sentinel_property.h" #include #include namespace Sentinel { namespace DataSource { template class ReadableDataSource { public: virtual ~ReadableDataSource() = default; virtual absl::optional LoadConfig() = 0; virtual S ReadSource() = 0; virtual Property::SentinelPropertySharedPtr GetProperty() = 0; }; } // namespace DataSource } // namespace Sentinel ================================================ FILE: sentinel-datasource-extension/test/mock/datasource/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "mock_lib", srcs = [ "mock.h", "mock.cc", ], copts = TEST_COPTS, deps = [ "//sentinel-datasource-extension/datasource:converter_interface", "//sentinel-datasource-extension/datasource:readable_data_source_interface", "@com_google_googletest//:gtest", ] ) ================================================ FILE: sentinel-datasource-extension/test/mock/datasource/mock.cc ================================================ namespace Sentinel { namespace DataSource {} // namespace DataSource } // namespace Sentinel ================================================ FILE: sentinel-datasource-extension/test/mock/datasource/mock.h ================================================ #pragma once #include #include "gmock/gmock.h" #include "gtest/gtest.h" #include "sentinel-datasource-extension/datasource/abstract_readable_data_source.h" #include "sentinel-datasource-extension/datasource/converter.h" namespace Sentinel { namespace DataSource { template class MockConverter : public Converter { public: virtual ~MockConverter() = default; MOCK_METHOD1_T(Convert, absl::optional(const S&)); }; template class MockAbstractReabableDataSource : public AbstractReadableDataSource { public: using AbstractReadableDataSource::AbstractReadableDataSource; virtual ~MockAbstractReabableDataSource() = default; MOCK_METHOD0_T(ReadSource, S()); }; } // namespace DataSource } // namespace Sentinel ================================================ FILE: tests/BUILD ================================================ load("//bazel:copts.bzl", "DEFAULT_COPTS", "TEST_COPTS") package(default_visibility = ["//visibility:public"]) cc_binary( name = "tsan-flow", srcs = ["tsan-flow.cc"], copts = DEFAULT_COPTS, deps = [ "//sentinel-core/public:sph_u_lib", "//sentinel-core/slot:global_slot_chain_header", "//sentinel-core/flow:flow_rule_manager_lib", "//sentinel-core/log:logger_lib", "//sentinel-core/log/metric:metric_log_task_lib", "//sentinel-core/statistic/node:resource_node_storage_lib", "//sentinel-core/init:init_target_registry_lib", "//sentinel-core/transport/command:http_command_center_init_target", ], ) ================================================ FILE: tests/tsan-flow.cc ================================================ #include #include #include #include #include #include "sentinel-core/flow/flow_rule_manager.h" #include "sentinel-core/init/init_target_registry.h" #include "sentinel-core/log/logger.h" #include "sentinel-core/log/metric/metric_log_task.h" #include "sentinel-core/public/sph_u.h" #include "sentinel-core/system/system_rule.h" #include "sentinel-core/system/system_rule_manager.h" #include "sentinel-core/system/system_status_listener.h" #include "sentinel-core/transport/command/http_server_init_target.h" std::atomic g_count{0}; void doEntry(const char* resource) { while (g_count < 10000) { auto r = Sentinel::SphU::Entry(resource); if (r->IsBlocked()) { std::this_thread::sleep_for(std::chrono::milliseconds(10)); } else { std::this_thread::sleep_for(std::chrono::milliseconds(10)); r->Exit(); } g_count.store(g_count.load() + 1); } } void doOneEntry() { doEntry("my_open_api_abc"); } void doAnotherEntry() { doEntry("big_brother_service:foo()"); } int main() { // Initialize for Sentinel. Sentinel::Log::Logger::InitDefaultLogger(); Sentinel::Transport::HttpCommandCenterInitTarget command_center_init; command_center_init.Initialize(); Sentinel::Log::MetricLogTask metric_log_task; metric_log_task.Initialize(); Sentinel::Flow::FlowRule rule{"my_open_api_abc"}; rule.set_metric_type(Sentinel::Flow::FlowMetricType::kThreadCount); rule.set_count(2); Sentinel::Flow::FlowRuleManager::GetInstance().LoadRules({rule}); Sentinel::System::SystemRule rule1, rule2, rule3; rule1.set_rule_type(Sentinel::System::MetricType::kQps); rule1.set_threshold(static_cast(300)); rule2.set_rule_type(Sentinel::System::MetricType::kConcurrency); rule2.set_threshold(2); rule3.set_rule_type(Sentinel::System::MetricType::kCpuUsage); rule3.set_threshold(static_cast(0.8)); Sentinel::System::SystemRule badRule{Sentinel::System::MetricType::kRt, -2}; Sentinel::System::SystemRuleManager::GetInstance().LoadRules( {rule1, rule2, rule3, badRule}); std::thread t1(doOneEntry); std::thread t2(doOneEntry); std::this_thread::sleep_for(std::chrono::milliseconds(5)); std::thread t3(doOneEntry); std::thread t4(doAnotherEntry); std::thread t5(doOneEntry); std::thread t6(doOneEntry); t1.join(); t2.join(); t3.join(); t4.join(); t5.join(); t6.join(); return 0; } ================================================ FILE: third_party/nlohmann/BUILD ================================================ licenses(["notice"]) # Apache 2 load("//bazel:copts.bzl", "DEFAULT_COPTS") package(default_visibility = ["//visibility:public"]) cc_library( name = "nlohmann_json_lib", srcs = [ "json.hpp", ], copts = DEFAULT_COPTS, visibility = ["//visibility:public"], ) ================================================ FILE: third_party/nlohmann/json.hpp ================================================ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ | | |__ | | | | | | version 3.6.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License . SPDX-License-Identifier: MIT Copyright (c) 2013-2019 Niels Lohmann . Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef INCLUDE_NLOHMANN_JSON_HPP_ #define INCLUDE_NLOHMANN_JSON_HPP_ #define NLOHMANN_JSON_VERSION_MAJOR 3 #define NLOHMANN_JSON_VERSION_MINOR 6 #define NLOHMANN_JSON_VERSION_PATCH 1 #include // all_of, find, for_each #include // assert #include // and, not, or #include // nullptr_t, ptrdiff_t, size_t #include // hash, less #include // initializer_list #include // istream, ostream #include // random_access_iterator_tag #include // unique_ptr #include // accumulate #include // string, stoi, to_string #include // declval, forward, move, pair, swap #include // vector // #include #include // #include #include // transform #include // array #include // and, not #include // forward_list #include // inserter, front_inserter, end #include // map #include // string #include // tuple, make_tuple #include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible #include // unordered_map #include // pair, declval #include // valarray // #include #include // exception #include // runtime_error #include // to_string // #include #include // size_t namespace nlohmann { namespace detail { /// struct to capture the start position of the current token struct position_t { /// the total number of characters read std::size_t chars_read_total = 0; /// the number of characters read in the current line std::size_t chars_read_current_line = 0; /// the number of lines read std::size_t lines_read = 0; /// conversion to size_t to preserve SAX interface constexpr operator size_t() const { return chars_read_total; } }; } // namespace detail } // namespace nlohmann namespace nlohmann { namespace detail { //////////////// // exceptions // //////////////// /*! @brief general exception of the @ref basic_json class This class is an extension of `std::exception` objects with a member @a id for exception ids. It is used as the base class for all exceptions thrown by the @ref basic_json class. This class can hence be used as "wildcard" to catch exceptions. Subclasses: - @ref parse_error for exceptions indicating a parse error - @ref invalid_iterator for exceptions indicating errors with iterators - @ref type_error for exceptions indicating executing a member function with a wrong type - @ref out_of_range for exceptions indicating access out of the defined range - @ref other_error for exceptions indicating other library errors @internal @note To have nothrow-copy-constructible exceptions, we internally use `std::runtime_error` which can cope with arbitrary-length error messages. Intermediate strings are built with static functions and then passed to the actual constructor. @endinternal @liveexample{The following code shows how arbitrary library exceptions can be caught.,exception} @since version 3.0.0 */ class exception : public std::exception { public: /// returns the explanatory string const char* what() const noexcept override { return m.what(); } /// the id of the exception const int id; protected: exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} static std::string name(const std::string& ename, int id_) { return "[json.exception." + ename + "." + std::to_string(id_) + "] "; } private: /// an exception object as storage for error messages std::runtime_error m; }; /*! @brief exception indicating a parse error This exception is thrown by the library when a parse error occurs. Parse errors can occur during the deserialization of JSON text, CBOR, MessagePack, as well as when using JSON Patch. Member @a byte holds the byte index of the last read character in the input file. Exceptions have ids 1xx. name / id | example message | description ------------------------------ | --------------- | ------------------------- json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). @note For an input with n bytes, 1 is the index of the first character and n+1 is the index of the terminating null byte or the end of file. This also holds true when reading a byte vector (CBOR or MessagePack). @liveexample{The following code shows how a `parse_error` exception can be caught.,parse_error} @sa - @ref exception for the base class of the library exceptions @sa - @ref invalid_iterator for exceptions indicating errors with iterators @sa - @ref type_error for exceptions indicating executing a member function with a wrong type @sa - @ref out_of_range for exceptions indicating access out of the defined range @sa - @ref other_error for exceptions indicating other library errors @since version 3.0.0 */ class parse_error : public exception { public: /*! @brief create a parse error exception @param[in] id_ the id of the exception @param[in] pos the position where the error occurred (or with chars_read_total=0 if the position cannot be determined) @param[in] what_arg the explanatory string @return parse_error object */ static parse_error create(int id_, const position_t& pos, const std::string& what_arg) { std::string w = exception::name("parse_error", id_) + "parse error" + position_string(pos) + ": " + what_arg; return parse_error(id_, pos.chars_read_total, w.c_str()); } static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) { std::string w = exception::name("parse_error", id_) + "parse error" + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + ": " + what_arg; return parse_error(id_, byte_, w.c_str()); } /*! @brief byte index of the parse error The byte index of the last read character in the input file. @note For an input with n bytes, 1 is the index of the first character and n+1 is the index of the terminating null byte or the end of file. This also holds true when reading a byte vector (CBOR or MessagePack). */ const std::size_t byte; private: parse_error(int id_, std::size_t byte_, const char* what_arg) : exception(id_, what_arg), byte(byte_) {} static std::string position_string(const position_t& pos) { return " at line " + std::to_string(pos.lines_read + 1) + ", column " + std::to_string(pos.chars_read_current_line); } }; /*! @brief exception indicating errors with iterators This exception is thrown if iterators passed to a library function do not match the expected semantics. Exceptions have ids 2xx. name / id | example message | description ----------------------------------- | --------------- | ------------------------- json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). @liveexample{The following code shows how an `invalid_iterator` exception can be caught.,invalid_iterator} @sa - @ref exception for the base class of the library exceptions @sa - @ref parse_error for exceptions indicating a parse error @sa - @ref type_error for exceptions indicating executing a member function with a wrong type @sa - @ref out_of_range for exceptions indicating access out of the defined range @sa - @ref other_error for exceptions indicating other library errors @since version 3.0.0 */ class invalid_iterator : public exception { public: static invalid_iterator create(int id_, const std::string& what_arg) { std::string w = exception::name("invalid_iterator", id_) + what_arg; return invalid_iterator(id_, w.c_str()); } private: invalid_iterator(int id_, const char* what_arg) : exception(id_, what_arg) {} }; /*! @brief exception indicating executing a member function with a wrong type This exception is thrown in case of a type error; that is, a library function is executed on a JSON value whose type does not match the expected semantics. Exceptions have ids 3xx. name / id | example message | description ----------------------------- | --------------- | ------------------------- json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t &. json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | @liveexample{The following code shows how a `type_error` exception can be caught.,type_error} @sa - @ref exception for the base class of the library exceptions @sa - @ref parse_error for exceptions indicating a parse error @sa - @ref invalid_iterator for exceptions indicating errors with iterators @sa - @ref out_of_range for exceptions indicating access out of the defined range @sa - @ref other_error for exceptions indicating other library errors @since version 3.0.0 */ class type_error : public exception { public: static type_error create(int id_, const std::string& what_arg) { std::string w = exception::name("type_error", id_) + what_arg; return type_error(id_, w.c_str()); } private: type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} }; /*! @brief exception indicating access out of the defined range This exception is thrown in case a library function is called on an input parameter that exceeds the expected range, for instance in case of array indices or nonexisting object keys. Exceptions have ids 4xx. name / id | example message | description ------------------------------- | --------------- | ------------------------- json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | @liveexample{The following code shows how an `out_of_range` exception can be caught.,out_of_range} @sa - @ref exception for the base class of the library exceptions @sa - @ref parse_error for exceptions indicating a parse error @sa - @ref invalid_iterator for exceptions indicating errors with iterators @sa - @ref type_error for exceptions indicating executing a member function with a wrong type @sa - @ref other_error for exceptions indicating other library errors @since version 3.0.0 */ class out_of_range : public exception { public: static out_of_range create(int id_, const std::string& what_arg) { std::string w = exception::name("out_of_range", id_) + what_arg; return out_of_range(id_, w.c_str()); } private: out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} }; /*! @brief exception indicating other library errors This exception is thrown in case of errors that cannot be classified with the other exception types. Exceptions have ids 5xx. name / id | example message | description ------------------------------ | --------------- | ------------------------- json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. @sa - @ref exception for the base class of the library exceptions @sa - @ref parse_error for exceptions indicating a parse error @sa - @ref invalid_iterator for exceptions indicating errors with iterators @sa - @ref type_error for exceptions indicating executing a member function with a wrong type @sa - @ref out_of_range for exceptions indicating access out of the defined range @liveexample{The following code shows how an `other_error` exception can be caught.,other_error} @since version 3.0.0 */ class other_error : public exception { public: static other_error create(int id_, const std::string& what_arg) { std::string w = exception::name("other_error", id_) + what_arg; return other_error(id_, w.c_str()); } private: other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} }; } // namespace detail } // namespace nlohmann // #include #include // pair // This file contains all internal macro definitions // You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them // exclude unsupported compilers #if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) #if defined(__clang__) #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" #endif #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" #endif #endif #endif // disable float-equal warnings on GCC/clang #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wfloat-equal" #endif // disable documentation warnings on clang #if defined(__clang__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdocumentation" #endif // allow for portable deprecation warnings #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #define JSON_DEPRECATED __attribute__((deprecated)) #elif defined(_MSC_VER) #define JSON_DEPRECATED __declspec(deprecated) #else #define JSON_DEPRECATED #endif // allow for portable nodiscard warnings #if defined(__has_cpp_attribute) #if __has_cpp_attribute(nodiscard) #define JSON_NODISCARD [[nodiscard]] #elif __has_cpp_attribute(gnu::warn_unused_result) #define JSON_NODISCARD [[gnu::warn_unused_result]] #else #define JSON_NODISCARD #endif #else #define JSON_NODISCARD #endif // allow to disable exceptions #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception #define JSON_TRY try #define JSON_CATCH(exception) catch(exception) #define JSON_INTERNAL_CATCH(exception) catch(exception) #else #include #define JSON_THROW(exception) std::abort() #define JSON_TRY if(true) #define JSON_CATCH(exception) if(false) #define JSON_INTERNAL_CATCH(exception) if(false) #endif // override exception macros #if defined(JSON_THROW_USER) #undef JSON_THROW #define JSON_THROW JSON_THROW_USER #endif #if defined(JSON_TRY_USER) #undef JSON_TRY #define JSON_TRY JSON_TRY_USER #endif #if defined(JSON_CATCH_USER) #undef JSON_CATCH #define JSON_CATCH JSON_CATCH_USER #undef JSON_INTERNAL_CATCH #define JSON_INTERNAL_CATCH JSON_CATCH_USER #endif #if defined(JSON_INTERNAL_CATCH_USER) #undef JSON_INTERNAL_CATCH #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER #endif // manual branch prediction #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #define JSON_LIKELY(x) __builtin_expect(x, 1) #define JSON_UNLIKELY(x) __builtin_expect(x, 0) #else #define JSON_LIKELY(x) x #define JSON_UNLIKELY(x) x #endif // C++ language standard detection #if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 #define JSON_HAS_CPP_17 #define JSON_HAS_CPP_14 #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) #define JSON_HAS_CPP_14 #endif /*! @brief macro to briefly define a mapping between an enum and JSON @def NLOHMANN_JSON_SERIALIZE_ENUM @since version 3.4.0 */ #define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ template \ inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ { \ static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ static const std::pair m[] = __VA_ARGS__; \ auto it = std::find_if(std::begin(m), std::end(m), \ [e](const std::pair& ej_pair) -> bool \ { \ return ej_pair.first == e; \ }); \ j = ((it != std::end(m)) ? it : std::begin(m))->second; \ } \ template \ inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ { \ static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ static const std::pair m[] = __VA_ARGS__; \ auto it = std::find_if(std::begin(m), std::end(m), \ [j](const std::pair& ej_pair) -> bool \ { \ return ej_pair.second == j; \ }); \ e = ((it != std::end(m)) ? it : std::begin(m))->first; \ } // Ugly macros to avoid uglier copy-paste when specializing basic_json. They // may be removed in the future once the class is split. #define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ template class ObjectType, \ template class ArrayType, \ class StringType, class BooleanType, class NumberIntegerType, \ class NumberUnsignedType, class NumberFloatType, \ template class AllocatorType, \ template class JSONSerializer> #define NLOHMANN_BASIC_JSON_TPL \ basic_json // #include #include // not #include // size_t #include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type namespace nlohmann { namespace detail { // alias templates to reduce boilerplate template using enable_if_t = typename std::enable_if::type; template using uncvref_t = typename std::remove_cv::type>::type; // implementation of C++14 index_sequence and affiliates // source: https://stackoverflow.com/a/32223343 template struct index_sequence { using type = index_sequence; using value_type = std::size_t; static constexpr std::size_t size() noexcept { return sizeof...(Ints); } }; template struct merge_and_renumber; template struct merge_and_renumber, index_sequence> : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; template struct make_index_sequence : merge_and_renumber < typename make_index_sequence < N / 2 >::type, typename make_index_sequence < N - N / 2 >::type > {}; template<> struct make_index_sequence<0> : index_sequence<> {}; template<> struct make_index_sequence<1> : index_sequence<0> {}; template using index_sequence_for = make_index_sequence; // dispatch utility (taken from ranges-v3) template struct priority_tag : priority_tag < N - 1 > {}; template<> struct priority_tag<0> {}; // taken from ranges-v3 template struct static_const { static constexpr T value{}; }; template constexpr T static_const::value; } // namespace detail } // namespace nlohmann // #include #include // not #include // numeric_limits #include // false_type, is_constructible, is_integral, is_same, true_type #include // declval // #include #include // random_access_iterator_tag // #include namespace nlohmann { namespace detail { template struct make_void { using type = void; }; template using void_t = typename make_void::type; } // namespace detail } // namespace nlohmann // #include namespace nlohmann { namespace detail { template struct iterator_types {}; template struct iterator_types < It, void_t> { using difference_type = typename It::difference_type; using value_type = typename It::value_type; using pointer = typename It::pointer; using reference = typename It::reference; using iterator_category = typename It::iterator_category; }; // This is required as some compilers implement std::iterator_traits in a way that // doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. template struct iterator_traits { }; template struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> : iterator_types { }; template struct iterator_traits::value>> { using iterator_category = std::random_access_iterator_tag; using value_type = T; using difference_type = ptrdiff_t; using pointer = T*; using reference = T&; }; } // namespace detail } // namespace nlohmann // #include // #include // #include #include // #include // http://en.cppreference.com/w/cpp/experimental/is_detected namespace nlohmann { namespace detail { struct nonesuch { nonesuch() = delete; ~nonesuch() = delete; nonesuch(nonesuch const&) = delete; nonesuch(nonesuch const&&) = delete; void operator=(nonesuch const&) = delete; void operator=(nonesuch&&) = delete; }; template class Op, class... Args> struct detector { using value_t = std::false_type; using type = Default; }; template class Op, class... Args> struct detector>, Op, Args...> { using value_t = std::true_type; using type = Op; }; template