Repository: chaitin/SafeLine Branch: main Commit: 6d871e638bf8 Files: 236 Total size: 649.6 KB Directory structure: gitextract_r0ep21tc/ ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug-report.yaml │ │ ├── config.yml │ │ ├── feature-request.yaml │ │ └── other.yaml │ └── workflows/ │ └── slmcp-docker.yml ├── .gitignore ├── .gitmodules ├── LICENSE.md ├── README.md ├── README_CN.md ├── compose.yaml ├── management/ │ ├── .gitignore │ ├── .golangci.yml │ ├── Makefile │ ├── README.md │ ├── scripts/ │ │ └── genproto.sh │ ├── tcontrollerd/ │ │ ├── README.md │ │ ├── config.yml │ │ ├── controller/ │ │ │ ├── controller.go │ │ │ ├── template.go │ │ │ └── website.go │ │ ├── go.mod │ │ ├── go.sum │ │ ├── main.go │ │ ├── model/ │ │ │ └── website.go │ │ ├── pkg/ │ │ │ ├── config/ │ │ │ │ ├── config.go │ │ │ │ ├── global.go │ │ │ │ └── log.go │ │ │ ├── constants/ │ │ │ │ ├── constants.go │ │ │ │ └── forbidden_page.go │ │ │ ├── cron/ │ │ │ │ ├── cron.go │ │ │ │ └── forbidden_page.go │ │ │ ├── log/ │ │ │ │ └── log.go │ │ │ └── ngcmd/ │ │ │ └── ngcmd.go │ │ ├── proto/ │ │ │ └── website/ │ │ │ └── website.proto │ │ └── utils/ │ │ ├── file.go │ │ └── random.go │ └── webserver/ │ ├── .gitignore │ ├── README.md │ ├── api/ │ │ ├── auth.go │ │ ├── behaviour.go │ │ ├── cert.go │ │ ├── common.go │ │ ├── dashboard.go │ │ ├── detectlog.go │ │ ├── endpoints.go │ │ ├── policygroup.go │ │ ├── policyrule.go │ │ ├── response/ │ │ │ ├── error.go │ │ │ ├── jsonbody.go │ │ │ └── png.go │ │ └── website.go │ ├── cmd/ │ │ ├── fake_logs.go │ │ ├── gen_certs.go │ │ ├── push_fsl.go │ │ ├── reset_user.go │ │ └── show_fsl.go │ ├── config.yml │ ├── go.mod │ ├── go.sum │ ├── main.go │ ├── middleware/ │ │ └── auth.go │ ├── model/ │ │ ├── base.go │ │ ├── behaviour.go │ │ ├── db_patch_1_4_0.go │ │ ├── detectlog.go │ │ ├── init.go │ │ ├── option.go │ │ ├── policygroup.go │ │ ├── policyrule.go │ │ ├── statistics.go │ │ ├── user.go │ │ └── website.go │ ├── pkg/ │ │ ├── config/ │ │ │ ├── config.go │ │ │ ├── db.go │ │ │ ├── detector.go │ │ │ ├── global.go │ │ │ ├── grpc.go │ │ │ ├── log.go │ │ │ ├── server.go │ │ │ └── telemetry.go │ │ ├── constants/ │ │ │ ├── constants.go │ │ │ ├── detectlog.go │ │ │ ├── detector.go │ │ │ ├── option.go │ │ │ ├── session.go │ │ │ └── telemetry.go │ │ ├── cron/ │ │ │ ├── cron.go │ │ │ └── update_policy.go │ │ ├── database/ │ │ │ └── postgres.go │ │ ├── fvm/ │ │ │ ├── fsl/ │ │ │ │ ├── action.go │ │ │ │ ├── quote.go │ │ │ │ ├── selector.go │ │ │ │ ├── state.go │ │ │ │ ├── table.go │ │ │ │ └── target.go │ │ │ ├── fvm.go │ │ │ ├── generator.go │ │ │ └── helper.go │ │ ├── log/ │ │ │ └── log.go │ │ └── telemetry.go │ ├── proto/ │ │ └── website/ │ │ └── website.proto │ ├── rpc/ │ │ ├── main.go │ │ └── website.go │ ├── tools/ │ │ └── init_db.sh │ └── utils/ │ ├── cert.go │ ├── file.go │ ├── healthy.go │ ├── httpclient.go │ ├── random.go │ └── url.go ├── mcp_server/ │ ├── Dockerfile │ ├── README.md │ ├── config.yaml │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ ├── internal/ │ │ ├── api/ │ │ │ ├── analyze/ │ │ │ │ └── get_event_list.go │ │ │ ├── app/ │ │ │ │ └── create_application.go │ │ │ ├── client.go │ │ │ ├── response.go │ │ │ ├── rule/ │ │ │ │ └── create_rule.go │ │ │ ├── service.go │ │ │ └── types.go │ │ ├── config/ │ │ │ └── config.go │ │ └── tools/ │ │ ├── analyze/ │ │ │ └── get_atttack_events.go │ │ ├── app/ │ │ │ └── create_application.go │ │ ├── example.go │ │ ├── init.go │ │ ├── rule/ │ │ │ ├── create_blacklist_rule.go │ │ │ └── create_whitelist_rule.go │ │ └── tool.go │ ├── main.go │ └── pkg/ │ ├── config/ │ │ └── config.go │ ├── errors/ │ │ └── errors.go │ ├── logger/ │ │ ├── field.go │ │ └── logger.go │ └── mcp/ │ ├── mcp.go │ ├── schema.go │ └── schema_test.go ├── scripts/ │ └── manage.py ├── sdk/ │ ├── ingress-nginx/ │ │ ├── README.md │ │ ├── ingress-nginx-safeline-1.0.2-1.rockspec │ │ ├── ingress-nginx-safeline-1.0.3-1.rockspec │ │ ├── ingress-nginx-safeline-1.0.4-1.rockspec │ │ └── lib/ │ │ └── safeline/ │ │ └── main.lua │ ├── kong/ │ │ ├── Readme.md │ │ ├── kong/ │ │ │ └── plugins/ │ │ │ └── safeline/ │ │ │ ├── handler.lua │ │ │ └── schema.lua │ │ ├── kong-safeline-1.0.0-1.rockspec │ │ ├── kong-safeline-1.0.1-1.rockspec │ │ ├── kong-safeline-1.0.2-1.rockspec │ │ ├── kong-safeline-1.0.3-1.rockspec │ │ ├── kong-safeline-1.0.4-1.rockspec │ │ ├── kong-safeline-1.0.5-1.rockspec │ │ ├── kong-safeline-1.0.6-1.rockspec │ │ └── kong-safeline-1.0.7-1.rockspec │ └── lua-resty-t1k/ │ ├── .github/ │ │ └── workflows/ │ │ ├── release.yml │ │ └── test.yml │ ├── .gitignore │ ├── .luacheckrc │ ├── LICENSE │ ├── README.md │ ├── ci/ │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── bytecode │ ├── lib/ │ │ └── resty/ │ │ ├── t1k/ │ │ │ ├── buffer.lua │ │ │ ├── constants.lua │ │ │ ├── file.lua │ │ │ ├── filter.lua │ │ │ ├── handler.lua │ │ │ ├── log.lua │ │ │ ├── request.lua │ │ │ ├── utils.lua │ │ │ └── uuid.lua │ │ └── t1k.lua │ ├── mainspec/ │ │ └── lua-resty-t1k-main-0-0.rockspec │ ├── rockspec/ │ │ ├── lua-resty-t1k-1.0.0-0.rockspec │ │ ├── lua-resty-t1k-1.0.1-0.rockspec │ │ ├── lua-resty-t1k-1.0.2-0.rockspec │ │ ├── lua-resty-t1k-1.0.3-0.rockspec │ │ ├── lua-resty-t1k-1.1.0-0.rockspec │ │ ├── lua-resty-t1k-1.1.1-0.rockspec │ │ ├── lua-resty-t1k-1.1.2-0.rockspec │ │ ├── lua-resty-t1k-1.1.3-0.rockspec │ │ ├── lua-resty-t1k-1.1.4-0.rockspec │ │ └── lua-resty-t1k-1.1.5-0.rockspec │ └── t/ │ ├── buffer.t │ ├── file.t │ ├── filter.t │ ├── handler.t │ ├── integration.t │ ├── log.t │ ├── option.t │ ├── request.t │ ├── utils.t │ └── uuid.t ├── version.json └── yanshi/ ├── .gitignore ├── Makefile ├── README.md ├── contrib/ │ ├── vim/ │ │ ├── compiler/ │ │ │ └── yanshi.vim │ │ ├── ftdetect/ │ │ │ └── yanshi.vim │ │ ├── ftplugin/ │ │ │ └── yanshi.vim │ │ ├── syntax/ │ │ │ └── yanshi.vim │ │ └── syntax_checkers/ │ │ └── yanshi/ │ │ └── yanshi.vim │ └── zsh/ │ └── _yanshi ├── src/ │ ├── common.cc │ ├── common.hh │ ├── compiler.cc │ ├── compiler.hh │ ├── fsa.cc │ ├── fsa.hh │ ├── fsa_anno.cc │ ├── fsa_anno.hh │ ├── lexer.l │ ├── lexer_helper.cc │ ├── lexer_helper.hh │ ├── loader.cc │ ├── loader.hh │ ├── location.cc │ ├── location.hh │ ├── main.cc │ ├── option.cc │ ├── option.hh │ ├── parser.y │ ├── repl.cc │ ├── repl.hh │ ├── syntax.cc │ └── syntax.hh └── unittest/ ├── determinize_test.cc ├── difference_test.cc ├── intersection_test.cc ├── minimize_test.cc ├── union_test.cc └── unittest_helper.hh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE/bug-report.yaml ================================================ name: Bug description: Report a bug # Create a report to help us improve title: "[Bug] " body: - type: markdown attributes: value: | Please search the [open issues](https://github.com/chaitin/SafeLine/issues) and [discussion](https://github.com/chaitin/SafeLine/discussions) for duplicate issue first. The more information you share, the faster we can identify and fix the bug. # Please check for duplicate issue first. - type: textarea id: Description attributes: label: What happened? # Describe the bug validations: required: false - type: textarea id: Reproduce attributes: label: How we reproduce? description: | Reports cannot be reproduced will Most likely be closed. # To Reproduce value: | 1. ... 2. ... 3. ... - type: textarea id: Expected attributes: label: Expected behavior # placeholder: | # Descript what you expected to happen. # Expected behavior. Descript what you expected to happen. - type: textarea id: Errorlog attributes: label: Error log placeholder: | Paste the error logs if any. # Expected behavior. Descript what you expected to happen. ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Discord url: https://discord.gg/wyshSVuvxC about: Ask questions and discuss with other SafeLine users in real time. - name: Home page 官网 url: https://waf.chaitin.com/ about: Get feature descriptions, technical documentation and more information. 获取功能描述、技术文档和更多信息。 - name: 绕过反馈 url: https://stack.chaitin.com/security-challenge/safeline/index about: Waf 绕过可在 CT Stack 安全挑战赛提交细节 ================================================ FILE: .github/ISSUE_TEMPLATE/feature-request.yaml ================================================ name: Suggestion # Feature request description: New feature or improvements. title: "[Suggestion] " body: - type: markdown attributes: value: | Please search the [open issues](https://github.com/chaitin/SafeLine/issues) and [discussion](https://github.com/chaitin/SafeLine/discussions) for duplicate issue first. Please rise only one suggestion in an issue. - type: textarea id: solution attributes: label: What would you like to be added or improved? # Describe the solution you'd like # placeholder: | # - type: textarea id: problem attributes: label: Why is it needed? # Background and the specific problem that frustrates you placeholder: | Background and the problem that frustrates you validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/other.yaml ================================================ name: Other description: Other issues such as Doc body: - type: markdown attributes: value: | Please search the [open issues](https://github.com/chaitin/SafeLine/issues) and [discussion](https://github.com/chaitin/SafeLine/discussions) for duplicate issue first. Please rise only one suggestion in an issue. - type: textarea id: content attributes: label: Content ================================================ FILE: .github/workflows/slmcp-docker.yml ================================================ name: MCP Docker on: push: branches: - main tags: - "v*" paths: - "mcp_server/**" - ".github/workflows/slmcp-docker.yml" jobs: build-and-push: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: platforms: linux/amd64,linux/arm64 - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERIO_USERNAME }} password: ${{ secrets.DOCKERIO_PASSWORD }} - name: Build and push uses: docker/build-push-action@v5 with: context: ./mcp_server push: true platforms: linux/amd64,linux/arm64 tags: | chaitin/safeline-mcp:latest chaitin/safeline-mcp:${{ github.ref_name }} cache-from: type=registry,ref=chaitin/safeline-mcp:buildcache cache-to: type=registry,ref=chaitin/safeline-mcp:buildcache,mode=max ================================================ FILE: .gitignore ================================================ *.Zone.Identifier .DS_Store *.zip *.tar *.tar.gz build.sh compose.yml __pycache__ .cursor .vscode ================================================ FILE: .gitmodules ================================================ [submodule "blazehttp"] path = blazehttp url = https://github.com/chaitin/blazehttp [submodule "sdk/traefik-safeline"] path = sdk/traefik-safeline url = https://github.com/chaitin/traefik-safeline ================================================ FILE: LICENSE.md ================================================ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . ================================================ FILE: README.md ================================================

SafeLine - Make your web apps secure

🏠 Website   |   📖 Docs   |   🔍 Live Demo   |   🙋‍♂️ Discord   |   中文版

## 👋 INTRODUCTION SafeLine is a self-hosted **`WAF(Web Application Firewall)`** to protect your web apps from attacks and exploits. A web application firewall helps protect web apps by filtering and monitoring HTTP traffic between a web application and the Internet. It typically protects web apps from attacks such as `SQL injection`, `XSS`, `code injection`, `os command injection`, `CRLF injection`, `ldap injection`, `xpath injection`, `RCE`, `XXE`, `SSRF`, `path traversal`, `backdoor`, `bruteforce`, `http-flood`, `bot abused`, among others. #### 💡 How It Works By deploying a WAF in front of a web application, a shield is placed between the web application and the Internet. While a proxy server protects a client machine’s identity by using an intermediary, a WAF is a type of reverse-proxy, protecting the server from exposure by having clients pass through the WAF before reaching the server. A WAF protects your web apps by filtering, monitoring, and blocking any malicious HTTP/S traffic traveling to the web application, and prevents any unauthorized data from leaving the app. It does this by adhering to a set of policies that help determine what traffic is malicious and what traffic is safe. Just as a proxy server acts as an intermediary to protect the identity of a client, a WAF operates in similar fashion but acting as a reverse proxy intermediary that protects the web app server from a potentially malicious client. its core capabilities include: - Defenses for web attacks - Proactive bot abused defense - HTML & JS code encryption - IP-based rate limiting - Web Access Control List #### ⚡️ Screenshots | | | | ------------------------------------------------- | ------------------------------------------------- | | | | Get [Live Demo](https://demo.waf.chaitin.com:9443/) ## 🔥 FEATURES List of the main features as follows: - **`Block Web Attacks`** - It defenses for all of web attacks, such as `SQL injection`, `XSS`, `code injection`, `os command injection`, `CRLF injection`, `XXE`, `SSRF`, `path traversal` and so on. - **`Rate Limiting`** - Defend your web apps against `DoS attacks`, `bruteforce attempts`, `traffic surges`, and other types of abuse by throttling traffic that exceeds defined limits. - **`Anti-Bot Challenge`** - Anti-Bot challenges to protect your website from `bot attacks`, humen users will be allowed, crawlers and bots will be blocked. - **`Authentication Challenge`** - When authentication challenge turned on, visitors need to enter the password, otherwise they will be blocked. - **`Dynamic Protection`** - When dynamic protection turned on, html and js codes in your web server will be dynamically encrypted by each time you visit. #### 🧩 Showcases | | Legitimate User | Malicious User | | ----------------------------- | --------------------------------------------------- | ---------------------------------------------------------------- | | **`Block Web Attacks`** | | | | **`Rate Limiting`** | | | | **`Anti-Bot Challenge`** | | | | **`Auth Challenge`** | | | | **`HTML Dynamic Protection`** | | | | **`JS Dynamic Protection`** | | | ## 🚀 Quickstart > [!WARNING] > 中国大陆用户安装国际版可能会导致无法连接云服务，请查看 [中文版安装文档](https://docs.waf-ce.chaitin.cn/zh/%E4%B8%8A%E6%89%8B%E6%8C%87%E5%8D%97/%E5%AE%89%E8%A3%85%E9%9B%B7%E6%B1%A0) #### 📦 Installing Information on how to install SafeLine can be found in the [Install Guide](https://docs.waf.chaitin.com/en/GetStarted/Deploy) #### ⚙️ Protecting Web Apps to see [Configuration](https://docs.waf.chaitin.com/en/GetStarted/AddApplication) ## 📋 More Informations #### Effect Evaluation | Metric | ModSecurity, Level 1 | CloudFlare, Free | SafeLine, Balance | SafeLine, Strict | | ----------------- | -------------------- | -------------------- | ---------------------- | --------------------- | | Total Samples | 33669 | 33669 | 33669 | 33669 | | **Detection** | 69.74% | 10.70% | 71.65% | **76.17%** | | **False Positive**| 17.58% | 0.07% | **0.07%** | 0.22% | | **Accuracy** | 82.20% | 98.40% | **99.45%** | 99.38% | #### Is SafeLine Production-Ready? Yes, SafeLine is production-ready. - Over 180,000 installations worldwide - Protecting over 1,000,000 Websites - Handling over 30,000,000,000 HTTP Requests Daily #### 🙋‍♂️ Community Join our [Discord](https://discord.gg/SVnZGzHFvn) to get community support, the core team members are identified by the STAFF role in Discord. - channel [#feedback](https://discord.com/channels/1243085666485534830/1243120292822253598): for new features discussion. - channel [#FAQ](https://discord.com/channels/1243085666485534830/1263761679619981413): for FAQ. - channel [#general](https://discord.com/channels/1243085666485534830/1243115843919806486): for any other questions. Several contact options exist for our community, the primary one being Discord. These are in addition to GitHub issues for creating a new issue.

   

#### 💪 PRO Edition Coming soon! #### 📝 License See [LICENSE](/LICENSE.md) for details. ================================================ FILE: README_CN.md ================================================

SafeLine - 雷池 - 不让黑客越过半步

🏠 官网   |   📖 文档   |   🔍 演示环境   |   🙋‍♂️ 社区微信群   |   国际版

## 👋 项目介绍 SafeLine，中文名 "雷池"，是一款简单好用, 效果突出的 **`Web 应用防火墙(WAF)`**，可以保护 Web 服务不受黑客攻击。 雷池通过过滤和监控 Web 应用与互联网之间的 HTTP 流量来保护 Web 服务。可以保护 Web 服务免受 `SQL 注入`、`XSS`、 `代码注入`、`命令注入`、`CRLF 注入`、`ldap 注入`、`xpath 注入`、`RCE`、`XXE`、`SSRF`、`路径遍历`、`后门`、`暴力破解`、`CC`、`爬虫` 等攻击。 #### 💡 工作原理 雷池通过阻断流向 Web 服务的恶意 HTTP 流量来保护 Web 服务。雷池作为反向代理接入网络，通过在 Web 服务前部署雷池，可在 Web 服务和互联网之间设置一道屏障。 雷池的核心功能如下: - 防护 Web 攻击 - 防爬虫, 防扫描 - 前端代码动态加密 - 基于源 IP 的访问速率限制 - HTTP 访问控制 #### ⚡️ 项目截图 | | | | ------------------------------------------------- | ------------------------------------------------- | | | | 查看 [演示环境](https://demo.waf-ce.chaitin.cn:9443/) ## 🔥 核心能力 对于你的网站而言, 雷池可以实现如下效果: - **`阻断 Web 攻击`** - 可以防御所有的 Web 攻击，例如 `SQL 注入`、`XSS`、`代码注入`、`操作系统命令注入`、`CRLF 注入`、`XXE`、`SSRF`、`路径遍历` 等等。 - **`限制访问频率`** - 限制用户的访问速率，让 Web 服务免遭 `CC 攻击`、`暴力破解`、`流量激增` 和其他类型的滥用。 - **`人机验证`** - 互联网上有来自真人用户的流量，但更多的是由爬虫, 漏洞扫描器, 蠕虫病毒, 漏洞利用程序等自动化程序发起的流量，开启雷池的人机验证功能后真人用户会被放行，恶意爬虫将会被阻断。 - **`身份认证`** - 雷池的 "身份认证" 功能可以很好的解决 "未授权访问" 漏洞，当用户访问您的网站时，需要输入您配置的用户名和密码信息，不持有认证信息的用户将被拒之门外。 - **`动态防护`** - 在用户浏览到的网页内容不变的情况下，将网页赋予动态特性，对 HTML 和 JavaScript 代码进行动态加密，确保每次访问时这些代码都以随机且独特的形态呈现。 #### 🧩 核心能力展示 | | Legitimate User | Malicious User | | ----------------------------- | --------------------------------------------------- | ---------------------------------------------------------------- | | **`阻断 Web 攻击`** | | | | **`限制访问频率`** | | | | **`人机验证`** | | | | **`身份认证`** | | | | **`HTML 动态防护`** | | | | **`JS 动态防护`** | | | ## 🚀 上手指南 #### 📦 安装 查看 [安装雷池](https://docs.waf-ce.chaitin.cn/zh/%E4%B8%8A%E6%89%8B%E6%8C%87%E5%8D%97/%E5%AE%89%E8%A3%85%E9%9B%B7%E6%B1%A0) #### ⚙️ 配置防护站点 查看 [快速配置](https://docs.waf-ce.chaitin.cn/zh/%E4%B8%8A%E6%89%8B%E6%8C%87%E5%8D%97/%E5%BF%AB%E9%80%9F%E9%85%8D%E7%BD%AE) ## 📋 更多信息 #### 防护效果测试 | Metric | ModSecurity, Level 1 | CloudFlare | 雷池, 平衡 | 雷池, 严格 | | ----------------- | -------------------- | -------------------- | ---------------------- | --------------------- | | 样本数量 | 33669 | 33669 | 33669 | 33669 | | **检出率** | 69.74% | 10.70% | 71.65% | **76.17%** | | **误报率** | 17.58% | 0.07% | **0.07%** | 0.22% | | **准确率** | 82.20% | 98.40% | **99.45%** | 99.38% | #### 雷池可以投入生产使用吗 是的，已经有不少用户将雷池投入生产使用，截至目前 - 全球累计装机量已超过 18 万台 - 防护的网站数量超过 100 万个 - 每天清洗 HTTP 请求超过 300 亿次 #### 🙋‍♂️ 用户社区 欢迎加入雷池 [社区微信群](/images/wechat.png) 进行技术交流。 也可以加入雷池 [Discord](https://discord.gg/SVnZGzHFvn) 来获取更多社区支持。

   

#### 💪 专业版 查看 [社区版 vs 专业版](https://waf-ce.chaitin.cn/version) ================================================ FILE: compose.yaml ================================================ networks: safeline-ce: name: safeline-ce driver: bridge ipam: driver: default config: - gateway: ${SUBNET_PREFIX:?SUBNET_PREFIX required}.1 subnet: ${SUBNET_PREFIX}.0/24 driver_opts: com.docker.network.bridge.name: safeline-ce services: postgres: container_name: safeline-pg restart: always image: ${IMAGE_PREFIX}/safeline-postgres${ARCH_SUFFIX}:15.2 volumes: - ${SAFELINE_DIR}/resources/postgres/data:/var/lib/postgresql/data - /etc/localtime:/etc/localtime:ro environment: - POSTGRES_USER=safeline-ce - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?postgres password required} networks: safeline-ce: ipv4_address: ${SUBNET_PREFIX}.2 command: [postgres, -c, max_connections=600] healthcheck: test: pg_isready -U safeline-ce -d safeline-ce mgt: container_name: safeline-mgt restart: always image: ${IMAGE_PREFIX}/safeline-mgt${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG:?image tag required} volumes: - /etc/localtime:/etc/localtime:ro - ${SAFELINE_DIR}/resources/mgt:/app/data - ${SAFELINE_DIR}/logs/nginx:/app/log/nginx:z - ${SAFELINE_DIR}/resources/sock:/app/sock - /var/run:/app/run ports: - ${MGT_PORT:-9443}:1443 healthcheck: test: curl -k -f https://localhost:1443/api/open/health environment: - MGT_PG=postgres://safeline-ce:${POSTGRES_PASSWORD}@safeline-pg/safeline-ce?sslmode=disable depends_on: - postgres - fvm logging: driver: "json-file" options: max-size: "100m" max-file: "5" networks: safeline-ce: ipv4_address: ${SUBNET_PREFIX}.4 detect: container_name: safeline-detector restart: always image: ${IMAGE_PREFIX}/safeline-detector${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} volumes: - ${SAFELINE_DIR}/resources/detector:/resources/detector - ${SAFELINE_DIR}/logs/detector:/logs/detector - /etc/localtime:/etc/localtime:ro environment: - LOG_DIR=/logs/detector networks: safeline-ce: ipv4_address: ${SUBNET_PREFIX}.5 tengine: container_name: safeline-tengine restart: always image: ${IMAGE_PREFIX}/safeline-tengine${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} volumes: - /etc/localtime:/etc/localtime:ro - /etc/resolv.conf:/etc/resolv.conf:ro - ${SAFELINE_DIR}/resources/nginx:/etc/nginx - ${SAFELINE_DIR}/resources/detector:/resources/detector - ${SAFELINE_DIR}/resources/chaos:/resources/chaos - ${SAFELINE_DIR}/logs/nginx:/var/log/nginx:z - ${SAFELINE_DIR}/resources/cache:/usr/local/nginx/cache - ${SAFELINE_DIR}/resources/sock:/app/sock environment: - TCD_MGT_API=https://${SUBNET_PREFIX}.4:1443/api/open/publish/server - TCD_SNSERVER=${SUBNET_PREFIX}.5:8000 # deprecated - SNSERVER_ADDR=${SUBNET_PREFIX}.5:8000 - CHAOS_ADDR=${SUBNET_PREFIX}.10 ulimits: nofile: 131072 network_mode: host luigi: container_name: safeline-luigi restart: always image: ${IMAGE_PREFIX}/safeline-luigi${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} environment: - MGT_IP=${SUBNET_PREFIX}.4 - LUIGI_PG=postgres://safeline-ce:${POSTGRES_PASSWORD}@safeline-pg/safeline-ce?sslmode=disable volumes: - /etc/localtime:/etc/localtime:ro - ${SAFELINE_DIR}/resources/luigi:/app/data logging: driver: "json-file" options: max-size: "100m" max-file: "5" depends_on: - detect - mgt networks: safeline-ce: ipv4_address: ${SUBNET_PREFIX}.7 fvm: container_name: safeline-fvm restart: always image: ${IMAGE_PREFIX}/safeline-fvm${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} volumes: - /etc/localtime:/etc/localtime:ro logging: driver: "json-file" options: max-size: "100m" max-file: "5" networks: safeline-ce: ipv4_address: ${SUBNET_PREFIX}.8 chaos: container_name: safeline-chaos restart: always image: ${IMAGE_PREFIX}/safeline-chaos${REGION}${ARCH_SUFFIX}${RELEASE}:${IMAGE_TAG} logging: driver: "json-file" options: max-size: "100m" max-file: "10" environment: - DB_ADDR=postgres://safeline-ce:${POSTGRES_PASSWORD}@safeline-pg/safeline-ce?sslmode=disable volumes: - ${SAFELINE_DIR}/resources/sock:/app/sock - ${SAFELINE_DIR}/resources/chaos:/app/chaos networks: safeline-ce: ipv4_address: ${SUBNET_PREFIX}.10 ================================================ FILE: management/.gitignore ================================================ # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out **/.idea *.iml #OSX system files. **/.DS_Store /build *.pb.go submodule/ ================================================ FILE: management/.golangci.yml ================================================ linters: disable-all: true enable: - deadcode - errcheck - gofmt - goimports - gosimple - govet - ineffassign - staticcheck - structcheck - typecheck - unused - varcheck ================================================ FILE: management/Makefile ================================================ GO = GO111MODULE=on go #GO = GO111MODULE=on GOOS=linux GOARCH=amd64 go GOBUILD = $(GO) build -mod=readonly GOTEST = $(GO) test -v -p 1 -coverprofile=coverage-management.out STAMP = $(shell date +%s) GITHASH = $(shell git rev-parse --short=8 HEAD) GITTAG = $(shell git describe --tags --abbrev=0) BUILDFLAGS := -ldflags "-X main.buildstamp=$(STAMP) -X main.githash=$(GITHASH) -X main.version=$(GITTAG)" pkgs = ./... all: build-all .PHONY: build-all build-all: proto build-webserver build-tcd .PHONY: build-webserver build-webserver: cd webserver && $(GOBUILD) $(BUILDFLAGS) -o ../build/webserver main.go .PHONY: build-tcd build-tcd: cd tcontrollerd && CGO_ENABLED=0 $(GOBUILD) $(BUILDFLAGS) -o ../build/tcontrollerd main.go .PHONY: test test: $(GOTEST) -failfast $(pkgs) .PHONY: proto proto: @./scripts/genproto.sh .PHONY: lint lint: # 'go list' needs to be executed before staticcheck to prepopulate the modules cache. # Otherwise staticcheck might fail randomly for some reason not yet explained. $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null goimports -local chaitin.cn -w $$(find . -type f -name '*.go' -not -path "./vendor/*") golangci-lint version cd webserver && golangci-lint run -v --skip-dirs vendor --deadline 10m .PHONY: clean clean: rm -rf build ================================================ FILE: management/README.md ================================================ # Management Micro Service ## Requirements Go 1.18+ ================================================ FILE: management/scripts/genproto.sh ================================================ #!/usr/bin/env bash # # Generate all protobuf bindings. # Run from repository root. set -u if ! [[ "$0" =~ scripts/genproto.sh ]]; then echo "must be run from repository root" exit 255 fi DIRS="webserver/proto tcontrollerd/proto" echo "generating code" protoc --version for dir in ${DIRS}; do pushd "${dir}" || return find . -type d -print0 | while IFS= read -r -d '' sdir ; do pushd "${sdir}" || return # shellcheck disable=SC2010 FS=$(ls | grep "\.proto\$") if [ -n "${FS}" ] ; then protoc --go_out=. --go_opt=paths=source_relative \ --go-grpc_out=. --go-grpc_opt=paths=source_relative \ "${FS}" goimports -local chaitin.cn -w ./*.pb.go fi popd || return done popd || return done ================================================ FILE: management/tcontrollerd/README.md ================================================ # TControllerD Tengine Controller Daemon (abbr. TCD) will be running in the Tengine container, designed to be in place with minion on the host machine. ## Requirements Go 1.18+ ## Development ### init protobuf ```shell # Refer: https://grpc.io/docs/languages/go/quickstart/ # 1. Install protoc # https://grpc.io/docs/protoc-installation/ # 2. Install Go plugins go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 # 3. Update your PATH so that the protoc compiler can find the plugins # export PATH="$PATH:$(go env GOPATH)/bin" # 4. Generate proto go code # cd /path/to/management ./scripts/genproto.sh ``` ================================================ FILE: management/tcontrollerd/config.yml ================================================ # develop use only. For production, refer to `package/build/tengine/tcontrollerd/config.yml` log: output: stdout # "stdout", "stderr" or file path level: debug # "debug", "info", "warn" or "error" mgt_addr: 169.254.0.4:9002 # gRPC addr of mgt-api webserver ================================================ FILE: management/tcontrollerd/controller/controller.go ================================================ package controller import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/config" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/log" pb "chaitin.cn/patronus/safeline-2/management/tcontrollerd/proto/website" ) var ( logger = log.GetLogger("controller") ) func Handle() error { logger.Infof("Connect mgt-webserver at %s", config.GlobalConfig.MgtWebserver) gRPCConn, err := grpc.Dial(config.GlobalConfig.MgtWebserver, []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), }...) if err != nil { logger.Errorf("Fail to dial: %v", err) return err } wsClient := pb.NewWebsiteClient(gRPCConn) defer func(conn *grpc.ClientConn) { err := conn.Close() if err != nil { logger.Errorf("Fail to close: %v", err) return } }(gRPCConn) if err = websiteHandler(wsClient); err != nil { return err } return nil } ================================================ FILE: management/tcontrollerd/controller/template.go ================================================ package controller var nginxConfigTpl = ` upstream %s { %s keepalive 128; keepalive_timeout 75; } server { %s %s %s %s location = /forbidden_page { internal; root /etc/nginx/forbidden_pages; try_files /default_forbidden_page.html =403; } location ^~ / { proxy_pass %s://%s; include proxy_params; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Accept-Encoding ""; t1k_append_header SL-CE-SUID %d; t1k_body_size 1024k; tx_body_size 4k; t1k_error_page 403 /forbidden_page; tx_error_page 403 /forbidden_page; } }` var upstreamAddrTpl = `server %s;` var serverListenTpl = `listen 0.0.0.0:%s%s%s;` var addrAnyPropertiesTpl = " default_server backlog=65536 reuseport" var serverNameTpl = `server_name %s;` var certTpl = `ssl_certificate /etc/nginx/certs/%s;` var certKeyTpl = `ssl_certificate_key /etc/nginx/certs/%s;` var backendTpl = `backend_%d` ================================================ FILE: management/tcontrollerd/controller/website.go ================================================ package controller import ( "context" "encoding/json" "fmt" "io" "io/fs" "io/ioutil" "net/url" "os" "path/filepath" "strings" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/model" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/ngcmd" pb "chaitin.cn/patronus/safeline-2/management/tcontrollerd/proto/website" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/utils" ) const ( Ping = "ping" Pong = "pong" EventTypeWebsite = "website" EventTypeDeleteWebsite = "deleteWebsite" EventTypeFullWebsite = "fullWebsite" nginxConfigPath = "/etc/nginx/sites-enabled/" // 修改的为 host /resources/nginx/ 中的文件 nginxFilePrefix = "IF_backend_" nginxBackupFilePrefix = "BAK_IF_backend_" nginxFileMode = 0644 HttpsScheme = "https" DefaultHttpsPort = "443" ) var pong = pb.Response{Type: Pong, Msg: nil, Err: false} func generateNginxConfig(website *model.WebsiteConfig) (string, error) { // only ONE upstream supported in v1.0 var upstreamAddr string scheme := "http" for _, upstream := range website.Upstreams { urlInfo, err := url.Parse(upstream) if err != nil { return "", err } if urlInfo.Scheme == HttpsScheme && urlInfo.Port() == "" { urlInfo.Host = urlInfo.Host + ":" + DefaultHttpsPort } upstreamAddr = fmt.Sprintf(upstreamAddrTpl, urlInfo.Host) if len(urlInfo.Scheme) > 0 { scheme = urlInfo.Scheme } } sslFlag := "" sslCertFilename := "" sslCertKeyFilename := "" if website.KeyFilename != "" && website.CertFilename != "" { sslFlag = " ssl" // with a blank ahead sslCertFilename = fmt.Sprintf(certTpl, website.CertFilename) sslCertKeyFilename = fmt.Sprintf(certKeyTpl, website.KeyFilename) } // only ONE server name supported in v1.0 var serverName string var addrAnyProperties string for _, sn := range website.ServerNames { if sn == "*" || sn == "" { sn = "_" addrAnyProperties = addrAnyPropertiesTpl } serverName = fmt.Sprintf(serverNameTpl, sn) } // only ONE port supported in v1.0 var serverListen string for _, port := range website.Ports { serverListen = fmt.Sprintf(serverListenTpl, port, sslFlag, addrAnyProperties) } upstreamName := fmt.Sprintf(backendTpl, website.Id) nginxConfig := strings.TrimSpace(fmt.Sprintf(nginxConfigTpl, upstreamName, upstreamAddr, serverListen, serverName, sslCertFilename, sslCertKeyFilename, scheme, upstreamName, website.Id)) return nginxConfig, nil } func nginxTestAndReload() error { err := ngcmd.NginxConfTest() if err != nil { return err } err = ngcmd.NginxConfReload() if err != nil { return err } return nil } func generateFullConfigAndReload(msg []byte) error { var websites []model.WebsiteConfig if err := json.Unmarshal(msg, &websites); err != nil { return err } configFilename := make(map[string]struct{}) for _, website := range websites { configFilename[fmt.Sprintf("%s%d", nginxFilePrefix, website.Id)] = struct{}{} } filepath.Walk(nginxConfigPath, func(path string, info fs.FileInfo, err error) error { _, ok := configFilename[info.Name()] if !ok && strings.HasPrefix(info.Name(), nginxFilePrefix) { if err := os.Remove(path); err != nil { // not return error only logged error in order to delete not exist website nginx conf logger.Warn(err) } } return nil }) for _, website := range websites { if err := generateConfigAndReload(&website); err != nil { // trigger a full site push when tcd starts, ignore some site errors, and push as much as possible logger.Warn(err) } } return nil } func generateConfigAndReload(website *model.WebsiteConfig) error { nginxConfig, err := generateNginxConfig(website) if err != nil { return err } configPath := filepath.Join(nginxConfigPath, fmt.Sprintf("%s%d", nginxFilePrefix, website.Id)) backupPath := filepath.Join(nginxConfigPath, fmt.Sprintf("%s%d", nginxBackupFilePrefix, website.Id)) oldConfigExists, err := utils.FileExist(configPath) if err != nil { return err } if oldConfigExists { oldConfig, err := ioutil.ReadFile(configPath) if err != nil { return err } if string(oldConfig) == nginxConfig { logger.Info("No changes in the new website config, skip nginx -s reload") return nil } // tmp save old config to the backup path if err = utils.CopyFile(configPath, backupPath); err != nil { return err } } if err = utils.EnsureWriteFile(configPath, []byte(nginxConfig), nginxFileMode); err != nil { return err } if err = nginxTestAndReload(); err != nil { nginxError := err if err = os.Remove(configPath); err != nil { return err } if oldConfigExists { // new config err, restore old config if err = utils.CopyFile(backupPath, configPath); err != nil { return err } if err = os.Remove(backupPath); err != nil { return err } } return nginxError } if oldConfigExists { if err = os.Remove(backupPath); err != nil { return err } } return nil } func deleteConfigAndReload(config []byte) error { var websiteIds []uint if err := json.Unmarshal(config, &websiteIds); err != nil { return err } for _, id := range websiteIds { configPath := filepath.Join(nginxConfigPath, fmt.Sprintf("%s%d", nginxFilePrefix, id)) exists, err := utils.FileExist(configPath) if err != nil { return err } if exists { if err := os.Remove(configPath); err != nil { return err } } } if err := nginxTestAndReload(); err != nil { return err } return nil } func sendResponse(stream pb.Website_SubscribeClient, eventType string, errMsg []byte) error { return stream.Send(&pb.Response{ Type: eventType, Msg: errMsg, Err: len(errMsg) != 0, }) } func websiteHandler(wc pb.WebsiteClient) error { stream, err := wc.Subscribe(context.Background()) if err != nil { logger.Errorf("Subscribe failed: %v", err) return err } for { event, err := stream.Recv() if err != nil { if err == io.EOF { // read done. logger.Infof("Recv EOF from webserver") return nil } else { logger.Errorf("Handle failed: %v", err) return err } } logger.Debugf("Got message Type %s, Msg: %s", event.GetType(), event.GetMsg()) if event.Type == Ping { if err = stream.Send(&pong); err != nil { return err } } else if event.Type == EventTypeWebsite { logger.Infof("Update website with config: %s", event.GetMsg()) var website model.WebsiteConfig if err := json.Unmarshal(event.GetMsg(), &website); err != nil { return err } if err = generateConfigAndReload(&website); err != nil { logger.Error(err) if err := sendResponse(stream, EventTypeDeleteWebsite, []byte(err.Error())); err != nil { return err } } else { if err = sendResponse(stream, EventTypeDeleteWebsite, nil); err != nil { return err } } } else if event.Type == EventTypeFullWebsite { if err = generateFullConfigAndReload(event.Msg); err != nil { logger.Error(err) sendErr := sendResponse(stream, EventTypeFullWebsite, []byte(err.Error())) if sendErr != nil { return sendErr } } else { if err = sendResponse(stream, EventTypeFullWebsite, nil); err != nil { return err } } } else if event.Type == EventTypeDeleteWebsite { if err = deleteConfigAndReload(event.Msg); err != nil { logger.Error(err) sendErr := sendResponse(stream, EventTypeDeleteWebsite, []byte(err.Error())) if sendErr != nil { return sendErr } } else { if err = sendResponse(stream, EventTypeDeleteWebsite, nil); err != nil { return err } } } } } ================================================ FILE: management/tcontrollerd/go.mod ================================================ module chaitin.cn/patronus/safeline-2/management/tcontrollerd go 1.21 toolchain go1.21.3 require ( chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6 chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.9.3 google.golang.org/grpc v1.65.0 ) require ( golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect ) ================================================ FILE: management/tcontrollerd/go.sum ================================================ chaitin.cn/dev/go/errors v0.0.0-20200717101723-df6132d53dc8/go.mod h1:u+ZD0shdyUt0UG9XfCgD1R5mSqXpTVoO5rwQXQeo3Eo= chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6 h1:1Qa9ABk907/9ZrOLbbRcS8Fqq9VhjAF/mLjbSP1qAJY= chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6/go.mod h1:u+ZD0shdyUt0UG9XfCgD1R5mSqXpTVoO5rwQXQeo3Eo= chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c h1:Xn9IYkxmnpDcEpV+7JIR5ufEIexd1dhqKwpOLG1mYOE= chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c/go.mod h1:xJIYwUoA2TX5mNg/RBrEPyE251BPwj+70/mM7UIhoxg= chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c h1:tXsraF7o9iUsQY6IwpDJusc6OFhB7iv/bBTfgR3MPUU= chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c/go.mod h1:fUvtmpG8Z8Zf5aciadL9a/vn5SB3knG7pdNJixDplPg= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.39.0-dev h1:K4VkkiYp4LCvQiW6OiGglzm5nO4Zyryf7pHhzP15cmI= google.golang.org/grpc v1.39.0-dev/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= ================================================ FILE: management/tcontrollerd/main.go ================================================ package main import ( "context" "flag" "fmt" "os" "os/signal" "strconv" "syscall" "time" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/controller" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/cron" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/ngcmd" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/config" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/constants" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/log" ) var ( logger = log.GetLogger("main") version = "undefined" githash = "undefined" buildstamp = "undefined" goVersion = "undefined" ) func init() { // do something that do not raise error } func handleLoop(ctx context.Context) { for { select { case <-ctx.Done(): os.Exit(0) default: if err := controller.Handle(); err != nil { logger.Error("Error occurred when handling controller: ", err) } time.Sleep(time.Second * 5) } } } func main() { log.SetLogFormatter() fs := flag.NewFlagSet("tcontrollerd", flag.ExitOnError) showVersion := fs.Bool("v", false, "show version") nginxTest := fs.Bool("t", false, "nginx -t") nginxReload := fs.Bool("r", false, "nginx -s reload") cfgFile := fs.String("c", constants.ConfigFilePath, "config file path") if err := fs.Parse(os.Args[1:]); err != nil { logger.Fatalln("Failed to parse args: ", err) } if *showVersion { i, _ := strconv.Atoi(buildstamp) t := time.Unix(int64(i), 0).Format("2006-01-02 15:04:05") fmt.Println("Version: ", version) fmt.Println("Githash: ", githash) fmt.Println("Build: ", t) fmt.Println("Go version: ", goVersion) return } // init configs if err := config.InitConfigs(*cfgFile); err != nil { logger.Fatalln("Failed to init configs: ", err) } if err := log.InitLogger(); err != nil { logger.Fatalln("Failed to init db: ", err) } if *nginxTest { if err := ngcmd.NginxConfTest(); err != nil { logger.Fatalln("Failed to test nginx conf: ", err) } return } if *nginxReload { if err := ngcmd.NginxConfReload(); err != nil { logger.Fatalln("Failed to reload nginx conf: ", err) } return } if err := cron.StartCron(); err != nil { logger.Fatalln("Failed to start cron: ", err) } ctx, cancel := context.WithCancel(context.Background()) sig := make(chan os.Signal, 1) signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) go func() { <-sig cancel() }() handleLoop(ctx) } ================================================ FILE: management/tcontrollerd/model/website.go ================================================ package model // WebsiteConfig is supposed to be same with webserver/model/website.go type WebsiteConfig struct { Id int `json:"id"` ServerNames []string `json:"server_names"` Ports []string `json:"ports"` Upstreams []string `json:"upstreams"` CertFilename string `json:"cert_filename"` KeyFilename string `json:"key_filename"` } ================================================ FILE: management/tcontrollerd/pkg/config/config.go ================================================ package config import ( "os" "chaitin.cn/dev/go/settings" ) var ( GlobalConfig = DefaultGlobalConfig() ) func InitConfigs(configFilePath string) error { s, err := settings.New(configFilePath) if err != nil { return err } if err = GlobalConfig.Log.Load(s); err != nil { return err } if err := s.Unmarshal("mgt_addr", &GlobalConfig.MgtWebserver); err != nil { return err } if v, ok := os.LookupEnv("MGT_ADDR"); ok { GlobalConfig.MgtWebserver = v } return nil } ================================================ FILE: management/tcontrollerd/pkg/config/global.go ================================================ package config type Config struct { Log LogConfig MgtWebserver string } func DefaultGlobalConfig() Config { return Config{ Log: DefaultLogConfig(), MgtWebserver: "169.254.0.4:9002", } } ================================================ FILE: management/tcontrollerd/pkg/config/log.go ================================================ package config import ( "chaitin.cn/dev/go/settings" ) type LogConfig struct { Output string `yaml:"output"` Level string `yaml:"level"` } func DefaultLogConfig() LogConfig { return LogConfig{ Output: "stdout", Level: "info", } } func (lc *LogConfig) Load(setting *settings.Setting) error { if err := setting.Unmarshal("log", lc); err != nil { return err } return nil } ================================================ FILE: management/tcontrollerd/pkg/constants/constants.go ================================================ package constants const ( ConfigFilePath = "config.yml" ) ================================================ FILE: management/tcontrollerd/pkg/constants/forbidden_page.go ================================================ package constants const ( DefaultForbiddenPageMd5 = "d9921f84f36a6cc92a6fc13946a18e98" DefaultForbiddenPage = `` ) ================================================ FILE: management/tcontrollerd/pkg/cron/cron.go ================================================ package cron import ( "github.com/robfig/cron/v3" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/log" ) var logger = log.GetLogger("cron") func newCronWithSeconds() *cron.Cron { secondParser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.DowOptional | cron.Descriptor) return cron.New(cron.WithParser(secondParser), cron.WithChain()) } func StartCron() error { cronInstance := newCronWithSeconds() _, err := cronInstance.AddFunc(specCheckForbiddenPage, checkAndUpdateForbiddenPage) if err != nil { return err } cronInstance.Start() return nil } ================================================ FILE: management/tcontrollerd/pkg/cron/forbidden_page.go ================================================ package cron import ( "crypto/md5" "encoding/hex" "io/ioutil" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/constants" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/utils" ) const ( // SpecUpdatePolicy http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html // Seconds Minutes Hours Day-of-Month Month Day-of-Week Year (optional field) // every 30 second (starting from 0s) specCheckForbiddenPage = "0/30 * * * * ?" forbiddenPagePath = "/etc/nginx/forbidden_pages/default_forbidden_page.html" ) func checkAndUpdateForbiddenPage() { existed, err := utils.FileExist(forbiddenPagePath) if err != nil { logger.Error(err) return } if !existed { err = utils.EnsureWriteFile(forbiddenPagePath, []byte(constants.DefaultForbiddenPage), 0644) if err != nil { logger.Error(err) } return } content, err := ioutil.ReadFile(forbiddenPagePath) if err != nil { logger.Error(err) return } hash := md5.New() hash.Write([]byte(content)) forbiddenMd5 := hex.EncodeToString(hash.Sum(nil)) if forbiddenMd5 == constants.DefaultForbiddenPageMd5 { return } err = ioutil.WriteFile(forbiddenPagePath, []byte(constants.DefaultForbiddenPage), 0644) if err != nil { logger.Error(err) return } } ================================================ FILE: management/tcontrollerd/pkg/log/log.go ================================================ package log import ( "fmt" "os" "runtime" "strings" "github.com/sirupsen/logrus" "chaitin.cn/dev/go/log" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/config" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/utils" ) func GetLogger(name string) *log.Logger { return log.GetLogger(name) } func LoadLogLevel() { lv, _ := log.ParseLevel(config.GlobalConfig.Log.Level) log.SetLevel(log.AllLoggers, lv) } func SetLogFormatter() { // format formatter := new(log.TextFormatter) formatter.FullTimestamp = true formatter.TimestampFormat = "2006/01/02 15:04:05" log.SetFormatter(log.AllLoggers, formatter) } func InitLogger() error { // output switch config.GlobalConfig.Log.Output { case "stdout": log.SetOutput(log.AllLoggers, os.Stdout) case "stderr": log.SetOutput(log.AllLoggers, os.Stderr) default: exist, err := utils.FileExist(config.GlobalConfig.Log.Output) if err != nil { return err } fileFlag := os.O_WRONLY | os.O_APPEND | os.O_SYNC if !exist { if err := utils.EnsureFileDir(config.GlobalConfig.Log.Output); err != nil { return err } fileFlag = fileFlag | os.O_CREATE } if fp, err := os.OpenFile(config.GlobalConfig.Log.Output, fileFlag, os.ModePerm); err != nil { return fmt.Errorf("failed to open log file: %s", err.Error()) } else { log.SetOutput(log.AllLoggers, log.NewLockOutput(fp)) } } // hook log.AddHook(log.AllLoggers, NewRuntimeHook()) log.AddHook(log.AllLoggers, log.NewErrorStackHook(true)) // level LoadLogLevel() return nil } type RuntimeHook struct{} func (h *RuntimeHook) Levels() []logrus.Level { return logrus.AllLevels } func (h *RuntimeHook) Fire(entry *logrus.Entry) error { file := "???" funcName := "???" line := 0 pc := make([]uintptr, 64) // Skip runtime.Callers, self, and another call from logrus n := runtime.Callers(3, pc) if n != 0 { pc = pc[:n] // pass only valid pcs to runtime.CallersFrames frames := runtime.CallersFrames(pc) // Loop to get frames. // A fixed number of pcs can expand to an indefinite number of Frames. for { frame, more := frames.Next() if !strings.Contains(frame.File, "github.com/sirupsen/logrus") && !strings.Contains(frame.Function, "chaitin.cn/dev/go") { file = frame.File funcName = frame.Function line = frame.Line break } if !more { break } } } slices := strings.Split(file, "/") file = slices[len(slices)-1] funcName = strings.ReplaceAll(funcName, "chaitin.cn", "") entry.Data["file"] = file entry.Data["func"] = funcName entry.Data["line"] = line return nil } func NewRuntimeHook() *RuntimeHook { return &RuntimeHook{} } ================================================ FILE: management/tcontrollerd/pkg/ngcmd/ngcmd.go ================================================ package ngcmd import ( "fmt" "os/exec" "strings" "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/tcontrollerd/pkg/log" ) var logger = log.GetLogger("ngcmd") // NginxConfTest exec nginx -t and return stderr func NginxConfTest() error { out, err := exec.Command("nginx", "-t").CombinedOutput() logger.Debugf("nginx -t output: %v", string(out)) if err != nil { return errors.Wrap(err, string(out)) } //logger.Debugf("nginx -t output: %v", out) if strings.Contains(string(out), "syntax is ok") && strings.Contains(string(out), "test is successful") { return nil } else { return errors.New(fmt.Sprintf("nginx conf test error: %s", string(out))) } } // NginxConfReload exec nginx -t and return stderr func NginxConfReload() error { out, err := exec.Command("nginx", "-s", "reload").CombinedOutput() logger.Debugf("nginx -s reload output: %v", string(out)) if err != nil { return errors.Wrap(err, string(out)) } if len(out) == 0 { return nil } else { return errors.New(fmt.Sprintf("nginx conf reload error: %s", string(out))) } } ================================================ FILE: management/tcontrollerd/proto/website/website.proto ================================================ syntax = "proto3"; package website; option go_package = "proto/website"; service Website { rpc Subscribe(stream Response) returns (stream Event) {} } // From client-side, may be "pong" message Response { string type = 1; bytes msg = 2; bool err = 3; } // From server-side, may be "ping" message Event { string type = 1; // ping/website bytes msg = 2; } ================================================ FILE: management/tcontrollerd/utils/file.go ================================================ package utils import ( "fmt" "io" "io/ioutil" "os" "path/filepath" ) func EnsureDir(dir string) error { if _, err := os.Stat(dir); os.IsNotExist(err) { return os.MkdirAll(dir, os.FileMode(0755)) } return nil } func EnsureFileDir(path string) error { return EnsureDir(filepath.Dir(path)) } func FileExist(path string) (bool, error) { stat, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return false, nil } else { return false, err } } else { if stat.IsDir() { return false, fmt.Errorf("%s is dir", path) } else { return true, nil } } } func FilesExist(paths ...string) (bool, error) { for _, path := range paths { exist, err := FileExist(path) if err != nil { return false, err } if !exist { return false, nil } } return true, nil } func RenameWriteFile(filename string, data []byte, perm os.FileMode) error { randFileName := filename + ".tmp." + RandStr(8) if err := ioutil.WriteFile(randFileName, data, perm); err != nil { return err } return os.Rename(randFileName, filename) } func EnsureRenameWriteFile(path string, data []byte, mode os.FileMode) error { err := EnsureFileDir(path) if err != nil { return err } return RenameWriteFile(path, data, mode) } func EnsureWriteFile(path string, data []byte, mode os.FileMode) error { err := EnsureFileDir(path) if err != nil { return err } return ioutil.WriteFile(path, data, mode) } func CopyFile(srcPath, dstPath string) error { srcFile, err := os.Open(srcPath) if err != nil { return err } defer func(srcFile *os.File) { err := srcFile.Close() if err != nil { } }(srcFile) fileInfo, err := srcFile.Stat() if err != nil { return err } return CopyFileFromIO(srcFile, dstPath, fileInfo.Mode()) } func CopyFileFromIO(src io.Reader, dstPath string, perm os.FileMode) error { if err := EnsureFileDir(dstPath); err != nil { return err } dstFile, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, perm) if err != nil { return err } defer func(dstFile *os.File) { err := dstFile.Close() if err != nil { } }(dstFile) _, err = io.Copy(dstFile, src) return err } func CopyFileIfNotExist(srcPath, dstPath string) error { if exist, err := FileExist(dstPath); err != nil { return err } else if !exist { return CopyFile(srcPath, dstPath) } else { return nil } } ================================================ FILE: management/tcontrollerd/utils/random.go ================================================ package utils import ( "math/rand" "time" ) var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") func RandStr(n int) string { rand.Seed(time.Now().UnixNano()) b := make([]rune, n) for i := range b { b[i] = letters[rand.Intn(len(letters))] } return string(b) } ================================================ FILE: management/webserver/.gitignore ================================================ tmp Taskfile.yml .air.toml ================================================ FILE: management/webserver/README.md ================================================ # Web Server web server for mgt-api ## Requirements Go 1.18+ ## Development ### Init protobuf ```shell # Refer: https://grpc.io/docs/languages/go/quickstart/ # 1. Install protoc # https://grpc.io/docs/protoc-installation/ # 2. Install Go plugins go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 # 3. Update your PATH so that the protoc compiler can find the plugins # export PATH="$PATH:$(go env GOPATH)/bin" # 4. Generate proto go code # cd /path/to/management ./scripts/genproto.sh ``` ### Init fvm libs ```shell # Due to the fvm c header files mkdir -p management/webserver/submodule/fvm/ mkdir -p management/webserver/submodule/libct/ cd management/webserver/submodule/fvm/ # Download https://chaitin.cn/patronus/fvm/-/tags 1.8.21 release:release, https://chaitin.cn/patronus/fvm/-/jobs/6716645 unzip artifacts.zip rm artifacts.zip cd management/webserver/submodule/libct/ # Download https://chaitin.cn/patronus/libct/-/tags 1.1.1.0 release, https://chaitin.cn/patronus/libct/-/jobs/7229201 # rename rm artifacts.zip cd management/webserver/submodule/ # Download https://chaitin.cn/patronus/fusion-2/-/tags 5.3.9-r1 build:release, https://chaitin.cn/patronus/fusion-2/-/jobs/7326007 # rename unzip artifacts.zip mv artifacts/lib/libfusion.so libfvm.so rm artifacts.zip rm -r artifacts/ ``` ### Build ```shell cd management/ docker run -it --rm -w="/mnt" --mount type=bind,source="$(pwd)",target=/mnt chaitin.cn/ci/golang:1.18 bash cp webserver/submodule/libfvm.so /usr/lib/ make build-webserver ``` ================================================ FILE: management/webserver/api/auth.go ================================================ package api import ( "math" "net/http" "time" "github.com/gin-contrib/sessions" "github.com/gin-gonic/gin" "github.com/pquerna/otp" "github.com/pquerna/otp/totp" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/log" ) var logger = log.GetLogger("api") var OtpOpts = totp.GenerateOpts{ Issuer: constants.ProductName, AccountName: constants.SuperUser, Period: 30, // seconds Digits: otp.DigitsSix, Algorithm: otp.AlgorithmSHA1, } type PostLoginRequest struct { Passcode string `json:"passcode"` Timestamp int64 `json:"timestamp"` } func PostLogin(c *gin.Context) { var params PostLoginRequest if err := c.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() // only SuperUser in v0.9 var user model.User db.Where(&model.User{Username: constants.SuperUser}).First(&user) valid := totp.Validate(params.Passcode, user.TFASecret) if !valid { millisecondTimestamp := params.Timestamp localTimeStamp := time.Now() if millisecondTimestamp > 0 { logger.Debugf("will valid otp frontend timestamp:%v, local timestamp:%v", millisecondTimestamp, localTimeStamp) otpTime := time.Unix(millisecondTimestamp/1000, (millisecondTimestamp%1000)*int64(time.Millisecond)) timeSub := localTimeStamp.Sub(otpTime) seconds := math.Abs(timeSub.Seconds()) if seconds >= 60 { logger.Errorf("otp timestamp gap is more than a minute") response.Error(c, response.JSONBody{Err: response.ErrWrongTimeGap, Msg: "otp timestamp gap is more than a minute"}, http.StatusUnauthorized) return } } response.Error(c, response.JSONBody{Err: response.ErrWrongPasscode, Msg: "Failed to verify your passcode"}, http.StatusUnauthorized) return } user.LastLoginTime = time.Now().Unix() user.IsEnabled = true db.Save(&user) session := sessions.Default(c) session.Options(sessions.Options{ Path: "/", MaxAge: 3600 * 24 * 7, //Domain: options.Domain, //HttpOnly: true, //SameSite: http.SameSiteLaxMode, //Secure: false, }) session.Set(constants.DefaultSessionUserKey, user.ID) if err := session.Save(); err != nil { logger.Error(err) response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when creating sessions"}, http.StatusInternalServerError) return } response.Success(c, nil) } func PostLogout(c *gin.Context) { session := sessions.Default(c) session.Clear() if err := session.Save(); err != nil { response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when creating sessions"}, http.StatusInternalServerError) return } response.Success(c, nil) } func GetOTPUrl(c *gin.Context) { otpKey, err := totp.Generate(OtpOpts) if err != nil { logger.Error(err) response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when generating otp qrcode"}, http.StatusInternalServerError) return } db := database.GetDB() // only SuperUser in v0.9 user := model.User{Username: constants.SuperUser} db.First(&user) if user.LastLoginTime > 0 { // already bind tfa, because tfa binding is mandatory when login. response.Success(c, gin.H{"url": ""}) return } user.TFASecret = otpKey.Secret() db.Save(&user) response.Success(c, gin.H{"url": otpKey.URL()}) } func GetUser(c *gin.Context) { db := database.GetDB() user := model.User{Username: constants.SuperUser} db.First(&user) response.Success(c, gin.H{"id": user.ID, "username": user.Username}) } ================================================ FILE: management/webserver/api/behaviour.go ================================================ package api import ( "net/http" "github.com/gin-gonic/gin" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" ) type PostBehaviourRequest struct { model.Behaviour } func PostBehaviour(c *gin.Context) { var params PostBehaviourRequest if err := c.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() db.Create(&model.Behaviour{SrcRouter: params.SrcRouter, DstRouter: params.DstRouter}) response.Success(c, nil) } ================================================ FILE: management/webserver/api/cert.go ================================================ package api import ( "crypto/x509/pkix" "fmt" "net/http" "path/filepath" "github.com/gin-gonic/gin" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) type postSSLCertRequest struct { Hostname string `json:"hostname"` } // SSLCertDir is the dir of tengine conf, not mgt-api nginx certs dir defined by constants.CertsPath const ( CRT = ".crt" PEM = ".pem" KEY = ".key" SSLCertDir = "certs" ) func PostUploadSSLCert(c *gin.Context) { file, err := c.FormFile("file") if err != nil { logger.Error(err) response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) return } switch filepath.Ext(file.Filename) { case CRT, PEM, KEY: logger.Debugf("File: %v is valid", file.Filename) default: logger.Errorf("Filename: %s, ext: %s", file.Filename, filepath.Ext(file.Filename)) response.Error(c, response.JSONBody{Err: response.ErrWrongFileType, Msg: "Wrong file type, please upload a file of .crt or .key"}, http.StatusUnsupportedMediaType) return } var dstPath string filename := fmt.Sprintf("%s_%s", utils.RandStr(16), file.Filename) if config.GlobalConfig.Server.DevMode { dstPath = filepath.Join("./nginx", SSLCertDir, filename) } else { dstPath = filepath.Join(config.GlobalConfig.NgxResDir, SSLCertDir, filename) } if err = c.SaveUploadedFile(file, dstPath); err != nil { logger.Error(err) response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when saving file"}, http.StatusInternalServerError) return } response.Success(c, gin.H{"filename": filename}) } func PostSSLCert(c *gin.Context) { var params postSSLCertRequest if err := c.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) return } filePrefix := utils.RandStr(16) certFilename := fmt.Sprintf("%s_backend.crt", filePrefix) keyFilename := fmt.Sprintf("%s_backend.key", filePrefix) var certPath, keyPath string if config.GlobalConfig.Server.DevMode { certPath = filepath.Join("./management", SSLCertDir, certFilename) keyPath = filepath.Join("./management", SSLCertDir, keyFilename) } else { certPath = filepath.Join(config.GlobalConfig.NgxResDir, SSLCertDir, certFilename) keyPath = filepath.Join(config.GlobalConfig.NgxResDir, SSLCertDir, keyFilename) } if err := utils.WriteCertIfNotExist( certPath, keyPath, func() ([]byte, []byte, error) { return utils.GenerateCert( []string{params.Hostname}, 3650, 4096, &pkix.Name{ Country: []string{}, Province: []string{}, Locality: []string{}, Organization: []string{}, OrganizationalUnit: []string{}, CommonName: "", }, false, ) }); err != nil { logger.Error(err) response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when generating certs"}, http.StatusInternalServerError) return } response.Success(c, gin.H{"crt": certFilename, "key": keyFilename}) } ================================================ FILE: management/webserver/api/common.go ================================================ package api import ( "encoding/json" "fmt" "io/ioutil" "net/http" "strings" "github.com/gin-gonic/gin" "github.com/rogpeppe/go-internal/semver" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) const VersionInfoEntrypoint = "/release/latest/version.json" type idsRequest struct { IDs []uint `json:"ids" form:"ids"` } type pageRequest struct { Page int `json:"page" form:"page,default=1" binding:"min=1"` PageSize int `json:"page_size" form:"page_size,default=10" binding:"min=1"` } type versionInfoResponse struct { LatestVersion string `json:"latest_version"` RecVersion string `json:"rec_version"` } func GetVersion(c *gin.Context) { response.Success(c, gin.H{"version": strings.TrimPrefix(constants.Version, "ce-")}) } func GetUpgradeTips(ctx *gin.Context) { client := utils.GetHTTPClient() logger.Debugf("GetUpgradeTips: %s", config.GlobalConfig.PlatformAddr+VersionInfoEntrypoint) versionInfoReq, err := http.NewRequest(http.MethodGet, config.GlobalConfig.PlatformAddr+VersionInfoEntrypoint, nil) if err != nil { logger.Warn(err) response.Success(ctx, gin.H{"upgrade_tips": constants.NotUpgrade}) return } versionInfoRsp, err := client.Do(versionInfoReq) if err != nil { logger.Warn(err) response.Success(ctx, gin.H{"upgrade_tips": constants.NotUpgrade}) return } body, err := ioutil.ReadAll(versionInfoRsp.Body) if err != nil { logger.Warn(err) response.Success(ctx, gin.H{"upgrade_tips": constants.NotUpgrade}) return } versionInfo := &versionInfoResponse{} err = json.Unmarshal(body, versionInfo) if err != nil { logger.Warnf("err: %v, body: %s", err, body) response.Success(ctx, gin.H{"upgrade_tips": constants.NotUpgrade}) return } currentVersion := fmt.Sprintf("v%s", constants.Version) latestVersionCmp := semver.Compare(currentVersion, versionInfo.LatestVersion) recVersionCmp := semver.Compare(currentVersion, versionInfo.RecVersion) if semver.Compare(versionInfo.LatestVersion, versionInfo.RecVersion) == -1 || latestVersionCmp == 1 { logger.Warnf("The version number is invalid, current version: %s, latest version: %s, rec version: %s", currentVersion, versionInfo.LatestVersion, versionInfo.RecVersion) response.Success(ctx, gin.H{"upgrade_tips": constants.NotUpgrade}) return } var upgradeTips int if recVersionCmp == -1 { upgradeTips = constants.MustUpgrade } else if recVersionCmp == 0 { if latestVersionCmp == 0 { upgradeTips = constants.NotUpgrade } else { upgradeTips = constants.RecommendedUpgrade } } else { if latestVersionCmp < 0 { upgradeTips = constants.RecommendedUpgrade } else { upgradeTips = constants.NotUpgrade } } response.Success(ctx, gin.H{"upgrade_tips": upgradeTips}) } ================================================ FILE: management/webserver/api/dashboard.go ================================================ package api import ( "math" "time" "github.com/gin-gonic/gin" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" ) func GetDashboardCounts(c *gin.Context) { var requested int64 = 0 var intercepted int64 = 0 db := database.GetDB() var statisticTotalReq model.SystemStatistics r := db.Where("type = 'total-req'").Where("created_at >= date_trunc('day',now())").First(&statisticTotalReq) if r.RowsAffected > 0 { requested = statisticTotalReq.Value } var statisticTotalDenied model.SystemStatistics r = db.Where("type = 'total-denied'").Where("created_at >= date_trunc('day',now())").First(&statisticTotalDenied) if r.RowsAffected > 0 { intercepted = statisticTotalDenied.Value } response.Success(c, gin.H{"requested": requested, "intercepted": intercepted}) } func GetDashboardSites(c *gin.Context) { response.Success(c, gin.H{"normal": 0, "abnormal": 0}) } func GetDashboardQps(c *gin.Context) { var statistics []model.SystemStatistics db := database.GetDB() db.Where("type = 'req'").Order("created_at desc").Limit(75).Find(&statistics) type Node struct { Label string `json:"label"` Value int64 `json:"value"` } var nodes = make([]Node, 0) for i := len(statistics) - 1; i >= 0; i-- { nodes = append(nodes, Node{ Label: statistics[i].CreatedAt.Format("2006-01-02 15:04:05"), Value: int64(math.Ceil(float64(statistics[i].Value) / 5)), }) } response.Success(c, gin.H{"nodes": nodes, "total": len(nodes)}) } func GetDashboardRequests(c *gin.Context) { var statistics []model.SystemStatistics db := database.GetDB() db.Where("type = 'total-req'").Order("created_at desc").Limit(30).Find(&statistics) type Node struct { Label string `json:"label"` Value int64 `json:"value"` } var nodes = make([]Node, 0) for i := 30; i > len(statistics); i-- { nodes = append(nodes, Node{ Label: time.Now().Add(-time.Duration(i-1) * time.Hour * 24).Format("2006-01-02"), Value: 0, }) } for i := len(statistics) - 1; i >= 0; i-- { nodes = append(nodes, Node{ Label: statistics[i].CreatedAt.Format("2006-01-02"), Value: statistics[i].Value, }) } response.Success(c, gin.H{"nodes": nodes, "total": len(nodes)}) } func GetDashboardIntercepts(c *gin.Context) { var statistics []model.SystemStatistics db := database.GetDB() db.Where("type = 'total-denied'").Order("created_at desc").Limit(30).Find(&statistics) type Node struct { Label string `json:"label"` Value int64 `json:"value"` } var nodes = make([]Node, 0) for i := 30; i > len(statistics); i-- { nodes = append(nodes, Node{ Label: time.Now().Add(-time.Duration(i-1) * time.Hour * 24).Format("2006-01-02"), Value: 0, }) } for i := len(statistics) - 1; i >= 0; i-- { nodes = append(nodes, Node{ Label: statistics[i].CreatedAt.Format("2006-01-02"), Value: statistics[i].Value, }) } response.Success(c, gin.H{"nodes": nodes, "total": len(nodes)}) } ================================================ FILE: management/webserver/api/detectlog.go ================================================ package api import ( "bytes" "encoding/json" "fmt" "net/http" "strconv" "strings" "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/webserver/pkg" "github.com/gin-gonic/gin" "chaitin.cn/dev/go/log" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) type ( GetDetectLogDetailRequest struct { EventId string `json:"event_id" form:"event_id"` } PostFalsePositivesRequest struct { EventId string `json:"event_id"` } telemetryFalsePositives struct { Telemetry struct { Id string `json:"id"` } `json:"telemetry"` Safeline struct { Id string `json:"id"` Type string `json:"type"` DetectLog model.DetectLog `json:"detect_log"` } `json:"safeline"` } ) func getDetectLog(eventId string) (*model.DetectLog, error) { db := database.GetDB() var detectLogBasic model.DetectLogBasic res := db.Where(&model.DetectLogBasic{EventId: eventId}).First(&detectLogBasic) if res.RowsAffected == 0 { return nil, errors.New("Data queried does not exist") } var detectLogDetail model.DetectLogDetail db.Where(&model.DetectLogDetail{EventId: eventId}).First(&detectLogDetail) detectLog, err := model.TransformDetectLog(&detectLogBasic, &detectLogDetail) if err != nil { return nil, err } return detectLog, nil } func GetDetectLogList(c *gin.Context) { var params pageRequest if err := c.BindQuery(¶ms); err != nil { logger.Error(err) response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() tx := db.Where("") // 按 ip 搜索条件 if ip := c.Query("ip"); ip != "" { tx = tx.Where("src_ip = ?", ip) } // 按 url 搜索条件 if url := c.Query("url"); url != "" { tx = tx.Where("url_path like ?", "%"+url+"%") } // 按 type 搜索条件 if at := c.Query("attack_type"); at != "" { ns := make([]int, 0) for _, s := range strings.Split(at, ",") { n, err := strconv.Atoi(s) if err == nil { ns = append(ns, n) } } tx = tx.Where("attack_type in (?)", ns) } var total int64 tx.Model(&model.DetectLogBasic{}).Count(&total) var basicList []model.DetectLogBasic tx.Limit(params.PageSize).Offset(params.PageSize * (params.Page - 1)).Order("id desc").Find(&basicList) var dLogList []*model.DetectLog for _, basic := range basicList { dLog, err := model.TransformDetectLog(&basic, nil) if err != nil { logger.Warn(err) continue } dLogList = append(dLogList, dLog) } response.Success(c, gin.H{"data": dLogList, "total": total}) } func GetDetectLogDetail(c *gin.Context) { var params GetDetectLogDetailRequest if err := c.BindQuery(¶ms); err != nil { logger.Error(err) response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) return } detectLog, err := getDetectLog(params.EventId) if err != nil { logger.Error(err) response.Error(c, response.ErrorDataNotExist, http.StatusNotFound) return } response.Success(c, detectLog) } func PostFalsePositives(c *gin.Context) { var params PostFalsePositivesRequest if err := c.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(c, response.ErrorParamNotOK, http.StatusInternalServerError) return } detectLog, err := getDetectLog(params.EventId) if err != nil { logger.Error(err) response.Error(c, response.ErrorDataNotExist, http.StatusNotFound) return } db := database.GetDB() var option model.Options db.Where(&model.Options{Key: constants.MachineID}).First(&option) var jsonData telemetryFalsePositives jsonData.Telemetry.Id = constants.TelemetryId jsonData.Safeline.Id = option.Value jsonData.Safeline.Type = constants.FalsePositives jsonData.Safeline.DetectLog = *detectLog data, err := json.Marshal(jsonData) if err != nil { log.Warn(err) response.Success(c, nil) return } reader := bytes.NewReader(data) client := utils.GetHTTPClient() addr := config.GlobalConfig.Telemetry.Addr rsp, err := pkg.DoPostTelemetry(client, addr, reader) if err != nil { log.Warn(err) response.Success(c, nil) return } if rsp.StatusCode != http.StatusOK && rsp.StatusCode != http.StatusCreated { log.Warn(fmt.Sprintf("Transfer telemetry failed, status code = %d", rsp.StatusCode), err) response.Success(c, nil) return } response.Success(c, nil) } ================================================ FILE: management/webserver/api/endpoints.go ================================================ package api const ( Version = "/Version" UpgradeTips = "/UpgradeTips" Login = "/Login" Logout = "/Logout" OTPUrl = "/OTPUrl" User = "/User" DetectLogList = "/DetectLogList" DetectLogDetail = "/DetectLogDetail" Behaviour = "/Behaviour" FalsePositives = "/FalsePositives" Website = "/Website" UploadSSLCert = "/UploadSSLCert" SSLCert = "/SSLCert" PolicyRule = "/PolicyRule" SwitchPolicyRule = "/SwitchPolicyRule" DashboardCounts = "/dashboard/counts" DashboardSites = "/dashboard/sites" DashboardQps = "/dashboard/qps" DashboardRequests = "/dashboard/requests" DashboardIntercepts = "/dashboard/intercepts" PolicyGroupGlobal = "/PolicyGroupGlobal" SrcIPConfig = "/SrcIPConfig" ) ================================================ FILE: management/webserver/api/policygroup.go ================================================ package api import ( "encoding/json" "net/http" "github.com/gin-gonic/gin" "gorm.io/gorm" "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" ) func PutPolicyGroupGlobal(ctx *gin.Context) { var params model.PolicyGroup if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { var pggOption model.Options res := tx.Where(&model.Options{Key: constants.PolicyGroupGlobal}).First(&pggOption) if res.Error != nil { return res.Error } if res.RowsAffected == 0 { return errors.New("Data queried does not exist") } pggStr, err := json.Marshal(params) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return err } pggOption.Value = string(pggStr) tx.Save(&pggOption) if err := fvm.PushFSL(tx); err != nil { return errors.New("Rules compile error, please check your params.") } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, nil) } func GetPolicyGroupGlobal(ctx *gin.Context) { var pggOption model.Options database.GetDB().Where(&model.Options{Key: constants.PolicyGroupGlobal}).First(&pggOption) var pgg model.PolicyGroup err := json.Unmarshal([]byte(pggOption.Value), &pgg) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, gin.H{"data": pgg}) } func PutSrcIPConfig(ctx *gin.Context) { var params model.SrcIPConfig if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { var scOption model.Options res := tx.Where(&model.Options{Key: constants.SrcIPConfig}).First(&scOption) if res.Error != nil { return res.Error } if res.RowsAffected == 0 { return errors.New("Data queried does not exist") } scStr, err := json.Marshal(params) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return err } scOption.Value = string(scStr) tx.Save(&scOption) if err := fvm.PushFSL(tx); err != nil { return errors.New("Rules compile error, please check your params.") } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, nil) } func GetSrcIPConfig(ctx *gin.Context) { srcIPConfig, err := model.GetSrcIPConfig(database.GetDB().DB) if err != nil { return } response.Success(ctx, gin.H{"data": srcIPConfig}) } ================================================ FILE: management/webserver/api/policyrule.go ================================================ package api import ( "net/http" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" "github.com/gin-gonic/gin" "gorm.io/gorm" "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" ) type putSwitchRequest struct { IDs []uint `json:"ids"` IsEnabled bool `json:"is_enabled"` } func PostPolicyRule(ctx *gin.Context) { var params model.PolicyRule if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { policyRule := &model.PolicyRule{Action: params.Action, Comment: params.Comment, IsEnabled: params.IsEnabled, Pattern: params.Pattern} res := tx.Create(policyRule) if res.Error != nil { return res.Error } if err := fvm.PushFSL(tx); err != nil { return errors.New("Rules compile error, please check your params.") } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, nil) } func PutSwitchPolicyRule(ctx *gin.Context) { var params putSwitchRequest if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { res := tx.Model(&model.PolicyRule{}).Where(params.IDs).Updates(model.PolicyRule{IsEnabled: params.IsEnabled}) if res.Error != nil { return res.Error } if res.RowsAffected == 0 { return errors.New("Data queried does not exist") } if err := fvm.PushFSL(tx); err != nil { return errors.New("Rules compile error, please check your params.") } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } } func PutPolicyRule(ctx *gin.Context) { var params model.PolicyRule if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { var policyRule model.PolicyRule res := tx.Where(params.ID).First(&policyRule) if res.Error != nil { return res.Error } if res.RowsAffected == 0 { return errors.New("Data queried does not exist") } policyRule.Action = params.Action policyRule.Comment = params.Comment policyRule.IsEnabled = params.IsEnabled policyRule.Pattern = params.Pattern tx.Save(&policyRule) if err := fvm.PushFSL(tx); err != nil { return errors.New("Rules compile error, please check your params.") } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, nil) } func DeletePolicyRule(ctx *gin.Context) { var params idsRequest if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { res := tx.Where(params.IDs).Delete(&model.PolicyRule{}) if res.Error != nil { return res.Error } if res.RowsAffected == 0 { return errors.New("Data queried does not exist") } if err := fvm.PushFSL(tx); err != nil { return errors.New("Rules compile error, please check your params.") } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, nil) } func GetPolicyRule(ctx *gin.Context) { var params pageRequest if err := ctx.BindQuery(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() var policyRuleList []model.PolicyRule db.Limit(params.PageSize).Offset(params.PageSize * (params.Page - 1)).Order("id desc").Find(&policyRuleList) var total int64 db.Model(&model.PolicyRule{}).Count(&total) response.Success(ctx, gin.H{"data": policyRuleList, "total": total}) } ================================================ FILE: management/webserver/api/response/error.go ================================================ package response const ( ErrLoginRequired = "login-required" ErrWrongPasscode = "wrong-passcode" ErrWrongTimeGap = "wrong-time-gap" ErrInternalError = "internal-error" ErrDataNotExist = "data-not-exist" ErrWrongFileType = "wrong-filetype" ErrReadOnly = "read-only" ) ================================================ FILE: management/webserver/api/response/jsonbody.go ================================================ package response import ( "net/http" "github.com/gin-gonic/gin" ) type JSONBody struct { Err string Msg string Data interface{} } var ( ErrorLoginRequired = JSONBody{ErrLoginRequired, "Login required", nil} ErrorParamNotOK = JSONBody{ErrInternalError, "Error occurred when extracting params", nil} ErrorDataNotExist = JSONBody{ErrDataNotExist, "Data queried does not exist", nil} ErrorReadOnly = JSONBody{ErrReadOnly, "This environment is read only", nil} ) func Success(c *gin.Context, data interface{}) { c.JSON(http.StatusOK, gin.H{ "data": data, "msg": "", "err": nil, }) //c.Abort() } func SuccessWithList(c *gin.Context, data interface{}) { c.JSON(http.StatusOK, gin.H{ "data": data, "msg": "", "err": nil, }) //c.Abort() } func SuccessWithMsg(c *gin.Context, data interface{}, msg string) { c.JSON(http.StatusOK, gin.H{ "data": data, "msg": msg, "err": nil, }) //c.Abort() } func Error(c *gin.Context, rsp JSONBody, status int) { c.JSON(status, gin.H{ "data": rsp.Data, "msg": rsp.Msg, "err": rsp.Err, }) //c.Abort() } ================================================ FILE: management/webserver/api/response/png.go ================================================ package response import ( "net/http" "github.com/gin-gonic/gin" ) const StreamContentType = "application/octet-stream" func PNG(c *gin.Context, bytes []byte) { c.Data(http.StatusOK, StreamContentType, bytes) } ================================================ FILE: management/webserver/api/website.go ================================================ package api import ( "encoding/json" "net/http" "strconv" "chaitin.cn/dev/go/errors" "github.com/gin-gonic/gin" "gorm.io/gorm" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/rpc" ) func publishWebsiteConfig(website *model.Website) error { byteWebsite, err := json.Marshal(website) if err != nil { return err } err = rpc.Publish(byteWebsite, rpc.EventTypeWebsite) if err != nil { return err } return nil } func publishDeleteWebsiteConfig(id []uint) error { byteId, err := json.Marshal(id) if err != nil { return err } err = rpc.Publish(byteId, rpc.EventTypeDeleteWebsite) if err != nil { return err } return nil } func PostWebsite(ctx *gin.Context) { var params model.Website if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { website := &model.Website{Comment: params.Comment, ServerNames: params.ServerNames, Upstreams: params.Upstreams, Ports: params.Ports, CertFilename: params.CertFilename, KeyFilename: params.KeyFilename, IsEnabled: true} res := tx.Create(website) if res.Error != nil { return res.Error } if config.GlobalConfig.Server.DevMode { return nil } err := publishWebsiteConfig(website) if err != nil { return err } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, nil) } func PutWebsite(ctx *gin.Context) { var params model.Website if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { var website model.Website res := tx.Where(params.ID).First(&website) if res.Error != nil { return res.Error } if res.RowsAffected == 0 { return errors.New("Data queried does not exist") } website.Comment = params.Comment website.ServerNames = params.ServerNames website.Upstreams = params.Upstreams website.Ports = params.Ports website.CertFilename = params.CertFilename website.KeyFilename = params.KeyFilename website.IsEnabled = true tx.Save(&website) if config.GlobalConfig.Server.DevMode { return nil } err := publishWebsiteConfig(¶ms) if err != nil { return err } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, nil) } func DeleteWebsite(ctx *gin.Context) { var params idsRequest if err := ctx.BindJSON(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { res := tx.Where(params.IDs).Delete(&model.Website{}) if res.Error != nil { return res.Error } if res.RowsAffected == 0 { return errors.New("Data queried does not exist") } if config.GlobalConfig.Server.DevMode { return nil } err := publishDeleteWebsiteConfig(params.IDs) if err != nil { return err } return nil }) if err != nil { logger.Error(err) response.Error(ctx, response.JSONBody{Err: response.ErrInternalError, Msg: err.Error()}, http.StatusInternalServerError) return } response.Success(ctx, nil) } func GetWebsite(ctx *gin.Context) { var params pageRequest if err := ctx.BindQuery(¶ms); err != nil { logger.Error(err) response.Error(ctx, response.ErrorParamNotOK, http.StatusInternalServerError) return } db := database.GetDB() type Website struct { model.Website ReqValue int64 `json:"req_value"` DeniedValue int64 `json:"denied_value"` } var websiteList []Website db.Limit(params.PageSize).Offset(params.PageSize * (params.Page - 1)).Order("id desc").Find(&websiteList) var statistics []model.SystemStatistics var websiteIds []string for _, i := range websiteList { websiteIds = append(websiteIds, strconv.Itoa(int(i.ID))) } db.Where("created_at >= date_trunc('day',now())").Where("website in (?)", websiteIds).Find(&statistics) for i, website := range websiteList { for _, j := range statistics { if strconv.Itoa(int(website.ID)) == j.Website { if j.Type == "website-req" { websiteList[i].ReqValue = j.Value } else if j.Type == "website-denied" { websiteList[i].DeniedValue = j.Value } } } } var total int64 db.Model(&model.Website{}).Count(&total) response.Success(ctx, gin.H{"data": websiteList, "total": total}) } ================================================ FILE: management/webserver/cmd/fake_logs.go ================================================ package cmd import "chaitin.cn/patronus/safeline-2/management/webserver/model" func FakeLogs() { model.InitDetectLogSamples() } ================================================ FILE: management/webserver/cmd/gen_certs.go ================================================ package cmd import ( "crypto/x509/pkix" "path/filepath" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) func GenCerts() error { if err := genServerCert(); err != nil { return err } if err := genClientCACert(); err != nil { return err } return nil } func genServerCert() error { certPath := filepath.Join(config.GlobalConfig.MgtResDir, constants.CertsPath, "server.crt") keyPath := filepath.Join(config.GlobalConfig.MgtResDir, constants.CertsPath, "server.key") if err := utils.WriteCertIfNotExist( certPath, keyPath, func() ([]byte, []byte, error) { return utils.GenerateCert( []string{}, 3650, 4096, &pkix.Name{ Country: []string{"CN"}, Province: []string{"Beijing"}, Locality: []string{"Beijing"}, Organization: []string{"Beijing WAF Technology Co., Ltd."}, OrganizationalUnit: []string{"Service Infrastructure Department"}, CommonName: "WAF Management Server", }, false, ) }); err != nil { return err } return nil } func genClientCACert() error { certPath := filepath.Join(config.GlobalConfig.MgtResDir, constants.CertsPath, "client_ca.crt") keyPath := filepath.Join(config.GlobalConfig.MgtResDir, constants.CertsPath, "client_ca.key") if err := utils.WriteCertIfNotExist( certPath, keyPath, func() ([]byte, []byte, error) { return utils.GenerateCert( []string{}, 3650, 4096, &pkix.Name{ Country: []string{"CN"}, Province: []string{"Beijing"}, Locality: []string{"Beijing"}, Organization: []string{"Beijing WAF Technology Co., Ltd."}, OrganizationalUnit: []string{"Service Infrastructure Department"}, CommonName: "WAF Client Certificate Authority", }, true, ) }); err != nil { return err } return nil } ================================================ FILE: management/webserver/cmd/push_fsl.go ================================================ package cmd import ( "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" ) func PushFSL() error { return fvm.PushFSL(database.GetDB().DB) } ================================================ FILE: management/webserver/cmd/reset_user.go ================================================ package cmd import ( "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" ) func ResetUser(username string) { db := database.GetDB() var user model.User db.Where(&model.User{Username: username}).First(&user) user.LastLoginTime = 0 db.Save(&user) } ================================================ FILE: management/webserver/cmd/show_fsl.go ================================================ package cmd import ( "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" ) func ShowFSL() (string, error) { return fvm.GenerateFullFSL(database.GetDB().DB) } ================================================ FILE: management/webserver/config.yml ================================================ # develop use only. For production, refer to `package/build/mgt-api/webserver/config.yml` log: output: stdout # "stdout", "stderr" or file path level: debug # "debug", "info", "warn" or "error" server: listen_addr: :9001 dev_mode: true db: url: postgres://safeline-ce:safeline-ce@127.0.0.1/safeline-ce log_sql: false detector: addr: "" fsl_bytecode: fvm/bytecode grpc_server: listen_addr: :9002 ================================================ FILE: management/webserver/go.mod ================================================ module chaitin.cn/patronus/safeline-2/management/webserver go 1.21 toolchain go1.21.3 require ( chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6 chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c github.com/gin-contrib/sessions v0.0.5 github.com/gin-gonic/gin v1.10.0 github.com/pquerna/otp v1.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/rogpeppe/go-internal v1.10.0 github.com/sirupsen/logrus v1.4.2 google.golang.org/grpc v1.65.0 gorm.io/datatypes v1.1.1 gorm.io/driver/postgres v1.5.9 gorm.io/gorm v1.25.10 ) require ( github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/securecookie v1.1.1 // indirect github.com/gorilla/sessions v1.2.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.4.7 // indirect ) ================================================ FILE: management/webserver/go.sum ================================================ chaitin.cn/dev/go/errors v0.0.0-20200717101723-df6132d53dc8/go.mod h1:u+ZD0shdyUt0UG9XfCgD1R5mSqXpTVoO5rwQXQeo3Eo= chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6 h1:1Qa9ABk907/9ZrOLbbRcS8Fqq9VhjAF/mLjbSP1qAJY= chaitin.cn/dev/go/errors v0.0.0-20210324055134-dc5247602af6/go.mod h1:u+ZD0shdyUt0UG9XfCgD1R5mSqXpTVoO5rwQXQeo3Eo= chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c h1:Xn9IYkxmnpDcEpV+7JIR5ufEIexd1dhqKwpOLG1mYOE= chaitin.cn/dev/go/log v0.0.0-20221220104336-05125760b10c/go.mod h1:xJIYwUoA2TX5mNg/RBrEPyE251BPwj+70/mM7UIhoxg= chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c h1:tXsraF7o9iUsQY6IwpDJusc6OFhB7iv/bBTfgR3MPUU= chaitin.cn/dev/go/settings v0.0.0-20221220104336-05125760b10c/go.mod h1:fUvtmpG8Z8Zf5aciadL9a/vn5SB3knG7pdNJixDplPg= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA= github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gin-contrib/sessions v0.0.5 h1:CATtfHmLMQrMNpJRgzjWXD7worTh7g7ritsQfmF+0jE= github.com/gin-contrib/sessions v0.0.5/go.mod h1:vYAuaUPqie3WUSsft6HUlCjlwwoJQs97miaG2+7neKY= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/datatypes v1.1.1 h1:XAjO7NNfUKVUvnS3+BkqMrPXxCAcxDlpOYbjnizxNCw= gorm.io/datatypes v1.1.1/go.mod h1:u8GEgFjJ+GpsGfgHmBUcQqHm/937t3sj/SO9dvbndTg= gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU= gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11 h1:9qNbmu21nNThCNnF5i2R3kw2aL27U8ZwbzccNjOmW0g= gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= ================================================ FILE: management/webserver/main.go ================================================ package main import ( "flag" "fmt" "net/http" "os" "strconv" "strings" "time" "github.com/gin-contrib/sessions" "github.com/gin-contrib/sessions/cookie" "github.com/gin-gonic/gin" "chaitin.cn/patronus/safeline-2/management/webserver/api" "chaitin.cn/patronus/safeline-2/management/webserver/cmd" "chaitin.cn/patronus/safeline-2/management/webserver/middleware" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/cron" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/log" "chaitin.cn/patronus/safeline-2/management/webserver/rpc" ) var ( logger = log.GetLogger("main") version = "undefined" githash = "undefined" buildstamp = "undefined" goVersion = "undefined" ) func init() { // do something that do not raise error } func main() { log.SetLogFormatter() fs := flag.NewFlagSet("webserver", flag.ExitOnError) showVersion := fs.Bool("v", false, "show version") cfgFile := fs.String("c", constants.ConfigFilePath, "config file path") genCerts := fs.Bool("gen_certs", false, "generate certs") showFSL := fs.Bool("show_fsl", false, "show full selectors") push_fsl := fs.Bool("push_fsl", false, "compile and push fsl") fakeLogs := fs.Bool("fake_logs", false, "fake logs") resetUsername := fs.String("reset_user", "", "reset user") if err := fs.Parse(os.Args[1:]); err != nil { logger.Fatalln("Failed to parse args: ", err) } if *showVersion { i, _ := strconv.Atoi(buildstamp) t := time.Unix(int64(i), 0).Format("2006-01-02 15:04:05") fmt.Println("Version: ", version) fmt.Println("Githash: ", githash) fmt.Println("Build: ", t) fmt.Println("Go version: ", goVersion) return } constants.Version = strings.TrimPrefix(version, "ce-") // init configs if err := config.InitConfigs(*cfgFile); err != nil { logger.Fatalln("Failed to init configs: ", err) } if err := log.InitLogger(); err != nil { logger.Fatalln("Failed to init db: ", err) } if *genCerts { if err := cmd.GenCerts(); err != nil { logger.Fatalln("Failed to generate certs: ", err) } return } logger.Info("Init database") if err := database.InitDB(); err != nil { logger.Fatalln("Failed to init db: ", err) } if *showFSL { if fullFSL, err := cmd.ShowFSL(); err != nil { logger.Fatalln("Failed to generate fsl: ", err) } else { logger.Info(strings.ReplaceAll(strings.ReplaceAll(fullFSL, ";", ";\n"), "CREATE", "\nCREATE")) } return } logger.Info("Init models") if err := model.InitModels(); err != nil { logger.Fatalln("Failed to init models: ", err) } if len(*resetUsername) > 0 { logger.Infoln("reset user:", *resetUsername) cmd.ResetUser(*resetUsername) logger.Infoln("success!") return } if *fakeLogs { logger.Infoln("faking logs...") cmd.FakeLogs() logger.Infoln("success!") return } if *push_fsl { logger.Infoln("push fsl...") if err := cmd.PushFSL(); err != nil { logger.Fatalln("Failed to generate fsl: ", err) } logger.Infoln("success!") return } logger.Info("Init FVM bytecode") fvm.InitFVMBytecode() if err := cron.StartCron(); err != nil { logger.Fatalln("Failed to start cron: ", err) } if err := rpc.StartGRPCSever(); err != nil { logger.Fatalln("Failed to start gRPC server: ", err) } gin.SetMode(gin.ReleaseMode) r := gin.Default() var option model.Options database.GetDB().Where(&model.Options{Key: constants.SecretKey}).First(&option) logger.Debugf("Secret: %s", option.Value) store := cookie.NewStore([]byte(option.Value)) r.Use(sessions.Sessions("session", store)) publicRouters := r.Group("/api") publicRouters.POST(api.Login, api.PostLogin) publicRouters.POST(api.Logout, api.PostLogout) publicRouters.POST(api.Behaviour, api.PostBehaviour) publicRouters.POST(api.FalsePositives, api.PostFalsePositives) publicRouters.GET(api.OTPUrl, api.GetOTPUrl) publicRouters.GET(api.Version, api.GetVersion) publicRouters.GET(api.UpgradeTips, api.GetUpgradeTips) // test use publicRouters.GET("/Ping", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ "message": "pong", }) }) limitedRouters := r.Group("/api") noAuth, existed := os.LookupEnv("NO_AUTH") if existed && len(noAuth) >= 0 { logger.Warn("No auth") } else { limitedRouters.Use(middleware.AuthRequired) } readOnly, existed := os.LookupEnv("READ_ONLY") if existed && len(readOnly) >= 0 { logger.Warn("Read only") limitedRouters.Use(middleware.ReadOnly) } limitedRouters.GET(api.User, api.GetUser) limitedRouters.GET(api.DetectLogList, api.GetDetectLogList) limitedRouters.GET(api.DetectLogDetail, api.GetDetectLogDetail) limitedRouters.POST(api.Website, api.PostWebsite) limitedRouters.PUT(api.Website, api.PutWebsite) limitedRouters.DELETE(api.Website, api.DeleteWebsite) limitedRouters.GET(api.Website, api.GetWebsite) limitedRouters.POST(api.UploadSSLCert, api.PostUploadSSLCert) limitedRouters.POST(api.SSLCert, api.PostSSLCert) limitedRouters.POST(api.PolicyRule, api.PostPolicyRule) limitedRouters.PUT(api.PolicyRule, api.PutPolicyRule) limitedRouters.DELETE(api.PolicyRule, api.DeletePolicyRule) limitedRouters.GET(api.PolicyRule, api.GetPolicyRule) limitedRouters.PUT(api.SwitchPolicyRule, api.PutSwitchPolicyRule) // 仪表盘接口 limitedRouters.GET(api.DashboardCounts, api.GetDashboardCounts) limitedRouters.GET(api.DashboardSites, api.GetDashboardSites) limitedRouters.GET(api.DashboardQps, api.GetDashboardQps) limitedRouters.GET(api.DashboardRequests, api.GetDashboardRequests) limitedRouters.GET(api.DashboardIntercepts, api.GetDashboardIntercepts) limitedRouters.GET(api.PolicyGroupGlobal, api.GetPolicyGroupGlobal) limitedRouters.PUT(api.PolicyGroupGlobal, api.PutPolicyGroupGlobal) limitedRouters.GET(api.SrcIPConfig, api.GetSrcIPConfig) limitedRouters.PUT(api.SrcIPConfig, api.PutSrcIPConfig) logger.Info("Staring...") if err := r.Run(config.GlobalConfig.Server.ListenAddr); err != nil { logger.Fatalln("Error occurred when running web server: ", err) } } ================================================ FILE: management/webserver/middleware/auth.go ================================================ package middleware import ( "net/http" "github.com/gin-contrib/sessions" "github.com/gin-gonic/gin" "chaitin.cn/patronus/safeline-2/management/webserver/api/response" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" ) func AuthRequired(c *gin.Context) { session := sessions.Default(c) user := session.Get(constants.DefaultSessionUserKey) if user == nil { response.Error(c, response.ErrorLoginRequired, http.StatusUnauthorized) c.Abort() return } // extend session expired time session.Options(sessions.Options{ Path: "/", MaxAge: 3600 * 24 * 7, //Domain: options.Domain, //HttpOnly: true, //SameSite: http.SameSiteLaxMode, //Secure: false, }) if err := session.Save(); err != nil { response.Error(c, response.JSONBody{Err: response.ErrInternalError, Msg: "Error occurred when creating sessions"}, http.StatusInternalServerError) return } c.Next() } func ReadOnly(c *gin.Context) { if c.Request.Method != "GET" && c.Request.Method != "HEAD" && c.Request.Method != "OPTIONS" { response.Error(c, response.ErrorReadOnly, http.StatusBadRequest) c.Abort() } } ================================================ FILE: management/webserver/model/base.go ================================================ package model import ( "time" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/log" ) // Base is a replacement for gorm.Model without DeletedAt, which is considered to be not good. type Base struct { ID uint `gorm:"primarykey" json:"id"` CreatedAt time.Time ` json:"created_at"` UpdatedAt time.Time ` json:"updated_at"` } var logger = log.GetLogger("model") ================================================ FILE: management/webserver/model/behaviour.go ================================================ package model type Behaviour struct { Base SrcRouter string `json:"src_router"` DstRouter string `json:"dst_router"` } ================================================ FILE: management/webserver/model/db_patch_1_4_0.go ================================================ package model import ( "strings" "chaitin.cn/dev/go/errors" "gorm.io/gorm" ) type sqlResult struct { Ids string `json:"ids"` } func DBPatch140(tx *gorm.DB) error { if !tx.Migrator().HasTable(&SystemStatistics{}) { return nil } var result []sqlResult //SELECT string_agg(id::text, ',') as ids FROM mgt_system_statistics GROUP BY (type, website, created_at) HAVING COUNT(*) > 1) as tmp res := tx.Model(&SystemStatistics{}).Select("string_agg(id::text, ',') as ids").Group("type, website, created_at").Having("COUNT(*) > 1").Find(&result) if res.Error != nil { return errors.Wrap(res.Error, "Failed to select data") } if len(result) <= 0 { return nil } var deleteIds []string for _, s := range result { tmpIds := strings.Split(s.Ids, ",") deleteIds = append(deleteIds, tmpIds...) } deleteRes := tx.Delete(&SystemStatistics{}, deleteIds) return errors.Wrap(deleteRes.Error, "Failed to delete") } ================================================ FILE: management/webserver/model/detectlog.go ================================================ package model import ( "fmt" "math/rand" "strings" "time" "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) // DetectLog is designed to be used in response, not a good naming. type DetectLog struct { DetectLogBasic DetectLogDetail EventId string `json:"event_id"` // to eliminate ambiguous Website string `json:"website"` AttackType string `json:"attack_type"` Module string `json:"module"` Reason string `json:"reason"` } func getRuleModule(ruleId string, attackType int) string { var module string if strings.HasPrefix(ruleId, "m_rule") { // `m_rule/65543` module = "m_rule" } else if strings.HasPrefix(ruleId, "/") { if attackType == -2 { module = "whitelist" } else { // if attackType == -3 module = "blacklist" } } else { module = ruleId } return constants.RuleModule[module] } func getRuleReason(ruleId string, attackType int) string { var reason string if strings.HasPrefix(ruleId, "m_rule") { // `m_rule/65543` reason = constants.RuleReason[strings.ReplaceAll(ruleId, "m_rule/", "")] } else if strings.HasPrefix(ruleId, "/") { reason = ruleId[1:] } else { reason = fmt.Sprintf("检测到 %s 攻击", getAttackType(attackType)) } return reason } func getAttackType(at int) string { atStr, ok := constants.AttackType[at] if !ok { atStr = constants.AttackType[62] // unknown } return atStr } func getCountry(country string) string { countryStr, ok := constants.CountryCode[country] if !ok { countryStr = "" } return countryStr } func TransformDetectLog(basic *DetectLogBasic, detail *DetectLogDetail) (*DetectLog, error) { if basic == nil { return nil, errors.New("basic *DetectLogBasic cannot be nil") } if detail != nil && basic.EventId != detail.EventId { return nil, errors.New("EventId field should be the same for basic and detail") } basic.Country = getCountry(basic.Country) dLog := DetectLog{ DetectLogBasic: *basic, EventId: basic.EventId, Website: utils.BuildUrl(basic.Protocol, basic.Host, basic.DstPort, basic.UrlPath), AttackType: getAttackType(basic.AttackType), Module: getRuleModule(basic.RuleId, basic.AttackType), Reason: getRuleReason(basic.RuleId, basic.AttackType), } if detail != nil { dLog.DetectLogDetail = *detail } return &dLog, nil } type DetectLogBasic struct { ID uint `json:"id" gorm:"primarykey"` EventId string `json:"event_id" gorm:"uniqueIndex;not null"` SiteUUID string `json:"site_uuid" gorm:"column:site_uuid"` SrcIp string `json:"src_ip" gorm:"index"` SocketIp string `json:"socket_ip" gorm:"index"` Protocol int `json:"protocol"` Host string `json:"host"` UrlPath string `json:"url_path"` DstPort uint `json:"dst_port"` Country string `json:"country"` Province string `json:"province"` City string `json:"city"` AttackType int `json:"attack_type" gorm:"index"` RiskLevel int `json:"risk_level" gorm:"index"` Action int `json:"action" gorm:"index"` RuleId string `json:"rule_id"` Timestamp int64 `json:"timestamp" gorm:"index"` } type DetectLogDetail struct { ID uint ` gorm:"primarykey"` EventId string ` gorm:"uniqueIndex;not null"` SrcPort uint `json:"src_port"` DstIp string `json:"dst_ip"` Method string `json:"method"` QueryString string `json:"query_string"` StatusCode uint `json:"status_code"` ReqHeader string `json:"req_header"` ReqBody string `json:"req_body"` RspHeader string `json:"rsp_header"` RspBody string `json:"rsp_body"` Payload string `json:"payload"` Location string `json:"location"` DecodePath string `json:"decode_path"` } func InitDetectLogSamples() { db := database.GetDB() var detectLogBasicList []DetectLogBasic var detectLogDetailList []DetectLogDetail timestamp := time.Now().Unix() detail := DetectLogDetail{ SrcPort: 58694, DstIp: "10.2.35.143", Method: "GET", QueryString: "", ReqHeader: "GET /webshell.php HTTP/1.1\nUser-Agent: curl/7.77.0\nAccept: */*\n\n\"", ReqBody: "", RspHeader: "", RspBody: "", Payload: "", Location: "urlpath", DecodePath: "", } protocolList := []int{constants.ProtocolHTTP, constants.ProtocolHTTPS} portList := []uint{80, 443} provinceList := []string{} cityList := []string{} ipList := []string{} ruleIdList := []string{} for i := 0; i < 100; i++ { randInt := rand.Intn(1000) eventId := utils.RandStr(32) basic := DetectLogBasic{} basic.EventId = eventId basic.SrcIp = ipList[randInt%len(ipList)] basic.SocketIp = ipList[randInt%len(ipList)] basic.Protocol = protocolList[randInt%len(protocolList)] basic.DstPort = portList[randInt%len(portList)] basic.Host = fmt.Sprintf("%s.com", utils.RandStr(5)) basic.UrlPath = fmt.Sprintf("/%s", utils.RandStr(10)) basic.Country = "CN" basic.Province = provinceList[randInt%len(provinceList)] basic.City = cityList[randInt%len(cityList)] basic.AttackType = randInt % 32 basic.RiskLevel = randInt % 4 basic.Action = randInt % 2 basic.RuleId = ruleIdList[randInt%len(ruleIdList)] basic.Timestamp = timestamp - int64(randInt*100) detailCopy := detail detailCopy.EventId = eventId detectLogBasicList = append(detectLogBasicList, basic) detectLogDetailList = append(detectLogDetailList, detailCopy) } db.CreateInBatches(detectLogBasicList, 100) db.CreateInBatches(detectLogDetailList, 100) } ================================================ FILE: management/webserver/model/init.go ================================================ package model import ( "gorm.io/gorm" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" ) func InitModels() error { db := database.GetDB() err := db.Transaction(func(tx *gorm.DB) error { // if err := DBPatch140(tx); err != nil { return err } return nil }) if err != nil { return err } if err := db.AutoMigrate(&User{}, &DetectLogBasic{}, &DetectLogDetail{}, &Behaviour{}, &Options{}, &Website{}, &PolicyRule{}, &SystemStatistics{}); err != nil { return err } if err := initAdminUser(); err != nil { return err } if err := initOptions(); err != nil { return err } if err := initPolicyGroupGlobal(); err != nil { return err } if err := initSrcIPConfig(); err != nil { return err } //InitDetectLogSamples() return nil } ================================================ FILE: management/webserver/model/option.go ================================================ package model import ( "bytes" "encoding/json" "net/http" "gorm.io/gorm/clause" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/pkg" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) type Options struct { Base Key string `gorm:"column:key;uniqueIndex"` Value string `gorm:"column:value;"` } func initOptions() error { db := database.GetDB() secretKey := Options{Key: constants.SecretKey, Value: utils.RandStr(32)} db.Clauses(clause.OnConflict{DoNothing: true}).Create(&secretKey) machineId := Options{Key: constants.MachineID, Value: utils.RandStr(32)} _ = db.Clauses(clause.OnConflict{DoNothing: true}).Create(&machineId) go NotifyInstallation(machineId.Value) return nil } func NotifyInstallation(machineId string) { logger.Info("Notify installation") tr := pkg.TelemetryRequest{ Telemetry: pkg.TelemetryInfo{ Id: constants.TelemetryId, }, Safeline: pkg.SafelineInfo{ Id: machineId, Type: constants.Installation, Version: constants.Version, }, } data, err := json.Marshal(tr) if err != nil { logger.Error(err) return } reader := bytes.NewReader(data) rsp, err := pkg.DoPostTelemetry(utils.GetHTTPClient(), config.GlobalConfig.Telemetry.Addr, reader) if err != nil { logger.Error(err) return } if rsp.StatusCode != http.StatusOK && rsp.StatusCode != http.StatusCreated { logger.Errorf("transfer telemetry %s failed, status code = %d", constants.Installation, rsp.StatusCode) return } } ================================================ FILE: management/webserver/model/policygroup.go ================================================ package model import ( "encoding/json" "gorm.io/gorm" "gorm.io/gorm/clause" "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" ) type PolicyGroup map[string]string const ( StrictMode = "strict" DefaultMode = "default" DisableMode = "disable" SocketIP = "socket_ip" HTTPHeader = "http_header" ) type SrcIPConfig struct { Source string `json:"source"` // socket_ip, http_header Value string `json:"value"` } type SkynetConfigModule struct { DetectConfig interface{} `json:"detect_config,omitempty"` HighRiskAction string `json:"high_risk_action"` MediumRiskAction string `json:"medium_risk_action"` LowRiskAction string `json:"low_risk_action"` HighRiskEnableLog int `json:"high_risk_enable_log"` MediumRiskEnableLog int `json:"medium_risk_enable_log"` LowRiskEnableLog int `json:"low_risk_enable_log"` State string `json:"state"` } type SkynetConfig struct { DetectConfig string `json:"detect_config"` DeepDetect bool `json:"deep_detect"` Timeout interface{} `json:"timeout"` Modules map[string]*SkynetConfigModule `json:"modules"` } func GetSrcIPConfig(db *gorm.DB) (*SrcIPConfig, error) { var scOption Options db.Where(&Options{Key: constants.SrcIPConfig}).First(&scOption) var sc SrcIPConfig err := json.Unmarshal([]byte(scOption.Value), &sc) if err != nil { return nil, err } return &sc, nil } func initSrcIPConfig() error { db := database.GetDB() // policyGroupGlobal config, three mode: strict/default/disable var srcIPConfig = SrcIPConfig{ Source: SocketIP, Value: "", } scStr, err := json.Marshal(srcIPConfig) if err != nil { return err } pgg := Options{Key: constants.SrcIPConfig, Value: string(scStr)} db.Clauses(clause.OnConflict{DoNothing: true}).Create(&pgg) return nil } func GetSkynetConfig(db *gorm.DB) (string, error) { // skynetConfigStr skynet config in 5.3.9 const skynetConfigStr = "{\"decode_config\":{\"decode_methods\":[\"url decode\",\"JSON\",\"base64\",\"hex\",\"eval\",\"XML\",\"PHP deserialize\",\"utf7\"]},\"deep_detect\":false,\"modules\":{\"m_asp_code_injection\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_cmd_injection\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_csrf\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_file_include\":{\"detect_config\":{\"detect_suspicious_schema\":true},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_file_upload\":{\"detect_config\":{\"action_of_handling_custome_file\":[\"deny transmission\"],\"detect_file_content\":true,\"detect_real_file_content\":false,\"enable_module_in_detecting_file_content\":[\"java\",\"asp_code_injection\",\"php_code_injection\"],\"file_type_of_custome_action\":[],\"forbidden_extra_token\":false,\"forbidden_multiple_filename\":true,\"forbidden_suspicious_filename\":true},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_http\":{\"detect_config\":{\"warning_http_parse_failed\":false,\"warning_suspicious_http_version\":false},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"disabled\"},\"m_java\":{\"detect_config\":{\"detect_lookup\":false},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_java_unserialize\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_php_code_injection\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_php_unserialize\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_response\":{\"detect_config\":{\"detection_configure\":[\"detect_jsp_code_leak\",\"detect_php_code_leak\",\"detect_webshell\"],\"error_types\":[\"directory indexing\",\"SQL execution error\",\"server exception\"]},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"disabled\"},\"m_rule\":{\"detect_config\":{\"check_info_leak_by_rsp\":false,\"compatibility_mode\":false,\"detection_configure\":[\"detect_phpinfo\",\"detect_admin_page\",\"detect_backdoor\",\"detect_php_vuln\",\"detect_nginx\",\"detect_dedecms\",\"detect_apache\",\"detect_xml\",\"detect_struts2\",\"detect_java_vuln\",\"detect_php168\",\"detect_wordpress\",\"detect_directory_traversal\",\"detect_iis\",\"detect_gogs_gitea\",\"detect_thinkphp\",\"detect_jenkins\",\"detect_ecshop\",\"detect_nexus\",\"detect_drupal\",\"detect_ghostscript\",\"detect_atlassian\",\"detect_weblogic\",\"detect_coremail\",\"detect_phpcms\",\"detect_spring\",\"detect_southidc\",\"detect_fastjson\",\"detect_tbk_dvr\",\"detect_joomla\",\"detect_ecology_oa\",\"detect_jackson\",\"detect_xstream\",\"detect_activemq\",\"detect_solr\",\"detect_csii\",\"detect_big_ip\",\"detect_apisix\",\"detect_druid\",\"detect_log4j\"],\"info_leak_types\":[\"test file\",\"backup file\",\"code repository\",\"server sensitive file\"],\"rules_config\":{\"disable_ruleid\":[],\"enable_ruleid\":[],\"rules_status\":[]}},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_scanner\":{\"detect_config\":{\"language_types\":[\"python_scanner\",\"go_scanner\"],\"other_types\":[\"normal_scanner\"],\"spider_types\":[]},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"disabled\"},\"m_sqli\":{\"detect_config\":{\"detect_non_injection_sql\":true},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_ssrf\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_ssti\":{\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"},\"m_xss\":{\"detect_config\":{\"detect_complete_html\":true},\"high_risk_action\":\"deny\",\"high_risk_enable_log\":1,\"low_risk_action\":\"continue\",\"low_risk_enable_log\":1,\"medium_risk_action\":\"continue\",\"medium_risk_enable_log\":1,\"state\":\"enabled\"}},\"timeout\":{\"threshold\":1000,\"log\":\"enabled\"}}" var sc SkynetConfig err := json.Unmarshal([]byte(skynetConfigStr), &sc) if err != nil { return "", err } var pggOption Options db.Where(&Options{Key: constants.PolicyGroupGlobal}).First(&pggOption) var pgg PolicyGroup err = json.Unmarshal([]byte(pggOption.Value), &pgg) if err != nil { return "", err } for module, mode := range pgg { switch mode { case StrictMode: scm := sc.Modules[module] scm.HighRiskAction = "deny" scm.MediumRiskAction = "deny" scm.LowRiskAction = "deny" scm.State = "enabled" sc.Modules[module] = scm case DefaultMode: scm := sc.Modules[module] scm.HighRiskAction = "deny" scm.MediumRiskAction = "continue" scm.LowRiskAction = "continue" scm.State = "enabled" sc.Modules[module] = scm case DisableMode: scm := sc.Modules[module] scm.HighRiskAction = "continue" scm.MediumRiskAction = "continue" scm.LowRiskAction = "continue" scm.State = "disabled" sc.Modules[module] = scm default: return "", errors.New("no such mode") } } scStr, err := json.Marshal(sc) if err != nil { return "", err } return string(scStr), nil } func initPolicyGroupGlobal() error { db := database.GetDB() // policyGroupGlobal config, three mode: strict/default/disable var policyGroupGlobal = PolicyGroup{ "m_asp_code_injection": DefaultMode, "m_cmd_injection": DefaultMode, "m_csrf": DefaultMode, "m_file_include": DefaultMode, "m_file_upload": DefaultMode, "m_http": DefaultMode, "m_java": DefaultMode, "m_java_unserialize": DefaultMode, "m_php_code_injection": DefaultMode, "m_php_unserialize": DefaultMode, "m_response": DefaultMode, "m_rule": DefaultMode, "m_scanner": DefaultMode, "m_sqli": DefaultMode, "m_ssrf": DefaultMode, "m_ssti": DefaultMode, "m_xss": DefaultMode, } pggStr, err := json.Marshal(policyGroupGlobal) if err != nil { return err } pgg := Options{Key: constants.PolicyGroupGlobal, Value: string(pggStr)} db.Clauses(clause.OnConflict{DoNothing: true}).Create(&pgg) return nil } ================================================ FILE: management/webserver/model/policyrule.go ================================================ package model import "gorm.io/datatypes" type PolicyRule struct { Base Action int `gorm:"action" json:"action"` Comment string `gorm:"comment" json:"comment"` Pattern datatypes.JSON `gorm:"pattern" json:"pattern"` IsEnabled bool `gorm:"is_enabled;default=true" json:"is_enabled"` } type PolicyRulePattern struct { K string `json:"k"` Op string `json:"op"` V string `json:"v"` } const ( KeySrcIP = "src_ip" KeyURI = "uri" KeyHost = "host" OpEq = "eq" // 完全相等 OpMatch = "match" // 模糊匹配 OpCIDR = "cidr" // CIDR OpHas = "has" // 关键字 OpPrefix = "prefix" // 前缀关键字 OpRe = "re" // 正则 ) ================================================ FILE: management/webserver/model/statistics.go ================================================ package model import "time" type SystemStatistics struct { ID uint `json:"id" gorm:"primarykey"` Type string `json:"type" gorm:"index;uniqueIndex:type_website_createdat"` Value int64 `json:"value"` CreatedAt time.Time `json:"created_at" gorm:"index;uniqueIndex:type_website_createdat"` Website string `json:"website" gorm:"index;uniqueIndex:type_website_createdat"` } ================================================ FILE: management/webserver/model/user.go ================================================ package model import ( "gorm.io/gorm/clause" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" ) type User struct { Base Username string `gorm:"uniqueIndex;not null"` Password string Comment string TFAEnabled bool `gorm:"column:tfa_enabled;default:true"` TFASecret string `gorm:"column:tfa_secret"` LastLoginTime int64 `gorm:"default:0"` IsEnabled bool `gorm:"default:true"` } func initAdminUser() error { db := database.GetDB() user := User{ Username: constants.SuperUser, } db.Clauses(clause.OnConflict{DoNothing: true}).Create(&user) return nil } ================================================ FILE: management/webserver/model/website.go ================================================ package model import "gorm.io/datatypes" type Website struct { Base Comment string `gorm:"comment" json:"comment"` ServerNames datatypes.JSON `gorm:"server_names" json:"server_names"` Ports datatypes.JSON `gorm:"ports" json:"ports"` Upstreams datatypes.JSON `gorm:"upstreams" json:"upstreams"` CertFilename string `gorm:"cert_filename" json:"cert_filename"` KeyFilename string `gorm:"key_filename" json:"key_filename"` IsEnabled bool `gorm:"is_enabled;default=true" json:"is_enabled"` } ================================================ FILE: management/webserver/pkg/config/config.go ================================================ package config import ( "os" "chaitin.cn/dev/go/settings" ) var ( GlobalConfig = DefaultGlobalConfig() ) func InitConfigs(configFilePath string) error { s, err := settings.New(configFilePath) if err != nil { return err } if err = GlobalConfig.DB.Load(s); err != nil { return err } if err = GlobalConfig.Log.Load(s); err != nil { return err } if err = GlobalConfig.Server.Load(s); err != nil { return err } if err = GlobalConfig.Detector.Load(s); err != nil { return err } if err = GlobalConfig.Telemetry.Load(s); err != nil { return err } if err = GlobalConfig.GPRC.Load(s); err != nil { return err } if err := settings.Unmarshal("platform_addr", &GlobalConfig.PlatformAddr); err != nil { return err } if v, ok := os.LookupEnv("MANAGEMENT_RESOURCES_DIR"); ok { GlobalConfig.MgtResDir = v } if v, ok := os.LookupEnv("NGINX_RESOURCES_DIR"); ok { GlobalConfig.NgxResDir = v } return nil } ================================================ FILE: management/webserver/pkg/config/db.go ================================================ package config import ( "net/url" "os" "chaitin.cn/dev/go/settings" ) type DBConfig struct { URL string `yaml:"url"` LogSQL bool `yaml:"log_sql"` SSLMode bool `yaml:"ssl_mode"` } func DefaultDBConfig() DBConfig { return DBConfig{ URL: "postgres://safeline-ce:safeline-ce@127.0.0.1/safeline-ce", LogSQL: false, SSLMode: false, } } func (dbc *DBConfig) Load(setting *settings.Setting) error { if err := setting.Unmarshal("db", dbc); err != nil { return err } if v, ok := os.LookupEnv("DATABASE_URL"); ok { dbc.URL = v } dbURL, err := url.Parse(dbc.URL) if err != nil { return err } q := dbURL.Query() if !dbc.SSLMode { q.Set("sslmode", "disable") } dbURL.RawQuery = q.Encode() dbc.URL = dbURL.String() return nil } ================================================ FILE: management/webserver/pkg/config/detector.go ================================================ package config import ( "chaitin.cn/dev/go/settings" ) type DetectorConfig struct { Addr string `yaml:"addr"` FslBytecode string `yaml:"fsl_bytecode"` } func DefaultDetectorConfig() DetectorConfig { return DetectorConfig{ Addr: "http://127.0.0.1:8001", FslBytecode: "bytecode", } } func (d *DetectorConfig) Load(setting *settings.Setting) error { if err := setting.Unmarshal("detector", d); err != nil { return err } return nil } ================================================ FILE: management/webserver/pkg/config/global.go ================================================ package config type Config struct { Log LogConfig DB DBConfig Server ServerConfig Detector DetectorConfig Telemetry TelemetryConfig GPRC GRPCConfig PlatformAddr string MgtResDir string NgxResDir string } func DefaultGlobalConfig() Config { return Config{ Log: DefaultLogConfig(), DB: DefaultDBConfig(), Server: DefaultServerConfig(), Detector: DefaultDetectorConfig(), Telemetry: DefaultTelemetryConfig(), GPRC: DefaultGRPCConfig(), PlatformAddr: "https://waf-ce.chaitin.cn", MgtResDir: "/resources/management", NgxResDir: "/resources/nginx", } } ================================================ FILE: management/webserver/pkg/config/grpc.go ================================================ package config import ( "chaitin.cn/dev/go/settings" ) type GRPCConfig struct { ListenAddr string `yaml:"listen_addr"` } func DefaultGRPCConfig() GRPCConfig { return GRPCConfig{ ListenAddr: ":9002", } } func (sc *GRPCConfig) Load(setting *settings.Setting) error { if err := setting.Unmarshal("grpc_server", sc); err != nil { return err } return nil } ================================================ FILE: management/webserver/pkg/config/log.go ================================================ package config import ( "chaitin.cn/dev/go/settings" ) type LogConfig struct { Output string `yaml:"output"` Level string `yaml:"level"` } func DefaultLogConfig() LogConfig { return LogConfig{ Output: "stdout", Level: "info", } } func (lc *LogConfig) Load(setting *settings.Setting) error { if err := setting.Unmarshal("log", lc); err != nil { return err } return nil } ================================================ FILE: management/webserver/pkg/config/server.go ================================================ package config import ( "chaitin.cn/dev/go/settings" ) type ServerConfig struct { ListenAddr string `yaml:"listen_addr"` DevMode bool `yaml:"dev_mode"` IntenseMode bool `yaml:"intense_mode"` } func DefaultServerConfig() ServerConfig { return ServerConfig{ ListenAddr: ":9001", DevMode: false, } } func (sc *ServerConfig) Load(setting *settings.Setting) error { if err := setting.Unmarshal("server", sc); err != nil { return err } return nil } ================================================ FILE: management/webserver/pkg/config/telemetry.go ================================================ package config import ( "chaitin.cn/dev/go/settings" ) type TelemetryConfig struct { Addr string `yaml:"addr"` } func DefaultTelemetryConfig() TelemetryConfig { return TelemetryConfig{ Addr: "rivers-telemetry:10086", } } func (t *TelemetryConfig) Load(setting *settings.Setting) error { if err := setting.Unmarshal("telemetry", t); err != nil { return err } return nil } ================================================ FILE: management/webserver/pkg/constants/constants.go ================================================ package constants const ( SuperUser = "admin" ProductName = "长亭雷池 WAF 社区版" ProductVersion = "" ConfigFilePath = "config.yml" CertsPath = "certs" ) const ( NotUpgrade int = iota RecommendedUpgrade MustUpgrade ) ================================================ FILE: management/webserver/pkg/constants/detectlog.go ================================================ package constants const ( ProtocolHTTP = 0 ProtocolHTTPS = 1 ProtocolHTTP2 = 2 ) var ( HTTPProtocol = map[int]string{ ProtocolHTTP: "http", ProtocolHTTPS: "https", // 在 WAF 中开启 HTTP2 的前提是开启 TLS，所以这里先把 HTTP2 都兼容显示为 HTTPS ProtocolHTTP2: "https", } AttackType = map[int]string{ -4: "超长数据", -3: "黑名单", -2: "白名单", -1: "非攻击", 0: "SQL 注入", 1: "XSS", 2: "CSRF", 3: "SSRF", 4: "拒绝服务", 5: "后门", 6: "反序列化", 7: "代码执行", 8: "代码注入", 9: "命令注入", 10: "文件上传", 11: "文件包含", 12: "重定向", 13: "权限不当", 14: "信息泄露", 15: "未授权访问", 16: "不安全的配置", 17: "XXE", 18: "XPath 注入", 19: "LDAP 注入", 20: "目录穿越", 21: "扫描器", 22: "水平权限绕过", 23: "垂直权限绕过", 24: "文件修改", 25: "文件读取", 26: "文件删除", 27: "逻辑错误", 28: "CRLF 注入", 29: "模板注入", 30: "点击劫持", 31: "缓冲区溢出", 32: "整数溢出", 33: "格式化字符串", 34: "条件竞争", 35: "HTTP 协议违规", 61: "超时", 62: "未知", 63: "威胁情报", 64: "Cookie 篡改", } CountryCode = map[string]string{ "CN": "中国", "AD": "安道尔", "AE": "阿联酋", "AF": "阿富汗", "AG": "安提瓜和巴布达", "AI": "安圭拉", "AL": "阿尔巴尼亚", "AM": "亚美尼亚", "AO": "安哥拉", "AQ": "南极洲", "AR": "阿根廷", "AS": "美属萨摩亚", "AT": "奥地利", "AU": "澳大利亚", "AW": "阿鲁巴", "AX": "奥兰", "AZ": "阿塞拜疆", "BA": "波斯尼亚和黑塞哥维那", "BB": "巴巴多斯", "BD": "孟加拉国", "BE": "比利时", "BF": "布基纳法索", "BG": "保加利亚", "BH": "巴林", "BI": "布隆迪", "BJ": "贝宁", "BL": "圣巴泰勒米", "BM": "百慕大", "BN": "文莱", "BO": "玻利维亚", "BQ": "加勒比荷兰", "BR": "巴西", "BS": "巴哈马", "BT": "不丹", "BV": "布韦岛", "BW": "博茨瓦纳", "BY": "白俄罗斯", "BZ": "伯利兹", "CA": "加拿大", "CC": "科科斯（基林）群岛", "CD": "刚果（金）", "CF": "中非", "CG": "刚果（布）", "CH": "瑞士", "CI": "科特迪瓦", "CK": "库克群岛", "CL": "智利", "CM": "喀麦隆", "CO": "哥伦比亚", "CR": "哥斯达黎加", "CU": "古巴", "CV": "佛得角", "CW": "库拉索", "CX": "圣诞岛", "CY": "塞浦路斯", "CZ": "捷克", "DE": "德国", "DJ": "吉布提", "DK": "丹麦", "DM": "多米尼克", "DO": "多米尼加", "DZ": "阿尔及利亚", "EC": "厄瓜多尔", "EE": "爱沙尼亚", "EG": "埃及", "EH": "阿拉伯撒哈拉民主共和国", "ER": "厄立特里亚", "ES": "西班牙", "ET": "埃塞俄比亚", "FI": "芬兰", "FJ": "斐济", "FK": "福克兰群岛", "FM": "密克罗尼西亚联邦", "FO": "法罗群岛", "FR": "法国", "GA": "加蓬", "GB": "英国", "GD": "格林纳达", "GE": "格鲁吉亚", "GF": "法属圭亚那", "GG": "根西", "GH": "加纳", "GI": "直布罗陀", "GL": "格陵兰", "GM": "冈比亚", "GN": "几内亚", "GP": "瓜德罗普", "GQ": "赤道几内亚", "GR": "希腊", "GS": "南乔治亚和南桑威奇群岛", "GT": "危地马拉", "GU": "关岛", "GW": "几内亚比绍", "GY": "圭亚那", "HM": "赫德岛和麦克唐纳群岛", "HN": "洪都拉斯", "HR": "克罗地亚", "HT": "海地", "HU": "匈牙利", "ID": "印尼", "IE": "爱尔兰", "IL": "以色列", "IM": "马恩岛", "IN": "印度", "IO": "英属印度洋领地", "IQ": "伊拉克", "IR": "伊朗", "IS": "冰岛", "IT": "意大利", "JE": "泽西", "JM": "牙买加", "JO": "约旦", "JP": "日本", "KE": "肯尼亚", "KG": "吉尔吉斯斯坦", "KH": "柬埔寨", "KI": "基里巴斯", "KM": "科摩罗", "KN": "圣基茨和尼维斯", "KP": "朝鲜", "KR": "韩国", "KW": "科威特", "KY": "开曼群岛", "KZ": "哈萨克斯坦", "LA": "老挝", "LB": "黎巴嫩", "LC": "圣卢西亚", "LI": "列支敦士登", "LK": "斯里兰卡", "LR": "利比里亚", "LS": "莱索托", "LT": "立陶宛", "LU": "卢森堡", "LV": "拉脱维亚", "LY": "利比亚", "MA": "摩洛哥", "MC": "摩纳哥", "MD": "摩尔多瓦", "ME": "黑山", "MF": "法属圣马丁", "MG": "马达加斯加", "MH": "马绍尔群岛", "MK": "马其顿", "ML": "马里", "MM": "缅甸", "MN": "蒙古", "MP": "北马里亚纳群岛", "MQ": "马提尼克", "MR": "毛里塔尼亚", "MS": "蒙特塞拉特", "MT": "马耳他", "MU": "毛里求斯", "MV": "马尔代夫", "MW": "马拉维", "MX": "墨西哥", "MY": "马来西亚", "MZ": "莫桑比克", "NA": "纳米比亚", "NC": "新喀里多尼亚", "NE": "尼日尔", "NF": "诺福克岛", "NG": "尼日利亚", "NI": "尼加拉瓜", "NL": "荷兰", "NO": "挪威", "NP": "尼泊尔", "NR": "瑙鲁", "NU": "纽埃", "NZ": "新西兰", "OM": "阿曼", "PA": "巴拿马", "PE": "秘鲁", "PF": "法属波利尼西亚", "PG": "巴布亚新几内亚", "PH": "菲律宾", "PK": "巴基斯坦", "PL": "波兰", "PM": "圣皮埃尔和密克隆", "PN": "皮特凯恩群岛", "PR": "波多黎各", "PS": "巴勒斯坦", "PT": "葡萄牙", "PW": "帕劳", "PY": "巴拉圭", "QA": "卡塔尔", "RE": "留尼汪", "RO": "罗马尼亚", "RS": "塞尔维亚", "RU": "俄罗斯", "RW": "卢旺达", "SA": "沙特阿拉伯", "SB": "所罗门群岛", "SC": "塞舌尔", "SD": "苏丹", "SE": "瑞典", "SG": "新加坡", "SH": "圣赫勒拿", "SI": "斯洛文尼亚", "SJ": "挪威", "SK": "斯洛伐克", "SL": "塞拉利昂", "SM": "圣马力诺", "SN": "塞内加尔", "SO": "索马里", "SR": "苏里南", "SS": "南苏丹", "ST": "圣多美和普林西比", "SV": "萨尔瓦多", "SX": "荷属圣马丁", "SY": "叙利亚", "SZ": "斯威士兰", "TC": "特克斯和凯科斯群岛", "TD": "乍得", "TF": "法属南方和南极洲领地", "TG": "多哥", "TH": "泰国", "TJ": "塔吉克斯坦", "TK": "托克劳", "TL": "东帝汶", "TM": "土库曼斯坦", "TN": "突尼斯", "TO": "汤加", "TR": "土耳其", "TT": "特立尼达和多巴哥", "TV": "图瓦卢", "TZ": "坦桑尼亚", "UA": "乌克兰", "UG": "乌干达", "UM": "美国本土外小岛屿", "US": "美国", "UY": "乌拉圭", "UZ": "乌兹别克斯坦", "VA": "梵蒂冈", "VC": "圣文森特和格林纳丁斯", "VE": "委内瑞拉", "VG": "英属维尔京群岛", "VI": "美属维尔京群岛", "VN": "越南", "VU": "瓦努阿图", "WF": "瓦利斯和富图纳", "WS": "萨摩亚", "YE": "也门", "YT": "马约特", "ZA": "南非", "ZM": "赞比亚", "ZW": "津巴布韦", } RuleModule = map[string]string{ "m_sqli": "SQL 注入检测模块", "m_xss": "XSS 检测模块", "m_csrf": "CSRF 检测模块", "m_ssrf": "SSRF 检测模块", "m_php_unserialize": "PHP 反序列化检测模块", "m_java_unserialize": "Java 反序列化检测模块", "m_file_upload": "文件上传攻击检测模块", "m_file_include": "文件包含攻击检测模块", "m_php_code_injection": "PHP 代码注入检测模块", "m_java": "Java 代码注入检测模块", "m_cmd_injection": "命令注入检测模块", "m_response": "服务器响应检测模块", "m_scanner": "机器人检测模块", "m_http": "畸形 HTTP 协议检测模块", "m_asp_code_injection": "ASP 代码注入检测模块", "m_ssti": "模板注入检测模块", "m_rule": "通用漏洞模块", "m_timeout": "", // 检测超时 "whitelist": "白名单", "blacklist": "黑名单", } // RuleReason TODO: get from libfusion.so, refer to skyview `Fusion.get_rule_detail_dict()` RuleReason = map[string]string{ "6f4922f45568161a8cdf4ad2299f6d23": "访问测试文件的请求", "c6b99e08e56911ecb18f00155d694977": "[HW2021] 深信服终端检测平台远程命令执行漏洞(CNVD-2020-46552)", "c9a2f2f5b1035a1ca3a5aa776914e393": "[HW2020] GitLab 文件读写 (CVE-2017-0915, CVE-2016-9086)", "97e264c2235c4ad0aa61ecd68fa53351": "jenkins 管理员权限开放 (CVE-2018-1999001)", "ee41775801ab11ed90a200155ddb8f4e": "Apache HTTPD SSRF (CVE-2021-40438)", "33e75ff09dd601bbe69f351039152189": "访问系统文件的请求", "1c383cd30b7c298ab50293adfecb7b18": "Struts2 Java 代码注入漏洞", "eff4046e0b0411ed976f00155dd454dd": "蓝凌OA漏洞", "a1d0c6e83f027327d8461063f4ac58a6": "webshell", "d92ebbf7683c5cafbc74456c73bc5d0c": "[HW2020] JIRA OAuth SSRF (CVE-2017-9506)", "dbc22aa2fce011eca0e800163e345065": "[HW2020] Nginx range filter overflow (CVE-2017-7529)", "ae971210f5f411ecae9200163e345065": "[HW2022] VMware 认证绕过漏洞 (CVE-2022-22972)", "5af4619c46045568b4d701ad5208eee0": "[HW2020] Druid 未授权访问", "fbd7939d674997cdb4692d34de8633c4": "Jackson 反序列化 (CVE-2017-17485)", "63400573dc155253869bc90e341708e9": "[HW2020] Apache Spark 未授权访问", "37274a94eadf11eca1a600155dcfc445": "[HW2022] Apache Solr SSRF漏洞 (CVE-2021-27905)", "c4ca4238a0b923820dcc509a6f75849b": "PHP 代码泄露", "3416a75f4cea9109507cacd8e2f2aefc": "phpinfo 泄露", "c9f0f895fb98ab9159f51fd0297e236d": "访问敏感文件的请求", "36564acfab005dcda413a0b489ce8a02": "[HW2020] ffmpeg SSRF (CVE-2016-1898)", "31a7ad75521152eca43e7eb15065e850": "[HW2020] Kibana远程代码执行(CVE-2019-7609)", "03d0598da679b4d68089730de888a945": "Laravel Debug 远程代码执行 (CVE-2021-3129)", "34173cb38f07f89ddbebc2ac9128303f": "XML 实体注入漏洞", "09afe350746411eda37500163e12b978": "Apache Kylin 未授权配置 (CVE-2020-13927)", "4096d530f60c11eca0ad00163e345065": "[HW2020] Horde Groupware Webmail Edition 远程命令执行漏洞", "c6b57d8ce56911ec980b00155d694977": "[HW2021] F5 BIG-IP 远程代码执行漏洞(CVE-2020-5902)", "8964339e01b411ed9bdc00163e345065": "Spring Cloud Config 路径穿越漏洞 (CVE-2020-5405)", "e336d501c94e556d8fa3f6df584e88e2": "[HW2020] Spring WebFlow RCE (CVE-2017-4971)", "c6b9fa2ee56911ecb64f00155d694977": "[HW2021] 通达OA权限提升漏洞(11.5.200417 之前的版本)", "8f1c07fe747c11ed8b7c00163e12b978": "Microsoft Exchange 远程代码执行 (CVE-2020-0688)", "3ef815416f775098fe977004015c6193": "Apache Solr 远程代码执行漏洞 (CVE-2019-0193)", "2e7f58dd541d5c478fc12bc8a64bef41": "[HW2020] S2-005 (2) (CVE-2010-1870)", "1ff1de774005f8da13f42943881c655f": "DEDECMS add suffix", "458c1b2af1f911ecafdb00163e345065": "[HW2021] Sitecore XP远程代码执行漏洞(CVE-2021-42237)", "ac3d6ee2714b11ed87ba00163e12b978": "通达OA V11.x远程代码执行漏洞", "9fa20cc8eabf11ec808600155dcfc445": "[HW2022] 360天擎信息泄露", "2a1f6e4292ee30c1a92d3e8333471534": "Jackson 反序列化", "e404c364f1f511ec9cc500163e345065": "[HW2021] Zabbix 5.0.17-Remote Code Execution (RCE) (Authenticated)", "c6b8cd66e56911ec8cc100155d694977": "[HW2021] 宝塔面板数据库管理未授权访问漏洞(Linux正式版7.4.2、Linux测试版7.5.13、Windows正式版6.8)", "3c27c530751511edbaf200163e12b978": "Citrix 未授权访问 ", "22aaa784da604abeb19b0aeee2b3cc6d": "Jenkins 非预期方法调用漏洞(SECURITY-595)", "a5bfc9e07964f8dddeb95fc584cd965d": "Java 畸形 double 数据 DOS 漏洞", "c6babe8ce56911eca76200155d694977": "[HW2021] 致远OA 文件上传漏洞(V8.0、V7.1、V7.1SP1、V7.0、V7.0SP1、V7.0SP2、V7.0SP3、V6.0、V6.1SP1、V6.1SP2、V5.x)", "d7e82c628f96541c924637f711555266": "[HW2020] PHPStudy 命令执行", "7f39f8317fbdb1988ef4c628eba02591": "访问 IIS 短文件名 / 文件夹", "44f683a84163b3523afe57c2e008bc8c": "扫描 IIS 短文件名 / 文件夹", "c38555f8746411eda4a400163e12b978": "快排CMS 未授权访问 ", "866cbe77d2884225a8fa19793cc8ad51": "ghostscript 命令执行 (CVE-2018-19475)", "17400b535f9f5db5a8424265ed8cc38c": "[HW2020] JeeCMS SSRF", "774b89ca853942cabc8b94bfff46d73c": "ECShop 2.x-3.x 远程代码执行漏洞", "b801a4a10a22422dff885812a5d0d417": "科蓝反序列化漏洞", "6c991e5e23215c589a86f13ae800c616": "[HW2020] Couchdb 垂直越权 (1) (CVE-2017-12635)", "7d5a620cf69311ecbf8400163e345065": "[HW2020] ShardingShpere远程命令执行漏洞 (CVE-2020-1947)", "a26afe82eade11ecb85f00155dcfc445": "[HW2022] Alibaba Nacos认证绕过", "3343e2def60b11ec964400163e345065": "[HW2021] 用友ERP-NC 任意文件读取漏洞 (2021-?-?)", "09189cf61c0e4774a4d8af7bfb0cf6de": "WebLogic wls9-async 和 wls-wsat 反序列化 (CNVD-C-2019-48814)", "738430de6f054031a7597c00eb1fd061": "Drupal Mail 命令注入", "2abf53869fd05cf5a69c11ea42d2cb54": "[HW2020] ActiveMQ 任意文件写入漏洞 (CVE-2016-3088)", "e2c420d928d4bf8ce0ff2ec19b371514": "Drupal 内核远程代码执行漏洞 (CVE-2018-7602)", "ac61a980eae411ec985100155dcfc445": "[HW2022] Zyxel NBG2105 身份验证绕过 (CVE-2021-3297)", "d67d8ab4f4c10bf22aa353e27879133c": "PHP168 代码执行漏洞", "e7629f3e750a11eda3a900163e12b978": "PbootCms v3.1.2 远程代码执行 ", "6d85a117690c32af50f9b47f98abf408": "骑士 CMS 远程代码执行", "4ebd2f6aee2811ec8e2600163e345065": "[HW2022] 齐治堡垒机任意用户登录漏洞", "688142e95d6d446ee901de2b4d087a5d": "F5 BIG-IP漏洞", "91d18ff6f2dc54618b1a28f15afe572f": "[HW2020] Couchdb 命令执行 (CVE-2017-12636)", "70efdf2ec9b086079795c442636b55fb": "访问管理后台的请求", "6d6bcd8ebc2c4d4a961ce2e8b4b3af67": "joomla 远程代码执行", "c6b4be74e56911eca25c00155d694977": "[HW2021] Apache Shiro身份认证绕过漏洞(CVE-2020-17523)", "1f0822d21d0a11edad0900163e345065": "[HW2020] Couchdb 垂直越权 (1) (CVE-2017-12635)", "2024042cf11511ecb17e00163e345065": "[HW2021] GoAhead Server 环境变量注入漏洞(CVE-2021-42342)", "abade038ebc211ec8a8800163e345065": "[HW2022] VMware SSRF XSS)", "32ac3f49feb65bada13cdf85722d237a": "[HW2020] Spring RCE (CVE-2018-1270)", "43ec517d68b6edd3015b3edc9a11367b": "Spring 框架漏洞", "6a48a52c11424cd68e89d26d212556c4": "ThinkPHP5 任意代码执行漏洞", "a5d9fe3b0172b4625a1fe6a41482aa8b": "Xstream 反序列化", "072b030ba126b2f4b2374f342be9ed44": "路径穿越攻击", "38529f8eebc211eca87c00163e345065": "[HW2022] VMware 任意文件读取", "3a3128d0e7e811ec909000155db4c628": "Java 代码注入", "e369853df766fa44e1ed0ff613f563bd": "Struts2 S2-020", "d3d9446802a44259755d38e6d163e820": "访问 Git 仓库的请求", "6e4b3bcf45295dcc9b1f3efef4353287": "[HW2020] JIRA SSRF (CVE-2019-8451)", "b58a180cf11111ec8a4600163e345065": "[HW2022] VMware 服务端模板注入漏洞(CVE-2022-22954)", "7cbbc409ec990f19c78c75bd1e06f215": "Jenkins 远程代码执行漏洞 (CVE-2018-1000861)", "c6b6f9d2e56911ec8e2e00155d694977": "[HW2021] phpStudy nginx 解析漏洞(phptsuy8.1.07的Nginx1.5.11版本)", "b4b143a12d0d5eeabef67b6597dc7cd9": "[HW2020] DedeCMS 密码重置漏洞", "028c274cfc4f11ecaedd00163e345065": "[HW2022] Oracle Access Manager RCE (CVE-2021-35587)", "1fc707c92da6519e9337f202ad1fd959": "[HW2020] 通达OA 前台任意用户登录", "2efd7872752f11ed97f600163e12b978": "XStream 反序列化代码执行 (CVE-2021-29505)", "9355da5d436e52cba6fd2012ad5a5832": "[HW2020] Spring Data Rest RCE (CVE-2017-8046)", "d09bf41544a3365a46c9077ebb5e35c3": "Jackson 反序列化 (CVE-2017-7525)", "c20ad4d76fe97759aa27a0c99bff6710": "访问 SVN 仓库的请求", "e1c17c16169b58078be071b37eda9f3c": "[HW2020] HFS RCE", "e4a2355fa844484bbdf799cc8795b415": "Apache Druid 远程命令执行漏洞", "72c04a1fbd6e5641bb33d0185dc36b9b": "[HW2020] Nexus 后台表达式注入 (CVE-2020-10199, CVE-2019-7238)", "c6b519e6e56911ecaa1c00155d694977": "[HW2021] Drupal任意PHP代码执行漏洞(CVE-2020-28948/28949)", "c81e728d9d4c2f636f067f89cc14862c": "访问敏感文件的请求", "a02582663945fa213788203624eb3b89": "Jackson-databind 反序列化 (CVE-2020-35790/CVE-2020-35491)", "45c48cce2e2d7fbdea1afc51c7c6ad26": "访问 Git 仓库的请求", "029e67ff376659daaea612ec5c3964ab": "[HW2020] Gitea LFS RCE", "e52769085be711ec859e00155dae391c": "Apache Log4j 远程命令执行漏洞", "19ca14e7ea6328a42e0eb13d585e4c22": "Java 代码注入漏洞 (针对 Struts 2 漏洞的通用防御)", "3c59dc048e8850243be8079a5c74d079": "CVE-2012-1823 PHP FastCGI 远程代码执行漏洞", "14766b63abf447ac808b92535dde9c09": "Gogs、Gitea 远程代码执行漏洞 (CVE-2018-18925)", "9bf31c7ff062936a96d3c8bd1f8f2ff3": "后门", "ae0196baf5f211ec98c400163e345065": "[HW2022] Sunlogin-11.0.0.33-RCE-1_获取token", "2838023a778dfaecdc212708f721b788": "访问备份或临时文件的请求", "110ad086eaff11ec87c500155d143f7b": "[HW2022] Gitlab exiftool 远程命令执行漏洞 (CVE-2021-22205)", "e3959e0e30494d6caa1e9b259073d96c": "南方数据管理员添加未授权访问漏洞", "98f13708210194c475687be6106a3b84": "Nginx code 解析漏洞", "887d1820a85a43cfbeee578a02f6a914": "fastjson 反序列化代码执行", "88a4a7227e81d064b551f52f7f51e6c8": "泛微 E-cology OA远程代码执行", "4c15abcbe6e8410aac559817925bfd11": "TBK DVR 验证绕过漏洞 (CVE-2018-9995)", "460072c82cca58999281247426c71bf8": "[HW2020] ffmpeg 文件读取 (CVE-2016-1897)", "96c4bc6a6adf4434b93d714106f4c67f": "ghostscript 任意文件读写 (CVE-2018-17961)", "9dbe327d8dd64f3a8d919cf19d5e3e00": "ThinkPHP 5.0.x _method 代码注入", "6512bd43d9caa6e02c990b0a82652dca": "访问 SVN 仓库的请求", "a12856ca51bb5ed69b6646dafd605e7c": "[HW2020] Apache Shiro 反序列化攻击 (CVE-2016-4437)", "9473dea2eb144fee9ddeac33b49358b9": "Nexus Repository Manager 3 远程代码执行漏洞(CVE-2019-7238)", "eccbc87e4b5ce2fe28308fd9f2a7baf3": "Apache 目录遍历", "e799871cf6b653db94b759b79deefe5d": "[HW2020] ImageMagick RCE (CVE-2016-3714)", "ca037bccfa7c5e7b822ea1889ba7be47": "[HW2020] MetInfo 前台 SQL 注入 (CNVD-2018-20024)", "638d9bb801b611eda81e00163e345065": "Atlassian Jira 服务端请求伪造漏洞 (CVE-2022-26135)", "8ef24d7501e85d1fa3ce4016dbf297d1": "[HW2020] Webmin RCE (CVE-2019-15107)", "81f360f5033c55179d34b37b61389503": "[HW2020] Palo Alto GlobalProtect SSL RCE (CVE-2019-1579)", "c374fb01134a4213be9ba67539523162": "IIS 扩展名漏洞 (CVE-2009-4444)", "21203602eafd11ec8dc300155d143f7b": "[HW2022] F5 BIG-IP 远程代码执行漏洞 (CVE-2021-22986)", "2cba1bd57c9603c07ff4f09f8e4fb0f9": "Apache AXIS 远程命令执行漏洞", "a0ff28f2f5f111eca2de00163e345065": "[HW2022] Weblogic Server 信息泄漏漏洞利用", "d645920e395fedad7bbbed0eca3fe2e0": "DEDECMS SQL 注入漏洞", "2b6e15ac747811eda03000163e12b978": "Apache DolphinScheduler 远程代码执行 (CVE-2020-11974)", "1f5449318bbc38d200fb9f3bcf87a50f": "WebLogic console 远程代码执行漏洞 (CVE-2020-14882)", "02e74f10e0327ad868d138f2b4fdd6f0": "利用 Apache 解析漏洞来执行 PHP 脚本", "3ff4aed51a7c506087c0b9df1b6ac0d3": "[HW2020] Couchdb 垂直越权 (2) (CVE-2017-12635)", "933cbfbbc71c417ab9d757e5dc4be4d4": "Jenkins Accept-Language 信息泄露 (CVE-2018-1999002)", "182be0c5cdcd5072bb1864cdee4d3d6e": "Struts2 S2-016", "6af6b17ceae711ecbd5100155dcfc445": "[HW2022] 三星 WLAN AP WEA453e路由器 RCE (query)", "7f41d19e081411edaca100163e345065": "[HW2022] 三星 WLAN AP WEA453e路由器 RCE (form)", "cfcd208495d565ef66e7dff9f98764da": "JSP 代码泄露", "3b57f3413df14b2380ae8e483cb0dcd5": "Drupalgeddon2 - Drupal核心远程代码执行", "6364d3f0f495b6ab9dcf8d3b5c6e0b01": "Struts2 S2-005", "a98e8c0cf12211ecb1d800163e345065": "[HW2021] VMware vCenter Server 远程代码执行漏洞(CVE-2021-21985)", "b42530a1ea025fa78bb0c6c7d2e1ed95": "[HW2020] PHP FPM RCE (CVE-2019-11043)", "bbe199e1efec469588fde16dce75df36": "coremail 信息泄露", "c6b7547ce56911ec9d3d00155d694977": "[HW2021] SaltStack多个高危漏洞(CVE-2020-16846,CVE-2020-17490,CVE-2020-25592)", "c6b7ae5ee56911ec9d1600155d694977": "[HW2021] SAP NetWeaver AS JAVA 高危漏洞(CVE-2020-6287)", "65d86160f1f711ec971800163e345065": "[HW2021] Total.js框架远程代码执行漏洞(CVE-2021-23389)", "aa4755eb0fdf47ec91e8f285613e7954": "Atlassian 漏洞", "c6b5dc64e56911ecbe7300155d694977": "[HW2021] Microsoft Exchange远程代码执行漏洞(CVE-2020-16875)", "b6f76f653ded54b494292eb6c285d464": "[HW2020] WebLogic 弱口令", "9778d5d219c5080b9a6a17bef029331c": "Apache Solr 远程代码执行漏洞 (CVE-2019-0192)", "c6bb1f3ae56911ec93a500155d694977": "[HW2021] WebSphere 权限提升漏洞(CVE-2020-4276)", "c6ba5e10e56911ec8f6d00155d694977": "[HW2021] 微软 SQL Server 报表服务远程代码执行漏洞(CVE-2020-0618)", "061de838eabf11ec8ee500155dcfc445": "[HW2022] 360天擎SQL注入漏洞", "6ea9ab1baa0efb9e19094440c317e21b": "XML 实体注入漏洞", "039c966635bd4ac286553f55d9457615": "dedecms 5.7 升级时间泄露", "b6d767d2f8ed5d21a44b0e5886680cb9": "DEDECMS add suffix", "f033ab37c30201f73f142449d037028d": "ActiveMQ 任意文件写入漏洞 (CVE-2016-3088)", "558fce096ca65beb8f7ca7d4b2bf6b4f": "[HW2020] ElasticSearch 命令执行 (CVE-2015-1427)", "b53b3a3d6ab90ce0268229151c9bde11": "代码注入漏洞 (针对 WordPress)", "c6b4239ce56911ec8b8a00155d694977": "[HW2021] Apache Shiro 权限绕过漏洞(CVE-2020-13933)", "2408e35abe40580286b6d9106b43357f": "[HW2020] S2-005 (1) (CVE-2010-1870)", "724390fd0bf29ab3adb92ee026474fe1": "Jackson-databind 反序列化 (CVE-2020-36179~CVE-2020-36189)", "ef268f2b31fd5943bd1882e2218e9924": "[HW2020] Tomcat RCE (CVE-2019-0232)", "d49bb7ebe41146deb48138f28d220c7f": "可疑远程调用协议", "aab3238922bcc25a6f606eb525ffdc56": "phpinfo 泄露", "582e036cee2811ec90c700163e345065": "[HW2022] H3C SecPath运维审计系统任意用户登录漏洞", "25d513abf19740b083f2262270c0a281": "PHPCMS v9.6.0 任意文件上传", "ab299838fbe1584d843fc004b0034e81": "[HW2020] GoAhead RCE (CVE-2017-17562)", "c6b81b78e56911ec989800155d694977": "[HW2021] SolarWinds 远程代码执行漏洞(CVE-2020-10148)", "daf7d4a0116711ed8d9c00163e35b03e": "Seacms php代码注入", "c6b93e5ee56911ec8e1500155d694977": "[HW2021] 深信服终端检测平台任意用户登录(CNVD-2020-46552)", "95a4035353ed405ba3418698fc06b544": "spring 目录遍历 (cve-2014-3625)", "eab53f221ee611edb57e00163e345065": "VMware Workspace ONE UEM SSRF (CVE-2021-22054)", "fca34064ebc211ec9b9900163e345065": "[HW2022] VMware vRealize SSRF (CVE-2021-21975)", "c51ce410c124a10e0db5e4b97fc2af39": "PHPSpy 后门", "2f7e97b9204b5239b089c5ab83242519": "[HW2020] ElasticSearch 未授权访问", "6e34a9cb6f5b43e4973b170e626a2f77": "Apache APISIX lua 远程代码执行", "299ce1c43af85c7987d8ddef915888a8": "[HW2020] Zimbra RCE", "529d97a8f5d711eca1e700163e345065": "[HW2022] Weblogic Server 信息泄漏漏洞利用 (CVE-2022-21371)", "49234868ee1f11ec833b00155de98ae1": "[HW2022] AVTECH监控后台命令注入漏洞", "1f0e3dad99908345f7439f8ffabdffc4": "PHP FastCGI 解析漏洞", } ) ================================================ FILE: management/webserver/pkg/constants/detector.go ================================================ package constants const ( ContentType = "application/octet-stream" UpdateEntrypoint = "/update/policy" StatEntrypoint = "/stat" DefaultPolicyVersion = "1" ) ================================================ FILE: management/webserver/pkg/constants/option.go ================================================ package constants const ( SecretKey = "secret_key" MachineID = "machine_id" PolicyGroupGlobal = "policy_group_global" SrcIPConfig = "src_ip_config" ) ================================================ FILE: management/webserver/pkg/constants/session.go ================================================ package constants const ( DefaultSessionUserKey = "user" ) ================================================ FILE: management/webserver/pkg/constants/telemetry.go ================================================ package constants const ( ApplicationJson = "application/json" TelemetryEntryPoint = "/telemetry" TelemetryId = "9e88109c-ebbf-4d82-8a25-f97f2ac5f3c4" Behaviour = "behaviour" FalsePositives = "false_positives" Installation = "Installation" ) var Version string ================================================ FILE: management/webserver/pkg/cron/cron.go ================================================ package cron import ( "github.com/robfig/cron/v3" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/log" ) var logger = log.GetLogger("cron") func newCronWithSeconds() *cron.Cron { secondParser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.DowOptional | cron.Descriptor) return cron.New(cron.WithParser(secondParser), cron.WithChain()) } func StartCron() error { cronInstance := newCronWithSeconds() _, err := cronInstance.AddFunc(SpecUpdatePolicy, CheckAndUpdatePolicy) if err != nil { return err } cronInstance.Start() return nil } ================================================ FILE: management/webserver/pkg/cron/update_policy.go ================================================ package cron import ( "bytes" "encoding/json" "io/ioutil" "net/http" "time" "chaitin.cn/patronus/safeline-2/management/webserver/utils" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" ) // SpecUpdatePolicy http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html // Seconds Minutes Hours Day-of-Month Month Day-of-Week Year (optional field) // every 5 second (starting from 0s) const SpecUpdatePolicy = "0/5 * * * * ?" type statResponseBody struct { PolicyVersion string `json:"policy_version"` } func getBytecode() ([]byte, error) { buff, err := ioutil.ReadFile(config.GlobalConfig.Detector.FslBytecode) if err != nil { return nil, err } return buff, nil } func CheckAndUpdatePolicy() { if config.GlobalConfig.Detector.Addr == "" { return } exist, err := utils.FileExist(config.GlobalConfig.Detector.FslBytecode) if !exist || err != nil { return } data, err := getBytecode() if err != nil { logger.Error(err) return } reader := bytes.NewReader(data) tr := &http.Transport{ MaxIdleConns: 10, IdleConnTimeout: 30 * time.Second, } client := &http.Client{Transport: tr} addr := config.GlobalConfig.Detector.Addr statReq, err := http.NewRequest(http.MethodGet, addr+constants.StatEntrypoint, nil) if err != nil { logger.Error(err) return } updateReq, err := http.NewRequest(http.MethodPost, addr+constants.UpdateEntrypoint, reader) if err != nil { logger.Error(err) return } updateReq.Header.Set("Content-Type", constants.ContentType) statRspData := statResponseBody{} statRsp, err := client.Do(statReq) if err != nil || statRsp.StatusCode != http.StatusOK { logger.Warn(err) return } body, err := ioutil.ReadAll(statRsp.Body) if err != nil { logger.Warn(err) return } err = json.Unmarshal(body, &statRspData) if err != nil { logger.Warn(err) return } if statRspData.PolicyVersion == constants.DefaultPolicyVersion { return } err = statRsp.Body.Close() if err != nil { logger.Warn(err) } logger.Info("Update fsl bytecode") updateRsp, err := client.Do(updateReq) if err != nil { logger.Warn(err) return } if updateRsp.StatusCode != http.StatusOK { logger.Warnf("%s update policy, return %d", addr, updateRsp.StatusCode) } err = updateRsp.Body.Close() if err != nil { logger.Warn(err) } } ================================================ FILE: management/webserver/pkg/database/postgres.go ================================================ package database import ( "gorm.io/driver/postgres" "gorm.io/gorm" "gorm.io/gorm/logger" "gorm.io/gorm/schema" "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" ) type PostgresDB struct { *gorm.DB } var ( // db is not designed to be used directly, use method `GetDB` instead. db PostgresDB ) func GetDB() *PostgresDB { return &db } func InitDB() error { dbConfig := config.GlobalConfig.DB if dbConfig.URL == "" { return errors.New("empty database url") } // URL also works, see https://godoc.org/github.com/lib/pq var ( gormDB *gorm.DB err error ) gormConfig := &gorm.Config{ NamingStrategy: schema.NamingStrategy{ TablePrefix: "mgt_", // table name prefix, table for `User` would be `t_users` SingularTable: true, // use singular table name, table for `User` would be `user` with this option enabled }, AllowGlobalUpdate: false, Logger: logger.Default.LogMode(logger.Silent), } if dbConfig.LogSQL { gormConfig.Logger = logger.Default.LogMode(logger.Info) } gormDB, err = gorm.Open(postgres.Open(dbConfig.URL), gormConfig) if err != nil { return err } db.DB = gormDB return nil } func (db *PostgresDB) SetDB(gormDB *gorm.DB) { db.DB = gormDB } ================================================ FILE: management/webserver/pkg/fvm/fsl/action.go ================================================ package fsl import ( "fmt" "strings" ) func Goto(m string) string { return fmt.Sprintf("GOTO %s", m) } func Wheres(items ...string) string { return strings.Join(items, " AND ") } func Actions(items ...string) string { return strings.Join(items, ", ") } func Set(k, t, v string) string { return fmt.Sprintf("SET %s::%s = %s", k, t, v) } ================================================ FILE: management/webserver/pkg/fvm/fsl/quote.go ================================================ package fsl import ( "fmt" ) // see : https://chaitin.cn/patronus/fvm/-/blob/master/src/util/StrUtil.cpp#L18 func appendEscapedByte(buf []byte, b byte) []byte { if b >= 0x20 && b <= 0x7e && b != '\'' && b != '"' && b != '\\' { return append(buf, b) } switch b { case '\'': return append(buf, `\'`...) case '"': return append(buf, `\"`...) case '\n': return append(buf, `\n`...) case '\t': return append(buf, `\t`...) case '\r': return append(buf, `\r`...) case '\b': return append(buf, `\b`...) case '\f': return append(buf, `\f`...) case '\\': return append(buf, `\\`...) default: return append(buf, fmt.Sprintf("\\x%x", b)...) } } func appendQuotedWith(buf []byte, s string, quote byte) []byte { buf = append(buf, quote) for i := 0; i < len(s); i++ { buf = appendEscapedByte(buf, s[i]) } buf = append(buf, quote) return buf } func quoteWith(s string, quote byte) string { return string(appendQuotedWith(make([]byte, 0, 3*len(s)/2), s, quote)) } func Quote(s string) string { return quoteWith(s, '\'') } ================================================ FILE: management/webserver/pkg/fvm/fsl/selector.go ================================================ package fsl import ( "fmt" ) type Selector struct { TableName string ID string Where string Action string } func NewSelector(tableName, id, where, action string) *Selector { return &Selector{ TableName: tableName, ID: id, Where: where, Action: action, } } func (s *Selector) String() string { ret := fmt.Sprintf("ID %s", Quote(s.ID)) if s.Where != "" { ret += " WHERE " + s.Where } ret += fmt.Sprintf(" ACTION %s", s.Action) return ret } func (s *Selector) AppendInto() string { return fmt.Sprintf("APPEND INTO %s %s;", s.TableName, s.String()) } func AppendInto(tableName, id, where, action string) string { return NewSelector(tableName, id, where, action).AppendInto() } ================================================ FILE: management/webserver/pkg/fvm/fsl/state.go ================================================ package fsl import ( "chaitin.cn/dev/go/errors" ) type State int const ( S_NOP State = iota S_ABORT S_RETURN ) func (s State) String() string { switch s { case S_NOP: return "NOP" case S_ABORT: return "ABORT" case S_RETURN: return "RETURN" default: panic(errors.New("wrong State")) } } ================================================ FILE: management/webserver/pkg/fvm/fsl/table.go ================================================ package fsl import ( "fmt" "strings" ) type Table struct { Name string Mappings map[State]State } func NewTable(name string, mappings map[State]State) *Table { return &Table{ Name: name, Mappings: mappings, } } func (t *Table) String() string { ret := fmt.Sprintf("TABLE %s", t.Name) if len(t.Mappings) > 0 { var mappings []string for from, to := range t.Mappings { mappings = append(mappings, fmt.Sprintf("%s TO %s", from.String(), to.String())) } ret += fmt.Sprintf(" MAPPING %s", strings.Join(mappings, ", ")) } return ret } func (t *Table) Create() string { return fmt.Sprintf("CREATE %s;", t.String()) } func CreateTable(name string, mappings map[State]State) string { return NewTable(name, mappings).Create() } ================================================ FILE: management/webserver/pkg/fvm/fsl/target.go ================================================ package fsl import ( "fmt" ) type Target struct { ID string Type string Args string } func NewTarget(id string, typ string, args string) *Target { return &Target{ ID: id, Type: typ, Args: args, } } func NewSkynetTarget(id, configJSONStr string) *Target { return NewTarget(id, "skynet", configJSONStr) } func (t *Target) String() string { return fmt.Sprintf("%s TYPE %s ARGS (%s)", t.ID, t.Type, t.Args) } func (t *Target) Create() string { return fmt.Sprintf("CREATE TARGET %s;", t.String()) } func CreateTarget(id, configJSONStr string) string { return NewSkynetTarget(id, configJSONStr).Create() } ================================================ FILE: management/webserver/pkg/fvm/fvm.go ================================================ package fvm /* #cgo CFLAGS: -I../../submodule/fvm/include -I../../submodule/libct/include #cgo LDFLAGS: -L../../submodule/fvm/lib -lfvm #include #include #include #include #include #include static inline fvm_update_t *build_update(const void* buf, size_t len) { ct_string_t tmp = CT_STRING_FROM_PTR_LENGTH(buf, len); return fvm_update_create(tmp); } */ import ( "C" //nolint:typecheck ) import ( "crypto/tls" "fmt" "io" "net/http" "net/url" "unsafe" "chaitin.cn/dev/go/errors" "chaitin.cn/dev/go/log" ) // Constant for Output const ( OutputItemTable int = 0 OutputItemSelector int = 1 OutputItemTarget int = 2 ) // FVM wrapper type FVM struct { framework *C.fvm_framework_t apiTable *C.char version int64 } type FVMUpdate struct { update *C.fvm_update_t } type FVMRe struct { data []byte } func maybePointer(array []byte) unsafe.Pointer { if len(array) == 0 { return nil } return unsafe.Pointer(&array[0]) } func (u *FVMUpdate) ToBytes() []byte { return C.GoBytes(unsafe.Pointer(u.update.buf.ptr), C.int(u.update.buf.length)) } func (u *FVMUpdate) FromBytes(out []byte) error { ptr := C.build_update(unsafe.Pointer(&out[0]), C.ulong(len(out))) if ptr == nil { return errors.New("failed to create update from db") } u.update = ptr return nil } func (u *FVMUpdate) MergeUpdate(patch *FVMUpdate) error { m := C.fvm_update_merge(u.update, patch.update) if m == nil { return errors.New("failed to merge two updates") } C.fvm_update_destroy(u.update) u.update = m return nil } func (u *FVMUpdate) Release() { C.fvm_update_destroy(u.update) } func (r *FVMRe) ToBytes() []byte { return r.data } func (r *FVMRe) FromBytes(o []byte) { r.data = o } type FVMOutput struct { output *C.fvm_output_t } // New creates FVM func New() (*FVM, error) { apiTable := C.fvm_api_table() fw := C.fvm_framework_acquire(nil) if fw == nil { return nil, errors.New("fvm_framework_acquire returns a null pointer") } return &FVM{framework: fw, apiTable: apiTable, version: 0}, nil } func (f *FVM) Update(upd *FVMUpdate) error { if upd == nil || upd.update == nil { return errors.New("null pointer passed to update") } ret := bool(C.fvm_update(f.framework, upd.update)) if !ret { return errors.New("update framework failed") } return nil } func (f *FVM) Compile(text string, re []byte) (*FVMOutput, *FVMRe, error) { cplPtr := C.fvm_compiler_acquire(f.apiTable) cText := C.CString(text) defer C.fvm_compiler_release(cplPtr) defer C.free(unsafe.Pointer(cText)) ok := C.fvm_compiler_load_re(cplPtr, (*C.char)(maybePointer(re)), C.ulong(len(re))) if !ok { return nil, nil, errors.New("failed to load re") } outputPtr := C.fvm_compile(cplPtr, cText) if outputPtr == nil { return nil, nil, errors.New("compiling FSL failed") } var cRePtr *C.char var cReLen C.ulong ok = C.fvm_compiler_dump_re(cplPtr, &cRePtr, &cReLen) if !ok { return nil, nil, errors.New("failed to dump re") } defer C.free(unsafe.Pointer(cRePtr)) cRe := &FVMRe{C.GoBytes(unsafe.Pointer(cRePtr), C.int(cReLen))} return &FVMOutput{outputPtr}, cRe, nil } func (f *FVM) Plot() string { plotP := C.fvm_plot(f.framework) defer C.fvm_plot_destroy(plotP) goBuf := C.GoBytes(unsafe.Pointer(plotP.buf.ptr), C.int(plotP.buf.length)) return string(goBuf) } func (f *FVM) Dump() *FVMUpdate { return &FVMUpdate{C.fvm_dump(f.framework)} } func (f *FVM) Release() { C.fvm_framework_release(f.framework) } func Serialize(out *FVMOutput, diff bool, base int64, target int64) *FVMUpdate { updatePtr := C.fvm_serialize(out.output, C.bool(diff), C.long(base), C.long(target)) if updatePtr == nil { return nil } ctStrPtr := updatePtr.buf.ptr if ctStrPtr == nil { return nil } return &FVMUpdate{updatePtr} } func ReleaseOutput(out *FVMOutput) { C.fvm_output_destroy(out.output) } func (f *FVM) PushFsl(server string, update *FVMUpdate) error { log.Infof("Push FSL to %s", server) var length = uint32(update.update.buf.length) var p = uintptr(unsafe.Pointer(update.update.buf.ptr)) log.Infof("Get update length %d, %d", length, C.int(update.update.buf.length)) u := &upload{ Length: length, P: p, } base, _ := url.Parse(server) base.Path = "/update/policy" snURL := base.String() req, err := http.NewRequest( "POST", snURL, u, ) if err != nil { return errors.Annotatef(err, "create request failed") } req.Header.Add("Content-Type", "application/octet-stream") // req.Header.Set("Connection", "close") httpClient := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } var ( respErr error resp *http.Response ) for i := 0; i < 3; i++ { resp, respErr = httpClient.Do(req) if respErr == nil { break } } if respErr != nil { return errors.Annotatef(respErr, "Get response failed") } if resp != nil && resp.StatusCode != 200 { bodyBytes, err := io.ReadAll(resp.Body) if err != nil { log.Fatal(err) } bodyString := string(bodyBytes) return errors.New(fmt.Sprintf("Push FSL response(%d %s)", resp.StatusCode, bodyString)) } return nil } type upload struct { Length uint32 off uint32 P uintptr } func (u *upload) Read(p []byte) (n int, err error) { if u.off >= u.Length { if len(p) == 0 { return 0, nil } return 0, io.EOF } plen := uint32(len(p)) var i uint32 for i = 0; i < plen; i++ { if i+u.off >= u.Length { break } p[i] = *(*byte)(unsafe.Pointer(u.P + unsafe.Sizeof(p[0])*uintptr(i+u.off))) } u.off = i + u.off return int(i), nil } ================================================ FILE: management/webserver/pkg/fvm/generator.go ================================================ package fvm import ( "encoding/json" "fmt" "strings" "gorm.io/gorm" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/dev/go/errors" "chaitin.cn/dev/go/log" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/fvm/fsl" ) const ( Main = "main" Preprocess = "preprocess" MassPackage = "mass_package" PolicyRule = "policy_rule" PolicyRuleGlobal = "policy_rule_global" PolicyGroup = "policy_group" PolicyGroupDetect = "policy_group_detect" WeblogGenerate = "weblog_generate" ReqWeblogGenerate = "req_weblog_generate" RspWeblogGenerate = "rsp_weblog_generate" CommonLog = "common_log" STR = "STRING" INT = "INTEGER" INET = "INET" BOOL = "BOOLEAN" ) var logger = log.GetLogger("fvm") func InitFVMBytecode() { go func() { fullFSL, err := GenerateFullFSL(database.GetDB().DB) if err != nil { logger.Fatalln("Failed to generate fsl", err) } if err = CompileAndSave(fullFSL); err != nil { logger.Fatalln("Failed to compile and save fsl", err) } }() } func PushFSL(tx *gorm.DB) error { fullFSL, err := GenerateFullFSL(tx) if err != nil { logger.Debugf("Full fsl: %s", fullFSL) logger.Error(err) return err } if err = CompileAndPush(fullFSL, config.GlobalConfig.Detector.Addr); err != nil { logger.Error(err) return err } return nil } func GenerateFullFSL(tx *gorm.DB) (fullFSL string, err error) { logger.Info("Generate FSL") pFSL, err := PreprocessTable(tx) if err != nil { return "", err } fullFSL += pFSL fullFSL += MassPackageTable() prFSL, err := PolicyRuleTable(tx) if err != nil { return "", err } fullFSL += prFSL pgFSL, err := PolicyGroupTable(tx) if err != nil { return "", err } fullFSL += pgFSL fullFSL += WeblogGenerateTable() fullFSL += MainTable() return fullFSL, err } func PreprocessTable(db *gorm.DB) (pFSL string, err error) { pFSL += fsl.CreateTable(Preprocess, nil) pFSL += fsl.AppendInto(Preprocess, "http_parse", "", "http_parse()") pFSL += fsl.AppendInto(Preprocess, "init_variables", "", fsl.Actions( fsl.Set("@site_uuid", STR, "get_http_all_headers('SL-CE-SUID')"), fsl.Set("@src_ip", INET, "socket.src_ip"), fsl.Set("@skip_remaining", BOOL, "false"), fsl.Set("@rule_id", STR, fsl.Quote("")), fsl.Set("@attack_type", INT, "-1"), fsl.Set("@final_action", INT, "0"), fsl.Set("@risk_level", INT, "0"), fsl.Set("@host", STR, "http.decayed_host"), fsl.Set("@start_time", INT, "now.timestamp_us"), fsl.Set("@transport_scheme", STR, "transport.scheme"), fsl.Set("@forward_log", INT, "0"), fsl.Set("@convert_to_alog", INT, "1"), fsl.Set("@alog_save_to_db", INT, "0"), fsl.Set("@convert_to_blog", INT, "0"), fsl.Set("@blog_save_to_db", INT, "0"), fsl.Set("@convert_to_dlog", INT, "0"), fsl.Set("@dlog_save_to_db", INT, "0"), )) pFSL += fsl.AppendInto(Preprocess, "set_host", "@host::STRING = ''", fsl.Actions(fsl.Set("@host", STR, "inet_to_string(socket.dst_ip)"))) pFSL += fsl.AppendInto(Preprocess, "adapt_http2", "@transport_scheme::STRING = 'http2'", fsl.Actions(fsl.Set("@transport_scheme", STR, fsl.Quote("https")))) // get src_ip sc, err := model.GetSrcIPConfig(db) if err != nil { return "", err } if sc.Source == model.HTTPHeader { key := fmt.Sprintf("string_to_inet(get_http_all_headers(%s))", fsl.Quote(sc.Value)) pFSL += fsl.AppendInto(Preprocess, "set_src_ip", fmt.Sprintf("%s != '::/0'::INET", key), fsl.Actions(fsl.Set("@src_ip", INET, key))) } return pFSL, nil } func MassPackageTable() (mpFSL string) { mpFSL += fsl.CreateTable(MassPackage, nil) mpFSL += fsl.AppendInto(MassPackage, "mass_package_detect", "string.length(http.raw_req_body) > 1048576", fsl.Actions( fsl.Set("@skip_remaining", BOOL, "true"), fsl.Set("@rule_id", STR, fsl.Quote("mass_package")), fsl.Set("@final_action", INT, "0"), fsl.Set("@convert_to_dlog", INT, "1"), fsl.Set("@dlog_save_to_db", INT, "1"), fsl.Set("@attack_type", INT, "-4"), )) return mpFSL } func PolicyRuleTable(db *gorm.DB) (prFSL string, err error) { prFSL += fsl.CreateTable(PolicyRuleGlobal, map[fsl.State]fsl.State{fsl.S_ABORT: fsl.S_RETURN}) var policyRuleList []model.PolicyRule res := db.Model(&model.PolicyRule{}).Where(&model.PolicyRule{IsEnabled: true}).Order("id desc").Find(&policyRuleList) if res.Error != nil { err = errors.New(fmt.Sprintf("Error occurred when fetch PolicyRule: %s", res.Error)) } for _, policyRule := range policyRuleList { var patternBytes []byte patternBytes, err = policyRule.Pattern.MarshalJSON() if err != nil { return "", err } var patterns []model.PolicyRulePattern err = json.Unmarshal(patternBytes, &patterns) if err != nil { return "", err } var wheres []string for _, pattern := range patterns { // transform key to string var key string switch pattern.K { case model.KeySrcIP: key = "inet_to_string(@src_ip::INET)" case model.KeyURI: key = "@uri_decoded::STRING" case model.KeyHost: key = "http.host" default: return "", errors.New("wrong Key") } switch pattern.Op { case model.OpEq: wheres = append(wheres, fmt.Sprintf("string.equals_case(%s, %s)", key, fsl.Quote(pattern.V))) case model.OpMatch: // . -> \. // * -> .* valueQuote := strings.ReplaceAll(fsl.Quote(strings.ReplaceAll(pattern.V, ".", "\\.")), "*", ".*") wheres = append(wheres, fmt.Sprintf("pcre_match(%s, %s)", valueQuote, key)) case model.OpCIDR: // use INET for CIDR compare key = "@src_ip::INET" wheres = append(wheres, fmt.Sprintf("%s IN CIDR(%s)", key, fsl.Quote(pattern.V))) case model.OpHas: wheres = append(wheres, fmt.Sprintf("string.contains_case(%s, %s)", key, fsl.Quote(pattern.V))) case model.OpPrefix: l := len(pattern.V) wheres = append(wheres, fmt.Sprintf("string.equals_case(string.substr(%s, 0, %d), %s)", key, l, fsl.Quote(pattern.V))) case model.OpRe: wheres = append(wheres, fmt.Sprintf("pcre_match(%s, %s)", fsl.Quote(pattern.V), key)) default: return "", errors.New("wrong Operator") } } // save log only when deny and dry_run logOption := "1" attackType := "-3" // DROP or ACCEPT appendix := "DROP" if policyRule.Action == 0 { logOption = "0" attackType = "-2" appendix = "set_ctx_allow(), ACCEPT" // this selector seems no use } prFSL += fsl.AppendInto(PolicyRuleGlobal, fmt.Sprintf("%s_%d", PolicyRuleGlobal, policyRule.ID), fsl.Wheres(wheres...), fsl.Actions( fsl.Set("@skip_remaining", BOOL, "true"), fsl.Set("@rule_id", STR, fsl.Quote(fmt.Sprintf("/%s", policyRule.Comment))), fsl.Set("@final_action", INT, fmt.Sprintf("%d", policyRule.Action)), fsl.Set("@convert_to_dlog", INT, logOption), fsl.Set("@dlog_save_to_db", INT, logOption), fsl.Set("@attack_type", INT, attackType), appendix, // no use in v1.1 //fsl.Set("@risk_level", INT, ""), )) } prFSL += fsl.CreateTable(PolicyRule, nil) prFSL += fsl.AppendInto(PolicyRule, "preprocess", "", fsl.Actions(fsl.Set("@uri_decoded", STR, "url_decode(http.uri)"))) prFSL += fsl.AppendInto(PolicyRule, "policy_rule_global", "", fsl.Goto(PolicyRuleGlobal)) prFSL += fsl.AppendInto(PolicyRule, "set_status_code", "@final_action::INTEGER = 1", "set_status_code(403)") return prFSL, err } func PolicyGroupTable(db *gorm.DB) (pgFSL string, err error) { skynetConfigStr, err := model.GetSkynetConfig(db) if err != nil { return "", err } pgFSL += fsl.CreateTarget("skynet_config", fsl.Quote(skynetConfigStr)) pgFSL += fsl.CreateTable(PolicyGroupDetect, map[fsl.State]fsl.State{fsl.S_ABORT: fsl.S_RETURN}) pgFSL += fsl.AppendInto(PolicyGroupDetect, "do_detect", "", "target 'skynet_config' type skynet do detect()") pgFSL += fsl.CreateTable(PolicyGroup, nil) pgFSL += fsl.AppendInto(PolicyGroup, "do_detect", "", fsl.Goto(PolicyGroupDetect)) pgFSL += fsl.AppendInto(PolicyGroup, "dlog_persistence", "skynet.enable_log = true", fsl.Actions( fsl.Set("@convert_to_dlog", INT, "1"), fsl.Set("@dlog_save_to_db", INT, "1"), )) pgFSL += fsl.AppendInto(PolicyGroup, "set_status_code", "skynet.action = 1", "set_status_code(403)") pgFSL += fsl.AppendInto(PolicyGroup, "hit_policy_group", "skynet.rule_id != ''", fsl.Actions( fsl.Set("@skip_remaining", BOOL, "true"), fsl.Set("@rule_id", STR, "skynet.rule_id"), fsl.Set("@attack_type", INT, "skynet.main_attack"), fsl.Set("@final_action", INT, "skynet.action"), fsl.Set("@risk_level", INT, "skynet.risk_level"), fsl.Set("@convert_to_dlog", INT, "1"), )) return pgFSL, nil } func CommonLogTable() (clFSL string) { clFSL += fsl.CreateTable(CommonLog, nil) clFSL += fsl.AppendInto(CommonLog, "batch_set_payload_string", "", "weblog_payload_set_string_batch({'event_id', 'scheme', 'dst_ip', 'socket_ip', 'src_ip'}::ARRAY(STRING), {extra.uuid, transport.scheme, inet_to_string(socket.dst_ip), inet_to_string(socket.src_ip), inet_to_string(@src_ip::INET)}::ARRAY(STRING))") clFSL += fsl.AppendInto(CommonLog, "batch_set_payload_int", "", "weblog_payload_set_int_batch({'src_port', 'dst_port'}::ARRAY(STRING), {socket.src_port, socket.dst_port}::ARRAY(INTEGER))") clFSL += fsl.AppendInto(CommonLog, "batch_set_extra_bool", "", "weblog_extra_set_bool_batch({'alog_save_to_db', 'convert_to_blog', 'forward_log', 'convert_to_alog', 'convert_to_dlog', 'dlog_save_to_db', 'blog_save_to_db'}::ARRAY(STRING), {@alog_save_to_db::INTEGER, @convert_to_blog::INTEGER, @forward_log::INTEGER, @convert_to_alog::INTEGER, @convert_to_dlog::INTEGER, @dlog_save_to_db::INTEGER, @blog_save_to_db::INTEGER}::ARRAY(INTEGER))") clFSL += fsl.AppendInto(CommonLog, "generate", "", "weblog_generate()") return clFSL } func ReqWeblogGenerateTable() (reqWGFSL string) { reqWGFSL += fsl.CreateTable(ReqWeblogGenerate, nil) reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "init_http_req_body", "", fsl.Actions(fsl.Set("@raw_req_body", STR, "http.raw_req_body"))) reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "cut_http_req_body", "string.length(@raw_req_body::STRING) > 1048576", fsl.Actions(fsl.Set("@raw_req_body", STR, "string.substr(http.raw_req_body, 0, 1048576)"))) reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "batch_set_payload_binary", "", "weblog_payload_set_binary_batch({'req_payload', 'req_body'}::ARRAY(STRING), {skynet.payload, @raw_req_body::STRING}::ARRAY(STRING))") reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "batch_set_payload_string", "", "weblog_payload_set_string_batch({'site_uuid', 'req_header', 'req_detector_name', 'req_proxy_name', 'req_rule_id', 'req_location', 'req_decode_path', 'user_agent', 'query_string', 'method', 'host', 'referer', 'url_path', 'cookie'}::ARRAY(STRING), {@site_uuid::STRING, http.raw_req_header, fusion.node_id, extra.proxy_name, @rule_id::STRING, skynet.location, skynet.decode_path, http.user_agent, http.decoded_query, http.raw_method, @host::STRING, http.referer, http.path, http.raw_cookie}::ARRAY(STRING))") reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "batch_set_payload_int", "", "weblog_payload_set_int_batch({'req_start_time', 'req_detect_time', 'req_block_reason', 'req_attack_type', 'req_risk_level', 'req_end_time', 'req_action'}::ARRAY(STRING), {extra.req_start_time, now.timestamp_us - @start_time::INTEGER, fusion.block_reason, @attack_type::INTEGER, @risk_level::INTEGER, extra.req_end_time, @final_action::INTEGER}::ARRAY(INTEGER))") reqWGFSL += fsl.AppendInto(ReqWeblogGenerate, "common_log", "", fsl.Goto(CommonLog)) return reqWGFSL } func RspWeblogGenerateTable() (rspWGFSL string) { rspWGFSL += fsl.CreateTable(RspWeblogGenerate, nil) rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "init_http_rsp_body", "", fsl.Actions(fsl.Set("@raw_rsp_body", STR, "http.raw_rsp_body"))) rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "cut_http_rsp_body", "string.length(@raw_rsp_body::STRING) > 1048576", fsl.Actions(fsl.Set("@raw_rsp_body", STR, "string.substr(http.raw_rsp_body, 0, 1048576)"))) rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "batch_set_payload_binary", "", "weblog_payload_set_binary_batch({'rsp_payload', 'rsp_body'}::ARRAY(STRING), {skynet.payload, @raw_rsp_body::STRING}::ARRAY(STRING))") rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "batch_set_payload_string", "", "weblog_payload_set_string_batch({'rsp_rule_id', 'rsp_location', 'rsp_decode_path', 'rsp_header', 'rsp_detector_name', 'rsp_proxy_name'}::ARRAY(STRING), {@rule_id::STRING, skynet.location, skynet.decode_path, http.raw_rsp_header, fusion.node_id, extra.proxy_name}::ARRAY(STRING))") rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "batch_set_payload_int", "", "weblog_payload_set_int_batch({'rsp_detect_time', 'status_code', 'rsp_start_time', 'rsp_end_time', 'rsp_action', 'rsp_attack_type', 'rsp_risk_level', 'rsp_block_reason'}::ARRAY(STRING), {now.timestamp_us - @start_time::INTEGER, http.status_code, extra.rsp_start_time, extra.rsp_end_time, @final_action::INTEGER, @attack_type::INTEGER, @risk_level::INTEGER, fusion.block_reason}::ARRAY(INTEGER))") rspWGFSL += fsl.AppendInto(RspWeblogGenerate, "common_log", "", fsl.Goto(CommonLog)) return rspWGFSL } func WeblogGenerateTable() (wgFSL string) { wgFSL += CommonLogTable() wgFSL += ReqWeblogGenerateTable() wgFSL += RspWeblogGenerateTable() wgFSL += fsl.CreateTable(WeblogGenerate, nil) wgFSL += fsl.AppendInto(WeblogGenerate, "req", "fusion.stage = 0", fsl.Goto(ReqWeblogGenerate)) wgFSL += fsl.AppendInto(WeblogGenerate, "rsp", "fusion.stage = 1", fsl.Goto(RspWeblogGenerate)) return wgFSL } func MainTable() (mainFSL string) { mainFSL += fsl.CreateTable(Main, nil) mainFSL += fsl.AppendInto(Main, "m_preprocess", "", fsl.Goto(Preprocess)) mainFSL += fsl.AppendInto(Main, "m_mass_package", "", fsl.Goto(MassPackage)) mainFSL += fsl.AppendInto(Main, "m_policy_rule", "@skip_remaining::BOOLEAN = false", fsl.Goto(PolicyRule)) mainFSL += fsl.AppendInto(Main, "m_policy_group", "@skip_remaining::BOOLEAN = false", fsl.Goto(PolicyGroup)) mainFSL += fsl.AppendInto(Main, "m_weblog_generate", "", fsl.Goto(WeblogGenerate)) mainFSL += "ENTRYPOINT TABLE main;" return mainFSL } ================================================ FILE: management/webserver/pkg/fvm/helper.go ================================================ package fvm import ( "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) var GlobalFVM *FVM func init() { var err error GlobalFVM, err = New() if err != nil { panic(err) } } func CompileAndSave(text string) error { output, _, err := GlobalFVM.Compile(text, nil) if err != nil { return errors.Annotate(err, "failed to compile rules") } defer ReleaseOutput(output) update := Serialize(output, false, 0, 1) defer update.Release() // save to file if err = utils.EnsureWriteFile(config.GlobalConfig.Detector.FslBytecode, update.ToBytes(), 0666); err != nil { return errors.Annotate(err, "failed to save bytecode") } return nil } func CompileAndPush(text, serverAddr string) error { output, _, err := GlobalFVM.Compile(text, nil) if err != nil { return errors.Annotate(err, "failed to compile rules") } defer ReleaseOutput(output) update := Serialize(output, false, 0, 1) defer update.Release() // save to file if err = utils.EnsureWriteFile(config.GlobalConfig.Detector.FslBytecode, update.ToBytes(), 0666); err != nil { return errors.Annotate(err, "failed to save bytecode") } if err := GlobalFVM.PushFsl(serverAddr, update); err != nil { return errors.Annotatef(err, "failed to push rule to '%s'", serverAddr) } return nil } ================================================ FILE: management/webserver/pkg/log/log.go ================================================ package log import ( "fmt" "os" "runtime" "strings" "github.com/sirupsen/logrus" "chaitin.cn/dev/go/log" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) func GetLogger(name string) *log.Logger { return log.GetLogger(name) } func LoadLogLevel() { lv, _ := log.ParseLevel(config.GlobalConfig.Log.Level) log.SetLevel(log.AllLoggers, lv) } func SetLogFormatter() { // format formatter := new(log.TextFormatter) formatter.FullTimestamp = true formatter.TimestampFormat = "2006/01/02 15:04:05" log.SetFormatter(log.AllLoggers, formatter) } func InitLogger() error { // output switch config.GlobalConfig.Log.Output { case "stdout": log.SetOutput(log.AllLoggers, os.Stdout) case "stderr": log.SetOutput(log.AllLoggers, os.Stderr) default: exist, err := utils.FileExist(config.GlobalConfig.Log.Output) if err != nil { return err } fileFlag := os.O_WRONLY | os.O_APPEND | os.O_SYNC if !exist { if err := utils.EnsureFileDir(config.GlobalConfig.Log.Output); err != nil { return err } fileFlag = fileFlag | os.O_CREATE } if fp, err := os.OpenFile(config.GlobalConfig.Log.Output, fileFlag, os.ModePerm); err != nil { return fmt.Errorf("failed to open log file: %s", err.Error()) } else { log.SetOutput(log.AllLoggers, log.NewLockOutput(fp)) } } // hook log.AddHook(log.AllLoggers, NewRuntimeHook()) log.AddHook(log.AllLoggers, log.NewErrorStackHook(true)) // level LoadLogLevel() return nil } type RuntimeHook struct{} func (h *RuntimeHook) Levels() []logrus.Level { return logrus.AllLevels } func (h *RuntimeHook) Fire(entry *logrus.Entry) error { file := "???" funcName := "???" line := 0 pc := make([]uintptr, 64) // Skip runtime.Callers, self, and another call from logrus n := runtime.Callers(3, pc) if n != 0 { pc = pc[:n] // pass only valid pcs to runtime.CallersFrames frames := runtime.CallersFrames(pc) // Loop to get frames. // A fixed number of pcs can expand to an indefinite number of Frames. for { frame, more := frames.Next() if !strings.Contains(frame.File, "github.com/sirupsen/logrus") && !strings.Contains(frame.Function, "chaitin.cn/dev/go") { file = frame.File funcName = frame.Function line = frame.Line break } if !more { break } } } slices := strings.Split(file, "/") file = slices[len(slices)-1] funcName = strings.ReplaceAll(funcName, "chaitin.cn", "") entry.Data["file"] = file entry.Data["func"] = funcName entry.Data["line"] = line return nil } func NewRuntimeHook() *RuntimeHook { return &RuntimeHook{} } ================================================ FILE: management/webserver/pkg/telemetry.go ================================================ package pkg import ( "fmt" "io" "net/http" "strconv" "time" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" "chaitin.cn/patronus/safeline-2/management/webserver/utils" ) type WebsiteResult struct { Id uint UpdatedAt time.Time } type TelemetryInfo struct { Id string `json:"id"` } type SafelineInfo struct { Id string `json:"id"` Type string `json:"type"` Version string `json:"version"` ReqCnt int `json:"req_cnt"` DetectLogCnt int `json:"detect_log_cnt"` SiteCnt int `json:"site_cnt"` HealthySiteCnt int `json:"healthy_site_cnt"` RuleCnt int `json:"rule_cnt"` IsHealthy bool `json:"is_healthy"` Behavior map[string]int `json:"behavior"` Websites []WebsiteResult `json:"websites"` WebsitesCnt int `json:"websites_cnt"` } type TelemetryRequest struct { Telemetry TelemetryInfo `json:"telemetry"` Safeline SafelineInfo `json:"safeline"` } func GetUploadTimestamp() string { yesterday := time.Now().AddDate(0, 0, -1) yesterdayNoon := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 12, 0, 0, 0, yesterday.Location()) return strconv.FormatInt(yesterdayNoon.Unix(), 10) } func GetUploadNonce() string { now := strconv.FormatInt(time.Now().Unix(), 10) randStr := utils.RandStr(6) return fmt.Sprintf("%s%s", now, randStr) } func DoPostTelemetry(client *http.Client, addr string, reader io.Reader) (*http.Response, error) { req, err := http.NewRequest(http.MethodPost, "https://"+addr+constants.TelemetryEntryPoint, reader) if err != nil { return nil, err } req.Header.Set("accept", constants.ApplicationJson) req.Header.Set("Content-Type", constants.ApplicationJson) rsp, err := client.Do(req) if err != nil { return nil, err } return rsp, nil } ================================================ FILE: management/webserver/proto/website/website.proto ================================================ syntax = "proto3"; package website; option go_package = "proto/website"; service Website { rpc Subscribe(stream Response) returns (stream Event) {} } // From client-side, may be "pong" message Response { string type = 1; bytes msg = 2; bool err = 3; } // From server-side, may be "ping" message Event { string type = 1; // ping/website bytes msg = 2; } ================================================ FILE: management/webserver/rpc/main.go ================================================ package rpc import ( "net" "time" "google.golang.org/grpc" "chaitin.cn/dev/go/errors" "chaitin.cn/dev/go/log" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/config" pb "chaitin.cn/patronus/safeline-2/management/webserver/proto/website" ) const ( WaitRspTimeout = 30 * time.Second KeepaliveTime = 5 * time.Second KeepaliveTimeout = 30 * time.Second Ping = "ping" Pong = "pong" EventTypeWebsite = "website" EventTypeDeleteWebsite = "deleteWebsite" EventTypeFullWebsite = "fullWebsite" ) var logger = log.GetLogger("grpc") func StartGRPCSever() error { lis, err := net.Listen("tcp", config.GlobalConfig.GPRC.ListenAddr) if err != nil { return errors.Wrap(err, "Failed to listen") } var opts []grpc.ServerOption grpcServer := grpc.NewServer(opts...) pb.RegisterWebsiteServer(grpcServer, GetWebsiteServer()) go func() { err := grpcServer.Serve(lis) if err != nil { logger.Fatalln("Failed to server") } }() return nil } ================================================ FILE: management/webserver/rpc/website.go ================================================ package rpc import ( "encoding/json" "io" "time" "chaitin.cn/dev/go/errors" "chaitin.cn/patronus/safeline-2/management/webserver/model" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/database" pb "chaitin.cn/patronus/safeline-2/management/webserver/proto/website" ) var ( ping = pb.Event{Type: Ping, Msg: nil} Subscriber *StreamClient // only ONE sub ) func Publish(msg []byte, eventType string) error { if Subscriber == nil { return errors.New("Service is abnormal, and the nginx conf cannot be updated for the time being. Please go to the shell to check the relevant logs") } err := Subscriber.stream.Send(&pb.Event{ Type: eventType, Msg: msg, }) if err != nil { select { case <-Subscriber.stream.Context().Done(): err = Subscriber.stream.Context().Err() // context canceled Subscriber = nil return err case Subscriber.errCh <- errors.Wrapf(err, "Send event err: %s", msg): return errors.Wrapf(err, "Send event err: %s", msg) } } rspTimeoutTicker := time.NewTicker(WaitRspTimeout) defer rspTimeoutTicker.Stop() for { select { case rsp := <-Subscriber.rspCh: if rsp.Err { return errors.New(string(rsp.Msg)) } else { // success return nil } case <-rspTimeoutTicker.C: return errors.New("Wait timeout for updating result") } } } // StreamClient is the instance for every client stream connected type StreamClient struct { stream pb.Website_SubscribeServer timer *time.Timer // timer for timeout errCh chan error rspCh chan *pb.Response quit chan struct{} // quit stream client gracefully } func newStreamClient(stream pb.Website_SubscribeServer) *StreamClient { return &StreamClient{ stream: stream, timer: time.NewTimer(KeepaliveTimeout), errCh: make(chan error, 1), rspCh: make(chan *pb.Response, 1), quit: make(chan struct{}), } } func (sc *StreamClient) pingLoop() { pingTicker := time.NewTicker(KeepaliveTime) defer pingTicker.Stop() for { if Subscriber != nil && sc != Subscriber { // new subscriber in replace of the old one. logger.Debug("New subscriber in replace of the old one") return } select { case <-sc.stream.Context().Done(): return case <-pingTicker.C: err := sc.stream.Send(&ping) if err != nil { select { case <-sc.stream.Context().Done(): case sc.errCh <- errors.Wrapf(err, "Send ping err"): } return } } } } func (sc *StreamClient) recvLoop() { for { if Subscriber != nil && sc != Subscriber { // new subscriber in replace of the old one. close(sc.quit) logger.Debug("New subscriber in replace of the old one") return } rsp, err := sc.stream.Recv() if err != nil { if err != io.EOF { select { case <-sc.stream.Context().Done(): case sc.errCh <- errors.Wrapf(err, "Recv pong err"): } } return } logger.Debugf("Got message Type %s", rsp.GetType()) if rsp.Type != Pong { // receive response logger.Infof("Recv updating website rsp: err(%t), msg(%s)", rsp.GetErr(), rsp.GetMsg()) sc.rspCh <- rsp continue } sc.timer.Reset(KeepaliveTimeout) } } // WebsiteServer is the gRPC server implementation type WebsiteServer struct { *pb.UnimplementedWebsiteServer } func GetWebsiteServer() *WebsiteServer { return &WebsiteServer{} } func publishFullWebsite() error { var websites []model.Website db := database.GetDB().DB db.Model(&model.Website{}).Find(&websites) byteWebsites, err := json.Marshal(&websites) if err != nil { return err } return Publish(byteWebsites, EventTypeFullWebsite) } // Subscribe is gRPC API entrypoint func (ws *WebsiteServer) Subscribe(stream pb.Website_SubscribeServer) error { Subscriber = newStreamClient(stream) defer Subscriber.timer.Stop() go Subscriber.pingLoop() go Subscriber.recvLoop() if err := publishFullWebsite(); err != nil { // triggered when tcd starts, ignore push error messages logger.Warn(err) } select { case <-Subscriber.quit: logger.Infof("Disconnected gracefully") return nil case <-Subscriber.timer.C: logger.Error("Keepalive timeout") case err := <-Subscriber.errCh: logger.WithError(err).Error() return err case <-Subscriber.stream.Context().Done(): logger.Infof("Subscribe context done: %s", Subscriber.stream.Context().Err()) Subscriber = nil } return nil } ================================================ FILE: management/webserver/tools/init_db.sh ================================================ docker rm -f mgt-postgres-dev #docker run -it -d -e POSTGRES_DB=safeline-ce -e POSTGRES_USER=safeline-ce -e POSTGRES_PASSWORD=safeline-ce -p 127.0.0.1:5432:5432 --name mgt-postgres-dev chaitin.cn/library/postgres:15.2 docker run -it -d -e POSTGRES_DB=safeline-ce -e POSTGRES_USER=safeline-ce -e POSTGRES_PASSWORD=safeline-ce -p 127.0.0.1:5432:5432 --name mgt-postgres-dev postgres:15.2 ================================================ FILE: management/webserver/utils/cert.go ================================================ package utils import ( "bytes" "crypto/rand" "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "encoding/pem" "io/ioutil" "math/big" "time" ) func GenerateCert(hostnames []string, days int64, keyBits int, subject *pkix.Name, isCA bool) ([]byte, []byte, error) { privateKey, err := rsa.GenerateKey(rand.Reader, keyBits) if err != nil { return nil, nil, err } notBefore := time.Now() duration := int64(time.Hour) * 24 * days notAfter := notBefore.Add(time.Duration(duration)) serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) if err != nil { return nil, nil, err } var keyUsage x509.KeyUsage if isCA { keyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign } else { keyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment } template := x509.Certificate{ SerialNumber: serialNumber, NotBefore: notBefore, NotAfter: notAfter, DNSNames: hostnames, BasicConstraintsValid: true, IsCA: isCA, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, KeyUsage: keyUsage, } if subject != nil { template.Subject = *subject } derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) if err != nil { return nil, nil, err } var certBuffer bytes.Buffer err = pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) if err != nil { return nil, nil, err } var keyBuffer bytes.Buffer err = pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}) if err != nil { return nil, nil, err } return certBuffer.Bytes(), keyBuffer.Bytes(), nil } func WriteCertIfNotExist(certFilePath, keyFilePath string, generator func() ([]byte, []byte, error)) error { cert, key, err := genCertIfNotExist(certFilePath, keyFilePath, generator) if err != nil { return err } if err = EnsureRenameWriteFile(certFilePath, cert, 0644); err != nil { return err } if err = EnsureRenameWriteFile(keyFilePath, key, 0644); err != nil { return err } return nil } func genCertIfNotExist(certFilePath, keyFilePath string, generator func() ([]byte, []byte, error)) ([]byte, []byte, error) { exist, err := FilesExist(certFilePath, keyFilePath) if err != nil { return nil, nil, err } if !exist { return generator() } else { cert, err := ioutil.ReadFile(certFilePath) if err != nil { return nil, nil, err } key, err := ioutil.ReadFile(keyFilePath) if err != nil { return nil, nil, err } return cert, key, nil } } ================================================ FILE: management/webserver/utils/file.go ================================================ package utils import ( "fmt" "io/ioutil" "os" "path/filepath" ) func EnsureDir(dir string) error { if _, err := os.Stat(dir); os.IsNotExist(err) { return os.MkdirAll(dir, os.FileMode(0755)) } return nil } func EnsureFileDir(path string) error { return EnsureDir(filepath.Dir(path)) } func FileExist(path string) (bool, error) { stat, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return false, nil } else { return false, err } } else { if stat.IsDir() { return false, fmt.Errorf("%s is dir", path) } else { return true, nil } } } func FilesExist(paths ...string) (bool, error) { for _, path := range paths { exist, err := FileExist(path) if err != nil { return false, err } if !exist { return false, nil } } return true, nil } func RenameWriteFile(filename string, data []byte, perm os.FileMode) error { randFileName := filename + ".tmp." + RandStr(8) if err := ioutil.WriteFile(randFileName, data, perm); err != nil { return err } return os.Rename(randFileName, filename) } func EnsureRenameWriteFile(path string, data []byte, mode os.FileMode) error { err := EnsureFileDir(path) if err != nil { return err } return RenameWriteFile(path, data, mode) } func EnsureWriteFile(path string, data []byte, mode os.FileMode) error { err := EnsureFileDir(path) if err != nil { return err } return ioutil.WriteFile(path, data, mode) } ================================================ FILE: management/webserver/utils/healthy.go ================================================ package utils import ( "fmt" "net/http" ) func CheckHealthy(url string) bool { method := "GET" client := &http.Client{} req, err := http.NewRequest(method, url, nil) if err != nil { fmt.Println(err) return false } res, err := client.Do(req) if err != nil { fmt.Println(err) return false } defer res.Body.Close() return res.StatusCode == 200 } func CheckWafHealthy() bool { // todo: not check the healthy status return CheckHealthy("http://safeline-mario:3335") && CheckHealthy("http://safeline-detector:8001/stat") } ================================================ FILE: management/webserver/utils/httpclient.go ================================================ package utils import ( "crypto/tls" "net/http" "net/url" "os" "time" ) const proxyName = "HTTPS_PROXY" var httpClient *http.Client func GetHTTPClient() *http.Client { if httpClient == nil { tr := &http.Transport{ MaxIdleConns: 10, IdleConnTimeout: 30 * time.Second, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, } proxyUrl, existed := os.LookupEnv(proxyName) if existed { uri, _ := url.Parse(proxyUrl) tr.Proxy = http.ProxyURL(uri) } httpClient = &http.Client{Transport: tr} } return httpClient } ================================================ FILE: management/webserver/utils/random.go ================================================ package utils import ( "math/rand" "time" ) var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") func RandStr(n int) string { rand.Seed(time.Now().UnixNano()) b := make([]rune, n) for i := range b { b[i] = letters[rand.Intn(len(letters))] } return string(b) } ================================================ FILE: management/webserver/utils/url.go ================================================ package utils import ( "fmt" "net" "strings" "chaitin.cn/patronus/safeline-2/management/webserver/pkg/constants" ) func IsIPv6(str string) bool { ip := net.ParseIP(str) return ip != nil && strings.Contains(str, ":") } func BuildUrl(protocol int, host string, port uint, path string) string { portStr := "" if len(host) > 0 && IsIPv6(host) && host[:1] != "[" { host = fmt.Sprintf("[%s]", host) } if protocol == constants.ProtocolHTTP { if port != 80 { portStr = fmt.Sprintf(":%d", port) } } else if protocol == constants.ProtocolHTTPS || protocol == constants.ProtocolHTTP2 { if port != 443 { portStr = fmt.Sprintf(":%d", port) } } else { // use HTTP as default protocol protocol = constants.ProtocolHTTP } return fmt.Sprintf("%s://%s%s%s", constants.HTTPProtocol[protocol], host, portStr, path) } ================================================ FILE: mcp_server/Dockerfile ================================================ FROM golang:1.24-alpine AS builder WORKDIR /app RUN apk add --no-cache git COPY go.mod go.sum ./ RUN go mod download COPY . . RUN CGO_ENABLED=0 go build -o mcp-server . FROM alpine:latest WORKDIR /app RUN apk add --no-cache ca-certificates tzdata COPY --from=builder /app/mcp-server . COPY --from=builder /app/config.yaml . ENV TZ=Asia/Shanghai EXPOSE 5678 CMD ["./mcp-server"] ================================================ FILE: mcp_server/README.md ================================================ # SafeLine MCP Server SafeLine MCP Server is an implementation of the [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) that provides complete management and control capabilities for SafeLine WAF. [![Docker](https://img.shields.io/badge/Docker-Supported-2496ED?style=flat-square&logo=docker&logoColor=white)](docker-compose.yml) [![Go Version](https://img.shields.io/badge/Go-1.21+-00ADD8?style=flat-square&logo=go&logoColor=white)](go.mod) ## Use Cases - Automated management and control of SafeLine WAF instances - WAF configuration and policy management through API - Building AI-based security protection tools and applications ## Prerequisites 1. Install [Docker](https://www.docker.com/) (if running in container) 2. Configure SafeLine API Token (obtained from SafeLine console) ## Features - Complete MCP (Management Control Protocol) server implementation - Support for SafeLine WAF instance management and control - Flexible configuration system supporting file configuration and environment variables - Docker containerization support - Secure API communication ## Quick Start ### Environment Variables | Environment Variable | Description | Default Value | Required | |---------|------|--------|-----| | LISTEN_PORT | Service listening port | 5678 | No | | LISTEN_ADDRESS | Service listening address | 0.0.0.0 | No | | SAFELINE_SECRET | SSE server secret | - | No | | SAFELINE_ADDRESS | SafeLine API address | - | Yes | | SAFELINE_API_TOKEN | SafeLine API authentication token | - | Yes | ### Using Docker #### Method 1: Using docker run ```bash docker run -d \ --name safeline-mcp \ -p 5678:5678 \ -e SAFELINE_API_TOKEN="your_api_token" \ -e SAFELINE_ADDRESS="https://your.safeline.com" \ -e LISTEN_PORT=5678 \ -e LISTEN_ADDRESS="0.0.0.0" \ chaitin/safeline-mcp:latest ``` #### Method 2: Using docker-compose ```bash # 1. Clone repository git clone https://github.com/chaitin/safeline-mcp.git cd safeline-mcp # 2. Edit docker-compose.yml to configure environment variables # Example docker-compose.yml: # version: '3' # services: # mcp: # image: chaitin/safeline-mcp:latest # container_name: safeline-mcp # ports: # - "5678:5678" # environment: # - SAFELINE_API_TOKEN=your_api_token # - SAFELINE_ADDRESS=https://your.safeline.com # - LISTEN_PORT=5678 # - LISTEN_ADDRESS=0.0.0.0 # 3. Start service docker compose -f docker-compose.yml up -d ``` #### Method 3: Using Go ```bash # 1. Clone repository git clone https://github.com/chaitin/SafeLine.git cd safeline-mcp # 2. Install dependencies go mod download # 3. Configure config.yaml cp config.yaml.example config.yaml # Edit config.yaml with necessary configurations # 4. Run service go run main.go ``` For more API details, please refer to the [API Documentation](https://demo.waf.chaitin.com:9443/swagger/index.html). ## Tools ### Application Management - **create_application** ### Rule Management - **create_blacklist_rule** - **create_whitelist_rule** ### Analyze - **get_attack_events** ## Development Guide The Go API in this project is currently under development, and APIs may change. If you have specific requirements, please submit an Issue for discussion. ### Directory Structure ``` internal/ ├── api/ # API implementation │ ├── app/ # Application-related APIs │ │ └── create_application.go │ └── rule/ # Rule-related APIs │ └── create_rule.go └── tools/ # MCP tool implementation ├── app/ # Application-related tools │ └── create_application.go └── rule/ # Rule-related tools └── create_rule.go ``` ### Adding New Tools 1. **Create Tool File** - Create corresponding directory and file under `internal/tools` - File name should match tool name - Use separate file for each tool - Example: `internal/tools/app/create_application.go` 2. **Tool Implementation Template** ```go package app type ToolName struct{} type ToolParams struct { // Parameter definitions Param1 string `json:"param1" desc:"parameter description" required:"true"` Param2 int `json:"param2" desc:"parameter description" required:"false"` } type ToolResult struct { Field1 string `json:"field1"` } func (t *ToolName) Name() string { return "tool_name" } func (t *ToolName) Description() string { return "tool description" } func (t *ToolName) Validate(params ToolParams) error { // Parameter validation logic return nil } func (t *ToolName) Execute(ctx context.Context, params ToolParams) (result ToolResult, err error) { // Tool execution logic return result, nil } ``` 3. **[Optional]Create API Implementation** If you need to use some APIs that have not been implemented yet, you need to create corresponding files in the api directory for implementation - Create same directory structure under `internal/api` - File name should match tool func - Example: `internal/api/app/create_application.go` **API Implementation Template** ```go package app type RequestType struct { // Request parameter definitions Param1 string `json:"param1"` Param2 int `json:"param2"` } func APIName(ctx context.Context, req *RequestType) (ResultType, error) { if req == nil { return nil, errors.New("request is required") } var resp api.Response[ResultType] err := api.Service().Post(ctx, "/api/path", req, &resp) if err != nil { return nil, errors.Wrap(err, "failed to execute") } if resp.Err != nil { return nil, errors.New(resp.Msg) } return resp.Data, nil } ``` 4. **Tool Registration (init.go)** The tool registration file `internal/tools/init.go` is used to centrally manage all tool registrations - Register all tools uniformly in the `init()` function - Use the `AppendTool()` method for registration - Example: ```go // Register create application tool AppendTool(&app.CreateApp{}) // Register create blacklist rule tool AppendTool(&rule.CreateBlacklistRule{}) ``` ### Development Standards 1. **Naming Conventions** - Use lowercase letters and underscores for tool names - File names should match tool names 2. **Directory Organization** - Divide directories by functional modules (e.g., app, rule, etc.) - Maintain consistent structure between tools and api directories - Keep related functionality in the same directory 3. **Code Standards** - Follow Go standard code conventions - Add necessary parameter validation - Use unified error handling approach - Add appropriate logging 4. **Documentation Requirements** - Provide clear functional description in tool Description - Add detailed description for parameters - Update API toolkit documentation in README ### Example Refer to the implementation of the `create_application` tool: - Tool implementation: `internal/tools/app/create_application.go` - API implementation: `internal/api/app/create_application.go` ================================================ FILE: mcp_server/config.yaml ================================================ # Server Configuration server: name: "SafeLine MCP Server" version: "1.0.0" # Can be overridden by environment variable LISTEN_PORT port: 5678 # Can be overridden by environment variable LISTEN_ADDRESS host: "0.0.0.0" # Can be overridden by environment variable SAFELINE_SECRET secret: "" # Secret for SSE server # Logger Configuration logger: level: "info" # Log level: debug, info, warn, error file_path: "" # Log file path console: true # Whether to output to console caller: false # Whether to record caller information development: true # Whether to use development mode # API Configuration api: # Can be overridden by environment variable SAFELINE_ADDRESS base_url: "" # API service address # Can be overridden by environment variable SAFELINE_API_TOKEN token: "" # Authentication token timeout: 30 # Timeout in seconds debug: false # Whether to enable debug mode insecure_skip_verify: true # Whether to skip certificate verification ================================================ FILE: mcp_server/docker-compose.yml ================================================ version: '3.8' services: mcp_server: image: chaitin/safeline-mcp:latest container_name: mcp_server restart: always ports: - "5678:5678" environment: - SAFELINE_SECRET=your_secret_key # optional, if you want to use secret key to authenticate - SAFELINE_ADDRESS=https://your_safeline_ip:9443 # required, your SafeLine WAF address - SAFELINE_API_TOKEN=your_safeline_api_token # required, your SafeLine WAF api token - LISTEN_PORT=5678 # optional, default is 5678 - LISTEN_ADDRESS=0.0.0.0 # optional, default is 0.0.0.0 ================================================ FILE: mcp_server/go.mod ================================================ module github.com/chaitin/SafeLine/mcp_server go 1.24.1 require ( github.com/mark3labs/mcp-go v0.18.0 go.uber.org/zap v1.27.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/google/uuid v1.6.0 // indirect github.com/mcuadros/go-defaults v1.2.0 github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.uber.org/multierr v1.11.0 // indirect ) ================================================ FILE: mcp_server/go.sum ================================================ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mark3labs/mcp-go v0.18.0 h1:YuhgIVjNlTG2ZOwmrkORWyPTp0dz1opPEqvsPtySXao= github.com/mark3labs/mcp-go v0.18.0/go.mod h1:KmJndYv7GIgcPVwEKJjNcbhVQ+hJGJhrCCB/9xITzpE= github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc= github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: mcp_server/internal/api/analyze/get_event_list.go ================================================ package analyze import ( "context" "fmt" "github.com/chaitin/SafeLine/mcp_server/internal/api" ) type GetEventListRequest struct { Page int `json:"page"` PageSize int `json:"page_size"` IP string `json:"ip"` Start int64 `json:"start"` End int64 `json:"end"` } type GetEventListResponse struct { Nodes []Event `json:"nodes"` Total int64 `json:"total"` } type Event struct { ID uint `json:"id"` IP string `json:"ip"` Protocol int `json:"protocol"` Host string `json:"host"` DstPort uint64 `json:"dst_port"` UpdatedAt int64 `json:"updated_at"` StartAt int64 `json:"start_at"` EndAt int64 `json:"end_at"` DenyCount int64 `json:"deny_count"` PassCount int64 `json:"pass_count"` Finished bool `json:"finished"` Country string `json:"country"` Province string `json:"province"` City string `json:"city"` } func GetEventList(ctx context.Context, req *GetEventListRequest) (*GetEventListResponse, error) { var resp api.Response[GetEventListResponse] err := api.Service().Get(ctx, fmt.Sprintf("/api/open/events?page=%d&page_size=%d&ip=%s&start=%d&end=%d", req.Page, req.PageSize, req.IP, req.Start, req.End), &resp) if err != nil { return nil, err } return &resp.Data, nil } ================================================ FILE: mcp_server/internal/api/app/create_application.go ================================================ package app import ( "context" "github.com/chaitin/SafeLine/mcp_server/internal/api" "github.com/chaitin/SafeLine/mcp_server/pkg/errors" ) type CreateAppRequest struct { ServerNames []string `json:"server_names"` Ports []string `json:"ports"` Upstreams []string `json:"upstreams"` Comment string `json:"comment"` } // CreateApp Create new website or app func CreateApp(ctx context.Context, req *CreateAppRequest) (int64, error) { if req == nil { return 0, errors.New("request is required") } var resp api.Response[int64] err := api.Service().Post(ctx, "/api/open/site", req, &resp) if err != nil { return 0, errors.Wrap(err, "failed to create app") } if resp.Err != nil { return 0, errors.New(resp.Msg) } return resp.Data, nil } ================================================ FILE: mcp_server/internal/api/client.go ================================================ package api import ( "bytes" "context" "crypto/tls" "encoding/json" "fmt" "io" "net/http" "time" "github.com/chaitin/SafeLine/mcp_server/pkg/errors" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" ) // Client API client type Client struct { baseURL string httpClient *http.Client headers map[string]string } // ClientOption Client configuration options type ClientOption func(*Client) // WithTimeout Set timeout duration func WithTimeout(timeout time.Duration) ClientOption { return func(c *Client) { c.httpClient.Timeout = timeout } } // WithHeader Set request header func WithHeader(key, value string) ClientOption { return func(c *Client) { c.headers[key] = value } } // WithBaseURL Set base URL func WithBaseURL(baseURL string) ClientOption { return func(c *Client) { c.baseURL = baseURL } } // WithInsecureSkipVerify Set whether to skip certificate verification func WithInsecureSkipVerify(skip bool) ClientOption { return func(c *Client) { if transport, ok := c.httpClient.Transport.(*http.Transport); ok { if transport.TLSClientConfig == nil { transport.TLSClientConfig = &tls.Config{} } transport.TLSClientConfig.InsecureSkipVerify = skip } } } // NewClient Create new API client func NewClient(opts ...ClientOption) *Client { transport := &http.Transport{ TLSClientConfig: &tls.Config{}, } c := &Client{ httpClient: &http.Client{ Timeout: 30 * time.Second, Transport: transport, }, headers: make(map[string]string), } for _, opt := range opts { opt(c) } return c } // Request Send request func (c *Client) Request(ctx context.Context, method, path string, body interface{}, result interface{}) error { reqURL := fmt.Sprintf("%s%s", c.baseURL, path) var bodyReader io.Reader if body != nil { bodyBytes, err := json.Marshal(body) if err != nil { return errors.Wrap(err, "marshal request body failed") } bodyReader = bytes.NewReader(bodyBytes) } logger.With("url", reqURL).Debug("request url") req, err := http.NewRequestWithContext(ctx, method, reqURL, bodyReader) if err != nil { return errors.Wrap(err, "create request failed") } // Set common headers req.Header.Set("Content-Type", "application/json") for k, v := range c.headers { req.Header.Set(k, v) } resp, err := c.httpClient.Do(req) if err != nil { return errors.Wrap(err, "send request failed") } defer resp.Body.Close() // Read response body respBody, err := io.ReadAll(resp.Body) if err != nil { return errors.Wrap(err, "read response body failed") } // Check status code if resp.StatusCode < 200 || resp.StatusCode >= 300 { return errors.New(fmt.Sprintf("request failed with status %d: %s", resp.StatusCode, string(respBody))) } // Parse response if result != nil { if err := json.Unmarshal(respBody, result); err == nil { return nil } var respData map[string]interface{} if err := json.Unmarshal(respBody, &respData); err != nil { return errors.Wrap(err, "unmarshal response failed") } if respData["err"] != nil || respData["msg"] != nil { return errors.New(respData["msg"].(string)) } } return nil } // Get Send GET request func (c *Client) Get(ctx context.Context, path string, result interface{}) error { return c.Request(ctx, http.MethodGet, path, nil, result) } // Post Send POST request func (c *Client) Post(ctx context.Context, path string, body interface{}, result interface{}) error { return c.Request(ctx, http.MethodPost, path, body, result) } // Put Send PUT request func (c *Client) Put(ctx context.Context, path string, body interface{}, result interface{}) error { return c.Request(ctx, http.MethodPut, path, body, result) } // Delete Send DELETE request func (c *Client) Delete(ctx context.Context, path string, result interface{}) error { return c.Request(ctx, http.MethodDelete, path, nil, result) } ================================================ FILE: mcp_server/internal/api/response.go ================================================ package api // Response Common API response structure type Response[T any] struct { // Response data Data T `json:"data"` // Error message Err any `json:"err"` // Prompt message Msg string `json:"msg"` } ================================================ FILE: mcp_server/internal/api/rule/create_rule.go ================================================ package rule import ( "context" "github.com/chaitin/SafeLine/mcp_server/internal/api" "github.com/chaitin/SafeLine/mcp_server/pkg/errors" ) type CreateRuleRequest struct { Name string `json:"name"` IP []string `json:"ip"` IsEnabled bool `json:"is_enabled"` Pattern [][]api.Pattern `json:"pattern"` Action int `json:"action"` } // CreateRule Create new rule func CreateRule(ctx context.Context, req *CreateRuleRequest) (int64, error) { if req == nil { return 0, errors.New("request is required") } var resp api.Response[int64] err := api.Service().Post(ctx, "/api/open/policy", req, &resp) if err != nil { return 0, errors.Wrap(err, "failed to create policy rule") } if resp.Err != nil { return 0, errors.New(resp.Msg) } return resp.Data, nil } ================================================ FILE: mcp_server/internal/api/service.go ================================================ package api import ( "context" "sync" "time" "github.com/chaitin/SafeLine/mcp_server/internal/config" "github.com/chaitin/SafeLine/mcp_server/pkg/errors" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" ) // APIClient API client implementation type APIClient struct { client *Client config *config.APIConfig } var ( instance *APIClient once sync.Once ) // Init Initialize API service func Init(cfg *config.APIConfig) error { var err error once.Do(func() { instance, err = newAPIClient(cfg) if err != nil { logger.With("error", err).Error("failed to initialize API service") return } logger.Info("API service initialized successfully") }) return err } // Service Get API service instance func Service() *APIClient { if instance == nil { logger.Error("API service not initialized") panic("API service not initialized") } return instance } // newAPIClient Create new API client func newAPIClient(config *config.APIConfig) (*APIClient, error) { if config == nil { return nil, errors.New("config is required") } if config.BaseURL == "" { return nil, errors.New("base_url is required") } timeout := 30 if config.Timeout > 0 { timeout = config.Timeout } opts := []ClientOption{ WithBaseURL(config.BaseURL), WithTimeout(time.Duration(timeout) * time.Second), WithHeader("User-Agent", "SafeLine-MCP/1.0"), WithInsecureSkipVerify(config.InsecureSkipVerify), } // If token is configured, add authentication header if config.Token != "" { opts = append(opts, WithHeader("X-SLCE-API-TOKEN", config.Token)) } client := NewClient(opts...) return &APIClient{ client: client, config: config, }, nil } // Post Send POST request func (c *APIClient) Post(ctx context.Context, path string, body interface{}, result interface{}) error { return c.client.Request(ctx, "POST", path, body, result) } // Get Send GET request func (c *APIClient) Get(ctx context.Context, path string, result interface{}) error { return c.client.Request(ctx, "GET", path, nil, result) } // Put Send PUT request func (c *APIClient) Put(ctx context.Context, path string, body interface{}, result interface{}) error { return c.client.Request(ctx, "PUT", path, body, result) } // Delete Send DELETE request func (c *APIClient) Delete(ctx context.Context, path string, result interface{}) error { return c.client.Request(ctx, "DELETE", path, nil, result) } ================================================ FILE: mcp_server/internal/api/types.go ================================================ package api type PolicyRuleAction int const ( PolicyRuleActionAllow PolicyRuleAction = iota PolicyRuleActionDeny PolicyRuleActionMax ) type Key = string const ( KeySrcIP Key = "src_ip" KeyURI Key = "uri" KeyURINoQuery Key = "uri_no_query" KeyHost Key = "host" KeyMethod Key = "method" KeyReqHeader Key = "req_header" KeyReqBody Key = "req_body" KeyGetParam Key = "get_param" KeyPostParam Key = "post_param" ) type Op = string const ( OpEq Op = "eq" // equal OpNotEq Op = "not_eq" // not equal OpMatch Op = "match" // match OpCIDR Op = "cidr" // cidr OpHas Op = "has" // has OpNotHas Op = "not_has" // not has OpPrefix Op = "prefix" // prefix OpRe Op = "re" // regex OpIn Op = "in" // in OpNotIn Op = "not_in" // not in OpNotCIDR Op = "not_cidr" // not cidr OpExist Op = "exist" // exist OpNotExist Op = "not_exist" // not exist OpGeoEq Op = "geo_eq" // geo equal OpGeoNotEq Op = "geo_not_eq" // geo not equal ) type Pattern struct { K Key `json:"k"` Op Op `json:"op"` V []string `json:"v"` SubK string `json:"sub_k"` } ================================================ FILE: mcp_server/internal/config/config.go ================================================ package config import ( "os" "strconv" "github.com/chaitin/SafeLine/mcp_server/pkg/errors" "gopkg.in/yaml.v3" ) // Config Global configuration structure type Config struct { Server *ServerConfig `yaml:"server"` Logger *LoggerConfig `yaml:"logger"` API *APIConfig `yaml:"api"` } // APIConfig API configuration type APIConfig struct { // API base URL BaseURL string `yaml:"base_url"` // API token Token string `yaml:"token"` // API timeout Timeout int `yaml:"timeout"` // API debug mode Debug bool `yaml:"debug"` // API insecure skip verify InsecureSkipVerify bool `yaml:"insecure_skip_verify"` } // ServerConfig Server configuration type ServerConfig struct { Name string `yaml:"name"` Version string `yaml:"version"` Port int `yaml:"port"` Host string `yaml:"host"` Secret string `yaml:"secret"` } // LoggerConfig Logger configuration type LoggerConfig struct { Level string `yaml:"level"` FilePath string `yaml:"file_path"` Console bool `yaml:"console"` Caller bool `yaml:"caller"` Development bool `yaml:"development"` } var config *Config // getEnvString Get string value from environment variable, return default value if not exists func getEnvString(key, defaultValue string) string { if value, exists := os.LookupEnv(key); exists { return value } return defaultValue } // getEnvInt Get integer value from environment variable, return default value if not exists or cannot be parsed func getEnvInt(key string, defaultValue int) int { if value, exists := os.LookupEnv(key); exists { if intValue, err := strconv.Atoi(value); err == nil { return intValue } } return defaultValue } // Load Load configuration file func Load(path string) error { data, err := os.ReadFile(path) if err != nil { return errors.Wrap(err, "read config file failed") } config = &Config{} if err := yaml.Unmarshal(data, config); err != nil { return errors.Wrap(err, "unmarshal config failed") } // Override configuration from environment variables if config.Server != nil { config.Server.Host = getEnvString("LISTEN_ADDRESS", config.Server.Host) config.Server.Port = getEnvInt("LISTEN_PORT", config.Server.Port) } if config.API != nil { config.API.BaseURL = getEnvString("SAFELINE_ADDRESS", config.API.BaseURL) config.API.Token = getEnvString("SAFELINE_API_TOKEN", config.API.Token) } return nil } // GetServer Get server configuration func GetServer() *ServerConfig { if config == nil { return nil } return config.Server } // GetLogger Get logger configuration func GetLogger() *LoggerConfig { if config == nil { return nil } return config.Logger } // GetAPI Get API configuration func GetAPI() *APIConfig { if config == nil { return nil } return config.API } ================================================ FILE: mcp_server/internal/tools/analyze/get_atttack_events.go ================================================ package analyze import ( "context" "github.com/chaitin/SafeLine/mcp_server/internal/api/analyze" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" ) type GetAttackEventsParams struct { IP string `json:"ip" desc:"ip" required:"false"` Page int `json:"page" desc:"page" required:"false" default:"1"` PageSize int `json:"page_size" desc:"page size" required:"false" default:"10"` Start int64 `json:"start" desc:"start unix timestamp in milliseconds" required:"false"` End int64 `json:"end" desc:"end unix timestamp in milliseconds" required:"false"` } type GetAttackEvents struct{} func (t *GetAttackEvents) Name() string { return "get_attack_events" } func (t *GetAttackEvents) Description() string { return "get attack events" } func (t *GetAttackEvents) Validate(params GetAttackEventsParams) error { return nil } func (t *GetAttackEvents) Execute(ctx context.Context, params GetAttackEventsParams) (analyze.GetEventListResponse, error) { resp, err := analyze.GetEventList(ctx, &analyze.GetEventListRequest{ IP: params.IP, PageSize: params.PageSize, Page: params.Page, Start: params.Start, End: params.End, }) if err != nil { return analyze.GetEventListResponse{}, err } logger.With("total", resp.Total).Info("get attack events") return *resp, nil } ================================================ FILE: mcp_server/internal/tools/app/create_application.go ================================================ package app import ( "context" "github.com/chaitin/SafeLine/mcp_server/internal/api/app" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" ) type CreateApp struct{} type CreateAppParams struct { ServerNames []string `json:"server_names" desc:"domain list" required:"true"` Ports []string `json:"ports" desc:"port list" required:"true"` Upstreams []string `json:"upstreams" desc:"upstream list" required:"true"` } func (t *CreateApp) Name() string { return "create_http_application" } func (t *CreateApp) Description() string { return "create a new website or app" } func (t *CreateApp) Validate(params CreateAppParams) error { return nil } func (t *CreateApp) Execute(ctx context.Context, params CreateAppParams) (int64, error) { id, err := app.CreateApp(ctx, &app.CreateAppRequest{ ServerNames: params.ServerNames, Ports: params.Ports, Upstreams: params.Upstreams, }) if err != nil { return 0, err } logger.Info("create app success", logger.Int64("id", id)) return id, nil } ================================================ FILE: mcp_server/internal/tools/example.go ================================================ package tools import ( "context" "github.com/chaitin/SafeLine/mcp_server/pkg/errors" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" ) type CalculateSum struct{} func (t *CalculateSum) Name() string { return "calculate_sum" } func (t *CalculateSum) Description() string { return "Add two numbers together" } type MyToolInput struct { A int `json:"a" desc:"number a" required:"true"` B int `json:"b" desc:"number b" required:"true"` } type MyToolOutput struct { C int `json:"c"` } func (t *CalculateSum) Validate(params MyToolInput) error { return nil } func (t *CalculateSum) Execute(ctx context.Context, params MyToolInput) (MyToolOutput, error) { logger.With("a", params.A). With("b", params.B). Debug("Executing calculation") result := MyToolOutput{ C: params.A + params.B, } logger.With("result", result.C). Debug("Calculation completed") return result, errors.New("test error") } ================================================ FILE: mcp_server/internal/tools/init.go ================================================ package tools import ( "github.com/chaitin/SafeLine/mcp_server/internal/tools/analyze" "github.com/chaitin/SafeLine/mcp_server/internal/tools/app" "github.com/chaitin/SafeLine/mcp_server/internal/tools/rule" ) func init() { // app AppendTool(&app.CreateApp{}) // rule AppendTool(&rule.CreateBlacklistRule{}) AppendTool(&rule.CreateWhitelistRule{}) // analyze AppendTool(&analyze.GetAttackEvents{}) } ================================================ FILE: mcp_server/internal/tools/rule/create_blacklist_rule.go ================================================ package rule import ( "context" "github.com/chaitin/SafeLine/mcp_server/internal/api" "github.com/chaitin/SafeLine/mcp_server/internal/api/rule" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" ) type CreateBlacklistRule struct{} type CreateBlacklistRuleParams struct { Name string `json:"name" desc:"name" required:"true"` IP []string `json:"ip" desc:"ip" required:"false"` URINoQuery []string `json:"uri_no_query" desc:"uri_no_query" required:"false"` } func (t *CreateBlacklistRule) Name() string { return "create_blacklist_rule" } func (t *CreateBlacklistRule) Description() string { return "create a new blacklist rule" } func (t *CreateBlacklistRule) Validate(params CreateBlacklistRuleParams) error { return nil } func (t *CreateBlacklistRule) Execute(ctx context.Context, params CreateBlacklistRuleParams) (int64, error) { var pattern [][]api.Pattern if len(params.IP) > 0 { pattern = append(pattern, []api.Pattern{ { K: api.KeySrcIP, Op: api.OpEq, V: params.IP, SubK: "", }, }) } if len(params.URINoQuery) > 0 { pattern = append(pattern, []api.Pattern{ { K: api.KeyURINoQuery, Op: api.OpEq, V: params.URINoQuery, SubK: "", }, }) } id, err := rule.CreateRule(ctx, &rule.CreateRuleRequest{ Name: params.Name, IP: params.IP, IsEnabled: true, Action: int(api.PolicyRuleActionDeny), Pattern: pattern, }) if err != nil { return 0, err } logger.With("id", id).Info("create blacklist rule success") return id, nil } ================================================ FILE: mcp_server/internal/tools/rule/create_whitelist_rule.go ================================================ package rule import ( "context" "github.com/chaitin/SafeLine/mcp_server/internal/api" "github.com/chaitin/SafeLine/mcp_server/internal/api/rule" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" ) type CreateWhitelistRule struct{} type CreateWhitelistRuleParams struct { Name string `json:"name" desc:"name" required:"true"` IP []string `json:"ip" desc:"ip" required:"false"` URINoQuery []string `json:"uri_no_query" desc:"uri_no_query" required:"false"` } func (t *CreateWhitelistRule) Name() string { return "create_whitelist_rule" } func (t *CreateWhitelistRule) Description() string { return "create a new whitelist rule" } func (t *CreateWhitelistRule) Validate(params CreateWhitelistRuleParams) error { return nil } func (t *CreateWhitelistRule) Execute(ctx context.Context, params CreateWhitelistRuleParams) (int64, error) { var pattern [][]api.Pattern if len(params.IP) > 0 { pattern = append(pattern, []api.Pattern{ { K: api.KeySrcIP, Op: api.OpEq, V: params.IP, SubK: "", }, }) } if len(params.URINoQuery) > 0 { pattern = append(pattern, []api.Pattern{ { K: api.KeyURINoQuery, Op: api.OpEq, V: params.URINoQuery, SubK: "", }, }) } id, err := rule.CreateRule(ctx, &rule.CreateRuleRequest{ Name: params.Name, IP: params.IP, IsEnabled: true, Action: int(api.PolicyRuleActionAllow), Pattern: pattern, }) if err != nil { return 0, err } logger.With("id", id).Info("create whitelist rule success") return id, nil } ================================================ FILE: mcp_server/internal/tools/tool.go ================================================ package tools import ( "github.com/chaitin/SafeLine/mcp_server/pkg/logger" "github.com/chaitin/SafeLine/mcp_server/pkg/mcp" ) // By deferring the concretization of generic types to the Register method, // we avoid type inference issues. // Each Tool is wrapped in a toolWrapper that knows its concrete type, // allowing correct passing of generic parameters during registration. type ToolWrapper interface { Register(s *mcp.MCPServer) error } var ( tools = []ToolWrapper{} ) func AppendTool[T any, R any](tool ...mcp.Tool[T, R]) { for _, t := range tool { tools = append(tools, &toolWrapper[T, R]{tool: t}) } } func Tools() []ToolWrapper { return tools } type toolWrapper[T any, R any] struct { tool mcp.Tool[T, R] } func (w *toolWrapper[T, R]) Register(s *mcp.MCPServer) error { logger.Info("Registering tool", logger.String("name", w.tool.Name()), logger.String("description", w.tool.Description()), ) return mcp.RegisterTool(s, w.tool) } ================================================ FILE: mcp_server/main.go ================================================ package main import ( "flag" "fmt" "github.com/chaitin/SafeLine/mcp_server/internal/api" "github.com/chaitin/SafeLine/mcp_server/internal/config" "github.com/chaitin/SafeLine/mcp_server/internal/tools" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" "github.com/chaitin/SafeLine/mcp_server/pkg/mcp" ) func main() { configPath := flag.String("config", "config.yaml", "path to config file") flag.Parse() if err := config.Load(*configPath); err != nil { panic(fmt.Errorf("failed to load config: %v", err)) } logConfig := config.GetLogger() if err := logger.Init(&logger.Config{ Level: logConfig.Level, FilePath: logConfig.FilePath, Console: logConfig.Console, Caller: logConfig.Caller, Development: logConfig.Development, }); err != nil { panic(fmt.Errorf("failed to init logger: %v", err)) } logger.With("base_url", config.GetAPI().BaseURL).Info("Initializing API service...") if err := api.Init(config.GetAPI()); err != nil { panic(fmt.Errorf("failed to init API service: %v", err)) } logger.Info("Starting MCP Server...") serverConfig := config.GetServer() s := mcp.NewMCPServer( serverConfig.Name, serverConfig.Version, serverConfig.Secret, ) logger.Info("Registering tools...") for _, tool := range tools.Tools() { if err := tool.Register(s); err != nil { logger.With("error", err). Error("Failed to register tool") panic(err) } } addr := fmt.Sprintf("%s:%d", serverConfig.Host, serverConfig.Port) logger.With("addr", addr).Info("Starting server") if err := s.Start(addr); err != nil { logger.With("error", err). Error("Server failed to start") panic(err) } } ================================================ FILE: mcp_server/pkg/config/config.go ================================================ package config import ( "os" "gopkg.in/yaml.v3" ) // Config Global configuration structure type Config struct { Server ServerConfig `yaml:"server"` Logger LoggerConfig `yaml:"logger"` } // ServerConfig Server configuration type ServerConfig struct { Name string `yaml:"name"` Version string `yaml:"version"` Port int `yaml:"port"` Host string `yaml:"host"` Secret string `yaml:"secret"` } // LoggerConfig Logger configuration type LoggerConfig struct { Level string `yaml:"level"` FilePath string `yaml:"file_path"` Console bool `yaml:"console"` Caller bool `yaml:"caller"` Development bool `yaml:"development"` } var ( globalConfig *Config ) // Load Load configuration from file func Load(filename string) error { data, err := os.ReadFile(filename) if err != nil { return err } config := &Config{} if err := yaml.Unmarshal(data, config); err != nil { return err } globalConfig = config return nil } // Get Get global configuration func Get() *Config { return globalConfig } // GetServer Get server configuration func GetServer() ServerConfig { if globalConfig == nil { return ServerConfig{} } return globalConfig.Server } // GetLogger Get logger configuration func GetLogger() LoggerConfig { if globalConfig == nil { return LoggerConfig{} } return globalConfig.Logger } ================================================ FILE: mcp_server/pkg/errors/errors.go ================================================ package errors import ( "errors" "fmt" "runtime" "strings" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" ) var ( // Common errors ErrInternal = New("internal error") ErrInvalidParam = New("invalid parameter") ErrNotFound = New("resource not found") ErrUnauthorized = New("unauthorized") ErrForbidden = New("forbidden") ErrTimeout = New("timeout") ) // Error Custom error structure type Error struct { err error stack []string msg string location string } // Error Implement error interface func (e *Error) Error() string { if e.msg != "" { return fmt.Sprintf("%s: %v (at %s)", e.msg, e.err, e.location) } return fmt.Sprintf("%v (at %s)", e.err, e.location) } // Unwrap Return original error func (e *Error) Unwrap() error { if e.err == nil { return nil } if wrapped, ok := e.err.(*Error); ok { return wrapped.Unwrap() } return e.err } // Stack Return error stack func (e *Error) Stack() []string { return e.stack } // Location Return error location func (e *Error) Location() string { return e.location } // getCallerLocation Get caller location func getCallerLocation(skip int) string { _, file, line, ok := runtime.Caller(skip) if !ok { return "unknown" } return fmt.Sprintf("%s:%d", file, line) } // WrapL Wrap error and print log func WrapL(err error, msg string) error { if err == nil { return nil } // Get stack trace information var stack []string for i := 1; i < 32; i++ { pc, file, line, ok := runtime.Caller(i) if !ok { break } fn := runtime.FuncForPC(pc) if fn == nil { break } name := fn.Name() if strings.Contains(name, "runtime.") { break } stack = append(stack, fmt.Sprintf("%s:%d", file, line)) } wrappedErr := &Error{ err: err, stack: stack, msg: msg, location: getCallerLocation(2), } // Print error information and stack using logger logger.With("error", err). With("location", wrappedErr.location). With("stack", strings.Join(stack, "\n")). Error(msg) return wrappedErr } // Is Check error type func Is(err, target error) bool { return errors.Is(err, target) } // As Type assertion func As(err error, target interface{}) bool { return errors.As(err, target) } // Wrap Wrap error without printing log func Wrap(err error, msg string) error { if err == nil { return nil } return &Error{ err: err, msg: msg, location: getCallerLocation(2), } } // New Create new error func New(text string) error { return &Error{ err: errors.New(text), location: getCallerLocation(2), } } ================================================ FILE: mcp_server/pkg/logger/field.go ================================================ package logger import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" ) // Field 日志字段 type Field = zapcore.Field // String 创建字符串字段 func String(key string, val string) Field { return zap.String(key, val) } // Int 创建整数字段 func Int(key string, val int) Field { return zap.Int(key, val) } // Int64 创建 int64 字段 func Int64(key string, val int64) Field { return zap.Int64(key, val) } // Float64 创建浮点数字段 func Float64(key string, val float64) Field { return zap.Float64(key, val) } // Bool 创建布尔字段 func Bool(key string, val bool) Field { return zap.Bool(key, val) } // Err 创建错误字段 func Err(err error) Field { return zap.Error(err) } // Any 创建任意类型字段 func Any(key string, val interface{}) Field { return zap.Any(key, val) } // Duration 创建时间段字段 func Duration(key string, val float64) Field { return zap.Float64(key, val) } ================================================ FILE: mcp_server/pkg/logger/logger.go ================================================ package logger import ( "os" "path/filepath" "sync" "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) // Logger 封装 zap.Logger type Logger struct { zl *zap.Logger } var ( defaultLogger *Logger once sync.Once ) // Config 日志配置 type Config struct { // 日志级别 Level string `json:"level" yaml:"level"` // 日志文件路径 FilePath string `json:"file_path" yaml:"file_path"` // 是否输出到控制台 Console bool `json:"console" yaml:"console"` // 是否记录调用者信息 Caller bool `json:"caller" yaml:"caller"` // 是否使用开发模式（更详细的日志） Development bool `json:"development" yaml:"development"` } // 默认配置 var defaultConfig = Config{ Level: "info", FilePath: "logs/mcp.log", Console: true, Caller: true, Development: false, } // Init 初始化日志系统 func Init(cfg *Config) error { var err error once.Do(func() { if cfg == nil { cfg = &defaultConfig } // 确保日志目录存在 if cfg.FilePath != "" { dir := filepath.Dir(cfg.FilePath) if err = os.MkdirAll(dir, 0755); err != nil { return } } // 配置编码器 encoderConfig := zapcore.EncoderConfig{ TimeKey: "time", LevelKey: "level", NameKey: "logger", CallerKey: "caller", MessageKey: "msg", StacktraceKey: "stacktrace", LineEnding: zapcore.DefaultLineEnding, EncodeLevel: zapcore.LowercaseLevelEncoder, EncodeTime: func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { enc.AppendString(t.Format("2006-01-02 15:04:05.000")) }, EncodeDuration: zapcore.SecondsDurationEncoder, EncodeCaller: zapcore.ShortCallerEncoder, } // 设置日志级别 var level zapcore.Level if err = level.UnmarshalText([]byte(cfg.Level)); err != nil { level = zapcore.InfoLevel } // 创建Core var cores []zapcore.Core // 文件输出 if cfg.FilePath != "" { fileWriter, err := os.OpenFile(cfg.FilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { return } cores = append(cores, zapcore.NewCore( zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(fileWriter), level, )) } // 控制台输出 if cfg.Console { cores = append(cores, zapcore.NewCore( zapcore.NewConsoleEncoder(encoderConfig), zapcore.AddSync(os.Stdout), level, )) } // 创建Logger core := zapcore.NewTee(cores...) zl := zap.New(core) // 是否记录调用者信息 if cfg.Caller { zl = zl.WithOptions(zap.AddCaller()) } // 是否使用开发模式 if cfg.Development { zl = zl.WithOptions(zap.Development()) } defaultLogger = &Logger{zl: zl} }) return err } // GetLogger 获取日志实例 func GetLogger() *Logger { if defaultLogger == nil { Init(nil) } return defaultLogger } // With 创建带有字段的新Logger func (l *Logger) With(key string, value interface{}) *Logger { return &Logger{zl: l.zl.With(Any(key, value))} } // Debug level func (l *Logger) Debug(msg string, fields ...Field) { l.zl.Debug(msg, fields...) } // Info level func (l *Logger) Info(msg string, fields ...Field) { l.zl.Info(msg, fields...) } // Warn level func (l *Logger) Warn(msg string, fields ...Field) { l.zl.Warn(msg, fields...) } // Error level func (l *Logger) Error(msg string, fields ...Field) { l.zl.Error(msg, fields...) } // Fatal level func (l *Logger) Fatal(msg string, fields ...Field) { l.zl.Fatal(msg, fields...) } // 全局函数 // With 创建带有字段的新Logger func With(key string, value interface{}) *Logger { l := GetLogger() // 为链式调用创建新的logger实例，并添加caller skip return &Logger{zl: l.zl.WithOptions(zap.AddCallerSkip(1)).With(Any(key, value))} } // Debug level func Debug(msg string, fields ...Field) { l := GetLogger() l.zl.WithOptions(zap.AddCallerSkip(1)).Debug(msg, fields...) } // Info level func Info(msg string, fields ...Field) { l := GetLogger() l.zl.WithOptions(zap.AddCallerSkip(1)).Info(msg, fields...) } // Warn level func Warn(msg string, fields ...Field) { l := GetLogger() l.zl.WithOptions(zap.AddCallerSkip(1)).Warn(msg, fields...) } // Error level func Error(msg string, fields ...Field) { l := GetLogger() l.zl.WithOptions(zap.AddCallerSkip(1)).Error(msg, fields...) } // Fatal level func Fatal(msg string, fields ...Field) { l := GetLogger() l.zl.WithOptions(zap.AddCallerSkip(1)).Fatal(msg, fields...) } ================================================ FILE: mcp_server/pkg/mcp/mcp.go ================================================ package mcp import ( "context" "encoding/json" "fmt" "net/http" "strconv" "github.com/chaitin/SafeLine/mcp_server/pkg/errors" "github.com/chaitin/SafeLine/mcp_server/pkg/logger" "github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/server" "github.com/mcuadros/go-defaults" ) type Tool[T any, R any] interface { Name() string Description() string Execute(ctx context.Context, params T) (R, error) Validate(params T) error } type SSEServer struct { sse *server.SSEServer secret string } func (s *SSEServer) Start(addr string) error { srv := &http.Server{ Addr: addr, Handler: s, } return srv.ListenAndServe() } func (s *SSEServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { if s.secret == "" { s.sse.ServeHTTP(w, r) return } messagePath := s.sse.CompleteMessagePath() if messagePath != "" && r.URL.Path == messagePath { secret := r.Header.Get("Secret") if secret != s.secret { http.Error(w, "Unauthorized", http.StatusUnauthorized) return } } s.sse.ServeHTTP(w, r) } type MCPServer struct { server *server.MCPServer sse *SSEServer } func NewMCPServer(name, version string, secret string) *MCPServer { s := server.NewMCPServer( name, version, server.WithLogging(), ) return &MCPServer{ server: s, sse: &SSEServer{sse: server.NewSSEServer(s), secret: secret}, } } func (s *MCPServer) Start(addr string) error { return s.sse.Start(addr) } func handleToolCall[T any, R any](ctx context.Context, request mcp.CallToolRequest, tool Tool[T, R]) (result *mcp.CallToolResult, err error) { defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic recovered: %v", r) } }() var raw []byte raw, err = json.Marshal(request.Params.Arguments) if err != nil { return nil, errors.Wrap(err, "marshal arguments failed") } var params T defaults.SetDefaults(¶ms) if err = json.Unmarshal(raw, ¶ms); err != nil { return nil, errors.Wrap(err, "unmarshal parameters failed") } if err = tool.Validate(params); err != nil { return nil, err } var execResult R execResult, err = tool.Execute(ctx, params) if err != nil { return nil, err } v := any(execResult) switch v := v.(type) { case string: return mcp.NewToolResultText(v), nil case []byte: return mcp.NewToolResultText(string(v)), nil case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: return mcp.NewToolResultText(json.Number(fmt.Sprint(v)).String()), nil case bool: return mcp.NewToolResultText(strconv.FormatBool(v)), nil default: bytes, err := json.Marshal(v) if err != nil { return nil, errors.New("invalid result type") } return mcp.NewToolResultText(string(bytes)), nil } } func RegisterTool[T any, R any](s *MCPServer, tool Tool[T, R]) error { var v T opts, err := SchemaToOptions(v) if err != nil { return err } opts = append(opts, mcp.WithDescription(tool.Description())) t := mcp.NewTool(tool.Name(), opts..., ) s.server.AddTool(t, func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { result, err := handleToolCall(ctx, request, tool) if err != nil { logger.With("error", err).Error("handle tool call failed") if wrapped, ok := err.(*errors.Error); ok { return nil, wrapped.Unwrap() } return nil, err } return result, nil }) return nil } ================================================ FILE: mcp_server/pkg/mcp/schema.go ================================================ package mcp import ( "encoding/json" "reflect" "strconv" "strings" "github.com/mark3labs/mcp-go/mcp" ) // SchemaToOptions Convert struct to MCP ToolOption list func SchemaToOptions(schema any) ([]mcp.ToolOption, error) { t := reflect.TypeOf(schema) if t.Kind() == reflect.Ptr { t = t.Elem() } var options []mcp.ToolOption for i := 0; i < t.NumField(); i++ { field := t.Field(i) jsonTag := field.Tag.Get("json") if jsonTag == "" { continue } desc := field.Tag.Get("desc") required := field.Tag.Get("required") == "true" enumTag := field.Tag.Get("enum") defaultTag := field.Tag.Get("default") minTag := field.Tag.Get("min") maxTag := field.Tag.Get("max") opts := []mcp.PropertyOption{} if desc != "" { opts = append(opts, mcp.Description(desc)) } if required { opts = append(opts, mcp.Required()) } if enumTag != "" && field.Type.Kind() == reflect.String { enumValues := strings.Split(enumTag, ",") opts = append(opts, mcp.Enum(enumValues...)) } switch field.Type.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: if defaultTag != "" { if defaultValue, err := strconv.Atoi(defaultTag); err == nil { opts = append(opts, mcp.DefaultNumber(float64(defaultValue))) } } if minTag != "" { if minValue, err := strconv.Atoi(minTag); err == nil { opts = append(opts, mcp.Min(float64(minValue))) } } if maxTag != "" { if maxValue, err := strconv.Atoi(maxTag); err == nil { opts = append(opts, mcp.Max(float64(maxValue))) } } options = append(options, mcp.WithNumber(jsonTag, opts...)) case reflect.Bool: if defaultTag != "" { if defaultValue, err := strconv.ParseBool(defaultTag); err == nil { opts = append(opts, mcp.DefaultBool(defaultValue)) } } options = append(options, mcp.WithBoolean(jsonTag, opts...)) case reflect.String: if defaultTag != "" { opts = append(opts, mcp.DefaultString(defaultTag)) } options = append(options, mcp.WithString(jsonTag, opts...)) case reflect.Struct: subSchema := reflect.New(field.Type).Interface() subOptions, err := SchemaToOptions(subSchema) if err != nil { return nil, err } // Create a temporary Tool to get JSON Schema of sub-struct tempTool := mcp.NewTool("temp", subOptions...) tempJSON, _ := tempTool.MarshalJSON() var tempMap map[string]any if err := json.Unmarshal(tempJSON, &tempMap); err != nil { continue } // Extract properties from temporary Tool if inputSchema, ok := tempMap["inputSchema"].(map[string]any); ok { if properties, ok := inputSchema["properties"].(map[string]any); ok { // Check if there are required fields if required, ok := inputSchema["required"].([]any); ok { // Add required field information to corresponding properties for _, req := range required { if reqStr, ok := req.(string); ok { if prop, ok := properties[reqStr].(map[string]any); ok { prop["required"] = true } } } } opts = append(opts, mcp.Properties(properties)) } } options = append(options, mcp.WithObject(jsonTag, opts...)) case reflect.Slice: elemType := field.Type.Elem() var items map[string]any switch elemType.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64: items = map[string]any{ "type": "number", } case reflect.Bool: items = map[string]any{ "type": "boolean", } case reflect.String: items = map[string]any{ "type": "string", } case reflect.Struct: subSchema := reflect.New(elemType).Interface() subOptions, err := SchemaToOptions(subSchema) if err != nil { return nil, err } // Create a temporary Tool to get JSON Schema of sub-struct tempTool := mcp.NewTool("temp", subOptions...) tempJSON, _ := tempTool.MarshalJSON() var tempMap map[string]any if err := json.Unmarshal(tempJSON, &tempMap); err != nil { continue } // Extract properties from temporary Tool if inputSchema, ok := tempMap["inputSchema"].(map[string]any); ok { if properties, ok := inputSchema["properties"].(map[string]any); ok { // Check if there are required fields if required, ok := inputSchema["required"].([]any); ok { // Add required field information to corresponding properties for _, req := range required { if reqStr, ok := req.(string); ok { if prop, ok := properties[reqStr].(map[string]any); ok { prop["required"] = true } } } } items = map[string]any{ "type": "object", "properties": properties, } } } } opts = append(opts, mcp.Items(items)) options = append(options, mcp.WithArray(jsonTag, opts...)) } } return options, nil } ================================================ FILE: mcp_server/pkg/mcp/schema_test.go ================================================ package mcp import ( "reflect" "testing" "github.com/mark3labs/mcp-go/mcp" ) func TestSchemaToOptions(t *testing.T) { tests := []struct { name string args any want mcp.Tool }{ { name: "test number", args: struct { A int `json:"a" desc:"number a" required:"true"` }{}, want: mcp.NewTool("test number", mcp.WithNumber("a", mcp.Required(), mcp.Description("number a")), ), }, { name: "test number int64", args: struct { A int64 `json:"a" desc:"number a" required:"true"` }{}, want: mcp.NewTool("test number int64", mcp.WithNumber("a", mcp.Required(), mcp.Description("number a")), ), }, { name: "test number float64", args: struct { A float64 `json:"a" desc:"number a" required:"true"` }{}, want: mcp.NewTool("test number float64", mcp.WithNumber("a", mcp.Required(), mcp.Description("number a")), ), }, { name: "test number default", args: struct { A int `json:"a" desc:"number a" required:"true" default:"10"` }{}, want: mcp.NewTool("test number default", mcp.WithNumber("a", mcp.Required(), mcp.Description("number a"), mcp.DefaultNumber(10)), ), }, { name: "test number min max", args: struct { A int `json:"a" desc:"number a" required:"true" min:"10" max:"20"` }{}, want: mcp.NewTool("test number min max", mcp.WithNumber("a", mcp.Required(), mcp.Description("number a"), mcp.Min(10), mcp.Max(20)), ), }, { name: "test number optional", args: struct { A int `json:"a" desc:"number a"` }{}, want: mcp.NewTool("test number optional", mcp.WithNumber("a", mcp.Description("number a")), ), }, { name: "test boolean", args: struct { A bool `json:"a" desc:"boolean a" required:"true"` }{}, want: mcp.NewTool("test boolean", mcp.WithBoolean("a", mcp.Required(), mcp.Description("boolean a")), ), }, { name: "test string", args: struct { A string `json:"a" desc:"string a" required:"true"` }{}, want: mcp.NewTool("test string", mcp.WithString("a", mcp.Required(), mcp.Description("string a")), ), }, { name: "test string default", args: struct { A string `json:"a" desc:"string a" required:"true" default:"hello"` }{}, want: mcp.NewTool("test string default", mcp.WithString("a", mcp.Required(), mcp.Description("string a"), mcp.DefaultString("hello")), ), }, { name: "test string enum", args: struct { A string `json:"a" desc:"string a" required:"true" enum:"1,2,3"` }{}, want: mcp.NewTool("test string enum", mcp.WithString("a", mcp.Required(), mcp.Description("string a"), mcp.Enum("1", "2", "3")), ), }, { name: "test object", args: struct { A struct { B int `json:"b" desc:"number b" required:"true"` } `json:"a" desc:"object a" required:"true"` }{}, want: mcp.NewTool("test object", mcp.WithObject("a", mcp.Required(), mcp.Description("object a"), mcp.Properties(map[string]any{ "b": map[string]any{ "type": "number", "description": "number b", "required": true, }, }), ), ), }, { name: "test object optional", args: struct { A struct { B int `json:"b" desc:"number b" required:"true"` C int `json:"c" desc:"number c"` } `json:"a" desc:"object a" required:"true"` }{}, want: mcp.NewTool("test object optional", mcp.WithObject("a", mcp.Required(), mcp.Description("object a"), mcp.Properties(map[string]any{ "b": map[string]any{ "type": "number", "description": "number b", "required": true, }, "c": map[string]any{ "type": "number", "description": "number c", }, }), ), ), }, { name: "test nested object", args: struct { A struct { B struct { C int `json:"c" desc:"number c" required:"true"` } `json:"b" desc:"object b" required:"true"` } `json:"a" desc:"object a" required:"true"` }{}, want: mcp.NewTool("test nested object", mcp.WithObject("a", mcp.Required(), mcp.Description("object a"), mcp.Properties(map[string]any{ "b": map[string]any{ "type": "object", "description": "object b", "required": true, "properties": map[string]any{ "c": map[string]any{ "type": "number", "description": "number c", "required": true, }, }, }, }), ), ), }, { name: "test array", args: struct { A []int `json:"a" desc:"array a" required:"true"` }{}, want: mcp.NewTool("test array", mcp.WithArray("a", mcp.Required(), mcp.Description("array a"), mcp.Items(map[string]any{ "type": "number", }), ), ), }, { name: "test array of object", args: struct { A []struct { B int `json:"b" desc:"number b" required:"true"` } `json:"a" desc:"array of object a" required:"true"` }{}, want: mcp.NewTool("test array of object", mcp.WithArray("a", mcp.Required(), mcp.Description("array of object a"), mcp.Items(map[string]any{ "type": "object", "properties": map[string]any{ "b": map[string]any{ "type": "number", "description": "number b", "required": true, }, }, }), ), ), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := SchemaToOptions(tt.args) if err != nil { t.Errorf("SchemaToOptions() error = %v", err) return } s1, _ := mcp.NewTool(tt.name, got...).MarshalJSON() s2, _ := tt.want.MarshalJSON() if !reflect.DeepEqual(s1, s2) { t.Errorf("\n got %v\n want %v", string(s1), string(s2)) } }) } } ================================================ FILE: scripts/manage.py ================================================ #!/usr/bin/env python3 import argparse import base64 import json import shutil import ssl import sys import datetime import platform import os from urllib.request import urlopen, HTTPError import re import subprocess import string import time import random import socket texts = { 'hello1': { 'en': 'SafeLine is a self-hosted WAF(Web Application Firewall) to protect your web apps from attacks and exploits.', 'zh': 'SafeLine，中文名 "雷池"，是一款简单好用, 效果突出的 Web 应用防火墙(WAF)，可以保护 Web 服务不受黑客攻击。' }, 'hello2': { 'en': 'A web application firewall helps protect web apps by filtering and monitoring HTTP traffic between a web application and the Internet. It typically protects web apps from attacks such as SQL injection, XSS, code injection, os command injection, CRLF injection, ldap injection, xpath injection, RCE, XXE, SSRF, path traversal, backdoor, bruteforce, http-flood, bot abused, among others.', 'zh': '雷池通过过滤和监控 Web 应用与互 联网之间的 HTTP 流量来保护 Web 服务。可以保护 Web 服务免受 SQL 注入、XSS 、 代码注入、命令注入、CRLF 注入、ldap 注入、xpath 注入、RCE、XXE、SSRF、路径遍历、后门、暴力破解、CC、爬虫 等攻击。' }, 'talking-group': { 'en': '\n' 'https://discord.gg/SVnZGzHFvn\n' '\n' 'Join discord group for more informations of SafeLine by above address', 'zh': '\n' '▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄\n' '█ ▄▄▄▄▄ █▀ █▀▀██▀▄▀▀▄▀▄▀▄██ ▄▄▄▄▄ █\n' '█ █ █ █▀ ▄ █▀▄▄▀▀ ▄█▄ ▀█ █ █ █\n' '█ █▄▄▄█ █▀█ █▄█▄▀▀▄▀▄ ▀▀▄▄█ █▄▄▄█ █\n' '█▄▄▄▄▄▄▄█▄█▄█ █▄▀ █ ▀▄▀ █▄█▄▄▄▄▄▄▄█\n' '█▄ ▄▄ █▄▄ ▄█▄▄▄▄▀▄▀▀▄██ ▄▄▀▄█▄▀ ▀█\n' '█▄ ▄▀▄ ▄▀▄ ▀ ▄█▀ ▀▄ █▀▀ ▀█▀▄██▄▀▄██\n' '██ ▀▄█ ▄ ▄▄▀▄▀▀█▄▀▄▄▀▄▀▄ ▄ ▀▄▄▄█▀▀█\n' '█ █▀▄▀ ▄▀▄▄▀█▀ ▄▄ █▄█▀▀▄▀▀█▄█▄█▀▄██\n' '█ █ ▀ ▄▀▀ ██▄█▄▄▄▄▄▀▄▀▀▀▄▄▀█▄▀█ ▀█\n' '█ █ ▀▄ ▄██▀▀ ▄█▀ ▀███▄ ▀▄▀▄▄ ▄▀▄██\n' '█▀▄▄█ ▄▀▄▀ ▄▀▀▀▄▀▄▀ ▄▀▄ ▄▀ ▄▀█ ▀█\n' '█ █ █ █▄▀ █▄█▀ ▄▄███▀▀▀▄█▀▄ ▀ ▀▄██\n' '█▄███▄█▄▄▀▄ █▄█▄▄▄▄▀▀▄█▀▀ ▄▄▄ ▀█ █\n' '█ ▄▄▄▄▄ █▄▀█ ▄█▀▄ █▀█▄ ▀ █▄█ ▀▄▀█\n' '█ █ █ █ █▄▀▀▀▄▄▄▀▀▀▀▀▀ ▄▄ ▀█ █\n' '█ █▄▄▄█ █ ▀█▀ ▄▄▄▄ ▀█ ▀▀▄▀ ▀▀ ▀███\n' '█▄▄▄▄▄▄▄█▄▄██▄█▄▄█▄██▄██▄▄█▄▄█▄█▄██\n' '\n' '微信扫描上方二维码加入雷池项目讨论组' }, 'input-target-path': { 'en': 'Input the path to install %s', 'zh': '请输入%s 的安装目录' }, 'input-mgt-port': { 'en': 'Input the %s mgt port', 'zh': '请输入%s 的管理端口' }, 'python-version-too-low': { 'en': 'The Python version is too low, Python 3.5 above is required', 'zh': 'Python 版本过低, 脚本无法运行, 需要 Python 3.5 以上' }, 'not-a-tty': { 'en': 'Stdin is not a standard TTY', 'zh': '运行脚本的方式不对, STDIN 不是标准的 TTY' }, 'not-root': { 'en': 'Requires root privileges to run', 'zh': '需要 root 权限才能运行' }, 'not-linux': { 'en': '%s does not support %s OS yet', 'zh': '%s 暂时不支持 %s 操作系统' }, 'unsupported-arch': { 'en': '%s does not support %s processor yet', 'zh': '%s 暂时不支持 %s 处理器' }, 'prepare-to-install': { 'en': 'Will be going to installing %s for you.', 'zh': '即将为您安装%s' }, 'choice-action': { 'en': 'Choice what do you want to do', 'zh': '选择你要执行的动作' }, 'default-value': { 'en': 'Keep blank default to', 'zh': '留空则为默认值' }, 'ssse3-not-support': { 'en': 'SSSE3 instruction set not enabled in your CPU', 'zh': '当前 CPU 未启用 SSSE3 指令集' }, 'precheck-failed': { 'en': 'The environment does not meet the installation conditions of %s', 'zh': '当前环境不符合%s 的安装条件' }, 'precheck-passed': { 'en': 'Installation environment check passed', 'zh': '检查安装环境已完成' }, 'insufficient-memory': { 'en': 'Remaining memory is less than 1 GB', 'zh': '剩余内存不足 1 GB' }, 'docker-not-installed': { 'en': 'Running %s requires Docker, but Docker is not installed', 'zh': '运行%s 依赖 Docker, 但是 Docker 没安装' }, 'docker-compose-not-installed': { 'en': 'Running %s requires Docker Compose, but Docker Compose is not installed', 'zh': '运行%s 依赖 Docker Compose, 但是 Docker Compose 没安装' }, 'docker-version-too-low': { 'en': 'Docker version is too low, it does not match %s', 'zh': 'Docker 版本太低, 不满足%s 的安装需求' }, 'if-install-docker': { 'en': 'Do you want the latest version of Docker to be automatically installed for you?', 'zh': '是否需要为你自动安装 Docker 的最新版本' }, 'if-restart-docker': { 'en': 'Do you want to restart %s Docker container', 'zh': '是否需要重启%s 的容器' }, 'if-update-docker': { 'en': 'Do you want to update your Docker version?', 'zh': '是否需要为你自动更新 Docker 版本' }, 'install-docker-failed': { 'en': 'Failed to install Docker. Please try to install Docker manually before installing %s', 'zh': '安装 Docker 失败, 请尝试手动安装 Docker 后再来安装%s' }, 'install-docker': { 'en': 'Docker is being installed for you. It will take a few minutes. Please wait patiently.', 'zh': '正在为你安装 Docker, 需要几分钟时间, 请耐心等待' }, 'get-space-failed': { 'en': 'Unable to query disk capacity of "%s"', 'zh': '无法查询 "%s" 的磁盘容量' }, 'remain-disk-capacity': { 'en': 'Disk capacity of "%s" has %s avaiable', 'zh': '"%s" 路径有 %s 的空间可用' }, 'insufficient-disk-capacity': { 'en': 'Insufficient disk capacity of "%s", at least 5 GB is required to install %s', 'zh': '"%s" 的磁盘容量不足，安装%s 至少需要 5 GB' }, 'pg-pass-contains-invalid-char': { 'en': 'The POSTGRES_PASSWORD variable contains special characters. Please choose repair to reset password', 'zh': 'POSTGRES_PASSWORD 变量包含特殊字符, 请选择修复重置密码' }, 'invalid-path': { 'en': '"%s" is not a valid absolute path', 'zh': '"%s" 不是合法的绝对路径' }, 'path-exists': { 'en': '"%s" already exists, please select a new directory', 'zh': '路径 "%s" 已存在，请选择一个全新的目录' }, 'fail-to-parse-route': { 'en': 'Unable to parse /proc/net/route file', 'zh': '无法解析 /proc/net/route 文件' }, 'fail-to-download-compose': { 'en': 'Failed to download docker compose script', 'zh': '下载 docker compose 脚本失败' }, 'fail-to-create-dir': { 'en': 'Unable to create the "%s" directory', 'zh': '无法创建 "%s" 目录' }, 'docker-pull': { 'en': 'Pulling Docker image', 'zh': '正在拉取 Docker 镜像' }, 'try-another-image-source': { 'en': 'Try another image source', 'zh': '尝试使用其他镜像源' }, 'image-clean': { 'en': 'Cleaning Docker image', "zh": '正在清理 Docker 镜像' }, 'update-config': { 'en': 'Updating .env configuration files', 'zh': '正在更新 .env 配置文件' }, 'download-compose': { 'en': 'Downloading the docker-compose.yaml file', 'zh': '正在下载 docker-compose.yaml 文件' }, 'fail-to-pull-image': { 'en': 'Failed to pull Docker image', 'zh': '拉取 Docker 镜像失败' }, 'docker-up': { 'en': 'Starting Docker containers', 'zh': '正在启动 Docker 容器' }, 'fail-to-up': { 'en': 'Failed to start Docker containers', 'zh': '启动 Docker 容器失败' }, 'fail-to-down': { 'en': 'Failed to stop Docker containers', 'zh': '停止 Docker 容器失败' }, 'install-finish': { 'en': '%s installation completed', 'zh': '%s 安装完成' }, 'upgrade-finish': { 'en': '%s upgrade completed', 'zh': '%s 升级完成' }, 'go-to-panel': { 'en': '%s management panel: https://%s:%s/', 'zh': '%s 管理面板: https://%s:%s/' } ,'install': { 'en': 'INSTALL', 'zh': '安装' }, 'repair': { 'en': 'REPAIR', 'zh': '修复' }, 'uninstall': { 'en': 'UNINSTALL', 'zh': '卸载' }, 'upgrade': { 'en': 'UPGRADE', 'zh': '升级' }, 'backup': { 'en': 'BACKUP', 'zh': '备份' }, 'yes': { 'en': 'Yes', 'zh': '是' }, 'no': { 'en': 'No', 'zh': '否' }, 'fail-to-get-installed-dir': { 'en': 'Failed to get installed dir', 'zh': '未能找到安装目录', }, 'fail-to-connect-image-source': { 'en': 'Failed to connect image source', 'zh': '无法连接到任何镜像源' }, 'fail-to-connect-docker-source': { 'en': 'Failed to connect docker source', 'zh': '无法连接到任何 docker 源' }, 'fail-to-download-docker-installation': { 'en': 'Failed to download docker installation', "zh": '下载 docker 安装脚本失败' }, 'docker-source': { 'en': 'Docker source', "zh": 'docker 源' }, 'reset-admin': { 'en': 'Setup admin', "zh": '设置 admin' }, 'docker-version': { 'en': 'Checking docker version', 'zh': '检查 docker 版本' }, 'docker-compose-version': { 'en': 'Checking docker compose version', 'zh': '检查 docker compose 版本' }, 'keyboard-interrupt': { 'en': 'Installation cancelled', "zh": '取消安装' }, 'docker-up-iptables-failed': { 'en': 'Iptables policy error, try to restart docker', 'zh': 'iptables 规则错误，尝试重启 docker' }, 'install-channel': { 'en': 'Installing: %s', 'zh': '安装通道：%s' }, 'preview-release': { 'en': 'Preview', 'zh': '预览版' }, 'lts-release': { 'en': 'LTS', 'zh': 'LTS 版' }, 'fail-to-docker-down': { 'en': 'Failed to stop container', 'zh': '停止 docker 容器失败' }, 'fail-to-remove-dir': { 'en': 'Failed to remove %s installation directory', 'zh': '删除 %s 安装目录失败' }, 'uninstall-finish': { 'en': '%s uninstall completed', 'zh': '%s 卸载完成' }, 'docker-down': { 'en': 'Stopping %s container', 'zh': '正在停止%s 容器' }, 'reset-tengine': { 'en': 'RESET TENGINE CONFIG', 'zh': '重置 tengine 配置', }, 'reset-postgres': { 'en': 'RESET DATABASE PASSWORD', 'zh': '重置数据库密码' }, 'fail-to-find-nginx': { 'en': 'Failed to find tengine config path', 'zh': '未找到 tengine 配置目录' }, 'nginx-backup-dir': { 'en': 'Tengine config backup directory', 'zh': 'tengine 配置备份目录' }, 'fail-to-backup-nginx': { 'en': 'Failed to backup tengine config', 'zh': '备份 tengine 目录失败' }, 'docker-restart': { 'en': 'Restart docker container', 'zh': '重启 docker 容器' }, 'docker-exec': { 'en': 'Executing docker command', 'zh': '执行 docker 命令' }, 'fail-to-recover-static': { 'en': 'Failed to recover tengine static config', 'zh': '恢复 tengine 静态站点资源失败' }, 'fail-to-find-env': { 'en': 'Failed to find .env file', 'zh': '未找到 .env 文件' }, 'fail-to-find-postgres-password': { 'en': 'Failed to find postgres password', 'zh': '未找到数据库密码' }, 'fail-to-reset-postgres-password': { 'en': 'Failed to reset postgres password', 'zh': '重置数据库密码失败' }, 'reset-postgres-password-finish': { 'en': 'Reset postgres password completed', 'zh': '重置数据库密码完成' }, 'reset-tengine-finish': { 'en': 'Reset tengine finish completed', 'zh': '重置 tengine 配置完成' }, 'if-remove-waf': { 'en': 'Do you want to uninstall %s, this operation will delete all data in the directory: %s', 'zh': '是否确认卸载%s，该操作会删除目录下所有数据：%s' }, 'restart-docker-finish': { 'en': 'Restart %s docker container completed', 'zh': '重启%s 容器完成' }, 'restart': { 'en': 'RESTART', 'zh': '重启' }, 'wait-mgt-health': { 'en': 'Wait for mgt healthy', 'zh': '等待 mgt 启动' }, 'if-remove-ipv6-scope': { 'en': '/etc/resolv.conf have ipv6 nameserver with scope. Do you want to remove these nameserver', 'zh': '/etc/resolv.conf 文件中存在 ipv6 地址包含区域 ID，是否删除包含区域 ID 的 ipv6 nameserver' }, 'input-target-version': { 'en': 'Input the %s version', 'zh': '请输入 %s 的版本' }, 'version-format-error': { 'en': 'version %s format error', 'zh': '版本 %s 格式错误' }, 'can-not-downgrade': { 'en': '%s can not downgrade %s to %s', 'zh': '%s 不支持从 %s 降级到 %s' }, 'get-version': { 'en': 'Getting %s latest version', 'zh': '正在获取%s 最新版本' }, 'get-version-from-mgt': { 'en': 'try to get version from mgt', 'zh': '尝试从 mgt 获取安装版本' }, 'skip-version-compare': { 'en': 'skip %s version compare', 'zh': '跳过%s 版本匹配' }, 'fail-to-get-version-from-mgt': { 'en': 'failed to get install version from mgt', 'zh': '从 mgt 获取安装版本失败' }, 'target-version': { 'en': 'target version: %s', 'zh': '目标版本：%s' }, 'fail-to-parse-assets': { 'en': 'failed to parse assets', 'zh': '解析补丁包失败' }, 'snap-docker-should-use-home-path': { 'en': 'Docker installed via snap can only be configured to set the installation directory under the user\'s home directory(%s)', 'zh': 'snap 安装的 docker 只能设置安装目录在用户的主目录(%s)下' }, 'fail-to-get-docker-path': { 'en': 'find docker binary path failed', 'zh': '获取 docker 二进制目录失败' } } BOLD = 1 DIM = 1 BLINK = 5 REVERSE = 7 RED = 31 GREEN = 32 YELLOW = 33 BLUE = 34 CYAN = 36 INSTALL = False DOMAIN = 'waf-ce.chaitin.cn' REQUEST_CTX = None LANG = 'zh' PRODUCT = '' DEBUG = False SELF = True def parse_assets(args): global PRODUCT, SELF if args.patch == '': return True assets = '' with open(args.patch, 'r') as f: for line in f.readlines(): assets += line split_assets = assets.strip().split('.') if len(split_assets) != 3: return False try: assets_info = json.loads(base64.b64decode(split_assets[1].replace('_','/').replace('-','+') + '=' * ((4 - len(split_assets[1]) % 4) % 4))) if args.en: PRODUCT = assets_info['fullname_en'] else: PRODUCT = assets_info['fullname'] if PRODUCT == '': return False if assets_info['self'] is None: return False if not assets_info['self']: SELF = False except Exception as e: log.debug('parse assets failed: ' + str(e)) return False return True def init_global_config(): global REQUEST_CTX, LANG, DOMAIN, PRODUCT, DEBUG REQUEST_CTX = ssl.create_default_context() REQUEST_CTX.check_hostname = False REQUEST_CTX.verify_mode = ssl.CERT_NONE parser = argparse.ArgumentParser( prog='installer-management', description='installer-management', allow_abbrev=False ) parser.add_argument('--debug', action='store_true', help='install with debug log') parser.add_argument("--lts", action='store_true', help='install lts version') parser.add_argument('--image-clean', action='store_true', help='clean image when upgrade done') parser.add_argument('--en', action='store_true', help='install international version') parser.add_argument('--patch', default='', type=str, help='patch path') args = parser.parse_args() if args.en: LANG = 'en' DOMAIN = 'waf.chaitin.com' PRODUCT = 'SafeLine WAF' else: PRODUCT = '雷池 WAF' if args.debug: DEBUG = True if args.patch != '' and not os.path.exists(args.patch): log.fatal('assets %s not exists' % args.patch) if not parse_assets(args): log.fatal(text('fail-to-parse-assets')) return args class log(): @staticmethod def _log(c, l, s): t = datetime.datetime.now().strftime('%H:%M:%S') print('\r\033[0;%dm[%-5s %s]: %s\033[0m' % (c, l, t, s)) @staticmethod def debug(s): if DEBUG: log._log(DIM, 'DEBUG', s) @staticmethod def info(s): log._log(CYAN, 'INFO', s) @staticmethod def warning(s): log._log(YELLOW, 'WARN', s) @staticmethod def error(s): log._log(RED, 'ERROR', s) @staticmethod def fatal(s): log._log(RED, 'ERROR', s) sys.exit(1) def text(label, var=()): t = texts.get(label, { 'en': 'Unknown "%s" (%s)' % (label, var), 'zh': '未知变量 "%s" (%s)' % (label, var) }) return t[LANG if LANG in t else 'en'] % var def color(t, attrs=[], end=True): t = '\x1B[%sm%s' % (';'.join([str(i) for i in attrs]), t) if end: t = t + '\x1B[m' return t GLOBAL_ARGS = init_global_config() def banner(): t = r''' ______ ___ _____ _ ____ ____ _ ________ .' ____ \ .' ..] |_ _| (_) |_ _| |_ _| / \ |_ __ | | (___ \_| ,--. _| |_ .---. | | __ _ .--. .---. \ \ /\ / / / _ \ | |_ \_| _.____`. `'_\ : '-| |-' / /__\\ | | _ [ | [ `.-. | / /__\\ \ \/ \/ / / ___ \ | _| | \____) | // | |, | | | \__., _| |__/ | | | | | | | | \__., \ /\ / _/ / \ \_ _| |_ \______.' \'-;__/ [___] '.__.' |________| [___] [___||__] '.__.' \/ \/ |____| |____| |_____| '''.strip('\n') print(color(t + '\n', [GREEN, BLINK])) def get_url(url): try: response = urlopen(url, timeout=10, context=REQUEST_CTX) content = response.read() return content.decode('utf-8') except Exception as e: log.error('get url %s failed: %s' % (url, str(e))) def ui_read(question, default): while True: if default is None: sys.stdout.write('%s: ' % ( color(question, [GREEN]), )) else: sys.stdout.write('%s %s: ' % ( color(question, [GREEN]), color('(%s %s)' % (text('default-value'), default), [YELLOW]), )) r = input().strip() if len(r) == 0: if default is None or len(default) == 0: continue r = default return r def ui_choice(question, options): while True: s_options = '[ %s ]' % ' '.join(['%s.%s' % option for option in options]) s_choices = '(%s)' % '/'.join([option[0] for option in options]) sys.stdout.write('%s %s %s: ' % (color(question, [GREEN]), color(s_options, [YELLOW]), color(s_choices, [YELLOW]))) r = input().strip() if r in [i[0] for i in options]: return r def humen_size(x): if x >= 1024 * 1024 * 1024 * 1024: return '%.02f TB' % (x / 1024 / 1024 / 1024 / 1024) elif x >= 1024 * 1024 * 1024: return '%.02f GB' % (x / 1024 / 1024 / 1024) elif x >= 1024 * 1024: return '%.02f MB' % (x / 1024 / 1024) elif x >= 1024: return '%.02f KB' % (x / 1024) else: return '%d Bytes' def rand_subnet(): routes = [] try: with open('/proc/net/route', 'r') as f: next(f) for line in f: parts = line.split() if len(parts) < 8: continue destination = int(parts[1], 0x10) if destination == 0: continue mask = int(parts[7], 0x10) routes.append((destination, mask)) except Exception as e: log.error(text('fail-to-parse-route') + ' ' + str(e)) for i in range(256): t = 192 t += 168 << 8 t += i << 16 for route in routes: if t & route[1] == route[0]: break else: return '%d.%d.%d' % (t & 0xFF, (t >> 8) & 0xFF, (t >> 16) & 0xFF) return '172.22.222' def free_space(path): while not os.path.exists(path) and path != '/': path = os.path.dirname(path) try: st = os.statvfs(path) free_bytes = st.f_bavail * st.f_frsize return free_bytes except Exception as e: log.error(text('get-space-failed', path) + ' ' + str(e)) return None def free_memory(): t = filter(lambda x: 'MemAvailable' in x, open('/proc/meminfo', 'r').readlines()) return int(next(t).split()[1]) * 1024 def exec_command(*args,shell=False): try: proc = subprocess.run(args, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,shell=shell) subprocess_output(proc.stdout.strip()) return proc.returncode, proc.stdout, proc.stderr except Exception as e: return -1, '', str(e) def exec_command_with_loading(*args, cwd=None, env=None): try: with subprocess.Popen(args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, env=env, cwd=cwd) as proc: if not DEBUG: loading = ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"] iloading = 0 while proc.poll() is None: sys.stderr.write('\r' + loading[iloading]) sys.stderr.flush() iloading = (iloading + 1) % len(loading) time.sleep(0.1) sys.stderr.write('\r') sys.stderr.flush() else: for line in iter(proc.stdout.readline, b''): if line.strip() != '': log.debug(" -->> "+line.strip()) if proc.poll() is not None and line == '': break return proc.returncode, proc.stdout.read(), proc.stderr.read() except Exception as e: return -1, '', str(e) def subprocess_output(stdout): if stdout != '': log.debug(" -->> "+stdout) else: log.debug(" -->> subprocess empty output") def start_docker(): return exec_command('systemctl enable docker && systemctl daemon-reload && systemctl restart docker',shell=True) def check_port(port): if not INSTALL: return True try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(('0.0.0.0', int(port))) return True except Exception as e: log.debug("try listen mgt port failed: "+str(e)) return False def install_docker(): log.info(text('install-docker')) log.debug("downloading get-docker.sh") if not save_file_from_url('https://'+DOMAIN+'/release/latest/get-docker.sh','get-docker.sh'): raise Exception(text('fail-to-download-docker-installation')) source = docker_source() if source == '': raise Exception(text('fail-to-connect-docker-source')) log.debug(text('docker-source')+': '+source) env = { "DOWNLOAD_URL": source, "https_proxy": os.environ.get('https_proxy',''), } log.debug("installing docker") r = exec_command_with_loading('bash get-docker.sh',env=env) if r[0] == 0: p = start_docker() subprocess_output(p[1].strip()) if p[0] != 0: log.error("start docker failed: "+p[2].strip()) return p[0] == 0 else: log.error("install docker error: "+r[2].strip()) return False compose_command = '' def precheck_docker_compose(): log.info(text("docker-compose-version")) global compose_command while True: version_output = '' proc = exec_command('docker', 'compose', 'version') if proc[0] == 0: help_proc = exec_command('docker', 'compose', 'up', '--help') if help_proc[0] == 0 and '--detach' in help_proc[1]: compose_command = 'docker compose -f docker-compose.yaml' version_output = proc[1].strip() else: log.debug('docker compose can not find detach argument') else: compose_proc = exec_command('docker-compose', 'version') if compose_proc[0] == 0: help_proc = exec_command('docker-compose', 'up', '--help') if help_proc[0] == 0 and '--detach' in help_proc[1]: compose_command = 'docker-compose -f docker-compose.yaml' version_output = compose_proc[1].strip() else: log.debug('docker-compose can not find detach argument') else: log.warning(text('docker-compose-not-installed', PRODUCT)) if version_output != '': t = re.findall(r'^Docker Compose version v?(\d+)\.', version_output) if len(t) == 0: log.warning(text('docker-compose-not-installed', PRODUCT)) elif int(t[0]) < 2: log.warning(text('docker-version-too-low', PRODUCT)) else: return True action = ui_choice(text('if-update-docker'), [ ('y', text('yes')), ('n', text('no')), ]) if action.lower() == 'n': return False elif action.lower() == 'y': if not install_docker(): log.warning(text('install-docker-failed', PRODUCT)) return False def precheck_dns_scope(): resolve_file = '/etc/resolv.conf' if not os.path.exists(resolve_file): return True have_scope = False raw_lines = [] with open(resolve_file, 'r') as f: for line in f.readlines(): strip_line = line.strip() if not strip_line.startswith('nameserver') or '%' not in strip_line.lstrip('nameserver').strip(): raw_lines.append(line) continue have_scope = True if not have_scope: return True action = ui_choice(text('if-remove-ipv6-scope'),[ ('y', text('yes')), ('n', text('no')), ]) if action.lower() != 'y': return False shutil.copyfile(resolve_file, resolve_file+'.bak') with open(resolve_file, 'w') as f: for line in raw_lines: f.write(line) return True def precheck(): if platform.machine() in ('x86_64', 'AMD64') and 'ssse3' not in open('/proc/cpuinfo', 'r').read().lower(): log.warning(text('ssse3-not-support')) return False if free_memory() < 1024 * 1024 * 1024: log.warning(text('insufficient-memory')) return False log.info(text("docker-version")) while True: proc = exec_command('docker', '--version') if proc[0] == 0: t = re.findall(r'^Docker version (\d+)\.', proc[1]) if len(t) == 0: log.warning(text('docker-not-installed', PRODUCT)) elif int(t[0]) < 20: log.warning(text('docker-version-too-low', PRODUCT)) else: break else: log.warning(text('docker-not-installed', PRODUCT)) action = ui_choice(text('if-install-docker'), [ ('y', text('yes')), ('n', text('no')), ]) if action.lower() == 'n': return False elif action.lower() == 'y': if not install_docker(): log.warning(text('install-docker-failed', PRODUCT)) return False if not precheck_docker_compose(): return False if not precheck_dns_scope(): return False return True def docker_pull(cwd): log.info(text('docker-pull')) try: subprocess.check_call(compose_command+' pull', cwd=cwd, shell=True) return True except Exception as e: log.warning("docker pull error: "+str(e)) return False def docker_restart(container): log.info(text('docker-restart')+": "+container) try: subprocess.check_call('docker restart '+container, shell=True) return True except Exception as e: log.error("docker restart error: "+str(e)) return False def docker_exec(container, command): log.info(text('docker-exec')+": ("+container+") "+command) try: subprocess.check_call('docker exec '+container+' '+command, shell=True) return True except Exception as e: log.error("docker exec error: "+str(e)) return False def image_clean(): log.info(text('image-clean')) proc = exec_command('docker image prune -f --filter="label=maintainer=SafeLine-CE"', shell=True) if proc[0] != 0: log.warning("remove docker image failed: "+proc[2]) def docker_up(cwd): log.info(text('docker-up')) while True: p = exec_command_with_loading(compose_command+' up -d --remove-orphans', cwd=cwd) if p[0] == 0: return True if 'iptables failed' in p[2]: log.warning("docker up error: "+p[2]) while True: action = ui_choice(text('docker-up-iptables-failed'),[ ('y', text('yes')), ('n', text('no')), ]) if action.lower() == 'y': start_docker() break elif action.lower() == 'n': return False else: log.error("docker up error: "+p[2]) return False def docker_down(cwd): log.info(text('docker-down', PRODUCT)) try: subprocess.check_call(compose_command+' down', cwd=cwd, shell=True) return True except Exception: return False def get_url_time(url): now = datetime.datetime.now() try: urlopen(url, timeout=10, context=REQUEST_CTX) except HTTPError as e: log.debug("get url "+url+" status: "+str(e.status)) if e.status > 499: return 999999 except Exception as e: log.debug("get url "+url+" failed: "+str(e)) return 999999 return (datetime.datetime.now() - now).microseconds / 1000 def get_avg_delay(url): log.debug("test url avg delay: "+url) total_delay = 0 for i in range(3): total_delay += get_url_time(url) avg_delay = total_delay / 3 log.debug("url "+url+" avg delay: "+str(avg_delay)) return avg_delay pull_failed_prefix = [] def image_source(): source = { 'https://registry-1.docker.io': 'chaitin', "https://swr.cn-east-3.myhuaweicloud.com": 'swr.cn-east-3.myhuaweicloud.com/chaitin-safeline' } min_delay = -1 image_prefix = '' for url, prefix in source.items(): if prefix in pull_failed_prefix: continue delay = get_avg_delay(url) if delay > 0 and (min_delay < 0 or delay < min_delay): min_delay = delay image_prefix = prefix log.debug("use image_prefix: "+image_prefix) return image_prefix def docker_source(): sources = [ "https://mirrors.aliyun.com/docker-ce/", "https://mirrors.tencent.com/docker-ce/", "https://download.docker.com" ] min_delay = -1 source = '' for v in sources: delay = get_avg_delay(v) if delay > 0 and (min_delay < 0 or delay < min_delay): min_delay = delay source = v return source def read_config(path,config): with open(path, 'r') as f: for line in f.readlines(): if line.strip() == '': continue try: s = line.index('=') if s > 0: k = line[:s].strip() v = line[s + 1:].strip() config[k] = v except ValueError: continue def write_config(path,config): with open(path, 'w') as f: for k in config: f.write('%s=%s\n' % (k, config[k])) def generate_config(path): log.info(text('update-config')) config = { 'SAFELINE_DIR': path, 'POSTGRES_PASSWORD': '', 'MGT_PORT': '', 'RELEASE': '', 'CHANNEL': '', 'REGION': '', 'IMAGE_PREFIX': '', 'IMAGE_TAG': '', 'SUBNET_PREFIX': '', 'ARCH_SUFFIX': '' } env_path = os.path.join(path,'.env') if os.path.exists(env_path): read_config(env_path,config) if config['ARCH_SUFFIX'] == '': if platform.machine() == 'aarch64': config['ARCH_SUFFIX'] = '-arm' if config['POSTGRES_PASSWORD'] == '': config['POSTGRES_PASSWORD'] = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(20)]) if config['SUBNET_PREFIX'] == '': config['SUBNET_PREFIX'] = rand_subnet() if config['RELEASE'] == '' and GLOBAL_ARGS.lts: config['RELEASE'] = '-lts' config['CHANNEL'] = '-lts' if config['RELEASE'] == '-lts': GLOBAL_ARGS.lts = True default_try = False if config['MGT_PORT'] == '9443': default_try = True while not config['MGT_PORT'].isnumeric() or int(config['MGT_PORT']) >= 65536 or int(config['MGT_PORT']) <= 0 or not check_port(config['MGT_PORT']): if not default_try: config['MGT_PORT'] = '9443' default_try = True else: config['MGT_PORT'] = ui_read(text('input-mgt-port', PRODUCT),None) if config['REGION'] == '' and GLOBAL_ARGS.en: config['REGION'] = '-g' if not config['POSTGRES_PASSWORD'].isalnum(): log.info(text('pg-pass-contains-invalid-char')) raise Exception(text('pg-pass-contains-invalid-char')) if config['IMAGE_PREFIX'] == '' or config['IMAGE_PREFIX'] in pull_failed_prefix: config['IMAGE_PREFIX'] = image_source() if config['IMAGE_PREFIX'] == '': raise Exception(text('fail-to-connect-image-source')) config['IMAGE_TAG'] = get_version(config['IMAGE_TAG']) log.info(text('target-version', config['IMAGE_TAG'])) write_config(env_path, config) return config def show_address(mgt_port): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(("8.8.8.8", 80)) local_ip = s.getsockname()[0] log.info(text('go-to-panel', (PRODUCT, local_ip, mgt_port))) log.info(text('go-to-panel', (PRODUCT, '0.0.0.0', mgt_port))) def init_mgt(): while True: p = exec_command('docker', 'inspect','--format=\'{{.State.Health.Status}}\'', 'safeline-mgt') if p[0] == 0 and p[1].strip().replace("'",'') == 'healthy': break elif p[0] != 0: log.debug("get safeline-mgt status error: "+str(p[2])) log.info(text('wait-mgt-health')) time.sleep(5) log.info(text('reset-admin')) proc = exec_command('docker exec safeline-mgt /app/mgt-cli reset-admin --once',shell=True) if proc[0] != 0: log.warning(proc[2]) elif proc[1].strip() != '': log.info('\n'+proc[1].strip()) def check_install_path(safeline_path): if not safeline_path.startswith('/'): return False proc = exec_command('which', 'docker') if proc[0] != 0: log.debug('get docker path failed: '+proc[2]) raise Exception(text('fail-to-get-docker-path')) if not proc[1].startswith('/snap'): return True home_path = os.path.expanduser('~') if not safeline_path.startswith(home_path): log.warning(text('snap-docker-should-use-home-path', home_path)) return False return True def install(): global INSTALL INSTALL = True log.info(text('prepare-to-install', PRODUCT)) if not precheck(): log.error(text('precheck-failed', PRODUCT)) return log.info(text('precheck-passed')) default_path = '/data/safeline' if not SELF: default_path = '/data/waf' while True: safeline_path = ui_read(text('input-target-path', PRODUCT), default_path) if not check_install_path(safeline_path): log.warning(text('invalid-path', safeline_path)) continue if os.path.exists(safeline_path): log.warning(text('path-exists', safeline_path)) continue if free_space(safeline_path) < 5 * 1024 * 1024 * 1024: log.warning(text('insufficient-disk-capacity', (safeline_path, PRODUCT))) continue break try: os.makedirs(safeline_path) except Exception as e: log.error(text('fail-to-create-dir', safeline_path) + ' ' + str(e)) return mgt_path = os.path.join(safeline_path,'resources','mgt') try: os.makedirs(mgt_path, exist_ok=True) except Exception as e: log.error(text('fail-to-create-dir', mgt_path) + ' ' + str(e)) return if GLOBAL_ARGS.patch != '': shutil.copyfile(GLOBAL_ARGS.patch, os.path.join(mgt_path, 'product.data')) log.info(text('remain-disk-capacity', (safeline_path, humen_size(free_space(safeline_path))))) log.info(text('download-compose')) if not save_file_from_url('https://'+DOMAIN+'/release/latest/compose.yaml',os.path.join(safeline_path, 'docker-compose.yaml')): log.error(text('fail-to-download-compose')) return rename_file(os.path.join(safeline_path, 'compose.yaml'),os.path.join(safeline_path, 'compose.yaml.bak')) mgt_port = generate_config_and_run(safeline_path) if mgt_port is None: return log.info(text('install-finish', PRODUCT)) finish(mgt_port) def get_installed_dir(): safeline_path = '' safeline_path_proc = exec_command('docker','inspect','--format','\'{{index .Config.Labels "com.docker.compose.project.working_dir"}}\'', 'safeline-mgt') if safeline_path_proc[0] == 0: safeline_path = safeline_path_proc[1].strip().replace("'",'') else: log.debug("get installed dir error: "+ safeline_path_proc[2]) log.debug("find safeline installed path: " + safeline_path) if safeline_path == '' or not os.path.exists(safeline_path): log.warning(text('fail-to-get-installed-dir')) return ui_read(text('input-target-path', PRODUCT),None) return safeline_path def save_file_from_url(url, path): log.debug('saving '+url+' to '+path) data = get_url(url) if data is None: return False with open(path, 'w') as f: f.write(data) return True def rename_file(src, dst): if os.path.exists(src): os.rename(src, dst) def remove_file(src): if os.path.exists(src): os.remove(src) def generate_config_and_run(safeline_path): env_file = os.path.join(safeline_path, '.env') env_bak_file = os.path.join(safeline_path, '.env.bak') if os.path.exists(env_file): shutil.copyfile(env_file, env_bak_file) try: while True: config = generate_config(safeline_path) if docker_pull(safeline_path): break pull_failed_prefix.append(config['IMAGE_PREFIX']) log.info(text('try-another-image-source')) if not docker_up(safeline_path): log.error(text('fail-to-up')) rename_file(env_bak_file, env_file) return None except KeyboardInterrupt: log.warning(text('keyboard-interrupt')) rename_file(env_bak_file, env_file) return None except Exception as e: log.error('start WAF failed: '+str(e)) rename_file(env_bak_file, env_file) return None remove_file(env_bak_file) return config['MGT_PORT'] def upgrade(): safeline_path = get_installed_dir() if not precheck_docker_compose() or not precheck_dns_scope(): log.error(text('precheck-failed', PRODUCT)) return log.info(text('download-compose')) rename_file(os.path.join(safeline_path, 'compose.yaml'), os.path.join(safeline_path, 'compose.yaml.bak')) if not save_file_from_url('https://'+DOMAIN+'/release/latest/compose.yaml', os.path.join(safeline_path, 'docker-compose.yaml')): log.error(text('fail-to-download-compose')) return mgt_port = generate_config_and_run(safeline_path) if mgt_port is None: return if GLOBAL_ARGS.image_clean: image_clean() log.info(text('upgrade-finish', PRODUCT)) finish(mgt_port) def finish(mgt_port): init_mgt() show_address(mgt_port) def reset_tengine(): safeline_path = get_installed_dir() resources_path = os.path.join(safeline_path, 'resources') nginx_path = os.path.join(resources_path,'nginx') if not os.path.exists(nginx_path): log.error(text('fail-to-find-nginx')) return backup_path = os.path.join(resources_path, 'nginx.'+str(datetime.datetime.now().timestamp())) log.info(text('nginx-backup-dir') +': '+ backup_path) try: shutil.move(nginx_path, backup_path) except Exception as e: log.error(text('fail-to-backup-nginx')+': '+str(e)) return if docker_restart('safeline-tengine'): docker_exec('safeline-mgt', 'gentenginewebsite') if os.path.exists(os.path.join(backup_path, 'static')): try: shutil.copy(os.path.join(backup_path, 'static'), os.path.join(nginx_path, 'static')) except Exception as e: log.error(text('fail-to-recover-static')+': '+str(e)) return log.info(text('reset-tengine-finish')) def docker_restart_all(cwd): if not docker_down(cwd): log.error(text('fail-to-down')) return False if not docker_up(cwd): log.error(text('fail-to-up')) return False return True def reset_postgres(): safeline_path = get_installed_dir() env_file = os.path.join(safeline_path, '.env') if not os.path.exists(env_file): log.error(text('fail-to-find-env')) return config = {} read_config(env_file, config) config['POSTGRES_PASSWORD'] = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(20)]) write_config(env_file, config) if not docker_exec('safeline-pg','psql -U safeline-ce -c "ALTER USER \\"safeline-ce\\" WITH PASSWORD \''+config['POSTGRES_PASSWORD']+'\';"'): log.error(text('fail-to-reset-postgres-password')) return action = ui_choice(text('if-restart-docker', PRODUCT), [ ('y', text('yes')), ('n', text('no')), ]) if action.lower() == 'y': if not precheck_docker_compose(): log.error(text('precheck-failed', PRODUCT)) return if not docker_restart_all(safeline_path): return log.info(text('reset-postgres-password-finish')) def repair(): action = ui_choice(text('choice-action'),[ ('1', text('reset-tengine')), ('2', text('reset-postgres')), ]) if action =='1': reset_tengine() elif action =='2': reset_postgres() def restart(): safeline_path = get_installed_dir() if not precheck_docker_compose(): log.error(text('precheck-failed', PRODUCT)) return if not docker_restart_all(safeline_path): return log.info(text('restart-docker-finish', PRODUCT)) def backup(): pass def uninstall(): safeline_path = get_installed_dir() action = ui_choice(text('if-remove-waf', (PRODUCT, safeline_path)),[ ('y', text('yes')), ('n', text('no')), ]) if action == 'n': return if not precheck_docker_compose(): log.error(text('precheck-failed', PRODUCT)) return if not docker_down(safeline_path): log.error(text('fail-to-docker-down')) return image_clean() try: shutil.rmtree(safeline_path) except Exception as e: log.debug("remove dir failed: "+str(e)) log.error(text('fail-to-remove-dir', PRODUCT)) log.info(text('uninstall-finish', PRODUCT)) ACTION = '' def get_version_from_input(old_version): while True: version = ui_read(text('input-target-version', ACTION),None) if not check_version_format(version): log.warning(text('version-format-error', version)) continue if not compare_version(old_version, version): log.warning(text('can-not-downgrade', (PRODUCT, old_version, version))) continue return version.lstrip('v') def check_version_format(version): if GLOBAL_ARGS.lts and not version.endswith('-lts'): log.debug('lts version should end with -lts') return False split_version = version.lstrip('v').rstrip('-lts').split('.') if len(split_version) != 3: log.debug('split version length is not 3') return False try: for v in split_version: int(v) except ValueError as e: log.debug('check version %s format failed: %s' % (version, str(e))) return False return True def compare_version(old_version, new_version): if old_version == 'latest': try: log.info(text('get-version-from-mgt')) old_version = get_version_from_mgt() except Exception as e: log.debug('get version from mgt failed: '+str(e)) log.warning(text('fail-to-get-version-from-mgt')) log.warning(text('skip-version-compare', PRODUCT)) return True elif old_version == '': return True if not check_version_format(old_version): log.warning(text('version-format-error', old_version)) log.warning(text('skip-version-compare', PRODUCT)) return True split_old_version = old_version.lstrip('v').rstrip('-lts').split('.') split_new_version = new_version.lstrip('v').rstrip('-lts').split('.') for index in range(len(split_old_version)): int_old_version = int(split_old_version[index]) int_new_version = int(split_new_version[index]) if int_old_version > int_new_version: return False elif int_old_version < int_new_version: return True return True def get_version_from_mgt(): proc = exec_command('docker exec safeline-mgt /app/mgt version',shell=True) if proc[0] != 0: raise Exception('stderr: ' + proc[2]) for line in proc[1].split('\n'): strip_line = line.strip() if not strip_line.startswith('version'): continue return strip_line.lstrip('version').strip() raise Exception('mgt version not found') TARGET_VERSION = '' def get_version(old_version): global TARGET_VERSION if TARGET_VERSION != '': return TARGET_VERSION log.info(text('get-version', PRODUCT)) try: data = get_url('https://'+DOMAIN+'/release/latest/version.json') if data is None: TARGET_VERSION = get_version_from_input(old_version) return TARGET_VERSION version = json.loads(data) if GLOBAL_ARGS.lts: latest_version = version['lts_version'] else: latest_version = version['latest_version'] if not check_version_format(latest_version): log.warning(text('version-format-error', latest_version)) TARGET_VERSION = get_version_from_input(old_version) elif not compare_version(old_version, latest_version): log.warning(text('can-not-downgrade', (old_version, latest_version))) TARGET_VERSION = get_version_from_input(old_version) else: TARGET_VERSION = latest_version.lstrip('v') return TARGET_VERSION except Exception as e: log.warning('get version failed: %s' % str(e)) TARGET_VERSION = get_version_from_input(old_version) return TARGET_VERSION def main(): if SELF: banner() log.info(text('hello1')) log.info(text('hello2')) print() if GLOBAL_ARGS.lts: log.info(text('install-channel', text('lts-release'))) if sys.version_info.major == 2 or (sys.version_info.major == 3 and sys.version_info.minor <= 5): log.error(text('python-version-too-low')) return if not sys.stdin.isatty(): log.error(text('not-a-tty')) return if os.geteuid() != 0: log.error(text('not-root')) return if platform.system() != 'Linux': log.error(text('not-linux', (PRODUCT, platform.system()))) return if platform.machine() not in ('aarch64', 'x86_64', 'AMD64'): log.error(text('unsupported-arch', (PRODUCT, platform.machine()))) return action = ui_choice(text('choice-action'), [ ('1', text('install')), ('2', text('upgrade')), ('3', text('uninstall')), ('4', text('repair')), ('5', text('restart')), # ('4', text('backup')) ]) global ACTION if action == '1': ACTION = text('install') install() elif action == '2': ACTION = text('upgrade') upgrade() elif action == '3': uninstall() elif action == '4': repair() elif action == '5': restart() # elif action == '4': # backup() if __name__ == '__main__': try: main() except KeyboardInterrupt: log.warning(text('keyboard-interrupt')) pass except Exception as e: log.error(e) finally: if SELF: print(color(text('talking-group') + '\n', [GREEN])) ================================================ FILE: sdk/ingress-nginx/README.md ================================================ # ingress-nginx-safeline [Ingress-nginx](https://kubernetes.github.io/ingress-nginx/) plugin for Chaitin SafeLine Web Application Firewall (WAF). This plugin is used to protect your API from malicious requests. It can be used to block requests that contain malicious content in the request body, query parameters, headers, or URI. ## Safeline Prepare The detection engine of the SafeLine provides services by default via Unix socket. We need to modify it to use TCP, so it can be called by the t1k plugin. 1.Navigate to the configuration directory of the SafeLine detection engine: ```shell cd /data/safeline/resources/detector/ ``` 2.Open the `detector.yml` file in a text editor. Modify the bind configuration from Unix socket to TCP by adding the following settings: ```yaml bind_addr: 0.0.0.0 listen_port: 8000 ``` These configuration values will override the default settings in the container, making the SafeLine engine listen on port 8000. 3.Next, map the container’s port 8000 to the host machine. First, navigate to the SafeLine installation directory: ```shell cd /data/safeline ``` 4.Open the compose.yaml file in a text editor and add the ports field to the detector container to expose port 8000: ```yaml ... detect: ports: - 8000:8000 ... ``` 5.Save the changes and restart SafeLine with the following commands: ```shell docker-compose down docker-compose up -d ``` This will apply the changes and activate the new configuration. ## Plugin Usage ### Step 1: Install the plugin way 1: Build your own ingress-nginx/controller image with the plugin installed. ```dockerfile FROM registry.k8s.io/ingress-nginx/controller:v1.10.1 USER root RUN apk add --no-cache make gcc unzip wget # install luaroncks RUN wget https://luarocks.org/releases/luarocks-3.11.0.tar.gz && \ tar zxpf luarocks-3.11.0.tar.gz && \ cd luarocks-3.11.0 && \ ./configure && \ make && \ make install && \ cd .. && \ rm -rf luarocks-3.11.0 luarocks-3.11.0.tar.gz RUN luarocks install ingress-nginx-safeline && \ ln -s /usr/local/share/lua/5.1/safeline /etc/nginx/lua/plugins/safeline USER www-data ``` way 2: Use the chaitin ingress-nginx-controller image. replace image ingress-nginx-controller with `docker.io/chaitin/ingress-nginx-controller:v1.10.1` in your deployment. ### Step 2: Configure the plugin use a ConfigMap to configure the plugin ```yaml apiVersion: v1 kind: ConfigMap metadata: name: safeline namespace: default data: host: "YOUR_DETECTOR_HOST" port: "YOUR_DETECTOR_PORT" ``` ### Step 3: Configure the ingress-controller inject env `SAFELINE_HOST` and `SAFELINE_PORT` to the ingress-controller deployment ```yaml ... env: - name: SAFELINE_HOST valueFrom: configMapKeyRef: name: safeline key: host - name: SAFELINE_PORT valueFrom: configMapKeyRef: name: safeline key: port ... ``` ### Step 3: Enable the plugin enable safeline plugin in configmap ```yaml apiVersion: v1 data: allow-snippet-annotations: "false" plugins: "safeline" kind: ConfigMap metadata: name: ingress-nginx-controller namespace: default ``` ### Step 4: Set externalTrafficPolicy to Local by default, the ingress-nginx-controller service is of type LoadBalancer, which means the source IP of the request will be the IP of the LoadBalancer. If you want to get the real source IP, you can set the externalTrafficPolicy to Local. ### Step 5: Test the plugin use a simple http sql injection test ```bash curl -X POST http://localhost/ -d "select * from users where id=1 or 1=1" ``` you should get a 403 response. ```bash {"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "b53eb5b95796475699c52a019abb8e6a"} ``` ================================================ FILE: sdk/ingress-nginx/ingress-nginx-safeline-1.0.2-1.rockspec ================================================ package = "ingress-nginx-safeline" version = "1.0.2-1" source = { url = "git://github.com/xbingW/ingress-nginx-safeline.git" } description = { summary = "Ingress-Nginx plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/xbingW/ingress-nginx-safeline", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k" } build = { type = "builtin", modules = { ["safeline.main"] = "lib/safeline/main.lua" } } ================================================ FILE: sdk/ingress-nginx/ingress-nginx-safeline-1.0.3-1.rockspec ================================================ package = "ingress-nginx-safeline" version = "1.0.3-1" source = { url = "git://github.com/xbingW/ingress-nginx-safeline.git" } description = { summary = "Ingress-Nginx plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/xbingW/ingress-nginx-safeline", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k >= 1.1.5" } build = { type = "builtin", modules = { ["safeline.main"] = "lib/safeline/main.lua" } } ================================================ FILE: sdk/ingress-nginx/ingress-nginx-safeline-1.0.4-1.rockspec ================================================ package = "ingress-nginx-safeline" version = "1.0.4-1" source = { url = "git://github.com/chaitin/ingress-nginx-safeline.git" } description = { summary = "Ingress-Nginx plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/chaitin/ingress-nginx-safeline", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k >= 1.1.5" } build = { type = "builtin", modules = { ["safeline.main"] = "lib/safeline/main.lua" } } ================================================ FILE: sdk/ingress-nginx/lib/safeline/main.lua ================================================ local t1k = require "resty.t1k" local t1k_constants = require "resty.t1k.constants" local ngx = ngx local fmt = string.format local blocked_message = [[{"code": %s, "success":false, ]] .. [["message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "%s"}]] local _M = {} local mode = os.getenv("SAFELINE_MODE") local host = os.getenv("SAFELINE_HOST") local port = os.getenv("SAFELINE_PORT") local connect_timeout = os.getenv("SAFELINE_CONNECT_TIMEOUT") local send_timeout = os.getenv("SAFELINE_SEND_TIMEOUT") local read_timeout = os.getenv("SAFELINE_READ_TIMEOUT") local req_body_size = os.getenv("SAFELINE_REQ_BODY_SIZE") local keepalive_size = os.getenv("SAFELINE_KEEPALIVE_SIZE") local keepalive_timeout = os.getenv("SAFELINE_KEEPALIVE_TIMEOUT") local remote_addr = os.getenv("SAFELINE_REMOTE_ADDR") local function get_conf() local t = { mode = mode or "block", host = host, port = port, connect_timeout = connect_timeout or 1000, send_timeout = send_timeout or 1000, read_timeout = read_timeout or 1000, req_body_size = req_body_size or 1024, keepalive_size = keepalive_size or 256, keepalive_timeout = keepalive_timeout or 60000, remote_addr = remote_addr } return t end function _M.rewrite() local t = get_conf() if not t.host then ngx.log(ngx.ERR, "safeline host is required") return end local ok, err, result = t1k.do_access(t, false) if not ok then ngx.log(ngx.ERR, "failed to detector req: ", err) return end if result then if result.action == t1k_constants.ACTION_BLOCKED then local msg = fmt(blocked_message, result.status, result.event_id) ngx.log(ngx.ERR, "blocked by safeline waf: ",msg) ngx.status = tonumber(result.status,10) ngx.say(msg) return ngx.exit(ngx.HTTP_OK) end end end return _M ================================================ FILE: sdk/kong/Readme.md ================================================ # Kong Safeline Plugin Kong plugin for Chaitin SafeLine Web Application Firewall (WAF). This plugin is used to protect your API from malicious requests. It can be used to block requests that contain malicious content in the request body, query parameters, headers, or URI. ## Safeline Prepare The detection engine of the SafeLine provides services by default via Unix socket. We need to modify it to use TCP, so it can be called by the t1k plugin. 1.Navigate to the configuration directory of the SafeLine detection engine: ```shell cd /data/safeline/resources/detector/ ``` 2.Open the `detector.yml` file in a text editor. Modify the bind configuration from Unix socket to TCP by adding the following settings: ```yaml bind_addr: 0.0.0.0 listen_port: 8000 ``` These configuration values will override the default settings in the container, making the SafeLine engine listen on port 8000. 3.Next, map the container’s port 8000 to the host machine. First, navigate to the SafeLine installation directory: ```shell cd /data/safeline ``` 4.Open the compose.yaml file in a text editor and add the ports field to the detector container to expose port 8000: ```yaml ... detect: ports: - 8000:8000 ... ``` 5.Save the changes and restart SafeLine with the following commands: ```shell docker-compose down docker-compose up -d ``` This will apply the changes and activate the new configuration. ## Plugin Installation To install the plugin, run the following command in your Kong server: ```shell $ luarocks install kong-safeline ``` ## Plugin Configuration You can add the plugin to your API by making the following request: ```shell # if your detector is running on tcp port $ curl -X POST http://kong:8001/services/{name}/plugins \ --data "name=safeline" \ --data "config.host=your_detector_host" \ --data "config.port=your_detector_port" # if your detector is running on unix socket $ curl -X POST http://kong:8001/services/{name}/plugins \ --data "name=safeline" \ --data "config.host=unix:/path/to/your/unix/socket" ``` ## Test You can test the plugin by sending a request to your API with malicious content. If the request is blocked, you will receive a `403 Forbidden` response. ```shell $ curl -X POST http://kong:8000?1=1%20and%202=2 # you will receive a 403 Forbidden response {"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "8b41a021ea9541c89bb88f3773b4da24"} ``` ================================================ FILE: sdk/kong/kong/plugins/safeline/handler.lua ================================================ local t1k = require "resty.t1k" local t1k_constants = require "resty.t1k.constants" local fmt = string.format local SafelineHandler = { VERSION = "0.0.1", PRIORITY = 1000 } local blocked_message = [[{"code": %s, "success":false, ]] .. [["message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "%s"}]] local function get_conf(conf) local t = { mode = conf.mode, host = conf.host, port = conf.port, connect_timeout = conf.connect_timeout, send_timeout = conf.send_timeout, read_timeout = conf.read_timeout, req_body_size = conf.req_body_size, keepalive_size = conf.keepalive_size, keepalive_timeout = conf.keepalive_timeout, remote_addr = conf.remote_addr } return t end function SafelineHandler:access(conf) -- your custom code here local t = get_conf(conf) local ok, err, result = t1k.do_access(t, false) if not ok then kong.log.err("failed to detector req: ", err) end if result and result.status then if result.action == t1k_constants.ACTION_BLOCKED then local msg = fmt(blocked_message, result.status, result.event_id) kong.log.debug("blocked by safeline: ",msg) return kong.response.exit(tonumber(result.status), msg) end end end return SafelineHandler ================================================ FILE: sdk/kong/kong/plugins/safeline/schema.lua ================================================ local typedefs = require "kong.db.schema.typedefs" return { name = "kong-safeline", fields = {{ consumer = typedefs.no_consumer }, { protocols = typedefs.protocols_http }, { config = { type = "record", fields = {{ host = { type = "string", required = true } }, { port = { type = "number", required = false } }, { mode = { type = "string", required = false, default = "block", one_of = {"monitor", "block", "off"} } }, { connect_timeout = { type = "number", required = false, default = 1000 } }, { send_timeout = { type = "number", required = false, default = 1000 } }, { read_timeout = { type = "number", required = false, default = 1000 } }, { req_body_size = { type = "number", required = false, default = 1000 } }, { keepalive_size = { type = "number", required = false, default = 1000 } }, { keepalive_timeout = { type = "number", required = false, default = 1000 } }, { remote_addr = { type = "string", required = false } }} } }} } ================================================ FILE: sdk/kong/kong-safeline-1.0.0-1.rockspec ================================================ package = "kong-safeline" version = "1.0.0-1" source = { url = "git://github.com/xbingW/kong-safeline.git" } description = { summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/xbingW/kong-safeline", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k" } build = { type = "builtin", modules = { ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" } } ================================================ FILE: sdk/kong/kong-safeline-1.0.1-1.rockspec ================================================ package = "kong-safeline" version = "1.0.1-1" source = { url = "git://github.com/xbingW/kong-safeline.git" } description = { summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/xbingW/kong-safeline", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k" } build = { type = "builtin", modules = { ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" } } ================================================ FILE: sdk/kong/kong-safeline-1.0.2-1.rockspec ================================================ package = "kong-safeline" version = "1.0.2-1" source = { url = "git://github.com/xbingW/kong-safeline.git" } description = { summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/xbingW/kong-safeline", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k >= 1.1.5" } build = { type = "builtin", modules = { ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" } } ================================================ FILE: sdk/kong/kong-safeline-1.0.3-1.rockspec ================================================ package = "kong-safeline" version = "1.0.3-1" source = { url = "file://kong-safeline-1.0.3.tar.gz", } build = { type = "script", rockspec = { build = { "git clone https://github.com/chaitin/SafeLine.git", "cp -r sdk/kong .", "rm -rf SafeLine" } } } description = { summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/chaitin/SafeLine", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k >= 1.1.5" } build = { type = "builtin", modules = { ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" } } ================================================ FILE: sdk/kong/kong-safeline-1.0.4-1.rockspec ================================================ package = "kong-safeline" version = "1.0.4-1" source = { url = "git://github.com/chaitin/SafeLine.git", } build = { type = "script", rockspec = { build = { "git clone https://github.com/chaitin/SafeLine.git", "cp -r sdk/kong .", "rm -rf SafeLine" } } } description = { summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/chaitin/SafeLine", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k >= 1.1.5" } build = { type = "builtin", modules = { ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" } } ================================================ FILE: sdk/kong/kong-safeline-1.0.5-1.rockspec ================================================ package = "kong-safeline" version = "1.0.5-1" source = { url = "git://github.com/chaitin/SafeLine.git", } build = { type = "script", rockspec = { build = { "git clone https://github.com/chaitin/SafeLine.git", "cp -r sdk/kong .", "rm -rf SafeLine" } } } description = { summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/chaitin/SafeLine", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k >= 1.1.5" } build = { type = "builtin", modules = { ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" } } ================================================ FILE: sdk/kong/kong-safeline-1.0.6-1.rockspec ================================================ package = "kong-safeline" version = "1.0.6-1" source = { url = "git://github.com/xbingW/kong-safeline.git", } description = { summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/chaitin/SafeLine", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k >= 1.1.5" } build = { type = "builtin", modules = { ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" } } ================================================ FILE: sdk/kong/kong-safeline-1.0.7-1.rockspec ================================================ package = "kong-safeline" version = "1.0.7-1" source = { url = "git://github.com/chaitin/kong-safeline.git", } description = { summary = "Kong plugin for Chaitin SafeLine Web Application Firewall", homepage = "https://github.com/chaitin/SafeLine", license = "Apache License 2.0", maintainer = "Xiaobing Wang " } dependencies = { "lua-resty-t1k >= 1.1.5" } build = { type = "builtin", modules = { ["kong.plugins.safeline.handler"] = "kong/plugins/safeline/handler.lua", ["kong.plugins.safeline.schema"] = "kong/plugins/safeline/schema.lua" } } ================================================ FILE: sdk/lua-resty-t1k/.github/workflows/release.yml ================================================ name: Release on: push: tags: - "v*" jobs: release: name: Release runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Install Lua uses: leafo/gh-actions-lua@v10 - name: Install Luarocks uses: leafo/gh-actions-luarocks@v4 - name: Extract release tag id: release_tag run: | # Extract the tag name from the ref tag="${GITHUB_REF#refs/tags/}" version_without_v="${tag#v}" echo "version=${tag}" >> $GITHUB_ENV echo "version_without_v=${version_without_v}" >> $GITHUB_ENV - name: Create Release uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ env.version }} release_name: ${{ env.version }} draft: false prerelease: false - name: Upload to luarocks env: LUAROCKS_TOKEN: ${{ secrets.LUAROCKS_TOKEN }} run: | luarocks install dkjson luarocks upload rockspec/lua-resty-t1k-${{ env.version_without_v }}-0.rockspec --api-key=${{ secrets.LUAROCKS_API_KEY }} ================================================ FILE: sdk/lua-resty-t1k/.github/workflows/test.yml ================================================ name: Test on: [ push, pull_request ] jobs: luacheck: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: leafo/gh-actions-lua@v10 with: luaVersion: "luajit-openresty" - uses: leafo/gh-actions-luarocks@v4 - run: luarocks install luacheck - run: luacheck lib run_tests: strategy: matrix: openresty_version: - 1.17.8.2 - 1.19.9.1 - 1.21.4.3 - 1.25.3.1 services: detector: image: chaitin/safeline-detector:t1k-ci-1.6.0 runs-on: ubuntu-latest container: image: openresty/openresty:${{ matrix.openresty_version }}-alpine-fat # --init runs tinit as PID 1 and prevents the 'WARNING: killing the child process' spam from the test suite options: --init steps: - name: Install deps run: | apk add --no-cache bash bind-tools curl git git-lfs libarchive-tools perl perl-dev wget ln -s /usr/bin/bsdtar /usr/bin/tar - name: Install CPAN run: curl -s -L http://xrl.us/cpanm > /bin/cpanm && chmod +x /bin/cpanm - name: Cache uses: actions/cache@v3 with: path: | ~/.cpan ~/.cache key: ${{ runner.os }}-${{ matrix.openresty_version }}-cache - name: Install Test::Nginx run: cpanm -q -n Test::Nginx - uses: actions/checkout@v4 with: lfs: true - name: Run tests run: | curl -fs -X POST -H "Content-Type: application/octet-stream" --data-binary "@ci/bytecode" "http://detector:8001/update/policy" env DETECTOR_IP=$(dig detector +short) prove -r t/ ================================================ FILE: sdk/lua-resty-t1k/.gitignore ================================================ # IntelliJ project files .idea *.iml out gen # Visual Studio Code .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json !.vscode/*.code-snippets # Local History for Visual Studio Code .history/ # Built Visual Studio Code Extensions *.vsix # Test::Nginx files t/servroot # luarocks build files *.src.rock ================================================ FILE: sdk/lua-resty-t1k/.luacheckrc ================================================ std = "ngx_lua" redefined = false max_line_length = 130 max_code_line_length = 130 max_string_line_length = 130 max_comment_line_length = 130 ================================================ FILE: sdk/lua-resty-t1k/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2023 Beijing Chaitin Technology Co., Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: sdk/lua-resty-t1k/README.md ================================================ # lua-resty-t1k [![LuaRocks](https://img.shields.io/luarocks/v/blaisewang/lua-resty-t1k?style=flat-square)](https://luarocks.org/modules/blaisewang/lua-resty-t1k) [![Releases](https://img.shields.io/github/v/release/chaitin/lua-resty-t1k?style=flat-square)](https://github.com/chaitin/lua-resty-t1k/releases) [![License](https://img.shields.io/github/license/chaitin/lua-resty-t1k?color=ff69b4&style=flat-square)](https://github.com/chaitin/lua-resty-t1k/blob/main/LICENSE) ## Name Lua implementation of the T1K protocol for [Chaitin/SafeLine](https://github.com/chaitin/safeline) Web Application Firewall. ## Status Production ready. [![Test](https://img.shields.io/github/actions/workflow/status/chaitin/lua-resty-t1k/test.yml?logo=github&style=flat-square)](https://github.com/chaitin/lua-resty-t1k/actions) ## Installation ```bash luarocks install lua-resty-t1k ``` If you are in Mainland China ```bash luarocks install lua-resty-t1k --server https://luarocks.cn ``` ## Synopsis ```lua location / { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", -- block or monitor or off, default off host = "unix:/workdir/snserver.sock", -- required, SafeLine WAF detection service host, unix domain socket, IP, or domain is supported, string port = 8000, -- required when the host is an IP or domain, SafeLine WAF detection service port, integer connect_timeout = 1000, -- connect timeout, in milliseconds, integer, default 1s (1000ms) send_timeout = 1000, -- send timeout, in milliseconds, integer, default 1s (1000ms) read_timeout = 1000, -- read timeout, in milliseconds, integer, default 1s (1000ms) req_body_size = 1024, -- request body size, in KB, integer, default 1MB (1024KB) keepalive_size = 256, -- maximum concurrent idle connections to the SafeLine WAF detection service, integer, default 256 keepalive_timeout = 60000, -- idle connection timeout, in milliseconds, integer, default 60s (60000ms) remote_addr = "http_x_forwarded_for: 1", -- remote address from ngx.var.VARIABLE, string, default from ngx.var.remote_addr } local ok, err, _ = t1k.do_access(t, true) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } } ``` ## Lua Resty T1K vs. C T1K [C T1K](https://t1k.chaitin.com/), as part of SafeLine's enterprise edition, is a deployment mode crafted in C language for enhanced performance. It is compatible with all versions of Nginx and does not require deployment via OpenResty (lua_nginx_module). | | Lua Resty T1K | C T1K | |-----------------------|---------------|-------| | Request Detection | ✅ | ✅ | | Response Detection | ❌ | ✅ | | Health Checks* | ❌ | ✅ | | Cookie Protection | ❌ | ✅ | | Bot Protection | ❌ | ✅ | | Proxy-side Statistics | ❌ | ✅ | * APISIX implements health check functionality for the `chaitin-waf` plugin. For more information, please see the [chaitin-waf documentation](https://apisix.apache.org/docs/apisix/next/plugins/chaitin-waf/). ================================================ FILE: sdk/lua-resty-t1k/ci/.dockerignore ================================================ /bytecode ================================================ FILE: sdk/lua-resty-t1k/ci/Dockerfile ================================================ # Usage: # docker build -t chaitin/safeline-detector:t1k-ci-1.6.0 . # docker push chaitin/safeline-detector:t1k-ci-1.6.0 FROM chaitin/safeline-detector:1.6.0 RUN sed -i "s/^# bind_addr/bind_addr/; s/^# listen_port/listen_port/; s/^bind_addr: unix/# bind_addr: unix/;" /detector/snserver.yml ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/buffer.lua ================================================ local _M = { _VERSION = '1.0.0', } function _M:new(o) o = o or {} setmetatable(o, self) self.__index = self return o end function _M:add(v) self[#self + 1] = v end function _M:len() local len = 0 for _, v in ipairs(self) do len = len + #v end return len end return _M ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/constants.lua ================================================ local t = {} t.ACTION_PASSED = "." t.ACTION_BLOCKED = "?" t.MODE_OFF = "off" t.MODE_BLOCK = "block" t.MODE_MONITOR = "monitor" t.T1K_HEADER_SIZE = 5 t.TAG_HEAD = 0x01 t.TAG_BODY = 0x02 t.TAG_EXTRA = 0x03 t.TAG_VERSION = 0x20 t.TAG_EXTRA_HEADER = 0x23 t.TAG_EXTRA_BODY = 0x24 t.MASK_FIRST = 0x40 t.MASK_LAST = 0x80 t.NGX_HTTP_HEADER_PREFIX = "http_" t.BLOCK_CONTENT_TYPE = "application/json" t.BLOCK_CONTENT_FORMAT = [[ {"code": %s, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "%s"}]] t.UNIX_SOCK_PREFIX = "unix:" return t ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/file.lua ================================================ local buffer = require "resty.t1k.buffer" local _M = { _VERSION = '1.0.0' } local buffer_size = 2 ^ 13 function _M.read(p, size) size = (not size or size < 0) and 0 or size local f, err = io.open(p, "rb") if not f or err then return nil, err, nil end local left = size local buf = buffer:new() while left ~= 0 do local block_size = math.min(left, buffer_size) local block = f:read(block_size) if not block then break end buf:add(block) left = math.max(left - block_size, 0) end f:close() return true, nil, buf end return _M ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/filter.lua ================================================ local _M = { _VERSION = '1.0.0' } local find = string.find local sub = string.sub local ngx = ngx local function parse_extra_header(extra_header) local t = {} local idx = 1 while (idx <= #extra_header) do local key, val local _, to = find(extra_header, ":", idx) if to == 0 then break else key = sub(extra_header, idx, to - 1) end idx = to + 1 _, to = find(extra_header, "\n", idx) if to == 0 then break else val = sub(extra_header, idx, to - 1) end t[key] = val idx = to + 1 end return t end function _M.do_header_filter() local extra_header = ngx.ctx.t1k_extra_header if extra_header ~= nil then local header_table = parse_extra_header(extra_header) for k, v in pairs(header_table) do if k ~= nil and v ~= nil then ngx.header[k] = v end end end end return _M ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/handler.lua ================================================ local consts = require "resty.t1k.constants" local log = require "resty.t1k.log" local fmt = string.format local ngx = ngx local log_fmt = log.fmt local _M = { _VERSION = '1.0.0' } function _M.handle(t) local t_type = type(t) if t_type ~= "table" then local err = log_fmt("invalid result type: %s", t_type) return nil, err end local action = t["action"] if action == consts.ACTION_PASSED then return true, nil elseif action == consts.ACTION_BLOCKED then ngx.status = t["status"] or ngx.HTTP_FORBIDDEN ngx.header.content_type = consts.BLOCK_CONTENT_TYPE ngx.say(fmt(consts.BLOCK_CONTENT_FORMAT, ngx.status, t["event_id"])) return ngx.exit(ngx.status) else local err = log_fmt("unknown action from t1k server: %s", action) return nil, err end end return _M ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/log.lua ================================================ local _M = { _VERSION = '1.0.0' } local fmt = string.format local ERR = ngx.ERR local WARN = ngx.WARN local DEBUG = ngx.DEBUG function _M.fmt(formatstring, ...) return fmt("lua-resty-t1k: " .. formatstring, ...) end function _M.err_fmt(formatstring, ...) return ERR, _M.fmt(formatstring, ...) end function _M.warn_fmt(formatstring, ...) return WARN, _M.fmt(formatstring, ...) end function _M.debug_fmt(formatstring, ...) return DEBUG, _M.fmt(formatstring, ...) end return _M ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/request.lua ================================================ local bit = require "bit" local buffer = require "resty.t1k.buffer" local consts = require "resty.t1k.constants" local file = require "resty.t1k.file" local log = require "resty.t1k.log" local utils = require "resty.t1k.utils" local uuid = require "resty.t1k.uuid" local _M = { _VERSION = '1.0.0', } local bor = bit.bor local byte = string.byte local char = string.char local fmt = string.format local sub = string.sub local concat = table.concat local ngx = ngx local nlog = ngx.log local ngx_req = ngx.req local ngx_socket = ngx.socket local ngx_var = ngx.var local warn_fmt = log.warn_fmt local debug_fmt = log.debug_fmt local KEY_EXTRA_UUID = "UUID" local KEY_EXTRA_LOCAL_ADDR = "LocalAddr" local KEY_EXTRA_LOCAL_PORT = "LocalPort" local KEY_EXTRA_REMOTE_ADDR = "RemoteAddr" local KEY_EXTRA_REMOTE_PORT = "RemotePort" local KEY_EXTRA_SCHEME = "Scheme" local KEY_EXTRA_SERVER_NAME = "ServerName" local KEY_EXTRA_PROXY_NAME = "ProxyName" local KEY_EXTRA_REQ_BEGIN_TIME = "ReqBeginTime" local KEY_EXTRA_HAS_RSP_IF_OK = "HasRspIfOK" local KEY_EXTRA_HAS_RSP_IF_BLOCK = "HasRspIfBlock" local TAG_HEAD_WITH_MASK_FIRST = bor(consts.TAG_HEAD, consts.MASK_FIRST) local TAG_EXTRA_WITH_MASK_LAST = bor(consts.TAG_EXTRA, consts.MASK_LAST) local T1K_PROTO = "Proto:2\n" local T1K_PROTO_DATA = fmt("%s%s%s", char(consts.TAG_VERSION), utils.int_to_char_length(#T1K_PROTO), T1K_PROTO) local function read_request_body(opt_req_body_size) local ok, err local req_body, req_body_size local content_length = tonumber(ngx_var.http_content_length) or 0 local transfer_encoding = ngx_var.http_transfer_encoding if content_length == 0 and not transfer_encoding then return true, nil, nil end ngx_req.read_body() req_body = ngx_req.get_body_data() if req_body then req_body_size = #req_body if req_body_size > opt_req_body_size then nlog(debug_fmt("request body is too long: %d bytes, cut to %d bytes", req_body_size, opt_req_body_size)) req_body = sub(req_body, 1, opt_req_body_size) end return true, nil, req_body end local path = ngx_req.get_body_file() if not path then return true, nil, nil end ok, err, req_body = file.read(path, opt_req_body_size) if not ok then err = fmt("failed to read temporary file %s: %s", path, err) return ok, err, nil end return true, nil, req_body end local function get_remote_addr(remote_addr_var, remote_addr_idx) local addr if remote_addr_var then addr = utils.get_indexed_element(ngx_var[remote_addr_var], remote_addr_idx) end return addr or ngx_var.remote_addr end local function parse_v(v) if type(v) == "table" then return concat(v, ", ") end return tostring(v) end local function build_header() local http_version = ngx_req.http_version() if http_version < 2.0 then return true, nil, ngx_req.raw_header() end local headers, err = ngx_req.get_headers(0, true) if err then err = fmt("failed to call ngx_req.get_headers: %s", err) return nil, err, nil end local buf = buffer:new() buf:add(fmt("%s %s HTTP/%.1f\r\n", ngx_req.get_method(), ngx_var.request_uri, http_version)) for k, v in pairs(headers) do buf:add(fmt("%s: %s\r\n", k, parse_v(v))) end buf:add("\r\n") return true, nil, buf end local function build_body(opts) local ok, err local body local req_body_size = opts.req_body_size * 1024 ok, err, body = read_request_body(req_body_size) if not ok then return ok, err, nil end return true, nil, body end local function build_extra(opts) local err local src_ip = get_remote_addr(opts.remote_addr_var, opts.remote_addr_idx) if not src_ip then err = fmt("failed to get remote_addr, var: %s, idx %d", opts.remote_addr_var, opts.remote_addr_idx) return nil, err end local src_port = ngx_var.remote_port if not src_port then err = "failed to get ngx_var.remote_port" return nil, err, nil end local local_ip = ngx_var.server_addr if not local_ip then err = "failed to get ngx_var.server_addr" return nil, err, nil end local local_port = ngx_var.server_port if not local_port then err = "failed to get ngx_var.server_port" return nil, err, nil end local extra = buffer:new({ KEY_EXTRA_UUID, ":", uuid.generate_v4(), "\n", KEY_EXTRA_REMOTE_ADDR, ":", src_ip, "\n", KEY_EXTRA_REMOTE_PORT, ":", src_port, "\n", KEY_EXTRA_LOCAL_ADDR, ":", local_ip, "\n", KEY_EXTRA_LOCAL_PORT, ":", local_port, "\n", KEY_EXTRA_SCHEME, ":", ngx_var.scheme, "\n", KEY_EXTRA_SERVER_NAME, ":", ngx_var.server_name, "\n", KEY_EXTRA_PROXY_NAME, ":", ngx_var.hostname, "\n", KEY_EXTRA_REQ_BEGIN_TIME, ":", fmt("%.0f", ngx_req.start_time() * 1000000), "\n", KEY_EXTRA_HAS_RSP_IF_OK, ":n\n", KEY_EXTRA_HAS_RSP_IF_BLOCK, ":n\n" }) return true, nil, extra end local function do_send(sock, data) local ok, err = sock:send(data) if not ok then return ok, err end return true, nil end local function receive_data(s, srv) local t = {} local ft = true local finished repeat local err local tag, length, packet, rsp_body packet, err = s:receive(consts.T1K_HEADER_SIZE) if err then err = fmt("failed to receive info packet from t1k server %s: %s", srv, err) return nil, err, nil end if not packet then err = fmt("empty packet from t1k server %s", srv) return nil, err, nil end if ft then if not utils.is_mask_first(byte(packet, 1, 1)) then err = fmt("first packet is not MASK_FIRST from t1k server %s", srv) return nil, err, nil end ft = false end finished, tag, length = utils.packet_parser(packet) if length > 0 then rsp_body, err = s:receive(length) if not rsp_body or #rsp_body ~= length then err = fmt("failed to receive payload from t1k server %s: %s", srv, err) return nil, err, nil end t[tag] = rsp_body end until (finished) return true, nil, t end local function get_socket(opts) local ok, err local count, sock, server sock, err = ngx_socket.tcp() if not sock then err = fmt("failed to create socket: %s", err) return nil, err, nil, nil end sock:settimeouts(opts.connect_timeout, opts.send_timeout, opts.read_timeout) if opts.uds then server = opts.host ok, err = sock:connect(opts.host) else server = fmt("%s:%d", opts.host, opts.port) ok, err = sock:connect(opts.host, opts.port) end if not ok then sock:close() err = fmt("failed to connect to t1k server %s: %s", server, err) return ok, err, nil, nil end nlog(debug_fmt("successfully connected to t1k server %s", server)) count, err = sock:getreusedtimes() if not count then nlog(warn_fmt("failed to get reused times from t1k server %s: %s", server, err)) end if count == 0 then ok, err = sock:setoption("keepalive", true) if not ok then nlog(warn_fmt("failed to set keepalive for t1k server %s: %s", server, err)) end ok, err = sock:setoption("reuseaddr", true) if not ok then nlog(warn_fmt("failed to set reuseaddr for t1k server %s: %s", server, err)) end ok, err = sock:setoption("tcp-nodelay", true) if not ok then nlog(warn_fmt("failed to set tcp-nodelay for t1k server %s: %s", server, err)) end end return true, nil, sock, server end local function do_socket(opts, header, body, extra) local ok, err local t, sock, server ok, err, sock, server = get_socket(opts) if not ok then err = fmt("failed to get socket: %s", err) return ok, err, nil end ok, err = do_send(sock, { char(TAG_HEAD_WITH_MASK_FIRST), utils.int_to_char_length(header:len()), header }) if not ok then sock:close() err = fmt("failed to send header data to t1k server %s: %s", server, err) return ok, err, nil end if body ~= nil then ok, err = do_send(sock, { char(consts.TAG_BODY), utils.int_to_char_length(body:len()), body }) if not ok then sock:close() err = fmt("failed to send body data to t1k server %s: %s", server, err) return ok, err, nil end end ok, err = do_send(sock, { T1K_PROTO_DATA, char(TAG_EXTRA_WITH_MASK_LAST), utils.int_to_char_length(extra:len()), extra }) if not ok then sock:close() err = fmt("failed to send extra data to t1k server %s: %s", server, err) return ok, err, nil end ok, err, t = receive_data(sock, server) if not ok then return ok, err, nil end ok, err = sock:setkeepalive(opts.keepalive_timeout, opts.keepalive_size) if not ok then nlog(warn_fmt("failed to set keepalive: %s", err)) sock:close() end return true, nil, t end function _M.do_request(opts) local ok, err local header, body, extra, t ok, err, header = build_header(opts) if not ok then return ok, err, nil end ok, err, body = build_body(opts) if not ok then return ok, err, nil end ok, err, extra = build_extra(opts) if not ok then return ok, err, nil end ok, err, t = do_socket(opts, header, body, extra) if not ok then return ok, err, nil end if opts.mode == consts.MODE_BLOCK then local extra_header = t[consts.TAG_EXTRA_HEADER] if extra_header then ngx.ctx.t1k_extra_header = extra_header end end local result = { action = t[consts.TAG_HEAD], status = t[consts.TAG_BODY], event_id = utils.get_event_id(t[consts.TAG_EXTRA_BODY]), } return true, nil, result end return _M ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/utils.lua ================================================ local consts = require "resty.t1k.constants" local _M = { _VERSION = '1.0.0' } local band = bit.band local bnot = bit.bnot local lshift = bit.lshift local rshift = bit.rshift local abs = math.abs local char = string.char local ngx = ngx local ngx_re = ngx.re local re_match = ngx_re.match local re_gmatch = ngx_re.gmatch local re_gsub = ngx_re.gsub local NOT_MASK_FIRST = bnot(consts.MASK_FIRST) local NOT_MASK_LAST = bnot(consts.MASK_LAST) function _M.int_to_char_length(x) return char(band(x, 0xff)) .. char(band(rshift(x, 8), 0xff)) .. char(band(rshift(x, 16), 0xff)) .. char(band(rshift(x, 24), 0xff)) end function _M.char_to_int_length(l) return l:byte(1, 1) + lshift(l:byte(2, 2), 8) + lshift(l:byte(3, 3), 16) + lshift(l:byte(4, 4), 24) end function _M.is_mask_first(b) return band(b, consts.MASK_FIRST) == consts.MASK_FIRST end function _M.is_mask_last(b) return band(b, consts.MASK_LAST) == consts.MASK_LAST end local function tag_parser(tag) return band(band(tag, NOT_MASK_FIRST), NOT_MASK_LAST) end function _M.packet_parser(header) if #header ~= consts.T1K_HEADER_SIZE then return true, nil, 0 end local fb = header:byte(1, 1) local finished = _M.is_mask_last(fb) local tag = tag_parser(fb) local length = _M.char_to_int_length(header:sub(2, 5)) return finished, tag, length end function _M.starts_with(str, start) return str:sub(1, #start) == start end function _M.to_var_idx(o) local var = o local idx local _, p = o:find(":") if p then var = o:sub(1, p - 1) idx = tonumber(o:sub(p + 1)) end var = re_gsub(var:lower(), "-", "_") if not _M.starts_with(var, consts.NGX_HTTP_HEADER_PREFIX) then var = consts.NGX_HTTP_HEADER_PREFIX .. var end return var, idx end function _M.get_indexed_element(str, idx) if not str or not idx or idx == 0 then return str end local it, err = re_gmatch(str, [[([^,\s]+)]], "jo") if err then return nil end local t = {} for m, e in it do if e then return nil end table.insert(t, m[1]) end local len = #t if len < abs(idx) then return nil end return t[idx > 0 and idx or (len + idx + 1)] end function _M.get_event_id(str) if not str then return nil end local m, err = re_match(str, [[]], "jo") if err then return nil end if m then return m[1] end return nil end return _M ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k/uuid.lua ================================================ local bit = require "bit" local ffi = require "ffi" local log = require "resty.t1k.log" local _M = { _VERSION = '1.0.0' } local C = ffi.C local N_BYTES = 32 local random = math.random local nlog = ngx.log local warn_fmt = log.warn_fmt ffi.cdef [[ int RAND_bytes(unsigned char *buf, int num); ]] local function _rand_bytes(buf, len) return C.RAND_bytes(buf, len) end local function rand_bytes(len) local buf = ffi.new("char[?]", len) local ok, ret = pcall(_rand_bytes, buf, len) if not ok or ret ~= 1 then nlog(warn_fmt("call RAND_bytes failed: %s", ret)) return nil end return ffi.string(buf, len) end do local band = bit.band local bor = bit.bor local tohex = bit.tohex local fmt = string.format local byte = string.byte function _M.generate_v4() local bytes = rand_bytes(N_BYTES) -- fallback to math.random based method if not bytes then return (fmt('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s', tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(bor(band(random(0, 255), 0x0F), 0x40), 2), tohex(random(0, 255), 2), tohex(bor(band(random(0, 255), 0x3F), 0x80), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2), tohex(random(0, 255), 2))) end return fmt('%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s', tohex(byte(bytes, 1, 2), 2), tohex(byte(bytes, 3, 4), 2), tohex(byte(bytes, 5, 6), 2), tohex(byte(bytes, 7, 8), 2), tohex(byte(bytes, 9, 10), 2), tohex(byte(bytes, 11, 12), 2), tohex(bor(band(byte(bytes, 13, 14), 0x0F), 0x40), 2), tohex(byte(bytes, 15, 16), 2), tohex(bor(band(byte(bytes, 17, 18), 0x3F), 0x80), 2), tohex(byte(bytes, 19, 20), 2), tohex(byte(bytes, 21, 22), 2), tohex(byte(bytes, 23, 24), 2), tohex(byte(bytes, 25, 26), 2), tohex(byte(bytes, 27, 28), 2), tohex(byte(bytes, 29, 30), 2), tohex(byte(bytes, 31, 32), 2)) end end return setmetatable(_M, { __call = _M.generate_v4 }) ================================================ FILE: sdk/lua-resty-t1k/lib/resty/t1k.lua ================================================ local consts = require "resty.t1k.constants" local filter = require "resty.t1k.filter" local handler = require "resty.t1k.handler" local log = require "resty.t1k.log" local request = require "resty.t1k.request" local utils = require "resty.t1k.utils" local lower = string.lower local ngx = ngx local nlog = ngx.log local log_fmt = log.fmt local debug_fmt = log.debug_fmt local _M = { _VERSION = '1.0.0' } local DEFAULT_T1K_CONNECT_TIMEOUT = 1000 -- 1s local DEFAULT_T1K_SEND_TIMEOUT = 1000 -- 1s local DEFAULT_T1K_READ_TIMEOUT = 1000 -- 1s local DEFAULT_T1K_REQ_BODY_SIZE = 1024 -- 1024 KB local DEFAULT_T1K_KEEPALIVE_SIZE = 256 local DEFAULT_T1K_KEEPALIVE_TIMEOUT = 60 * 1000 -- 60s function _M.do_access(t, handle) local ok, err, result local opts = {} t = t or {} if not t.mode then return true, nil, nil end opts.mode = lower(t.mode) if opts.mode == consts.MODE_OFF then nlog(debug_fmt("t1k is not enabled")) return true, nil, nil end if opts.mode ~= consts.MODE_OFF and opts.mode ~= consts.MODE_BLOCK and opts.mode ~= consts.MODE_MONITOR then err = log_fmt("invalid t1k mode: %s", t.mode) return nil, err, nil end if not t.host then err = log_fmt("invalid t1k host: %s", t.host) return nil, err, nil end opts.host = t.host if utils.starts_with(opts.host, consts.UNIX_SOCK_PREFIX) then opts.uds = true else if not tonumber(t.port) then err = log_fmt("invalid t1k port: %s", t.port) return nil, err, nil end opts.port = tonumber(t.port) end opts.connect_timeout = t.connect_timeout or DEFAULT_T1K_CONNECT_TIMEOUT opts.send_timeout = t.send_timeout or DEFAULT_T1K_SEND_TIMEOUT opts.read_timeout = t.read_timeout or DEFAULT_T1K_READ_TIMEOUT opts.req_body_size = t.req_body_size or DEFAULT_T1K_REQ_BODY_SIZE opts.keepalive_size = t.keepalive_size or DEFAULT_T1K_KEEPALIVE_SIZE opts.keepalive_timeout = t.keepalive_timeout or DEFAULT_T1K_KEEPALIVE_TIMEOUT if t.remote_addr then local var, idx = utils.to_var_idx(t.remote_addr) opts.remote_addr_var = var opts.remote_addr_idx = idx end ok, err, result = request.do_request(opts) if not ok then return ok, err, result end if handle and opts.mode == consts.MODE_BLOCK then ok, err = _M.do_handle(result) end return ok, err, result end function _M.do_handle(t) local ok, err = handler.handle(t) return ok, err end function _M.do_header_filter() filter.do_header_filter() end return _M ================================================ FILE: sdk/lua-resty-t1k/mainspec/lua-resty-t1k-main-0-0.rockspec ================================================ package = "lua-resty-t1k-main" version = "0-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", branch = "main", } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.0-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.0.0-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.0.0" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.1-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.0.1-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.0.1" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.2-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.0.2-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.0.2" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.0.3-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.0.3-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.0.3" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.0-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.1.0-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.1.0" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.1-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.1.1-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.1.1" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.2-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.1.2-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.1.2" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.3-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.1.3-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.1.3" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.4-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.1.4-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.1.4" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/rockspec/lua-resty-t1k-1.1.5-0.rockspec ================================================ package = "lua-resty-t1k" version = "1.1.5-0" source = { url = "git://github.com/chaitin/lua-resty-t1k", tag = "v1.1.5" } description = { summary = "Lua implementation of the T1K protocol for Chaitin SafeLine Web Application Firewall", detailed = [[ Check https://waf-ce.chaitin.cn/ for more information about Chaitin SafeLine Web Application Firewall. ]], homepage = "https://github.com/chaitin/lua-resty-t1k", license = "Apache License 2.0", maintainer = "Xudong Wang " } build = { type = "builtin", modules = { ["resty.t1k"] = "lib/resty/t1k.lua", ["resty.t1k.buffer"] = "lib/resty/t1k/buffer.lua", ["resty.t1k.constants"] = "lib/resty/t1k/constants.lua", ["resty.t1k.file"] = "lib/resty/t1k/file.lua", ["resty.t1k.filter"] = "lib/resty/t1k/filter.lua", ["resty.t1k.handler"] = "lib/resty/t1k/handler.lua", ["resty.t1k.log"] = "lib/resty/t1k/log.lua", ["resty.t1k.request"] = "lib/resty/t1k/request.lua", ["resty.t1k.utils"] = "lib/resty/t1k/utils.lua", ["resty.t1k.uuid"] = "lib/resty/t1k/uuid.lua", }, } ================================================ FILE: sdk/lua-resty-t1k/t/buffer.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 3); run_tests(); __DATA__ === TEST 1: buffer add --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local buffer = require "resty.t1k.buffer" local b = buffer:new() b:add("hello") b:add(" ") b:add("world") b:add("!") ngx.say(b[1], b[2], b[3], b[4]) ngx.say(b:len()) } } --- request GET /t --- response_body hello world! 12 --- no_error_log [error] ================================================ FILE: sdk/lua-resty-t1k/t/file.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 3); run_tests(); __DATA__ === TEST 1: read full file --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local file = require "resty.t1k.file" local path = ngx.var.document_root .. "/foo.bar" local ok, err, content = file.read(path, 2 ^ 15) if not ok then ngx.say(err) end ngx.print(table.concat(content)) } } --- user_files eval [ ["foo.bar" => "a" x (2 ** 14) ], ] --- request GET /t --- response_body eval "a" x (2 ** 14) --- no_error_log [error] === TEST 2: read partial file --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local file = require "resty.t1k.file" local path = ngx.var.document_root .. "/foo.bar" local ok, err, content = file.read(path, 2 ^ 13) if not ok then ngx.say(err) end ngx.print(table.concat(content)) } } --- user_files eval [ ["foo.bar" => "a" x (2 ** 14) ], ] --- request GET /t --- response_body eval "a" x (2 ** 13) --- no_error_log [error] === TEST 3: read negative bytes --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local file = require "resty.t1k.file" local path = ngx.var.document_root .. "/foo.bar" local ok, err, content = file.read(path, -1) if not ok then ngx.say(err) end ngx.print(table.concat(content)) } } --- user_files >>> foo.bar --- request GET /t --- response_body eval "" --- no_error_log [error] === TEST 4: read empty file --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local file = require "resty.t1k.file" local path = ngx.var.document_root .. "/foo.bar" local ok, err, content = file.read(path, 1) if not ok then ngx.say(err) end ngx.print(table.concat(content)) } } --- user_files >>> foo.bar --- request GET /t --- response_body eval "" --- no_error_log [error] === TEST 5: read non-existent file --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local file = require "resty.t1k.file" local ok, err, buffer = file.read("/opt/non_existent_file", 0) if not ok then ngx.say(err) end } } --- request GET /t --- response_body /opt/non_existent_file: No such file or directory --- no_error_log [error] ================================================ FILE: sdk/lua-resty-t1k/t/filter.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 5); run_tests(); __DATA__ === TEST 1: do_header_filter --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { ngx.ctx.t1k_extra_header = "k1:v1\nk2:v2\nk3:v3\n" } header_filter_by_lua_block { local filter = require "resty.t1k.filter" filter.do_header_filter() } content_by_lua_block { ngx.say("hi") } } --- request GET /t --- response_headers k1: v1 k2: v2 k3: v3 --- no_error_log [error] ================================================ FILE: sdk/lua-resty-t1k/t/handler.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 3); run_tests(); __DATA__ === TEST 1: handle passed action --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local handler = require "resty.t1k.handler" local t = { action = ".", } local ok, err = handler.handle(t) if not ok then ngx.log(ngx.ERR, err) end } content_by_lua_block { ngx.say("passed") } } --- request GET /t --- response_body passed --- no_error_log [error] === TEST 2: handle blocked action --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local handler = require "resty.t1k.handler" local t = { action = "?", status = 405, event_id = "c0c039a7c348486eaffd9e2f9846b66b", } local ok, err = handler.handle(t) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local filter = require "resty.t1k.filter" filter.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- request GET /t --- response_body {"code": 405, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "c0c039a7c348486eaffd9e2f9846b66b"} --- error_code eval "405" --- no_error_log [error] --- no_error_log [error] === TEST 3: handle unknown action --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local handler = require "resty.t1k.handler" local t = { action = "~" } local ok, err = handler.handle(t) if not ok then ngx.log(ngx.ERR, err) end } content_by_lua_block { ngx.say("passed") } } --- request GET /t --- response_body passed --- error_log lua-resty-t1k: unknown action from t1k server: ~ === TEST 4: handle nil result --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local handler = require "resty.t1k.handler" local ok, err = handler.handle(nil) if not ok then ngx.log(ngx.ERR, err) end } content_by_lua_block { ngx.say("passed") } } --- request GET /t --- response_body passed --- error_log lua-resty-t1k: invalid result type: nil ================================================ FILE: sdk/lua-resty-t1k/t/integration.t ================================================ use Test::Nginx::Socket 'no_plan'; our $MainConfig = <<'_EOC_'; env DETECTOR_IP; _EOC_ our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; lua_socket_log_errors off; _EOC_ run_tests(); __DATA__ === TEST 1: integration test blocked --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- request GET /t/shell.php --- response_headers Content-Type: application/json --- response_body_like eval '^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$' --- error_code: 403 --- no_error_log [error] --- error_log eval "lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" --- log_level: debug --- skip_eval 4: not exists($ENV{DETECTOR_IP}) === TEST 2: integration test blocked internal handle --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t, true) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- request GET /t/shell.php --- response_headers Content-Type: application/json --- response_body_like eval '^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$' --- error_code: 403 --- no_error_log [error] --- error_log eval "lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" --- log_level: debug --- skip_eval 4: not exists($ENV{DETECTOR_IP}) === TEST 3: integration test blocked http2 --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- http2 --- request GET /t/shell.php --- response_headers Content-Type: application/json --- response_body_like eval '^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$' --- error_code: 403 --- no_error_log [error] --- error_log eval "lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" --- log_level: debug --- skip_eval 4: not exists($ENV{DETECTOR_IP}) === TEST 4: integration test blocked http2 internal handle --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t, true) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- http2 --- request GET /t/shell.php --- response_headers Content-Type: application/json --- response_body_like eval '^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$' --- error_code: 403 --- no_error_log [error] --- error_log eval "lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" --- log_level: debug --- skip_eval 4: not exists($ENV{DETECTOR_IP}) === TEST 5: integration test monitor --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "monitor", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- request GET /t/shell.php --- response_body passed --- no_error_log [error] --- error_log eval "lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" --- log_level: debug --- skip_eval 4: not exists($ENV{DETECTOR_IP}) === TEST 6: integration test monitor internal handle --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "monitor", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t, true) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- request GET /t/shell.php --- response_body passed --- no_error_log [error] --- error_log eval "lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" --- log_level: debug --- skip_eval 4: not exists($ENV{DETECTOR_IP}) === TEST 7: integration test monitor http2 --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "monitor", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- http2 --- request GET /t/shell.php --- response_body passed --- no_error_log [error] --- error_log eval ["lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000", "skip blocking"] --- log_level: debug --- skip_eval 4: not exists($ENV{DETECTOR_IP}) === TEST 8: integration test monitor http2 internal handle --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "monitor", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t, true) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- http2 --- request GET /t/shell.php --- response_body passed --- no_error_log [error] --- error_log eval "lua-resty-t1k: successfully connected to t1k server $ENV{DETECTOR_IP}:8000" --- log_level: debug --- skip_eval 4: not exists($ENV{DETECTOR_IP}) === TEST 9: integration test disabled --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "off", } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- request GET /t/shell.php --- response_body passed --- no_error_log [error] --- error_log lua-resty-t1k: t1k is not enabled skip blocking --- log_level: debug === TEST 10: integration test configuration priority --- main_config eval: $::MainConfig --- http_config eval: $::HttpConfig --- config access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = os.getenv("DETECTOR_IP"), port = 8000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } location /pass { access_by_lua_block { } content_by_lua_block { ngx.say("passed") } } location /block { content_by_lua_block { ngx.say("there must be a problem when you see this line") } } --- request eval ["GET /pass/shell.php", "GET /block/shell.php"] --- response_body_like eval ["passed", '^{"code": 403, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": ".*"}$'] --- error_code eval [200, 403] --- no_error_log [error] --- skip_eval 6: not exists($ENV{DETECTOR_IP}) === TEST 11: integration test blocked extra headers --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: 18000 --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- response_headers k1: v1 k2: v2 k3: v3 --- response_headers Content-Type: application/json --- response_body {"code": 405, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "c0c039a7c348486eaffd9e2f9846b66b"} --- error_code eval "405" --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 12: integration test blocked extra headers internal handle --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t, true) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: 18000 --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- response_headers k1: v1 k2: v2 k3: v3 --- response_headers Content-Type: application/json --- response_body {"code": 405, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "c0c039a7c348486eaffd9e2f9846b66b"} --- error_code eval "405" --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 13: integration test blocked extra headers with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = "unix:t1k.sock", connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: t1k.sock --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- response_headers k1: v1 k2: v2 k3: v3 --- response_headers Content-Type: application/json --- response_body {"code": 405, "success":false, "message": "blocked by Chaitin SafeLine Web Application Firewall", "event_id": "c0c039a7c348486eaffd9e2f9846b66b"} --- error_code eval "405" --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server unix:t1k.sock --- log_level: debug === TEST 14: integration test passed extra headers --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: 18000 --- tcp_reply eval "\x41\x01\x00\x00\x00.\xa3\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a" --- request GET /t --- response_headers k1: v1 k2: v2 k3: v3 --- response_body passed --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 15: integration test passed extra headers internal handle --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t, true) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: 18000 --- tcp_reply eval "\x41\x01\x00\x00\x00.\xa3\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a" --- request GET /t --- response_headers k1: v1 k2: v2 k3: v3 --- response_body passed --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 16: integration test passed extra headers with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = "unix:t1k.sock", connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: t1k.sock --- tcp_reply eval "\x41\x01\x00\x00\x00.\xa3\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a" --- request GET /t --- response_headers k1: v1 k2: v2 k3: v3 --- response_body passed --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server unix:t1k.sock --- log_level: debug === TEST 17: integration test monitor extra headers --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "monitor", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: 18000 --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- raw_response_headers_unlike eval '.*k1: v1\r\n.*' --- response_body passed --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 skip blocking --- log_level: debug === TEST 18: integration test monitor extra headers internal handle --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "monitor", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t, true) if not ok then ngx.log(ngx.ERR, err) end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: 18000 --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- raw_response_headers_unlike eval '.*k1: v1\r\n.*' --- response_body passed --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 19: integration test monitor extra headers with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "monitor", host = "unix:t1k.sock", connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end if t.mode ~= "block" then ngx.log(ngx.DEBUG, "skip blocking") return end ok, err = t1k.do_handle(result) if not ok then ngx.log(ngx.ERR, err) return end } header_filter_by_lua_block { local t1k = require "resty.t1k" t1k.do_header_filter() } content_by_lua_block { ngx.say("passed") } } --- tcp_listen: t1k.sock --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\x23\x12\x00\x00\x00k1:v1\x0ak2:v2\x0ak3:v3\x0a\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- raw_response_headers_unlike eval '.*k1: v1\r\n.*' --- response_body passed --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server unix:t1k.sock skip blocking --- log_level: debug ================================================ FILE: sdk/lua-resty-t1k/t/log.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 2 + 2); run_tests(); __DATA__ === TEST 1: err_fmt --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local log = require "resty.t1k.log" ngx.log(log.err_fmt("%s - %04d - %.4f", "test", 1, 1)) } } --- request GET /t [error] --- error_log lua-resty-t1k: test - 0001 - 1.0000 --- log_level: error === TEST 2: warn_fmt --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local log = require "resty.t1k.log" ngx.log(log.warn_fmt("%s - %04d - %.4f", "test", 1, 1)) } } --- request GET /t --- no_error_log [error] --- error_log lua-resty-t1k: test - 0001 - 1.0000 --- log_level: warn === TEST 3: debug_fmt --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local log = require "resty.t1k.log" ngx.log(log.debug_fmt("%s - %04d - %.4f", "test", 1, 1)) } } --- request GET /t --- no_error_log [error] --- error_log lua-resty-t1k: test - 0001 - 1.0000 --- log_level: debug ================================================ FILE: sdk/lua-resty-t1k/t/option.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 3 + 1); run_tests(); __DATA__ === TEST 1: do_access nil option --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local ok, err, _ = t1k.do_access(nil) if not ok then ngx.log(ngx.ERR, err) return end } content_by_lua_block { ngx.say("passed") } } --- request GET /t/shell.php --- response_body passed --- no_error_log [debug] --- log_level: debug === TEST 2: do_access disabled --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "off", } local ok, err, _ = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end } content_by_lua_block { ngx.say("passed") } } --- request GET /t/shell.php --- response_body passed --- no_error_log [error] --- error_log lua-resty-t1k: t1k is not enabled --- log_level: debug === TEST 3: do_access invalid mode --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "invalid", } local ok, err, _ = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end } content_by_lua_block { ngx.say("passed") } } --- request GET /t/shell.php --- response_body passed --- error_log lua-resty-t1k: invalid t1k mode: invalid === TEST 4: do_access invalid host --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block" } local ok, err, _ = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end } content_by_lua_block { ngx.say("passed") } } --- request GET /t --- response_body passed --- error_log lua-resty-t1k: invalid t1k host: nil --- log_level: debug === TEST 5: do_access invalid port --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local t1k = require "resty.t1k" local t = { mode = "block", host = "127.0.0.1" } local ok, err, _ = t1k.do_access(t) if not ok then ngx.log(ngx.ERR, err) return end } content_by_lua_block { ngx.say("passed") } } --- request GET /t --- response_body passed --- error_log lua-resty-t1k: invalid t1k port: nil --- log_level: debug ================================================ FILE: sdk/lua-resty-t1k/t/request.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 3 + 14); run_tests(); __DATA__ === TEST 1: do_request blocked --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say(result["action"]) ngx.say(result["status"]) ngx.say(result["event_id"]) } } --- tcp_listen: 18000 --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- response_body ? 405 c0c039a7c348486eaffd9e2f9846b66b --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 2: do_request blocked with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "unix:t1k.sock", uds = true, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say(result["action"]) ngx.say(result["status"]) ngx.say(result["event_id"]) } } --- tcp_listen: t1k.sock --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- response_body ? 405 c0c039a7c348486eaffd9e2f9846b66b --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server unix:t1k.sock --- log_level: debug === TEST 3: do_request passed --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say(result["action"]) } } --- tcp_listen: 18000 --- tcp_reply eval "\xc1\x01\x00\x00\x00." --- request GET /t --- response_body . --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 4: do_request passed with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "unix:t1k.sock", uds = true, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say(result["action"]) } } --- tcp_listen: t1k.sock --- tcp_reply eval "\xc1\x01\x00\x00\x00." --- request GET /t --- response_body . --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server unix:t1k.sock --- log_level: debug === TEST 5: do_request trim request body --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 0.0625, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say(result["action"]) } } --- tcp_listen: 18000 --- tcp_reply eval "\xc1\x01\x00\x00\x00." --- request GET /t Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. --- response_body . --- no_error_log [error] --- error_log lua-resty-t1k: request body is too long: 123 bytes, cut to 64 bytes lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 6: do_request trim request body with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "unix:t1k.sock", uds = true, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 0.0625, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say(result["action"]) } } --- tcp_listen: t1k.sock --- tcp_reply eval "\xc1\x01\x00\x00\x00." --- request GET /t Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. --- response_body . --- no_error_log [error] --- error_log lua-resty-t1k: request body is too long: 123 bytes, cut to 64 bytes lua-resty-t1k: successfully connected to t1k server unix:t1k.sock --- log_level: debug === TEST 7: do_request refuse connection --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 0.0625, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say("result: ", result) } } --- request GET /t --- response_body result: nil --- error_log failed to connect to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 8: do_request refuse connection with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "unix:t1k.sock", uds = true, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 0.0625, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say("result: ", result) } } --- request GET /t --- response_body result: nil --- error_log failed to connect to t1k server unix:t1k.sock --- log_level: debug === TEST 9: do_request timeout --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 100, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say("result: ", result) } } --- tcp_listen: 18000 --- tcp_reply_delay: 200ms --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- response_body result: nil --- error_log failed to receive info packet from t1k server 127.0.0.1:18000: timeout --- log_level: debug === TEST 10: do_request timeout with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "unix:t1k.sock", uds = true, connect_timeout = 1000, send_timeout = 1000, read_timeout = 100, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say("result: ", result) } } --- tcp_listen: t1k.sock --- tcp_reply_delay: 200ms --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" --- request GET /t/shell.php --- response_body result: nil --- error_log failed to receive info packet from t1k server unix:t1k.sock: timeout --- log_level: debug === TEST 11: do_request invalid action --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 0.0625, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say("action: ", result["action"]) } } --- tcp_listen: 18000 --- tcp_reply eval "\xc1\x01\x00\x00\x00~" --- request GET /t --- response_body action: ~ --- no_error_log [error] --- error_log successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 12: do_request invalid action with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "unix:t1k.sock", uds = true, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 0.0625, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say("action: ", result["action"]) } } --- tcp_listen: t1k.sock --- tcp_reply eval "\xc1\x01\x00\x00\x00~" --- request GET /t --- response_body action: ~ --- no_error_log [error] --- error_log successfully connected to t1k server unix:t1k.sock --- log_level: debug === TEST 13: do_request remote address --- http_config eval: $::HttpConfig --- config location /t { access_by_lua_block { local utils = require "resty.t1k.utils" ngx.say("ngx.var.http_x_real_ip or ngx.var.remote_addr is ", utils.get_indexed_element(ngx.var.http_x_real_ip) or ngx.var.remote_addr) ngx.say("ngx.var.http_x_forwarded_for: 2 or ngx.var.remote_addr is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 2) or ngx.var.remote_addr) ngx.say("ngx.var.http_x_forwarded_for: -2 or ngx.var.remote_addr is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -2) or ngx.var.remote_addr) ngx.say("ngx.var.http_non_existent_header or ngx.var.remote_addr is ", utils.get_indexed_element(ngx.var.http_non_existent_header) or ngx.var.remote_addr) } } --- request GET /t --- more_headers X-Forwarded-For: 1.1.1.1, 2.2.2.2, 2001:db8:3333:4444:5555:6666:7777:8888, 3.3.3.3 X-Real-IP: 100.100.100.100 --- response_body ngx.var.http_x_real_ip or ngx.var.remote_addr is 100.100.100.100 ngx.var.http_x_forwarded_for: 2 or ngx.var.remote_addr is 2.2.2.2 ngx.var.http_x_forwarded_for: -2 or ngx.var.remote_addr is 2001:db8:3333:4444:5555:6666:7777:8888 ngx.var.http_non_existent_header or ngx.var.remote_addr is 127.0.0.1 --- no_error_log [error] === TEST 14: do_request http2 --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "127.0.0.1", port = 18000, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say(result["action"]) ngx.say(result["status"]) ngx.say(result["event_id"]) } } --- tcp_listen: 18000 --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" --- http2 --- request GET /t/shell.php --- tcp_query eval qr/.*HTTP\/2.0.*/ --- response_body ? 405 c0c039a7c348486eaffd9e2f9846b66b --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server 127.0.0.1:18000 --- log_level: debug === TEST 15: do_request http2 with unix domain socket --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local request = require "resty.t1k.request" local t = { mode = "block", host = "unix:t1k.sock", uds = true, connect_timeout = 1000, send_timeout = 1000, read_timeout = 1000, req_body_size = 1024, keepalive_size = 16, keepalive_timeout = 10000, } local ok, err, result = request.do_request(t) if not ok then ngx.log(ngx.ERR, err) end ngx.say(result["action"]) ngx.say(result["status"]) ngx.say(result["event_id"]) } } --- tcp_listen: t1k.sock --- tcp_reply eval "\x41\x01\x00\x00\x00?\x02\x03\x00\x00\x00405\xa4\x33\x00\x00\x00" --- http2 --- request GET /t/shell.php --- tcp_query eval qr/.*HTTP\/2.0.*/ --- response_body ? 405 c0c039a7c348486eaffd9e2f9846b66b --- no_error_log [error] --- error_log lua-resty-t1k: successfully connected to t1k server unix:t1k.sock --- log_level: debug ================================================ FILE: sdk/lua-resty-t1k/t/utils.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 3); run_tests(); __DATA__ === TEST 1: int_to_char_length --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" ngx.say("255 to char length: ", utils.int_to_char_length(255)) ngx.print("16777216 to char length: ", utils.int_to_char_length(16777216)) } } --- request GET /t --- response_body eval "255 to char length: \x{ff}\x{00}\x{00}\x{00} 16777216 to char length: \x{00}\x{00}\x{00}\x{01}" --- no_error_log [error] === TEST 2: int_to_char_length --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" ngx.say("0xff 0x00 0x00 0x00 to int length: ", utils.char_to_int_length("\xff\x00\x00\x00")) ngx.say("0x00 0x00 0x00 0x01 to int length: ", utils.char_to_int_length("\x00\x00\x00\x01")) } } --- request GET /t --- response_body 0xff 0x00 0x00 0x00 to int length: 255 0x00 0x00 0x00 0x01 to int length: 16777216 --- no_error_log [error] === TEST 3: is_mask_first --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" ngx.say("0x30 is MASK FIRST: ", utils.is_mask_first(0x30)) ngx.say("0x41 is MASK FIRST: ", utils.is_mask_first(0x41)) ngx.say("0xc0 is MASK FIRST: ", utils.is_mask_first(0xc0)) ngx.say("0xc1 is MASK FIRST: ", utils.is_mask_first(0xc1)) } } --- request GET /t --- response_body 0x30 is MASK FIRST: false 0x41 is MASK FIRST: true 0xc0 is MASK FIRST: true 0xc1 is MASK FIRST: true --- no_error_log [error] === TEST 4: is_mask_last --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" ngx.say("0x41 is MASK LAST: ", utils.is_mask_last(65)) ngx.say("0x80 is MASK LAST: ", utils.is_mask_last(128)) } } --- request GET /t --- response_body 0x41 is MASK LAST: false 0x80 is MASK LAST: true --- no_error_log [error] === TEST 5: packet_parser unfinished --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" local finished, tag, length = utils.packet_parser("\x41\x59\x00\x00\x00") ngx.say("finished: ", finished) ngx.say("tag == TAG_HEAD: ", tag == 1) ngx.say("length: ", 89) } } --- request GET /t --- response_body finished: false tag == TAG_HEAD: true length: 89 --- no_error_log [error] === TEST 6: packet_parser finished --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" local finished, tag, length = utils.packet_parser("\xa0\x08\x00\x00\x00") ngx.say("finished: ", finished) ngx.say("tag == TAG_VERSION: ", tag == 32) ngx.say("length: ", 8) } } --- request GET /t --- response_body finished: true tag == TAG_VERSION: true length: 8 --- no_error_log [error] === TEST 7: starts_with --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" ngx.say("http://www.baidu.com starts with \"http\": ", utils.starts_with("http://www.baidu.com", "http")) ngx.say("http://www.baidu.com starts with \"https\": ", utils.starts_with("http://www.baidu.com", "https")) } } --- request GET /t --- response_body http://www.baidu.com starts with "http": true http://www.baidu.com starts with "https": false --- no_error_log [error] === TEST 8: to_var_idx --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" local fmt = string.format ngx.say(fmt("http_x_real_ip is %s and %s", utils.to_var_idx("http_x_real_ip"))) ngx.say(fmt("X_REAL_IP is %s and %s", utils.to_var_idx("X_REAL_IP"))) ngx.say(fmt("X-Forwarded-For: 1 is %s and %d", utils.to_var_idx("X-Forwarded-For: 1"))) ngx.say(fmt("X-Forwarded-For: -1 is %s and %d", utils.to_var_idx("X-Forwarded-For: -1"))) ngx.say(fmt("X-FORWARDED-FOR:100 is %s and %d", utils.to_var_idx("X-FORWARDED-FOR:-100"))) } } --- request GET /t --- response_body http_x_real_ip is http_x_real_ip and nil X_REAL_IP is http_x_real_ip and nil X-Forwarded-For: 1 is http_x_forwarded_for and 1 X-Forwarded-For: -1 is http_x_forwarded_for and -1 X-FORWARDED-FOR:100 is http_x_forwarded_for and -100 --- no_error_log [error] === TEST 9: get_indexed_element --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" ngx.say("X-Forwarded-For: 1 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 1)) ngx.say("X-Forwarded-For: 2 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 2)) ngx.say("X-Forwarded-For: 3 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 3)) ngx.say("X-Forwarded-For: 4 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 4)) ngx.say("X-Forwarded-For: 5 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 5)) ngx.say("X-Forwarded-For: -1 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -1)) ngx.say("X-Forwarded-For: -2 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -2)) ngx.say("X-Forwarded-For: -3 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -3)) ngx.say("X-Forwarded-For: -4 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -4)) ngx.say("X-Forwarded-For: -5 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, -5)) ngx.say("X-Forwarded-For: 0 is ", utils.get_indexed_element(ngx.var.http_x_forwarded_for, 0)) ngx.say("X-Non-Existent-Header is ", utils.get_indexed_element(ngx.var.http_non_existent_header)) ngx.say("X-Real-IP is ", utils.get_indexed_element(ngx.var.http_x_real_ip)) ngx.say("X-Real-IP: 1 is ", utils.get_indexed_element(ngx.var.http_x_real_ip, 1)) ngx.say("X-Real-IP: 2 is ", utils.get_indexed_element(ngx.var.http_x_real_ip, 2)) ngx.say("X-Real-IP: -1 is ", utils.get_indexed_element(ngx.var.http_x_real_ip, -1)) ngx.say("X-Real-IP: -2 is ", utils.get_indexed_element(ngx.var.http_x_real_ip, -2)) } } --- request GET /t --- more_headers X-Forwarded-For: 1.1.1.1, 2.2.2.2 X-Forwarded-For: 3.3.3.3, 4.4.4.4 X-Real-IP: 10.10.10.10 --- response_body X-Forwarded-For: 1 is 1.1.1.1 X-Forwarded-For: 2 is 2.2.2.2 X-Forwarded-For: 3 is 3.3.3.3 X-Forwarded-For: 4 is 4.4.4.4 X-Forwarded-For: 5 is nil X-Forwarded-For: -1 is 4.4.4.4 X-Forwarded-For: -2 is 3.3.3.3 X-Forwarded-For: -3 is 2.2.2.2 X-Forwarded-For: -4 is 1.1.1.1 X-Forwarded-For: -5 is nil X-Forwarded-For: 0 is 1.1.1.1, 2.2.2.2, 3.3.3.3, 4.4.4.4 X-Non-Existent-Header is nil X-Real-IP is 10.10.10.10 X-Real-IP: 1 is 10.10.10.10 X-Real-IP: 2 is nil X-Real-IP: -1 is 10.10.10.10 X-Real-IP: -2 is nil --- no_error_log [error] === TEST 10: get_event_id --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local utils = require "resty.t1k.utils" ngx.say(utils.get_event_id("")) ngx.say(utils.get_event_id("")) ngx.say(utils.get_event_id("")) } } --- request GET /t --- response_body 0988987de04844c7a3ce6d27865c9513 8bae6adf33864c7f8bf715a9b7a65b2c nil --- no_error_log [error] ================================================ FILE: sdk/lua-resty-t1k/t/uuid.t ================================================ use Test::Nginx::Socket; our $HttpConfig = <<'_EOC_'; lua_package_path "lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; _EOC_ repeat_each(3); plan tests => repeat_each() * (blocks() * 3); run_tests(); __DATA__ === TEST 1: generate_v4 --- http_config eval: $::HttpConfig --- config location /t { content_by_lua_block { local uuid = require "resty.t1k.uuid" ngx.say(uuid.generate_v4()) } } --- request GET /t --- response_body_like ^[0-9a-f]{8}[0-9a-f]{4}4[0-9a-f]{3}[89ab][0-9a-f]{3}[0-9a-f]{12}$ --- no_error_log [error] ================================================ FILE: version.json ================================================ { "latest_version": "v9.2.7", "rec_version": "v9.2.7", "lts_version": "v9.1.0-lts" } ================================================ FILE: yanshi/.gitignore ================================================ *.o *.dot *.output /build /src/parser.cc /src/parser.hh /src/lexer.cc /src/lexer.hh ================================================ FILE: yanshi/Makefile ================================================ CPPFLAGS := -g3 -std=c++1y -Isrc -I. -DHAVE_READLINE ifeq ($(build),release) BUILD := release CPPFLAGS += -Os else BUILD := build CPPFLAGS += -fsanitize=undefined,address -DDEBUG LDLIBS := -lasan -lubsan endif LDLIBS += -licuuc -lreadline SRC := $(filter-out src/lexer.cc src/parser.cc, $(wildcard src/*.cc)) src/lexer.cc src/parser.cc OBJ := $(addprefix $(BUILD)/,$(subst src/,,$(SRC:.cc=.o))) UNITTEST_SRC := $(wildcard unittest/*.cc) UNITTEST_EXE := $(subst unittest/,,$(UNITTEST_SRC:.cc=)) all: $(BUILD)/yanshi # unittest unittest: $(addprefix $(BUILD)/unittest/,$(UNITTEST_EXE)) $(foreach x,$(addprefix $(BUILD)/unittest/,$(UNITTEST_EXE)),$x && ) : sinclude $(OBJ:.o=.d) # FIXME $(BUILD)/repl.o: src/lexer.hh $(BUILD) $(BUILD)/unittest: mkdir -p $@ $(BUILD)/yanshi: $(OBJ) $(LINK.cc) $^ $(LDLIBS) -o $@ $(BUILD)/%.o: src/%.cc | $(BUILD) $(CXX) $(CPPFLAGS) -MM -MP -MT $@ -MF $(@:.o=.d) $< $(COMPILE.cc) $< -o $@ $(BUILD)/unittest/%: unittest/%.cc $(wildcard unittest/*.hh) $(filter-out $(BUILD)/main.o,$(OBJ)) | $(BUILD)/unittest $(CXX) $(CPPFLAGS) -MM -MP -MT $@ -MF $(@:.o=.d) $< $(LINK.cc) $(filter-out %.hh,$^) $(LDLIBS) -o $@ src/lexer.cc src/lexer.hh: src/lexer.l flex --header-file=src/lexer.hh -o src/lexer.cc $< src/parser.cc src/parser.hh: src/parser.y src/common.hh src/location.hh src/option.hh src/syntax.hh bison --defines=src/parser.hh -o src/parser.cc $< $(BUILD)/loader.o: src/parser.hh $(BUILD)/parser.o: src/lexer.hh $(BUILD)/lexer.o: src/parser.hh clean: $(RM) -r build release distclean: clean $(RM) src/{lexer,parser}.{cc,hh} .PHONY: all clean distclean ================================================ FILE: yanshi/README.md ================================================ # 偃师 (yanshi) yanshi is a finite state automaton generator like Ragel. Use inline operators to embed C++ code in the recognition of a language. yanshi is enhanced with features to approximate context-free grammar: - Approximation of substring grammar - Approximation of recursive automaton to match expressions with recursion. The motivation to create yanshi is that Ragel does not provide a mechanism to serialize its representation of finite state automata, making it difficult to post-process generated automata and obtain the substring grammar recognizer. Later on, I found a simplified SQL grammar might contain more than 10000 states. It was not only slow to generate the automaton, making it hard to do trial-and-error experiments, but wasted memory to store the automaton. I introduced `CollapseExpr` to allow circular references. `CallExpr` takes one step further, maintains a return address stack to imitate function calls. It can be seen as an augmented `CollapseExpr`, removing a lot of false positive cases. ## Name From :
In ancient China, a curious account of automata is found in the Lie Zi text (列子), written in the 3rd century BC. Within it there is a description of a much earlier encounter between King Mu of Zhou (周穆王, 1023-957 BC) and a mechanical engineer known as Yan Shi (偃师), an 'artificer'.
## Build * Debug: `make` * Release: `make build=release` ## Getting Started * Create file `a.ys`: ``` export foo = 'hello' ``` Run `yanshi -S a.ys -o /tmp/a.cc` to generate a C++ file from the yanshi source file `a.ys`. + `yanshi_foo_start`: the start state is 0. States are represented by natural numbers. + `yanshi_foo_is_final`: leave aside `ret_stack` and look at the last line, it checks whether `u` is one of the final states. + `yanshi_foo_transit`: leave aside `ret_stack`, `u` is the current state and `c` is the next input codepoint or label. With the `-S` option, yanshi will generate a standalone C++ file. ``` % make -C /tmp a make: Entering directory '/tmp' g++ a.cc -o a make: Leaving directory '/tmp' % /tmp/a hello 0 h 1 e 2 l 3 l 4 o 5 len: 5 pref: 5 state: 5 final: true % /tmp/a hello0 h 1 e 2 l 3 l 4 o 5 len: 5 pref: 5 state: 5 final: true ``` States are yellow and interleaved with transition labels. Final states are bold yellow. + `len`: the length of input codepoints or labels + `pref`: the length of the longest prefix that does not enter the dead state + `state`: the state entered after consuming the input + `final`: whether the state is one of final states * Interactive mode The `-i` option enables interactive mode. ``` % yanshi -i a.ys Testing foo foo :: DefineStmt .integer mode Commands available from the prompt: .automaton dump automaton .assoc dump associated AST Expr for each state .help display this help .integer input is a list of non-negative integers, macros(#define) or '' quoted strings .macro display defined macros .string input is a string .stmt change target DefineStmt to .quit exit interactive mode λ 104 101 108 108 111 0 104 1 101 2 108 3 108 4 111 5 export foo = 'hello': λ .string .string mode λ hello 0 h 1 e 2 l 3 l 4 o 5 export foo = 'hello': λ ``` * Regex-like syntax ``` export hello = [gh] 'e' l{2} 'o' l = 'l' ``` `[gh]` is a bracket expression and `l{2}` denotes to matches `l` at least twice. This grammar matches `hello`, `gello`, `helllo`, ... + Union: `c = a | b` + Intersection: `c = a && b` + Difference: `c = a - b` + Concatenation: `c = a b` + Complement: `c = ~ a` * Actions (embedded C++ code) ``` c++ { #include } export hello = '喵' @ { puts("meow"); } {2} ``` I have not thought clearly on the implementation. The executing point may be counter-intuitive. * Modules ``` # a.ys import 'b.ys' as B # B::bar import 'b.ys' # qux export foo = B::bar | qux bar = '4' # b.ys bar = '3' qux = '5' ``` * Substring grammar Specify the `--substring-grammar` option to generate code for substring grammar. That is, the generated code matches every substring of the grammar. The implementation creates a new start state and a new final state, connects the start state to the old start state, and old final states to the new final state. * `EmbedExpr`, reference a nonterminal without modifiers ``` foo = bar bar = [0-9] ``` The complete automaton of bar will be duplicated in each reference site. If the referenced automaton is large, `EmbedExpr` will significantly increase the number of states. `EmbedExpr` defines dependencies among states and no cyclic dependency is allowed. * `CollapseExpr`, reference a nonterminal with the `!` modifier ``` export foo = 'pre' !bar 'post' bar = [\u0300-\u034E] quz = 'meow' !bar 'meow' ``` The final state of `'pre'` and the start state of `'post'` will be connected by a special directed arc. When exporting, an epsilon transition will be added from the tail of the arc to the start state of `bar`, others will be added from the final states of `bar` to the head of the arc. `CollapseExpr` behaves like function calls, however, the return address is not preserved (hence the name `CollapseExpr`) and the state may go to other call sites. In this example, the state may go to either `foo` or `quz` after traveling through `bar`, causing false positives. * `CallExpr`, reference a nonterminal with the `&` modifier ``` export foo = 'pre' &bar 'post' bar = '4' ``` This is a refinement of `CollapseExpr`. Suppose state `&B` is contained in `A`'s definition (`A` calls `B`). `&B` will be represented as an pseudo arc (`u -> v`), where `u` is the state before `&B` and `v` is the state after `&B`. If arcs of `u` do not collide with arcs of `B`, the transition function will push `v` to the return stack if current state set contains `u` and there is no other transition. Note `B` is disconnected from `A`, which is different from the `CollapseExpr` case. The machine will transit on automaton `B` greedily. If there is no transition, it will pop a return address(`v` in this case) and jumps to it. ## Contrib ### Vim Syntax highlighting, and a syntax checking plugin for Synaptics ```zsh ln -sr contrib/vim/compiler/yanshi.vim ~/.vim/compiler/ ln -sr contrib/vim/ftdetect/yanshi.vim ~/.vim/ftdetect/ ln -sr contrib/vim/ftplugin/yanshi.vim ~/.vim/ftplugin/ ln -sr contrib/vim/syntax/yanshi.vim ~/.vim/syntax/ ln -sr contrib/vim/syntax_checker/yanshi ~/.vim/syntax_checker/ ``` ### Zsh Command line completion ``` # ~/.zshrc fpath=(~/.zsh $fpath) # ln -sr contrib/zsh/_yanshi ~/.zsh/ ``` ## Internals ``` src common.{cc,hh} main.{cc,hh} syntax.{cc,hh} loader.{cc,hh} fsa.{cc,hh} fsa_anno.{cc,hh} compiler.{cc,hh} parser.y lexer.l location.cc ``` * Lex `lexer.l` * Parse and generate a syntax tree `parser.y` * `loader.cc` + Get a list of definitions + Recursively load for each `import` + Resolve references and associate uses to definitions + Build a dependency graph from `EmbedExpr` + Compile automaton for each nonterminal in topological order. `CollapseExpr` and `CallExpr` are represented by special directed arcs. + Generate code for `export` nonterminals, resolving `CollapseExpr` and `CallExpr` ### Finite state automaton Each node of the syntax tree corresponds to an automaton. The parent builds an automaton from its children according to the semantics. The automaton of the parent may contain states from the automaton of one of the children, or it is a state introduced by the parent. `assoc[i]` records the associative nodes in the automaton tree (which part of the syntax tree has associations with this state) and positions (start state, final state or inner state) for state `i`. It serves three purposes: * Check which action should be triggered * Look for inner states (neither start nor final) in the implementation of substring grammar * Check whether it is associated to a `CallExpr` or `CollapseExpr` ### `CollapseExpr` ================================================ FILE: yanshi/contrib/vim/compiler/yanshi.vim ================================================ "if exists('current_compiler') " finish "endif let current_compiler = 'yanshi' if exists(':CompilerSet') != 2 command -nargs=* CompilerSet setlocal endif CompilerSet errorformat= \%E%f\ %l:%c-%*\\d\ error\ %m, \%E%f\ %l-%*\\d:%c-%*\\d\ error\ %m, \%W%f\ %l:%c-%*\\d\ warning\ %m, \%W%f\ %l-%*\\d:%c-%*\\d\ warning\ %m, \%C%.%# CompilerSet makeprg=yanshi\ -d0\ -c\ $*\ % ================================================ FILE: yanshi/contrib/vim/ftdetect/yanshi.vim ================================================ au BufRead,BufNewFile *.yanshi setf yanshi au BufRead,BufNewFile *.ys setf yanshi ================================================ FILE: yanshi/contrib/vim/ftplugin/yanshi.vim ================================================ if exists('b:did_ftplugin') finish endif let b:did_ftplugin = 1 compiler yanshi ================================================ FILE: yanshi/contrib/vim/syntax/yanshi.vim ================================================ if exists('b:current_syntax') finish endif syn cluster yanshiCommentGroup contains=yanshiTodo syn include @yanshiCcode syntax/cpp.vim syn keyword yanshiAction action syn keyword yanshiMacro semicolon nosemicolon syn keyword yanshiStorageClass export intact syn keyword yanshiTodo contained TODO FIXME XXX syn match yanshiCpp 'c++' syn match yanshiActionOperator '[>$@%]' syn match yanshiCall '\^\w\+\(::\w\+\)\?' syn match yanshiCollapse '!\w\+\(::\w\+\)\?' syn match yanshiHighOp '[+\*?]' syn match yanshiIdent '\w\+\(::\w\+\)\?' syn match yanshiCpp display "^c++\s*" skipwhite nextgroup=yanshiBrace syn match yanshiImport display "^import\s*" contains=yanshiImported syn match yanshiLowOp '[-&|]' syn match yanshiSpecial display contained "\\\(x\x\x\|.\|$\)" syn region yanshiBrace matchgroup=Delimiter start='{' end='}' fold contains=@yanshiCcode syn region yanshiBracket start='\[' skip=+\\\\\|\\]+ end=']' syn region yanshiComment start='/\*' end='\*/' keepend contains=@yanshiCommentGroup,@Spell syn region yanshiImported display contained start="+" skip=+\\\\\|\\"+ end=+"+ syn region yanshiLineComment start='#\|//' skip='\\$' end='$' keepend contains=@yanshiCommentGroup,@Spell syn region yanshiPreprocess start="#define" skip="\\$" end="$" keepend syn region yanshiQQString start=+"+ skip=+\\.+ end=+"+ contains=yanshiSpecial syn region yanshiQString start=+'+ skip=+\\.+ end=+'+ syn region yanshiDefineStmt start='^\w\+\s*[=:]' end='$' skipnl contains=@yanshiExpr,yanshiComment,yanshiLineComment,yanshiParen0 syn cluster yanshiExpr contains=yanshiActionOperator,yanshiBrace,yanshiBracket,yanshiCall,yanshiCollapse,yanshiIdent,yanshiHighOp,yanshiLowOp,yanshiQString,yanshiQQString, sy region yanshiParen0 matchgroup=yanshiParen0 start='(' end=')' contains=@yanshiExpr,yanshiParen1 sy region yanshiParen1 matchgroup=yanshiParen1 start='(' end=')' contains=@yanshiExpr,yanshiParen2 contained sy region yanshiParen2 matchgroup=yanshiParen2 start='(' end=')' contains=@yanshiExpr,yanshiParen3 contained sy region yanshiParen3 matchgroup=yanshiParen3 start='(' end=')' contains=@yanshiExpr,yanshiParen4 contained sy region yanshiParen4 matchgroup=yanshiParen4 start='(' end=')' contains=@yanshiExpr,yanshiParen5 contained sy region yanshiParen5 matchgroup=yanshiParen5 start='(' end=')' contains=@yanshiExpr,yanshiParen0 contained hi yanshiParen0 ctermfg=brown guifg=#3bb9ff hi yanshiParen1 ctermfg=DarkBlue guifg=#f88017 hi yanshiParen2 ctermfg=darkgray guifg=#5efb6e hi yanshiParen3 ctermfg=darkgreen guifg=#f62817 hi yanshiParen4 ctermfg=darkcyan guifg=#fdd017 hi yanshiParen5 ctermfg=darkmagenta guifg=#faafba hi link yanshiIdent Identifier "TODO color mismatch of {} "hi link yanshiBrace Statement "hi link yanshiDefineStmt Statement hi def link yanshiCall Constant hi def link yanshiCollapse Constant hi def link yanshiAction Structure hi def link yanshiActionOperator Type hi def link yanshiBracket Function hi def link yanshiCpp Structure hi def link yanshiComment Comment hi def link yanshiHighOp Operator hi def link yanshiImport Include hi def link yanshiImported String hi def link yanshiLineComment Comment hi def link yanshiLowOp Conditional hi def link yanshiMacro Macro hi def link yanshiPreprocess Macro hi def link yanshiQQString String hi def link yanshiQString String hi def link yanshiSpecial SpecialChar hi def link yanshiStorageClass StorageClass hi def link yanshiTodo Todo let b:current_syntax = 'yanshi' ================================================ FILE: yanshi/contrib/vim/syntax_checkers/yanshi/yanshi.vim ================================================ if exists('g:loaded_syntastic_yanshi_yanshi_checker') finish endif let g:loaded_syntastic_yanshi_yanshi_checker = 1 let s:save_cpo = &cpo set cpo&vim fu! SyntaxCheckers_yanshi_yanshi_GetLocList() dict let makeprg = self.makeprgBuild({ 'args': '-d0 -c' }) let errorformat = \ '%C %.%#,'. \ '%E%f %l:%c-%*\d error %m,'. \ '%E%f %l-%*\d:%c-%*\d error %m,'. \ '%W%f %l:%c-%*\d warning %m,'. \ '%W%f %l-%*\d:%c-%*\d warning %m' return SyntasticMake({ \ 'makeprg': makeprg, \ 'errorformat': errorformat }) endf call g:SyntasticRegistry.CreateAndRegisterChecker({ \ 'filetype': 'yanshi', \ 'name': 'yanshi'}) let &cpo = s:save_cpo unlet s:save_cpo ================================================ FILE: yanshi/contrib/zsh/_yanshi ================================================ #compdef yanshi _arguments \ '(-b --bytes)'{-b,--bytes}'[make labels range over \[0,256), Unicode literals will be treated as UTF-8 bytes]' \ '(-c --check)'{-c,--check}'[check syntax & use/def]' \ '-C[generate C source code (default: C++)]' \ '(-d --debug)'{-d,--debug}'+[debug level]:level:(0 1 2 3 4 5)' \ '--dump-action[dump associated actions for each edge]' \ '--dump-assoc[dump associated AST Expr for each state]' \ '--dump-automaton[dump automata]' \ '--dump-embed[dump statistics of EmbedExpr]' \ '--dump-module[dump module use/def/...]' \ '--dump-tree[dump AST]' \ '(-G --graph)'{-G,--graph}'[output a Graphviz dot file]' \ '(-I --import)'{-I,--import}'=[add to search path for "import"]' \ '(-i --interactive)'{-i,--interactive}'[interactive mode]' \ '(-k --keep-inaccessible)'{-k,--keep-inaccessible}'[do not perform accessible/co-accessible]' \ '(-l --debug-output)'{-l,--debug-output}'=[filename for debug output]:file:_files' \ '--max-return-stack=[max length of return stack in C generator]:len:' \ '(-o --output)'{-o,--output}'=[.cc output filename]:file:_files' \ '(-O --output-header)'{-O,--output-header}'=[.hh output filename]:file:_files' \ '(-s --substring-grammar)'{-s,--substring-grammar}'[construct regular approximation of the substring grammar. Inner states of nonterminals labeled 'intact' are not connected to start/final]' \ '(-h --help)'{-h,--help}'[display this help]' \ '1:file:_files -g "*.{ys,yanshi}"'\ ================================================ FILE: yanshi/src/common.cc ================================================ #include "common.hh" #include "option.hh" #include #include #include #include #include #include #include #include ///// Error static const char *ENAME[] = { /* 0 */ "", /* 1 */ "EPERM", "ENOENT", "ESRCH", "EINTR", "EIO", "ENXIO", /* 7 */ "E2BIG", "ENOEXEC", "EBADF", "ECHILD", /* 11 */ "EAGAIN/EWOULDBLOCK", "ENOMEM", "EACCES", "EFAULT", /* 15 */ "ENOTBLK", "EBUSY", "EEXIST", "EXDEV", "ENODEV", /* 20 */ "ENOTDIR", "EISDIR", "EINVAL", "ENFILE", "EMFILE", /* 25 */ "ENOTTY", "ETXTBSY", "EFBIG", "ENOSPC", "ESPIPE", /* 30 */ "EROFS", "EMLINK", "EPIPE", "EDOM", "ERANGE", /* 35 */ "EDEADLK/EDEADLOCK", "ENAMETOOLONG", "ENOLCK", "ENOSYS", /* 39 */ "ENOTEMPTY", "ELOOP", "", "ENOMSG", "EIDRM", "ECHRNG", /* 45 */ "EL2NSYNC", "EL3HLT", "EL3RST", "ELNRNG", "EUNATCH", /* 50 */ "ENOCSI", "EL2HLT", "EBADE", "EBADR", "EXFULL", "ENOANO", /* 56 */ "EBADRQC", "EBADSLT", "", "EBFONT", "ENOSTR", "ENODATA", /* 62 */ "ETIME", "ENOSR", "ENONET", "ENOPKG", "EREMOTE", /* 67 */ "ENOLINK", "EADV", "ESRMNT", "ECOMM", "EPROTO", /* 72 */ "EMULTIHOP", "EDOTDOT", "EBADMSG", "EOVERFLOW", /* 76 */ "ENOTUNIQ", "EBADFD", "EREMCHG", "ELIBACC", "ELIBBAD", /* 81 */ "ELIBSCN", "ELIBMAX", "ELIBEXEC", "EILSEQ", "ERESTART", /* 86 */ "ESTRPIPE", "EUSERS", "ENOTSOCK", "EDESTADDRREQ", /* 90 */ "EMSGSIZE", "EPROTOTYPE", "ENOPROTOOPT", /* 93 */ "EPROTONOSUPPORT", "ESOCKTNOSUPPORT", /* 95 */ "EOPNOTSUPP/ENOTSUP", "EPFNOSUPPORT", "EAFNOSUPPORT", /* 98 */ "EADDRINUSE", "EADDRNOTAVAIL", "ENETDOWN", "ENETUNREACH", /* 102 */ "ENETRESET", "ECONNABORTED", "ECONNRESET", "ENOBUFS", /* 106 */ "EISCONN", "ENOTCONN", "ESHUTDOWN", "ETOOMANYREFS", /* 110 */ "ETIMEDOUT", "ECONNREFUSED", "EHOSTDOWN", "EHOSTUNREACH", /* 114 */ "EALREADY", "EINPROGRESS", "ESTALE", "EUCLEAN", /* 118 */ "ENOTNAM", "ENAVAIL", "EISNAM", "EREMOTEIO", "EDQUOT", /* 123 */ "ENOMEDIUM", "EMEDIUMTYPE", "ECANCELED", "ENOKEY", /* 127 */ "EKEYEXPIRED", "EKEYREVOKED", "EKEYREJECTED", /* 130 */ "EOWNERDEAD", "ENOTRECOVERABLE", "ERFKILL", "EHWPOISON" }; #define MAX_ENAME 133 long action_label_base, action_label, call_label_base, call_label, collapse_label_base, collapse_label; void output_error(bool use_err, const char *format, va_list ap) { char text[BUF_SIZE], msg[BUF_SIZE], buf[BUF_SIZE]; vsnprintf(msg, BUF_SIZE, format, ap); if (use_err) snprintf(text, BUF_SIZE, "[%s %s] ", 0 < errno && errno < MAX_ENAME ? ENAME[errno] : "?UNKNOWN?", strerror(errno)); else strcpy(text, ""); snprintf(buf, BUF_SIZE, RED "%s%s\n", text, msg); fputs(buf, stderr); fputs(SGR0, stderr); fflush(stderr); } void err_msg(const char *format, ...) { va_list ap; va_start(ap, format); int saved = errno; output_error(errno > 0, format, ap); errno = saved; va_end(ap); } #define err_msg_g(...) ({err_msg(__VA_ARGS__); goto quit;}) void err_exit(int exitno, const char *format, ...) { va_list ap; va_start(ap, format); int saved = errno; output_error(errno > 0, format, ap); errno = saved; va_end(ap); void *bt[99]; char buf[1024]; int nptrs = backtrace(bt, LEN(buf)); int i = sprintf(buf, "addr2line -Cfipe %s", program_invocation_name), j = 0; while (j < nptrs && i+30 < sizeof buf) i += sprintf(buf+i, " %p", bt[j++]); strcat(buf, ">&2"); fputs("\n", stderr); system(buf); //backtrace_symbols_fd(buf, nptrs, STDERR_FILENO); exit(exitno); } long get_long(const char *arg) { char *end; errno = 0; long ret = strtol(arg, &end, 0); if (errno) err_exit(EX_USAGE, "get_long: %s", arg); if (*end) err_exit(EX_USAGE, "get_long: nonnumeric character"); return ret; } //// log // void log_generic(const char *prefix, const char *format, va_list ap) { char buf[BUF_SIZE]; timeval tv; tm tm; gettimeofday(&tv, NULL); fputs(prefix, stdout); if (localtime_r(&tv.tv_sec, &tm)) { strftime(buf, sizeof buf, "%T.%%06u ", &tm); printf(buf, tv.tv_usec); } vprintf(format, ap); fputs(SGR0, stdout); fflush(stdout); } void log_event(const char *format, ...) { va_list ap; va_start(ap, format); log_generic(CYAN, format, ap); va_end(ap); } void log_action(const char *format, ...) { va_list ap; va_start(ap, format); log_generic(GREEN, format, ap); va_end(ap); } void log_status(const char *format, ...) { va_list ap; va_start(ap, format); log_generic(YELLOW, format, ap); va_end(ap); } void bold(long fd) { if (isatty(fd)) fputs("\x1b[1m", fd == STDOUT_FILENO ? stdout : stderr); } void blue(long fd) { if (isatty(fd)) fputs(BLUE, fd == STDOUT_FILENO ? stdout : stderr); } void cyan(long fd) { if (isatty(fd)) fputs(CYAN, fd == STDOUT_FILENO ? stdout : stderr); } void green(long fd) { if (isatty(fd)) fputs(GREEN, fd == STDOUT_FILENO ? stdout : stderr); } void magenta(long fd) { if (isatty(fd)) fputs(MAGENTA, fd == STDOUT_FILENO ? stdout : stderr); } void red(long fd) { if (isatty(fd)) fputs(RED, fd == STDOUT_FILENO ? stdout : stderr); } void sgr0(long fd) { if (isatty(fd)) fputs(SGR0, fd == STDOUT_FILENO ? stdout : stderr); } void yellow(long fd) { if (isatty(fd)) fputs(YELLOW, fd == STDOUT_FILENO ? stdout : stderr); } void normal_yellow(long fd) { if (isatty(fd)) fputs(NORMAL_YELLOW, fd == STDOUT_FILENO ? stdout : stderr); } void indent(FILE* f, int d) { fprintf(f, "%*s", 2*d, ""); } void DisjointIntervals::flip() { long i = 0; map to2; for (auto &x: to) { if (i < x.first) to2.emplace(i, x.first); i = x.second; } if (i < AB) to2.emplace(i, AB); to = move(to2); } void DisjointIntervals::print() { for (auto& x: to) printf("(%ld,%ld) ", x.first, x.second); puts(""); } ================================================ FILE: yanshi/src/common.hh ================================================ #pragma once #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif #include #include #include #include #include #include #include using std::map; using std::vector; typedef int8_t i8; typedef int16_t i16; typedef int32_t i32; typedef int64_t i64; typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; typedef unsigned long ulong; #ifdef __APPLE__ #include extern char*** _NSGetArgv(void); #define program_invocation_name (((char **)*_NSGetArgv())[0]) #define program_invocation_short_name (((char **)*_NSGetArgv())[0]) #endif #define LEN(x) (sizeof(x)/sizeof(*x)) #define ALL(x) (x).begin(), (x).end() #define REP(i, n) FOR(i, 0, n) #define FOR(i, a, b) for (typename std::remove_cv::type>::type i = (a); i < (b); i++) #define ROF(i, a, b) for (typename std::remove_cv::type>::type i = (b); --i >= (a); ) #define SGR0 "\x1b[m" #define RED "\x1b[1;31m" #define GREEN "\x1b[1;32m" #define YELLOW "\x1b[1;33m" #define BLUE "\x1b[1;34m" #define MAGENTA "\x1b[1;35m" #define CYAN "\x1b[1;36m" #define NORMAL_YELLOW "\x1b[33m" const long MAX_CODEPOINT = 0x10ffff; extern long action_label_base, action_label, call_label_base, call_label, collapse_label_base, collapse_label; void bold(long fd = 1); void blue(long fd = 1); void cyan(long fd = 1); void green(long fd = 1); void magenta(long fd = 1); void red(long fd = 1); void sgr0(long fd = 1); void yellow(long fd = 1); void normal_yellow(long fd = 1); void indent(FILE* f, int d); const size_t BUF_SIZE = 512; void output_error(bool use_err, const char *format, va_list ap); void err_msg(const char *format, ...); void err_exit(int exitno, const char *format, ...); long get_long(const char *arg); void log_generic(const char *prefix, const char *format, va_list ap); void log_event(const char *format, ...); void log_action(const char *format, ...); void log_status(const char *format, ...); extern long debug_level; extern FILE* debug_file; #define DP(level, ...) do { \ if (level <= debug_level) { \ fprintf(debug_file, "%s:%d ", __FILE__, __LINE__); \ fprintf(debug_file, __VA_ARGS__);\ fprintf(debug_file, "\n"); \ fflush(debug_file); \ } \ } while (0) template void emplace_front(vector& a, Args&&... args) { a.emplace(a.begin(), args...); } template void sorted_insert(vector& a, const T& x) { a.emplace_back(); auto it = a.end(); while (a.begin() != --it && x < it[-1]) *it = it[-1]; *it = x; } template void sorted_emplace(vector& a, Args&&... args) { T x{args...}; a.emplace_back(); auto it = a.end(); while (a.begin() != --it && x < it[-1]) *it = it[-1]; *it = x; } struct DisjointIntervals { typedef std::pair value_type; std::map to; template void emplace(Args&&... args) { value_type x{args...}; auto it = to.lower_bound(x.first); if (it != to.begin() && x.first <= prev(it)->second) x.first = (--it)->first; auto it2 = to.upper_bound(x.second); if (it2 != to.begin() && prev(it2)->first <= x.second && x.second < prev(it2)->second) x.second = prev(it2)->second; while (it != it2) it = to.erase(it); to.emplace(x); } void flip(); void print(); // XXX }; ================================================ FILE: yanshi/src/compiler.cc ================================================ #include "compiler.hh" #include "fsa_anno.hh" #include "loader.hh" #include "option.hh" #include #include #include #include #include #include #include using namespace std; unordered_map compiled; static unordered_map>> stmt2call_addr; static unordered_map> stmt2final; void print_assoc(const FsaAnno& anno) { magenta(); printf("=== Associated Expr of each state\n"); sgr0(); REP(i, anno.fsa.n()) { printf("%ld:", i); for (auto aa: anno.assoc[i]) { auto a = aa.first; printf(" %s%s%s%s(%ld-%ld", a->name().c_str(), has_start(aa.second) ? "^" : "", has_inner(aa.second) ? "." : "", has_final(aa.second) ? "$" : "", a->loc.start, a->loc.end); if (a->entering.size()) printf(",>%zd", a->entering.size()); if (a->leaving.size()) printf(",%%%zd", a->leaving.size()); if (a->finishing.size()) printf(",@%zd", a->finishing.size()); if (a->transiting.size()) printf(",$%zd", a->transiting.size()); printf(")"); } puts(""); } puts(""); } void print_automaton(const Fsa& fsa) { magenta(); printf("=== Automaton\n"); sgr0(); green(); printf("start: %ld\n", fsa.start); red(); printf("finals:"); for (long i: fsa.finals) printf(" %ld", i); puts(""); sgr0(); puts("edges:"); REP(i, fsa.n()) { printf("%ld:", i); for (auto it = fsa.adj[i].begin(); it != fsa.adj[i].end(); ) { long from = it->first.first, to = it->first.second, v = it->second; while (++it != fsa.adj[i].end() && to == it->first.first && it->second == v) to = it->first.second; if (from == to-1) printf(" (%ld,%ld)", from, v); else printf(" (%ld-%ld,%ld)", from, to-1, v); } puts(""); } puts(""); } Expr* find_lca(Expr* u, Expr* v) { if (u->depth > v->depth) swap(u, v); if (u->depth < v->depth) for (long k = 63-__builtin_clzl(v->depth-u->depth); k >= 0; k--) if (u->depth <= v->depth-(1L<anc[k]; if (u == v) return u; if (v->depth) for (long k = 63-__builtin_clzl(v->depth); k >= 0; k--) if (k < u->anc.size() && u->anc[k] != v->anc[k]) u = u->anc[k], v = v->anc[k]; return u->anc[0]; // NULL if two trees } struct Compiler : Visitor { stack st; stack path; long tick = 0; void pre_expr(Expr& expr) { expr.pre = tick++; expr.depth = path.size(); if (path.size()) { expr.anc.assign(1, path.top()); for (long k = 1; 1L << k <= expr.depth; k++) expr.anc.push_back(expr.anc[k-1]->anc[k-1]); } else expr.anc.assign(1, nullptr); path.push(&expr); DP(5, "%s(%ld-%ld)", expr.name().c_str(), expr.loc.start, expr.loc.end); } void post_expr(Expr& expr) { path.pop(); expr.post = tick; #ifdef DEBUG st.top().fsa.check(); #endif } void visit(Expr& expr) override { pre_expr(expr); expr.accept(*this); post_expr(expr); } void visit(BracketExpr& expr) override { st.push(FsaAnno::bracket(expr)); } void visit(CallExpr& expr) override { st.push(FsaAnno::call(expr)); } void visit(CollapseExpr& expr) override { st.push(FsaAnno::collapse(expr)); } void visit(ComplementExpr& expr) override { visit(*expr.inner); st.top().complement(&expr); } void visit(ConcatExpr& expr) override { visit(*expr.rhs); FsaAnno rhs = move(st.top()); visit(*expr.lhs); st.top().concat(rhs, &expr); } void visit(DifferenceExpr& expr) override { visit(*expr.rhs); FsaAnno rhs = move(st.top()); visit(*expr.lhs); st.top().difference(rhs, &expr); } void visit(DotExpr& expr) override { st.push(FsaAnno::dot(&expr)); } void visit(EmbedExpr& expr) override { st.push(FsaAnno::embed(expr)); } void visit(EpsilonExpr& expr) override { st.push(FsaAnno::epsilon_fsa(&expr)); } void visit(IntersectExpr& expr) override { visit(*expr.rhs); FsaAnno rhs = move(st.top()); visit(*expr.lhs); st.top().intersect(rhs, &expr); } void visit(LiteralExpr& expr) override { st.push(FsaAnno::literal(expr)); } void visit(PlusExpr& expr) override { visit(*expr.inner); st.top().plus(&expr); } void visit(QuestionExpr& expr) override { visit(*expr.inner); st.top().question(&expr); } void visit(RepeatExpr& expr) override { visit(*expr.inner); st.top().repeat(expr); } void visit(StarExpr& expr) override { visit(*expr.inner); st.top().star(&expr); } void visit(UnionExpr& expr) override { visit(*expr.rhs); FsaAnno rhs = move(st.top()); visit(*expr.lhs); st.top().union_(rhs, &expr); } }; void compile(DefineStmt* stmt) { if (compiled.count(stmt)) return; FsaAnno& anno = compiled[stmt]; Compiler comp; comp.visit(*stmt->rhs); anno = move(comp.st.top()); anno.determinize(NULL, NULL); anno.minimize(NULL); DP(4, "size(%s::%s) = %ld", stmt->module->filename.c_str(), stmt->lhs.c_str(), anno.fsa.n()); } void generate_transitions(DefineStmt* stmt) { FsaAnno& anno = compiled[stmt]; auto& call_addr = stmt2call_addr[stmt]; auto& sub_final = stmt2final[stmt]; auto find_within = [&](long u) { vector> within; Expr* last = NULL; sort(ALL(anno.assoc[u]), [](const pair& x, const pair& y) { if (x.first->pre != y.first->pre) return x.first->pre < y.first->pre; return x.second < y.second; }); for (auto aa: anno.assoc[u]) { Expr* stop = last ? find_lca(last, aa.first) : NULL; last = aa.first; for (Expr* x = aa.first; x != stop; x = x->anc[0]) within.emplace_back(x, aa.second); } sort(ALL(within)); auto j = within.begin(); for (auto i = within.begin(); i != within.end(); ) { Expr* x = i->first; long t = long(i->second); while (++i != within.end() && x == i->first) t |= long(i->second); *j++ = {x, ExprTag(t)}; } within.erase(j, within.end()); return within; }; decltype(anno.assoc) withins(anno.fsa.n()); REP(i, anno.fsa.n()) withins[i] = move(find_within(i)); auto get_code = [](Action* action) { if (auto t = dynamic_cast(action)) return t->code; else if (auto t = dynamic_cast(action)) return t->define_stmt->code; else assert(0); return string(); }; #define D(S) if (opt_dump_action) { \ if (auto t = dynamic_cast(action.first)) { \ if (from == to-1) \ printf(S " %ld %ld %ld %s\n", u, from, v, t->code.c_str()); \ else \ printf(S " %ld %ld-%ld %ld %s\n", u, from, to-1, v, t->code.c_str()); \ } else if (auto t = dynamic_cast(action.first)) { \ if (from == to-1) \ printf(S " %ld %ld %ld %s\n", u, from, v, t->define_stmt->code.c_str()); \ else \ printf(S " %ld %ld-%ld %ld %s\n", u, from, to-1, v, t->define_stmt->code.c_str()); \ } \ } if (output_header) { if (opt_gen_c) { if (opt_gen_extern_c) fputs("extern \"C\" ", output_header); fprintf(output_header, "long yanshi_%s_transit(long* ret_stack, long* ret_stack_len, long u, long c", stmt->lhs.c_str()); } else fprintf(output_header, "long yanshi_%s_transit(vector& ret_stack, long u, long c", stmt->lhs.c_str()); if (stmt->export_params.size()) fprintf(output_header, ", %s", stmt->export_params.c_str()); fprintf(output_header, ");\n"); } if (opt_gen_c) { if (opt_gen_extern_c) fputs("extern \"C\" ", output); fprintf(output, "long yanshi_%s_transit(long* ret_stack, long* ret_stack_len, long u, long c", stmt->lhs.c_str()); } else fprintf(output, "long yanshi_%s_transit(vector& ret_stack, long u, long c", stmt->lhs.c_str()); if (stmt->export_params.size()) fprintf(output, ", %s", stmt->export_params.c_str()); fprintf(output, ")\n" "{\n" " long v = -1;\n" "again:\n" " switch (u) {\n"); REP(u, anno.fsa.n()) { if (call_addr[u].first >= 0) { // no other transitions fprintf(output, " case %ld:\n" " u = %ld;\n" , u, call_addr[u].first); if (opt_gen_c) fprintf(output, " if (*ret_stack_len >= %ld) return -1;\n" " ret_stack[(*ret_stack_len)++] = %ld;\n" , opt_max_return_stack, call_addr[u].second); else fprintf(output, " ret_stack.push_back(%ld);\n" , call_addr[u].second); fprintf(output, " goto again;\n"); continue; } if (anno.fsa.adj[u].empty() && ! sub_final[u]) continue; indent(output, 1); fprintf(output, "case %ld:\n", u); indent(output, 2); fprintf(output, "switch (c) {\n"); unordered_map>, vector>>> v2case; for (auto it = anno.fsa.adj[u].begin(); it != anno.fsa.adj[u].end(); ) { long from = it->first.first, to = it->first.second, v = it->second; while (++it != anno.fsa.adj[u].end() && to == it->first.first && it->second == v) to = it->first.second; v2case[v].first.emplace_back(from, to); auto& body = v2case[v].second; auto ie = withins[u].end(), je = withins[v].end(); // leaving = Expr(u) - Expr(v) for (auto i = withins[u].begin(), j = withins[v].begin(); i != ie; ++i) { while (j != je && i->first > j->first) ++j; if (j == je || i->first != j->first) for (auto action: i->first->leaving) { D("%%"); body.push_back(action); } } // entering = Expr(v) - Expr(u) for (auto i = withins[u].begin(), j = withins[v].begin(); j != je; ++j) { while (i != ie && i->first < j->first) ++i; if (i == ie || i->first != j->first) for (auto action: j->first->entering) { D(">"); body.push_back(action); } } // transiting = intersect(Expr(u), Expr(v)) for (auto i = withins[u].begin(), j = withins[v].begin(); j != je; ++j) { while (i != ie && i->first < j->first) ++i; if (i != ie && i->first == j->first) for (auto action: j->first->transiting) { D("$"); body.push_back(action); } } // finishing = intersect(Expr(u), Expr(v)) & Expr(v).has_final(v) for (auto i = withins[u].begin(), j = withins[v].begin(); j != je; ++j) { while (i != ie && i->first < j->first) ++i; if (i != ie && i->first == j->first && has_final(j->second)) for (auto action: j->first->finishing) { D("@"); body.push_back(action); } } } for (auto& x: v2case) { for (auto& y: x.second.first) { indent(output, 2); if (y.first == y.second-1) fprintf(output, "case %ld:\n", y.first); else fprintf(output, "case %ld ... %ld:\n", y.first, y.second-1); } indent(output, 3); fprintf(output, "v = %ld;\n", x.first); // actions sort(ALL(x.second.second), [](const pair& a0, const pair& a1) { return a0.second != a1.second ? a0.second < a1.second : a0.first < a1.first; }); x.second.second.erase(unique(ALL(x.second.second)), x.second.second.end()); for (auto a: x.second.second) fprintf(output, "{%s}\n", get_code(a.first).c_str()); indent(output, 3); fprintf(output, "break;\n"); } // return from finals of DefineStmt called by CallExpr if (sub_final[u]) { indent(output, 2); fprintf(output, "default:\n"); indent(output, 3); fprintf(output, opt_gen_c ? "if (*ret_stack_len) { u = ret_stack[--*ret_stack_len]; goto again; }\n" : "if (ret_stack.size()) { u = ret_stack.back(); ret_stack.pop_back(); goto again; }\n"); indent(output, 3); fprintf(output, "break;\n"); } indent(output, 2); fprintf(output, "}\n"); indent(output, 2); fprintf(output, "break;\n"); } indent(output, 1); fprintf(output, "}\n"); indent(output, 1); fprintf(output, "return v;\n"); fprintf(output, "}\n\n"); } bool compile_export(DefineStmt* stmt) { DP(2, "Exporting %s", stmt->lhs.c_str()); FsaAnno& anno = compiled[stmt]; DP(3, "Construct automaton with all DefineStmt associated to referenced CallExpr/CollapseExpr"); vector> adj; decltype(anno.assoc) assoc; vector> cllps; long allo = 0; unordered_map stmt2offset; unordered_map stmt2start; unordered_map start2stmt; vector starts; vector sub_final; function allocate = [&](DefineStmt* stmt) { if (stmt2offset.count(stmt)) return; DP(4, "Allocate %ld to %s", allo, stmt->lhs.c_str()); FsaAnno& anno = compiled[stmt]; long base = stmt2offset[stmt] = allo; allo += anno.fsa.n(); sub_final.resize(allo); if (used_as_call.count(stmt)) { stmt2start[stmt] = base+anno.fsa.start; start2stmt[base+anno.fsa.start] = stmt; starts.push_back(base+anno.fsa.start); for (long f: anno.fsa.finals) sub_final[base+f] = true; } adj.insert(adj.end(), ALL(anno.fsa.adj)); REP(i, anno.fsa.n()) for (auto& e: adj[base+i]) e.second += base; assoc.insert(assoc.end(), ALL(anno.assoc)); FOR(i, base, base+anno.fsa.n()) { for (auto aa: assoc[i]) if (has_start(aa.second)) { if (auto* e = dynamic_cast(aa.first)) { DefineStmt* v = e->define_stmt; allocate(v); } else if (auto* e = dynamic_cast(aa.first)) { DefineStmt* v = e->define_stmt; allocate(v); // (i@{CollapseExpr,...}, special, _) -> ({CollapseExpr,...}, epsilon, CollapseExpr.define_stmt.start) sorted_emplace(adj[i], epsilon, stmt2offset[v]+compiled[v].fsa.start); } } long j = adj[i].size(); while (j && collapse_label_base < adj[i][j-1].first.second) { long v = adj[i][j-1].second; if (adj[i][j-1].first.first < collapse_label_base) adj[i][j-1].first.second = collapse_label_base; else j--; CollapseExpr* e; for (auto aa: assoc[v]) if (has_final(aa.second) && (e = dynamic_cast(aa.first))) { DefineStmt* w = e->define_stmt; allocate(w); // (_, special, v@{CollapseExpr,...}) -> (CollapseExpr.define_stmt.final, epsilon, v) for (long f: compiled[w].fsa.finals) { long g = stmt2offset[w]+f; sorted_emplace(adj[g], epsilon, v); if (g == i) j++; } } } // remove (i, special, _) adj[i].resize(j); } }; allocate(stmt); anno.fsa.adj = move(adj); anno.assoc = move(assoc); anno.deterministic = false; DP(3, "# of states: %ld", anno.fsa.n()); // substring grammar & this nonterminal is not marked as intact if (opt_substring_grammar && ! stmt->intact) { DP(3, "Constructing substring grammar"); anno.substring_grammar(); DP(3, "# of states: %ld", anno.fsa.n()); } vector> map0; DP(3, "Determinize"); anno.determinize(&starts, &map0); vector sub_final2(anno.fsa.n()); REP(i, anno.fsa.n()) for (long u: map0[i]) { if (sub_final[u]) sub_final2[i] = true; if (start2stmt.count(u)) { DefineStmt* stmt = start2stmt[u]; if (stmt2start[stmt] < 0) { stmt->module->locfile.error_context(stmt->loc, "the start has been included in multiple DFA states"); return false; } stmt2start[stmt] = ~ i; } } sub_final = move(sub_final2); start2stmt.clear(); for (auto& it: stmt2start) { it.second = ~ it.second; start2stmt[it.second] = it.first; } DP(3, "# of states: %ld", anno.fsa.n()); DP(3, "Minimize"); map0.clear(); anno.minimize(&map0); sub_final2.assign(anno.fsa.n(), false); REP(i, anno.fsa.n()) for (long u: map0[i]) { if (sub_final[u]) sub_final2[i] = true; if (start2stmt.count(u)) { DefineStmt* stmt = start2stmt[u]; stmt2start[stmt] = i; } } sub_final = move(sub_final2); start2stmt.clear(); for (auto& it: stmt2start) start2stmt[it.second] = it.first; DP(3, "# of states: %ld", anno.fsa.n()); if (! opt_keep_inaccessible) { DP(3, "Keep accessible states"); // roots: start, starts of DefineStmt associated to CallExpr starts.clear(); for (auto& it: stmt2start) starts.push_back(it.second); vector map1; anno.accessible(&starts, map1); sub_final2.assign(anno.fsa.n(), false); REP(i, anno.fsa.n()) { long u = map1[i]; sub_final2[i] = sub_final[u]; if (start2stmt.count(u)) stmt2start[start2stmt[u]] = i; } sub_final = move(sub_final2); start2stmt.clear(); for (auto& it: stmt2start) start2stmt[it.second] = it.first; DP(3, "# of states: %ld", anno.fsa.n()); DP(3, "Keep co-accessible states"); // roots: finals, finals of DefineStmt associated to CallExpr map1.clear(); anno.co_accessible(&sub_final, map1); sub_final2.assign(anno.fsa.n(), false); REP(i, anno.fsa.n()) { long u = map1[i]; sub_final2[i] = sub_final[u]; if (start2stmt.count(u)) stmt2start[start2stmt[u]] = i; } sub_final = move(sub_final2); start2stmt.clear(); for (auto& it: stmt2start) start2stmt[it.second] = it.first; DP(3, "# of states: %ld", anno.fsa.n()); } stmt2final[stmt] = sub_final; auto& call_addr = stmt2call_addr[stmt]; call_addr.assign(anno.fsa.n(), make_pair(-1L, -1L)); DP(3, "CallExpr"); REP(i, anno.fsa.n()) if (anno.fsa.has_call(i)) { if (anno.fsa.adj[i].size() != 1 || anno.fsa.adj[i][0].first.second-anno.fsa.adj[i][0].first.first > 1) { stmt->module->locfile.error_context(stmt->loc, "state %ld: CallExpr cannot coexist with other transitions", i); for (auto it = anno.fsa.adj[i].begin(); it != anno.fsa.adj[i].end(); ) { long from = it->first.first, to = it->first.second, v = it->second; while (++it != anno.fsa.adj[i].end() && to == it->first.first && it->second == v) to = it->first.second; fprintf(stderr, " (%ld,%ld)\n", from, to-1); } return false; } for (auto aa: anno.assoc[i]) if (has_start(aa.second)) if (auto* e = dynamic_cast(aa.first)) // unique call_addr[i] = {stmt2start[e->define_stmt], anno.fsa.adj[i][0].second}; } DP(3, "Removing action/CallExpr labels"); REP(i, anno.fsa.n()) { long j = anno.fsa.adj[i].size(); while (j && action_label_base < anno.fsa.adj[i][j-1].first.second) if (anno.fsa.adj[i][j-1].first.first < action_label_base) anno.fsa.adj[i][j-1].first.second = action_label_base; else j--; anno.fsa.adj[i].resize(j); } return true; } //// Graphviz dot renderer void generate_graphviz(Module* mo) { fprintf(output, "// Generated by 偃师, %s\n", mo->filename.c_str()); for (Stmt* x = mo->toplevel; x; x = x->next) if (auto stmt = dynamic_cast(x)) { if (stmt->export_) { FsaAnno& anno = compiled[stmt]; fprintf(output, "digraph \"%s\" {\n", mo->filename.c_str()); bool start_is_final = false; // finals indent(output, 1); fprintf(output, "node[shape=doublecircle,color=olivedrab1,style=filled,fontname=Monospace];"); for (long f: anno.fsa.finals) if (f == anno.fsa.start) start_is_final = true; else fprintf(output, " %ld", f); fprintf(output, "\n"); // start indent(output, 1); if (start_is_final) fprintf(output, "node[shape=doublecircle,color=orchid];"); else fprintf(output, "node[shape=circle,color=orchid];"); fprintf(output, " %ld\n", anno.fsa.start); // other states indent(output, 1); fprintf(output, "node[shape=circle,color=black,style=\"\"]\n"); // edges REP(u, anno.fsa.n()) { unordered_map labels; bool first = true; auto it = anno.fsa.adj[u].begin(); for (; it != anno.fsa.adj[u].end(); ++it) { stringstream& lb = labels[it->second]; if (! lb.str().empty()) lb << ','; if (it->first.first == it->first.second-1) lb << it->first.first; else lb << it->first.first << '-' << it->first.second-1; } for (auto& lb: labels) { indent(output, 1); fprintf(output, "%ld -> %ld[label=\"%s\"]\n", u, lb.first, lb.second.str().c_str()); } } } } fprintf(output, "}\n"); } //// C++ renderer static void generate_final(const char* name, const vector& final) { // comment fprintf(output, " //static const long %sfinals[] = {", name); bool first = true; REP(i, final.size()) if (final[i]) { if (first) first = false; else fprintf(output, ","); fprintf(output, "%ld", i); } fprintf(output, "};\n"); first = true; fprintf(output, " static const unsigned long %sfinal[] = {", name); for (long j = 0, i = 0; i < final.size(); i += CHAR_BIT*sizeof(long)) { ulong mask = 0; for (; j < final.size() && j < i+CHAR_BIT*sizeof(long); j++) if (final[j]) { mask |= 1uL << (j-i); } if (i) fprintf(output, ","); fprintf(output, "%#lx", mask); } fprintf(output, "};\n"); } static void generate_cxx_export(DefineStmt* stmt) { FsaAnno& anno = compiled[stmt]; // yanshi_%s_init if (output_header) fprintf(output_header, "extern long yanshi_%s_start;\n", stmt->lhs.c_str()); fprintf(output, "long yanshi_%s_start = %ld;\n\n", stmt->lhs.c_str(), anno.fsa.start); // yanshi_%s_is_final if (output_header) { if (opt_gen_extern_c) fputs("extern \"C\" ", output_header); fprintf(output_header, opt_gen_c ? "bool yanshi_%s_is_final(const long* ret_stack, long ret_stack_len, long u);\n" : "bool yanshi_%s_is_final(const vector& ret_stack, long u);\n" , stmt->lhs.c_str()); } if (opt_gen_extern_c) fputs("extern \"C\" ", output); fprintf(output, opt_gen_c ? "bool yanshi_%s_is_final(const long* ret_stack, long ret_stack_len, long u)\n" : "bool yanshi_%s_is_final(const vector& ret_stack, long u)\n" , stmt->lhs.c_str()); fprintf(output, "{\n"); vector final(anno.fsa.n()); for (long f: anno.fsa.finals) final[f] = true; generate_final("", final); generate_final("sub_", stmt2final[stmt]); fprintf(output, opt_gen_c ? " for (long i = ret_stack_len; i; u = ret_stack[--i])\n" : " for (auto i = ret_stack.size(); i; u = ret_stack[--i])\n" ); fprintf(output, " if (! (0 <= u && u < %ld && sub_final[u/(CHAR_BIT*sizeof(long))] >> (u%%(CHAR_BIT*sizeof(long))) & 1))\n" " return false;\n" " return 0 <= u && u < %ld && final[u/(CHAR_BIT*sizeof(long))] >> (u%%(CHAR_BIT*sizeof(long))) & 1;\n" "};\n\n" , anno.fsa.n() , anno.fsa.n() ); generate_transitions(stmt); } void generate_cxx(Module* mo) { fprintf(output, "// Generated by 偃师, %s\n", mo->filename.c_str()); fprintf(output, "#include \n"); if (! opt_gen_c) { fprintf(output, "#include \n"); fprintf(output, "using namespace std;\n"); } else { fprintf(output, "#include \n"); } if (opt_standalone) { fputs( "#include \n" "#include \n" "#include \n" "#include \n" "#include \n" "#include \n" "#include \n" "#include \n" "#include \n" "#include \n" "#include \n" "using namespace std;\n" , output); } if (output_header) { fputs("#pragma once\n", output_header); if (! opt_gen_c) { fprintf(output_header, "#include \n"); fprintf(output_header, "using std::vector;\n"); } else { fprintf(output_header, "#include \n"); } } fprintf(output, "\n"); DefineStmt* main_export = NULL; for (Stmt* x = mo->toplevel; x; x = x->next) if (auto xx = dynamic_cast(x)) { if (xx->export_) { if (! main_export) main_export = xx; generate_cxx_export(xx); } } else if (auto xx = dynamic_cast(x)) fprintf(output, "%s", xx->code.c_str()); if (opt_standalone && main_export) { fprintf(output, "\n" "int main(int argc, char* argv[])\n" "{\n" " setlocale(LC_ALL, \"\");\n" " string utf8;\n" " const char* p;\n" " long c, u = yanshi_%s_start, pref = 0;\n" , main_export->lhs.c_str()); if (opt_gen_c) fprintf(output, " long ret_stack[%ld], ret_stack_len = 0;\n", opt_max_return_stack); else fprintf(output, " vector ret_stack;\n"); fprintf(output, " if (argc == 2)\n" " utf8 = argv[1];\n" " else {\n" " FILE* f = argc == 1 ? stdin : fopen(argv[1], \"r\");\n" " while ((c = fgetc(f)) != EOF)\n" " utf8 += c;\n" " fclose(f);\n" " }\n" " u32string utf32 = wstring_convert, char32_t>{}.from_bytes(utf8);\n" ); fprintf(output, opt_gen_c ? " printf(\"\\033[%%s33m%%ld \\033[m\", yanshi_%s_is_final(ret_stack, ret_stack_len, u) ? \"1;\" : \"\", u);\n" : " printf(\"\\033[%%s33m%%ld \\033[m\", yanshi_%s_is_final(ret_stack, u) ? \"1;\" : \"\", u);\n" , main_export->lhs.c_str() ); fprintf(output, " for (char32_t c: utf32) {\n"); fprintf(output, opt_gen_c ? " u = yanshi_%s_transit(ret_stack, &ret_stack_len, u, c);\n" : " u = yanshi_%s_transit(ret_stack, u, c);\n" , main_export->lhs.c_str()); fprintf(output, " if (c > WCHAR_MAX || iswcntrl(c)) printf(\"%%\" PRIuLEAST32 \" \", c);\n" " else cout << wstring_convert, char32_t>{}.to_bytes(c) << ' ';\n"); fprintf(output, opt_gen_c ? " printf(\"\\033[%%s33m%%ld \\033[m\", yanshi_%s_is_final(ret_stack, ret_stack_len, u) ? \"1;\" : \"\", u);\n" : " printf(\"\\033[%%s33m%%ld \\033[m\", yanshi_%s_is_final(ret_stack, u) ? \"1;\" : \"\", u);\n" , main_export->lhs.c_str()); fprintf(output, " if (u < 0) break;\n" " pref++;\n" " }\n"); fprintf(output, opt_gen_c ? " printf(\"\\nlen: %%zd\\npref: %%ld\\nstate: %%ld\\nfinal: %%s\\n\", utf32.size(), pref, u, yanshi_%s_is_final(ret_stack, ret_stack_len, u) ? \"true\" : \"false\");\n" "}\n" : " printf(\"\\nlen: %%zd\\npref: %%ld\\nstate: %%ld\\nfinal: %%s\\n\", utf32.size(), pref, u, yanshi_%s_is_final(ret_stack, u) ? \"true\" : \"false\");\n" "}\n" , main_export->lhs.c_str()); } } ================================================ FILE: yanshi/src/compiler.hh ================================================ #pragma once #include "fsa_anno.hh" #include "syntax.hh" #include using std::unordered_map; void print_assoc(const FsaAnno& anno); void print_automaton(const Fsa& fsa); void compile(DefineStmt*); bool compile_export(DefineStmt* stmt); void generate_cxx(Module* mo); void generate_graphviz(Module* mo); extern unordered_map compiled; ================================================ FILE: yanshi/src/fsa.cc ================================================ #include "common.hh" #include "fsa.hh" #include "option.hh" #include #include #include #include #include #include #include #include #include #include #include using namespace std; namespace std { template struct hash> { size_t operator()(const vector& v) const { hash h; size_t r = 0; for (auto x: v) r = r*17+h(x); return r; } }; } void Fsa::check() const { REP(i, n()) FOR(j, 1, adj[i].size()) assert(adj[i][j-1].first.second == 0 && adj[i][j].first.second == 0 || adj[i][j-1].first.second <= adj[i][j].first.first); } bool Fsa::has(long u, long c) const { auto it = upper_bound(ALL(adj[u]), make_pair(make_pair(c, LONG_MAX), LONG_MAX)); return it != adj[u].begin() && c < (--it)->first.second; } bool Fsa::has_call(long u) const { auto it = upper_bound(ALL(adj[u]), make_pair(make_pair(call_label_base, LONG_MAX), LONG_MAX)); return (it != adj[u].end() && it->first.first < call_label) || (it != adj[u].begin() && call_label_base < (--it)->first.second); } bool Fsa::has_call_or_collapse(long u) const { auto it = upper_bound(ALL(adj[u]), make_pair(make_pair(call_label_base, LONG_MAX), LONG_MAX)); return it != adj[u].end() || (it != adj[u].begin() && call_label_base < (--it)->first.second); } long Fsa::transit(long u, long c) const { auto it = upper_bound(ALL(adj[u]), make_pair(make_pair(c, LONG_MAX), LONG_MAX)); return it != adj[u].begin() && c < (--it)->first.second ? it->second : -1; } bool Fsa::is_final(long x) const { return binary_search(ALL(finals), x); } void Fsa::epsilon_closure(vector& src) const { static vector vis; if (n() > vis.size()) vis.resize(n()); for (long i: src) vis[i] = true; REP(i, src.size()) { long u = src[i]; for (auto& e: adj[u]) { if (-1 < e.first.first) break; if (! vis[e.second]) { vis[e.second] = true; src.push_back(e.second); } } } for (long i: src) vis[i] = false; sort(ALL(src)); } Fsa Fsa::operator~() const { long accept = n(); Fsa r; r.start = start; r.adj.resize(accept+1); REP(i, accept) { long j = 0; for (auto& e: adj[i]) { if (j < e.first.first) r.adj[i].emplace_back(make_pair(j, e.first.first), accept); r.adj[i].emplace_back(e.first, e.second); j = e.first.second; } if (j < AB) r.adj[i].emplace_back(make_pair(j, AB), accept); } r.adj[accept].emplace_back(make_pair(0, AB), accept); vector new_finals; auto j = finals.begin(); REP(i, accept+1) { while (j != finals.end() && *j < i) ++j; if (j == finals.end() || *j != i) new_finals.push_back(i); } r.finals = move(new_finals); return r; } void Fsa::accessible(const vector* starts, function relate) { vector q{start}, id(n(), 0); id[start] = 1; if (starts) for (long u: *starts) if (! id[u]) { id[u] = 1; q.push_back(u); } REP(i, q.size()) { long u = q[i]; for (auto& e: adj[u]) { //if (e.first.first >= AB) break; if (! id[e.second]) { id[e.second] = 1; q.push_back(e.second); } } } long j = 0; REP(i, n()) id[i] = id[i] ? j++ : -1; auto it = finals.begin(), it2 = it; REP(i, n()) if (id[i] >= 0) { relate(i); if (start == i) start = id[i]; while (it != finals.end() && *it < i) ++it; if (it != finals.end() && *it == i) *it2++ = id[i]; long k = 0; for (auto& e: adj[i]) if (id[e.second] >= 0) adj[i][k++] = {e.first, id[e.second]}; // unordered unless deterministic adj[i].resize(k); if (id[i] != i) adj[id[i]] = move(adj[i]); } finals.erase(it2, finals.end()); adj.resize(j); } void Fsa::co_accessible(const vector*final, function relate) { vector> radj(n()); REP(i, n()) for (auto& e: adj[i]) { //if (e.first.first >= AB) break; radj[e.second].push_back(i); } REP(i, n()) sort(ALL(radj[i])); vector q = finals, id(n(), 0); for (long f: finals) id[f] = 1; if (final) REP(i, n()) if ((*final)[i] && ! id[i]) { id[i] = 1; q.push_back(i); } REP(i, q.size()) { long u = q[i]; for (auto& v: radj[u]) if (! id[v]) { id[v] = 1; q.push_back(v); } } if (! id[start]) { start = 0; finals.clear(); adj.assign(1, {}); return; } long j = 0; REP(i, n()) id[i] = id[i] ? j++ : -1; auto it = finals.begin(), it2 = it; REP(i, n()) if (id[i] >= 0) { relate(i); if (start == i) start = id[i]; while (it != finals.end() && *it < i) ++it; if (it != finals.end() && *it == i) *it2++ = id[i]; long k = 0; for (auto& e: adj[i]) if (id[e.second] >= 0) adj[i][k++] = {e.first, id[e.second]}; // unordered unless deterministic adj[i].resize(k); if (id[i] != i) adj[id[i]] = move(adj[i]); } finals.erase(it2, finals.end()); adj.resize(j); } Fsa Fsa::difference(const Fsa& rhs, function relate) const { Fsa r; vector> q; unordered_map m; q.emplace_back(start, rhs.start); m[(rhs.n()+1) * start + rhs.start] = 0; r.start = 0; REP(i, q.size()) { long u0, u1, v0, v1; tie(u0, u1) = q[i]; if (is_final(u0) && ! rhs.is_final(u1)) r.finals.push_back(i); r.adj.emplace_back(); relate(u0); vector::const_iterator it0 = adj[u0].begin(), it1, it1e; if (u1 == rhs.n()) it1 = it1e = rhs.adj[0].end(); else { it1 = rhs.adj[u1].begin(); it1e = rhs.adj[u1].end(); } long last = LONG_MIN; while (it0 != adj[u0].end()) { long from = max(last, it0->first.first), to = it0->first.second; while (it1 != it1e && it1->first.second <= from) ++it1; if (it1 != it1e) to = min(to, from < it1->first.first ? it1->first.first : it1->first.second); last = to; long v1 = it1 != it1e && it1->first.first <= from ? it1->second : rhs.n(), t = (rhs.n()+1) * it0->second + v1; auto mit = m.find(t); if (mit == m.end()) { mit = m.emplace(t, m.size()).first; q.emplace_back(it0->second, v1); } r.adj[i].emplace_back(make_pair(from, to), mit->second); if (to == it0->first.second) ++it0; } } return r; } Fsa Fsa::intersect(const Fsa& rhs, function relate) const { Fsa r; vector> q; long u0, u1, v0, v1; unordered_map m; q.emplace_back(start, rhs.start); m[rhs.n() * start + rhs.start] = 0; r.start = 0; REP(i, q.size()) { tie(u0, u1) = q[i]; if (is_final(u0) && rhs.is_final(u1)) r.finals.push_back(i); r.adj.emplace_back(); relate(u0, u1); auto it0 = adj[u0].begin(), it1 = rhs.adj[u1].begin(); while (it0 != adj[u0].end() && it1 != rhs.adj[u1].end()) { if (it0->first.second <= it1->first.first) ++it0; else if (it1->first.second <= it0->first.first) ++it1; else { long t = rhs.n() * it0->second + it1->second; auto mit = m.find(t); if (mit == m.end()) { mit = m.emplace(t, m.size()).first; q.emplace_back(it0->second, it1->second); } r.adj[i].emplace_back(make_pair(max(it0->first.first, it1->first.first), min(it0->first.second, it1->first.second)), mit->second); if (it0->first.second < it1->first.second) ++it0; else if (it0->first.second > it1->first.second) ++it1; else ++it0, ++it1; } } } return r; } Fsa Fsa::determinize(const vector* starts, function&)> relate) const { Fsa r; r.start = 0; unordered_map, long> m; vector::const_iterator> its(n()); vector vs{start}; vector> events; stack> st; epsilon_closure(vs); m[vs] = 0; st.push(move(vs)); if (starts) for (long u: *starts) { vs.assign(1, u); epsilon_closure(vs); if (! m.count(vs)) { m.emplace(vs, m.size()); st.push(move(vs)); } } while (st.size()) { vector x = move(st.top()); st.pop(); long id = m[x]; if (id+1 > r.adj.size()) r.adj.resize(id+1); relate(id, x); bool final = false; events.clear(); for (long u: x) { if (is_final(u)) final = true; for (auto& e: adj[u]) { events.emplace_back(e.first.first, e.second); events.emplace_back(e.first.second, ~ e.second); } } if (final) r.finals.push_back(id); long last = 0; multiset live; sort(ALL(events)); for (auto& ev: events) { if (last < ev.first) { if (live.size()) { vs.assign(ALL(live)); vs.erase(unique(ALL(vs)), vs.end()); epsilon_closure(vs); auto mit = m.find(vs); if (mit == m.end()) { mit = m.emplace(vs, m.size()).first; st.push(vs); } if (r.adj[id].size() && r.adj[id].back().first.second == last && r.adj[id].back().second == mit->second) // coalesce two edges r.adj[id].back().first.second = ev.first; else r.adj[id].emplace_back(make_pair(last, ev.first), mit->second); } last = ev.first; } if (ev.second >= 0) live.insert(ev.second); else live.erase(live.find(~ ev.second)); } } sort(ALL(r.finals)); return r; } Fsa Fsa::distinguish(function&)> relate) const { vector scale; REP(i, n()) for (auto& e: adj[i]) { scale.push_back(e.first.first); scale.push_back(e.first.second); } sort(ALL(scale)); scale.erase(unique(ALL(scale)), scale.end()); vector>> radj(n()); REP(i, n()) for (auto& e: adj[i]) { long from = lower_bound(ALL(scale), e.first.first) - scale.begin(), to = lower_bound(ALL(scale), e.first.second) - scale.begin(); FOR(j, from, to) radj[e.second].emplace_back(j, i); } REP(i, n()) sort(ALL(radj[i])); vector L(n()), R(n()), B(n()), C(n(), 0), CC(n(), 0); vector mark(n(), false); // distinguish finals & non-finals long fx = -1, x = -1, fy = -1, y = -1, j = 0; REP(i, n()) if (j < finals.size() && finals[j] == i) { j++; if (y < 0) fy = i; else R[y] = i; C[B[i] = fy]++; L[i] = y; y = i; } else { if (x < 0) fx = i; else R[x] = i; C[B[i] = fx]++; L[i] = x; x = i; } if (x >= 0) L[fx] = x, R[x] = fx; if (y >= 0) L[fy] = y, R[y] = fy; set> refines; auto labels = [&](long fx) { vector lb; for (long x = fx; ; ) { for (auto& e: radj[x]) lb.push_back(e.first); if ((x = R[x]) == fx) break; } sort(ALL(lb)); lb.erase(unique(ALL(lb)), lb.end()); return lb; }; if (fx >= 0) for (long a: labels(fx)) refines.emplace(a, fx); if (fy >= 0) for (long a: labels(fy)) refines.emplace(a, fy); while (refines.size()) { long a; tie(a, fx) = *refines.begin(); refines.erase(refines.begin()); // count vector bs; for (x = fx; ; ) { auto it = lower_bound(ALL(radj[x]), make_pair(a, 0L)), ite = upper_bound(ALL(radj[x]), make_pair(a, n())); for (; it != ite; ++it) { y = it->second; if (! CC[B[y]]++) bs.push_back(B[y]); mark[y] = true; } if ((x = R[x]) == fx) break; } // for each refinable set for (long fy: bs) { if (CC[fy] < C[fy]) { long fu = -1, u = -1, cu = 0, fv = -1, v = -1, cv = 0; vector lb = labels(fy); for (long i = fy; ; ) { if (mark[i]) { mark[i] = false; if (u < 0) C[fu = i] = 0; else R[u] = i; C[fu]++; B[i] = fu; L[i] = u; u = i; } else { if (v < 0) C[fv = i] = 0; else R[v] = i; C[fv]++; B[i] = fv; L[i] = v; v = i; } if ((i = R[i]) == fy) break; } L[fu] = u, R[u] = fu; L[fv] = v, R[v] = fv; //REP(a, AB+1) for (long a: lb) if (refines.count({a, fy})) refines.emplace(a, fu != fy ? fu : fv); else refines.emplace(a, C[fu] < C[fv] ? fu : fv); } else for (long i = fy; ; ) { mark[i] = false; if ((i = R[i]) == fy) break; } CC[fy] = 0; } // clear marks for (x = fx; ; ) { auto it = lower_bound(ALL(radj[x]), make_pair(a, 0L)), ite = upper_bound(ALL(radj[x]), make_pair(a, n())); for (; it != ite; ++it) { y = it->second; CC[B[y]] = 0; mark[y] = false; } if ((x = R[x]) == fx) break; } } Fsa r; long nn = 0; vector vs; REP(i, n()) if (B[i] == i) { vs.clear(); for (long j = i; ; ) { B[j] = nn; vs.push_back(j); if ((j = R[j]) == i) break; } relate(vs); if (binary_search(ALL(finals), i)) r.finals.push_back(nn); nn++; } r.start = B[start]; r.adj.resize(nn); REP(i, n()) for (auto& e: adj[i]) r.adj[B[i]].emplace_back(e.first, B[e.second]); REP(i, nn) { // merge edges with the same destination sort(ALL(r.adj[i]), [](const Edge& x, const Edge& y) { return x.second != y.second ? x.second < y.second : x.first < y.first; }); auto it2 = r.adj[i].begin(); for (auto it = r.adj[i].begin(); it != r.adj[i].end(); ) { long v = it->second, from = it->first.first, to = it->first.second; while (++it != r.adj[i].end() && it->second == v) if (it->first.first <= to) to = max(to, it->first.second); else { *it2++ = make_pair(make_pair(from, to), v); tie(from, to) = it->first; } *it2++ = make_pair(make_pair(from, to), v); } r.adj[i].erase(it2, r.adj[i].end()); sort(ALL(r.adj[i])); } return r; } ================================================ FILE: yanshi/src/fsa.hh ================================================ #pragma once #include #include #include using std::function; using std::pair; using std::vector; typedef pair Label; typedef pair Edge; const Label epsilon{-1L, 0L}; struct Fsa { long start; vector finals; // sorted vector> adj; // sorted void check() const; long n() const { return adj.size(); } bool is_final(long x) const; bool has(long u, long c) const; bool has_call(long u) const; bool has_call_or_collapse(long u) const; long transit(long u, long c) const; void epsilon_closure(vector& src) const; Fsa operator~() const; // a -> a void accessible(const vector* starts, function relate); // a -> a void co_accessible(const vector* final, function relate); // DFA -> DFA -> DFA Fsa intersect(const Fsa& rhs, function relate) const; // DFA -> DFA -> DFA Fsa difference(const Fsa& rhs, function relate) const; // DFA -> DFA Fsa distinguish(function&)> relate) const; // * -> DFA Fsa determinize(const vector* starts, function&)> relate) const; }; ================================================ FILE: yanshi/src/fsa_anno.cc ================================================ #include "common.hh" #include "compiler.hh" #include "fsa_anno.hh" #include "loader.hh" #include "option.hh" #include #include #include #include #include using namespace std; bool operator<(ExprTag x, ExprTag y) { return long(x) < long(y); } bool assoc_has_expr(vector>& as, Expr* x) { auto it = lower_bound(ALL(as), make_pair(x, ExprTag(0))); return it != as.end() && it->first == x; } void sort_assoc(vector>& as) { sort(ALL(as)); auto i = as.begin(), j = i, k = i; for (; i != as.end(); i = j) { while (++j != as.end() && i->first == j->first) i->second = ExprTag(long(i->second) | long(j->second)); *k++ = *i; } as.erase(k, as.end()); } void FsaAnno::add_assoc(Expr& expr) { // has actions: actions need tags to differentiate 'entering', 'leaving', ... // 'intact': states with the 'inner' tag cannot be connected to start/final in substring grammar // 'CallExpr' 'CollapseExpr': differentiate states representing 'CallExpr' 'CollapseExpr' (u, special, v) // 'opt_mode': displaying possible positions for given strings in interactive mode if (expr.no_action() && ! expr.stmt->intact && ! dynamic_cast(&expr) && ! dynamic_cast(&expr) && opt_mode != Mode::interactive) return; auto j = fsa.finals.begin(); REP(i, fsa.n()) { ExprTag tag = ExprTag(0); if (i == fsa.start) tag = ExprTag::start; while (j != fsa.finals.end() && *j < i) ++j; if (j != fsa.finals.end() && *j == i) tag = ExprTag(long(tag) | long(ExprTag::final)); if (tag == ExprTag(0)) tag = ExprTag::inner; sorted_insert(assoc[i], make_pair(&expr, tag)); } // Add pseudo transitions with labels [ACTION_LABEL_BASE, COLLAPSE_LABEL_BASE) to prevent its merge with other states if (expr.leaving.size() || expr.entering.size() || expr.transiting.size()) for (auto action: expr.transiting) REP(i, fsa.n()) { fsa.adj[i].emplace_back(make_pair(action_label, action_label+1), i); action_label++; } else if (expr.finishing.size()) for (long f: fsa.finals) { fsa.adj[f].emplace_back(make_pair(action_label, action_label+1), f); action_label++; } } void FsaAnno::accessible(const vector* starts, vector& mapping) { long allo = 0; auto relate = [&](long x) { if (allo != x) assoc[allo] = move(assoc[x]); allo++; mapping.push_back(x); }; fsa.accessible(starts, relate); assoc.resize(allo); } void FsaAnno::co_accessible(const vector* final, vector& mapping) { long allo = 0; auto relate = [&](long x) { if (allo != x) assoc[allo] = move(assoc[x]); allo++; mapping.push_back(x); }; fsa.co_accessible(final, relate); if (fsa.finals.empty()) { // 'start' does not produce acceptable strings assoc.assign(1, {}); mapping.assign(1, 0); deterministic = true; return; } if (! deterministic) REP(i, fsa.n()) sort(ALL(fsa.adj[i])); assoc.resize(allo); } void FsaAnno::complement(ComplementExpr* expr) { if (! deterministic) fsa = fsa.determinize(NULL, [&](long, const vector&){}); fsa = ~ fsa; assoc.assign(fsa.n(), {}); deterministic = true; } void FsaAnno::concat(FsaAnno& rhs, ConcatExpr* expr) { long ln = fsa.n(), rn = rhs.fsa.n(); for (long f: fsa.finals) emplace_front(fsa.adj[f], epsilon, ln+rhs.fsa.start); for (auto& es: rhs.fsa.adj) { for (auto& e: es) e.second += ln; fsa.adj.emplace_back(move(es)); } fsa.finals = move(rhs.fsa.finals); for (long& f: fsa.finals) f += ln; assoc.resize(fsa.n()); REP(i, rhs.fsa.n()) assoc[ln+i] = move(rhs.assoc[i]); if (expr) add_assoc(*expr); deterministic = false; } void FsaAnno::determinize(const vector* starts, vector>* mapping) { if (deterministic) return; decltype(assoc) new_assoc; auto relate = [&](long id, const vector& xs) { if (id+1 > new_assoc.size()) { new_assoc.resize(id+1); if (mapping) mapping->resize(id+1); } auto& as = new_assoc[id]; for (long x: xs) as.insert(as.end(), ALL(assoc[x])); sort_assoc(as); if (mapping) (*mapping)[id] = xs; }; fsa = fsa.determinize(starts, relate); assoc = move(new_assoc); deterministic = true; } void FsaAnno::difference(FsaAnno& rhs, DifferenceExpr* expr) { vector> rel0; decltype(rhs.assoc) new_assoc; auto relate0 = [&](long id, const vector& xs) { if (id+1 > rel0.size()) rel0.resize(id+1); rel0[id] = xs; }; auto relate = [&](long x) { if (rel0.empty()) new_assoc.emplace_back(assoc[x]); else { new_assoc.emplace_back(); auto& as = new_assoc.back(); for (long u: rel0[x]) as.insert(as.end(), ALL(assoc[u])); sort_assoc(as); } }; if (! deterministic) fsa = fsa.determinize(NULL, relate0); if (! rhs.deterministic) rhs.fsa = rhs.fsa.determinize(NULL, [](long, const vector&) {}); fsa = fsa.difference(rhs.fsa, relate); assoc = move(new_assoc); if (expr) add_assoc(*expr); deterministic = true; } FsaAnno FsaAnno::epsilon_fsa(EpsilonExpr* expr) { FsaAnno r; r.fsa.start = 0; r.fsa.finals.push_back(0); r.fsa.adj.resize(1); r.assoc.resize(1); if (expr) r.add_assoc(*expr); r.deterministic = true; return r; } void FsaAnno::intersect(FsaAnno& rhs, IntersectExpr* expr) { decltype(rhs.assoc) new_assoc; vector> rel0, rel1; auto relate0 = [&](long id, const vector& xs) { if (id+1 > rel0.size()) rel0.resize(id+1); rel0[id] = xs; }; auto relate1 = [&](long id, const vector& xs) { if (id+1 > rel1.size()) rel1.resize(id+1); rel1[id] = xs; }; auto relate = [&](long x, long y) { new_assoc.emplace_back(); auto& as = new_assoc.back(); if (rel0.empty()) as.insert(as.end(), ALL(assoc[x])); else for (long u: rel0[x]) as.insert(as.end(), ALL(assoc[u])); if (rel1.empty()) as.insert(as.end(), ALL(rhs.assoc[y])); else for (long v: rel1[y]) as.insert(as.end(), ALL(rhs.assoc[v])); sort_assoc(as); }; if (! deterministic) fsa = fsa.determinize(NULL, relate0); if (! rhs.deterministic) rhs.fsa = rhs.fsa.determinize(NULL, relate1); fsa = fsa.intersect(rhs.fsa, relate); assoc = move(new_assoc); if (expr) add_assoc(*expr); deterministic = true; } void FsaAnno::minimize(vector>* mapping) { assert(deterministic); decltype(assoc) new_assoc; auto relate = [&](vector& xs) { new_assoc.emplace_back(); auto& as = new_assoc.back(); for (long x: xs) as.insert(as.end(), ALL(assoc[x])); sort_assoc(as); if (mapping) mapping->push_back(xs); }; fsa = fsa.distinguish(relate); assoc = move(new_assoc); } void FsaAnno::union_(FsaAnno& rhs, UnionExpr* expr) { long ln = fsa.n(), rn = rhs.fsa.n(), src = ln+rn, old_lsrc = fsa.start; fsa.start = src; for (long f: rhs.fsa.finals) fsa.finals.push_back(ln+f); for (auto& es: rhs.fsa.adj) { for (auto& e: es) e.second += ln; fsa.adj.emplace_back(move(es)); } fsa.adj.emplace_back(); fsa.adj[src].emplace_back(epsilon, old_lsrc); fsa.adj[src].emplace_back(epsilon, ln+rhs.fsa.start); assoc.resize(fsa.n()); REP(i, rhs.fsa.n()) assoc[ln+i] = move(rhs.assoc[i]); if (expr) add_assoc(*expr); deterministic = false; } void FsaAnno::plus(PlusExpr* expr) { for (long f: fsa.finals) emplace_front(fsa.adj[f], epsilon, fsa.start); if (expr) add_assoc(*expr); deterministic = false; } void FsaAnno::question(QuestionExpr* expr) { long src = fsa.n(), sink = src+1, old_src = fsa.start; fsa.start = src; fsa.adj.emplace_back(); fsa.adj.emplace_back(); fsa.adj[src].emplace_back(epsilon, old_src); fsa.adj[src].emplace_back(epsilon, sink); fsa.finals.push_back(sink); assoc.resize(fsa.n()); if (expr) add_assoc(*expr); deterministic = false; } void FsaAnno::repeat(RepeatExpr& expr) { FsaAnno r = epsilon_fsa(NULL); REP(i, expr.low) { FsaAnno t = *this; r.concat(t, NULL); } if (expr.high == LONG_MAX) { star(NULL); r.concat(*this, NULL); } else if (expr.low < expr.high) { FsaAnno rhs = epsilon_fsa(NULL), x = *this; ROF(i, 0, expr.high-expr.low) { FsaAnno t = x; rhs.union_(t, NULL); if (i) { t = *this; x.concat(t, NULL); } } r.concat(rhs, NULL); } r.deterministic = false; *this = move(r); } void FsaAnno::star(StarExpr* expr) { long src = fsa.n(), sink = src+1, old_src = fsa.start; fsa.start = src; fsa.adj.emplace_back(); fsa.adj.emplace_back(); fsa.adj[src].emplace_back(epsilon, old_src); fsa.adj[src].emplace_back(epsilon, sink); for (long f: fsa.finals) { sorted_emplace(fsa.adj[f], epsilon, old_src); sorted_emplace(fsa.adj[f], epsilon, sink); } fsa.finals.assign(1, sink); assoc.resize(fsa.n()); if (expr) add_assoc(*expr); deterministic = false; } FsaAnno FsaAnno::bracket(BracketExpr& expr) { FsaAnno r; r.fsa.start = 0; r.fsa.finals = {1}; r.fsa.adj.resize(2); for (auto& x: expr.intervals.to) r.fsa.adj[0].emplace_back(x, 1); r.assoc.resize(2); r.add_assoc(expr); r.deterministic = true; return r; } FsaAnno FsaAnno::call(CallExpr& expr) { // represented by (0, special, 1) FsaAnno r; r.fsa.start = 0; r.fsa.finals = {1}; r.fsa.adj.resize(2); r.fsa.adj[0].emplace_back(make_pair(call_label, call_label+1), 1); call_label++; r.assoc.resize(2); r.add_assoc(expr); r.deterministic = true; return r; } FsaAnno FsaAnno::collapse(CollapseExpr& expr) { // represented by (0, special, 1) FsaAnno r; r.fsa.start = 0; r.fsa.finals = {1}; r.fsa.adj.resize(2); r.fsa.adj[0].emplace_back(make_pair(collapse_label, collapse_label+1), 1); collapse_label++; r.assoc.resize(2); r.add_assoc(expr); r.deterministic = true; return r; } FsaAnno FsaAnno::dot(DotExpr* expr) { FsaAnno r; r.fsa.start = 0; r.fsa.finals = {1}; r.fsa.adj.resize(2); r.fsa.adj[0].emplace_back(make_pair(0L, AB), 1); r.assoc.resize(2); if (expr) r.add_assoc(*expr); r.deterministic = true; return r; } FsaAnno FsaAnno::embed(EmbedExpr& expr) { if (expr.define_stmt) { FsaAnno r = compiled[expr.define_stmt]; // change the labels to differentiate instances of CallExpr REP(i, r.fsa.n()) { auto it = upper_bound(ALL(r.fsa.adj[i]), make_pair(make_pair(call_label_base, LONG_MAX), LONG_MAX)); if (it != r.fsa.adj[i].begin() && call_label_base < (it-1)->first.second) --it; for (; it != r.fsa.adj[i].end() && it->first.first < call_label; ++it) { assert(call_label_base <= it->first.first); long t = it->first.second-it->first.first; it->first.first = call_label; call_label += t; it->first.second = call_label; assert(it->first.second <= call_label); } } r.add_assoc(expr); return r; } else { // macro FsaAnno r; r.fsa.start = 0; r.fsa.finals = {1}; r.fsa.adj.resize(2); r.fsa.adj[0].emplace_back(make_pair(expr.macro_value, expr.macro_value+1), 1); r.assoc.resize(2); r.add_assoc(expr); r.deterministic = true; return r; } } FsaAnno FsaAnno::literal(LiteralExpr& expr) { FsaAnno r; r.fsa.start = 0; long len = 0; if (opt_bytes) { len = expr.literal.size(); r.fsa.adj.resize(len+1); REP(i, expr.literal.size()) { long c = (u8)expr.literal[i]; r.fsa.adj[i].emplace_back(make_pair(c, c+1), i+1); } } else { for (i32 c, i = 0; i < expr.literal.size(); len++) { U8_NEXT_OR_FFFD(expr.literal.c_str(), i, expr.literal.size(), c); r.fsa.adj.emplace_back(); r.fsa.adj[len].emplace_back(make_pair(c, c+1), len+1); } r.fsa.adj.emplace_back(); } r.fsa.finals.push_back(len); r.assoc.resize(len+1); r.add_assoc(expr); r.deterministic = true; return r; } void FsaAnno::substring_grammar() { long src = fsa.n(), sink = src+1, old_src = fsa.start; fsa.start = src; fsa.adj.emplace_back(); fsa.adj.emplace_back(); REP(i, src) { bool ok = true; for (auto aa: assoc[i]) if (auto e = dynamic_cast(aa.first)) { if (e->define_stmt->intact && has_inner(aa.second)) { ok = false; break; } } else if (aa.first->stmt->intact && has_inner(aa.second)) { ok = false; break; } if (ok || i == old_src) fsa.adj[src].emplace_back(epsilon, i); if (ok || fsa.is_final(i)) emplace_front(fsa.adj[i], epsilon, sink); } fsa.finals.assign(1, sink); assoc.resize(fsa.n()); deterministic = false; } ================================================ FILE: yanshi/src/fsa_anno.hh ================================================ #pragma once #include "fsa.hh" #include "syntax.hh" enum class ExprTag { start = 1, inner = 2, final = 4, }; extern inline bool has_start(ExprTag x) { return long(x) & long(ExprTag::start); } extern inline bool has_inner(ExprTag x) { return long(x) & long(ExprTag::inner); } extern inline bool has_final(ExprTag x) { return long(x) & long(ExprTag::final); } bool operator<(ExprTag x, ExprTag y); bool assoc_has_expr(vector>& as, const Expr* x); struct FsaAnno { bool deterministic; Fsa fsa; vector>> assoc; void accessible(const vector* starts, vector& mapping); void add_assoc(Expr& expr); void complement(ComplementExpr* expr); void co_accessible(const vector* final, vector& mapping); void concat(FsaAnno& rhs, ConcatExpr* expr); void determinize(const vector* starts, vector>* mapping); void difference(FsaAnno& rhs, DifferenceExpr* expr); void intersect(FsaAnno& rhs, IntersectExpr* expr); void minimize(vector>* mapping); void plus(PlusExpr* expr); void question(QuestionExpr* expr); void repeat(RepeatExpr& expr); void star(StarExpr* expr); void substring_grammar(); void union_(FsaAnno& rhs, UnionExpr* expr); static FsaAnno bracket(BracketExpr& expr); static FsaAnno call(CallExpr& expr); static FsaAnno collapse(CollapseExpr& expr); static FsaAnno dot(DotExpr* expr); static FsaAnno embed(EmbedExpr& expr); static FsaAnno epsilon_fsa(EpsilonExpr* expr); static FsaAnno literal(LiteralExpr& expr); }; ================================================ FILE: yanshi/src/lexer.l ================================================ %{ #include "lexer_helper.hh" #include "option.hh" #include "parser.hh" #include "syntax.hh" #include #include #include #include #include using namespace std; #define YY_USER_ACTION \ do { \ yylloc->start = yyget_extra(yyscanner); \ yylloc->end = yylloc->start + yyleng; \ yyset_extra(yylloc->end, yyscanner); \ } while (0); static string tmp_bracket, tmp_str; static long tmp_str_pos; static bool semicolon; static long invalid_escape(YYSTYPE* yylval, const char* text) { yylval->errmsg = aprintf("invalid \\-escape: %s", text); return INVALID_CHARACTER; } static int invalid_escape_octonary(YYSTYPE* yylval, const char* text) { yylval->errmsg = aprintf("invalid number after \\-escape: %s", text); return INVALID_CHARACTER; } static int invalid_escape_x(YYSTYPE* yylval, const char* text) { yylval->errmsg = aprintf("invalid number after \\x-escape: %s", text); return INVALID_CHARACTER; } static int invalid_escape_u(YYSTYPE* yylval, const char* text) { yylval->errmsg = aprintf("invalid number after \\u-escape: %s", text); return INVALID_CHARACTER; } static int invalid_escape_U(YYSTYPE* yylval, const char* text) { yylval->errmsg = aprintf("invalid number after \\U-escape: %s", text); return INVALID_CHARACTER; } static void unexpected_eof(YYSTYPE* yylval, const char* token_end) { yylval->errmsg = aprintf("missing %s at end of file", token_end); } static void unexpected_newline(YYSTYPE* yylval, const char* token_end) { yylval->errmsg = aprintf("missing %s at end of line", token_end); } static int unexpected_codepoint(YYSTYPE* yylval) { yylval->errmsg = aprintf("cannot use Unicode codepoints"); return INVALID_CHARACTER; } extern "C" int raw_yywrap(yyscan_t yyscanner) { semicolon = false; return 1; } %} %option yywrap noinput %option reentrant %option extra-type="long" %option bison-bridge bison-locations %option prefix="raw_yy" %option stack %x EXPECT_CODE %x AFTER_ACTION_OP %x AFTER_EXPORT %x IN_BRACE %x IN_CODE %x IN_COMMENT %x IN_BRACKET %x IN_BRACKET_FIRST %x IN_LINE_COMMENT %s IN_PAREN %x IN_Q_STRING %x IN_QQ_STRING D [0-9] H [0-9A-Fa-f] L [a-zA-Z_\x80-\xff] %% "::" return COLONCOLON; ".." return DOTDOT; "&&" return AMPERAMPER; ";" if (semicolon) return '\n'; [-~!&*=+,.?|{}:] return yytext[0]; "action" yy_push_state(EXPECT_CODE, yyscanner); return ACTION; "as" return AS; "c++" yy_push_state(EXPECT_CODE, yyscanner); return CPP; "epsilon" return EPSILON; "export" yy_push_state(AFTER_EXPORT, yyscanner); return EXPORT; "import" return IMPORT; "intact" return INTACT; "semicolon" semicolon = true; "nosemicolon" semicolon = false; {L}({L}|{D})* yylval->str = new string(yytext); return IDENT; {D}+ yylval->integer = atol(yytext); return INTEGER; "#define" return PREPROCESS_DEFINE; "#" yy_push_state(IN_LINE_COMMENT, yyscanner); "//" yy_push_state(IN_LINE_COMMENT, yyscanner); { "\n" yy_pop_state(yyscanner); unput('\n'); yyset_extra(yylloc->end-1, yyscanner); <> yy_pop_state(yyscanner); . {} } "/*" yy_push_state(IN_COMMENT, yyscanner); { "*/" yy_pop_state(yyscanner); <> yy_pop_state(yyscanner); .|\n {} } "(" yy_push_state(IN_PAREN, yyscanner); return '('; ")" { if (YY_START != IN_PAREN) { unexpected_newline(yylval, ")"); return INVALID_CHARACTER; } yy_pop_state(yyscanner); return ')'; } "[" yy_push_state(IN_BRACKET_FIRST, yyscanner); return '['; { "^" BEGIN IN_BRACKET; return '^'; [^-\\\]\n] { yy_pop_state(yyscanner); yy_push_state(IN_BRACKET, yyscanner); yylval->integer = yytext[0]; return CHAR; } "-" { yylval->integer = '-'; return CHAR; } } { "]" { yy_pop_state(yyscanner); if (YY_START == INITIAL || YY_START == IN_PAREN) return ']'; } [^-\\\]\n] yylval->integer = yytext[0]; return CHAR; } { \\[0-7]+ { BEGIN IN_BRACKET; long c = strtol(yytext+1, NULL, 8); if (UCHAR_MAX < c) return invalid_escape_octonary(yylval, yytext); yylval->integer = c; return CHAR; } \\u[0-9a-fA-F]+ { BEGIN IN_BRACKET; if (opt_bytes) return unexpected_codepoint(yylval); long c = strtol(yytext+2, NULL, 16), len = 0; if (UINT16_MAX < c) return invalid_escape_u(yylval, yytext); yylval->integer = c; return CHAR; } \\U[0-9a-fA-F]+ { BEGIN IN_BRACKET; if (opt_bytes) return unexpected_codepoint(yylval); long c = strtol(yytext+2, NULL, 16), len = 0; if (MAX_CODEPOINT < c) return invalid_escape_U(yylval, yytext); yylval->integer = c; return CHAR; } \\x[0-9a-fA-F]+ { BEGIN IN_BRACKET; long c = strtol(yytext+2, NULL, 16); if (UCHAR_MAX < c) return invalid_escape_x(yylval, yytext); yylval->integer = c; return CHAR; } \\a BEGIN IN_BRACKET; yylval->integer = '\a'; return CHAR; \\b BEGIN IN_BRACKET; yylval->integer = '\b'; return CHAR; \\f BEGIN IN_BRACKET; yylval->integer = '\f'; return CHAR; \\n BEGIN IN_BRACKET; yylval->integer = '\n'; return CHAR; \\r BEGIN IN_BRACKET; yylval->integer = '\r'; return CHAR; \\t BEGIN IN_BRACKET; yylval->integer = '\t'; return CHAR; \\v BEGIN IN_BRACKET; yylval->integer = '\v'; return CHAR; \\. BEGIN IN_BRACKET; yylval->integer = yytext[1]; return CHAR; - BEGIN IN_BRACKET; return '-'; "\n" unexpected_newline(yylval, "]"); return INVALID_CHARACTER; <> yy_pop_state(yyscanner); unexpected_eof(yylval, "]"); } { // optional 'BRACED_CODE' to specify extra parameters "intact" yy_pop_state(yyscanner); return INTACT; {L}({L}|{D})* yy_pop_state(yyscanner); yylval->str = new string(yytext); return IDENT; "{" BEGIN IN_CODE; tmp_bracket.clear(); [ \t\n] {} <> yy_pop_state(yyscanner); } { {L}({L}|{D})* yylval->str = new string(yytext); return IDENT; "{" BEGIN IN_CODE; tmp_bracket.clear(); [ \t\n] {} <> yy_pop_state(yyscanner); } [>@%$] yy_push_state(AFTER_ACTION_OP, yyscanner); return yytext[0]; { -?{D}+ yylval->integer = atol(yytext); return INTEGER; {L}({L}|{D})* yy_pop_state(yyscanner); yylval->str = new string(yytext); return IDENT; "{" BEGIN IN_CODE; tmp_bracket.clear(); [ \t\n]+ {} <> yy_pop_state(yyscanner); . yylval->errmsg = strdup("invalid character"); return INVALID_CHARACTER; } { "'" { tmp_bracket += '\''; yy_push_state(IN_Q_STRING, yyscanner); } "\"" { tmp_bracket += '"'; yy_push_state(IN_QQ_STRING, yyscanner); } "{" { tmp_bracket += '{'; yy_push_state(IN_CODE, yyscanner); } "}" { yy_pop_state(yyscanner); if (YY_START == INITIAL || YY_START == IN_PAREN) { yylval->str = new string(tmp_bracket); return BRACED_CODE; } else tmp_bracket += '}'; } .|"\n" tmp_bracket += yytext[0]; <> yy_pop_state(yyscanner); unexpected_eof(yylval, "}"); } ' tmp_str.clear(); tmp_str_pos = yylloc->start; yy_push_state(IN_Q_STRING, yyscanner); "\"" tmp_str.clear(); tmp_str_pos = yylloc->start; yy_push_state(IN_QQ_STRING, yyscanner); { ' { yy_pop_state(yyscanner); if (YY_START == INITIAL || YY_START == IN_PAREN) { yylval->str = new string(tmp_str); yylloc->start = tmp_str_pos; return STRING_LITERAL; } tmp_bracket += yytext; } <> yy_pop_state(yyscanner); unexpected_eof(yylval, "'"); } { "\"" { yy_pop_state(yyscanner); if (YY_START == INITIAL || YY_START == IN_PAREN) { yylval->str = new string(tmp_str); yylloc->start = tmp_str_pos; return STRING_LITERAL; } tmp_bracket += yytext; } <> yy_pop_state(yyscanner); unexpected_eof(yylval, "\""); } { \\[0-7]+ { long c = strtol(yytext+1, NULL, 8); if (UCHAR_MAX < c) return invalid_escape_octonary(yylval, yytext); tmp_str.push_back(c); tmp_bracket += yytext; } \\x[0-9a-fA-F]+ { long c = strtol(yytext+2, NULL, 16); if (UCHAR_MAX < c) return invalid_escape_x(yylval, yytext); tmp_str.push_back(c); tmp_bracket += yytext; } \\u[0-9a-fA-F]+ { char s[4]; long c = strtol(yytext+2, NULL, 16), len = 0; if (UINT16_MAX < c) return invalid_escape_u(yylval, yytext); U8_APPEND_UNSAFE(s, len, c); tmp_str.insert(tmp_str.end(), s, s+len); tmp_bracket += yytext; } \\U[0-9a-fA-F]+ { char s[4]; long c = strtol(yytext+2, NULL, 16), len = 0; if (MAX_CODEPOINT < c) return invalid_escape_U(yylval, yytext); U8_APPEND_UNSAFE(s, len, c); tmp_str.insert(tmp_str.end(), s, s+len); tmp_bracket += yytext; } \\a tmp_str += '\a'; tmp_bracket += yytext; \\b tmp_str += '\b'; tmp_bracket += yytext; \\f tmp_str += '\f'; tmp_bracket += yytext; \\n tmp_str += '\n'; tmp_bracket += yytext; \\r tmp_str += '\r'; tmp_bracket += yytext; \\t tmp_str += '\t'; tmp_bracket += yytext; \\v tmp_str += '\v'; tmp_bracket += yytext; \\[\n\"\'?\\] tmp_str += yytext[1]; tmp_bracket += yytext; \\. return invalid_escape(yylval, yytext); .|\n tmp_str += yytext[0]; tmp_bracket += yytext[0]; } \\\n {} "\n" if (YY_START == INITIAL && ! semicolon) return '\n'; [ \t]+ {} . { yylval->errmsg = strdup("invalid character"); return INVALID_CHARACTER; } ================================================ FILE: yanshi/src/lexer_helper.cc ================================================ #include "common.hh" #include #include using namespace std; char* aprintf(const char* fmt, ...) { va_list va; va_start(va, fmt); char* r = NULL; vasprintf(&r, fmt, va); va_end(va); return r; } ================================================ FILE: yanshi/src/lexer_helper.hh ================================================ #pragma once char* aprintf(const char* fmt, ...) __attribute__((format(printf, 1, 2))); ================================================ FILE: yanshi/src/loader.cc ================================================ #include "common.hh" #include "compiler.hh" #include "loader.hh" #include "option.hh" #include "parser.hh" #include "repl.hh" #include #include #include #include #include #include #include #include #include #include #include using namespace std; static map, Module> inode2module; static unordered_map> depended_by; // key ranges over all DefineStmt map> used_as_call, used_as_collapse, used_as_embed; static DefineStmt* main_export; Module* main_module; FILE *output, *output_header; void print_module_info(Module& mo) { yellow(); printf("filename: %s\n", mo.filename.c_str()); cyan(); puts("qualified imports:"); sgr0(); for (auto& x: mo.qualified_import) printf(" %s as %s\n", x.second->filename.c_str(), x.first.c_str()); cyan(); puts("unqualified imports:"); sgr0(); for (auto& x: mo.unqualified_import) printf(" %s\n", x->filename.c_str()); cyan(); puts("defined actions:"); sgr0(); for (auto& x: mo.defined_action) printf(" %s\n", x.first.c_str()); cyan(); puts("defined:"); sgr0(); for (auto& x: mo.defined) printf(" %s\n", x.first.c_str()); } Stmt* resolve(Module& mo, const string qualified, const string& ident) { if (qualified.size()) { if (! mo.qualified_import.count(qualified)) return NULL; auto it = mo.qualified_import[qualified]->defined.find(ident); if (it == mo.qualified_import[qualified]->defined.end()) return NULL; return it->second; } else { Stmt* r = NULL; if (mo.macro.count(ident)) r = mo.macro[ident]; if (mo.defined.count(ident)) { if (r) return (Stmt*)1; r = mo.defined[ident]; } for (auto* import: mo.unqualified_import) { if (import->macro.count(ident)) { if (r) return (Stmt*)1; r = import->macro[ident]; } if (import->defined.count(ident)) { if (r) return (Stmt*)1; r = import->defined[ident]; } } return r; } } ActionStmt* resolve_action(Module& mo, const string qualified, const string& ident) { if (qualified.size()) { if (! mo.qualified_import.count(qualified)) return NULL; auto it = mo.qualified_import[qualified]->defined_action.find(ident); if (it == mo.qualified_import[qualified]->defined_action.end()) return NULL; return it->second; } else { ActionStmt* r = NULL; if (mo.defined_action.count(ident)) r = mo.defined_action[ident]; for (auto* import: mo.unqualified_import) if (import->defined_action.count(ident)) { if (r) return (ActionStmt*)1; r = import->defined_action[ident]; } return r; } } struct ModuleImportDef : PreorderStmtVisitor { Module& mo; long& n_errors; ModuleImportDef(Module& mo, long& n_errors) : mo(mo), n_errors(n_errors) {} void visit(ActionStmt& stmt) override { if (mo.defined_action.count(stmt.ident)) { n_errors++; mo.locfile.error(stmt.loc, "redefined '%s'", stmt.ident.c_str()); } else mo.defined_action[stmt.ident] = &stmt; } // TODO report error: import 'aa.hs' (#define d 3) ; #define d 4 void visit(DefineStmt& stmt) override { if (mo.defined.count(stmt.lhs) || mo.macro.count(stmt.lhs)) { n_errors++; mo.locfile.error(stmt.loc, "redefined '%s'", stmt.lhs.c_str()); } else { mo.defined.emplace(stmt.lhs, &stmt); stmt.module = &mo; depended_by[&stmt]; // empty } } void visit(ImportStmt& stmt) override { Module* m = load_module(n_errors, stmt.filename); if (! m) { n_errors++; mo.locfile.error(stmt.loc, "'%s': %s", stmt.filename.c_str(), errno ? strerror(errno) : "parse error"); return; } if (stmt.qualified.size()) mo.qualified_import[stmt.qualified] = m; else if (count(ALL(mo.unqualified_import), m) == 0) mo.unqualified_import.push_back(m); } void visit(PreprocessDefineStmt& stmt) override { if (mo.defined.count(stmt.ident) || mo.macro.count(stmt.ident)) { n_errors++; mo.locfile.error(stmt.loc, "redefined '%s'", stmt.ident.c_str()); } else mo.macro[stmt.ident] = &stmt; } }; struct ModuleUse : PrePostActionExprStmtVisitor { Module& mo; long& n_errors; DefineStmt* stmt = NULL; ModuleUse(Module& mo, long& n_errors) : mo(mo), n_errors(n_errors) {} void pre_expr(Expr& expr) override { expr.stmt = stmt; } void post_expr(Expr& expr) override { for (auto a: expr.entering) PrePostActionExprStmtVisitor::visit(*a.first); for (auto a: expr.finishing) PrePostActionExprStmtVisitor::visit(*a.first); for (auto a: expr.leaving) PrePostActionExprStmtVisitor::visit(*a.first); for (auto a: expr.transiting) PrePostActionExprStmtVisitor::visit(*a.first); } void visit(RefAction& action) override { ActionStmt* r = resolve_action(mo, action.qualified, action.ident); if (! r) { n_errors++; if (action.qualified.size()) mo.locfile.error(action.loc, "'%s::%s' undefined", action.qualified.c_str(), action.ident.c_str()); else mo.locfile.error(action.loc, "'%s' undefined", action.ident.c_str()); } else if (r == (Stmt*)1) { n_errors++; mo.locfile.error(action.loc, "'%s' redefined", action.ident.c_str()); } else action.define_stmt = r; } void visit(BracketExpr& expr) override { for (auto& x: expr.intervals.to) AB = max(AB, x.second); } void visit(CallExpr& expr) override { Stmt* r = resolve(mo, expr.qualified, expr.ident); if (! r) error_undefined(expr.loc, expr.qualified, expr.ident); else if (r == (Stmt*)1) error_ambiguous(expr.loc, expr.ident); else if (auto d = dynamic_cast(r)) error_misuse_macro("CallExpr", expr.loc, expr.qualified, expr.ident); else if (auto d = dynamic_cast(r)) { used_as_call[d].push_back(&expr); expr.define_stmt = d; } else assert(0); } void visit(CollapseExpr& expr) override { Stmt* r = resolve(mo, expr.qualified, expr.ident); if (! r) error_undefined(expr.loc, expr.qualified, expr.ident); else if (r == (Stmt*)1) error_ambiguous(expr.loc, expr.ident); else if (auto d = dynamic_cast(r)) error_misuse_macro("CollapseExpr", expr.loc, expr.qualified, expr.ident); else if (auto d = dynamic_cast(r)) { used_as_collapse[d].push_back(&expr); expr.define_stmt = d; } else assert(0); } void visit(DefineStmt& stmt) override { this->stmt = &stmt; PrePostActionExprStmtVisitor::visit(*stmt.rhs); this->stmt = NULL; } void visit(EmbedExpr& expr) override { // introduce dependency Stmt* r = resolve(mo, expr.qualified, expr.ident); if (! r) error_undefined(expr.loc, expr.qualified, expr.ident); else if (r == (Stmt*)1) error_ambiguous(expr.loc, expr.ident); else if (auto d = dynamic_cast(r)) { // enlarge alphabet expr.define_stmt = NULL; expr.macro_value = d->value; AB = max(AB, d->value+1); } else if (auto d = dynamic_cast(r)) { depended_by[d].push_back(stmt); used_as_embed[d].push_back(&expr); expr.define_stmt = d; } else assert(0); } private: void error_undefined(const Location& loc, const string& qualified, const string& ident) { n_errors++; if (qualified.size()) mo.locfile.error(loc, "'%s::%s' undefined", qualified.c_str(), ident.c_str()); else mo.locfile.error(loc, "'%s' undefined", ident.c_str()); } void error_ambiguous(const Location& loc, const string& ident) { n_errors++; mo.locfile.error(loc, "ambiguous '%s'", ident.c_str()); } void error_misuse_macro(const char* name, const Location& loc, const string& qualified, const string& ident) { n_errors++; if (qualified.size()) mo.locfile.error(loc, "macro '%s::%s' used as %s", qualified.c_str(), ident.c_str(), name); else mo.locfile.error(loc, "macro '%s' used as %s", ident.c_str(), name); } }; Module* load_module(long& n_errors, const string& filename) { FILE* file = stdin; if (filename != "-") { file = fopen(filename.c_str(), "r"); for (string& include: opt_include_paths) { if (file) break; file = fopen((include+'/'+filename).c_str(), "r"); } } if (! file) { n_errors++; return NULL; } pair inode{0, 0}; // stdin -> {0, 0} if (file != stdin) { struct stat sb; if (fstat(fileno(file), &sb) < 0) err_exit(EX_OSFILE, "fstat '%s'", filename.c_str()); inode = {sb.st_dev, sb.st_ino}; } if (inode2module.count(inode)) { fclose(file); return &inode2module[inode]; } Module& mo = inode2module[inode]; string module{file != stdin ? filename : "main"}; string::size_type t = module.find('.'); if (t != string::npos) module.erase(t, module.size()-t); long r; char buf[BUF_SIZE]; string data; while ((r = fread(buf, 1, sizeof buf, file)) > 0) { data += string(buf, buf+r); if (r < sizeof buf) break; } fclose(file); if (data.empty() || data.back() != '\n') data.push_back('\n'); LocationFile locfile(filename, data); Stmt* toplevel = NULL; mo.locfile = locfile; mo.filename = filename; long errors = parse(locfile, toplevel); if (! toplevel) { n_errors += errors; mo.status = BAD; mo.toplevel = NULL; return &mo; } mo.toplevel = toplevel; return &mo; } static vector topo_define_stmts(long& n_errors) { vector topo; vector st; unordered_map vis; // 0: unvisited; 1: in stack; 2: visited; 3: in a cycle unordered_map cnt; function dfs = [&](DefineStmt* u) { if (vis[u] == 2) return false; if (vis[u] == 3) return true; if (vis[u] == 1) { u->module->locfile.error_context(u->loc, "'%s': circular embedding", u->lhs.c_str()); long i = st.size(); while (st[i-1] != u) i--; st.push_back(st[i-1]); for (; i < st.size(); i++) { vis[st[i]] = 3; fputs(" ", stderr); st[i]->module->locfile.error_context(st[i]->loc, "required by %s", st[i]->lhs.c_str()); } fputs("\n", stderr); return true; } cnt[u] = u->export_ ? 1 : 0; vis[u] = 1; st.push_back(u); bool cycle = false; for (auto v: depended_by[u]) if (dfs(v)) cycle = true; else cnt[u] += cnt[v]; st.pop_back(); vis[u] = 2; topo.push_back(u); return cycle; }; for (auto& d: depended_by) if (! vis[d.first] && dfs(d.first)) // detected cycle n_errors++; reverse(ALL(topo)); if (opt_dump_embed) { magenta(); printf("=== Embed\n"); sgr0(); for (auto stmt: topo) if (cnt[stmt] > 0) printf("count(%s::%s) = %ld\n", stmt->module->filename.c_str(), stmt->lhs.c_str(), cnt[stmt]); } return topo; } long load(const string& filename) { long n_errors = 0; Module* mo = load_module(n_errors, filename); if (! mo) { err_exit(EX_OSFILE, "fopen", filename.c_str()); return n_errors; } if (mo->status == BAD) return n_errors; main_module = mo; DP(1, "Processing import & def"); for(;;) { bool done = true; for (auto& it: inode2module) if (it.second.status == UNPROCESSED) { done = false; Module& mo = it.second; mo.status = GOOD; long old = n_errors; ModuleImportDef p{mo, n_errors}; for (Stmt* x = mo.toplevel; x; x = x->next) x->accept(p); mo.status = old == n_errors ? GOOD : BAD; } if (done) break; } if (n_errors) return n_errors; DP(1, "Processing use"); for (auto& it: inode2module) if (it.second.status == GOOD) { Module& mo = it.second; ModuleUse p{mo, n_errors}; for (Stmt* x = mo.toplevel; x; x = x->next) x->accept(p); } if (n_errors) return n_errors; // warning: not used solely as CallExpr, CollapseExpr or EmbedExpr { auto it0 = used_as_call.begin(), it0e = used_as_call.end(), it1 = used_as_collapse.begin(), it1e = used_as_collapse.end(), it2 = used_as_embed.begin(), it2e = used_as_embed.end(); while (it0 != it0e || it1 != it1e || it2 != it2e) { long k = 0; long c = 0; DefineStmt* x = NULL; if (it0 != it0e && (! x || it0->first < x)) x = it0->first; if (it1 != it1e && (! x || it1->first < x)) x = it1->first; if (it2 != it2e && (! x || it2->first < x)) x = it2->first; if (it0 != it0e && it0->first == x) c++; if (it1 != it1e && it1->first == x) c++; if (it2 != it2e && it2->first == x) c++; if (c > 1) { x->module->locfile.warning(x->loc, "'%s' is not used solely as CallExpr, CollapseExpr or EmbedExpr", x->lhs.c_str()); if (it0 != it0e && it0->first == x) for (auto* y: it0->second) { fputs(" ", stderr); y->stmt->module->locfile.warning_context(y->loc, "required by %s", y->stmt->lhs.c_str()); } if (it1 != it1e && it1->first == x) for (auto* y: it1->second) { fputs(" ", stderr); y->stmt->module->locfile.warning_context(y->loc, "required by %s", y->stmt->lhs.c_str()); } if (it2 != it2e && it2->first == x) for (auto* y: it2->second) { fputs(" ", stderr); y->stmt->module->locfile.warning_context(y->loc, "required by %s", y->stmt->lhs.c_str()); } } if (it0 != it0e && it0->first == x) ++it0, c++; if (it1 != it1e && it1->first == x) ++it1, c++; if (it2 != it2e && it2->first == x) ++it2, c++; } } if (opt_dump_module) { magenta(); printf("=== Module\n"); sgr0(); for (auto& it: inode2module) if (it.second.status == GOOD) { Module& mo = it.second; print_module_info(mo); } puts(""); } if (opt_dump_tree) { magenta(); printf("=== Tree\n"); sgr0(); StmtPrinter p; for (auto& it: inode2module) if (it.second.status == GOOD) { Module& mo = it.second; yellow(); printf("filename: %s\n", mo.filename.c_str()); sgr0(); for (Stmt* x = mo.toplevel; x; x = x->next) x->accept(p); } puts(""); } DP(1, "Topological sorting"); vector topo = topo_define_stmts(n_errors); if (n_errors) return n_errors; if (opt_check) return 0; // AB has been updated by ModuleUse action_label_base = action_label = AB; call_label_base = call_label = action_label+1000000; collapse_label_base = collapse_label = call_label+1000000; DP(1, "Compiling DefineStmt"); for (auto stmt: topo) compile(stmt); output = strcmp(opt_output_filename, "-") ? fopen(opt_output_filename, "w") : stdout; if (! output) { n_errors++; err_exit(EX_OSFILE, "fopen", opt_output_filename); return n_errors; } unordered_map>> stmt2call_addr; DP(1, "Compiling exporting DefineStmt (coalescing referenced CallExpr/CollapseExpr)"); for (Stmt* x = main_module->toplevel; x; x = x->next) if (auto xx = dynamic_cast(x)) if (xx->export_ && ! compile_export(xx)) n_errors++; if (n_errors) return n_errors; for (Stmt* x = main_module->toplevel; x; x = x->next) if (auto xx = dynamic_cast(x)) if (xx->export_) { FsaAnno& anno = compiled[xx]; if (opt_dump_automaton) print_automaton(anno.fsa); if (opt_dump_assoc) print_assoc(anno); } if (opt_mode == Mode::cxx) { if (opt_output_header_filename) { output_header = fopen(opt_output_header_filename, "w"); if (! output_header) { n_errors++; err_exit(EX_OSFILE, "fopen", opt_output_header_filename); return n_errors; } } DP(1, "Generating C++"); generate_cxx(mo); if (output_header) fclose(output_header); } else if (opt_mode == Mode::graphviz) { DP(1, "Generating Graphviz dot"); generate_graphviz(mo); } else if (opt_mode == Mode::interactive) { DP(1, "Testing given string"); DefineStmt* main_export = NULL; for (Stmt* x = main_module->toplevel; x; x = x->next) if (auto xx = dynamic_cast(x)) if (xx->export_) { main_export = xx; break; } if (! main_export) puts("no exporting DefineStmt"); else { printf("Testing %s\n", main_export->lhs.c_str()); repl(main_export); } } fclose(output); return n_errors; } void unload_all() { for (auto& it: inode2module) { Module& mo = it.second; stmt_free(mo.toplevel); } } ================================================ FILE: yanshi/src/loader.hh ================================================ #pragma once #include "syntax.hh" #include #include #include #include #include using std::map; using std::set; using std::string; using std::unordered_map; using std::vector; enum ModuleStatus { UNPROCESSED = 0, BAD, GOOD }; struct Module { ModuleStatus status; LocationFile locfile; string filename; Stmt* toplevel; unordered_map defined; vector unqualified_import; unordered_map qualified_import; unordered_map defined_action; unordered_map macro; }; Stmt* resolve(Module& mo, const string qualified, const string& ident); long load(const string& filename); Module* load_module(long& n_errors, const string& filename); void unload_all(); extern Module* main_module; extern FILE *output, *output_header; extern map> used_as_call, used_as_collapse, used_as_embed; ================================================ FILE: yanshi/src/location.cc ================================================ #include "common.hh" #include "location.hh" #include #include using namespace std; LocationFile::LocationFile(const string& filename, const string& data) : filename(filename), data(data) { // data ends with '\n' long nlines = count(data.begin(), data.end(), '\n'); linemap.assign(nlines+1, 0); long line = 1; for (long i = 0; i < data.size(); i++) if (data[i] == '\n') linemap[line++] = i+1; } void LocationFile::context(const Location& loc) const { long line1, col1, line2, col2; locate(loc, line1, col1, line2, col2); if (line1 == line2) { fputs(" ", stderr); FOR(i, linemap[line1], line1+1 < linemap.size() ? linemap[line1+1] : data.size()) { if (i == loc.start) magenta(); fputc(data[i], stderr); if (i+1 == loc.end) sgr0(); } } else { bool first = true; fputs(" ", stderr); FOR(i, linemap[line1], linemap[line1+1]) { if (i == loc.start) magenta(); fputc(data[i], stderr); } if (line2-line1 < 8) { FOR(i, linemap[line1+1], linemap[line2]) { if (first) { first = false; fputs(" ", stderr); } fputc(data[i], stderr); if (data[i] == '\n') first = true; } } else { FOR(i, linemap[line1+1], linemap[line1+4]) { if (first) { first = false; fputs(" ", stderr); } fputc(data[i], stderr); if (data[i] == '\n') first = true; } fputs(" ........\n", stderr); FOR(i, linemap[line2-3], linemap[line2]) { if (first) { first = false; fputs(" ", stderr); } fputc(data[i], stderr); if (data[i] == '\n') first = true; } } FOR(i, linemap[line2], line2+1 < linemap.size() ? linemap[line2+1] : data.size()) { if (first) { first = false; fputs(" ", stderr); } fputc(data[i], stderr); if (i+1 == loc.end) sgr0(); } } } void LocationFile::locate(const Location& loc, long& line1, long& col1, long& line2, long& col2) const { line1 = upper_bound(ALL(linemap), loc.start) - linemap.begin() - 1; line2 = upper_bound(ALL(linemap), max(loc.end-1, 0L)) - linemap.begin() - 1; col1 = loc.start - linemap[line1]; col2 = loc.end - linemap[line2]; } void LocationFile::report_location(const Location& loc) const { long line1, col1, line2, col2; locate(loc, line1, col1, line2, col2); yellow(2); fprintf(stderr, "%s ", filename.c_str()); cyan(2); if (line1 == line2) fprintf(stderr, "%ld:%ld-%ld ", line1+1, col1+1, col2); else fprintf(stderr, "%ld-%ld:%ld-%ld ", line1+1, line2+1, col1+1, col2); } void LocationFile::error(const Location& loc, const char* fmt, ...) const { report_location(loc); red(2); fprintf(stderr, "error "); va_list va; va_start(va, fmt); vfprintf(stderr, fmt, va); va_end(va); fputs("\n", stderr); sgr0(2); } void LocationFile::warning(const Location& loc, const char* fmt, ...) const { report_location(loc); yellow(2); fprintf(stderr, "warning "); va_list va; va_start(va, fmt); vfprintf(stderr, fmt, va); va_end(va); fputs("\n", stderr); sgr0(2); } ================================================ FILE: yanshi/src/location.hh ================================================ #pragma once #include "common.hh" #include #include #include struct Location { long start, end; }; struct LocationFile { std::string filename, data; std::vector linemap; LocationFile() = default; LocationFile(const std::string& filename, const std::string& data); LocationFile& operator=(const LocationFile&) = default; void context(const Location& loc) const; void locate(const Location& loc, long& line1, long& col1, long& line2, long& col2) const; void report_location(const Location& loc) const; void error(const Location& loc, const char* fmt, ...) const; void warning(const Location& loc, const char* fmt, ...) const; template void error_context(const Location& loc, const char* fmt, Args&&... args) const { error(loc, fmt, std::forward(args)...); context(loc); } template void warning_context(const Location& loc, const char* fmt, Args&&... args) const { warning(loc, fmt, std::forward(args)...); context(loc); } }; ================================================ FILE: yanshi/src/main.cc ================================================ #include "common.hh" #include "fsa.hh" #include "loader.hh" #include "option.hh" #include #include #include #include #include #include #include #include #include using namespace std; void print_help(FILE *fh) { fprintf(fh, "Usage: %s [OPTIONS] dir\n", program_invocation_short_name); fputs( "\n" "Options:\n" " -b,--bytes make labels range over [0,256), Unicode literals will be treated as UTF-8 bytes\n" " -C generate C source code (default: C++)\n" " --check check syntax & use/def\n" " --debug debug level\n" " --debug-output filename for debug output\n" " --dump-action dump associated actions for each edge\n" " --dump-assoc dump associated AST Expr for each state\n" " --dump-automaton dump automata\n" " --dump-embed dump statistics of EmbedExpr\n" " --dump-module dump module use/def/...\n" " --dump-tree dump AST\n" " --extern-c generate extern \"C\" specifier\n" " -G,--graph output a Graphviz dot file\n" " -I,--import add to search path for 'import'\n" " -i,--interactive interactive mode\n" " --max-return-stack max length of return stack in C generator (default: 100)\n" " -k,--keep-inaccessible do not perform accessible/co-accessible\n" " -S,--standalone generate header and 'main()'\n" " --substring-grammar construct regular approximation of the substring grammar. Inner states of nonterminals labeled 'intact' are not connected to start/final\n" " -o,--output .cc output filename\n" " -O,--output-header .hh output filename\n" " -h, --help display this help and exit\n" "\n" , fh); exit(fh == stdout ? 0 : EX_USAGE); } int main(int argc, char *argv[]) { setlocale(LC_ALL, ""); int opt; static struct option long_options[] = { {"bytes", no_argument, 0, 'b'}, {"check", required_argument, 0, 'c'}, {"debug", required_argument, 0, 'd'}, {"debug-output", required_argument, 0, 'l'}, {"dump-action", no_argument, 0, 1000}, {"dump-assoc", no_argument, 0, 1001}, {"dump-automaton", no_argument, 0, 1002}, {"dump-embed", no_argument, 0, 1003}, {"dump-module", no_argument, 0, 1004}, {"dump-tree", no_argument, 0, 1005}, {"extern-c", no_argument, 0, 1007}, {"graph", no_argument, 0, 'G'}, {"import", required_argument, 0, 'I'}, {"interactive", no_argument, 0, 'i'}, {"max-return-stack", required_argument, 0, 1006}, {"keep-inaccessible", no_argument, 0, 'k'}, {"standalone", no_argument, 0, 'S'}, {"substring-grammar", no_argument, 0, 's'}, {"output", required_argument, 0, 'o'}, {"output-header", required_argument, 0, 'O'}, {"help", no_argument, 0, 'h'}, {0, 0, 0, 0}, }; while ((opt = getopt_long(argc, argv, "bCDcd:GhI:ikl:O:o:Ss", long_options, NULL)) != -1) { switch (opt) { case 'b': opt_bytes = true; AB = 256; break; case 'C': opt_gen_c = true; break; case 'D': break; case 'c': opt_check = true; break; case 'd': debug_level = get_long(optarg); break; case 'G': opt_mode = Mode::graphviz; break; case 'h': print_help(stdout); break; case 'I': opt_include_paths.push_back(string(optarg)); break; case 'i': opt_mode = Mode::interactive; break; case 'k': opt_keep_inaccessible = true; break; case 'l': if (debug_file) err_exit(EX_USAGE, "multiple '-l'"); debug_file = fopen(optarg, "w"); if (! debug_file) err_exit(EX_OSFILE, "fopen"); break; case 'O': opt_output_header_filename = optarg; break; case 'o': opt_output_filename = optarg; break; case 'S': opt_standalone = true; break; case 's': opt_substring_grammar = true; break; case 1000: opt_dump_action = true; break; case 1001: opt_dump_assoc = true; break; case 1002: opt_dump_automaton = true; break; case 1003: opt_dump_embed = true; break; case 1004: opt_dump_module = true; break; case 1005: opt_dump_tree = true; break; case 1006: opt_max_return_stack = get_long(optarg); break; case 1007: opt_gen_extern_c = true; break; case '?': print_help(stderr); break; } } if (! debug_file) debug_file = stderr; argc -= optind; argv += optind; long n_errors = load(argc ? argv[0] : "-"); unload_all(); fclose(debug_file); return n_errors ? 2 : 0; } ================================================ FILE: yanshi/src/option.cc ================================================ #include "common.hh" #include "option.hh" #include bool opt_bytes, opt_check, opt_dump_action, opt_dump_assoc, opt_dump_automaton, opt_dump_embed, opt_dump_module, opt_dump_tree, opt_gen_c, opt_gen_extern_c, opt_keep_inaccessible, opt_standalone, opt_substring_grammar; long AB = MAX_CODEPOINT+1, opt_max_return_stack = 100; long debug_level = 3; FILE* debug_file; const char* opt_output_filename = "-"; const char* opt_output_header_filename; Mode opt_mode = Mode::cxx; vector opt_include_paths; ================================================ FILE: yanshi/src/option.hh ================================================ #pragma once #include #include using std::string; using std::vector; extern bool opt_bytes, opt_check, opt_dump_action, opt_dump_assoc, opt_dump_automaton, opt_dump_embed, opt_dump_module, opt_dump_tree, opt_gen_c, opt_gen_extern_c, opt_keep_inaccessible, opt_standalone, opt_substring_grammar; extern long AB, opt_max_return_stack; extern const char* opt_output_filename; extern const char* opt_output_header_filename; enum class Mode {cxx, graphviz, interactive}; extern Mode opt_mode; extern vector opt_include_paths; ================================================ FILE: yanshi/src/parser.y ================================================ %code requires { #include "common.hh" #include "location.hh" #include "option.hh" #include "syntax.hh" #include #include #define YYINITDEPTH 1000 #define YYLTYPE Location #define YYLLOC_DEFAULT(Loc, Rhs, N) \ do { \ if (N) { \ (Loc).start = YYRHSLOC(Rhs, 1).start; \ (Loc).end = YYRHSLOC(Rhs, N).end; \ } else { \ (Loc).start = YYRHSLOC(Rhs, 0).start; \ (Loc).end = YYRHSLOC(Rhs, 0).end; \ } \ } while (0) int parse(const LocationFile& locfile, Stmt*& res); } %locations %error-verbose %define api.pure %parse-param {Stmt*& res} %parse-param {long& errors} %parse-param {const LocationFile& locfile} %parse-param {void** lexer} %lex-param {Stmt*& res} %lex-param {long& errors} %lex-param {const LocationFile& locfile} %lex-param {void** lexer} %union { long integer; string* str; DisjointIntervals* intervals; Action* action; Expr* expr; Stmt* stmt; char* errmsg; } %destructor { delete $$; } %destructor { delete $$; } %destructor { delete $$; } %destructor { delete $$; } %destructor { delete $$; } %token ACTION AMPERAMPER AS COLONCOLON CPP DOTDOT EPSILON EXPORT IMPORT INTACT INVALID_CHARACTER PREPROCESS_DEFINE %token CHAR INTEGER %token IDENT %token BRACED_CODE %token STRING_LITERAL %type action %type concat_expr difference_expr factor repeat intersect_expr union_expr union_expr2 unop_expr %type bracket bracket_items %type define_stmt preprocess stmt stmt_list %{ #include "lexer.hh" #define FAIL(loc, errmsg) \ do { \ Location l = loc; \ yyerror(&l, res, errors, locfile, lexer, errmsg); \ } while (0) void yyerror(YYLTYPE* loc, Stmt*& res, long& errors, const LocationFile& locfile, yyscan_t* lexer, const char *errmsg) { errors++; locfile.error_context(*loc, "%s", errmsg); } int yylex(YYSTYPE* yylval, YYLTYPE* loc, Stmt*& res, long& errors, const LocationFile& locfile, yyscan_t* lexer) { int token = raw_yylex(yylval, loc, *lexer); if (token == INVALID_CHARACTER) { FAIL(*loc, yylval->errmsg ? yylval->errmsg : "invalid character"); free(yylval->errmsg); } return token; } #define gen_repeat(x, inner, low, high) \ if (low < 0) { \ FAIL(yyloc, "negative"); \ } \ if (low > high) { \ FAIL(yyloc, "low > high"); \ } \ x = new RepeatExpr(inner, low, high) %} %% toplevel: stmt_list { res = $1; } stmt_list: %empty { $$ = new EmptyStmt; } | '\n' stmt_list { $$ = $2; } | stmt stmt_list { $1->next = $2; $2->prev = $1; $$ = $1; } | error stmt_list { $$ = $2; } stmt: define_stmt { $$ = $1; } | preprocess '\n' { $$ = $1; } | IMPORT STRING_LITERAL AS IDENT '\n' { $$ = new ImportStmt(*$2, *$4); delete $2; delete $4; $$->loc = yyloc; } | IMPORT STRING_LITERAL '\n' { string t; $$ = new ImportStmt(*$2, t); delete $2; $$->loc = yyloc; } | ACTION IDENT BRACED_CODE '\n' { $$ = new ActionStmt(*$2, *$3); delete $2; delete $3; $$->loc = yyloc; } | CPP BRACED_CODE '\n' { $$ = new CppStmt(*$2); delete $2; $$->loc = yyloc; } preprocess: PREPROCESS_DEFINE IDENT INTEGER { $$ = new PreprocessDefineStmt(*$2, $3); delete $2; $$->loc = yyloc; } eq: '=' | ':' define_stmt: IDENT eq union_expr '\n' { $$ = new DefineStmt(*$1, $3); delete $1; $$->loc = yyloc; } | IDENT eq '|' union_expr '\n' { $$ = new DefineStmt(*$1, $4); delete $1; $$->loc = yyloc; } | IDENT eq '\n' union_expr2 '\n' { $$ = new DefineStmt(*$1, $4); delete $1; $$->loc = yyloc; } | IDENT eq '\n' '|' union_expr2 '\n' { $$ = new DefineStmt(*$1, $5); delete $1; $$->loc = yyloc; } | EXPORT define_stmt { $$ = $2; ((DefineStmt*)$$)->export_ = true; $$->loc = yyloc; } | EXPORT BRACED_CODE define_stmt { $$ = $3; ((DefineStmt*)$$)->export_ = true; ((DefineStmt*)$$)->export_params = *$2; delete $2; $$->loc = yyloc; } | INTACT define_stmt { $$ = $2; ((DefineStmt*)$$)->intact = true; $$->loc = yyloc; } union_expr: intersect_expr { $$ = $1; } | union_expr '|' intersect_expr { $$ = new UnionExpr($1, $3); $$->loc = yyloc; } union_expr2: intersect_expr { $$ = $1; } | union_expr2 '|' intersect_expr { $$ = new UnionExpr($1, $3); $$->loc = yyloc; } | union_expr2 '\n' '|' intersect_expr { $$ = new UnionExpr($1, $4); $$->loc = yyloc; } intersect_expr: difference_expr { $$ = $1; } | intersect_expr AMPERAMPER difference_expr { $$ = new IntersectExpr($1, $3); $$->loc = yyloc; } difference_expr: concat_expr { $$ = $1; } | difference_expr '-' concat_expr { $$ = new DifferenceExpr($1, $3); $$->loc = yyloc; } concat_expr: unop_expr { $$ = $1; } | concat_expr unop_expr { $$ = new ConcatExpr($1, $2); $$->loc = yyloc; } unop_expr: factor { $$ = $1; } | '~' unop_expr { $$ = new ComplementExpr($2); $$->loc = yyloc; } factor: EPSILON { $$ = new EpsilonExpr; $$->loc = yyloc; } | IDENT { string t; $$ = new EmbedExpr(t, *$1); delete $1; $$->loc = yyloc; } | IDENT COLONCOLON IDENT { $$ = new EmbedExpr(*$1, *$3); delete $1; delete $3; $$->loc = yyloc; } | '!' IDENT { string t; $$ = new CollapseExpr(t, *$2); delete $2; $$->loc = yyloc; } | '!' IDENT COLONCOLON IDENT { $$ = new CollapseExpr(*$2, *$4); delete $2; delete $4; $$->loc = yyloc; } | '&' IDENT { string t; $$ = new CallExpr(t, *$2); delete $2; $$->loc = yyloc; } | '&' IDENT COLONCOLON IDENT { $$ = new CallExpr(*$2, *$4); delete $2; delete $4; $$->loc = yyloc; } | STRING_LITERAL { $$ = new LiteralExpr(*$1); delete $1; $$->loc = yyloc; } | '.' { $$ = new DotExpr(); $$->loc = yyloc; } | INTEGER { if (opt_bytes && 256 <= $1) { FAIL(yyloc, "literal integers should be less than 256 in bytes mode"); $$ = new DotExpr; } else { auto t = new DisjointIntervals; t->emplace($1, $1+1); $$ = new BracketExpr(t); $$->loc = yyloc; } } | bracket { $$ = new BracketExpr($1); $$->loc = yyloc; } | STRING_LITERAL DOTDOT STRING_LITERAL { i32 c0, c1, i = 0, j = 0; if (opt_bytes) { c0 = u8((*$1)[0]); c1 = u8((*$3)[0]); i = j = 1; } else { U8_NEXT($1->c_str(), i, $1->size(), c0); U8_NEXT($3->c_str(), j, $3->size(), c1); } delete $1; delete $3; if (i != $1->size() || j != $3->size()) { FAIL(yyloc, "endpoints of Unicode range should be of length 1"); $$ = new DotExpr; } else if (c0 > c1) { FAIL(yyloc, "negative Unicode range"); $$ = new DotExpr; } else { auto t = new DisjointIntervals; t->emplace(c0, c1+1); $$ = new BracketExpr(t); $$->loc = yyloc; } } | '(' union_expr ')' { $$ = $2; } | '(' error ')' { $$ = new DotExpr; } | repeat { $$ = $1; } | factor '>' action { $$ = $1; $$->entering.emplace_back($3, 0L); } | factor '>' INTEGER action { $$ = $1; $$->entering.emplace_back($4, $3); } | factor '@' action { $$ = $1; $$->finishing.emplace_back($3, 0L); } | factor '@' INTEGER action { $$ = $1; $$->finishing.emplace_back($4, $3); } | factor '%' action { $$ = $1; $$->leaving.emplace_back($3, 0L); } | factor '%' INTEGER action { $$ = $1; $$->leaving.emplace_back($4, $3); } | factor '$' action { $$ = $1; $$->transiting.emplace_back($3, 0L); } | factor '$' INTEGER action { $$ = $1; $$->transiting.emplace_back($4, $3); } | factor '+' { $$ = new PlusExpr($1); $$->loc = yyloc; } | factor '?' { $$ = new QuestionExpr($1); $$->loc = yyloc; } | factor '*' { $$ = new StarExpr($1); $$->loc = yyloc; } repeat: factor '{' INTEGER ',' INTEGER '}' { gen_repeat($$, $1, $3, $5); $$->loc = yyloc; } | factor '{' INTEGER ',' '}' { gen_repeat($$, $1, $3, LONG_MAX); $$->loc = yyloc; } | factor '{' INTEGER '}' { gen_repeat($$, $1, $3, $3); $$->loc = yyloc; } | factor '{' ',' INTEGER '}' { gen_repeat($$, $1, 0, $4); $$->loc = yyloc; } action: IDENT { string t; $$ = new RefAction(t, *$1); delete $1; $$->loc = yyloc; } | IDENT COLONCOLON IDENT { $$ = new RefAction(*$1, *$3); delete $1; delete $3; $$->loc = yyloc; } | BRACED_CODE { $$ = new InlineAction(*$1); delete $1; $$->loc = yyloc; } bracket: '[' bracket_items ']' { $$ = $2; } | '[' '^' bracket_items ']' { $$ = $3; $$->flip(); } bracket_items: bracket_items CHAR '-' CHAR { $$ = $1; if ($2 > $4) FAIL(yyloc, "negative range in character class"); else $$->emplace($2, $4+1); } | bracket_items CHAR { $$ = $1; $$->emplace($2, $2+1); } | %empty { $$ = new DisjointIntervals; } %% int parse(const LocationFile& locfile, Stmt*& res) { yyscan_t lexer; raw_yylex_init_extra(0, &lexer); YY_BUFFER_STATE buf = raw_yy_scan_bytes(locfile.data.c_str(), locfile.data.size(), lexer); long errors = 0; yyparse(res, errors, locfile, &lexer); raw_yy_delete_buffer(buf, lexer); raw_yylex_destroy(lexer); if (errors > 0) { stmt_free(res); res = NULL; } return errors; } ================================================ FILE: yanshi/src/repl.cc ================================================ #include "compiler.hh" #include "fsa_anno.hh" #include "loader.hh" #include "parser.hh" #include "lexer.hh" // after parser.hh #include "syntax.hh" #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_READLINE # include # include #endif using namespace std; enum class ReplMode {string, integer}; static ReplMode mode = ReplMode::string; static const FsaAnno* anno; static bool quit; struct Command { const char* name; function fn; } commands[] = { {".automaton", [](const char*) {print_automaton(anno->fsa); }}, {".assoc", [](const char*) {print_assoc(*anno); }}, {".help", [](const char*) { fputs("Commands available from the prompt:\n" " .automaton dump automaton\n" " .assoc dump associated AST Expr for each state\n" " .help display this help\n" " .integer input is a list of non-negative integers, macros(#define) or '' "" quoted strings\n" " .macro display defined macros\n" " .string input is a string\n" " .stmt change target DefineStmt to \n" " .quit exit interactive mode\n" , stdout); }}, {".integer", [](const char*) { mode = ReplMode::integer; puts(".integer mode"); }}, {".macro", [](const char*) { for (auto& it: main_module->macro) printf("%s\t%ld\n", it.first.c_str(), it.second->value); for (auto* import: main_module->unqualified_import) for (auto& it: import->macro) printf("%s\t%ld\n", it.first.c_str(), it.second->value); }}, {".stmt", [](const char* arg) { Stmt* r = resolve(*main_module, "", arg); if (! r) printf("'%s' undefined\n", arg); else if (r == (Stmt*)1) printf("ambiguous '%s'\n", arg); else if (auto d = dynamic_cast(r)) printf("'%s' is a macro\n", arg); else if (auto d = dynamic_cast(r)) { anno = &compiled[d]; printf("%s :: DefineStmt\n", d->lhs.c_str()); } else assert(0); }}, {".string", [](const char*) { mode = ReplMode::string; puts(".string mode"); } }, {".quit", [](const char*) { puts("Leaving interactive mode"); quit = true; } }, }; #ifdef HAVE_READLINE static char* command_completer(const char* text, int state) { static long i = 0; if (! state) i = 0; while (i < LEN(commands)) { Command* x = &commands[i++]; if (! strncmp(x->name, text, strlen(text))) return strdup(x->name); } return NULL; } static char* macro_completer(const char* text, int state) { static Stmt* x; if (! state) x = main_module->toplevel; while (x) { auto xx = dynamic_cast(x); x = x->next; if (xx && ! strncmp(xx->ident.c_str(), text, strlen(text))) return strdup(xx->ident.c_str()); } return NULL; } static char* stmt_completer(const char* text, int state) { static Stmt* x; if (! state) x = main_module->toplevel; while (x) { auto xx = dynamic_cast(x); x = x->next; if (xx && ! strncmp(xx->lhs.c_str(), text, strlen(text))) return strdup(xx->lhs.c_str()); } return NULL; } static char** on_complete(const char* text, int start, int end) { rl_attempted_completion_over = 1; if (! start) return rl_completion_matches(text, command_completer); if (6 <= start && ! strncmp(rl_line_buffer, ".stmt ", 6)) return rl_completion_matches(text, stmt_completer); if (mode == ReplMode::integer) return rl_completion_matches(text, macro_completer); return NULL; } #else char* readline(const char* prompt) { char* r = NULL; size_t s = 0; ssize_t n; fputs(prompt, stdout); if ((n = getline(&r, &s, stdin)) > 0) r[n-1] = '\0'; else { free(r); r = NULL; } return r; } #endif static void run_command(char* line) { size_t p = 1; while (line[p] && isalnum(line[p])) p++; Command* com = NULL; REP(i, LEN(commands)) if (! strncmp(commands[i].name, line, p)) { if (com) com = (Command*)1; else com = &commands[i]; } if (! com) printf("Unknown command '%s'\n", line); else if (com == (Command*)1) printf("Ambiguous command '%s'\n", line); else { while (line[p] && isspace(line[p])) p++; size_t len = strlen(line); while (len && isspace(line[len-1])) line[--len] = '\0'; com->fn(line+p); } } void repl(DefineStmt* stmt) { #ifdef HAVE_READLINE rl_attempted_completion_function = on_complete; #endif char buf[BUF_SIZE]; snprintf(buf, sizeof buf, ".stmt %s", stmt->lhs.c_str()); run_command(buf); strcpy(buf, ".integer"); run_command(buf); strcpy(buf, ".help"); run_command(buf); if (! anno) return; char* line; stringstream ss; while ((line = readline("λ ")) != NULL) { #ifdef HAVE_READLINE if (line[0]) add_history(line); #endif if (line[0] == '.') { run_command(line); free(line); if (quit) break; continue; } long u = anno->fsa.start; if (mode == ReplMode::string) { i32 i = 0, len; long c; len = strlen(line); if (anno->fsa.is_final(u)) yellow(1); else normal_yellow(1); printf("%ld ", u); sgr0(); while (i < len) { U8_NEXT_OR_FFFD(line, i, len, c); if (iswcntrl(c)) printf("%ld ", c); else printf("%lc ", wint_t(c)); u = anno->fsa.transit(u, c); if (anno->fsa.is_final(u)) yellow(); else normal_yellow(); printf("%ld ", u); sgr0(); if (u < 0) break; } } else { vector input; int token; yyscan_t lexer; raw_yylex_init_extra(0, &lexer); YY_BUFFER_STATE buf = raw_yy_scan_bytes(line, strlen(line), lexer); YYSTYPE yylval; YYLTYPE yylloc; while (u >= 0 && (token = raw_yylex(&yylval, &yylloc, lexer)) != 0) { switch (token) { // all tokens with a destructor should be listed case IDENT: { Stmt* stmt = resolve(*main_module, "", yylval.str->c_str()); if (! stmt) { printf("'%s' undefined", yylval.str->c_str()); u = -1; } else if (stmt == (Stmt*)1) { printf("ambiguous '%s'", yylval.str->c_str()); u = -1; } else if (auto d = dynamic_cast(stmt)) input.push_back(d->value); else if (auto d = dynamic_cast(stmt)) { printf("'%s' is not a macro", yylval.str->c_str()); u = -1; } else assert(0); delete yylval.str; break; } case INTEGER: input.push_back(yylval.integer); break; case STRING_LITERAL: if (opt_bytes) for (unsigned char c: *yylval.str) input.push_back(c); else for (i32 c, i = 0; i < yylval.str->size(); ) { U8_NEXT_OR_FFFD(yylval.str->c_str(), i, yylval.str->size(), c); input.push_back(c); } delete yylval.str; break; case BRACED_CODE: delete yylval.str; // fall through default: printf("invalid token at column %ld-%ld\n", yylloc.start+1, yylloc.end); u = -1; break; } } raw_yy_delete_buffer(buf, lexer); raw_yylex_destroy(lexer); if (u >= 0) { if (anno->fsa.is_final(u)) yellow(1); else normal_yellow(1); printf("%ld ", u); sgr0(); for (long c: input) { printf("%ld ", c); u = anno->fsa.transit(u, c); if (anno->fsa.is_final(u)) yellow(); else normal_yellow(); printf("%ld ", u); sgr0(); if (u < 0) break; } } } free(line); puts(""); if (u >= 0) { unordered_map> start_finals; unordered_map>> inners; for (auto aa: anno->assoc[u]) { if (has_start(aa.second)) start_finals[aa.first->stmt].push_back(aa.first->loc.start); if (has_inner(aa.second)) { inners[aa.first->stmt].emplace_back(aa.first->loc.start, 1); inners[aa.first->stmt].emplace_back(aa.first->loc.end, -1); } if (has_final(aa.second)) start_finals[aa.first->stmt].push_back(aa.first->loc.end); } vector stmts; for (auto& it: start_finals) stmts.push_back(it.first); for (auto& it: inners) stmts.push_back(it.first); // sort DefineStmt by location sort(ALL(stmts), [](const DefineStmt* x, const DefineStmt* y) { if (x->module != y->module) return x->module < y->module; if (x->loc.start != y->loc.start) return x->loc.start < y->loc.start; return x->loc.end < y->loc.end; }); stmts.erase(unique(ALL(stmts)), stmts.end()); for (auto* stmt: stmts) { auto& start_final = start_finals[stmt]; auto& inner = inners[stmt]; sort(ALL(start_final)); sort(ALL(inner)); auto it0 = start_final.begin(); auto it1 = inner.begin(); long nest = 0; FOR(i, stmt->loc.start, stmt->loc.end) { for (; it0 != start_final.end() && *it0 < i; ++it0); if (it0 != start_final.end() && *it0 == i) { cyan(); putchar(':'); sgr0(); } for (; it1 != inner.end() && it1->first <= i; ++it1) nest += it1->second; if (nest) yellow(); putchar(stmt->module->locfile.data[i]); if (nest) sgr0(); } } } } } ================================================ FILE: yanshi/src/repl.hh ================================================ #pragma once #include "syntax.hh" void repl(DefineStmt* stmt); ================================================ FILE: yanshi/src/syntax.cc ================================================ #include "syntax.hh" void stmt_free(Stmt* stmt) { while (stmt) { auto x = stmt->next; delete stmt; stmt = x; } } ================================================ FILE: yanshi/src/syntax.hh ================================================ #pragma once #include "common.hh" #include "location.hh" #include #include #include #include #include #include using std::move; using std::pair; using std::string; using std::vector; //// Visitor template struct Visitor; template struct VisitableBase { virtual void accept(Visitor& visitor) = 0; }; template struct Visitable : Base { void accept(Visitor& visitor) override { visitor.visit(static_cast(*this)); } }; struct Action; struct InlineAction; struct RefAction; template<> struct Visitor { virtual void visit(Action& action) = 0; virtual void visit(InlineAction&) = 0; virtual void visit(RefAction&) = 0; }; struct Expr; struct BracketExpr; struct CallExpr; struct CollapseExpr; struct ComplementExpr; struct ConcatExpr; struct DifferenceExpr; struct DotExpr; struct EmbedExpr; struct EpsilonExpr; struct IntersectExpr; struct LiteralExpr; struct PlusExpr; struct RepeatExpr; struct QuestionExpr; struct StarExpr; struct UnionExpr; template<> struct Visitor { virtual void visit(Expr&) = 0; virtual void visit(BracketExpr&) = 0; virtual void visit(CallExpr&) = 0; virtual void visit(CollapseExpr&) = 0; virtual void visit(ComplementExpr&) = 0; virtual void visit(ConcatExpr&) = 0; virtual void visit(DifferenceExpr&) = 0; virtual void visit(DotExpr&) = 0; virtual void visit(EmbedExpr&) = 0; virtual void visit(EpsilonExpr&) = 0; virtual void visit(IntersectExpr&) = 0; virtual void visit(LiteralExpr&) = 0; virtual void visit(PlusExpr&) = 0; virtual void visit(RepeatExpr&) = 0; virtual void visit(QuestionExpr&) = 0; virtual void visit(StarExpr&) = 0; virtual void visit(UnionExpr&) = 0; }; struct Stmt; struct ActionStmt; struct CppStmt; struct DefineStmt; struct EmptyStmt; struct ImportStmt; struct PreprocessDefineStmt; template<> struct Visitor { virtual void visit(Stmt&) = 0; virtual void visit(ActionStmt&) = 0; virtual void visit(CppStmt&) = 0; virtual void visit(DefineStmt&) = 0; virtual void visit(EmptyStmt&) = 0; virtual void visit(ImportStmt&) = 0; virtual void visit(PreprocessDefineStmt&) = 0; }; //// Action struct Action : VisitableBase { Location loc; virtual ~Action() = default; }; struct InlineAction : Visitable { string code; InlineAction(string& code) : code(move(code)) {} }; struct Module; struct RefAction : Visitable { string qualified, ident; ActionStmt* define_stmt; // set by ModuleUse RefAction(string& qualified, string& ident) : qualified(move(qualified)), ident(move(ident)) {} }; //// Expr struct Expr : VisitableBase { Location loc; long pre, post, depth; // set by Compiler vector anc; // set by Compiler vector> entering, finishing, leaving, transiting; DefineStmt* stmt = NULL; // set by ModuleImportDef virtual ~Expr() { for (auto a: entering) delete a.first; for (auto a: finishing) delete a.first; for (auto a: leaving) delete a.first; for (auto a: transiting) delete a.first; } string name() const { int status; std::unique_ptr r{ abi::__cxa_demangle(typeid(*this).name(), NULL, NULL, &status), free }; std::string t = r.get(); t = t.substr(0, t.size()-4); // suffix 'Expr' return t; } bool no_action() const { return entering.empty() && finishing.empty() && leaving.empty() && transiting.empty(); } }; struct BracketExpr : Visitable { DisjointIntervals intervals; BracketExpr(DisjointIntervals* intervals) : intervals(std::move(*intervals)) { delete intervals; } }; struct CallExpr : Visitable { string qualified, ident; DefineStmt* define_stmt = NULL; // set by ModuleUse CallExpr(string& qualified, string& ident) : qualified(move(qualified)), ident(move(ident)) {} }; struct CollapseExpr : Visitable { string qualified, ident; DefineStmt* define_stmt = NULL; // set by ModuleUse CollapseExpr(string& qualified, string& ident) : qualified(move(qualified)), ident(move(ident)) {} }; struct ComplementExpr : Visitable { Expr* inner; ComplementExpr(Expr* inner) : inner(inner) {} ~ComplementExpr() { delete inner; } }; struct ConcatExpr : Visitable { Expr *lhs, *rhs; ConcatExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {} ~ConcatExpr() { delete lhs; delete rhs; } }; struct DifferenceExpr : Visitable { Expr *lhs, *rhs; DifferenceExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {} ~DifferenceExpr() { delete lhs; delete rhs; } }; struct DotExpr : Visitable {}; struct EmbedExpr : Visitable { string qualified, ident; DefineStmt* define_stmt = NULL; // set by ModuleUse long macro_value; // set by ModuleUse EmbedExpr(string& qualified, string& ident) : qualified(move(qualified)), ident(move(ident)) {} }; struct EpsilonExpr : Visitable {}; struct IntersectExpr : Visitable { Expr *lhs, *rhs; IntersectExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {} ~IntersectExpr() { delete lhs; delete rhs; } }; struct LiteralExpr : Visitable { string literal; LiteralExpr(string& literal) : literal(move(literal)) {} }; struct PlusExpr : Visitable { Expr* inner; PlusExpr(Expr* inner) : inner(inner) {} ~PlusExpr() { delete inner; } }; struct RepeatExpr : Visitable { Expr* inner; long low, high; RepeatExpr(Expr* inner, long low, long high) : inner(inner), low(low), high(high) {} ~RepeatExpr() { delete inner; } }; struct QuestionExpr : Visitable { Expr* inner; QuestionExpr(Expr* inner) : inner(inner) {} ~QuestionExpr() { delete inner; } }; struct StarExpr : Visitable { Expr* inner; StarExpr(Expr* inner) : inner(inner) {} ~StarExpr() { delete inner; } }; struct UnionExpr : Visitable { Expr *lhs, *rhs; UnionExpr(Expr* lhs, Expr* rhs) : lhs(lhs), rhs(rhs) {} ~UnionExpr() { delete lhs; delete rhs; } }; //// Stmt struct Stmt { Location loc; Stmt *prev = NULL, *next = NULL; virtual ~Stmt() = default; virtual void accept(Visitor& visitor) = 0; }; struct EmptyStmt : Visitable {}; struct ActionStmt : Visitable { string ident, code; ActionStmt(string& ident, string& code) : ident(move(ident)), code(move(code)) {} }; struct CppStmt : Visitable { string code; CppStmt(string& code) : code(move(code)) {} }; struct DefineStmt : Visitable { bool export_ = false, intact = false; string export_params, lhs; Expr* rhs; Module* module; // used in topological sort DefineStmt(string& lhs, Expr* rhs) : lhs(move(lhs)), rhs(rhs) {} ~DefineStmt() { delete rhs; } }; struct ImportStmt : Visitable { string filename, qualified; ImportStmt(string& filename, string& qualified) : filename(move(filename)), qualified(move(qualified)) {} }; struct PreprocessDefineStmt : Visitable { string ident; long value; PreprocessDefineStmt(string& ident, long value) : ident(move(ident)), value(value) {} }; void stmt_free(Stmt* stmt); //// Visitor implementations struct StmtPrinter : Visitor, Visitor, Visitor { int depth = 0; void visit(Action& action) override { action.accept(*this); } void visit(InlineAction& action) override { printf("%*s%s\n", 2*depth, "", "InlineAction"); printf("%*s%s\n", 2*(depth+1), "", action.code.c_str()); } void visit(RefAction& action) override { printf("%*s%s\n", 2*depth, "", "RefAction"); printf("%*s%s\n", 2*(depth+1), "", action.ident.c_str()); } void visit(Expr& expr) override { if (expr.entering.size()) { printf("%*s%s\n", 2*depth, "", "@entering"); depth++; for (auto a: expr.entering) { indent(stdout, depth); printf("%ld\n", a.second); a.first->accept(*this); } depth--; } if (expr.finishing.size()) { printf("%*s%s\n", 2*depth, "", "@finishing"); depth++; for (auto a: expr.finishing) { indent(stdout, depth); printf("%ld\n", a.second); a.first->accept(*this); } depth--; } if (expr.leaving.size()) { printf("%*s%s\n", 2*depth, "", "@entering"); depth++; for (auto a: expr.leaving) { indent(stdout, depth); printf("%ld\n", a.second); a.first->accept(*this); } depth--; } if (expr.transiting.size()) { printf("%*s%s\n", 2*depth, "", "@transiting"); depth++; for (auto a: expr.transiting) { indent(stdout, depth); printf("%ld\n", a.second); a.first->accept(*this); } depth--; } expr.accept(*this); } void visit(BracketExpr& expr) override { printf("%*s%s\n", 2*depth, "", "BracketExpr"); printf("%*s", 2*(depth+1), ""); for (auto& x: expr.intervals.to) printf("(%ld,%ld) ", x.first, x.second); puts(""); } void visit(CallExpr& expr) override { printf("%*s%s\n", 2*depth, "", "CallExpr"); printf("%*s", 2*(depth+1), ""); if (expr.qualified.size()) printf("%s::%s\n", expr.qualified.c_str(), expr.ident.c_str()); else printf("%s\n", expr.ident.c_str()); } void visit(CollapseExpr& expr) override { printf("%*s%s\n", 2*depth, "", "CollapseExpr"); printf("%*s", 2*(depth+1), ""); if (expr.qualified.size()) printf("%s::%s\n", expr.qualified.c_str(), expr.ident.c_str()); else printf("%s\n", expr.ident.c_str()); } void visit(ComplementExpr& expr) override { printf("%*s%s\n", 2*depth, "", "ComplementExpr"); depth++; visit(*expr.inner); depth--; } void visit(ConcatExpr& expr) override { printf("%*s%s\n", 2*depth, "", "ConcatExpr"); depth++; visit(*expr.lhs); visit(*expr.rhs); depth--; } void visit(DifferenceExpr& expr) override { printf("%*s%s\n", 2*depth, "", "DifferenceExpr"); depth++; visit(*expr.lhs); visit(*expr.rhs); depth--; } void visit(DotExpr& expr) override { printf("%*s%s\n", 2*depth, "", "DotExpr"); } void visit(EmbedExpr& expr) override { printf("%*s%s\n", 2*depth, "", "EmbedExpr"); printf("%*s", 2*(depth+1), ""); if (expr.qualified.size()) printf("%s::%s\n", expr.qualified.c_str(), expr.ident.c_str()); else printf("%s\n", expr.ident.c_str()); } void visit(EpsilonExpr& expr) override { printf("%*s%s\n", 2*depth, "", "EpsilonExpr"); } void visit(IntersectExpr& expr) override { printf("%*s%s\n", 2*depth, "", "IntersectExpr"); depth++; visit(*expr.lhs); visit(*expr.rhs); depth--; } void visit(LiteralExpr& expr) override { printf("%*s%s\n", 2*depth, "", "LiteralExpr"); printf("%*s%s\n", 2*(depth+1), "", expr.literal.c_str()); } void visit(PlusExpr& expr) override { printf("%*s%s\n", 2*depth, "", "PlusExpr"); depth++; visit(*expr.inner); depth--; } void visit(RepeatExpr& expr) override { printf("%*s%s\n", 2*depth, "", "RepeatExpr"); printf("%*s%ld,%ld\n", 2*(depth+1), "", expr.low, expr.high); depth++; visit(*expr.inner); depth--; } void visit(QuestionExpr& expr) override { printf("%*s%s\n", 2*depth, "", "QuestionExpr"); depth++; visit(*expr.inner); depth--; } void visit(StarExpr& expr) override { printf("%*s%s\n", 2*depth, "", "StarExpr"); depth++; visit(*expr.inner); depth--; } void visit(UnionExpr& expr) override { printf("%*s%s\n", 2*depth, "", "UnionExpr"); depth++; visit(*expr.lhs); visit(*expr.rhs); depth--; } void visit(Stmt& stmt) override { stmt.accept(*this); } void visit(ActionStmt& stmt) override { printf("%*s%s\n", 2*depth, "", "ActionStmt"); printf("%*s%s\n", 2*(depth+1), "", stmt.ident.c_str()); printf("%*s%s\n", 2*(depth+1), "", stmt.code.c_str()); } void visit(CppStmt& stmt) override { printf("%*s%s\n", 2*depth, "", "CppStmt"); printf("%*s%s\n", 2*(depth+1), "", stmt.code.c_str()); } void visit(DefineStmt& stmt) override { printf("%*s%s%s\n", 2*depth, "", "DefineStmt", stmt.export_ ? " export" : ""); depth++; indent(stdout, depth); if (stmt.export_params.size()) printf("(%s) ", stmt.export_params.c_str()); printf("%s\n", stmt.lhs.c_str()); visit(*stmt.rhs); depth--; } void visit(EmptyStmt& stmt) override { printf("%*s%s\n", 2*depth, "", "EmptyStmt"); } void visit(ImportStmt& stmt) override { printf("%*s%s\n", 2*depth, "", "ImportStmt"); printf("%*s%s\n", 2*(depth+1), "", stmt.filename.c_str()); if (stmt.qualified.size()) printf("%*sas %s\n", 2*(depth+1), "", stmt.qualified.c_str()); } void visit(PreprocessDefineStmt& stmt) override { printf("%*s%s\n", 2*depth, "", "PreprocessDefineStmt"); printf("%*s%s %ld\n", 2*(depth+1), "", stmt.ident.c_str(), stmt.value); } }; //// Visitor implementation struct PreorderStmtVisitor : Visitor { void visit(Stmt& stmt) override { stmt.accept(*this); } void visit(ActionStmt& stmt) override {} void visit(CppStmt& stmt) override {} void visit(DefineStmt& stmt) override {} void visit(EmptyStmt& stmt) override {} void visit(ImportStmt& stmt) override {} void visit(PreprocessDefineStmt&) override {} }; struct PrePostActionExprStmtVisitor : Visitor, Visitor, Visitor { virtual void pre_action(Action&) {} virtual void post_action(Action&) {} virtual void pre_expr(Expr&) {} virtual void post_expr(Expr&) {} virtual void pre_stmt(Stmt&) {} virtual void post_stmt(Stmt&) {} void visit(Action& action) override { pre_action(action); action.accept(*this); post_action(action); } void visit(InlineAction&) override {} void visit(RefAction&) override {} void visit(Expr& expr) override { pre_expr(expr); expr.accept(*this); post_expr(expr); } void visit(BracketExpr& expr) override {} void visit(CallExpr& expr) override {} void visit(CollapseExpr& expr) override {} void visit(ComplementExpr& expr) override { visit(*expr.inner); } void visit(ConcatExpr& expr) override { visit(*expr.lhs); visit(*expr.rhs); } void visit(DifferenceExpr& expr) override { visit(*expr.lhs); visit(*expr.rhs); } void visit(DotExpr& expr) override {} void visit(EmbedExpr& expr) override {} void visit(EpsilonExpr& expr) override {} void visit(IntersectExpr& expr) override { visit(*expr.lhs); visit(*expr.rhs); } void visit(LiteralExpr& expr) override {} void visit(PlusExpr& expr) override { visit(*expr.inner); } void visit(RepeatExpr& expr) override { visit(*expr.inner); } void visit(QuestionExpr& expr) override { visit(*expr.inner); } void visit(StarExpr& expr) override { visit(*expr.inner); } void visit(UnionExpr& expr) override { visit(*expr.lhs); visit(*expr.rhs); } void visit(Stmt& stmt) override { pre_stmt(stmt); stmt.accept(*this); post_stmt(stmt); } void visit(ActionStmt& stmt) override {} void visit(CppStmt& stmt) override {} void visit(DefineStmt& stmt) override { stmt.rhs->accept(*this); } void visit(EmptyStmt& stmt) override {} void visit(ImportStmt& stmt) override {} void visit(PreprocessDefineStmt&) override {} }; ================================================ FILE: yanshi/unittest/determinize_test.cc ================================================ #include "fsa.hh" #include "unittest/unittest_helper.hh" #include #include #include #include #include #include using namespace std; const char test[] = "4 7 2\n" "2 3 \n" "0 0 1\n" "0 -1 2\n" "1 1 1\n" "1 1 3\n" "2 -1 1\n" "2 0 3\n" "3 0 2\n" ; int main(int argc, char *argv[]) { if (argc == 1) { char filename[] = "/tmp/XXXXXX"; int fd = mkstemp(filename); write(fd, test, sizeof test-1); close(fd); freopen(filename, "r", stdin); unlink(filename); } auto relate = [](const vector&) {}; Fsa fsa = read_nfa().determinize(relate); print_fsa(fsa); if (argc == 1) return fsa.n() == 4 ? 0 : 1; } ================================================ FILE: yanshi/unittest/difference_test.cc ================================================ #include "fsa.hh" #include "unittest/unittest_helper.hh" #include #include #include #include #include using namespace std; const char test[] = "4 4 1\n" "3 \n" "0 0 1\n" "0 1 2\n" "1 0 3\n" "2 1 3\n" "4 4 1\n" "3 \n" "0 0 1\n" "0 1 2\n" "1 1 3\n" "2 1 3\n" ; int main(int argc, char *argv[]) { if (argc == 1) { char filename[] = "/tmp/XXXXXX"; int fd = mkstemp(filename); write(fd, test, sizeof test-1); close(fd); freopen(filename, "r", stdin); unlink(filename); } auto relate = [](long u, long v) {}; Fsa a = read_dfa(), b = read_dfa(), fsa = a.difference(b, relate); print_fsa(fsa); if (argc == 1) return 0; // fsa.n() == 4 ? 0 : 1; } ================================================ FILE: yanshi/unittest/intersection_test.cc ================================================ #include "fsa.hh" #include "unittest/unittest_helper.hh" #include #include #include #include #include using namespace std; const char test[] = "4 4 1\n" "3 \n" "0 0 1\n" "0 1 2\n" "1 0 3\n" "2 1 3\n" "4 4 1\n" "3 \n" "0 0 1\n" "0 1 2\n" "1 1 3\n" "2 1 3\n" ; int main(int argc, char *argv[]) { if (argc == 1) { char filename[] = "/tmp/XXXXXX"; int fd = mkstemp(filename); write(fd, test, sizeof test-1); close(fd); freopen(filename, "r", stdin); unlink(filename); } auto relate = [](long u, long v) {}; Fsa a = read_dfa(), b = read_dfa(), fsa = a.intersect(b, relate); print_fsa(fsa); if (argc == 1) return 0; // fsa.n() == 4 ? 0 : 1; } ================================================ FILE: yanshi/unittest/minimize_test.cc ================================================ #include "fsa.hh" #include "unittest/unittest_helper.hh" #include #include #include #include #include #include #include using namespace std; const char test[] = "4 4 1\n" "3 \n" "0 0 1\n" "0 1 2\n" "1 0 3\n" "2 0 3\n" ; int main(int argc, char *argv[]) { if (argc == 1) { char filename[] = "/tmp/XXXXXX"; int fd = mkstemp(filename); write(fd, test, sizeof test-1); close(fd); freopen(filename, "r", stdin); unlink(filename); } auto relate = [](const vector&) {}; Fsa fsa = read_dfa().minimize(relate); print_fsa(fsa); if (argc == 1) return fsa.n() == 3 ? 0 : 1; } ================================================ FILE: yanshi/unittest/union_test.cc ================================================ #include "fsa.hh" #include "unittest/unittest_helper.hh" #include #include #include #include #include #include #include using namespace std; const char test[] = "4 4 1\n" "3 \n" "0 0 1\n" "0 1 2\n" "1 0 3\n" "2 1 3\n" "4 4 1\n" "3 \n" "0 0 1\n" "0 1 2\n" "1 1 3\n" "2 1 3\n" ; int main(int argc, char *argv[]) { if (argc == 1) { char filename[] = "/tmp/XXXXXX"; int fd = mkstemp(filename); write(fd, test, sizeof test-1); close(fd); freopen(filename, "r", stdin); unlink(filename); } auto relate = [](long u, long v) {}; Fsa a = read_dfa(), b = read_dfa(), fsa = a.union_(b, relate); print_fsa(fsa); if (argc == 1) return fsa.n() == 4 ? 0 : 1; } ================================================ FILE: yanshi/unittest/unittest_helper.hh ================================================ #pragma once #include "common.hh" #include "fsa.hh" #include #include #include #include #include using namespace std; static Fsa read_nfa() { long n, m, k, u, a, v; cin >> n >> m >> k; Fsa r; r.start = 0; r.adj.resize(n); while (k--) { cin >> u; r.finals.push_back(u); } sort(ALL(r.finals)); while (m--) { cin >> u >> a >> v; if (u < 0 || u >= n) errx(EX_DATAERR, "%ld: 0 <= u < n", u); if (a < -1 || a >= 256) errx(EX_DATAERR, "%ld: -1 <= c < 256", a); if (v < 0 || v >= n) errx(EX_DATAERR, "%ld: 0 <= v < n", v); r.adj[u].emplace_back(a, v); } assert(cin.good()); REP(i, n) sort(ALL(r.adj[i])); return r; } static Fsa read_dfa() { Fsa r = read_nfa(); REP(i, r.n()) if (r.adj[i].size()) { if (r.adj[i][0].first < 0) errx(EX_DATAERR, "epsilon edge found for %ld", i); REP(j, r.adj[i].size()-1) if (r.adj[i][j].first == r.adj[i][j+1].first) errx(EX_DATAERR, "duplicate labels %ld found for %ld", r.adj[i][j].first, i); } assert(cin.good()); return r; } static void print_fsa(const Fsa& fsa) { printf("finals:"); for (long i: fsa.finals) printf(" %ld", i); puts(""); puts("edges:"); REP(i, fsa.n()) { printf("%ld:", i); for (auto& x: fsa.adj[i]) printf(" (%ld,%ld)", x.first, x.second); puts(""); } }