Repository: gobicycle/bicycle Branch: master Commit: 2af50e7a7295 Files: 62 Total size: 600.6 KB Directory structure: gitextract_x1h1gsmv/ ├── .github/ │ └── workflows/ │ └── go.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── api/ │ ├── example.http │ ├── handlers.go │ ├── http-client.env.json │ └── middleware.go ├── audit/ │ └── log.go ├── blockchain/ │ ├── blockchain.go │ ├── blockchain_test.go │ ├── limited_client.go │ └── shard_tracker.go ├── cmd/ │ ├── processor/ │ │ └── main.go │ ├── testutil/ │ │ ├── http.go │ │ ├── main.go │ │ ├── metrics.go │ │ └── utils.go │ └── testwebhook/ │ └── main.go ├── config/ │ └── config.go ├── core/ │ ├── block_scanner.go │ ├── models.go │ ├── proxy.go │ ├── wallets.go │ └── withdrawal_processor.go ├── db/ │ ├── db.go │ ├── db_test.go │ └── tests/ │ ├── get-jetton-internal-withdrawal-tasks/ │ │ └── 01_data.up.sql │ ├── get-ton-internal-withdrawal-tasks/ │ │ └── 01_data.up.sql │ └── set-expired/ │ └── 01_data.up.sql ├── deploy/ │ ├── db/ │ │ ├── 01_init.down.sql │ │ ├── 01_init.up.sql │ │ └── 02_create_readonly_user.sh │ ├── grafana/ │ │ ├── main/ │ │ │ ├── dashboards/ │ │ │ │ └── Payments.json │ │ │ └── provisioning/ │ │ │ ├── dashboards/ │ │ │ │ └── payments.yml │ │ │ └── datasources/ │ │ │ └── data_sources.yml │ │ └── test/ │ │ ├── dashboards/ │ │ │ ├── Processor A.json │ │ │ ├── Processor B.json │ │ │ └── Test util.json │ │ └── provisioning/ │ │ ├── dashboards/ │ │ │ └── payments.yml │ │ └── datasources/ │ │ └── data_sources.yml │ ├── manual_migrations/ │ │ ├── 0.1.x-0.2.0.sql │ │ └── 0.4.x-0.5.0.sql │ └── prometheus/ │ ├── main/ │ │ └── prometheus.yml │ └── test/ │ └── prometheus.yml ├── docker-compose.yml ├── docs/ │ ├── api.apib │ └── index.html ├── go.mod ├── go.sum ├── jettons.md ├── manual_migrations.md ├── manual_testing_plan.md ├── metrics/ │ └── metrics.go ├── queue/ │ └── queue.go ├── release_notes.md ├── technical_notes.md ├── tests/ │ └── docker-compose-tests.yml ├── threat_model.md ├── todo_list.md └── webhook/ └── webhook.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/go.yml ================================================ name: Go on: push: branches: [ "master" ] workflow_dispatch: {} jobs: build: runs-on: ubuntu-24.04 environment: TESTS services: postgres: image: postgres env: POSTGRES_DB: payment_processor POSTGRES_USER: pp_user POSTGRES_PASSWORD: postgres options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 ports: - 5432:5432 steps: - uses: actions/checkout@v3 with: persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token. fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. - name: Set up Go uses: actions/setup-go@v3 with: go-version: 1.24.0 - name: Build run: go build -v ./... - uses: actions/cache@v4 with: path: ~/go/pkg/mod key: ${{ runner.os }}-2go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-2go- - name: Install dependencies run: | sudo apt-get update sudo apt-get -y install openssl ca-certificates libsodium23 - name: Install libs run: | wget -O libemulator.so https://github.com/ton-blockchain/ton/releases/download/v2025.06/libemulator-linux-x86_64.so sudo cp libemulator.so /lib - name: Update Library Cache run: sudo ldconfig - name: Verify Library Presence run: ls /lib | grep libemulator.so - name: Run Test env: SEED: ${{ secrets.SEED }} SERVER: ${{ secrets.SERVER }} KEY: ${{ secrets.KEY }} DB_URI: postgresql://pp_user:postgres@localhost:5432/payment_processor?sslmode=disable run: | go test -v $(go list ./...) ================================================ FILE: Dockerfile ================================================ FROM docker.io/library/golang:1.24-bookworm AS builder WORKDIR /build-dir COPY go.mod . COPY go.sum . RUN go mod download all COPY api api COPY blockchain blockchain COPY cmd cmd COPY config config COPY core core COPY db db COPY audit audit COPY queue queue COPY webhook webhook COPY metrics metrics RUN apt-get update && apt-get install -y libsodium23 ARG GIT_TAG RUN go build -ldflags "-X main.Version=$GIT_TAG" -o /tmp/processor github.com/gobicycle/bicycle/cmd/processor RUN go build -ldflags "-X main.Version=$GIT_TAG" -o /tmp/testutil github.com/gobicycle/bicycle/cmd/testutil FROM docker.io/library/ubuntu:24.04 AS payment-processor RUN apt-get update && apt-get install -y openssl ca-certificates libsodium23 wget && rm -rf /var/lib/apt/lists/* RUN mkdir -p /app/lib COPY --from=builder /go/pkg/mod/github.com/tonkeeper/tongo*/lib/linux /app/lib/ ENV LD_LIBRARY_PATH=/app/lib COPY --from=builder /tmp/processor /app/processor CMD ["/app/processor", "-v"] FROM docker.io/library/ubuntu:24.04 AS payment-test RUN apt-get update && apt-get install -y openssl ca-certificates libsodium23 wget && rm -rf /var/lib/apt/lists/* RUN mkdir -p /app/lib COPY --from=builder /go/pkg/mod/github.com/tonkeeper/tongo*/lib/linux /app/lib/ ENV LD_LIBRARY_PATH=/app/lib COPY --from=builder /tmp/testutil /app/testutil CMD ["/app/testutil", "-v"] ================================================ FILE: LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . ================================================ FILE: Makefile ================================================ VERSION := latest GIT_TAG := $(shell git describe --tags --always) build: @echo "Building tag: $(GIT_TAG)" docker build --build-arg GIT_TAG=$(GIT_TAG) -t payment-processor:$(VERSION) --target payment-processor . docker build --build-arg GIT_TAG=$(GIT_TAG) -t payment-test:$(VERSION) --target payment-test . ================================================ FILE: README.md ================================================ # TON payment processor [![Based on TON][ton-svg]][ton] [![Go](https://github.com/gobicycle/bicycle/actions/workflows/go.yml/badge.svg)](https://github.com/gobicycle/bicycle/actions/workflows/go.yml) [![Telegram][telegram-svg]][telegram-url] Microservice for accepting payments and making withdrawals to wallets in TON blockchain. Supports TON coins and Jettons (conforming certain criteria) Provides REST API for integration. Service is ADNL based and interacts directly with node and do not use any third party API. **Warning** Manual withdrawals from hot wallet is prohibited **do not withdraw funds from a hot wallet bypassing the service, this will lead to a service error** - [How it works](#How-it-works) - [Features](#Features) - [Glossary](#Glossary) - [Prerequisites](#Prerequisites) - [Criteria for valid Jettons](#Criteria-for-valid-Jettons) - [Deployment](#Deployment) - [Configurable parameters](#Configurable-parameters) - [Service deploy](#Service-deploy) - [Payment notifications](#Payment-notifications) - [Binary comment support](#Binary-comment-support) - [REST API](https://gobicycle.github.io/bicycle/) - [Technical notes](/technical_notes.md) - [Threat model](/threat_model.md) - [Manual migrations](/manual_migrations.md) - [TODO list](/todo_list.md) ![test_dashboard](https://user-images.githubusercontent.com/120649456/211955983-698b12b8-eccf-45c5-85bb-f8f6364c154e.png) ![db_dashboard](https://user-images.githubusercontent.com/120649456/211955998-749772a6-10d2-4594-96f1-be6ab6051be5.png) ## How it works The service provides the following functionality: - `generate new deposit address` of wallet (TON or Jetton) for TON blockchain. This address you provide to customer for payment. Payments are accumulated in the hot-wallet. - `make withdrawal` of TONs or Jettons from hot-wallet to customer wallet at TON blockchain. ### Features * Deposit addresses can be reused for multiple payments * Sends withdrawals with comment * Sends withdrawals in batches using highload wallet * Aggregates part of TONs or Jettons at cold-wallet * Supports authorization by Bearer token * Service withdrawals (cancellation of incorrect payments) ## Glossary - `deposit-address` - address generated by the service to which users send payments. - `deposit` - blockchain account with `deposit-address` - `hot-wallet` - wallet for aggregation all incoming TONs and Jettons from deposit-addresses. - `cold-wallet` - wallet to which part of the funds from the hot wallet is sent for security. Cold-wallet seed phrase is not used by the service. - `user_id` - unique text value to identify deposit-addresses or withdrawal request for a specific user. - `query_id` - unique text value to identify withdrawal request for a specific user to prevent double spending. - `basic unit` - minimum indivisible unit for TON (e.g. for TON `basic unit` = nanoTONs) or Jetton. - `hot_wallet_minimum_balance` - minimum TON balance in nanoTONs at hot wallet to start service. - `hot_wallet_maximum_balance` - maximum balance (of TONs or Jettons) in basic units at hot wallet. Anything more than this amount will be withdrawn to a cold wallet. - `minimum_withdrawal_amount` - minimum balance (of TONs or Jettons) in basic units at deposit account to make withdrawal to hot wallet. It is necessary to prevent the case when the withdrawal fee will be close to the balance on the deposit. ## Prerequisites - Need minimum (configured) amount of TONs at HighloadV2 wallet address correlated with seed phrase or already deployed HighloadV2 wallet. - To ensure the reliability and security of the service, you need to provide your own TON node (with lite server) on the same machine as the service. If you want to use an untrusted node (such as a rented node), you need to set `PROOF_CHECK_ENABLED=true` and specify `NETWORK_CONFIG_URL`. - Jettons used must meet certain criteria ### Criteria for valid Jettons - conforming to the standard [TEP-74](https://github.com/ton-blockchain/TEPs/blob/master/text/0074-jettons-standard.md) - the Jetton wallet should not spontaneously change its balance, only with transfer. - fee for the withdrawal of Jettons from the wallet should not be too high and meet the internal setting of the service For more information on Jettons compatibility, see [Jettons compatibility](/jettons.md) ## Deployment ### Configurable parameters | ENV variable | Description | |------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `LITESERVER` | IP and port of lite server, example: `185.86.76.183:5815` | | `LITESERVER_KEY` | public key of lite server `5v2dHtSclsGsZVbNVwTj4hQDso5xvQjzL/yPEHJevHk=`.
Be careful with base64 encoding and ENV var. Use '' | | `LITESERVER_RATE_LIMIT` | If you have a rented node with an RPS limit, set the RPS value here equal to (or preferably slightly less than) the limit. Default: 100. | | `SEED` | seed phrase for main hot wallet. 24 words compatible with standard TON wallets | | `DB_URI` | URI for DB connection, example:
`postgresql://db_user:db_password@localhost:5432/payment_processor` | | `POSTGRES_DB` | name of database for storing payments data | | `POSTGRES_READONLY_PASSWORD` | password for grafana readonly db user | | `API_PORT` | port for REST API, example `8081` | | `API_TOKEN` | Bearer token for REST API, example `123` | | `IS_TESTNET` | `true` if service works in TESTNET, `false` - for MAINNET. Default: `true`. | | `JETTONS` | list of Jettons, processed by service in format:
`JETTON_SYMBOL_1:MASTER_CONTRACT_ADDR_1:hot_wallet_max_balance:min_withdrawal_amount:hot_wallet_residual_balance, JETTON_SYMBOL_2:MASTER_CONTRACT_ADDR_2:hot_wallet_max_balance:min_withdrawal_amount:hot_wallet_residual_balance`,
example: `TGR:kQCKt2WPGX-fh0cIAz38Ljd_OKQjoZE_cqk7QrYGsNP6wfP0:1000000:100000` | | `TON_CUTOFFS` | cutoffs in nanoTONs in format:
`hot_wallet_min_balance:hot_wallet_max_balance:min_withdrawal_amount:hot_wallet_residual_balance`,
example `1000000000:100000000000:1000000000:95000000000` | | `COLD_WALLET` | cold-wallet address, example `kQCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseH3XZ_YiH9Y1ufw`. If cold wallet is not active - use non-bounceable address (use https://ton.org/address for convert) | | `DEPOSIT_SIDE_BALANCE` | `true` - service calculates total income for user by deposit incoming, `false` - by hot wallet incoming. Default: `true`. | | `QUEUE_ENABLED` | `true` - service sends incoming notifications to queue, `false` - sending disabled. Default: `false`. | | `QUEUE_URI` | URI for queue client connection, example `amqp://guest:guest@payment_rabbitmq:5672/` | | `QUEUE_NAME` | name of exchange | | `WEBHOOK_ENDPOINT` | endpoint to send webhooks, example: `http://hostname:3333/webhook`. If the value is not set, then webhooks are not sent. | | `WEBHOOK_TOKEN` | Bearer token for webhook request. If not set then not used. | | `ALLOWABLE_LAG` | allowable time lag between service time and last block time in seconds, default: 15 | | `FORWARD_TON_AMOUNT` | forward ton amount for jetton withdrawals, default: 1 nanoton | | `PROOF_CHECK_ENABLED` | enable verification of all proofs to securely connect to an untrusted node, default: `false`. Also you need to define `NETWORK_CONFIG_URL`. | | `NETWORK_CONFIG_URL` | the path to load the network configuration to get the trusted key block from it. This is necessary for proof verification, example: `https://ton.org/global.config.json` | **! Be careful with `IS_TESTNET` variable.** This does not guarantee that a testnet node is being used. It is only for address checking purposes. There are also internal service settings (fees and timeouts) that are specified in the source code in the [Config](/config/config.go) package. Calibration parameters recommendations in [Technical notes](/technical_notes.md). #### `hot_wallet_residual_balance` and `hot_wallet_max_balance` In order to avoid triggering a withdrawal to a cold wallet with each receipt of funds, a hysteresis is introduced. `hot_wallet_max_balance` - this is the amount at which the withdrawal from the hot wallet to the cold one will be triggered `hot_wallet_residual_balance` is the amount that will remain on the hot wallet after the withdrawal `hot_wallet_max_balance` must be greater than `hot_wallet_residual_balance` If the `hot_wallet_residual_balance` is not set, then it is calculated using the formula: `hot_wallet_residual_balance` = `hot_wallet_max_balance` * `hysteresis`, where hysteresis is a hardcoded value (at the time of writing this is 0.95) ### Service deploy **Do not use same `.env` file for `payment-processor` and other services!** 1. Build docker images from makefile ```console make -f Makefile ``` 2. Prepare `.env` file for `payment-postgres` service or fill environment variables in `docker-compose.yml` file. Database scheme automatically init. ```console docker-compose -f docker-compose.yml up -d payment-postgres ``` 3. Prepare `.env` file for `payment-processor` service or fill environment variables in `docker-compose.yml` file. ```console docker-compose -f docker-compose.yml up -d payment-processor ``` 4. Optionally you can start Grafana for service monitoring. Prepare `.env` file for `payment-grafana` service or fill environment variables in `docker-compose.yml` file. ```console docker-compose -f docker-compose.yml up -d payment-grafana ``` 5. Optionally you can start RabbitMQ to collect payment notifications (if `QUEUE_ENABLED` env var is `true` for payment-processor). Prepare `.env` file for `payment-rabbitmq` service or fill environment variables in `docker-compose.yml` file. ```console docker-compose -f docker-compose.yml up -d payment-rabbitmq ``` ## Payment notifications ATTENTION! Sending notifications does not guarantee that all notifications will be sent. If the service is restarted after the data is saved to the database and before the notification data is sent, these notifications will not be sent after restart. The service has several mechanisms for notification of payments. These are webhooks and a AMQP (to RabbitMQ). Depending on the `DEPOSIT_SIDE_BALANCE` setting, a notification is received either about the payment to the deposit address, or about the withdrawal from the deposit to the hot wallet. Source address and comment returned if known. Message format when `DEPOSIT_SIDE_BALANCE` == true: ```json { "deposit_address":"0QCdsj-u39qVlfYdpPKuAY0hTe5VIsiJcpB5Rx4tOUOyBFhL", "time": 12345678, "amount":"100", "source_address":"0QAOp2OZwWdkF5HhJ0WVDspgh6HhpmHyQ3cBuBmfJ4q_AIVe", "comment":"hello", "tx_hash": "f9b9e7efd3a38da318a894576499f0b6af5ca2da97ccd15c5f1d291a808a0ebf", "user_id": "123" } ``` Message format when `DEPOSIT_SIDE_BALANCE` == false (there is fewer data, because the accumulated amount is withdrawn from the deposit): ```json { "deposit_address":"0QCdsj-u39qVlfYdpPKuAY0hTe5VIsiJcpB5Rx4tOUOyBFhL", "time": 12345678, "amount":"200", "tx_hash": "f9b9e7efd3a38da318a894576499f0b6af5ca2da97ccd15c5f1d291a808a0ebf", "user_id": "123" } ``` ### Using RabbitMQ 1. Set `QUEUE_ENABLED = true` env variable 2. Set `QUEUE_URI` as described at [Configurable parameters](#Configurable-parameters) 3. Set `QUEUE_NAME` env variable. Be careful, this is not the name of a specific queue in the rabbit, but the name of the exchange. 4. Start RabbitMQ as described at [Service deploy](#Service-deploy) ### Using webhooks 1. Set `WEBHOOK_ENDPOINT` env variable 2. Optionally set `WEBHOOK_TOKEN` env variable When the `payment-processor` is running, it will send a `POST` request to the webhook endpoint with each payment and wait for a response with a `200` code and an empty body. If a successful delivery response is not received after several attempts, the service will stop with an error. If the variable `WEBHOOK_TOKEN` is set, it will also add header `Authorization: Bearer {token}`. ## Binary comment support Method `/v1/withdrawal/send` also supports `binary_comment`. The comment is written in a hex form. If the bits qty is not a multiple of a byte, then the record form with a flip bit is supported, for example `9fe7_`. A `binary_comment` is writing directly to the body of the message (for the TON transfer) and to the `forward_payload` (for the Jetton transfer) with its opcode according to the following TLB scheme: `binary_comment#b3ddcf7d {n:#} data:(SnakeData ~n) = InternalMsgBody;` `crc32('binary_comment n:# data:SnakeData ~n = InternalMsgBody') = 0xb3ddcf7d` This comment will not be displayed by the explorer as text and can be useful for transmitting metadata that will be read by indexers. The documentation [contains](https://docs.ton.org/develop/smart-contracts/guidelines/internal-messages#simple-message-with-comment) a standard way of writing a binary comment, but due to the fact that it is not supported by services, an alternative recording method was chosen. [ton-svg]: https://img.shields.io/badge/Based%20on-TON-blue [ton]: https://ton.org [telegram-url]: https://t.me/tonbicycle [telegram-svg]: https://img.shields.io/badge/telegram-chat-blue?color=blue&logo=telegram&logoColor=white ================================================ FILE: api/example.http ================================================ ### POST {{url}}/v1/address/new Authorization: Bearer {{token}} Content-Type: application/json {"user_id": "TestUser", "currency": "TON"} ### POST {{url}}/v1/address/new Authorization: Bearer {{token}} Content-Type: application/json {"user_id": "TestUser", "currency": "TGR"} ### GET {{url}}/v1/address/all?user_id=TestUser Authorization: Bearer {{token}} ### GET {{url}}/v1/income?user_id=TestUser Authorization: Bearer {{token}} ### GET {{url}}/v1/deposit/history?user_id=TestUser¤cy=TON&limit=3&offset=0&sort_order=asc Authorization: Bearer {{token}} ### POST {{url}}/v1/withdrawal/send Authorization: Bearer {{token}} Content-Type: application/json {"user_id": "TestUser", "query_id": "1", "currency": "TON", "amount": 200000000, "destination": "kQBFETbGASx3-6QYpPuQAKQM1s32AfSkWzbsADqt3bKDlN1A", "comment": "test_ton_withdrawal"} ### POST {{url}}/v1/withdrawal/send Authorization: Bearer {{token}} Content-Type: application/json {"user_id": "TestUser", "query_id": "2", "currency": "TGR", "amount": 1000, "destination": "kQBFETbGASx3-6QYpPuQAKQM1s32AfSkWzbsADqt3bKDlN1A", "comment": "test_jetton_withdrawal"} ### POST {{url}}/v1/withdrawal/send Authorization: Bearer {{token}} Content-Type: application/json {"user_id": "TestUser", "query_id": "3", "currency": "TON", "amount": 1000, "destination": "kQBFETbGASx3-6QYpPuQAKQM1s32AfSkWzbsADqt3bKDlN1A", "binary_comment": "9fe7_"} ### POST {{url}}/v1/withdrawal/service/jetton Authorization: Bearer {{token}} Content-Type: application/json {"owner": "0QCdsj-u39qVlfYdpPKuAY0hTe5VIsiJcpB5Rx4tOUOyBFhL", "jetton_master": "kQAbMQzuuGiCne0R7QEj9nrXsjM7gNjeVmrlBZouyC-SCALE"} ### POST {{url}}/v1/withdrawal/service/ton Authorization: Bearer {{token}} Content-Type: application/json {"from": "0QAOp2OZwWdkF5HhJ0WVDspgh6HhpmHyQ3cBuBmfJ4q_AIVe"} ### GET {{url}}/v1/withdrawal/status?id=2 Authorization: Bearer {{token}} ### GET {{url}}/v1/system/sync ### GET {{url}}/metrics ### GET {{url}}//v1/deposit/income?tx_hash=54e61136c33b94372030de8c7d02bc23a60e3de7cfad46f26258e8e722dc66b1 Authorization: Bearer {{token}} ### GET {{url}}//v1/balance?currency=TON&address=kQAbMQzuuGiCne0R7QEj9nrXsjM7gNjeVmrlBZouyC-SCALE Authorization: Bearer {{token}} ### GET {{url}}//v1/balance?currency=TON Authorization: Bearer {{token}} ### GET {{url}}//v1/resolve?domain=wallet.ton Authorization: Bearer {{token}} ================================================ FILE: api/handlers.go ================================================ package api import ( "bytes" "context" "encoding/hex" "encoding/json" "errors" "fmt" "github.com/gobicycle/bicycle/config" "github.com/gobicycle/bicycle/core" "github.com/gobicycle/bicycle/metrics" "github.com/gofrs/uuid" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" "github.com/tonkeeper/tongo/boc" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton/wallet" "math/big" "net/http" "strconv" "strings" "sync" ) type Handler struct { storage storage blockchain blockchain token string shard byte mutex sync.Mutex hotWalletAddress address.Address } type WithdrawalRequest struct { UserID string `json:"user_id"` QueryID string `json:"query_id"` Currency string `json:"currency"` Amount core.Coins `json:"amount"` Destination string `json:"destination"` Comment string `json:"comment"` BinaryComment string `json:"binary_comment"` } type ServiceTonWithdrawalRequest struct { From string `json:"from"` } type ServiceJettonWithdrawalRequest struct { Owner string `json:"owner"` JettonMaster string `json:"jetton_master"` } type WalletAddress struct { Address string `json:"address"` Currency string `json:"currency"` } type GetAddressesResponse struct { Addresses []WalletAddress `json:"addresses"` } type WithdrawalResponse struct { ID int64 `json:"ID"` } type GetBalanceResponse struct { Balance string `json:"balance"` Status string `json:"status,omitempty"` ProcessingAmount string `json:"total_processing_amount,omitempty"` PendingAmount string `json:"total_pending_amount,omitempty"` } type ResolveResponse struct { Address string `json:"address"` } type WithdrawalStatusResponse struct { UserID string `json:"user_id"` QueryID string `json:"query_id"` Status core.WithdrawalStatus `json:"status"` TxHash string `json:"tx_hash,omitempty"` } type GetIncomeResponse struct { Side string `json:"counting_side"` TotalIncomes []totalIncome `json:"total_income"` } type GetHistoryResponse struct { Incomes []income `json:"incomes"` } type GetIncomeByTxResponse struct { Currency string `json:"currency"` Income income `json:"income"` } type totalIncome struct { Address string `json:"deposit_address"` Amount string `json:"amount"` Currency string `json:"currency"` } type income struct { DepositAddress string `json:"deposit_address"` Time int64 `json:"time"` SourceAddress string `json:"source_address,omitempty"` Amount string `json:"amount"` Comment string `json:"comment,omitempty"` TxHash string `json:"tx_hash,omitempty"` } func NewHandler(s storage, b blockchain, token string, shard byte, hotWalletAddress address.Address) *Handler { return &Handler{storage: s, blockchain: b, token: token, shard: shard, hotWalletAddress: hotWalletAddress} } func (h *Handler) getNewAddress(resp http.ResponseWriter, req *http.Request) { var data struct { UserID string `json:"user_id"` Currency string `json:"currency"` } err := json.NewDecoder(req.Body).Decode(&data) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("decode payload data err: %v", err)) return } if !isValidCurrency(data.Currency) { writeHttpError(resp, http.StatusBadRequest, "invalid currency type") return } h.mutex.Lock() defer h.mutex.Unlock() // To prevent data race addr, err := generateAddress(req.Context(), data.UserID, data.Currency, h.shard, h.storage, h.blockchain, h.hotWalletAddress) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("generate address err: %v", err)) return } res := struct { Address string `json:"address"` }{Address: addr} resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) err = json.NewEncoder(resp).Encode(res) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) getAddresses(resp http.ResponseWriter, req *http.Request) { userID := req.URL.Query().Get("user_id") if userID == "" { writeHttpError(resp, http.StatusBadRequest, "need to provide user ID") return } addresses, err := getAddresses(req.Context(), userID, h.storage) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get addresses err: %v", err)) return } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) err = json.NewEncoder(resp).Encode(addresses) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) sendWithdrawal(resp http.ResponseWriter, req *http.Request) { var body WithdrawalRequest err := json.NewDecoder(req.Body).Decode(&body) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("decode payload err: %v", err)) return } w, err := convertWithdrawal(body) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("convert withdrawal err: %v", err)) return } unique, err := h.storage.IsWithdrawalRequestUnique(req.Context(), w) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("check withdrawal uniquess err: %v", err)) return } else if !unique { writeHttpError(resp, http.StatusBadRequest, "(user_id,query_id) not unique") return } _, ok := h.storage.GetWalletType(w.Destination) if ok { writeHttpError(resp, http.StatusBadRequest, "withdrawal to service internal addresses not supported") return } id, err := h.storage.SaveWithdrawalRequest(req.Context(), w) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("save withdrawal request err: %v", err)) return } r := WithdrawalResponse{ID: id} resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) err = json.NewEncoder(resp).Encode(r) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) getSync(resp http.ResponseWriter, req *http.Request) { isSynced, utime, err := h.storage.IsActualBlockData(req.Context()) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get sync from db err: %v", err)) return } getSyncResponse := struct { IsSynced bool `json:"is_synced"` BlockTime int64 `json:"last_block_gen_utime"` }{ IsSynced: isSynced, BlockTime: utime, } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) err = json.NewEncoder(resp).Encode(getSyncResponse) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) getWithdrawalStatus(resp http.ResponseWriter, req *http.Request) { ids := req.URL.Query().Get("id") if ids == "" { writeHttpError(resp, http.StatusBadRequest, "need to provide request ID") return } id, err := strconv.ParseInt(ids, 10, 64) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("convert request ID err: %v", err)) return } status, err := h.storage.GetExternalWithdrawalStatus(req.Context(), id) if errors.Is(err, core.ErrNotFound) { writeHttpError(resp, http.StatusBadRequest, "request ID not found") return } if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get external withdrawal status err: %v", err)) return } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) res := WithdrawalStatusResponse{ UserID: status.UserID, QueryID: status.QueryID, Status: status.Status, } if status.TxHash != nil { res.TxHash = fmt.Sprintf("%x", status.TxHash) } err = json.NewEncoder(resp).Encode(res) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) getIncome(resp http.ResponseWriter, req *http.Request) { id := req.URL.Query().Get("user_id") if id == "" { writeHttpError(resp, http.StatusBadRequest, "need to provide user ID") return } totalIncomes, err := h.storage.GetIncome(req.Context(), id, config.Config.IsDepositSideCalculation) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get income err: %v", err)) return } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) err = json.NewEncoder(resp).Encode(convertIncome(h.storage, totalIncomes)) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) getIncomeHistory(resp http.ResponseWriter, req *http.Request) { id := req.URL.Query().Get("user_id") if id == "" { writeHttpError(resp, http.StatusBadRequest, "need to provide user ID") return } currency := req.URL.Query().Get("currency") if currency == "" { writeHttpError(resp, http.StatusBadRequest, "need to provide currency") return } if !isValidCurrency(currency) { writeHttpError(resp, http.StatusBadRequest, "invalid currency type") return } limit, err := strconv.Atoi(req.URL.Query().Get("limit")) if err != nil { writeHttpError(resp, http.StatusBadRequest, "invalid limit parameter") return } offset, err := strconv.Atoi(req.URL.Query().Get("offset")) if err != nil { writeHttpError(resp, http.StatusBadRequest, "invalid offset parameter") return } ascOrder := false sort := strings.ToLower(req.URL.Query().Get("sort_order")) if sort == "asc" { ascOrder = true } else if sort != "" && sort != "desc" { writeHttpError(resp, http.StatusBadRequest, "invalid sort order") return } history, err := h.storage.GetIncomeHistory(req.Context(), id, currency, limit, offset, ascOrder) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get history err: %v", err)) return } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) err = json.NewEncoder(resp).Encode(convertHistory(h.storage, currency, history)) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) serviceTonWithdrawal(resp http.ResponseWriter, req *http.Request) { var body ServiceTonWithdrawalRequest err := json.NewDecoder(req.Body).Decode(&body) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("decode payload err: %v", err)) return } w, err := convertTonServiceWithdrawal(h.storage, body) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("convert service withdrawal err: %v", err)) return } memo, err := h.storage.SaveServiceWithdrawalRequest(req.Context(), w) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("save service withdrawal request err: %v", err)) return } var response = struct { Memo uuid.UUID `json:"memo"` }{ Memo: memo, } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) err = json.NewEncoder(resp).Encode(response) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) serviceJettonWithdrawal(resp http.ResponseWriter, req *http.Request) { var body ServiceJettonWithdrawalRequest err := json.NewDecoder(req.Body).Decode(&body) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("decode payload err: %v", err)) return } w, err := convertJettonServiceWithdrawal(h.storage, body) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("convert service withdrawal err: %v", err)) return } memo, err := h.storage.SaveServiceWithdrawalRequest(req.Context(), w) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("save service withdrawal request err: %v", err)) return } var response = struct { Memo uuid.UUID `json:"memo"` }{ Memo: memo, } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) err = json.NewEncoder(resp).Encode(response) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) getMetrics(resp http.ResponseWriter, req *http.Request) { buf := new(bytes.Buffer) for _, m := range metrics.AllMetrics { err := m.Print(buf) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get metrics err: %v", err.Error())) return } } resp.Header().Add("Content-Type", "application/text") resp.WriteHeader(http.StatusOK) _, err := resp.Write(buf.Bytes()) if err != nil { log.Errorf("buffer writing error: %v", err) } } func (h *Handler) getIncomeByTx(resp http.ResponseWriter, req *http.Request) { txHash := strings.ToLower(req.URL.Query().Get("tx_hash")) hash, err := hex.DecodeString(txHash) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("get tx hash err: %v", err.Error())) return } if len(hash) != 32 { writeHttpError(resp, http.StatusBadRequest, "invalid hash len") return } oneIncome, currency, err := h.storage.GetIncomeByTx(req.Context(), hash) if errors.Is(err, core.ErrNotFound) { writeHttpError(resp, http.StatusNotFound, "transaction not found") return } if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get income by tx err: %v", err)) return } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) res := GetIncomeByTxResponse{Currency: currency, Income: convertOneIncome(h.storage, currency, *oneIncome)} err = json.NewEncoder(resp).Encode(res) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) getBalance(resp http.ResponseWriter, req *http.Request) { currency := req.URL.Query().Get("currency") if currency == "" { writeHttpError(resp, http.StatusBadRequest, "need to provide currency") return } if !isValidCurrency(currency) { writeHttpError(resp, http.StatusBadRequest, "invalid currency type") return } var ( tonWalletAddress core.Address err error balance *big.Int status tlb.AccountStatus res GetBalanceResponse ) addr := req.URL.Query().Get("address") if addr != "" { tonWalletAddress, _, err = validateAddress(addr) if err != nil { writeHttpError(resp, http.StatusBadRequest, fmt.Sprintf("invalid address: %s", err.Error())) return } } else { tonWalletAddress = core.AddressMustFromTonutilsAddress(&h.hotWalletAddress) amounts, err := h.storage.GetTotalWithdrawalAmounts(req.Context(), currency) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get total withdrawal amounts err: %v", err)) return } res.PendingAmount = amounts.Pending.String() res.ProcessingAmount = amounts.Processing.String() } if currency == core.TonSymbol { balance, status, err = h.blockchain.GetAccountCurrentState(req.Context(), tonWalletAddress.ToTonutilsAddressStd(0)) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get TON balance err: %v", err)) return } res.Status = strings.ToLower(string(status)) } else { jetton, _ := config.Config.Jettons[currency] // currency validate earlier balance, err = h.blockchain.GetJettonBalanceByOwner(req.Context(), tonWalletAddress.ToTonutilsAddressStd(0), jetton.Master) if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get jetton balance err: %v", err)) return } } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) res.Balance = balance.String() err = json.NewEncoder(resp).Encode(res) if err != nil { log.Errorf("json encode error: %v", err) } } func (h *Handler) getResolve(resp http.ResponseWriter, req *http.Request) { domain := req.URL.Query().Get("domain") if domain == "" { writeHttpError(resp, http.StatusBadRequest, "invalid domain") return } addr, err := h.blockchain.DnsResolveSmc(req.Context(), domain) if errors.Is(err, core.ErrNotFound) { writeHttpError(resp, http.StatusNotFound, "smart contract DNS record not found") return } if err != nil { writeHttpError(resp, http.StatusInternalServerError, fmt.Sprintf("get DNS record err: %v", err)) return } addr.SetTestnetOnly(config.Config.Testnet) addr.SetBounce(true) resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(http.StatusOK) res := ResolveResponse{Address: addr.String()} err = json.NewEncoder(resp).Encode(res) if err != nil { log.Errorf("json encode error: %v", err) } } func RegisterHandlers(mux *http.ServeMux, h *Handler) { mux.HandleFunc("/v1/address/new", recoverMiddleware(authMiddleware(post(h.getNewAddress)))) mux.HandleFunc("/v1/address/all", recoverMiddleware(authMiddleware(get(h.getAddresses)))) mux.HandleFunc("/v1/withdrawal/send", recoverMiddleware(authMiddleware(post(h.sendWithdrawal)))) mux.HandleFunc("/v1/withdrawal/service/ton", recoverMiddleware(authMiddleware(post(h.serviceTonWithdrawal)))) mux.HandleFunc("/v1/withdrawal/service/jetton", recoverMiddleware(authMiddleware(post(h.serviceJettonWithdrawal)))) mux.HandleFunc("/v1/withdrawal/status", recoverMiddleware(authMiddleware(get(h.getWithdrawalStatus)))) mux.HandleFunc("/v1/system/sync", recoverMiddleware(get(h.getSync))) mux.HandleFunc("/v1/income", recoverMiddleware(authMiddleware(get(h.getIncome)))) mux.HandleFunc("/v1/deposit/history", recoverMiddleware(authMiddleware(get(h.getIncomeHistory)))) mux.HandleFunc("/v1/deposit/income", recoverMiddleware(authMiddleware(get(h.getIncomeByTx)))) mux.HandleFunc("/v1/balance", recoverMiddleware(authMiddleware(get(h.getBalance)))) mux.HandleFunc("/v1/resolve", recoverMiddleware(authMiddleware(get(h.getResolve)))) mux.HandleFunc("/metrics", recoverMiddleware(get(h.getMetrics))) } func generateAddress( ctx context.Context, userID string, currency string, shard byte, dbConn storage, bc blockchain, hotWalletAddress address.Address, ) ( string, error, ) { subwalletID, err := dbConn.GetLastSubwalletID(ctx) if err != nil { return "", err } var res string if currency == core.TonSymbol { w, id, err := bc.GenerateSubWallet(config.Config.Seed, shard, subwalletID+1) if err != nil { return "", err } a, err := core.AddressFromTonutilsAddress(w.Address()) if err != nil { return "", err } err = dbConn.SaveTonWallet(ctx, core.WalletData{ SubwalletID: id, UserID: userID, Currency: core.TonSymbol, Type: core.TonDepositWallet, Address: a, }, ) if err != nil { return "", err } res = a.ToUserFormat() } else { jetton, ok := config.Config.Jettons[currency] if !ok { return "", fmt.Errorf("jetton address not found") } proxy, addr, err := bc.GenerateDepositJettonWalletForProxy(ctx, shard, &hotWalletAddress, jetton.Master, subwalletID+1) if err != nil { return "", err } jettonWalletAddr, err := core.AddressFromTonutilsAddress(addr) if err != nil { return "", err } proxyAddr, err := core.AddressFromTonutilsAddress(proxy.Address()) if err != nil { return "", err } err = dbConn.SaveJettonWallet( ctx, proxyAddr, core.WalletData{ UserID: userID, SubwalletID: proxy.SubwalletID, Currency: currency, Type: core.JettonDepositWallet, Address: jettonWalletAddr, }, false, ) if err != nil { return "", err } res = proxyAddr.ToUserFormat() } return res, nil } func getAddresses(ctx context.Context, userID string, dbConn storage) (GetAddressesResponse, error) { var res = GetAddressesResponse{ Addresses: []WalletAddress{}, } tonAddr, err := dbConn.GetTonWalletsAddresses(ctx, userID, []core.WalletType{core.TonDepositWallet}) if err != nil { return GetAddressesResponse{}, err } jettonAddr, err := dbConn.GetJettonOwnersAddresses(ctx, userID, []core.WalletType{core.JettonDepositWallet}) if err != nil { return GetAddressesResponse{}, err } for _, a := range tonAddr { res.Addresses = append(res.Addresses, WalletAddress{Address: a.ToUserFormat(), Currency: core.TonSymbol}) } for _, a := range jettonAddr { res.Addresses = append(res.Addresses, WalletAddress{Address: a.Address.ToUserFormat(), Currency: a.Currency}) } return res, nil } func isValidCommentLen(comment string) bool { return len(comment) < config.MaxCommentLength } func isValidCurrency(cur string) bool { if _, ok := config.Config.Jettons[cur]; ok || cur == core.TonSymbol { return true } return false } func convertWithdrawal(w WithdrawalRequest) (core.WithdrawalRequest, error) { if !isValidCurrency(w.Currency) { return core.WithdrawalRequest{}, fmt.Errorf("invalid currency") } addr, bounceable, err := validateAddress(w.Destination) if err != nil { return core.WithdrawalRequest{}, fmt.Errorf("invalid destination address: %v", err) } if !(w.Amount.Cmp(decimal.New(0, 0)) == 1) { return core.WithdrawalRequest{}, fmt.Errorf("amount must be > 0") } if w.Comment != "" && w.BinaryComment != "" { return core.WithdrawalRequest{}, fmt.Errorf("only one type of comment can be specified (comment OR binary comment)") } if !isValidCommentLen(w.Comment) || !isValidCommentLen(w.BinaryComment) { return core.WithdrawalRequest{}, fmt.Errorf("too long comment, max length allowed: %d", config.MaxCommentLength) } res := core.WithdrawalRequest{ UserID: w.UserID, QueryID: w.QueryID, Currency: w.Currency, Amount: w.Amount, Destination: addr, Bounceable: bounceable, Comment: w.Comment, IsInternal: false, } if w.BinaryComment != "" { _, err = boc.BitStringFromFiftHex(w.BinaryComment) if err != nil { return core.WithdrawalRequest{}, fmt.Errorf("decode binary comment error: %v", err) } res.BinaryComment = w.BinaryComment } return res, nil } func convertTonServiceWithdrawal(s storage, w ServiceTonWithdrawalRequest) (core.ServiceWithdrawalRequest, error) { from, _, err := validateAddress(w.From) if err != nil { return core.ServiceWithdrawalRequest{}, fmt.Errorf("invalid from address: %v", err) } t, ok := s.GetWalletType(from) if !ok { return core.ServiceWithdrawalRequest{}, fmt.Errorf("unknown deposit address") } if t != core.JettonOwner { return core.ServiceWithdrawalRequest{}, fmt.Errorf("service withdrawal allowed only for Jetton deposit owner") } return core.ServiceWithdrawalRequest{ From: from, }, nil } func convertJettonServiceWithdrawal(s storage, w ServiceJettonWithdrawalRequest) (core.ServiceWithdrawalRequest, error) { from, _, err := validateAddress(w.Owner) if err != nil { return core.ServiceWithdrawalRequest{}, fmt.Errorf("invalid from address: %v", err) } t, ok := s.GetWalletType(from) if !ok { return core.ServiceWithdrawalRequest{}, fmt.Errorf("unknown deposit address") } if t != core.JettonOwner && t != core.TonDepositWallet { return core.ServiceWithdrawalRequest{}, fmt.Errorf("service withdrawal allowed only for Jetton deposit owner or TON deposit") } jetton, _, err := validateAddress(w.JettonMaster) if err != nil { return core.ServiceWithdrawalRequest{}, fmt.Errorf("invalid jetton master address: %v", err) } // currency type checks by withdrawal processor return core.ServiceWithdrawalRequest{ From: from, JettonMaster: &jetton, }, nil } func convertIncome(dbConn storage, totalIncomes []core.TotalIncome) GetIncomeResponse { var res = GetIncomeResponse{ TotalIncomes: []totalIncome{}, } if config.Config.IsDepositSideCalculation { res.Side = core.SideDeposit } else { res.Side = core.SideHotWallet } for _, b := range totalIncomes { totIncome := totalIncome{ Amount: b.Amount.String(), Currency: b.Currency, } if b.Currency == core.TonSymbol { totIncome.Address = b.Deposit.ToUserFormat() } else { owner := dbConn.GetOwner(b.Deposit) if owner == nil { // TODO: remove fatal log.Fatalf("can not find owner for deposit: %s", b.Deposit.ToUserFormat()) } totIncome.Address = owner.ToUserFormat() } res.TotalIncomes = append(res.TotalIncomes, totIncome) } return res } func convertOneIncome(dbConn storage, currency string, oneIncome core.ExternalIncome) income { inc := income{ Time: int64(oneIncome.Utime), Amount: oneIncome.Amount.String(), Comment: oneIncome.Comment, TxHash: fmt.Sprintf("%x", oneIncome.TxHash), } if currency == core.TonSymbol { inc.DepositAddress = oneIncome.To.ToUserFormat() } else { owner := dbConn.GetOwner(oneIncome.To) if owner == nil { // TODO: remove fatal log.Fatalf("can not find owner for deposit: %s", oneIncome.To.ToUserFormat()) } inc.DepositAddress = owner.ToUserFormat() } // show only std address if len(oneIncome.From) == 32 && oneIncome.FromWorkchain != nil { addr := address.NewAddress(0, byte(*oneIncome.FromWorkchain), oneIncome.From) addr.SetTestnetOnly(config.Config.Testnet) inc.SourceAddress = addr.String() } return inc } func convertHistory(dbConn storage, currency string, incomes []core.ExternalIncome) GetHistoryResponse { var res = GetHistoryResponse{ Incomes: []income{}, } for _, i := range incomes { inc := convertOneIncome(dbConn, currency, i) res.Incomes = append(res.Incomes, inc) } return res } func validateAddress(addr string) (core.Address, bool, error) { if addr == "" { return core.Address{}, false, fmt.Errorf("empty address") } a, err := address.ParseAddr(addr) if err != nil { return core.Address{}, false, fmt.Errorf("invalid address: %v", err) } if a.IsTestnetOnly() && !config.Config.Testnet { return core.Address{}, false, fmt.Errorf("address for testnet only") } if a.Workchain() != core.DefaultWorkchain { return core.Address{}, false, fmt.Errorf("address must be in %d workchain", core.DefaultWorkchain) } res, err := core.AddressFromTonutilsAddress(a) return res, a.IsBounceable(), err } type storage interface { GetLastSubwalletID(ctx context.Context) (uint32, error) SaveTonWallet(ctx context.Context, walletData core.WalletData) error SaveJettonWallet(ctx context.Context, ownerAddress core.Address, walletData core.WalletData, notSaveOwner bool) error GetTonWalletsAddresses(ctx context.Context, userID string, types []core.WalletType) ([]core.Address, error) GetJettonOwnersAddresses(ctx context.Context, userID string, types []core.WalletType) ([]core.OwnerWallet, error) SaveWithdrawalRequest(ctx context.Context, w core.WithdrawalRequest) (int64, error) IsWithdrawalRequestUnique(ctx context.Context, w core.WithdrawalRequest) (bool, error) IsActualBlockData(ctx context.Context) (bool, int64, error) GetExternalWithdrawalStatus(ctx context.Context, id int64) (core.WithdrawalData, error) GetWalletType(address core.Address) (core.WalletType, bool) GetIncome(ctx context.Context, userID string, isDepositSide bool) ([]core.TotalIncome, error) SaveServiceWithdrawalRequest(ctx context.Context, w core.ServiceWithdrawalRequest) (uuid.UUID, error) GetIncomeHistory(ctx context.Context, userID string, currency string, limit int, offset int, ascOrder bool) ([]core.ExternalIncome, error) GetOwner(address core.Address) *core.Address GetIncomeByTx(ctx context.Context, txHash []byte) (*core.ExternalIncome, string, error) GetTotalWithdrawalAmounts(ctx context.Context, currency string) (*core.TotalWithdrawalsAmount, error) } type blockchain interface { GenerateSubWallet(seed string, shard byte, startSubWalletID uint32) (*wallet.Wallet, uint32, error) GenerateDepositJettonWalletForProxy( ctx context.Context, shard byte, proxyOwner, jettonMaster *address.Address, startSubWalletID uint32, ) ( proxy *core.JettonProxy, addr *address.Address, err error, ) GenerateDefaultWallet(seed string, isHighload bool) (*wallet.Wallet, byte, uint32, error) GetAccountCurrentState(ctx context.Context, address *address.Address) (*big.Int, tlb.AccountStatus, error) GetJettonBalanceByOwner(ctx context.Context, owner *address.Address, jettonMaster *address.Address) (*big.Int, error) DnsResolveSmc(ctx context.Context, domainName string) (*address.Address, error) } ================================================ FILE: api/http-client.env.json ================================================ { "local": { "url": "http://localhost:8081" } } ================================================ FILE: api/middleware.go ================================================ package api import ( "crypto/subtle" "encoding/json" "github.com/gobicycle/bicycle/config" log "github.com/sirupsen/logrus" "net/http" "runtime/debug" "strings" ) func recoverMiddleware(next func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { defer func() { if err := recover(); err != nil { w.WriteHeader(http.StatusInternalServerError) log.Errorf( "err: %v trace %v", err, debug.Stack(), ) } }() next(w, r) } } func authMiddleware(next func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { if !checkToken(r, config.Config.APIToken) { w.WriteHeader(http.StatusUnauthorized) return } next(w, r) } } func get(next func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { writeHttpError(w, http.StatusBadRequest, "only GET method is supported") return } next(w, r) } } func post(next func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { writeHttpError(w, http.StatusBadRequest, "only POST method is supported") return } next(w, r) } } func checkToken(req *http.Request, token string) bool { auth := strings.Split(req.Header.Get("authorization"), " ") if len(auth) != 2 { return false } if auth[0] != "Bearer" { return false } if x := subtle.ConstantTimeCompare([]byte(auth[1]), []byte(token)); x == 1 { return true } // constant time comparison to prevent time attack return false } func writeHttpError(resp http.ResponseWriter, status int, comment string) { body := struct { Error string `json:"error"` }{ Error: comment, } resp.Header().Add("Content-Type", "application/json") resp.WriteHeader(status) err := json.NewEncoder(resp).Encode(body) if err != nil { log.Errorf("json encode error: %v", err) } } ================================================ FILE: audit/log.go ================================================ package audit import ( "fmt" "github.com/gobicycle/bicycle/metrics" log "github.com/sirupsen/logrus" "time" ) type Severity string const ( Error Severity = "ERROR" Warning Severity = "WARNING" Info Severity = "INFO" ) type message struct { Severity Severity Text string } func pushLog(m message) { switch m.Severity { case Error: log.Printf("AUDIT|%v|%v|%s", m.Severity, time.Now().Format(time.RFC1123), m.Text) metrics.Errors.Inc() case Warning: log.Printf("AUDIT|%v|%v|%s", m.Severity, time.Now().Format(time.RFC1123), m.Text) metrics.Warnings.Inc() case Info: log.Printf("AUDIT|%v|%v|%s", m.Severity, time.Now().Format(time.RFC1123), m.Text) metrics.Info.Inc() } } func LogTX(severity Severity, location string, hash []byte, text string) { pushLog(message{ Severity: severity, Text: fmt.Sprintf("%s|TX:%x|%s", location, hash, text), }) } func Log(severity Severity, location, event, text string) { pushLog(message{ Severity: severity, Text: fmt.Sprintf("%s|%s|%s", location, event, text), }) } ================================================ FILE: blockchain/blockchain.go ================================================ package blockchain import ( "context" "errors" "fmt" "github.com/gobicycle/bicycle/config" "github.com/gobicycle/bicycle/core" log "github.com/sirupsen/logrus" "github.com/tonkeeper/tongo" "github.com/tonkeeper/tongo/boc" tongoTlb "github.com/tonkeeper/tongo/tlb" "github.com/tonkeeper/tongo/tvm" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/liteclient" "github.com/xssnick/tonutils-go/tl" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton" "github.com/xssnick/tonutils-go/ton/dns" "github.com/xssnick/tonutils-go/ton/jetton" "github.com/xssnick/tonutils-go/ton/wallet" "github.com/xssnick/tonutils-go/tvm/cell" "math" "math/big" "sort" "strings" "time" ) type Connection struct { client ton.APIClientWrapped resolver *dns.Client } func (c *Connection) WaitForBlock(seqno uint32) ton.APIClientWrapped { return c.client.WaitForBlock(seqno) } func (c *Connection) FindLastTransactionByInMsgHash(ctx context.Context, addr *address.Address, msgHash []byte, maxTxNumToScan ...int) (*tlb.Transaction, error) { //TODO implement me panic("implement me") } func (c *Connection) FindLastTransactionByOutMsgHash(ctx context.Context, addr *address.Address, msgHash []byte, maxTxNumToScan ...int) (*tlb.Transaction, error) { //TODO implement me panic("implement me") } type contract struct { Address tongo.AccountID Code *boc.Cell Data *boc.Cell } // NewConnection creates new Blockchain connection func NewConnection(addr, key string, rateLimit int) (*Connection, error) { client := liteclient.NewConnectionPool() ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) defer cancel() err := client.AddConnection(ctx, addr, key) if err != nil { return nil, fmt.Errorf("connection err: %v", err.Error()) } limitedClient := newLimitedClient(client, rateLimit) var wrappedClient ton.APIClientWrapped if config.Config.ProofCheckEnabled { if config.Config.NetworkConfigUrl == "" { return nil, fmt.Errorf("empty network config URL") } cfg, err := liteclient.GetConfigFromUrl(ctx, config.Config.NetworkConfigUrl) if err != nil { return nil, fmt.Errorf("get network config from url err: %s", err.Error()) } wrappedClient = ton.NewAPIClient(limitedClient, ton.ProofCheckPolicySecure).WithRetry() wrappedClient.SetTrustedBlockFromConfig(cfg) log.Infof("Fetching and checking proofs since config init block ...") _, err = wrappedClient.CurrentMasterchainInfo(ctx) // we fetch block just to trigger chain proof check if err != nil { return nil, fmt.Errorf("get masterchain info err: %s", err.Error()) } log.Infof("Proof checks are completed") } else { wrappedClient = ton.NewAPIClient(limitedClient, ton.ProofCheckPolicyUnsafe).WithRetry() } // TODO: replace after tonutils fix rootDNS, bcConfig, err := getConfigData(ctx, wrappedClient) if err != nil { return nil, fmt.Errorf("get config data err: %s", err.Error()) } config.Config.BlockchainConfig = bcConfig resolver := dns.NewDNSClient(wrappedClient, rootDNS) return &Connection{ client: wrappedClient, resolver: resolver, }, nil } func getConfigData(ctx context.Context, api ton.APIClientWrapped) (*address.Address, *boc.Cell, error) { b, err := api.CurrentMasterchainInfo(ctx) if err != nil { return nil, nil, fmt.Errorf("failed to get masterchain info: %w", err) } cfg, err := api.GetBlockchainConfig(ctx, b, 4) if err != nil { return nil, nil, fmt.Errorf("failed to get blockchain config: %w", err) } cfgDict, err := getBlockchainConfig(ctx, api.Client(), b) if err != nil { return nil, nil, fmt.Errorf("failed to get config dict: %w", err) } cfgCell, err := tlb.ToCell(cfgDict) if err != nil { return nil, nil, fmt.Errorf("failed to serialize blockchain config: %w", err) } configCell, err := boc.DeserializeBoc(cfgCell.ToBOC()) if err != nil { return nil, nil, fmt.Errorf("failed to deserialize blockchain config: %w", err) } if len(configCell) != 1 { return nil, nil, fmt.Errorf("blockchain config must conatins only one cell") } data := cfg.Get(4) if data == nil { return nil, nil, fmt.Errorf("failed to get root address from blockchain config") } hash, err := data.BeginParse().LoadSlice(256) if err != nil { return nil, nil, fmt.Errorf("failed to get root address from blockchain config 4, failed to load hash: %w", err) } return address.NewAddress(0, 255, hash), configCell[0], nil } // GenerateDefaultWallet generates HighloadV2R2 or V3R2 TON wallet with // default subwallet_id and returns wallet, shard and subwalletID func (c *Connection) GenerateDefaultWallet(seed string, isHighload bool) ( w *wallet.Wallet, shard byte, subwalletID uint32, err error, ) { words := strings.Split(seed, " ") if isHighload { w, err = wallet.FromSeed(c, words, wallet.HighloadV2R2) } else { w, err = wallet.FromSeed(c, words, wallet.V3) } if err != nil { return nil, 0, 0, err } return w, w.Address().Data()[0], uint32(wallet.DefaultSubwallet), nil } // GenerateSubWallet generates subwallet for custom shard and // subwallet_id >= startSubWalletId and returns wallet and new subwallet_id func (c *Connection) GenerateSubWallet(seed string, shard byte, startSubWalletID uint32) (*wallet.Wallet, uint32, error) { words := strings.Split(seed, " ") basic, err := wallet.FromSeed(c, words, wallet.V3) if err != nil { return nil, 0, err } for id := startSubWalletID; id < math.MaxUint32; id++ { subWallet, err := basic.GetSubwallet(id) if err != nil { return nil, 0, err } addr, err := core.AddressFromTonutilsAddress(subWallet.Address()) if err != nil { return nil, 0, err } if inShard(addr, shard) { return subWallet, id, nil } } return nil, 0, fmt.Errorf("subwallet not found") } // GetJettonWalletAddress generates jetton wallet address from owner and jetton master addresses func (c *Connection) GetJettonWalletAddress( ctx context.Context, owner *address.Address, jettonMaster *address.Address, ) (*address.Address, error) { contr, err := c.getContract(ctx, jettonMaster) if err != nil { return nil, err } emulator, err := newEmulator(contr.Code, contr.Data) if err != nil { return nil, err } addr, err := getJettonWalletAddressByTVM(owner, contr.Address, emulator) if err != nil { return nil, err } res := addr.ToTonutilsAddressStd(0) res.SetTestnetOnly(config.Config.Testnet) return res, nil } func (c *Connection) GetJettonBalanceByOwner( ctx context.Context, owner *address.Address, jettonMaster *address.Address, ) (*big.Int, error) { jettonMasterClient := jetton.NewJettonMasterClient(c.client, jettonMaster) jettonWalletClient, err := jettonMasterClient.GetJettonWallet(ctx, owner) if err != nil { return nil, err } return jettonWalletClient.GetBalance(ctx) } func (c *Connection) DnsResolveSmc( ctx context.Context, domainName string, ) (*address.Address, error) { // TODO: it is necessary to distinguish network errors from the impossibility of resolving domain, err := c.resolver.Resolve(ctx, domainName) if errors.Is(err, dns.ErrNoSuchRecord) { return nil, core.ErrNotFound } else if err != nil { return nil, err } // TODO: replace after tonutils fix smcAddr := getWalletRecord(domain) if smcAddr == nil { // not wallet return nil, core.ErrNotFound } return smcAddr, nil } func getWalletRecord(d *dns.Domain) *address.Address { // TODO: remove after tonutils fix rec := d.GetRecord("wallet") if rec == nil { return nil } p := rec.BeginParse() p, err := p.LoadRef() if err != nil { return nil } category, err := p.LoadUInt(16) if err != nil { return nil } if category != 0x9fd3 { // const _CategoryContractAddr = 0x9fd3 return nil } addr, err := p.LoadAddr() if err != nil { return nil } return addr } // GenerateDepositJettonWalletForProxy // Generates jetton wallet address for custom shard and proxy contract as owner with subwallet_id >= startSubWalletId func (c *Connection) GenerateDepositJettonWalletForProxy( ctx context.Context, shard byte, proxyOwner, jettonMaster *address.Address, startSubWalletID uint32, ) ( proxy *core.JettonProxy, addr *address.Address, err error, ) { contr, err := c.getContract(ctx, jettonMaster) if err != nil { return nil, nil, err } emulator, err := newEmulator(contr.Code, contr.Data) if err != nil { return nil, nil, err } for id := startSubWalletID; id < math.MaxUint32; id++ { proxy, err = core.NewJettonProxy(id, proxyOwner) if err != nil { return nil, nil, err } jettonWalletAddress, err := getJettonWalletAddressByTVM(proxy.Address(), contr.Address, emulator) if err != nil { return nil, nil, err } if inShard(jettonWalletAddress, shard) { addr = jettonWalletAddress.ToTonutilsAddressStd(0) addr.SetTestnetOnly(config.Config.Testnet) return proxy, addr, nil } } return nil, nil, fmt.Errorf("jetton wallet address not found") } func (c *Connection) getContract(ctx context.Context, addr *address.Address) (contract, error) { block, err := c.client.CurrentMasterchainInfo(ctx) if err != nil { return contract{}, err } account, err := c.WaitForBlock(block.SeqNo).GetAccount(ctx, block, addr) if err != nil { return contract{}, err } if account == nil || account.Code == nil || account.Data == nil { return contract{}, fmt.Errorf("empty account code or data") } accountID, err := tongo.ParseAccountID(addr.String()) if err != nil { return contract{}, err } codeCell, err := boc.DeserializeBoc(account.Code.ToBOC()) if err != nil { return contract{}, err } if len(codeCell) != 1 { return contract{}, fmt.Errorf("BOC must have only one root") } dataCell, err := boc.DeserializeBoc(account.Data.ToBOC()) if err != nil { return contract{}, err } if len(dataCell) != 1 { return contract{}, fmt.Errorf("BOC must have only one root") } return contract{ Address: accountID, Code: codeCell[0], Data: dataCell[0], }, nil } func getJettonWalletAddressByTVM( owner *address.Address, jettonMaster tongo.AccountID, emulator *tvm.Emulator, ) (core.Address, error) { ownerAccountID, err := tongo.ParseAccountID(owner.String()) if err != nil { return core.Address{}, err } slice, err := tongoTlb.TlbStructToVmCellSlice(ownerAccountID.ToMsgAddress()) if err != nil { return core.Address{}, err } code, result, err := emulator.RunSmcMethod(context.Background(), jettonMaster, "get_wallet_address", tongoTlb.VmStack{slice}) if err != nil { return core.Address{}, err } if code != 0 || len(result) != 1 || result[0].SumType != "VmStkSlice" { return core.Address{}, fmt.Errorf("tvm execution failed") } var msgAddress tongoTlb.MsgAddress err = result[0].VmStkSlice.UnmarshalToTlbStruct(&msgAddress) if err != nil { return core.Address{}, err } if msgAddress.SumType != "AddrStd" { return core.Address{}, fmt.Errorf("not std jetton wallet address") } if msgAddress.AddrStd.WorkchainId != core.DefaultWorkchain { return core.Address{}, fmt.Errorf("not default workchain for jetton wallet address") } return core.Address(msgAddress.AddrStd.Address), nil } func newEmulator(code, data *boc.Cell) (*tvm.Emulator, error) { emulator, err := tvm.NewEmulator(code, data, config.Config.BlockchainConfig, tvm.WithBalance(1_000_000_000)) if err != nil { return nil, err } // TODO: try tvm.WithLazyC7Optimization() return emulator, nil } // GetJettonBalance // Get method get_wallet_data() returns (int balance, slice owner, slice jetton, cell jetton_wallet_code) // Returns jetton balance for custom block in basic units func (c *Connection) GetJettonBalance(ctx context.Context, address core.Address, blockID *ton.BlockIDExt) (*big.Int, error) { jettonWallet := address.ToTonutilsAddressStd(0) stack, err := c.RunGetMethod(ctx, blockID, jettonWallet, "get_wallet_data") if err != nil { if strings.Contains(err.Error(), "contract is not initialized") { return big.NewInt(0), nil } return nil, fmt.Errorf("get wallet data error: %v", err) } res := stack.AsTuple() if len(res) != 4 { return nil, fmt.Errorf("invalid stack size") } switch res[0].(type) { case *big.Int: return res[0].(*big.Int), nil default: return nil, fmt.Errorf("invalid balance type") } } // GetLastJettonBalance // Returns jetton balance for last block in basic units func (c *Connection) GetLastJettonBalance(ctx context.Context, address *address.Address) (*big.Int, error) { masterID, err := c.client.CurrentMasterchainInfo(ctx) if err != nil { return nil, err } addr, err := core.AddressFromTonutilsAddress(address) if err != nil { return nil, err } return c.GetJettonBalance(ctx, addr, masterID) } // GetAccountCurrentState // Returns TON balance in nanoTONs and account status func (c *Connection) GetAccountCurrentState(ctx context.Context, address *address.Address) (*big.Int, tlb.AccountStatus, error) { masterID, err := c.client.CurrentMasterchainInfo(ctx) if err != nil { return nil, "", err } // TODO: fix waitForBlock and 651 error for { select { case <-ctx.Done(): return nil, "", core.ErrTimeoutExceeded default: account, err := c.client.GetAccount(ctx, masterID, address) if err != nil && isNotReadyError(err) { time.Sleep(time.Millisecond * 200) continue } else if err != nil { return nil, "", err } if !account.IsActive { return big.NewInt(0), tlb.AccountStatusNonExist, nil } return account.State.Balance.Nano(), account.State.Status, nil } } } // DeployTonWallet // Deploys wallet contract and wait its activation func (c *Connection) DeployTonWallet(ctx context.Context, wallet *wallet.Wallet) error { balance, status, err := c.GetAccountCurrentState(ctx, wallet.Address()) if err != nil { return err } if balance.Cmp(big.NewInt(0)) == 0 { return fmt.Errorf("empty balance") } if status != tlb.AccountStatusActive { err = wallet.TransferNoBounce(ctx, wallet.Address(), tlb.FromNanoTONU(0), "") if err != nil { return err } } else { return nil } return c.WaitStatus(ctx, wallet.Address(), tlb.AccountStatusActive) } // GetTransactionIDsFromBlock // Gets all transactions IDs from custom block func (c *Connection) GetTransactionIDsFromBlock(ctx context.Context, blockID *ton.BlockIDExt) ([]ton.TransactionShortInfo, error) { var ( txIDList []ton.TransactionShortInfo after *ton.TransactionID3 next = true ) for next { fetchedIDs, more, err := c.client.GetBlockTransactionsV2(ctx, blockID, 256, after) if err != nil { return nil, err } txIDList = append(txIDList, fetchedIDs...) next = more if more { // set load offset for next query (pagination) after = fetchedIDs[len(fetchedIDs)-1].ID3() } } // sort by LT sort.Slice(txIDList, func(i, j int) bool { return txIDList[i].LT < txIDList[j].LT }) return txIDList, nil } // GetTransactionFromBlock // Gets transaction from block func (c *Connection) GetTransactionFromBlock(ctx context.Context, blockID *ton.BlockIDExt, txID ton.TransactionShortInfo) (*tlb.Transaction, error) { tx, err := c.client.GetTransaction(ctx, blockID, address.NewAddress(0, byte(blockID.Workchain), txID.Account), txID.LT) if err != nil { return nil, err } return tx, nil } func inShard(addr core.Address, shard byte) bool { return addr[0] == shard } func (c *Connection) getCurrentNodeTime(ctx context.Context) (time.Time, error) { t, err := c.client.GetTime(ctx) if err != nil { return time.Time{}, err } res := time.Unix(int64(t), 0) return res, nil } // CheckTime // Checks time diff between node and local time. Due to the fact that the request to the node takes time, // the local time is defined as the average between the beginning and end of the request. // Returns true if time diff < cutoff. func (c *Connection) CheckTime(ctx context.Context, cutoff time.Duration) (bool, error) { prevTime := time.Now() nodeTime, err := c.getCurrentNodeTime(ctx) if err != nil { return false, err } nextTime := time.Now() midTime := prevTime.Add(nextTime.Sub(prevTime) / 2) nodeTimeShift := midTime.Sub(nodeTime) log.Infof("Service-Node time diff: %v", nodeTimeShift) if nodeTimeShift > cutoff || nodeTimeShift < -cutoff { return false, nil } return true, nil } // WaitStatus // Waits custom status for account. Returns error if context timeout is exceeded. // Context must be with timeout to avoid blocking! func (c *Connection) WaitStatus(ctx context.Context, addr *address.Address, status tlb.AccountStatus) error { for { select { case <-ctx.Done(): return core.ErrTimeoutExceeded default: _, st, err := c.GetAccountCurrentState(ctx, addr) if err != nil { return err } if st == status { return nil } time.Sleep(time.Millisecond * 200) } } } // tonutils TonAPI interface methods // GetAccount // The method is being redefined for more stable operation. // Gets account from prev block if impossible to get it from current block. Be careful with diff calculation between blocks. func (c *Connection) GetAccount(ctx context.Context, block *ton.BlockIDExt, addr *address.Address) (*tlb.Account, error) { res, err := c.client.GetAccount(ctx, block, addr) if err != nil && isNotReadyError(err) { prevBlock, err := c.client.LookupBlock(ctx, block.Workchain, block.Shard, block.SeqNo-1) if err != nil { return nil, err } return c.client.GetAccount(ctx, prevBlock, addr) } return res, err } func (c *Connection) SendExternalMessage(ctx context.Context, msg *tlb.ExternalMessage) error { return c.client.SendExternalMessage(ctx, msg) } // RunGetMethod // The method is being redefined for more stable operation // Wait until BlockIsApplied. Use context with timeout. func (c *Connection) RunGetMethod(ctx context.Context, block *ton.BlockIDExt, addr *address.Address, method string, params ...any) (*ton.ExecutionResult, error) { for { select { case <-ctx.Done(): return nil, core.ErrTimeoutExceeded default: res, err := c.client.RunGetMethod(ctx, block, addr, method, params...) if err != nil && isNotReadyError(err) { time.Sleep(time.Millisecond * 200) continue } return res, err } } } func (c *Connection) ListTransactions(ctx context.Context, addr *address.Address, num uint32, lt uint64, txHash []byte) ([]*tlb.Transaction, error) { return c.client.ListTransactions(ctx, addr, num, lt, txHash) } func (c *Connection) Client() ton.LiteClient { return c.client.Client() } func (c *Connection) CurrentMasterchainInfo(ctx context.Context) (*ton.BlockIDExt, error) { return c.client.CurrentMasterchainInfo(ctx) } func (c *Connection) GetMasterchainInfo(ctx context.Context) (*ton.BlockIDExt, error) { return c.client.GetMasterchainInfo(ctx) } func (c *Connection) SendExternalMessageWaitTransaction(ctx context.Context, ext *tlb.ExternalMessage) (*tlb.Transaction, *ton.BlockIDExt, []byte, error) { return c.client.SendExternalMessageWaitTransaction(ctx, ext) } func getBlockchainConfig(ctx context.Context, client ton.LiteClient, block *ton.BlockIDExt) (*cell.Dictionary, error) { var resp tl.Serializable var err error err = client.QueryLiteserver(ctx, ton.GetConfigAll{ Mode: 0b1111111111, BlockID: block, }, &resp) if err != nil { return nil, err } switch t := resp.(type) { case ton.ConfigAll: stateExtra, err := ton.CheckShardMcStateExtraProof(block, []*cell.Cell{t.StateProof, t.ConfigProof}) if err != nil { return nil, fmt.Errorf("incorrect proof: %w", err) } return stateExtra.ConfigParams.Config.Params, nil case ton.LSError: return nil, t } return nil, fmt.Errorf("unexpected response from node") } ================================================ FILE: blockchain/blockchain_test.go ================================================ package blockchain import ( "bytes" "context" "github.com/gobicycle/bicycle/core" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton/jetton" "github.com/xssnick/tonutils-go/ton/wallet" "math/big" "math/rand" "os" "testing" "time" ) var ( jettonMasterAddress, _ = address.ParseAddr("kQCKt2WPGX-fh0cIAz38Ljd_OKQjoZE_cqk7QrYGsNP6wfP0") // TGR in Testnet activeAccount, _ = address.ParseAddr("kQCOSEttz9aEGXkjd1h_NJsQqOca3T-Pld5zSIPHcYZIxsyf") notActiveAccount, _ = address.ParseAddr("kQAkRRJ1RiViVHY2UmUhWCFjdiZBeEYnhkhxI1JTJFNUNG9v") ) func connect(t *testing.T) *Connection { server := os.Getenv("SERVER") if server == "" { t.Fatal("empty server var") } key := os.Getenv("KEY") if key == "" { t.Fatal("empty key var") } c, err := NewConnection(server, key, 100) if err != nil { t.Fatal("connections err: ", err) } return c } func getSeed() string { seed := os.Getenv("SEED") if seed == "" { panic("empty seed") } return seed } func Test_NewConnection(t *testing.T) { connect(t) } func Test_GenerateDefaultWallet(t *testing.T) { c := connect(t) seed := getSeed() hlWallet, shard, id, err := c.GenerateDefaultWallet(seed, false) if err != nil { t.Fatal("gen default wallet err: ", err) } if hlWallet.Address().Data()[0] != shard { t.Fatal("invalid shard") } if id != wallet.DefaultSubwallet { t.Fatal("invalid subwallet ID") } w, shard, id, err := c.GenerateDefaultWallet(seed, true) if err != nil { t.Fatal("gen default wallet err: ", err) } if w.Address().Data()[0] != shard { t.Fatal("invalid shard") } if id != wallet.DefaultSubwallet { t.Fatal("invalid subwallet ID") } } func Test_GenerateSubWallet(t *testing.T) { c := connect(t) seed := getSeed() for i := 0; i < 10; i++ { shard := byte(rand.Intn(255)) startSubWalletID := rand.Uint32() subWallet, subWalletID, err := c.GenerateSubWallet(seed, shard, startSubWalletID) if err != nil { t.Fatal("gen sub wallet err: ", err) } if subWalletID <= startSubWalletID { t.Fatal("invalid subwallet ID") } if subWallet.Address().Data()[0] != shard { t.Fatal("invalid shard") } } } func Test_GetJettonWalletAddress(t *testing.T) { c := connect(t) seed := getSeed() ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() owner, _, _, err := c.GenerateDefaultWallet(seed, true) if err != nil { t.Fatal("gen owner wallet err: ", err) } jettonWalletAddr, err := c.GetJettonWalletAddress(ctx, owner.Address(), jettonMasterAddress) if err != nil { t.Fatal("get jetton wallet address err: ", err) } master := jetton.NewJettonMasterClient(c.client, jettonMasterAddress) jettonWallet, err := master.GetJettonWallet(ctx, owner.Address()) if err != nil { t.Fatal("get jetton wallet address by tonutils method err: ", err) } if !bytes.Equal(jettonWallet.Address().Data(), jettonWalletAddr.Data()) || jettonWallet.Address().Workchain() != jettonWalletAddr.Workchain() { t.Fatal("invalid jetton wallet address") } } func Test_GenerateJettonWalletAddressForProxy(t *testing.T) { c := connect(t) seed := getSeed() ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() owner, _, _, err := c.GenerateDefaultWallet(seed, true) if err != nil { t.Fatal("gen owner wallet err: ", err) } master := jetton.NewJettonMasterClient(c.client, jettonMasterAddress) for i := 0; i < 10; i++ { shard := byte(rand.Intn(255)) startSubWalletID := rand.Uint32() proxy, jettonWalletAddr, err := c.GenerateDepositJettonWalletForProxy(ctx, shard, owner.Address(), jettonMasterAddress, startSubWalletID) if err != nil { t.Fatal("gen sub wallet err: ", err) } if proxy == nil { t.Fatal("nil owner wallet") } if jettonWalletAddr == nil { t.Fatal("nil jetton wallet address") } if proxy.SubwalletID <= startSubWalletID { t.Fatal("invalid subwallet ID") } if jettonWalletAddr.Data()[0] != shard { t.Fatal("invalid shard") } jettonWallet, err := master.GetJettonWallet(ctx, proxy.Address()) if err != nil { t.Fatal("get jetton wallet address by tonutils method err: ", err) } if jettonWallet.Address().String() != jettonWalletAddr.String() { t.Fatal("invalid jetton wallet address") } } } func Test_GetJettonBalance(t *testing.T) { c := connect(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() block, err := c.client.CurrentMasterchainInfo(ctx) if err != nil { t.Fatal("get current masterchain err: ", err) } coreAddr1 := core.AddressMustFromTonutilsAddress(activeAccount) coreAddr2 := core.AddressMustFromTonutilsAddress(notActiveAccount) b1, err := c.GetJettonBalance(ctx, coreAddr1, block) if err != nil { t.Fatal("get balance: ", err) } if b1.Cmp(big.NewInt(0)) != 1 { t.Fatal("empty balance: ", err) } b2, err := c.GetJettonBalance(ctx, coreAddr2, block) if err != nil { t.Fatal("get balance: ", err) } if b2.Cmp(big.NewInt(0)) != 0 { t.Fatal("not empty balance: ", err) } } func Test_GetAccountCurrentState(t *testing.T) { c := connect(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() b1, st1, err := c.GetAccountCurrentState(ctx, activeAccount) if err != nil { t.Fatal("get acc current state err: ", err) } if b1.Cmp(big.NewInt(0)) != 1 || st1 != tlb.AccountStatusActive { t.Fatal("acc not active") } b2, st2, err := c.GetAccountCurrentState(ctx, notActiveAccount) if err != nil { t.Fatal("get acc current state err: ", err) } if b2.Cmp(big.NewInt(0)) != 0 || st2 != tlb.AccountStatusNonExist { t.Fatal("acc active") } } func Test_DeployTonWallet(t *testing.T) { c := connect(t) seed := getSeed() ctx, cancel := context.WithTimeout(context.Background(), time.Second*200) defer cancel() amount := tlb.FromNanoTONU(100_000_000) mainWallet, _, _, err := c.GenerateDefaultWallet(seed, true) if err != nil { t.Fatal("gen main wallet err: ", err) } b, st, err := c.GetAccountCurrentState(ctx, mainWallet.Address()) if err != nil { t.Fatal("get acc current state err: ", err) } if b.Cmp(amount.Nano()) != 1 || st != tlb.AccountStatusActive { t.Fatal("wallet not active") } newWallet, err := mainWallet.GetSubwallet(rand.Uint32()) if err != nil { t.Fatal("gen new wallet err: ", err) } //fmt.Printf("Main wallet: %v\n", mainWallet.Address().String()) //fmt.Printf("New wallet: %v\n", newWallet.Address().String()) _, st, err = c.GetAccountCurrentState(ctx, newWallet.Address()) if err != nil { t.Fatal("get acc current state err: ", err) } if st != tlb.AccountStatusNonExist { t.Log("wallet not empty") t.Skip() } err = mainWallet.TransferNoBounce( ctx, newWallet.Address(), amount, "", true, ) if err != nil { t.Fatal("transfer err: ", err) } err = c.WaitStatus(ctx, newWallet.Address(), tlb.AccountStatusUninit) if err != nil { t.Fatal("wait uninit err: ", err) } err = c.DeployTonWallet(ctx, newWallet) if err != nil { t.Fatal("deploy new wallet err: ", err) } err = newWallet.Send(ctx, &wallet.Message{ Mode: 128 + 32, // 128 + 32 send all and destroy InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: false, DstAddr: mainWallet.Address(), Amount: tlb.FromNanoTONU(0), Body: nil, }, }, false) if err != nil { t.Fatal("send withdrawal err: ", err) } err = c.WaitStatus(ctx, newWallet.Address(), tlb.AccountStatusNonExist) if err != nil { t.Fatal("wait empty err: ", err) } } func Test_GetTransactionIDsFromBlock(t *testing.T) { c := connect(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() masterID, err := c.client.CurrentMasterchainInfo(ctx) if err != nil { t.Fatal("get last block err: ", err) } _, err = c.GetTransactionIDsFromBlock(ctx, masterID) if err != nil { t.Fatal("get tx ids err: ", err) } } func Test_GetTransactionFromBlock(t *testing.T) { c := connect(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) defer cancel() for { masterID, err := c.client.CurrentMasterchainInfo(ctx) if err != nil { t.Fatal("get last block err: ", err) } txIDs, err := c.GetTransactionIDsFromBlock(ctx, masterID) if err != nil { t.Fatal("get tx ids err: ", err) } if len(txIDs) > 0 { tx, err := c.GetTransactionFromBlock(ctx, masterID, txIDs[0]) if err != nil { t.Fatal("get tx err: ", err) } if tx == nil { t.Fatal("nil tx") } break } } } func Test_CheckTime(t *testing.T) { c := connect(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) defer cancel() res, err := c.CheckTime(ctx, time.Second*0) if err != nil { t.Fatal("check time err: ", err) } if res == true { t.Fatal("time diff can not be 0") } res, err = c.CheckTime(ctx, time.Hour*1000) if err != nil { t.Fatal("check time err: ", err) } if res == false { t.Fatal("failed for extra large cutoff") } } func Test_WaitStatus(t *testing.T) { c := connect(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*7) defer cancel() err := c.WaitStatus(ctx, activeAccount, tlb.AccountStatusActive) if err != nil { t.Fatal("wait status err: ", err) } err = c.WaitStatus(ctx, activeAccount, tlb.AccountStatusNonExist) if err == nil { t.Fatal("must be timeout error") } } func Test_GetAccount(t *testing.T) { c := connect(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) defer cancel() for i := 0; i < 20; i++ { b, err := c.client.GetMasterchainInfo(ctx) if err != nil { t.Fatal("get masterchain info err: ", err) } _, err = c.GetAccount(ctx, b, activeAccount) if err != nil { t.Fatal("get account err: ", err) } } } func Test_RunGetMethod(t *testing.T) { c := connect(t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) defer cancel() for i := 0; i < 20; i++ { b, err := c.client.GetMasterchainInfo(ctx) if err != nil { t.Fatal("get masterchain info err: ", err) } _, err = c.RunGetMethod(ctx, b, jettonMasterAddress, "get_jetton_data") if err != nil { t.Fatal("run get method err: ", err) } } } func Test_NextBlock(t *testing.T) { c := connect(t) var shard byte = 123 st := NewShardTracker(shard, nil, c) for i := 0; i < 5; i++ { h, _, err := st.NextBlock() if err != nil { t.Fatal("get next block err: ", err) } if !isInShard(uint64(h.Shard), shard) { t.Fatal("next block not in shard") } } } func Test_Stop(t *testing.T) { c := connect(t) st := NewShardTracker(123, nil, c) for i := 0; i < 2; i++ { _, _, err := st.NextBlock() if err != nil { t.Fatal("get next block err: ", err) } } st.Stop() _, flag, err := st.NextBlock() if err != nil { t.Fatal("get next block err: ", err) } if !flag { t.Fatal("no shutdown flag") } } ================================================ FILE: blockchain/limited_client.go ================================================ package blockchain import ( "context" "fmt" "github.com/xssnick/tonutils-go/tl" "github.com/xssnick/tonutils-go/ton" "golang.org/x/time/rate" ) type limitedLiteClient struct { limiter *rate.Limiter original ton.LiteClient } func newLimitedClient(lc ton.LiteClient, rateLimit int) *limitedLiteClient { return &limitedLiteClient{ original: lc, limiter: rate.NewLimiter(rate.Limit(rateLimit), 1), } } func (w *limitedLiteClient) QueryLiteserver(ctx context.Context, payload tl.Serializable, result tl.Serializable) error { err := w.limiter.Wait(ctx) if err != nil { return fmt.Errorf("limiter err: %w", err) } return w.original.QueryLiteserver(ctx, payload, result) } func (w *limitedLiteClient) StickyContext(ctx context.Context) context.Context { return w.original.StickyContext(ctx) } func (w *limitedLiteClient) StickyNodeID(ctx context.Context) uint32 { return w.original.StickyNodeID(ctx) } func (w *limitedLiteClient) StickyContextNextNode(ctx context.Context) (context.Context, error) { return w.original.StickyContextNextNode(ctx) } func (w *limitedLiteClient) StickyContextNextNodeBalanced(ctx context.Context) (context.Context, error) { return w.original.StickyContextNextNodeBalanced(ctx) } ================================================ FILE: blockchain/shard_tracker.go ================================================ package blockchain import ( "context" "github.com/gobicycle/bicycle/core" log "github.com/sirupsen/logrus" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton" "math/bits" "strings" "time" ) const ErrBlockNotApplied = "block is not applied" const ErrBlockNotInDB = "code 651" type ShardTracker struct { connection *Connection shard byte lastKnownShardBlock *ton.BlockIDExt lastMasterBlock *ton.BlockIDExt buffer []core.ShardBlockHeader gracefulShutdown bool infoCounter, infoStep int infoLastTime time.Time } // NewShardTracker creates new tracker to get blocks with specific shard attribute func NewShardTracker(shard byte, startBlock *ton.BlockIDExt, connection *Connection) *ShardTracker { t := &ShardTracker{ connection: connection, shard: shard, lastKnownShardBlock: startBlock, buffer: make([]core.ShardBlockHeader, 0), infoCounter: 0, infoStep: 1000, infoLastTime: time.Now(), } return t } // NextBlock returns next block header and graceful shutdown flag. // (ShardBlockHeader, false) for normal operation and (empty block header, true) for graceful shutdown. func (s *ShardTracker) NextBlock() (core.ShardBlockHeader, bool, error) { if s.gracefulShutdown { return core.ShardBlockHeader{}, true, nil } h := s.getNext() if h != nil { return *h, false, nil } // the interval between blocks can be up to 40 seconds ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) defer cancel() masterBlockID, err := s.getNextMasterBlockID(ctx) if err != nil { return core.ShardBlockHeader{}, false, err } exit, err := s.loadShardBlocksBatch(masterBlockID) if err != nil { return core.ShardBlockHeader{}, false, err } if exit { log.Printf("Shard tracker sync stopped") return core.ShardBlockHeader{}, true, nil } return s.NextBlock() } // Stop initiates graceful shutdown func (s *ShardTracker) Stop() { s.gracefulShutdown = true } func (s *ShardTracker) getNext() *core.ShardBlockHeader { if len(s.buffer) != 0 { h := s.buffer[0] s.buffer = s.buffer[1:] return &h } return nil } func (s *ShardTracker) getNextMasterBlockID(ctx context.Context) (*ton.BlockIDExt, error) { for { masterBlockID, err := s.connection.client.GetMasterchainInfo(ctx) if err != nil { // exit by context timeout return nil, err } if s.lastMasterBlock == nil { s.lastMasterBlock = masterBlockID return masterBlockID, nil } if masterBlockID.SeqNo == s.lastMasterBlock.SeqNo { time.Sleep(time.Second) continue } s.lastMasterBlock = masterBlockID return masterBlockID, nil } } func (s *ShardTracker) loadShardBlocksBatch(masterBlockID *ton.BlockIDExt) (bool, error) { var ( shards []*ton.BlockIDExt err error ) ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) defer cancel() for { shards, err = s.connection.client.GetBlockShardsInfo(ctx, masterBlockID) if err != nil && isNotReadyError(err) { // TODO: clarify error type time.Sleep(time.Second) continue } else if err != nil { return false, err // exit by context timeout } break } s.infoCounter = 0 batch, exit, err := s.getShardBlocksRecursively(filterByShard(shards, s.shard), nil) if err != nil { return false, err } if exit { return true, nil } if len(batch) != 0 { s.lastKnownShardBlock = batch[0].BlockIDExt for i := len(batch) - 1; i >= 0; i-- { s.buffer = append(s.buffer, batch[i]) } } return false, nil } func (s *ShardTracker) getShardBlocksRecursively(i *ton.BlockIDExt, batch []core.ShardBlockHeader) ([]core.ShardBlockHeader, bool, error) { if s.gracefulShutdown { return nil, true, nil } if s.lastKnownShardBlock == nil { s.lastKnownShardBlock = i } isKnown := (s.lastKnownShardBlock.Shard == i.Shard) && (s.lastKnownShardBlock.SeqNo == i.SeqNo) if isKnown { return batch, false, nil } // compare seqno with filtered shard block // handle the case when a node may reference an old block if s.lastKnownShardBlock.SeqNo > i.SeqNo { return []core.ShardBlockHeader{}, false, nil } seqnoDiff := int(i.SeqNo - s.lastKnownShardBlock.SeqNo) if seqnoDiff > s.infoStep { if s.infoCounter%s.infoStep == 0 { estimatedTime := time.Duration(seqnoDiff/s.infoStep) * time.Since(s.infoLastTime) s.infoLastTime = time.Now() if s.infoCounter == 0 { log.Printf("Shard tracker syncing... Seqno diff: %v Estimated time: unknown\n", seqnoDiff) } else { log.Printf("Shard tracker syncing... Seqno diff: %v Estimated time: %v\n", seqnoDiff, estimatedTime) } } s.infoCounter++ } ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) defer cancel() h, err := s.connection.getShardBlocksHeader(ctx, i, s.shard) if err != nil { return nil, false, err } batch = append(batch, h) return s.getShardBlocksRecursively(h.Parent, batch) } func isInShard(blockShardPrefix uint64, shard byte) bool { if blockShardPrefix == 0 { log.Fatalf("invalid shard_prefix") } prefixLen := 64 - 1 - bits.TrailingZeros64(blockShardPrefix) // without one insignificant bit if prefixLen > 8 { log.Fatalf("more than 256 shards is not supported") } res := (uint64(shard) << (64 - 8)) ^ blockShardPrefix return bits.LeadingZeros64(res) >= prefixLen } func filterByShard(headers []*ton.BlockIDExt, shard byte) *ton.BlockIDExt { for _, h := range headers { if isInShard(uint64(h.Shard), shard) { return h } } log.Fatalf("must be at least one suitable shard block") return nil } func convertBlockToShardHeader(block *tlb.Block, info *ton.BlockIDExt, shard byte) (core.ShardBlockHeader, error) { parents, err := block.BlockInfo.GetParentBlocks() if err != nil { return core.ShardBlockHeader{}, err } parent := filterByShard(parents, shard) return core.ShardBlockHeader{ NotMaster: block.BlockInfo.NotMaster, GenUtime: block.BlockInfo.GenUtime, StartLt: block.BlockInfo.StartLt, EndLt: block.BlockInfo.EndLt, Parent: parent, BlockIDExt: info, }, nil } // get shard block header for specific shard attribute with one parent func (c *Connection) getShardBlocksHeader(ctx context.Context, shardBlockID *ton.BlockIDExt, shard byte) (core.ShardBlockHeader, error) { var ( err error block *tlb.Block ) for { block, err = c.client.GetBlockData(ctx, shardBlockID) if err != nil && isNotReadyError(err) { time.Sleep(time.Millisecond * 500) continue } else if err != nil { return core.ShardBlockHeader{}, err // exit by context timeout } break } return convertBlockToShardHeader(block, shardBlockID, shard) } func isNotReadyError(err error) bool { return strings.Contains(err.Error(), ErrBlockNotApplied) || strings.Contains(err.Error(), ErrBlockNotInDB) } ================================================ FILE: cmd/processor/main.go ================================================ package main import ( "context" "errors" "fmt" "github.com/gobicycle/bicycle/api" "github.com/gobicycle/bicycle/blockchain" "github.com/gobicycle/bicycle/config" "github.com/gobicycle/bicycle/core" "github.com/gobicycle/bicycle/db" "github.com/gobicycle/bicycle/queue" "github.com/gobicycle/bicycle/webhook" log "github.com/sirupsen/logrus" "net/http" "os" "os/signal" "sync" "syscall" "time" ) var Version = "dev" func main() { log.Infof("App version: %s", Version) config.GetConfig() sigChannel := make(chan os.Signal, 1) signal.Notify(sigChannel, os.Interrupt, syscall.SIGTERM) wg := new(sync.WaitGroup) bcClient, err := blockchain.NewConnection(config.Config.LiteServer, config.Config.LiteServerKey, config.Config.LiteServerRateLimit) if err != nil { log.Fatalf("blockchain connection error: %v", err) } dbClient, err := db.NewConnection(config.Config.DatabaseURI) if err != nil { log.Fatalf("DB connection error: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) defer cancel() err = dbClient.LoadAddressBook(ctx) if err != nil { log.Fatalf("address book loading error: %v", err) } isTimeSynced, err := bcClient.CheckTime(ctx, config.AllowableServiceToNodeTimeDiff) if err != nil { log.Fatalf("get node time err: %v", err) } if !isTimeSynced { log.Fatalf("Service and Node time not synced") } wallets, err := core.InitWallets(ctx, dbClient, bcClient, config.Config.Seed, config.Config.Jettons) if err != nil { log.Fatalf("Wallets initialization error: %v", err) } var notificators []core.Notificator if config.Config.QueueEnabled { queueClient, err := queue.NewAmqpClient(config.Config.QueueURI, config.Config.QueueEnabled, config.Config.QueueName) if err != nil { log.Fatalf("new queue client creating error: %v", err) } notificators = append(notificators, queueClient) } if config.Config.WebhookEndpoint != "" { webhookClient, err := webhook.NewWebhookClient(config.Config.WebhookEndpoint, config.Config.WebhookToken) if err != nil { log.Fatalf("new webhook client creating error: %v", err) } notificators = append(notificators, webhookClient) } var tracker *blockchain.ShardTracker block, err := dbClient.GetLastSavedBlockID(ctx) if !errors.Is(err, core.ErrNotFound) && err != nil { log.Fatalf("Get last saved block error: %v", err) } else if errors.Is(err, core.ErrNotFound) { tracker = blockchain.NewShardTracker(wallets.Shard, nil, bcClient) } else { tracker = blockchain.NewShardTracker(wallets.Shard, block, bcClient) } blockScanner := core.NewBlockScanner(wg, dbClient, bcClient, wallets.Shard, tracker, notificators) withdrawalsProcessor := core.NewWithdrawalsProcessor( wg, dbClient, bcClient, wallets, config.Config.ColdWallet) withdrawalsProcessor.Start() apiMux := http.NewServeMux() h := api.NewHandler(dbClient, bcClient, config.Config.APIToken, wallets.Shard, *wallets.TonHotWallet.Address()) api.RegisterHandlers(apiMux, h) go func() { err := http.ListenAndServe(fmt.Sprintf(":%d", config.Config.APIPort), apiMux) if err != nil { log.Fatalf("api error: %v", err) } }() go func() { <-sigChannel log.Printf("SIGTERM received") blockScanner.Stop() withdrawalsProcessor.Stop() }() wg.Wait() } ================================================ FILE: cmd/testutil/http.go ================================================ package main import ( "bytes" "encoding/json" "fmt" "github.com/gobicycle/bicycle/api" "github.com/gobicycle/bicycle/config" "github.com/gobicycle/bicycle/core" "github.com/gofrs/uuid" "github.com/shopspring/decimal" "io" "log" "net/http" "time" ) type Client struct { client *http.Client urlA string urlB string token string userID string } func NewClient(urlA, urlB, token, userID string) *Client { c := &Client{ client: &http.Client{Timeout: 10 * time.Second}, urlA: urlA, urlB: urlB, token: token, userID: userID, } return c } func (s *Client) InitDeposits(host string) (map[string][]string, error) { deposits := make(map[string][]string) addr, err := s.GetAllAddresses(host) if err != nil { return nil, err } if len(addr.Addresses) != 0 { for _, wa := range addr.Addresses { deposits[wa.Currency] = append(deposits[wa.Currency], wa.Address) } return deposits, nil } for i := 0; i < depositsQty; i++ { addr, err := s.GetNewAddress(host, core.TonSymbol) if err != nil { return nil, err } deposits[core.TonSymbol] = append(deposits[core.TonSymbol], addr) } for i := 0; i < depositsQty; i++ { for cur := range config.Config.Jettons { addr, err := s.GetNewAddress(host, cur) if err != nil { return nil, err } deposits[cur] = append(deposits[cur], addr) } } log.Printf("Deposits initialized for %s", host) return deposits, nil } func (s *Client) GetAllAddresses(host string) (api.GetAddressesResponse, error) { url := fmt.Sprintf("http://%s/v1/address/all?user_id=%s", host, s.userID) request, err := http.NewRequest("GET", url, nil) if err != nil { return api.GetAddressesResponse{}, err } request.Header.Add("Authorization", "Bearer "+s.token) response, err := s.client.Do(request) if err != nil { return api.GetAddressesResponse{}, err } defer func() { err := response.Body.Close() if err != nil { log.Fatalf("response body close error: %v", err) } }() if response.StatusCode >= 300 { return api.GetAddressesResponse{}, fmt.Errorf("response status: %v", response.Status) } content, err := io.ReadAll(response.Body) if err != nil { return api.GetAddressesResponse{}, err } var res api.GetAddressesResponse err = json.Unmarshal(content, &res) if err != nil { return api.GetAddressesResponse{}, err } return res, nil } func (s *Client) SendWithdrawal(host, currency, destination string, amount int64) (api.WithdrawalResponse, uuid.UUID, error) { url := fmt.Sprintf("http://%s/v1/withdrawal/send", host) u, err := uuid.NewV4() if err != nil { return api.WithdrawalResponse{}, uuid.UUID{}, err } reqData := api.WithdrawalRequest{ UserID: s.userID, QueryID: u.String(), Currency: currency, Amount: decimal.New(amount, 0), Destination: destination, Comment: u.String(), } jsonData, err := json.Marshal(reqData) if err != nil { return api.WithdrawalResponse{}, uuid.UUID{}, err } request, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) if err != nil { return api.WithdrawalResponse{}, uuid.UUID{}, err } request.Header.Set("Content-Type", "application/json; charset=UTF-8") request.Header.Add("Authorization", "Bearer "+s.token) response, err := s.client.Do(request) if err != nil { return api.WithdrawalResponse{}, uuid.UUID{}, err } defer func() { err := response.Body.Close() if err != nil { log.Fatalf("response body close error: %v", err) } }() if response.StatusCode >= 300 { return api.WithdrawalResponse{}, uuid.UUID{}, fmt.Errorf("response status: %v", response.Status) } content, err := io.ReadAll(response.Body) if err != nil { return api.WithdrawalResponse{}, uuid.UUID{}, err } var res api.WithdrawalResponse err = json.Unmarshal(content, &res) if err != nil { return api.WithdrawalResponse{}, uuid.UUID{}, err } return res, u, nil } func (s *Client) GetNewAddress(host, currency string) (string, error) { url := fmt.Sprintf("http://%s/v1/address/new", host) reqData := struct { UserID string `json:"user_id"` Currency string `json:"currency"` }{ UserID: s.userID, Currency: currency, } jsonData, err := json.Marshal(reqData) if err != nil { return "", err } request, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) if err != nil { return "", err } request.Header.Set("Content-Type", "application/json; charset=UTF-8") request.Header.Add("Authorization", "Bearer "+s.token) response, err := s.client.Do(request) if err != nil { return "", err } defer func() { err := response.Body.Close() if err != nil { log.Fatalf("response body close error: %v", err) } }() if response.StatusCode >= 300 { return "", fmt.Errorf("response status: %v", response.Status) } content, err := io.ReadAll(response.Body) if err != nil { return "", err } var res struct { Address string `json:"address"` } err = json.Unmarshal(content, &res) if err != nil { return "", err } return res.Address, nil } func (s *Client) GetWithdrawalStatus(host string, id int64) (api.WithdrawalStatusResponse, error) { url := fmt.Sprintf("http://%s/v1/withdrawal/status?id=%v", host, id) request, err := http.NewRequest("GET", url, nil) if err != nil { return api.WithdrawalStatusResponse{}, err } request.Header.Add("Authorization", "Bearer "+s.token) response, err := s.client.Do(request) if err != nil { return api.WithdrawalStatusResponse{}, err } defer func() { err := response.Body.Close() if err != nil { log.Fatalf("response body close error: %v", err) } }() if response.StatusCode >= 300 { return api.WithdrawalStatusResponse{}, fmt.Errorf("response status: %v", response.Status) } content, err := io.ReadAll(response.Body) if err != nil { return api.WithdrawalStatusResponse{}, err } var res api.WithdrawalStatusResponse err = json.Unmarshal(content, &res) if err != nil { return api.WithdrawalStatusResponse{}, err } return res, nil } ================================================ FILE: cmd/testutil/main.go ================================================ package main import ( "context" "github.com/gobicycle/bicycle/blockchain" "github.com/gobicycle/bicycle/config" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/xssnick/tonutils-go/address" "log" "net/http" "os" "time" ) var ( onlyMonitoring = true Version = "dev" ) const ( depositsQty = 10 tonWithdrawAmount = 550_000_000 jettonWithdrawAmount = 550_000_000 tonMinCutoff = 10_000_000_000 ) func main() { log.Printf("App version: %s", Version) config.GetConfig() if circulation := os.Getenv("CIRCULATION"); circulation == "true" { onlyMonitoring = false } urlA := os.Getenv("HOST_A") if urlA == "" { log.Fatalf("empty HOST_A env var") } urlB := os.Getenv("HOST_B") if urlB == "" { log.Fatalf("empty HOST_B env var") } hotWalletA, err := address.ParseAddr(os.Getenv("HOT_WALLET_A")) if err != nil { log.Fatalf("invalid HOT_WALLET_A env var") } hotWalletB, err := address.ParseAddr(os.Getenv("HOT_WALLET_B")) if err != nil { log.Fatalf("invalid HOT_WALLET_B env var") } bcClient, err := blockchain.NewConnection(config.Config.LiteServer, config.Config.LiteServerKey, config.Config.LiteServerRateLimit) if err != nil { log.Fatalf("blockchain connection error: %v", err) } httpClient := NewClient(urlA, urlB, config.Config.APIToken, "TestClient") http.Handle("/metrics", promhttp.Handler()) go func() { log.Fatal(http.ListenAndServe(":9101", nil)) }() depositsA, err := httpClient.InitDeposits(urlA) if err != nil { log.Fatalf("can not init deposits: %v", err) } depositsB, err := httpClient.InitDeposits(urlB) if err != nil { log.Fatalf("can not init deposits: %v", err) } payerProc := NewPayerProcessor(context.TODO(), httpClient, bcClient, depositsA, depositsB, hotWalletA, hotWalletB) payerProc.Start() for { time.Sleep(time.Hour) } } ================================================ FILE: cmd/testutil/metrics.go ================================================ package main import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) var ( hotWalletABalance = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "hot_wallet_a_balance", Help: "Hot wallet A balance", }, []string{"currency"}, ) hotWalletBBalance = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "hot_wallet_b_balance", Help: "Hot wallet B balance", }, []string{"currency"}, ) depositWalletABalance = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "deposit_wallet_a_balance", Help: "Deposit wallet A balance", }, []string{"currency", "address"}, ) depositWalletBBalance = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "deposit_wallet_b_balance", Help: "Deposit wallet B balance", }, []string{"currency", "address"}, ) totalBalance = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "total_balance", Help: "Total balance", }, []string{"currency"}, ) totalLosses = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "total_losses", Help: "Total losses", }, []string{"currency"}, ) predictedTonLoss = promauto.NewGauge( prometheus.GaugeOpts{ Name: "predicted_ton_loss", Help: "Predicted TON loss", }, ) totalProcessedAmount = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "total_processed_amount", Help: "Total processed amount", }, []string{"currency"}, ) ) ================================================ FILE: cmd/testutil/utils.go ================================================ package main import ( "bytes" "context" "fmt" "github.com/gobicycle/bicycle/blockchain" "github.com/gobicycle/bicycle/config" "github.com/gobicycle/bicycle/core" "github.com/gofrs/uuid" log "github.com/sirupsen/logrus" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "math" "sync" "time" ) type PayerProcessor struct { client *Client bcClient *blockchain.Connection depositsA []Deposit depositsB []Deposit hotWalletsAddrA map[string]*address.Address hotWalletsAddrB map[string]*address.Address lastTxIDA, lastTxIDB TxID balances *hotBalances knownUUIDsA, knownUUIDsB map[uuid.UUID]struct{} } const ( TonLossForJettonExternalWithdrawal int64 = 58_376_225 // without Jetton wallet deploy except of excess // SCALE:69_579_403 TGR:47_173_046 TonLossForJettonInternalWithdrawal int64 = 55_306_226 // proxy deploy without Jetton wallet deploy except of JettonForwardAmount // SCALE:57_873_578 TGR:52_738_873 TonLossForTonExternalWithdrawal int64 = 10_232_080 // only fees TonLossForTonInternalWithdrawal int64 = 8_034_998 // deposit wallet deploy ) type PaymentSide string const ( SideA PaymentSide = "A" SideB PaymentSide = "B" ) type TxID struct { Lt uint64 Hash []byte } type Deposit struct { Address *address.Address JettonWallet *address.Address Currency string } type hotBalances struct { mutex sync.Mutex hotWalletsBalancesA map[string]int64 hotWalletsBalancesB map[string]int64 } func newHotBalances() *hotBalances { return &hotBalances{ hotWalletsBalancesA: make(map[string]int64), hotWalletsBalancesB: make(map[string]int64), } } func (h *hotBalances) ReadBalance(walletSide PaymentSide, currency string) int64 { h.mutex.Lock() defer h.mutex.Unlock() switch walletSide { case SideA: b, ok := h.hotWalletsBalancesA[currency] if ok { return b } return 0 case SideB: b, ok := h.hotWalletsBalancesB[currency] if ok { return b } return 0 } log.Fatalf("invalid payment side") return 0 } func (h *hotBalances) WriteBalance(walletSide PaymentSide, currency string, balance int64) { h.mutex.Lock() defer h.mutex.Unlock() switch walletSide { case SideA: h.hotWalletsBalancesA[currency] = balance return case SideB: h.hotWalletsBalancesB[currency] = balance return } log.Fatalf("invalid payment side") } func NewPayerProcessor( ctx context.Context, client *Client, bcClient *blockchain.Connection, depositsA map[string][]string, depositsB map[string][]string, hotA, hotB *address.Address, ) *PayerProcessor { // hot jetton wallets addrA := make(map[string]*address.Address) addrB := make(map[string]*address.Address) addrA[core.TonSymbol] = hotA addrB[core.TonSymbol] = hotB lastTxIDA, err := getLastTxID(ctx, bcClient, hotA) if err != nil { log.Fatalf("get last TX ID A error: %v", err) } lastTxIDB, err := getLastTxID(ctx, bcClient, hotB) if err != nil { log.Fatalf("get last TX ID B error: %v", err) } for cur, jetton := range config.Config.Jettons { jwA, err := bcClient.GetJettonWalletAddress(ctx, hotA, jetton.Master) if err != nil { log.Fatalf("get hot jetton wallet A error: %v", err) } jwB, err := bcClient.GetJettonWalletAddress(ctx, hotB, jetton.Master) if err != nil { log.Fatalf("get hot jetton wallet B error: %v", err) } addrA[cur] = jwA addrB[cur] = jwB totalProcessedAmount.WithLabelValues(cur).Set(0) } totalProcessedAmount.WithLabelValues(core.TonSymbol).Set(0) p := &PayerProcessor{ client: client, bcClient: bcClient, depositsA: convertDeposits(bcClient, depositsA), depositsB: convertDeposits(bcClient, depositsB), hotWalletsAddrA: addrA, hotWalletsAddrB: addrB, balances: newHotBalances(), lastTxIDA: lastTxIDA, lastTxIDB: lastTxIDB, knownUUIDsA: make(map[uuid.UUID]struct{}), knownUUIDsB: make(map[uuid.UUID]struct{}), } return p } func convertDeposits(bcClient *blockchain.Connection, deposits map[string][]string) []Deposit { var dep []Deposit for cur, addresses := range deposits { if cur != core.TonSymbol { jetton := config.Config.Jettons[cur] for _, a := range addresses { addr, _ := address.ParseAddr(a) jw, err := bcClient.GetJettonWalletAddress(context.Background(), addr, jetton.Master) if err != nil { log.Fatalf("get jetton wallet error: %v", err) } dep = append(dep, Deposit{Address: addr, Currency: cur, JettonWallet: jw}) } } else { for _, a := range addresses { addr, err := address.ParseAddr(a) if err != nil { log.Fatalf("parse deposit address error: %v", err) } dep = append(dep, Deposit{Address: addr, Currency: cur}) } } } return dep } func (p *PayerProcessor) Start() { go p.balanceMonitor() if !onlyMonitoring { go p.startPayments(SideA) go p.startPayments(SideB) } } func (p *PayerProcessor) balanceMonitor() { log.Infof("Balance monitor started") startTotal := make(map[string]int64) for { ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) tot, err := p.updateHotWalletBalances(ctx) if err != nil { log.Errorf("can not update hot wallet balances: %v\n", err) cancel() continue } for _, d := range p.depositsA { b, err := p.getDepositBalance(ctx, d) if err != nil { log.Errorf("can not update deposit wallet A balance: %v\n", err) cancel() continue } tot[d.Currency] = tot[d.Currency] + b depositWalletABalance.WithLabelValues(d.Currency, d.Address.String()).Set(float64(b)) } for _, d := range p.depositsB { b, err := p.getDepositBalance(ctx, d) if err != nil { log.Errorf("can not update deposit wallet B balance: %v\n", err) cancel() continue } tot[d.Currency] = tot[d.Currency] + b depositWalletBBalance.WithLabelValues(d.Currency, d.Address.String()).Set(float64(b)) } for c, t := range tot { totalBalance.WithLabelValues(c).Set(float64(t)) if _, ok := startTotal[c]; !ok { startTotal[c] = t } totalLosses.WithLabelValues(c).Set(float64(t - startTotal[c])) } cancel() time.Sleep(time.Millisecond * 500) } } func (p *PayerProcessor) updateHotWalletBalances(ctx context.Context) (map[string]int64, error) { totBalance := make(map[string]int64) bA, _, err := p.bcClient.GetAccountCurrentState(ctx, p.hotWalletsAddrA[core.TonSymbol]) if err != nil { return nil, err } bB, _, err := p.bcClient.GetAccountCurrentState(ctx, p.hotWalletsAddrB[core.TonSymbol]) if err != nil { return nil, err } hotWalletABalance.WithLabelValues(core.TonSymbol).Set(float64(bA.Int64())) hotWalletBBalance.WithLabelValues(core.TonSymbol).Set(float64(bB.Int64())) totBalance[core.TonSymbol] = bA.Int64() + bB.Int64() p.balances.WriteBalance(SideA, core.TonSymbol, bA.Int64()) p.balances.WriteBalance(SideB, core.TonSymbol, bB.Int64()) for cur := range config.Config.Jettons { bA, err = p.bcClient.GetLastJettonBalance(ctx, p.hotWalletsAddrA[cur]) if err != nil { return nil, err } bB, err = p.bcClient.GetLastJettonBalance(ctx, p.hotWalletsAddrB[cur]) if err != nil { return nil, err } hotWalletABalance.WithLabelValues(cur).Set(float64(bA.Int64())) hotWalletBBalance.WithLabelValues(cur).Set(float64(bB.Int64())) totBalance[cur] = bA.Int64() + bB.Int64() p.balances.WriteBalance(SideA, cur, bA.Int64()) p.balances.WriteBalance(SideB, cur, bB.Int64()) } return totBalance, nil } func (p *PayerProcessor) getDepositBalance(ctx context.Context, d Deposit) (int64, error) { if d.Currency == core.TonSymbol { b, _, err := p.bcClient.GetAccountCurrentState(ctx, d.Address) if err != nil { return 0, err } return b.Int64(), nil } b, err := p.bcClient.GetLastJettonBalance(ctx, d.JettonWallet) if err != nil { return 0, err } return b.Int64(), nil } func (p *PayerProcessor) startPayments(side PaymentSide) { for { time.Sleep(time.Second * 60) if !p.checkBalances(side) { continue } ids, ww, err := p.withdrawToDeposits(side) if err != nil { log.Fatalf("withdraw to deposit error: %s", err) } log.Infof("Withdrawals sended for side %s", side) err = p.waitWithdrawals(side, ids) if err != nil { log.Fatalf("wait withdrawals error: %s", err) } err = p.validateWithdrawals(context.TODO(), side, ww) if err != nil { log.Fatalf("validate withdrawals error: %s", err) } log.Infof("Withdrawals validated for side %s", side) } } func (p *PayerProcessor) checkBalances(side PaymentSide) bool { tonRemained := p.balances.ReadBalance(side, core.TonSymbol) - tonWithdrawAmount*depositsQty if tonRemained < tonMinCutoff { return false } for cur := range config.Config.Jettons { jettonRemained := p.balances.ReadBalance(side, cur) - jettonWithdrawAmount*depositsQty tonRemained = tonRemained - depositsQty*config.JettonTransferTonAmount.Nano().Int64() if jettonRemained < 0 || tonRemained < tonMinCutoff { return false } } return true } type withdrawal struct { To *address.Address Amount int64 UUID uuid.UUID } func (p *PayerProcessor) withdrawToDeposits(fromSide PaymentSide) ([]int64, []withdrawal, error) { var ( url string deposits []Deposit ) switch fromSide { case SideA: deposits = p.depositsB url = p.client.urlA case SideB: deposits = p.depositsA url = p.client.urlB default: return nil, nil, fmt.Errorf("invalid side") } var ( ww []withdrawal ids []int64 ) for _, d := range deposits { if d.Currency == core.TonSymbol { r, u, err := p.client.SendWithdrawal(url, d.Currency, d.Address.String(), tonWithdrawAmount) if err != nil { return nil, nil, err } ww = append(ww, withdrawal{ To: d.Address, Amount: tonWithdrawAmount, UUID: u, }) ids = append(ids, r.ID) predictedTonLoss.Add(predictLoss(core.TonSymbol)) totalProcessedAmount.WithLabelValues(core.TonSymbol).Add(float64(tonWithdrawAmount)) } else { r, u, err := p.client.SendWithdrawal(url, d.Currency, d.Address.String(), jettonWithdrawAmount) if err != nil { return nil, nil, err } ww = append(ww, withdrawal{ To: d.Address, Amount: jettonWithdrawAmount, UUID: u, }) ids = append(ids, r.ID) predictedTonLoss.Add(predictLoss(d.Currency)) totalProcessedAmount.WithLabelValues(d.Currency).Add(float64(jettonWithdrawAmount)) } } return ids, ww, nil } func (p *PayerProcessor) waitWithdrawals(fromSide PaymentSide, ids []int64) error { var url string switch fromSide { case SideA: url = p.client.urlA case SideB: url = p.client.urlB default: return fmt.Errorf("invalid side") } completed := make(map[int64]struct{}) for { for _, id := range ids { _, ok := completed[id] if ok { continue } r, err := p.client.GetWithdrawalStatus(url, id) if err != nil { return err } if r.Status == core.ProcessedStatus { completed[id] = struct{}{} } else { time.Sleep(time.Millisecond * 200) break } } if len(completed) == len(ids) { break } } return nil } func (p *PayerProcessor) validateWithdrawals(ctx context.Context, fromSide PaymentSide, withdrawals []withdrawal) error { var ( addr *address.Address lastTxID TxID newUUIDs []uuid.UUID ) switch fromSide { case SideA: addr = p.hotWalletsAddrA[core.TonSymbol] lastTxID = p.lastTxIDA case SideB: addr = p.hotWalletsAddrB[core.TonSymbol] lastTxID = p.lastTxIDB default: return fmt.Errorf("invalid side") } txs, err := p.loadTXs(ctx, lastTxID, addr) if err != nil { return err } if len(txs) > 0 { lastTxID = TxID{ Lt: txs[0].LT, Hash: txs[0].Hash, } } remainingTXs := withdrawals for _, tx := range txs { ww, uuids, err := parseTX(tx) if err != nil { return err } newUUIDs = append(newUUIDs, uuids...) remainingTXs = compareWithdrawals(ww, remainingTXs) } if len(remainingTXs) > 0 { var s string for _, r := range remainingTXs { s = s + r.UUID.String() + ", " } return fmt.Errorf("can not find withdrawals: %s", s) } switch fromSide { case SideA: p.lastTxIDA = lastTxID err := checkDoubleSpending(newUUIDs, p.knownUUIDsA) if err != nil { return err } case SideB: p.lastTxIDB = lastTxID err := checkDoubleSpending(newUUIDs, p.knownUUIDsB) if err != nil { return err } } return nil } func checkDoubleSpending(newUUIDs []uuid.UUID, knownUUIDs map[uuid.UUID]struct{}) error { for _, u := range newUUIDs { _, ok := knownUUIDs[u] if ok { return fmt.Errorf("double spending: %s", u.String()) } knownUUIDs[u] = struct{}{} } return nil } func (p *PayerProcessor) loadTXs(ctx context.Context, lastTxID TxID, addr *address.Address) ([]*tlb.Transaction, error) { newTxID, err := getLastTxID(ctx, p.bcClient, addr) if err != nil { return nil, err } currentTxID := newTxID txs := make([]*tlb.Transaction, 0) for { // last transaction has 0 prev lt if currentTxID.Lt == 0 { break } list, err := p.bcClient.ListTransactions(ctx, addr, 3, currentTxID.Lt, currentTxID.Hash) if err != nil { return nil, err } // oldest = first in list for i := len(list) - 1; i >= 0; i-- { if bytes.Equal(list[i].Hash, lastTxID.Hash) { return txs, nil } txs = append(txs, list[i]) } // set previous info from the oldest transaction in list currentTxID.Hash = list[0].PrevTxHash currentTxID.Lt = list[0].PrevTxLT } return nil, fmt.Errorf("can not get txs") } func parseTX(tx *tlb.Transaction) ([]withdrawal, []uuid.UUID, error) { var ( ww []withdrawal uuids []uuid.UUID msgList []tlb.Message err error ) if tx.OutMsgCount > 0 { msgList, err = tx.IO.Out.ToSlice() if err != nil { return nil, nil, err } } for _, m := range msgList { if m.MsgType != tlb.MsgTypeInternal { continue } msg := m.AsInternal() jt, err := core.DecodeJettonTransfer(msg) if err == nil { u, err := uuid.FromString(jt.Comment) if err != nil { return nil, nil, err } ww = append(ww, withdrawal{ To: jt.Destination, Amount: jt.Amount.BigInt().Int64(), UUID: u, }) uuids = append(uuids, u) } else if msg.Body.BitsSize() > 32 { u, err := uuid.FromString(core.LoadComment(msg.Body)) if err != nil { return nil, nil, err } ww = append(ww, withdrawal{ To: msg.DstAddr, Amount: msg.Amount.Nano().Int64(), UUID: u, }) uuids = append(uuids, u) } } return ww, uuids, nil } func compareWithdrawals(all, target []withdrawal) []withdrawal { var res []withdrawal for _, t := range target { found := false for _, a := range all { if t.UUID == a.UUID { found = true break } } if !found { res = append(res, t) } } return res } func getLastTxID(ctx context.Context, bcClient *blockchain.Connection, address *address.Address) (TxID, error) { b, err := bcClient.GetMasterchainInfo(ctx) if err != nil { return TxID{}, err } res, err := bcClient.GetAccount(ctx, b, address) if err != nil { return TxID{}, err } return TxID{ Lt: res.LastTxLT, Hash: res.LastTxHash, }, nil } func predictLoss(currency string) float64 { if currency == core.TonSymbol { cutoff := config.Config.Ton.Withdrawal.Int64() n := math.Ceil(float64(cutoff) / float64(tonWithdrawAmount)) // number of replenishments of the deposit before withdrawal return float64(TonLossForTonExternalWithdrawal) + float64(TonLossForTonInternalWithdrawal)/n } else { cutoff := config.Config.Jettons[currency].WithdrawalCutoff.Int64() n := math.Ceil(float64(cutoff) / float64(jettonWithdrawAmount)) // number of replenishments of the deposit before withdrawal return float64(TonLossForJettonExternalWithdrawal) + float64(TonLossForJettonInternalWithdrawal)/n } } ================================================ FILE: cmd/testwebhook/main.go ================================================ package main import ( "crypto/subtle" "fmt" "io" "net/http" "strings" ) func main() { http.HandleFunc("/webhook", getNotification) fmt.Printf("webhook listener started\n") err := http.ListenAndServe(":3333", nil) if err != nil { panic(err) } } func getNotification(resp http.ResponseWriter, req *http.Request) { if req.Method != http.MethodPost { fmt.Printf("Not a post request!\n") } checkToken(req, "123") res, err := io.ReadAll(req.Body) if err != nil { fmt.Printf("notification read error: %v", err) return } _ = req.Body.Close() fmt.Printf("Notification: %s\n", res) resp.WriteHeader(http.StatusOK) } func checkToken(req *http.Request, token string) { header := req.Header.Get("authorization") if header == "" { fmt.Printf("no authorization header\n") return } auth := strings.Split(header, " ") if len(auth) != 2 || auth[0] != "Bearer" { fmt.Printf("not Bearer token\n") return } if x := subtle.ConstantTimeCompare([]byte(auth[1]), []byte(token)); x == 1 { return } // constant time comparison to prevent time attack fmt.Printf("invalid token\n") return } ================================================ FILE: config/config.go ================================================ package config import ( "log" "math/big" "strings" "time" "github.com/caarlos0/env/v6" "github.com/shopspring/decimal" "github.com/tonkeeper/tongo/boc" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" ) const MaxJettonForwardTonAmount = 20_000_000 var ( JettonTransferTonAmount = tlb.FromNanoTONU(100_000_000) JettonForwardAmount = tlb.FromNanoTONU(MaxJettonForwardTonAmount) // must be < JettonTransferTonAmount JettonInternalForwardAmount = tlb.FromNanoTONU(1) DefaultHotWalletHysteresis = decimal.NewFromFloat(0.95) // `hot_wallet_residual_balance` = `hot_wallet_max_balance` * `hysteresis` ExternalMessageLifetime = 50 * time.Second ExternalWithdrawalPeriod = 80 * time.Second // must be ExternalWithdrawalPeriod > ExternalMessageLifetime and some time for balance update InternalWithdrawalPeriod = 80 * time.Second ExpirationProcessorPeriod = 5 * time.Second AllowableBlockchainLagging = 40 * time.Second // TODO: use env var AllowableServiceToNodeTimeDiff = 2 * time.Second ) // JettonProxyContractCode source code at https://github.com/gobicycle/ton-proxy-contract const JettonProxyContractCode = "B5EE9C72410102010037000114FF00F4A413F4BCF2C80B010050D33331D0D3030171B0915BE0FA4030ED44D0FA4030C705F2E1939320D74A97D4018100A0FB00E8301E8A9040" const MaxCommentLength = 1000 // qty in chars var Config = struct { LiteServer string `env:"LITESERVER,required"` LiteServerKey string `env:"LITESERVER_KEY,required"` LiteServerRateLimit int `env:"LITESERVER_RATE_LIMIT" envDefault:"100"` Seed string `env:"SEED,required"` DatabaseURI string `env:"DB_URI,required"` APIPort int `env:"API_PORT,required"` APIToken string `env:"API_TOKEN,required"` Testnet bool `env:"IS_TESTNET" envDefault:"true"` ColdWalletString string `env:"COLD_WALLET"` JettonString string `env:"JETTONS"` TonString string `env:"TON_CUTOFFS,required"` IsDepositSideCalculation bool `env:"DEPOSIT_SIDE_BALANCE" envDefault:"true"` // TODO: rename to DEPOSIT_SIDE_CALCULATION QueueURI string `env:"QUEUE_URI"` QueueName string `env:"QUEUE_NAME"` QueueEnabled bool `env:"QUEUE_ENABLED" envDefault:"false"` ProofCheckEnabled bool `env:"PROOF_CHECK_ENABLED" envDefault:"false"` NetworkConfigUrl string `env:"NETWORK_CONFIG_URL"` WebhookEndpoint string `env:"WEBHOOK_ENDPOINT"` WebhookToken string `env:"WEBHOOK_TOKEN"` AllowableLaggingSec int `env:"ALLOWABLE_LAG"` ForwardTonAmount int `env:"FORWARD_TON_AMOUNT" envDefault:"1"` Jettons map[string]Jetton Ton Cutoffs ColdWallet *address.Address BlockchainConfig *boc.Cell }{} type Jetton struct { Master *address.Address WithdrawalCutoff *big.Int HotWalletMaxCutoff *big.Int HotWalletResidual *big.Int } type Cutoffs struct { HotWalletMin *big.Int HotWalletMax *big.Int Withdrawal *big.Int HotWalletResidual *big.Int } func GetConfig() { err := env.Parse(&Config) if err != nil { log.Fatalf("Can not load config: %v", err) } Config.Jettons = parseJettonString(Config.JettonString) Config.Ton = parseTonString(Config.TonString) if Config.ForwardTonAmount < 0 || Config.ForwardTonAmount > MaxJettonForwardTonAmount { log.Fatalf("Forward TON amount for jetton transfer must be positive and less than %d", MaxJettonForwardTonAmount) } else { JettonForwardAmount = tlb.FromNanoTONU(uint64(Config.ForwardTonAmount)) } if Config.ColdWalletString != "" { coldAddr, err := address.ParseAddr(Config.ColdWalletString) if err != nil { log.Fatalf("Can not parse cold wallet address: %v", err) } if coldAddr.Type() != address.StdAddress { log.Fatalf("Only std cold wallet address supported") } if coldAddr.IsTestnetOnly() && !Config.Testnet { log.Fatalf("Can not use testnet cold wallet address for mainnet") } Config.ColdWallet = coldAddr } if Config.AllowableLaggingSec != 0 { AllowableBlockchainLagging = time.Second * time.Duration(Config.AllowableLaggingSec) } } func parseJettonString(s string) map[string]Jetton { res := make(map[string]Jetton) if s == "" { return res } jettons := strings.Split(s, ",") for _, j := range jettons { data := strings.Split(j, ":") if len(data) != 4 && len(data) != 5 { log.Fatalf("invalid jetton data") } cur := data[0] addr, err := address.ParseAddr(data[1]) if err != nil { log.Fatalf("invalid jetton address: %v", err) } maxCutoff, err := decimal.NewFromString(data[2]) if err != nil { log.Fatalf("invalid %v jetton max cutoff: %v", data[0], err) } withdrawalCutoff, err := decimal.NewFromString(data[3]) if err != nil { log.Fatalf("invalid %v jetton withdrawal cutoff: %v", data[0], err) } residual := maxCutoff.Mul(DefaultHotWalletHysteresis) if len(data) == 5 { residual, err = decimal.NewFromString(data[4]) if err != nil { log.Fatalf("invalid hot_wallet_residual_balance parameter: %v", err) } } res[cur] = Jetton{ Master: addr, WithdrawalCutoff: withdrawalCutoff.BigInt(), HotWalletMaxCutoff: maxCutoff.BigInt(), HotWalletResidual: residual.BigInt(), } } return res } func parseTonString(s string) Cutoffs { data := strings.Split(s, ":") if len(data) != 3 && len(data) != 4 { log.Fatalf("invalid TON cuttofs") } hotWalletMin, err := decimal.NewFromString(data[0]) if err != nil { log.Fatalf("invalid TON hot wallet min cutoff: %v", err) } hotWalletMax, err := decimal.NewFromString(data[1]) if err != nil { log.Fatalf("invalid TON hot wallet max cutoff: %v", err) } withdrawal, err := decimal.NewFromString(data[2]) if err != nil { log.Fatalf("invalid TON withdrawal cutoff: %v", err) } if hotWalletMin.Cmp(hotWalletMax) == 1 { log.Fatalf("TON hot wallet max cutoff must be greater than TON hot wallet min cutoff") } residual := hotWalletMax.Mul(DefaultHotWalletHysteresis) if len(data) == 4 { residual, err = decimal.NewFromString(data[3]) if err != nil { log.Fatalf("invalid hot_wallet_residual_balance parameter: %v", err) } } return Cutoffs{ HotWalletMin: hotWalletMin.BigInt(), HotWalletMax: hotWalletMax.BigInt(), Withdrawal: withdrawal.BigInt(), HotWalletResidual: residual.BigInt(), } } ================================================ FILE: core/block_scanner.go ================================================ package core import ( "context" "fmt" "math/big" "sync" "time" "github.com/gobicycle/bicycle/audit" "github.com/gobicycle/bicycle/config" "github.com/gofrs/uuid" log "github.com/sirupsen/logrus" "github.com/tonkeeper/tongo" "github.com/tonkeeper/tongo/boc" tongoTlb "github.com/tonkeeper/tongo/tlb" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton" "github.com/xssnick/tonutils-go/tvm/cell" ) type BlockScanner struct { db storage blockchain blockchain shard byte tracker blocksTracker wg *sync.WaitGroup notificators []Notificator } type transactions struct { Address Address WalletType WalletType Transactions []*tlb.Transaction } type jettonTransferNotificationMsg struct { Amount Coins Sender *address.Address Comment string } type JettonTransferMsg struct { Amount Coins Destination *address.Address Comment string } type HighLoadWalletExtMsgInfo struct { UUID uuid.UUID TTL time.Time Messages *cell.Dictionary } type incomeNotification struct { Deposit string `json:"deposit_address"` Timestamp int64 `json:"time"` Amount string `json:"amount"` Source string `json:"source_address,omitempty"` Comment string `json:"comment,omitempty"` UserID string `json:"user_id"` TxHash string `json:"tx_hash"` } func NewBlockScanner( wg *sync.WaitGroup, db storage, blockchain blockchain, shard byte, tracker blocksTracker, notificators []Notificator, ) *BlockScanner { t := &BlockScanner{ db: db, blockchain: blockchain, shard: shard, tracker: tracker, wg: wg, notificators: notificators, } t.wg.Add(1) go t.Start() return t } func (s *BlockScanner) Start() { defer s.wg.Done() log.Printf("Block scanner started") for { block, exit, err := s.tracker.NextBlock() if err != nil { log.Fatalf("get block error: %v", err) } if exit { log.Printf("Block scanner stopped") break } ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) err = s.processBlock(ctx, block) if err != nil { log.Fatalf("block processing error: %v", err) } cancel() } } func (s *BlockScanner) Stop() { s.tracker.Stop() } func (s *BlockScanner) processBlock(ctx context.Context, block ShardBlockHeader) error { txIDs, err := s.blockchain.GetTransactionIDsFromBlock(ctx, block.BlockIDExt) if err != nil { return err } filteredTXs, err := s.filterTXs(ctx, block.BlockIDExt, txIDs) if err != nil { return err } e, err := s.processTXs(ctx, filteredTXs, block) if err != nil { return err } err = s.db.SaveParsedBlockData(ctx, e) if err != nil { return err } // Push notifications after saving to the database. // Prevents duplicate sending on restart, but may result in lost notifications. return s.pushNotifications(e) } func (s *BlockScanner) pushNotifications(e BlockEvents) error { if len(s.notificators) == 0 { return nil } if config.Config.IsDepositSideCalculation { for _, ei := range e.ExternalIncomes { err := s.pushNotification(ei.To, ei.Amount, ei.Utime, ei.From, ei.FromWorkchain, ei.Comment, ei.TxHash) if err != nil { return err } } } else { for _, ii := range e.InternalIncomes { err := s.pushNotification(ii.From, ii.Amount, ii.Utime, nil, nil, "", ii.TxHash) if err != nil { return err } } } return nil } func (s *BlockScanner) pushNotification( addr Address, amount Coins, timestamp uint32, from []byte, fromWorkchain *int32, comment string, txHash []byte, ) error { owner := s.db.GetOwner(addr) if owner != nil { addr = *owner } userID, ok := s.db.GetUserID(addr) if !ok { return fmt.Errorf("not found UserID for deposit %s", addr.ToUserFormat()) } notification := incomeNotification{ Deposit: addr.ToUserFormat(), Amount: amount.String(), Timestamp: int64(timestamp), Comment: comment, UserID: userID, TxHash: fmt.Sprintf("%x", txHash), } if len(from) == 32 && fromWorkchain != nil { // supports only std address src := address.NewAddress(0, byte(*fromWorkchain), from) src.SetTestnetOnly(config.Config.Testnet) notification.Source = src.String() } for _, n := range s.notificators { err := n.Publish(notification) if err != nil { return err } } return nil } func (s *BlockScanner) filterTXs( ctx context.Context, blockID *ton.BlockIDExt, ids []ton.TransactionShortInfo, ) ( []transactions, error, ) { txMap := make(map[Address][]*tlb.Transaction) for _, id := range ids { a, err := AddressFromBytes(id.Account) // must be int256 for lite api if err != nil { return nil, err } _, ok := s.db.GetWalletType(a) if ok { tx, err := s.blockchain.GetTransactionFromBlock(ctx, blockID, id) if err != nil { return nil, err } txMap[a] = append(txMap[a], tx) } } var res []transactions for a, txs := range txMap { wType, _ := s.db.GetWalletType(a) res = append(res, transactions{a, wType, txs}) } return res, nil } func checkTxForSuccess(tx *tlb.Transaction) (bool, error) { cell1, err := tlb.ToCell(tx.Description) if err != nil { return false, err } c, err := boc.DeserializeBoc(cell1.ToBOC()) if err != nil { return false, err } var desc tongoTlb.TransactionDescr err = tongoTlb.Unmarshal(c[0], &desc) if err != nil { return false, err } var fakeTx tongo.Transaction // need for check tx success via tongo fakeTx.Description = desc return fakeTx.IsSuccess(), nil } func (s *BlockScanner) processTXs( ctx context.Context, txs []transactions, block ShardBlockHeader, ) ( BlockEvents, error, ) { blockEvents := BlockEvents{Block: block} for _, t := range txs { switch t.WalletType { // TODO: check order of Lt for different accounts (it is important for intermediate tx Lt) case TonHotWallet: hotWalletEvents, err := s.processTonHotWalletTXs(t) if err != nil { return BlockEvents{}, err } blockEvents.Append(hotWalletEvents) case TonDepositWallet: tonDepositEvents, err := s.processTonDepositWalletTXs(t) if err != nil { return BlockEvents{}, err } blockEvents.Append(tonDepositEvents) case JettonDepositWallet: jettonDepositEvents, err := s.processJettonDepositWalletTXs(ctx, t, block.BlockIDExt, block.Parent) if err != nil { return BlockEvents{}, err } blockEvents.Append(jettonDepositEvents) } } return blockEvents, nil } func (s *BlockScanner) processTonHotWalletTXs(txs transactions) (Events, error) { var events Events for _, tx := range txs.Transactions { if tx.IO.In == nil { // impossible for standard highload TON wallet audit.LogTX(audit.Error, string(TonHotWallet), tx.Hash, "transaction without in message") return Events{}, fmt.Errorf("anomalous behavior of the TON hot wallet") } switch tx.IO.In.MsgType { case tlb.MsgTypeExternalIn: e, err := s.processTonHotWalletExternalInMsg(tx) if err != nil { return Events{}, err } events.Append(e) case tlb.MsgTypeInternal: e, err := s.processTonHotWalletInternalInMsg(tx) if err != nil { return Events{}, err } events.Append(e) default: audit.LogTX(audit.Error, string(TonHotWallet), tx.Hash, "transaction in message must be internal or external in") return Events{}, fmt.Errorf("anomalous behavior of the TON hot wallet") } } return events, nil } func (s *BlockScanner) processTonDepositWalletTXs(txs transactions) (Events, error) { var events Events for _, tx := range txs.Transactions { if tx.IO.In == nil { // impossible for standard TON V3 wallet audit.LogTX(audit.Error, string(TonDepositWallet), tx.Hash, "transaction without in message") return Events{}, fmt.Errorf("anomalous behavior of the deposit TON wallet") } switch tx.IO.In.MsgType { case tlb.MsgTypeExternalIn: // internal withdrawal. spam or invalid external cannot invoke tx // theoretically will be up to 4 out messages for TON V3 wallet // external_in msg without out_msg very rare or impossible // it is not critical for internal transfers (double spending not dangerous). success, err := checkTxForSuccess(tx) if err != nil { return Events{}, err } if !success { audit.LogTX(audit.Info, string(TonDepositWallet), tx.Hash, "failed transaction") continue } e, err := s.processTonDepositWalletExternalInMsg(tx) if err != nil { return Events{}, err } events.Append(e) case tlb.MsgTypeInternal: // external payment income // internal message can not invoke out message for TON wallet V3 except of bounce // bounced filtered by len(tx.IO.Out) != 0 if tx.OutMsgCount != 0 { audit.LogTX(audit.Info, string(TonDepositWallet), tx.Hash, "ton deposit filling is bounced") continue } e, err := s.processTonDepositWalletInternalInMsg(tx) if err != nil { return Events{}, err } events.Append(e) default: audit.LogTX(audit.Error, string(TonDepositWallet), tx.Hash, "transaction in message must be internal or external in") return Events{}, fmt.Errorf("anomalous behavior of the deposit TON wallet") } } return events, nil } func (s *BlockScanner) processJettonDepositWalletTXs( ctx context.Context, txs transactions, blockID, prevBlockID *ton.BlockIDExt, ) (Events, error) { var ( unknownTransactions []*tlb.Transaction events Events ) knownIncomeAmount := big.NewInt(0) totalWithdrawalsAmount := big.NewInt(0) for _, tx := range txs.Transactions { e, knownAmount, outUnknownFound, err := s.processJettonDepositOutMsgs(tx) if err != nil { return Events{}, err } knownIncomeAmount.Add(knownIncomeAmount, knownAmount) events.Append(e) e, totalAmount, inUnknownFound, err := s.processJettonDepositInMsg(tx) if err != nil { return Events{}, err } totalWithdrawalsAmount.Add(totalWithdrawalsAmount, totalAmount) events.Append(e) if outUnknownFound || inUnknownFound { // if found some unknown messages that potentially can change Jetton balance unknownTransactions = append(unknownTransactions, tx) } } unknownIncomeAmount, err := s.calculateJettonAmounts(ctx, txs.Address, prevBlockID, blockID, knownIncomeAmount, totalWithdrawalsAmount) if err != nil { return Events{}, err } if unknownIncomeAmount.Cmp(big.NewInt(0)) == 1 { // unknownIncomeAmount > 0 unknownIncomes, err := convertUnknownJettonTxs(unknownTransactions, txs.Address, unknownIncomeAmount) if err != nil { return Events{}, err } events.ExternalIncomes = append(events.ExternalIncomes, unknownIncomes...) } return events, nil } func (s *BlockScanner) calculateJettonAmounts( ctx context.Context, address Address, prevBlockID, blockID *ton.BlockIDExt, knownIncomeAmount, totalWithdrawalsAmount *big.Int, ) ( unknownIncomeAmount *big.Int, err error, ) { prevBalance, err := s.blockchain.GetJettonBalance(ctx, address, prevBlockID) if err != nil { return nil, err } currentBalance, err := s.blockchain.GetJettonBalance(ctx, address, blockID) if err != nil { return nil, err } diff := big.NewInt(0) diff.Sub(currentBalance, prevBalance) // diff = currentBalance - prevBalance totalIncomeAmount := big.NewInt(0) totalIncomeAmount.Add(diff, totalWithdrawalsAmount) // totalIncomeAmount = diff + totalWithdrawalsAmount unknownIncomeAmount = big.NewInt(0) unknownIncomeAmount.Sub(totalIncomeAmount, knownIncomeAmount) // unknownIncomeAmount = totalIncomeAmount - knownIncomeAmount return unknownIncomeAmount, nil } func convertUnknownJettonTxs(txs []*tlb.Transaction, addr Address, amount *big.Int) ([]ExternalIncome, error) { var incomes []ExternalIncome for _, tx := range txs { // unknown sender (jetton wallet owner). do not save message sender as from. incomes = append(incomes, ExternalIncome{ Utime: tx.Now, Lt: tx.LT, To: addr, Amount: ZeroCoins(), TxHash: tx.Hash, }) } if len(txs) > 0 { incomes = append(incomes, ExternalIncome{ Utime: txs[0].Now, // mark unknown tx with first tx time Lt: txs[0].LT, To: addr, Amount: NewCoins(amount), TxHash: txs[0].Hash, }) } return incomes, nil } func decodeJettonTransferNotification(msg *tlb.InternalMessage) (jettonTransferNotificationMsg, error) { if msg == nil { return jettonTransferNotificationMsg{}, fmt.Errorf("nil msg") } payload := msg.Payload() if payload == nil { return jettonTransferNotificationMsg{}, fmt.Errorf("empty payload") } var notification struct { _ tlb.Magic `tlb:"#7362d09c"` QueryID uint64 `tlb:"## 64"` Amount tlb.Coins `tlb:"."` Sender *address.Address `tlb:"addr"` ForwardPayload *cell.Cell `tlb:"either . ^"` } err := tlb.LoadFromCell(¬ification, payload.BeginParse()) if err != nil { return jettonTransferNotificationMsg{}, err } return jettonTransferNotificationMsg{ Sender: notification.Sender, Amount: NewCoins(notification.Amount.Nano()), Comment: LoadComment(notification.ForwardPayload), }, nil } func DecodeJettonTransfer(msg *tlb.InternalMessage) (JettonTransferMsg, error) { if msg == nil { return JettonTransferMsg{}, fmt.Errorf("nil msg") } payload := msg.Payload() if payload == nil { return JettonTransferMsg{}, fmt.Errorf("empty payload") } var transfer struct { _ tlb.Magic `tlb:"#0f8a7ea5"` QueryID uint64 `tlb:"## 64"` Amount tlb.Coins `tlb:"."` Destination *address.Address `tlb:"addr"` ResponseDestination *address.Address `tlb:"addr"` CustomPayload *cell.Cell `tlb:"maybe ^"` ForwardTonAmount tlb.Coins `tlb:"."` ForwardPayload *cell.Cell `tlb:"either . ^"` } err := tlb.LoadFromCell(&transfer, payload.BeginParse()) if err != nil { return JettonTransferMsg{}, err } return JettonTransferMsg{ NewCoins(transfer.Amount.Nano()), transfer.Destination, LoadComment(transfer.ForwardPayload), }, nil } func decodeJettonExcesses(msg *tlb.InternalMessage) (int64, error) { if msg == nil { return 0, fmt.Errorf("nil msg") } payload := msg.Payload() if payload == nil { return 0, fmt.Errorf("empty payload") } var excesses struct { _ tlb.Magic `tlb:"#d53276db"` QueryID uint64 `tlb:"## 64"` } err := tlb.LoadFromCell(&excesses, payload.BeginParse()) if err != nil { return 0, err } return int64(excesses.QueryID), nil } func parseExternalMessage(msg *tlb.ExternalMessage) ( u uuid.UUID, addrMap map[Address]struct{}, isValidWithdrawal bool, err error, ) { if msg == nil { return uuid.UUID{}, nil, false, fmt.Errorf("nil msg") } addrMap = make(map[Address]struct{}) info, err := getHighLoadWalletExtMsgInfo(msg) if err != nil { return uuid.UUID{}, nil, false, err } for _, m := range info.Messages.All() { var ( intMsg tlb.InternalMessage addr Address ) msgCell, err := m.Value.BeginParse().LoadRef() if err != nil { return uuid.UUID{}, nil, false, err } err = tlb.LoadFromCell(&intMsg, msgCell) if err != nil { return uuid.UUID{}, nil, false, err } jettonTransfer, err := DecodeJettonTransfer(&intMsg) if err == nil { addr, err = AddressFromTonutilsAddress(jettonTransfer.Destination) if err != nil { return uuid.UUID{}, nil, false, nil } } else { addr, err = AddressFromTonutilsAddress(intMsg.DstAddr) if err != nil { return uuid.UUID{}, nil, false, nil } } _, ok := addrMap[addr] if ok { // not unique addresses return uuid.UUID{}, nil, false, nil } addrMap[addr] = struct{}{} } return info.UUID, addrMap, true, nil } func (s *BlockScanner) failedWithdrawals(inMap map[Address]struct{}, outMap map[Address]struct{}, u uuid.UUID, txHash []byte) []ExternalWithdrawal { var w []ExternalWithdrawal for i := range inMap { _, dstOk := s.db.GetWalletType(i) if _, ok := outMap[i]; !ok && !dstOk { // !dstOk - not failed internal fee payments w = append(w, ExternalWithdrawal{ExtMsgUuid: u, To: i, IsFailed: true, TxHash: txHash}) audit.LogTX(audit.Error, string(TonHotWallet), txHash, fmt.Sprintf("Failed external withdrawal to %v", i.ToUserFormat())) } else if !ok && dstOk { // failed internal fee payments // TODO: cause a fatal error or increment error counter audit.LogTX(audit.Error, string(TonHotWallet), txHash, fmt.Sprintf("Failed internal withdrawal to %v", i.ToUserFormat())) } } return w } func getHighLoadWalletExtMsgInfo(extMsg *tlb.ExternalMessage) (HighLoadWalletExtMsgInfo, error) { body := extMsg.Payload() if body == nil { return HighLoadWalletExtMsgInfo{}, fmt.Errorf("nil body for external message") } hash := body.Hash() // must be 32 bytes u, err := uuid.FromBytes(hash[:16]) if err != nil { return HighLoadWalletExtMsgInfo{}, err } var data struct { Sign []byte `tlb:"bits 512"` SubwalletID uint32 `tlb:"## 32"` BoundedID uint64 `tlb:"## 64"` Messages *cell.Dictionary `tlb:"dict 16"` } err = tlb.LoadFromCell(&data, body.BeginParse()) if err != nil { return HighLoadWalletExtMsgInfo{}, err } ttl := time.Unix(int64((data.BoundedID>>32)&0x00_00_00_00_FF_FF_FF_FF), 0) return HighLoadWalletExtMsgInfo{UUID: u, TTL: ttl, Messages: data.Messages}, nil } func (s *BlockScanner) processTonHotWalletExternalInMsg(tx *tlb.Transaction) (Events, error) { var events Events inMsg := tx.IO.In.AsExternalIn() // withdrawal messages must be only with different recipients for identification u, addrMapIn, isValid, err := parseExternalMessage(inMsg) if err != nil { return Events{}, err } if !isValid { audit.LogTX(audit.Error, string(TonHotWallet), tx.Hash, "not valid external message") return Events{}, fmt.Errorf("not valid message") } addrMapOut := make(map[Address]struct{}) var outList []tlb.Message if tx.OutMsgCount > 0 { outList, err = tx.IO.Out.ToSlice() if err != nil { return Events{}, err } } for _, m := range outList { if m.MsgType != tlb.MsgTypeInternal { audit.LogTX(audit.Error, string(TonHotWallet), tx.Hash, "not internal out message for transaction") return Events{}, fmt.Errorf("anomalous behavior of the TON hot wallet") } msg := m.AsInternal() addr, err := AddressFromTonutilsAddress(msg.DstAddr) if err != nil { return Events{}, fmt.Errorf("invalid address in withdrawal message") } dstType, dstOk := s.db.GetWalletTypeByTonutilsAddress(msg.DstAddr) if dstOk && dstType == JettonHotWallet { // Jetton external withdrawal jettonTransfer, err := DecodeJettonTransfer(msg) if err != nil { audit.LogTX(audit.Error, string(TonHotWallet), tx.Hash, "invalid jetton transfer message to hot jetton wallet") return Events{}, fmt.Errorf("invalid jetton transfer message to hot jetton wallet") } a, err := AddressFromTonutilsAddress(jettonTransfer.Destination) if err != nil { return Events{}, fmt.Errorf("invalid address in withdrawal message") } events.ExternalWithdrawals = append(events.ExternalWithdrawals, ExternalWithdrawal{ ExtMsgUuid: u, Utime: msg.CreatedAt, Lt: msg.CreatedLT, To: a, Amount: jettonTransfer.Amount, Comment: jettonTransfer.Comment, IsFailed: false, TxHash: tx.Hash, }) addrMapOut[a] = struct{}{} continue } if dstOk && dstType == JettonOwner { // Jetton internal withdrawal or service withdrawal e, err := s.processTonHotWalletProxyMsg(msg) if err != nil { return Events{}, fmt.Errorf("jetton withdrawal error: %v", err) } events.Append(e) addrMapOut[addr] = struct{}{} continue } if !dstOk { // hot_wallet -> unknown_address. to filter internal fee payments events.ExternalWithdrawals = append(events.ExternalWithdrawals, ExternalWithdrawal{ ExtMsgUuid: u, Utime: msg.CreatedAt, Lt: msg.CreatedLT, To: addr, Amount: NewCoins(msg.Amount.Nano()), Comment: msg.Comment(), IsFailed: false, TxHash: tx.Hash, }) } addrMapOut[addr] = struct{}{} } events.ExternalWithdrawals = append(events.ExternalWithdrawals, s.failedWithdrawals(addrMapIn, addrMapOut, u, tx.Hash)...) return events, nil } func (s *BlockScanner) processTonHotWalletProxyMsg(msg *tlb.InternalMessage) (Events, error) { var events Events body := msg.Payload() internalPayload, err := body.BeginParse().LoadRef() if err != nil { return Events{}, fmt.Errorf("no internal payload to proxy contract: %v", err) } var intMsg tlb.InternalMessage err = tlb.LoadFromCell(&intMsg, internalPayload) if err != nil { return Events{}, fmt.Errorf("can not decode payload message for proxy contract: %v", err) } destType, ok := s.db.GetWalletTypeByTonutilsAddress(intMsg.DstAddr) // ok && destType == TonHotWallet - service TON withdrawal // !ok - service Jetton withdrawal if ok && destType == JettonDepositWallet { // Jetton internal withdrawal jettonTransfer, err := DecodeJettonTransfer(&intMsg) if err != nil { return Events{}, fmt.Errorf("invalid jetton transfer message to deposit jetton wallet: %v", err) } a, err := AddressFromTonutilsAddress(jettonTransfer.Destination) if err != nil { return Events{}, fmt.Errorf("invalid address in withdrawal message") } events.SendingConfirmations = append(events.SendingConfirmations, SendingConfirmation{ Lt: msg.CreatedLT, From: a, Memo: jettonTransfer.Comment, }) } return events, nil } func (s *BlockScanner) processTonHotWalletInternalInMsg(tx *tlb.Transaction) (Events, error) { var events Events inMsg := tx.IO.In.AsInternal() srcAddr, err := AddressFromTonutilsAddress(inMsg.SrcAddr) if err != nil { return Events{}, err } dstAddr, err := AddressFromTonutilsAddress(inMsg.DstAddr) if err != nil { return Events{}, err } srcType, srcOk := s.db.GetWalletType(srcAddr) if !srcOk { // unknown_address -> hot_wallet. to check for external jetton transfer confirmation via excess message queryID, err := decodeJettonExcesses(inMsg) if err == nil { events.WithdrawalConfirmations = append(events.WithdrawalConfirmations, JettonWithdrawalConfirmation{queryID}) } } else if srcOk && srcType == TonDepositWallet { // income TONs from deposit income := InternalIncome{ Lt: inMsg.CreatedLT, Utime: inMsg.CreatedAt, From: srcAddr, To: dstAddr, Amount: NewCoins(inMsg.Amount.Nano()), Memo: inMsg.Comment(), IsFailed: false, TxHash: tx.Hash, } success, err := checkTxForSuccess(tx) if err != nil { return Events{}, err } // TODO: check for partially failed message if success { events.InternalIncomes = append(events.InternalIncomes, income) } else { income.IsFailed = true events.InternalIncomes = append(events.InternalIncomes, income) } } else if srcOk && srcType == JettonHotWallet { // income Jettons notification from Jetton hot wallet income, err := decodeJettonTransferNotification(inMsg) if err == nil { sender, err := AddressFromTonutilsAddress(income.Sender) if err != nil { return Events{}, err } fromType, fromOk := s.db.GetWalletType(sender) if !fromOk || fromType != JettonOwner { // skip transfers not from deposit wallets return events, nil } events.InternalIncomes = append(events.InternalIncomes, InternalIncome{ Lt: inMsg.CreatedLT, Utime: inMsg.CreatedAt, From: sender, // sender == owner of jetton deposit wallet To: srcAddr, Amount: income.Amount, Memo: income.Comment, IsFailed: false, TxHash: tx.Hash, }) } } return events, nil } func (s *BlockScanner) processTonDepositWalletExternalInMsg(tx *tlb.Transaction) (Events, error) { var events Events dstAddr, err := AddressFromTonutilsAddress(tx.IO.In.AsExternalIn().DstAddr) if err != nil { return Events{}, err } var outList []tlb.Message if tx.OutMsgCount > 0 { outList, err = tx.IO.Out.ToSlice() if err != nil { return Events{}, err } } for _, o := range outList { if o.MsgType != tlb.MsgTypeInternal { audit.LogTX(audit.Error, string(TonDepositWallet), tx.Hash, "not internal out message for transaction") return Events{}, fmt.Errorf("anomalous behavior of the deposit TON wallet") } msg := o.AsInternal() t, srcOk := s.db.GetWalletTypeByTonutilsAddress(msg.DstAddr) if !srcOk || t != TonHotWallet { audit.LogTX(audit.Warning, string(TonDepositWallet), tx.Hash, fmt.Sprintf("TONs withdrawal from %v to %v (not to hot wallet)", msg.SrcAddr.String(), msg.DstAddr.String())) continue } events.SendingConfirmations = append(events.SendingConfirmations, SendingConfirmation{ Lt: msg.CreatedLT, From: dstAddr, Memo: msg.Comment(), }) events.InternalWithdrawals = append(events.InternalWithdrawals, InternalWithdrawal{ Utime: msg.CreatedAt, Lt: msg.CreatedLT, From: dstAddr, Amount: NewCoins(msg.Amount.Nano()), Memo: msg.Comment(), IsFailed: false, }) } return events, nil } func (s *BlockScanner) processTonDepositWalletInternalInMsg(tx *tlb.Transaction) (Events, error) { var ( events Events from Address err error fromWorkchain *int32 ) inMsg := tx.IO.In.AsInternal() dstAddr, err := AddressFromTonutilsAddress(inMsg.DstAddr) if err != nil { return Events{}, err } isKnownSender := false // support only std address if inMsg.SrcAddr.Type() == address.StdAddress { from, err = AddressFromTonutilsAddress(inMsg.SrcAddr) if err != nil { return Events{}, err } _, isKnownSender = s.db.GetWalletType(from) wc := inMsg.SrcAddr.Workchain() fromWorkchain = &wc } if !isKnownSender { // income TONs from payer. exclude internal (hot->deposit, deposit->deposit) transfers. events.ExternalIncomes = append(events.ExternalIncomes, ExternalIncome{ Lt: tx.LT, Utime: tx.Now, From: from.ToBytes(), FromWorkchain: fromWorkchain, To: dstAddr, Amount: NewCoins(inMsg.Amount.Nano()), Comment: inMsg.Comment(), TxHash: tx.Hash, }) } return events, nil } func (s *BlockScanner) processJettonDepositOutMsgs(tx *tlb.Transaction) (Events, *big.Int, bool, error) { var events Events knownIncomeAmount := big.NewInt(0) unknownMsgFound := false var ( outList []tlb.Message err error ) if tx.OutMsgCount > 0 { outList, err = tx.IO.Out.ToSlice() if err != nil { return Events{}, nil, false, err } } for _, m := range outList { // checks for JettonTransferNotification if m.MsgType != tlb.MsgTypeInternal { audit.LogTX(audit.Info, string(JettonDepositWallet), tx.Hash, "sends external out message") unknownMsgFound = true continue } // skip external_out msg outMsg := m.AsInternal() srcAddr, err := AddressFromTonutilsAddress(outMsg.SrcAddr) if err != nil { return Events{}, nil, false, err } notify, err := decodeJettonTransferNotification(outMsg) if err != nil { unknownMsgFound = true continue } // need not check success. impossible for failed txs. _, senderOk := s.db.GetWalletTypeByTonutilsAddress(notify.Sender) if senderOk { // TODO: check balance calculation for unknown transactions for service transfers audit.LogTX(audit.Info, string(JettonDepositWallet), tx.Hash, "service Jetton transfer") // not set unknownMsgFound = true to prevent service transfers interpretation as unknown continue } // some kind of internal transfer dstAddr, err := AddressFromTonutilsAddress(outMsg.DstAddr) if err != nil { return Events{}, nil, false, err } owner := s.db.GetOwner(srcAddr) if owner == nil { return Events{}, nil, false, fmt.Errorf("no owner for Jetton deposit in addressbook") } if dstAddr != *owner { audit.LogTX(audit.Info, string(JettonDepositWallet), tx.Hash, "sends transfer notification message not to owner") // interpret it as an unknown message unknownMsgFound = true continue } var ( from []byte fromWorkchain *int32 ) if notify.Sender != nil && (notify.Sender.Type() == address.StdAddress || notify.Sender.Type() == address.VarAddress) { from = notify.Sender.Data() wc := notify.Sender.Workchain() fromWorkchain = &wc } events.ExternalIncomes = append(events.ExternalIncomes, ExternalIncome{ Utime: outMsg.CreatedAt, Lt: outMsg.CreatedLT, From: from, FromWorkchain: fromWorkchain, To: srcAddr, Amount: notify.Amount, Comment: notify.Comment, TxHash: tx.Hash, }) knownIncomeAmount.Add(knownIncomeAmount, notify.Amount.BigInt()) } return events, knownIncomeAmount, unknownMsgFound, nil } func (s *BlockScanner) processJettonDepositInMsg(tx *tlb.Transaction) (Events, *big.Int, bool, error) { var events Events unknownMsgFound := false totalWithdrawalsAmount := big.NewInt(0) if tx.IO.In == nil { // skip not decodable in_msg audit.LogTX(audit.Info, string(JettonDepositWallet), tx.Hash, "transaction without in message") // interpret it as an unknown message return events, totalWithdrawalsAmount, true, nil } if tx.IO.In.MsgType != tlb.MsgTypeInternal { // skip not decodable in_msg audit.LogTX(audit.Info, string(JettonDepositWallet), tx.Hash, "not internal in message") // interpret it as an unknown message return events, totalWithdrawalsAmount, true, nil } success, err := checkTxForSuccess(tx) if err != nil { return Events{}, nil, false, err } inMsg := tx.IO.In.AsInternal() dstAddr, err := AddressFromTonutilsAddress(inMsg.DstAddr) if err != nil { return Events{}, nil, false, err } transfer, err := DecodeJettonTransfer(inMsg) if err != nil { unknownMsgFound = true return events, totalWithdrawalsAmount, unknownMsgFound, nil } if !success { // failed withdrawal from deposit jetton wallet events.InternalWithdrawals = append(events.InternalWithdrawals, InternalWithdrawal{ Utime: inMsg.CreatedAt, Lt: inMsg.CreatedLT, From: dstAddr, Amount: transfer.Amount, Memo: transfer.Comment, IsFailed: true, }) return events, totalWithdrawalsAmount, unknownMsgFound, nil } // success withdrawal from deposit jetton wallet if tx.OutMsgCount < 1 { audit.LogTX(audit.Error, string(JettonDepositWallet), tx.Hash, "success Jettons transfer TX without out message") return Events{}, nil, true, fmt.Errorf("anomalous behavior of the deposit Jetton wallet") } totalWithdrawalsAmount.Add(totalWithdrawalsAmount, transfer.Amount.BigInt()) destType, destOk := s.db.GetWalletTypeByTonutilsAddress(transfer.Destination) if !destOk || destType != TonHotWallet { audit.LogTX(audit.Warning, string(JettonDepositWallet), tx.Hash, fmt.Sprintf("Jettons withdrawal from %v to %v (not to hot wallet)", inMsg.DstAddr.String(), transfer.Destination.String())) // TODO: check balance calculation for unknown transactions for service transfers // not set unknownMsgFound = true to prevent service transfers interpretation as unknown return Events{}, totalWithdrawalsAmount, false, nil } events.InternalWithdrawals = append(events.InternalWithdrawals, InternalWithdrawal{ Utime: inMsg.CreatedAt, Lt: inMsg.CreatedLT, From: dstAddr, Amount: transfer.Amount, Memo: transfer.Comment, IsFailed: false, }) return events, totalWithdrawalsAmount, unknownMsgFound, nil } ================================================ FILE: core/models.go ================================================ package core import ( "context" "database/sql/driver" "errors" "fmt" "github.com/gobicycle/bicycle/config" "github.com/gofrs/uuid" "github.com/shopspring/decimal" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton" "github.com/xssnick/tonutils-go/ton/wallet" "math/big" "time" ) const ( TonSymbol = "TON" DefaultWorkchain = 0 // use only 0 workchain ) type IncomeSide = string const ( SideHotWallet IncomeSide = "hot_wallet" SideDeposit IncomeSide = "deposit" ) type EventName = string const ( ServiceWithdrawalEvent EventName = "service withdrawal" InternalWithdrawalEvent EventName = "internal withdrawal" ExternalWithdrawalEvent EventName = "external withdrawal" InitEvent EventName = "initialization" ) type WalletType string const ( TonHotWallet WalletType = "ton_hot" JettonHotWallet WalletType = "jetton_hot" TonDepositWallet WalletType = "ton_deposit" JettonDepositWallet WalletType = "jetton_deposit" JettonOwner WalletType = "owner" ) type WithdrawalStatus string const ( PendingStatus WithdrawalStatus = "pending" ProcessingStatus WithdrawalStatus = "processing" ProcessedStatus WithdrawalStatus = "processed" FailedStatus WithdrawalStatus = "failed" ) var ( ErrNotFound = errors.New("not found") ErrTimeoutExceeded = errors.New("timeout exceeded") ) type Address [32]byte // supports only MsgAddressInt addr_std$10 without anycast and 0 workchain // Scan implements Scanner for database/sql. func (a *Address) Scan(src interface{}) error { srcB, ok := src.([]byte) if !ok { return fmt.Errorf("can't scan %T into Address", src) } if len(srcB) != 32 { return fmt.Errorf("can't scan []byte of len %d into Address, want %d", len(srcB), 32) } copy(a[:], srcB) return nil } // Value implements valuer for database/sql. func (a Address) Value() (driver.Value, error) { return a[:], nil } // ToTonutilsAddressStd implements converter to ton-utils std Address type for default workchain ! func (a Address) ToTonutilsAddressStd(flags byte) *address.Address { return address.NewAddress(flags, DefaultWorkchain, a[:]) } // ToUserFormat converts to user-friendly text format with testnet and bounce flags func (a Address) ToUserFormat() string { addr := a.ToTonutilsAddressStd(0) addr.SetTestnetOnly(config.Config.Testnet) addr.SetBounce(false) return addr.String() } func (a Address) ToBytes() []byte { return a[:] } func TonutilsAddressToUserFormat(addr *address.Address) string { addr.SetTestnetOnly(config.Config.Testnet) addr.SetBounce(false) return addr.String() } func AddressFromBytes(data []byte) (Address, error) { if len(data) != 32 { return Address{}, fmt.Errorf("invalid address len. Std addr len must be 32 bytes") } var res Address copy(res[:], data) return res, nil } func AddressFromTonutilsAddress(addr *address.Address) (Address, error) { if addr == nil { return Address{}, fmt.Errorf("nil tonutils address") } if addr.Type() != address.StdAddress { return Address{}, fmt.Errorf("only std address supported") } return AddressFromBytes(addr.Data()) } func AddressMustFromTonutilsAddress(addr *address.Address) Address { res, err := AddressFromTonutilsAddress(addr) if err != nil { panic(err) } return res } type AddressInfo struct { Type WalletType Owner *Address UserID string } type JettonWallet struct { Address *address.Address Currency string } type OwnerWallet struct { Address Address Currency string } type WalletData struct { SubwalletID uint32 UserID string Currency string Type WalletType Address Address } type WithdrawalRequest struct { QueryID string UserID string Currency string Amount Coins Bounceable bool IsInternal bool Destination Address Comment string BinaryComment string } type WithdrawalData struct { QueryID string UserID string Status WithdrawalStatus TxHash []byte } type ServiceWithdrawalRequest struct { From Address JettonMaster *Address } type ServiceWithdrawalTask struct { ServiceWithdrawalRequest JettonAmount Coins Memo uuid.UUID SubwalletID uint32 } type ExternalWithdrawalTask struct { QueryID int64 Currency string Amount Coins Destination Address Bounceable bool Comment string BinaryComment string } type InternalWithdrawal struct { Utime uint32 Lt uint64 From Address Amount Coins Memo string // uuid from comment IsFailed bool } type SendingConfirmation struct { Lt uint64 // Lt of outgoing wallet message From Address Memo string // uuid from comment } type ExternalWithdrawal struct { ExtMsgUuid uuid.UUID Utime uint32 Lt uint64 To Address Amount Coins Comment string IsFailed bool TxHash []byte } type JettonWithdrawalConfirmation struct { QueryId int64 } type InternalIncome struct { Utime uint32 Lt uint64 // will not fit in db bigint after 1.5 billion years From Address To Address Amount Coins Memo string IsFailed bool TxHash []byte } type ExternalIncome struct { Utime uint32 Lt uint64 From []byte FromWorkchain *int32 To Address Amount Coins Comment string TxHash []byte } type Events struct { ExternalIncomes []ExternalIncome InternalIncomes []InternalIncome SendingConfirmations []SendingConfirmation InternalWithdrawals []InternalWithdrawal ExternalWithdrawals []ExternalWithdrawal WithdrawalConfirmations []JettonWithdrawalConfirmation } func (e *Events) Append(ae Events) { e.ExternalIncomes = append(e.ExternalIncomes, ae.ExternalIncomes...) e.InternalIncomes = append(e.InternalIncomes, ae.InternalIncomes...) e.SendingConfirmations = append(e.SendingConfirmations, ae.SendingConfirmations...) e.InternalWithdrawals = append(e.InternalWithdrawals, ae.InternalWithdrawals...) e.ExternalWithdrawals = append(e.ExternalWithdrawals, ae.ExternalWithdrawals...) e.WithdrawalConfirmations = append(e.WithdrawalConfirmations, ae.WithdrawalConfirmations...) } type BlockEvents struct { Events Block ShardBlockHeader } type InternalWithdrawalTask struct { From Address SubwalletID uint32 Lt uint64 Currency string } type TotalIncome struct { Deposit Address Amount Coins Currency string } type TotalWithdrawalsAmount struct { Pending Coins Processing Coins } type Coins = decimal.Decimal func NewCoins(int *big.Int) Coins { return decimal.NewFromBigInt(int, 0) } func ZeroCoins() Coins { return decimal.New(0, 0) } // ShardBlockHeader // Block header for a specific shard mask attribute. Has only one parent. type ShardBlockHeader struct { *ton.BlockIDExt NotMaster bool GenUtime uint32 StartLt uint64 EndLt uint64 Parent *ton.BlockIDExt } type storage interface { GetExternalWithdrawalTasks(ctx context.Context, limit int) ([]ExternalWithdrawalTask, error) SaveTonWallet(ctx context.Context, walletData WalletData) error SaveJettonWallet(ctx context.Context, ownerAddress Address, walletData WalletData, notSaveOwner bool) error GetWalletType(address Address) (WalletType, bool) GetOwner(address Address) *Address GetUserID(address Address) (string, bool) GetWalletTypeByTonutilsAddress(address *address.Address) (WalletType, bool) SaveParsedBlockData(ctx context.Context, events BlockEvents) error GetTonInternalWithdrawalTasks(ctx context.Context, limit int) ([]InternalWithdrawalTask, error) GetJettonInternalWithdrawalTasks(ctx context.Context, forbiddenAddresses []Address, limit int) ([]InternalWithdrawalTask, error) CreateExternalWithdrawals(ctx context.Context, tasks []ExternalWithdrawalTask, extMsgUuid uuid.UUID, expiredAt time.Time) error GetTonHotWalletAddress(ctx context.Context) (Address, error) SetExpired(ctx context.Context) error SaveInternalWithdrawalTask(ctx context.Context, task InternalWithdrawalTask, expiredAt time.Time, memo uuid.UUID) error IsActualBlockData(ctx context.Context) (bool, int64, error) SaveWithdrawalRequest(ctx context.Context, w WithdrawalRequest) (int64, error) IsInProgressInternalWithdrawalRequest(ctx context.Context, dest Address, currency string) (bool, error) GetServiceHotWithdrawalTasks(ctx context.Context, limit int) ([]ServiceWithdrawalTask, error) UpdateServiceWithdrawalRequest(ctx context.Context, t ServiceWithdrawalTask, tonAmount Coins, expiredAt time.Time, filled bool) error GetServiceDepositWithdrawalTasks(ctx context.Context, limit int) ([]ServiceWithdrawalTask, error) GetJettonWallet(ctx context.Context, address Address) (*WalletData, bool, error) } type blockchain interface { GetJettonWalletAddress(ctx context.Context, ownerWallet *address.Address, jettonMaster *address.Address) (*address.Address, error) GetTransactionIDsFromBlock(ctx context.Context, blockID *ton.BlockIDExt) ([]ton.TransactionShortInfo, error) GetTransactionFromBlock(ctx context.Context, blockID *ton.BlockIDExt, txID ton.TransactionShortInfo) (*tlb.Transaction, error) GenerateDefaultWallet(seed string, isHighload bool) (*wallet.Wallet, byte, uint32, error) GetJettonBalance(ctx context.Context, address Address, blockID *ton.BlockIDExt) (*big.Int, error) SendExternalMessage(ctx context.Context, msg *tlb.ExternalMessage) error GetAccountCurrentState(ctx context.Context, address *address.Address) (*big.Int, tlb.AccountStatus, error) GetLastJettonBalance(ctx context.Context, address *address.Address) (*big.Int, error) DeployTonWallet(ctx context.Context, wallet *wallet.Wallet) error } type blocksTracker interface { NextBlock() (ShardBlockHeader, bool, error) Stop() } type Notificator interface { Publish(payload any) error } ================================================ FILE: core/proxy.go ================================================ package core import ( "encoding/hex" "fmt" "github.com/gobicycle/bicycle/config" log "github.com/sirupsen/logrus" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/tvm/cell" ) // JettonProxy is a special contract wrapper that allow to control jetton wallet from TON wallet. // It is possible create few jetton proxies for single TON wallet (as owner) and control multiple jetton wallets. // Read about JettonProxy smart contract at README.md and https://github.com/gobicycle/ton-proxy-contract type JettonProxy struct { Owner *address.Address SubwalletID uint32 address *address.Address stateInit *tlb.StateInit } func NewJettonProxy(subwalletId uint32, owner *address.Address) (*JettonProxy, error) { if owner == nil { return nil, fmt.Errorf("nil owner") } stateInit := buildJettonProxyStateInit(subwalletId, owner) stateCell, err := tlb.ToCell(stateInit) if err != nil { return nil, fmt.Errorf("failed to get state cell: %w", err) } addr := address.NewAddress(0, DefaultWorkchain, stateCell.Hash()) return &JettonProxy{ Owner: owner, SubwalletID: subwalletId, address: addr, stateInit: stateInit, }, nil } func buildJettonProxyStateInit(subwalletId uint32, owner *address.Address) *tlb.StateInit { h, err := hex.DecodeString(config.JettonProxyContractCode) if err != nil { log.Fatalf("decode JettonProxyContractCode hex error: %v", err) } code, err := cell.FromBOC(h) if err != nil { log.Fatalf("parsing JettonProxyContractCode boc error: %v", err) } data := cell.BeginCell(). MustStoreAddr(owner). MustStoreUInt(uint64(subwalletId), 32). EndCell() res := &tlb.StateInit{ Code: code, Data: data, } return res } // Address returns address of jetton proxy contract func (p *JettonProxy) Address() *address.Address { return p.address } // StateInit returns state init structure of jetton proxy contract func (p *JettonProxy) StateInit() *tlb.StateInit { return p.stateInit } // BuildMessage wraps custom body payload to resend by proxy contract func (p *JettonProxy) BuildMessage(destination *address.Address, body *cell.Cell) *tlb.InternalMessage { return &tlb.InternalMessage{ IHRDisabled: true, Bounce: true, DstAddr: destination, Amount: tlb.FromNanoTONU(0), // proxy sends all TONs with mode == 128 + 32 Body: body, } } ================================================ FILE: core/wallets.go ================================================ package core import ( "context" "errors" "fmt" "github.com/gobicycle/bicycle/audit" "github.com/gobicycle/bicycle/config" "github.com/gofrs/uuid" log "github.com/sirupsen/logrus" "github.com/tonkeeper/tongo/boc" tongoTlb "github.com/tonkeeper/tongo/tlb" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton/jetton" "github.com/xssnick/tonutils-go/ton/wallet" "github.com/xssnick/tonutils-go/tvm/cell" "math/big" "math/rand" ) type Wallets struct { Shard byte TonHotWallet *wallet.Wallet TonBasicWallet *wallet.Wallet // basic V3 wallet to make other wallets with different subwallet_id JettonHotWallets map[string]JettonWallet } // InitWallets // Generates highload hot-wallet and map[currency]JettonWallet Jetton wallets, and saves to DB // TON highload hot-wallet (for seed and default subwallet_id) must be already active for success initialization. func InitWallets( ctx context.Context, db storage, bc blockchain, seed string, jettons map[string]config.Jetton, ) (Wallets, error) { if config.Config.ColdWallet != nil && config.Config.ColdWallet.IsBounceable() { _, status, err := bc.GetAccountCurrentState(ctx, config.Config.ColdWallet) if err != nil { return Wallets{}, err } log.Infof("Cold wallet status: %s", status) if status != tlb.AccountStatusActive { return Wallets{}, fmt.Errorf("cold wallet address must be non-bounceable for not active wallet") } } tonHotWallet, shard, subwalletId, err := initTonHotWallet(ctx, db, bc, seed) if err != nil { return Wallets{}, err } tonBasicWallet, _, _, err := bc.GenerateDefaultWallet(seed, false) if err != nil { return Wallets{}, err } // don't set TTL here because spec is not inherited by GetSubwallet method jettonHotWallets := make(map[string]JettonWallet) for currency, j := range jettons { w, err := initJettonHotWallet(ctx, db, bc, tonHotWallet.Address(), j.Master, currency, subwalletId) if err != nil { return Wallets{}, err } jettonHotWallets[currency] = w } return Wallets{ Shard: shard, TonHotWallet: tonHotWallet, TonBasicWallet: tonBasicWallet, JettonHotWallets: jettonHotWallets, }, nil } func initTonHotWallet( ctx context.Context, db storage, bc blockchain, seed string, ) ( tonHotWallet *wallet.Wallet, shard byte, subwalletId uint32, err error, ) { tonHotWallet, shard, subwalletId, err = bc.GenerateDefaultWallet(seed, true) if err != nil { return nil, 0, 0, err } hotSpec := tonHotWallet.GetSpec().(*wallet.SpecHighloadV2R2) hotSpec.SetMessagesTTL(uint32(config.ExternalMessageLifetime.Seconds())) addr := AddressMustFromTonutilsAddress(tonHotWallet.Address()) alreadySaved := false addrFromDb, err := db.GetTonHotWalletAddress(ctx) if err == nil && addr != addrFromDb { audit.Log(audit.Error, string(TonHotWallet), InitEvent, fmt.Sprintf("Hot TON wallet address is not equal to the one stored in the database. Maybe seed was being changed. %s != %s", tonHotWallet.Address().String(), addrFromDb.ToTonutilsAddressStd(0).String())) return nil, 0, 0, fmt.Errorf("saved hot wallet not equal generated hot wallet. Maybe seed was being changed") } else if !errors.Is(err, ErrNotFound) && err != nil { return nil, 0, 0, err } else if err == nil { alreadySaved = true } log.Infof("Shard: %v", shard) log.Infof("TON hot wallet address: %v", tonHotWallet.Address().String()) balance, status, err := bc.GetAccountCurrentState(ctx, tonHotWallet.Address()) if err != nil { return nil, 0, 0, err } if balance.Cmp(config.Config.Ton.HotWalletMin) == -1 { // hot wallet balance < TonHotWalletMinimumBalance return nil, 0, 0, fmt.Errorf("hot wallet balance must be at least %v nanoTON", config.Config.Ton.HotWalletMin) } if status != tlb.AccountStatusActive { err = bc.DeployTonWallet(ctx, tonHotWallet) if err != nil { return nil, 0, 0, err } } if !alreadySaved { err = db.SaveTonWallet(ctx, WalletData{ SubwalletID: uint32(wallet.DefaultSubwallet), Currency: TonSymbol, Type: TonHotWallet, Address: addr, }) if err != nil { return nil, 0, 0, err } } return tonHotWallet, shard, subwalletId, nil } func initJettonHotWallet( ctx context.Context, db storage, bc blockchain, tonHotWallet, jettonMaster *address.Address, currency string, subwalletId uint32, ) (JettonWallet, error) { // not init or check balances of Jetton wallets, it is not required for the service to work a, err := bc.GetJettonWalletAddress(ctx, tonHotWallet, jettonMaster) if err != nil { return JettonWallet{}, err } res := JettonWallet{Address: a, Currency: currency} log.Infof("%v jetton hot wallet address: %v", currency, a.String()) ownerAddr, err := AddressFromTonutilsAddress(tonHotWallet) if err != nil { return JettonWallet{}, err } jettonWalletAddr, err := AddressFromTonutilsAddress(a) if err != nil { return JettonWallet{}, err } walletData, isPresented, err := db.GetJettonWallet(ctx, jettonWalletAddr) if err != nil { return JettonWallet{}, err } if isPresented && walletData.Currency == currency { return res, nil } else if isPresented && walletData.Currency != currency { audit.Log(audit.Error, string(JettonHotWallet), InitEvent, fmt.Sprintf("Hot Jetton wallets %s and %s have the same address %s", walletData.Currency, currency, a.String())) return JettonWallet{}, fmt.Errorf("jetton hot wallet address duplication") } err = db.SaveJettonWallet( ctx, ownerAddr, WalletData{ SubwalletID: subwalletId, Currency: currency, Type: JettonHotWallet, Address: jettonWalletAddr, }, true, ) if err != nil { return JettonWallet{}, err } return res, nil } func buildComment(comment string) *cell.Cell { root := cell.BeginCell().MustStoreUInt(0, 32) if err := root.StoreStringSnake(comment); err != nil { log.Fatalf("memo must fit into cell") } return root.EndCell() } func LoadComment(cell *cell.Cell) string { if cell == nil { return "" } l := cell.BeginParse() if val, err := l.LoadUInt(32); err == nil && val == 0 { str, err := l.LoadStringSnake() if err != nil { log.Errorf("load comment error: %v", err) return "" } return str } return "" } // WithdrawTONs // Send all TON from one wallet (and deploy it if needed) to another and destroy "from" wallet contract. // Wallet must be not empty. func WithdrawTONs(ctx context.Context, from, to *wallet.Wallet, comment string) error { if from == nil || to == nil || to.Address() == nil { return fmt.Errorf("nil wallet") } var body *cell.Cell if comment != "" { body = buildComment(comment) } return from.Send(ctx, &wallet.Message{ Mode: 128 + 32, // 128 + 32 send all and destroy InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: false, DstAddr: to.Address(), Amount: tlb.FromNanoTONU(0), Body: body, }, }, false) } func WithdrawJettons( ctx context.Context, from, to *wallet.Wallet, jettonWallet *address.Address, forwardAmount tlb.Coins, amount Coins, comment string, ) error { if from == nil || to == nil || to.Address() == nil { return fmt.Errorf("nil wallet") } body := MakeJettonTransferMessage( to.Address(), to.Address(), amount.BigInt(), forwardAmount, rand.Int63(), comment, "", ) return from.Send(ctx, &wallet.Message{ Mode: 128 + 32, // 128 + 32 send all and destroy InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: true, DstAddr: jettonWallet, // jetton wallet address Amount: tlb.FromNanoTONU(0), Body: body, }, }, false) } func MakeJettonTransferMessage( destination, responseDest *address.Address, amount *big.Int, forwardAmount tlb.Coins, queryId int64, comment string, binaryComment string, ) *cell.Cell { forwardPayload := cell.BeginCell().EndCell() if binaryComment != "" { c, err := decodeBinaryComment(binaryComment) if err != nil { log.Fatalf("decode binary comment error : %s", err.Error()) } forwardPayload = c } else if comment != "" { forwardPayload = buildComment(comment) } payload, err := tlb.ToCell(jetton.TransferPayload{ QueryID: uint64(queryId), Amount: tlb.FromNanoTON(amount), Destination: destination, ResponseDestination: responseDest, CustomPayload: nil, ForwardTONAmount: forwardAmount, ForwardPayload: forwardPayload, }) if err != nil { log.Fatalf("jetton transfer message serialization error: %s", err.Error()) } return payload } // decodeBinaryComment implements decoding of hex string and put it into cell with TLB scheme: // `binary_comment#b3ddcf7d {n:#} data:(SnakeData ~n) = InternalMsgBody;` func decodeBinaryComment(comment string) (*cell.Cell, error) { bitString, err := boc.BitStringFromFiftHex(comment) if err != nil { return nil, err } c := boc.NewCell() err = c.WriteUint(0xb3ddcf7d, 32) // binary_comment#b3ddcf7d if err != nil { return nil, err } err = tongoTlb.Marshal(c, tongoTlb.SnakeData(*bitString)) if err != nil { return nil, err } b, err := c.ToBoc() if err != nil { return nil, err } return cell.FromBOC(b) } func BuildTonWithdrawalMessage(t ExternalWithdrawalTask) *wallet.Message { internalMessage := tlb.InternalMessage{ IHRDisabled: true, Bounce: t.Bounceable, DstAddr: t.Destination.ToTonutilsAddressStd(0), Amount: tlb.FromNanoTON(t.Amount.BigInt()), } if t.BinaryComment != "" { c, err := decodeBinaryComment(t.BinaryComment) if err != nil { log.Fatalf("decode binary comment error : %s", err.Error()) } internalMessage.Body = c } else if t.Comment != "" { internalMessage.Body = buildComment(t.Comment) } else { internalMessage.Body = cell.BeginCell().EndCell() } return &wallet.Message{ Mode: 3, InternalMessage: &internalMessage, } } func BuildJettonWithdrawalMessage( t ExternalWithdrawalTask, highloadWallet *wallet.Wallet, fromJettonWallet *address.Address, ) *wallet.Message { body := MakeJettonTransferMessage( t.Destination.ToTonutilsAddressStd(0), highloadWallet.Address(), t.Amount.BigInt(), config.JettonForwardAmount, t.QueryID, t.Comment, t.BinaryComment, ) return &wallet.Message{ Mode: 3, InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: true, DstAddr: fromJettonWallet, Amount: config.JettonTransferTonAmount, Body: body, }, } } func BuildJettonProxyWithdrawalMessage( proxy JettonProxy, jettonWallet, tonWallet *address.Address, forwardAmount tlb.Coins, amount *big.Int, comment string, ) *wallet.Message { jettonTransferPayload := MakeJettonTransferMessage( tonWallet, tonWallet, amount, forwardAmount, rand.Int63(), comment, "", ) msg, err := tlb.ToCell(proxy.BuildMessage(jettonWallet, jettonTransferPayload)) if err != nil { log.Fatalf("build proxy message cell error: %v", err) } body := cell.BeginCell().MustStoreRef(msg).EndCell() return &wallet.Message{ Mode: 3, InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: true, DstAddr: proxy.Address(), Amount: config.JettonTransferTonAmount, Body: body, StateInit: proxy.StateInit(), }, } } func buildJettonProxyServiceTonWithdrawalMessage( proxy JettonProxy, tonWallet *address.Address, memo uuid.UUID, ) *wallet.Message { msg, err := tlb.ToCell(proxy.BuildMessage(tonWallet, buildComment(memo.String()))) if err != nil { log.Fatalf("build proxy message cell error: %v", err) } body := cell.BeginCell().MustStoreRef(msg).EndCell() return &wallet.Message{ Mode: 3, InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: true, DstAddr: proxy.Address(), Amount: config.JettonTransferTonAmount, Body: body, StateInit: proxy.StateInit(), }, } } func buildTonFillMessage( to *address.Address, amount tlb.Coins, memo uuid.UUID, ) *wallet.Message { return &wallet.Message{ Mode: 3, InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: false, DstAddr: to, Amount: amount, Body: buildComment(memo.String()), }, } } ================================================ FILE: core/withdrawal_processor.go ================================================ package core import ( "context" "fmt" "math/big" "sync" "sync/atomic" "time" "github.com/gobicycle/bicycle/audit" "github.com/gobicycle/bicycle/config" "github.com/gofrs/uuid" log "github.com/sirupsen/logrus" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton/wallet" ) type WithdrawalsProcessor struct { db storage bc blockchain wallets Wallets coldWallet *address.Address wg *sync.WaitGroup gracefulShutdown atomic.Bool } type internalWithdrawal struct { Memo uuid.UUID Task InternalWithdrawalTask } type serviceWithdrawal struct { TonAmount Coins Filled bool Task ServiceWithdrawalTask } type withdrawals struct { Messages []*wallet.Message External []ExternalWithdrawalTask Internal []internalWithdrawal Service []serviceWithdrawal } func NewWithdrawalsProcessor( wg *sync.WaitGroup, db storage, bc blockchain, wallets Wallets, coldWallet *address.Address, ) *WithdrawalsProcessor { w := &WithdrawalsProcessor{ db: db, bc: bc, wallets: wallets, coldWallet: coldWallet, wg: wg, } return w } func (p *WithdrawalsProcessor) Start() { p.wg.Add(3) go p.startWithdrawalsProcessor() go p.startInternalTonWithdrawalsProcessor() go p.startExpirationProcessor() } func (p *WithdrawalsProcessor) Stop() { p.gracefulShutdown.Store(true) } func (p *WithdrawalsProcessor) startWithdrawalsProcessor() { defer p.wg.Done() log.Infof("External withdrawal processor started") for { p.waitSync() // gracefulShutdown break must be after waitSync if p.gracefulShutdown.Load() { log.Infof("External withdrawal processor stopped") break } time.Sleep(config.ExternalWithdrawalPeriod) ctx, cancel := context.WithTimeout(context.Background(), time.Second*50) // must be < ExternalWithdrawalPeriod err := p.makeColdWalletWithdrawals(ctx) if err != nil { log.Fatalf("make withdrawals to cold wallet error: %v\n", err) } w, err := p.buildWithdrawalMessages(ctx) if err != nil { log.Fatalf("make withdrawal messages error: %v\n", err) } if len(w.Messages) == 0 { cancel() continue } extMsg, err := p.wallets.TonHotWallet.BuildExternalMessageForMany(ctx, w.Messages) if err != nil { log.Fatalf("build hotwallet external msg error: %v\n", err) } info, err := getHighLoadWalletExtMsgInfo(extMsg) if err != nil { log.Fatalf("get external message uuid error: %v\n", err) } err = p.db.CreateExternalWithdrawals(ctx, w.External, info.UUID, info.TTL) if err != nil { log.Fatalf("save external withdrawals error: %v\n", err) } for _, sw := range w.Service { err = p.db.UpdateServiceWithdrawalRequest(ctx, sw.Task, sw.TonAmount, info.TTL, sw.Filled) if err != nil { log.Fatalf("update service withdrawal error: %v\n", err) } } for _, iw := range w.Internal { err = p.db.SaveInternalWithdrawalTask(ctx, iw.Task, info.TTL, iw.Memo) if err != nil { log.Fatalf("save internal withdrawal error: %v\n", err) } } err = p.bc.SendExternalMessage(ctx, extMsg) if err != nil { log.Errorf("send external msg error: %v\n", err) } cancel() } } func (p *WithdrawalsProcessor) buildWithdrawalMessages(ctx context.Context) (withdrawals, error) { var ( usedAddresses []Address res withdrawals ) balances, err := p.getHotWalletBalances(ctx) if err != nil { return withdrawals{}, fmt.Errorf("get hot wallet balance error: %s", err.Error()) } serviceTasks, err := p.db.GetServiceHotWithdrawalTasks(ctx, 250) if err != nil { return withdrawals{}, err } for _, t := range serviceTasks { if decreaseBalances(balances, TonSymbol, config.JettonTransferTonAmount.Nano()) { continue } msg, w, err := p.buildServiceWithdrawalMessage(ctx, t) if err != nil { return withdrawals{}, err } if len(msg) != 0 { // block scanner determines the uniqueness of the message in the batch by the dest address // the dest address will be the address of the proxy contract // TON deposit address is the dest addr for TON deposit filling message // so the address `t.From` is the dest address when checking the uniqueness usedAddresses = append(usedAddresses, t.From) res.Messages = append(res.Messages, msg...) res.Service = append(res.Service, w) } else { // save rejected service withdrawals err = p.db.UpdateServiceWithdrawalRequest(ctx, w.Task, w.TonAmount, time.Now(), w.Filled) if err != nil { return withdrawals{}, err } } } // `internalTask.From` address is the address of deposit Jetton wallet // the dest address for uniqueness check is proxy contract address // so the proxy contract address must be deduplicated with usedAddresses in db query internalTasks, err := p.db.GetJettonInternalWithdrawalTasks(ctx, usedAddresses, 250) if err != nil { return withdrawals{}, err } for _, t := range internalTasks { if len(res.Messages) > 250 { break } if decreaseBalances(balances, TonSymbol, config.JettonTransferTonAmount.Nano()) { continue } msg, memo, err := p.buildJettonInternalWithdrawalMessage(ctx, t) if err != nil { return withdrawals{}, err } if len(msg) != 0 { res.Messages = append(res.Messages, msg...) res.Internal = append(res.Internal, internalWithdrawal{ Task: t, Memo: memo, }) } } // not filter usedAddresses by DB and perform internal addresses checking and logging externalTasks, err := p.db.GetExternalWithdrawalTasks(ctx, 250) if err != nil { return withdrawals{}, err } for _, w := range externalTasks { if len(res.Messages) > 250 { break } t, ok := p.db.GetWalletType(w.Destination) if ok { audit.Log(audit.Warning, string(TonHotWallet), ExternalWithdrawalEvent, fmt.Sprintf("withdrawal task to internal %s address %s", t, w.Destination.ToUserFormat())) continue } if decreaseBalances(balances, w.Currency, w.Amount.BigInt()) { continue } msg := p.buildExternalWithdrawalMessage(w) res.Messages = append(res.Messages, msg) res.External = append(res.External, w) } return res, nil } func (p *WithdrawalsProcessor) getHotWalletBalances(ctx context.Context) (map[string]*big.Int, error) { res := make(map[string]*big.Int) balance, _, err := p.bc.GetAccountCurrentState(ctx, p.wallets.TonHotWallet.Address()) if err != nil { return nil, err } res[TonSymbol] = balance for cur, w := range p.wallets.JettonHotWallets { balance, err := p.bc.GetLastJettonBalance(ctx, w.Address) if err != nil { return nil, err } res[cur] = balance } return res, nil } // decreaseBalances returns true if balance < amount func decreaseBalances(balances map[string]*big.Int, currency string, amount *big.Int) bool { if currency == TonSymbol { if balances[TonSymbol].Cmp(amount) == -1 { // balance < amount return true } balances[TonSymbol].Sub(balances[TonSymbol], amount) return false } if balances[currency].Cmp(amount) == -1 || // balance < amount balances[TonSymbol].Cmp(config.JettonTransferTonAmount.Nano()) == -1 { // balance < JettonTransferTonAmount return true } balances[currency].Sub(balances[currency], amount) balances[TonSymbol].Sub(balances[TonSymbol], config.JettonTransferTonAmount.Nano()) return false } func (p *WithdrawalsProcessor) buildJettonInternalWithdrawalMessage( ctx context.Context, task InternalWithdrawalTask, ) ( []*wallet.Message, uuid.UUID, error, ) { proxy, err := NewJettonProxy(task.SubwalletID, p.wallets.TonHotWallet.Address()) if err != nil { return nil, uuid.UUID{}, err } jettonWalletAddress := task.From.ToTonutilsAddressStd(0) balance, err := p.bc.GetLastJettonBalance(ctx, jettonWalletAddress) if err != nil { return nil, uuid.UUID{}, err } if balance.Cmp(config.Config.Jettons[task.Currency].WithdrawalCutoff) == 1 { // balance > MinimalJettonWithdrawalAmount memo, err := uuid.NewV4() if err != nil { return nil, uuid.UUID{}, err } msg := BuildJettonProxyWithdrawalMessage( *proxy, jettonWalletAddress, p.wallets.TonHotWallet.Address(), config.JettonInternalForwardAmount, balance, memo.String(), ) return []*wallet.Message{msg}, memo, nil } return []*wallet.Message{}, uuid.UUID{}, nil } func (p *WithdrawalsProcessor) buildServiceWithdrawalMessage( ctx context.Context, task ServiceWithdrawalTask, ) ( []*wallet.Message, serviceWithdrawal, error, ) { t, ok := p.db.GetWalletType(task.From) if !ok || !(t == JettonOwner || t == TonDepositWallet) { return nil, serviceWithdrawal{}, fmt.Errorf("invalid service withdrawal address") } if t == TonDepositWallet { // only fill TON deposit to send Jetton transfer message later return p.buildServiceFilling(ctx, task) } if task.JettonMaster == nil { // full TON withdrawal from Jetton proxy return p.buildServiceTonWithdrawal(ctx, task) } // Jetton withdrawal from Jetton wallet return p.buildServiceJettonWithdrawal(ctx, task) } func (p *WithdrawalsProcessor) buildServiceFilling( ctx context.Context, task ServiceWithdrawalTask, ) ( []*wallet.Message, serviceWithdrawal, error, ) { deposit := task.From.ToTonutilsAddressStd(0) jettonWallet, err := p.bc.GetJettonWalletAddress( ctx, deposit, task.JettonMaster.ToTonutilsAddressStd(0)) if err != nil { return nil, serviceWithdrawal{}, err } jettonBalance, err := p.bc.GetLastJettonBalance(ctx, jettonWallet) if err != nil { return nil, serviceWithdrawal{}, err } if jettonBalance.Cmp(big.NewInt(0)) == 0 { audit.Log(audit.Warning, string(TonDepositWallet), ServiceWithdrawalEvent, fmt.Sprintf("zero balance of Jettons %s on TON deposit address %s", task.JettonMaster.ToTonutilsAddressStd(0).String(), TonutilsAddressToUserFormat(deposit))) return nil, serviceWithdrawal{ TonAmount: ZeroCoins(), Task: task, }, nil } msg := buildTonFillMessage(deposit, config.JettonTransferTonAmount, task.Memo) task.JettonAmount = NewCoins(jettonBalance) return []*wallet.Message{msg}, serviceWithdrawal{ TonAmount: ZeroCoins(), Task: task, Filled: true, }, nil } func (p *WithdrawalsProcessor) buildServiceTonWithdrawal( ctx context.Context, task ServiceWithdrawalTask, ) ( []*wallet.Message, serviceWithdrawal, error, ) { proxy, err := NewJettonProxy(task.SubwalletID, p.wallets.TonHotWallet.Address()) if err != nil { return nil, serviceWithdrawal{}, err } tonBalance, _, err := p.bc.GetAccountCurrentState(ctx, proxy.address) if err != nil { return nil, serviceWithdrawal{}, err } res := serviceWithdrawal{ TonAmount: NewCoins(tonBalance), Task: task, } if tonBalance.Cmp(big.NewInt(0)) == 0 { audit.Log(audit.Warning, string(JettonOwner), ServiceWithdrawalEvent, fmt.Sprintf("zero balance of TONs on proxy address %s", TonutilsAddressToUserFormat(proxy.address))) return nil, res, nil } msg := buildJettonProxyServiceTonWithdrawalMessage(*proxy, p.wallets.TonHotWallet.Address(), task.Memo) return []*wallet.Message{msg}, res, nil } func (p *WithdrawalsProcessor) buildServiceJettonWithdrawal( ctx context.Context, task ServiceWithdrawalTask, ) ( []*wallet.Message, serviceWithdrawal, error, ) { proxy, err := NewJettonProxy(task.SubwalletID, p.wallets.TonHotWallet.Address()) if err != nil { return nil, serviceWithdrawal{}, err } jettonWallet, err := p.bc.GetJettonWalletAddress(ctx, proxy.address, task.JettonMaster.ToTonutilsAddressStd(0)) if err != nil { return nil, serviceWithdrawal{}, err } t, ok := p.db.GetWalletTypeByTonutilsAddress(jettonWallet) if ok { audit.Log(audit.Warning, string(JettonOwner), ServiceWithdrawalEvent, fmt.Sprintf("service withdrawal from known internal %s address %s rejected", t, TonutilsAddressToUserFormat(jettonWallet))) return nil, serviceWithdrawal{ TonAmount: ZeroCoins(), Task: task, }, nil } jettonBalance, err := p.bc.GetLastJettonBalance(ctx, jettonWallet) if err != nil { return nil, serviceWithdrawal{}, err } if jettonBalance.Cmp(big.NewInt(0)) == 0 { audit.Log(audit.Warning, string(JettonOwner), ServiceWithdrawalEvent, fmt.Sprintf("zero %s Jetton balance on proxy address %s", task.JettonMaster.ToTonutilsAddressStd(0).String(), TonutilsAddressToUserFormat(proxy.address))) return nil, serviceWithdrawal{ TonAmount: ZeroCoins(), Task: task, }, nil } task.JettonAmount = NewCoins(jettonBalance) res := serviceWithdrawal{ TonAmount: ZeroCoins(), Task: task, } msg := BuildJettonProxyWithdrawalMessage( *proxy, jettonWallet, p.wallets.TonHotWallet.Address(), tlb.FromNanoTONU(0), // zero forward amount to prevent notification sending and incorrect internal income invoking jettonBalance, task.Memo.String(), ) return []*wallet.Message{msg}, res, nil } func (p *WithdrawalsProcessor) buildExternalWithdrawalMessage(wt ExternalWithdrawalTask) *wallet.Message { if wt.Currency == TonSymbol { return BuildTonWithdrawalMessage(wt) } jw := p.wallets.JettonHotWallets[wt.Currency] return BuildJettonWithdrawalMessage(wt, p.wallets.TonHotWallet, jw.Address) } func (p *WithdrawalsProcessor) startExpirationProcessor() { log.Infof("Expiration processor started") defer p.wg.Done() for { p.waitSync() // gracefulShutdown break must be after waitSync if p.gracefulShutdown.Load() { log.Infof("Expiration processor stopped") break } ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) // must be < ExpirationProcessorPeriod err := p.db.SetExpired(ctx) if err != nil { log.Fatalf("set expired withdrawals error: %v", err) } cancel() time.Sleep(config.ExpirationProcessorPeriod) } } func (p *WithdrawalsProcessor) startInternalTonWithdrawalsProcessor() { defer p.wg.Done() log.Infof("Internal TON withdrawal processor started") for { p.waitSync() // gracefulShutdown break must be after waitSync if p.gracefulShutdown.Load() { log.Infof("Internal TON withdrawal processor stopped") break } ctx, cancel := context.WithTimeout(context.Background(), time.Second*120) // TODO: split context serviceTasks, err := p.db.GetServiceDepositWithdrawalTasks(ctx, 5) if err != nil { log.Fatalf("get service withdrawal tasks error: %v", err) } for _, task := range serviceTasks { err = p.serviceWithdrawJettons(ctx, task) if err != nil { log.Fatalf("Jettons service internal withdrawal error: %v", err) } time.Sleep(time.Millisecond * 50) } internalTasks, err := p.db.GetTonInternalWithdrawalTasks(ctx, 40) // context limitation if err != nil { log.Fatalf("get internal withdrawal tasks error: %v", err) } for _, task := range internalTasks { err = p.withdrawTONsFromDeposit(ctx, task) if err != nil { log.Fatalf("TONs internal withdrawal error: %v", err) } time.Sleep(time.Millisecond * 50) } cancel() time.Sleep(config.InternalWithdrawalPeriod) } } func (p *WithdrawalsProcessor) withdrawTONsFromDeposit(ctx context.Context, task InternalWithdrawalTask) error { subwallet, err := p.wallets.TonBasicWallet.GetSubwallet(task.SubwalletID) if err != nil { return err } spec := subwallet.GetSpec().(*wallet.SpecV3) spec.SetMessagesTTL(uint32(config.ExternalMessageLifetime.Seconds())) balance, state, err := p.bc.GetAccountCurrentState(ctx, subwallet.Address()) if err != nil { return err } if state == tlb.AccountStatusNonExist { return nil } if balance.Cmp(config.Config.Ton.Withdrawal) == 1 { // Balance > MinimalTonWithdrawalAmount memo, err := uuid.NewV4() if err != nil { return err } err = p.db.SaveInternalWithdrawalTask(ctx, task, time.Now().Add(config.ExternalMessageLifetime), memo) if err != nil { return err } // time.Now().Add(config.ExternalMessageLifetime) and real TTL // should be very close since the withdrawal occurs immediately err = WithdrawTONs(ctx, subwallet, p.wallets.TonHotWallet, memo.String()) if err != nil { audit.Log(audit.Info, string(TonDepositWallet), InternalWithdrawalEvent, fmt.Sprintf("TONs internal withdrawal from deposit %s error: %s", task.From.ToUserFormat(), err.Error())) } } return nil } func (p *WithdrawalsProcessor) serviceWithdrawJettons(ctx context.Context, task ServiceWithdrawalTask) error { subwallet, err := p.wallets.TonBasicWallet.GetSubwallet(task.SubwalletID) if err != nil { return err } spec := subwallet.GetSpec().(*wallet.SpecV3) spec.SetMessagesTTL(uint32(config.ExternalMessageLifetime.Seconds())) _, state, err := p.bc.GetAccountCurrentState(ctx, subwallet.Address()) if err != nil { return err } if state == tlb.AccountStatusNonExist { return nil } jettonWallet, err := p.bc.GetJettonWalletAddress(ctx, subwallet.Address(), task.JettonMaster.ToTonutilsAddressStd(0)) if err != nil { return err } err = p.db.UpdateServiceWithdrawalRequest(ctx, task, ZeroCoins(), time.Now().Add(config.ExternalMessageLifetime), false) if err != nil { return err } // time.Now().Add(config.ExternalMessageLifetime) and real TTL // should be very close since the withdrawal occurs immediately err = WithdrawJettons(ctx, subwallet, p.wallets.TonHotWallet, jettonWallet, tlb.FromNanoTONU(0), task.JettonAmount, task.Memo.String()) // zero forward TON amount to prevent notify message invoking if err != nil { log.Errorf("Jettons service withdrawal error: %v", err) audit.Log(audit.Info, string(TonDepositWallet), ServiceWithdrawalEvent, fmt.Sprintf("Jettons service withdrawal from deposit %s error: %s", task.From.ToUserFormat(), err.Error())) } return nil } func (p *WithdrawalsProcessor) waitSync() { for { if p.gracefulShutdown.Load() { log.Infof("WaitSync interrupted") break } ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) isSynced, _, err := p.db.IsActualBlockData(ctx) if err != nil { log.Fatalf("check sync error: %v", err) } if isSynced { cancel() break } cancel() time.Sleep(time.Second * 3) } } func (p *WithdrawalsProcessor) makeColdWalletWithdrawals(ctx context.Context) error { if p.coldWallet == nil { return nil } tonBalance, _, err := p.bc.GetAccountCurrentState(ctx, p.wallets.TonHotWallet.Address()) if err != nil { return err } dest := AddressMustFromTonutilsAddress(p.coldWallet) for cur, jw := range p.wallets.JettonHotWallets { inProgress, err := p.db.IsInProgressInternalWithdrawalRequest(ctx, dest, cur) if err != nil { return err } if inProgress { continue } jettonBalance, err := p.bc.GetLastJettonBalance(ctx, jw.Address) if err != nil { return err } if jettonBalance.Cmp(config.Config.Jettons[cur].HotWalletMaxCutoff) != 1 { // jettonBalance <= HotWalletMaxCutoff continue } jettonAmount := big.NewInt(0) u, err := uuid.NewV4() if err != nil { return err } jettonAmount.Sub(jettonBalance, config.Config.Jettons[cur].HotWalletResidual) tonBalance.Sub(tonBalance, config.JettonTransferTonAmount.Nano()) req := WithdrawalRequest{ Currency: jw.Currency, Amount: NewCoins(jettonAmount), Bounceable: true, Destination: dest, IsInternal: true, QueryID: u.String(), } _, err = p.db.SaveWithdrawalRequest(ctx, req) if err != nil { return err } log.Infof("%v withdrawal to cold wallet saved", cur) } inProgress, err := p.db.IsInProgressInternalWithdrawalRequest(ctx, dest, TonSymbol) if err != nil { return err } if inProgress { return nil } if tonBalance.Cmp(config.Config.Ton.HotWalletMax) != 1 { // tonBalance <= HotWalletMax return nil } tonAmount := big.NewInt(0) u, err := uuid.NewV4() if err != nil { return err } tonAmount.Sub(tonBalance, config.Config.Ton.HotWalletResidual) req := WithdrawalRequest{ Currency: TonSymbol, Amount: NewCoins(tonAmount), Bounceable: p.coldWallet.IsBounceable(), Destination: dest, IsInternal: true, QueryID: u.String(), } _, err = p.db.SaveWithdrawalRequest(ctx, req) if err != nil { return err } log.Infof("TON withdrawal to cold wallet saved") return nil } ================================================ FILE: db/db.go ================================================ package db import ( "context" "errors" "fmt" "strings" "sync" "time" "unicode/utf8" "github.com/gobicycle/bicycle/audit" "github.com/gobicycle/bicycle/config" "github.com/gobicycle/bicycle/core" "github.com/gofrs/uuid" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" log "github.com/sirupsen/logrus" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/ton" "github.com/xssnick/tonutils-go/ton/wallet" ) type Connection struct { client *pgxpool.Pool addressBook addressBook } type addressBook struct { addresses map[core.Address]core.AddressInfo mutex sync.Mutex } func (ab *addressBook) get(address core.Address) (core.AddressInfo, bool) { ab.mutex.Lock() t, ok := ab.addresses[address] ab.mutex.Unlock() return t, ok } func (ab *addressBook) put(address core.Address, t core.AddressInfo) { ab.mutex.Lock() ab.addresses[address] = t ab.mutex.Unlock() } func NewConnection(URI string) (*Connection, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) defer cancel() client, err := pgxpool.Connect(ctx, URI) if err != nil { return nil, fmt.Errorf("connection err: %v", err) } conn := Connection{client, addressBook{}} return &conn, nil } func (c *Connection) GetWalletType(address core.Address) (core.WalletType, bool) { info, ok := c.addressBook.get(address) return info.Type, ok } func (c *Connection) GetUserID(address core.Address) (string, bool) { info, ok := c.addressBook.get(address) return info.UserID, ok } // GetOwner returns owner for jetton deposit from address book and nil for other types func (c *Connection) GetOwner(address core.Address) *core.Address { info, ok := c.addressBook.get(address) if ok && info.Type == core.JettonDepositWallet && info.Owner == nil { log.Fatalf("must be owner address in address book for jetton deposit") } return info.Owner } func (c *Connection) GetWalletTypeByTonutilsAddress(address *address.Address) (core.WalletType, bool) { a, err := core.AddressFromTonutilsAddress(address) if err != nil { return "", false } return c.GetWalletType(a) } // GetLastSubwalletID returns last (greatest) used subwallet_id from DB // numeration starts from wallet.DefaultSubwallet (this number reserved for main hot wallet) // returns wallet.DefaultSubwallet if table is empty func (c *Connection) GetLastSubwalletID(ctx context.Context) (uint32, error) { var id uint32 err := c.client.QueryRow(ctx, ` SELECT COALESCE(MAX(subwallet_id), $1) FROM payments.ton_wallets `, wallet.DefaultSubwallet).Scan(&id) return id, err } func (c *Connection) SaveTonWallet(ctx context.Context, walletData core.WalletData) error { _, err := c.client.Exec(ctx, ` INSERT INTO payments.ton_wallets ( user_id, subwallet_id, type, address) VALUES ($1, $2, $3,$4) `, walletData.UserID, walletData.SubwalletID, walletData.Type, walletData.Address, ) if err != nil { return err } c.addressBook.put(walletData.Address, core.AddressInfo{Type: walletData.Type, Owner: nil, UserID: walletData.UserID}) return nil } func (c *Connection) GetJettonWallet(ctx context.Context, address core.Address) (*core.WalletData, bool, error) { d := core.WalletData{ Address: address, } err := c.client.QueryRow(ctx, ` SELECT subwallet_id, user_id, currency, type FROM payments.jetton_wallets WHERE address = $1 `, address).Scan(&d.SubwalletID, &d.UserID, &d.Currency, &d.Type) if errors.Is(err, pgx.ErrNoRows) { return nil, false, nil } if err != nil { return nil, false, err } return &d, true, nil } func (c *Connection) SaveJettonWallet( ctx context.Context, ownerAddress core.Address, walletData core.WalletData, notSaveOwner bool, ) error { tx, err := c.client.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) if !notSaveOwner { _, err = tx.Exec(ctx, ` INSERT INTO payments.ton_wallets ( user_id, subwallet_id, type, address) VALUES ($1, $2, $3,$4) `, walletData.UserID, walletData.SubwalletID, core.JettonOwner, ownerAddress, ) if err != nil { return err } } _, err = tx.Exec(ctx, ` INSERT INTO payments.jetton_wallets ( user_id, subwallet_id, currency, type, address) VALUES ($1, $2, $3, $4, $5) `, walletData.UserID, walletData.SubwalletID, walletData.Currency, walletData.Type, walletData.Address, ) if err != nil { return err } err = tx.Commit(ctx) if err != nil { return err } if walletData.Type == core.JettonDepositWallet { // only jetton deposit owners tracked by address book // hot TON wallet also owner of jetton hot wallets // cold wallets excluded from address book c.addressBook.put(ownerAddress, core.AddressInfo{Type: core.JettonOwner, Owner: nil, UserID: walletData.UserID}) } c.addressBook.put(walletData.Address, core.AddressInfo{Type: walletData.Type, Owner: &ownerAddress, UserID: walletData.UserID}) return nil } func (c *Connection) GetTonWalletsAddresses( ctx context.Context, userID string, types []core.WalletType, ) ( []core.Address, error, ) { if types == nil { types = make([]core.WalletType, 0) } rows, err := c.client.Query(ctx, ` SELECT address FROM payments.ton_wallets WHERE user_id = $1 AND type=ANY($2) `, userID, types) if err != nil { return nil, err } defer rows.Close() var res []core.Address for rows.Next() { var a core.Address err = rows.Scan(&a) if err != nil { return nil, err } res = append(res, a) } if rows.Err() != nil { return nil, rows.Err() } return res, nil } func (c *Connection) GetJettonOwnersAddresses( ctx context.Context, userID string, types []core.WalletType, ) ( []core.OwnerWallet, error, ) { if types == nil { types = make([]core.WalletType, 0) } rows, err := c.client.Query(ctx, ` SELECT tw.address, jw.currency FROM payments.jetton_wallets jw LEFT JOIN payments.ton_wallets tw ON jw.subwallet_id = tw.subwallet_id WHERE jw.user_id = $1 AND jw.type=ANY($2) `, userID, types) if err != nil { return nil, err } defer rows.Close() var res []core.OwnerWallet for rows.Next() { var ow core.OwnerWallet err = rows.Scan(&ow.Address, &ow.Currency) if err != nil { return nil, err } res = append(res, ow) } if rows.Err() != nil { return nil, rows.Err() } return res, nil } func (c *Connection) LoadAddressBook(ctx context.Context) error { res := make(map[core.Address]core.AddressInfo) var ( addr core.Address t core.WalletType userID string ) rows, err := c.client.Query(ctx, ` SELECT address, type, user_id FROM payments.ton_wallets `) if err != nil { return err } defer rows.Close() for rows.Next() { err = rows.Scan(&addr, &t, &userID) if err != nil { return err } res[addr] = core.AddressInfo{Type: t, Owner: nil, UserID: userID} } if rows.Err() != nil { return rows.Err() } rows, err = c.client.Query(ctx, ` SELECT jw.address, jw.type, tw.address, jw.user_id FROM payments.jetton_wallets jw LEFT JOIN payments.ton_wallets tw ON jw.subwallet_id = tw.subwallet_id `) if err != nil { return err } defer rows.Close() for rows.Next() { var owner core.Address err = rows.Scan(&addr, &t, &owner, &userID) if err != nil { return err } res[addr] = core.AddressInfo{Type: t, Owner: &owner, UserID: userID} } if rows.Err() != nil { return rows.Err() } c.addressBook.addresses = res log.Info("Address book loaded") return nil } func stripInvalidUTF8(s string) string { b := []byte(s) out := b[:0] for len(b) > 0 { r, size := utf8.DecodeRune(b) if r != utf8.RuneError || size > 1 { out = append(out, b[:size]...) } b = b[size:] } return strings.Replace(string(out), "\x00", "", -1) // PostgreSQL doesn't support storing NULL (\0x00) characters in text fields } func saveExternalIncome(ctx context.Context, tx pgx.Tx, inc core.ExternalIncome) error { inc.Comment = stripInvalidUTF8(inc.Comment) // PostgreSQL doesn't support storing invalid utf-8 characters _, err := tx.Exec(ctx, ` INSERT INTO payments.external_incomes ( lt, utime, deposit_address, payer_address, amount, comment, payer_workchain, tx_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) `, inc.Lt, time.Unix(int64(inc.Utime), 0), inc.To, inc.From, inc.Amount, inc.Comment, inc.FromWorkchain, inc.TxHash, ) return err } func (c *Connection) saveInternalIncome(ctx context.Context, tx pgx.Tx, inc core.InternalIncome) error { memo, err := uuid.FromString(inc.Memo) if err != nil { return err } wType, ok := c.GetWalletType(inc.From) var from core.Address if ok && wType == core.JettonOwner { // convert jetton owner address to jetton wallet address err = tx.QueryRow(ctx, ` SELECT jw.address FROM payments.ton_wallets tw LEFT JOIN payments.jetton_wallets jw ON tw.subwallet_id = jw.subwallet_id WHERE tw.address = $1 `, inc.From).Scan(&from) if err != nil { return err } } else { from = inc.From } _, err = tx.Exec(ctx, ` INSERT INTO payments.internal_incomes ( lt, utime, deposit_address, amount, memo) VALUES ($1, $2, $3, $4, $5) `, inc.Lt, time.Unix(int64(inc.Utime), 0), from, inc.Amount, memo, ) return err } func (c *Connection) SaveWithdrawalRequest(ctx context.Context, w core.WithdrawalRequest) (int64, error) { var queryID int64 err := c.client.QueryRow(ctx, ` INSERT INTO payments.withdrawal_requests ( user_id, user_query_id, amount, currency, bounceable, dest_address, comment, is_internal, binary_comment ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING query_id `, w.UserID, w.QueryID, w.Amount, w.Currency, w.Bounceable, w.Destination, w.Comment, w.IsInternal, w.BinaryComment, ).Scan(&queryID) return queryID, err } func (c *Connection) SaveServiceWithdrawalRequest(ctx context.Context, w core.ServiceWithdrawalRequest) ( uuid.UUID, error, ) { var memo uuid.UUID err := c.client.QueryRow(ctx, ` INSERT INTO payments.service_withdrawal_requests ( from_address, jetton_master ) VALUES ($1, $2) RETURNING memo `, w.From, w.JettonMaster, ).Scan(&memo) return memo, err } func (c *Connection) UpdateServiceWithdrawalRequest( ctx context.Context, t core.ServiceWithdrawalTask, tonAmount core.Coins, expiredAt time.Time, filled bool, ) error { _, err := c.client.Exec(ctx, ` UPDATE payments.service_withdrawal_requests SET ton_amount = $1, jetton_amount = $2, processed = not $4, expired_at = $3, filled = $4 WHERE memo = $5 `, tonAmount, t.JettonAmount, expiredAt, filled, t.Memo) return err } func (c *Connection) IsWithdrawalRequestUnique(ctx context.Context, w core.WithdrawalRequest) (bool, error) { var queryID int64 err := c.client.QueryRow(ctx, ` SELECT query_id FROM payments.withdrawal_requests WHERE user_id = $1 AND user_query_id = $2 AND is_internal = false `, w.UserID, w.QueryID, ).Scan(&queryID) if errors.Is(err, pgx.ErrNoRows) { return true, nil } if err != nil { return false, err } return false, nil } func (c *Connection) GetExternalWithdrawalTasks(ctx context.Context, limit int) ([]core.ExternalWithdrawalTask, error) { var res []core.ExternalWithdrawalTask rows, err := c.client.Query(ctx, ` SELECT DISTINCT ON (dest_address) dest_address, query_id, currency, bounceable, comment, amount, binary_comment FROM payments.withdrawal_requests WHERE processing = false ORDER BY dest_address, query_id LIMIT $1 `, limit) if err != nil { return nil, err } defer rows.Close() for rows.Next() { var w core.ExternalWithdrawalTask err = rows.Scan(&w.Destination, &w.QueryID, &w.Currency, &w.Bounceable, &w.Comment, &w.Amount, &w.BinaryComment) if err != nil { return nil, err } res = append(res, w) } if rows.Err() != nil { return nil, rows.Err() } return res, nil } // GetServiceHotWithdrawalTasks return tasks for Hot wallet withdrawals func (c *Connection) GetServiceHotWithdrawalTasks(ctx context.Context, limit int) ([]core.ServiceWithdrawalTask, error) { var tasks []core.ServiceWithdrawalTask rows, err := c.client.Query(ctx, ` SELECT DISTINCT ON (from_address) swr.from_address, swr.memo, swr.jetton_master, tw.subwallet_id FROM payments.service_withdrawal_requests swr LEFT JOIN payments.ton_wallets tw ON swr.from_address = tw.address WHERE processed = false and type = ANY($1) and filled = false ORDER BY from_address LIMIT $2 `, []core.WalletType{core.JettonOwner, core.TonDepositWallet}, limit) if err != nil { return nil, err } defer rows.Close() for rows.Next() { var w core.ServiceWithdrawalTask err = rows.Scan(&w.From, &w.Memo, &w.JettonMaster, &w.SubwalletID) if err != nil { return nil, err } tasks = append(tasks, w) } if rows.Err() != nil { return nil, rows.Err() } return tasks, nil } // GetServiceDepositWithdrawalTasks return tasks for TON deposit wallets func (c *Connection) GetServiceDepositWithdrawalTasks(ctx context.Context, limit int) ([]core.ServiceWithdrawalTask, error) { var tasks []core.ServiceWithdrawalTask rows, err := c.client.Query(ctx, ` SELECT DISTINCT ON (from_address) swr.from_address, swr.memo, swr.jetton_master, swr.jetton_amount, tw.subwallet_id FROM payments.service_withdrawal_requests swr LEFT JOIN payments.ton_wallets tw ON swr.from_address = tw.address WHERE processed = false AND filled = true AND type = $1 ORDER BY from_address LIMIT $2 `, core.TonDepositWallet, limit) if err != nil { return nil, err } defer rows.Close() for rows.Next() { var w core.ServiceWithdrawalTask err = rows.Scan(&w.From, &w.Memo, &w.JettonMaster, &w.JettonAmount, &w.SubwalletID) if err != nil { return nil, err } tasks = append(tasks, w) } if rows.Err() != nil { return nil, rows.Err() } return tasks, nil } func saveBlock(ctx context.Context, tx pgx.Tx, block core.ShardBlockHeader) error { _, err := tx.Exec(ctx, ` INSERT INTO payments.block_data ( shard, seqno, root_hash, file_hash, gen_utime ) VALUES ($1, $2, $3, $4, $5) `, block.Shard, block.SeqNo, block.RootHash, block.FileHash, time.Unix(int64(block.GenUtime), 0), ) return err } func updateInternalWithdrawal(ctx context.Context, tx pgx.Tx, w core.InternalWithdrawal) error { memo, err := uuid.FromString(w.Memo) if err != nil { return err } var ( sendingLt *int64 alreadyFailed bool ) err = tx.QueryRow(ctx, ` SELECT failed, sending_lt FROM payments.internal_withdrawals WHERE memo = $1 `, memo).Scan(&alreadyFailed, &sendingLt) if alreadyFailed { audit.Log(audit.Error, "internal withdrawal message", core.InternalWithdrawalEvent, fmt.Sprintf("successful withdrawal for expired internal withdrawal message. memo: %v", w.Memo)) return fmt.Errorf("invalid behavior of the expiration processor") } if sendingLt == nil { audit.Log(audit.Error, "internal withdrawal message", core.InternalWithdrawalEvent, fmt.Sprintf("successful withdrawal without sending confirmation. memo: %v", w.Memo)) return fmt.Errorf("invalid event order") } if w.IsFailed { _, err = tx.Exec(ctx, ` UPDATE payments.internal_withdrawals SET failed = true WHERE memo = $1 `, memo) return err } _, err = tx.Exec(ctx, ` UPDATE payments.internal_withdrawals SET finish_lt = $1, finished_at = $2, amount = amount + $3 WHERE memo = $4 `, w.Lt, time.Unix(int64(w.Utime), 0), w.Amount, memo) return err } func (c *Connection) SaveInternalWithdrawalTask( ctx context.Context, task core.InternalWithdrawalTask, expiredAt time.Time, memo uuid.UUID, ) error { _, err := c.client.Exec(ctx, ` INSERT INTO payments.internal_withdrawals ( since_lt, from_address, expired_at, memo ) VALUES ($1, $2, $3, $4) `, task.Lt, task.From, expiredAt, memo, ) return err } func (c *Connection) SaveParsedBlockData(ctx context.Context, events core.BlockEvents) error { tx, err := c.client.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) for _, ei := range events.ExternalIncomes { err = saveExternalIncome(ctx, tx, ei) if err != nil { return err } } for _, ii := range events.InternalIncomes { err = c.saveInternalIncome(ctx, tx, ii) if err != nil { return err } } for _, sc := range events.SendingConfirmations { err = applySendingConfirmations(ctx, tx, sc) if err != nil { return err } } for _, iw := range events.InternalWithdrawals { err = updateInternalWithdrawal(ctx, tx, iw) if err != nil { return err } } for _, ew := range events.ExternalWithdrawals { err = updateExternalWithdrawal(ctx, tx, ew) if err != nil { return err } } for _, wc := range events.WithdrawalConfirmations { err = applyJettonWithdrawalConfirmation(ctx, tx, wc) if err != nil { return err } } err = saveBlock(ctx, tx, events.Block) if err != nil { return err } err = tx.Commit(ctx) return err } func (c *Connection) GetTonInternalWithdrawalTasks(ctx context.Context, limit int) ([]core.InternalWithdrawalTask, error) { var tasks []core.InternalWithdrawalTask // lt > finish_lt condition because all TONs withdraws rows, err := c.client.Query(ctx, ` SELECT deposit_address, MAX(lt) AS last_lt, tw.subwallet_id FROM payments.external_incomes di LEFT JOIN ( SELECT iw1.from_address, iw1.since_lt, iw1.finish_lt FROM payments.internal_withdrawals iw1 JOIN ( SELECT from_address, MAX(since_lt) AS max_since_lt FROM payments.internal_withdrawals WHERE failed = false GROUP BY from_address ) iw2 ON iw2.from_address = iw1.from_address AND iw2.max_since_lt = iw1.since_lt WHERE iw1.failed = false ) as iw3 ON from_address = deposit_address JOIN payments.ton_wallets tw ON di.deposit_address = tw.address WHERE ((since_lt IS NOT NULL AND finish_lt IS NOT NULL AND lt > finish_lt) OR (since_lt IS NULL)) AND type = $1 GROUP BY deposit_address, tw.subwallet_id ORDER BY MAX(di.amount) DESC LIMIT $2 `, core.TonDepositWallet, limit) if err != nil { return nil, err } defer rows.Close() for rows.Next() { var task core.InternalWithdrawalTask err = rows.Scan(&task.From, &task.Lt, &task.SubwalletID) if err != nil { return nil, err } task.Currency = core.TonSymbol tasks = append(tasks, task) } if rows.Err() != nil { return nil, rows.Err() } return tasks, nil } func (c *Connection) GetJettonInternalWithdrawalTasks( ctx context.Context, forbiddenAddresses []core.Address, limit int, ) ( []core.InternalWithdrawalTask, error, ) { var tasks []core.InternalWithdrawalTask excludedAddr := make([][]byte, 0) // it is important for 'deposit_address = ANY($2)' sql constraint for _, a := range forbiddenAddresses { excludedAddr = append(excludedAddr, a[:]) // array of core.Address not supported by driver } rows, err := c.client.Query(ctx, ` SELECT deposit_address, MAX(lt) AS last_lt, jw.subwallet_id, jw.currency FROM payments.external_incomes di LEFT JOIN ( SELECT iw1.from_address, iw1.since_lt, iw1.finish_lt FROM payments.internal_withdrawals iw1 JOIN ( SELECT from_address, MAX(since_lt) AS max_since_lt FROM payments.internal_withdrawals WHERE failed = false GROUP BY from_address ) iw2 ON iw2.from_address = iw1.from_address AND iw2.max_since_lt = iw1.since_lt WHERE iw1.failed = false ) as iw3 ON from_address = deposit_address JOIN payments.jetton_wallets jw ON di.deposit_address = jw.address LEFT JOIN payments.ton_wallets tw ON jw.subwallet_id = tw.subwallet_id WHERE ((since_lt IS NOT NULL AND lt > since_lt AND finish_lt IS NOT NULL) OR (since_lt IS NULL)) AND jw.type = $1 AND NOT tw.address = ANY($2) GROUP BY deposit_address, jw.subwallet_id, jw.currency ORDER BY MAX(di.amount) DESC LIMIT $3 `, core.JettonDepositWallet, excludedAddr, limit) if err != nil { return nil, err } defer rows.Close() for rows.Next() { var task core.InternalWithdrawalTask err = rows.Scan(&task.From, &task.Lt, &task.SubwalletID, &task.Currency) if err != nil { return nil, err } tasks = append(tasks, task) } if rows.Err() != nil { return nil, rows.Err() } return tasks, nil } func applyJettonWithdrawalConfirmation( ctx context.Context, tx pgx.Tx, confirm core.JettonWithdrawalConfirmation, ) error { _, err := tx.Exec(ctx, ` UPDATE payments.external_withdrawals SET confirmed = true WHERE query_id = $1 AND processed_lt IS NOT NULL `, confirm.QueryId) return err } func updateExternalWithdrawal(ctx context.Context, tx pgx.Tx, w core.ExternalWithdrawal) error { var queryID int64 var alreadyFailed bool err := tx.QueryRow(ctx, ` SELECT failed FROM payments.external_withdrawals WHERE msg_uuid = $1 AND address = $2 `, w.ExtMsgUuid, w.To).Scan(&alreadyFailed) if alreadyFailed { audit.Log(audit.Error, "external withdrawal message", core.ExternalWithdrawalEvent, fmt.Sprintf("successful withdrawal for expired external withdrawal message. msg uuid: %v", w.ExtMsgUuid.String())) return fmt.Errorf("invalid behavior of the expiration processor") } // if there was a transaction on the hot wallet but the message was not sent // set processed = true to prevent burn balance if w.IsFailed { err := tx.QueryRow(ctx, ` UPDATE payments.external_withdrawals SET failed = true, tx_hash = $1 WHERE msg_uuid = $2 AND address = $3 RETURNING query_id `, w.TxHash, w.ExtMsgUuid, w.To).Scan(&queryID) if err != nil { return err } _, err = tx.Exec(ctx, ` UPDATE payments.withdrawal_requests SET processed = true WHERE query_id = $1 `, queryID) return err } err = tx.QueryRow(ctx, ` UPDATE payments.external_withdrawals SET processed_lt = $1, processed_at = $2, tx_hash = $3 WHERE msg_uuid = $4 AND address = $5 RETURNING query_id `, w.Lt, time.Unix(int64(w.Utime), 0), w.TxHash, w.ExtMsgUuid, w.To).Scan(&queryID) if errors.Is(err, pgx.ErrNoRows) { audit.Log(audit.Error, "external withdrawal message", core.ExternalWithdrawalEvent, fmt.Sprintf("successful withdrawal not linked to any known withdrawal request; possibly a manual hot wallet withdrawal. tx hash: %x", w.TxHash)) return fmt.Errorf("anomalous behavior of the TON hot wallet") } if err != nil { return err } _, err = tx.Exec(ctx, ` UPDATE payments.withdrawal_requests SET processed = true WHERE query_id = $1 `, queryID) return err } func applySendingConfirmations(ctx context.Context, tx pgx.Tx, w core.SendingConfirmation) error { var alreadyFailed bool memo, err := uuid.FromString(w.Memo) if err != nil { return err } err = tx.QueryRow(ctx, ` UPDATE payments.internal_withdrawals SET sending_lt = $1 WHERE memo = $2 RETURNING failed `, w.Lt, memo).Scan(&alreadyFailed) if err != nil { return err } if alreadyFailed { audit.Log(audit.Error, "internal withdrawal message", core.InternalWithdrawalEvent, fmt.Sprintf("successful sending for expired internal withdrawal message. memo: %v", w.Memo)) return fmt.Errorf("invalid behavior of the expiration processor") } return err } func (c *Connection) CreateExternalWithdrawals( ctx context.Context, tasks []core.ExternalWithdrawalTask, extMsgUuid uuid.UUID, expiredAt time.Time, ) error { tx, err := c.client.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) for _, t := range tasks { _, err = tx.Exec(ctx, ` INSERT INTO payments.external_withdrawals ( msg_uuid, query_id, expired_at, address ) VALUES ($1, $2, $3, $4) `, extMsgUuid, t.QueryID, expiredAt, t.Destination) if err != nil { return err } _, err = tx.Exec(ctx, ` UPDATE payments.withdrawal_requests SET processing = true WHERE query_id = $1 `, t.QueryID) if err != nil { return err } } return tx.Commit(ctx) } func (c *Connection) GetTonHotWalletAddress(ctx context.Context) (core.Address, error) { var addr core.Address err := c.client.QueryRow(ctx, ` SELECT address FROM payments.ton_wallets WHERE TYPE = $1 `, core.TonHotWallet).Scan(&addr) if errors.Is(err, pgx.ErrNoRows) { err = core.ErrNotFound } return addr, err } func (c *Connection) GetLastSavedBlockID(ctx context.Context) (*ton.BlockIDExt, error) { var blockID ton.BlockIDExt err := c.client.QueryRow(ctx, ` SELECT seqno, shard, root_hash, file_hash FROM payments.block_data ORDER BY seqno DESC LIMIT 1 `).Scan( &blockID.SeqNo, &blockID.Shard, &blockID.RootHash, &blockID.FileHash, ) if errors.Is(err, pgx.ErrNoRows) { return nil, core.ErrNotFound } if err != nil { return nil, err } blockID.Workchain = core.DefaultWorkchain return &blockID, nil } // SetExpired TODO: maybe add block related expiration func (c *Connection) SetExpired(ctx context.Context) error { _, err := c.client.Exec(ctx, ` UPDATE payments.internal_withdrawals SET failed = true WHERE expired_at < $1 AND sending_lt IS NULL AND failed = false `, time.Now().Add(-config.AllowableBlockchainLagging)) if err != nil { return err } tx, err := c.client.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) // processed_lt IS NULL AND failed = false - for lost external messages rows, err := tx.Query(ctx, ` UPDATE payments.external_withdrawals SET failed = true WHERE expired_at < $1 AND processed_lt IS NULL AND failed = false RETURNING query_id `, time.Now().Add(-config.AllowableBlockchainLagging)) if err != nil { return err } defer rows.Close() var ids []int64 for rows.Next() { var queryID int64 err = rows.Scan(&queryID) if err != nil { return err } ids = append(ids, queryID) } if rows.Err() != nil { return rows.Err() } for _, id := range ids { _, err = tx.Exec(ctx, ` UPDATE payments.withdrawal_requests SET processing = false WHERE query_id = $1 `, id) if err != nil { return err } } return tx.Commit(ctx) } func (c *Connection) IsActualBlockData(ctx context.Context) (bool, int64, error) { var lastBlockTime time.Time err := c.client.QueryRow(ctx, ` SELECT gen_utime FROM payments.block_data ORDER BY seqno DESC LIMIT 1 `).Scan(&lastBlockTime) if errors.Is(err, pgx.ErrNoRows) { return false, 0, nil } if err != nil { return false, 0, err } return time.Since(lastBlockTime) < config.AllowableBlockchainLagging, lastBlockTime.Unix(), nil } func (c *Connection) IsInProgressInternalWithdrawalRequest( ctx context.Context, dest core.Address, currency string, ) ( bool, error, ) { var queryID int64 err := c.client.QueryRow(ctx, ` SELECT query_id FROM payments.withdrawal_requests WHERE dest_address = $1 AND currency = $2 AND is_internal = true AND processed = false LIMIT 1 `, dest, currency, ).Scan(&queryID) if errors.Is(err, pgx.ErrNoRows) { return false, nil } if err != nil { return false, err } return true, nil } // GetExternalWithdrawalStatus returns status and hash of transaction for external withdrawal func (c *Connection) GetExternalWithdrawalStatus(ctx context.Context, id int64) (core.WithdrawalData, error) { var ( processing, processed bool data core.WithdrawalData ) err := c.client.QueryRow(ctx, ` SELECT processing, processed, user_id, user_query_id FROM payments.withdrawal_requests WHERE query_id = $1 AND is_internal = false LIMIT 1 `, id).Scan(&processing, &processed, &data.UserID, &data.QueryID) if errors.Is(err, pgx.ErrNoRows) { return core.WithdrawalData{}, core.ErrNotFound } if err != nil { return core.WithdrawalData{}, err } if processing && processed { var ( txHash []byte isFailed bool ) // must be only one record // OR processed_lt IS NOT NULL for DB up to 0.5.0 version err = c.client.QueryRow(ctx, ` SELECT tx_hash, failed FROM payments.external_withdrawals WHERE query_id = $1 AND (tx_hash IS NOT NULL OR processed_lt IS NOT NULL) LIMIT 1 `, id).Scan(&txHash, &isFailed) if err != nil { return core.WithdrawalData{}, err } if isFailed { data.Status = core.FailedStatus data.TxHash = txHash return data, nil } data.Status = core.ProcessedStatus data.TxHash = txHash return data, nil } else if processing && !processed { data.Status = core.ProcessingStatus return data, nil } else if !processing && !processed { data.Status = core.PendingStatus return data, nil } return core.WithdrawalData{}, fmt.Errorf("bad status") } // GetIncome returns list of incomes by user_id func (c *Connection) GetIncome( ctx context.Context, userID string, isDepositSide bool, ) ( []core.TotalIncome, error, ) { var sqlStatement string if isDepositSide { sqlStatement = ` SELECT COALESCE(jw.address,tw.address) as deposit, COALESCE(SUM(i.amount),0) as balance, COALESCE(jw.currency,$1) as currency FROM payments.ton_wallets tw LEFT JOIN payments.jetton_wallets jw ON jw.subwallet_id = tw.subwallet_id LEFT JOIN payments.external_incomes i ON i.deposit_address = COALESCE(jw.address,tw.address) WHERE tw.user_id = $2 AND tw.type = ANY($3) GROUP BY deposit, tw.address, jw.currency ` } else { sqlStatement = ` SELECT COALESCE(jw.address,tw.address) as deposit, COALESCE(SUM(i.amount),0) as balance, COALESCE(jw.currency,$1) as currency FROM payments.ton_wallets tw LEFT JOIN payments.jetton_wallets jw ON jw.subwallet_id = tw.subwallet_id LEFT JOIN payments.internal_incomes i ON i.deposit_address = COALESCE(jw.address,tw.address) WHERE tw.user_id = $2 AND tw.type = ANY($3) GROUP BY deposit, tw.address, jw.currency ` } rows, err := c.client.Query( ctx, sqlStatement, core.TonSymbol, userID, []core.WalletType{core.TonDepositWallet, core.JettonOwner}, ) if err != nil { return nil, err } defer rows.Close() res := make([]core.TotalIncome, 0) for rows.Next() { var deposit core.TotalIncome err = rows.Scan(&deposit.Deposit, &deposit.Amount, &deposit.Currency) if err != nil { return nil, err } res = append(res, deposit) } if rows.Err() != nil { return nil, rows.Err() } return res, nil } // GetIncomeHistory returns list of external incomes for deposit side by user_id and currency func (c *Connection) GetIncomeHistory( ctx context.Context, userID string, currency string, limit int, offset int, ascOrder bool, ) ( []core.ExternalIncome, error, ) { var ( res []core.ExternalIncome sqlStatement string walletType core.WalletType ) order := "DESC" if ascOrder { order = "ASC" } if currency == core.TonSymbol { sqlStatement = fmt.Sprintf(` SELECT utime, lt, payer_address, deposit_address, amount, comment, payer_workchain, tx_hash FROM payments.external_incomes i LEFT JOIN payments.ton_wallets tw ON i.deposit_address = tw.address WHERE tw.type = $1 AND tw.user_id = $2 AND $3 = $3 ORDER BY lt %s LIMIT $4 OFFSET $5 `, order) walletType = core.TonDepositWallet } else { sqlStatement = fmt.Sprintf(` SELECT utime, lt, payer_address, deposit_address, amount, comment, payer_workchain, tx_hash FROM payments.external_incomes i LEFT JOIN payments.jetton_wallets jw ON i.deposit_address = jw.address WHERE jw.type = $1 AND jw.user_id = $2 AND jw.currency = $3 ORDER BY lt %s LIMIT $4 OFFSET $5 `, order) walletType = core.JettonDepositWallet } rows, err := c.client.Query(ctx, sqlStatement, walletType, userID, currency, limit, offset) if err != nil { return nil, err } defer rows.Close() for rows.Next() { var ( income core.ExternalIncome t time.Time ) err = rows.Scan(&t, &income.Lt, &income.From, &income.To, &income.Amount, &income.Comment, &income.FromWorkchain, &income.TxHash) if err != nil { return nil, err } income.Utime = uint32(t.Unix()) res = append(res, income) } if rows.Err() != nil { return nil, rows.Err() } return res, nil } // GetIncomeByTx returns external income and currency for deposit side by transaction hash func (c *Connection) GetIncomeByTx( ctx context.Context, txHash []byte, ) ( *core.ExternalIncome, string, error, ) { var ( income core.ExternalIncome t time.Time ) err := c.client.QueryRow(ctx, ` SELECT utime, lt, payer_address, deposit_address, amount, comment, payer_workchain FROM payments.external_incomes i LEFT JOIN payments.ton_wallets tw ON i.deposit_address = tw.address WHERE tw.type = $1 AND i.tx_hash = $2 LIMIT 1 `, core.TonDepositWallet, txHash).Scan(&t, &income.Lt, &income.From, &income.To, &income.Amount, &income.Comment, &income.FromWorkchain) if errors.Is(err, pgx.ErrNoRows) { var currency string // amount > 0 means receiving an aggregated transaction for an unidentified jetton replenishment err = c.client.QueryRow(ctx, ` SELECT utime, lt, payer_address, deposit_address, amount, comment, payer_workchain, jw.currency FROM payments.external_incomes i LEFT JOIN payments.jetton_wallets jw ON i.deposit_address = jw.address WHERE jw.type = $1 AND i.tx_hash = $2 AND i.amount > 0 LIMIT 1 `, core.JettonDepositWallet, txHash).Scan(&t, &income.Lt, &income.From, &income.To, &income.Amount, &income.Comment, &income.FromWorkchain, ¤cy) if errors.Is(err, pgx.ErrNoRows) { return nil, "", core.ErrNotFound // not found } if err != nil { return nil, "", err } income.Utime = uint32(t.Unix()) income.TxHash = txHash return &income, currency, nil } if err != nil { return nil, "", err } income.Utime = uint32(t.Unix()) income.TxHash = txHash return &income, core.TonSymbol, nil } func (c *Connection) GetTotalWithdrawalAmounts(ctx context.Context, currency string) (*core.TotalWithdrawalsAmount, error) { tx, err := c.client.Begin(ctx) if err != nil { return nil, err } defer tx.Rollback(ctx) var totalAmounts core.TotalWithdrawalsAmount err = tx.QueryRow(ctx, ` SELECT COALESCE(SUM(amount), 0 ) as total_processing_amount FROM payments.withdrawal_requests WHERE currency = $1 AND processed = false AND processing = true `, currency).Scan( &totalAmounts.Processing, ) if err != nil { return nil, err } err = tx.QueryRow(ctx, ` SELECT COALESCE(SUM(amount), 0 ) as total_pending_amount FROM payments.withdrawal_requests WHERE currency = $1 AND processed = false AND processing = false `, currency).Scan( &totalAmounts.Pending, ) if err != nil { return nil, err } err = tx.Commit(ctx) if err != nil { return nil, err } return &totalAmounts, nil } ================================================ FILE: db/db_test.go ================================================ package db import ( "context" "encoding/hex" "github.com/gobicycle/bicycle/core" "os" "strings" "testing" "time" ) var dbURI string func execMultiStatement(c *Connection, ctx context.Context, query string) error { query = strings.TrimPrefix(query, "BEGIN;") query = strings.TrimSuffix(query, "COMMIT;") queries := strings.Split(query, ";") tx, err := c.client.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) for _, q := range queries { _, err := tx.Exec(ctx, q) if err != nil { return err } } err = tx.Commit(ctx) return err } func migrateUp(c *Connection, t *testing.T, source string) error { migrateDown(c, t) deploy, err := os.ReadFile("../deploy/db/01_init.up.sql") if err != nil { return err } err = execMultiStatement(c, context.Background(), string(deploy)) if err != nil { return err } test, err := os.ReadFile("tests/" + source + "/01_data.up.sql") if err != nil { return err } err = execMultiStatement(c, context.Background(), string(test)) if err != nil { return err } return nil } func migrateDown(c *Connection, t *testing.T) { drop, err := os.ReadFile("../deploy/db/01_init.down.sql") if err != nil { t.Fatal("migrate down err: ", err) } err = execMultiStatement(c, context.Background(), string(drop)) if err != nil { t.Fatal("migrate down err: ", err) } } func init() { dbURI = os.Getenv("DB_URI") if dbURI == "" { panic("empty db uri var") } } func connect(t *testing.T) *Connection { c, err := NewConnection(dbURI) if err != nil { t.Fatal("connections err: ", err) } return c } func Test_NewConnection(t *testing.T) { connect(t) } func Test_GetTonInternalWithdrawalTasks(t *testing.T) { c := connect(t) source := "get-ton-internal-withdrawal-tasks" err := migrateUp(c, t, source) if err != nil { t.Fatal("migrate up err: ", err) } defer migrateDown(c, t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() res, err := c.GetTonInternalWithdrawalTasks(ctx, 100) if err != nil { t.Fatal("get tasks err: ", err) } if len(res) != 1 { t.Fatal("only one task must be loaded") } if res[0].SubwalletID != 2 { t.Fatal("task must be loaded only for deposit A") } if res[0].Lt != 3 { t.Fatal("task must be loaded only for second payment") } } func Test_GetJettonInternalWithdrawalTasks(t *testing.T) { c := connect(t) source := "get-jetton-internal-withdrawal-tasks" err := migrateUp(c, t, source) if err != nil { t.Fatal("migrate up err: ", err) } defer migrateDown(c, t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() res, err := c.GetJettonInternalWithdrawalTasks(ctx, []core.Address{}, 250) if err != nil { t.Fatal("get tasks err: ", err) } if len(res) != 2 { t.Fatal("two tasks must be loaded") } if res[0].SubwalletID != 2 || res[1].SubwalletID != 4 { t.Fatal("tasks must be loaded only for deposits A and C") } if res[0].Lt != 2 { t.Fatal("task must be loaded only for second payment for deposit A") } if res[1].Lt != 1 { t.Fatal("task must be loaded only for first payment for deposit C") } } func Test_GetJettonInternalWithdrawalTasksForbidden(t *testing.T) { c := connect(t) source := "get-jetton-internal-withdrawal-tasks" err := migrateUp(c, t, source) if err != nil { t.Fatal("migrate up err: ", err) } defer migrateDown(c, t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() b, _ := hex.DecodeString("01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab3") // owner of Jetton deposit A var forbiddenAddress core.Address copy(forbiddenAddress[:], b) res, err := c.GetJettonInternalWithdrawalTasks(ctx, []core.Address{forbiddenAddress}, 250) if err != nil { t.Fatal("get tasks err: ", err) } if len(res) != 1 { t.Fatal("one tasks must be loaded") } if res[0].SubwalletID != 4 { t.Fatal("tasks must be loaded only for deposits C") } if res[0].Lt != 1 { t.Fatal("task must be loaded only for first payment for deposit C") } } func Test_SetExpired(t *testing.T) { type extResult struct { queryID int failed bool processing bool processed bool } externalResult := [7]extResult{ {1, true, false, false}, {2, true, false, false}, {3, true, false, false}, {4, false, true, false}, {5, true, false, false}, {6, false, true, true}, {7, false, true, true}, } internalResult := [6]bool{true, true, false, true, false, false} c := connect(t) source := "set-expired" err := migrateUp(c, t, source) if err != nil { t.Fatal("migrate up err: ", err) } defer migrateDown(c, t) ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) defer cancel() err = c.SetExpired(ctx) if err != nil { t.Fatal("set expired err: ", err) } // load external withdrawals data rows, err := c.client.Query(ctx, ` SELECT ew.query_id, failed, wr.processing, wr.processed FROM payments.external_withdrawals ew LEFT JOIN payments.withdrawal_requests wr ON wr.query_id = ew.query_id ORDER BY ew.query_id `) if err != nil { t.Fatal("get data err: ", err) } defer rows.Close() var ( extRes [7]extResult i = 0 ) for rows.Next() { var r extResult var err = rows.Scan(&r.queryID, &r.failed, &r.processing, &r.processed) if err != nil { t.Fatal("scan err: ", err) } extRes[i] = r i++ } if rows.Err() != nil { t.Fatal("rows err: ", rows.Err()) } if externalResult != extRes { t.Fatalf("invalid external result pattern: %v", extRes) } // load internal withdrawals data rows, err = c.client.Query(ctx, ` SELECT failed FROM payments.internal_withdrawals ORDER BY since_lt `) if err != nil { t.Fatal("get data err: ", err) } defer rows.Close() var intRes [6]bool i = 0 for rows.Next() { var r bool var err = rows.Scan(&r) if err != nil { t.Fatal("scan err: ", err) } intRes[i] = r i++ } if rows.Err() != nil { t.Fatal("rows err: ", rows.Err()) } if internalResult != intRes { t.Fatalf("invalid internal result pattern: %v", intRes) } } func TestStripInvalidUTF8(t *testing.T) { tests := []struct { input string expected string }{ { input: "hello world", expected: "hello world", }, { input: "привет мир", expected: "привет мир", }, { input: string([]byte{'h', 'e', 'l', 'l', 0xb3, 'o'}), expected: "hello", }, { input: string([]byte{0xff, 0xfe, 0xfd}), expected: "", }, { input: "valid \xf0\x9f\x98\x81 invalid \xff text", expected: "valid 😁 invalid text", }, } for i, tt := range tests { result := stripInvalidUTF8(tt.input) if result != tt.expected { t.Errorf("test %d failed: expected %q, got %q", i, tt.expected, result) } } } ================================================ FILE: db/tests/get-jetton-internal-withdrawal-tasks/01_data.up.sql ================================================ BEGIN; INSERT INTO payments.external_incomes ( lt, utime, deposit_address, payer_address, amount, comment ) VALUES ( -- first payment to TON deposit A 1, '2021-03-10 08:10:00 UTC', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab4', 'hex'), 123, 'test_comment_1' ), ( -- second payment to TON deposit A 2, '2021-03-10 08:11:00 UTC', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab5', 'hex'), 123, 'test_comment_2' ), ( -- first payment to TON deposit B 1, '2021-03-10 08:12:00 UTC', decode('01bb00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab6', 'hex'), 123, 'test_comment_3' ), ( -- first payment to Jetton deposit C 1, '2021-03-10 08:13:00 UTC', decode('01cc00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab7', 'hex'), 123, 'test_comment_4' ); INSERT INTO payments.internal_withdrawals ( failed, since_lt, finish_lt, created_at, finished_at, expired_at, amount, from_address, memo ) VALUES ( -- finished withdrawal from TON deposit A after first payment false, 1, 2, '2021-03-10 08:14:00 UTC', '2021-03-10 08:15:00 UTC', '2021-03-10 08:17:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7831' ), ( -- failed withdrawal from TON deposit A after second payment true, 2, NULL, '2021-03-10 08:16:00 UTC', NULL, '2021-03-10 08:19:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7832' ), ( -- not finished withdrawal from TON deposit B after payment false, 1, NULL, '2021-03-10 08:14:00 UTC', NULL, '2021-03-10 08:17:00 UTC', 100, decode('01bb00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7833' ); INSERT INTO payments.jetton_wallets ( subwallet_id, created_at, user_id, currency, type, address ) VALUES ( -- Jetton hot wallet currency A 1, '2021-03-10 08:00:00 UTC', '', 'A', 'jetton_hot', decode('00aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ), ( -- Jetton deposit A currency A 2, '2021-03-10 08:00:00 UTC', 'test_user', 'A', 'jetton_deposit', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ), ( -- Jetton deposit B currency B 3, '2021-03-10 08:00:00 UTC', 'test_user', 'B', 'jetton_deposit', decode('01bb00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ), ( -- Jetton deposit C currency C 4, '2021-03-10 08:00:00 UTC', 'test_user', 'C', 'jetton_deposit', decode('01cc00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ); INSERT INTO payments.ton_wallets ( subwallet_id, created_at, user_id, type, address ) VALUES ( -- TON hot wallet 1, '2021-03-10 08:00:00 UTC', '', 'ton_hot', decode('00aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab3', 'hex') ), ( -- Jetton deposit A owner 2, '2021-03-10 08:00:00 UTC', 'test_user', 'owner', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab3', 'hex') ), ( -- Jetton deposit B owner 3, '2021-03-10 08:00:00 UTC', 'test_user', 'owner', decode('01bb00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab3', 'hex') ), ( -- Jetton deposit C owner 4, '2021-03-10 08:00:00 UTC', 'test_user', 'owner', decode('01cc00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab3', 'hex') ); COMMIT; ================================================ FILE: db/tests/get-ton-internal-withdrawal-tasks/01_data.up.sql ================================================ BEGIN; INSERT INTO payments.external_incomes ( lt, utime, deposit_address, payer_address, amount, comment ) VALUES ( -- first payment to TON deposit A 1, '2021-03-10 08:10:00 UTC', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab4', 'hex'), 123, 'test_comment_1' ), ( -- second payment to TON deposit A 3, '2021-03-10 08:11:00 UTC', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab5', 'hex'), 123, 'test_comment_2' ), ( -- first payment to TON deposit B 1, '2021-03-10 08:12:00 UTC', decode('01bb00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab6', 'hex'), 123, 'test_comment_3' ), ( -- first payment to Jetton deposit C 1, '2021-03-10 08:13:00 UTC', decode('01cc00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab7', 'hex'), 123, 'test_comment_4' ); INSERT INTO payments.internal_withdrawals ( failed, since_lt, sending_lt, finish_lt, created_at, finished_at, expired_at, amount, from_address, memo ) VALUES ( -- finished withdrawal from TON deposit A after first payment false, 1, 2, 2, '2021-03-10 08:14:00 UTC', '2021-03-10 08:15:00 UTC', '2021-03-10 08:17:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7831' ), ( -- failed withdrawal from TON deposit A after second payment true, 2, 2, NULL, '2021-03-10 08:16:00 UTC', NULL, '2021-03-10 08:19:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7832' ), ( -- not finished withdrawal from TON deposit B after payment false, 1, 1, NULL, '2021-03-10 08:14:00 UTC', NULL, '2021-03-10 08:17:00 UTC', 100, decode('01bb00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7833' ); INSERT INTO payments.ton_wallets ( subwallet_id, created_at, user_id, type, address ) VALUES ( -- TON hot wallet 1, '2021-03-10 08:00:00 UTC', '', 'ton_hot', decode('00aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ), ( -- TON deposit A 2, '2021-03-10 08:00:00 UTC', 'test_user', 'ton_deposit', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ), ( -- TON deposit B 3, '2021-03-10 08:00:00 UTC', 'test_user', 'ton_deposit', decode('01bb00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ), ( -- Jetton deposit C 4, '2021-03-10 08:00:00 UTC', 'test_user', 'owner', decode('01cc00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ); COMMIT; ================================================ FILE: db/tests/set-expired/01_data.up.sql ================================================ BEGIN; INSERT INTO payments.external_withdrawals ( msg_uuid, query_id, created_at, expired_at, processed_at, processed_lt, confirmed, failed, address ) VALUES ( -- expired and not marked as expired 1st 'c2d29867-3d0b-d497-9191-18a9d8ee7831', 1, '2000-01-01 08:00:00 UTC', '2000-01-01 08:03:00 UTC', NULL, NULL, false, false, decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab1', 'hex') ), ( -- expired and not marked as expired 2nd 'c2d29867-3d0b-d497-9191-18a9d8ee7831', 2, '2000-01-01 08:00:00 UTC', '2000-01-01 08:04:00 UTC', NULL, NULL, false, false, decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ), ( -- expired and already marked as expired 'c2d29867-3d0b-d497-9191-18a9d8ee7831', 3, '2000-01-01 08:00:00 UTC', '2000-01-01 08:05:00 UTC', NULL, NULL, false, true, decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab3', 'hex') ), ( -- not expired 'c2d29867-3d0b-d497-9191-18a9d8ee7831', 4, '2000-01-01 08:00:00 UTC', '3000-01-01 08:00:00 UTC', NULL, NULL, false, false, decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab4', 'hex') ), ( -- not expired but failed 'c2d29867-3d0b-d497-9191-18a9d8ee7831', 5, '2000-01-01 08:00:00 UTC', '3000-01-01 08:00:00 UTC', NULL, NULL, false, true, decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab5', 'hex') ), ( -- not expired and already processed 'c2d29867-3d0b-d497-9191-18a9d8ee7831', 6, '2000-01-01 08:00:00 UTC', '3000-01-01 08:00:00 UTC', '2000-01-01 08:10:00 UTC', 1, false, false, decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab6', 'hex') ), ( -- expired and already processed 'c2d29867-3d0b-d497-9191-18a9d8ee7831', 7, '2000-01-01 08:00:00 UTC', '2000-01-01 08:03:00 UTC', '2000-01-01 08:01:00 UTC', 2, false, false, decode('007424ee4767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab7', 'hex') ); INSERT INTO payments.internal_withdrawals ( failed, since_lt, sending_lt, finish_lt, created_at, finished_at, expired_at, amount, from_address, memo ) VALUES ( -- expired and not marked as expired false, 1, NULL, NULL, '2020-03-10 08:00:00 UTC', NULL, '2020-03-10 08:03:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7831' ), ( -- expired and already marked as expired true, 2, NULL, NULL, '2020-03-10 08:00:00 UTC', NULL, '2020-03-10 08:03:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7832' ), ( -- not expired false, 3, 4, NULL, '2020-03-10 08:00:00 UTC', NULL, '3020-03-10 08:00:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7833' ), ( -- not expired but failed true, 4, 5, NULL, '2020-03-10 08:00:00 UTC', NULL, '3020-03-10 08:00:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7834' ), ( -- not expired but already processed false, 5, 6, 6, '2020-03-10 08:00:00 UTC', '2020-03-10 08:01:00 UTC', '3020-03-10 08:00:00 UTC', 100, decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7835' ), ( -- expired and already processed false, 6, 7, 7, '2020-03-10 08:00:00 UTC', '2020-03-10 08:01:00 UTC', '2020-03-10 08:03:00 UTC', 100, decode('01bb00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex'), 'c2d29867-3d0b-d497-9191-18a9d8ee7836' ); INSERT INTO payments.withdrawal_requests ( query_id, bounceable, processing, processed, is_internal, amount, user_id, user_query_id, currency, comment, dest_address ) VALUES ( -- request with query_id 1 for external_withdrawals 1, false, true, false, false, 100, 'test_user', '1', 'TON', 'test_comment', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab1', 'hex') ), ( -- request with query_id 2 for external_withdrawals 2, false, true, false, false, 100, 'test_user', '2', 'TON', 'test_comment', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab2', 'hex') ), ( -- request with query_id 3 for external_withdrawals 3, false, false, false, false, 100, 'test_user', '3', 'TON', 'test_comment', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab3', 'hex') ), ( -- request with query_id 4 for external_withdrawals 4, false, true, false, false, 100, 'test_user', '4', 'TON', 'test_comment', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab4', 'hex') ), ( -- request with query_id 5 for external_withdrawals 5, false, false, false, false, 100, 'test_user', '5', 'TON', 'test_comment', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab5', 'hex') ), ( -- request with query_id 6 for external_withdrawals 6, false, true, true, false, 100, 'test_user', '6', 'TON', 'test_comment', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab6', 'hex') ), ( -- request with query_id 7 for external_withdrawals 7, false, true, true, false, 100, 'test_user', '7', 'TON', 'test_comment', decode('01aa00004767fbcf859609200910269446980f4d27bd8f4e3faa6e4d74792ab7', 'hex') ); COMMIT; ================================================ FILE: deploy/db/01_init.down.sql ================================================ BEGIN; DROP TABLE IF EXISTS payments.ton_wallets; DROP TABLE IF EXISTS payments.jetton_wallets; DROP TABLE IF EXISTS payments.internal_incomes; DROP TABLE IF EXISTS payments.external_withdrawals; DROP TABLE IF EXISTS payments.withdrawal_requests; DROP TABLE IF EXISTS payments.external_incomes; DROP TABLE IF EXISTS payments.block_data; DROP TABLE IF EXISTS payments.internal_withdrawals; DROP TABLE IF EXISTS payments.service_withdrawal_requests; DROP SCHEMA IF EXISTS payments; COMMIT; ================================================ FILE: deploy/db/01_init.up.sql ================================================ BEGIN; CREATE SCHEMA IF NOT EXISTS payments; CREATE TABLE IF NOT EXISTS payments.ton_wallets ( subwallet_id bigint not null unique, -- store uint32 created_at timestamptz not null default now(), user_id text not null, type text not null, address bytea not null ); CREATE INDEX IF NOT EXISTS ton_wallets_address_index ON payments.ton_wallets (address); CREATE INDEX IF NOT EXISTS ton_wallets_type_index ON payments.ton_wallets (type); CREATE INDEX IF NOT EXISTS ton_wallets_user_id_index ON payments.ton_wallets (user_id); CREATE TABLE IF NOT EXISTS payments.jetton_wallets ( subwallet_id bigint not null, -- store uint32 created_at timestamptz not null default now(), user_id text not null, currency text not null, type text not null, address bytea not null unique ); CREATE INDEX IF NOT EXISTS jetton_wallets_subwallet_id_index ON payments.jetton_wallets (subwallet_id); CREATE TABLE IF NOT EXISTS payments.internal_incomes ( lt bigint not null, utime timestamptz not null, deposit_address bytea not null, amount numeric not null, memo uuid not null, unique (memo, lt) ); CREATE INDEX IF NOT EXISTS internal_incomes_deposit_address_index ON payments.internal_incomes (deposit_address); CREATE TABLE IF NOT EXISTS payments.external_withdrawals ( msg_uuid uuid not null, query_id bigint not null, created_at timestamptz not null default now(), expired_at timestamptz, processed_at timestamptz, processed_lt bigint, confirmed bool not null default false, failed bool not null default false, address bytea not null, tx_hash bytea, unique (msg_uuid, address) ); CREATE INDEX IF NOT EXISTS external_withdrawals_expired_at_index ON payments.external_withdrawals (expired_at); CREATE INDEX IF NOT EXISTS external_withdrawals_msg_uuid_index ON payments.external_withdrawals (msg_uuid); CREATE INDEX IF NOT EXISTS external_withdrawals_address_index ON payments.external_withdrawals (address); CREATE INDEX IF NOT EXISTS external_withdrawals_query_id_index ON payments.external_withdrawals (query_id); CREATE TABLE IF NOT EXISTS payments.withdrawal_requests ( query_id bigserial constraint withdrawal_requests_pk primary key, bounceable bool not null, processing bool not null default false, processed bool not null default false, is_internal bool default false, amount numeric not null, user_id text not null, user_query_id text not null, currency text not null, dest_address bytea not null, comment text, binary_comment text, unique (user_id, user_query_id, is_internal) ); CREATE INDEX IF NOT EXISTS withdrawal_requests_user_id_index ON payments.withdrawal_requests (user_id); CREATE INDEX IF NOT EXISTS withdrawal_requests_user_query_id_index ON payments.withdrawal_requests (user_query_id); CREATE INDEX IF NOT EXISTS withdrawal_requests_dest_address_index ON payments.withdrawal_requests (dest_address); CREATE TABLE IF NOT EXISTS payments.external_incomes ( lt bigint not null, utime timestamptz not null, payer_workchain integer, deposit_address bytea not null, payer_address bytea, amount numeric not null, comment text not null, tx_hash bytea ); CREATE INDEX IF NOT EXISTS external_incomes_deposit_address_index ON payments.external_incomes (deposit_address); CREATE TABLE IF NOT EXISTS payments.block_data ( saved_at timestamptz not null default now(), shard bigint not null, seqno bigint not null, gen_utime timestamptz not null, root_hash bytea not null, file_hash bytea not null, unique (shard, seqno) ); CREATE INDEX IF NOT EXISTS block_data_seqno_index ON payments.block_data (seqno); CREATE TABLE IF NOT EXISTS payments.internal_withdrawals ( failed bool not null default false, since_lt bigint not null, -- amount for this LT sending_lt bigint, -- on wallet side finish_lt bigint, -- on deposit side finished_at timestamptz, -- on deposit side created_at timestamptz not null default now(), expired_at timestamptz, amount numeric not null default 0, from_address bytea not null, memo uuid not null unique, unique (from_address, since_lt, memo) ); CREATE INDEX IF NOT EXISTS internal_withdrawals_from_address_index ON payments.internal_withdrawals (from_address); CREATE INDEX IF NOT EXISTS internal_withdrawals_since_lt_index ON payments.internal_withdrawals (since_lt); CREATE INDEX IF NOT EXISTS internal_withdrawals_expired_at_index ON payments.internal_withdrawals (expired_at); CREATE TABLE IF NOT EXISTS payments.service_withdrawal_requests ( memo uuid not null default gen_random_uuid() unique, created_at timestamptz not null default now(), expired_at timestamptz, filled bool not null default false, processed bool not null default false, ton_amount numeric not null default 0, jetton_amount numeric not null default 0, from_address bytea not null, jetton_master bytea ); COMMIT; ================================================ FILE: deploy/db/02_create_readonly_user.sh ================================================ #!/bin/bash if [ -z "$POSTGRES_READONLY_PASSWORD" ]; then echo "Environment variable POSTGRES_READONLY_PASSWORD is not set. Exiting." exit 1 fi psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL CREATE USER pp_readonly WITH PASSWORD '$POSTGRES_READONLY_PASSWORD'; GRANT CONNECT ON DATABASE $POSTGRES_DB TO pp_readonly; GRANT USAGE ON SCHEMA payments TO pp_readonly; GRANT SELECT ON ALL TABLES IN SCHEMA payments TO pp_readonly; ALTER DEFAULT PRIVILEGES IN SCHEMA payments GRANT SELECT ON TABLES TO pp_readonly; EOSQL ================================================ FILE: deploy/grafana/main/dashboards/Payments.json ================================================ { "__inputs": [ { "name": "DS_POSTGRES", "label": "Postgres", "description": "", "type": "datasource", "pluginId": "postgres", "pluginName": "PostgreSQL" } ], "__elements": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "8.3.6" }, { "type": "datasource", "id": "postgres", "name": "PostgreSQL", "version": "1.0.0" }, { "type": "panel", "id": "table", "name": "Table", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": true, "panels": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "saved_at" }, "properties": [ { "id": "custom.width", "value": 184 } ] }, { "matcher": { "id": "byName", "options": "seqno" }, "properties": [ { "id": "custom.width", "value": 210 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, "id": 2, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT saved_at, seqno, gen_utime\nFROM payments.block_data\nORDER BY seqno desc\nLIMIT 4", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "paymnets.block_data", "timeColumn": "time", "where": [] } ], "title": "Block data", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 71 } ] }, { "matcher": { "id": "byName", "options": "processing" }, "properties": [ { "id": "custom.width", "value": 89 } ] }, { "matcher": { "id": "byName", "options": "processed" }, "properties": [ { "id": "custom.width", "value": 84 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 104 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 78 } ] }, { "matcher": { "id": "byName", "options": "comment" }, "properties": [ { "id": "custom.width", "value": 358 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, "id": 4, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n query_id, processing, processed, amount, currency,comment\nFROM payments.withdrawal_requests\nORDER BY query_id DESC", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.withdrawal_requests", "timeColumn": "time", "where": [] } ], "title": "Withdrawal requests", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "utime" }, "properties": [ { "id": "custom.width", "value": 176 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 103 } ] }, { "matcher": { "id": "byName", "options": "comment" }, "properties": [ { "id": "custom.width", "value": 134 } ] }, { "matcher": { "id": "byName", "options": "subwallet_id" }, "properties": [ { "id": "custom.width", "value": 112 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 75 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, "id": 10, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n utime, amount, coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, coalesce(currency, 'TON') as currency, comment\nFROM\n payments.external_incomes ei\nLEFT JOIN payments.ton_wallets tw ON ei.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ei.deposit_address = jw.address\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.external_incomes", "timeColumn": "time", "where": [] } ], "title": "External imcomes", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "subwallet_id" }, "properties": [ { "id": "custom.width", "value": 105 } ] }, { "matcher": { "id": "byName", "options": "user_id" }, "properties": [ { "id": "custom.width", "value": 100 } ] }, { "matcher": { "id": "byName", "options": "ton_type" }, "properties": [ { "id": "custom.width", "value": 214 } ] }, { "matcher": { "id": "byName", "options": "jetton_type" }, "properties": [ { "id": "custom.width", "value": 104 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 80 } ] } ] }, "gridPos": { "h": 12, "w": 8, "x": 0, "y": 6 }, "id": 30, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n tw.subwallet_id, tw.user_id, tw.type as ton_type,jw.type as jetton_type, coalesce(currency, 'TON') as Currency\nFROM payments.ton_wallets tw\nLEFT JOIN payments.jetton_wallets jw ON tw.subwallet_id = jw.subwallet_id\nORDER BY 1", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.ton_wallets", "timeColumn": "time", "where": [] } ], "title": "Subwallets", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 70 } ] }, { "matcher": { "id": "byName", "options": "confirmed" }, "properties": [ { "id": "custom.width", "value": 87 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 151 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 73 } ] }, { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "custom.width", "value": 56 } ] }, { "matcher": { "id": "byName", "options": "created_at" }, "properties": [ { "id": "custom.width", "value": 148 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 6 }, "id": 12, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n ew.query_id,wr.amount,wr.currency,confirmed,failed,processed_at \nFROM\n payments.external_withdrawals ew\nLEFT JOIN payments.withdrawal_requests wr ON ew.query_id = wr.query_id\nWHERE wr.is_internal=false\n\n \n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "External withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "finished_at" }, "properties": [ { "id": "custom.width", "value": 149 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 92 } ] }, { "matcher": { "id": "byName", "options": "from" }, "properties": [ { "id": "custom.width", "value": 89 } ] }, { "matcher": { "id": "byName", "options": "created_at" }, "properties": [ { "id": "custom.width", "value": 150 } ] }, { "matcher": { "id": "byName", "options": "memo" }, "properties": [ { "id": "custom.width", "value": 123 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 76 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 6 }, "id": 14, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n memo,amount,coalesce(jw.currency, 'TON') as currency,coalesce(tw.subwallet_id, jw.subwallet_id) as from, finished_at, failed\nFROM\n payments.internal_withdrawals iw\nLEFT JOIN payments.ton_wallets tw ON iw.from_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON iw.from_address = jw.address \n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Internal withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 73 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 78 } ] }, { "matcher": { "id": "byName", "options": "confirmed" }, "properties": [ { "id": "custom.width", "value": 86 } ] }, { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "custom.width", "value": 60 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 12 }, "id": 18, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n ew.query_id,wr.amount,wr.currency,confirmed,failed,processed_at \nFROM\n payments.external_withdrawals ew\nLEFT JOIN payments.withdrawal_requests wr ON ew.query_id = wr.query_id\nWHERE wr.is_internal=true\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Cold wallet withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 139 } ] }, { "matcher": { "id": "byName", "options": "memo" }, "properties": [ { "id": "custom.width", "value": 287 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 76 } ] }, { "matcher": { "id": "byName", "options": "from" }, "properties": [ { "id": "custom.width", "value": 92 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 12 }, "id": 16, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n memo,amount,coalesce(jw.currency, 'TON') as currency,coalesce(tw.subwallet_id, jw.subwallet_id) as from\nFROM\n payments.internal_incomes ii\nLEFT JOIN payments.ton_wallets tw ON ii.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ii.deposit_address = jw.address \n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Internal incomes", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 18 }, "id": 20, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, SUM(amount) as total_amount, coalesce(jw.currency, 'TON') as currency\nFROM\n payments.internal_incomes ii\nLEFT JOIN payments.ton_wallets tw ON ii.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ii.deposit_address = jw.address\nGROUP BY ii.deposit_address, tw.subwallet_id, jw.subwallet_id, jw.currency\n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Hot side deposit balances", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "processed" }, "properties": [ { "id": "custom.width", "value": 87 } ] }, { "matcher": { "id": "byName", "options": "ton_amount" }, "properties": [ { "id": "custom.width", "value": 103 } ] }, { "matcher": { "id": "byName", "options": "jetton_amount" }, "properties": [ { "id": "custom.width", "value": 111 } ] } ] }, "gridPos": { "h": 5, "w": 8, "x": 8, "y": 18 }, "id": 29, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT memo, processed, from_address, ton_amount, jetton_amount\nFROM payments.service_withdrawal_requests", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "service_withdrawal_requests", "timeColumn": "time", "where": [] } ], "title": "Service withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 16, "y": 18 }, "id": 21, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, SUM(amount) as total_amount, coalesce(jw.currency, 'TON') as currency\nFROM\n payments.external_incomes ei\nLEFT JOIN payments.ton_wallets tw ON ei.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ei.deposit_address = jw.address\nGROUP BY ei.deposit_address, tw.subwallet_id, jw.subwallet_id, jw.currency\n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Deposit side balances", "type": "table" } ], "refresh": "5s", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-5m", "to": "now" }, "timepicker": { "hidden": false, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "type": "timepicker" }, "timezone": "browser", "title": "Payments", "uid": "_QoyuqtVz", "version": 7, "weekStart": "" } ================================================ FILE: deploy/grafana/main/provisioning/dashboards/payments.yml ================================================ apiVersion: 1 providers: - name: dashboards type: file updateIntervalSeconds: 30 options: path: /etc/dashboards foldersFromFilesStructure: true ================================================ FILE: deploy/grafana/main/provisioning/datasources/data_sources.yml ================================================ apiVersion: 1 datasources: - name: Prometheus uid: DS_PROMETHEUS type: prometheus access: proxy url: http://payment_prometheus:9090 isDefault: false - name: Postgres uid: DS_POSTGRES type: postgres url: payment_processor_db:5432 database: $POSTGRES_DB user: pp_readonly secureJsonData: password: $POSTGRES_READONLY_PASSWORD isDefault: true jsonData: sslmode: disable # disable/require/verify-ca/verify-full maxOpenConns: 0 # Grafana v5.4+ maxIdleConns: 2 # Grafana v5.4+ connMaxLifetime: 14400 # Grafana v5.4+ postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 timescaledb: false ================================================ FILE: deploy/grafana/test/dashboards/Processor A.json ================================================ { "__inputs": [ { "name": "DS_POSTGRES_A", "label": "Postgres A", "description": "", "type": "datasource", "pluginId": "postgres", "pluginName": "PostgreSQL" } ], "__elements": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "8.3.6" }, { "type": "datasource", "id": "postgres", "name": "PostgreSQL", "version": "1.0.0" }, { "type": "panel", "id": "table", "name": "Table", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": true, "panels": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "saved_at" }, "properties": [ { "id": "custom.width", "value": 184 } ] }, { "matcher": { "id": "byName", "options": "seqno" }, "properties": [ { "id": "custom.width", "value": 210 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, "id": 2, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT saved_at, seqno, gen_utime\nFROM payments.block_data\nORDER BY seqno desc\nLIMIT 4", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "paymnets.block_data", "timeColumn": "time", "where": [] } ], "title": "Block data", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 71 } ] }, { "matcher": { "id": "byName", "options": "processing" }, "properties": [ { "id": "custom.width", "value": 89 } ] }, { "matcher": { "id": "byName", "options": "processed" }, "properties": [ { "id": "custom.width", "value": 84 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 104 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 78 } ] }, { "matcher": { "id": "byName", "options": "comment" }, "properties": [ { "id": "custom.width", "value": 358 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, "id": 4, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n query_id, processing, processed, amount, currency,comment\nFROM payments.withdrawal_requests\nORDER BY query_id DESC", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.withdrawal_requests", "timeColumn": "time", "where": [] } ], "title": "Withdrawal requests", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "utime" }, "properties": [ { "id": "custom.width", "value": 176 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 103 } ] }, { "matcher": { "id": "byName", "options": "comment" }, "properties": [ { "id": "custom.width", "value": 134 } ] }, { "matcher": { "id": "byName", "options": "subwallet_id" }, "properties": [ { "id": "custom.width", "value": 112 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 75 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, "id": 10, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n utime, amount, coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, coalesce(currency, 'TON') as currency, comment\nFROM\n payments.external_incomes ei\nLEFT JOIN payments.ton_wallets tw ON ei.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ei.deposit_address = jw.address\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.external_incomes", "timeColumn": "time", "where": [] } ], "title": "External imcomes", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "subwallet_id" }, "properties": [ { "id": "custom.width", "value": 105 } ] }, { "matcher": { "id": "byName", "options": "user_id" }, "properties": [ { "id": "custom.width", "value": 100 } ] }, { "matcher": { "id": "byName", "options": "ton_type" }, "properties": [ { "id": "custom.width", "value": 214 } ] }, { "matcher": { "id": "byName", "options": "jetton_type" }, "properties": [ { "id": "custom.width", "value": 104 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 80 } ] } ] }, "gridPos": { "h": 12, "w": 8, "x": 0, "y": 6 }, "id": 30, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n tw.subwallet_id, tw.user_id, tw.type as ton_type,jw.type as jetton_type, coalesce(currency, 'TON') as Currency\nFROM payments.ton_wallets tw\nLEFT JOIN payments.jetton_wallets jw ON tw.subwallet_id = jw.subwallet_id\nORDER BY 1", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.ton_wallets", "timeColumn": "time", "where": [] } ], "title": "Subwallets", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 70 } ] }, { "matcher": { "id": "byName", "options": "confirmed" }, "properties": [ { "id": "custom.width", "value": 87 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 151 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 73 } ] }, { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "custom.width", "value": 56 } ] }, { "matcher": { "id": "byName", "options": "created_at" }, "properties": [ { "id": "custom.width", "value": 148 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 6 }, "id": 12, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n ew.query_id,wr.amount,wr.currency,confirmed,failed,processed_at \nFROM\n payments.external_withdrawals ew\nLEFT JOIN payments.withdrawal_requests wr ON ew.query_id = wr.query_id\nWHERE wr.is_internal=false\n\n \n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "External withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "finished_at" }, "properties": [ { "id": "custom.width", "value": 149 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 92 } ] }, { "matcher": { "id": "byName", "options": "from" }, "properties": [ { "id": "custom.width", "value": 89 } ] }, { "matcher": { "id": "byName", "options": "created_at" }, "properties": [ { "id": "custom.width", "value": 150 } ] }, { "matcher": { "id": "byName", "options": "memo" }, "properties": [ { "id": "custom.width", "value": 123 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 76 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 6 }, "id": 14, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n memo,amount,coalesce(jw.currency, 'TON') as currency,coalesce(tw.subwallet_id, jw.subwallet_id) as from, finished_at, failed\nFROM\n payments.internal_withdrawals iw\nLEFT JOIN payments.ton_wallets tw ON iw.from_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON iw.from_address = jw.address \n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Internal withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 73 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 78 } ] }, { "matcher": { "id": "byName", "options": "confirmed" }, "properties": [ { "id": "custom.width", "value": 86 } ] }, { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "custom.width", "value": 60 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 12 }, "id": 18, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n ew.query_id,wr.amount,wr.currency,confirmed,failed,processed_at \nFROM\n payments.external_withdrawals ew\nLEFT JOIN payments.withdrawal_requests wr ON ew.query_id = wr.query_id\nWHERE wr.is_internal=true\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Cold wallet withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 139 } ] }, { "matcher": { "id": "byName", "options": "memo" }, "properties": [ { "id": "custom.width", "value": 287 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 76 } ] }, { "matcher": { "id": "byName", "options": "from" }, "properties": [ { "id": "custom.width", "value": 92 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 12 }, "id": 16, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n memo,amount,coalesce(jw.currency, 'TON') as currency,coalesce(tw.subwallet_id, jw.subwallet_id) as from\nFROM\n payments.internal_incomes ii\nLEFT JOIN payments.ton_wallets tw ON ii.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ii.deposit_address = jw.address \n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Internal incomes", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 18 }, "id": 20, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, SUM(amount) as total_amount, coalesce(jw.currency, 'TON') as currency\nFROM\n payments.internal_incomes ii\nLEFT JOIN payments.ton_wallets tw ON ii.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ii.deposit_address = jw.address\nGROUP BY ii.deposit_address, tw.subwallet_id, jw.subwallet_id, jw.currency\n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Hot side deposit balances", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "processed" }, "properties": [ { "id": "custom.width", "value": 87 } ] }, { "matcher": { "id": "byName", "options": "ton_amount" }, "properties": [ { "id": "custom.width", "value": 103 } ] }, { "matcher": { "id": "byName", "options": "jetton_amount" }, "properties": [ { "id": "custom.width", "value": 111 } ] } ] }, "gridPos": { "h": 5, "w": 8, "x": 8, "y": 18 }, "id": 29, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT memo, processed, from_address, ton_amount, jetton_amount\nFROM payments.service_withdrawal_requests", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "service_withdrawal_requests", "timeColumn": "time", "where": [] } ], "title": "Service withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 16, "y": 18 }, "id": 21, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_A" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, SUM(amount) as total_amount, coalesce(jw.currency, 'TON') as currency\nFROM\n payments.external_incomes ei\nLEFT JOIN payments.ton_wallets tw ON ei.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ei.deposit_address = jw.address\nGROUP BY ei.deposit_address, tw.subwallet_id, jw.subwallet_id, jw.currency\n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Deposit side balances", "type": "table" } ], "refresh": "5s", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-5m", "to": "now" }, "timepicker": { "hidden": false, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "type": "timepicker" }, "timezone": "browser", "title": "Processor A", "uid": "_QoyuqtVz", "version": 7, "weekStart": "" } ================================================ FILE: deploy/grafana/test/dashboards/Processor B.json ================================================ { "__inputs": [ { "name": "DS_POSTGRES_B", "label": "Postgres B", "description": "", "type": "datasource", "pluginId": "postgres", "pluginName": "PostgreSQL" } ], "__elements": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "8.3.6" }, { "type": "datasource", "id": "postgres", "name": "PostgreSQL", "version": "1.0.0" }, { "type": "panel", "id": "table", "name": "Table", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": true, "panels": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "saved_at" }, "properties": [ { "id": "custom.width", "value": 184 } ] }, { "matcher": { "id": "byName", "options": "seqno" }, "properties": [ { "id": "custom.width", "value": 210 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, "id": 2, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT saved_at, seqno, gen_utime\nFROM payments.block_data\nORDER BY seqno desc\nLIMIT 4", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "paymnets.block_data", "timeColumn": "time", "where": [] } ], "title": "Block data", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 71 } ] }, { "matcher": { "id": "byName", "options": "processing" }, "properties": [ { "id": "custom.width", "value": 89 } ] }, { "matcher": { "id": "byName", "options": "processed" }, "properties": [ { "id": "custom.width", "value": 84 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 104 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 78 } ] }, { "matcher": { "id": "byName", "options": "comment" }, "properties": [ { "id": "custom.width", "value": 358 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, "id": 4, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n query_id, processing, processed, amount, currency,comment\nFROM payments.withdrawal_requests\nORDER BY query_id DESC", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.withdrawal_requests", "timeColumn": "time", "where": [] } ], "title": "Withdrawal requests", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "utime" }, "properties": [ { "id": "custom.width", "value": 176 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 103 } ] }, { "matcher": { "id": "byName", "options": "comment" }, "properties": [ { "id": "custom.width", "value": 134 } ] }, { "matcher": { "id": "byName", "options": "subwallet_id" }, "properties": [ { "id": "custom.width", "value": 112 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 75 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, "id": 10, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n utime, amount, coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, coalesce(currency, 'TON') as currency, comment\nFROM\n payments.external_incomes ei\nLEFT JOIN payments.ton_wallets tw ON ei.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ei.deposit_address = jw.address\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.external_incomes", "timeColumn": "time", "where": [] } ], "title": "External imcomes", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "subwallet_id" }, "properties": [ { "id": "custom.width", "value": 105 } ] }, { "matcher": { "id": "byName", "options": "user_id" }, "properties": [ { "id": "custom.width", "value": 100 } ] }, { "matcher": { "id": "byName", "options": "ton_type" }, "properties": [ { "id": "custom.width", "value": 214 } ] }, { "matcher": { "id": "byName", "options": "jetton_type" }, "properties": [ { "id": "custom.width", "value": 104 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 80 } ] } ] }, "gridPos": { "h": 12, "w": 8, "x": 0, "y": 6 }, "id": 30, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n tw.subwallet_id, tw.user_id, tw.type as ton_type,jw.type as jetton_type, coalesce(currency, 'TON') as Currency\nFROM payments.ton_wallets tw\nLEFT JOIN payments.jetton_wallets jw ON tw.subwallet_id = jw.subwallet_id\nORDER BY 1", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "payments.ton_wallets", "timeColumn": "time", "where": [] } ], "title": "Subwallets", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 70 } ] }, { "matcher": { "id": "byName", "options": "confirmed" }, "properties": [ { "id": "custom.width", "value": 87 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 151 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 73 } ] }, { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "custom.width", "value": 56 } ] }, { "matcher": { "id": "byName", "options": "created_at" }, "properties": [ { "id": "custom.width", "value": 148 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 6 }, "id": 12, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n ew.query_id,wr.amount,wr.currency,confirmed,failed,processed_at \nFROM\n payments.external_withdrawals ew\nLEFT JOIN payments.withdrawal_requests wr ON ew.query_id = wr.query_id\nWHERE wr.is_internal=false\n\n \n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "External withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "finished_at" }, "properties": [ { "id": "custom.width", "value": 149 } ] }, { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 92 } ] }, { "matcher": { "id": "byName", "options": "from" }, "properties": [ { "id": "custom.width", "value": 89 } ] }, { "matcher": { "id": "byName", "options": "created_at" }, "properties": [ { "id": "custom.width", "value": 150 } ] }, { "matcher": { "id": "byName", "options": "memo" }, "properties": [ { "id": "custom.width", "value": 123 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 76 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 6 }, "id": 14, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n memo,amount,coalesce(jw.currency, 'TON') as currency,coalesce(tw.subwallet_id, jw.subwallet_id) as from, finished_at, failed\nFROM\n payments.internal_withdrawals iw\nLEFT JOIN payments.ton_wallets tw ON iw.from_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON iw.from_address = jw.address \n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Internal withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "query_id" }, "properties": [ { "id": "custom.width", "value": 73 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 78 } ] }, { "matcher": { "id": "byName", "options": "confirmed" }, "properties": [ { "id": "custom.width", "value": 86 } ] }, { "matcher": { "id": "byName", "options": "failed" }, "properties": [ { "id": "custom.width", "value": 60 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 12 }, "id": 18, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n ew.query_id,wr.amount,wr.currency,confirmed,failed,processed_at \nFROM\n payments.external_withdrawals ew\nLEFT JOIN payments.withdrawal_requests wr ON ew.query_id = wr.query_id\nWHERE wr.is_internal=true\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Cold wallet withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "amount" }, "properties": [ { "id": "custom.width", "value": 139 } ] }, { "matcher": { "id": "byName", "options": "memo" }, "properties": [ { "id": "custom.width", "value": 287 } ] }, { "matcher": { "id": "byName", "options": "currency" }, "properties": [ { "id": "custom.width", "value": 76 } ] }, { "matcher": { "id": "byName", "options": "from" }, "properties": [ { "id": "custom.width", "value": 92 } ] } ] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 12 }, "id": 16, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n memo,amount,coalesce(jw.currency, 'TON') as currency,coalesce(tw.subwallet_id, jw.subwallet_id) as from\nFROM\n payments.internal_incomes ii\nLEFT JOIN payments.ton_wallets tw ON ii.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ii.deposit_address = jw.address \n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Internal incomes", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 18 }, "id": 20, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, SUM(amount) as total_amount, coalesce(jw.currency, 'TON') as currency\nFROM\n payments.internal_incomes ii\nLEFT JOIN payments.ton_wallets tw ON ii.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ii.deposit_address = jw.address\nGROUP BY ii.deposit_address, tw.subwallet_id, jw.subwallet_id, jw.currency\n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Hot side deposit balances", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "processed" }, "properties": [ { "id": "custom.width", "value": 87 } ] }, { "matcher": { "id": "byName", "options": "ton_amount" }, "properties": [ { "id": "custom.width", "value": 103 } ] }, { "matcher": { "id": "byName", "options": "jetton_amount" }, "properties": [ { "id": "custom.width", "value": 111 } ] } ] }, "gridPos": { "h": 5, "w": 8, "x": 8, "y": 18 }, "id": 29, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT memo, processed, from_address, ton_amount, jetton_amount\nFROM payments.service_withdrawal_requests", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "table": "service_withdrawal_requests", "timeColumn": "time", "where": [] } ], "title": "Service withdrawals", "type": "table" }, { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 16, "y": 18 }, "id": 21, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "postgres", "uid": "DS_POSTGRES_B" }, "format": "table", "group": [], "hide": false, "metricColumn": "none", "rawQuery": true, "rawSql": "SELECT\n coalesce(tw.subwallet_id, jw.subwallet_id) as subwallet_id, SUM(amount) as total_amount, coalesce(jw.currency, 'TON') as currency\nFROM\n payments.external_incomes ei\nLEFT JOIN payments.ton_wallets tw ON ei.deposit_address = tw.address \nLEFT JOIN payments.jetton_wallets jw ON ei.deposit_address = jw.address\nGROUP BY ei.deposit_address, tw.subwallet_id, jw.subwallet_id, jw.currency\n\n", "refId": "A", "select": [ [ { "params": [ "value" ], "type": "column" } ] ], "timeColumn": "time", "where": [ { "name": "$__timeFilter", "params": [], "type": "macro" } ] } ], "title": "Deposit side balances", "type": "table" } ], "refresh": "5s", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-5m", "to": "now" }, "timepicker": { "hidden": false, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "type": "timepicker" }, "timezone": "browser", "title": "Processor B", "uid": "DaYu9qtVk", "version": 2, "weekStart": "" } ================================================ FILE: deploy/grafana/test/dashboards/Test util.json ================================================ { "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": [], "__requires": [ { "type": "panel", "id": "bargauge", "name": "Bar gauge", "version": "" }, { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "8.3.6" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": true, "panels": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 3, "mappings": [], "max": 50, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 20 }, { "color": "green", "value": 50 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 7, "x": 0, "y": 0 }, "id": 8, "options": { "displayMode": "gradient", "orientation": "vertical", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": false, "expr": "hot_wallet_a_balance{}/1000000000", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{currency}}: A", "refId": "Payer" }, { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": true, "expr": "hot_wallet_b_balance{}/1000000000", "hide": false, "interval": "", "legendFormat": "{{currency}}: B", "refId": "A" } ], "title": "Hot wallets balance", "transformations": [], "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 3, "mappings": [], "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 7, "y": 0 }, "id": 28, "options": { "displayMode": "gradient", "orientation": "vertical", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": true, "expr": "total_balance{}/1000000000", "hide": false, "interval": "", "legendFormat": "{{currency}}", "refId": "A" } ], "title": "Total balance", "transformations": [], "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 3, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": -1 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 12, "y": 0 }, "id": 32, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": true, "expr": "total_processed_amount{}/1000000000", "hide": false, "interval": "", "legendFormat": "{{currency}}", "refId": "A" } ], "title": "Processed amount", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": -1 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 16, "y": 0 }, "id": 25, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": true, "expr": "total_losses{}/-1000000000", "hide": false, "interval": "", "legendFormat": "{{currency}}", "refId": "A" } ], "title": "Losses", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "displayName": "TON", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": -1 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 2, "x": 20, "y": 0 }, "id": 31, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "vertical", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": false, "expr": "predicted_ton_loss{}/1000000000", "format": "time_series", "hide": false, "instant": false, "interval": "", "legendFormat": "", "refId": "B" } ], "title": "Predicted loss", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "m" }, "overrides": [] }, "gridPos": { "h": 7, "w": 2, "x": 22, "y": 0 }, "id": 27, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value" }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": true, "expr": "(time() - process_start_time_seconds{})/60", "hide": false, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Uptime", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 3, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 16, "w": 12, "x": 0, "y": 7 }, "id": 23, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": true, "expr": "deposit_wallet_a_balance{}/1000000000", "hide": false, "interval": "", "legendFormat": "{{currency}}: {{address}} ", "refId": "A" } ], "title": "Deposit A balances", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 3, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 16, "w": 12, "x": 12, "y": 7 }, "id": 30, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "8.3.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" }, "exemplar": true, "expr": "deposit_wallet_b_balance{}/1000000000", "hide": false, "interval": "", "legendFormat": "{{currency}}: {{address}} ", "refId": "A" } ], "title": "Deposit B balances", "type": "bargauge" } ], "refresh": "5s", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "hidden": false, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "type": "timepicker" }, "timezone": "browser", "title": "Test util", "uid": "o-Hru3t4k", "version": 4, "weekStart": "" } ================================================ FILE: deploy/grafana/test/provisioning/dashboards/payments.yml ================================================ apiVersion: 1 providers: - name: dashboards type: file updateIntervalSeconds: 30 options: path: /etc/dashboards foldersFromFilesStructure: true ================================================ FILE: deploy/grafana/test/provisioning/datasources/data_sources.yml ================================================ apiVersion: 1 datasources: - name: Prometheus uid: DS_PROMETHEUS type: prometheus access: proxy url: http://payment_prometheus:9090 isDefault: false - name: Postgres A uid: DS_POSTGRES_A type: postgres url: payment_processor_db_a:5432 database: payment_processor user: pp_user secureJsonData: password: $POSTGRES_PASSWORD isDefault: false jsonData: sslmode: disable # disable/require/verify-ca/verify-full maxOpenConns: 0 # Grafana v5.4+ maxIdleConns: 2 # Grafana v5.4+ connMaxLifetime: 14400 # Grafana v5.4+ postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 timescaledb: false - name: Postgres B uid: DS_POSTGRES_B type: postgres url: payment_processor_db_b:5432 database: payment_processor user: pp_user secureJsonData: password: $POSTGRES_PASSWORD isDefault: false jsonData: sslmode: disable # disable/require/verify-ca/verify-full maxOpenConns: 0 # Grafana v5.4+ maxIdleConns: 2 # Grafana v5.4+ connMaxLifetime: 14400 # Grafana v5.4+ postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 timescaledb: false ================================================ FILE: deploy/manual_migrations/0.1.x-0.2.0.sql ================================================ BEGIN; ALTER TABLE payments.external_incomes ADD COLUMN IF NOT EXISTS payer_workchain integer; UPDATE payments.external_incomes SET payer_workchain = 0 WHERE payer_address IS NOT NULL AND payer_workchain IS NULL; -- all existing addresses will be marked as 0 workchain COMMIT; ================================================ FILE: deploy/manual_migrations/0.4.x-0.5.0.sql ================================================ BEGIN; ALTER TABLE payments.external_withdrawals ADD COLUMN IF NOT EXISTS tx_hash bytea; ALTER TABLE payments.withdrawal_requests ADD COLUMN IF NOT EXISTS binary_comment text default ''; ALTER TABLE payments.external_incomes ADD COLUMN IF NOT EXISTS tx_hash bytea; COMMIT; ================================================ FILE: deploy/prometheus/main/prometheus.yml ================================================ scrape_configs: - job_name: audit-metrics scrape_interval: 5s static_configs: - targets: ['payment-processor:8081'] ================================================ FILE: deploy/prometheus/test/prometheus.yml ================================================ scrape_configs: - job_name: test-utils scrape_interval: 5s static_configs: - targets: ['payment_test:9101'] ================================================ FILE: docker-compose.yml ================================================ version: '3' services: payment-postgres: image: postgres:14 container_name: payment_processor_db volumes: - ./deploy/db/01_init.up.sql:/docker-entrypoint-initdb.d/01_init.up.sql - ./deploy/db/02_create_readonly_user.sh:/docker-entrypoint-initdb.d/02_create_readonly_user.sh - bicycle-postgres:/var/lib/postgresql/data restart: always environment: POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_READONLY_PASSWORD: ${POSTGRES_READONLY_PASSWORD} # For grafana networks: - p-network payment-processor: image: payment-processor container_name: payment_processor ports: - "127.0.0.1:8081:${API_PORT}" restart: unless-stopped environment: DB_URI: ${DB_URI} # example: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@payment_processor_db:5432/${POSTGRES_DB}" API_PORT: ${API_PORT} API_TOKEN: ${API_TOKEN} COLD_WALLET: ${COLD_WALLET} JETTONS: ${JETTONS} LITESERVER: ${LITESERVER} LITESERVER_KEY: ${LITESERVER_KEY} SEED: ${SEED} TON_CUTOFFS: ${TON_CUTOFFS} IS_TESTNET: ${IS_TESTNET} networks: - p-network payment-grafana: image: grafana/grafana:latest container_name: payment_grafana restart: always ports: - '127.0.0.1:3001:3000' volumes: - ./deploy/grafana/main/provisioning/datasources:/etc/grafana/provisioning/datasources - ./deploy/grafana/main/provisioning/dashboards:/etc/grafana/provisioning/dashboards - ./deploy/grafana/main/dashboards:/etc/dashboards environment: GF_SECURITY_ADMIN_USER: admin # TODO: change GF_SECURITY_ADMIN_PASSWORD: admin # TODO: change POSTGRES_DB: ${POSTGRES_DB} POSTGRES_READONLY_PASSWORD: ${POSTGRES_READONLY_PASSWORD} networks: - p-network payment-rabbitmq: image: library/rabbitmq:3-management container_name: payment_rabbitmq restart: always ports: - '127.0.0.1:5672:5672' networks: - p-network payment-prometheus: image: prom/prometheus:latest container_name: payment_prometheus restart: always volumes: - ./deploy/prometheus/main:/etc/prometheus user: "$UID:$GID" command: - '--config.file=/etc/prometheus/prometheus.yml' networks: - p-network networks: p-network: driver: bridge volumes: bicycle-postgres: external: false ================================================ FILE: docs/api.apib ================================================ FORMAT: 1A # Payment processor API This API describes endpoints of payment processor. ## New address [POST /v1/address/new] Generates new deposit address + Request (application/json) + Headers Authorization: Bearer + Body {"user_id": "123", "currency": "TON"} + Response 200 (application/json) + Body { "address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR" } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Get all addresses [GET /v1/address/all{?user_id}] Get all created addresses by `user_id` + Parameters + user_id (string) - an unique identifier of the user + Request + Headers Authorization: Bearer + Response 200 (application/json) + Body { "addresses": [ { "address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR", "currency": "TON" }, { "address": "0QCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseHynZ_YiH9Y1oSe", "currency": "TGR" } ] } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Get income [GET /v1/income{?user_id}] Get income for deposits by `user_id`. The total amount of funds that came to the deposit for the entire time. Funds sent from the deposit to the hot wallet or all funds received to the deposit are taken into account, depending on the service settings. Counting side field options: "hot_wallet", "deposit". + Parameters + user_id (string) - an unique identifier of the user + Request + Headers Authorization: Bearer + Response 200 (application/json) + Body { "counting_side": "deposit", "total_income": [ { "deposit_address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR", "amount": "1000000", "currency": "TON" }, { "deposit_address": "0QCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseHynZ_YiH9Y1oSe", "amount": "1023000", "currency": "TGR" } ] } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Get history [GET /v1/deposit/history{?user_id,currency,limit,offset,sort_order}] Get history for deposits by `user_id` and `currency`. Returns the history of all deposits replenishments with the sender's address as `source_address` (if it could be determined). + Parameters + user_id (string) - an unique identifier of the user + currency (string) - the text identifier of the currency specified in the processor configuration. `TON` for TON coin. + limit (number) - the maximum value of returned records + offset (number) - offset for returned records + sort_order (string) - asc or desc. desc by default if the parameter is not specified. + Request + Headers Authorization: Bearer + Response 200 (application/json) + Body { "incomes": [ { "deposit_address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR", "time": 1680604643, "source_address": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP", "amount": "1000000", "comment": "hello", "tx_hash": "9d0fb69b1ca9371bc9e5260a248cda12a1c42916f9051fe9fc21b4abdd41d744" }, { "deposit_address": "0QCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseHynZ_YiH9Y1oSe", "time": 1680604648, "source_address": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP", "amount": "1000000", "comment": "hello", "tx_hash": "7d0fb69b1ca9371bc9e5260a248cda12a1c42916f9051fe9fc21b4abdd41d744" } ] } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Send withdrawal [POST /v1/withdrawal/send] Send withdrawal request. **Amount must be in base units without decimal point (NanoTONs for TONs)** Instead of a `comment`, you can specify a `binary_comment` in hex notation format. `binary_comment` supports binary string hex representation form with flip bit (example: `9fe7_`) + Request (application/json) + Headers Authorization: Bearer + Body { "user_id": "123", "query_id": "321", "currency": "TON", "amount": "100", "destination": "0QCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseHynZ_YiH9Y1oSe", "comment": "hello" } + Response 200 (application/json) + Body { "ID": 1 } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Send service withdrawal [POST /v1/withdrawal/service/ton] Send service withdrawal request. Withdraw all TONs from `from` address to hot wallet. Returns `memo` as comment for transfer message. + Request (application/json) + Headers Authorization: Bearer + Body { "from": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP", } + Response 200 (application/json) + Body { "memo": "123e4567-e89b-12d3-a456-426655440000" } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Send service withdrawal [POST /v1/withdrawal/service/jetton] Send service withdrawal request. Withdraw all Jettons from Jetton wallet. Address calculated through owner and Jetton master. Returns `memo` as comment for transfer message. + Request (application/json) + Headers Authorization: Bearer + Body { "owner": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP", "jetton_master": "kQCKt2WPGX-fh0cIAz38Ljd_OKQjoZE_cqk7QrYGsNP6wfP0", } + Response 200 (application/json) + Body { "memo": "123e4567-e89b-12d3-a456-426655440000" } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Get withdrawal status [GET /v1/withdrawal/status{?id}] Get withdrawal status. Returns `pending`, `processing`, `processed`/'failed', transaction hash for processed withdrawal and request meta (user_id and query_id). + Parameters + id (number) - An unique identifier of the withdrawal. + Request (application/json) + Headers Authorization: Bearer + Response 200 (application/json) + Body { "user_id": "123", "query_id": "321", "status": "processed", "tx_hash": "9d0fb69b1ca9371bc9e5260a248cda12a1c42916f9051fe9fc21b4abdd41d744" } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Get balance [GET /v1/balance{?currency,address}] Get balance for account by `address` and `currency`. Returns the balance of hot wallet if `address` not specified. For hot wallet also returns `total_processing_amount` and `total_pending_amount` for withdrawals in queue. For TON also returns account status: "active", "uninit", "frozen", "non_exist". + Parameters + currency (string) - the text identifier of the currency specified in the processor configuration. `TON` for TON coin. + address (string) - address in URL-safe, user-friendly Base64 form. Hot wallet address by default if the parameter is not specified. + Request + Headers Authorization: Bearer + Response 200 (application/json) + Body { "balance": "1000000", "status": "active", "total_processing_amount": "1000", "total_pending_amount": "1000" } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 500 (application/json) + Body { "error": "error text" } ## Get sync flag [GET /v1/system/sync] Get blockchain sync flag. Returns `true` if the service has up-to-date data from the blockchain. As long as the flag is equal to `false`, no withdrawals are made. Also returns last scanned workchain block `gen_utime` (unix time) to evaluate the processor's lag from the blockchain. + Response 200 (application/json) + Body { "is_synced": false, "last_block_gen_utime": 1718490850, } + Response 500 (application/json) + Body { "error": "error text" } ## Get income by transaction hash [GET /v1/deposit/income{?tx_hash}] Find income for deposit by `tx_hash`. Returns the currency and the deposit replenishment with the sender's address as `source_address` (if it could be determined). + Parameters + tx_hash (string) - an unique hash of replenishment transaction + Request + Headers Authorization: Bearer + Response 200 (application/json) + Body { "currency": "TON", "income": { "deposit_address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR", "time": 1680604643, "source_address": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP", "amount": "1000000", "comment": "hello", "tx_hash": "9d0fb69b1ca9371bc9e5260a248cda12a1c42916f9051fe9fc21b4abdd41d744" } } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 404 (application/json) + Body { "error": "error text" } + Response 500 (application/json) + Body { "error": "error text" } ## Resolve domain name [GET /v1/resolve{?domain}] Resolve domain name. Returns bounceable user-friendly address for smart-contract (`dns_smc_address#9fd3` DNS record). + Parameters + domain (string) - domain name (for example: wallet.ton) + Request + Headers Authorization: Bearer + Response 200 (application/json) + Body { "address": "kQCKt2WPGX-fh0cIAz38Ljd_OKQjoZE_cqk7QrYGsNP6wfP0" } + Response 400 (application/json) + Body { "error": "error text" } + Response 401 + Response 404 (application/json) + Body { "error": "error text" } + Response 500 (application/json) + Body { "error": "error text" } ================================================ FILE: docs/index.html ================================================ Payment processor API Back to top

Payment processor API

This API describes endpoints of payment processor.

Resource Group

New address

New address
POST/v1/address/new

Generates new deposit address

Example URI

POST /v1/address/new
Request
HideShow
Headers
Content-Type: application/json
Authorization: Bearer
Body
{
  "user_id": "123",
  "currency": "TON"
}
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR"
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Get all addresses

Get all addresses
GET/v1/address/all{?user_id}

Get all created addresses by user_id

Example URI

GET /v1/address/all?user_id=
URI Parameters
HideShow
user_id
string (required) 

an unique identifier of the user

Request
HideShow
Headers
Authorization: Bearer
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "addresses": [
    {
      "address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR",
      "currency": "TON"
    },
    {
      "address": "0QCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseHynZ_YiH9Y1oSe",
      "currency": "TGR"
    }
  ]
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Get income

Get income
GET/v1/income{?user_id}

Get income for deposits by user_id. The total amount of funds that came to the deposit for the entire time. Funds sent from the deposit to the hot wallet or all funds received to the deposit are taken into account, depending on the service settings. Counting side field options: “hot_wallet”, “deposit”.

Example URI

GET /v1/income?user_id=
URI Parameters
HideShow
user_id
string (required) 

an unique identifier of the user

Request
HideShow
Headers
Authorization: Bearer
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "counting_side": "deposit",
  "total_income": [
    {
      "deposit_address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR",
      "amount": "1000000",
      "currency": "TON"
    },
    {
      "deposit_address": "0QCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseHynZ_YiH9Y1oSe",
      "amount": "1023000",
      "currency": "TGR"
    }
  ]
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Get history

Get history
GET/v1/deposit/history{?user_id,currency,limit,offset,sort_order}

Get history for deposits by user_id and currency. Returns the history of all deposits replenishments with the sender’s address as source_address (if it could be determined).

Example URI

GET /v1/deposit/history?user_id=&currency=&limit=&offset=&sort_order=
URI Parameters
HideShow
user_id
string (required) 

an unique identifier of the user

currency
string (required) 

the text identifier of the currency specified in the processor configuration. TON for TON coin.

limit
number (required) 

the maximum value of returned records

offset
number (required) 

offset for returned records

sort_order
string (required) 

asc or desc. desc by default if the parameter is not specified.

Request
HideShow
Headers
Authorization: Bearer
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "incomes": [
    {
      "deposit_address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR",
      "time": 1680604643,
      "source_address": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP",
      "amount": "1000000",
      "comment": "hello",
      "tx_hash": "9d0fb69b1ca9371bc9e5260a248cda12a1c42916f9051fe9fc21b4abdd41d744"
    },
    {
      "deposit_address": "0QCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseHynZ_YiH9Y1oSe",
      "time": 1680604648,
      "source_address": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP",
      "amount": "1000000",
      "comment": "hello",
      "tx_hash": "7d0fb69b1ca9371bc9e5260a248cda12a1c42916f9051fe9fc21b4abdd41d744"
    }
  ]
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Send withdrawal

Send withdrawal
POST/v1/withdrawal/send

Send withdrawal request. Amount must be in base units without decimal point (NanoTONs for TONs) Instead of a comment, you can specify a binary_comment in hex notation format. binary_comment supports binary string hex representation form with flip bit (example: 9fe7_)

Example URI

POST /v1/withdrawal/send
Request
HideShow
Headers
Content-Type: application/json
Authorization: Bearer
Body
{
  "user_id": "123",
  "query_id": "321",
  "currency": "TON",
  "amount": "100",
  "destination": "0QCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseHynZ_YiH9Y1oSe",
  "comment": "hello"
}
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "ID": 1
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Send service withdrawal

Send service withdrawal
POST/v1/withdrawal/service/ton

Send service withdrawal request. Withdraw all TONs from from address to hot wallet. Returns memo as comment for transfer message.

Example URI

POST /v1/withdrawal/service/ton
Request
HideShow
Headers
Content-Type: application/json
Authorization: Bearer
Body
{
  "from": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP",
}
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "memo": "123e4567-e89b-12d3-a456-426655440000"
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Send service withdrawal

Send service withdrawal
POST/v1/withdrawal/service/jetton

Send service withdrawal request. Withdraw all Jettons from Jetton wallet. Address calculated through owner and Jetton master. Returns memo as comment for transfer message.

Example URI

POST /v1/withdrawal/service/jetton
Request
HideShow
Headers
Content-Type: application/json
Authorization: Bearer
Body
{
  "owner": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP",
  "jetton_master": "kQCKt2WPGX-fh0cIAz38Ljd_OKQjoZE_cqk7QrYGsNP6wfP0",
}
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "memo": "123e4567-e89b-12d3-a456-426655440000"
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Get withdrawal status

Get withdrawal status
GET/v1/withdrawal/status{?id}

Get withdrawal status. Returns pending, processing, processed/‘failed’, transaction hash for processed withdrawal and request meta (user_id and query_id).

Example URI

GET /v1/withdrawal/status?id=
URI Parameters
HideShow
id
number (required) 

An unique identifier of the withdrawal.

Request
HideShow
Headers
Content-Type: application/json
Authorization: Bearer
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "user_id": "123",
  "query_id": "321",
  "status": "processed",
  "tx_hash": "9d0fb69b1ca9371bc9e5260a248cda12a1c42916f9051fe9fc21b4abdd41d744"
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Get balance

Get balance
GET/v1/balance{?currency,address}

Get balance for account by address and currency. Returns the balance of hot wallet if address not specified. For hot wallet also returns total_processing_amount and total_pending_amount for withdrawals in queue. For TON also returns account status: “active”, “uninit”, “frozen”, “non_exist”.

Example URI

GET /v1/balance?currency=&address=
URI Parameters
HideShow
currency
string (required) 

the text identifier of the currency specified in the processor configuration. TON for TON coin.

address
string (required) 

address in URL-safe, user-friendly Base64 form. Hot wallet address by default if the parameter is not specified.

Request
HideShow
Headers
Authorization: Bearer
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "balance": "1000000",
  "status": "active",
  "total_processing_amount": "1000",
  "total_pending_amount": "1000"
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Get sync flag

Get sync flag
GET/v1/system/sync

Get blockchain sync flag. Returns true if the service has up-to-date data from the blockchain. As long as the flag is equal to false, no withdrawals are made. Also returns last scanned workchain block gen_utime (unix time) to evaluate the processor’s lag from the blockchain.

Example URI

GET /v1/system/sync
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "is_synced": false,
  "last_block_gen_utime": 1718490850,
}
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Get income by transaction hash

Get income by transaction hash
GET/v1/deposit/income{?tx_hash}

Find income for deposit by tx_hash. Returns the currency and the deposit replenishment with the sender’s address as source_address (if it could be determined).

Example URI

GET /v1/deposit/income?tx_hash=
URI Parameters
HideShow
tx_hash
string (required) 

an unique hash of replenishment transaction

Request
HideShow
Headers
Authorization: Bearer
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "currency": "TON",
  "income": {
    "deposit_address": "0QB7BSerVyP9xAKnxp3QpqR8JO2HKwZhl10zsfwg7aJ281ZR",
    "time": 1680604643,
    "source_address": "0QAUuul9LdYcyJuBHernHo3JkbWTduH_FuEb2H8jCDdGesOP",
    "amount": "1000000",
    "comment": "hello",
    "tx_hash": "9d0fb69b1ca9371bc9e5260a248cda12a1c42916f9051fe9fc21b4abdd41d744"
  }
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  404
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Resolve domain name

Resolve domain name
GET/v1/resolve{?domain}

Resolve domain name. Returns bounceable user-friendly address for smart-contract (dns_smc_address#9fd3 DNS record).

Example URI

GET /v1/resolve?domain=
URI Parameters
HideShow
domain
string (required) 

domain name (for example: wallet.ton)

Request
HideShow
Headers
Authorization: Bearer
Response  200
HideShow
Headers
Content-Type: application/json
Body
{
  "address": "kQCKt2WPGX-fh0cIAz38Ljd_OKQjoZE_cqk7QrYGsNP6wfP0"
}
Response  400
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  401
Response  404
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}
Response  500
HideShow
Headers
Content-Type: application/json
Body
{
  "error": "error text"
}

Generated by aglio on 17 Jun 2024

================================================ FILE: go.mod ================================================ module github.com/gobicycle/bicycle go 1.24.0 toolchain go1.24.11 require ( github.com/caarlos0/env/v6 v6.10.1 github.com/gofrs/uuid v4.4.0+incompatible github.com/jackc/pgx/v4 v4.18.2 github.com/prometheus/client_golang v1.19.1 github.com/rabbitmq/amqp091-go v1.10.0 github.com/shopspring/decimal v1.4.0 github.com/sirupsen/logrus v1.9.3 github.com/tonkeeper/tongo v1.16.12 github.com/xssnick/tonutils-go v1.13.1 golang.org/x/time v0.10.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.14.3 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.3 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/oasisprotocol/curve25519-voi v0.0.0-20220328075252-7dd334e3daae // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3 // indirect github.com/snksoft/crc v1.1.0 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/exp v0.0.0-20230116083435-1de6713980de // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect google.golang.org/protobuf v1.33.0 // indirect ) ================================================ FILE: go.sum ================================================ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/oasisprotocol/curve25519-voi v0.0.0-20220328075252-7dd334e3daae h1:7smdlrfdcZic4VfsGKD2ulWL804a4GVphr4s7WZxGiY= github.com/oasisprotocol/curve25519-voi v0.0.0-20220328075252-7dd334e3daae/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3 h1:aQKxg3+2p+IFXXg97McgDGT5zcMrQoi0EICZs8Pgchs= github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3/go.mod h1:9/etS5gpQq9BJsJMWg1wpLbfuSnkm8dPF6FdW2JXVhA= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/snksoft/crc v1.1.0 h1:HkLdI4taFlgGGG1KvsWMpz78PkOC9TkPVpTV/cuWn48= github.com/snksoft/crc v1.1.0/go.mod h1:5/gUOsgAm7OmIhb6WJzw7w5g2zfJi4FrHYgGPdshE+A= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tonkeeper/tongo v1.16.12 h1:oAS9z0Kj7okTwVUV7QKAq3E6XjYmx+6DXfYaP8gg4S4= github.com/tonkeeper/tongo v1.16.12/go.mod h1:MjgIgAytFarjCoVjMLjYEtpZNN1f2G/pnZhKjr28cWs= github.com/xssnick/tonutils-go v1.13.1 h1:eWMD3KoRDX29gjAQIcLJU2Lnnzojr97xpzaMOEFjOeE= github.com/xssnick/tonutils-go v1.13.1/go.mod h1:EDe/9D/HZpAenbR+WPMQHICOF0BZWAe01TU5+Vpg08k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20230116083435-1de6713980de h1:DBWn//IJw30uYCgERoxCg84hWtA97F4wMiKOIh00Uf0= golang.org/x/exp v0.0.0-20230116083435-1de6713980de/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= ================================================ FILE: jettons.md ================================================ # Jettons compatibility information **! Be careful with Jettons metadata, the information may not be up-to-date, see the current one in the explorer or https://github.com/tonkeeper/ton-assets** ## TGR ### mainnet - Address: 0:2F0DF5851B4A185F5F63C0D0CD0412F5ACA353F577DA18FF47C936F99DBD849A - Code hash: 394e236fdaa478d218aa923b06dc0d3ad08ce65cbce5b0ab3ead62c61da69920 - Metadata: * name: Tegro * symbol: TGR * description: Cross-platform DeFi ecosystem Tegro * image: https://tegro.io/tgr.png - Tested with payment processor: - [x] Deposit filling from external address and withdrawal to hot wallet - [x] Withdrawal to new external address with wallet init - [ ] Highload test - Presented in https://github.com/tonkeeper/ton-assets: yes - Comments: sends notification with bounce flag ### testnet - Address: 0:8AB7658F197F9F874708033DFC2E377F38A423A1913F72A93B42B606B0D3FAC1 - Code hash: failed to calc via tonutils - Metadata: * name: Tegro * symbol: TGR * description: Cross-platform payment token on the TON blockchain. * image: https://tegro.io/tgr.png - Tested with payment processor: - [x] Deposit filling from external address and withdrawal to hot wallet - [x] Withdrawal to new external address with wallet init - [x] Highload test - Presented in https://github.com/tonkeeper/ton-assets: no - Comments: code not equal to actual TGR jetton in mainnet! Jetton with the equivalent code was not found in the testnet. ## SCALE ### mainnet - Address: 0:65aac9b5e380eae928db3c8e238d9bc0d61a9320fdc2bc7a2f6c87d6fedf9208 - Code hash: c81828dcd4df2a3a7516762c154dd973b92a26408db3bc143c7adda0576a9d6c - Metadata: * name: Scaleton * symbol: SCALE * description: SCALE is a utility token that will be used to support all independent developers. * image: ipfs://QmSMiXsZYMefwrTQ3P6HnDQaCpecS4EWLpgKK5EX1G8iA8 - Tested with payment processor: - [ ] Deposit filling from external address and withdrawal to hot wallet - [ ] Withdrawal to new external address with wallet init - [ ] Highload test - Presented in https://github.com/tonkeeper/ton-assets: yes - Comments: ### testnet - Address: 0:1b310ceeb868829ded11ed0123f67ad7b2333b80d8de566ae5059a2ec82f9208 - Code hash: f95ba0330b38cdf3459b1e811e5fc6fa6cfee566d7b764455c0468140365a737 - Metadata: * name: Scaleton * symbol: SCALE * description: * image: ipfs://QmSMiXsZYMefwrTQ3P6HnDQaCpecS4EWLpgKK5EX1G8iA8 - Tested with payment processor: - [x] Deposit filling from external address and withdrawal to hot wallet - [x] Withdrawal to new external address with wallet init - [x] Highload test - Presented in https://github.com/tonkeeper/ton-assets: no - Comments: code not equal to actual SCALE jetton in mainnet! ## FNZ ### mainnet - Address: 0:C224BD22407A1F70106F1411DD546DB7DD18B657890234581F453FA0A8077739 - Code hash: b4804ee49db9823eb5e9bdd98ff3784913c5dd01762dfd6e257f15be97b9fcf4 - Metadata: * name: Fanzee Token * symbol: FNZ * description: fanz.ee is a web3 fan engagement platform designed to help sports and entertainment organisations meaningfully connect with their fans through immersive gamification experiences * image: https://media.fanz.ee/images/91ee938a92934656a01131c569b377b6.png * decimals: 9 - Tested with payment processor: - [x] Deposit filling from external address and withdrawal to hot wallet - [x] Withdrawal to new external address with wallet init - [ ] Highload test - Presented in https://github.com/tonkeeper/ton-assets: yes - Comments: sends notification with non-bounce flag ### testnet - Address: 0:FCEE09C7BA28DDBA7B3F83EEBC6EC1B36B3AA3E40B98C3A69372DC1EA711353D - Code hash: b4804ee49db9823eb5e9bdd98ff3784913c5dd01762dfd6e257f15be97b9fcf4 - Metadata: * name: FANZEE test Coin * symbol: FNZT * description: This is an example jetton for the TON network * image: https://media.fanz.ee/images/91ee938a92934656a01131c569b377b6.png * decimals: 9 - Tested with payment processor: - [ ] Deposit filling from external address and withdrawal to hot wallet - [ ] Withdrawal to new external address with wallet init - [ ] Highload test - Presented in https://github.com/tonkeeper/ton-assets: no - Comments: code is equal with the mainnet jetton. ================================================ FILE: manual_migrations.md ================================================ # Manual migrations between versions ## v0.1.x -> v0.2.0 1. Apply [DB migration](/deploy/manual_migrations/0.1.x-0.2.0.sql) 2. Build new docker image and recreate container for `payment-processor` as described in `Service deploy` chapter in [Readme](/README.md) Note that this query creates a new column in the `external_incomes` DB table. All existing values for the payer address will be filled with a 0 workchain. ## v0.4.x -> v0.5.0 1. Apply [DB migration](/deploy/manual_migrations/0.4.x-0.5.0.sql) 2. Build new docker image and recreate container for `payment-processor` as described in `Service deploy` chapter in [Readme](/README.md) Note that this query creates a new nullable column in the `external_withdrawals` and `external_incomes` DB tables and `binary_comment` column in `withdrawal_requests` table. ================================================ FILE: manual_testing_plan.md ================================================ ## Manual testing plan for v0.5.0 Template: -[x] Checked - TEST : test description - RESULT : expected result - COMMENT : some comment ### Initialization 1. -[X] Checked - TEST : Run with not deployed (and zero balance) hot wallet (new seed phrase) - RESULT : There must be an insufficient balance error - COMMENT : 2. -[X] Checked - TEST : Run with uninit hot wallet with balance > minimum balance - RESULT : Hot wallet must be initialized at first start of service - COMMENT : 3. -[X] Checked - TEST : Run with new seed phrase when hot wallet already exist in DB - RESULT : There must be an incorrect seed phrase error - COMMENT : 4. -[X] Checked - TEST : Run service with empty DB and stop after few minutes. Check time of first and last block in `block_data` table - RESULT : Time `saved_at` and `gen_utime` must correlate with system time - COMMENT : 5. -[X] Checked - TEST : Run with nonexist hot jetton wallet and receive external jetton transfer at jetton deposit (> minimal withdrawal amount) - RESULT : Jetton hot wallet must be initialized by Jetton withdrawal from deposit, if jetton deposit successfully initialized (it depends on transfer sender) - COMMENT : 6. -[X] Checked - TEST : Run with testnet cold wallet address at mainnet (`IS_TESTNET=false`) - RESULT : There must be "Can not use testnet cold wallet address for mainnet" error - COMMENT : 7. -[X] Checked - TEST : Run service with empty `JETTONS` env variable - RESULT : Service must start and process TONs - COMMENT : 8. -[X] Checked - TEST : Run service with `JETTONS` env variable with different currencies and same master contract address. Like `TGR:ABC...,FNZ:ABC...`. - RESULT : Service must stop. Must be address duplication error message in audit log. - COMMENT : 9. -[X] Checked - TEST : Run service with one `JETTONS` env variable, then rename currency for one of Jetton and restart. Like `TGR:ABC...,FNZ:CDE...` -> `SCALE:ABC...,FNZ:CDE...`. - RESULT : Service must stop. Must be address duplication error message in audit log. - COMMENT : 10. -[X] Checked - TEST : Start service with uninitialized cold wallet and bounceable address for cold wallet. - RESULT : Service must stop. Must be invalid address format error message in log. - COMMENT : 11. -[X] Checked - TEST : Start service with `PROOF_CHECK_ENABLED=true` and empty or invalid `NETWORK_CONFIG_URL` ENV variable. - RESULT : Service must stop. Must be blockchain connection error message in log. - COMMENT : 12. -[X] Checked - TEST : Start service with `PROOF_CHECK_ENABLED=false` ENV variable. - RESULT : Service must start normally. - COMMENT : 13. -[ ] Checked - TEST : Start service with `PROOF_CHECK_ENABLED=true` and valid `NETWORK_CONFIG_URL` ENV variable. - RESULT : Service must start normally. Must be `Proof checks are completed` message in log. - COMMENT : Proof check can not get account state for each workchain block. ### API 1. -[X] Checked - TEST : Use `/v1/address/new` method (few for TONs and few for Jettons for different users). Check new addresses in DB - RESULT : You must receive different addresses in user-friendly format with `bounce = false` flag and testnet flag correlated with `IS_TESTNET` env var and raw in DB. For Jetton deposits it must be an owner address. - COMMENT : 2. -[X] Checked - TEST : Use `/v1/address/all{?user_id}` method and compare with addresses created at 1. And check it by DB - RESULT : All addresses must be received and equal to those created earlier - COMMENT : 3. -[X] Checked - TEST : Check `/v1/income{?user_id}` for new empty deposits - RESULT : Income must be zero. The addresses must match the addresses obtained by method `/v1/address/all{?user_id}`. - COMMENT : 4. -[X] Checked - TEST : Make some payments at deposits and check it by `/v1/income{?user_id}` method with different `DEPOSIT_SIDE_BALANCE` env var - RESULT : Income must correlate with payments sum - COMMENT : 5. -[X] Checked - TEST : Make some withdrawals by `/v1/withdrawal/send` method for TONs and Jettons with amount > hot wallet balance and check it by `/v1/withdrawal/status{?id}` few times. Check status of withdrawals by transaction explorer (e.g. https://testnet.tonapi.io/ or https://tonapi.io/). Check withdrawal in DB. - RESULT : The withdrawal must be in the `pending` state and the wallet must not send any messages. There is no any correlated messages in `external withdrawals` table. - COMMENT : 6. -[X] Checked - TEST : Make some withdrawals by `/v1/withdrawal/send` method for TONs and check it by `/v1/withdrawal/status{?id}` few times and try to catch all statuses: `pending`, `processing`, `processed`. Check status of withdrawals by transaction explorer (e.g. https://testnet.tonapi.io/ or https://tonapi.io/). Check withdrawal in DB. - RESULT : Withdrawal status and transaction hash must correlate with explorer status. In `withdrawal_requests` DB table must be `processing=true`, `processed=true` and `failed=false`, `confirmed=false` in `external_withdrawals` table as final state. - COMMENT : 7. -[X] Checked - TEST : Make some withdrawals by `/v1/withdrawal/send` method for Jettons with not deployed Jetton hot wallets and withdrawals by `/v1/withdrawal/status{?id}` few times. Check status of withdrawals by transaction explorer (e.g. https://testnet.tonapi.io/ or https://tonapi.io/). Check withdrawal in DB. - RESULT : The withdrawal must be in the `pending` state and the wallet must not send any messages. There is no any correlated messages in `external withdrawals` table. - COMMENT : 8. -[X] Checked - TEST : Make some withdrawals by `/v1/withdrawal/send` method for Jettons and TONs with invalid `binary_comment` (two comments, invalid data, too long) - RESULT : The withdrawal must be rejected with bad request error. - COMMENT : 9. -[X] Checked - TEST : Make some withdrawals by `/v1/withdrawal/send` method for Jettons and TONs with short and long `comment` (not fitted into one cell) - RESULT : The withdrawal must be finished successfully. Comment in explorer must correlate with request. - COMMENT : 10. -[X] Checked - TEST : Make some withdrawals by `/v1/withdrawal/send` method for Jettons and TONs with short and long `binary_comment` (not fitted into one cell) - RESULT : The withdrawal must be finished successfully. Payload in explorer must correlate with request. - COMMENT : 11. -[X] Checked - TEST : Make some withdrawals by `/v1/withdrawal/send` method for Jettons with deployed Jetton hot wallets and check it by `/v1/withdrawal/status{?id}` few times and try to catch all statuses: `pending`, `processing`, `processed`. Check status of withdrawals by transaction explorer (e.g. https://testnet.tonapi.io/ or https://tonapi.io/). Check withdrawal in DB. - RESULT : Withdrawal status must correlate with explorer status. In `withdrawal_requests` DB table must be `processing=true`, `processed=true` and `failed=false`, `confirmed=true` in `external_withdrawals` table as final state. - COMMENT : 12. -[X] Checked - TEST : Start the service after some downtime. Check sync status by `/v1/system/sync` few times. - RESULT : Start status should be `"is_synced": false` then become `"is_synced": true` - COMMENT : 13. -[X] Checked - TEST : Start service with `IS_TESTNET=true` env var. Make some withdrawals by `/v1/withdrawal/send` method to TESTNET and MAINNET user-friendly form addresses. - RESULT : All withdrawals must be accepted - COMMENT : 14. -[X] Checked - TEST : Start service with `IS_TESTNET=false` env var. Make some withdrawals by `/v1/withdrawal/send` method to TESTNET user-friendly form addresses. - RESULT : All withdrawals must be rejected - COMMENT : 15. -[X] Checked - TEST : Make some withdrawals by `/v1/withdrawal/send` method to service internal addresses (hot wallet, jetton hot wallet, owner, ton deposit, jetton deposit). - RESULT : All withdrawals must be rejected - COMMENT : 16. -[X] Checked - TEST : Try all methods with auth using wrong token - RESULT : All requests must be rejected - COMMENT : 17. -[X] Checked - TEST : Make TON and Jetton withdrawal to -1 workchain - RESULT : Withdrawals must be rejected - COMMENT : 18. -[X] Checked - TEST : Make withdrawals by `/v1/withdrawal/service/ton` and `/v1/withdrawal/service/jetton` from unknown address and another network (testnet addr for mainnet address and -1 workchain address) - RESULT : Withdrawals must be rejected - COMMENT : 19. -[X] Checked - TEST : Make withdrawal by `/v1/withdrawal/service/ton` and `/v1/withdrawal/service/jetton` from known internal but not Jetton deposit owner and not TON deposit address (hot wallet and Jetton wallet) - RESULT : Withdrawal must be rejected - COMMENT : 20. -[X] Checked - TEST : Make TON withdrawal by `/v1/withdrawal/service/ton` from known Jetton deposit owner address. Check for unusual transactions in the database - RESULT : Withdrawal must be accepted. All TONs must be sent from Jetton deposit owner to hot wallet. There is no any deposit transactions (incomes/withdrawals) in the database. In `service_withdrawal_request` DB table must be `processed = true` - COMMENT : 21. -[X] Checked - TEST : Make Jetton (not deposit Jetton type) withdrawal by `/v1/withdrawal/service/jetton` from known internal Jetton deposit owner address. Check for unusual transactions in the database - RESULT : Withdrawal must be accepted. Jettons must be sent from Jetton wallet to hot wallet. There is no any deposit transactions (incomes/withdrawals) in the database. In `service_withdrawal_request` DB table must be `processed = true` - COMMENT : Should be zero forward TON amount in transfer message to prevent invoking notification message and incorrect interpretation hot wallet incoming message 22. -[X] Checked - TEST : Make Jetton withdrawal by `/v1/withdrawal/service/jetton` from known internal TON deposit address. Check for unusual transactions in the database - RESULT : Withdrawal must be accepted. First, there must be a TON filling transaction from hot wallet to TON deposit. The status in `service_withdrawal_request` DB should be changed to `filled=true`. Later, the Jettons should be sent from Jetton wallet to the hot wallet and in DB table should be `processed = true` and `filled=false`. There is no any deposit transactions (incomes/withdrawals) in the database. There should be audit log warning about withdrawal from TON deposit to non-hot wallet. - COMMENT : The balance on the deposit side is calculated correctly, but the withdrawal of Jettons (and TONs) from the TON deposit occurs through the Jetton wallet and is not detected by the block scanner as an internal TON withdrawal. The deposit balance on the hot wallet side is not replenished. 23. -[X] Checked - TEST : Make Jetton (for deposit Jetton type) withdrawal by `/v1/withdrawal/service/jetton` from known internal Jetton deposit owner address. - RESULT : In `service_withdrawal_request` DB table should be `processed = true` and `filled=false` with zero balances. Must be warning about rejected withdrawal in audit log - COMMENT : 24. -[X] Checked - TEST : Make TON withdrawal by `/v1/withdrawal/service/ton` from Jetton owner address with zero TON balance - RESULT : Withdrawal must be accepted. There should be audit log info about zero balance. There is no any deposit transactions (incomes/withdrawals) in the database and no messages from hot wallet. In `service_withdrawal_request` DB table must be `processed = true` - COMMENT : 25. -[X] Checked - TEST : Make Jetton withdrawal by `/v1/withdrawal/service/jetton` from Jetton owner address and from TON deposit address with zero Jetton balance - RESULT : Withdrawal must be accepted. There should be audit log info about zero balance. There is no any deposit transactions (incomes/withdrawals) in the database and no messages from hot wallet. In `service_withdrawal_request` DB table must be `processed = true` - COMMENT : 26. -[X] Checked - TEST : Set some Jetton in `JETTONS` env variable. Start service to init jetton hot wallet in DB. Remove Jetton from env variable and restart. Try `/v1/address/new`, `/v1/withdrawal/send` for removed Jetton. - RESULT : Must be currency error for `/v1/address/new`, `/v1/withdrawal/send`. - COMMENT : 27. -[ ] Checked - TEST : Set some Jetton in `JETTONS` env variable. Start service to init jetton hot wallet in DB. Remove Jetton from env variable and restart. Try `/v1/address/all`, `/v1/income` for user with removed Jetton deposits. - RESULT : Removed Jetton should not appear in `/v1/address/all`, `/v1/income`. - COMMENT : Not implemented yet 28. -[X] Checked - TEST : Make some payments at deposits and check it by `/v1/history{?user_id,currency,limit,offset}` method with different `DEPOSIT_SIDE_BALANCE` env var - RESULT : Incomes must correlate with payments and DB `external_incomes` table. The history on the deposits side should always be displayed. - COMMENT : 29. -[ ] Checked - TEST : Replenish the TON deposit from the masterchain wallet and check it by `/v1/deposit/history{?user_id,currency,limit,offset}` method. - RESULT : The sender's address must be displayed correctly in the history. - COMMENT : 30. -[X] Checked - TEST : Replenish the TON deposit (when it in nonexist status) with a bounceable message and check it by `/v1/deposit/history{?user_id,currency,limit,offset}` method. Also check logs. - RESULT : The bounced payment should not be in the history. There should be no errors in the logs, only a warning about a bounced message. - COMMENT : 31. -[X] Checked - TEST : Replenish the Jetton deposit with zero forward amount and check it by `/v1/deposit/history{?user_id,currency,limit,offset}` method. - RESULT : The sender's address must be not presented in the history. - COMMENT : 32. -[X] Checked - TEST : Replenish the TON and Jetton deposit with some amount and check it by `/v1/deposit/income{?tx_hash}` method. - RESULT : Incomes must correlate with payments and DB `external_incomes` table. - COMMENT : 33. -[X] Checked - TEST : Replenish the Jetton deposit with zero forward amount and check it by `/v1/deposit/income{?tx_hash}` method. - RESULT : The sender's address must be not presented in the response value must be nonzero. - COMMENT : 34. -[X] Checked - TEST : Try to find non-existent `tx_hash` by `/v1/deposit/income{?tx_hash}` method. - RESULT : The method should return a 404 error. - COMMENT : 35. -[X] Checked - TEST : Check balance of hot wallet by `/v1/balance{?currency}` (without address parameter) method for TON and jettons. - RESULT : The method should return actual balance for hot wallet and correct withdrawal amounts. - COMMENT : 36. -[X] Checked - TEST : Check balance of custom account by `/v1/balance{?currency,address}` method for TON and jettons. - RESULT : The method should return actual balance for this account. - COMMENT : 37. -[X] Checked - TEST : Check balance of custom noexist account without jetton wallet by `/v1/balance{?currency,address}` method for TON and jettons. - RESULT : The method should return zero balance for this account. - COMMENT : 38. -[X] Checked - TEST : Check balance of custom account by `/v1/balance{?currency,address}` with invalid parameters (unknown currency, testnet address for mainnet). - RESULT : The method should return bad request error. - COMMENT : 39. -[X] Checked - TEST : Resolve address of custom account by `/v1/resolve{?domain}` with valid `dns_smc_address` record. - RESULT : The method should return valid user-friendly address with bounceable flag and testnet/mainnet flag. - COMMENT : 40. -[X] Checked - TEST : Resolve address of custom account by `/v1/resolve{?domain}` with invalid domain or without `dns_smc_address` DNS record. - RESULT : The method should return not found error. - COMMENT : 41. -[X] Checked - TEST : Check last block time by `/v1/system/sync`. - RESULT : Last block time must correlate with DB records. - COMMENT : 42. -[X] Checked - TEST : Check balance of hot wallet by `/v1/balance{?currency}` (without address parameter) method for TON and jettons. And check amounts of processing and pending amount. - RESULT : The method should return actual balance for hot wallet and correct withdrawal amounts. - COMMENT : ### Internal logic 1. -[X] Checked - TEST : Replenish the deposit with TONs and Jettons so that as a result the amount on the hot wallet is greater than `hot_wallet_max_balance` when cold wallet is not active and cold wallet address in non-bounceable format. Check withdrawals in DB - RESULT : You must find new withdrawal in `withdrawal_requests` table with `is_internal=true` and `bounceable=false`. And final status must correlate with explorer. - COMMENT : 2. -[ ] Checked - TEST : Start the service while a workchain merges and splits. Check the integrity of the chain of blocks in the table by comparing with the explorer. - RESULT : There should be no missing blocks in the DB. - COMMENT : 3. -[X] Checked - TEST : Replenish the deposit with TONs and Jettons so that as a result the amount on the hot wallet is greater than `hot_wallet_max_balance`. Try with and without `hot_wallet_residual_balance` parameter. Check withdrawals in DB - RESULT : You must find new withdrawal in `withdrawal_requests` table with `is_internal=true`. And final status must correlate with explorer. Withdrawal amount must correlate with hysteresis formula (and `hot_wallet_residual_balance` parameter). - COMMENT : 4. -[X] Checked - TEST : Set invalid `FORWARD_TON_AMOUNT` env (negative, > max value) - RESULT : Service failed with load config error. - COMMENT : 5. -[X] Checked - TEST : Set `FORWARD_TON_AMOUNT` env to 1 nanoton. And check external withdrawals and internal withdrawals. - RESULT : All withdrawals must be successful and confirmed. - COMMENT : 6. -[X] Checked - TEST : Not set `FORWARD_TON_AMOUNT` env. And check external withdrawals and internal withdrawals. - RESULT : All withdrawals must be successful and confirmed. Forward ton amount must be 1 nanoton. - COMMENT : 7. -[X] Checked - TEST : Set `FORWARD_TON_AMOUNT` env to 0 nanoton. And check external withdrawals and internal withdrawals. - RESULT : All withdrawals must be successful and confirmed and there is no transfer notification message for external withdrawals and 1 nanoton forward amount for internal transfer. - COMMENT : ### Deploy 1. -[X] Checked - TEST : Build docker images and start `payment-postgres`, `payment-processor` services using README.md instructions Check availability and functionality of service. - RESULT : The API must be accessible and functional - COMMENT : 2. -[X] Checked - TEST : Start optional `payment-grafana` service using README.md instructions Check availability and functionality of service. - RESULT : The `payments` Grafana dashboard must be accessible and show DB data - COMMENT : 3. -[X] Checked - TEST : Start `payment-processor` with `QUEUE_ENABLED=true` env var and optional `payment-rabbitmq` service using README.md instructions. Make some payments to deposits. Check availability and functionality of service by RabbitMQ dashboard - RESULT : Must be some message activity in RabbitMQ dashboard for exchange - COMMENT : 4. -[ ] Checked - TEST : Start `payment-test` service using technical_notes.md instructions with `CIRCULATION=false` env variable. Check availability and functionality of service by Grafana dashboard. - RESULT : Grafana must show prometheus metrics from `payment-test` service (deposit and total balances) - COMMENT : 5. -[ ] Checked - TEST : Start `payment-test` service using technical_notes.md instructions with `CIRCULATION=true` env variable. Check availability and functionality of service by Grafana dashboard. - RESULT : Grafana must show prometheus metrics from `payment-test` service (deposit and total balances) and payment activity - COMMENT : 6. -[X] Checked - TEST : Start `payment-processor` with `WEBHOOK_ENDPOINT=http://localhost:3333/webhook` env var. Start test webserver from `cmd/testwebhook/main.go`. Make some payments to deposits. Check payments data at webserver side. Add env variable `WEBHOOK_TOKEN=123` and restart `payment-processor`. Make some payments to deposits. Check payments data at webserver side. - RESULT : Must be payments log activity in webserver and warning about the absence of a token when the variable `WEBHOOK_TOKEN` is not set. - COMMENT : 7. -[X] Checked - TEST : Start `payment-processor` with webhooks. Make Jetton payment to deposits with zero froward amount. Check payments data at webserver side. - RESULT : The sender's address must be not presented. - COMMENT : 8. -[X] Checked - TEST : Up `payment-postgres` from `docker-compose.yml` with named volume. Write some data to DB. Remove the container via down (without -v flag) and up it again. Check that the data is available in the DB. - RESULT : The data in the database should be preserved after the container is recreated. - COMMENT : 9. -[X] Checked - TEST : Check availability of Prometheus metrics. Run the prometheus container from the `docker-compose.yml` file and check the metrics via the web interface. - RESULT : Error counter metrics should be available in the Prometheus web interface. - COMMENT : 10. -[X] Checked - TEST : Check migration 0.4.x-0.5.0.sql. Create version v0.4.0 DB, then make several withdrawals (preferably with lost messages) that are in the status `processed` and `processing`. Then stop the processor and apply the migration. Launch the new version of the processor and check that the outputs were processed normally and no duplicate outputs occurred. - RESULT : Withdrawals must be made correctly, there should be no duplicate withdrawals. - COMMENT : ### Stability test 1. -[X] Checked - TEST : Start `payment-test` service using technical_notes.md instructions with `CIRCULATION=true` env variable for long time (with enough amount of test TONs on wallet). Periodically check availability and functionality of service by Grafana dashboard and docker logs. - RESULT : There should be no abnormal behavior of service and errors in log - COMMENT : Add reconnect when timeout expires ### Highload test 1. -[ ] Checked - TEST : Start `payment-test` service using technical_notes.md instructions with `CIRCULATION=true` env variable and depositsQty = 100 x 3 types of deposits (with enough amount of test TONs on wallet). Periodically check availability and functionality of service by Grafana dashboard and docker logs. - RESULT : There should be no abnormal behavior of service and errors in log - COMMENT : 2. -[ ] Checked - TEST : Generate a large number of deposits (1 million) and run the processor for a long time. periodically make payments on deposits. Periodically check availability and functionality of service by Grafana dashboard and docker logs. - RESULT : There should be no abnormal behavior of service and errors in log - COMMENT : ================================================ FILE: metrics/metrics.go ================================================ package metrics import ( "fmt" "io" "strconv" "sync/atomic" ) type counter struct { name, description string counter uint64 } func (c *counter) Inc() { atomic.AddUint64(&c.counter, 1) } func (c *counter) Add(n uint64) { atomic.AddUint64(&c.counter, n) } func (c *counter) Print(w io.Writer) error { if c.description != "" { _, err := fmt.Fprintf(w, "# HELP %s %s\n", c.name, c.description) if err != nil { return err } } _, err := fmt.Fprintf(w, "# TYPE %s counter\n%s{} %s\n", c.name, c.name, strconv.FormatUint(atomic.LoadUint64(&c.counter), 10)) return err } type printer interface { Print(w io.Writer) error } var ( Errors = &counter{name: "errors", description: "number of errors since the service was launched. see logs for details", counter: 0} Warnings = &counter{name: "warnings", description: "number of warnings since the service was launched. see logs for details", counter: 0} Info = &counter{name: "info", description: "number of infos since the service was launched. see logs for details", counter: 0} AllMetrics = []printer{Errors, Warnings, Info} ) ================================================ FILE: queue/queue.go ================================================ package queue import ( "encoding/json" "fmt" amqp "github.com/rabbitmq/amqp091-go" ) type AmqpClient struct { connection *amqp.Connection channel *amqp.Channel exchange string enabled bool Subscription <-chan amqp.Delivery } // NewAmqpClient creates new AMQP client and declare new exchange func NewAmqpClient(uri string, enabled bool, queueName string) (*AmqpClient, error) { if !enabled { return &AmqpClient{enabled: false}, nil } conn, err := amqp.Dial(uri) if err != nil { return nil, err } ch, err := conn.Channel() if err != nil { return nil, err } client := &AmqpClient{connection: conn, channel: ch, enabled: enabled} err = client.declareExchange(queueName) if err != nil { return nil, err } return client, nil } func (c *AmqpClient) declareExchange(exchangeName string) error { err := c.channel.ExchangeDeclare( exchangeName, "fanout", false, false, false, false, nil, ) if err != nil { return err } c.exchange = exchangeName return nil } // Publish publishes any payload to queue func (c *AmqpClient) Publish(payload any) error { if !c.enabled { return nil } if c.exchange == "" { return fmt.Errorf("exchange not init") } body, err := json.Marshal(payload) if err != nil { return err } err = c.channel.Publish( c.exchange, "", false, false, amqp.Publishing{ ContentType: "application/json", Body: body, }) return err } ================================================ FILE: release_notes.md ================================================ # v0.5.0 release notes 1. Add `tx_hash` field, `sort_order` parameter to `/v1/deposit/history{?user_id,currency,limit,offset}` method 2. Add `tx_hash`, `user_id`, `query_id` fields to `/v1/withdrawal/status{?id}` method response 3. Add `failed` status to `/v1/withdrawal/status{?id}` method and withdrawal processor do not retry failed withdrawals (retry only for lost external messages) 4. New method `/v1/deposit/income{?tx_hash}` to find income by `tx_hash` 5. New method `/v1/balance{?currency,address}` to get account balance from blockchain (and account status for TON) and pending and processing amounts for hot wallet 6. Push notifications (webhook or rabbitmq) after save to db (see new notification logic in `README.md` and `technical_notes.md`) 7. Postgres docker volume permanent by default 8. Audit log error counters with prometheus metrics 9. Update dependencies 10. Bugfixes 11. Improve stability 12. New method `/v1/resolve{?domain}` to resolve wallet DNS record 13. Add `last_block_gen_utime` field to `/v1/system/sync` method to get unix time of the last scanned block 14. Add `FORWARD_TON_AMOUNT` env variable to customize `forward_ton_amount` for Jetton withdrawals 15. Binary comment support # v0.6.0 release notes 1. Displaying the application version at startup # v0.7.0 release notes 1. Emulator fix 2. Total proof check available # v0.8.0 release notes 1. Allow to set `FORWARD_TON_AMOUNT = 0` env variable to disable transfer notification message for external jetton transfer 2. Download last blockchain config at start ================================================ FILE: technical_notes.md ================================================ # Technical notes - [Glossary](#Glossary) - [Limitations](#Limitations) - [API blueprint](/docs/api.apib) - [Wallets generation](#Wallets-generation) - [Healthcheck](#Healthcheck) - [Transfers layouts](#Transfers-layouts) - [Withdrawal mechanism](#Withdrawal-mechanism) - [Shard tracker algorithm](#Shard-tracker-algorithm) - [Block scanner algorithm](#Block-scanner-algorithm) - [Restart policy](#Restart-policy) - [Service withdrawals](#Service-withdrawals) - [Calibration parameters](#Calibration-parameters) - [Freezing and deleting unused accounts](#Freezing-and-deleting-unused-accounts) - [Highload wallet message deduplication](#Highload-wallet-message-deduplication) - [Audit log](#Audit-log) - [Sharding](#Sharding) - [Notification](#Notifications) - [Running the test util for payment processor](#Running-the-test-util-for-payment-processor) ## Glossary * `sharding` - when the network needs to process a large number of account transactions, in order to distribute the load, accounts begin to be grouped into separate blocks (shard blocks). The shard block contains accounts that have the same bit prefix in the address. * `subwallets` - a wallets (addresses) derived from the same seed but with a different subwallet_id. * `jetton_wallet_owner` - TON wallet (Highload V2 for hot) OR special proxy contract owner of the jetton wallet smartcontract. * `ext_msg` - external in message in TON blockchain * `int_msg` - internal message in TON blockchain * `external wallet` - some wallet not hot and not deposit * `internal withdrawal` - withdrawal from deposit wallet to hot wallet * `external withdrawal` - withdrawal from hot wallet to external wallet * `internal income` - transfer from deposit wallet to hot wallet * `external income` - transfer from external wallet to deposit wallet * `external withdrawals processor` - service that performs external withdrawals * `internal withdrawals processor` - service that performs internal withdrawals * `expiration processor` - service that tracks expired and unconfirmed withdrawals * `block scanner` - service that extracts and decodes transactions and messages from blocks * `shard tracker` - utility for receiving blocks from blockchain with custom shard prefix ## Limitations * Supports up to 256 shards only * Supports only 0 workchain * For deposit addresses used only 32 byte addr_std * Var address of senders saves as nil * Withdrawals to deposit addresses is prohibited * Manual withdrawals from hot wallet is prohibited **do not withdraw funds from a hot wallet bypassing the service, this will lead to a service error** ### Sub-wallets (deposit) qty limitation - `subwallet_id` Go type: `uint32` (postgresql bigint type) - `max(uint32) = 4294967295` - `default_subwallet_id = 698983191` (for main hot wallet) - DB numeration starts from default `subwallet_id` - part of wallets in shard (for 1 byte shard prefix): 1/256 - maximum qty of subwallets: `(max(uint32) - default_subwallet_id)/256 = 14_046_812` ## Wallets generation 1. Generates `hot_ton_wallet` (if needed) from seed phrase as Highload V2 wallet. The first byte of the address define the shard. 2. Generates `hot_jetton_wallet` for each Jetton. (not in shard) `hot_ton_wallet` - owner for all of this jetton wallets. 3. Generates `deposit_ton_wallet` as V3R2 from hot wallet seed phrase and subwallet_id to set the desired shard. 4. Generates `jetton_wallet_owner` as proxy contract with `hot_ton_wallet` address as owner and subwallet_id to set the desired shard for `deposit_jetton_wallet`. ## Healthcheck 1. Checks time on liteserver after start. If service-liteserver time diff > configured value then service not starts. To avoid fail of message expiration checks. 2. REST API provides method to check blockchain sync flag. ## Transfers layouts ### Replenishment of the TON deposit by payer Standard case. Instead of `payer_ton_wallet` there may be another smart contract. 1. `ext_msg` -> `payer_ton_wallet` 2. `payer_ton_wallet` -> `msg (internal)` -> `deposit_ton_wallet` ### Replenishment of the Jetton deposit by payer Standard case. Instead of `payer_jetton_wallet_owner` there may be another smart contract. 1. `ext_msg (payload = transfer)` -> `payer_jetton_wallet_owner` 2. `payer_jetton_wallet_owner` -> `int_msg (value > fees, body = transfer)` -> `payer_jetton_wallet` 3. `payer_jetton_wallet` -> `int_msg (body = internal_transfer, init)` -> `deposit_jetton_wallet` 4. *optional* `deposit_jetton_wallet` -> `int_msg (value = excesses, body = excesses)` -> `response_destination` 5. *optional* `deposit_jetton_wallet` -> `int_msg (body = transfer_notification)` -> `deposit_jetton_wallet_owner` ### Withdraw TONs from deposit to hot wallet 1. `ext_msg (init, mode = 128 + 32, comment = memo)` -> `deposit_ton_wallet` // init wallet, send all TONs and destroy 2. `deposit_ton_wallet` -> `int_msg (value = all_TONs, body = memo)` -> `hot_ton_wallet` ### Withdraw Jettons from deposit to hot wallet 1. `ext_msg` -> `hot_ton_wallet` 2. `hot_ton_wallet` -> `int_msg (init, value > fees, body = msg_with_transfer_body)` -> `deposit_jetton_wallet_owner` // fees for notify and excesses. init of proxy contract 3. `deposit_jetton_wallet_owner` -> `int_msg (mode = 128 + 32, body = transfer, comment = memo, forward_ton_amount > fees)` -> `deposit_jetton_wallet` 4. `deposit_jetton_wallet` -> `int_msg (jetton_value = all_jettons, forward_ton_amount > fees, body = internal_transfer, comment = memo)` -> `hot_jetton_wallet` 5. `hot_jetton_wallet` -> `int_msg (body = transfer_notification, comment = memo)` -> `hot_ton_wallet` 6. `hot_jetton_wallet` -> `int_msg (value = excesses, body = excesses)` -> `hot_ton_wallet` ## Withdrawal mechanism ### Criteria for successful withdrawal #### TON internal withdrawal 1. Incoming int_msg with unique memo is found at hot_ton_wallet 2. Transaction is success (message not bounced and computation phase is success) #### Jetton internal withdrawal 1. Incoming transfer_notification int_msg with unique memo is found at hot_ton_wallet (as owner of hot_jetton_wallet) #### TON external withdrawal 1. Outgoing int_msg is found at hot_ton_wallet. Transaction success on the receiver's side is not checked! (it depends on user request) #### Jetton external withdrawal 1. Incoming int_msg (excesses) with unique query_id is found at hot_ton_wallet (as response destination address) ### Algorithm of internal withdrawal 1. Get transactions on deposit_wallets from blockchain and save to DB as external incomes 2. Periodically get last (by LT) external income for deposit from DB 3. Check for pending internal withdrawals in DB 4. If there is no pending internal withdrawals and last success internal withdrawal LT < last external income LT then create internal withdrawal task 5. Check balance of deposit wallet 6. If balance > minimum value for withdrawal then save internal withdrawal task to DB with sinceLT = last external income LT, expiration time and unique memo 7. Make withdrawal message with memo and send 8. *Get message from blockchain (out in_msg for TON withdrawal or success transfer in_msg for Jetton) and save to DB `finish_lt` and `failed` if needed 9. ** Mark expired (and not found in blockchain) withdrawals in DB as `failed` 10. Get internal income (success in_msg for TON or transfer_notification in_msg for Jetton) from blockchain at hot_wallet 11. Save internal income to DB *- There is a specific behavior for TON withdrawal. When we make a withdrawal with the 128+32 mode from deposit wallet (i.e. the destruction of the account), we reset its data and seqno. However, the external message may be in node mem-pool for some time (about 10 minutes). If during this time the account balance is replenished, then this message will apply again by creating a duplicate message with the same memo. On the hot wallet side, we scan all messages with a unique (memo+LT). For withdrawal in DB we save total amount and last LT for memo. **- Since the message lifetime refers to an external message to the wallet, and the success of the withdrawal of Jettons is checked in the transaction on the Jetton wallet, then in order to correctly check expiration, it is necessary to save the time/Lt of the intermediate transaction (on hot wallet). If the entire transaction chain ends up in one block, then the order in which the intermediate and final messages are saved will also be important. ### Using a proxy contract to withdraw Jettons If a regular wallet (V3R2, which is stored in an empty state to avoid storage fees) is used as the jetton_wallet_owner, then the withdrawal of Jettons takes place in three stages. 1. Replenishment of a jetton_wallet_owner from hot wallet 2. Waiting for the replenishment to arrive and the account status will be uninit 3. Sending a Jetton transfer message To avoid such a complex sequence of actions, a proxy contract is used as the jetton_wallet_owner. This proxy forwards Jetton transfer message from hot wallet and self destroy. In order for the Jetton wallet to have a suitable address for the shard, the proxy contract applies subwallet_id. [Proxy contract source code](https://github.com/gobicycle/ton-proxy-contract) ### Algorithm of external withdrawal 1. Get batch of TON withdrawal requests from DB with different destination addresses (in raw form) 2. Mark withdrawal requests as `processing` in DB 3. Make ext_msg for hot_ton_wallet with withdrawals and expiration time 4. Save withdrawals, ext_msg uuid (as part of msg hash) and expiration time to DB 5. Send ext_msg 6. Get transaction with ext_msg from blockchain 7. Get out messages from transaction, save and mark correlated withdrawals as `processed` in DB 8. If some withdrawals is presented in ext_msg and not presented in out in_msg - mark correlated withdrawals as `failed` in DB 9. For Jetton withdrawals: get transaction with excesses in_msg with unique query_id from blockchain and mark correlated withdrawal as `confirmed` in DB 10. Mark expired (and not found in blockchain) withdrawals in DB as `failed` and reset withdrawal requests `processing` flag ## Shard tracker algorithm 1. Get last masterchain block 2. Get all shard blocks from masterchain block 3. Filter shard blocks by custom shard prefix 4. Get parent shard block for filtered shard block. If block has two parents then filter parents by shard prefix. 5. Repeat 4. until find last known shard block 6. Save all found shard blocks in memory 7. Provide the following block on request by Next() method and remove from memory 8. If there is no blocks in memory goto 1 ## Block scanner algorithm 1. Get next shard block with custom shard prefix from shard tracker 2. Get TxIDs for block 3. Filter TxIDs by known addresses 4. Get TXs from blockchain by TxIDs 5. Decode TXs and messages and save to DB 6. Goto 1. ### Nuances of detecting Jetton transactions Since the contract code of Jetton wallets can be different, and the standard describes only the format of the transfer message (the internal transfer is not standardized), and there may be no excess and notification messages, that is, the nuances of detecting Jetton transactions. Transactions are tracked on the deposit Jetton wallet. If there is a transfer notification message in the transaction, then it is decoded and the decoded data is stored in the database. But since there may be events that change the balance of the wallet (depending on the wallet code), the balance is additionally checked for the previous block and the current one, the known value is subtracted and the result is written to the database in the form of replenishment with an unknown sender. ## Restart policy The service must be resistant to restart and long downtime. All operations before being sent to the blockchain must be saved in the database with the status and expiration time. Withdrawal processors and expiration processor are suspended until the block scanner is synchronized. The block scanner is considered synchronized when (now - last_saved_block_gen_utime) < custom preset. ## Service withdrawals Method for service withdrawals available. The method is used in the following cases: #### 1. Mistaken transfer of TONs to the address of the proxy contract (the owner of the deposit Jetton wallet) when the TONs are still there If an internal withdrawal has already been made from the Jetton wallet, then the tones from the contract proxy should have already returned to the hot wallet and no additional actions are needed. Use `/v1/withdrawal/service/ton` if the TONs are still at proxy contract. It makes direct withdrawal of all TONs from proxy contract to hot wallet. #### 2. Mistaken transfer of Jettons to the address of TON deposit Use `/v1/withdrawal/service/jetton`. It makes withdrawal of all Jettons from Jetton wallet (not deposit) to hot wallet. **! Be careful with this method.** This method withdraw all TONs from deposit to hot wallet, but balance replenish at hot wallet side not detect. Use this method with zero or near zero deposit TON balance. #### 3. Mistaken transfer of unexpected Jetton type to the address of the proxy contract Use `/v1/withdrawal/service/jetton`. It makes withdrawal of all Jettons from Jetton wallet (not deposit) to hot wallet. ## Calibration parameters ### TESTNET Single highload message Jetton transfer to not deployed Jetton wallet (SCALE Jetton): - transfer message value - 0.1 TON - forward TON amount - 0.02 TON (for notification message) - excess - 0.033 TON - total loss = 0.1 - 0.033 = 0.067 TON Single highload message Jetton transfer to already deployed Jetton wallet (SCALE Jetton): - transfer message value - 0.1 TON - forward TON amount - 0.02 TON (for notification message) - excess - 0.042 TON - total loss = 0.1 - 0.042 = 0.058 TON ### MAINNET Single highload message Jetton transfer to not deployed Jetton wallet (TGR Jetton): - transfer message value - 0.1 TON - forward TON amount - 0.02 TON (for notification message) - excess - 0.033 TON - total loss = 0.1 - 0.033 = 0.067 TON Single highload message Jetton transfer to not deployed Jetton wallet (FNZ Jetton): - transfer message value - 0.1 TON - forward TON amount - 0.02 TON (for notification message) - excess - 0.022 TON - total loss = 0.1 - 0.022 = 0.078 TON ## Freezing and deleting unused accounts If account do not used by a long time, and its balance under 0 by storage fee, this account freezes (by the next transaction) and then deletes by node (by the next transaction if balance still < 0). It is dangerous for Jetton wallets (hot and cold) and when account data drops Jetton balance drops too. Recommended to check hot and Jetton wallet balances periodically and fill it (or use special software). ## Highload wallet message deduplication In order to check the success of sending separate messages in a batch, we need to identify them. Adding a memo to a message to make it unique distorts the user's comment. Use control of the uniqueness of the destination address in the batch instead of adding memo. Destination address is a message dest address for TON transfers, and it is a destination from ton transfer message payload for Jetton transfer (to avoid deduplication by the Jetton hot wallet address). Because the wrapped payload is sent to the proxy contract, then the destination address is the address of the proxy contract (for service and internal Jetton withdrawals). ## Audit log There is an audit log to detect anomalous service behavior and unusual events in the blockchain. Service errors of a technical nature fall into the ordinary log. The audit log message contains the location where the event was detected, the transaction hash (if this event was detected by the block scanner) and the text of the message. There are three levels of warnings: * INFO - the event is not dangerous, but unusual * WARNING - the event may be potentially hazardous and should be attended to * ERROR - the event can pose a critical threat to the operation of the service ## Sharding ### Examples of block shard_prefix from lite client * `0000 100000000000000000000000000000000000000000000000000000000000` * `0001 100000000000000000000000000000000000000000000000000000000000` * `0010 100000000000000000000000000000000000000000000000000000000000` ### Only one shard example * ` 1000000000000000000000000000000000000000000000000000000000000000` OR `0x8000000000000000` It is equivalent of empty bitstring. ### Default SHARD We use a fixed-size address prefix (8 bits). The first 8 bits of 256 bit std_address (not workchain) and workchain = 0. And all addresses will be in the same shard up to 2^8 (256) shards. The default `SHARD` value is taken from the hot wallet address. Hot wallet address generates from seed phrase and default subwallet_id. #### Example: * `hot_ton_wallet_address = 0:60573d8db98cc369b7ce4ca1dadbfcbd17e82952938857a6cf14e1f8d77c811a` (raw form) * `SHARD = 01100000` (0x60) * `address_binary_prefix = 01100000` (for all deposit-addresses) ##### Suitable block shard prefixes (for these addresses): * ` 1000000000000000000000000000000000000000000000000000000000000000` - 1 shard * `0110 100000000000000000000000000000000000000000000000000000000000` - 16 shards * `01100000 10000000000000000000000000000000000000000000000000000000` - 256 shards ##### Not suitable block shard prefixes (for these addresses): * `0010 100000000000000000000000000000000000000000000000000000000000` - invalid prefix * `01100000 0 100000000000000000000000000000000000000000000000000000` - more than 256 shards. No guarantees that the address will be in the right shard. ## Notifications The system state is saved to the database in a single transaction, which allows you to restore the full state in the event of an abnormal restart and avoid data loss or duplication. However, sending notifications to channels (webhook or queue) is a separate operation that can be performed before or after saving the state to the database. If notifications are sent before being saved to the database, and the service is restarted between these events, then after the restart the notification will be sent again, which will lead to duplication. If notifications are sent after saving to the database, and the service is restarted between these events, then after the restart the notification will not be sent, which will lead to the loss of the notification. Because the service stores all data about operations in the database and, if necessary, you can make clarifying queries, then the second scenario was chosen. The first scenario requires the high-tier service to have notification deduplication logic. ## Running the test util for payment processor **It is strictly recommended to run the test utility with the processor configured for the testnet.** Optionally you can start test environment for payment-processor. This utility generates deposits via API, sends TONs and Jettons from `payment-processor A` to deposits of `payment-processor B` and vice versa. Thus, the utility circulates TONs and Jettons in a closed loop between `payment-processor A`->`payment-processor B`->`payment-processor A`. The utility also allows to evaluate the loss of TONs for the circulation of funds, check the completeness of the withdrawals and the presence of double withdrawals of funds. ### Configurable parameters | ENV variable | Description | |------------------------|--------------------------------------------------------------------------------------------------------| | `LITESERVER` | same as for payment-processor A and B (must be the same for A and B) | | `LITESERVER_KEY` | same as for payment-processor A and B (must be the same for A and B) | | `DB_URI` | same as for payment-processor A | | `HOST_A` | host of payment-processor A, example `payment_processor_a:8081` | | `HOST_B` | host of payment-processor B, example `payment_processor_b:8081` | | `API_TOKEN` | same as for payment-processor A and B (must be the same for A and B) | | `IS_TESTNET` | same as for payment-processor A and B (must be the same for A and B) | | `JETTONS` | same as for payment-processor A and B (must be the same for A and B) | | `TON_CUTOFFS` | same as for payment-processor A and B (must be the same for A and B) | | `HOT_WALLET_A` | hot-wallet address for payment-processor A, example `kQCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseH3XZ_YiH9Y1ufw` | | `HOT_WALLET_B` | hot-wallet address for payment-processor B, example `kQCdyiS-fIV9UVfI9Phswo4l2MA-hm8YseH3XZ_YiH9Y1ufw` | | `CIRCULATION` | `true` for funds circulation in closed loop. Default: `false`. | To turn on TON and Jetton circulation set `CIRCULATION=true` ENV variable. If you need only balance monitoring without TON and Jetton circulation set `CIRCULATION=false` ENV variable. 1. The test util image of the utility is built from the same makefile as the payment processor ```console make -f Makefile ``` 2. Prepare `.env` file for `payment-postgres` A and B services or fill environment variables in `docker-compose-test.yml` file. Database scheme automatically init. ```console docker-compose -f docker-compose-test.yml up -d payment-postgres-a docker-compose -f docker-compose-test.yml up -d payment-postgres-b ``` 3. Prepare `.env` file for `payment-processor` A and B services or fill environment variables in `docker-compose-test.yml` file. Seeds for A and B must be different. ```console docker-compose -f docker-compose-test.yml up -d payment-processor-a docker-compose -f docker-compose-test.yml up -d payment-processor-b ``` 4. Start Grafana for services monitoring. Prepare `.env` file for `payment-grafana` service or fill environment variables in `docker-compose-test.yml` file. ```console docker-compose -f docker-compose-test.yml up -d payment-grafana ``` 5. Start `payment-prometheus` container ```console docker-compose -f docker-compose-test.yml up -d payment-prometheus ``` 6. Prepare `.env` file for `payment-test` service or fill environment variables in `docker-compose-test.yml` file. ```console docker-compose -f docker-compose-test.yml up -d payment-test ``` ================================================ FILE: tests/docker-compose-tests.yml ================================================ version: '3' services: payment-postgres-a: image: postgres:14 container_name: payment_processor_db_a volumes: - ./deploy/db/01_init.up.sql:/docker-entrypoint-initdb.d/init.sql ports: - "5433:5432" restart: always env_file: - ../.env environment: POSTGRES_DB: "payment_processor" POSTGRES_USER: "pp_user" POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} networks: - p-network payment-postgres-b: image: postgres:14 container_name: payment_processor_db_b volumes: - ./deploy/db/01_init.up.sql:/docker-entrypoint-initdb.d/init.sql ports: - "5434:5432" restart: always env_file: - ../.env environment: POSTGRES_DB: "payment_processor" POSTGRES_USER: "pp_user" POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} networks: - p-network payment-processor-a: image: payment-processor container_name: payment_processor_a ports: - "8082:8081" restart: unless-stopped env_file: - ../.env environment: DB_URI: "postgres://pp_user:${POSTGRES_PASSWORD}@payment_processor_db_a:5432/payment_processor" API_TOKEN: ${API_TOKEN} COLD_WALLET: ${COLD_WALLET} JETTONS: ${JETTONS} LITESERVER: ${LITESERVER} LITESERVER_KEY: ${LITESERVER_KEY} SEED: ${SEED} TON_CUTOFFS: ${TON_CUTOFFS} IS_TESTNET: ${IS_TESTNET} networks: - p-network payment-processor-b: image: payment-processor container_name: payment_processor_b ports: - "8083:8081" restart: unless-stopped env_file: - ../.env environment: DB_URI: "postgres://pp_user:${POSTGRES_PASSWORD}@payment_processor_db_b:5432/payment_processor" API_TOKEN: ${API_TOKEN} COLD_WALLET: ${COLD_WALLET} JETTONS: ${JETTONS} LITESERVER: ${LITESERVER} LITESERVER_KEY: ${LITESERVER_KEY} SEED: ${SEED} TON_CUTOFFS: ${TON_CUTOFFS} IS_TESTNET: ${IS_TESTNET} networks: - p-network payment-grafana: image: grafana/grafana:latest container_name: payment_grafana restart: always ports: - '3001:3000' volumes: - ./deploy/grafana/test/provisioning/datasources:/etc/grafana/provisioning/datasources - ./deploy/grafana/test/provisioning/dashboards:/etc/grafana/provisioning/dashboards - ./deploy/grafana/test/dashboards:/etc/dashboards env_file: - ../.env environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} networks: - p-network payment-prometheus: image: prom/prometheus:latest container_name: payment_prometheus restart: always ports: - '9090:9090' volumes: - ./deploy/prometheus/test:/etc/prometheus user: "$UID:$GID" command: - '--config.file=/etc/prometheus/prometheus.yml' - '--web.external-url=http://localhost:9090' networks: - p-network payment-test: image: payment-test container_name: payment_test restart: unless-stopped env_file: - ../.env environment: DB_URI: "postgres://pp_user:${POSTGRES_PASSWORD}@payment_processor_db:5432/payment_processor" API_TOKEN: ${API_TOKEN} COLD_WALLET: ${COLD_WALLET} JETTONS: ${JETTONS} LITESERVER: ${LITESERVER} LITESERVER_KEY: ${LITESERVER_KEY} SEED: ${SEED} TON_CUTOFFS: ${TON_CUTOFFS} IS_TESTNET: ${IS_TESTNET} CIRCULATION: ${CIRCULATION} HOST_A: ${HOST_A} HOST_B: ${HOST_B} HOT_WALLET_A: ${HOT_WALLET_A} HOT_WALLET_B: ${HOT_WALLET_B} networks: - p-network networks: p-network: driver: bridge ================================================ FILE: threat_model.md ================================================ ### List of possible vulnerabilities In the format: - P: problem - T: threat - S: possible solutions - D: decision #### Untrusted node - P: unreliable data from the node lite server - T: the node operator can control the behavior of the service - S: use your trusted node or use a web3 library with proof checking (like tolibgo) - D: add a recommendation for launching your node in the readme #### Time out of sync - P: time out of sync between a node and service - T: incorrect expiration check and double withdrawals - S: use your trusted node and check time diff between node and service - D: add a recommendation for launching your node in the readme and service stops if time diff too big #### Blockchain out of sync - P: out of sync between a blockchain and service (by blocks) - T: the service may mark some transactions that have not yet been found as expired and make double withdrawals - S: check time diff between last processed block and actual time (time of last block) - D: service checks time diff between last the processed block and the actual time and does not make any withdrawals until service is synchronized #### Repeated withdrawal of TONs from the deposit - P: when a message is sent to wallet with 128+32 mode, the wallet contract is deleted, but the message itself remains in the node's mempool for some time. If TONs arrive at the wallet address at this time, the message will be applied again (because seqno is being reset). - T: repeated withdrawal of TONs from the deposit to the hot wallet ignoring the cutoff for the minimum withdrawal and growth of internal fees - S: reducing the valid_until time for message - D: small valid_until time for message #### Bruteforce API Token through Time Attack - P: the api token can be bruteforced by the difference in the token verification time - T: an attacker can control the service through the API - S: isolation of the payment processor from the external network and use constant time function to check token - D: isolation recommendation in readme file and service uses constant time function to check token #### Forgery of messages with internal service data - P: an attacker can copy service messages with their data (like memo) - T: incorrect operation of the service or external behavior control from the blockchain - S: checking the addresses of the sender and recipient of the message and checking hash for some messages - D: checking the addresses of the sender and recipient of the message and checking hash for some messages #### Unexpected tonutils package functionality - P: the behavior of some functions may differ from what is expected - T: incorrect operation of the service or malicious behavior of the library - S: open source code review or trust the tonutils library - D: trust the tonutils library #### Modified wallet code in the tonutils package - P: the library may contain a modified wallet code with additional functionality - T: unexpected behavior of the wallet in the blockchain and control of the wallet from the outside of service - S: compare the wallet code with a trusted source and fix the library version or trust the tonutils library - D: trust the tonutils library #### Untrusted binary libs in tongo package - P: the behavior of some functions (TVM emulation) may differ from what is expected - T: incorrect operation of the service or malicious behavior of the tongo library - S: build binary libraries from the official TON repository or trust the tongo library - D: build binary libraries from the official TON repository #### Jetton wallets with unexpected behavior - P: custom tokens may have unusual behavior on blockchain - T: incorrect calculation of Jettons balances or large internal commissions of the service - S: use Jettons with known behavior and conforming to the standard - D: add a recommendations for valid Jettons to readme file #### A lot of expensive withdrawals - P: the service is trying to make a lot of expensive withdrawals (more than wallet balance) as a result, they displace other withdrawals from the message batch of 255 messages - T: the service stops processing external withdrawals - S: increasing the number of withdrawals requests from the database - D: the number of requested withdrawals requests is configured during operation #### Service withdrawals changes incoming Jetton balance - P: service withdrawal may be detected as negative incoming (and interprets as unknown tx) - T: incorrect calculation of Jettons balances - S: check dest address and not set "unknown tx" flag - D: check dest address and not set "unknown tx" flag #### DDOS and blocking requests - P: service can not process a lot of requests or may wait for blocked request (where mutex used) - T: the service stops processing new requests - S: all requests must be limited by the user - D: recommendations for requests limitations in the readme file #### SQL injections via `comment`, `user_id` and other text fields - P: the danger of injection through the request fields - T: executing an arbitrary query on the database - S: sanitize user input - D: use `go pgx` with sanitize #### Withdrawals to internal address - P: it is possible to make a withdrawal to an address and then generate the same address as the deposit address, thereby making a withdrawal to the internal address - T: this can break the uniqueness of the addresses in the wallet message batch and break the checking of the correspondence of incoming and outgoing messages - S: it is rare case, warning in technical notes, uniqueness check in withdrawal processor - D: warning in technical notes, uniqueness check in withdrawal processor #### Duplicate randomly generated UUIDs - P: DB error for not unique UUID in internal or service withdrawals - T: service crashes with fatal error - S: the probability to find a duplicate within 103 trillion version-4 UUIDs is one in a billion - D: the probability of error is too small, no action is required #### Freezing and deleting unused account - P: if account do not used by long time and its balance under 0 by storage fee, this account freezes and then deletes by node - T: if Jetton wallet do not used by a long time it may be dropped by node, data with Jetton balance is cleaning - S: all Jettons (> cutoff) withdraws from deposits when service works normally. It is dangerous for Jetton cold/hot wallets, that do not use for a long time. Needs to check and fill balances periodically. - D: recommendation in technical_notes file to periodically check and fill TON balances on hot and cold wallets. Recommendation to use software to do it automatically. #### Deposit side vs hot side balance shifting when service withdrawals - P: service withdrawal of Jettons from the TON deposit occurs through the Jetton wallet and is not detected by the block scanner as an internal TON withdrawal - T: TON balance shift between deposit side and hot wallet side - S: use service withdrawal of Jettons from TON deposit wallet only with zero or near-zero TON balance - D: warning about this behavior in technical_notes file for method description #### Setting the value to "expired" without taking into account the allowable delay - P: it is impossible to absolutely precisely synchronize in time with the blockchain, so there is an allowable time delay value. If you get into this gap, the "expired" may be incorrectly set. - T: double spending for external withdrawals or unnecessary internal withdrawals - S: check expiration taking into account time delay - D: check expiration taking into account time delay #### Repetitive failed transactions burning fees - P: with periodic withdrawal cycles, there may be situations where the transaction fails every time. For example, when withdrawing to an uninitialized cold wallet with the bounce flag. - T: constant burning of a certain amount of TON on fees - S: additional checks to predict the success of the transaction and additional messages in the audit log - D: made an additional check on the state of the cold wallet and checking the bounce flag for withdrawal #### Too frequent withdrawals from a hot wallet to a cold wallet - P: if you set only the maximum cutoff for funds on the hot wallet, then the withdrawal to the cold wallet will occur if this amount is exceeded, even if the amount of the excess is less than the amount of the withdrawal fee - T: there may be withdrawals of the amount of funds at which the amount of funds is unreasonably small, which will lead to unnecessary burning of funds on fees - S: it is necessary to set some delta between the amount of triggering the withdrawal to the cold wallet and the amount that will remain after the withdrawal - D: one more parameter has been added to the cutoffs - `hot_wallet_residual_balance` #### Message queue overflow in highload v2 hot wallet contract - P: When messages with a long expiration time are sent frequently, the queue inside the wallet contract can grow to large sizes. At a certain queue size, transactions on the contract will begin to fail with the out of gas error, which will lead to the contract not working and burning funds on fees. Some details here: [highload-wallet-v2-code.fc](https://github.com/ton-blockchain/ton/blob/cf83bd18933143da31a37c1e0d1d67f999d5f9ec/crypto/smartcont/highload-wallet-v2-code.fc) - T: The hot wallet contract will stop processing external messages correctly and will burn funds on fees. - S: reducing the valid_until time for message, combining messages into batches, increasing the interval between withdrawals - D: small valid_until time for message, use of batches, large interval between withdrawals ================================================ FILE: todo_list.md ================================================ ## TODO - [x] Withdraw TON method - [x] Withdraw jetton method - [x] Generate new address API method - [x] Get addresses for user API method - [x] Send TON/Jetton API method - [x] Shard block scanner and tx parsing - [x] Batched send TONs method - [x] Batched send Jettons method - [x] TON transfer comment saving - [x] TON deposit withdrawal - [x] Custom withdrawal comment support - [x] Jetton transfer comment saving - [x] Jetton deposit withdrawal - [x] Deposit withdrawal validation - [x] Restart policy (repair after reconnect) - [x] Time sync with node - [x] Cold wallets support - [x] Shard merge/split detecting - [x] Graceful shutdown - [x] Healthcheck - [x] License - [x] Deploy (in need) hot wallet on start - [x] Queue/webhook notifications - [x] Deposits balances get method - [x] Deposit balance calculation flag (after deposit filling or hot-wallet filling) - [x] Threat model draft - [x] Anomalous behavior detecting and audit log - [x] Docs - [x] Refactoring - [x] Deploy scripts - [x] Unit tests - [x] Validate wallets code for tonutils v1.4.1 - [x] Manual testing plan - [x] Service methods for API (cancellation of incorrect payments) - [x] Build emulator lib from sources - [x] Integration tests - [x] Hot wallets metrics - [x] Manual testing - [x] Jettons test list - [x] Fix timeouts - [x] Allow to start with empty Jetton env var - [x] Deposit side balances by default - [x] Fix "outgoing message from internal incoming" for bounced TON payment - [x] Add history method - [x] Rename balance to income and return owner address instead of jetton wallet (for queue too) - [x] Add history method to test plan - [x] Add filling deposit with bounce to test plan - [x] Update to tonutils-go 1.6.2 - [x] Process masterchain addresses for external incomes - [x] Cold wallet withdrawal fix - [x] Add hysteresis to cold wallet withdrawal - [x] Add user id to notifications - [x] Add transaction hash to notifications - [x] Save tx hash to DB for incomes - [x] Add `failed` status for withdrawals and do not retry failed (at hot wallet) withdrawals - [x] Save tx hash to DB for withdrawals - [x] Get incoming by tx hash method - [x] Add asc-desc flag for get history method - [x] Add error counter as prometheus metrics - [x] Send events after saving to the database (there is a possibility of losing events instead of duplicating them) - [x] Use stable branch for emulator - [x] Get balance method - [x] Add meta to get withdrawals status method - [x] DNS resolver - [x] Check proofs - [x] Total withdrawals amount for get balance method - [x] Add last block time to /v1/system/sync method - [x] Forward ton amount customization - [x] Binary comment support for withdrawals - [x] Show app ver at start - [x] Download blockchain config at start - [x] Add reconnect to node when timeout expires - [ ] Get withdrawal by tx hash method - [ ] Add incorrect processing of some TON deposit replenishments for failed transaction to threat model - [ ] Duplicates of external withdrawals for DB backup problem - [ ] Avoid blocking withdrawals to an address if there is a very large amount in the queue for withdrawals to this address - [ ] Withdrawal cancellation mechanism - [ ] Jetton threat model - [ ] TNX compatibility test - [ ] Installation video manual - [ ] Node deploy - [ ] Performance optimization - [ ] Fix base64 public key format in .env file - [ ] Describe recovery scenarios - [ ] BOLT compatibility test - [ ] Not process removed Jettons - [ ] Separate .env files for services - [ ] Automatic migrations - [ ] SDK - [ ] Migration from blueprint to openapi - [ ] Refactor config and cutoff parameters - [ ] Get balances via states and check proof (not via get method) - [ ] Remove scam jettons from examples ================================================ FILE: webhook/webhook.go ================================================ package webhook import ( "bytes" "encoding/json" "fmt" log "github.com/sirupsen/logrus" "net/http" "time" ) type Client struct { client *http.Client uri string token string } // NewWebhookClient creates new webhook client func NewWebhookClient(uri string, token string) (*Client, error) { if uri == "" { return nil, fmt.Errorf("emty uri") } if token == "" { log.Infof("empty token for webhook") } return &Client{ client: &http.Client{Timeout: 10 * time.Second}, uri: uri, token: token, }, nil } func (s *Client) Publish(payload any) error { jsonData, err := json.Marshal(payload) if err != nil { return err } request, err := http.NewRequest("POST", s.uri, bytes.NewBuffer(jsonData)) if err != nil { return err } request.Header.Set("Content-Type", "application/json; charset=UTF-8") if s.token != "" { request.Header.Add("Authorization", "Bearer "+s.token) } for i := 0; i < 3; i++ { err := send(s.client, request) if err != nil { log.Errorf("webhook sending error: %v", err) continue } return nil } return fmt.Errorf("attempts to send a webhook ended") } func send(client *http.Client, request *http.Request) error { response, err := client.Do(request) if err != nil { return fmt.Errorf("webhook sending error: %v", err) } defer func() { err := response.Body.Close() if err != nil { log.Fatalf("response body close error: %v", err) } }() if response.StatusCode == 200 { return nil } else { return fmt.Errorf("webhook response status: %v", response.Status) } }